2023-09-23T17:24:42.527 INFO:root:teuthology version: 0.0.1.dev194+g54e62bc 2023-09-23T17:24:42.528 DEBUG:teuthology.run:Teuthology command: teuthology --owner scheduled_teuthology@teuthology --verbose --archive /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763 --name teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi --description upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} -- /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763/orig.config.yaml 2023-09-23T17:24:42.589 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T17:24:42.676 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' first_in_suite: false job_id: '7401763' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'run basic import/export cli tests for rbd ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: rhel os_version: '8.6' overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 4997 sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh env: RBD_FEATURES: '61' - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 targets: smithi055.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= smithi099.front.sepia.ceph.com: ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools:rhel8 --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 54e62bcbac4e53d9685e08328b790d3b20d71cae timestamp: 2023-09-23_14:23:10 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.2830 2023-09-23T17:24:42.676 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa; will attempt to use it 2023-09-23T17:24:42.677 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks 2023-09-23T17:24:42.677 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-09-23T17:24:42.679 INFO:teuthology.task.internal:Checking packages... 2023-09-23T17:24:42.701 INFO:teuthology.task.internal:Checking packages for os_type 'rhel', flavor 'default' and ceph hash 'bb0daeff97f020f686596fca6eba20c310e26fb7' 2023-09-23T17:24:42.701 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T17:24:42.701 INFO:teuthology.packaging:ref: None 2023-09-23T17:24:42.702 INFO:teuthology.packaging:tag: None 2023-09-23T17:24:42.702 INFO:teuthology.packaging:branch: reef 2023-09-23T17:24:42.702 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:24:42.702 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-09-23T17:24:43.027 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-460.gbb0daeff 2023-09-23T17:24:43.027 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-09-23T17:24:43.033 INFO:teuthology.task.internal:no buildpackages task found 2023-09-23T17:24:43.033 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-09-23T17:24:43.057 INFO:teuthology.task.internal:Saving configuration 2023-09-23T17:24:43.079 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-09-23T17:24:43.085 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T17:24:43.108 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi055.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 17:20:13.686066', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T17:24:43.130 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi099.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 17:20:13.687547', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T17:24:43.131 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-09-23T17:24:43.137 INFO:teuthology.task.internal:roles: ubuntu@smithi055.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-09-23T17:24:43.137 INFO:teuthology.task.internal:roles: ubuntu@smithi099.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-09-23T17:24:43.137 INFO:teuthology.run_tasks:Running task console_log... 2023-09-23T17:24:43.218 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f94406c1af0>, signals=[15]) 2023-09-23T17:24:43.218 INFO:teuthology.run_tasks:Running task internal.connect... 2023-09-23T17:24:43.224 INFO:teuthology.task.internal:Opening connections... 2023-09-23T17:24:43.224 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi055.front.sepia.ceph.com 2023-09-23T17:24:43.225 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi055.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:24:43.293 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi099.front.sepia.ceph.com 2023-09-23T17:24:43.294 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:24:43.358 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-09-23T17:24:43.364 DEBUG:teuthology.orchestra.run.smithi055:> uname -m 2023-09-23T17:24:43.386 INFO:teuthology.orchestra.run.smithi055.stdout:x86_64 2023-09-23T17:24:43.386 DEBUG:teuthology.orchestra.run.smithi055:> cat /etc/os-release 2023-09-23T17:24:43.445 INFO:teuthology.orchestra.run.smithi055.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T17:24:43.446 INFO:teuthology.orchestra.run.smithi055.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T17:24:43.446 INFO:teuthology.orchestra.run.smithi055.stdout:ID="rhel" 2023-09-23T17:24:43.446 INFO:teuthology.orchestra.run.smithi055.stdout:ID_LIKE="fedora" 2023-09-23T17:24:43.446 INFO:teuthology.orchestra.run.smithi055.stdout:VERSION_ID="8.6" 2023-09-23T17:24:43.446 INFO:teuthology.orchestra.run.smithi055.stdout:PLATFORM_ID="platform:el8" 2023-09-23T17:24:43.446 INFO:teuthology.orchestra.run.smithi055.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T17:24:43.446 INFO:teuthology.orchestra.run.smithi055.stdout:ANSI_COLOR="0;31" 2023-09-23T17:24:43.446 INFO:teuthology.orchestra.run.smithi055.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T17:24:43.446 INFO:teuthology.orchestra.run.smithi055.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T17:24:43.446 INFO:teuthology.orchestra.run.smithi055.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T17:24:43.447 INFO:teuthology.orchestra.run.smithi055.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T17:24:43.447 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:24:43.447 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T17:24:43.447 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T17:24:43.447 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T17:24:43.447 INFO:teuthology.orchestra.run.smithi055.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T17:24:43.448 INFO:teuthology.lock.ops:Updating smithi055.front.sepia.ceph.com on lock server 2023-09-23T17:24:43.469 DEBUG:teuthology.orchestra.run.smithi099:> uname -m 2023-09-23T17:24:43.486 INFO:teuthology.orchestra.run.smithi099.stdout:x86_64 2023-09-23T17:24:43.486 DEBUG:teuthology.orchestra.run.smithi099:> cat /etc/os-release 2023-09-23T17:24:43.542 INFO:teuthology.orchestra.run.smithi099.stdout:NAME="Red Hat Enterprise Linux" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION="8.6 (Ootpa)" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:ID="rhel" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:ID_LIKE="fedora" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:VERSION_ID="8.6" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:PLATFORM_ID="platform:el8" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:PRETTY_NAME="Red Hat Enterprise Linux 8.6 (Ootpa)" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:ANSI_COLOR="0;31" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:CPE_NAME="cpe:/o:redhat:enterprise_linux:8::baseos" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:HOME_URL="https://www.redhat.com/" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:DOCUMENTATION_URL="https://access.redhat.com/documentation/red_hat_enterprise_linux/8/" 2023-09-23T17:24:43.543 INFO:teuthology.orchestra.run.smithi099.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-09-23T17:24:43.544 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:24:43.544 INFO:teuthology.orchestra.run.smithi099.stdout:REDHAT_BUGZILLA_PRODUCT="Red Hat Enterprise Linux 8" 2023-09-23T17:24:43.544 INFO:teuthology.orchestra.run.smithi099.stdout:REDHAT_BUGZILLA_PRODUCT_VERSION=8.6 2023-09-23T17:24:43.544 INFO:teuthology.orchestra.run.smithi099.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux" 2023-09-23T17:24:43.544 INFO:teuthology.orchestra.run.smithi099.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="8.6" 2023-09-23T17:24:43.544 INFO:teuthology.lock.ops:Updating smithi099.front.sepia.ceph.com on lock server 2023-09-23T17:24:43.569 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-09-23T17:24:43.579 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-09-23T17:24:43.598 INFO:teuthology.task.internal:Checking for old test directory... 2023-09-23T17:24:43.598 DEBUG:teuthology.orchestra.run.smithi055:> test '!' -e /home/ubuntu/cephtest 2023-09-23T17:24:43.600 DEBUG:teuthology.orchestra.run.smithi099:> test '!' -e /home/ubuntu/cephtest 2023-09-23T17:24:43.618 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-09-23T17:24:43.632 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-09-23T17:24:43.632 DEBUG:teuthology.orchestra.run.smithi055:> test -z $(ls -A /var/lib/ceph) 2023-09-23T17:24:43.661 DEBUG:teuthology.orchestra.run.smithi099:> test -z $(ls -A /var/lib/ceph) 2023-09-23T17:24:43.684 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-09-23T17:24:43.733 INFO:teuthology.run_tasks:Running task kernel... 2023-09-23T17:24:43.748 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-09-23T17:24:43.748 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-09-23T17:24:43.748 DEBUG:teuthology.orchestra.run.smithi055:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T17:24:43.749 DEBUG:teuthology.orchestra.run.smithi099:> test -f /run/.containerenv -o -f /.dockerenv 2023-09-23T17:24:43.764 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T17:24:43.765 DEBUG:teuthology.orchestra.run.smithi099:> uname -r 2023-09-23T17:24:43.768 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T17:24:43.768 DEBUG:teuthology.orchestra.run.smithi055:> uname -r 2023-09-23T17:24:43.821 INFO:teuthology.orchestra.run.smithi099.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T17:24:43.821 INFO:teuthology.task.kernel:Running kernel on smithi099: 4.18.0-372.9.1.el8.x86_64 2023-09-23T17:24:43.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum install -y kernel 2023-09-23T17:24:43.828 INFO:teuthology.orchestra.run.smithi055.stdout:4.18.0-372.9.1.el8.x86_64 2023-09-23T17:24:43.828 INFO:teuthology.task.kernel:Running kernel on smithi055: 4.18.0-372.9.1.el8.x86_64 2023-09-23T17:24:43.829 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum install -y kernel 2023-09-23T17:24:45.015 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:24:45.016 INFO:teuthology.orchestra.run.smithi055.stdout:Unable to read consumer identity 2023-09-23T17:24:45.163 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:24:45.163 INFO:teuthology.orchestra.run.smithi099.stdout:Unable to read consumer identity 2023-09-23T17:24:46.513 INFO:teuthology.orchestra.run.smithi055.stdout:Copr repo for python3-asyncssh owned by ceph 18 kB/s | 3.5 kB 00:00 2023-09-23T17:24:46.554 INFO:teuthology.orchestra.run.smithi099.stdout:Copr repo for python3-asyncssh owned by ceph 23 kB/s | 3.5 kB 00:00 2023-09-23T17:24:47.221 INFO:teuthology.orchestra.run.smithi055.stdout:Extra Packages for Enterprise Linux 23 MB/s | 16 MB 00:00 2023-09-23T17:24:47.596 INFO:teuthology.orchestra.run.smithi099.stdout:Extra Packages for Enterprise Linux 16 MB/s | 16 MB 00:01 2023-09-23T17:24:50.879 INFO:teuthology.orchestra.run.smithi055.stdout:lab-extras 438 kB/s | 24 kB 00:00 2023-09-23T17:24:51.037 INFO:teuthology.orchestra.run.smithi099.stdout:lab-extras 436 kB/s | 24 kB 00:00 2023-09-23T17:24:51.618 INFO:teuthology.orchestra.run.smithi055.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T17:24:51.668 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:24:51.669 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2023-09-23T17:24:51.669 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:24:51.723 INFO:teuthology.orchestra.run.smithi099.stdout:Package kernel-4.18.0-372.9.1.el8.x86_64 is already installed. 2023-09-23T17:24:51.770 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:24:51.771 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2023-09-23T17:24:51.771 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:24:51.797 DEBUG:teuthology.orchestra.run.smithi055:> echo no | sudo yum reinstall kernel || true 2023-09-23T17:24:51.900 DEBUG:teuthology.orchestra.run.smithi099:> echo no | sudo yum reinstall kernel || true 2023-09-23T17:24:52.147 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:24:52.147 INFO:teuthology.orchestra.run.smithi055.stdout:Unable to read consumer identity 2023-09-23T17:24:52.231 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:24:52.231 INFO:teuthology.orchestra.run.smithi099.stdout:Unable to read consumer identity 2023-09-23T17:24:52.423 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 05:24:50 PM UTC. 2023-09-23T17:24:52.470 INFO:teuthology.orchestra.run.smithi055.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T17:24:52.490 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 05:24:50 PM UTC. 2023-09-23T17:24:52.513 INFO:teuthology.orchestra.run.smithi055.stderr:Error: No packages marked for reinstall. 2023-09-23T17:24:52.533 INFO:teuthology.orchestra.run.smithi099.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T17:24:52.559 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum reinstall -y kernel || true 2023-09-23T17:24:52.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error: No packages marked for reinstall. 2023-09-23T17:24:52.616 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum reinstall -y kernel || true 2023-09-23T17:24:52.910 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:24:52.910 INFO:teuthology.orchestra.run.smithi055.stdout:Unable to read consumer identity 2023-09-23T17:24:52.944 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:24:52.944 INFO:teuthology.orchestra.run.smithi099.stdout:Unable to read consumer identity 2023-09-23T17:24:53.178 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:03 ago on Sat 23 Sep 2023 05:24:50 PM UTC. 2023-09-23T17:24:53.197 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:00:02 ago on Sat 23 Sep 2023 05:24:50 PM UTC. 2023-09-23T17:24:53.223 INFO:teuthology.orchestra.run.smithi055.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T17:24:53.237 INFO:teuthology.orchestra.run.smithi099.stdout:Installed package kernel-4.18.0-372.9.1.el8.x86_64 (from anaconda) not available. 2023-09-23T17:24:53.261 INFO:teuthology.orchestra.run.smithi055.stderr:Error: No packages marked for reinstall. 2023-09-23T17:24:53.278 INFO:teuthology.orchestra.run.smithi099.stderr:Error: No packages marked for reinstall. 2023-09-23T17:24:53.308 DEBUG:teuthology.orchestra.run.smithi055:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T17:24:53.325 DEBUG:teuthology.orchestra.run.smithi099:> rpm -q kernel | sort -rV | head -n 1 2023-09-23T17:24:53.392 INFO:teuthology.orchestra.run.smithi055.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T17:24:53.393 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T17:24:53.393 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T17:24:53.393 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-09-23T17:24:53.393 DEBUG:teuthology.orchestra.run.smithi055:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T17:24:53.404 INFO:teuthology.orchestra.run.smithi099.stdout:kernel-4.18.0-372.9.1.el8.x86_64 2023-09-23T17:24:53.405 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-372.9.1.el8.x86_64 2023-09-23T17:24:53.405 INFO:teuthology.task.kernel:Newest distro kernel installed and running 2023-09-23T17:24:53.405 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-09-23T17:24:53.406 DEBUG:teuthology.orchestra.run.smithi099:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-09-23T17:24:53.425 INFO:teuthology.orchestra.run.smithi055.stdout:ttyS1 2023-09-23T17:24:53.431 INFO:teuthology.orchestra.run.smithi099.stdout:ttyS1 2023-09-23T17:24:53.438 DEBUG:teuthology.parallel:result is None 2023-09-23T17:24:53.444 DEBUG:teuthology.parallel:result is None 2023-09-23T17:24:53.444 INFO:teuthology.run_tasks:Running task internal.base... 2023-09-23T17:24:53.453 INFO:teuthology.task.internal:Creating test directory... 2023-09-23T17:24:53.453 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T17:24:53.480 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-09-23T17:24:53.500 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-09-23T17:24:53.507 INFO:teuthology.run_tasks:Running task internal.archive... 2023-09-23T17:24:53.523 INFO:teuthology.task.internal:Creating archive directory... 2023-09-23T17:24:53.524 DEBUG:teuthology.orchestra.run.smithi055:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T17:24:53.538 DEBUG:teuthology.orchestra.run.smithi099:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-09-23T17:24:53.571 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-09-23T17:24:53.578 INFO:teuthology.task.internal:Enabling coredump saving... 2023-09-23T17:24:53.578 DEBUG:teuthology.orchestra.run.smithi055:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T17:24:53.609 DEBUG:teuthology.orchestra.run.smithi099:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-09-23T17:24:53.646 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T17:24:53.653 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T17:24:53.657 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T17:24:53.663 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-09-23T17:24:53.664 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-09-23T17:24:53.672 INFO:teuthology.task.internal:Configuring sudo... 2023-09-23T17:24:53.672 DEBUG:teuthology.orchestra.run.smithi055:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T17:24:53.701 DEBUG:teuthology.orchestra.run.smithi099:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-09-23T17:24:53.727 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-09-23T17:24:53.735 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-09-23T17:24:53.736 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T17:24:53.769 DEBUG:teuthology.orchestra.run.smithi099:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-09-23T17:24:53.785 DEBUG:teuthology.orchestra.run.smithi055:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T17:24:53.867 DEBUG:teuthology.orchestra.run.smithi055:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T17:24:53.923 DEBUG:teuthology.orchestra.run.smithi055:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T17:24:54.001 DEBUG:teuthology.orchestra.run.smithi055:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T17:24:54.048 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:24:54.049 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T17:24:54.116 DEBUG:teuthology.orchestra.run.smithi099:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T17:24:54.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-09-23T17:24:54.209 DEBUG:teuthology.orchestra.run.smithi099:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T17:24:54.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-09-23T17:24:54.334 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:24:54.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-09-23T17:24:54.401 DEBUG:teuthology.orchestra.run.smithi055:> sudo service rsyslog restart 2023-09-23T17:24:54.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo service rsyslog restart 2023-09-23T17:24:54.506 INFO:teuthology.orchestra.run.smithi055.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T17:24:54.532 INFO:teuthology.orchestra.run.smithi099.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T17:24:54.977 INFO:teuthology.run_tasks:Running task internal.timer... 2023-09-23T17:24:54.984 INFO:teuthology.task.internal:Starting timer... 2023-09-23T17:24:54.985 INFO:teuthology.run_tasks:Running task pcp... 2023-09-23T17:24:55.019 INFO:teuthology.run_tasks:Running task selinux... 2023-09-23T17:24:55.034 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-09-23T17:24:55.034 DEBUG:teuthology.orchestra.run.smithi055:> sudo service auditd rotate 2023-09-23T17:24:55.117 INFO:teuthology.orchestra.run.smithi055.stdout:Rotating logs: 2023-09-23T17:24:55.118 DEBUG:teuthology.orchestra.run.smithi099:> sudo service auditd rotate 2023-09-23T17:24:55.182 INFO:teuthology.orchestra.run.smithi099.stdout:Rotating logs: 2023-09-23T17:24:55.184 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-09-23T17:24:55.184 DEBUG:teuthology.orchestra.run.smithi055:> /usr/sbin/getenforce 2023-09-23T17:24:55.205 INFO:teuthology.orchestra.run.smithi055.stdout:Permissive 2023-09-23T17:24:55.205 DEBUG:teuthology.orchestra.run.smithi099:> /usr/sbin/getenforce 2023-09-23T17:24:55.227 INFO:teuthology.orchestra.run.smithi099.stdout:Permissive 2023-09-23T17:24:55.228 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi055.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi099.front.sepia.ceph.com': 'permissive'} 2023-09-23T17:24:55.228 DEBUG:teuthology.orchestra.run.smithi055:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T17:24:55.273 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T17:24:55.274 DEBUG:teuthology.orchestra.run.smithi099:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T17:24:55.301 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T17:24:55.301 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-09-23T17:24:55.302 DEBUG:teuthology.orchestra.run.smithi055:> sudo /usr/sbin/setenforce permissive 2023-09-23T17:24:55.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo /usr/sbin/setenforce permissive 2023-09-23T17:24:55.385 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-09-23T17:24:55.396 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-09-23T17:24:55.396 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-09-23T17:24:55.413 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-09-23T17:24:55.414 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi055.front.sepia.ceph.com,smithi099.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-09-23T17:34:02.822 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi055.front.sepia.ceph.com'), Remote(name='ubuntu@smithi099.front.sepia.ceph.com')] 2023-09-23T17:34:02.823 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi055.front.sepia.ceph.com' 2023-09-23T17:34:02.825 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi055.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:34:02.895 DEBUG:teuthology.orchestra.run.smithi055:> true 2023-09-23T17:34:02.969 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi055.front.sepia.ceph.com' 2023-09-23T17:34:02.970 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi099.front.sepia.ceph.com' 2023-09-23T17:34:02.970 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi099.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-09-23T17:34:03.039 DEBUG:teuthology.orchestra.run.smithi099:> true 2023-09-23T17:34:03.064 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi099.front.sepia.ceph.com' 2023-09-23T17:34:03.064 INFO:teuthology.run_tasks:Running task clock... 2023-09-23T17:34:03.077 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-09-23T17:34:03.077 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T17:34:03.077 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T17:34:03.080 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-09-23T17:34:03.080 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T17:34:03.113 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T17:34:03.129 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T17:34:03.136 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-09-23T17:34:03.150 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-09-23T17:34:03.201 INFO:teuthology.orchestra.run.smithi055.stderr:sudo: ntpd: command not found 2023-09-23T17:34:03.215 INFO:teuthology.orchestra.run.smithi055.stdout:506 Cannot talk to daemon 2023-09-23T17:34:03.227 INFO:teuthology.orchestra.run.smithi099.stderr:sudo: ntpd: command not found 2023-09-23T17:34:03.231 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T17:34:03.239 INFO:teuthology.orchestra.run.smithi099.stdout:506 Cannot talk to daemon 2023-09-23T17:34:03.246 INFO:teuthology.orchestra.run.smithi055.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T17:34:03.253 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-09-23T17:34:03.267 INFO:teuthology.orchestra.run.smithi099.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-09-23T17:34:03.315 INFO:teuthology.orchestra.run.smithi055.stderr:bash: ntpq: command not found 2023-09-23T17:34:03.318 INFO:teuthology.orchestra.run.smithi055.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T17:34:03.318 INFO:teuthology.orchestra.run.smithi055.stdout:=============================================================================== 2023-09-23T17:34:03.318 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T17:34:03.318 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T17:34:03.318 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T17:34:03.319 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T17:34:03.336 INFO:teuthology.orchestra.run.smithi099.stderr:bash: ntpq: command not found 2023-09-23T17:34:03.339 INFO:teuthology.orchestra.run.smithi099.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T17:34:03.339 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================== 2023-09-23T17:34:03.339 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T17:34:03.340 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T17:34:03.340 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T17:34:03.340 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T17:34:03.341 INFO:teuthology.run_tasks:Running task pexec... 2023-09-23T17:34:03.350 INFO:teuthology.task.pexec:Executing custom commands... 2023-09-23T17:34:03.350 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi055.front.sepia.ceph.com 2023-09-23T17:34:03.350 DEBUG:teuthology.orchestra.run.smithi055:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T17:34:03.351 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi099.front.sepia.ceph.com 2023-09-23T17:34:03.351 DEBUG:teuthology.orchestra.run.smithi099:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-09-23T17:34:03.754 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:34:03.754 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:34:04.604 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:54 ago on Sat 23 Sep 2023 05:32:10 PM UTC. 2023-09-23T17:34:04.642 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:24 ago on Sat 23 Sep 2023 05:32:40 PM UTC. 2023-09-23T17:34:06.344 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:34:06.345 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:34:06.345 INFO:teuthology.orchestra.run.smithi099.stdout: Package Architecture Version Repository Size 2023-09-23T17:34:06.345 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:34:06.345 INFO:teuthology.orchestra.run.smithi099.stdout:Resetting modules: 2023-09-23T17:34:06.345 INFO:teuthology.orchestra.run.smithi099.stdout: container-tools 2023-09-23T17:34:06.345 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:34:06.345 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:34:06.345 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:34:06.345 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:34:06.672 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:34:06.673 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:34:06.673 INFO:teuthology.orchestra.run.smithi055.stdout: Package Architecture Version Repository Size 2023-09-23T17:34:06.674 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:34:06.674 INFO:teuthology.orchestra.run.smithi055.stdout:Resetting modules: 2023-09-23T17:34:06.674 INFO:teuthology.orchestra.run.smithi055.stdout: container-tools 2023-09-23T17:34:06.674 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:34:06.674 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:34:06.674 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:34:06.674 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:34:07.313 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:56 ago on Sat 23 Sep 2023 05:32:10 PM UTC. 2023-09-23T17:34:07.675 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:27 ago on Sat 23 Sep 2023 05:32:40 PM UTC. 2023-09-23T17:34:10.004 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:34:10.405 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:34:10.542 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:34:10.877 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:34:11.336 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:00 ago on Sat 23 Sep 2023 05:32:10 PM UTC. 2023-09-23T17:34:11.718 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:31 ago on Sat 23 Sep 2023 05:32:40 PM UTC. 2023-09-23T17:34:13.052 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:34:13.145 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================================================ 2023-09-23T17:34:13.146 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-09-23T17:34:13.146 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================================================ 2023-09-23T17:34:13.146 INFO:teuthology.orchestra.run.smithi099.stdout:Installing group/module packages: 2023-09-23T17:34:13.146 INFO:teuthology.orchestra.run.smithi099.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-09-23T17:34:13.146 INFO:teuthology.orchestra.run.smithi099.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-09-23T17:34:13.146 INFO:teuthology.orchestra.run.smithi099.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-09-23T17:34:13.146 INFO:teuthology.orchestra.run.smithi099.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-09-23T17:34:13.146 INFO:teuthology.orchestra.run.smithi099.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-09-23T17:34:13.146 INFO:teuthology.orchestra.run.smithi099.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout:Installing module profiles: 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout: container-tools/common 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout:Enabling module streams: 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout: container-tools rhel8 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:34:13.147 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:34:13.148 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================================================ 2023-09-23T17:34:13.148 INFO:teuthology.orchestra.run.smithi099.stdout:Install 10 Packages 2023-09-23T17:34:13.148 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:34:13.148 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 18 M 2023-09-23T17:34:13.148 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 65 M 2023-09-23T17:34:13.149 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:34:13.214 INFO:teuthology.orchestra.run.smithi099.stdout:(1/10): yajl-2.1.0-10.el8.x86_64.rpm 621 kB/s | 41 kB 00:00 2023-09-23T17:34:13.239 INFO:teuthology.orchestra.run.smithi099.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 273 kB/s | 25 kB 00:00 2023-09-23T17:34:13.265 INFO:teuthology.orchestra.run.smithi099.stdout:(3/10): udica-0.2.6-4.module+el8.6.0+16771+28df 981 kB/s | 49 kB 00:00 2023-09-23T17:34:13.290 INFO:teuthology.orchestra.run.smithi099.stdout:(4/10): python3-pyxdg-0.25-16.el8.noarch.rpm 663 kB/s | 94 kB 00:00 2023-09-23T17:34:13.332 INFO:teuthology.orchestra.run.smithi099.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 5.5 MB/s | 524 kB 00:00 2023-09-23T17:34:13.349 INFO:teuthology.orchestra.run.smithi099.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 2.5 MB/s | 149 kB 00:00 2023-09-23T17:34:13.382 INFO:teuthology.orchestra.run.smithi099.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 4.1 MB/s | 213 kB 00:00 2023-09-23T17:34:13.500 INFO:teuthology.orchestra.run.smithi099.stdout:(8/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 19 MB/s | 2.2 MB 00:00 2023-09-23T17:34:13.600 INFO:teuthology.orchestra.run.smithi099.stdout:(9/10): buildah-1.26.2-1.module+el8.6.0+16771+2 24 MB/s | 8.0 MB 00:00 2023-09-23T17:34:13.683 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:34:13.692 INFO:teuthology.orchestra.run.smithi099.stdout:(10/10): skopeo-1.9.1-1.module+el8.6.0+16771+28 20 MB/s | 7.0 MB 00:00 2023-09-23T17:34:13.692 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:34:13.692 INFO:teuthology.orchestra.run.smithi099.stdout:Total 33 MB/s | 18 MB 00:00 2023-09-23T17:34:13.783 INFO:teuthology.orchestra.run.smithi055.stdout:============================================================================================================ 2023-09-23T17:34:13.784 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2023-09-23T17:34:13.784 INFO:teuthology.orchestra.run.smithi055.stdout:============================================================================================================ 2023-09-23T17:34:13.784 INFO:teuthology.orchestra.run.smithi055.stdout:Installing group/module packages: 2023-09-23T17:34:13.784 INFO:teuthology.orchestra.run.smithi055.stdout: buildah x86_64 1:1.26.2-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 8.0 M 2023-09-23T17:34:13.784 INFO:teuthology.orchestra.run.smithi055.stdout: cockpit-podman noarch 49.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 524 k 2023-09-23T17:34:13.784 INFO:teuthology.orchestra.run.smithi055.stdout: crun x86_64 1.5-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 213 k 2023-09-23T17:34:13.784 INFO:teuthology.orchestra.run.smithi055.stdout: python3-podman noarch 4.0.1-1.module+el8.6.0+16837+85b82211 rhel-8-for-x86_64-appstream-rpms 149 k 2023-09-23T17:34:13.784 INFO:teuthology.orchestra.run.smithi055.stdout: skopeo x86_64 2:1.9.1-1.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 7.0 M 2023-09-23T17:34:13.784 INFO:teuthology.orchestra.run.smithi055.stdout: toolbox x86_64 0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 2.2 M 2023-09-23T17:34:13.784 INFO:teuthology.orchestra.run.smithi055.stdout: udica noarch 0.2.6-4.module+el8.6.0+16771+28dfca77 rhel-8-for-x86_64-appstream-rpms 49 k 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 rhel-8-for-x86_64-appstream-rpms 25 k 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyxdg noarch 0.25-16.el8 rhel-8-for-x86_64-appstream-rpms 94 k 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout: yajl x86_64 2.1.0-10.el8 rhel-8-for-x86_64-appstream-rpms 41 k 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout:Installing module profiles: 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout: container-tools/common 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout:Enabling module streams: 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout: container-tools rhel8 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:34:13.785 INFO:teuthology.orchestra.run.smithi055.stdout:============================================================================================================ 2023-09-23T17:34:13.786 INFO:teuthology.orchestra.run.smithi055.stdout:Install 10 Packages 2023-09-23T17:34:13.786 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:34:13.786 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 18 M 2023-09-23T17:34:13.786 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 65 M 2023-09-23T17:34:13.787 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:34:13.874 INFO:teuthology.orchestra.run.smithi055.stdout:(1/10): python3-pyxdg-0.25-16.el8.noarch.rpm 1.0 MB/s | 94 kB 00:00 2023-09-23T17:34:13.894 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:34:13.902 INFO:teuthology.orchestra.run.smithi055.stdout:(2/10): python3-pytoml-0.1.14-5.git7dea353.el8. 214 kB/s | 25 kB 00:00 2023-09-23T17:34:13.913 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:34:13.913 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:34:13.928 INFO:teuthology.orchestra.run.smithi055.stdout:(3/10): yajl-2.1.0-10.el8.x86_64.rpm 288 kB/s | 41 kB 00:00 2023-09-23T17:34:13.953 INFO:teuthology.orchestra.run.smithi055.stdout:(4/10): udica-0.2.6-4.module+el8.6.0+16771+28df 630 kB/s | 49 kB 00:00 2023-09-23T17:34:13.987 INFO:teuthology.orchestra.run.smithi055.stdout:(5/10): cockpit-podman-49.1-1.module+el8.6.0+16 6.1 MB/s | 524 kB 00:00 2023-09-23T17:34:14.012 INFO:teuthology.orchestra.run.smithi055.stdout:(6/10): python3-podman-4.0.1-1.module+el8.6.0+1 2.5 MB/s | 149 kB 00:00 2023-09-23T17:34:14.046 INFO:teuthology.orchestra.run.smithi055.stdout:(7/10): crun-1.5-1.module+el8.6.0+16771+28dfca7 3.6 MB/s | 213 kB 00:00 2023-09-23T17:34:14.096 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:34:14.099 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:34:14.154 INFO:teuthology.orchestra.run.smithi055.stdout:(8/10): toolbox-0.0.99.3-0.6.module+el8.6.0+167 20 MB/s | 2.2 MB 00:00 2023-09-23T17:34:14.288 INFO:teuthology.orchestra.run.smithi055.stdout:(9/10): skopeo-1.9.1-1.module+el8.6.0+16771+28d 25 MB/s | 7.0 MB 00:00 2023-09-23T17:34:14.380 INFO:teuthology.orchestra.run.smithi055.stdout:(10/10): buildah-1.26.2-1.module+el8.6.0+16771+ 18 MB/s | 8.0 MB 00:00 2023-09-23T17:34:14.380 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:34:14.380 INFO:teuthology.orchestra.run.smithi055.stdout:Total 31 MB/s | 18 MB 00:00 2023-09-23T17:34:14.590 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:34:14.600 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:34:14.611 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:34:14.611 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:34:14.775 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-09-23T17:34:14.806 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:34:14.810 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:34:14.961 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T17:34:15.147 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-09-23T17:34:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:34:15.363 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-09-23T17:34:15.580 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 1/10 2023-09-23T17:34:15.713 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-09-23T17:34:15.746 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T17:34:15.953 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 3/10 2023-09-23T17:34:16.126 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 4/10 2023-09-23T17:34:16.487 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 5/10 2023-09-23T17:34:16.490 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-09-23T17:34:17.342 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 6/10 2023-09-23T17:34:17.367 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-09-23T17:34:17.567 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-09-23T17:34:17.718 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-09-23T17:34:17.894 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T17:34:18.163 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T17:34:18.163 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-09-23T17:34:18.163 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T17:34:18.163 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-09-23T17:34:18.163 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-09-23T17:34:18.163 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-09-23T17:34:18.164 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-09-23T17:34:18.164 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-09-23T17:34:18.164 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-09-23T17:34:18.164 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-09-23T17:34:18.264 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 7/10 2023-09-23T17:34:18.496 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 8/10 2023-09-23T17:34:18.677 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 9/10 2023-09-23T17:34:18.990 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T17:34:19.286 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 10/10 2023-09-23T17:34:19.286 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 1/10 2023-09-23T17:34:19.286 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : yajl-2.1.0-10.el8.x86_64 2/10 2023-09-23T17:34:19.286 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/10 2023-09-23T17:34:19.286 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 4/10 2023-09-23T17:34:19.287 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca7 5/10 2023-09-23T17:34:19.287 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x 6/10 2023-09-23T17:34:19.287 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-podman-4.0.1-1.module+el8.6.0+16837+85b822 7/10 2023-09-23T17:34:19.287 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 8/10 2023-09-23T17:34:19.287 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86 9/10 2023-09-23T17:34:19.391 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-09-23T17:34:19.391 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:08 ago on Sat 23 Sep 2023 05:32:10 PM UTC. 2023-09-23T17:34:20.432 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77 10/10 2023-09-23T17:34:20.432 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:39 ago on Sat 23 Sep 2023 05:32:40 PM UTC. 2023-09-23T17:34:22.108 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:34:22.299 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:34:22.299 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:34:22.299 INFO:teuthology.orchestra.run.smithi099.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T17:34:22.299 INFO:teuthology.orchestra.run.smithi099.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T17:34:22.299 INFO:teuthology.orchestra.run.smithi099.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T17:34:22.299 INFO:teuthology.orchestra.run.smithi099.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-09-23T17:34:22.299 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-09-23T17:34:22.299 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-09-23T17:34:22.300 INFO:teuthology.orchestra.run.smithi099.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T17:34:22.300 INFO:teuthology.orchestra.run.smithi099.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T17:34:22.300 INFO:teuthology.orchestra.run.smithi099.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T17:34:22.300 INFO:teuthology.orchestra.run.smithi099.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T17:34:22.300 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:34:22.300 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:34:22.451 DEBUG:teuthology.parallel:result is None 2023-09-23T17:34:23.044 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:34:23.255 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:34:23.255 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:34:23.255 INFO:teuthology.orchestra.run.smithi055.stdout: buildah-1:1.26.2-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T17:34:23.255 INFO:teuthology.orchestra.run.smithi055.stdout: cockpit-podman-49.1-1.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T17:34:23.256 INFO:teuthology.orchestra.run.smithi055.stdout: crun-1.5-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T17:34:23.256 INFO:teuthology.orchestra.run.smithi055.stdout: python3-podman-4.0.1-1.module+el8.6.0+16837+85b82211.noarch 2023-09-23T17:34:23.256 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-09-23T17:34:23.256 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-09-23T17:34:23.256 INFO:teuthology.orchestra.run.smithi055.stdout: skopeo-2:1.9.1-1.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T17:34:23.256 INFO:teuthology.orchestra.run.smithi055.stdout: toolbox-0.0.99.3-0.6.module+el8.6.0+16771+28dfca77.x86_64 2023-09-23T17:34:23.256 INFO:teuthology.orchestra.run.smithi055.stdout: udica-0.2.6-4.module+el8.6.0+16771+28dfca77.noarch 2023-09-23T17:34:23.256 INFO:teuthology.orchestra.run.smithi055.stdout: yajl-2.1.0-10.el8.x86_64 2023-09-23T17:34:23.256 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:34:23.256 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:34:23.537 DEBUG:teuthology.parallel:result is None 2023-09-23T17:34:23.537 INFO:teuthology.run_tasks:Running task install... 2023-09-23T17:34:23.546 DEBUG:teuthology.task.install:project ceph 2023-09-23T17:34:23.546 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T17:34:23.546 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T17:34:23.546 INFO:teuthology.task.install:Using flavor: default 2023-09-23T17:34:23.552 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-09-23T17:34:23.552 INFO:teuthology.task.install:extra packages: [] 2023-09-23T17:34:23.553 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T17:34:23.553 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T17:34:23.553 INFO:teuthology.packaging:ref: None 2023-09-23T17:34:23.553 INFO:teuthology.packaging:tag: None 2023-09-23T17:34:23.553 INFO:teuthology.packaging:branch: quincy 2023-09-23T17:34:23.553 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:34:23.554 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T17:34:23.556 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7', 'tag': None, 'wait_for_package': False} 2023-09-23T17:34:23.556 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T17:34:23.556 INFO:teuthology.packaging:ref: None 2023-09-23T17:34:23.556 INFO:teuthology.packaging:tag: None 2023-09-23T17:34:23.556 INFO:teuthology.packaging:branch: quincy 2023-09-23T17:34:23.556 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:34:23.556 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T17:34:23.957 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T17:34:23.957 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T17:34:23.962 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/ 2023-09-23T17:34:23.963 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1123.g61431f6b 2023-09-23T17:34:24.068 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T17:34:24.068 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:34:24.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T17:34:24.073 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/61431f6ba2752649b506867ec856b4a6236f90dd/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-09-23T17:34:24.073 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:34:24.073 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-09-23T17:34:24.101 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T17:34:24.101 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T17:34:24.101 INFO:teuthology.packaging:ref: None 2023-09-23T17:34:24.101 INFO:teuthology.packaging:tag: None 2023-09-23T17:34:24.101 INFO:teuthology.packaging:branch: quincy 2023-09-23T17:34:24.101 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:34:24.101 DEBUG:teuthology.orchestra.run.smithi099:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T17:34:24.104 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-09-23T17:34:24.104 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T17:34:24.104 INFO:teuthology.packaging:ref: None 2023-09-23T17:34:24.104 INFO:teuthology.packaging:tag: None 2023-09-23T17:34:24.104 INFO:teuthology.packaging:branch: quincy 2023-09-23T17:34:24.104 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:34:24.104 DEBUG:teuthology.orchestra.run.smithi055:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-09-23T17:34:24.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T17:34:24.175 DEBUG:teuthology.orchestra.run.smithi055:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-09-23T17:34:24.251 DEBUG:teuthology.orchestra.run.smithi099:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T17:34:24.256 DEBUG:teuthology.orchestra.run.smithi055:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-09-23T17:34:24.319 INFO:teuthology.orchestra.run.smithi099.stdout:check_obsoletes = 1 2023-09-23T17:34:24.321 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum clean all 2023-09-23T17:34:24.325 INFO:teuthology.orchestra.run.smithi055.stdout:check_obsoletes = 1 2023-09-23T17:34:24.327 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum clean all 2023-09-23T17:34:24.693 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:34:24.724 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:34:24.972 INFO:teuthology.orchestra.run.smithi099.stdout:44 files removed 2023-09-23T17:34:25.000 INFO:teuthology.orchestra.run.smithi055.stdout:44 files removed 2023-09-23T17:34:25.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-radosgw 2023-09-23T17:34:25.048 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-radosgw 2023-09-23T17:34:25.349 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:34:25.398 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:34:26.241 INFO:teuthology.orchestra.run.smithi055.stdout:ceph packages for x86_64 185 kB/s | 78 kB 00:00 2023-09-23T17:34:26.303 INFO:teuthology.orchestra.run.smithi099.stdout:ceph packages for x86_64 148 kB/s | 78 kB 00:00 2023-09-23T17:34:26.591 INFO:teuthology.orchestra.run.smithi055.stdout:ceph noarch packages 29 kB/s | 9.1 kB 00:00 2023-09-23T17:34:26.645 INFO:teuthology.orchestra.run.smithi099.stdout:ceph noarch packages 29 kB/s | 9.1 kB 00:00 2023-09-23T17:34:26.879 INFO:teuthology.orchestra.run.smithi099.stdout:ceph source packages 6.8 kB/s | 1.5 kB 00:00 2023-09-23T17:34:26.942 INFO:teuthology.orchestra.run.smithi055.stdout:ceph source packages 4.4 kB/s | 1.5 kB 00:00 2023-09-23T17:34:27.080 INFO:teuthology.orchestra.run.smithi099.stdout:Copr repo for python3-asyncssh owned by ceph 19 kB/s | 3.5 kB 00:00 2023-09-23T17:34:27.142 INFO:teuthology.orchestra.run.smithi055.stdout:Copr repo for python3-asyncssh owned by ceph 19 kB/s | 3.5 kB 00:00 2023-09-23T17:34:29.138 INFO:teuthology.orchestra.run.smithi055.stdout:Extra Packages for Enterprise Linux 8.0 MB/s | 16 MB 00:01 2023-09-23T17:34:32.768 INFO:teuthology.orchestra.run.smithi055.stdout:lab-extras 616 kB/s | 24 kB 00:00 2023-09-23T17:34:33.728 INFO:teuthology.orchestra.run.smithi055.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 60 MB/s | 53 MB 00:00 2023-09-23T17:34:34.566 INFO:teuthology.orchestra.run.smithi099.stdout:Extra Packages for Enterprise Linux 2.1 MB/s | 16 MB 00:07 2023-09-23T17:34:38.097 INFO:teuthology.orchestra.run.smithi099.stdout:lab-extras 437 kB/s | 24 kB 00:00 2023-09-23T17:34:38.998 INFO:teuthology.orchestra.run.smithi099.stdout:Red Hat Enterprise Linux 8 for x86_64 - BaseOS 64 MB/s | 53 MB 00:00 2023-09-23T17:34:42.371 INFO:teuthology.orchestra.run.smithi055.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 56 MB/s | 47 MB 00:00 2023-09-23T17:34:47.365 INFO:teuthology.orchestra.run.smithi099.stdout:Red Hat Enterprise Linux 8 for x86_64 - AppStre 57 MB/s | 47 MB 00:00 2023-09-23T17:34:54.241 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:34:54.243 INFO:teuthology.orchestra.run.smithi055.stdout:==================================================================================================== 2023-09-23T17:34:54.244 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2023-09-23T17:34:54.244 INFO:teuthology.orchestra.run.smithi055.stdout:==================================================================================================== 2023-09-23T17:34:54.244 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:34:54.244 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T17:34:54.244 INFO:teuthology.orchestra.run.smithi055.stdout:Upgrading: 2023-09-23T17:34:54.244 INFO:teuthology.orchestra.run.smithi055.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T17:34:54.244 INFO:teuthology.orchestra.run.smithi055.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T17:34:54.244 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2023-09-23T17:34:54.244 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T17:34:54.245 INFO:teuthology.orchestra.run.smithi055.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout:==================================================================================================== 2023-09-23T17:34:54.246 INFO:teuthology.orchestra.run.smithi055.stdout:Install 21 Packages 2023-09-23T17:34:54.247 INFO:teuthology.orchestra.run.smithi055.stdout:Upgrade 2 Packages 2023-09-23T17:34:54.247 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:34:54.247 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 63 M 2023-09-23T17:34:54.247 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:34:55.106 INFO:teuthology.orchestra.run.smithi055.stdout:(1/23): ceph-base-17.2.6-1123.g61431f6b.el8.x86 11 MB/s | 5.2 MB 00:00 2023-09-23T17:34:55.139 INFO:teuthology.orchestra.run.smithi055.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 733 kB/s | 24 kB 00:00 2023-09-23T17:34:55.214 INFO:teuthology.orchestra.run.smithi055.stdout:(3/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 10 MB/s | 796 kB 00:00 2023-09-23T17:34:55.273 INFO:teuthology.orchestra.run.smithi055.stdout:(4/23): libradosstriper1-17.2.6-1123.g61431f6b. 8.5 MB/s | 506 kB 00:00 2023-09-23T17:34:55.624 INFO:teuthology.orchestra.run.smithi055.stdout:(5/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_6 16 MB/s | 5.5 MB 00:00 2023-09-23T17:34:55.674 INFO:teuthology.orchestra.run.smithi055.stdout:(6/23): python3-ceph-argparse-17.2.6-1123.g6143 897 kB/s | 45 kB 00:00 2023-09-23T17:34:55.732 INFO:teuthology.orchestra.run.smithi055.stdout:(7/23): python3-ceph-common-17.2.6-1123.g61431f 1.9 MB/s | 115 kB 00:00 2023-09-23T17:34:55.799 INFO:teuthology.orchestra.run.smithi055.stdout:(8/23): python3-cephfs-17.2.6-1123.g61431f6b.el 2.8 MB/s | 190 kB 00:00 2023-09-23T17:34:55.858 INFO:teuthology.orchestra.run.smithi055.stdout:(9/23): python3-rados-17.2.6-1123.g61431f6b.el8 6.2 MB/s | 371 kB 00:00 2023-09-23T17:34:55.933 INFO:teuthology.orchestra.run.smithi055.stdout:(10/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 4.7 MB/s | 358 kB 00:00 2023-09-23T17:34:55.992 INFO:teuthology.orchestra.run.smithi055.stdout:(11/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 1.8 MB/s | 106 kB 00:00 2023-09-23T17:34:56.259 INFO:teuthology.orchestra.run.smithi055.stdout:(12/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.1 MB/s | 306 kB 00:00 2023-09-23T17:34:56.317 INFO:teuthology.orchestra.run.smithi055.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 1.0 MB/s | 59 kB 00:00 2023-09-23T17:34:56.359 INFO:teuthology.orchestra.run.smithi055.stdout:(14/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.8 MB/s | 75 kB 00:00 2023-09-23T17:34:57.060 INFO:teuthology.orchestra.run.smithi055.stdout:(15/23): ceph-common-17.2.6-1123.g61431f6b.el8. 9.7 MB/s | 23 MB 00:02 2023-09-23T17:34:57.093 INFO:teuthology.orchestra.run.smithi055.stdout:(16/23): thrift-0.13.0-2.el8.x86_64.rpm 2.4 MB/s | 1.7 MB 00:00 2023-09-23T17:34:57.160 INFO:teuthology.orchestra.run.smithi055.stdout:(17/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 468 kB/s | 47 kB 00:00 2023-09-23T17:34:57.194 INFO:teuthology.orchestra.run.smithi055.stdout:(18/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.8 MB/s | 259 kB 00:00 2023-09-23T17:34:57.219 INFO:teuthology.orchestra.run.smithi055.stdout:(19/23): python3-prettytable-0.7.2-14.el8.noarc 752 kB/s | 44 kB 00:00 2023-09-23T17:34:57.253 INFO:teuthology.orchestra.run.smithi055.stdout:(20/23): librdkafka-0.11.4-3.el8.x86_64.rpm 5.9 MB/s | 354 kB 00:00 2023-09-23T17:34:57.695 INFO:teuthology.orchestra.run.smithi055.stdout:(21/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8 5.5 MB/s | 17 MB 00:03 2023-09-23T17:34:57.762 INFO:teuthology.orchestra.run.smithi055.stdout:(22/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 7.1 MB/s | 3.6 MB 00:00 2023-09-23T17:34:57.820 INFO:teuthology.orchestra.run.smithi055.stdout:(23/23): librados2-17.2.6-1123.g61431f6b.el8.x8 5.8 MB/s | 3.5 MB 00:00 2023-09-23T17:34:57.824 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:34:57.824 INFO:teuthology.orchestra.run.smithi055.stdout:Total 18 MB/s | 63 MB 00:03 2023-09-23T17:34:57.874 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:34:57.923 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:34:57.923 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:34:58.298 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:34:58.302 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:34:58.619 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:34:58.621 INFO:teuthology.orchestra.run.smithi099.stdout:==================================================================================================== 2023-09-23T17:34:58.621 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-09-23T17:34:58.622 INFO:teuthology.orchestra.run.smithi099.stdout:==================================================================================================== 2023-09-23T17:34:58.622 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:34:58.622 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-radosgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T17:34:58.622 INFO:teuthology.orchestra.run.smithi099.stdout:Upgrading: 2023-09-23T17:34:58.622 INFO:teuthology.orchestra.run.smithi099.stdout: librados2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T17:34:58.623 INFO:teuthology.orchestra.run.smithi099.stdout: librbd1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.6 M 2023-09-23T17:34:58.623 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-09-23T17:34:58.623 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-base x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.2 M 2023-09-23T17:34:58.623 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 23 M 2023-09-23T17:34:58.623 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-selinux x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 24 k 2023-09-23T17:34:58.624 INFO:teuthology.orchestra.run.smithi099.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-09-23T17:34:58.624 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 796 k 2023-09-23T17:34:58.624 INFO:teuthology.orchestra.run.smithi099.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-09-23T17:34:58.624 INFO:teuthology.orchestra.run.smithi099.stdout: librabbitmq x86_64 0.9.0-3.el8 rhel-8-for-x86_64-baseos-rpms 47 k 2023-09-23T17:34:58.624 INFO:teuthology.orchestra.run.smithi099.stdout: libradosstriper1 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 506 k 2023-09-23T17:34:58.625 INFO:teuthology.orchestra.run.smithi099.stdout: librdkafka x86_64 0.11.4-3.el8 rhel-8-for-x86_64-appstream-rpms 354 k 2023-09-23T17:34:58.625 INFO:teuthology.orchestra.run.smithi099.stdout: librgw2 x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 5.5 M 2023-09-23T17:34:58.625 INFO:teuthology.orchestra.run.smithi099.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-09-23T17:34:58.625 INFO:teuthology.orchestra.run.smithi099.stdout: lttng-ust x86_64 2.8.1-11.el8 rhel-8-for-x86_64-appstream-rpms 259 k 2023-09-23T17:34:58.625 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-argparse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 45 k 2023-09-23T17:34:58.625 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-common x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 115 k 2023-09-23T17:34:58.626 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cephfs x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 190 k 2023-09-23T17:34:58.626 INFO:teuthology.orchestra.run.smithi099.stdout: python3-prettytable noarch 0.7.2-14.el8 rhel-8-for-x86_64-appstream-rpms 44 k 2023-09-23T17:34:58.626 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rados x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 371 k 2023-09-23T17:34:58.626 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 358 k 2023-09-23T17:34:58.626 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rgw x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 106 k 2023-09-23T17:34:58.627 INFO:teuthology.orchestra.run.smithi099.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-09-23T17:34:58.627 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:34:58.627 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:34:58.627 INFO:teuthology.orchestra.run.smithi099.stdout:==================================================================================================== 2023-09-23T17:34:58.627 INFO:teuthology.orchestra.run.smithi099.stdout:Install 21 Packages 2023-09-23T17:34:58.627 INFO:teuthology.orchestra.run.smithi099.stdout:Upgrade 2 Packages 2023-09-23T17:34:58.628 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:34:58.628 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 63 M 2023-09-23T17:34:58.629 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:34:59.395 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:34:59.742 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T17:35:00.116 INFO:teuthology.orchestra.run.smithi099.stdout:(1/23): ceph-common-17.2.6-1123.g61431f6b.el8.x 18 MB/s | 23 MB 00:01 2023-09-23T17:35:00.147 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T17:35:00.157 INFO:teuthology.orchestra.run.smithi099.stdout:(2/23): ceph-selinux-17.2.6-1123.g61431f6b.el8. 590 kB/s | 24 kB 00:00 2023-09-23T17:35:00.258 INFO:teuthology.orchestra.run.smithi099.stdout:(3/23): libcephfs2-17.2.6-1123.g61431f6b.el8.x8 7.8 MB/s | 796 kB 00:00 2023-09-23T17:35:00.316 INFO:teuthology.orchestra.run.smithi099.stdout:(4/23): libradosstriper1-17.2.6-1123.g61431f6b. 8.5 MB/s | 506 kB 00:00 2023-09-23T17:35:00.342 INFO:teuthology.orchestra.run.smithi055.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T17:35:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T17:35:00.633 INFO:teuthology.orchestra.run.smithi099.stdout:(5/23): librgw2-17.2.6-1123.g61431f6b.el8.x86_6 18 MB/s | 5.5 MB 00:00 2023-09-23T17:35:00.675 INFO:teuthology.orchestra.run.smithi099.stdout:(6/23): python3-ceph-argparse-17.2.6-1123.g6143 1.1 MB/s | 45 kB 00:00 2023-09-23T17:35:00.723 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T17:35:00.725 INFO:teuthology.orchestra.run.smithi099.stdout:(7/23): python3-ceph-common-17.2.6-1123.g61431f 2.3 MB/s | 115 kB 00:00 2023-09-23T17:35:00.784 INFO:teuthology.orchestra.run.smithi099.stdout:(8/23): python3-cephfs-17.2.6-1123.g61431f6b.el 3.2 MB/s | 190 kB 00:00 2023-09-23T17:35:00.851 INFO:teuthology.orchestra.run.smithi099.stdout:(9/23): python3-rados-17.2.6-1123.g61431f6b.el8 5.4 MB/s | 371 kB 00:00 2023-09-23T17:35:00.926 INFO:teuthology.orchestra.run.smithi099.stdout:(10/23): python3-rbd-17.2.6-1123.g61431f6b.el8. 4.7 MB/s | 358 kB 00:00 2023-09-23T17:35:00.964 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T17:35:01.010 INFO:teuthology.orchestra.run.smithi099.stdout:(11/23): ceph-base-17.2.6-1123.g61431f6b.el8.x8 2.3 MB/s | 5.2 MB 00:02 2023-09-23T17:35:01.060 INFO:teuthology.orchestra.run.smithi099.stdout:(12/23): python3-rgw-17.2.6-1123.g61431f6b.el8. 798 kB/s | 106 kB 00:00 2023-09-23T17:35:01.259 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T17:35:01.486 INFO:teuthology.orchestra.run.smithi099.stdout:(13/23): liboath-2.6.2-3.el8.x86_64.rpm 138 kB/s | 59 kB 00:00 2023-09-23T17:35:01.628 INFO:teuthology.orchestra.run.smithi099.stdout:(14/23): libunwind-1.3.1-3.el8.x86_64.rpm 531 kB/s | 75 kB 00:00 2023-09-23T17:35:01.653 INFO:teuthology.orchestra.run.smithi099.stdout:(15/23): gperftools-libs-2.7-9.el8.x86_64.rpm 477 kB/s | 306 kB 00:00 2023-09-23T17:35:01.712 INFO:teuthology.orchestra.run.smithi099.stdout:(16/23): librabbitmq-0.9.0-3.el8.x86_64.rpm 804 kB/s | 47 kB 00:00 2023-09-23T17:35:01.714 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T17:35:01.753 INFO:teuthology.orchestra.run.smithi099.stdout:(17/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 6.1 MB/s | 259 kB 00:00 2023-09-23T17:35:01.795 INFO:teuthology.orchestra.run.smithi099.stdout:(18/23): python3-prettytable-0.7.2-14.el8.noarc 1.0 MB/s | 44 kB 00:00 2023-09-23T17:35:01.837 INFO:teuthology.orchestra.run.smithi099.stdout:(19/23): librdkafka-0.11.4-3.el8.x86_64.rpm 8.3 MB/s | 354 kB 00:00 2023-09-23T17:35:01.926 INFO:teuthology.orchestra.run.smithi055.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T17:35:02.030 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T17:35:02.155 INFO:teuthology.orchestra.run.smithi099.stdout:(20/23): ceph-radosgw-17.2.6-1123.g61431f6b.el8 5.0 MB/s | 17 MB 00:03 2023-09-23T17:35:02.196 INFO:teuthology.orchestra.run.smithi099.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 3.1 MB/s | 1.7 MB 00:00 2023-09-23T17:35:02.259 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T17:35:02.328 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T17:35:02.430 INFO:teuthology.orchestra.run.smithi099.stdout:(22/23): librbd1-17.2.6-1123.g61431f6b.el8.x86_ 13 MB/s | 3.6 MB 00:00 2023-09-23T17:35:02.639 INFO:teuthology.orchestra.run.smithi099.stdout:(23/23): librados2-17.2.6-1123.g61431f6b.el8.x8 4.3 MB/s | 3.5 MB 00:00 2023-09-23T17:35:02.642 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:35:02.643 INFO:teuthology.orchestra.run.smithi099.stdout:Total 16 MB/s | 63 MB 00:04 2023-09-23T17:35:02.661 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T17:35:02.688 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:35:02.732 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:35:02.733 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:35:03.079 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:35:03.083 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:35:03.366 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T17:35:03.517 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T17:35:03.593 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T17:35:03.721 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T17:35:03.904 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T17:35:04.127 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T17:35:04.137 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:35:04.337 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T17:35:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T17:35:04.508 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T17:35:04.581 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T17:35:04.717 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 1/25 2023-09-23T17:35:04.760 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T17:35:04.920 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T17:35:04.944 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T17:35:05.022 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T17:35:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T17:35:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T17:35:05.351 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T17:35:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : liboath-2.6.2-3.el8.x86_64 4/25 2023-09-23T17:35:05.678 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T17:35:06.063 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T17:35:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: Upgrading : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T17:35:06.386 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 6/25 2023-09-23T17:35:06.570 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T17:35:06.621 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 7/25 2023-09-23T17:35:06.940 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librabbitmq-0.9.0-3.el8.x86_64 8/25 2023-09-23T17:35:07.640 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : thrift-0.13.0-2.el8.x86_64 9/25 2023-09-23T17:35:07.794 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T17:35:07.867 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T17:35:07.996 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 11/25 2023-09-23T17:35:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T17:35:08.337 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T17:35:08.522 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 14/25 2023-09-23T17:35:08.669 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T17:35:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 15/25 2023-09-23T17:35:08.870 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-09-23T17:35:09.045 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 17/25 2023-09-23T17:35:09.268 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-09-23T17:35:09.445 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 19/25 2023-09-23T17:35:09.539 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T17:35:09.783 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T17:35:10.386 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T17:35:10.629 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T17:35:10.713 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T17:35:10.841 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T17:35:13.205 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T17:35:13.452 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T17:35:14.018 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 20/25 2023-09-23T17:35:14.282 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T17:35:14.362 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 21/25 2023-09-23T17:35:14.542 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T17:35:30.721 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T17:35:30.722 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /sys 2023-09-23T17:35:30.722 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /proc 2023-09-23T17:35:30.722 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /dev 2023-09-23T17:35:30.722 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /run 2023-09-23T17:35:30.722 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /mnt 2023-09-23T17:35:30.722 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /var/tmp 2023-09-23T17:35:30.722 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /home 2023-09-23T17:35:30.722 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /tmp 2023-09-23T17:35:30.722 INFO:teuthology.orchestra.run.smithi055.stdout:skipping the directory /dev 2023-09-23T17:35:30.722 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:35:32.502 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T17:35:32.564 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T17:35:32.564 INFO:teuthology.orchestra.run.smithi055.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T17:35:32.766 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T17:35:32.766 INFO:teuthology.orchestra.run.smithi055.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T17:35:33.071 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T17:35:33.071 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /sys 2023-09-23T17:35:33.072 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /proc 2023-09-23T17:35:33.072 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /dev 2023-09-23T17:35:33.072 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /run 2023-09-23T17:35:33.072 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /mnt 2023-09-23T17:35:33.072 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /var/tmp 2023-09-23T17:35:33.072 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /home 2023-09-23T17:35:33.072 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /tmp 2023-09-23T17:35:33.072 INFO:teuthology.orchestra.run.smithi099.stdout:skipping the directory /dev 2023-09-23T17:35:33.072 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:35:33.405 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T17:35:33.405 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T17:35:33.405 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T17:35:33.405 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T17:35:33.406 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T17:35:33.406 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T17:35:33.406 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T17:35:33.406 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T17:35:33.406 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T17:35:33.406 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T17:35:33.406 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T17:35:33.406 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T17:35:33.406 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T17:35:33.406 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T17:35:33.407 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T17:35:34.736 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T17:35:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T17:35:34.790 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:53 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:35:34.815 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 23/25 2023-09-23T17:35:34.815 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T17:35:35.007 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-09-23T17:35:35.007 INFO:teuthology.orchestra.run.smithi099.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T17:35:35.598 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T17:35:35.598 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 1/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 3/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 4/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 5/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_6 6/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 7/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8. 8/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x8 9/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 10/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 11/25 2023-09-23T17:35:35.599 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 12/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 13/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 14/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 15/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 16/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 17/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librabbitmq-0.9.0-3.el8.x86_64 18/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 19/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 20/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 21/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 22/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-09-23T17:35:35.600 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 24/25 2023-09-23T17:35:36.905 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-09-23T17:35:36.905 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:00:50 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:35:37.279 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:35:37.494 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:35:37.494 INFO:teuthology.orchestra.run.smithi055.stdout:Upgraded: 2023-09-23T17:35:37.494 INFO:teuthology.orchestra.run.smithi055.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.494 INFO:teuthology.orchestra.run.smithi055.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.494 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.495 INFO:teuthology.orchestra.run.smithi055.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:35:37.496 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:35:37.683 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-test 2023-09-23T17:35:38.055 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:35:38.889 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:00:57 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:35:39.372 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:35:39.560 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:35:39.560 INFO:teuthology.orchestra.run.smithi099.stdout:Upgraded: 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-radosgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-selinux-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: liboath-2.6.2-3.el8.x86_64 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: librabbitmq-0.9.0-3.el8.x86_64 2023-09-23T17:35:39.561 INFO:teuthology.orchestra.run.smithi099.stdout: libradosstriper1-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: librgw2-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: libunwind-1.3.1-3.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-argparse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: python3-ceph-common-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:39.562 INFO:teuthology.orchestra.run.smithi099.stdout: thrift-0.13.0-2.el8.x86_64 2023-09-23T17:35:39.563 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:35:39.563 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:35:39.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-test 2023-09-23T17:35:40.086 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:35:40.413 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:35:40.415 INFO:teuthology.orchestra.run.smithi055.stdout:============================================================================================ 2023-09-23T17:35:40.415 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2023-09-23T17:35:40.415 INFO:teuthology.orchestra.run.smithi055.stdout:============================================================================================ 2023-09-23T17:35:40.415 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:35:40.415 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T17:35:40.415 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2023-09-23T17:35:40.415 INFO:teuthology.orchestra.run.smithi055.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T17:35:40.415 INFO:teuthology.orchestra.run.smithi055.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T17:35:40.415 INFO:teuthology.orchestra.run.smithi055.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T17:35:40.415 INFO:teuthology.orchestra.run.smithi055.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T17:35:40.416 INFO:teuthology.orchestra.run.smithi055.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T17:35:40.416 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:35:40.416 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:35:40.416 INFO:teuthology.orchestra.run.smithi055.stdout:============================================================================================ 2023-09-23T17:35:40.416 INFO:teuthology.orchestra.run.smithi055.stdout:Install 6 Packages 2023-09-23T17:35:40.416 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:35:40.416 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 53 M 2023-09-23T17:35:40.417 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 233 M 2023-09-23T17:35:40.417 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:35:40.447 INFO:teuthology.orchestra.run.smithi055.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 2.3 MB/s | 70 kB 00:00 2023-09-23T17:35:40.514 INFO:teuthology.orchestra.run.smithi055.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.7 MB/s | 323 kB 00:00 2023-09-23T17:35:40.557 INFO:teuthology.orchestra.run.smithi055.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-09-23T17:35:40.615 INFO:teuthology.orchestra.run.smithi055.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 3.4 MB/s | 202 kB 00:00 2023-09-23T17:35:40.773 INFO:teuthology.orchestra.run.smithi055.stdout:(5/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 490 kB/s | 174 kB 00:00 2023-09-23T17:35:40.887 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:00:54 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:35:42.347 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:35:42.348 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================================ 2023-09-23T17:35:42.348 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-09-23T17:35:42.348 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================================ 2023-09-23T17:35:42.348 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-test x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 52 M 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout: jq x86_64 1.6-3.el8 rhel-8-for-x86_64-appstream-rpms 202 k 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout: libcephsqlite x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 174 k 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout: oniguruma x86_64 6.8.2-2.el8 rhel-8-for-x86_64-appstream-rpms 187 k 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout: socat x86_64 1.7.4.1-1.el8 rhel-8-for-x86_64-appstream-rpms 323 k 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout: xmlstarlet x86_64 1.6.1-20.el8 lab-extras 70 k 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout:============================================================================================ 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout:Install 6 Packages 2023-09-23T17:35:42.349 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:35:42.350 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 53 M 2023-09-23T17:35:42.350 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 233 M 2023-09-23T17:35:42.350 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:35:42.391 INFO:teuthology.orchestra.run.smithi099.stdout:(1/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 1.7 MB/s | 70 kB 00:00 2023-09-23T17:35:42.459 INFO:teuthology.orchestra.run.smithi099.stdout:(2/6): socat-1.7.4.1-1.el8.x86_64.rpm 4.7 MB/s | 323 kB 00:00 2023-09-23T17:35:42.509 INFO:teuthology.orchestra.run.smithi099.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 4.4 MB/s | 187 kB 00:00 2023-09-23T17:35:42.551 INFO:teuthology.orchestra.run.smithi099.stdout:(4/6): jq-1.6-3.el8.x86_64.rpm 4.7 MB/s | 202 kB 00:00 2023-09-23T17:35:42.876 INFO:teuthology.orchestra.run.smithi099.stdout:(5/6): libcephsqlite-17.2.6-1123.g61431f6b.el8. 332 kB/s | 174 kB 00:00 2023-09-23T17:35:43.190 INFO:teuthology.orchestra.run.smithi055.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 19 MB/s | 52 MB 00:02 2023-09-23T17:35:43.192 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:35:43.192 INFO:teuthology.orchestra.run.smithi055.stdout:Total 19 MB/s | 53 MB 00:02 2023-09-23T17:35:43.231 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:35:43.254 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:35:43.254 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:35:43.545 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:35:43.549 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:35:44.010 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:35:44.137 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T17:35:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T17:35:44.385 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T17:35:44.530 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T17:35:44.661 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T17:35:44.820 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T17:35:45.201 INFO:teuthology.orchestra.run.smithi099.stdout:(6/6): ceph-test-17.2.6-1123.g61431f6b.el8.x86_ 18 MB/s | 52 MB 00:02 2023-09-23T17:35:45.203 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:35:45.203 INFO:teuthology.orchestra.run.smithi099.stdout:Total 18 MB/s | 53 MB 00:02 2023-09-23T17:35:45.241 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:35:45.262 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:35:45.262 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:35:45.535 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:35:45.539 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:35:45.979 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:35:46.111 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T17:35:46.204 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 1/6 2023-09-23T17:35:46.380 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : jq-1.6-3.el8.x86_64 2/6 2023-09-23T17:35:46.561 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 3/6 2023-09-23T17:35:46.701 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 4/6 2023-09-23T17:35:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T17:35:50.090 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T17:35:50.434 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T17:35:50.656 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T17:35:50.656 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T17:35:50.656 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T17:35:50.656 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T17:35:50.656 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T17:35:50.656 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T17:35:51.746 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 5/6 2023-09-23T17:35:51.871 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T17:35:51.871 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:10 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:35:51.990 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T17:35:52.214 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 6/6 2023-09-23T17:35:52.215 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 1/6 2023-09-23T17:35:52.215 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2/6 2023-09-23T17:35:52.215 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 3/6 2023-09-23T17:35:52.215 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 4/6 2023-09-23T17:35:52.215 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 5/6 2023-09-23T17:35:53.312 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : jq-1.6-3.el8.x86_64 6/6 2023-09-23T17:35:53.312 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:07 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:35:54.405 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:35:54.605 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:35:54.605 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:35:54.605 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:54.605 INFO:teuthology.orchestra.run.smithi055.stdout: jq-1.6-3.el8.x86_64 2023-09-23T17:35:54.605 INFO:teuthology.orchestra.run.smithi055.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:54.605 INFO:teuthology.orchestra.run.smithi055.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T17:35:54.605 INFO:teuthology.orchestra.run.smithi055.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T17:35:54.605 INFO:teuthology.orchestra.run.smithi055.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T17:35:54.605 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:35:54.606 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:35:54.859 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph 2023-09-23T17:35:55.213 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:35:55.778 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:35:55.963 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:35:55.964 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:35:55.964 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-test-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:55.964 INFO:teuthology.orchestra.run.smithi099.stdout: jq-1.6-3.el8.x86_64 2023-09-23T17:35:55.964 INFO:teuthology.orchestra.run.smithi099.stdout: libcephsqlite-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:35:55.964 INFO:teuthology.orchestra.run.smithi099.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-09-23T17:35:55.964 INFO:teuthology.orchestra.run.smithi099.stdout: socat-1.7.4.1-1.el8.x86_64 2023-09-23T17:35:55.964 INFO:teuthology.orchestra.run.smithi099.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-09-23T17:35:55.964 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:35:55.964 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:35:56.105 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph 2023-09-23T17:35:56.124 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:15 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:35:56.438 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:35:57.293 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:11 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:35:57.640 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:35:57.644 INFO:teuthology.orchestra.run.smithi055.stdout:======================================================================================================= 2023-09-23T17:35:57.644 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2023-09-23T17:35:57.644 INFO:teuthology.orchestra.run.smithi055.stdout:======================================================================================================= 2023-09-23T17:35:57.644 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:35:57.644 INFO:teuthology.orchestra.run.smithi055.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T17:35:57.644 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2023-09-23T17:35:57.645 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T17:35:57.645 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T17:35:57.645 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T17:35:57.645 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T17:35:57.645 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T17:35:57.645 INFO:teuthology.orchestra.run.smithi055.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T17:35:57.645 INFO:teuthology.orchestra.run.smithi055.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T17:35:57.645 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T17:35:57.645 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T17:35:57.645 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T17:35:57.646 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T17:35:57.646 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T17:35:57.646 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T17:35:57.646 INFO:teuthology.orchestra.run.smithi055.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T17:35:57.646 INFO:teuthology.orchestra.run.smithi055.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T17:35:57.646 INFO:teuthology.orchestra.run.smithi055.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T17:35:57.646 INFO:teuthology.orchestra.run.smithi055.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T17:35:57.646 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T17:35:57.647 INFO:teuthology.orchestra.run.smithi055.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T17:35:57.647 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T17:35:57.647 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T17:35:57.647 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T17:35:57.647 INFO:teuthology.orchestra.run.smithi055.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T17:35:57.647 INFO:teuthology.orchestra.run.smithi055.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T17:35:57.647 INFO:teuthology.orchestra.run.smithi055.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T17:35:57.647 INFO:teuthology.orchestra.run.smithi055.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T17:35:57.647 INFO:teuthology.orchestra.run.smithi055.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T17:35:57.647 INFO:teuthology.orchestra.run.smithi055.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T17:35:57.648 INFO:teuthology.orchestra.run.smithi055.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T17:35:57.648 INFO:teuthology.orchestra.run.smithi055.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T17:35:57.648 INFO:teuthology.orchestra.run.smithi055.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T17:35:57.648 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:35:57.648 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:35:57.648 INFO:teuthology.orchestra.run.smithi055.stdout:======================================================================================================= 2023-09-23T17:35:57.648 INFO:teuthology.orchestra.run.smithi055.stdout:Install 32 Packages 2023-09-23T17:35:57.648 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:35:57.649 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 28 M 2023-09-23T17:35:57.649 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 110 M 2023-09-23T17:35:57.649 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:35:57.924 INFO:teuthology.orchestra.run.smithi055.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 27 kB/s | 6.4 kB 00:00 2023-09-23T17:35:58.049 INFO:teuthology.orchestra.run.smithi055.stdout:(2/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86_ 4.3 MB/s | 1.6 MB 00:00 2023-09-23T17:35:58.750 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:35:58.763 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================================= 2023-09-23T17:35:58.763 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-09-23T17:35:58.763 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================================= 2023-09-23T17:35:58.763 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:35:58.763 INFO:teuthology.orchestra.run.smithi099.stdout: ceph x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 6.4 k 2023-09-23T17:35:58.763 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-09-23T17:35:58.763 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mds x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 2.3 M 2023-09-23T17:35:58.763 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 1.6 M 2023-09-23T17:35:58.763 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 238 k 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mon x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.9 M 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-osd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 17 M 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cffi x86_64 1.11.5-5.el8 rhel-8-for-x86_64-baseos-rpms 238 k 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cryptography x86_64 3.2.1-5.el8 rhel-8-for-x86_64-baseos-rpms 559 k 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-09-23T17:35:58.764 INFO:teuthology.orchestra.run.smithi099.stdout: python3-mako noarch 1.0.6-13.el8 rhel-8-for-x86_64-appstream-rpms 157 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-markupsafe x86_64 0.23-19.el8 rhel-8-for-x86_64-appstream-rpms 39 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 rhel-8-for-x86_64-appstream-rpms 103 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pycparser noarch 2.14-14.el8 rhel-8-for-x86_64-baseos-rpms 109 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pytz noarch 2017.2-9.el8 rhel-8-for-x86_64-appstream-rpms 54 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-09-23T17:35:58.765 INFO:teuthology.orchestra.run.smithi099.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout: python3-werkzeug noarch 0.12.2-4.el8 rhel-8-for-x86_64-appstream-rpms 457 k 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================================= 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout:Install 32 Packages 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 28 M 2023-09-23T17:35:58.766 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 110 M 2023-09-23T17:35:58.767 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:35:58.858 INFO:teuthology.orchestra.run.smithi055.stdout:(3/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86_ 4.2 MB/s | 3.9 MB 00:00 2023-09-23T17:35:59.066 INFO:teuthology.orchestra.run.smithi055.stdout:(4/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86_ 16 MB/s | 17 MB 00:01 2023-09-23T17:35:59.117 INFO:teuthology.orchestra.run.smithi055.stdout:(5/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 1.6 MB/s | 2.3 MB 00:01 2023-09-23T17:35:59.126 INFO:teuthology.orchestra.run.smithi099.stdout:(1/32): ceph-17.2.6-1123.g61431f6b.el8.x86_64.r 39 kB/s | 6.4 kB 00:00 2023-09-23T17:35:59.142 INFO:teuthology.orchestra.run.smithi055.stdout:(6/32): ceph-mgr-modules-core-17.2.6-1123.g6143 840 kB/s | 238 kB 00:00 2023-09-23T17:35:59.384 INFO:teuthology.orchestra.run.smithi055.stdout:(7/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 138 kB/s | 44 kB 00:00 2023-09-23T17:35:59.460 INFO:teuthology.orchestra.run.smithi099.stdout:(2/32): ceph-mon-17.2.6-1123.g61431f6b.el8.x86_ 12 MB/s | 3.9 MB 00:00 2023-09-23T17:35:59.492 INFO:teuthology.orchestra.run.smithi055.stdout:(8/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 500 kB/s | 173 kB 00:00 2023-09-23T17:35:59.501 INFO:teuthology.orchestra.run.smithi099.stdout:(3/32): ceph-mds-17.2.6-1123.g61431f6b.el8.x86_ 4.2 MB/s | 2.3 MB 00:00 2023-09-23T17:35:59.526 INFO:teuthology.orchestra.run.smithi055.stdout:(9/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 489 kB/s | 185 kB 00:00 2023-09-23T17:35:59.559 INFO:teuthology.orchestra.run.smithi055.stdout:(10/32): python3-jaraco-6.2-6.el8.noarch.rpm 166 kB/s | 11 kB 00:00 2023-09-23T17:35:59.593 INFO:teuthology.orchestra.run.smithi055.stdout:(11/32): python3-jaraco-functools-2.0-4.el8.noa 263 kB/s | 18 kB 00:00 2023-09-23T17:35:59.602 INFO:teuthology.orchestra.run.smithi099.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1123.g6143 2.3 MB/s | 238 kB 00:00 2023-09-23T17:35:59.635 INFO:teuthology.orchestra.run.smithi055.stdout:(12/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.5 MB/s | 384 kB 00:00 2023-09-23T17:35:59.652 INFO:teuthology.orchestra.run.smithi055.stdout:(13/32): python3-logutils-0.3.5-11.el8.noarch.r 528 kB/s | 49 kB 00:00 2023-09-23T17:35:59.669 INFO:teuthology.orchestra.run.smithi055.stdout:(14/32): python3-more-itertools-7.2.0-3.el8.noa 777 kB/s | 59 kB 00:00 2023-09-23T17:35:59.744 INFO:teuthology.orchestra.run.smithi055.stdout:(15/32): python3-pecan-1.3.2-9.el8.noarch.rpm 2.5 MB/s | 283 kB 00:00 2023-09-23T17:35:59.761 INFO:teuthology.orchestra.run.smithi055.stdout:(16/32): python3-portend-2.6-1.el8.noarch.rpm 143 kB/s | 16 kB 00:00 2023-09-23T17:35:59.812 INFO:teuthology.orchestra.run.smithi055.stdout:(17/32): python3-simplegeneric-0.8.1-17.el8.noa 131 kB/s | 19 kB 00:00 2023-09-23T17:35:59.829 INFO:teuthology.orchestra.run.smithi055.stdout:(18/32): python3-singledispatch-3.4.0.3-18.el8. 284 kB/s | 24 kB 00:00 2023-09-23T17:35:59.836 INFO:teuthology.orchestra.run.smithi099.stdout:(5/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rpm 187 kB/s | 44 kB 00:00 2023-09-23T17:35:59.854 INFO:teuthology.orchestra.run.smithi055.stdout:(19/32): python3-tempora-1.14.1-5.el8.noarch.rp 317 kB/s | 29 kB 00:00 2023-09-23T17:35:59.896 INFO:teuthology.orchestra.run.smithi055.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 328 kB/s | 27 kB 00:00 2023-09-23T17:35:59.938 INFO:teuthology.orchestra.run.smithi055.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.3 MB/s | 255 kB 00:00 2023-09-23T17:35:59.971 INFO:teuthology.orchestra.run.smithi055.stdout:(22/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.1 MB/s | 251 kB 00:00 2023-09-23T17:35:59.989 INFO:teuthology.orchestra.run.smithi055.stdout:(23/32): python3-webtest-2.0.33-1.el8.noarch.rp 927 kB/s | 85 kB 00:00 2023-09-23T17:36:00.011 INFO:teuthology.orchestra.run.smithi099.stdout:(6/32): python3-beautifulsoup4-4.6.3-2.el8.1.no 1.0 MB/s | 185 kB 00:00 2023-09-23T17:36:00.014 INFO:teuthology.orchestra.run.smithi055.stdout:(24/32): python3-zc-lockfile-2.0-2.el8.noarch.r 304 kB/s | 23 kB 00:00 2023-09-23T17:36:00.081 INFO:teuthology.orchestra.run.smithi055.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 1.0 MB/s | 109 kB 00:00 2023-09-23T17:36:00.106 INFO:teuthology.orchestra.run.smithi055.stdout:(26/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 3.1 MB/s | 238 kB 00:00 2023-09-23T17:36:00.119 INFO:teuthology.orchestra.run.smithi099.stdout:(7/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 1.6 MB/s | 173 kB 00:00 2023-09-23T17:36:00.123 INFO:teuthology.orchestra.run.smithi055.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 1.3 MB/s | 54 kB 00:00 2023-09-23T17:36:00.140 INFO:teuthology.orchestra.run.smithi055.stdout:(28/32): python3-markupsafe-0.23-19.el8.x86_64. 1.1 MB/s | 39 kB 00:00 2023-09-23T17:36:00.165 INFO:teuthology.orchestra.run.smithi055.stdout:(29/32): python3-cryptography-3.2.1-5.el8.x86_6 3.1 MB/s | 559 kB 00:00 2023-09-23T17:36:00.190 INFO:teuthology.orchestra.run.smithi055.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 2.3 MB/s | 157 kB 00:00 2023-09-23T17:36:00.224 INFO:teuthology.orchestra.run.smithi055.stdout:(31/32): python3-werkzeug-0.12.2-4.el8.noarch.r 5.3 MB/s | 457 kB 00:00 2023-09-23T17:36:00.249 INFO:teuthology.orchestra.run.smithi055.stdout:(32/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.2 MB/s | 103 kB 00:00 2023-09-23T17:36:00.254 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:36:00.254 INFO:teuthology.orchestra.run.smithi055.stdout:Total 11 MB/s | 28 MB 00:02 2023-09-23T17:36:00.295 INFO:teuthology.orchestra.run.smithi099.stdout:(8/32): ceph-osd-17.2.6-1123.g61431f6b.el8.x86_ 20 MB/s | 17 MB 00:00 2023-09-23T17:36:00.320 INFO:teuthology.orchestra.run.smithi099.stdout:(9/32): python3-cherrypy-18.4.0-1.el8.noarch.rp 1.9 MB/s | 384 kB 00:00 2023-09-23T17:36:00.350 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:36:00.387 INFO:teuthology.orchestra.run.smithi099.stdout:(10/32): python3-jaraco-functools-2.0-4.el8.noa 264 kB/s | 18 kB 00:00 2023-09-23T17:36:00.390 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:36:00.390 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:36:00.429 INFO:teuthology.orchestra.run.smithi099.stdout:(11/32): ceph-mgr-17.2.6-1123.g61431f6b.el8.x86 1.1 MB/s | 1.6 MB 00:01 2023-09-23T17:36:00.454 INFO:teuthology.orchestra.run.smithi099.stdout:(12/32): python3-logutils-0.3.5-11.el8.noarch.r 733 kB/s | 49 kB 00:00 2023-09-23T17:36:00.479 INFO:teuthology.orchestra.run.smithi099.stdout:(13/32): python3-jaraco-6.2-6.el8.noarch.rpm 60 kB/s | 11 kB 00:00 2023-09-23T17:36:00.530 INFO:teuthology.orchestra.run.smithi099.stdout:(14/32): python3-pecan-1.3.2-9.el8.noarch.rpm 3.7 MB/s | 283 kB 00:00 2023-09-23T17:36:00.564 INFO:teuthology.orchestra.run.smithi099.stdout:(15/32): python3-portend-2.6-1.el8.noarch.rpm 196 kB/s | 16 kB 00:00 2023-09-23T17:36:00.597 INFO:teuthology.orchestra.run.smithi099.stdout:(16/32): python3-simplegeneric-0.8.1-17.el8.noa 282 kB/s | 19 kB 00:00 2023-09-23T17:36:00.609 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:36:00.613 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:36:00.648 INFO:teuthology.orchestra.run.smithi099.stdout:(17/32): python3-singledispatch-3.4.0.3-18.el8. 285 kB/s | 24 kB 00:00 2023-09-23T17:36:00.673 INFO:teuthology.orchestra.run.smithi099.stdout:(18/32): python3-tempora-1.14.1-5.el8.noarch.rp 386 kB/s | 29 kB 00:00 2023-09-23T17:36:00.698 INFO:teuthology.orchestra.run.smithi099.stdout:(19/32): python3-more-itertools-7.2.0-3.el8.noa 218 kB/s | 59 kB 00:00 2023-09-23T17:36:00.732 INFO:teuthology.orchestra.run.smithi099.stdout:(20/32): python3-trustme-0.6.0-4.el8.noarch.rpm 329 kB/s | 27 kB 00:00 2023-09-23T17:36:00.757 INFO:teuthology.orchestra.run.smithi099.stdout:(21/32): python3-waitress-1.4.3-1.el8.noarch.rp 3.0 MB/s | 255 kB 00:00 2023-09-23T17:36:00.824 INFO:teuthology.orchestra.run.smithi099.stdout:(22/32): python3-zc-lockfile-2.0-2.el8.noarch.r 345 kB/s | 23 kB 00:00 2023-09-23T17:36:00.858 INFO:teuthology.orchestra.run.smithi099.stdout:(23/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.5 MB/s | 251 kB 00:00 2023-09-23T17:36:00.892 INFO:teuthology.orchestra.run.smithi099.stdout:(24/32): python3-webtest-2.0.33-1.el8.noarch.rp 536 kB/s | 85 kB 00:00 2023-09-23T17:36:00.917 INFO:teuthology.orchestra.run.smithi099.stdout:(25/32): python3-pycparser-2.14-14.el8.noarch.r 1.2 MB/s | 109 kB 00:00 2023-09-23T17:36:00.975 INFO:teuthology.orchestra.run.smithi099.stdout:(26/32): python3-cryptography-3.2.1-5.el8.x86_6 4.9 MB/s | 559 kB 00:00 2023-09-23T17:36:01.001 INFO:teuthology.orchestra.run.smithi099.stdout:(27/32): python3-pytz-2017.2-9.el8.noarch.rpm 747 kB/s | 54 kB 00:00 2023-09-23T17:36:01.026 INFO:teuthology.orchestra.run.smithi099.stdout:(28/32): python3-cffi-1.11.5-5.el8.x86_64.rpm 1.7 MB/s | 238 kB 00:00 2023-09-23T17:36:01.051 INFO:teuthology.orchestra.run.smithi099.stdout:(29/32): python3-markupsafe-0.23-19.el8.x86_64. 513 kB/s | 39 kB 00:00 2023-09-23T17:36:01.102 INFO:teuthology.orchestra.run.smithi099.stdout:(30/32): python3-mako-1.0.6-13.el8.noarch.rpm 1.5 MB/s | 157 kB 00:00 2023-09-23T17:36:01.127 INFO:teuthology.orchestra.run.smithi099.stdout:(31/32): python3-pyOpenSSL-19.0.0-1.el8.noarch. 1.3 MB/s | 103 kB 00:00 2023-09-23T17:36:01.152 INFO:teuthology.orchestra.run.smithi099.stdout:(32/32): python3-werkzeug-0.12.2-4.el8.noarch.r 3.6 MB/s | 457 kB 00:00 2023-09-23T17:36:01.156 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:36:01.156 INFO:teuthology.orchestra.run.smithi099.stdout:Total 12 MB/s | 28 MB 00:02 2023-09-23T17:36:01.245 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:36:01.281 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:36:01.281 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:36:01.485 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:36:01.489 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:36:01.850 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:36:02.062 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T17:36:02.324 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T17:36:02.548 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T17:36:02.571 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:36:02.749 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T17:36:02.770 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-09-23T17:36:02.987 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T17:36:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-09-23T17:36:03.166 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T17:36:03.469 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 3/32 2023-09-23T17:36:03.532 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T17:36:03.710 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 4/32 2023-09-23T17:36:03.894 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T17:36:03.910 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-mako-1.0.6-13.el8.noarch 5/32 2023-09-23T17:36:04.077 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pytz-2017.2-9.el8.noarch 6/32 2023-09-23T17:36:04.181 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T17:36:04.283 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 7/32 2023-09-23T17:36:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T17:36:04.526 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T17:36:04.620 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cffi-1.11.5-5.el8.x86_64 8/32 2023-09-23T17:36:04.744 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T17:36:04.901 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cryptography-3.2.1-5.el8.x86_64 9/32 2023-09-23T17:36:05.011 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T17:36:05.047 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 10/32 2023-09-23T17:36:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T17:36:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 11/32 2023-09-23T17:36:05.372 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T17:36:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 12/32 2023-09-23T17:36:05.572 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T17:36:05.655 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 13/32 2023-09-23T17:36:05.854 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 14/32 2023-09-23T17:36:06.035 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 15/32 2023-09-23T17:36:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 16/32 2023-09-23T17:36:06.359 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T17:36:06.413 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 17/32 2023-09-23T17:36:06.478 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T17:36:06.557 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 18/32 2023-09-23T17:36:06.672 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T17:36:06.800 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 19/32 2023-09-23T17:36:06.892 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T17:36:07.014 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 20/32 2023-09-23T17:36:07.086 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T17:36:07.207 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-09-23T17:36:07.365 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T17:36:07.505 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-09-23T17:36:07.601 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T17:36:07.733 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-09-23T17:36:07.763 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T17:36:07.912 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 24/32 2023-09-23T17:36:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T17:36:08.215 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 25/32 2023-09-23T17:36:08.351 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T17:36:08.689 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-09-23T17:36:08.699 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T17:36:09.036 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 27/32 2023-09-23T17:36:09.230 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T17:36:09.470 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T17:36:10.801 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T17:36:10.944 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 28/32 2023-09-23T17:36:11.005 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T17:36:11.136 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T17:36:11.421 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T17:36:11.528 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 29/32 2023-09-23T17:36:11.575 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T17:36:11.664 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T17:36:11.804 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T17:36:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 30/32 2023-09-23T17:36:12.047 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T17:36:12.064 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T17:36:12.178 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T17:36:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T17:36:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 31/32 2023-09-23T17:36:12.499 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T17:36:12.529 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T17:36:12.529 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T17:36:12.529 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T17:36:12.529 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T17:36:12.529 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T17:36:12.529 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T17:36:12.530 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T17:36:12.531 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T17:36:12.531 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T17:36:12.531 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T17:36:12.531 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T17:36:12.531 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T17:36:12.531 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T17:36:12.531 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T17:36:12.531 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T17:36:12.531 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T17:36:12.531 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T17:36:12.532 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T17:36:12.532 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T17:36:12.532 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T17:36:12.532 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T17:36:12.875 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 32/32 2023-09-23T17:36:12.875 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 1/32 2023-09-23T17:36:12.875 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 3/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 4/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 5/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8. 6/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 7/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 8/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 9/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 10/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 11/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 12/32 2023-09-23T17:36:12.876 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 13/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 14/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 15/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-portend-2.6-1.el8.noarch 16/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 17/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 18/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 19/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 20/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 21/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 22/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 23/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 24/32 2023-09-23T17:36:12.877 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 25/32 2023-09-23T17:36:12.878 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cryptography-3.2.1-5.el8.x86_64 26/32 2023-09-23T17:36:12.878 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cffi-1.11.5-5.el8.x86_64 27/32 2023-09-23T17:36:12.878 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pytz-2017.2-9.el8.noarch 28/32 2023-09-23T17:36:12.878 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 29/32 2023-09-23T17:36:12.878 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-mako-1.0.6-13.el8.noarch 30/32 2023-09-23T17:36:12.879 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 31/32 2023-09-23T17:36:13.925 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T17:36:13.926 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:27 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:36:14.570 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 32/32 2023-09-23T17:36:14.570 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:33 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:36:16.878 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:36:17.069 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:17.069 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:36:17.069 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:17.069 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:17.069 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:17.069 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T17:36:17.070 INFO:teuthology.orchestra.run.smithi099.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T17:36:17.071 INFO:teuthology.orchestra.run.smithi099.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T17:36:17.071 INFO:teuthology.orchestra.run.smithi099.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T17:36:17.071 INFO:teuthology.orchestra.run.smithi099.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T17:36:17.071 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T17:36:17.071 INFO:teuthology.orchestra.run.smithi099.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T17:36:17.071 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T17:36:17.071 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T17:36:17.071 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T17:36:17.071 INFO:teuthology.orchestra.run.smithi099.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T17:36:17.071 INFO:teuthology.orchestra.run.smithi099.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T17:36:17.072 INFO:teuthology.orchestra.run.smithi099.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T17:36:17.072 INFO:teuthology.orchestra.run.smithi099.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T17:36:17.072 INFO:teuthology.orchestra.run.smithi099.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T17:36:17.072 INFO:teuthology.orchestra.run.smithi099.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T17:36:17.072 INFO:teuthology.orchestra.run.smithi099.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T17:36:17.072 INFO:teuthology.orchestra.run.smithi099.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T17:36:17.072 INFO:teuthology.orchestra.run.smithi099.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T17:36:17.072 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:17.072 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:36:17.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-base 2023-09-23T17:36:17.373 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:36:17.575 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:17.579 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:17.580 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:36:17.580 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:17.580 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mds-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:17.580 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:17.580 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-modules-core-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:36:17.580 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mon-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:17.580 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-osd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:17.580 INFO:teuthology.orchestra.run.smithi055.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-09-23T17:36:17.580 INFO:teuthology.orchestra.run.smithi055.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-09-23T17:36:17.581 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cffi-1.11.5-5.el8.x86_64 2023-09-23T17:36:17.581 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-09-23T17:36:17.581 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-09-23T17:36:17.581 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cryptography-3.2.1-5.el8.x86_64 2023-09-23T17:36:17.581 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-6.2-6.el8.noarch 2023-09-23T17:36:17.581 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-09-23T17:36:17.581 INFO:teuthology.orchestra.run.smithi055.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-09-23T17:36:17.581 INFO:teuthology.orchestra.run.smithi055.stdout: python3-mako-1.0.6-13.el8.noarch 2023-09-23T17:36:17.581 INFO:teuthology.orchestra.run.smithi055.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-09-23T17:36:17.581 INFO:teuthology.orchestra.run.smithi055.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-09-23T17:36:17.582 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-09-23T17:36:17.582 INFO:teuthology.orchestra.run.smithi055.stdout: python3-portend-2.6-1.el8.noarch 2023-09-23T17:36:17.582 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-09-23T17:36:17.582 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pycparser-2.14-14.el8.noarch 2023-09-23T17:36:17.582 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pytz-2017.2-9.el8.noarch 2023-09-23T17:36:17.582 INFO:teuthology.orchestra.run.smithi055.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-09-23T17:36:17.582 INFO:teuthology.orchestra.run.smithi055.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-09-23T17:36:17.582 INFO:teuthology.orchestra.run.smithi055.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-09-23T17:36:17.582 INFO:teuthology.orchestra.run.smithi055.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-09-23T17:36:17.582 INFO:teuthology.orchestra.run.smithi055.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-09-23T17:36:17.583 INFO:teuthology.orchestra.run.smithi055.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-09-23T17:36:17.583 INFO:teuthology.orchestra.run.smithi055.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-09-23T17:36:17.583 INFO:teuthology.orchestra.run.smithi055.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-09-23T17:36:17.583 INFO:teuthology.orchestra.run.smithi055.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-09-23T17:36:17.583 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:17.583 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:36:19.209 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:33 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:36:19.455 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-base 2023-09-23T17:36:19.825 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:20.569 INFO:teuthology.orchestra.run.smithi099.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:36:20.679 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:36:20.680 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2023-09-23T17:36:20.680 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:36:21.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install cephadm 2023-09-23T17:36:21.321 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:40 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:36:21.648 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:22.741 INFO:teuthology.orchestra.run.smithi055.stdout:Package ceph-base-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:36:22.863 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:36:22.864 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2023-09-23T17:36:22.865 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:36:22.996 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install cephadm 2023-09-23T17:36:23.157 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:37 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:36:23.365 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:24.565 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:36:24.566 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:36:24.566 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-09-23T17:36:24.566 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:36:24.566 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:36:24.566 INFO:teuthology.orchestra.run.smithi099.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T17:36:24.566 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:24.566 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:36:24.566 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:36:24.567 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2023-09-23T17:36:24.567 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:24.567 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 80 k 2023-09-23T17:36:24.567 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 363 k 2023-09-23T17:36:24.567 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:36:24.734 INFO:teuthology.orchestra.run.smithi099.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 481 kB/s | 80 kB 00:00 2023-09-23T17:36:24.734 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:36:24.734 INFO:teuthology.orchestra.run.smithi099.stdout:Total 477 kB/s | 80 kB 00:00 2023-09-23T17:36:24.735 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:36:24.743 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:36:24.743 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:36:24.745 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:36:24.748 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:36:24.789 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:36:25.003 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:44 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:36:26.414 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T17:36:26.567 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T17:36:26.592 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:36:26.593 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:36:26.593 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2023-09-23T17:36:26.593 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:36:26.593 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:36:26.593 INFO:teuthology.orchestra.run.smithi055.stdout: cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 80 k 2023-09-23T17:36:26.593 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:26.593 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:36:26.593 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:36:26.593 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2023-09-23T17:36:26.593 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:26.594 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 80 k 2023-09-23T17:36:26.594 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 363 k 2023-09-23T17:36:26.594 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:36:26.732 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T17:36:26.922 INFO:teuthology.orchestra.run.smithi055.stdout:cephadm-17.2.6-1123.g61431f6b.el8.noarch.rpm 244 kB/s | 80 kB 00:00 2023-09-23T17:36:26.923 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:36:26.923 INFO:teuthology.orchestra.run.smithi055.stdout:Total 243 kB/s | 80 kB 00:00 2023-09-23T17:36:26.923 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:36:26.932 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:36:26.932 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:36:26.934 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:36:26.938 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:36:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:36:27.794 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T17:36:27.794 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:41 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:36:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T17:36:28.922 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T17:36:29.098 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T17:36:30.145 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/1 2023-09-23T17:36:30.145 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:49 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:36:32.037 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:36:32.222 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:32.222 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:36:32.222 INFO:teuthology.orchestra.run.smithi099.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:36:32.222 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:32.222 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:36:32.357 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T17:36:32.715 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:34.300 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:48 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:36:35.674 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:36:35.674 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:36:35.674 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repo Size 2023-09-23T17:36:35.674 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:36:35.675 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:36:35.675 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T17:36:35.675 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:35.675 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:36:35.675 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:36:35.675 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2023-09-23T17:36:35.675 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:35.676 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 161 k 2023-09-23T17:36:35.676 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 490 k 2023-09-23T17:36:35.676 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:36:35.883 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:36:36.086 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:36.086 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:36:36.086 INFO:teuthology.orchestra.run.smithi055.stdout: cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:36:36.086 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:36.087 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:36:36.181 INFO:teuthology.orchestra.run.smithi099.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 319 kB/s | 161 kB 00:00 2023-09-23T17:36:36.181 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:36:36.182 INFO:teuthology.orchestra.run.smithi099.stdout:Total 318 kB/s | 161 kB 00:00 2023-09-23T17:36:36.182 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:36:36.192 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:36:36.193 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:36:36.226 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-immutable-object-cache 2023-09-23T17:36:36.244 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:36:36.248 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:36:36.355 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:36:36.500 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T17:36:36.606 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:36.838 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T17:36:37.516 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:01:56 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:36:37.943 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T17:36:37.944 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:51 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:36:39.076 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:36:39.076 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:36:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repo Size 2023-09-23T17:36:39.076 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:36:39.076 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:36:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 161 k 2023-09-23T17:36:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:39.077 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:36:39.077 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:36:39.077 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2023-09-23T17:36:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:39.077 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 161 k 2023-09-23T17:36:39.078 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 490 k 2023-09-23T17:36:39.078 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:36:39.263 INFO:teuthology.orchestra.run.smithi055.stdout:ceph-immutable-object-cache-17.2.6-1123.g61431f 872 kB/s | 161 kB 00:00 2023-09-23T17:36:39.263 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:36:39.263 INFO:teuthology.orchestra.run.smithi055.stdout:Total 866 kB/s | 161 kB 00:00 2023-09-23T17:36:39.264 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:36:39.275 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:36:39.275 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:36:39.332 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:36:39.336 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:36:39.451 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:36:39.597 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T17:36:40.010 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T17:36:41.118 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:36:41.301 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:41.301 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:36:41.301 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:41.301 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:41.302 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:36:41.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-mgr 2023-09-23T17:36:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b. 1/1 2023-09-23T17:36:41.505 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:02:00 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:36:41.777 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:43.152 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:01:57 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:36:44.445 INFO:teuthology.orchestra.run.smithi099.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:36:44.553 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:36:44.554 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2023-09-23T17:36:44.555 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:36:44.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T17:36:45.008 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:46.273 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:00 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:36:46.548 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:36:46.752 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:46.752 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:36:46.752 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-immutable-object-cache-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:36:46.752 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:46.752 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:36:46.903 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-mgr 2023-09-23T17:36:47.272 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:47.646 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:36:47.647 INFO:teuthology.orchestra.run.smithi099.stdout:=================================================================================================== 2023-09-23T17:36:47.647 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-09-23T17:36:47.647 INFO:teuthology.orchestra.run.smithi099.stdout:=================================================================================================== 2023-09-23T17:36:47.647 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:36:47.647 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T17:36:47.647 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-09-23T17:36:47.647 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T17:36:47.647 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T17:36:47.648 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T17:36:47.648 INFO:teuthology.orchestra.run.smithi099.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T17:36:47.648 INFO:teuthology.orchestra.run.smithi099.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T17:36:47.648 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:47.648 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:36:47.648 INFO:teuthology.orchestra.run.smithi099.stdout:=================================================================================================== 2023-09-23T17:36:47.648 INFO:teuthology.orchestra.run.smithi099.stdout:Install 6 Packages 2023-09-23T17:36:47.648 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:47.649 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 2.0 M 2023-09-23T17:36:47.649 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 8.0 M 2023-09-23T17:36:47.649 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:36:47.954 INFO:teuthology.orchestra.run.smithi099.stdout:(1/6): ceph-prometheus-alerts-17.2.6-1123.g6143 103 kB/s | 14 kB 00:00 2023-09-23T17:36:47.979 INFO:teuthology.orchestra.run.smithi099.stdout:(2/6): ceph-grafana-dashboards-17.2.6-1123.g614 140 kB/s | 23 kB 00:00 2023-09-23T17:36:48.113 INFO:teuthology.orchestra.run.smithi099.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 5.7 MB/s | 1.7 MB 00:00 2023-09-23T17:36:48.213 INFO:teuthology.orchestra.run.smithi099.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 130 kB/s | 34 kB 00:00 2023-09-23T17:36:48.272 INFO:teuthology.orchestra.run.smithi099.stdout:(5/6): python3-jwt-1.6.1-2.el8.noarch.rpm 269 kB/s | 43 kB 00:00 2023-09-23T17:36:48.330 INFO:teuthology.orchestra.run.smithi099.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 560 kB/s | 196 kB 00:00 2023-09-23T17:36:48.333 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:36:48.334 INFO:teuthology.orchestra.run.smithi099.stdout:Total 2.9 MB/s | 2.0 MB 00:00 2023-09-23T17:36:48.345 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:36:48.354 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:36:48.355 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:36:48.411 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:36:48.415 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:36:48.683 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:36:48.839 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T17:36:48.906 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:02:07 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:36:49.019 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T17:36:49.216 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T17:36:49.336 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T17:36:49.647 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T17:36:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T17:36:50.077 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T17:36:50.078 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T17:36:50.078 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T17:36:50.078 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T17:36:50.078 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T17:36:50.078 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T17:36:50.366 INFO:teuthology.orchestra.run.smithi055.stdout:Package ceph-mgr-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:36:50.487 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:36:50.488 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2023-09-23T17:36:50.488 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:36:50.619 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-mgr-dashboard 2023-09-23T17:36:50.986 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:51.102 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T17:36:51.102 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:05 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:36:52.305 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:02:11 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:36:53.865 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:36:53.866 INFO:teuthology.orchestra.run.smithi055.stdout:=================================================================================================== 2023-09-23T17:36:53.866 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2023-09-23T17:36:53.867 INFO:teuthology.orchestra.run.smithi055.stdout:=================================================================================================== 2023-09-23T17:36:53.867 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:36:53.867 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 1.7 M 2023-09-23T17:36:53.867 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2023-09-23T17:36:53.867 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 23 k 2023-09-23T17:36:53.867 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 14 k 2023-09-23T17:36:53.867 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jwt noarch 1.6.1-2.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T17:36:53.867 INFO:teuthology.orchestra.run.smithi055.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-09-23T17:36:53.868 INFO:teuthology.orchestra.run.smithi055.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-09-23T17:36:53.868 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:53.868 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:36:53.868 INFO:teuthology.orchestra.run.smithi055.stdout:=================================================================================================== 2023-09-23T17:36:53.868 INFO:teuthology.orchestra.run.smithi055.stdout:Install 6 Packages 2023-09-23T17:36:53.868 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:36:53.869 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 2.0 M 2023-09-23T17:36:53.869 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 8.0 M 2023-09-23T17:36:53.869 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:36:54.143 INFO:teuthology.orchestra.run.smithi055.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1123.g614 94 kB/s | 23 kB 00:00 2023-09-23T17:36:54.261 INFO:teuthology.orchestra.run.smithi055.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1123.g6143 54 kB/s | 14 kB 00:00 2023-09-23T17:36:54.451 INFO:teuthology.orchestra.run.smithi055.stdout:(3/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 109 kB/s | 34 kB 00:00 2023-09-23T17:36:54.519 INFO:teuthology.orchestra.run.smithi055.stdout:(4/6): python3-jwt-1.6.1-2.el8.noarch.rpm 640 kB/s | 43 kB 00:00 2023-09-23T17:36:54.560 INFO:teuthology.orchestra.run.smithi055.stdout:(5/6): python3-routes-2.4.1-12.el8.noarch.rpm 501 kB/s | 196 kB 00:00 2023-09-23T17:36:54.719 INFO:teuthology.orchestra.run.smithi055.stdout:(6/6): ceph-mgr-dashboard-17.2.6-1123.g61431f6b 2.1 MB/s | 1.7 MB 00:00 2023-09-23T17:36:54.722 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:36:54.723 INFO:teuthology.orchestra.run.smithi055.stdout:Total 2.3 MB/s | 2.0 MB 00:00 2023-09-23T17:36:54.737 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:36:54.748 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:36:54.748 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:36:54.814 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:36:54.818 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:36:55.069 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:36:55.246 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 1/6 2023-09-23T17:36:55.359 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:36:55.420 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 2/6 2023-09-23T17:36:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:55.546 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:36:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:36:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:36:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:36:55.546 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T17:36:55.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T17:36:55.547 INFO:teuthology.orchestra.run.smithi099.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T17:36:55.547 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:55.547 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:36:55.572 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 3/6 2023-09-23T17:36:55.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T17:36:55.693 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 4/6 2023-09-23T17:36:56.014 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 5/6 2023-09-23T17:36:56.014 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:36:56.190 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T17:36:56.433 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 6/6 2023-09-23T17:36:56.433 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8. 1/6 2023-09-23T17:36:56.433 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarc 2/6 2023-09-23T17:36:56.433 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.n 3/6 2023-09-23T17:36:56.433 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 4/6 2023-09-23T17:36:56.433 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 5/6 2023-09-23T17:36:56.939 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:10 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:36:57.512 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 6/6 2023-09-23T17:36:57.512 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:02:16 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:36:58.351 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:36:58.352 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================================================= 2023-09-23T17:36:58.352 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-09-23T17:36:58.352 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================================================= 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:58.353 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:36:58.354 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================================================= 2023-09-23T17:36:58.354 INFO:teuthology.orchestra.run.smithi099.stdout:Install 8 Packages 2023-09-23T17:36:58.354 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:36:58.354 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 35 M 2023-09-23T17:36:58.354 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 210 M 2023-09-23T17:36:58.354 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:36:58.470 INFO:teuthology.orchestra.run.smithi099.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 1.5 MB/s | 171 kB 00:00 2023-09-23T17:36:58.503 INFO:teuthology.orchestra.run.smithi099.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 4.3 MB/s | 644 kB 00:00 2023-09-23T17:36:58.686 INFO:teuthology.orchestra.run.smithi099.stdout:(3/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 20 MB/s | 3.7 MB 00:00 2023-09-23T17:36:58.920 INFO:teuthology.orchestra.run.smithi099.stdout:(4/8): python3-scipy-1.0.0-21.module+el8.5.0+10 30 MB/s | 14 MB 00:00 2023-09-23T17:36:58.945 INFO:teuthology.orchestra.run.smithi099.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 871 kB/s | 225 kB 00:00 2023-09-23T17:36:59.154 INFO:teuthology.orchestra.run.smithi099.stdout:(6/8): ceph-mgr-diskprediction-local-17.2.6-112 9.2 MB/s | 7.4 MB 00:00 2023-09-23T17:36:59.238 INFO:teuthology.orchestra.run.smithi099.stdout:(7/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 15 MB/s | 4.9 MB 00:00 2023-09-23T17:36:59.321 INFO:teuthology.orchestra.run.smithi099.stdout:(8/8): openblas-0.3.15-3.el8.x86_64.rpm 13 MB/s | 4.7 MB 00:00 2023-09-23T17:36:59.322 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:36:59.323 INFO:teuthology.orchestra.run.smithi099.stdout:Total 37 MB/s | 35 MB 00:00 2023-09-23T17:36:59.607 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:36:59.625 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:36:59.626 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:36:59.916 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:36:59.919 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:37:00.450 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:37:00.594 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T17:37:00.611 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:37:00.717 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T17:37:00.817 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:00.817 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:37:00.817 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-grafana-dashboards-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:37:00.817 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-dashboard-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:37:00.817 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-prometheus-alerts-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:37:00.817 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-09-23T17:37:00.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-09-23T17:37:00.818 INFO:teuthology.orchestra.run.smithi055.stdout: python3-routes-2.4.1-12.el8.noarch 2023-09-23T17:37:00.818 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:00.818 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:37:00.877 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T17:37:00.951 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-mgr-diskprediction-local 2023-09-23T17:37:01.321 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:37:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T17:37:01.644 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T17:37:02.205 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:02:21 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:37:02.227 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T17:37:02.402 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T17:37:03.078 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T17:37:03.404 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T17:37:03.626 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T17:37:03.783 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:37:03.785 INFO:teuthology.orchestra.run.smithi055.stdout:======================================================================================================================= 2023-09-23T17:37:03.785 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2023-09-23T17:37:03.785 INFO:teuthology.orchestra.run.smithi055.stdout:======================================================================================================================= 2023-09-23T17:37:03.785 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:37:03.785 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 7.4 M 2023-09-23T17:37:03.785 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2023-09-23T17:37:03.785 INFO:teuthology.orchestra.run.smithi055.stdout: libgfortran x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 644 k 2023-09-23T17:37:03.785 INFO:teuthology.orchestra.run.smithi055.stdout: libquadmath x86_64 8.5.0-10.1.el8_6 rhel-8-for-x86_64-baseos-rpms 171 k 2023-09-23T17:37:03.786 INFO:teuthology.orchestra.run.smithi055.stdout: openblas x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.7 M 2023-09-23T17:37:03.786 INFO:teuthology.orchestra.run.smithi055.stdout: openblas-threads x86_64 0.3.15-3.el8 rhel-8-for-x86_64-appstream-rpms 4.9 M 2023-09-23T17:37:03.786 INFO:teuthology.orchestra.run.smithi055.stdout: python3-numpy x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 3.7 M 2023-09-23T17:37:03.786 INFO:teuthology.orchestra.run.smithi055.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 rhel-8-for-x86_64-appstream-rpms 225 k 2023-09-23T17:37:03.786 INFO:teuthology.orchestra.run.smithi055.stdout: python3-scipy x86_64 1.0.0-21.module+el8.5.0+10916+41bd434d rhel-8-for-x86_64-appstream-rpms 14 M 2023-09-23T17:37:03.786 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:03.786 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:37:03.786 INFO:teuthology.orchestra.run.smithi055.stdout:======================================================================================================================= 2023-09-23T17:37:03.786 INFO:teuthology.orchestra.run.smithi055.stdout:Install 8 Packages 2023-09-23T17:37:03.786 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:03.787 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 35 M 2023-09-23T17:37:03.787 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 210 M 2023-09-23T17:37:03.787 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:37:03.991 INFO:teuthology.orchestra.run.smithi055.stdout:(1/8): libquadmath-8.5.0-10.1.el8_6.x86_64.rpm 841 kB/s | 171 kB 00:00 2023-09-23T17:37:04.025 INFO:teuthology.orchestra.run.smithi055.stdout:(2/8): libgfortran-8.5.0-10.1.el8_6.x86_64.rpm 2.7 MB/s | 644 kB 00:00 2023-09-23T17:37:04.442 INFO:teuthology.orchestra.run.smithi055.stdout:(3/8): python3-scipy-1.0.0-21.module+el8.5.0+10 30 MB/s | 14 MB 00:00 2023-09-23T17:37:04.517 INFO:teuthology.orchestra.run.smithi055.stdout:(4/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 7.6 MB/s | 3.7 MB 00:00 2023-09-23T17:37:04.626 INFO:teuthology.orchestra.run.smithi055.stdout:(5/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 1.2 MB/s | 225 kB 00:00 2023-09-23T17:37:04.759 INFO:teuthology.orchestra.run.smithi055.stdout:(6/8): openblas-threads-0.3.15-3.el8.x86_64.rpm 20 MB/s | 4.9 MB 00:00 2023-09-23T17:37:04.834 INFO:teuthology.orchestra.run.smithi055.stdout:(7/8): openblas-0.3.15-3.el8.x86_64.rpm 23 MB/s | 4.7 MB 00:00 2023-09-23T17:37:04.985 INFO:teuthology.orchestra.run.smithi055.stdout:(8/8): ceph-mgr-diskprediction-local-17.2.6-112 6.2 MB/s | 7.4 MB 00:01 2023-09-23T17:37:04.986 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:37:04.986 INFO:teuthology.orchestra.run.smithi055.stdout:Total 30 MB/s | 35 MB 00:01 2023-09-23T17:37:05.234 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:37:05.254 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:37:05.255 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:37:05.390 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T17:37:05.568 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:37:05.572 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:37:06.136 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:37:06.259 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T17:37:06.408 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: libquadmath-8.5.0-10.1.el8_6.x86_64 1/8 2023-09-23T17:37:06.526 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T17:37:06.664 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T17:37:06.834 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T17:37:07.094 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T17:37:07.094 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T17:37:07.095 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T17:37:07.095 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T17:37:07.095 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T17:37:07.095 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T17:37:07.095 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T17:37:07.096 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T17:37:07.164 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T17:37:07.326 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T17:37:07.962 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: openblas-threads-0.3.15-3.el8.x86_64 3/8 2023-09-23T17:37:08.134 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T17:37:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T17:37:08.142 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:22 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:37:08.873 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: openblas-0.3.15-3.el8.x86_64 4/8 2023-09-23T17:37:09.168 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T17:37:09.376 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T17:37:11.259 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T17:37:12.541 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 7/8 2023-09-23T17:37:12.767 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T17:37:13.022 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 8/8 2023-09-23T17:37:13.023 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6 1/8 2023-09-23T17:37:13.023 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libgfortran-8.5.0-10.1.el8_6.x86_64 2/8 2023-09-23T17:37:13.023 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libquadmath-8.5.0-10.1.el8_6.x86_64 3/8 2023-09-23T17:37:13.023 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d 4/8 2023-09-23T17:37:13.023 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-09-23T17:37:13.023 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-09-23T17:37:13.023 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : openblas-threads-0.3.15-3.el8.x86_64 7/8 2023-09-23T17:37:14.138 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : openblas-0.3.15-3.el8.x86_64 8/8 2023-09-23T17:37:14.138 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:02:33 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:37:14.305 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:37:14.491 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:14.491 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:37:14.491 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:37:14.491 INFO:teuthology.orchestra.run.smithi099.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T17:37:14.492 INFO:teuthology.orchestra.run.smithi099.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T17:37:14.492 INFO:teuthology.orchestra.run.smithi099.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T17:37:14.492 INFO:teuthology.orchestra.run.smithi099.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T17:37:14.492 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T17:37:14.492 INFO:teuthology.orchestra.run.smithi099.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T17:37:14.492 INFO:teuthology.orchestra.run.smithi099.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T17:37:14.492 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:14.492 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:37:14.629 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-mgr-rook 2023-09-23T17:37:14.974 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:37:16.435 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:30 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:37:17.825 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:37:17.827 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================================== 2023-09-23T17:37:17.827 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-09-23T17:37:17.827 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================================== 2023-09-23T17:37:17.827 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:37:17.827 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T17:37:17.827 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-09-23T17:37:17.827 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T17:37:17.828 INFO:teuthology.orchestra.run.smithi099.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T17:37:17.829 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:17.829 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:37:17.829 INFO:teuthology.orchestra.run.smithi099.stdout:======================================================================================================== 2023-09-23T17:37:17.829 INFO:teuthology.orchestra.run.smithi099.stdout:Install 13 Packages 2023-09-23T17:37:17.829 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:17.829 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 1.9 M 2023-09-23T17:37:17.830 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 25 M 2023-09-23T17:37:17.830 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:37:18.185 INFO:teuthology.orchestra.run.smithi099.stdout:(1/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 156 kB/s | 47 kB 00:00 2023-09-23T17:37:18.310 INFO:teuthology.orchestra.run.smithi099.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 77 kB/s | 33 kB 00:00 2023-09-23T17:37:18.335 INFO:teuthology.orchestra.run.smithi099.stdout:(3/13): python3-certifi-2018.10.15-7.el8.noarch 36 kB/s | 16 kB 00:00 2023-09-23T17:37:18.552 INFO:teuthology.orchestra.run.smithi099.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 297 kB/s | 64 kB 00:00 2023-09-23T17:37:18.636 INFO:teuthology.orchestra.run.smithi099.stdout:(5/13): python3-google-auth-1.1.1-10.el8.noarch 180 kB/s | 81 kB 00:00 2023-09-23T17:37:18.670 INFO:teuthology.orchestra.run.smithi099.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 523 kB/s | 61 kB 00:00 2023-09-23T17:37:18.770 INFO:teuthology.orchestra.run.smithi099.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 319 kB/s | 43 kB 00:00 2023-09-23T17:37:18.862 INFO:teuthology.orchestra.run.smithi099.stdout:(8/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 808 kB/s | 155 kB 00:00 2023-09-23T17:37:18.912 INFO:teuthology.orchestra.run.smithi099.stdout:(9/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 531 kB/s | 27 kB 00:00 2023-09-23T17:37:18.938 INFO:teuthology.orchestra.run.smithi099.stdout:(10/13): python3-pyasn1-modules-0.3.7-6.el8.noa 658 kB/s | 110 kB 00:00 2023-09-23T17:37:18.971 INFO:teuthology.orchestra.run.smithi099.stdout:(11/13): python3-kubernetes-11.0.0-6.el8.noarch 1.8 MB/s | 1.2 MB 00:00 2023-09-23T17:37:19.013 INFO:teuthology.orchestra.run.smithi099.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 1.2 MB/s | 126 kB 00:00 2023-09-23T17:37:19.055 INFO:teuthology.orchestra.run.smithi099.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 171 kB/s | 20 kB 00:00 2023-09-23T17:37:19.056 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:37:19.056 INFO:teuthology.orchestra.run.smithi099.stdout:Total 1.6 MB/s | 1.9 MB 00:01 2023-09-23T17:37:19.122 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:37:19.138 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:37:19.139 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:37:19.210 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:37:19.214 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:37:19.715 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:37:19.745 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:37:19.914 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T17:37:19.951 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:19.951 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:37:19.951 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:37:19.952 INFO:teuthology.orchestra.run.smithi055.stdout: libgfortran-8.5.0-10.1.el8_6.x86_64 2023-09-23T17:37:19.952 INFO:teuthology.orchestra.run.smithi055.stdout: libquadmath-8.5.0-10.1.el8_6.x86_64 2023-09-23T17:37:19.952 INFO:teuthology.orchestra.run.smithi055.stdout: openblas-0.3.15-3.el8.x86_64 2023-09-23T17:37:19.952 INFO:teuthology.orchestra.run.smithi055.stdout: openblas-threads-0.3.15-3.el8.x86_64 2023-09-23T17:37:19.952 INFO:teuthology.orchestra.run.smithi055.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-09-23T17:37:19.952 INFO:teuthology.orchestra.run.smithi055.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-09-23T17:37:19.952 INFO:teuthology.orchestra.run.smithi055.stdout: python3-scipy-1.0.0-21.module+el8.5.0+10916+41bd434d.x86_64 2023-09-23T17:37:19.952 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:19.952 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:37:20.105 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-mgr-rook 2023-09-23T17:37:20.125 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T17:37:20.312 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T17:37:20.475 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:37:20.555 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T17:37:20.742 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T17:37:20.969 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T17:37:21.159 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T17:37:21.355 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T17:37:21.547 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:02:40 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:37:21.561 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T17:37:21.768 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T17:37:22.829 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T17:37:23.079 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:37:23.081 INFO:teuthology.orchestra.run.smithi055.stdout:======================================================================================================== 2023-09-23T17:37:23.081 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2023-09-23T17:37:23.081 INFO:teuthology.orchestra.run.smithi055.stdout:======================================================================================================== 2023-09-23T17:37:23.081 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:37:23.081 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-rook noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 47 k 2023-09-23T17:37:23.081 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jsonpatch noarch 1.21-2.el8 rhel-8-for-x86_64-appstream-rpms 27 k 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jsonpointer noarch 1.10-11.el8 rhel-8-for-x86_64-appstream-rpms 20 k 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-oauthlib noarch 2.1.0-1.el8 rhel-8-for-x86_64-baseos-rpms 155 k 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyasn1 noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 126 k 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 rhel-8-for-x86_64-appstream-rpms 110 k 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 rhel-8-for-x86_64-baseos-rpms 43 k 2023-09-23T17:37:23.082 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-09-23T17:37:23.083 INFO:teuthology.orchestra.run.smithi055.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-09-23T17:37:23.083 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:23.083 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:37:23.083 INFO:teuthology.orchestra.run.smithi055.stdout:======================================================================================================== 2023-09-23T17:37:23.083 INFO:teuthology.orchestra.run.smithi055.stdout:Install 13 Packages 2023-09-23T17:37:23.083 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:23.083 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 1.9 M 2023-09-23T17:37:23.084 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 25 M 2023-09-23T17:37:23.084 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:37:23.089 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T17:37:23.247 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T17:37:23.349 INFO:teuthology.orchestra.run.smithi055.stdout:(1/13): python3-certifi-2018.10.15-7.el8.noarch 191 kB/s | 16 kB 00:00 2023-09-23T17:37:23.391 INFO:teuthology.orchestra.run.smithi055.stdout:(2/13): python3-cachetools-3.1.1-4.el8.noarch.r 259 kB/s | 33 kB 00:00 2023-09-23T17:37:23.416 INFO:teuthology.orchestra.run.smithi055.stdout:(3/13): python3-google-auth-1.1.1-10.el8.noarch 1.2 MB/s | 81 kB 00:00 2023-09-23T17:37:23.464 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T17:37:23.465 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T17:37:23.466 INFO:teuthology.orchestra.run.smithi055.stdout:(4/13): python3-rsa-4.9-2.el8.noarch.rpm 1.3 MB/s | 64 kB 00:00 2023-09-23T17:37:23.516 INFO:teuthology.orchestra.run.smithi055.stdout:(5/13): python3-kubernetes-11.0.0-6.el8.noarch. 9.3 MB/s | 1.2 MB 00:00 2023-09-23T17:37:23.533 INFO:teuthology.orchestra.run.smithi055.stdout:(6/13): python3-websocket-client-0.56.0-5.el8.n 912 kB/s | 61 kB 00:00 2023-09-23T17:37:23.675 INFO:teuthology.orchestra.run.smithi055.stdout:(7/13): ceph-mgr-rook-17.2.6-1123.g61431f6b.el8 115 kB/s | 47 kB 00:00 2023-09-23T17:37:23.776 INFO:teuthology.orchestra.run.smithi055.stdout:(8/13): python3-requests-oauthlib-1.0.0-1.el8.n 165 kB/s | 43 kB 00:00 2023-09-23T17:37:23.793 INFO:teuthology.orchestra.run.smithi055.stdout:(9/13): python3-pyasn1-modules-0.3.7-6.el8.noar 1.1 MB/s | 110 kB 00:00 2023-09-23T17:37:23.818 INFO:teuthology.orchestra.run.smithi055.stdout:(10/13): python3-oauthlib-2.1.0-1.el8.noarch.rp 545 kB/s | 155 kB 00:00 2023-09-23T17:37:23.835 INFO:teuthology.orchestra.run.smithi055.stdout:(11/13): python3-jsonpatch-1.21-2.el8.noarch.rp 451 kB/s | 27 kB 00:00 2023-09-23T17:37:23.852 INFO:teuthology.orchestra.run.smithi055.stdout:(12/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 2.1 MB/s | 126 kB 00:00 2023-09-23T17:37:23.869 INFO:teuthology.orchestra.run.smithi055.stdout:(13/13): python3-jsonpointer-1.10-11.el8.noarch 396 kB/s | 20 kB 00:00 2023-09-23T17:37:23.869 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:37:23.869 INFO:teuthology.orchestra.run.smithi055.stdout:Total 2.5 MB/s | 1.9 MB 00:00 2023-09-23T17:37:23.937 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:37:23.953 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:37:23.954 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:37:24.032 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:37:24.036 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:37:24.548 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T17:37:24.548 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:38 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:37:24.586 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:37:24.784 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-09-23T17:37:24.988 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-rsa-4.9-2.el8.noarch 2/13 2023-09-23T17:37:25.173 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 3/13 2023-09-23T17:37:25.323 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 4/13 2023-09-23T17:37:25.532 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 5/13 2023-09-23T17:37:25.838 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-09-23T17:37:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-09-23T17:37:26.266 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 8/13 2023-09-23T17:37:26.449 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-09-23T17:37:26.676 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 10/13 2023-09-23T17:37:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 11/13 2023-09-23T17:37:28.184 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 12/13 2023-09-23T17:37:28.324 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T17:37:28.581 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 13/13 2023-09-23T17:37:28.581 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 1/13 2023-09-23T17:37:28.581 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 2/13 2023-09-23T17:37:28.581 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 3/13 2023-09-23T17:37:28.581 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 4/13 2023-09-23T17:37:28.582 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 5/13 2023-09-23T17:37:28.582 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 6/13 2023-09-23T17:37:28.582 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 7/13 2023-09-23T17:37:28.582 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 8/13 2023-09-23T17:37:28.582 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 9/13 2023-09-23T17:37:28.582 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 10/13 2023-09-23T17:37:28.582 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 11/13 2023-09-23T17:37:28.582 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 12/13 2023-09-23T17:37:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 13/13 2023-09-23T17:37:29.703 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:02:48 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:37:30.057 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:37:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:30.239 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:37:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:37:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T17:37:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T17:37:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T17:37:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T17:37:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T17:37:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T17:37:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T17:37:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T17:37:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T17:37:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T17:37:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T17:37:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T17:37:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:30.240 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:37:30.373 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T17:37:30.726 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:37:32.149 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:46 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:37:33.596 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:37:33.597 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================================== 2023-09-23T17:37:33.597 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-09-23T17:37:33.598 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================================== 2023-09-23T17:37:33.598 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:37:33.598 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T17:37:33.598 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-09-23T17:37:33.598 INFO:teuthology.orchestra.run.smithi099.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T17:37:33.598 INFO:teuthology.orchestra.run.smithi099.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T17:37:33.598 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T17:37:33.599 INFO:teuthology.orchestra.run.smithi099.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T17:37:33.599 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:33.599 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:37:33.599 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================================== 2023-09-23T17:37:33.599 INFO:teuthology.orchestra.run.smithi099.stdout:Install 5 Packages 2023-09-23T17:37:33.599 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:33.600 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 6.0 M 2023-09-23T17:37:33.600 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 26 M 2023-09-23T17:37:33.600 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:37:33.987 INFO:teuthology.orchestra.run.smithi099.stdout:(1/5): python3-natsort-7.1.1-2.el8.noarch.rpm 282 kB/s | 60 kB 00:00 2023-09-23T17:37:34.104 INFO:teuthology.orchestra.run.smithi099.stdout:(2/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 366 kB/s | 121 kB 00:00 2023-09-23T17:37:34.204 INFO:teuthology.orchestra.run.smithi099.stdout:(3/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.4 MB/s | 538 kB 00:00 2023-09-23T17:37:34.255 INFO:teuthology.orchestra.run.smithi099.stdout:(4/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 989 kB/s | 476 kB 00:00 2023-09-23T17:37:34.505 INFO:teuthology.orchestra.run.smithi099.stdout:(5/5): python3-babel-2.5.1-7.el8.noarch.rpm 12 MB/s | 4.8 MB 00:00 2023-09-23T17:37:34.506 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:37:34.507 INFO:teuthology.orchestra.run.smithi099.stdout:Total 6.6 MB/s | 6.0 MB 00:00 2023-09-23T17:37:34.559 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:37:34.571 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:37:34.571 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:37:34.650 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:37:34.653 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:37:35.594 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:37:36.618 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:37:36.621 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T17:37:36.823 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:36.823 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:37:36.823 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-rook-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-09-23T17:37:36.824 INFO:teuthology.orchestra.run.smithi055.stdout: python3-rsa-4.9-2.el8.noarch 2023-09-23T17:37:36.825 INFO:teuthology.orchestra.run.smithi055.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-09-23T17:37:36.825 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:36.825 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:37:36.970 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-mgr-cephadm 2023-09-23T17:37:37.346 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:37:37.931 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T17:37:38.229 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T17:37:38.513 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T17:37:38.702 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T17:37:38.819 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:02:57 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:37:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T17:37:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T17:37:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T17:37:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T17:37:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T17:37:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T17:37:39.960 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:02:53 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:37:40.369 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:37:40.370 INFO:teuthology.orchestra.run.smithi055.stdout:=============================================================================================== 2023-09-23T17:37:40.370 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2023-09-23T17:37:40.370 INFO:teuthology.orchestra.run.smithi055.stdout:=============================================================================================== 2023-09-23T17:37:40.370 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:37:40.370 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1123.g61431f6b.el8 ceph-noarch 121 k 2023-09-23T17:37:40.370 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2023-09-23T17:37:40.370 INFO:teuthology.orchestra.run.smithi055.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-09-23T17:37:40.370 INFO:teuthology.orchestra.run.smithi055.stdout: python3-babel noarch 2.5.1-7.el8 rhel-8-for-x86_64-appstream-rpms 4.8 M 2023-09-23T17:37:40.370 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jinja2 noarch 2.10.1-3.el8 rhel-8-for-x86_64-appstream-rpms 538 k 2023-09-23T17:37:40.371 INFO:teuthology.orchestra.run.smithi055.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-09-23T17:37:40.371 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:40.371 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:37:40.371 INFO:teuthology.orchestra.run.smithi055.stdout:=============================================================================================== 2023-09-23T17:37:40.371 INFO:teuthology.orchestra.run.smithi055.stdout:Install 5 Packages 2023-09-23T17:37:40.371 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:40.372 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 6.0 M 2023-09-23T17:37:40.372 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 26 M 2023-09-23T17:37:40.372 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:37:40.594 INFO:teuthology.orchestra.run.smithi055.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1123.g61431f6b.e 636 kB/s | 121 kB 00:00 2023-09-23T17:37:40.786 INFO:teuthology.orchestra.run.smithi055.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 2.7 MB/s | 538 kB 00:00 2023-09-23T17:37:40.903 INFO:teuthology.orchestra.run.smithi055.stdout:(3/5): python3-natsort-7.1.1-2.el8.noarch.rpm 121 kB/s | 60 kB 00:00 2023-09-23T17:37:41.012 INFO:teuthology.orchestra.run.smithi055.stdout:(4/5): python3-babel-2.5.1-7.el8.noarch.rpm 21 MB/s | 4.8 MB 00:00 2023-09-23T17:37:41.154 INFO:teuthology.orchestra.run.smithi055.stdout:(5/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 634 kB/s | 476 kB 00:00 2023-09-23T17:37:41.156 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:37:41.156 INFO:teuthology.orchestra.run.smithi055.stdout:Total 7.6 MB/s | 6.0 MB 00:00 2023-09-23T17:37:41.214 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:37:41.226 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:37:41.227 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:37:41.311 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:37:41.315 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:37:42.319 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:37:42.677 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 1/5 2023-09-23T17:37:42.891 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 2/5 2023-09-23T17:37:43.186 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T17:37:43.484 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-09-23T17:37:43.646 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T17:37:43.883 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 5/5 2023-09-23T17:37:43.883 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 1/5 2023-09-23T17:37:43.883 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-09-23T17:37:43.883 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 3/5 2023-09-23T17:37:43.884 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-09-23T17:37:45.986 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 5/5 2023-09-23T17:37:45.986 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:03:04 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:37:47.498 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:37:47.681 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:47.681 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:37:47.681 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:37:47.681 INFO:teuthology.orchestra.run.smithi099.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T17:37:47.681 INFO:teuthology.orchestra.run.smithi099.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T17:37:47.681 INFO:teuthology.orchestra.run.smithi099.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T17:37:47.681 INFO:teuthology.orchestra.run.smithi099.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T17:37:47.681 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:47.681 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:37:47.819 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install ceph-fuse 2023-09-23T17:37:48.172 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:37:49.521 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:03 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:37:50.902 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:37:50.903 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:37:50.903 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repository Size 2023-09-23T17:37:50.903 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:37:50.903 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:37:50.903 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-fuse 2023-09-23T17:37:50.904 INFO:teuthology.orchestra.run.smithi099.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T17:37:50.904 INFO:teuthology.orchestra.run.smithi099.stdout:Installing dependencies: 2023-09-23T17:37:50.904 INFO:teuthology.orchestra.run.smithi099.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T17:37:50.904 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:50.904 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:37:50.904 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:37:50.904 INFO:teuthology.orchestra.run.smithi099.stdout:Install 2 Packages 2023-09-23T17:37:50.904 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:50.905 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 957 k 2023-09-23T17:37:50.905 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 3.0 M 2023-09-23T17:37:50.905 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:37:51.132 INFO:teuthology.orchestra.run.smithi099.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 364 kB/s | 83 kB 00:00 2023-09-23T17:37:51.307 INFO:teuthology.orchestra.run.smithi099.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 2.1 MB/s | 874 kB 00:00 2023-09-23T17:37:51.308 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:37:51.308 INFO:teuthology.orchestra.run.smithi099.stdout:Total 2.3 MB/s | 957 kB 00:00 2023-09-23T17:37:51.319 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:37:51.340 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:37:51.340 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:37:51.414 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:37:51.418 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:37:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:37:51.825 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T17:37:51.955 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:37:51.964 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T17:37:52.159 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:52.159 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:37:52.159 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-mgr-cephadm-2:17.2.6-1123.g61431f6b.el8.noarch 2023-09-23T17:37:52.159 INFO:teuthology.orchestra.run.smithi055.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-09-23T17:37:52.159 INFO:teuthology.orchestra.run.smithi055.stdout: python3-babel-2.5.1-7.el8.noarch 2023-09-23T17:37:52.159 INFO:teuthology.orchestra.run.smithi055.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-09-23T17:37:52.159 INFO:teuthology.orchestra.run.smithi055.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-09-23T17:37:52.160 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:52.160 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:37:52.300 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install ceph-fuse 2023-09-23T17:37:52.308 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T17:37:52.308 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T17:37:52.677 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:37:53.370 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T17:37:53.370 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:07 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:37:54.038 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:03:13 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:37:55.575 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:37:55.575 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:37:55.575 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repository Size 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-fuse 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout: x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 874 k 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout:Installing dependencies: 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout: fuse x86_64 2.9.7-15.el8 rhel-8-for-x86_64-baseos-rpms 83 k 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout:Install 2 Packages 2023-09-23T17:37:55.576 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:37:55.577 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 957 k 2023-09-23T17:37:55.577 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 3.0 M 2023-09-23T17:37:55.577 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:37:55.740 INFO:teuthology.orchestra.run.smithi055.stdout:(1/2): fuse-2.9.7-15.el8.x86_64.rpm 510 kB/s | 83 kB 00:00 2023-09-23T17:37:55.965 INFO:teuthology.orchestra.run.smithi055.stdout:(2/2): ceph-fuse-17.2.6-1123.g61431f6b.el8.x86_ 2.2 MB/s | 874 kB 00:00 2023-09-23T17:37:55.965 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:37:55.966 INFO:teuthology.orchestra.run.smithi055.stdout:Total 2.4 MB/s | 957 kB 00:00 2023-09-23T17:37:55.977 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:37:55.999 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:37:55.999 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:37:56.078 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:37:56.082 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:37:56.242 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:37:56.510 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : fuse-2.9.7-15.el8.x86_64 1/2 2023-09-23T17:37:56.669 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T17:37:57.017 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2/2 2023-09-23T17:37:57.017 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/2 2023-09-23T17:37:58.161 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : fuse-2.9.7-15.el8.x86_64 2/2 2023-09-23T17:37:58.162 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:03:17 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:37:59.328 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:37:59.510 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:59.510 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:37:59.510 INFO:teuthology.orchestra.run.smithi099.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T17:37:59.510 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:37:59.510 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:37:59.639 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install librados-devel 2023-09-23T17:37:59.997 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:01.551 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:15 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:02.947 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:38:02.947 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:38:02.947 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repo Size 2023-09-23T17:38:02.947 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:38:02.947 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:38:02.947 INFO:teuthology.orchestra.run.smithi099.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T17:38:02.948 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:02.948 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:38:02.948 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:38:02.948 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2023-09-23T17:38:02.948 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:02.948 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 130 k 2023-09-23T17:38:02.948 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 464 k 2023-09-23T17:38:02.948 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:38:03.205 INFO:teuthology.orchestra.run.smithi099.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 501 kB/s | 130 kB 00:00 2023-09-23T17:38:03.206 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:38:03.206 INFO:teuthology.orchestra.run.smithi099.stdout:Total 499 kB/s | 130 kB 00:00 2023-09-23T17:38:03.206 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:38:03.212 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:38:03.212 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:38:03.267 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:38:03.270 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:38:03.395 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:38:03.552 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:03.785 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:04.195 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:38:04.396 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:38:04.397 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:38:04.397 INFO:teuthology.orchestra.run.smithi055.stdout: ceph-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 fuse-2.9.7-15.el8.x86_64 2023-09-23T17:38:04.397 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:38:04.397 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:38:04.541 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install librados-devel 2023-09-23T17:38:04.805 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:04.805 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:18 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:04.919 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:06.355 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:03:25 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:07.917 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:38:07.918 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:38:07.918 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repo Size 2023-09-23T17:38:07.918 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:38:07.918 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:38:07.918 INFO:teuthology.orchestra.run.smithi055.stdout: librados-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 130 k 2023-09-23T17:38:07.918 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:38:07.919 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:38:07.919 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:38:07.919 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2023-09-23T17:38:07.919 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:38:07.919 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 130 k 2023-09-23T17:38:07.919 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 464 k 2023-09-23T17:38:07.920 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:38:08.387 INFO:teuthology.orchestra.run.smithi055.stdout:librados-devel-17.2.6-1123.g61431f6b.el8.x86_64 278 kB/s | 130 kB 00:00 2023-09-23T17:38:08.388 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:38:08.388 INFO:teuthology.orchestra.run.smithi055.stdout:Total 277 kB/s | 130 kB 00:00 2023-09-23T17:38:08.388 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:38:08.395 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:38:08.395 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:38:08.456 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:38:08.460 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:38:08.696 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:38:08.866 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:09.108 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:09.421 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:38:09.608 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:09.608 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:38:09.608 INFO:teuthology.orchestra.run.smithi099.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:38:09.608 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:09.608 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:38:09.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install libcephfs2 2023-09-23T17:38:10.192 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:10.192 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:03:29 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:10.321 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:12.076 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:26 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:13.357 INFO:teuthology.orchestra.run.smithi099.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:13.471 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:38:13.472 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2023-09-23T17:38:13.472 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:38:13.584 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install libcephfs-devel 2023-09-23T17:38:13.939 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:15.738 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:29 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:17.124 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:38:17.124 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:38:17.125 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repo Size 2023-09-23T17:38:17.125 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:38:17.125 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:38:17.125 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T17:38:17.125 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:17.125 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:38:17.125 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:38:17.125 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2023-09-23T17:38:17.125 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:17.126 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 25 k 2023-09-23T17:38:17.126 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 109 k 2023-09-23T17:38:17.126 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:38:17.276 INFO:teuthology.orchestra.run.smithi099.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 165 kB/s | 25 kB 00:00 2023-09-23T17:38:17.276 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:38:17.277 INFO:teuthology.orchestra.run.smithi099.stdout:Total 164 kB/s | 25 kB 00:00 2023-09-23T17:38:17.277 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:38:17.281 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:38:17.281 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:38:17.283 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:38:17.286 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:38:17.333 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:38:17.491 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:17.685 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:18.361 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:38:18.566 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:38:18.566 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:38:18.566 INFO:teuthology.orchestra.run.smithi055.stdout: librados-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:38:18.567 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:38:18.567 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:38:18.667 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:18.667 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:32 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:18.711 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install libcephfs2 2023-09-23T17:38:19.088 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:20.758 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:03:39 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:22.172 INFO:teuthology.orchestra.run.smithi055.stdout:Package libcephfs2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:22.293 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:38:22.295 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2023-09-23T17:38:22.295 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:38:22.431 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install libcephfs-devel 2023-09-23T17:38:22.808 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:24.383 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:03:43 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:24.768 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:38:24.953 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:24.953 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:38:24.953 INFO:teuthology.orchestra.run.smithi099.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:38:24.953 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:24.953 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:38:25.091 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install librados2 2023-09-23T17:38:25.444 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:25.896 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:38:25.897 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:38:25.897 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repo Size 2023-09-23T17:38:25.897 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:38:25.897 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:38:25.897 INFO:teuthology.orchestra.run.smithi055.stdout: libcephfs-devel x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 25 k 2023-09-23T17:38:25.897 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:38:25.897 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:38:25.897 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:38:25.897 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2023-09-23T17:38:25.898 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:38:25.898 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 25 k 2023-09-23T17:38:25.898 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 109 k 2023-09-23T17:38:25.898 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:38:26.041 INFO:teuthology.orchestra.run.smithi055.stdout:libcephfs-devel-17.2.6-1123.g61431f6b.el8.x86_6 174 kB/s | 25 kB 00:00 2023-09-23T17:38:26.041 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:38:26.042 INFO:teuthology.orchestra.run.smithi055.stdout:Total 172 kB/s | 25 kB 00:00 2023-09-23T17:38:26.042 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:38:26.046 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:38:26.046 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:38:26.048 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:38:26.052 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:38:26.099 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:38:26.264 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:26.489 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:26.795 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:40 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:27.521 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:27.522 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:03:46 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:28.093 INFO:teuthology.orchestra.run.smithi099.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:28.203 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:38:28.204 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2023-09-23T17:38:28.204 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:38:28.330 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install librbd1 2023-09-23T17:38:28.681 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:30.251 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:44 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:31.600 INFO:teuthology.orchestra.run.smithi099.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:31.709 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:38:31.711 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2023-09-23T17:38:31.711 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:38:31.830 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install python3-rados 2023-09-23T17:38:32.185 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:33.640 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:47 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:34.902 INFO:teuthology.orchestra.run.smithi099.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:35.012 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:38:35.013 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2023-09-23T17:38:35.013 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:38:35.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install python3-rgw 2023-09-23T17:38:35.484 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:36.827 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:38:36.978 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:50 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:37.028 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:38:37.028 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:38:37.028 INFO:teuthology.orchestra.run.smithi055.stdout: libcephfs-devel-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:38:37.029 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:38:37.029 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:38:37.170 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install librados2 2023-09-23T17:38:37.548 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:38.321 INFO:teuthology.orchestra.run.smithi099.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:38.431 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:38:38.432 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2023-09-23T17:38:38.432 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:38:38.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install python3-cephfs 2023-09-23T17:38:39.011 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:39.233 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:03:58 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:40.512 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:54 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:40.658 INFO:teuthology.orchestra.run.smithi055.stdout:Package librados2-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:40.781 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:38:40.783 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2023-09-23T17:38:40.783 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:38:40.912 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install librbd1 2023-09-23T17:38:41.290 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:41.779 INFO:teuthology.orchestra.run.smithi099.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:41.889 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:38:41.891 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2023-09-23T17:38:41.891 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:38:42.323 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install python3-rbd 2023-09-23T17:38:42.675 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:42.769 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:01 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:44.002 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:03:58 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:44.211 INFO:teuthology.orchestra.run.smithi055.stdout:Package librbd1-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:44.331 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:38:44.333 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2023-09-23T17:38:44.333 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:38:44.471 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install python3-rados 2023-09-23T17:38:44.849 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:45.349 INFO:teuthology.orchestra.run.smithi099.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:45.459 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:38:45.460 INFO:teuthology.orchestra.run.smithi099.stdout:Nothing to do. 2023-09-23T17:38:45.460 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:38:45.593 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install rbd-fuse 2023-09-23T17:38:45.946 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:46.305 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:05 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:47.428 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:04:01 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:47.751 INFO:teuthology.orchestra.run.smithi055.stdout:Package python3-rados-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:47.873 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:38:47.874 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2023-09-23T17:38:47.874 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:38:47.994 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install python3-rgw 2023-09-23T17:38:48.382 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:48.812 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:38:48.812 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:38:48.812 INFO:teuthology.orchestra.run.smithi099.stdout: Package Architecture Version Repository Size 2023-09-23T17:38:48.813 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:38:48.813 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:38:48.813 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T17:38:48.813 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:48.813 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:38:48.813 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:38:48.813 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2023-09-23T17:38:48.813 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:48.814 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 90 k 2023-09-23T17:38:48.814 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 264 k 2023-09-23T17:38:48.814 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:38:49.217 INFO:teuthology.orchestra.run.smithi099.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 224 kB/s | 90 kB 00:00 2023-09-23T17:38:49.218 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:38:49.218 INFO:teuthology.orchestra.run.smithi099.stdout:Total 223 kB/s | 90 kB 00:00 2023-09-23T17:38:49.218 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:38:49.228 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:38:49.228 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:38:49.281 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:38:49.285 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:38:49.390 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:38:49.550 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:49.778 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:49.925 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:08 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:50.798 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:38:50.798 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:04:04 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:38:51.352 INFO:teuthology.orchestra.run.smithi055.stdout:Package python3-rgw-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:51.474 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:38:51.475 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2023-09-23T17:38:51.476 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:38:51.593 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install python3-cephfs 2023-09-23T17:38:51.972 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:53.654 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:12 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:55.079 INFO:teuthology.orchestra.run.smithi055.stdout:Package python3-cephfs-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:55.199 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:38:55.201 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2023-09-23T17:38:55.201 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:38:55.333 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install python3-rbd 2023-09-23T17:38:55.713 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:57.410 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:16 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:38:58.409 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:38:58.593 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:58.593 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:38:58.593 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:38:58.593 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:38:58.593 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:38:58.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install rbd-mirror 2023-09-23T17:38:58.812 INFO:teuthology.orchestra.run.smithi055.stdout:Package python3-rbd-2:17.2.6-1123.g61431f6b.el8.x86_64 is already installed. 2023-09-23T17:38:58.935 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:38:58.936 INFO:teuthology.orchestra.run.smithi055.stdout:Nothing to do. 2023-09-23T17:38:58.936 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:38:59.049 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install rbd-fuse 2023-09-23T17:38:59.069 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:38:59.427 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:39:00.486 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:04:14 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:39:01.206 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:20 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:39:01.863 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:39:01.863 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:39:01.863 INFO:teuthology.orchestra.run.smithi099.stdout: Package Arch Version Repo Size 2023-09-23T17:39:01.863 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:39:01.864 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:39:01.864 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T17:39:01.864 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:39:01.864 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:39:01.864 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:39:01.864 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2023-09-23T17:39:01.864 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:39:01.865 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 3.5 M 2023-09-23T17:39:01.865 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 16 M 2023-09-23T17:39:01.865 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:39:02.427 INFO:teuthology.orchestra.run.smithi099.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 6.1 MB/s | 3.5 MB 00:00 2023-09-23T17:39:02.428 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:39:02.428 INFO:teuthology.orchestra.run.smithi099.stdout:Total 6.1 MB/s | 3.5 MB 00:00 2023-09-23T17:39:02.429 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:39:02.440 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:39:02.440 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:39:02.507 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:39:02.511 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:39:02.771 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:39:02.771 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:39:02.771 INFO:teuthology.orchestra.run.smithi055.stdout: Package Architecture Version Repository Size 2023-09-23T17:39:02.772 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:39:02.772 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:39:02.772 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-fuse x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 90 k 2023-09-23T17:39:02.772 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:02.772 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:39:02.772 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:39:02.772 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2023-09-23T17:39:02.772 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:02.773 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 90 k 2023-09-23T17:39:02.773 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 264 k 2023-09-23T17:39:02.773 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:39:02.953 INFO:teuthology.orchestra.run.smithi055.stdout:rbd-fuse-17.2.6-1123.g61431f6b.el8.x86_64.rpm 502 kB/s | 90 kB 00:00 2023-09-23T17:39:02.953 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:39:02.954 INFO:teuthology.orchestra.run.smithi055.stdout:Total 498 kB/s | 90 kB 00:00 2023-09-23T17:39:02.954 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:39:02.957 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:39:02.965 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:39:02.965 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:39:03.023 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:39:03.027 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:39:03.138 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:39:03.159 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:03.486 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:03.496 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:03.727 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:04.582 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:04.583 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:04:18 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:39:04.787 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:04.788 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:23 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:39:11.378 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:39:11.579 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:11.579 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:39:11.579 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-fuse-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:39:11.579 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:11.579 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:39:11.719 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install rbd-mirror 2023-09-23T17:39:12.097 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:39:12.757 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:39:12.941 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:39:12.941 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:39:12.942 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:39:12.942 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:39:12.942 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:39:13.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo yum -y install rbd-nbd 2023-09-23T17:39:13.431 INFO:teuthology.orchestra.run.smithi099.stdout:Updating Subscription Management repositories. 2023-09-23T17:39:13.658 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:32 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:39:14.833 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:04:28 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:39:15.213 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:39:15.214 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:39:15.214 INFO:teuthology.orchestra.run.smithi055.stdout: Package Arch Version Repo Size 2023-09-23T17:39:15.214 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:39:15.214 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:39:15.214 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-mirror x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 3.5 M 2023-09-23T17:39:15.214 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:15.214 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:39:15.214 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:39:15.214 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2023-09-23T17:39:15.214 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:15.215 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 3.5 M 2023-09-23T17:39:15.215 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 16 M 2023-09-23T17:39:15.215 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:39:15.742 INFO:teuthology.orchestra.run.smithi055.stdout:rbd-mirror-17.2.6-1123.g61431f6b.el8.x86_64.rpm 6.6 MB/s | 3.5 MB 00:00 2023-09-23T17:39:15.743 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:39:15.743 INFO:teuthology.orchestra.run.smithi055.stdout:Total 6.5 MB/s | 3.5 MB 00:00 2023-09-23T17:39:15.743 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:39:15.756 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:39:15.756 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:39:15.828 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:39:15.832 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:39:16.285 INFO:teuthology.orchestra.run.smithi099.stdout:Dependencies resolved. 2023-09-23T17:39:16.286 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:39:16.287 INFO:teuthology.orchestra.run.smithi099.stdout: Package Architecture Version Repository Size 2023-09-23T17:39:16.287 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:39:16.287 INFO:teuthology.orchestra.run.smithi099.stdout:Installing: 2023-09-23T17:39:16.287 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T17:39:16.287 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:39:16.287 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction Summary 2023-09-23T17:39:16.287 INFO:teuthology.orchestra.run.smithi099.stdout:================================================================================ 2023-09-23T17:39:16.287 INFO:teuthology.orchestra.run.smithi099.stdout:Install 1 Package 2023-09-23T17:39:16.287 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:39:16.288 INFO:teuthology.orchestra.run.smithi099.stdout:Total download size: 173 k 2023-09-23T17:39:16.288 INFO:teuthology.orchestra.run.smithi099.stdout:Installed size: 552 k 2023-09-23T17:39:16.288 INFO:teuthology.orchestra.run.smithi099.stdout:Downloading Packages: 2023-09-23T17:39:16.299 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:39:16.466 INFO:teuthology.orchestra.run.smithi099.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 974 kB/s | 173 kB 00:00 2023-09-23T17:39:16.466 INFO:teuthology.orchestra.run.smithi099.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:39:16.466 INFO:teuthology.orchestra.run.smithi099.stdout:Total 967 kB/s | 173 kB 00:00 2023-09-23T17:39:16.466 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction check 2023-09-23T17:39:16.476 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction check succeeded. 2023-09-23T17:39:16.477 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction test 2023-09-23T17:39:16.485 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:16.530 INFO:teuthology.orchestra.run.smithi099.stdout:Transaction test succeeded. 2023-09-23T17:39:16.533 INFO:teuthology.orchestra.run.smithi099.stdout:Running transaction 2023-09-23T17:39:16.651 INFO:teuthology.orchestra.run.smithi099.stdout: Preparing : 1/1 2023-09-23T17:39:16.803 INFO:teuthology.orchestra.run.smithi099.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:16.845 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:17.018 INFO:teuthology.orchestra.run.smithi099.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:17.917 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:17.917 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:36 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:39:18.042 INFO:teuthology.orchestra.run.smithi099.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:18.042 INFO:teuthology.orchestra.run.smithi099.stdout:Last metadata expiration check: 0:04:32 ago on Sat 23 Sep 2023 05:34:46 PM UTC. 2023-09-23T17:39:22.934 INFO:teuthology.orchestra.run.smithi099.stdout:Installed products updated. 2023-09-23T17:39:23.116 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:39:23.116 INFO:teuthology.orchestra.run.smithi099.stdout:Installed: 2023-09-23T17:39:23.116 INFO:teuthology.orchestra.run.smithi099.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:39:23.116 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:39:23.116 INFO:teuthology.orchestra.run.smithi099.stdout:Complete! 2023-09-23T17:39:23.267 DEBUG:teuthology.parallel:result is None 2023-09-23T17:39:24.615 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:39:24.816 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:24.817 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:39:24.817 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-mirror-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:39:24.817 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:24.817 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:39:24.962 DEBUG:teuthology.orchestra.run.smithi055:> sudo yum -y install rbd-nbd 2023-09-23T17:39:25.343 INFO:teuthology.orchestra.run.smithi055.stdout:Updating Subscription Management repositories. 2023-09-23T17:39:26.902 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:45 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:39:28.479 INFO:teuthology.orchestra.run.smithi055.stdout:Dependencies resolved. 2023-09-23T17:39:28.480 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:39:28.480 INFO:teuthology.orchestra.run.smithi055.stdout: Package Architecture Version Repository Size 2023-09-23T17:39:28.480 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:39:28.480 INFO:teuthology.orchestra.run.smithi055.stdout:Installing: 2023-09-23T17:39:28.480 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-nbd x86_64 2:17.2.6-1123.g61431f6b.el8 ceph 173 k 2023-09-23T17:39:28.480 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:28.481 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction Summary 2023-09-23T17:39:28.481 INFO:teuthology.orchestra.run.smithi055.stdout:================================================================================ 2023-09-23T17:39:28.481 INFO:teuthology.orchestra.run.smithi055.stdout:Install 1 Package 2023-09-23T17:39:28.481 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:28.481 INFO:teuthology.orchestra.run.smithi055.stdout:Total download size: 173 k 2023-09-23T17:39:28.481 INFO:teuthology.orchestra.run.smithi055.stdout:Installed size: 552 k 2023-09-23T17:39:28.481 INFO:teuthology.orchestra.run.smithi055.stdout:Downloading Packages: 2023-09-23T17:39:28.788 INFO:teuthology.orchestra.run.smithi055.stdout:rbd-nbd-17.2.6-1123.g61431f6b.el8.x86_64.rpm 563 kB/s | 173 kB 00:00 2023-09-23T17:39:28.789 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:39:28.789 INFO:teuthology.orchestra.run.smithi055.stdout:Total 561 kB/s | 173 kB 00:00 2023-09-23T17:39:28.790 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction check 2023-09-23T17:39:28.802 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction check succeeded. 2023-09-23T17:39:28.802 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction test 2023-09-23T17:39:28.861 INFO:teuthology.orchestra.run.smithi055.stdout:Transaction test succeeded. 2023-09-23T17:39:28.866 INFO:teuthology.orchestra.run.smithi055.stdout:Running transaction 2023-09-23T17:39:28.986 INFO:teuthology.orchestra.run.smithi055.stdout: Preparing : 1/1 2023-09-23T17:39:29.206 INFO:teuthology.orchestra.run.smithi055.stdout: Installing : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:29.438 INFO:teuthology.orchestra.run.smithi055.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:30.520 INFO:teuthology.orchestra.run.smithi055.stdout: Verifying : rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 1/1 2023-09-23T17:39:30.520 INFO:teuthology.orchestra.run.smithi055.stdout:Last metadata expiration check: 0:04:49 ago on Sat 23 Sep 2023 05:34:41 PM UTC. 2023-09-23T17:39:37.169 INFO:teuthology.orchestra.run.smithi055.stdout:Installed products updated. 2023-09-23T17:39:37.368 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:37.368 INFO:teuthology.orchestra.run.smithi055.stdout:Installed: 2023-09-23T17:39:37.368 INFO:teuthology.orchestra.run.smithi055.stdout: rbd-nbd-2:17.2.6-1123.g61431f6b.el8.x86_64 2023-09-23T17:39:37.368 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:39:37.368 INFO:teuthology.orchestra.run.smithi055.stdout:Complete! 2023-09-23T17:39:37.595 DEBUG:teuthology.parallel:result is None 2023-09-23T17:39:37.596 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T17:39:37.596 INFO:teuthology.packaging:ref: None 2023-09-23T17:39:37.596 INFO:teuthology.packaging:tag: None 2023-09-23T17:39:37.596 INFO:teuthology.packaging:branch: quincy 2023-09-23T17:39:37.596 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:39:37.596 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T17:39:37.830 DEBUG:teuthology.orchestra.run.smithi055:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T17:39:37.853 INFO:teuthology.orchestra.run.smithi055.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T17:39:37.853 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T17:39:37.853 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T17:39:37.854 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-09-23T17:39:37.854 INFO:teuthology.packaging:ref: None 2023-09-23T17:39:37.854 INFO:teuthology.packaging:tag: None 2023-09-23T17:39:37.854 INFO:teuthology.packaging:branch: quincy 2023-09-23T17:39:37.854 INFO:teuthology.packaging:sha1: bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:39:37.854 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-09-23T17:39:38.105 DEBUG:teuthology.orchestra.run.smithi099:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-09-23T17:39:38.132 INFO:teuthology.orchestra.run.smithi099.stdout:17.2.6-1123.g61431f6b.el8 2023-09-23T17:39:38.132 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1123.g61431f6b.el8 2023-09-23T17:39:38.132 INFO:teuthology.task.install:The correct ceph version 17.2.6-1123.g61431f6b is installed. 2023-09-23T17:39:38.132 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-09-23T17:39:38.133 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:39:38.133 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T17:39:38.162 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:39:38.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-09-23T17:39:38.206 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-09-23T17:39:38.206 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:39:38.206 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T17:39:38.234 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T17:39:38.302 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:39:38.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/daemon-helper 2023-09-23T17:39:38.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-09-23T17:39:38.403 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-09-23T17:39:38.403 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:39:38.403 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T17:39:38.431 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T17:39:38.498 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:39:38.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/adjust-ulimits 2023-09-23T17:39:38.526 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-09-23T17:39:38.595 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-09-23T17:39:38.595 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:39:38.595 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T17:39:38.622 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T17:39:38.689 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:39:38.689 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/usr/bin/stdin-killer 2023-09-23T17:39:38.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-09-23T17:39:38.783 INFO:teuthology.run_tasks:Running task cephadm... 2023-09-23T17:39:38.853 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'} 2023-09-23T17:39:38.853 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-09-23T17:39:38.854 INFO:tasks.cephadm:Cluster fsid is 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:39:38.854 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-09-23T17:39:38.854 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.55', 'mon.c': '[v2:172.21.15.55:3301,v1:172.21.15.55:6790]', 'mon.b': '172.21.15.99'} 2023-09-23T17:39:38.854 INFO:tasks.cephadm:First mon is mon.a on smithi055 2023-09-23T17:39:38.854 INFO:tasks.cephadm:First mgr is y 2023-09-23T17:39:38.855 INFO:tasks.cephadm:Normalizing hostnames... 2023-09-23T17:39:38.855 DEBUG:teuthology.orchestra.run.smithi055:> sudo hostname $(hostname -s) 2023-09-23T17:39:38.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo hostname $(hostname -s) 2023-09-23T17:39:38.910 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-09-23T17:39:38.910 DEBUG:teuthology.orchestra.run.smithi055:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T17:39:39.168 INFO:teuthology.orchestra.run.smithi055.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 17:39 /home/ubuntu/cephtest/cephadm 2023-09-23T17:39:39.169 DEBUG:teuthology.orchestra.run.smithi099:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-09-23T17:39:39.313 INFO:teuthology.orchestra.run.smithi099.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Sep 23 17:39 /home/ubuntu/cephtest/cephadm 2023-09-23T17:39:39.313 DEBUG:teuthology.orchestra.run.smithi055:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T17:39:39.332 DEBUG:teuthology.orchestra.run.smithi099:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-09-23T17:39:39.361 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-09-23T17:39:39.361 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T17:39:39.374 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-09-23T17:39:40.695 INFO:teuthology.orchestra.run.smithi099.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T17:39:40.877 INFO:teuthology.orchestra.run.smithi055.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T17:40:05.484 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:40:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T17:40:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T17:40:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "repo_digests": [ 2023-09-23T17:40:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T17:40:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:40:05.484 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:40:06.169 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:40:06.169 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph_version": "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)", 2023-09-23T17:40:06.169 INFO:teuthology.orchestra.run.smithi055.stdout: "image_id": "0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64", 2023-09-23T17:40:06.169 INFO:teuthology.orchestra.run.smithi055.stdout: "repo_digests": [ 2023-09-23T17:40:06.169 INFO:teuthology.orchestra.run.smithi055.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:1d807b57f44e899861bfb3500d17f81ef0ce7a664575560e306bb83d8fe31c44" 2023-09-23T17:40:06.169 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:40:06.169 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:40:06.188 DEBUG:teuthology.orchestra.run.smithi055:> sudo mkdir -p /etc/ceph 2023-09-23T17:40:06.218 DEBUG:teuthology.orchestra.run.smithi099:> sudo mkdir -p /etc/ceph 2023-09-23T17:40:06.252 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod 777 /etc/ceph 2023-09-23T17:40:06.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 777 /etc/ceph 2023-09-23T17:40:06.319 INFO:tasks.cephadm:Writing seed config... 2023-09-23T17:40:06.320 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-09-23T17:40:06.320 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-09-23T17:40:06.320 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-09-23T17:40:06.320 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-09-23T17:40:06.320 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-09-23T17:40:06.321 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-09-23T17:40:06.321 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-09-23T17:40:06.321 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-09-23T17:40:06.321 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-09-23T17:40:06.321 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-09-23T17:40:06.321 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-09-23T17:40:06.321 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:40:06.322 DEBUG:teuthology.orchestra.run.smithi055:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-09-23T17:40:06.342 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = 2b70c030-5a38-11ee-8db4-212e2dc638e7 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-09-23T17:40:06.342 DEBUG:teuthology.orchestra.run.smithi055:mon.a> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.a.service 2023-09-23T17:40:06.384 DEBUG:teuthology.orchestra.run.smithi055:mgr.y> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.y.service 2023-09-23T17:40:06.411 INFO:journalctl@ceph.mon.a.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T17:40:06.412 INFO:tasks.cephadm:Bootstrapping... 2023-09-23T17:40:06.412 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.55 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-09-23T17:40:06.442 INFO:journalctl@ceph.mgr.y.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T17:40:06.620 INFO:teuthology.orchestra.run.smithi055.stdout:-------------------------------------------------------------------------------- 2023-09-23T17:40:06.620 INFO:teuthology.orchestra.run.smithi055.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', '2b70c030-5a38-11ee-8db4-212e2dc638e7', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.55', '--skip-admin-label'] 2023-09-23T17:40:06.766 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stdout 4.1.1 2023-09-23T17:40:06.766 INFO:teuthology.orchestra.run.smithi055.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-09-23T17:40:06.766 INFO:teuthology.orchestra.run.smithi055.stdout:Verifying podman|docker is present... 2023-09-23T17:40:06.909 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stdout 4.1.1 2023-09-23T17:40:06.909 INFO:teuthology.orchestra.run.smithi055.stdout:Verifying lvm2 is present... 2023-09-23T17:40:06.910 INFO:teuthology.orchestra.run.smithi055.stdout:Verifying time synchronization is in place... 2023-09-23T17:40:06.917 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T17:40:06.917 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T17:40:06.923 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T17:40:06.923 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout inactive 2023-09-23T17:40:06.929 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout enabled 2023-09-23T17:40:06.935 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout active 2023-09-23T17:40:06.935 INFO:teuthology.orchestra.run.smithi055.stdout:Unit chronyd.service is enabled and running 2023-09-23T17:40:06.935 INFO:teuthology.orchestra.run.smithi055.stdout:Repeating the final host check... 2023-09-23T17:40:07.078 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stdout 4.1.1 2023-09-23T17:40:07.079 INFO:teuthology.orchestra.run.smithi055.stdout:podman (/bin/podman) version 4.1.1 is present 2023-09-23T17:40:07.079 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl is present 2023-09-23T17:40:07.079 INFO:teuthology.orchestra.run.smithi055.stdout:lvcreate is present 2023-09-23T17:40:07.085 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-09-23T17:40:07.085 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-09-23T17:40:07.091 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-09-23T17:40:07.091 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout inactive 2023-09-23T17:40:07.097 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout enabled 2023-09-23T17:40:07.103 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout active 2023-09-23T17:40:07.103 INFO:teuthology.orchestra.run.smithi055.stdout:Unit chronyd.service is enabled and running 2023-09-23T17:40:07.103 INFO:teuthology.orchestra.run.smithi055.stdout:Host looks OK 2023-09-23T17:40:07.103 INFO:teuthology.orchestra.run.smithi055.stdout:Cluster fsid: 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:07.103 INFO:teuthology.orchestra.run.smithi055.stdout:Acquiring lock 140502532303336 on /run/cephadm/2b70c030-5a38-11ee-8db4-212e2dc638e7.lock 2023-09-23T17:40:07.104 INFO:teuthology.orchestra.run.smithi055.stdout:Lock 140502532303336 acquired on /run/cephadm/2b70c030-5a38-11ee-8db4-212e2dc638e7.lock 2023-09-23T17:40:07.104 INFO:teuthology.orchestra.run.smithi055.stdout:Verifying IP 172.21.15.55 port 3300 ... 2023-09-23T17:40:07.104 INFO:teuthology.orchestra.run.smithi055.stdout:Verifying IP 172.21.15.55 port 6789 ... 2023-09-23T17:40:07.104 INFO:teuthology.orchestra.run.smithi055.stdout:Base mon IP(s) is [172.21.15.55:3300, 172.21.15.55:6789], mon addrv is [v2:172.21.15.55:3300,v1:172.21.15.55:6789] 2023-09-23T17:40:07.108 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout default via 172.21.15.254 dev ens1f0 proto dhcp metric 100 2023-09-23T17:40:07.108 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout 172.21.0.0/20 dev ens1f0 proto kernel scope link src 172.21.15.55 metric 100 2023-09-23T17:40:07.111 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-09-23T17:40:07.111 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout fe80::/64 dev ens1f0 proto kernel metric 256 pref medium 2023-09-23T17:40:07.111 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev ens1f0 proto ra metric 1024 expires 1722sec hoplimit 64 pref medium 2023-09-23T17:40:07.113 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-09-23T17:40:07.113 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-09-23T17:40:07.113 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T17:40:07.114 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout 4: ens1f0: mtu 1500 state UP qlen 1000 2023-09-23T17:40:07.114 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout inet6 fe80::ae1f:6bff:fef8:2606/64 scope link 2023-09-23T17:40:07.114 INFO:teuthology.orchestra.run.smithi055.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-09-23T17:40:07.114 INFO:teuthology.orchestra.run.smithi055.stdout:Mon IP `172.21.15.55` is in CIDR network `172.21.0.0/20` 2023-09-23T17:40:07.114 INFO:teuthology.orchestra.run.smithi055.stdout:Mon IP `172.21.15.55` is in CIDR network `172.21.0.0/20` 2023-09-23T17:40:07.114 INFO:teuthology.orchestra.run.smithi055.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-09-23T17:40:07.115 INFO:teuthology.orchestra.run.smithi055.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-09-23T17:40:07.115 INFO:teuthology.orchestra.run.smithi055.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T17:40:07.565 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stdout 0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T17:40:07.565 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-09-23T17:40:07.565 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stderr Getting image source signatures 2023-09-23T17:40:07.565 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stderr Copying blob sha256:af7d028f70e220a466755c6119effe51eebe359d9c00eb6b4883db3fabc6becd 2023-09-23T17:40:07.566 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stderr Copying blob sha256:3f4fd5c3c800c6da9ef94691df203a05a5bbbf32c483d09d620ec06eef42ea2b 2023-09-23T17:40:07.566 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stderr Copying config sha256:0db14aa010946f310e3ede6ffd96448192b98ccc76c262396fe9cf91887d0d64 2023-09-23T17:40:07.566 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stderr Writing manifest to image destination 2023-09-23T17:40:07.566 INFO:teuthology.orchestra.run.smithi055.stdout:/bin/podman: stderr Storing signatures 2023-09-23T17:40:08.314 INFO:teuthology.orchestra.run.smithi055.stdout:ceph: stdout ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T17:40:08.314 INFO:teuthology.orchestra.run.smithi055.stdout:Ceph version: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable) 2023-09-23T17:40:08.314 INFO:teuthology.orchestra.run.smithi055.stdout:Extracting ceph user uid/gid from container image... 2023-09-23T17:40:08.959 INFO:teuthology.orchestra.run.smithi055.stdout:stat: stdout 167 167 2023-09-23T17:40:08.960 INFO:teuthology.orchestra.run.smithi055.stdout:Creating initial keys... 2023-09-23T17:40:09.607 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph-authtool: stdout AQD5Ig9lGjOrEhAAPwiMtb6Khp2Ju2fWFR+IGg== 2023-09-23T17:40:10.256 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph-authtool: stdout AQD5Ig9lo9OyORAAWjsYNt5XEim1ERYYJoQvXQ== 2023-09-23T17:40:10.922 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph-authtool: stdout AQD6Ig9lXU1DIxAAaYnd5j5Tu49ZRAckAJ39Jg== 2023-09-23T17:40:10.923 INFO:teuthology.orchestra.run.smithi055.stdout:Creating initial monmap... 2023-09-23T17:40:11.575 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T17:40:11.575 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-09-23T17:40:11.575 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:11.575 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T17:40:11.575 INFO:teuthology.orchestra.run.smithi055.stdout:monmaptool for a [v2:172.21.15.55:3300,v1:172.21.15.55:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-09-23T17:40:11.576 INFO:teuthology.orchestra.run.smithi055.stdout:setting min_mon_release = octopus 2023-09-23T17:40:11.576 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/monmaptool: set fsid to 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:11.576 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-09-23T17:40:11.576 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:40:11.576 INFO:teuthology.orchestra.run.smithi055.stdout:Creating mon... 2023-09-23T17:40:12.321 INFO:teuthology.orchestra.run.smithi055.stdout:create mon.a on 2023-09-23T17:40:12.424 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-09-23T17:40:12.513 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-09-23T17:40:12.613 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7.target. 2023-09-23T17:40:12.613 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7.target → /etc/systemd/system/ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7.target. 2023-09-23T17:40:12.741 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T17:40:12.741 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.a.service: Unit ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.a.service not loaded. 2023-09-23T17:40:12.832 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7.target.wants/ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.a.service → /etc/systemd/system/ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@.service. 2023-09-23T17:40:12.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:12 smithi055 systemd[1]: Starting Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:40:13.341 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T17:40:13.341 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout disabled 2023-09-23T17:40:13.348 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T17:40:13.348 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout inactive 2023-09-23T17:40:13.348 INFO:teuthology.orchestra.run.smithi055.stdout:firewalld.service is not enabled 2023-09-23T17:40:13.348 INFO:teuthology.orchestra.run.smithi055.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T17:40:13.349 INFO:teuthology.orchestra.run.smithi055.stdout:Waiting for mon to start... 2023-09-23T17:40:13.349 INFO:teuthology.orchestra.run.smithi055.stdout:Waiting for mon... 2023-09-23T17:40:13.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:13 smithi055 podman[95313]: 2023-09-23T17:40:13.387 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:13 smithi055 bash[95313]: 584266df7d2f2b9298abfc011f3be147420f67d0bc9b7bb5b513f466ecee6409 2023-09-23T17:40:13.387 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:13 smithi055 systemd[1]: Started Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:40:13.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:13 smithi055 ceph-mon[95352]: mkfs 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:13.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:13 smithi055 ceph-mon[95352]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T17:40:14.296 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout cluster: 2023-09-23T17:40:14.298 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout id: 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:14.298 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout services: 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.579224s) 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout data: 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout pgs: 2023-09-23T17:40:14.299 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:14.300 INFO:teuthology.orchestra.run.smithi055.stdout:mon is available 2023-09-23T17:40:14.300 INFO:teuthology.orchestra.run.smithi055.stdout:Assimilating anything we can from ceph.conf... 2023-09-23T17:40:14.533 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:14 smithi055 ceph-mon[95352]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T17:40:14.533 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:14 smithi055 ceph-mon[95352]: monmap e1: 1 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0]} removed_ranks: {} 2023-09-23T17:40:14.533 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:14 smithi055 ceph-mon[95352]: fsmap 2023-09-23T17:40:14.533 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:14 smithi055 ceph-mon[95352]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T17:40:14.533 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:14 smithi055 ceph-mon[95352]: mgrmap e1: no daemons active 2023-09-23T17:40:14.533 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:14 smithi055 ceph-mon[95352]: from='client.? 172.21.15.55:0/996061938' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-09-23T17:40:15.244 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:15.244 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T17:40:15.244 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout fsid = 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:15.244 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.55:3300,v1:172.21.15.55:6789] 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T17:40:15.245 INFO:teuthology.orchestra.run.smithi055.stdout:Generating new minimal ceph.conf... 2023-09-23T17:40:15.478 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:15 smithi055 ceph-mon[95352]: from='client.? 172.21.15.55:0/2322433361' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T17:40:15.478 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:15 smithi055 ceph-mon[95352]: from='client.? 172.21.15.55:0/2322433361' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-09-23T17:40:16.184 INFO:teuthology.orchestra.run.smithi055.stdout:Restarting the monitor... 2023-09-23T17:40:16.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:16 smithi055 systemd[1]: Stopping Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:40:16.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:16 smithi055 ceph-mon[95352]: from='client.? 172.21.15.55:0/3504480283' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:40:16.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:16 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[95328]: 2023-09-23T17:40:16.527+0000 7fa7f1c6d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:40:16.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:16 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[95328]: 2023-09-23T17:40:16.527+0000 7fa7f1c6d700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-09-23T17:40:17.076 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:16 smithi055 bash[95684]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T17:40:17.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:17 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T17:40:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:17 smithi055 systemd[1]: Stopped Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:40:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:17 smithi055 systemd[1]: Starting Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:40:18.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:18 smithi055 podman[95808]: 2023-09-23T17:40:19.372 INFO:teuthology.orchestra.run.smithi055.stdout:Setting mon public_network to 172.21.0.0/20 2023-09-23T17:40:19.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 bash[95808]: 3ea34fb03ea803202f9b5a03b46cacd4d2f7ce136eff5cc193a126f2478eba06 2023-09-23T17:40:19.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T17:40:19.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T17:40:19.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: pidfile_write: ignore empty --pid-file 2023-09-23T17:40:19.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: load: jerasure load: lrc 2023-09-23T17:40:19.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: RocksDB version: 6.15.5 2023-09-23T17:40:19.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T17:40:19.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Compile date Sep 22 2023 2023-09-23T17:40:19.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: DB SUMMARY 2023-09-23T17:40:19.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: DB Session ID: 1KW45A80MB0ZJI40G5BA 2023-09-23T17:40:19.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: CURRENT file: CURRENT 2023-09-23T17:40:19.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T17:40:19.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-09-23T17:40:19.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-09-23T17:40:19.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83585 ; 2023-09-23T17:40:19.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.error_if_exists: 0 2023-09-23T17:40:19.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.create_if_missing: 0 2023-09-23T17:40:19.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.paranoid_checks: 1 2023-09-23T17:40:19.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T17:40:19.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.env: 0x56284436ba80 2023-09-23T17:40:19.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.fs: Posix File System 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.info_log: 0x562845cd7d80 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.statistics: (nil) 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.use_fsync: 0 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_log_file_size: 0 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.allow_fallocate: 1 2023-09-23T17:40:19.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.use_direct_reads: 0 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.db_log_dir: 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T17:40:19.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.write_buffer_manager: 0x5628469ca960 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T17:40:19.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T17:40:19.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T17:40:19.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T17:40:19.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.unordered_write: 0 2023-09-23T17:40:19.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T17:40:19.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T17:40:19.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T17:40:19.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T17:40:19.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.row_cache: None 2023-09-23T17:40:19.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.wal_filter: None 2023-09-23T17:40:19.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.preserve_deletes: 0 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.two_write_queues: 0 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.atomic_flush: 0 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.log_readahead_size: 0 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T17:40:19.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_background_jobs: 2 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_background_compactions: -1 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_subcompactions: 1 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T17:40:19.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_open_files: -1 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_background_flushes: -1 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Compression algorithms supported: 2023-09-23T17:40:19.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T17:40:19.634 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: kZSTD supported: 0 2023-09-23T17:40:19.634 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: kXpressCompression supported: 0 2023-09-23T17:40:19.634 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T17:40:19.634 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: kZlibCompression supported: 1 2023-09-23T17:40:19.634 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: kSnappyCompression supported: 1 2023-09-23T17:40:19.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: kLZ4Compression supported: 1 2023-09-23T17:40:19.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: kBZip2Compression supported: 0 2023-09-23T17:40:19.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T17:40:19.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-09-23T17:40:19.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T17:40:19.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T17:40:19.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.merge_operator: 2023-09-23T17:40:19.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_filter: None 2023-09-23T17:40:19.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T17:40:19.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T17:40:19.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T17:40:19.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T17:40:19.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562845caafa0) 2023-09-23T17:40:19.636 INFO:journalctl@ceph.mon.a.smithi055.stdout: cache_index_and_filter_blocks: 1 2023-09-23T17:40:19.636 INFO:journalctl@ceph.mon.a.smithi055.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T17:40:19.636 INFO:journalctl@ceph.mon.a.smithi055.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T17:40:19.636 INFO:journalctl@ceph.mon.a.smithi055.stdout: pin_top_level_index_and_filter: 1 2023-09-23T17:40:19.637 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_type: 0 2023-09-23T17:40:19.637 INFO:journalctl@ceph.mon.a.smithi055.stdout: data_block_index_type: 0 2023-09-23T17:40:19.637 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_shortening: 1 2023-09-23T17:40:19.637 INFO:journalctl@ceph.mon.a.smithi055.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T17:40:19.637 INFO:journalctl@ceph.mon.a.smithi055.stdout: hash_index_allow_collision: 1 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: checksum: 1 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: no_block_cache: 0 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache: 0x562845cdd090 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_name: BinnedLRUCache 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_options: 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: capacity : 536870912 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: num_shard_bits : 4 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: strict_capacity_limit : 0 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: high_pri_pool_ratio: 0.000 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_compressed: (nil) 2023-09-23T17:40:19.638 INFO:journalctl@ceph.mon.a.smithi055.stdout: persistent_cache: (nil) 2023-09-23T17:40:19.639 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_size: 4096 2023-09-23T17:40:19.639 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_size_deviation: 10 2023-09-23T17:40:19.639 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_restart_interval: 16 2023-09-23T17:40:19.639 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_block_restart_interval: 1 2023-09-23T17:40:19.639 INFO:journalctl@ceph.mon.a.smithi055.stdout: metadata_block_size: 4096 2023-09-23T17:40:19.639 INFO:journalctl@ceph.mon.a.smithi055.stdout: partition_filters: 0 2023-09-23T17:40:19.639 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_delta_encoding: 1 2023-09-23T17:40:19.639 INFO:journalctl@ceph.mon.a.smithi055.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T17:40:19.639 INFO:journalctl@ceph.mon.a.smithi055.stdout: whole_key_filtering: 1 2023-09-23T17:40:19.639 INFO:journalctl@ceph.mon.a.smithi055.stdout: verify_compression: 0 2023-09-23T17:40:19.640 INFO:journalctl@ceph.mon.a.smithi055.stdout: read_amp_bytes_per_bit: 0 2023-09-23T17:40:19.640 INFO:journalctl@ceph.mon.a.smithi055.stdout: format_version: 4 2023-09-23T17:40:19.640 INFO:journalctl@ceph.mon.a.smithi055.stdout: enable_index_compression: 1 2023-09-23T17:40:19.640 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_align: 0 2023-09-23T17:40:19.640 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T17:40:19.640 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T17:40:19.640 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compression: NoCompression 2023-09-23T17:40:19.640 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T17:40:19.640 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T17:40:19.641 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T17:40:19.641 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.num_levels: 7 2023-09-23T17:40:19.641 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T17:40:19.641 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T17:40:19.641 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T17:40:19.641 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T17:40:19.641 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T17:40:19.641 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T17:40:19.641 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T17:40:19.641 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:40:19.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T17:40:19.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T17:40:19.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T17:40:19.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T17:40:19.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T17:40:19.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T17:40:19.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:40:19.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T17:40:19.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T17:40:19.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T17:40:19.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T17:40:19.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T17:40:19.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T17:40:19.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T17:40:19.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T17:40:19.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T17:40:19.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T17:40:19.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T17:40:19.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T17:40:19.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T17:40:19.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T17:40:19.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T17:40:19.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T17:40:19.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T17:40:19.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T17:40:19.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T17:40:19.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T17:40:19.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T17:40:19.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T17:40:19.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T17:40:19.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.table_properties_collectors: 2023-09-23T17:40:19.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.inplace_update_support: 0 2023-09-23T17:40:19.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T17:40:19.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.bloom_locality: 0 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.max_successive_merges: 0 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.ttl: 2592000 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T17:40:19.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.enable_blob_files: false 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.min_blob_size: 0 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695490819360100, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T17:40:19.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695490819360927, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 80132, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78400, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10136, "raw_average_key_size": 47, "raw_value_size": 72608, "raw_average_value_size": 340, "num_data_blocks": 10, "num_entries": 213, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695490819, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "e3c8ea17-9219-4534-b4c8-de259f78cf01", "db_session_id": "1KW45A80MB0ZJI40G5BA"}} 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695490819366318, "job": 1, "event": "recovery_finished"} 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x562845c71180 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: DB pointer 0x562845d46000 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** DB Stats ** 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T17:40:19.649 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: L0 2/0 80.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: Sum 2/0 80.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2023-09-23T17:40:19.650 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T17:40:19.651 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative compaction: 0.00 GB write, 8.39 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:40:19.652 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval compaction: 0.00 GB write, 8.39 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:40:19.652 INFO:journalctl@ceph.mon.a.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T17:40:19.652 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:40:19.652 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T17:40:19.652 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:40:19.652 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:40:19.652 INFO:journalctl@ceph.mon.a.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:40:19.653 INFO:journalctl@ceph.mon.a.smithi055.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:40:19.653 INFO:journalctl@ceph.mon.a.smithi055.stdout: L0 2/0 80.01 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2023-09-23T17:40:19.653 INFO:journalctl@ceph.mon.a.smithi055.stdout: Sum 2/0 80.01 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2023-09-23T17:40:19.653 INFO:journalctl@ceph.mon.a.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T17:40:19.653 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:40:19.654 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:40:19.654 INFO:journalctl@ceph.mon.a.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:40:19.654 INFO:journalctl@ceph.mon.a.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:40:19.654 INFO:journalctl@ceph.mon.a.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 148.7 0.00 0.00 1 0.001 0 0 2023-09-23T17:40:19.654 INFO:journalctl@ceph.mon.a.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:40:19.654 INFO:journalctl@ceph.mon.a.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T17:40:19.654 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T17:40:19.654 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T17:40:19.654 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T17:40:19.654 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative compaction: 0.00 GB write, 8.34 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: starting mon.a rank 0 at public addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] at bind addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: mon.a@-1(???) e1 preinit fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: mon.a@-1(???).mds e1 new map 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: mon.a@-1(???).mds e1 print_map 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout: e1 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T17:40:19.655 INFO:journalctl@ceph.mon.a.smithi055.stdout: legacy client fscid: -1 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout: No filesystems configured 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 systemd[1]: Started Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: mon.a is new leader, mons a in quorum (ranks 0) 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: monmap e1: 1 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0]} removed_ranks: {} 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: fsmap 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: osdmap e1: 0 total, 0 up, 0 in 2023-09-23T17:40:19.656 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:19 smithi055 ceph-mon[95848]: mgrmap e1: no daemons active 2023-09-23T17:40:20.422 INFO:teuthology.orchestra.run.smithi055.stdout:Wrote config to /etc/ceph/ceph.conf 2023-09-23T17:40:20.422 INFO:teuthology.orchestra.run.smithi055.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-09-23T17:40:20.422 INFO:teuthology.orchestra.run.smithi055.stdout:Creating mgr... 2023-09-23T17:40:20.423 INFO:teuthology.orchestra.run.smithi055.stdout:Verifying port 9283 ... 2023-09-23T17:40:20.556 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T17:40:20.556 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Failed to reset failed state of unit ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.y.service: Unit ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.y.service not loaded. 2023-09-23T17:40:20.651 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7.target.wants/ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.y.service → /etc/systemd/system/ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@.service. 2023-09-23T17:40:20.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:20 smithi055 systemd[1]: Starting Ceph mgr.y for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:40:21.249 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:20 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1488503678' entity='client.admin' 2023-09-23T17:40:21.249 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:20 smithi055 podman[96076]: 2023-09-23T17:40:21.249 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:21 smithi055 bash[96076]: 61647c07eed69c22a6d1d1c68612fc730900cdf9038ee315ef966af7be59fe44 2023-09-23T17:40:21.262 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T17:40:21.262 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout disabled 2023-09-23T17:40:21.270 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T17:40:21.270 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout inactive 2023-09-23T17:40:21.270 INFO:teuthology.orchestra.run.smithi055.stdout:firewalld.service is not enabled 2023-09-23T17:40:21.271 INFO:teuthology.orchestra.run.smithi055.stdout:Not possible to enable service . firewalld.service is not available 2023-09-23T17:40:21.280 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T17:40:21.281 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout disabled 2023-09-23T17:40:21.289 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T17:40:21.289 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout inactive 2023-09-23T17:40:21.289 INFO:teuthology.orchestra.run.smithi055.stdout:firewalld.service is not enabled 2023-09-23T17:40:21.290 INFO:teuthology.orchestra.run.smithi055.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-09-23T17:40:21.290 INFO:teuthology.orchestra.run.smithi055.stdout:Waiting for mgr to start... 2023-09-23T17:40:21.290 INFO:teuthology.orchestra.run.smithi055.stdout:Waiting for mgr... 2023-09-23T17:40:21.527 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:21 smithi055 systemd[1]: Started Ceph mgr.y for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:40:21.528 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:21 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:21.338+0000 7fe7204b21c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:40:21.528 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:21 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:21.433+0000 7fe7204b21c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:40:21.865 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:21 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:21.544+0000 7fe7204b21c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:40:21.865 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:21 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:21.659+0000 7fe7204b21c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:40:21.865 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:21 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:21.865+0000 7fe7204b21c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:40:22.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:21 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1692521641' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T17:40:22.238 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:22.241 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout { 2023-09-23T17:40:22.241 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "fsid": "2b70c030-5a38-11ee-8db4-212e2dc638e7", 2023-09-23T17:40:22.241 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T17:40:22.241 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 0 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum_age": 2, 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T17:40:22.243 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T17:40:22.244 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T17:40:22.247 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:22.247 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T17:40:22.247 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T17:40:22.247 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T17:40:22.247 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T17:40:22.247 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T17:40:22.247 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T17:40:22.247 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T17:40:22.247 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T17:40:22.248 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T17:40:13.381607+0000", 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:22.249 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T17:40:22.250 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout } 2023-09-23T17:40:22.250 INFO:teuthology.orchestra.run.smithi055.stdout:mgr not available, waiting (1/15)... 2023-09-23T17:40:22.526 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:22.274+0000 7fe7204b21c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:40:22.526 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:22.379+0000 7fe7204b21c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:40:22.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:22.527+0000 7fe7204b21c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:40:22.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:22.639+0000 7fe7204b21c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:40:23.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:23.156+0000 7fe7204b21c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:40:24.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:23.843+0000 7fe7204b21c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:40:24.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:24.031+0000 7fe7204b21c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:40:25.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:25.015+0000 7fe7204b21c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:40:25.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:25.124+0000 7fe7204b21c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:40:25.189 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:25.189 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout { 2023-09-23T17:40:25.189 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "fsid": "2b70c030-5a38-11ee-8db4-212e2dc638e7", 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 0 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T17:40:25.190 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum_age": 5, 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T17:40:25.191 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T17:40:25.192 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T17:40:25.192 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T17:40:25.195 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:25.195 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T17:40:25.195 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T17:40:25.195 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T17:40:25.195 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T17:40:25.195 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T17:40:25.195 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T17:40:25.195 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T17:40:25.195 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T17:40:25.196 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T17:40:13.381607+0000", 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout } 2023-09-23T17:40:25.197 INFO:teuthology.orchestra.run.smithi055.stdout:mgr not available, waiting (2/15)... 2023-09-23T17:40:25.677 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:25.338+0000 7fe7204b21c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:40:25.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:25 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3920582622' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T17:40:25.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:25.678+0000 7fe7204b21c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:40:25.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:25.781+0000 7fe7204b21c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:40:26.425 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:26.108+0000 7fe7204b21c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:40:26.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:26.425+0000 7fe7204b21c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:40:27.647 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:27.249+0000 7fe7204b21c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:40:27.647 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:27.410+0000 7fe7204b21c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:40:27.647 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:27.646+0000 7fe7204b21c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:40:27.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:27 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2831557431' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T17:40:27.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:27.785+0000 7fe7204b21c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:40:28.142 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:28.142 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout { 2023-09-23T17:40:28.142 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "fsid": "2b70c030-5a38-11ee-8db4-212e2dc638e7", 2023-09-23T17:40:28.142 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T17:40:28.142 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T17:40:28.142 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T17:40:28.142 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T17:40:28.142 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:28.142 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 0 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum_age": 8, 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T17:40:28.143 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T17:40:28.144 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T17:40:28.145 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "available": false, 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T17:40:28.146 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:28.147 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T17:40:13.381607+0000", 2023-09-23T17:40:28.147 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T17:40:28.147 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:28.148 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T17:40:28.148 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout } 2023-09-23T17:40:28.148 INFO:teuthology.orchestra.run.smithi055.stdout:mgr not available, waiting (3/15)... 2023-09-23T17:40:28.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:28 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:28.057+0000 7fe7204b21c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:40:29.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:29.088+0000 7fe7204b21c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:40:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: Activating manager daemon y 2023-09-23T17:40:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: mgrmap e2: y(active, starting, since 0.0030398s) 2023-09-23T17:40:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: from='mgr.14100 172.21.15.55:0/980162605' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:40:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: from='mgr.14100 172.21.15.55:0/980162605' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:40:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: from='mgr.14100 172.21.15.55:0/980162605' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:40:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: from='mgr.14100 172.21.15.55:0/980162605' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:40:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: from='mgr.14100 172.21.15.55:0/980162605' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:40:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: Manager daemon y is now available 2023-09-23T17:40:29.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: from='mgr.14100 172.21.15.55:0/980162605' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:40:29.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: from='mgr.14100 172.21.15.55:0/980162605' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:40:29.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: from='mgr.14100 172.21.15.55:0/980162605' entity='mgr.y' 2023-09-23T17:40:29.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: from='mgr.14100 172.21.15.55:0/980162605' entity='mgr.y' 2023-09-23T17:40:29.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:29 smithi055 ceph-mon[95848]: from='mgr.14100 172.21.15.55:0/980162605' entity='mgr.y' 2023-09-23T17:40:31.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:31 smithi055 ceph-mon[95848]: mgrmap e3: y(active, since 1.01202s) 2023-09-23T17:40:33.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:33 smithi055 ceph-mon[95848]: mgrmap e4: y(active, since 3s) 2023-09-23T17:40:33.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:33 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3147802089' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-09-23T17:40:35.704 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:35.707 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout { 2023-09-23T17:40:35.707 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "fsid": "2b70c030-5a38-11ee-8db4-212e2dc638e7", 2023-09-23T17:40:35.707 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "health": { 2023-09-23T17:40:35.707 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 0 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "a" 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:35.708 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "quorum_age": 13, 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "monmap": { 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-09-23T17:40:35.709 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-09-23T17:40:35.713 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:35.713 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-09-23T17:40:35.713 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-09-23T17:40:35.713 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-09-23T17:40:35.713 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-09-23T17:40:35.713 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-09-23T17:40:35.713 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-09-23T17:40:35.713 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-09-23T17:40:35.713 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-09-23T17:40:35.714 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "modules": [ 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "iostat", 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "nfs", 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "restful" 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ], 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "modified": "2023-09-23T17:40:13.381607+0000", 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "services": {} 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout }, 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-09-23T17:40:35.715 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout } 2023-09-23T17:40:35.716 INFO:teuthology.orchestra.run.smithi055.stdout:mgr is available 2023-09-23T17:40:38.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:38 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/4035553928' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-09-23T17:40:38.865 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:38.865 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout [global] 2023-09-23T17:40:38.865 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout fsid = 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:38.865 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-09-23T17:40:38.865 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-09-23T17:40:38.865 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-09-23T17:40:38.865 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-09-23T17:40:38.865 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:38.865 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout [mgr] 2023-09-23T17:40:38.865 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-09-23T17:40:38.866 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 2023-09-23T17:40:38.866 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout [osd] 2023-09-23T17:40:38.866 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-09-23T17:40:38.866 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-09-23T17:40:38.866 INFO:teuthology.orchestra.run.smithi055.stdout:Enabling cephadm module... 2023-09-23T17:40:40.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:40 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/790383095' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-09-23T17:40:40.667 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ignoring --setuser ceph since I am not root 2023-09-23T17:40:40.667 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ignoring --setgroup ceph since I am not root 2023-09-23T17:40:40.667 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:40.573+0000 7f28b65791c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:40:40.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:40.666+0000 7f28b65791c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:40:40.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:40.785+0000 7f28b65791c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:40:40.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:40.894+0000 7f28b65791c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:40:41.387 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:41 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:41.113+0000 7f28b65791c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:40:41.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:41 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/790383095' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-09-23T17:40:41.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:41 smithi055 ceph-mon[95848]: mgrmap e5: y(active, since 11s) 2023-09-23T17:40:41.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:41 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:41.511+0000 7f28b65791c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:40:41.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:41 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:41.617+0000 7f28b65791c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:40:41.962 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout { 2023-09-23T17:40:41.962 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-09-23T17:40:41.962 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T17:40:41.962 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T17:40:41.962 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T17:40:41.962 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout } 2023-09-23T17:40:41.962 INFO:teuthology.orchestra.run.smithi055.stdout:Waiting for the mgr to restart... 2023-09-23T17:40:41.962 INFO:teuthology.orchestra.run.smithi055.stdout:Waiting for mgr epoch 5... 2023-09-23T17:40:41.999 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:41 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:41.774+0000 7f28b65791c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:40:41.999 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:41 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:41.889+0000 7f28b65791c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:40:42.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:42 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2497672230' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T17:40:42.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:42 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:42.405+0000 7f28b65791c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:40:43.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:43.083+0000 7f28b65791c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:40:43.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:43.271+0000 7f28b65791c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:40:44.561 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:44 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:44.239+0000 7f28b65791c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:40:44.562 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:44 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:44.347+0000 7f28b65791c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:40:44.904 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:44 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:44.561+0000 7f28b65791c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:40:45.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:44 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:44.904+0000 7f28b65791c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:40:45.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:45 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:45.006+0000 7f28b65791c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:40:45.653 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:45 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:45.333+0000 7f28b65791c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:40:45.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:45 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:45.653+0000 7f28b65791c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:40:46.867 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:46 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:46.472+0000 7f28b65791c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:40:46.867 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:46 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:46.629+0000 7f28b65791c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:40:47.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:46 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:46.866+0000 7f28b65791c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:40:47.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:47.001+0000 7f28b65791c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:40:47.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:47.266+0000 7f28b65791c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:40:48.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:40:48.289+0000 7f28b65791c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:40:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: Active manager daemon y restarted 2023-09-23T17:40:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: Activating manager daemon y 2023-09-23T17:40:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: osdmap e2: 0 total, 0 up, 0 in 2023-09-23T17:40:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: mgrmap e6: y(active, starting, since 0.00642486s) 2023-09-23T17:40:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:40:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:40:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:40:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:40:48.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:40:48.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: Manager daemon y is now available 2023-09-23T17:40:48.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:40:48.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:40:48.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:40:48.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:48 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:40:49.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:49 smithi055 ceph-mon[95848]: Found migration_current of "None". Setting to last migration. 2023-09-23T17:40:49.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:49 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:40:49.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:49 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:40:49.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:49 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:40:49.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:49 smithi055 ceph-mon[95848]: mgrmap e7: y(active, since 1.00847s) 2023-09-23T17:40:49.716 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout { 2023-09-23T17:40:49.716 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-09-23T17:40:49.716 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T17:40:49.716 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout } 2023-09-23T17:40:49.717 INFO:teuthology.orchestra.run.smithi055.stdout:mgr epoch 5 is available 2023-09-23T17:40:49.717 INFO:teuthology.orchestra.run.smithi055.stdout:Setting orchestrator backend to cephadm... 2023-09-23T17:40:50.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:50 smithi055 ceph-mon[95848]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T17:40:50.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:50 smithi055 ceph-mon[95848]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T17:40:50.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:50 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:40:50.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:50 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:40:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:51 smithi055 ceph-mon[95848]: [23/Sep/2023:17:40:49] ENGINE Bus STARTING 2023-09-23T17:40:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:51 smithi055 ceph-mon[95848]: [23/Sep/2023:17:40:49] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:40:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:51 smithi055 ceph-mon[95848]: [23/Sep/2023:17:40:49] ENGINE Bus STARTED 2023-09-23T17:40:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:51 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:40:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:51 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:40:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:51 smithi055 ceph-mon[95848]: mgrmap e8: y(active, since 2s) 2023-09-23T17:40:51.997 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout value unchanged 2023-09-23T17:40:51.998 INFO:teuthology.orchestra.run.smithi055.stdout:Generating ssh key... 2023-09-23T17:40:52.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:52 smithi055 ceph-mon[95848]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:40:53.664 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-mon[95848]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:40:53.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: Generating public/private rsa key pair. 2023-09-23T17:40:53.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: Your identification has been saved in /tmp/tmpqpnyx5u0/key. 2023-09-23T17:40:53.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: Your public key has been saved in /tmp/tmpqpnyx5u0/key.pub. 2023-09-23T17:40:53.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: The key fingerprint is: 2023-09-23T17:40:53.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: SHA256:WW7TsRevto8qqzkWGKFtcVNJq6ek52J6r7DfVrIu++U ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:53.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: The key's randomart image is: 2023-09-23T17:40:53.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: +---[RSA 3072]----+ 2023-09-23T17:40:53.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: | oo. | 2023-09-23T17:40:53.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: | o o .. | 2023-09-23T17:40:53.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: | o + .o . . | 2023-09-23T17:40:53.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: | . + = . o o | 2023-09-23T17:40:53.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: | . oS = o . .| 2023-09-23T17:40:53.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: | .oo+.. . . | 2023-09-23T17:40:53.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: | . . o=. o | 2023-09-23T17:40:53.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: | o=+=+. . o | 2023-09-23T17:40:53.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: | o=+@OoEo..o..| 2023-09-23T17:40:53.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:40:53 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: +----[SHA256]-----+ 2023-09-23T17:40:54.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:54 smithi055 ceph-mon[95848]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:40:54.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:54 smithi055 ceph-mon[95848]: Generating ssh key... 2023-09-23T17:40:54.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:54 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:40:54.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:54 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:40:55.183 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDdvCpjqu5xB6mwIFCklDcDHk4Bc3nV+9oIq7fCNeSlBRsC/bzyfju1fD1r/M17+hWZ40js87eFM7WvFTfAkOdHF0w8KtJPrkTeY815k8S6FvgtXtZgWQY8GfSxXGdZIUxAPJ7d0X7gCPj9jnTbOp4p3xVztqcjD9D3oSgYTO+C0wU6rdA7Wf4yBBO7IbBzca3P2PMOgNwlXhVieUZkGCSbpeQcU1DRycBe/Ecnt+dB+DGAS29+buIgFs3j/srCGcbIUBBZuKqwpOCUS8eVShhPSyXnCVh/LXO9DOkLdnNxe8do+Z01DEBcG6kfm5xqNatSyloumFDiSj4aer+oq0HtSnkbCdy05g5o91Shi8P5p58A9qENPWuafJFmjroTMM8WFWvfiHXMQwHmKAUy/6h5WqOFMYOT5maT9GoLB7PKVLHcNUq7tsLjsGUg+xXQK+JdsusAsS5gzDQQkI8n61yg0YeTkE1neCY2E84x/CdjwXQWclcj1pT650VyuL/43sk= ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:40:55.183 INFO:teuthology.orchestra.run.smithi055.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-09-23T17:40:55.183 INFO:teuthology.orchestra.run.smithi055.stdout:Adding key to root@localhost authorized_keys... 2023-09-23T17:40:55.184 INFO:teuthology.orchestra.run.smithi055.stdout:Adding host smithi055... 2023-09-23T17:40:55.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:55 smithi055 ceph-mon[95848]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:40:56.539 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:56 smithi055 ceph-mon[95848]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi055", "addr": "172.21.15.55", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:40:57.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:57 smithi055 ceph-mon[95848]: Deploying cephadm binary to smithi055 2023-09-23T17:40:58.330 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout Added host 'smithi055' with addr '172.21.15.55' 2023-09-23T17:40:58.330 INFO:teuthology.orchestra.run.smithi055.stdout:Deploying unmanaged mon service... 2023-09-23T17:40:59.328 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:59 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:40:59.329 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:59 smithi055 ceph-mon[95848]: Added host smithi055 2023-09-23T17:40:59.329 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:40:59 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:40:59.672 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-09-23T17:40:59.672 INFO:teuthology.orchestra.run.smithi055.stdout:Deploying unmanaged mgr service... 2023-09-23T17:41:00.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:00 smithi055 ceph-mon[95848]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:41:00.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:00 smithi055 ceph-mon[95848]: Saving service mon spec with placement count:5 2023-09-23T17:41:00.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:00 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:41:01.015 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-09-23T17:41:01.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:01 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:41:01.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:01 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:41:01.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:01 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:41:02.466 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:02 smithi055 ceph-mon[95848]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:41:02.466 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:02 smithi055 ceph-mon[95848]: Saving service mgr spec with placement count:2 2023-09-23T17:41:02.466 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:02 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2811784363' entity='client.admin' 2023-09-23T17:41:03.350 INFO:teuthology.orchestra.run.smithi055.stdout:Enabling the dashboard module... 2023-09-23T17:41:03.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:03 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:41:03.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:03 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/4248386100' entity='client.admin' 2023-09-23T17:41:03.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:03 smithi055 ceph-mon[95848]: from='mgr.14120 172.21.15.55:0/1918943946' entity='mgr.y' 2023-09-23T17:41:04.874 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:04 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1797932719' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-09-23T17:41:04.874 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ignoring --setuser ceph since I am not root 2023-09-23T17:41:04.874 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ignoring --setgroup ceph since I am not root 2023-09-23T17:41:04.874 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:04.873+0000 7fb5cdbfa1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:41:05.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:04.971+0000 7fb5cdbfa1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:41:05.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:05.078+0000 7fb5cdbfa1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:41:05.456 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:05.187+0000 7fb5cdbfa1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:41:05.457 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:05.395+0000 7fb5cdbfa1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:41:05.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:05 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1797932719' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-09-23T17:41:05.802 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:05 smithi055 ceph-mon[95848]: mgrmap e9: y(active, since 16s) 2023-09-23T17:41:06.057 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:05.801+0000 7fb5cdbfa1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:41:06.057 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:05.908+0000 7fb5cdbfa1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:41:06.396 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:06.057+0000 7fb5cdbfa1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:41:06.396 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:06.169+0000 7fb5cdbfa1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:41:06.542 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout { 2023-09-23T17:41:06.543 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-09-23T17:41:06.543 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "available": true, 2023-09-23T17:41:06.543 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-09-23T17:41:06.543 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-09-23T17:41:06.543 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout } 2023-09-23T17:41:06.543 INFO:teuthology.orchestra.run.smithi055.stdout:Waiting for the mgr to restart... 2023-09-23T17:41:06.543 INFO:teuthology.orchestra.run.smithi055.stdout:Waiting for mgr epoch 9... 2023-09-23T17:41:06.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:06 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3079176584' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-09-23T17:41:06.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:06.686+0000 7fb5cdbfa1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:41:07.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:07 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:07.380+0000 7fb5cdbfa1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:41:07.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:07 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:07.569+0000 7fb5cdbfa1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:41:08.843 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:08 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:08.520+0000 7fb5cdbfa1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:41:08.843 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:08 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:08.628+0000 7fb5cdbfa1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:41:09.615 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:08 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:08.842+0000 7fb5cdbfa1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:41:09.616 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:09 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:09.183+0000 7fb5cdbfa1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:41:09.616 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:09 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:09.287+0000 7fb5cdbfa1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:41:09.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:09 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:09.615+0000 7fb5cdbfa1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:41:10.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:09 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:09.936+0000 7fb5cdbfa1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:41:11.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:10.788+0000 7fb5cdbfa1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:41:11.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:10.946+0000 7fb5cdbfa1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:41:11.584 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:11 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:11.184+0000 7fb5cdbfa1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:41:11.585 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:11 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:11.320+0000 7fb5cdbfa1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:41:11.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:11 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:11.585+0000 7fb5cdbfa1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:41:12.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:41:12.610+0000 7fb5cdbfa1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:41:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-mon[95848]: Active manager daemon y restarted 2023-09-23T17:41:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-mon[95848]: Activating manager daemon y 2023-09-23T17:41:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-mon[95848]: osdmap e3: 0 total, 0 up, 0 in 2023-09-23T17:41:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-mon[95848]: mgrmap e10: y(active, starting, since 0.0124957s) 2023-09-23T17:41:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:41:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:41:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:41:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:41:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:41:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:12 smithi055 ceph-mon[95848]: Manager daemon y is now available 2023-09-23T17:41:13.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:13 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:13.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:13 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:13 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:41:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:13 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:41:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:13 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:13 smithi055 ceph-mon[95848]: mgrmap e11: y(active, since 1.0142s) 2023-09-23T17:41:14.000 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout { 2023-09-23T17:41:14.000 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-09-23T17:41:14.000 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout "initialized": true 2023-09-23T17:41:14.000 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout } 2023-09-23T17:41:14.000 INFO:teuthology.orchestra.run.smithi055.stdout:mgr epoch 9 is available 2023-09-23T17:41:14.001 INFO:teuthology.orchestra.run.smithi055.stdout:Generating a dashboard self-signed certificate... 2023-09-23T17:41:14.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:14 smithi055 ceph-mon[95848]: [23/Sep/2023:17:41:13] ENGINE Bus STARTING 2023-09-23T17:41:14.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:14 smithi055 ceph-mon[95848]: [23/Sep/2023:17:41:13] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:41:14.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:14 smithi055 ceph-mon[95848]: [23/Sep/2023:17:41:13] ENGINE Bus STARTED 2023-09-23T17:41:14.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:14 smithi055 ceph-mon[95848]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-09-23T17:41:14.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:14 smithi055 ceph-mon[95848]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-09-23T17:41:14.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:14 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:15.244 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-09-23T17:41:15.244 INFO:teuthology.orchestra.run.smithi055.stdout:Creating initial admin user... 2023-09-23T17:41:16.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:15 smithi055 ceph-mon[95848]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:41:16.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:15 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:15 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:15 smithi055 ceph-mon[95848]: mgrmap e12: y(active, since 2s) 2023-09-23T17:41:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:15 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:16.696 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$6wJQdnX/oLkzHjiUef2.k.get2WV4AgN8Hw9zSj73.d2A4qE2Ri2y", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1695490876, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-09-23T17:41:16.696 INFO:teuthology.orchestra.run.smithi055.stdout:Fetching dashboard port number... 2023-09-23T17:41:17.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:17 smithi055 ceph-mon[95848]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:41:17.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:17 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:18.139 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stdout 8443 2023-09-23T17:41:18.149 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-09-23T17:41:18.149 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout disabled 2023-09-23T17:41:18.156 INFO:teuthology.orchestra.run.smithi055.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-09-23T17:41:18.156 INFO:teuthology.orchestra.run.smithi055.stdout:systemctl: stdout inactive 2023-09-23T17:41:18.156 INFO:teuthology.orchestra.run.smithi055.stdout:firewalld.service is not enabled 2023-09-23T17:41:18.156 INFO:teuthology.orchestra.run.smithi055.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-09-23T17:41:18.158 INFO:teuthology.orchestra.run.smithi055.stdout:Ceph Dashboard is now available at: 2023-09-23T17:41:18.158 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:18.158 INFO:teuthology.orchestra.run.smithi055.stdout: URL: https://smithi055:8443/ 2023-09-23T17:41:18.158 INFO:teuthology.orchestra.run.smithi055.stdout: User: admin 2023-09-23T17:41:18.158 INFO:teuthology.orchestra.run.smithi055.stdout: Password: 3gsi50dgkr 2023-09-23T17:41:18.159 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:18.159 INFO:teuthology.orchestra.run.smithi055.stdout:Saving cluster configuration to /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config directory 2023-09-23T17:41:18.159 INFO:teuthology.orchestra.run.smithi055.stdout:Enabling autotune for osd_memory_target 2023-09-23T17:41:18.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:18 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/524294730' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-09-23T17:41:20.645 INFO:teuthology.orchestra.run.smithi055.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-09-23T17:41:20.645 INFO:teuthology.orchestra.run.smithi055.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-09-23T17:41:20.645 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:20.645 INFO:teuthology.orchestra.run.smithi055.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-09-23T17:41:20.645 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:20.645 INFO:teuthology.orchestra.run.smithi055.stdout:Or, if you are only running a single cluster on this host: 2023-09-23T17:41:20.645 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout:Please consider enabling telemetry to help improve Ceph: 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout: ceph telemetry on 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout:For more information see: 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:20.646 INFO:teuthology.orchestra.run.smithi055.stdout:Bootstrap complete. 2023-09-23T17:41:20.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:20 smithi055 ceph-mon[95848]: mgrmap e13: y(active, since 6s) 2023-09-23T17:41:20.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:20 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2988952287' entity='client.admin' 2023-09-23T17:41:20.682 INFO:tasks.cephadm:Fetching config... 2023-09-23T17:41:20.682 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:41:20.682 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-09-23T17:41:20.700 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-09-23T17:41:20.700 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:41:20.700 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-09-23T17:41:20.757 INFO:tasks.cephadm:Fetching mon keyring... 2023-09-23T17:41:20.757 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:41:20.757 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/keyring of=/dev/stdout 2023-09-23T17:41:20.825 INFO:tasks.cephadm:Fetching pub ssh key... 2023-09-23T17:41:20.825 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:41:20.825 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-09-23T17:41:20.883 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-09-23T17:41:20.883 DEBUG:teuthology.orchestra.run.smithi055:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDdvCpjqu5xB6mwIFCklDcDHk4Bc3nV+9oIq7fCNeSlBRsC/bzyfju1fD1r/M17+hWZ40js87eFM7WvFTfAkOdHF0w8KtJPrkTeY815k8S6FvgtXtZgWQY8GfSxXGdZIUxAPJ7d0X7gCPj9jnTbOp4p3xVztqcjD9D3oSgYTO+C0wU6rdA7Wf4yBBO7IbBzca3P2PMOgNwlXhVieUZkGCSbpeQcU1DRycBe/Ecnt+dB+DGAS29+buIgFs3j/srCGcbIUBBZuKqwpOCUS8eVShhPSyXnCVh/LXO9DOkLdnNxe8do+Z01DEBcG6kfm5xqNatSyloumFDiSj4aer+oq0HtSnkbCdy05g5o91Shi8P5p58A9qENPWuafJFmjroTMM8WFWvfiHXMQwHmKAUy/6h5WqOFMYOT5maT9GoLB7PKVLHcNUq7tsLjsGUg+xXQK+JdsusAsS5gzDQQkI8n61yg0YeTkE1neCY2E84x/CdjwXQWclcj1pT650VyuL/43sk= ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T17:41:20.965 INFO:teuthology.orchestra.run.smithi055.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDdvCpjqu5xB6mwIFCklDcDHk4Bc3nV+9oIq7fCNeSlBRsC/bzyfju1fD1r/M17+hWZ40js87eFM7WvFTfAkOdHF0w8KtJPrkTeY815k8S6FvgtXtZgWQY8GfSxXGdZIUxAPJ7d0X7gCPj9jnTbOp4p3xVztqcjD9D3oSgYTO+C0wU6rdA7Wf4yBBO7IbBzca3P2PMOgNwlXhVieUZkGCSbpeQcU1DRycBe/Ecnt+dB+DGAS29+buIgFs3j/srCGcbIUBBZuKqwpOCUS8eVShhPSyXnCVh/LXO9DOkLdnNxe8do+Z01DEBcG6kfm5xqNatSyloumFDiSj4aer+oq0HtSnkbCdy05g5o91Shi8P5p58A9qENPWuafJFmjroTMM8WFWvfiHXMQwHmKAUy/6h5WqOFMYOT5maT9GoLB7PKVLHcNUq7tsLjsGUg+xXQK+JdsusAsS5gzDQQkI8n61yg0YeTkE1neCY2E84x/CdjwXQWclcj1pT650VyuL/43sk= ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:41:20.977 DEBUG:teuthology.orchestra.run.smithi099:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDdvCpjqu5xB6mwIFCklDcDHk4Bc3nV+9oIq7fCNeSlBRsC/bzyfju1fD1r/M17+hWZ40js87eFM7WvFTfAkOdHF0w8KtJPrkTeY815k8S6FvgtXtZgWQY8GfSxXGdZIUxAPJ7d0X7gCPj9jnTbOp4p3xVztqcjD9D3oSgYTO+C0wU6rdA7Wf4yBBO7IbBzca3P2PMOgNwlXhVieUZkGCSbpeQcU1DRycBe/Ecnt+dB+DGAS29+buIgFs3j/srCGcbIUBBZuKqwpOCUS8eVShhPSyXnCVh/LXO9DOkLdnNxe8do+Z01DEBcG6kfm5xqNatSyloumFDiSj4aer+oq0HtSnkbCdy05g5o91Shi8P5p58A9qENPWuafJFmjroTMM8WFWvfiHXMQwHmKAUy/6h5WqOFMYOT5maT9GoLB7PKVLHcNUq7tsLjsGUg+xXQK+JdsusAsS5gzDQQkI8n61yg0YeTkE1neCY2E84x/CdjwXQWclcj1pT650VyuL/43sk= ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-09-23T17:41:21.022 INFO:teuthology.orchestra.run.smithi099.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDdvCpjqu5xB6mwIFCklDcDHk4Bc3nV+9oIq7fCNeSlBRsC/bzyfju1fD1r/M17+hWZ40js87eFM7WvFTfAkOdHF0w8KtJPrkTeY815k8S6FvgtXtZgWQY8GfSxXGdZIUxAPJ7d0X7gCPj9jnTbOp4p3xVztqcjD9D3oSgYTO+C0wU6rdA7Wf4yBBO7IbBzca3P2PMOgNwlXhVieUZkGCSbpeQcU1DRycBe/Ecnt+dB+DGAS29+buIgFs3j/srCGcbIUBBZuKqwpOCUS8eVShhPSyXnCVh/LXO9DOkLdnNxe8do+Z01DEBcG6kfm5xqNatSyloumFDiSj4aer+oq0HtSnkbCdy05g5o91Shi8P5p58A9qENPWuafJFmjroTMM8WFWvfiHXMQwHmKAUy/6h5WqOFMYOT5maT9GoLB7PKVLHcNUq7tsLjsGUg+xXQK+JdsusAsS5gzDQQkI8n61yg0YeTkE1neCY2E84x/CdjwXQWclcj1pT650VyuL/43sk= ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:41:21.035 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-09-23T17:41:21.644 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:41:23.557 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:23.557 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:23.557 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi055", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:41:23.558 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:23.558 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:23.558 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:23.558 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:23 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2898647875' entity='client.admin' 2023-09-23T17:41:23.558 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:23.559 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:23.562 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-09-23T17:41:23.562 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-09-23T17:41:24.289 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:41:25.952 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi099 2023-09-23T17:41:25.952 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:41:25.952 DEBUG:teuthology.orchestra.run.smithi099:> dd of=/etc/ceph/ceph.conf 2023-09-23T17:41:25.972 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:41:25.973 DEBUG:teuthology.orchestra.run.smithi099:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:41:26.032 INFO:tasks.cephadm:Adding host smithi099 to orchestrator... 2023-09-23T17:41:26.033 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch host add smithi099 2023-09-23T17:41:26.646 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:41:26.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:26 smithi055 ceph-mon[95848]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:41:26.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:26 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:26.841 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:26 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:26.841 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:26 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:26.841 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:26 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:41:26.841 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:26 smithi055 ceph-mon[95848]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:41:27.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:27 smithi055 ceph-mon[95848]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:41:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:27 smithi055 ceph-mon[95848]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:41:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:27 smithi055 ceph-mon[95848]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:41:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:27 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:27 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:27 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:29.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:28 smithi055 ceph-mon[95848]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi099", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:41:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:28 smithi055 ceph-mon[95848]: Deploying cephadm binary to smithi099 2023-09-23T17:41:29.998 INFO:teuthology.orchestra.run.smithi055.stdout:Added host 'smithi099' with addr '172.21.15.99' 2023-09-23T17:41:30.296 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch host ls --format=json 2023-09-23T17:41:30.900 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:41:31.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:30 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:31.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:30 smithi055 ceph-mon[95848]: Added host smithi099 2023-09-23T17:41:31.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:30 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:31.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:30 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:32.366 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:41:32.366 INFO:teuthology.orchestra.run.smithi055.stdout:[{"addr": "172.21.15.55", "hostname": "smithi055", "labels": [], "status": ""}, {"addr": "172.21.15.99", "hostname": "smithi099", "labels": [], "status": ""}] 2023-09-23T17:41:32.392 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:32 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:32.757 INFO:tasks.cephadm:Setting crush tunables to default 2023-09-23T17:41:32.757 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd crush tunables default 2023-09-23T17:41:33.360 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:41:33.745 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:33 smithi055 ceph-mon[95848]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:41:33.745 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:33 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:33.745 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:33 smithi055 ceph-mon[95848]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:35.400 INFO:teuthology.orchestra.run.smithi055.stderr:adjusted tunables profile to default 2023-09-23T17:41:35.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:35 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2552106868' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-09-23T17:41:35.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:35 smithi055 ceph-mon[95848]: pgmap v4: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:35.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:35 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:35.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:35 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:35.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:35 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:35.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:35 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:35.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:35 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi099", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:41:35.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:35 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:35.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:35 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:41:35.708 INFO:tasks.cephadm:Adding mon.a on smithi055 2023-09-23T17:41:35.708 INFO:tasks.cephadm:Adding mon.c on smithi055 2023-09-23T17:41:35.708 INFO:tasks.cephadm:Adding mon.b on smithi099 2023-09-23T17:41:35.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch apply mon '3;smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi099:172.21.15.99=b' 2023-09-23T17:41:36.352 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:41:36.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:36 smithi055 ceph-mon[95848]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:41:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:36 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2552106868' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-09-23T17:41:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:36 smithi055 ceph-mon[95848]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T17:41:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:36 smithi055 ceph-mon[95848]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:41:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:36 smithi055 ceph-mon[95848]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:41:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:36 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:36 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:36 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:37.645 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled mon update... 2023-09-23T17:41:37.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:37 smithi055 ceph-mon[95848]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:41:37.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:37 smithi055 ceph-mon[95848]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:38.028 DEBUG:teuthology.orchestra.run.smithi055:mon.c> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.c.service 2023-09-23T17:41:38.031 DEBUG:teuthology.orchestra.run.smithi099:mon.b> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.b.service 2023-09-23T17:41:38.034 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T17:41:38.034 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T17:41:38.061 INFO:journalctl@ceph.mon.b.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T17:41:38.082 INFO:journalctl@ceph.mon.c.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T17:41:38.903 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:41:38.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:38 smithi055 ceph-mon[95848]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi099:172.21.15.99=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:41:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:38 smithi055 ceph-mon[95848]: Saving service mon spec with placement smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi099:172.21.15.99=b;count:3 2023-09-23T17:41:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:41:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:38 smithi055 ceph-mon[95848]: Deploying daemon mon.b on smithi099 2023-09-23T17:41:39.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:39 smithi055 ceph-mon[95848]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:40.796 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:40 smithi099 systemd[1]: Starting Ceph mon.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:41:40.835 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:41:40.835 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":1,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","modified":"2023-09-23T17:40:11.277823Z","created":"2023-09-23T17:40:11.277823Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3300","nonce":0},{"type":"v1","addr":"172.21.15.55:6789","nonce":0}]},"addr":"172.21.15.55:6789/0","public_addr":"172.21.15.55:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-09-23T17:41:40.837 INFO:teuthology.orchestra.run.smithi099.stderr:dumped monmap epoch 1 2023-09-23T17:41:41.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:40 smithi099 podman[100204]: 2023-09-23T17:41:41.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: pidfile_write: ignore empty --pid-file 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: load: jerasure load: lrc 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: RocksDB version: 6.15.5 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Compile date Sep 22 2023 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: DB SUMMARY 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: DB Session ID: OQ3LVZ4HNN9UZGOHCGHO 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: CURRENT file: CURRENT 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T17:41:41.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.error_if_exists: 0 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.create_if_missing: 0 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.paranoid_checks: 1 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.env: 0x55ec30e07a80 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.fs: Posix File System 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.info_log: 0x55ec319e7d80 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T17:41:41.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.statistics: (nil) 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.use_fsync: 0 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_log_file_size: 0 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 bash[100204]: 5f0fb2e87ea104a4656dd215e36723cc0308c338ec0630e7e36d835fbbe00b33 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.allow_fallocate: 1 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T17:41:41.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.use_direct_reads: 0 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.db_log_dir: 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T17:41:41.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.write_buffer_manager: 0x55ec326da960 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T17:41:41.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.unordered_write: 0 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.row_cache: None 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.wal_filter: None 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.preserve_deletes: 0 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.two_write_queues: 0 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T17:41:41.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.atomic_flush: 0 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.log_readahead_size: 0 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T17:41:41.085 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_background_jobs: 2 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_background_compactions: -1 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_subcompactions: 1 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T17:41:41.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_open_files: -1 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_background_flushes: -1 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Compression algorithms supported: 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: kZSTD supported: 0 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: kXpressCompression supported: 0 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: kZlibCompression supported: 1 2023-09-23T17:41:41.087 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: kSnappyCompression supported: 1 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: kLZ4Compression supported: 1 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: kBZip2Compression supported: 0 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.merge_operator: 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_filter: None 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T17:41:41.088 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55ec319bafa0) 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: cache_index_and_filter_blocks: 1 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: pin_top_level_index_and_filter: 1 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_type: 0 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_block_index_type: 0 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_shortening: 1 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: hash_index_allow_collision: 1 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: checksum: 1 2023-09-23T17:41:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout: no_block_cache: 0 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache: 0x55ec319ed090 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_name: BinnedLRUCache 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_options: 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: capacity : 536870912 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: num_shard_bits : 4 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: strict_capacity_limit : 0 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: high_pri_pool_ratio: 0.000 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_compressed: (nil) 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: persistent_cache: (nil) 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_size: 4096 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_size_deviation: 10 2023-09-23T17:41:41.090 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_restart_interval: 16 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_block_restart_interval: 1 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: metadata_block_size: 4096 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: partition_filters: 0 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: use_delta_encoding: 1 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: whole_key_filtering: 1 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: verify_compression: 0 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: read_amp_bytes_per_bit: 0 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: format_version: 4 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: enable_index_compression: 1 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_align: 0 2023-09-23T17:41:41.091 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compression: NoCompression 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.num_levels: 7 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T17:41:41.092 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T17:41:41.093 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T17:41:41.094 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T17:41:41.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.table_properties_collectors: 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.inplace_update_support: 0 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T17:41:41.096 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.bloom_locality: 0 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.max_successive_merges: 0 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.ttl: 2592000 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.enable_blob_files: false 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.min_blob_size: 0 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T17:41:41.097 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695490901021892, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695490901022157, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695490901, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "c77aa3da-3c8d-4a63-a689-12e97feda96e", "db_session_id": "OQ3LVZ4HNN9UZGOHCGHO"}} 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T17:41:41.098 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695490901022461, "job": 1, "event": "recovery_finished"} 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55ec31981180 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: DB pointer 0x55ec31a56000 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** DB Stats ** 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T17:41:41.099 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T17:41:41.100 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:41:41.101 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:41:41.102 INFO:journalctl@ceph.mon.b.smithi099.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative compaction: 0.00 GB write, 0.59 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: using public_addr v2:172.21.15.99:0/0 -> [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] 2023-09-23T17:41:41.103 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: starting mon.b rank -1 at public addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] at bind addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(???) e0 preinit fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).mds e1 new map 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).mds e1 print_map 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout: e1 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout: legacy client fscid: -1 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout: No filesystems configured 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T17:41:41.104 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi099:172.21.15.99=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: Saving service mon spec with placement smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi099:172.21.15.99=b;count:3 2023-09-23T17:41:41.105 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:41.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:41.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:41.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:41:41.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:41.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:41.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:41.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: Deploying daemon mon.b on smithi099 2023-09-23T17:41:41.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:41.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T17:41:41.106 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 ceph-mon[100270]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-09-23T17:41:41.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:41 smithi099 systemd[1]: Started Ceph mon.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:41:42.263 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T17:41:42.263 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T17:41:42.866 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:41:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:41:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: mon.a calling monitor election 2023-09-23T17:41:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: mon.b calling monitor election 2023-09-23T17:41:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:46.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T17:41:46.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: monmap e2: 2 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0]} removed_ranks: {} 2023-09-23T17:41:46.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: fsmap 2023-09-23T17:41:46.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T17:41:46.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: mgrmap e13: y(active, since 33s) 2023-09-23T17:41:46.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: overall HEALTH_OK 2023-09-23T17:41:46.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:46.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:46.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:41:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: mon.a calling monitor election 2023-09-23T17:41:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: mon.b calling monitor election 2023-09-23T17:41:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:46.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T17:41:46.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: monmap e2: 2 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0]} removed_ranks: {} 2023-09-23T17:41:46.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: fsmap 2023-09-23T17:41:46.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T17:41:46.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: mgrmap e13: y(active, since 33s) 2023-09-23T17:41:46.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: overall HEALTH_OK 2023-09-23T17:41:46.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:46.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:47.222 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:41:47.222 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":2,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","modified":"2023-09-23T17:41:41.042456Z","created":"2023-09-23T17:40:11.277823Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3300","nonce":0},{"type":"v1","addr":"172.21.15.55:6789","nonce":0}]},"addr":"172.21.15.55:6789/0","public_addr":"172.21.15.55:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:3300","nonce":0},{"type":"v1","addr":"172.21.15.99:6789","nonce":0}]},"addr":"172.21.15.99:6789/0","public_addr":"172.21.15.99:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T17:41:47.224 INFO:teuthology.orchestra.run.smithi099.stderr:dumped monmap epoch 2 2023-09-23T17:41:47.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:47 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:47 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:47 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:47 smithi055 ceph-mon[95848]: Deploying daemon mon.c on smithi055 2023-09-23T17:41:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:47 smithi055 ceph-mon[95848]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:47 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:47.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:47 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:47.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:47 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:47.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:47 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:47.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:47 smithi099 ceph-mon[100270]: Deploying daemon mon.c on smithi055 2023-09-23T17:41:47.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:47 smithi099 ceph-mon[100270]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:47.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:47 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/422472481' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T17:41:48.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 systemd[1]: Starting Ceph mon.c for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:41:48.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:48 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/422472481' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T17:41:48.615 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-09-23T17:41:48.615 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph mon dump -f json 2023-09-23T17:41:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 podman[109872]: 2023-09-23T17:41:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 bash[109872]: 549a518b554d1ad562e8cb52215e42da42938f3d9658d5d816c241bfcf4b42d2 2023-09-23T17:41:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable), process ceph-mon, pid 7 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: pidfile_write: ignore empty --pid-file 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: load: jerasure load: lrc 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: RocksDB version: 6.15.5 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Compile date Sep 22 2023 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: DB SUMMARY 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: DB Session ID: FYXMNENAI02GR295IPGF 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: CURRENT file: CURRENT 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T17:41:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-09-23T17:41:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-09-23T17:41:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-09-23T17:41:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.error_if_exists: 0 2023-09-23T17:41:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.create_if_missing: 0 2023-09-23T17:41:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.paranoid_checks: 1 2023-09-23T17:41:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T17:41:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.env: 0x55c067a9aa80 2023-09-23T17:41:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.fs: Posix File System 2023-09-23T17:41:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.info_log: 0x55c068895d80 2023-09-23T17:41:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.statistics: (nil) 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.use_fsync: 0 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_log_file_size: 0 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.allow_fallocate: 1 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T17:41:48.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.use_direct_reads: 0 2023-09-23T17:41:48.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T17:41:48.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T17:41:48.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.db_log_dir: 2023-09-23T17:41:48.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-09-23T17:41:48.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T17:41:48.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T17:41:48.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T17:41:48.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T17:41:48.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T17:41:48.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T17:41:48.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T17:41:48.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T17:41:48.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.write_buffer_manager: 0x55c069588930 2023-09-23T17:41:48.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T17:41:48.936 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.unordered_write: 0 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T17:41:48.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T17:41:48.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T17:41:48.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.row_cache: None 2023-09-23T17:41:48.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.wal_filter: None 2023-09-23T17:41:48.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T17:41:48.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T17:41:48.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.preserve_deletes: 0 2023-09-23T17:41:48.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.two_write_queues: 0 2023-09-23T17:41:48.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T17:41:48.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.atomic_flush: 0 2023-09-23T17:41:48.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T17:41:48.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T17:41:48.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T17:41:48.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.log_readahead_size: 0 2023-09-23T17:41:48.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T17:41:48.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T17:41:48.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T17:41:48.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T17:41:48.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T17:41:48.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T17:41:48.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_background_jobs: 2 2023-09-23T17:41:48.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_background_compactions: -1 2023-09-23T17:41:48.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_subcompactions: 1 2023-09-23T17:41:48.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T17:41:48.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T17:41:48.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T17:41:48.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T17:41:48.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T17:41:48.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T17:41:48.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T17:41:48.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T17:41:48.941 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_open_files: -1 2023-09-23T17:41:48.941 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T17:41:48.941 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T17:41:48.941 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T17:41:48.941 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T17:41:48.941 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_background_flushes: -1 2023-09-23T17:41:48.941 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Compression algorithms supported: 2023-09-23T17:41:48.941 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T17:41:48.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: kZSTD supported: 0 2023-09-23T17:41:48.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: kXpressCompression supported: 0 2023-09-23T17:41:48.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T17:41:48.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: kZlibCompression supported: 1 2023-09-23T17:41:48.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: kSnappyCompression supported: 1 2023-09-23T17:41:48.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: kLZ4Compression supported: 1 2023-09-23T17:41:48.942 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: kBZip2Compression supported: 0 2023-09-23T17:41:48.943 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T17:41:48.943 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-09-23T17:41:48.943 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-09-23T17:41:48.943 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T17:41:48.943 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.merge_operator: 2023-09-23T17:41:48.943 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_filter: None 2023-09-23T17:41:48.944 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T17:41:48.944 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T17:41:48.944 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T17:41:48.944 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T17:41:48.944 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55c068868fa0) 2023-09-23T17:41:48.944 INFO:journalctl@ceph.mon.c.smithi055.stdout: cache_index_and_filter_blocks: 1 2023-09-23T17:41:48.944 INFO:journalctl@ceph.mon.c.smithi055.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T17:41:48.944 INFO:journalctl@ceph.mon.c.smithi055.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T17:41:48.945 INFO:journalctl@ceph.mon.c.smithi055.stdout: pin_top_level_index_and_filter: 1 2023-09-23T17:41:48.945 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_type: 0 2023-09-23T17:41:48.945 INFO:journalctl@ceph.mon.c.smithi055.stdout: data_block_index_type: 0 2023-09-23T17:41:48.945 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_shortening: 1 2023-09-23T17:41:48.945 INFO:journalctl@ceph.mon.c.smithi055.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T17:41:48.946 INFO:journalctl@ceph.mon.c.smithi055.stdout: hash_index_allow_collision: 1 2023-09-23T17:41:48.946 INFO:journalctl@ceph.mon.c.smithi055.stdout: checksum: 1 2023-09-23T17:41:48.946 INFO:journalctl@ceph.mon.c.smithi055.stdout: no_block_cache: 0 2023-09-23T17:41:48.946 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache: 0x55c06889b090 2023-09-23T17:41:48.946 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_name: BinnedLRUCache 2023-09-23T17:41:48.947 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_options: 2023-09-23T17:41:48.947 INFO:journalctl@ceph.mon.c.smithi055.stdout: capacity : 536870912 2023-09-23T17:41:48.947 INFO:journalctl@ceph.mon.c.smithi055.stdout: num_shard_bits : 4 2023-09-23T17:41:48.947 INFO:journalctl@ceph.mon.c.smithi055.stdout: strict_capacity_limit : 0 2023-09-23T17:41:48.947 INFO:journalctl@ceph.mon.c.smithi055.stdout: high_pri_pool_ratio: 0.000 2023-09-23T17:41:48.947 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_compressed: (nil) 2023-09-23T17:41:48.948 INFO:journalctl@ceph.mon.c.smithi055.stdout: persistent_cache: (nil) 2023-09-23T17:41:48.948 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_size: 4096 2023-09-23T17:41:48.948 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_size_deviation: 10 2023-09-23T17:41:48.948 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_restart_interval: 16 2023-09-23T17:41:48.948 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_block_restart_interval: 1 2023-09-23T17:41:48.948 INFO:journalctl@ceph.mon.c.smithi055.stdout: metadata_block_size: 4096 2023-09-23T17:41:48.948 INFO:journalctl@ceph.mon.c.smithi055.stdout: partition_filters: 0 2023-09-23T17:41:48.948 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_delta_encoding: 1 2023-09-23T17:41:48.949 INFO:journalctl@ceph.mon.c.smithi055.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-09-23T17:41:48.949 INFO:journalctl@ceph.mon.c.smithi055.stdout: whole_key_filtering: 1 2023-09-23T17:41:48.949 INFO:journalctl@ceph.mon.c.smithi055.stdout: verify_compression: 0 2023-09-23T17:41:48.949 INFO:journalctl@ceph.mon.c.smithi055.stdout: read_amp_bytes_per_bit: 0 2023-09-23T17:41:48.949 INFO:journalctl@ceph.mon.c.smithi055.stdout: format_version: 4 2023-09-23T17:41:48.950 INFO:journalctl@ceph.mon.c.smithi055.stdout: enable_index_compression: 1 2023-09-23T17:41:48.950 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_align: 0 2023-09-23T17:41:48.950 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T17:41:48.950 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T17:41:48.950 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compression: NoCompression 2023-09-23T17:41:48.951 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T17:41:48.951 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T17:41:48.951 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T17:41:48.951 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.num_levels: 7 2023-09-23T17:41:48.951 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T17:41:48.952 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T17:41:48.952 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T17:41:48.952 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T17:41:48.952 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T17:41:48.952 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T17:41:48.952 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T17:41:48.952 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:41:48.952 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T17:41:48.952 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T17:41:48.953 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T17:41:48.954 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T17:41:48.954 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T17:41:48.954 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T17:41:48.954 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T17:41:48.954 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T17:41:48.954 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T17:41:48.954 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T17:41:48.954 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T17:41:48.954 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T17:41:48.954 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.arena_block_size: 4194304 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T17:41:48.955 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.table_properties_collectors: 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.inplace_update_support: 0 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T17:41:48.956 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.bloom_locality: 0 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.max_successive_merges: 0 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.ttl: 2592000 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.enable_blob_files: false 2023-09-23T17:41:48.957 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.min_blob_size: 0 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695490908728216, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-09-23T17:41:48.958 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695490908730214, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1695490908, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "778fb6bd-2907-4c6a-b25c-fd01e770e8f6", "db_session_id": "FYXMNENAI02GR295IPGF"}} 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695490908730844, "job": 1, "event": "recovery_finished"} 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55c06882f180 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: DB pointer 0x55c068904000 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** DB Stats ** 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:41:48.959 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2023-09-23T17:41:48.960 INFO:journalctl@ceph.mon.c.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T17:41:48.961 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative compaction: 0.00 GB write, 0.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:41:48.962 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval compaction: 0.00 GB write, 0.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:41:48.962 INFO:journalctl@ceph.mon.c.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T17:41:48.962 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:41:48.962 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T17:41:48.962 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:41:48.962 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:41:48.962 INFO:journalctl@ceph.mon.c.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:41:48.962 INFO:journalctl@ceph.mon.c.smithi055.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:41:48.962 INFO:journalctl@ceph.mon.c.smithi055.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2023-09-23T17:41:48.962 INFO:journalctl@ceph.mon.c.smithi055.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.8 0.00 0.00 1 0.002 0 0 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T17:41:48.963 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T17:41:48.964 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative compaction: 0.00 GB write, 0.31 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:41:48.965 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:41:48.965 INFO:journalctl@ceph.mon.c.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T17:41:48.965 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:41:48.965 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T17:41:48.965 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: using public_addrv [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] 2023-09-23T17:41:48.965 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: starting mon.c rank -1 at public addrs [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] at bind addrs [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:41:48.965 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(???) e0 preinit fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 systemd[1]: Started Ceph mon.c for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).mds e1 new map 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).mds e1 print_map 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout: e1 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout: legacy client fscid: -1 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout: No filesystems configured 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-09-23T17:41:48.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi099:172.21.15.99=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:41:48.967 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: Saving service mon spec with placement smithi055:172.21.15.55=a;smithi055:[v2:172.21.15.55:3301,v1:172.21.15.55:6790]=c;smithi099:172.21.15.99=b;count:3 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: Deploying daemon mon.b on smithi099 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.a calling monitor election 2023-09-23T17:41:48.968 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:48.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:48.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:48.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:48.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.b calling monitor election 2023-09-23T17:41:48.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:48.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: pgmap v10: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:48.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:48.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:48.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T17:41:48.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: monmap e2: 2 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0]} removed_ranks: {} 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: fsmap 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mgrmap e13: y(active, since 33s) 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: overall HEALTH_OK 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: Deploying daemon mon.c on smithi055 2023-09-23T17:41:48.970 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:48.971 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:48.971 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/422472481' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T17:41:48.971 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-09-23T17:41:48.971 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:48 smithi055 ceph-mon[109912]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-09-23T17:41:49.222 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:41:54.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:41:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: mon.b calling monitor election 2023-09-23T17:41:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: mon.a calling monitor election 2023-09-23T17:41:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: fsmap 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: mgrmap e13: y(active, since 41s) 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: mon.c (rank 2) addr [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] is down (out of quorum) 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:54.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:54.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:54.261 INFO:teuthology.orchestra.run.smithi099.stdout: 2023-09-23T17:41:54.261 INFO:teuthology.orchestra.run.smithi099.stdout:{"epoch":3,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","modified":"2023-09-23T17:41:48.774281Z","created":"2023-09-23T17:40:11.277823Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3300","nonce":0},{"type":"v1","addr":"172.21.15.55:6789","nonce":0}]},"addr":"172.21.15.55:6789/0","public_addr":"172.21.15.55:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:3300","nonce":0},{"type":"v1","addr":"172.21.15.99:6789","nonce":0}]},"addr":"172.21.15.99:6789/0","public_addr":"172.21.15.99:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:3301","nonce":0},{"type":"v1","addr":"172.21.15.55:6790","nonce":0}]},"addr":"172.21.15.55:6790/0","public_addr":"172.21.15.55:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-09-23T17:41:54.264 INFO:teuthology.orchestra.run.smithi099.stderr:dumped monmap epoch 3 2023-09-23T17:41:54.293 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:41:54.293 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: mon.b calling monitor election 2023-09-23T17:41:54.293 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: mon.a calling monitor election 2023-09-23T17:41:54.293 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:54.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:54.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:54.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: fsmap 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: mgrmap e13: y(active, since 41s) 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: mon.c (rank 2) addr [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] is down (out of quorum) 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:54.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:54.296 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:54.296 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:54.570 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-09-23T17:41:54.570 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph config generate-minimal-conf 2023-09-23T17:41:55.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:54 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:54 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/3692811320' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T17:41:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:54 smithi055 ceph-mon[95848]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:54 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:55.189 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:41:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:54 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:54 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/3692811320' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T17:41:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:54 smithi099 ceph-mon[100270]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:54 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:56.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:41:56.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: mon.b calling monitor election 2023-09-23T17:41:56.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: mon.a calling monitor election 2023-09-23T17:41:56.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:41:56.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:56.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:56.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:56.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:56.015 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: fsmap 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: mgrmap e13: y(active, since 41s) 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: Health check failed: 1/3 mons down, quorum a,b (MON_DOWN) 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: Health detail: HEALTH_WARN 1/3 mons down, quorum a,b 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: [WRN] MON_DOWN: 1/3 mons down, quorum a,b 2023-09-23T17:41:56.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: mon.c (rank 2) addr [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] is down (out of quorum) 2023-09-23T17:41:56.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:41:56.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/3692811320' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T17:41:56.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:56.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:56.681 INFO:teuthology.orchestra.run.smithi055.stdout:# minimal ceph.conf for 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:41:56.681 INFO:teuthology.orchestra.run.smithi055.stdout:[global] 2023-09-23T17:41:56.681 INFO:teuthology.orchestra.run.smithi055.stdout: fsid = 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:41:56.681 INFO:teuthology.orchestra.run.smithi055.stdout: mon_host = [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] 2023-09-23T17:41:56.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: mon.c calling monitor election 2023-09-23T17:41:56.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: mon.c calling monitor election 2023-09-23T17:41:56.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: mon.a calling monitor election 2023-09-23T17:41:56.993 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: mon.b calling monitor election 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: fsmap 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: mgrmap e13: y(active, since 43s) 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: Cluster is now healthy 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: overall HEALTH_OK 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:56.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:41:56.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:56.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3325821257' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:56.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: mon.c calling monitor election 2023-09-23T17:41:56.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: mon.c calling monitor election 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: mon.a calling monitor election 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: mon.b calling monitor election 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: fsmap 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: mgrmap e13: y(active, since 43s) 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: Cluster is now healthy 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: overall HEALTH_OK 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:56.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:56.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:41:56.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:56.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:56 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3325821257' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:56.998 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-09-23T17:41:56.998 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:41:56.998 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T17:41:57.028 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:41:57.028 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:41:57.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: mon.c calling monitor election 2023-09-23T17:41:57.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: mon.c calling monitor election 2023-09-23T17:41:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: mon.a calling monitor election 2023-09-23T17:41:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: mon.b calling monitor election 2023-09-23T17:41:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:41:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:41:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: fsmap 2023-09-23T17:41:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: osdmap e4: 0 total, 0 up, 0 in 2023-09-23T17:41:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: mgrmap e13: y(active, since 43s) 2023-09-23T17:41:57.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T17:41:57.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: Cluster is now healthy 2023-09-23T17:41:57.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: overall HEALTH_OK 2023-09-23T17:41:57.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:57.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:57.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:57.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:57.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:57.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:57.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:41:57.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:57.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:56 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3325821257' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:57.096 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:41:57.096 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.conf 2023-09-23T17:41:57.128 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:41:57.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:41:57.196 INFO:tasks.cephadm:Adding mgr.y on smithi055 2023-09-23T17:41:57.196 INFO:tasks.cephadm:Adding mgr.x on smithi099 2023-09-23T17:41:57.197 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch apply mgr '2;smithi055=y;smithi099=x' 2023-09-23T17:41:57.822 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:41:57.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:41:57.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:41:57.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T17:41:57.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: Reconfiguring daemon mon.a on smithi055 2023-09-23T17:41:57.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:57.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:57.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:57.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:57.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:57.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:41:57.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:57.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:57.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:41:57.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:41:57.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T17:41:57.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: Reconfiguring daemon mon.a on smithi055 2023-09-23T17:41:57.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:57.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:57.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:57.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:57.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:57.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:41:57.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:57.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:57 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:58.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:41:58.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:41:58.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: Reconfiguring mon.a (unknown last config time)... 2023-09-23T17:41:58.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: Reconfiguring daemon mon.a on smithi055 2023-09-23T17:41:58.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:58.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:41:58.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:58.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:58.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:58.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:41:58.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:58.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:57 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:58.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:58 smithi099 ceph-mon[100270]: Reconfiguring mon.c (monmap changed)... 2023-09-23T17:41:58.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:58 smithi099 ceph-mon[100270]: Reconfiguring daemon mon.c on smithi055 2023-09-23T17:41:58.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:58 smithi055 ceph-mon[95848]: Reconfiguring mon.c (monmap changed)... 2023-09-23T17:41:58.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:58 smithi055 ceph-mon[95848]: Reconfiguring daemon mon.c on smithi055 2023-09-23T17:41:58.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:58 smithi055 ceph-mon[109912]: Reconfiguring mon.c (monmap changed)... 2023-09-23T17:41:58.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:58 smithi055 ceph-mon[109912]: Reconfiguring daemon mon.c on smithi055 2023-09-23T17:41:59.192 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled mgr update... 2023-09-23T17:41:59.764 DEBUG:teuthology.orchestra.run.smithi099:mgr.x> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.x.service 2023-09-23T17:41:59.767 INFO:tasks.cephadm:Deploying OSDs... 2023-09-23T17:41:59.767 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:41:59.767 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T17:41:59.789 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T17:41:59.790 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_1 2023-09-23T17:41:59.795 INFO:journalctl@ceph.mgr.x.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T17:41:59.851 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T17:41:59.851 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T17:41:59.851 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 6h/6d Inode: 389387 Links: 1 2023-09-23T17:41:59.851 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T17:41:59.851 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T17:41:59.851 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2023-09-23 17:41:48.202441537 +0000 2023-09-23T17:41:59.851 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2023-09-23 17:33:34.693013900 +0000 2023-09-23T17:41:59.851 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2023-09-23 17:33:34.693013900 +0000 2023-09-23T17:41:59.851 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: - 2023-09-23T17:41:59.852 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T17:41:59.923 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:59 smithi099 ceph-mon[100270]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:41:59.923 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:59 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:59.923 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:59 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:59.923 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:59 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:41:59.924 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:59 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:41:59.924 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:59 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:41:59.924 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:41:59 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:41:59.924 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2023-09-23T17:41:59.925 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2023-09-23T17:41:59.925 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000309027 s, 1.7 MB/s 2023-09-23T17:41:59.926 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T17:41:59.987 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_2 2023-09-23T17:42:00.046 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T17:42:00.046 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T17:42:00.047 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 6h/6d Inode: 392260 Links: 1 2023-09-23T17:42:00.047 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T17:42:00.047 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T17:42:00.047 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2023-09-23 17:41:48.202441537 +0000 2023-09-23T17:42:00.047 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2023-09-23 17:33:35.205004057 +0000 2023-09-23T17:42:00.047 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2023-09-23 17:33:35.205004057 +0000 2023-09-23T17:42:00.047 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: - 2023-09-23T17:42:00.049 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T17:42:00.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[109912]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:00.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:42:00.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:42:00.115 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:00.115 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[95848]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:00.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:42:00.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:42:00.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:00.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:41:59 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.117 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2023-09-23T17:42:00.117 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2023-09-23T17:42:00.117 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000315667 s, 1.6 MB/s 2023-09-23T17:42:00.118 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T17:42:00.177 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_3 2023-09-23T17:42:00.236 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T17:42:00.236 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T17:42:00.236 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 6h/6d Inode: 392464 Links: 1 2023-09-23T17:42:00.237 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T17:42:00.237 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T17:42:00.237 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2023-09-23 17:41:48.202441537 +0000 2023-09-23T17:42:00.237 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2023-09-23 17:33:35.740993752 +0000 2023-09-23T17:42:00.237 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2023-09-23 17:33:35.740993752 +0000 2023-09-23T17:42:00.237 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: - 2023-09-23T17:42:00.237 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T17:42:00.303 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2023-09-23T17:42:00.303 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2023-09-23T17:42:00.303 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.000309322 s, 1.7 MB/s 2023-09-23T17:42:00.304 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T17:42:00.362 DEBUG:teuthology.orchestra.run.smithi055:> stat /dev/vg_nvme/lv_4 2023-09-23T17:42:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T17:42:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T17:42:00.420 INFO:teuthology.orchestra.run.smithi055.stdout:Device: 6h/6d Inode: 366556 Links: 1 2023-09-23T17:42:00.420 INFO:teuthology.orchestra.run.smithi055.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T17:42:00.420 INFO:teuthology.orchestra.run.smithi055.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T17:42:00.420 INFO:teuthology.orchestra.run.smithi055.stdout:Access: 2023-09-23 17:41:48.203441518 +0000 2023-09-23T17:42:00.420 INFO:teuthology.orchestra.run.smithi055.stdout:Modify: 2023-09-23 17:33:36.251983929 +0000 2023-09-23T17:42:00.420 INFO:teuthology.orchestra.run.smithi055.stdout:Change: 2023-09-23 17:33:36.251983929 +0000 2023-09-23T17:42:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: Birth: - 2023-09-23T17:42:00.420 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T17:42:00.486 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records in 2023-09-23T17:42:00.486 INFO:teuthology.orchestra.run.smithi055.stderr:1+0 records out 2023-09-23T17:42:00.486 INFO:teuthology.orchestra.run.smithi055.stderr:512 bytes copied, 0.00029437 s, 1.7 MB/s 2023-09-23T17:42:00.487 DEBUG:teuthology.orchestra.run.smithi055:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T17:42:00.545 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:42:00.545 DEBUG:teuthology.orchestra.run.smithi099:> dd if=/scratch_devs of=/dev/stdout 2023-09-23T17:42:00.563 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-09-23T17:42:00.563 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_1 2023-09-23T17:42:00.620 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-09-23T17:42:00.621 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T17:42:00.621 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 6h/6d Inode: 383599 Links: 1 2023-09-23T17:42:00.621 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T17:42:00.621 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T17:42:00.621 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2023-09-23 17:41:40.456784832 +0000 2023-09-23T17:42:00.621 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2023-09-23 17:33:03.612169894 +0000 2023-09-23T17:42:00.621 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2023-09-23 17:33:03.612169894 +0000 2023-09-23T17:42:00.621 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2023-09-23T17:42:00.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-09-23T17:42:00.687 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2023-09-23T17:42:00.687 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2023-09-23T17:42:00.687 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000231921 s, 2.2 MB/s 2023-09-23T17:42:00.688 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-09-23T17:42:00.747 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_2 2023-09-23T17:42:00.806 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-09-23T17:42:00.806 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T17:42:00.806 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 6h/6d Inode: 384663 Links: 1 2023-09-23T17:42:00.806 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T17:42:00.806 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T17:42:00.806 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2023-09-23 17:41:40.456784832 +0000 2023-09-23T17:42:00.806 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2023-09-23 17:33:04.126158146 +0000 2023-09-23T17:42:00.806 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2023-09-23 17:33:04.126158146 +0000 2023-09-23T17:42:00.807 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2023-09-23T17:42:00.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-09-23T17:42:00.871 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2023-09-23T17:42:00.872 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2023-09-23T17:42:00.872 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000243984 s, 2.1 MB/s 2023-09-23T17:42:00.873 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-09-23T17:42:00.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: Reconfiguring mon.b (monmap changed)... 2023-09-23T17:42:00.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: Reconfiguring daemon mon.b on smithi099 2023-09-23T17:42:00.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi055=y;smithi099=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:00.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: Saving service mgr spec with placement smithi055=y;smithi099=x;count:2 2023-09-23T17:42:00.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:00.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:00.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:00.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:42:00.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T17:42:00.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:42:00.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:00.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: Reconfiguring mon.b (monmap changed)... 2023-09-23T17:42:00.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: Reconfiguring daemon mon.b on smithi099 2023-09-23T17:42:00.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi055=y;smithi099=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:00.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: Saving service mgr spec with placement smithi055=y;smithi099=x;count:2 2023-09-23T17:42:00.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:00.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:00.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:00.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:00.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:42:00.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T17:42:00.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:42:00.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:00.935 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_3 2023-09-23T17:42:00.992 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-09-23T17:42:00.993 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T17:42:00.993 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 6h/6d Inode: 390280 Links: 1 2023-09-23T17:42:00.993 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T17:42:00.993 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T17:42:00.993 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2023-09-23 17:41:40.456784832 +0000 2023-09-23T17:42:00.993 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2023-09-23 17:33:04.647146238 +0000 2023-09-23T17:42:00.993 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2023-09-23 17:33:04.647146238 +0000 2023-09-23T17:42:00.993 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2023-09-23T17:42:00.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-09-23T17:42:01.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: Reconfiguring mon.b (monmap changed)... 2023-09-23T17:42:01.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: Reconfiguring daemon mon.b on smithi099 2023-09-23T17:42:01.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='client.24103 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi055=y;smithi099=x", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:01.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: Saving service mgr spec with placement smithi055=y;smithi099=x;count:2 2023-09-23T17:42:01.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:01.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:01.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:01.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:01.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:01.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:01.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:42:01.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-09-23T17:42:01.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:42:01.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:01.043 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2023-09-23T17:42:01.043 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2023-09-23T17:42:01.043 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000257198 s, 2.0 MB/s 2023-09-23T17:42:01.044 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-09-23T17:42:01.102 DEBUG:teuthology.orchestra.run.smithi099:> stat /dev/vg_nvme/lv_4 2023-09-23T17:42:01.158 INFO:teuthology.orchestra.run.smithi099.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-09-23T17:42:01.158 INFO:teuthology.orchestra.run.smithi099.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-09-23T17:42:01.158 INFO:teuthology.orchestra.run.smithi099.stdout:Device: 6h/6d Inode: 391753 Links: 1 2023-09-23T17:42:01.159 INFO:teuthology.orchestra.run.smithi099.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-09-23T17:42:01.159 INFO:teuthology.orchestra.run.smithi099.stdout:Context: system_u:object_r:device_t:s0 2023-09-23T17:42:01.159 INFO:teuthology.orchestra.run.smithi099.stdout:Access: 2023-09-23 17:41:40.456784832 +0000 2023-09-23T17:42:01.159 INFO:teuthology.orchestra.run.smithi099.stdout:Modify: 2023-09-23 17:33:05.135135084 +0000 2023-09-23T17:42:01.159 INFO:teuthology.orchestra.run.smithi099.stdout:Change: 2023-09-23 17:33:05.135135084 +0000 2023-09-23T17:42:01.159 INFO:teuthology.orchestra.run.smithi099.stdout: Birth: - 2023-09-23T17:42:01.159 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-09-23T17:42:01.228 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records in 2023-09-23T17:42:01.229 INFO:teuthology.orchestra.run.smithi099.stderr:1+0 records out 2023-09-23T17:42:01.229 INFO:teuthology.orchestra.run.smithi099.stderr:512 bytes copied, 0.000205866 s, 2.5 MB/s 2023-09-23T17:42:01.230 DEBUG:teuthology.orchestra.run.smithi099:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-09-23T17:42:01.292 INFO:tasks.cephadm:Deploying osd.0 on smithi055 with /dev/vg_nvme/lv_4... 2023-09-23T17:42:01.292 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T17:42:01.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:01 smithi099 systemd[1]: Starting Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:42:01.902 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:42:02.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:01 smithi099 ceph-mon[100270]: Deploying daemon mgr.x on smithi099 2023-09-23T17:42:02.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:01 smithi099 ceph-mon[100270]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:02.026 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:01 smithi099 podman[102360]: 2023-09-23T17:42:02.027 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:01 smithi099 ceph-mgr[102400]: -- 172.21.15.99:0/3824753264 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x559a49c4c9c0 con 0x559a49c22800 2023-09-23T17:42:02.027 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:01 smithi099 bash[102360]: 6d7959b6f2fc63b80c2d88a37568915f07ee0d1f2a3e819c977f996f94f274b3 2023-09-23T17:42:02.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:01 smithi055 ceph-mon[109912]: Deploying daemon mgr.x on smithi099 2023-09-23T17:42:02.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:01 smithi055 ceph-mon[109912]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:02.106 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:01 smithi055 ceph-mon[95848]: Deploying daemon mgr.x on smithi099 2023-09-23T17:42:02.106 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:01 smithi055 ceph-mon[95848]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:02.327 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:02 smithi099 systemd[1]: Started Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:42:02.327 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:02.069+0000 7f907b61f1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:42:02.327 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:02.148+0000 7f907b61f1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:42:02.327 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:02.240+0000 7f907b61f1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:42:02.826 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:02.343+0000 7f907b61f1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:42:02.826 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:02.520+0000 7f907b61f1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:42:03.174 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:02.858+0000 7f907b61f1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:42:03.174 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:02.950+0000 7f907b61f1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:42:03.174 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:03 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:03.080+0000 7f907b61f1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:42:03.174 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:03 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:03.174+0000 7f907b61f1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:42:03.175 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.175 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.175 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.175 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.175 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:03.175 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:03 smithi099 ceph-mon[100270]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:03.175 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.324 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T17:42:03.324 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T17:42:03.324 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2023-09-23T17:42:03.324 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2023-09-23T17:42:03.324 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0122762 s, 854 MB/s 2023-09-23T17:42:03.324 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2023-09-23T17:42:03.362 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:03.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[95848]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:03.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.363 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.363 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.364 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.364 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.364 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:03.364 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[109912]: pgmap v19: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:03.364 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:03.658 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi055:vg_nvme/lv_4 2023-09-23T17:42:04.076 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:03 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:03.616+0000 7f907b61f1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:42:04.266 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:42:04.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:04.195+0000 7f907b61f1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:42:04.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:04.362+0000 7f907b61f1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:42:05.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:05.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:05.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:05.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:05.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:05.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T17:42:05.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:42:05.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:42:05.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:05.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: Reconfiguring daemon mgr.y on smithi055 2023-09-23T17:42:05.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[109912]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:05.017 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:05.017 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:05.017 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:05.017 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:05.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:05.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T17:42:05.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:42:05.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:42:05.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:05.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: Reconfiguring daemon mgr.y on smithi055 2023-09-23T17:42:05.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:04 smithi055 ceph-mon[95848]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:05.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:05.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:05.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:05.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: Reconfiguring mgr.y (unknown last config time)... 2023-09-23T17:42:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:42:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:42:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: Reconfiguring daemon mgr.y on smithi055 2023-09-23T17:42:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:04 smithi099 ceph-mon[100270]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:05.378 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:05 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:05.124+0000 7f907b61f1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:42:05.378 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:05 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:05.212+0000 7f907b61f1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:42:05.652 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:05 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:05.378+0000 7f907b61f1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:42:06.021 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:05 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:05.652+0000 7f907b61f1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:42:06.022 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:05 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:05.737+0000 7f907b61f1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:42:06.274 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:06.021+0000 7f907b61f1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:42:06.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:06.274+0000 7f907b61f1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:42:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:42:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:42:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:42:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:42:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:42:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:42:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:07.306 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:06 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:06.990+0000 7f907b61f1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:42:07.306 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:07 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:07.120+0000 7f907b61f1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:42:07.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:07 smithi099 ceph-mon[100270]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:07 smithi099 ceph-mon[100270]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:07.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:07 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:07.306+0000 7f907b61f1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:42:07.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:07 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:07.417+0000 7f907b61f1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:42:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:07 smithi055 ceph-mon[95848]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:07 smithi055 ceph-mon[95848]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:07 smithi055 ceph-mon[109912]: from='client.14217 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:07 smithi055 ceph-mon[109912]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:08.076 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:07 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:07.626+0000 7f907b61f1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:42:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:08 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1633620018' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99e7cae6-389e-461a-8a86-a5789673908b"}]: dispatch 2023-09-23T17:42:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:08 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99e7cae6-389e-461a-8a86-a5789673908b"}]: dispatch 2023-09-23T17:42:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:08 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "99e7cae6-389e-461a-8a86-a5789673908b"}]': finished 2023-09-23T17:42:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:08 smithi055 ceph-mon[95848]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T17:42:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:08 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:08 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1633620018' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99e7cae6-389e-461a-8a86-a5789673908b"}]: dispatch 2023-09-23T17:42:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:08 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99e7cae6-389e-461a-8a86-a5789673908b"}]: dispatch 2023-09-23T17:42:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:08 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "99e7cae6-389e-461a-8a86-a5789673908b"}]': finished 2023-09-23T17:42:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:08 smithi055 ceph-mon[109912]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T17:42:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:08 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:08.826 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:42:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:42:08.491+0000 7f907b61f1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:42:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:08 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1633620018' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99e7cae6-389e-461a-8a86-a5789673908b"}]: dispatch 2023-09-23T17:42:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:08 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "99e7cae6-389e-461a-8a86-a5789673908b"}]: dispatch 2023-09-23T17:42:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:08 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "99e7cae6-389e-461a-8a86-a5789673908b"}]': finished 2023-09-23T17:42:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:08 smithi099 ceph-mon[100270]: osdmap e5: 1 total, 0 up, 1 in 2023-09-23T17:42:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:08 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:09.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[95848]: Standby manager daemon x started 2023-09-23T17:42:09.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:42:09.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:42:09.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:42:09.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:42:09.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[95848]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:09.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1928832781' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:42:09.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T17:42:09.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:42:09.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:42:09.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:42:09.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:42:09.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[109912]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:09.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:09 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1928832781' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:42:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:09 smithi099 ceph-mon[100270]: Standby manager daemon x started 2023-09-23T17:42:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:09 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:42:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:09 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:42:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:09 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:42:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:09 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/2826909689' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:42:09.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:09 smithi099 ceph-mon[100270]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:09.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:09 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1928832781' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:42:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:10 smithi055 ceph-mon[95848]: mgrmap e14: y(active, since 56s), standbys: x 2023-09-23T17:42:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:10 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:42:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:10 smithi055 ceph-mon[109912]: mgrmap e14: y(active, since 56s), standbys: x 2023-09-23T17:42:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:10 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:42:10.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:10 smithi099 ceph-mon[100270]: mgrmap e14: y(active, since 56s), standbys: x 2023-09-23T17:42:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:10 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:42:11.472 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:11 smithi055 ceph-mon[95848]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:11.473 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:11 smithi055 ceph-mon[109912]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:11.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:11 smithi099 ceph-mon[100270]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:13.826 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:13 smithi055 ceph-mon[109912]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:13.827 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:13 smithi055 ceph-mon[95848]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:14.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:13 smithi099 ceph-mon[100270]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:14.804 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:14 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T17:42:14.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:14 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:14.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:14 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T17:42:14.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:14 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:15.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:14 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T17:42:15.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:14 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:16.010 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:15 smithi055 ceph-mon[109912]: Deploying daemon osd.0 on smithi055 2023-09-23T17:42:16.010 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:15 smithi055 ceph-mon[109912]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:16.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:15 smithi055 ceph-mon[95848]: Deploying daemon osd.0 on smithi055 2023-09-23T17:42:16.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:15 smithi055 ceph-mon[95848]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:16.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:15 smithi099 ceph-mon[100270]: Deploying daemon osd.0 on smithi055 2023-09-23T17:42:16.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:15 smithi099 ceph-mon[100270]: pgmap v26: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:18.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:17 smithi099 ceph-mon[100270]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:17 smithi055 ceph-mon[95848]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:18.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:17 smithi055 ceph-mon[109912]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:18.980 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:18 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:18.981 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:18 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:18.981 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:18 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:18.981 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:18 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:18.981 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:18 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:18.981 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:18 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:19.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:18 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:18 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:18 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:19.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:19 smithi055 ceph-mon[95848]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:19.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:19 smithi055 ceph-mon[109912]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:20.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:19 smithi099 ceph-mon[100270]: pgmap v28: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:20.499 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 0 on host 'smithi055' 2023-09-23T17:42:20.829 DEBUG:teuthology.orchestra.run.smithi055:osd.0> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.0.service 2023-09-23T17:42:20.831 INFO:tasks.cephadm:Deploying osd.1 on smithi055 with /dev/vg_nvme/lv_3... 2023-09-23T17:42:20.832 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T17:42:20.864 INFO:journalctl@ceph.osd.0.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T17:42:21.429 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[114371]: 2023-09-23T17:42:21.120+0000 7f592950d540 -1 osd.0 0 log_to_monitors true 2023-09-23T17:42:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[95848]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[95848]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T17:42:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[109912]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:21 smithi055 ceph-mon[109912]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T17:42:21.501 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:42:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:21 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:21 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:21 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:21 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:21 smithi099 ceph-mon[100270]: pgmap v29: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:21 smithi099 ceph-mon[100270]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T17:42:22.665 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:42:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[114371]: 2023-09-23T17:42:22.491+0000 7f591fcf3700 -1 osd.0 0 waiting for initial osdmap 2023-09-23T17:42:22.665 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:42:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[114371]: 2023-09-23T17:42:22.499+0000 7f591c2f1700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:42:22.665 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:22 smithi055 ceph-mon[95848]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T17:42:22.665 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:22 smithi055 ceph-mon[95848]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T17:42:22.665 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:22 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:22.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:22 smithi055 ceph-mon[95848]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:42:22.666 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:22 smithi055 ceph-mon[109912]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T17:42:22.666 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:22 smithi055 ceph-mon[109912]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T17:42:22.666 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:22 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:22.666 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:22 smithi055 ceph-mon[109912]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:42:22.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:22 smithi099 ceph-mon[100270]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T17:42:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:22 smithi099 ceph-mon[100270]: osdmap e6: 1 total, 0 up, 1 in 2023-09-23T17:42:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:22 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:22 smithi099 ceph-mon[100270]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:42:23.404 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T17:42:23.404 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T17:42:23.405 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2023-09-23T17:42:23.405 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2023-09-23T17:42:23.405 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0157931 s, 664 MB/s 2023-09-23T17:42:23.405 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2023-09-23T17:42:23.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:23 smithi055 ceph-mon[95848]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:42:23.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:23 smithi055 ceph-mon[95848]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T17:42:23.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:23.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:23.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:23 smithi055 ceph-mon[95848]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:23.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:23 smithi055 ceph-mon[109912]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:42:23.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:23 smithi055 ceph-mon[109912]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T17:42:23.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:23 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:23.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:23 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:23.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:23 smithi055 ceph-mon[109912]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:23.811 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi055:vg_nvme/lv_3 2023-09-23T17:42:23.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:23 smithi099 ceph-mon[100270]: from='osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716]' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:42:23.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:23 smithi099 ceph-mon[100270]: osdmap e7: 1 total, 0 up, 1 in 2023-09-23T17:42:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:23 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:23 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:23 smithi099 ceph-mon[100270]: pgmap v32: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-09-23T17:42:24.422 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:42:24.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: purged_snaps scrub starts 2023-09-23T17:42:24.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: purged_snaps scrub ok 2023-09-23T17:42:24.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:24.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716] boot 2023-09-23T17:42:24.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T17:42:24.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:24.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:42:24.764 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.764 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:24.764 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:24.764 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.764 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:24.764 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:24.764 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:24.764 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.764 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T17:42:24.764 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T17:42:24.765 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:24.765 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716] boot 2023-09-23T17:42:24.765 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T17:42:24.765 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:24.765 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.765 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.765 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:42:24.765 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.765 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:24.765 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:24.766 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.766 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:24.766 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:24.766 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:24.766 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: purged_snaps scrub starts 2023-09-23T17:42:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: purged_snaps scrub ok 2023-09-23T17:42:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: osd.0 [v2:172.21.15.55:6802/3018551716,v1:172.21.15.55:6803/3018551716] boot 2023-09-23T17:42:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: osdmap e8: 1 total, 1 up, 1 in 2023-09-23T17:42:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:42:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:42:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:25.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:25 smithi099 ceph-mon[100270]: Detected new or changed devices on smithi055 2023-09-23T17:42:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:25 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi055 to 16149M 2023-09-23T17:42:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:25 smithi099 ceph-mon[100270]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T17:42:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:25 smithi099 ceph-mon[100270]: pgmap v35: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:25.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:25 smithi055 ceph-mon[95848]: Detected new or changed devices on smithi055 2023-09-23T17:42:25.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:25 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi055 to 16149M 2023-09-23T17:42:25.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:25 smithi055 ceph-mon[95848]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T17:42:25.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:25 smithi055 ceph-mon[95848]: pgmap v35: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:25.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:25 smithi055 ceph-mon[109912]: Detected new or changed devices on smithi055 2023-09-23T17:42:25.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:25 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi055 to 16149M 2023-09-23T17:42:25.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:25 smithi055 ceph-mon[109912]: osdmap e9: 1 total, 1 up, 1 in 2023-09-23T17:42:25.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:25 smithi055 ceph-mon[109912]: pgmap v35: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:26.798 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:26 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:42:26.798 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:26 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:42:26.798 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:26 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:26.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:26 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:42:26.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:26 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:42:26.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:26 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:26.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:26 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:42:26.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:26 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:42:26.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:26 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:27.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:27 smithi099 ceph-mon[100270]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:27.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:27 smithi099 ceph-mon[100270]: pgmap v36: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:27 smithi055 ceph-mon[95848]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:27 smithi055 ceph-mon[95848]: pgmap v36: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:27.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:27 smithi055 ceph-mon[109912]: from='client.24137 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:27.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:27 smithi055 ceph-mon[109912]: pgmap v36: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:28.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:28 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/288734622' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfb0da01-7a06-430b-b973-209f22ac30d4"}]: dispatch 2023-09-23T17:42:28.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:28 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfb0da01-7a06-430b-b973-209f22ac30d4"}]: dispatch 2023-09-23T17:42:28.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:28 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfb0da01-7a06-430b-b973-209f22ac30d4"}]': finished 2023-09-23T17:42:28.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:28 smithi099 ceph-mon[100270]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T17:42:28.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:28 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:28 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/288734622' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfb0da01-7a06-430b-b973-209f22ac30d4"}]: dispatch 2023-09-23T17:42:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:28 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfb0da01-7a06-430b-b973-209f22ac30d4"}]: dispatch 2023-09-23T17:42:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:28 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfb0da01-7a06-430b-b973-209f22ac30d4"}]': finished 2023-09-23T17:42:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:28 smithi055 ceph-mon[95848]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T17:42:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:28 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:28.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:28 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/288734622' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfb0da01-7a06-430b-b973-209f22ac30d4"}]: dispatch 2023-09-23T17:42:28.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:28 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "bfb0da01-7a06-430b-b973-209f22ac30d4"}]: dispatch 2023-09-23T17:42:28.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:28 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "bfb0da01-7a06-430b-b973-209f22ac30d4"}]': finished 2023-09-23T17:42:28.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:28 smithi055 ceph-mon[109912]: osdmap e10: 2 total, 1 up, 2 in 2023-09-23T17:42:28.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:28 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:29.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:29 smithi099 ceph-mon[100270]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:29 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2960112687' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:42:29.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:29 smithi055 ceph-mon[109912]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:29.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:29 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2960112687' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:42:29.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:29 smithi055 ceph-mon[95848]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:29.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:29 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2960112687' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:42:32.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:31 smithi099 ceph-mon[100270]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:32.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:31 smithi055 ceph-mon[109912]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:32.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:31 smithi055 ceph-mon[95848]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:34.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:33 smithi099 ceph-mon[100270]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:34.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:33 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T17:42:34.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:33 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:33 smithi055 ceph-mon[95848]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:33 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T17:42:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:33 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:33 smithi055 ceph-mon[109912]: pgmap v40: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:33 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T17:42:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:33 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:34.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:34 smithi055 ceph-mon[95848]: Deploying daemon osd.1 on smithi055 2023-09-23T17:42:34.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:34 smithi055 ceph-mon[109912]: Deploying daemon osd.1 on smithi055 2023-09-23T17:42:35.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:34 smithi099 ceph-mon[100270]: Deploying daemon osd.1 on smithi055 2023-09-23T17:42:35.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:35 smithi055 ceph-mon[95848]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:35.989 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:35 smithi055 ceph-mon[109912]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:35 smithi099 ceph-mon[100270]: pgmap v41: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:36 smithi099 ceph-mon[100270]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:37.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:36 smithi055 ceph-mon[95848]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:37.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:36 smithi055 ceph-mon[109912]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:38.976 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:38.977 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:38.977 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:38.977 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:38 smithi055 ceph-mon[95848]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:38.977 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:38 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:38.977 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:38 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:38.977 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:38 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:38.978 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:38 smithi055 ceph-mon[109912]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:39.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:38 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:39.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:38 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:39.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:38 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:39.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:38 smithi099 ceph-mon[100270]: pgmap v43: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:40.365 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 1 on host 'smithi055' 2023-09-23T17:42:40.715 DEBUG:teuthology.orchestra.run.smithi055:osd.1> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T17:42:40.717 INFO:tasks.cephadm:Deploying osd.2 on smithi055 with /dev/vg_nvme/lv_2... 2023-09-23T17:42:40.718 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T17:42:40.746 INFO:journalctl@ceph.osd.1.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T17:42:41.248 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:42:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[121485]: 2023-09-23T17:42:40.986+0000 7f21a8398540 -1 osd.1 0 log_to_monitors true 2023-09-23T17:42:41.442 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:42:41.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:41.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:41.527 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.527 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.527 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.527 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:41.527 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[95848]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:41.527 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[95848]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T17:42:41.527 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.527 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.527 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:41.527 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:41.528 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.528 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.528 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.528 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:41.528 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[109912]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:41.528 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:41 smithi055 ceph-mon[109912]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T17:42:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:41 smithi099 ceph-mon[100270]: pgmap v44: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:41 smithi099 ceph-mon[100270]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T17:42:42.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:42:42 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[121485]: 2023-09-23T17:42:42.344+0000 7f219eb7e700 -1 osd.1 0 waiting for initial osdmap 2023-09-23T17:42:42.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:42:42 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[121485]: 2023-09-23T17:42:42.349+0000 7f219a17a700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:42:42.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:42 smithi055 ceph-mon[109912]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T17:42:42.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:42 smithi055 ceph-mon[109912]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T17:42:42.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:42.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:42 smithi055 ceph-mon[109912]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:42:42.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:42 smithi055 ceph-mon[95848]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T17:42:42.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:42 smithi055 ceph-mon[95848]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T17:42:42.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:42.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:42 smithi055 ceph-mon[95848]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:42:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:42 smithi099 ceph-mon[100270]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T17:42:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:42 smithi099 ceph-mon[100270]: osdmap e11: 2 total, 1 up, 2 in 2023-09-23T17:42:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:42 smithi099 ceph-mon[100270]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:42:43.202 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T17:42:43.203 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T17:42:43.203 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2023-09-23T17:42:43.203 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2023-09-23T17:42:43.203 INFO:teuthology.orchestra.run.smithi055.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0103803 s, 1.0 GB/s 2023-09-23T17:42:43.203 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2023-09-23T17:42:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:43 smithi055 ceph-mon[109912]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:42:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:43 smithi055 ceph-mon[109912]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T17:42:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:43 smithi055 ceph-mon[109912]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:43 smithi055 ceph-mon[95848]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:42:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:43 smithi055 ceph-mon[95848]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T17:42:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:43 smithi055 ceph-mon[95848]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:43.748 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi055:vg_nvme/lv_2 2023-09-23T17:42:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:43 smithi099 ceph-mon[100270]: from='osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:42:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:43 smithi099 ceph-mon[100270]: osdmap e12: 2 total, 1 up, 2 in 2023-09-23T17:42:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:43 smithi099 ceph-mon[100270]: pgmap v47: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-09-23T17:42:44.354 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:42:44.620 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T17:42:44.620 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846] boot 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: Detected new or changed devices on smithi055 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi055 to 8074M 2023-09-23T17:42:44.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:44.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:44.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: purged_snaps scrub starts 2023-09-23T17:42:44.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: purged_snaps scrub ok 2023-09-23T17:42:44.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:44.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846] boot 2023-09-23T17:42:44.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T17:42:44.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:44.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: Detected new or changed devices on smithi055 2023-09-23T17:42:44.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:42:44.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:42:44.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi055 to 8074M 2023-09-23T17:42:44.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:44.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:44.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:44 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: purged_snaps scrub starts 2023-09-23T17:42:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: purged_snaps scrub ok 2023-09-23T17:42:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: osd.1 [v2:172.21.15.55:6810/1171163846,v1:172.21.15.55:6811/1171163846] boot 2023-09-23T17:42:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: osdmap e13: 2 total, 2 up, 2 in 2023-09-23T17:42:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:42:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: Detected new or changed devices on smithi055 2023-09-23T17:42:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:42:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:42:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi055 to 8074M 2023-09-23T17:42:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:42:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:44 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:45.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:45 smithi099 ceph-mon[100270]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T17:42:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:45 smithi099 ceph-mon[100270]: pgmap v50: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:45.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:45 smithi055 ceph-mon[95848]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T17:42:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:45 smithi055 ceph-mon[95848]: pgmap v50: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:45.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:45 smithi055 ceph-mon[109912]: osdmap e14: 2 total, 2 up, 2 in 2023-09-23T17:42:45.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:45 smithi055 ceph-mon[109912]: pgmap v50: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:46.784 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:46 smithi055 ceph-mon[95848]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:46.784 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:42:46.784 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:42:46.784 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:46.785 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:46 smithi055 ceph-mon[109912]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:46.785 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:46 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:42:46.785 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:46 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:42:46.785 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:46 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:46.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:46 smithi099 ceph-mon[100270]: from='client.14271 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:42:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:42:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:42:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:47.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:47 smithi099 ceph-mon[100270]: pgmap v51: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:47.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:47 smithi055 ceph-mon[95848]: pgmap v51: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:47.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:47 smithi055 ceph-mon[109912]: pgmap v51: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:48.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:48 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2851435003' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f766d604-db54-4a09-9993-fe1a3ce0048e"}]: dispatch 2023-09-23T17:42:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:48 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2851435003' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f766d604-db54-4a09-9993-fe1a3ce0048e"}]': finished 2023-09-23T17:42:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:48 smithi099 ceph-mon[100270]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T17:42:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:48 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:42:48.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:48 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2851435003' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f766d604-db54-4a09-9993-fe1a3ce0048e"}]: dispatch 2023-09-23T17:42:48.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:48 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2851435003' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f766d604-db54-4a09-9993-fe1a3ce0048e"}]': finished 2023-09-23T17:42:48.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:48 smithi055 ceph-mon[95848]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T17:42:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:48 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:42:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:48 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2851435003' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f766d604-db54-4a09-9993-fe1a3ce0048e"}]: dispatch 2023-09-23T17:42:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:48 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2851435003' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f766d604-db54-4a09-9993-fe1a3ce0048e"}]': finished 2023-09-23T17:42:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:48 smithi055 ceph-mon[109912]: osdmap e15: 3 total, 2 up, 3 in 2023-09-23T17:42:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:42:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:49 smithi055 ceph-mon[95848]: pgmap v53: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:49 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2527941474' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:42:49.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:49 smithi055 ceph-mon[109912]: pgmap v53: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:49.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:49 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2527941474' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:42:50.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:49 smithi099 ceph-mon[100270]: pgmap v53: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:50.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:49 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2527941474' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:42:52.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:51 smithi099 ceph-mon[100270]: pgmap v54: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:52.148 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:51 smithi055 ceph-mon[109912]: pgmap v54: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:51 smithi055 ceph-mon[95848]: pgmap v54: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:53.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:53 smithi055 ceph-mon[95848]: pgmap v55: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:53.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T17:42:53.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:53.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:53 smithi055 ceph-mon[109912]: pgmap v55: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:53.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:53 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T17:42:53.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:53 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:54.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:53 smithi099 ceph-mon[100270]: pgmap v55: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T17:42:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:42:54.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:54 smithi055 ceph-mon[95848]: Deploying daemon osd.2 on smithi055 2023-09-23T17:42:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:54 smithi055 ceph-mon[109912]: Deploying daemon osd.2 on smithi055 2023-09-23T17:42:55.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:54 smithi099 ceph-mon[100270]: Deploying daemon osd.2 on smithi055 2023-09-23T17:42:56.034 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:55 smithi055 ceph-mon[109912]: pgmap v56: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:56.034 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:55 smithi055 ceph-mon[95848]: pgmap v56: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:56.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:55 smithi099 ceph-mon[100270]: pgmap v56: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:58.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:57 smithi099 ceph-mon[100270]: pgmap v57: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:58.164 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:57 smithi055 ceph-mon[109912]: pgmap v57: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:58.166 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:57 smithi055 ceph-mon[95848]: pgmap v57: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:58.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:58 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:58.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:58 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:58.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:58 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:58.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:42:58 smithi055 ceph-mon[95848]: pgmap v58: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:58.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:58 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:58.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:58 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:58.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:58 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:58.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:42:58 smithi055 ceph-mon[109912]: pgmap v58: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:42:59.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:58 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:58 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:42:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:58 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:42:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:42:58 smithi099 ceph-mon[100270]: pgmap v58: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:43:00.406 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 2 on host 'smithi055' 2023-09-23T17:43:00.747 DEBUG:teuthology.orchestra.run.smithi055:osd.2> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T17:43:00.749 INFO:tasks.cephadm:Deploying osd.3 on smithi055 with /dev/vg_nvme/lv_1... 2023-09-23T17:43:00.750 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T17:43:00.779 INFO:journalctl@ceph.osd.2.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T17:43:01.064 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[129328]: 2023-09-23T17:43:00.869+0000 7efd884c1540 -1 osd.2 0 log_to_monitors true 2023-09-23T17:43:01.326 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.326 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.326 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:01.326 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:01.326 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.326 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.326 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.326 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:01.327 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: pgmap v59: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:43:01.327 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: from='osd.2 [v2:172.21.15.55:6818/3805819981,v1:172.21.15.55:6819/3805819981]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:43:01.327 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[95848]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:43:01.327 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.327 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.327 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:01.327 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:01.327 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.327 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.327 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.328 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:01.328 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: pgmap v59: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:43:01.328 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: from='osd.2 [v2:172.21.15.55:6818/3805819981,v1:172.21.15.55:6819/3805819981]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:43:01.328 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:01 smithi055 ceph-mon[109912]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:43:01.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:01.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:01.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: pgmap v59: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:43:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: from='osd.2 [v2:172.21.15.55:6818/3805819981,v1:172.21.15.55:6819/3805819981]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:43:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:01 smithi099 ceph-mon[100270]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:43:01.464 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:43:02.405 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-mon[95848]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T17:43:02.405 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-mon[95848]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T17:43:02.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:02.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-mon[95848]: from='osd.2 [v2:172.21.15.55:6818/3805819981,v1:172.21.15.55:6819/3805819981]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:02.406 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-mon[95848]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:02.681 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[129328]: 2023-09-23T17:43:02.409+0000 7efd7eca7700 -1 osd.2 0 waiting for initial osdmap 2023-09-23T17:43:02.682 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[129328]: 2023-09-23T17:43:02.412+0000 7efd77a9e700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:43:02.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-mon[109912]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T17:43:02.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-mon[109912]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T17:43:02.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:02.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-mon[109912]: from='osd.2 [v2:172.21.15.55:6818/3805819981,v1:172.21.15.55:6819/3805819981]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:02.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:02 smithi055 ceph-mon[109912]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:02.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:02 smithi099 ceph-mon[100270]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T17:43:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:02 smithi099 ceph-mon[100270]: osdmap e16: 3 total, 2 up, 3 in 2023-09-23T17:43:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:02 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:02 smithi099 ceph-mon[100270]: from='osd.2 [v2:172.21.15.55:6818/3805819981,v1:172.21.15.55:6819/3805819981]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:02 smithi099 ceph-mon[100270]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:03.310 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T17:43:03.311 INFO:teuthology.orchestra.run.smithi055.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T17:43:03.311 INFO:teuthology.orchestra.run.smithi055.stderr: stderr: 10+0 records in 2023-09-23T17:43:03.311 INFO:teuthology.orchestra.run.smithi055.stderr:10+0 records out 2023-09-23T17:43:03.311 INFO:teuthology.orchestra.run.smithi055.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0128724 s, 815 MB/s 2023-09-23T17:43:03.311 INFO:teuthology.orchestra.run.smithi055.stderr:--> Zapping successful for: 2023-09-23T17:43:03.548 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:03 smithi055 ceph-mon[109912]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:43:03.548 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:03 smithi055 ceph-mon[109912]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T17:43:03.548 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:03.548 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:03.548 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:03 smithi055 ceph-mon[109912]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:43:03.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:03 smithi055 ceph-mon[95848]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:43:03.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:03 smithi055 ceph-mon[95848]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T17:43:03.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:03.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:03.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:03 smithi055 ceph-mon[95848]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:43:03.821 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi055:vg_nvme/lv_1 2023-09-23T17:43:03.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:03 smithi099 ceph-mon[100270]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:43:03.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:03 smithi099 ceph-mon[100270]: osdmap e17: 3 total, 2 up, 3 in 2023-09-23T17:43:03.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:03.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:03.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:03 smithi099 ceph-mon[100270]: pgmap v62: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-09-23T17:43:04.430 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:43:04.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: purged_snaps scrub starts 2023-09-23T17:43:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: purged_snaps scrub ok 2023-09-23T17:43:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: osd.2 [v2:172.21.15.55:6818/3805819981,v1:172.21.15.55:6819/3805819981] boot 2023-09-23T17:43:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T17:43:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T17:43:04.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T17:43:04.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:04.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: osd.2 [v2:172.21.15.55:6818/3805819981,v1:172.21.15.55:6819/3805819981] boot 2023-09-23T17:43:04.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T17:43:04.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:04.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:04.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:04.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:04.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:04.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:04.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: purged_snaps scrub starts 2023-09-23T17:43:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: purged_snaps scrub ok 2023-09-23T17:43:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: osd.2 [v2:172.21.15.55:6818/3805819981,v1:172.21.15.55:6819/3805819981] boot 2023-09-23T17:43:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: osdmap e18: 3 total, 3 up, 3 in 2023-09-23T17:43:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:43:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:04.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:04.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:04.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:04.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:05 smithi099 ceph-mon[100270]: Detected new or changed devices on smithi055 2023-09-23T17:43:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:05 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi055 to 5383M 2023-09-23T17:43:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:05 smithi099 ceph-mon[100270]: pgmap v64: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:05 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T17:43:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:05 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T17:43:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:05 smithi099 ceph-mon[100270]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T17:43:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:05 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:43:05.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[95848]: Detected new or changed devices on smithi055 2023-09-23T17:43:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi055 to 5383M 2023-09-23T17:43:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[95848]: pgmap v64: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T17:43:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T17:43:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[95848]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T17:43:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:43:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[109912]: Detected new or changed devices on smithi055 2023-09-23T17:43:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi055 to 5383M 2023-09-23T17:43:05.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[109912]: pgmap v64: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:05.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-09-23T17:43:05.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-09-23T17:43:05.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[109912]: osdmap e19: 3 total, 3 up, 3 in 2023-09-23T17:43:05.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:05 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:43:06.711 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:43:06 smithi055 sudo[135210]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T17:43:06.711 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:43:06 smithi055 sudo[135210]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:06.711 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:43:06 smithi055 sudo[135210]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:06.711 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:43:06 smithi055 sudo[135210]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:06.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[109912]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:43:06.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:43:06.711 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:43:06.712 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:06.712 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T17:43:06.712 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[109912]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T17:43:06.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[95848]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:43:06.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:43:06.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:43:06.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:06.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T17:43:06.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:06 smithi055 ceph-mon[95848]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T17:43:06.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:06 smithi099 ceph-mon[100270]: from='client.14298 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi055:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:43:06.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:43:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:43:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:06 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-09-23T17:43:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:06 smithi099 ceph-mon[100270]: osdmap e20: 3 total, 3 up, 3 in 2023-09-23T17:43:07.381 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:43:07 smithi055 sudo[135245]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T17:43:07.381 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:43:07 smithi055 sudo[135245]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:07.381 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:43:07 smithi055 sudo[135245]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:07.381 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:43:07 smithi055 sudo[135245]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:07.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:07 smithi055 ceph-mon[95848]: pgmap v67: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:07.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:07 smithi055 ceph-mon[95848]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T17:43:07.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:07 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:07.637 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:07 smithi055 ceph-mon[109912]: pgmap v67: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:07.637 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:07 smithi055 ceph-mon[109912]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T17:43:07.637 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:07 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:07.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:07 smithi099 ceph-mon[100270]: pgmap v67: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:07 smithi099 ceph-mon[100270]: osdmap e21: 3 total, 3 up, 3 in 2023-09-23T17:43:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:07 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:07.929 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:43:07 smithi055 sudo[135300]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T17:43:07.930 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:43:07 smithi055 sudo[135300]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:07.930 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:43:07 smithi055 sudo[135300]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:07.930 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:43:07 smithi055 sudo[135300]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:08.429 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135335]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T17:43:08.430 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135335]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:08.430 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135335]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:08.430 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135335]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:08.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:08 smithi099 ceph-mon[100270]: mgrmap e15: y(active, since 114s), standbys: x 2023-09-23T17:43:08.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:08 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/335862231' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f7bc62d-c839-4d05-89c7-98e5abd0a121"}]: dispatch 2023-09-23T17:43:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:08 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/335862231' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f7bc62d-c839-4d05-89c7-98e5abd0a121"}]': finished 2023-09-23T17:43:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:08 smithi099 ceph-mon[100270]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T17:43:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:08 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:08.929 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135373]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T17:43:08.929 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135373]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:08.930 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135373]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:08.930 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135373]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:08.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:08 smithi055 ceph-mon[109912]: mgrmap e15: y(active, since 114s), standbys: x 2023-09-23T17:43:08.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:08 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/335862231' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f7bc62d-c839-4d05-89c7-98e5abd0a121"}]: dispatch 2023-09-23T17:43:08.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:08 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/335862231' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f7bc62d-c839-4d05-89c7-98e5abd0a121"}]': finished 2023-09-23T17:43:08.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:08 smithi055 ceph-mon[109912]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T17:43:08.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:08 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:08.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:08 smithi055 ceph-mon[95848]: mgrmap e15: y(active, since 114s), standbys: x 2023-09-23T17:43:08.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:08 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/335862231' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9f7bc62d-c839-4d05-89c7-98e5abd0a121"}]: dispatch 2023-09-23T17:43:08.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:08 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/335862231' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9f7bc62d-c839-4d05-89c7-98e5abd0a121"}]': finished 2023-09-23T17:43:08.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:08 smithi055 ceph-mon[95848]: osdmap e22: 4 total, 3 up, 4 in 2023-09-23T17:43:08.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:08 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:09.429 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135425]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T17:43:09.429 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135425]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:09.430 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135425]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:09.430 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:43:08 smithi055 sudo[135425]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:09.882 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:09 smithi055 sudo[135617]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T17:43:09.882 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:09 smithi055 sudo[135617]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:09.882 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:09 smithi055 sudo[135617]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:09.882 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:09 smithi055 sudo[135617]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:09 smithi055 sudo[135807]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T17:43:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:09 smithi055 sudo[135807]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:09 smithi055 sudo[135807]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:09 smithi055 sudo[135807]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:10.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 sudo[103087]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T17:43:10.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 sudo[103087]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:10.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 sudo[103087]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:10.379 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 sudo[103087]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:10.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:10 smithi055 ceph-mon[95848]: pgmap v70: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:10 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1115156297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:43:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:10 smithi055 ceph-mon[95848]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T17:43:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:10 smithi055 ceph-mon[109912]: pgmap v70: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:10 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1115156297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:43:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:10 smithi055 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T17:43:10.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 sudo[103090]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T17:43:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 sudo[103090]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 sudo[103090]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 sudo[103090]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 ceph-mon[100270]: pgmap v70: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1115156297' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:43:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:10 smithi099 ceph-mon[100270]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T17:43:11.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:10 smithi055 sudo[136193]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-09-23T17:43:11.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:10 smithi055 sudo[136193]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:11.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:10 smithi055 sudo[136193]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:11.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:10 smithi055 sudo[136193]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T17:43:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:43:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:43:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:43:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T17:43:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:43:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:43:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:43:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T17:43:11.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:11.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T17:43:11.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:43:11.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:43:11.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:43:11.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: pgmap v71: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:11.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[95848]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T17:43:11.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 sudo[136361]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-09-23T17:43:11.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 sudo[136361]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-09-23T17:43:11.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 sudo[136361]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 sudo[136361]: pam_unix(sudo:session): session closed for user root 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T17:43:11.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:43:11.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:43:11.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:43:11.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: pgmap v71: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:11.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:11 smithi055 ceph-mon[109912]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T17:43:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T17:43:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:43:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:43:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:43:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T17:43:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:43:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:43:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:43:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T17:43:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-09-23T17:43:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:43:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:43:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:43:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: pgmap v71: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:11 smithi099 ceph-mon[100270]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-09-23T17:43:13.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:13 smithi055 ceph-mon[95848]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:13.816 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:13 smithi055 ceph-mon[109912]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:14.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:13 smithi099 ceph-mon[100270]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:14.755 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:14 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T17:43:14.755 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:14 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:14.755 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:14 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T17:43:14.756 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:14 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:15.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:14 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T17:43:15.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:14 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:15.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:15 smithi055 ceph-mon[95848]: Deploying daemon osd.3 on smithi055 2023-09-23T17:43:15.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:15 smithi055 ceph-mon[95848]: pgmap v73: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2023-09-23T17:43:15.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:15 smithi055 ceph-mon[109912]: Deploying daemon osd.3 on smithi055 2023-09-23T17:43:15.857 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:15 smithi055 ceph-mon[109912]: pgmap v73: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2023-09-23T17:43:16.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:15 smithi099 ceph-mon[100270]: Deploying daemon osd.3 on smithi055 2023-09-23T17:43:16.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:15 smithi099 ceph-mon[100270]: pgmap v73: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 4.5 KiB/s rd, 196 KiB/s wr, 13 op/s 2023-09-23T17:43:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:17 smithi099 ceph-mon[100270]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-09-23T17:43:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:17 smithi055 ceph-mon[95848]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-09-23T17:43:18.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:17 smithi055 ceph-mon[109912]: pgmap v74: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-09-23T17:43:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:18 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:18 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:18 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:18 smithi055 ceph-mon[109912]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-09-23T17:43:19.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:18 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:19.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:18 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:18 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:18 smithi055 ceph-mon[95848]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-09-23T17:43:19.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:18 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:19.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:18 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:18 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:18 smithi099 ceph-mon[100270]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-09-23T17:43:20.322 INFO:teuthology.orchestra.run.smithi055.stdout:Created osd(s) 3 on host 'smithi055' 2023-09-23T17:43:20.708 DEBUG:teuthology.orchestra.run.smithi055:osd.3> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T17:43:20.710 INFO:tasks.cephadm:Deploying osd.4 on smithi099 with /dev/vg_nvme/lv_4... 2023-09-23T17:43:20.711 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-09-23T17:43:20.743 INFO:journalctl@ceph.osd.3.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T17:43:21.313 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:43:21.430 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[137230]: 2023-09-23T17:43:21.141+0000 7f1f717cf540 -1 osd.3 0 log_to_monitors true 2023-09-23T17:43:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[109912]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T17:43:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[109912]: from='osd.3 [v2:172.21.15.55:6826/2133183145,v1:172.21.15.55:6827/2133183145]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T17:43:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[109912]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T17:43:21.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[95848]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T17:43:21.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[95848]: from='osd.3 [v2:172.21.15.55:6826/2133183145,v1:172.21.15.55:6827/2133183145]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T17:43:21.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:21 smithi055 ceph-mon[95848]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T17:43:21.570 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:21 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.571 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:21 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.571 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:21 smithi099 ceph-mon[100270]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T17:43:21.571 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:21 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.571 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:21 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:21.571 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:21 smithi099 ceph-mon[100270]: from='osd.3 [v2:172.21.15.55:6826/2133183145,v1:172.21.15.55:6827/2133183145]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T17:43:21.571 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:21 smithi099 ceph-mon[100270]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T17:43:22.798 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-09-23T17:43:22.798 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-09-23T17:43:22.799 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2023-09-23T17:43:22.799 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2023-09-23T17:43:22.799 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0110595 s, 948 MB/s 2023-09-23T17:43:22.799 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2023-09-23T17:43:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:22 smithi099 ceph-mon[100270]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T17:43:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:22 smithi099 ceph-mon[100270]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T17:43:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:22 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:22 smithi099 ceph-mon[100270]: from='osd.3 [v2:172.21.15.55:6826/2133183145,v1:172.21.15.55:6827/2133183145]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:22 smithi099 ceph-mon[100270]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:23.166 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi099:vg_nvme/lv_4 2023-09-23T17:43:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-mon[109912]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T17:43:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-mon[109912]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T17:43:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-mon[109912]: from='osd.3 [v2:172.21.15.55:6826/2133183145,v1:172.21.15.55:6827/2133183145]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-mon[109912]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:23.182 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[137230]: 2023-09-23T17:43:22.747+0000 7f1f67fb5700 -1 osd.3 0 waiting for initial osdmap 2023-09-23T17:43:23.182 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[137230]: 2023-09-23T17:43:22.753+0000 7f1f60dac700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:43:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-mon[95848]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T17:43:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-mon[95848]: osdmap e23: 4 total, 3 up, 4 in 2023-09-23T17:43:23.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:23.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-mon[95848]: from='osd.3 [v2:172.21.15.55:6826/2133183145,v1:172.21.15.55:6827/2133183145]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:23.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:22 smithi055 ceph-mon[95848]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:43:23.754 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:43:24.013 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:23 smithi099 ceph-mon[100270]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T17:43:24.013 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:23 smithi099 ceph-mon[100270]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:43:24.013 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:23 smithi099 ceph-mon[100270]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T17:43:24.013 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:23 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:24.013 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:23 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:24.100 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:23 smithi055 ceph-mon[95848]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T17:43:24.100 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:23 smithi055 ceph-mon[95848]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:43:24.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:23 smithi055 ceph-mon[95848]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T17:43:24.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:24.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:24.101 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:23 smithi055 ceph-mon[109912]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-09-23T17:43:24.101 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:23 smithi055 ceph-mon[109912]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]': finished 2023-09-23T17:43:24.101 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:23 smithi055 ceph-mon[109912]: osdmap e24: 4 total, 3 up, 4 in 2023-09-23T17:43:24.101 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:23 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:24.101 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:23 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:25.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: purged_snaps scrub starts 2023-09-23T17:43:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: purged_snaps scrub ok 2023-09-23T17:43:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: osd.3 [v2:172.21.15.55:6826/2133183145,v1:172.21.15.55:6827/2133183145] boot 2023-09-23T17:43:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T17:43:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: Detected new or changed devices on smithi055 2023-09-23T17:43:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi055 to 4037M 2023-09-23T17:43:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:25.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:25.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:25.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:25.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:24 smithi099 ceph-mon[100270]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:25.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: purged_snaps scrub starts 2023-09-23T17:43:25.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: purged_snaps scrub ok 2023-09-23T17:43:25.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:25.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: osd.3 [v2:172.21.15.55:6826/2133183145,v1:172.21.15.55:6827/2133183145] boot 2023-09-23T17:43:25.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T17:43:25.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:25.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: Detected new or changed devices on smithi055 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi055 to 4037M 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:25.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:25.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:25.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:25.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[95848]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:25.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T17:43:25.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T17:43:25.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:25.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: osd.3 [v2:172.21.15.55:6826/2133183145,v1:172.21.15.55:6827/2133183145] boot 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: osdmap e25: 4 total, 4 up, 4 in 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: Detected new or changed devices on smithi055 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi055 to 4037M 2023-09-23T17:43:25.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:25.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:25.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:25.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:25.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:25.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:25.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:24 smithi055 ceph-mon[109912]: pgmap v81: 1 pgs: 1 active+clean; 577 KiB data, 18 MiB used, 268 GiB / 268 GiB avail 2023-09-23T17:43:26.029 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:25 smithi099 ceph-mon[100270]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T17:43:26.030 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:25 smithi099 ceph-mon[100270]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:43:26.030 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:25 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:43:26.030 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:25 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:43:26.030 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:25 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:26.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:25 smithi055 ceph-mon[95848]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T17:43:26.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:25 smithi055 ceph-mon[95848]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:43:26.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:25 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:43:26.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:25 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:43:26.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:25 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:26.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:25 smithi055 ceph-mon[109912]: osdmap e26: 4 total, 4 up, 4 in 2023-09-23T17:43:26.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:25 smithi055 ceph-mon[109912]: from='client.24214 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:43:26.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:25 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:43:26.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:25 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:43:26.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:25 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:27.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:26 smithi099 ceph-mon[100270]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:27.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:26 smithi055 ceph-mon[95848]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:27.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:26 smithi055 ceph-mon[109912]: pgmap v83: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:28.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:27 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/2130472369' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9c0950d-2618-4a47-ad13-035e2bd227f1"}]: dispatch 2023-09-23T17:43:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:27 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9c0950d-2618-4a47-ad13-035e2bd227f1"}]: dispatch 2023-09-23T17:43:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:27 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a9c0950d-2618-4a47-ad13-035e2bd227f1"}]': finished 2023-09-23T17:43:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:27 smithi099 ceph-mon[100270]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T17:43:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:27 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:27 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/2130472369' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9c0950d-2618-4a47-ad13-035e2bd227f1"}]: dispatch 2023-09-23T17:43:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:27 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9c0950d-2618-4a47-ad13-035e2bd227f1"}]: dispatch 2023-09-23T17:43:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:27 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a9c0950d-2618-4a47-ad13-035e2bd227f1"}]': finished 2023-09-23T17:43:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:27 smithi055 ceph-mon[95848]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T17:43:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:27 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:28.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:27 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/2130472369' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9c0950d-2618-4a47-ad13-035e2bd227f1"}]: dispatch 2023-09-23T17:43:28.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:27 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "a9c0950d-2618-4a47-ad13-035e2bd227f1"}]: dispatch 2023-09-23T17:43:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:27 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "a9c0950d-2618-4a47-ad13-035e2bd227f1"}]': finished 2023-09-23T17:43:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:27 smithi055 ceph-mon[109912]: osdmap e27: 5 total, 4 up, 5 in 2023-09-23T17:43:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:27 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:29.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:28 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/2008094731' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:43:29.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:28 smithi099 ceph-mon[100270]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:29.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:28 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/2008094731' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:43:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:28 smithi055 ceph-mon[95848]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:29.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:28 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/2008094731' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:43:29.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:28 smithi055 ceph-mon[109912]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 23 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:31.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:30 smithi055 ceph-mon[109912]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:30 smithi055 ceph-mon[95848]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:31.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:30 smithi099 ceph-mon[100270]: pgmap v86: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:33.979 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:33 smithi099 ceph-mon[100270]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:33.980 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:33 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T17:43:33.980 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:33 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:33 smithi055 ceph-mon[95848]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:33 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T17:43:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:33 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:33 smithi055 ceph-mon[109912]: pgmap v87: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:33 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T17:43:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:33 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:34 smithi099 ceph-mon[100270]: Deploying daemon osd.4 on smithi099 2023-09-23T17:43:35.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:34 smithi055 ceph-mon[95848]: Deploying daemon osd.4 on smithi099 2023-09-23T17:43:35.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:34 smithi055 ceph-mon[109912]: Deploying daemon osd.4 on smithi099 2023-09-23T17:43:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:35 smithi099 ceph-mon[100270]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:36.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:35 smithi055 ceph-mon[95848]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:36.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:35 smithi055 ceph-mon[109912]: pgmap v88: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:37.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:36 smithi099 ceph-mon[100270]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:37.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:36 smithi055 ceph-mon[109912]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:37.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:36 smithi055 ceph-mon[95848]: pgmap v89: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:38.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:38 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:38.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:38 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:38.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:38 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:38 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:38 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:38 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:38.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:38.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:38 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:39.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:39 smithi099 ceph-mon[100270]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:39.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:39 smithi055 ceph-mon[95848]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:39.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:39 smithi055 ceph-mon[109912]: pgmap v90: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:39.861 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 4 on host 'smithi099' 2023-09-23T17:43:40.239 DEBUG:teuthology.orchestra.run.smithi099:osd.4> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.4.service 2023-09-23T17:43:40.241 INFO:tasks.cephadm:Deploying osd.5 on smithi099 with /dev/vg_nvme/lv_3... 2023-09-23T17:43:40.241 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-09-23T17:43:40.269 INFO:journalctl@ceph.osd.4.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T17:43:40.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:40 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:40 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:40 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:40 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.577 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 17:43:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T17:43:40.362+0000 7fbd429b6540 -1 osd.4 0 log_to_monitors true 2023-09-23T17:43:40.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:40 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:40 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:40 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:40 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:40 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:40 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:40 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:40 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:41.021 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:43:41.444 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:41 smithi099 ceph-mon[100270]: from='osd.4 [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T17:43:41.444 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:41 smithi099 ceph-mon[100270]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T17:43:41.444 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:41 smithi099 ceph-mon[100270]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:41 smithi055 ceph-mon[109912]: from='osd.4 [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T17:43:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:41 smithi055 ceph-mon[109912]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T17:43:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:41 smithi055 ceph-mon[109912]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:41.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:41 smithi055 ceph-mon[95848]: from='osd.4 [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T17:43:41.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:41 smithi055 ceph-mon[95848]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-09-23T17:43:41.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:41 smithi055 ceph-mon[95848]: pgmap v91: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:42.428 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:42 smithi099 ceph-mon[100270]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T17:43:42.428 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:42 smithi099 ceph-mon[100270]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T17:43:42.429 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:42.429 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:42 smithi099 ceph-mon[100270]: from='osd.4 [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:43:42.429 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:42 smithi099 ceph-mon[100270]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:43:42.429 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 17:43:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T17:43:42.295+0000 7fbd3919c700 -1 osd.4 0 waiting for initial osdmap 2023-09-23T17:43:42.429 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 17:43:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T17:43:42.301+0000 7fbd34798700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:43:42.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:42 smithi055 ceph-mon[95848]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T17:43:42.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:42 smithi055 ceph-mon[95848]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T17:43:42.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:42.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:42 smithi055 ceph-mon[95848]: from='osd.4 [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:43:42.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:42 smithi055 ceph-mon[95848]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:43:42.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:42 smithi055 ceph-mon[109912]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-09-23T17:43:42.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:42 smithi055 ceph-mon[109912]: osdmap e28: 5 total, 4 up, 5 in 2023-09-23T17:43:42.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:42.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:42 smithi055 ceph-mon[109912]: from='osd.4 [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:43:42.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:42 smithi055 ceph-mon[109912]: from='osd.4 ' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:43:42.931 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-09-23T17:43:42.932 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-09-23T17:43:42.932 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2023-09-23T17:43:42.932 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2023-09-23T17:43:42.932 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0124673 s, 841 MB/s 2023-09-23T17:43:42.932 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2023-09-23T17:43:43.271 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi099:vg_nvme/lv_3 2023-09-23T17:43:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:43:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T17:43:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: osd.4 [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307] boot 2023-09-23T17:43:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T17:43:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:43:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T17:43:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:43.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:43.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:43.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:43.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:43.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:43.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: osd.4 [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307] boot 2023-09-23T17:43:43.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T17:43:43.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:43.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='osd.4 ' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:43:43.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: osdmap e29: 5 total, 4 up, 5 in 2023-09-23T17:43:43.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:43.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:43.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-09-23T17:43:43.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:43:43.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:43.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:43.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:43.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:43.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:43:43.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:43.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: osd.4 [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307] boot 2023-09-23T17:43:43.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: osdmap e30: 5 total, 5 up, 5 in 2023-09-23T17:43:43.686 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:43:43.871 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:43:44.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:44 smithi099 ceph-mon[100270]: purged_snaps scrub starts 2023-09-23T17:43:44.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:44 smithi099 ceph-mon[100270]: purged_snaps scrub ok 2023-09-23T17:43:44.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:44 smithi099 ceph-mon[100270]: Detected new or changed devices on smithi099 2023-09-23T17:43:44.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:44 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi099 to 17168M 2023-09-23T17:43:44.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:44 smithi055 ceph-mon[95848]: purged_snaps scrub starts 2023-09-23T17:43:44.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:44 smithi055 ceph-mon[95848]: purged_snaps scrub ok 2023-09-23T17:43:44.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:44 smithi055 ceph-mon[95848]: Detected new or changed devices on smithi099 2023-09-23T17:43:44.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:44 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi099 to 17168M 2023-09-23T17:43:44.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:44 smithi055 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T17:43:44.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:44 smithi055 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T17:43:44.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:44 smithi055 ceph-mon[109912]: Detected new or changed devices on smithi099 2023-09-23T17:43:44.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:44 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi099 to 17168M 2023-09-23T17:43:45.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:45 smithi055 ceph-mon[95848]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T17:43:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:45 smithi055 ceph-mon[95848]: pgmap v97: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:43:45.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:45 smithi055 ceph-mon[109912]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T17:43:45.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:45 smithi055 ceph-mon[109912]: pgmap v97: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:43:45.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:45 smithi099 ceph-mon[100270]: osdmap e31: 5 total, 5 up, 5 in 2023-09-23T17:43:45.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:45 smithi099 ceph-mon[100270]: pgmap v97: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:43:46.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:46 smithi099 ceph-mon[100270]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T17:43:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:46 smithi099 ceph-mon[100270]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:43:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:43:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:43:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:46 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:46 smithi055 ceph-mon[95848]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T17:43:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:46 smithi055 ceph-mon[95848]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:43:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:43:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:43:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:46 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:46.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:46 smithi055 ceph-mon[109912]: osdmap e32: 5 total, 5 up, 5 in 2023-09-23T17:43:46.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:46 smithi055 ceph-mon[109912]: from='client.24241 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:43:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:46 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:43:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:46 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:43:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:46 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:47 smithi055 ceph-mon[95848]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T17:43:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:47 smithi055 ceph-mon[95848]: pgmap v100: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:43:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:47 smithi055 ceph-mon[95848]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T17:43:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:47 smithi055 ceph-mon[109912]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T17:43:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:47 smithi055 ceph-mon[109912]: pgmap v100: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:43:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:47 smithi055 ceph-mon[109912]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T17:43:47.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:47 smithi099 ceph-mon[100270]: osdmap e33: 5 total, 5 up, 5 in 2023-09-23T17:43:47.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:47 smithi099 ceph-mon[100270]: pgmap v100: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:43:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:47 smithi099 ceph-mon[100270]: osdmap e34: 5 total, 5 up, 5 in 2023-09-23T17:43:48.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:48 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/290887606' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64ac7fc4-adc3-4e7f-8512-a458d0092f49"}]: dispatch 2023-09-23T17:43:48.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:48 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64ac7fc4-adc3-4e7f-8512-a458d0092f49"}]: dispatch 2023-09-23T17:43:48.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:48 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "64ac7fc4-adc3-4e7f-8512-a458d0092f49"}]': finished 2023-09-23T17:43:48.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:48 smithi055 ceph-mon[95848]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T17:43:48.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:48 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:43:48.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:48 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/290887606' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64ac7fc4-adc3-4e7f-8512-a458d0092f49"}]: dispatch 2023-09-23T17:43:48.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:48 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64ac7fc4-adc3-4e7f-8512-a458d0092f49"}]: dispatch 2023-09-23T17:43:48.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:48 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "64ac7fc4-adc3-4e7f-8512-a458d0092f49"}]': finished 2023-09-23T17:43:48.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:48 smithi055 ceph-mon[109912]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T17:43:48.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:48 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:43:48.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:48 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/290887606' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64ac7fc4-adc3-4e7f-8512-a458d0092f49"}]: dispatch 2023-09-23T17:43:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:48 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "64ac7fc4-adc3-4e7f-8512-a458d0092f49"}]: dispatch 2023-09-23T17:43:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:48 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "64ac7fc4-adc3-4e7f-8512-a458d0092f49"}]': finished 2023-09-23T17:43:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:48 smithi099 ceph-mon[100270]: osdmap e35: 6 total, 5 up, 6 in 2023-09-23T17:43:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:48 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:43:49.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:49 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/2069529464' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:43:49.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:49 smithi055 ceph-mon[95848]: pgmap v103: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:43:49.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:49 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/2069529464' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:43:49.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:49 smithi055 ceph-mon[109912]: pgmap v103: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:43:49.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:49 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/2069529464' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:43:49.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:49 smithi099 ceph-mon[100270]: pgmap v103: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:43:51.052 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:50 smithi099 ceph-mon[100270]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 108 KiB/s, 0 objects/s recovering 2023-09-23T17:43:51.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:50 smithi055 ceph-mon[95848]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 108 KiB/s, 0 objects/s recovering 2023-09-23T17:43:51.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:50 smithi055 ceph-mon[109912]: pgmap v104: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 108 KiB/s, 0 objects/s recovering 2023-09-23T17:43:54.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:53 smithi099 ceph-mon[100270]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2023-09-23T17:43:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T17:43:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:54.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:53 smithi055 ceph-mon[109912]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2023-09-23T17:43:54.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:53 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T17:43:54.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:53 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:53 smithi055 ceph-mon[95848]: pgmap v105: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 91 KiB/s, 0 objects/s recovering 2023-09-23T17:43:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T17:43:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:43:54.998 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:54 smithi099 ceph-mon[100270]: Deploying daemon osd.5 on smithi099 2023-09-23T17:43:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:54 smithi055 ceph-mon[95848]: Deploying daemon osd.5 on smithi099 2023-09-23T17:43:55.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:54 smithi055 ceph-mon[109912]: Deploying daemon osd.5 on smithi099 2023-09-23T17:43:55.951 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:55 smithi099 ceph-mon[100270]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T17:43:56.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:55 smithi055 ceph-mon[95848]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T17:43:56.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:55 smithi055 ceph-mon[109912]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T17:43:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:57 smithi099 ceph-mon[100270]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-09-23T17:43:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:57 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:57 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:57 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:58.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:57 smithi055 ceph-mon[95848]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-09-23T17:43:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:57 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:57 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:57 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:57 smithi055 ceph-mon[109912]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 62 KiB/s, 0 objects/s recovering 2023-09-23T17:43:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:57 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:57 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:43:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:57 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:43:59.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:43:59 smithi099 ceph-mon[100270]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-09-23T17:44:00.064 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 5 on host 'smithi099' 2023-09-23T17:44:00.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:43:59 smithi055 ceph-mon[109912]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-09-23T17:44:00.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:43:59 smithi055 ceph-mon[95848]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 53 KiB/s, 0 objects/s recovering 2023-09-23T17:44:00.445 DEBUG:teuthology.orchestra.run.smithi099:osd.5> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T17:44:00.447 INFO:tasks.cephadm:Deploying osd.6 on smithi099 with /dev/vg_nvme/lv_2... 2023-09-23T17:44:00.448 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-09-23T17:44:00.477 INFO:journalctl@ceph.osd.5.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T17:44:01.004 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[112818]: 2023-09-23T17:44:00.751+0000 7f3a50651540 -1 osd.5 0 log_to_monitors true 2023-09-23T17:44:01.004 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.004 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.004 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:01.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:01.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:01.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T17:44:01.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: from='osd.5 [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T17:44:01.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:00 smithi099 ceph-mon[100270]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T17:44:01.164 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:44:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T17:44:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: from='osd.5 [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T17:44:01.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[95848]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T17:44:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T17:44:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: from='osd.5 [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T17:44:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:00 smithi055 ceph-mon[109912]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T17:44:02.280 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 17:44:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[112818]: 2023-09-23T17:44:02.052+0000 7f3a46e37700 -1 osd.5 0 waiting for initial osdmap 2023-09-23T17:44:02.281 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 17:44:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[112818]: 2023-09-23T17:44:02.058+0000 7f3a43c36700 -1 osd.5 37 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:44:02.281 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:02 smithi099 ceph-mon[100270]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T17:44:02.281 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:02 smithi099 ceph-mon[100270]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T17:44:02.281 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:02 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:02.281 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:02 smithi099 ceph-mon[100270]: from='osd.5 [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:02.281 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:02 smithi099 ceph-mon[100270]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:02 smithi055 ceph-mon[95848]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T17:44:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:02 smithi055 ceph-mon[95848]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T17:44:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:02 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:02 smithi055 ceph-mon[95848]: from='osd.5 [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:02 smithi055 ceph-mon[95848]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:02 smithi055 ceph-mon[109912]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T17:44:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:02 smithi055 ceph-mon[109912]: osdmap e36: 6 total, 5 up, 6 in 2023-09-23T17:44:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:02 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:02.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:02 smithi055 ceph-mon[109912]: from='osd.5 [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:02.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:02 smithi055 ceph-mon[109912]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:02.805 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-09-23T17:44:02.805 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-09-23T17:44:02.805 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2023-09-23T17:44:02.805 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2023-09-23T17:44:02.805 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0120078 s, 873 MB/s 2023-09-23T17:44:02.805 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2023-09-23T17:44:03.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:03 smithi099 ceph-mon[100270]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:44:03.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:03 smithi099 ceph-mon[100270]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T17:44:03.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:03.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:03.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:03 smithi099 ceph-mon[100270]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:44:03.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:03 smithi099 ceph-mon[100270]: osd.5 [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267] boot 2023-09-23T17:44:03.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:03 smithi099 ceph-mon[100270]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T17:44:03.168 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:03.168 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi099:vg_nvme/lv_2 2023-09-23T17:44:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[95848]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:44:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[95848]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T17:44:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[95848]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:44:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[95848]: osd.5 [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267] boot 2023-09-23T17:44:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[95848]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T17:44:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[109912]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:44:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[109912]: osdmap e37: 6 total, 5 up, 6 in 2023-09-23T17:44:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[109912]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-09-23T17:44:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[109912]: osd.5 [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267] boot 2023-09-23T17:44:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[109912]: osdmap e38: 6 total, 6 up, 6 in 2023-09-23T17:44:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:44:03.771 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:44:04.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: purged_snaps scrub starts 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: purged_snaps scrub ok 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: Detected new or changed devices on smithi099 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi099 to 8584M 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:04 smithi099 ceph-mon[100270]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T17:44:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T17:44:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T17:44:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: Detected new or changed devices on smithi099 2023-09-23T17:44:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi099 to 8584M 2023-09-23T17:44:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:04.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:04.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[109912]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T17:44:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: purged_snaps scrub starts 2023-09-23T17:44:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: purged_snaps scrub ok 2023-09-23T17:44:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: Detected new or changed devices on smithi099 2023-09-23T17:44:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi099 to 8584M 2023-09-23T17:44:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:04.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:04 smithi055 ceph-mon[95848]: osdmap e39: 6 total, 6 up, 6 in 2023-09-23T17:44:05.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:05 smithi055 ceph-mon[95848]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:05.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:05 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:44:05.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:05 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:44:05.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:05 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:05.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:05 smithi055 ceph-mon[95848]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T17:44:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:05 smithi055 ceph-mon[109912]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:05.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:05 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:44:05.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:05 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:44:05.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:05 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:05.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:05 smithi055 ceph-mon[109912]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T17:44:05.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:05 smithi099 ceph-mon[100270]: pgmap v115: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:05 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:44:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:05 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:44:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:05 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:05 smithi099 ceph-mon[100270]: osdmap e40: 6 total, 6 up, 6 in 2023-09-23T17:44:06.665 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:06 smithi099 ceph-mon[100270]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:06.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:06 smithi055 ceph-mon[95848]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:06 smithi055 ceph-mon[109912]: from='client.24268 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[95848]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T17:44:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[95848]: pgmap v118: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/3391991126' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9be63962-c98b-4df9-8781-270556e51119"}]: dispatch 2023-09-23T17:44:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9be63962-c98b-4df9-8781-270556e51119"}]: dispatch 2023-09-23T17:44:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9be63962-c98b-4df9-8781-270556e51119"}]': finished 2023-09-23T17:44:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[95848]: osdmap e42: 7 total, 6 up, 7 in 2023-09-23T17:44:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[109912]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T17:44:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[109912]: pgmap v118: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/3391991126' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9be63962-c98b-4df9-8781-270556e51119"}]: dispatch 2023-09-23T17:44:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9be63962-c98b-4df9-8781-270556e51119"}]: dispatch 2023-09-23T17:44:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9be63962-c98b-4df9-8781-270556e51119"}]': finished 2023-09-23T17:44:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[109912]: osdmap e42: 7 total, 6 up, 7 in 2023-09-23T17:44:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:07 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:07 smithi099 ceph-mon[100270]: osdmap e41: 6 total, 6 up, 6 in 2023-09-23T17:44:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:07 smithi099 ceph-mon[100270]: pgmap v118: 1 pgs: 1 remapped+peering; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:07 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/3391991126' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9be63962-c98b-4df9-8781-270556e51119"}]: dispatch 2023-09-23T17:44:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:07 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9be63962-c98b-4df9-8781-270556e51119"}]: dispatch 2023-09-23T17:44:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:07 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9be63962-c98b-4df9-8781-270556e51119"}]': finished 2023-09-23T17:44:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:07 smithi099 ceph-mon[100270]: osdmap e42: 7 total, 6 up, 7 in 2023-09-23T17:44:07.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:07 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:08 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/948073646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:44:08.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:08 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/948073646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:44:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:08 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/948073646' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:44:09.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:09 smithi099 ceph-mon[100270]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:09 smithi055 ceph-mon[109912]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:09 smithi055 ceph-mon[95848]: pgmap v120: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:11.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:10 smithi055 ceph-mon[95848]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T17:44:11.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:10 smithi055 ceph-mon[109912]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T17:44:11.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:10 smithi099 ceph-mon[100270]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T17:44:13.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:13 smithi099 ceph-mon[100270]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-09-23T17:44:13.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:13 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T17:44:13.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:13 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:14.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:13 smithi055 ceph-mon[95848]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-09-23T17:44:14.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:13 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T17:44:14.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:13 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:14.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:13 smithi055 ceph-mon[109912]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 76 KiB/s, 0 objects/s recovering 2023-09-23T17:44:14.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:13 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T17:44:14.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:13 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:15.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:14 smithi099 ceph-mon[100270]: Deploying daemon osd.6 on smithi099 2023-09-23T17:44:15.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:14 smithi055 ceph-mon[95848]: Deploying daemon osd.6 on smithi099 2023-09-23T17:44:15.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:14 smithi055 ceph-mon[109912]: Deploying daemon osd.6 on smithi099 2023-09-23T17:44:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:15 smithi099 ceph-mon[100270]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-09-23T17:44:16.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:15 smithi055 ceph-mon[95848]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-09-23T17:44:16.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:15 smithi055 ceph-mon[109912]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 70 KiB/s, 0 objects/s recovering 2023-09-23T17:44:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:17 smithi099 ceph-mon[100270]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T17:44:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:17 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:17 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:17 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:18.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:17 smithi055 ceph-mon[95848]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T17:44:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:17 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:17 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:17 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:18.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:17 smithi055 ceph-mon[109912]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T17:44:18.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:17 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:18.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:17 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:18.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:17 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:19.745 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 6 on host 'smithi099' 2023-09-23T17:44:19.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:19 smithi099 ceph-mon[100270]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 51 KiB/s, 0 objects/s recovering 2023-09-23T17:44:19.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:19 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:19.888 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:19 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:19.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:19 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:19.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:19 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:19.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:19 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:19.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:19 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:19.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:19 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:19.889 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:19 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:20.160 DEBUG:teuthology.orchestra.run.smithi099:osd.6> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T17:44:20.162 INFO:tasks.cephadm:Deploying osd.7 on smithi099 with /dev/vg_nvme/lv_1... 2023-09-23T17:44:20.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-09-23T17:44:20.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[95848]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 51 KiB/s, 0 objects/s recovering 2023-09-23T17:44:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:20.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:20.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:20.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[109912]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail; 51 KiB/s, 0 objects/s recovering 2023-09-23T17:44:20.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:20.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:20.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:20.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:20.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:20.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:20.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:20.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:19 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:20.195 INFO:journalctl@ceph.osd.6.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T17:44:20.576 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:44:20 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[119847]: 2023-09-23T17:44:20.199+0000 7f9471f28540 -1 osd.6 0 log_to_monitors true 2023-09-23T17:44:21.151 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:44:21.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:20 smithi099 ceph-mon[100270]: from='osd.6 [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:44:21.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:20 smithi099 ceph-mon[100270]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:44:21.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:20 smithi099 ceph-mon[100270]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:20 smithi055 ceph-mon[95848]: from='osd.6 [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:44:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:20 smithi055 ceph-mon[95848]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:44:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:20 smithi055 ceph-mon[95848]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:20 smithi055 ceph-mon[109912]: from='osd.6 [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:44:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:20 smithi055 ceph-mon[109912]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:44:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:20 smithi055 ceph-mon[109912]: pgmap v126: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:21 smithi099 ceph-mon[100270]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T17:44:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:21 smithi099 ceph-mon[100270]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T17:44:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:21 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:21 smithi099 ceph-mon[100270]: from='osd.6 [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:21 smithi099 ceph-mon[100270]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:22.328 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:44:21 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[119847]: 2023-09-23T17:44:21.946+0000 7f946870e700 -1 osd.6 0 waiting for initial osdmap 2023-09-23T17:44:22.328 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:44:21 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[119847]: 2023-09-23T17:44:21.954+0000 7f9463509700 -1 osd.6 44 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:44:22.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:21 smithi055 ceph-mon[95848]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T17:44:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:21 smithi055 ceph-mon[95848]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T17:44:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:21 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:21 smithi055 ceph-mon[95848]: from='osd.6 [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:21 smithi055 ceph-mon[95848]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:21 smithi055 ceph-mon[109912]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T17:44:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:21 smithi055 ceph-mon[109912]: osdmap e43: 7 total, 6 up, 7 in 2023-09-23T17:44:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:21 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:21 smithi055 ceph-mon[109912]: from='osd.6 [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:22.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:21 smithi055 ceph-mon[109912]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:22.854 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-09-23T17:44:22.854 INFO:teuthology.orchestra.run.smithi099.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-09-23T17:44:22.854 INFO:teuthology.orchestra.run.smithi099.stderr: stderr: 10+0 records in 2023-09-23T17:44:22.855 INFO:teuthology.orchestra.run.smithi099.stderr:10+0 records out 2023-09-23T17:44:22.855 INFO:teuthology.orchestra.run.smithi099.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0110647 s, 948 MB/s 2023-09-23T17:44:22.855 INFO:teuthology.orchestra.run.smithi099.stderr:--> Zapping successful for: 2023-09-23T17:44:23.162 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:44:23.162 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T17:44:23.163 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:23.163 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:23.163 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:23.163 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:23.163 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:23.163 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:23.163 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:23.163 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:23.163 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:22 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:23.206 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch daemon add osd smithi099:vg_nvme/lv_1 2023-09-23T17:44:23.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:44:23.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T17:44:23.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:23.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:23.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:23.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:23.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:23.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:23.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:23.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:23.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:23.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:44:23.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: osdmap e44: 7 total, 6 up, 7 in 2023-09-23T17:44:23.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:23.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:23.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: pgmap v129: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-09-23T17:44:23.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:23.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:23.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:23.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:23.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:23.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:22 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:23.800 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:44:24.277 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: purged_snaps scrub starts 2023-09-23T17:44:24.277 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: purged_snaps scrub ok 2023-09-23T17:44:24.277 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: Detected new or changed devices on smithi099 2023-09-23T17:44:24.277 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi099 to 5722M 2023-09-23T17:44:24.277 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:24.277 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:24.277 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: osd.6 [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865] boot 2023-09-23T17:44:24.277 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: osdmap e45: 7 total, 7 up, 7 in 2023-09-23T17:44:24.278 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:24.278 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:24.278 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:23 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:24.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: purged_snaps scrub starts 2023-09-23T17:44:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: purged_snaps scrub ok 2023-09-23T17:44:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: Detected new or changed devices on smithi099 2023-09-23T17:44:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi099 to 5722M 2023-09-23T17:44:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: osd.6 [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865] boot 2023-09-23T17:44:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: osdmap e45: 7 total, 7 up, 7 in 2023-09-23T17:44:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T17:44:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T17:44:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: Detected new or changed devices on smithi099 2023-09-23T17:44:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi099 to 5722M 2023-09-23T17:44:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: osd.6 [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865] boot 2023-09-23T17:44:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: osdmap e45: 7 total, 7 up, 7 in 2023-09-23T17:44:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:44:24.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:24.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:23 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:25.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:24 smithi099 ceph-mon[100270]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T17:44:25.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:24 smithi099 ceph-mon[100270]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:24 smithi099 ceph-mon[100270]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T17:44:25.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:24 smithi055 ceph-mon[95848]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T17:44:25.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:24 smithi055 ceph-mon[95848]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:25.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:24 smithi055 ceph-mon[95848]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T17:44:25.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:24 smithi055 ceph-mon[109912]: osdmap e46: 7 total, 7 up, 7 in 2023-09-23T17:44:25.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:24 smithi055 ceph-mon[109912]: pgmap v132: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:25.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:24 smithi055 ceph-mon[109912]: osdmap e47: 7 total, 7 up, 7 in 2023-09-23T17:44:26.247 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:25 smithi099 ceph-mon[100270]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:26.247 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:25 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:44:26.247 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:25 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:44:26.247 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:25 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:26.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:25 smithi055 ceph-mon[95848]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:26.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:25 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:44:26.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:25 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:44:26.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:25 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:26.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:25 smithi055 ceph-mon[109912]: from='client.24295 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi099:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:26.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:25 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-09-23T17:44:26.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:25 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-09-23T17:44:26.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:25 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:27.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:27 smithi099 ceph-mon[100270]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T17:44:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:27 smithi099 ceph-mon[100270]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:27.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:26 smithi055 ceph-mon[95848]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T17:44:27.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:26 smithi055 ceph-mon[95848]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:27.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:27 smithi055 ceph-mon[109912]: osdmap e48: 7 total, 7 up, 7 in 2023-09-23T17:44:27.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:27 smithi055 ceph-mon[109912]: pgmap v135: 1 pgs: 1 remapped+peering; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:28 smithi099 ceph-mon[100270]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T17:44:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:28 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/995422860' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea011d8c-d9f8-4619-9ac0-575813be808a"}]: dispatch 2023-09-23T17:44:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:28 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea011d8c-d9f8-4619-9ac0-575813be808a"}]: dispatch 2023-09-23T17:44:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:28 smithi099 ceph-mon[100270]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea011d8c-d9f8-4619-9ac0-575813be808a"}]': finished 2023-09-23T17:44:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:28 smithi099 ceph-mon[100270]: osdmap e50: 8 total, 7 up, 8 in 2023-09-23T17:44:28.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:28 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[95848]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T17:44:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/995422860' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea011d8c-d9f8-4619-9ac0-575813be808a"}]: dispatch 2023-09-23T17:44:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea011d8c-d9f8-4619-9ac0-575813be808a"}]: dispatch 2023-09-23T17:44:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[95848]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea011d8c-d9f8-4619-9ac0-575813be808a"}]': finished 2023-09-23T17:44:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[95848]: osdmap e50: 8 total, 7 up, 8 in 2023-09-23T17:44:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[109912]: osdmap e49: 7 total, 7 up, 7 in 2023-09-23T17:44:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/995422860' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea011d8c-d9f8-4619-9ac0-575813be808a"}]: dispatch 2023-09-23T17:44:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ea011d8c-d9f8-4619-9ac0-575813be808a"}]: dispatch 2023-09-23T17:44:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[109912]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ea011d8c-d9f8-4619-9ac0-575813be808a"}]': finished 2023-09-23T17:44:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[109912]: osdmap e50: 8 total, 7 up, 8 in 2023-09-23T17:44:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:28 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:29.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:29 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/760076195' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:44:29.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:29 smithi099 ceph-mon[100270]: pgmap v138: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:29 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/760076195' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:44:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:29 smithi055 ceph-mon[95848]: pgmap v138: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:29 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/760076195' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-09-23T17:44:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:29 smithi055 ceph-mon[109912]: pgmap v138: 1 pgs: 1 remapped+peering; 577 KiB data, 40 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:31.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:30 smithi099 ceph-mon[100270]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T17:44:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:30 smithi055 ceph-mon[95848]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T17:44:31.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:30 smithi055 ceph-mon[109912]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 101 KiB/s, 0 objects/s recovering 2023-09-23T17:44:34.008 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:33 smithi099 ceph-mon[100270]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-09-23T17:44:34.008 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:33 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T17:44:34.009 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:33 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:33 smithi055 ceph-mon[95848]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-09-23T17:44:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:33 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T17:44:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:33 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:33 smithi055 ceph-mon[109912]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 86 KiB/s, 0 objects/s recovering 2023-09-23T17:44:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:33 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T17:44:34.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:33 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:34.916 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:34 smithi099 ceph-mon[100270]: Deploying daemon osd.7 on smithi099 2023-09-23T17:44:35.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:34 smithi055 ceph-mon[95848]: Deploying daemon osd.7 on smithi099 2023-09-23T17:44:35.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:34 smithi055 ceph-mon[109912]: Deploying daemon osd.7 on smithi099 2023-09-23T17:44:36.000 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:35 smithi099 ceph-mon[100270]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T17:44:36.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:35 smithi055 ceph-mon[95848]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T17:44:36.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:35 smithi055 ceph-mon[109912]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-09-23T17:44:38.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:37 smithi099 ceph-mon[100270]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T17:44:38.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:37 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:37 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:37 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:38.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:37 smithi055 ceph-mon[95848]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T17:44:38.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:37 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:38.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:37 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:38.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:37 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:37 smithi055 ceph-mon[109912]: pgmap v142: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-09-23T17:44:38.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:37 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:38.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:37 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:38.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:37 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:39.825 INFO:teuthology.orchestra.run.smithi099.stdout:Created osd(s) 7 on host 'smithi099' 2023-09-23T17:44:39.949 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:39 smithi099 ceph-mon[100270]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T17:44:40.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:39 smithi055 ceph-mon[95848]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T17:44:40.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:39 smithi055 ceph-mon[109912]: pgmap v143: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 52 KiB/s, 0 objects/s recovering 2023-09-23T17:44:40.183 DEBUG:teuthology.orchestra.run.smithi099:osd.7> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T17:44:40.185 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-09-23T17:44:40.185 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T17:44:40.217 INFO:journalctl@ceph.osd.7.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T17:44:40.730 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 17:44:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[127690]: 2023-09-23T17:44:40.471+0000 7f2c125b5540 -1 osd.7 0 log_to_monitors true 2023-09-23T17:44:40.803 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:44:41.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[95848]: from='osd.7 [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T17:44:41.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[95848]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T17:44:41.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[95848]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T17:44:41.052 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.052 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.052 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.052 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[109912]: from='osd.7 [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T17:44:41.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[109912]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T17:44:41.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:40 smithi055 ceph-mon[109912]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T17:44:41.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:40 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:40 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:40 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:40 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:41.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:40 smithi099 ceph-mon[100270]: from='osd.7 [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T17:44:41.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:40 smithi099 ceph-mon[100270]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T17:44:41.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:40 smithi099 ceph-mon[100270]: pgmap v144: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T17:44:42.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:41 smithi055 ceph-mon[95848]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T17:44:42.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:41 smithi055 ceph-mon[95848]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T17:44:42.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:41 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:42.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:41 smithi055 ceph-mon[95848]: from='osd.7 [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:42.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:41 smithi055 ceph-mon[95848]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:42.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:41 smithi055 ceph-mon[109912]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T17:44:42.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:41 smithi055 ceph-mon[109912]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T17:44:42.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:41 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:42.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:41 smithi055 ceph-mon[109912]: from='osd.7 [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:42.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:41 smithi055 ceph-mon[109912]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:42.221 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:44:42.326 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 17:44:41 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[127690]: 2023-09-23T17:44:41.948+0000 7f2c08d9b700 -1 osd.7 0 waiting for initial osdmap 2023-09-23T17:44:42.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 17:44:41 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[127690]: 2023-09-23T17:44:41.955+0000 7f2c05399700 -1 osd.7 52 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:44:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:41 smithi099 ceph-mon[100270]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T17:44:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:41 smithi099 ceph-mon[100270]: osdmap e51: 8 total, 7 up, 8 in 2023-09-23T17:44:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:41 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:41 smithi099 ceph-mon[100270]: from='osd.7 [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:41 smithi099 ceph-mon[100270]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:44:42.550 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":52,"num_osds":8,"num_up_osds":7,"osd_up_since":1695491062,"num_in_osds":8,"osd_in_since":1695491067,"num_remapped_pgs":0} 2023-09-23T17:44:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:44:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T17:44:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3981781021' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T17:44:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: Detected new or changed devices on smithi099 2023-09-23T17:44:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi099 to 4292M 2023-09-23T17:44:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:43.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:42 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:44:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T17:44:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3981781021' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T17:44:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: Detected new or changed devices on smithi099 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi099 to 4292M 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:43.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]': finished 2023-09-23T17:44:43.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: osdmap e52: 8 total, 7 up, 8 in 2023-09-23T17:44:43.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:43.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:43.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3981781021' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T17:44:43.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: Detected new or changed devices on smithi099 2023-09-23T17:44:43.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:44:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi099 to 4292M 2023-09-23T17:44:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: pgmap v147: 1 pgs: 1 active+clean; 577 KiB data, 42 MiB used, 626 GiB / 626 GiB avail 2023-09-23T17:44:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:42 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:43.551 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd stat -f json 2023-09-23T17:44:44.155 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:44:44.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:43 smithi099 ceph-mon[100270]: purged_snaps scrub starts 2023-09-23T17:44:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:43 smithi099 ceph-mon[100270]: purged_snaps scrub ok 2023-09-23T17:44:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:43 smithi099 ceph-mon[100270]: osd.7 [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889] boot 2023-09-23T17:44:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:43 smithi099 ceph-mon[100270]: osdmap e53: 8 total, 8 up, 8 in 2023-09-23T17:44:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:43 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:44.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[95848]: purged_snaps scrub starts 2023-09-23T17:44:44.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[95848]: purged_snaps scrub ok 2023-09-23T17:44:44.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:44.334 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[95848]: osd.7 [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889] boot 2023-09-23T17:44:44.334 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[95848]: osdmap e53: 8 total, 8 up, 8 in 2023-09-23T17:44:44.334 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:44.334 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:44.334 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:44.334 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:44.334 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[109912]: purged_snaps scrub starts 2023-09-23T17:44:44.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[109912]: purged_snaps scrub ok 2023-09-23T17:44:44.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:44.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[109912]: osd.7 [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889] boot 2023-09-23T17:44:44.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[109912]: osdmap e53: 8 total, 8 up, 8 in 2023-09-23T17:44:44.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:44:44.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:44.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:44.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:43 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:45.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:44 smithi099 ceph-mon[100270]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T17:44:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:44 smithi099 ceph-mon[100270]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:44:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:44 smithi099 ceph-mon[100270]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T17:44:45.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:44 smithi055 ceph-mon[95848]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T17:44:45.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:44 smithi055 ceph-mon[95848]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:44:45.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:44 smithi055 ceph-mon[95848]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T17:44:45.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:44 smithi055 ceph-mon[109912]: osdmap e54: 8 total, 8 up, 8 in 2023-09-23T17:44:45.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:44 smithi055 ceph-mon[109912]: pgmap v150: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:44:45.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:44 smithi055 ceph-mon[109912]: osdmap e55: 8 total, 8 up, 8 in 2023-09-23T17:44:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:44:45.806 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":55,"num_osds":8,"num_up_osds":8,"osd_up_since":1695491082,"num_in_osds":8,"osd_in_since":1695491067,"num_remapped_pgs":1} 2023-09-23T17:44:45.807 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T17:44:46.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:45 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1151495143' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T17:44:46.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:45 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1151495143' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T17:44:46.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:45 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1151495143' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-09-23T17:44:46.411 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:44:47.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:47 smithi055 ceph-mon[95848]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T17:44:47.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:47 smithi055 ceph-mon[95848]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:44:47.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:47 smithi055 ceph-mon[95848]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T17:44:47.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:47 smithi055 ceph-mon[109912]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T17:44:47.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:47 smithi055 ceph-mon[109912]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:44:47.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:47 smithi055 ceph-mon[109912]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T17:44:47.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:47 smithi099 ceph-mon[100270]: osdmap e56: 8 total, 8 up, 8 in 2023-09-23T17:44:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:47 smithi099 ceph-mon[100270]: pgmap v153: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:44:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:47 smithi099 ceph-mon[100270]: osdmap e57: 8 total, 8 up, 8 in 2023-09-23T17:44:47.701 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:44:47.701 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":57,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","created":"2023-09-23T17:40:13.381005+0000","modified":"2023-09-23T17:44:46.978402+0000","last_up_change":"2023-09-23T17:44:42.935311+0000","last_in_change":"2023-09-23T17:44:27.554515+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T17:43:05.158393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"99e7cae6-389e-461a-8a86-a5789673908b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6803","nonce":3018551716}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6805","nonce":3018551716}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6809","nonce":3018551716}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6807","nonce":3018551716}]},"public_addr":"172.21.15.55:6803/3018551716","cluster_addr":"172.21.15.55:6805/3018551716","heartbeat_back_addr":"172.21.15.55:6809/3018551716","heartbeat_front_addr":"172.21.15.55:6807/3018551716","state":["exists","up"]},{"osd":1,"uuid":"bfb0da01-7a06-430b-b973-209f22ac30d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6811","nonce":1171163846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6813","nonce":1171163846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6817","nonce":1171163846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6815","nonce":1171163846}]},"public_addr":"172.21.15.55:6811/1171163846","cluster_addr":"172.21.15.55:6813/1171163846","heartbeat_back_addr":"172.21.15.55:6817/1171163846","heartbeat_front_addr":"172.21.15.55:6815/1171163846","state":["exists","up"]},{"osd":2,"uuid":"f766d604-db54-4a09-9993-fe1a3ce0048e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6819","nonce":3805819981}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6821","nonce":3805819981}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6825","nonce":3805819981}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6823","nonce":3805819981}]},"public_addr":"172.21.15.55:6819/3805819981","cluster_addr":"172.21.15.55:6821/3805819981","heartbeat_back_addr":"172.21.15.55:6825/3805819981","heartbeat_front_addr":"172.21.15.55:6823/3805819981","state":["exists","up"]},{"osd":3,"uuid":"9f7bc62d-c839-4d05-89c7-98e5abd0a121","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6827","nonce":2133183145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6829","nonce":2133183145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6833","nonce":2133183145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6831","nonce":2133183145}]},"public_addr":"172.21.15.55:6827/2133183145","cluster_addr":"172.21.15.55:6829/2133183145","heartbeat_back_addr":"172.21.15.55:6833/2133183145","heartbeat_front_addr":"172.21.15.55:6831/2133183145","state":["exists","up"]},{"osd":4,"uuid":"a9c0950d-2618-4a47-ad13-035e2bd227f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6801","nonce":2570729307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6803","nonce":2570729307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6807","nonce":2570729307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6805","nonce":2570729307}]},"public_addr":"172.21.15.99:6801/2570729307","cluster_addr":"172.21.15.99:6803/2570729307","heartbeat_back_addr":"172.21.15.99:6807/2570729307","heartbeat_front_addr":"172.21.15.99:6805/2570729307","state":["exists","up"]},{"osd":5,"uuid":"64ac7fc4-adc3-4e7f-8512-a458d0092f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6809","nonce":3690413267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6811","nonce":3690413267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6815","nonce":3690413267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6813","nonce":3690413267}]},"public_addr":"172.21.15.99:6809/3690413267","cluster_addr":"172.21.15.99:6811/3690413267","heartbeat_back_addr":"172.21.15.99:6815/3690413267","heartbeat_front_addr":"172.21.15.99:6813/3690413267","state":["exists","up"]},{"osd":6,"uuid":"9be63962-c98b-4df9-8781-270556e51119","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6817","nonce":2102700865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6819","nonce":2102700865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6823","nonce":2102700865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6821","nonce":2102700865}]},"public_addr":"172.21.15.99:6817/2102700865","cluster_addr":"172.21.15.99:6819/2102700865","heartbeat_back_addr":"172.21.15.99:6823/2102700865","heartbeat_front_addr":"172.21.15.99:6821/2102700865","state":["exists","up"]},{"osd":7,"uuid":"ea011d8c-d9f8-4619-9ac0-575813be808a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6825","nonce":453077889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6827","nonce":453077889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6831","nonce":453077889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6829","nonce":453077889}]},"public_addr":"172.21.15.99:6825/453077889","cluster_addr":"172.21.15.99:6827/453077889","heartbeat_back_addr":"172.21.15.99:6831/453077889","heartbeat_front_addr":"172.21.15.99:6829/453077889","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:22.099373+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:42.018683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:01.856304+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:22.152206+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:41.385241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:01.725749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:21.176273+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:41.516585+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/238250704":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/2173356416":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/658967358":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1826983981":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/880528732":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6800/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/591596032":"2023-09-24T17:41:12.624150+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T17:44:48.066 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-09-23T17:43:05.158393+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '21', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-09-23T17:44:48.066 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool get .mgr pg_num 2023-09-23T17:44:48.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:48 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/398135600' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:44:48.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:48 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/398135600' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:44:48.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:48 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/398135600' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:44:48.668 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:44:49.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:49 smithi055 ceph-mon[95848]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:44:49.102 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:49 smithi055 ceph-mon[109912]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:44:49.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:49 smithi099 ceph-mon[100270]: pgmap v155: 1 pgs: 1 remapped+peering; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:44:49.961 INFO:teuthology.orchestra.run.smithi055.stdout:pg_num: 1 2023-09-23T17:44:50.328 INFO:tasks.cephadm:Adding prometheus.a on smithi099 2023-09-23T17:44:50.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch apply prometheus '1;smithi099=a' 2023-09-23T17:44:50.931 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:44:51.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:50 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1258494134' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T17:44:51.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:50 smithi055 ceph-mon[95848]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T17:44:51.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:50 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1258494134' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T17:44:51.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:50 smithi055 ceph-mon[109912]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T17:44:51.278 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:50 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1258494134' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-09-23T17:44:51.278 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:50 smithi099 ceph-mon[100270]: pgmap v156: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 96 KiB/s, 0 objects/s recovering 2023-09-23T17:44:52.228 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled prometheus update... 2023-09-23T17:44:52.584 DEBUG:teuthology.orchestra.run.smithi099:prometheus.a> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@prometheus.a.service 2023-09-23T17:44:52.586 INFO:tasks.cephadm:Adding node-exporter.a on smithi055 2023-09-23T17:44:52.586 INFO:tasks.cephadm:Adding node-exporter.b on smithi099 2023-09-23T17:44:52.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch apply node-exporter '2;smithi055=a;smithi099=b' 2023-09-23T17:44:52.615 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T17:44:53.230 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:44:53.494 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:53 smithi099 ceph-mon[100270]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:53.494 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:53 smithi099 ceph-mon[100270]: Saving service prometheus spec with placement smithi099=a;count:1 2023-09-23T17:44:53.494 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:53.494 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:53.494 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:53.494 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:53.494 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:53 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:53.494 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:53 smithi099 ceph-mon[100270]: Deploying daemon prometheus.a on smithi099 2023-09-23T17:44:53.494 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:53 smithi099 ceph-mon[100270]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2023-09-23T17:44:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[109912]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[109912]: Saving service prometheus spec with placement smithi099=a;count:1 2023-09-23T17:44:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[109912]: Deploying daemon prometheus.a on smithi099 2023-09-23T17:44:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[109912]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2023-09-23T17:44:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[95848]: from='client.24346 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[95848]: Saving service prometheus spec with placement smithi099=a;count:1 2023-09-23T17:44:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:44:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:44:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:44:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[95848]: Deploying daemon prometheus.a on smithi099 2023-09-23T17:44:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:53 smithi055 ceph-mon[95848]: pgmap v157: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 75 KiB/s, 0 objects/s recovering 2023-09-23T17:44:54.778 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled node-exporter update... 2023-09-23T17:44:55.390 DEBUG:teuthology.orchestra.run.smithi055:node-exporter.a> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@node-exporter.a.service 2023-09-23T17:44:55.392 DEBUG:teuthology.orchestra.run.smithi099:node-exporter.b> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@node-exporter.b.service 2023-09-23T17:44:55.394 INFO:tasks.cephadm:Adding alertmanager.a on smithi055 2023-09-23T17:44:55.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch apply alertmanager '1;smithi055=a' 2023-09-23T17:44:55.424 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T17:44:55.430 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T17:44:56.058 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:44:56.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:55 smithi099 ceph-mon[100270]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-09-23T17:44:56.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:55 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:56.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:55 smithi055 ceph-mon[95848]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-09-23T17:44:56.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:55 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:56.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:55 smithi055 ceph-mon[109912]: pgmap v158: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 66 KiB/s, 0 objects/s recovering 2023-09-23T17:44:56.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:55 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:44:57.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:56 smithi099 ceph-mon[100270]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi055=a;smithi099=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:57.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:56 smithi099 ceph-mon[100270]: Saving service node-exporter spec with placement smithi055=a;smithi099=b;count:2 2023-09-23T17:44:57.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:56 smithi055 ceph-mon[95848]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi055=a;smithi099=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:57.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:56 smithi055 ceph-mon[95848]: Saving service node-exporter spec with placement smithi055=a;smithi099=b;count:2 2023-09-23T17:44:57.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:56 smithi055 ceph-mon[109912]: from='client.24352 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi055=a;smithi099=b", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:44:57.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:56 smithi055 ceph-mon[109912]: Saving service node-exporter spec with placement smithi055=a;smithi099=b;count:2 2023-09-23T17:44:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:57 smithi099 ceph-mon[100270]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T17:44:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:57 smithi055 ceph-mon[109912]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T17:44:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:57 smithi055 ceph-mon[95848]: pgmap v159: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-09-23T17:44:59.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:44:59 smithi099 ceph-mon[100270]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-09-23T17:44:59.963 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled alertmanager update... 2023-09-23T17:45:00.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:44:59 smithi055 ceph-mon[109912]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-09-23T17:45:00.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:44:59 smithi055 ceph-mon[95848]: pgmap v160: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 49 KiB/s, 0 objects/s recovering 2023-09-23T17:45:00.420 DEBUG:teuthology.orchestra.run.smithi055:alertmanager.a> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@alertmanager.a.service 2023-09-23T17:45:00.422 INFO:tasks.cephadm:Adding grafana.a on smithi099 2023-09-23T17:45:00.423 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph orch apply grafana '1;smithi099=a' 2023-09-23T17:45:00.456 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T17:45:01.268 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:45:01.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:00 smithi099 ceph-mon[100270]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi055=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:45:01.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:00 smithi099 ceph-mon[100270]: Saving service alertmanager spec with placement smithi055=a;count:1 2023-09-23T17:45:01.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:00 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:01.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:00 smithi099 ceph-mon[100270]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T17:45:01.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:00 smithi055 ceph-mon[95848]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi055=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:45:01.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:00 smithi055 ceph-mon[95848]: Saving service alertmanager spec with placement smithi055=a;count:1 2023-09-23T17:45:01.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:00 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:01.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:00 smithi055 ceph-mon[95848]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T17:45:01.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:00 smithi055 ceph-mon[109912]: from='client.24358 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi055=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:45:01.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:00 smithi055 ceph-mon[109912]: Saving service alertmanager spec with placement smithi055=a;count:1 2023-09-23T17:45:01.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:00 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:01.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:00 smithi055 ceph-mon[109912]: pgmap v161: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 48 KiB/s, 0 objects/s recovering 2023-09-23T17:45:02.327 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 systemd[1]: Starting Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:45:02.766 INFO:teuthology.orchestra.run.smithi099.stdout:Scheduled grafana update... 2023-09-23T17:45:02.874 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 podman[133582]: 2023-09-23T17:45:02.874 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.873Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T17:45:02.874 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.873Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T17:45:02.874 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.873Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi099 (none))" 2023-09-23T17:45:02.874 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.873Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T17:45:02.874 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.873Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T17:45:02.874 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.874Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T17:45:03.178 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.874Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T17:45:03.179 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.875Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T17:45:03.179 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.875Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T17:45:03.179 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.877Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T17:45:03.179 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.877Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=3.628µs 2023-09-23T17:45:03.179 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.877Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T17:45:03.179 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.878Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=0 2023-09-23T17:45:03.179 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.878Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=37.828µs wal_replay_duration=623.638µs wbl_replay_duration=117ns total_replay_duration=689.652µs 2023-09-23T17:45:03.179 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.879Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T17:45:03.180 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.879Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T17:45:03.180 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.879Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T17:45:03.180 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.893Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.553401ms db_storage=839ns remote_storage=1.1µs web_handler=416ns query_engine=524ns scrape=262.126µs scrape_sd=18.298µs notify=1.554µs notify_sd=1.965µs rules=13.107013ms tracing=5.585µs 2023-09-23T17:45:03.180 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.893Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T17:45:03.180 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:45:02.893Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T17:45:03.180 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:03 smithi099 bash[133582]: d1f2db0179af654ab3317be5787e2ab799db7f8dbf967cf95c93973e52ff4d04 2023-09-23T17:45:03.180 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:45:03 smithi099 systemd[1]: Started Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:45:03.232 DEBUG:teuthology.orchestra.run.smithi099:grafana.a> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@grafana.a.service 2023-09-23T17:45:03.234 INFO:tasks.cephadm:Setting up client nodes... 2023-09-23T17:45:03.234 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T17:45:03.263 INFO:journalctl@ceph.grafana.a.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T17:45:03.849 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:04.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[109912]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:04.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[109912]: from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:45:04.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[109912]: Saving service grafana spec with placement smithi099=a;count:1 2023-09-23T17:45:04.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T17:45:04.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[95848]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:04.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[95848]: from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:45:04.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[95848]: Saving service grafana spec with placement smithi099=a;count:1 2023-09-23T17:45:04.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:03 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T17:45:04.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:03 smithi099 ceph-mon[100270]: pgmap v162: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:04.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:03 smithi099 ceph-mon[100270]: from='client.24364 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi099=a", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:45:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:03 smithi099 ceph-mon[100270]: Saving service grafana spec with placement smithi099=a;count:1 2023-09-23T17:45:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' 2023-09-23T17:45:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:03 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-09-23T17:45:04.324 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ignoring --setuser ceph since I am not root 2023-09-23T17:45:04.324 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ignoring --setgroup ceph since I am not root 2023-09-23T17:45:04.325 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:04 smithi055 ceph-mgr[96116]: -- 172.21.15.55:0/1193884759 <== mon.0 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d109d409c0 con 0x55d109d16800 2023-09-23T17:45:04.530 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ignoring --setuser ceph since I am not root 2023-09-23T17:45:04.530 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ignoring --setgroup ceph since I am not root 2023-09-23T17:45:04.531 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:04 smithi099 ceph-mgr[102400]: -- 172.21.15.99:0/679991050 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5652985a09c0 con 0x565298577000 2023-09-23T17:45:04.531 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:04.366+0000 7fbd5a3f31c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:45:04.531 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:04.443+0000 7fbd5a3f31c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:45:04.531 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:04.530+0000 7fbd5a3f31c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:45:04.594 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:04.392+0000 7f169024a1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:45:04.594 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:04.486+0000 7f169024a1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:45:04.785 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:04.620+0000 7fbd5a3f31c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:45:04.855 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:04.593+0000 7f169024a1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:45:04.856 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:04.711+0000 7f169024a1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:45:05.076 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:04.785+0000 7fbd5a3f31c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:45:05.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:04.922+0000 7f169024a1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:45:05.325 INFO:teuthology.orchestra.run.smithi055.stdout:[client.0] 2023-09-23T17:45:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: key = AQAhJA9lBPAIExAA6LapekioULquwigXumOVVw== 2023-09-23T17:45:05.393 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:05 smithi099 ceph-mon[100270]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T17:45:05.393 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:05 smithi099 ceph-mon[100270]: mgrmap e16: y(active, since 3m), standbys: x 2023-09-23T17:45:05.393 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:05 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:05.088+0000 7fbd5a3f31c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:45:05.393 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:05 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:05.174+0000 7fbd5a3f31c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:45:05.393 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:05 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:05.300+0000 7fbd5a3f31c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:45:05.393 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:05 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:05.393+0000 7fbd5a3f31c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:45:05.588 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:05.323+0000 7f169024a1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:45:05.588 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:05.436+0000 7f169024a1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:45:05.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:05 smithi055 ceph-mon[95848]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T17:45:05.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:05 smithi055 ceph-mon[95848]: mgrmap e16: y(active, since 3m), standbys: x 2023-09-23T17:45:05.588 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:05 smithi055 ceph-mon[109912]: from='mgr.14152 172.21.15.55:0/3996141057' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-09-23T17:45:05.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:05 smithi055 ceph-mon[109912]: mgrmap e16: y(active, since 3m), standbys: x 2023-09-23T17:45:05.708 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:45:05.709 DEBUG:teuthology.orchestra.run.smithi055:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-09-23T17:45:05.709 DEBUG:teuthology.orchestra.run.smithi055:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-09-23T17:45:05.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-09-23T17:45:05.875 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:05.587+0000 7f169024a1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:45:05.875 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:05.702+0000 7f169024a1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:45:06.076 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:05 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:05.813+0000 7fbd5a3f31c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:45:06.381 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:45:06.569 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:06 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:06.393+0000 7fbd5a3f31c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:45:06.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:06 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/361033115' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:06.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:06 smithi099 ceph-mon[100270]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:06.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:06 smithi099 ceph-mon[100270]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T17:45:06.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:06.215+0000 7f169024a1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:45:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:06 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/361033115' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:06 smithi055 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:06 smithi055 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T17:45:06.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:06 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/361033115' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:06.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:06 smithi055 ceph-mon[95848]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:06.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:06 smithi055 ceph-mon[95848]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T17:45:06.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:06 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:06.568+0000 7fbd5a3f31c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:45:07.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:06.894+0000 7f169024a1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:45:07.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:07 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:07.085+0000 7f169024a1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:45:07.389 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:07 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:07.362+0000 7fbd5a3f31c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:45:07.640 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:07 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:07.460+0000 7fbd5a3f31c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:45:07.852 INFO:teuthology.orchestra.run.smithi099.stdout:[client.1] 2023-09-23T17:45:07.852 INFO:teuthology.orchestra.run.smithi099.stdout: key = AQAjJA9l5Mx8MhAAyQtzp/52Q2mrUa09+mDvJw== 2023-09-23T17:45:07.901 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:07 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:07.639+0000 7fbd5a3f31c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:45:08.154 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:08 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:08.044+0000 7f169024a1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:45:08.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:07 smithi055 ceph-mon[109912]: from='client.? 172.21.15.99:0/4287917800' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:08.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:07 smithi055 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:08.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:07 smithi055 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T17:45:08.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:07 smithi055 ceph-mon[95848]: from='client.? 172.21.15.99:0/4287917800' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:08.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:07 smithi055 ceph-mon[95848]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:08.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:07 smithi055 ceph-mon[95848]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T17:45:08.206 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:07 smithi099 ceph-mon[100270]: from='client.? 172.21.15.99:0/4287917800' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:08.206 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:07 smithi099 ceph-mon[100270]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-09-23T17:45:08.206 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:07 smithi099 ceph-mon[100270]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-09-23T17:45:08.207 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:07 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:07.922+0000 7fbd5a3f31c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:45:08.207 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:08.015+0000 7fbd5a3f31c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:45:08.231 DEBUG:teuthology.orchestra.run.smithi099:> set -ex 2023-09-23T17:45:08.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-09-23T17:45:08.232 DEBUG:teuthology.orchestra.run.smithi099:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-09-23T17:45:08.275 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-09-23T17:45:08.275 DEBUG:teuthology.orchestra.run.smithi055:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-09-23T17:45:08.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:08 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:08.153+0000 7f169024a1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:45:08.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:08 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:08.373+0000 7f169024a1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:45:08.569 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:08.310+0000 7fbd5a3f31c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:45:08.605 INFO:teuthology.misc.health.smithi055.stdout: 2023-09-23T17:45:08.606 INFO:teuthology.misc.health.smithi055.stdout:{"epoch":57,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","created":"2023-09-23T17:40:13.381005+0000","modified":"2023-09-23T17:44:46.978402+0000","last_up_change":"2023-09-23T17:44:42.935311+0000","last_in_change":"2023-09-23T17:44:27.554515+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T17:43:05.158393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"99e7cae6-389e-461a-8a86-a5789673908b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6803","nonce":3018551716}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6805","nonce":3018551716}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6809","nonce":3018551716}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6807","nonce":3018551716}]},"public_addr":"172.21.15.55:6803/3018551716","cluster_addr":"172.21.15.55:6805/3018551716","heartbeat_back_addr":"172.21.15.55:6809/3018551716","heartbeat_front_addr":"172.21.15.55:6807/3018551716","state":["exists","up"]},{"osd":1,"uuid":"bfb0da01-7a06-430b-b973-209f22ac30d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6811","nonce":1171163846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6813","nonce":1171163846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6817","nonce":1171163846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6815","nonce":1171163846}]},"public_addr":"172.21.15.55:6811/1171163846","cluster_addr":"172.21.15.55:6813/1171163846","heartbeat_back_addr":"172.21.15.55:6817/1171163846","heartbeat_front_addr":"172.21.15.55:6815/1171163846","state":["exists","up"]},{"osd":2,"uuid":"f766d604-db54-4a09-9993-fe1a3ce0048e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6819","nonce":3805819981}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6821","nonce":3805819981}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6825","nonce":3805819981}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6823","nonce":3805819981}]},"public_addr":"172.21.15.55:6819/3805819981","cluster_addr":"172.21.15.55:6821/3805819981","heartbeat_back_addr":"172.21.15.55:6825/3805819981","heartbeat_front_addr":"172.21.15.55:6823/3805819981","state":["exists","up"]},{"osd":3,"uuid":"9f7bc62d-c839-4d05-89c7-98e5abd0a121","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6827","nonce":2133183145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6829","nonce":2133183145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6833","nonce":2133183145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6831","nonce":2133183145}]},"public_addr":"172.21.15.55:6827/2133183145","cluster_addr":"172.21.15.55:6829/2133183145","heartbeat_back_addr":"172.21.15.55:6833/2133183145","heartbeat_front_addr":"172.21.15.55:6831/2133183145","state":["exists","up"]},{"osd":4,"uuid":"a9c0950d-2618-4a47-ad13-035e2bd227f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6801","nonce":2570729307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6803","nonce":2570729307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6807","nonce":2570729307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6805","nonce":2570729307}]},"public_addr":"172.21.15.99:6801/2570729307","cluster_addr":"172.21.15.99:6803/2570729307","heartbeat_back_addr":"172.21.15.99:6807/2570729307","heartbeat_front_addr":"172.21.15.99:6805/2570729307","state":["exists","up"]},{"osd":5,"uuid":"64ac7fc4-adc3-4e7f-8512-a458d0092f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":41,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6809","nonce":3690413267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6811","nonce":3690413267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6815","nonce":3690413267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6813","nonce":3690413267}]},"public_addr":"172.21.15.99:6809/3690413267","cluster_addr":"172.21.15.99:6811/3690413267","heartbeat_back_addr":"172.21.15.99:6815/3690413267","heartbeat_front_addr":"172.21.15.99:6813/3690413267","state":["exists","up"]},{"osd":6,"uuid":"9be63962-c98b-4df9-8781-270556e51119","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":48,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6817","nonce":2102700865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6819","nonce":2102700865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6823","nonce":2102700865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6821","nonce":2102700865}]},"public_addr":"172.21.15.99:6817/2102700865","cluster_addr":"172.21.15.99:6819/2102700865","heartbeat_back_addr":"172.21.15.99:6823/2102700865","heartbeat_front_addr":"172.21.15.99:6821/2102700865","state":["exists","up"]},{"osd":7,"uuid":"ea011d8c-d9f8-4619-9ac0-575813be808a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6825","nonce":453077889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6827","nonce":453077889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6831","nonce":453077889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6829","nonce":453077889}]},"public_addr":"172.21.15.99:6825/453077889","cluster_addr":"172.21.15.99:6827/453077889","heartbeat_back_addr":"172.21.15.99:6831/453077889","heartbeat_front_addr":"172.21.15.99:6829/453077889","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:22.099373+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:42.018683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:01.856304+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:22.152206+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:41.385241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:01.725749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:21.176273+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:41.516585+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/238250704":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/2173356416":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/658967358":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1826983981":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/880528732":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6800/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/591596032":"2023-09-24T17:41:12.624150+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T17:45:08.619 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-09-23T17:45:08.619 INFO:tasks.cephadm:Creating RBD pool 2023-09-23T17:45:08.619 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-09-23T17:45:08.826 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:08.568+0000 7fbd5a3f31c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:45:08.901 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:08 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:08.728+0000 7f169024a1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:45:08.901 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:08 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:08.839+0000 7f169024a1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:45:09.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:08 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3913766961' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:45:09.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:08 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3913766961' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:45:09.226 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:09.264 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:08 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3913766961' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:45:09.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:09 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:09.178+0000 7f169024a1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:45:09.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:09.263+0000 7fbd5a3f31c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:45:09.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:09.393+0000 7fbd5a3f31c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:45:09.706 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:09 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:09.515+0000 7f169024a1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:45:09.841 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:09.582+0000 7fbd5a3f31c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:45:09.841 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:09.695+0000 7fbd5a3f31c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:45:10.326 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:09.910+0000 7fbd5a3f31c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:45:10.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:10.370+0000 7f169024a1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:45:10.911 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:10.531+0000 7f169024a1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:45:10.912 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:10.772+0000 7f169024a1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:45:11.032 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'rbd' created 2023-09-23T17:45:11.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:10 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3020131040' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T17:45:11.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:10 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:10.756+0000 7fbd5a3f31c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:45:11.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:10 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: [23/Sep/2023:17:45:10] ENGINE Bus STARTING 2023-09-23T17:45:11.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:10 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: CherryPy Checker: 2023-09-23T17:45:11.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:10 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: The Application mounted at '' has an empty config. 2023-09-23T17:45:11.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:10 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: 2023-09-23T17:45:11.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:10 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: [23/Sep/2023:17:45:10] ENGINE Serving on http://:::9283 2023-09-23T17:45:11.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:10 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: [23/Sep/2023:17:45:10] ENGINE Bus STARTED 2023-09-23T17:45:11.165 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:10.911+0000 7f169024a1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:45:11.165 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:10 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3020131040' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T17:45:11.165 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:10 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3020131040' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T17:45:11.380 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-09-23T17:45:11.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:11.191+0000 7f169024a1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:45:11.994 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:12.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[95848]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T17:45:12.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:45:12.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:45:12.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:45:12.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:45:12.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[95848]: Standby manager daemon x restarted 2023-09-23T17:45:12.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[95848]: Standby manager daemon x started 2023-09-23T17:45:12.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[95848]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T17:45:12.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[95848]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T17:45:12.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[95848]: mgrmap e17: y(active, since 3m), standbys: x 2023-09-23T17:45:12.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T17:45:12.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:45:12.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:45:12.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:45:12.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:45:12.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[109912]: Standby manager daemon x restarted 2023-09-23T17:45:12.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T17:45:12.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T17:45:12.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[109912]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T17:45:12.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:11 smithi055 ceph-mon[109912]: mgrmap e17: y(active, since 3m), standbys: x 2023-09-23T17:45:12.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:11 smithi099 ceph-mon[100270]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-09-23T17:45:12.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:11 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:45:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:11 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:45:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:11 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:45:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:11 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/3965359567' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:45:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:11 smithi099 ceph-mon[100270]: Standby manager daemon x restarted 2023-09-23T17:45:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:11 smithi099 ceph-mon[100270]: Standby manager daemon x started 2023-09-23T17:45:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:11 smithi099 ceph-mon[100270]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-09-23T17:45:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:11 smithi099 ceph-mon[100270]: osdmap e58: 8 total, 8 up, 8 in 2023-09-23T17:45:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:11 smithi099 ceph-mon[100270]: mgrmap e17: y(active, since 3m), standbys: x 2023-09-23T17:45:12.514 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:12 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:12.262+0000 7f169024a1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:45:12.515 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:12 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:45:12] ENGINE Bus STARTING 2023-09-23T17:45:12.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:12 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: CherryPy Checker: 2023-09-23T17:45:12.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:12 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: The Application mounted at '' has an empty config. 2023-09-23T17:45:12.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:12 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:45:12.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:12 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:45:12] ENGINE Serving on http://:::9283 2023-09-23T17:45:12.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:12 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:45:12] ENGINE Bus STARTED 2023-09-23T17:45:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T17:45:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: Active manager daemon y restarted 2023-09-23T17:45:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: Activating manager daemon y 2023-09-23T17:45:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T17:45:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: mgrmap e18: y(active, starting, since 0.00867576s), standbys: x 2023-09-23T17:45:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:45:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:45:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:45:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:45:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:45:13.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:45:13.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: Manager daemon y is now available 2023-09-23T17:45:13.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:45:13.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:45:13.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:13.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:13.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:45:13.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:45:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T17:45:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: Active manager daemon y restarted 2023-09-23T17:45:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: Activating manager daemon y 2023-09-23T17:45:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: mgrmap e18: y(active, starting, since 0.00867576s), standbys: x 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:45:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: Manager daemon y is now available 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:45:13.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:45:13.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: osdmap e59: 8 total, 8 up, 8 in 2023-09-23T17:45:13.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: Active manager daemon y restarted 2023-09-23T17:45:13.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: Activating manager daemon y 2023-09-23T17:45:13.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: osdmap e60: 8 total, 8 up, 8 in 2023-09-23T17:45:13.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: mgrmap e18: y(active, starting, since 0.00867576s), standbys: x 2023-09-23T17:45:13.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:45:13.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:45:13.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:45:13.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:45:13.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:45:13.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:45:13.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: Manager daemon y is now available 2023-09-23T17:45:13.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:45:13.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:45:13.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:13.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:13.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:45:13.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:45:14.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:14 smithi099 ceph-mon[100270]: [23/Sep/2023:17:45:13] ENGINE Bus STARTING 2023-09-23T17:45:14.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:14 smithi099 ceph-mon[100270]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T17:45:14.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:14 smithi099 ceph-mon[100270]: mgrmap e19: y(active, since 1.02405s), standbys: x 2023-09-23T17:45:14.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:14 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:14.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:14 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2879724487' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:45:14.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:14 smithi099 ceph-mon[100270]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:45:14.354 INFO:teuthology.orchestra.run.smithi055.stderr:enabled application 'rbd' on pool 'rbd' 2023-09-23T17:45:14.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[95848]: [23/Sep/2023:17:45:13] ENGINE Bus STARTING 2023-09-23T17:45:14.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[95848]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T17:45:14.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[95848]: mgrmap e19: y(active, since 1.02405s), standbys: x 2023-09-23T17:45:14.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:14.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2879724487' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:45:14.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[95848]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:45:14.366 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[109912]: [23/Sep/2023:17:45:13] ENGINE Bus STARTING 2023-09-23T17:45:14.366 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[109912]: osdmap e61: 8 total, 8 up, 8 in 2023-09-23T17:45:14.366 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[109912]: mgrmap e19: y(active, since 1.02405s), standbys: x 2023-09-23T17:45:14.366 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:14.366 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2879724487' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:45:14.366 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:14 smithi055 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:45:14.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:45:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:45:14.764 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T17:45:14.764 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T17:45:14.765 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T17:45:15.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:15 smithi099 ceph-mon[100270]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:15.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:15 smithi099 ceph-mon[100270]: [23/Sep/2023:17:45:13] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:45:15.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:15 smithi099 ceph-mon[100270]: [23/Sep/2023:17:45:13] ENGINE Bus STARTED 2023-09-23T17:45:15.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:15 smithi099 ceph-mon[100270]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T17:45:15.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:15 smithi099 ceph-mon[100270]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T17:45:15.382 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:15.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:15 smithi055 ceph-mon[109912]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:15.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:15 smithi055 ceph-mon[109912]: [23/Sep/2023:17:45:13] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:45:15.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:15 smithi055 ceph-mon[109912]: [23/Sep/2023:17:45:13] ENGINE Bus STARTED 2023-09-23T17:45:15.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:15 smithi055 ceph-mon[109912]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T17:45:15.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:15 smithi055 ceph-mon[109912]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T17:45:15.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:15 smithi055 ceph-mon[95848]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:15.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:15 smithi055 ceph-mon[95848]: [23/Sep/2023:17:45:13] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:45:15.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:15 smithi055 ceph-mon[95848]: [23/Sep/2023:17:45:13] ENGINE Bus STARTED 2023-09-23T17:45:15.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:15 smithi055 ceph-mon[95848]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-09-23T17:45:15.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:15 smithi055 ceph-mon[95848]: osdmap e62: 8 total, 8 up, 8 in 2023-09-23T17:45:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[95848]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[95848]: mgrmap e20: y(active, since 2s), standbys: x 2023-09-23T17:45:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[109912]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:16.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[109912]: mgrmap e20: y(active, since 2s), standbys: x 2023-09-23T17:45:16.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:16 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:16 smithi099 ceph-mon[100270]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:16.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:16 smithi099 ceph-mon[100270]: mgrmap e20: y(active, since 2s), standbys: x 2023-09-23T17:45:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:16 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:16 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:16 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:16 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:16.812 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:45:17.232 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":20,"active_gid":24338,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":32209804},{"type":"v1","addr":"172.21.15.55:6801","nonce":32209804}]},"active_addr":"172.21.15.55:6801/32209804","active_change":"2023-09-23T17:45:12.277064+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24373,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":2880364962}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":880894918}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":573100440}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":2064251389}]}]} 2023-09-23T17:45:17.237 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T17:45:17.237 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T17:45:17.237 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T17:45:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:17.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:45:17.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2028308070' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T17:45:17.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:17.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:45:17.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:17 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2028308070' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T17:45:17.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:45:17.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:17.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:17.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:45:17.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:17 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2028308070' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T17:45:17.872 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:18.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:18.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi099 to 4036M 2023-09-23T17:45:18.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:45:18.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:45:18.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:45:18.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:45:18.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:45:18.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:45:18.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:45:18.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:45:18.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:18.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi099 to 4036M 2023-09-23T17:45:18.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:18 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:18.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi099 to 4036M 2023-09-23T17:45:18.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:45:18.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:45:18.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:45:18.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:45:18.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:45:18.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:45:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:45:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:45:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:18 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:19.391 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:45:19.391 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":62,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","created":"2023-09-23T17:40:13.381005+0000","modified":"2023-09-23T17:45:14.347468+0000","last_up_change":"2023-09-23T17:44:42.935311+0000","last_in_change":"2023-09-23T17:44:27.554515+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T17:43:05.158393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T17:45:10.969164+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"99e7cae6-389e-461a-8a86-a5789673908b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6803","nonce":3018551716}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6805","nonce":3018551716}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6809","nonce":3018551716}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6807","nonce":3018551716}]},"public_addr":"172.21.15.55:6803/3018551716","cluster_addr":"172.21.15.55:6805/3018551716","heartbeat_back_addr":"172.21.15.55:6809/3018551716","heartbeat_front_addr":"172.21.15.55:6807/3018551716","state":["exists","up"]},{"osd":1,"uuid":"bfb0da01-7a06-430b-b973-209f22ac30d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6811","nonce":1171163846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6813","nonce":1171163846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6817","nonce":1171163846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6815","nonce":1171163846}]},"public_addr":"172.21.15.55:6811/1171163846","cluster_addr":"172.21.15.55:6813/1171163846","heartbeat_back_addr":"172.21.15.55:6817/1171163846","heartbeat_front_addr":"172.21.15.55:6815/1171163846","state":["exists","up"]},{"osd":2,"uuid":"f766d604-db54-4a09-9993-fe1a3ce0048e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6819","nonce":3805819981}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6821","nonce":3805819981}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6825","nonce":3805819981}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6823","nonce":3805819981}]},"public_addr":"172.21.15.55:6819/3805819981","cluster_addr":"172.21.15.55:6821/3805819981","heartbeat_back_addr":"172.21.15.55:6825/3805819981","heartbeat_front_addr":"172.21.15.55:6823/3805819981","state":["exists","up"]},{"osd":3,"uuid":"9f7bc62d-c839-4d05-89c7-98e5abd0a121","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6827","nonce":2133183145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6829","nonce":2133183145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6833","nonce":2133183145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6831","nonce":2133183145}]},"public_addr":"172.21.15.55:6827/2133183145","cluster_addr":"172.21.15.55:6829/2133183145","heartbeat_back_addr":"172.21.15.55:6833/2133183145","heartbeat_front_addr":"172.21.15.55:6831/2133183145","state":["exists","up"]},{"osd":4,"uuid":"a9c0950d-2618-4a47-ad13-035e2bd227f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6801","nonce":2570729307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6803","nonce":2570729307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6807","nonce":2570729307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6805","nonce":2570729307}]},"public_addr":"172.21.15.99:6801/2570729307","cluster_addr":"172.21.15.99:6803/2570729307","heartbeat_back_addr":"172.21.15.99:6807/2570729307","heartbeat_front_addr":"172.21.15.99:6805/2570729307","state":["exists","up"]},{"osd":5,"uuid":"64ac7fc4-adc3-4e7f-8512-a458d0092f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6809","nonce":3690413267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6811","nonce":3690413267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6815","nonce":3690413267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6813","nonce":3690413267}]},"public_addr":"172.21.15.99:6809/3690413267","cluster_addr":"172.21.15.99:6811/3690413267","heartbeat_back_addr":"172.21.15.99:6815/3690413267","heartbeat_front_addr":"172.21.15.99:6813/3690413267","state":["exists","up"]},{"osd":6,"uuid":"9be63962-c98b-4df9-8781-270556e51119","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6817","nonce":2102700865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6819","nonce":2102700865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6823","nonce":2102700865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6821","nonce":2102700865}]},"public_addr":"172.21.15.99:6817/2102700865","cluster_addr":"172.21.15.99:6819/2102700865","heartbeat_back_addr":"172.21.15.99:6823/2102700865","heartbeat_front_addr":"172.21.15.99:6821/2102700865","state":["exists","up"]},{"osd":7,"uuid":"ea011d8c-d9f8-4619-9ac0-575813be808a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6825","nonce":453077889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6827","nonce":453077889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6831","nonce":453077889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6829","nonce":453077889}]},"public_addr":"172.21.15.99:6825/453077889","cluster_addr":"172.21.15.99:6827/453077889","heartbeat_back_addr":"172.21.15.99:6831/453077889","heartbeat_front_addr":"172.21.15.99:6829/453077889","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:22.099373+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:42.018683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:01.856304+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:22.152206+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:41.385241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:01.725749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:21.176273+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:41.516585+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/4290529019":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/1663054712":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6801/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6800/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/238250704":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/2173356416":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/658967358":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1826983981":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/3953275940":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/880528732":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6800/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/218576037":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/591596032":"2023-09-24T17:41:12.624150+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T17:45:19.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:19 smithi099 ceph-mon[100270]: Deploying daemon alertmanager.a on smithi055 2023-09-23T17:45:19.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:19 smithi055 ceph-mon[95848]: Deploying daemon alertmanager.a on smithi055 2023-09-23T17:45:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:19 smithi055 ceph-mon[109912]: Deploying daemon alertmanager.a on smithi055 2023-09-23T17:45:20.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:20 smithi099 ceph-mon[100270]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:20 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2372735844' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:45:20.605 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:20 smithi055 ceph-mon[95848]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:20.605 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:20 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2372735844' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:45:20.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:20 smithi055 ceph-mon[109912]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:20.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:20 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2372735844' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:45:20.624 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T17:45:20.624 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T17:45:21.427 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:21.505 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:21 smithi055 ceph-mon[109912]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:21.505 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:21 smithi055 ceph-mon[95848]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:21.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:21 smithi099 ceph-mon[100270]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:23.352 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 systemd[1]: Starting Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:45:23.539 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:45:23.539 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":62,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","created":"2023-09-23T17:40:13.381005+0000","modified":"2023-09-23T17:45:14.347468+0000","last_up_change":"2023-09-23T17:44:42.935311+0000","last_in_change":"2023-09-23T17:44:27.554515+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T17:43:05.158393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T17:45:10.969164+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"99e7cae6-389e-461a-8a86-a5789673908b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":55,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6803","nonce":3018551716}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6805","nonce":3018551716}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6809","nonce":3018551716}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6807","nonce":3018551716}]},"public_addr":"172.21.15.55:6803/3018551716","cluster_addr":"172.21.15.55:6805/3018551716","heartbeat_back_addr":"172.21.15.55:6809/3018551716","heartbeat_front_addr":"172.21.15.55:6807/3018551716","state":["exists","up"]},{"osd":1,"uuid":"bfb0da01-7a06-430b-b973-209f22ac30d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6811","nonce":1171163846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6813","nonce":1171163846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6817","nonce":1171163846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6815","nonce":1171163846}]},"public_addr":"172.21.15.55:6811/1171163846","cluster_addr":"172.21.15.55:6813/1171163846","heartbeat_back_addr":"172.21.15.55:6817/1171163846","heartbeat_front_addr":"172.21.15.55:6815/1171163846","state":["exists","up"]},{"osd":2,"uuid":"f766d604-db54-4a09-9993-fe1a3ce0048e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6819","nonce":3805819981}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6821","nonce":3805819981}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6825","nonce":3805819981}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6823","nonce":3805819981}]},"public_addr":"172.21.15.55:6819/3805819981","cluster_addr":"172.21.15.55:6821/3805819981","heartbeat_back_addr":"172.21.15.55:6825/3805819981","heartbeat_front_addr":"172.21.15.55:6823/3805819981","state":["exists","up"]},{"osd":3,"uuid":"9f7bc62d-c839-4d05-89c7-98e5abd0a121","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6827","nonce":2133183145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6829","nonce":2133183145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6833","nonce":2133183145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6831","nonce":2133183145}]},"public_addr":"172.21.15.55:6827/2133183145","cluster_addr":"172.21.15.55:6829/2133183145","heartbeat_back_addr":"172.21.15.55:6833/2133183145","heartbeat_front_addr":"172.21.15.55:6831/2133183145","state":["exists","up"]},{"osd":4,"uuid":"a9c0950d-2618-4a47-ad13-035e2bd227f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6801","nonce":2570729307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6803","nonce":2570729307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6807","nonce":2570729307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6805","nonce":2570729307}]},"public_addr":"172.21.15.99:6801/2570729307","cluster_addr":"172.21.15.99:6803/2570729307","heartbeat_back_addr":"172.21.15.99:6807/2570729307","heartbeat_front_addr":"172.21.15.99:6805/2570729307","state":["exists","up"]},{"osd":5,"uuid":"64ac7fc4-adc3-4e7f-8512-a458d0092f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6809","nonce":3690413267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6811","nonce":3690413267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6815","nonce":3690413267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6813","nonce":3690413267}]},"public_addr":"172.21.15.99:6809/3690413267","cluster_addr":"172.21.15.99:6811/3690413267","heartbeat_back_addr":"172.21.15.99:6815/3690413267","heartbeat_front_addr":"172.21.15.99:6813/3690413267","state":["exists","up"]},{"osd":6,"uuid":"9be63962-c98b-4df9-8781-270556e51119","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6817","nonce":2102700865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6819","nonce":2102700865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6823","nonce":2102700865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6821","nonce":2102700865}]},"public_addr":"172.21.15.99:6817/2102700865","cluster_addr":"172.21.15.99:6819/2102700865","heartbeat_back_addr":"172.21.15.99:6823/2102700865","heartbeat_front_addr":"172.21.15.99:6821/2102700865","state":["exists","up"]},{"osd":7,"uuid":"ea011d8c-d9f8-4619-9ac0-575813be808a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":58,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6825","nonce":453077889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6827","nonce":453077889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6831","nonce":453077889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6829","nonce":453077889}]},"public_addr":"172.21.15.99:6825/453077889","cluster_addr":"172.21.15.99:6827/453077889","heartbeat_back_addr":"172.21.15.99:6831/453077889","heartbeat_front_addr":"172.21.15.99:6829/453077889","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:22.099373+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:42.018683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:01.856304+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:22.152206+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:41.385241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:01.725749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:21.176273+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:41.516585+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/4290529019":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/1663054712":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6801/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6800/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/238250704":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/2173356416":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/658967358":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1826983981":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/3953275940":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/880528732":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6800/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/218576037":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/591596032":"2023-09-24T17:41:12.624150+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T17:45:23.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:23 smithi055 ceph-mon[95848]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:23 smithi055 ceph-mon[109912]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:23.681 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 podman[146523]: 2023-09-23T17:45:23.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:23 smithi099 ceph-mon[100270]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:24.092 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:45:23.828Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T17:45:24.092 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:45:23.828Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T17:45:24.092 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 bash[146523]: 63ddddaef99d394d4631ca38fe5dbfac3dca6da8a7e235cb2154a9e37a590483 2023-09-23T17:45:24.093 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:45:23.830Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.55 port=9094 2023-09-23T17:45:24.093 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:45:23.831Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T17:45:24.093 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:45:23.858Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T17:45:24.093 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:45:23.859Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T17:45:24.093 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:45:23.862Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T17:45:24.093 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:45:23.862Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T17:45:24.093 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:23 smithi055 systemd[1]: Started Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:45:24.095 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T17:45:24.095 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T17:45:24.096 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T17:45:24.096 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T17:45:24.097 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T17:45:24.097 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T17:45:24.098 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T17:45:24.098 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T17:45:24.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/227968601' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:45:24.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.381 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.384 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/227968601' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:45:24.384 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.384 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.384 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.384 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.385 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.385 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:45:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:45:24.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:24 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/227968601' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:45:24.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:24 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:24 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:24 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:24 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:24 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:24 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:24.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:45:24] ENGINE Bus STOPPING 2023-09-23T17:45:24.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:45:24] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T17:45:24.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:45:24] ENGINE Bus STOPPED 2023-09-23T17:45:24.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:45:24] ENGINE Bus STARTING 2023-09-23T17:45:24.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:45:24] ENGINE Serving on http://:::9283 2023-09-23T17:45:24.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:45:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:45:24] ENGINE Bus STARTED 2023-09-23T17:45:24.897 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:24.917 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:24.934 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:25.006 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:25.008 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:25.008 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:25.054 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:25.055 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:25.381 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-mon[95848]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:25.381 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T17:45:25.381 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-mon[95848]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T17:45:25.381 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:25.381 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-mon[95848]: Deploying daemon grafana.a on smithi099 2023-09-23T17:45:25.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-mon[109912]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T17:45:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-mon[109912]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T17:45:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-mon[109912]: Deploying daemon grafana.a on smithi099 2023-09-23T17:45:25.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:25 smithi099 ceph-mon[100270]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:25 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T17:45:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:25 smithi099 ceph-mon[100270]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-09-23T17:45:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:25 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:25 smithi099 ceph-mon[100270]: Deploying daemon grafana.a on smithi099 2023-09-23T17:45:25.930 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:45:25.831Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000186569s 2023-09-23T17:45:27.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:27 smithi055 ceph-mon[95848]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:27.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:27 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:27.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:27 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:27.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:27 smithi055 ceph-mon[109912]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:27.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:27 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:27.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:27 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:27.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:27 smithi099 ceph-mon[100270]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:27 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:27 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:29.644 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:29 smithi055 ceph-mon[109912]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:29.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:29 smithi055 ceph-mon[95848]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:29.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:29 smithi099 ceph-mon[100270]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:30.280 INFO:teuthology.orchestra.run.smithi055.stdout:55834574883 2023-09-23T17:45:30.280 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T17:45:30.281 INFO:teuthology.orchestra.run.smithi055.stdout:227633266700 2023-09-23T17:45:30.281 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T17:45:30.347 INFO:teuthology.orchestra.run.smithi055.stdout:193273528336 2023-09-23T17:45:30.348 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T17:45:30.600 INFO:teuthology.orchestra.run.smithi055.stdout:107374182427 2023-09-23T17:45:30.600 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T17:45:30.847 INFO:teuthology.orchestra.run.smithi055.stdout:163208757267 2023-09-23T17:45:30.847 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T17:45:31.549 INFO:teuthology.orchestra.run.smithi055.stdout:34359738407 2023-09-23T17:45:31.549 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T17:45:32.062 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:31 smithi055 ceph-mon[95848]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:32.062 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:31 smithi055 ceph-mon[109912]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:32.063 INFO:teuthology.orchestra.run.smithi055.stdout:128849018901 2023-09-23T17:45:32.063 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T17:45:32.215 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:32.262 INFO:teuthology.orchestra.run.smithi055.stdout:77309411359 2023-09-23T17:45:32.263 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T17:45:32.263 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:32.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:31 smithi099 ceph-mon[100270]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:32.376 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:32.572 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:32.575 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:32.579 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:33.929 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:45:33 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:45:33.832Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001672011s 2023-09-23T17:45:34.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:33 smithi099 ceph-mon[100270]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:34.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:33 smithi055 ceph-mon[95848]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:34.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:33 smithi055 ceph-mon[109912]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:34.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:45:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:45:35.195 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:35.538 INFO:teuthology.orchestra.run.smithi055.stdout:193273528337 2023-09-23T17:45:35.700 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:35.891 INFO:teuthology.orchestra.run.smithi055.stdout:227633266701 2023-09-23T17:45:35.924 INFO:teuthology.orchestra.run.smithi055.stdout:34359738408 2023-09-23T17:45:36.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:35 smithi055 ceph-mon[95848]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:36.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:35 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1013569276' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T17:45:36.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:35 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2336320349' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T17:45:36.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:35 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/99435326' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T17:45:36.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:35 smithi055 ceph-mon[109912]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:36.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:35 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1013569276' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T17:45:36.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:35 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2336320349' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T17:45:36.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:35 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/99435326' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T17:45:36.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:35 smithi099 ceph-mon[100270]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:36.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:35 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1013569276' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T17:45:36.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:35 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2336320349' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T17:45:36.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:35 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/99435326' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T17:45:36.447 INFO:tasks.cephadm.ceph_manager.ceph:need seq 193273528336 got 193273528337 for osd.6 2023-09-23T17:45:36.447 DEBUG:teuthology.parallel:result is None 2023-09-23T17:45:36.760 INFO:teuthology.orchestra.run.smithi055.stdout:107374182428 2023-09-23T17:45:36.825 INFO:tasks.cephadm.ceph_manager.ceph:need seq 227633266700 got 227633266701 for osd.7 2023-09-23T17:45:36.825 DEBUG:teuthology.parallel:result is None 2023-09-23T17:45:37.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:36 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3008473937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T17:45:37.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:36 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3008473937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T17:45:37.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:36 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3008473937' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T17:45:37.904 INFO:teuthology.orchestra.run.smithi055.stdout:55834574884 2023-09-23T17:45:37.943 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182427 got 107374182428 for osd.3 2023-09-23T17:45:37.943 DEBUG:teuthology.parallel:result is None 2023-09-23T17:45:37.955 INFO:teuthology.orchestra.run.smithi055.stdout:163208757268 2023-09-23T17:45:38.006 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738407 got 34359738408 for osd.0 2023-09-23T17:45:38.006 DEBUG:teuthology.parallel:result is None 2023-09-23T17:45:38.112 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:37 smithi055 ceph-mon[109912]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:38.113 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:37 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2783340679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T17:45:38.113 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:37 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/455343467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T17:45:38.113 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:37 smithi055 ceph-mon[95848]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:38.113 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:37 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2783340679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T17:45:38.113 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:37 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/455343467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T17:45:38.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:37 smithi099 ceph-mon[100270]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:38.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:37 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2783340679' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T17:45:38.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:37 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/455343467' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T17:45:38.511 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757267 got 163208757268 for osd.5 2023-09-23T17:45:38.511 DEBUG:teuthology.parallel:result is None 2023-09-23T17:45:38.746 INFO:teuthology.orchestra.run.smithi055.stdout:128849018902 2023-09-23T17:45:38.891 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574883 got 55834574884 for osd.1 2023-09-23T17:45:38.892 DEBUG:teuthology.parallel:result is None 2023-09-23T17:45:39.034 INFO:teuthology.orchestra.run.smithi055.stdout:77309411361 2023-09-23T17:45:39.063 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:39 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1761139153' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T17:45:39.063 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:39 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1761139153' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T17:45:39.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:39 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1761139153' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T17:45:39.129 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018901 got 128849018902 for osd.4 2023-09-23T17:45:39.129 DEBUG:teuthology.parallel:result is None 2023-09-23T17:45:39.399 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411359 got 77309411361 for osd.2 2023-09-23T17:45:39.399 DEBUG:teuthology.parallel:result is None 2023-09-23T17:45:39.399 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T17:45:39.399 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:45:40.003 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:40.262 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:40 smithi055 ceph-mon[109912]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:40.262 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:40 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3906779323' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T17:45:40.263 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:40 smithi055 ceph-mon[95848]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:40.263 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:40 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3906779323' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T17:45:40.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:40 smithi099 ceph-mon[100270]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:40.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:40 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3906779323' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T17:45:41.316 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:45:41.317 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:45:41.763 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2023-09-23T17:45:40.310259+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50208,"kb_used_data":6560,"kb_used_omap":0,"kb_used_meta":43584,"kb_avail":749911008,"statfs":{"total":767960285184,"available":767908872192,"internally_reserved":0,"allocated":6717440,"data_stored":3837639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44630016},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002255"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389987+0000","last_change":"2023-09-23T17:45:12.029937+0000","last_active":"2023-09-23T17:45:13.389987+0000","last_peered":"2023-09-23T17:45:13.389987+0000","last_clean":"2023-09-23T17:45:13.389987+0000","last_became_active":"2023-09-23T17:45:12.029619+0000","last_became_peered":"2023-09-23T17:45:12.029619+0000","last_unstale":"2023-09-23T17:45:13.389987+0000","last_undegraded":"2023-09-23T17:45:13.389987+0000","last_fullsized":"2023-09-23T17:45:13.389987+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:49.407425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389529+0000","last_change":"2023-09-23T17:45:12.288804+0000","last_active":"2023-09-23T17:45:13.389529+0000","last_peered":"2023-09-23T17:45:13.389529+0000","last_clean":"2023-09-23T17:45:13.389529+0000","last_became_active":"2023-09-23T17:45:12.288496+0000","last_became_peered":"2023-09-23T17:45:12.288496+0000","last_unstale":"2023-09-23T17:45:13.389529+0000","last_undegraded":"2023-09-23T17:45:13.389529+0000","last_fullsized":"2023-09-23T17:45:13.389529+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:50.938577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389711+0000","last_change":"2023-09-23T17:45:12.291380+0000","last_active":"2023-09-23T17:45:13.389711+0000","last_peered":"2023-09-23T17:45:13.389711+0000","last_clean":"2023-09-23T17:45:13.389711+0000","last_became_active":"2023-09-23T17:45:12.290977+0000","last_became_peered":"2023-09-23T17:45:12.290977+0000","last_unstale":"2023-09-23T17:45:13.389711+0000","last_undegraded":"2023-09-23T17:45:13.389711+0000","last_fullsized":"2023-09-23T17:45:13.389711+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:31.931088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389694+0000","last_change":"2023-09-23T17:45:12.030797+0000","last_active":"2023-09-23T17:45:13.389694+0000","last_peered":"2023-09-23T17:45:13.389694+0000","last_clean":"2023-09-23T17:45:13.389694+0000","last_became_active":"2023-09-23T17:45:12.030530+0000","last_became_peered":"2023-09-23T17:45:12.030530+0000","last_unstale":"2023-09-23T17:45:13.389694+0000","last_undegraded":"2023-09-23T17:45:13.389694+0000","last_fullsized":"2023-09-23T17:45:13.389694+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:58:27.428598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389660+0000","last_change":"2023-09-23T17:44:46.990708+0000","last_active":"2023-09-23T17:45:13.389660+0000","last_peered":"2023-09-23T17:45:13.389660+0000","last_clean":"2023-09-23T17:45:13.389660+0000","last_became_active":"2023-09-23T17:44:46.990139+0000","last_became_peered":"2023-09-23T17:44:46.990139+0000","last_unstale":"2023-09-23T17:45:13.389660+0000","last_undegraded":"2023-09-23T17:45:13.389660+0000","last_fullsized":"2023-09-23T17:45:13.389660+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:51.013639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.288289+0000","last_change":"2023-09-23T17:45:12.031891+0000","last_active":"2023-09-23T17:45:13.288289+0000","last_peered":"2023-09-23T17:45:13.288289+0000","last_clean":"2023-09-23T17:45:13.288289+0000","last_became_active":"2023-09-23T17:45:12.031569+0000","last_became_peered":"2023-09-23T17:45:12.031569+0000","last_unstale":"2023-09-23T17:45:13.288289+0000","last_undegraded":"2023-09-23T17:45:13.288289+0000","last_fullsized":"2023-09-23T17:45:13.288289+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:08.345244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.294085+0000","last_change":"2023-09-23T17:45:12.621717+0000","last_active":"2023-09-23T17:45:13.294085+0000","last_peered":"2023-09-23T17:45:13.294085+0000","last_clean":"2023-09-23T17:45:13.294085+0000","last_became_active":"2023-09-23T17:45:12.621356+0000","last_became_peered":"2023-09-23T17:45:12.621356+0000","last_unstale":"2023-09-23T17:45:13.294085+0000","last_undegraded":"2023-09-23T17:45:13.294085+0000","last_fullsized":"2023-09-23T17:45:13.294085+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:35.490791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.288287+0000","last_change":"2023-09-23T17:45:12.031241+0000","last_active":"2023-09-23T17:45:13.288287+0000","last_peered":"2023-09-23T17:45:13.288287+0000","last_clean":"2023-09-23T17:45:13.288287+0000","last_became_active":"2023-09-23T17:45:12.030905+0000","last_became_peered":"2023-09-23T17:45:12.030905+0000","last_unstale":"2023-09-23T17:45:13.288287+0000","last_undegraded":"2023-09-23T17:45:13.288287+0000","last_fullsized":"2023-09-23T17:45:13.288287+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:45.853453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389508+0000","last_change":"2023-09-23T17:45:12.030639+0000","last_active":"2023-09-23T17:45:13.389508+0000","last_peered":"2023-09-23T17:45:13.389508+0000","last_clean":"2023-09-23T17:45:13.389508+0000","last_became_active":"2023-09-23T17:45:12.030334+0000","last_became_peered":"2023-09-23T17:45:12.030334+0000","last_unstale":"2023-09-23T17:45:13.389508+0000","last_undegraded":"2023-09-23T17:45:13.389508+0000","last_fullsized":"2023-09-23T17:45:13.389508+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:35.756901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266702,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6308,"kb_used_data":1180,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":1208320,"data_stored":848483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51700000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.318}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40699999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46100000000000002}]}]},{"osd":6,"up_from":45,"seq":193273528338,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6372,"kb_used_data":1180,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":1208320,"data_stored":848483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.56999999999999995}]},{"osd":1,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.443},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.55400000000000005}]},{"osd":3,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.47999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52900000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574885,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6500,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738652,"statfs":{"total":95995035648,"available":95988379648,"internally_reserved":0,"allocated":618496,"data_stored":258438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:44:45 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.26600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.377,"15min":0.36299999999999999},"min":{"1min":0.29999999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.46000000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.43099999999999999,"15min":0.42099999999999999},"min":{"1min":0.32300000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.442,"15min":0.432},"min":{"1min":0.313,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.51800000000000002,"15min":0.52200000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.69899999999999995,"5min":0.871,"15min":0.871},"last":0.499},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.496,"15min":0.499},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.64600000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.44}]},{"osd":4,"last update":"Sat Sep 23 17:44:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.52700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55100000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738409,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7144,"kb_used_data":1184,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738008,"statfs":{"total":95995035648,"available":95987720192,"internally_reserved":0,"allocated":1212416,"data_stored":848806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:44:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.38600000000000001,"15min":0.38},"min":{"1min":0.27600000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26300000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.30099999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.42999999999999999,"15min":0.433},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.505},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.441,"15min":0.439},"min":{"1min":0.33600000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.59499999999999997,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.40899999999999997}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.49399999999999999,"15min":0.504},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.60299999999999998,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51900000000000002,"15min":0.52600000000000002},"min":{"1min":0.37,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.64700000000000002,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.44800000000000001}]},{"osd":4,"last update":"Sat Sep 23 17:44:45 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:45:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.60599999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.56299999999999994}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411361,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6436,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738716,"statfs":{"total":95995035648,"available":95988445184,"internally_reserved":0,"allocated":618496,"data_stored":258438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:07 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.39500000000000002,"15min":0.39600000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.432,"15min":0.42599999999999999},"min":{"1min":0.376,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.622,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.497}]},{"osd":1,"last update":"Sat Sep 23 17:45:07 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.42999999999999999,"15min":0.42699999999999999},"min":{"1min":0.33300000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.58699999999999997,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46000000000000002,"15min":0.45900000000000002},"min":{"1min":0.29099999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.53500000000000003}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.53600000000000003,"15min":0.54500000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.52200000000000002,"15min":0.53300000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.72899999999999998,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.57999999999999996}]},{"osd":4,"last update":"Sat Sep 23 17:44:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.623}]},{"osd":5,"last update":"Sat Sep 23 17:45:07 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.69799999999999995}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.72199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68700000000000006}]}]},{"osd":3,"up_from":25,"seq":107374182429,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5796,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739356,"statfs":{"total":95995035648,"available":95989100544,"internally_reserved":0,"allocated":618496,"data_stored":258438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.58499999999999996,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.42199999999999999,"15min":0.42399999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.43099999999999999,"15min":0.435},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.45200000000000001,"15min":0.45400000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.48299999999999998,"15min":0.48799999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.47299999999999998,"15min":0.47899999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.64000000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:44:45 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.372},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":5.2069999999999999,"5min":5.2069999999999999,"15min":5.2069999999999999},"last":0.47599999999999998}]},{"osd":5,"last update":"Sat Sep 23 17:45:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.45300000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018903,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5860,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739292,"statfs":{"total":95995035648,"available":95989035008,"internally_reserved":0,"allocated":618496,"data_stored":258438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:44:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.495},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.41199999999999998}]},{"osd":1,"last update":"Sat Sep 23 17:44:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:44:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.5129999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:44:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.097,"5min":1.097,"15min":1.097},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.45900000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:45:03 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.38},{"interface":"front","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.52800000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50600000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757269,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5792,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739360,"statfs":{"total":95995035648,"available":95989104640,"internally_reserved":0,"allocated":614400,"data_stored":258115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.5960000000000001,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":0.497},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.45300000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":1.5680000000000001,"5min":1.5680000000000001,"15min":1.5680000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.581,"5min":1.581,"15min":1.581},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.6060000000000001,"5min":1.6060000000000001,"15min":1.6060000000000001},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.5569999999999999,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.6279999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.371}]},{"osd":4,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.738,"5min":1.738,"15min":1.738},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.778,"5min":1.778,"15min":1.778},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":0.51400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:45:41.766 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:45:42.135 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:42 smithi099 ceph-mon[100270]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:42.372 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:42.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:42 smithi055 ceph-mon[95848]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:42.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:42 smithi055 ceph-mon[109912]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:42.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:42 smithi099 systemd[1]: Starting Ceph grafana.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:45:43.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:43 smithi055 ceph-mon[95848]: from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:45:43.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:43 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:43.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:43 smithi055 ceph-mon[109912]: from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:45:43.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:43 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:43.198 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-mon[100270]: from='client.24506 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:45:43.198 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:43.199 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:42 smithi099 podman[135758]: 2023-09-23T17:45:43.199 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 bash[135758]: 59417245e1d8a7fadb08aee8a59aa61cab14a07591da97490996757dfd810323 2023-09-23T17:45:43.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 systemd[1]: Started Ceph grafana.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:45:43.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.300919298Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T17:45:43.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301087271Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T17:45:43.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301103955Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T17:45:43.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301110543Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T17:45:43.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301117148Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T17:45:43.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301122945Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T17:45:43.452 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.30112839Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301134125Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301140072Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301145541Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301151004Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301156388Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301161799Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=settings t=2023-09-23T17:45:43.301167441Z level=info msg="App mode production" 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=sqlstore t=2023-09-23T17:45:43.301215613Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=sqlstore t=2023-09-23T17:45:43.301233589Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.302131401Z level=info msg="Starting DB migrations" 2023-09-23T17:45:43.453 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.302208387Z level=info msg="Executing migration" id="create migration_log table" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.302733187Z level=info msg="Executing migration" id="create user table" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.303154591Z level=info msg="Executing migration" id="add unique index user.login" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.304035257Z level=info msg="Executing migration" id="add unique index user.email" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.306915558Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.309703735Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.310140123Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.31089687Z level=info msg="Executing migration" id="create user table v2" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.311344223Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.311764926Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.312199698Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.312521683Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-09-23T17:45:43.454 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.312890131Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.313274887Z level=info msg="Executing migration" id="Update user table charset" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.313546833Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.313966098Z level=info msg="Executing migration" id="Add missing user data" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.314303584Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.314741704Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.315148848Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.315538098Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.316627482Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.317096694Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.317548233Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-09-23T17:45:43.455 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.317974517Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.318400473Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.318826905Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.319117078Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.319495632Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.319916222Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.320307667Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.320722183Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.321336033Z level=info msg="Executing migration" id="create temp_user v2" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.321758902Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.322163951Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.322583263Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-09-23T17:45:43.456 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.322990726Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.323421206Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.323777829Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.324132947Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.324461055Z level=info msg="Executing migration" id="create star table" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.324902168Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.325355326Z level=info msg="Executing migration" id="create org table v1" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.325756071Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.326163363Z level=info msg="Executing migration" id="create org_user table v1" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.326582292Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.326989146Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-09-23T17:45:43.457 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.329833005Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.33043376Z level=info msg="Executing migration" id="Update org table charset" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.333065669Z level=info msg="Executing migration" id="Update org_user table charset" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.335831557Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.339004011Z level=info msg="Executing migration" id="create dashboard table" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.341694249Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.3444214Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.347332922Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.350177678Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.35301375Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.355758105Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.35840792Z level=info msg="Executing migration" id="create dashboard v2" 2023-09-23T17:45:43.458 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.361094435Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.363799916Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.366464819Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.369136543Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.371689242Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.374408329Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.377126818Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.379671684Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.3823681Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.384792077Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.387617902Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-09-23T17:45:43.459 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.39029295Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.393037867Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.395539056Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.398366854Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.399079241Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.400667399Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.401227018Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.403526526Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.404213042Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.4050734Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.405888153Z level=info msg="Executing migration" id="Update dashboard title length" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.406699249Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.407582984Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-09-23T17:45:43.460 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.408449744Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.409564883Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.410341699Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.411258887Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.412280507Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.412892789Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.413775531Z level=info msg="Executing migration" id="Add check_sum column" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.414510336Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.415359153Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.416186875Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.416944695Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-09-23T17:45:43.461 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.417715896Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.41872089Z level=info msg="Executing migration" id="create data_source table" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.419453758Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.42024282Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.420676639Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.421538732Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.422526713Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.423673368Z level=info msg="Executing migration" id="create data_source table v2" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.424546897Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.425493032Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.426304625Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-09-23T17:45:43.462 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.427102958Z level=info msg="Executing migration" id="Add column with_credentials" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.42766915Z level=info msg="Executing migration" id="Add secure json data column" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.428239284Z level=info msg="Executing migration" id="Update data_source table charset" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.429085657Z level=info msg="Executing migration" id="Update initial version to 1" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.429436068Z level=info msg="Executing migration" id="Add read_only data column" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.430034254Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.430959309Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.431714187Z level=info msg="Executing migration" id="Add uid column" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.434738054Z level=info msg="Executing migration" id="Update uid value" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.437576204Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.440437394Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.44337384Z level=info msg="Executing migration" id="create api_key table" 2023-09-23T17:45:43.463 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.445889885Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-09-23T17:45:43.464 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.44913776Z level=info msg="Executing migration" id="add index api_key.key" 2023-09-23T17:45:43.464 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.451885919Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-09-23T17:45:43.702 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.454630636Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-09-23T17:45:43.702 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.457305351Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-09-23T17:45:43.702 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.460131111Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-09-23T17:45:43.702 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.463266825Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-09-23T17:45:43.702 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.46606638Z level=info msg="Executing migration" id="create api_key table v2" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.466470669Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.466912552Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.46978453Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.470250477Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.470534619Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.470921052Z level=info msg="Executing migration" id="Update api_key table charset" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.471197558Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.471767283Z level=info msg="Executing migration" id="Add service account foreign key" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.472335342Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.472616175Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-09-23T17:45:43.703 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.4731953Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.473766746Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.474147161Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.477105364Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.477545884Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.478016278Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.478444429Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.478854949Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.479145951Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.479419919Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.479998727Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.480589133Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.483546551Z level=info msg="Executing migration" id="create quota table v1" 2023-09-23T17:45:43.704 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.486444015Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.489041928Z level=info msg="Executing migration" id="Update quota table charset" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.489361153Z level=info msg="Executing migration" id="create plugin_setting table" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.49224068Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.49516515Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.497798855Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.498101739Z level=info msg="Executing migration" id="create session table" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.501044424Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.503857954Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.506901406Z level=info msg="Executing migration" id="create playlist table v2" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.509614937Z level=info msg="Executing migration" id="create playlist item table v2" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.512428952Z level=info msg="Executing migration" id="Update playlist table charset" 2023-09-23T17:45:43.705 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.515456926Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.518084883Z level=info msg="Executing migration" id="drop preferences table v2" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.520667442Z level=info msg="Executing migration" id="drop preferences table v3" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.523449422Z level=info msg="Executing migration" id="create preferences table v3" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.526207181Z level=info msg="Executing migration" id="Update preferences table charset" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.528723735Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.531483419Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.534263402Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.536964227Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.539619276Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.542245733Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.544870625Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-09-23T17:45:43.706 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.547650905Z level=info msg="Executing migration" id="create alert table v1" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.550490574Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.553141963Z level=info msg="Executing migration" id="add index alert state" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.555719729Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.558671937Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.561196246Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.563851282Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.566674927Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.568067497Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.568825425Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.569820475Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.570509502Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-09-23T17:45:43.707 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.571351027Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.572290221Z level=info msg="Executing migration" id="Add column is_default" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.573135646Z level=info msg="Executing migration" id="Add column frequency" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.573895302Z level=info msg="Executing migration" id="Add column send_reminder" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.574718283Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.575750919Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.576416326Z level=info msg="Executing migration" id="Update alert table charset" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.577302201Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.578069954Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.578856834Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.579804397Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-09-23T17:45:43.708 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.580719097Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-09-23T17:45:43.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.581457351Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-09-23T17:45:43.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.582318057Z level=info msg="Executing migration" id="Add for to alert table" 2023-09-23T17:45:43.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.583185629Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-09-23T17:45:43.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.584024599Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-09-23T17:45:43.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.584845705Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-09-23T17:45:43.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.585816059Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-09-23T17:45:43.709 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.586493416Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.58736005Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.588158593Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.588615883Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.589071504Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.589892679Z level=info msg="Executing migration" id="create annotation table v5" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.590384851Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.590810955Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.591636566Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.592465358Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.593449456Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-09-23T17:45:43.710 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.594293124Z level=info msg="Executing migration" id="Update annotation table charset" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.595058579Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.597799059Z level=info msg="Executing migration" id="Drop category_id index" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.600610498Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.603456104Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.606082619Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.608848286Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.611581547Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.614296116Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.617311581Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.619914928Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.622709982Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.625528392Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-09-23T17:45:43.711 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.628452679Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.631412979Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.634028527Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.636973504Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.639667496Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.642328257Z level=info msg="Executing migration" id="Add epoch_end column" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.645306023Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.647885997Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.650618747Z level=info msg="Executing migration" id="Move region to single row" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.653175578Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.655881235Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.658494601Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.661215562Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-09-23T17:45:43.712 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.663829635Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.666436149Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.6692229Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.671956307Z level=info msg="Executing migration" id="create test_data table" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.674667152Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.677518116Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.680306241Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.682782736Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.685804211Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.688229348Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.691021596Z level=info msg="Executing migration" id="create team table" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.691744437Z level=info msg="Executing migration" id="add index team.org_id" 2023-09-23T17:45:43.713 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.692692314Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.693365133Z level=info msg="Executing migration" id="create team member table" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.694205516Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.695049426Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.695825651Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.696637048Z level=info msg="Executing migration" id="Add column email to team table" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.697511682Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.698338826Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.699163088Z level=info msg="Executing migration" id="create dashboard acl table" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.700127285Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.70080481Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-09-23T17:45:43.714 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.70165732Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-09-23T17:45:43.715 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.702456516Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-09-23T17:45:43.715 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.703340804Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-09-23T17:45:43.715 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.704126219Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-09-23T17:45:43.809 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:45:43.814 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:45:43.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.70495355Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-09-23T17:45:43.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.705397478Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-09-23T17:45:43.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.70624661Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-09-23T17:45:43.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.707097683Z level=info msg="Executing migration" id="create tag table" 2023-09-23T17:45:43.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.70794969Z level=info msg="Executing migration" id="add index tag.key_value" 2023-09-23T17:45:43.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.708373698Z level=info msg="Executing migration" id="create login attempt table" 2023-09-23T17:45:43.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.709281357Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-09-23T17:45:43.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.710087121Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-09-23T17:45:43.955 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.710923072Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.712897437Z level=info msg="Executing migration" id="create login_attempt v2" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.715700416Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.718495746Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.721451582Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.72192165Z level=info msg="Executing migration" id="create user auth table" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.724705918Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.725229353Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.727232924Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.727991674Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.729848165Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.73060769Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-09-23T17:45:43.956 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.733166609Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.733611547Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.734361683Z level=info msg="Executing migration" id="create server_lock table" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.734745297Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.735165843Z level=info msg="Executing migration" id="create user auth token table" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.735578389Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.73598872Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.736396686Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.736816775Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.737551059Z level=info msg="Executing migration" id="create cache_data table" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.737957777Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.738369266Z level=info msg="Executing migration" id="create short_url table v1" 2023-09-23T17:45:43.957 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.738799757Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.741595869Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.741895571Z level=info msg="Executing migration" id="delete alert_definition table" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.742192152Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.74261261Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.745260311Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.745742707Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.746027083Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.746406325Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.746814175Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.747230398Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-09-23T17:45:43.958 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.747636419Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.74839218Z level=info msg="Executing migration" id="drop alert_definition table" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.748799031Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.749094028Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.749461715Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.749893609Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.750315987Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.750590501Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.750997744Z level=info msg="Executing migration" id="create alert_instance table" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.751382013Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.751828522Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.752254671Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-09-23T17:45:43.959 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.753045149Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.753477898Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.753941158Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.757342714Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.76066663Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.761104472Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.761524804Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.762325415Z level=info msg="Executing migration" id="create alert_rule table" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.762749657Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.763177631Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.763608816Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.764018676Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-09-23T17:45:43.960 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.764292886Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.765130466Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.765984685Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.766813711Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.767245681Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.767649406Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.768427239Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.769274256Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.769698011Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.770516924Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.77134302Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.771646516Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-09-23T17:45:43.961 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.772073935Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-09-23T17:45:43.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.772483421Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-09-23T17:45:43.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.772900368Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-09-23T17:45:43.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.773188166Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-09-23T17:45:43.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.775948448Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-09-23T17:45:43.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.77680761Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-09-23T17:45:43.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.779574206Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-09-23T17:45:43.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.782149524Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-09-23T17:45:43.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.784924903Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-09-23T17:45:43.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.78535621Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-09-23T17:45:43.962 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.788318218Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.791094286Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.793775706Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.794725008Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.795582486Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.796430274Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.797279253Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.797728033Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.798587423Z level=info msg="Executing migration" id="create provenance_type table" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.799420595Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.800225749Z level=info msg="Executing migration" id="create alert_image table" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.800648731Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-09-23T17:45:43.963 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.801410915Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.802387682Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.802809349Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.803212215Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.803286259Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.804017234Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.804386416Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.804854624Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.804972167Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.805081425Z level=info msg="alerts found to migrate" alerts=0 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.805742464Z level=info msg="Executing migration" id="create library_element table v1" 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.806598659Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-09-23T17:45:43.964 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.807263705Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.808080083Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.808886909Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.809358241Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.810170213Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.81059248Z level=info msg="Executing migration" id="create data_keys table" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.811101347Z level=info msg="Executing migration" id="create secrets table" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.811957241Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.816473266Z level=info msg="Executing migration" id="add name column into data_keys" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.817384365Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.818366979Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.822735195Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-09-23T17:45:43.965 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.826916548Z level=info msg="Executing migration" id="create kv_store table v1" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.827799756Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.828766458Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.829511296Z level=info msg="Executing migration" id="create permission table" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.829938116Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.83079247Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.831673513Z level=info msg="Executing migration" id="create role table" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.834611445Z level=info msg="Executing migration" id="add column display_name" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.83551971Z level=info msg="Executing migration" id="add column group_name" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.838355168Z level=info msg="Executing migration" id="add index role.org_id" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.841194314Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-09-23T17:45:43.966 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.844004082Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.844602285Z level=info msg="Executing migration" id="create team role table" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.847481898Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.850196939Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.852959494Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.853412757Z level=info msg="Executing migration" id="create user role table" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.856407079Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.859056341Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.861739575Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.862261532Z level=info msg="Executing migration" id="create builtin role table" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.865150583Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.867710066Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-09-23T17:45:43.967 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.870597991Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.871680325Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.872172318Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.874745573Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.87522463Z level=info msg="Executing migration" id="add unique index role.uid" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.875645564Z level=info msg="Executing migration" id="create seed assignment table" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.876053462Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.876522374Z level=info msg="Executing migration" id="add column hidden to role table" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.877500347Z level=info msg="Executing migration" id="create query_history table v1" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.878356275Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.878840148Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.879781094Z level=info msg="Executing migration" id="teams permissions migration" 2023-09-23T17:45:43.968 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.880529822Z level=info msg="Executing migration" id="dashboard permissions" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.88150247Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.884380096Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.887099928Z level=info msg="Executing migration" id="alerting notification permissions" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.88988088Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.892600687Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.893065001Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.895856245Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.898611651Z level=info msg="Executing migration" id="create correlation table v1" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.901303799Z level=info msg="Executing migration" id="add index correlations.uid" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.90176997Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.904740107Z level=info msg="Executing migration" id="add correlation config column" 2023-09-23T17:45:43.969 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.907744116Z level=info msg="Executing migration" id="create entity_events table" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.910328325Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.910821762Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.910930493Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.913590573Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.913697773Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.916425887Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.919131124Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.919640753Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.922326446Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.925189791Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.927724424Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T17:45:43.970 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.928195495Z level=info msg="Executing migration" id="Drop public config table" 2023-09-23T17:45:43.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.931021621Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-09-23T17:45:43.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.931563457Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-09-23T17:45:43.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.93374827Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-09-23T17:45:43.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.93665056Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-09-23T17:45:43.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.937126626Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-09-23T17:45:43.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.940192852Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-09-23T17:45:43.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.942973609Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-09-23T17:45:43.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.945725387Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-09-23T17:45:43.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.946079093Z level=info msg="Executing migration" id="add share column" 2023-09-23T17:45:43.971 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.948933501Z level=info msg="Executing migration" id="create default alerting folders" 2023-09-23T17:45:43.972 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.951608114Z level=info msg="Executing migration" id="create file table" 2023-09-23T17:45:43.972 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.95427551Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-09-23T17:45:43.972 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.95482867Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-09-23T17:45:44.137 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":20,"stamp":"2023-09-23T17:45:42.310660+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":50208,"kb_used_data":6560,"kb_used_omap":0,"kb_used_meta":43584,"kb_avail":749911008,"statfs":{"total":767960285184,"available":767908872192,"internally_reserved":0,"allocated":6717440,"data_stored":3837639,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44630016},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002289"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389987+0000","last_change":"2023-09-23T17:45:12.029937+0000","last_active":"2023-09-23T17:45:13.389987+0000","last_peered":"2023-09-23T17:45:13.389987+0000","last_clean":"2023-09-23T17:45:13.389987+0000","last_became_active":"2023-09-23T17:45:12.029619+0000","last_became_peered":"2023-09-23T17:45:12.029619+0000","last_unstale":"2023-09-23T17:45:13.389987+0000","last_undegraded":"2023-09-23T17:45:13.389987+0000","last_fullsized":"2023-09-23T17:45:13.389987+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:49.407425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389529+0000","last_change":"2023-09-23T17:45:12.288804+0000","last_active":"2023-09-23T17:45:13.389529+0000","last_peered":"2023-09-23T17:45:13.389529+0000","last_clean":"2023-09-23T17:45:13.389529+0000","last_became_active":"2023-09-23T17:45:12.288496+0000","last_became_peered":"2023-09-23T17:45:12.288496+0000","last_unstale":"2023-09-23T17:45:13.389529+0000","last_undegraded":"2023-09-23T17:45:13.389529+0000","last_fullsized":"2023-09-23T17:45:13.389529+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:50.938577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389711+0000","last_change":"2023-09-23T17:45:12.291380+0000","last_active":"2023-09-23T17:45:13.389711+0000","last_peered":"2023-09-23T17:45:13.389711+0000","last_clean":"2023-09-23T17:45:13.389711+0000","last_became_active":"2023-09-23T17:45:12.290977+0000","last_became_peered":"2023-09-23T17:45:12.290977+0000","last_unstale":"2023-09-23T17:45:13.389711+0000","last_undegraded":"2023-09-23T17:45:13.389711+0000","last_fullsized":"2023-09-23T17:45:13.389711+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:31.931088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389694+0000","last_change":"2023-09-23T17:45:12.030797+0000","last_active":"2023-09-23T17:45:13.389694+0000","last_peered":"2023-09-23T17:45:13.389694+0000","last_clean":"2023-09-23T17:45:13.389694+0000","last_became_active":"2023-09-23T17:45:12.030530+0000","last_became_peered":"2023-09-23T17:45:12.030530+0000","last_unstale":"2023-09-23T17:45:13.389694+0000","last_undegraded":"2023-09-23T17:45:13.389694+0000","last_fullsized":"2023-09-23T17:45:13.389694+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:58:27.428598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":271,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389660+0000","last_change":"2023-09-23T17:44:46.990708+0000","last_active":"2023-09-23T17:45:13.389660+0000","last_peered":"2023-09-23T17:45:13.389660+0000","last_clean":"2023-09-23T17:45:13.389660+0000","last_became_active":"2023-09-23T17:44:46.990139+0000","last_became_peered":"2023-09-23T17:44:46.990139+0000","last_unstale":"2023-09-23T17:45:13.389660+0000","last_undegraded":"2023-09-23T17:45:13.389660+0000","last_fullsized":"2023-09-23T17:45:13.389660+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:51.013639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.288289+0000","last_change":"2023-09-23T17:45:12.031891+0000","last_active":"2023-09-23T17:45:13.288289+0000","last_peered":"2023-09-23T17:45:13.288289+0000","last_clean":"2023-09-23T17:45:13.288289+0000","last_became_active":"2023-09-23T17:45:12.031569+0000","last_became_peered":"2023-09-23T17:45:12.031569+0000","last_unstale":"2023-09-23T17:45:13.288289+0000","last_undegraded":"2023-09-23T17:45:13.288289+0000","last_fullsized":"2023-09-23T17:45:13.288289+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:08.345244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.294085+0000","last_change":"2023-09-23T17:45:12.621717+0000","last_active":"2023-09-23T17:45:13.294085+0000","last_peered":"2023-09-23T17:45:13.294085+0000","last_clean":"2023-09-23T17:45:13.294085+0000","last_became_active":"2023-09-23T17:45:12.621356+0000","last_became_peered":"2023-09-23T17:45:12.621356+0000","last_unstale":"2023-09-23T17:45:13.294085+0000","last_undegraded":"2023-09-23T17:45:13.294085+0000","last_fullsized":"2023-09-23T17:45:13.294085+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:35.490791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.288287+0000","last_change":"2023-09-23T17:45:12.031241+0000","last_active":"2023-09-23T17:45:13.288287+0000","last_peered":"2023-09-23T17:45:13.288287+0000","last_clean":"2023-09-23T17:45:13.288287+0000","last_became_active":"2023-09-23T17:45:12.030905+0000","last_became_peered":"2023-09-23T17:45:12.030905+0000","last_unstale":"2023-09-23T17:45:13.288287+0000","last_undegraded":"2023-09-23T17:45:13.288287+0000","last_fullsized":"2023-09-23T17:45:13.288287+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:45.853453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":12,"reported_epoch":61,"state":"active+clean","last_fresh":"2023-09-23T17:45:13.389508+0000","last_change":"2023-09-23T17:45:12.030639+0000","last_active":"2023-09-23T17:45:13.389508+0000","last_peered":"2023-09-23T17:45:13.389508+0000","last_clean":"2023-09-23T17:45:13.389508+0000","last_became_active":"2023-09-23T17:45:12.030334+0000","last_became_peered":"2023-09-23T17:45:12.030334+0000","last_unstale":"2023-09-23T17:45:13.389508+0000","last_undegraded":"2023-09-23T17:45:13.389508+0000","last_fullsized":"2023-09-23T17:45:13.389508+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:35.756901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266702,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6308,"kb_used_data":1180,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738844,"statfs":{"total":95995035648,"available":95988576256,"internally_reserved":0,"allocated":1208320,"data_stored":848483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51700000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49399999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47199999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39500000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.318}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40699999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46100000000000002}]}]},{"osd":6,"up_from":45,"seq":193273528338,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6372,"kb_used_data":1180,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738780,"statfs":{"total":95995035648,"available":95988510720,"internally_reserved":0,"allocated":1208320,"data_stored":848483,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.56999999999999995}]},{"osd":1,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.443},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":0.55400000000000005}]},{"osd":3,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.53800000000000003}]},{"osd":4,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.47999999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52900000000000003}]}]},{"osd":1,"up_from":13,"seq":55834574885,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6500,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738652,"statfs":{"total":95995035648,"available":95988379648,"internally_reserved":0,"allocated":618496,"data_stored":258438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:44:45 2023","interfaces":[{"interface":"back","average":{"1min":0.432,"5min":0.372,"15min":0.36199999999999999},"min":{"1min":0.26600000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.377,"15min":0.36299999999999999},"min":{"1min":0.29999999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.66500000000000004,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.46000000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.43099999999999999,"15min":0.42099999999999999},"min":{"1min":0.32300000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.503,"5min":0.442,"15min":0.432},"min":{"1min":0.313,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.51800000000000002,"15min":0.52200000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.69899999999999995,"5min":0.871,"15min":0.871},"last":0.499},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.496,"15min":0.499},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.64600000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.44}]},{"osd":4,"last update":"Sat Sep 23 17:44:45 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.99099999999999999,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.67300000000000004,"5min":0.67300000000000004,"15min":0.67300000000000004},"min":{"1min":0.48299999999999998,"5min":0.48299999999999998,"15min":0.48299999999999998},"max":{"1min":0.95799999999999996,"5min":0.95799999999999996,"15min":0.95799999999999996},"last":0.57499999999999996}]},{"osd":5,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.42699999999999999,"5min":0.42699999999999999,"15min":0.42699999999999999},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.51500000000000001},{"interface":"front","average":{"1min":0.61599999999999999,"5min":0.61599999999999999,"15min":0.61599999999999999},"min":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"max":{"1min":1.018,"5min":1.018,"15min":1.018},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.52700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55100000000000005}]}]},{"osd":0,"up_from":8,"seq":34359738409,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7144,"kb_used_data":1184,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738008,"statfs":{"total":95995035648,"available":95987720192,"internally_reserved":0,"allocated":1212416,"data_stored":848806,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:44:53 2023","interfaces":[{"interface":"back","average":{"1min":0.42199999999999999,"5min":0.38600000000000001,"15min":0.38},"min":{"1min":0.27600000000000002,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"min":{"1min":0.26300000000000001,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.74099999999999999,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.30099999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:45:09 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.42999999999999999,"15min":0.433},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.505},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.441,"15min":0.439},"min":{"1min":0.33600000000000002,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":0.59499999999999997,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.40899999999999997}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.49399999999999999,"15min":0.504},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.60299999999999998,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.35299999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51900000000000002,"15min":0.52600000000000002},"min":{"1min":0.37,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.64700000000000002,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.44800000000000001}]},{"osd":4,"last update":"Sat Sep 23 17:44:45 2023","interfaces":[{"interface":"back","average":{"1min":0.64400000000000002,"5min":0.64400000000000002,"15min":0.64400000000000002},"min":{"1min":0.502,"5min":0.502,"15min":0.502},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.49399999999999999,"5min":0.49399999999999999,"15min":0.49399999999999999},"max":{"1min":0.78800000000000003,"5min":0.78800000000000003,"15min":0.78800000000000003},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:45:03 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.64000000000000001,"5min":0.64000000000000001,"15min":0.64000000000000001},"min":{"1min":0.496,"5min":0.496,"15min":0.496},"max":{"1min":0.79600000000000004,"5min":0.79600000000000004,"15min":0.79600000000000004},"last":0.60599999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.56299999999999994}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.59199999999999997}]}]},{"osd":2,"up_from":18,"seq":77309411361,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6436,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738716,"statfs":{"total":95995035648,"available":95988445184,"internally_reserved":0,"allocated":618496,"data_stored":258438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:07 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.39500000000000002,"15min":0.39600000000000002},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.46800000000000003,"5min":0.432,"15min":0.42599999999999999},"min":{"1min":0.376,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.622,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.497}]},{"osd":1,"last update":"Sat Sep 23 17:45:07 2023","interfaces":[{"interface":"back","average":{"1min":0.44700000000000001,"5min":0.42999999999999999,"15min":0.42699999999999999},"min":{"1min":0.33300000000000002,"5min":0.218,"15min":0.218},"max":{"1min":0.58699999999999997,"5min":0.60899999999999999,"15min":0.60899999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46000000000000002,"15min":0.45900000000000002},"min":{"1min":0.29099999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"last":0.53500000000000003}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.53600000000000003,"15min":0.54500000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.52200000000000002,"15min":0.53300000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.72899999999999998,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.57999999999999996}]},{"osd":4,"last update":"Sat Sep 23 17:44:47 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.60899999999999999,"15min":0.60899999999999999},"min":{"1min":0.46000000000000002,"5min":0.46000000000000002,"15min":0.46000000000000002},"max":{"1min":0.94199999999999995,"5min":0.94199999999999995,"15min":0.94199999999999995},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"max":{"1min":0.97699999999999998,"5min":0.97699999999999998,"15min":0.97699999999999998},"last":0.623}]},{"osd":5,"last update":"Sat Sep 23 17:45:07 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.91900000000000004,"5min":0.91900000000000004,"15min":0.91900000000000004},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"last":0.69799999999999995}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.71099999999999997},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.72199999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.68700000000000006}]}]},{"osd":3,"up_from":25,"seq":107374182429,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5796,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739356,"statfs":{"total":95995035648,"available":95989100544,"internally_reserved":0,"allocated":618496,"data_stored":258438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.58499999999999996,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.42199999999999999,"15min":0.42399999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.27600000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.43099999999999999,"15min":0.435},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.24099999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.45200000000000001,"15min":0.45400000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.48299999999999998,"15min":0.48799999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.441,"5min":0.47299999999999998,"15min":0.47899999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.64000000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:44:45 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.78900000000000003,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.372},{"interface":"front","average":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":5.2069999999999999,"5min":5.2069999999999999,"15min":5.2069999999999999},"last":0.47599999999999998}]},{"osd":5,"last update":"Sat Sep 23 17:45:04 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.85899999999999999,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":0.873,"5min":0.873,"15min":0.873},"last":0.41099999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.45300000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46700000000000003}]}]},{"osd":4,"up_from":30,"seq":128849018903,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5860,"kb_used_data":604,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739292,"statfs":{"total":95995035648,"available":95989035008,"internally_reserved":0,"allocated":618496,"data_stored":258438,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:44:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.495},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.41199999999999998}]},{"osd":1,"last update":"Sat Sep 23 17:44:44 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81000000000000005,"5min":0.81000000000000005,"15min":0.81000000000000005},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:44:44 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":1.5129999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":0.57999999999999996,"5min":0.57999999999999996,"15min":0.57999999999999996},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":1.073,"5min":1.073,"15min":1.073},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:44:44 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.097,"5min":1.097,"15min":1.097},"last":0.39900000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.83099999999999996,"5min":0.83099999999999996,"15min":0.83099999999999996},"last":0.45900000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:45:03 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.60499999999999998,"5min":0.60499999999999998,"15min":0.60499999999999998},"last":0.38},{"interface":"front","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"last":0.55600000000000005}]},{"osd":6,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.52800000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50600000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757269,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5792,"kb_used_data":600,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739360,"statfs":{"total":95995035648,"available":95989104640,"internally_reserved":0,"allocated":614400,"data_stored":258115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":1.5960000000000001,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":0.497},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.77400000000000002,"5min":0.77400000000000002,"15min":0.77400000000000002},"last":0.45300000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":1.5680000000000001,"5min":1.5680000000000001,"15min":1.5680000000000001},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.75600000000000001,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.42399999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":1.581,"5min":1.581,"15min":1.581},"last":0.54100000000000004},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":1.6060000000000001,"5min":1.6060000000000001,"15min":1.6060000000000001},"last":0.34699999999999998}]},{"osd":3,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":1.5569999999999999,"5min":1.5569999999999999,"15min":1.5569999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":1.6279999999999999,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.371}]},{"osd":4,"last update":"Sat Sep 23 17:45:05 2023","interfaces":[{"interface":"back","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.738,"5min":1.738,"15min":1.738},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.778,"5min":1.778,"15min":1.778},"last":0.47799999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":0.51400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41399999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:45:44.139 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T17:45:44.139 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T17:45:44.139 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T17:45:44.139 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T17:45:44.294 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:44 smithi055 ceph-mon[95848]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:44.294 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:44 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.294 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:44 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.294 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:44 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.295 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:44 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.295 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:44 smithi055 ceph-mon[109912]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:44.295 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:44 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.295 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:44 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.295 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:44 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.295 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:44 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-mon[100270]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:44.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:44.322 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.957518364Z level=info msg="Executing migration" id="create file_meta table" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.958360431Z level=info msg="Executing migration" id="file table idx: path key" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.959105587Z level=info msg="Executing migration" id="set path collation in file table" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.959427943Z level=info msg="Executing migration" id="managed permissions migration" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.960227891Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.961051168Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.961848198Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.962890244Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.963702775Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.96452509Z level=info msg="Executing migration" id="update group index for alert rules" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.965320201Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-09-23T17:45:44.323 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.965643694Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.966047216Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.967083654Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.968144878Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.969024029Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.978808524Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.979682607Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.980530221Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.983848096Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.984649371Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.985534466Z level=info msg="Executing migration" id="create folder table" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.98641195Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.987255332Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-09-23T17:45:44.324 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.987736539Z level=info msg="Executing migration" id="Update folder title length" 2023-09-23T17:45:44.325 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.988604616Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-09-23T17:45:44.325 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=migrator t=2023-09-23T17:45:43.989426296Z level=info msg="migrations completed" performed=485 skipped=0 duration=687.239697ms 2023-09-23T17:45:44.325 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:43 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=sqlstore t=2023-09-23T17:45:43.989855175Z level=info msg="Created default organization" 2023-09-23T17:45:44.325 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=plugin.loader t=2023-09-23T17:45:44.048879178Z level=info msg="Plugin registered" pluginID=input 2023-09-23T17:45:44.325 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=plugin.loader t=2023-09-23T17:45:44.07566138Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T17:45:44.325 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=plugin.loader t=2023-09-23T17:45:44.075685323Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T17:45:44.325 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=secrets t=2023-09-23T17:45:44.075749446Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T17:45:44.325 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=query_data t=2023-09-23T17:45:44.077429854Z level=info msg="Query Service initialization" 2023-09-23T17:45:44.325 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=live.push_http t=2023-09-23T17:45:44.080032857Z level=info msg="Live Push Gateway initialization" 2023-09-23T17:45:44.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:45:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:45:44.679 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:44 smithi055 bash[150740]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T17:45:44.763 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:45.050 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=infra.usagestats.collector t=2023-09-23T17:45:44.785351239Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T17:45:45.050 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=provisioning.datasources t=2023-09-23T17:45:44.785817415Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-09-23T17:45:45.050 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=provisioning.datasources t=2023-09-23T17:45:44.819617221Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T17:45:45.050 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=provisioning.datasources t=2023-09-23T17:45:44.851376845Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-09-23T17:45:45.050 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=provisioning.alerting t=2023-09-23T17:45:44.872640572Z level=info msg="starting to provision alerting" 2023-09-23T17:45:45.050 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=provisioning.alerting t=2023-09-23T17:45:44.872664808Z level=info msg="finished to provision alerting" 2023-09-23T17:45:45.050 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=grafanaStorageLogger t=2023-09-23T17:45:44.872835626Z level=info msg="storage starting" 2023-09-23T17:45:45.050 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=ngalert.state.manager t=2023-09-23T17:45:44.872861338Z level=info msg="Warming state cache for startup" 2023-09-23T17:45:45.050 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=http.server t=2023-09-23T17:45:44.876022119Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T17:45:45.050 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=sqlstore.transactions t=2023-09-23T17:45:44.883151864Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T17:45:45.051 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=ngalert.state.manager t=2023-09-23T17:45:44.883217629Z level=info msg="State cache has been initialized" states=0 duration=10.348904ms 2023-09-23T17:45:45.051 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=ngalert.multiorg.alertmanager t=2023-09-23T17:45:44.883302947Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T17:45:45.051 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=ticker t=2023-09-23T17:45:44.883334105Z level=info msg=starting first_tick=2023-09-23T17:45:50Z 2023-09-23T17:45:45.051 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=sqlstore.transactions t=2023-09-23T17:45:44.929368935Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T17:45:45.051 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:45:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[135773]: logger=sqlstore.transactions t=2023-09-23T17:45:44.939918727Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=1 code="database is locked" 2023-09-23T17:45:45.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:45 smithi055 ceph-mon[109912]: Deploying daemon node-exporter.a on smithi055 2023-09-23T17:45:45.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:45 smithi055 ceph-mon[109912]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:45:45.245 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:45 smithi055 bash[150740]: Getting image source signatures 2023-09-23T17:45:45.245 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:45 smithi055 bash[150740]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T17:45:45.245 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:45 smithi055 bash[150740]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T17:45:45.245 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:45 smithi055 bash[150740]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T17:45:45.245 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:45 smithi055 bash[150740]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T17:45:45.246 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:45 smithi055 bash[150740]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T17:45:45.246 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:45 smithi055 bash[150740]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T17:45:45.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:45 smithi055 ceph-mon[95848]: Deploying daemon node-exporter.a on smithi055 2023-09-23T17:45:45.246 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:45 smithi055 ceph-mon[95848]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:45:45.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:45 smithi099 ceph-mon[100270]: Deploying daemon node-exporter.a on smithi055 2023-09-23T17:45:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:45 smithi099 ceph-mon[100270]: from='client.24512 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:45:46.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:46 smithi099 ceph-mon[100270]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:46.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:46 smithi055 ceph-mon[109912]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:46 smithi055 ceph-mon[95848]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:46.595 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:45:46.595 INFO:teuthology.orchestra.run.smithi055.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T17:45:46.828 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:46 smithi055 bash[150740]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T17:45:46.829 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:46 smithi055 bash[150740]: Writing manifest to image destination 2023-09-23T17:45:46.829 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:46 smithi055 bash[150740]: Storing signatures 2023-09-23T17:45:47.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3227248539' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T17:45:47.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3227248539' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T17:45:47.230 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T17:45:47.230 INFO:tasks.cephadm:Setup complete, yielding 2023-09-23T17:45:47.230 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-09-23T17:45:47.240 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi055.front.sepia.ceph.com 2023-09-23T17:45:47.241 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- bash -c 'ceph fs volume create foo' 2023-09-23T17:45:47.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:47 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3227248539' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T17:45:47.553 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 podman[150740]: 2023-09-23T17:45:47.553 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.458Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T17:45:47.554 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.458Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T17:45:47.554 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.458Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T17:45:47.554 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.458Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T17:45:47.554 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T17:45:47.554 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T17:45:47.554 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T17:45:47.554 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T17:45:47.554 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T17:45:47.554 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T17:45:47.554 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T17:45:47.555 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.459Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T17:45:47.556 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T17:45:47.557 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T17:45:47.558 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T17:45:47.558 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T17:45:47.558 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T17:45:47.558 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T17:45:47.558 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T17:45:47.558 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T17:45:47.558 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[151014]: ts=2023-09-23T17:45:47.460Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T17:45:47.558 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 bash[150740]: d17a99f5c12ed71daf315cc9b4b3d6930238522e34525a2cec4593c8d6bed74f 2023-09-23T17:45:47.907 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:47.929 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:45:47 smithi055 systemd[1]: Started Ceph node-exporter.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:45:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:48 smithi099 ceph-mon[100270]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:48 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:48 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:48 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:48 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:48 smithi055 ceph-mon[95848]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:48.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:48 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:48 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:48 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.387 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:48 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:48 smithi055 ceph-mon[109912]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:48.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:48 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:48 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:48 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.387 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:48 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:48.826 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:48 smithi099 systemd[1]: Starting Ceph node-exporter.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:45:48.827 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:48 smithi099 bash[136027]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-09-23T17:45:49.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:49 smithi099 ceph-mon[100270]: Deploying daemon node-exporter.b on smithi099 2023-09-23T17:45:49.327 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:49 smithi099 bash[136027]: Getting image source signatures 2023-09-23T17:45:49.327 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:49 smithi099 bash[136027]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T17:45:49.327 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:49 smithi099 bash[136027]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-09-23T17:45:49.327 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:49 smithi099 bash[136027]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T17:45:49.327 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:49 smithi099 bash[136027]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-09-23T17:45:49.327 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:49 smithi099 bash[136027]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T17:45:49.327 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:49 smithi099 bash[136027]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-09-23T17:45:49.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:49 smithi055 ceph-mon[95848]: Deploying daemon node-exporter.b on smithi099 2023-09-23T17:45:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:49 smithi055 ceph-mon[109912]: Deploying daemon node-exporter.b on smithi099 2023-09-23T17:45:50.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:50 smithi055 ceph-mon[95848]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:50.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:50 smithi055 ceph-mon[95848]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:45:50.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:50 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T17:45:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:50 smithi055 ceph-mon[109912]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:50 smithi055 ceph-mon[109912]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:45:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:50 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T17:45:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:50 smithi099 ceph-mon[100270]: pgmap v23: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:50 smithi099 ceph-mon[100270]: from='client.14640 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:45:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:50 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-09-23T17:45:51.076 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:50 smithi099 bash[136027]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-09-23T17:45:51.076 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:50 smithi099 bash[136027]: Writing manifest to image destination 2023-09-23T17:45:51.077 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:50 smithi099 bash[136027]: Storing signatures 2023-09-23T17:45:51.386 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T17:45:51.386 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-mon[100270]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T17:45:51.386 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T17:45:51.387 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 podman[136027]: 2023-09-23T17:45:51.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:51 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T17:45:51.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:51 smithi055 ceph-mon[95848]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T17:45:51.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:51 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T17:45:51.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:51 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[95824]: 2023-09-23T17:45:51.127+0000 7f175eeea700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T17:45:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:51 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-09-23T17:45:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:51 smithi055 ceph-mon[109912]: osdmap e63: 8 total, 8 up, 8 in 2023-09-23T17:45:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:51 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-09-23T17:45:51.461 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-09-23T17:45:51.827 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.512Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T17:45:51.827 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.512Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T17:45:51.827 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T17:45:51.827 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T17:45:51.827 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T17:45:51.827 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T17:45:51.827 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T17:45:51.827 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T17:45:51.828 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T17:45:51.829 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T17:45:51.830 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[136076]: ts=2023-09-23T17:45:51.513Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 bash[136027]: 1dcf070a4b2567d49d1c5cfd539ed5a522dc518acc92a709b1f787d3683e45aa 2023-09-23T17:45:51.831 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:45:51 smithi099 systemd[1]: Started Ceph node-exporter.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:45:52.079 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: pgmap v25: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T17:45:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T17:45:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T17:45:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: fsmap foo:0 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: Saving service mds.foo spec with placement count:2 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: pgmap v25: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: fsmap foo:0 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: Saving service mds.foo spec with placement count:2 2023-09-23T17:45:52.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:52 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:45:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: pgmap v25: 41 pgs: 32 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-09-23T17:45:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: osdmap e64: 8 total, 8 up, 8 in 2023-09-23T17:45:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-09-23T17:45:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-09-23T17:45:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-09-23T17:45:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: osdmap e65: 8 total, 8 up, 8 in 2023-09-23T17:45:52.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-09-23T17:45:52.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: fsmap foo:0 2023-09-23T17:45:52.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: Saving service mds.foo spec with placement count:2 2023-09-23T17:45:52.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:52.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:52 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:45:53.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:53 smithi055 ceph-mon[95848]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T17:45:53.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:53 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:53.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:53 smithi055 ceph-mon[109912]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T17:45:53.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:53 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:53.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:53 smithi099 ceph-mon[100270]: osdmap e66: 8 total, 8 up, 8 in 2023-09-23T17:45:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:53 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:54.377 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:54 smithi055 ceph-mon[95848]: pgmap v29: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:54.377 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:54 smithi055 ceph-mon[95848]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T17:45:54.377 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:54 smithi055 ceph-mon[109912]: pgmap v29: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:54.377 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:54 smithi055 ceph-mon[109912]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T17:45:54.385 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-09-23T17:45:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:54 smithi099 ceph-mon[100270]: pgmap v29: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:54 smithi099 ceph-mon[100270]: osdmap e67: 8 total, 8 up, 8 in 2023-09-23T17:45:54.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:45:54 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:45:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:45:55.000 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:56.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:56 smithi055 ceph-mon[109912]: pgmap v31: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:56.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:56 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:56.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:56 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:56 smithi055 ceph-mon[95848]: pgmap v31: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:56 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:56 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:56.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:56 smithi099 ceph-mon[100270]: pgmap v31: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:56.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:56 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:56.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:56 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:56.955 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T17:45:56.966 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T17:45:56.966 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T17:45:56.966 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T17:45:57.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:57 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:57.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:57 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:57.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:57 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:57.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:57 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:45:57.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:57 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:57.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:57 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.exjftp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:45:57.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:57 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.exjftp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T17:45:57.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:57 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:57.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:57.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:57.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:57.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:45:57.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:57.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.exjftp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:45:57.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.exjftp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T17:45:57.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:57.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:57.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:57.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:57.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:45:57.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:57.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.exjftp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:45:57.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.exjftp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T17:45:57.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:57 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:57.969 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:45:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:58 smithi099 ceph-mon[100270]: Deploying daemon mds.foo.smithi055.exjftp on smithi055 2023-09-23T17:45:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:58 smithi099 ceph-mon[100270]: pgmap v32: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:58 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:58 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:58.608 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:58 smithi055 ceph-mon[95848]: Deploying daemon mds.foo.smithi055.exjftp on smithi055 2023-09-23T17:45:58.609 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:58 smithi055 ceph-mon[95848]: pgmap v32: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:58.609 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:58 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:58.609 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:58 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:58.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:58 smithi055 ceph-mon[109912]: Deploying daemon mds.foo.smithi055.exjftp on smithi055 2023-09-23T17:45:58.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:58 smithi055 ceph-mon[109912]: pgmap v32: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:58.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:58 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:58.609 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:58 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:45:59.560 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:45:59.928 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: pgmap v33: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:59.928 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:59.928 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:59.928 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:59.928 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.djovrl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.djovrl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: Deploying daemon mds.foo.smithi099.djovrl on smithi099 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: mds.? [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] up:boot 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: daemon mds.foo.smithi055.exjftp assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: Cluster is now healthy 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: fsmap foo:0 1 up:standby 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:45:59.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: fsmap foo:1 {0=foo.smithi055.exjftp=up:creating} 2023-09-23T17:45:59.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: daemon mds.foo.smithi055.exjftp is now active in filesystem foo as rank 0 2023-09-23T17:45:59.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3053090435' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T17:45:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: pgmap v33: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:45:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:45:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.djovrl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:45:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.djovrl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T17:45:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: Deploying daemon mds.foo.smithi099.djovrl on smithi099 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: mds.? [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] up:boot 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: daemon mds.foo.smithi055.exjftp assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: Cluster is now healthy 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: fsmap foo:0 1 up:standby 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi055.exjftp=up:creating} 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: daemon mds.foo.smithi055.exjftp is now active in filesystem foo as rank 0 2023-09-23T17:45:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:45:59 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3053090435' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T17:45:59.932 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":20,"active_gid":24338,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":32209804},{"type":"v1","addr":"172.21.15.55:6801","nonce":32209804}]},"active_addr":"172.21.15.55:6801/32209804","active_change":"2023-09-23T17:45:12.277064+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":24373,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":60,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":2880364962}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":880894918}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":573100440}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":2064251389}]}]} 2023-09-23T17:45:59.937 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T17:45:59.937 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T17:45:59.937 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T17:46:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: pgmap v33: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.djovrl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:46:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.djovrl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-09-23T17:46:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: Deploying daemon mds.foo.smithi099.djovrl on smithi099 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: mds.? [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] up:boot 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: daemon mds.foo.smithi055.exjftp assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: Cluster is now healthy 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: fsmap foo:0 1 up:standby 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: fsmap foo:1 {0=foo.smithi055.exjftp=up:creating} 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: daemon mds.foo.smithi055.exjftp is now active in filesystem foo as rank 0 2023-09-23T17:46:00.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:45:59 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3053090435' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T17:46:00.544 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:01 smithi099 ceph-mon[100270]: mds.? [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] up:active 2023-09-23T17:46:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:01 smithi099 ceph-mon[100270]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 2023-09-23T17:46:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:01 smithi099 ceph-mon[100270]: pgmap v34: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:01 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:01 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:01 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:01 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:01 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:01 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:01.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[95848]: mds.? [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] up:active 2023-09-23T17:46:01.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[95848]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 2023-09-23T17:46:01.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[95848]: pgmap v34: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:01.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:01.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[109912]: mds.? [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] up:active 2023-09-23T17:46:01.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 2023-09-23T17:46:01.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[109912]: pgmap v34: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:01.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:01.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:01 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:01.823 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:46:01.824 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":67,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","created":"2023-09-23T17:40:13.381005+0000","modified":"2023-09-23T17:45:53.148798+0000","last_up_change":"2023-09-23T17:44:42.935311+0000","last_in_change":"2023-09-23T17:44:27.554515+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T17:43:05.158393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T17:45:10.969164+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T17:45:49.622328+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T17:45:50.526600+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"99e7cae6-389e-461a-8a86-a5789673908b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6803","nonce":3018551716}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6805","nonce":3018551716}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6809","nonce":3018551716}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6807","nonce":3018551716}]},"public_addr":"172.21.15.55:6803/3018551716","cluster_addr":"172.21.15.55:6805/3018551716","heartbeat_back_addr":"172.21.15.55:6809/3018551716","heartbeat_front_addr":"172.21.15.55:6807/3018551716","state":["exists","up"]},{"osd":1,"uuid":"bfb0da01-7a06-430b-b973-209f22ac30d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6811","nonce":1171163846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6813","nonce":1171163846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6817","nonce":1171163846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6815","nonce":1171163846}]},"public_addr":"172.21.15.55:6811/1171163846","cluster_addr":"172.21.15.55:6813/1171163846","heartbeat_back_addr":"172.21.15.55:6817/1171163846","heartbeat_front_addr":"172.21.15.55:6815/1171163846","state":["exists","up"]},{"osd":2,"uuid":"f766d604-db54-4a09-9993-fe1a3ce0048e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6819","nonce":3805819981}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6821","nonce":3805819981}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6825","nonce":3805819981}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6823","nonce":3805819981}]},"public_addr":"172.21.15.55:6819/3805819981","cluster_addr":"172.21.15.55:6821/3805819981","heartbeat_back_addr":"172.21.15.55:6825/3805819981","heartbeat_front_addr":"172.21.15.55:6823/3805819981","state":["exists","up"]},{"osd":3,"uuid":"9f7bc62d-c839-4d05-89c7-98e5abd0a121","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6827","nonce":2133183145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6829","nonce":2133183145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6833","nonce":2133183145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6831","nonce":2133183145}]},"public_addr":"172.21.15.55:6827/2133183145","cluster_addr":"172.21.15.55:6829/2133183145","heartbeat_back_addr":"172.21.15.55:6833/2133183145","heartbeat_front_addr":"172.21.15.55:6831/2133183145","state":["exists","up"]},{"osd":4,"uuid":"a9c0950d-2618-4a47-ad13-035e2bd227f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6801","nonce":2570729307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6803","nonce":2570729307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6807","nonce":2570729307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6805","nonce":2570729307}]},"public_addr":"172.21.15.99:6801/2570729307","cluster_addr":"172.21.15.99:6803/2570729307","heartbeat_back_addr":"172.21.15.99:6807/2570729307","heartbeat_front_addr":"172.21.15.99:6805/2570729307","state":["exists","up"]},{"osd":5,"uuid":"64ac7fc4-adc3-4e7f-8512-a458d0092f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6809","nonce":3690413267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6811","nonce":3690413267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6815","nonce":3690413267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6813","nonce":3690413267}]},"public_addr":"172.21.15.99:6809/3690413267","cluster_addr":"172.21.15.99:6811/3690413267","heartbeat_back_addr":"172.21.15.99:6815/3690413267","heartbeat_front_addr":"172.21.15.99:6813/3690413267","state":["exists","up"]},{"osd":6,"uuid":"9be63962-c98b-4df9-8781-270556e51119","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6817","nonce":2102700865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6819","nonce":2102700865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6823","nonce":2102700865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6821","nonce":2102700865}]},"public_addr":"172.21.15.99:6817/2102700865","cluster_addr":"172.21.15.99:6819/2102700865","heartbeat_back_addr":"172.21.15.99:6823/2102700865","heartbeat_front_addr":"172.21.15.99:6821/2102700865","state":["exists","up"]},{"osd":7,"uuid":"ea011d8c-d9f8-4619-9ac0-575813be808a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6825","nonce":453077889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6827","nonce":453077889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6831","nonce":453077889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6829","nonce":453077889}]},"public_addr":"172.21.15.99:6825/453077889","cluster_addr":"172.21.15.99:6827/453077889","heartbeat_back_addr":"172.21.15.99:6831/453077889","heartbeat_front_addr":"172.21.15.99:6829/453077889","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:22.099373+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:42.018683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:01.856304+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:22.152206+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:41.385241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:01.725749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:21.176273+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:41.516585+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/4290529019":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/1663054712":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6801/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6800/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/238250704":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/2173356416":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/658967358":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1826983981":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/3953275940":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/880528732":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6800/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/218576037":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/591596032":"2023-09-24T17:41:12.624150+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T17:46:02.209 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T17:46:02.209 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T17:46:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:02 smithi055 ceph-mon[109912]: mds.? [v2:172.21.15.99:6832/2997750460,v1:172.21.15.99:6833/2997750460] up:boot 2023-09-23T17:46:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:02 smithi055 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:02 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:46:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:02 smithi055 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:02 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/882474878' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:46:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:02 smithi055 ceph-mon[95848]: mds.? [v2:172.21.15.99:6832/2997750460,v1:172.21.15.99:6833/2997750460] up:boot 2023-09-23T17:46:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:02 smithi055 ceph-mon[95848]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:02.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:02 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:46:02.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:02 smithi055 ceph-mon[95848]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:02.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:02 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/882474878' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:46:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:02 smithi099 ceph-mon[100270]: mds.? [v2:172.21.15.99:6832/2997750460,v1:172.21.15.99:6833/2997750460] up:boot 2023-09-23T17:46:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:02 smithi099 ceph-mon[100270]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:02 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:46:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:02 smithi099 ceph-mon[100270]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:02 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/882474878' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:46:02.831 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:03.676 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:03 smithi055 ceph-mon[109912]: pgmap v35: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:03.676 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:03 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:03.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:03 smithi055 ceph-mon[95848]: pgmap v35: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:03.677 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:03 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:03.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:03 smithi099 ceph-mon[100270]: pgmap v35: 73 pgs: 73 active+clean; 577 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:03.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:03 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:46:04.321 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":67,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","created":"2023-09-23T17:40:13.381005+0000","modified":"2023-09-23T17:45:53.148798+0000","last_up_change":"2023-09-23T17:44:42.935311+0000","last_in_change":"2023-09-23T17:44:27.554515+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T17:43:05.158393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T17:45:10.969164+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"62","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T17:45:49.622328+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T17:45:50.526600+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"99e7cae6-389e-461a-8a86-a5789673908b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6803","nonce":3018551716}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6805","nonce":3018551716}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6809","nonce":3018551716}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3018551716},{"type":"v1","addr":"172.21.15.55:6807","nonce":3018551716}]},"public_addr":"172.21.15.55:6803/3018551716","cluster_addr":"172.21.15.55:6805/3018551716","heartbeat_back_addr":"172.21.15.55:6809/3018551716","heartbeat_front_addr":"172.21.15.55:6807/3018551716","state":["exists","up"]},{"osd":1,"uuid":"bfb0da01-7a06-430b-b973-209f22ac30d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6811","nonce":1171163846}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6813","nonce":1171163846}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6817","nonce":1171163846}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":1171163846},{"type":"v1","addr":"172.21.15.55:6815","nonce":1171163846}]},"public_addr":"172.21.15.55:6811/1171163846","cluster_addr":"172.21.15.55:6813/1171163846","heartbeat_back_addr":"172.21.15.55:6817/1171163846","heartbeat_front_addr":"172.21.15.55:6815/1171163846","state":["exists","up"]},{"osd":2,"uuid":"f766d604-db54-4a09-9993-fe1a3ce0048e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6819","nonce":3805819981}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6821","nonce":3805819981}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6825","nonce":3805819981}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":3805819981},{"type":"v1","addr":"172.21.15.55:6823","nonce":3805819981}]},"public_addr":"172.21.15.55:6819/3805819981","cluster_addr":"172.21.15.55:6821/3805819981","heartbeat_back_addr":"172.21.15.55:6825/3805819981","heartbeat_front_addr":"172.21.15.55:6823/3805819981","state":["exists","up"]},{"osd":3,"uuid":"9f7bc62d-c839-4d05-89c7-98e5abd0a121","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6827","nonce":2133183145}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6829","nonce":2133183145}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6833","nonce":2133183145}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":2133183145},{"type":"v1","addr":"172.21.15.55:6831","nonce":2133183145}]},"public_addr":"172.21.15.55:6827/2133183145","cluster_addr":"172.21.15.55:6829/2133183145","heartbeat_back_addr":"172.21.15.55:6833/2133183145","heartbeat_front_addr":"172.21.15.55:6831/2133183145","state":["exists","up"]},{"osd":4,"uuid":"a9c0950d-2618-4a47-ad13-035e2bd227f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6801","nonce":2570729307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6803","nonce":2570729307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6807","nonce":2570729307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6805","nonce":2570729307}]},"public_addr":"172.21.15.99:6801/2570729307","cluster_addr":"172.21.15.99:6803/2570729307","heartbeat_back_addr":"172.21.15.99:6807/2570729307","heartbeat_front_addr":"172.21.15.99:6805/2570729307","state":["exists","up"]},{"osd":5,"uuid":"64ac7fc4-adc3-4e7f-8512-a458d0092f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6809","nonce":3690413267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6811","nonce":3690413267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6815","nonce":3690413267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6813","nonce":3690413267}]},"public_addr":"172.21.15.99:6809/3690413267","cluster_addr":"172.21.15.99:6811/3690413267","heartbeat_back_addr":"172.21.15.99:6815/3690413267","heartbeat_front_addr":"172.21.15.99:6813/3690413267","state":["exists","up"]},{"osd":6,"uuid":"9be63962-c98b-4df9-8781-270556e51119","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":64,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6817","nonce":2102700865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6819","nonce":2102700865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6823","nonce":2102700865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6821","nonce":2102700865}]},"public_addr":"172.21.15.99:6817/2102700865","cluster_addr":"172.21.15.99:6819/2102700865","heartbeat_back_addr":"172.21.15.99:6823/2102700865","heartbeat_front_addr":"172.21.15.99:6821/2102700865","state":["exists","up"]},{"osd":7,"uuid":"ea011d8c-d9f8-4619-9ac0-575813be808a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":63,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6825","nonce":453077889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6827","nonce":453077889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6831","nonce":453077889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6829","nonce":453077889}]},"public_addr":"172.21.15.99:6825/453077889","cluster_addr":"172.21.15.99:6827/453077889","heartbeat_back_addr":"172.21.15.99:6831/453077889","heartbeat_front_addr":"172.21.15.99:6829/453077889","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:22.099373+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:42.018683+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:01.856304+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:22.152206+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:41.385241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:01.725749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:21.176273+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:41.516585+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/4290529019":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/1663054712":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6801/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6800/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/238250704":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/2173356416":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/658967358":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6801/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1826983981":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/3953275940":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/880528732":"2023-09-24T17:40:48.304146+0000","172.21.15.55:6800/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/218576037":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/591596032":"2023-09-24T17:41:12.624150+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T17:46:04.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:04 smithi099 ceph-mon[100270]: mds.? [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] up:active 2023-09-23T17:46:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:04 smithi099 ceph-mon[100270]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:04 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2531457971' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:46:04.578 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:46:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:46:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:46:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:04 smithi055 ceph-mon[95848]: mds.? [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] up:active 2023-09-23T17:46:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:04 smithi055 ceph-mon[95848]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:04 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2531457971' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:46:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:04 smithi055 ceph-mon[109912]: mds.? [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] up:active 2023-09-23T17:46:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:04 smithi055 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:04 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2531457971' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:46:04.695 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T17:46:04.695 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T17:46:04.696 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T17:46:04.696 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T17:46:04.697 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T17:46:04.697 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T17:46:04.697 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T17:46:04.698 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T17:46:05.753 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:05.888 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:05.967 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:06.098 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:06.103 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:06.104 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:06.107 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:06.224 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:06.530 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[109912]: pgmap v36: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2023-09-23T17:46:06.530 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[109912]: mds.? [v2:172.21.15.99:6832/2997750460,v1:172.21.15.99:6833/2997750460] up:standby 2023-09-23T17:46:06.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:06.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:06.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:06.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[95848]: pgmap v36: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2023-09-23T17:46:06.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[95848]: mds.? [v2:172.21.15.99:6832/2997750460,v1:172.21.15.99:6833/2997750460] up:standby 2023-09-23T17:46:06.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[95848]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:06.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:06.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:06.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:06 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:06 smithi099 ceph-mon[100270]: pgmap v36: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.2 KiB/s wr, 3 op/s 2023-09-23T17:46:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:06 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:06 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:06 smithi099 ceph-mon[100270]: mds.? [v2:172.21.15.99:6832/2997750460,v1:172.21.15.99:6833/2997750460] up:standby 2023-09-23T17:46:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:06 smithi099 ceph-mon[100270]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:46:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:06 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:06 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:06.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:06 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:06.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:06 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:06.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:06 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:07 smithi099 ceph-mon[100270]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T17:46:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:07 smithi099 ceph-mon[100270]: Reconfiguring daemon alertmanager.a on smithi055 2023-09-23T17:46:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:07 smithi055 ceph-mon[109912]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T17:46:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:07 smithi055 ceph-mon[109912]: Reconfiguring daemon alertmanager.a on smithi055 2023-09-23T17:46:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:07 smithi055 ceph-mon[95848]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T17:46:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:07 smithi055 ceph-mon[95848]: Reconfiguring daemon alertmanager.a on smithi055 2023-09-23T17:46:08.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:08 smithi055 ceph-mon[95848]: pgmap v37: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:08.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:08 smithi055 ceph-mon[109912]: pgmap v37: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:08.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:08 smithi099 ceph-mon[100270]: pgmap v37: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:09.544 INFO:teuthology.orchestra.run.smithi055.stdout:163208757276 2023-09-23T17:46:09.545 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T17:46:10.388 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:10 smithi055 ceph-mon[109912]: pgmap v38: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:10.388 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:10 smithi055 ceph-mon[95848]: pgmap v38: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:10.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:10 smithi099 ceph-mon[100270]: pgmap v38: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:12.089 INFO:teuthology.orchestra.run.smithi055.stdout:55834574892 2023-09-23T17:46:12.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T17:46:12.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:12 smithi055 ceph-mon[109912]: pgmap v39: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:12 smithi055 ceph-mon[95848]: pgmap v39: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:12.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:12 smithi099 ceph-mon[100270]: pgmap v39: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:12.635 INFO:teuthology.orchestra.run.smithi055.stdout:227633266709 2023-09-23T17:46:12.636 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T17:46:12.725 INFO:teuthology.orchestra.run.smithi055.stdout:193273528345 2023-09-23T17:46:12.725 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T17:46:13.145 INFO:teuthology.orchestra.run.smithi055.stdout:34359738416 2023-09-23T17:46:13.145 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T17:46:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T17:46:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T17:46:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T17:46:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:13 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T17:46:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T17:46:13.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T17:46:13.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:13.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:13 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]: dispatch 2023-09-23T17:46:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]: dispatch 2023-09-23T17:46:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-09-23T17:46:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:13 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:13.623 INFO:teuthology.orchestra.run.smithi055.stdout:77309411368 2023-09-23T17:46:13.624 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T17:46:13.868 INFO:teuthology.orchestra.run.smithi055.stdout:128849018910 2023-09-23T17:46:13.869 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T17:46:13.873 INFO:teuthology.orchestra.run.smithi055.stdout:107374182436 2023-09-23T17:46:13.873 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T17:46:13.921 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:14.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:14 smithi055 ceph-mon[109912]: pgmap v40: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:14.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:14 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T17:46:14.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:14 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T17:46:14.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:14 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T17:46:14.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:14 smithi055 ceph-mon[109912]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T17:46:14.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:14 smithi055 ceph-mon[95848]: pgmap v40: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:14.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:14 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T17:46:14.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:14 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T17:46:14.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:14 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T17:46:14.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:14 smithi055 ceph-mon[95848]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T17:46:14.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:14 smithi099 ceph-mon[100270]: pgmap v40: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-09-23T17:46:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:14 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 2]}]': finished 2023-09-23T17:46:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:14 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.d", "id": [1, 5]}]': finished 2023-09-23T17:46:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:14 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-09-23T17:46:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:14 smithi099 ceph-mon[100270]: osdmap e68: 8 total, 8 up, 8 in 2023-09-23T17:46:14.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:46:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:46:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:46:14.865 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:15.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:15 smithi055 ceph-mon[109912]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T17:46:15.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:15 smithi055 ceph-mon[109912]: pgmap v43: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T17:46:15.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:15 smithi055 ceph-mon[95848]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T17:46:15.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:15 smithi055 ceph-mon[95848]: pgmap v43: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T17:46:15.535 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:15.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:15 smithi099 ceph-mon[100270]: osdmap e69: 8 total, 8 up, 8 in 2023-09-23T17:46:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:15 smithi099 ceph-mon[100270]: pgmap v43: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail; 0 B/s wr, 0 op/s 2023-09-23T17:46:15.786 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:15 smithi055 systemd[1]: Stopping Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:46:15.807 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:15.808 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:15.818 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:16.117 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:16.207 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:16.272 INFO:teuthology.orchestra.run.smithi055.stdout:163208757277 2023-09-23T17:46:16.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:16 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1800669815' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T17:46:16.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:16 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1800669815' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T17:46:16.675 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:16 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[146553]: ts=2023-09-23T17:46:16.361Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T17:46:16.675 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:16 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1800669815' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T17:46:17.308 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:17 smithi055 bash[156533]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T17:46:17.569 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:17 smithi055 ceph-mon[109912]: pgmap v44: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:17.570 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:17 smithi055 ceph-mon[95848]: pgmap v44: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:17.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:17 smithi099 ceph-mon[100270]: pgmap v44: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:17.992 INFO:teuthology.orchestra.run.smithi055.stdout:227633266710 2023-09-23T17:46:18.544 INFO:teuthology.orchestra.run.smithi055.stdout:128849018911 2023-09-23T17:46:18.574 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:18 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/514059527' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T17:46:18.575 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:18 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/514059527' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T17:46:18.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:18 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/514059527' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T17:46:19.343 INFO:tasks.cephadm.ceph_manager.ceph:need seq 227633266709 got 227633266710 for osd.7 2023-09-23T17:46:19.344 DEBUG:teuthology.parallel:result is None 2023-09-23T17:46:19.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:19 smithi055 ceph-mon[95848]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:19.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:19 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2729734843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T17:46:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:19 smithi055 ceph-mon[109912]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:19 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2729734843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T17:46:19.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:19 smithi099 ceph-mon[100270]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:19 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2729734843' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T17:46:19.891 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757276 got 163208757277 for osd.5 2023-09-23T17:46:19.892 DEBUG:teuthology.parallel:result is None 2023-09-23T17:46:20.180 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:19 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T17:46:20.180 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:19 smithi055 systemd[1]: Stopped Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:46:20.180 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:19 smithi055 systemd[1]: Starting Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:46:20.738 INFO:teuthology.orchestra.run.smithi055.stdout:77309411370 2023-09-23T17:46:20.792 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018910 got 128849018911 for osd.4 2023-09-23T17:46:20.792 DEBUG:teuthology.parallel:result is None 2023-09-23T17:46:21.087 INFO:teuthology.orchestra.run.smithi055.stdout:193273528347 2023-09-23T17:46:21.230 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:20 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1627165701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T17:46:21.231 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:20 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1627165701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T17:46:21.290 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411368 got 77309411370 for osd.2 2023-09-23T17:46:21.290 DEBUG:teuthology.parallel:result is None 2023-09-23T17:46:21.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:20 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1627165701' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T17:46:21.809 INFO:teuthology.orchestra.run.smithi055.stdout:34359738418 2023-09-23T17:46:21.836 INFO:teuthology.orchestra.run.smithi055.stdout:107374182438 2023-09-23T17:46:21.840 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:21 smithi055 podman[157370]: 2023-09-23T17:46:22.092 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:21 smithi055 ceph-mon[95848]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:22.093 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:21 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1913044712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T17:46:22.093 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:21 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3544115308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T17:46:22.093 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:21 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3376109882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T17:46:22.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:21 smithi055 ceph-mon[109912]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:22.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:21 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1913044712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T17:46:22.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:21 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3544115308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T17:46:22.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:21 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3376109882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T17:46:22.094 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:46:22.062Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T17:46:22.094 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:46:22.062Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T17:46:22.094 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:46:22.063Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.55 port=9094 2023-09-23T17:46:22.094 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:46:22.064Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T17:46:22.094 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:46:22.093Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T17:46:22.262 INFO:tasks.cephadm.ceph_manager.ceph:need seq 193273528345 got 193273528347 for osd.6 2023-09-23T17:46:22.262 DEBUG:teuthology.parallel:result is None 2023-09-23T17:46:22.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:21 smithi099 ceph-mon[100270]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:21 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1913044712' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T17:46:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:21 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3544115308' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T17:46:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:21 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3376109882' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T17:46:22.429 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:46:22.093Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T17:46:22.430 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:46:22.096Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T17:46:22.430 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:46:22.096Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T17:46:22.430 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:22 smithi055 bash[157370]: 2593820a0ba04eb4aaf619b4ab7291bf3a1a234ab96dffdda1296bb858213ff5 2023-09-23T17:46:22.430 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:22 smithi055 systemd[1]: Started Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:46:22.509 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738416 got 34359738418 for osd.0 2023-09-23T17:46:22.509 DEBUG:teuthology.parallel:result is None 2023-09-23T17:46:22.568 INFO:teuthology.orchestra.run.smithi055.stdout:55834574894 2023-09-23T17:46:22.604 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182436 got 107374182438 for osd.3 2023-09-23T17:46:22.604 DEBUG:teuthology.parallel:result is None 2023-09-23T17:46:22.909 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574892 got 55834574894 for osd.1 2023-09-23T17:46:22.909 DEBUG:teuthology.parallel:result is None 2023-09-23T17:46:22.909 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T17:46:22.909 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:46:23.517 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:23 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:23 smithi099 ceph-mon[100270]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:23 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:23 smithi099 ceph-mon[100270]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T17:46:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:23 smithi099 ceph-mon[100270]: Reconfiguring daemon prometheus.a on smithi099 2023-09-23T17:46:23.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:23 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2038704014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T17:46:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[109912]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[109912]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T17:46:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[109912]: Reconfiguring daemon prometheus.a on smithi099 2023-09-23T17:46:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2038704014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T17:46:23.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:23.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[95848]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:23.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:23.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[95848]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T17:46:23.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[95848]: Reconfiguring daemon prometheus.a on smithi099 2023-09-23T17:46:23.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:23 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2038704014' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T17:46:24.430 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:46:24.064Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000482734s 2023-09-23T17:46:24.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:46:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:46:24.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 systemd[1]: Stopping Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:46:24.910 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:46:24.912 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:46:25.099 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T17:46:25.099 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T17:46:25.099 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T17:46:25.099 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T17:46:25.099 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T17:46:25.099 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T17:46:25.099 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T17:46:25.100 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T17:46:25.100 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T17:46:25.100 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T17:46:25.100 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.839Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T17:46:25.100 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[133618]: ts=2023-09-23T17:46:24.840Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T17:46:25.277 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":48,"stamp":"2023-09-23T17:46:24.321922+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52576,"kb_used_data":7328,"kb_used_omap":0,"kb_used_meta":45184,"kb_avail":749908640,"statfs":{"total":767960285184,"available":767906447360,"internally_reserved":0,"allocated":7503872,"data_stored":4247973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46268416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.048959"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273285+0000","last_change":"2023-09-23T17:45:51.300186+0000","last_active":"2023-09-23T17:46:13.273285+0000","last_peered":"2023-09-23T17:46:13.273285+0000","last_clean":"2023-09-23T17:46:13.273285+0000","last_became_active":"2023-09-23T17:45:51.299890+0000","last_became_peered":"2023-09-23T17:45:51.299890+0000","last_unstale":"2023-09-23T17:46:13.273285+0000","last_undegraded":"2023-09-23T17:46:13.273285+0000","last_fullsized":"2023-09-23T17:46:13.273285+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:44.957690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490625+0000","last_change":"2023-09-23T17:45:51.135920+0000","last_active":"2023-09-23T17:46:13.490625+0000","last_peered":"2023-09-23T17:46:13.490625+0000","last_clean":"2023-09-23T17:46:13.490625+0000","last_became_active":"2023-09-23T17:45:51.132206+0000","last_became_peered":"2023-09-23T17:45:51.132206+0000","last_unstale":"2023-09-23T17:46:13.490625+0000","last_undegraded":"2023-09-23T17:46:13.490625+0000","last_fullsized":"2023-09-23T17:46:13.490625+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:12.704099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276259+0000","last_change":"2023-09-23T17:45:52.157852+0000","last_active":"2023-09-23T17:46:13.276259+0000","last_peered":"2023-09-23T17:46:13.276259+0000","last_clean":"2023-09-23T17:46:13.276259+0000","last_became_active":"2023-09-23T17:45:52.157517+0000","last_became_peered":"2023-09-23T17:45:52.157517+0000","last_unstale":"2023-09-23T17:46:13.276259+0000","last_undegraded":"2023-09-23T17:46:13.276259+0000","last_fullsized":"2023-09-23T17:46:13.276259+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:16.420615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277334+0000","last_change":"2023-09-23T17:45:51.132569+0000","last_active":"2023-09-23T17:46:13.277334+0000","last_peered":"2023-09-23T17:46:13.277334+0000","last_clean":"2023-09-23T17:46:13.277334+0000","last_became_active":"2023-09-23T17:45:51.132269+0000","last_became_peered":"2023-09-23T17:45:51.132269+0000","last_unstale":"2023-09-23T17:46:13.277334+0000","last_undegraded":"2023-09-23T17:46:13.277334+0000","last_fullsized":"2023-09-23T17:46:13.277334+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:05.731777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273575+0000","last_change":"2023-09-23T17:45:51.299474+0000","last_active":"2023-09-23T17:46:13.273575+0000","last_peered":"2023-09-23T17:46:13.273575+0000","last_clean":"2023-09-23T17:46:13.273575+0000","last_became_active":"2023-09-23T17:45:51.299180+0000","last_became_peered":"2023-09-23T17:45:51.299180+0000","last_unstale":"2023-09-23T17:46:13.273575+0000","last_undegraded":"2023-09-23T17:46:13.273575+0000","last_fullsized":"2023-09-23T17:46:13.273575+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:54:15.507654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270240+0000","last_change":"2023-09-23T17:45:51.121328+0000","last_active":"2023-09-23T17:46:13.270240+0000","last_peered":"2023-09-23T17:46:13.270240+0000","last_clean":"2023-09-23T17:46:13.270240+0000","last_became_active":"2023-09-23T17:45:51.121034+0000","last_became_peered":"2023-09-23T17:45:51.121034+0000","last_unstale":"2023-09-23T17:46:13.270240+0000","last_undegraded":"2023-09-23T17:46:13.270240+0000","last_fullsized":"2023-09-23T17:46:13.270240+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:13.256406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273805+0000","last_change":"2023-09-23T17:45:51.301129+0000","last_active":"2023-09-23T17:46:13.273805+0000","last_peered":"2023-09-23T17:46:13.273805+0000","last_clean":"2023-09-23T17:46:13.273805+0000","last_became_active":"2023-09-23T17:45:51.300499+0000","last_became_peered":"2023-09-23T17:45:51.300499+0000","last_unstale":"2023-09-23T17:46:13.273805+0000","last_undegraded":"2023-09-23T17:46:13.273805+0000","last_fullsized":"2023-09-23T17:46:13.273805+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:53.744384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270091+0000","last_change":"2023-09-23T17:45:51.136076+0000","last_active":"2023-09-23T17:46:13.270091+0000","last_peered":"2023-09-23T17:46:13.270091+0000","last_clean":"2023-09-23T17:46:13.270091+0000","last_became_active":"2023-09-23T17:45:51.135752+0000","last_became_peered":"2023-09-23T17:45:51.135752+0000","last_unstale":"2023-09-23T17:46:13.270091+0000","last_undegraded":"2023-09-23T17:46:13.270091+0000","last_fullsized":"2023-09-23T17:46:13.270091+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:45.068329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.271298+0000","last_change":"2023-09-23T17:45:52.158937+0000","last_active":"2023-09-23T17:46:13.271298+0000","last_peered":"2023-09-23T17:46:13.271298+0000","last_clean":"2023-09-23T17:46:13.271298+0000","last_became_active":"2023-09-23T17:45:52.158673+0000","last_became_peered":"2023-09-23T17:45:52.158673+0000","last_unstale":"2023-09-23T17:46:13.271298+0000","last_undegraded":"2023-09-23T17:46:13.271298+0000","last_fullsized":"2023-09-23T17:46:13.271298+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:17.880952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490556+0000","last_change":"2023-09-23T17:45:51.136229+0000","last_active":"2023-09-23T17:46:13.490556+0000","last_peered":"2023-09-23T17:46:13.490556+0000","last_clean":"2023-09-23T17:46:13.490556+0000","last_became_active":"2023-09-23T17:45:51.135863+0000","last_became_peered":"2023-09-23T17:45:51.135863+0000","last_unstale":"2023-09-23T17:46:13.490556+0000","last_undegraded":"2023-09-23T17:46:13.490556+0000","last_fullsized":"2023-09-23T17:46:13.490556+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:50.649397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273531+0000","last_change":"2023-09-23T17:45:51.148003+0000","last_active":"2023-09-23T17:46:13.273531+0000","last_peered":"2023-09-23T17:46:13.273531+0000","last_clean":"2023-09-23T17:46:13.273531+0000","last_became_active":"2023-09-23T17:45:51.147815+0000","last_became_peered":"2023-09-23T17:45:51.147815+0000","last_unstale":"2023-09-23T17:46:13.273531+0000","last_undegraded":"2023-09-23T17:46:13.273531+0000","last_fullsized":"2023-09-23T17:46:13.273531+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:32.374670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490625+0000","last_change":"2023-09-23T17:45:51.136857+0000","last_active":"2023-09-23T17:46:13.490625+0000","last_peered":"2023-09-23T17:46:13.490625+0000","last_clean":"2023-09-23T17:46:13.490625+0000","last_became_active":"2023-09-23T17:45:51.136547+0000","last_became_peered":"2023-09-23T17:45:51.136547+0000","last_unstale":"2023-09-23T17:46:13.490625+0000","last_undegraded":"2023-09-23T17:46:13.490625+0000","last_fullsized":"2023-09-23T17:46:13.490625+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:27.945281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276139+0000","last_change":"2023-09-23T17:45:12.029937+0000","last_active":"2023-09-23T17:46:13.276139+0000","last_peered":"2023-09-23T17:46:13.276139+0000","last_clean":"2023-09-23T17:46:13.276139+0000","last_became_active":"2023-09-23T17:45:12.029619+0000","last_became_peered":"2023-09-23T17:45:12.029619+0000","last_unstale":"2023-09-23T17:46:13.276139+0000","last_undegraded":"2023-09-23T17:46:13.276139+0000","last_fullsized":"2023-09-23T17:46:13.276139+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:49.407425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276173+0000","last_change":"2023-09-23T17:45:52.379329+0000","last_active":"2023-09-23T17:46:13.276173+0000","last_peered":"2023-09-23T17:46:13.276173+0000","last_clean":"2023-09-23T17:46:13.276173+0000","last_became_active":"2023-09-23T17:45:52.378998+0000","last_became_peered":"2023-09-23T17:45:52.378998+0000","last_unstale":"2023-09-23T17:46:13.276173+0000","last_undegraded":"2023-09-23T17:46:13.276173+0000","last_fullsized":"2023-09-23T17:46:13.276173+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:56.353879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273852+0000","last_change":"2023-09-23T17:45:12.288804+0000","last_active":"2023-09-23T17:46:13.273852+0000","last_peered":"2023-09-23T17:46:13.273852+0000","last_clean":"2023-09-23T17:46:13.273852+0000","last_became_active":"2023-09-23T17:45:12.288496+0000","last_became_peered":"2023-09-23T17:45:12.288496+0000","last_unstale":"2023-09-23T17:46:13.273852+0000","last_undegraded":"2023-09-23T17:46:13.273852+0000","last_fullsized":"2023-09-23T17:46:13.273852+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:50.938577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277302+0000","last_change":"2023-09-23T17:45:51.139906+0000","last_active":"2023-09-23T17:46:13.277302+0000","last_peered":"2023-09-23T17:46:13.277302+0000","last_clean":"2023-09-23T17:46:13.277302+0000","last_became_active":"2023-09-23T17:45:51.139637+0000","last_became_peered":"2023-09-23T17:45:51.139637+0000","last_unstale":"2023-09-23T17:46:13.277302+0000","last_undegraded":"2023-09-23T17:46:13.277302+0000","last_fullsized":"2023-09-23T17:46:13.277302+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:46.457695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490486+0000","last_change":"2023-09-23T17:45:52.766994+0000","last_active":"2023-09-23T17:46:13.490486+0000","last_peered":"2023-09-23T17:46:13.490486+0000","last_clean":"2023-09-23T17:46:13.490486+0000","last_became_active":"2023-09-23T17:45:52.766698+0000","last_became_peered":"2023-09-23T17:45:52.766698+0000","last_unstale":"2023-09-23T17:46:13.490486+0000","last_undegraded":"2023-09-23T17:46:13.490486+0000","last_fullsized":"2023-09-23T17:46:13.490486+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:44.996827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273581+0000","last_change":"2023-09-23T17:45:51.136259+0000","last_active":"2023-09-23T17:46:13.273581+0000","last_peered":"2023-09-23T17:46:13.273581+0000","last_clean":"2023-09-23T17:46:13.273581+0000","last_became_active":"2023-09-23T17:45:51.135902+0000","last_became_peered":"2023-09-23T17:45:51.135902+0000","last_unstale":"2023-09-23T17:46:13.273581+0000","last_undegraded":"2023-09-23T17:46:13.273581+0000","last_fullsized":"2023-09-23T17:46:13.273581+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:18.046084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278159+0000","last_change":"2023-09-23T17:45:12.291380+0000","last_active":"2023-09-23T17:46:13.278159+0000","last_peered":"2023-09-23T17:46:13.278159+0000","last_clean":"2023-09-23T17:46:13.278159+0000","last_became_active":"2023-09-23T17:45:12.290977+0000","last_became_peered":"2023-09-23T17:45:12.290977+0000","last_unstale":"2023-09-23T17:46:13.278159+0000","last_undegraded":"2023-09-23T17:46:13.278159+0000","last_fullsized":"2023-09-23T17:46:13.278159+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:31.931088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490686+0000","last_change":"2023-09-23T17:45:52.158826+0000","last_active":"2023-09-23T17:46:13.490686+0000","last_peered":"2023-09-23T17:46:13.490686+0000","last_clean":"2023-09-23T17:46:13.490686+0000","last_became_active":"2023-09-23T17:45:52.158524+0000","last_became_peered":"2023-09-23T17:45:52.158524+0000","last_unstale":"2023-09-23T17:46:13.490686+0000","last_undegraded":"2023-09-23T17:46:13.490686+0000","last_fullsized":"2023-09-23T17:46:13.490686+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:56.901906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490373+0000","last_change":"2023-09-23T17:45:51.134487+0000","last_active":"2023-09-23T17:46:13.490373+0000","last_peered":"2023-09-23T17:46:13.490373+0000","last_clean":"2023-09-23T17:46:13.490373+0000","last_became_active":"2023-09-23T17:45:51.132072+0000","last_became_peered":"2023-09-23T17:45:51.132072+0000","last_unstale":"2023-09-23T17:46:13.490373+0000","last_undegraded":"2023-09-23T17:46:13.490373+0000","last_fullsized":"2023-09-23T17:46:13.490373+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:26.916208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277721+0000","last_change":"2023-09-23T17:45:12.030797+0000","last_active":"2023-09-23T17:46:13.277721+0000","last_peered":"2023-09-23T17:46:13.277721+0000","last_clean":"2023-09-23T17:46:13.277721+0000","last_became_active":"2023-09-23T17:45:12.030530+0000","last_became_peered":"2023-09-23T17:45:12.030530+0000","last_unstale":"2023-09-23T17:46:13.277721+0000","last_undegraded":"2023-09-23T17:46:13.277721+0000","last_fullsized":"2023-09-23T17:46:13.277721+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:58:27.428598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276016+0000","last_change":"2023-09-23T17:45:51.297621+0000","last_active":"2023-09-23T17:46:13.276016+0000","last_peered":"2023-09-23T17:46:13.276016+0000","last_clean":"2023-09-23T17:46:13.276016+0000","last_became_active":"2023-09-23T17:45:51.297322+0000","last_became_peered":"2023-09-23T17:45:51.297322+0000","last_unstale":"2023-09-23T17:46:13.276016+0000","last_undegraded":"2023-09-23T17:46:13.276016+0000","last_fullsized":"2023-09-23T17:46:13.276016+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:14.268027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276488+0000","last_change":"2023-09-23T17:45:51.125208+0000","last_active":"2023-09-23T17:46:13.276488+0000","last_peered":"2023-09-23T17:46:13.276488+0000","last_clean":"2023-09-23T17:46:13.276488+0000","last_became_active":"2023-09-23T17:45:51.124806+0000","last_became_peered":"2023-09-23T17:45:51.124806+0000","last_unstale":"2023-09-23T17:46:13.276488+0000","last_undegraded":"2023-09-23T17:46:13.276488+0000","last_fullsized":"2023-09-23T17:46:13.276488+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:47.770626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":278,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277723+0000","last_change":"2023-09-23T17:44:46.990708+0000","last_active":"2023-09-23T17:46:13.277723+0000","last_peered":"2023-09-23T17:46:13.277723+0000","last_clean":"2023-09-23T17:46:13.277723+0000","last_became_active":"2023-09-23T17:44:46.990139+0000","last_became_peered":"2023-09-23T17:44:46.990139+0000","last_unstale":"2023-09-23T17:46:13.277723+0000","last_undegraded":"2023-09-23T17:46:13.277723+0000","last_fullsized":"2023-09-23T17:46:13.277723+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:51.013639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270011+0000","last_change":"2023-09-23T17:45:12.031891+0000","last_active":"2023-09-23T17:46:13.270011+0000","last_peered":"2023-09-23T17:46:13.270011+0000","last_clean":"2023-09-23T17:46:13.270011+0000","last_became_active":"2023-09-23T17:45:12.031569+0000","last_became_peered":"2023-09-23T17:45:12.031569+0000","last_unstale":"2023-09-23T17:46:13.270011+0000","last_undegraded":"2023-09-23T17:46:13.270011+0000","last_fullsized":"2023-09-23T17:46:13.270011+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:08.345244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273943+0000","last_change":"2023-09-23T17:45:52.765915+0000","last_active":"2023-09-23T17:46:13.273943+0000","last_peered":"2023-09-23T17:46:13.273943+0000","last_clean":"2023-09-23T17:46:13.273943+0000","last_became_active":"2023-09-23T17:45:52.765499+0000","last_became_peered":"2023-09-23T17:45:52.765499+0000","last_unstale":"2023-09-23T17:46:13.273943+0000","last_undegraded":"2023-09-23T17:46:13.273943+0000","last_fullsized":"2023-09-23T17:46:13.273943+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:50.326906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.272992+0000","last_change":"2023-09-23T17:45:51.134996+0000","last_active":"2023-09-23T17:46:13.272992+0000","last_peered":"2023-09-23T17:46:13.272992+0000","last_clean":"2023-09-23T17:46:13.272992+0000","last_became_active":"2023-09-23T17:45:51.134548+0000","last_became_peered":"2023-09-23T17:45:51.134548+0000","last_unstale":"2023-09-23T17:46:13.272992+0000","last_undegraded":"2023-09-23T17:46:13.272992+0000","last_fullsized":"2023-09-23T17:46:13.272992+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:03.600399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.271408+0000","last_change":"2023-09-23T17:45:12.621717+0000","last_active":"2023-09-23T17:46:13.271408+0000","last_peered":"2023-09-23T17:46:13.271408+0000","last_clean":"2023-09-23T17:46:13.271408+0000","last_became_active":"2023-09-23T17:45:12.621356+0000","last_became_peered":"2023-09-23T17:45:12.621356+0000","last_unstale":"2023-09-23T17:46:13.271408+0000","last_undegraded":"2023-09-23T17:46:13.271408+0000","last_fullsized":"2023-09-23T17:46:13.271408+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:35.490791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273539+0000","last_change":"2023-09-23T17:45:52.159430+0000","last_active":"2023-09-23T17:46:13.273539+0000","last_peered":"2023-09-23T17:46:13.273539+0000","last_clean":"2023-09-23T17:46:13.273539+0000","last_became_active":"2023-09-23T17:45:52.159106+0000","last_became_peered":"2023-09-23T17:45:52.159106+0000","last_unstale":"2023-09-23T17:46:13.273539+0000","last_undegraded":"2023-09-23T17:46:13.273539+0000","last_fullsized":"2023-09-23T17:46:13.273539+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:48:56.492342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273738+0000","last_change":"2023-09-23T17:45:51.137475+0000","last_active":"2023-09-23T17:46:13.273738+0000","last_peered":"2023-09-23T17:46:13.273738+0000","last_clean":"2023-09-23T17:46:13.273738+0000","last_became_active":"2023-09-23T17:45:51.137123+0000","last_became_peered":"2023-09-23T17:45:51.137123+0000","last_unstale":"2023-09-23T17:46:13.273738+0000","last_undegraded":"2023-09-23T17:46:13.273738+0000","last_fullsized":"2023-09-23T17:46:13.273738+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:07.986191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.269916+0000","last_change":"2023-09-23T17:45:12.031241+0000","last_active":"2023-09-23T17:46:13.269916+0000","last_peered":"2023-09-23T17:46:13.269916+0000","last_clean":"2023-09-23T17:46:13.269916+0000","last_became_active":"2023-09-23T17:45:12.030905+0000","last_became_peered":"2023-09-23T17:45:12.030905+0000","last_unstale":"2023-09-23T17:46:13.269916+0000","last_undegraded":"2023-09-23T17:46:13.269916+0000","last_fullsized":"2023-09-23T17:46:13.269916+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:45.853453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273046+0000","last_change":"2023-09-23T17:45:52.767267+0000","last_active":"2023-09-23T17:46:13.273046+0000","last_peered":"2023-09-23T17:46:13.273046+0000","last_clean":"2023-09-23T17:46:13.273046+0000","last_became_active":"2023-09-23T17:45:52.766834+0000","last_became_peered":"2023-09-23T17:45:52.766834+0000","last_unstale":"2023-09-23T17:46:13.273046+0000","last_undegraded":"2023-09-23T17:46:13.273046+0000","last_fullsized":"2023-09-23T17:46:13.273046+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:43.240389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270228+0000","last_change":"2023-09-23T17:45:51.129058+0000","last_active":"2023-09-23T17:46:13.270228+0000","last_peered":"2023-09-23T17:46:13.270228+0000","last_clean":"2023-09-23T17:46:13.270228+0000","last_became_active":"2023-09-23T17:45:51.128594+0000","last_became_peered":"2023-09-23T17:45:51.128594+0000","last_unstale":"2023-09-23T17:46:13.270228+0000","last_undegraded":"2023-09-23T17:46:13.270228+0000","last_fullsized":"2023-09-23T17:46:13.270228+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:07.796675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273501+0000","last_change":"2023-09-23T17:45:12.030639+0000","last_active":"2023-09-23T17:46:13.273501+0000","last_peered":"2023-09-23T17:46:13.273501+0000","last_clean":"2023-09-23T17:46:13.273501+0000","last_became_active":"2023-09-23T17:45:12.030334+0000","last_became_peered":"2023-09-23T17:45:12.030334+0000","last_unstale":"2023-09-23T17:46:13.273501+0000","last_undegraded":"2023-09-23T17:46:13.273501+0000","last_fullsized":"2023-09-23T17:46:13.273501+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:35.756901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T17:46:14.510791+0000","last_change":"2023-09-23T17:46:14.510791+0000","last_active":"2023-09-23T17:46:14.510791+0000","last_peered":"2023-09-23T17:46:14.510791+0000","last_clean":"2023-09-23T17:46:14.510791+0000","last_became_active":"2023-09-23T17:46:14.510517+0000","last_became_peered":"2023-09-23T17:46:14.510517+0000","last_unstale":"2023-09-23T17:46:14.510791+0000","last_undegraded":"2023-09-23T17:46:14.510791+0000","last_fullsized":"2023-09-23T17:46:14.510791+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:21.618776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276196+0000","last_change":"2023-09-23T17:45:51.136550+0000","last_active":"2023-09-23T17:46:13.276196+0000","last_peered":"2023-09-23T17:46:13.276196+0000","last_clean":"2023-09-23T17:46:13.276196+0000","last_became_active":"2023-09-23T17:45:51.136224+0000","last_became_peered":"2023-09-23T17:45:51.136224+0000","last_unstale":"2023-09-23T17:46:13.276196+0000","last_undegraded":"2023-09-23T17:46:13.276196+0000","last_fullsized":"2023-09-23T17:46:13.276196+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:13.719399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.274128+0000","last_change":"2023-09-23T17:45:51.300454+0000","last_active":"2023-09-23T17:46:13.274128+0000","last_peered":"2023-09-23T17:46:13.274128+0000","last_clean":"2023-09-23T17:46:13.274128+0000","last_became_active":"2023-09-23T17:45:51.300252+0000","last_became_peered":"2023-09-23T17:45:51.300252+0000","last_unstale":"2023-09-23T17:46:13.274128+0000","last_undegraded":"2023-09-23T17:46:13.274128+0000","last_fullsized":"2023-09-23T17:46:13.274128+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:16.235580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273541+0000","last_change":"2023-09-23T17:45:51.129274+0000","last_active":"2023-09-23T17:46:13.273541+0000","last_peered":"2023-09-23T17:46:13.273541+0000","last_clean":"2023-09-23T17:46:13.273541+0000","last_became_active":"2023-09-23T17:45:51.128820+0000","last_became_peered":"2023-09-23T17:45:51.128820+0000","last_unstale":"2023-09-23T17:46:13.273541+0000","last_undegraded":"2023-09-23T17:46:13.273541+0000","last_fullsized":"2023-09-23T17:46:13.273541+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:36.246847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T17:46:14.281164+0000","last_change":"2023-09-23T17:46:14.281164+0000","last_active":"2023-09-23T17:46:14.281164+0000","last_peered":"2023-09-23T17:46:14.281164+0000","last_clean":"2023-09-23T17:46:14.281164+0000","last_became_active":"2023-09-23T17:46:14.280823+0000","last_became_peered":"2023-09-23T17:46:14.280823+0000","last_unstale":"2023-09-23T17:46:14.281164+0000","last_undegraded":"2023-09-23T17:46:14.281164+0000","last_fullsized":"2023-09-23T17:46:14.281164+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:42.311620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276423+0000","last_change":"2023-09-23T17:45:51.136208+0000","last_active":"2023-09-23T17:46:13.276423+0000","last_peered":"2023-09-23T17:46:13.276423+0000","last_clean":"2023-09-23T17:46:13.276423+0000","last_became_active":"2023-09-23T17:45:51.135905+0000","last_became_peered":"2023-09-23T17:45:51.135905+0000","last_unstale":"2023-09-23T17:46:13.276423+0000","last_undegraded":"2023-09-23T17:46:13.276423+0000","last_fullsized":"2023-09-23T17:46:13.276423+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:16.208671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273773+0000","last_change":"2023-09-23T17:45:51.299058+0000","last_active":"2023-09-23T17:46:13.273773+0000","last_peered":"2023-09-23T17:46:13.273773+0000","last_clean":"2023-09-23T17:46:13.273773+0000","last_became_active":"2023-09-23T17:45:51.298721+0000","last_became_peered":"2023-09-23T17:45:51.298721+0000","last_unstale":"2023-09-23T17:46:13.273773+0000","last_undegraded":"2023-09-23T17:46:13.273773+0000","last_fullsized":"2023-09-23T17:46:13.273773+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:36:25.490320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276623+0000","last_change":"2023-09-23T17:45:51.138661+0000","last_active":"2023-09-23T17:46:13.276623+0000","last_peered":"2023-09-23T17:46:13.276623+0000","last_clean":"2023-09-23T17:46:13.276623+0000","last_became_active":"2023-09-23T17:45:51.138368+0000","last_became_peered":"2023-09-23T17:45:51.138368+0000","last_unstale":"2023-09-23T17:46:13.276623+0000","last_undegraded":"2023-09-23T17:46:13.276623+0000","last_fullsized":"2023-09-23T17:46:13.276623+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:53.073009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490403+0000","last_change":"2023-09-23T17:45:52.156070+0000","last_active":"2023-09-23T17:46:13.490403+0000","last_peered":"2023-09-23T17:46:13.490403+0000","last_clean":"2023-09-23T17:46:13.490403+0000","last_became_active":"2023-09-23T17:45:52.155769+0000","last_became_peered":"2023-09-23T17:45:52.155769+0000","last_unstale":"2023-09-23T17:46:13.490403+0000","last_undegraded":"2023-09-23T17:46:13.490403+0000","last_fullsized":"2023-09-23T17:46:13.490403+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:48.021708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270041+0000","last_change":"2023-09-23T17:45:51.128931+0000","last_active":"2023-09-23T17:46:13.270041+0000","last_peered":"2023-09-23T17:46:13.270041+0000","last_clean":"2023-09-23T17:46:13.270041+0000","last_became_active":"2023-09-23T17:45:51.127293+0000","last_became_peered":"2023-09-23T17:45:51.127293+0000","last_unstale":"2023-09-23T17:46:13.270041+0000","last_undegraded":"2023-09-23T17:46:13.270041+0000","last_fullsized":"2023-09-23T17:46:13.270041+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:44.738332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T17:46:15.277248+0000","last_change":"2023-09-23T17:46:15.277248+0000","last_active":"2023-09-23T17:46:15.277248+0000","last_peered":"2023-09-23T17:46:15.277248+0000","last_clean":"2023-09-23T17:46:15.277248+0000","last_became_active":"2023-09-23T17:46:15.276843+0000","last_became_peered":"2023-09-23T17:46:15.276843+0000","last_unstale":"2023-09-23T17:46:15.277248+0000","last_undegraded":"2023-09-23T17:46:15.277248+0000","last_fullsized":"2023-09-23T17:46:15.277248+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:26.911992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278212+0000","last_change":"2023-09-23T17:45:51.129514+0000","last_active":"2023-09-23T17:46:13.278212+0000","last_peered":"2023-09-23T17:46:13.278212+0000","last_clean":"2023-09-23T17:46:13.278212+0000","last_became_active":"2023-09-23T17:45:51.129154+0000","last_became_peered":"2023-09-23T17:45:51.129154+0000","last_unstale":"2023-09-23T17:46:13.278212+0000","last_undegraded":"2023-09-23T17:46:13.278212+0000","last_fullsized":"2023-09-23T17:46:13.278212+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:58.292683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273407+0000","last_change":"2023-09-23T17:45:51.301969+0000","last_active":"2023-09-23T17:46:13.273407+0000","last_peered":"2023-09-23T17:46:13.273407+0000","last_clean":"2023-09-23T17:46:13.273407+0000","last_became_active":"2023-09-23T17:45:51.301698+0000","last_became_peered":"2023-09-23T17:45:51.301698+0000","last_unstale":"2023-09-23T17:46:13.273407+0000","last_undegraded":"2023-09-23T17:46:13.273407+0000","last_fullsized":"2023-09-23T17:46:13.273407+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:41.747178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278309+0000","last_change":"2023-09-23T17:45:51.134783+0000","last_active":"2023-09-23T17:46:13.278309+0000","last_peered":"2023-09-23T17:46:13.278309+0000","last_clean":"2023-09-23T17:46:13.278309+0000","last_became_active":"2023-09-23T17:45:51.134525+0000","last_became_peered":"2023-09-23T17:45:51.134525+0000","last_unstale":"2023-09-23T17:46:13.278309+0000","last_undegraded":"2023-09-23T17:46:13.278309+0000","last_fullsized":"2023-09-23T17:46:13.278309+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:16.716194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.269862+0000","last_change":"2023-09-23T17:45:51.297370+0000","last_active":"2023-09-23T17:46:13.269862+0000","last_peered":"2023-09-23T17:46:13.269862+0000","last_clean":"2023-09-23T17:46:13.269862+0000","last_became_active":"2023-09-23T17:45:51.297055+0000","last_became_peered":"2023-09-23T17:45:51.297055+0000","last_unstale":"2023-09-23T17:46:13.269862+0000","last_undegraded":"2023-09-23T17:46:13.269862+0000","last_fullsized":"2023-09-23T17:46:13.269862+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:52.209267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278310+0000","last_change":"2023-09-23T17:45:51.138022+0000","last_active":"2023-09-23T17:46:13.278310+0000","last_peered":"2023-09-23T17:46:13.278310+0000","last_clean":"2023-09-23T17:46:13.278310+0000","last_became_active":"2023-09-23T17:45:51.137698+0000","last_became_peered":"2023-09-23T17:45:51.137698+0000","last_unstale":"2023-09-23T17:46:13.278310+0000","last_undegraded":"2023-09-23T17:46:13.278310+0000","last_fullsized":"2023-09-23T17:46:13.278310+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:53.900270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277244+0000","last_change":"2023-09-23T17:45:52.379176+0000","last_active":"2023-09-23T17:46:13.277244+0000","last_peered":"2023-09-23T17:46:13.277244+0000","last_clean":"2023-09-23T17:46:13.277244+0000","last_became_active":"2023-09-23T17:45:52.378904+0000","last_became_peered":"2023-09-23T17:45:52.378904+0000","last_unstale":"2023-09-23T17:46:13.277244+0000","last_undegraded":"2023-09-23T17:46:13.277244+0000","last_fullsized":"2023-09-23T17:46:13.277244+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:18.724594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":22,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.275671+0000","last_change":"2023-09-23T17:45:51.136689+0000","last_active":"2023-09-23T17:46:13.275671+0000","last_peered":"2023-09-23T17:46:13.275671+0000","last_clean":"2023-09-23T17:46:13.275671+0000","last_became_active":"2023-09-23T17:45:51.136395+0000","last_became_peered":"2023-09-23T17:45:51.136395+0000","last_unstale":"2023-09-23T17:46:13.275671+0000","last_undegraded":"2023-09-23T17:46:13.275671+0000","last_fullsized":"2023-09-23T17:46:13.275671+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:37.621501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490499+0000","last_change":"2023-09-23T17:45:52.379468+0000","last_active":"2023-09-23T17:46:13.490499+0000","last_peered":"2023-09-23T17:46:13.490499+0000","last_clean":"2023-09-23T17:46:13.490499+0000","last_became_active":"2023-09-23T17:45:52.379118+0000","last_became_peered":"2023-09-23T17:45:52.379118+0000","last_unstale":"2023-09-23T17:46:13.490499+0000","last_undegraded":"2023-09-23T17:46:13.490499+0000","last_fullsized":"2023-09-23T17:46:13.490499+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:55.577646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277779+0000","last_change":"2023-09-23T17:45:51.127190+0000","last_active":"2023-09-23T17:46:13.277779+0000","last_peered":"2023-09-23T17:46:13.277779+0000","last_clean":"2023-09-23T17:46:13.277779+0000","last_became_active":"2023-09-23T17:45:51.126903+0000","last_became_peered":"2023-09-23T17:45:51.126903+0000","last_unstale":"2023-09-23T17:46:13.277779+0000","last_undegraded":"2023-09-23T17:46:13.277779+0000","last_fullsized":"2023-09-23T17:46:13.277779+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:55.349259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270103+0000","last_change":"2023-09-23T17:45:51.298930+0000","last_active":"2023-09-23T17:46:13.270103+0000","last_peered":"2023-09-23T17:46:13.270103+0000","last_clean":"2023-09-23T17:46:13.270103+0000","last_became_active":"2023-09-23T17:45:51.298621+0000","last_became_peered":"2023-09-23T17:45:51.298621+0000","last_unstale":"2023-09-23T17:46:13.270103+0000","last_undegraded":"2023-09-23T17:46:13.270103+0000","last_fullsized":"2023-09-23T17:46:13.270103+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:41.683535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490407+0000","last_change":"2023-09-23T17:45:51.134974+0000","last_active":"2023-09-23T17:46:13.490407+0000","last_peered":"2023-09-23T17:46:13.490407+0000","last_clean":"2023-09-23T17:46:13.490407+0000","last_became_active":"2023-09-23T17:45:51.134625+0000","last_became_peered":"2023-09-23T17:45:51.134625+0000","last_unstale":"2023-09-23T17:46:13.490407+0000","last_undegraded":"2023-09-23T17:46:13.490407+0000","last_fullsized":"2023-09-23T17:46:13.490407+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:31:15.544172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276941+0000","last_change":"2023-09-23T17:45:52.379718+0000","last_active":"2023-09-23T17:46:13.276941+0000","last_peered":"2023-09-23T17:46:13.276941+0000","last_clean":"2023-09-23T17:46:13.276941+0000","last_became_active":"2023-09-23T17:45:52.379447+0000","last_became_peered":"2023-09-23T17:45:52.379447+0000","last_unstale":"2023-09-23T17:46:13.276941+0000","last_undegraded":"2023-09-23T17:46:13.276941+0000","last_fullsized":"2023-09-23T17:46:13.276941+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:45.018672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278060+0000","last_change":"2023-09-23T17:45:51.130546+0000","last_active":"2023-09-23T17:46:13.278060+0000","last_peered":"2023-09-23T17:46:13.278060+0000","last_clean":"2023-09-23T17:46:13.278060+0000","last_became_active":"2023-09-23T17:45:51.130269+0000","last_became_peered":"2023-09-23T17:45:51.130269+0000","last_unstale":"2023-09-23T17:46:13.278060+0000","last_undegraded":"2023-09-23T17:46:13.278060+0000","last_fullsized":"2023-09-23T17:46:13.278060+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:24.403510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273695+0000","last_change":"2023-09-23T17:45:51.301314+0000","last_active":"2023-09-23T17:46:13.273695+0000","last_peered":"2023-09-23T17:46:13.273695+0000","last_clean":"2023-09-23T17:46:13.273695+0000","last_became_active":"2023-09-23T17:45:51.300858+0000","last_became_peered":"2023-09-23T17:45:51.300858+0000","last_unstale":"2023-09-23T17:46:13.273695+0000","last_undegraded":"2023-09-23T17:46:13.273695+0000","last_fullsized":"2023-09-23T17:46:13.273695+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:50.761311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"67'6","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273852+0000","last_change":"2023-09-23T17:45:51.124742+0000","last_active":"2023-09-23T17:46:13.273852+0000","last_peered":"2023-09-23T17:46:13.273852+0000","last_clean":"2023-09-23T17:46:13.273852+0000","last_became_active":"2023-09-23T17:45:51.124342+0000","last_became_peered":"2023-09-23T17:45:51.124342+0000","last_unstale":"2023-09-23T17:46:13.273852+0000","last_undegraded":"2023-09-23T17:46:13.273852+0000","last_fullsized":"2023-09-23T17:46:13.273852+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:24.351549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.272958+0000","last_change":"2023-09-23T17:45:52.766448+0000","last_active":"2023-09-23T17:46:13.272958+0000","last_peered":"2023-09-23T17:46:13.272958+0000","last_clean":"2023-09-23T17:46:13.272958+0000","last_became_active":"2023-09-23T17:45:52.766105+0000","last_became_peered":"2023-09-23T17:45:52.766105+0000","last_unstale":"2023-09-23T17:46:13.272958+0000","last_undegraded":"2023-09-23T17:46:13.272958+0000","last_fullsized":"2023-09-23T17:46:13.272958+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:57.814977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278125+0000","last_change":"2023-09-23T17:45:51.129380+0000","last_active":"2023-09-23T17:46:13.278125+0000","last_peered":"2023-09-23T17:46:13.278125+0000","last_clean":"2023-09-23T17:46:13.278125+0000","last_became_active":"2023-09-23T17:45:51.128769+0000","last_became_peered":"2023-09-23T17:45:51.128769+0000","last_unstale":"2023-09-23T17:46:13.278125+0000","last_undegraded":"2023-09-23T17:46:13.278125+0000","last_fullsized":"2023-09-23T17:46:13.278125+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:26.573582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276354+0000","last_change":"2023-09-23T17:45:52.766380+0000","last_active":"2023-09-23T17:46:13.276354+0000","last_peered":"2023-09-23T17:46:13.276354+0000","last_clean":"2023-09-23T17:46:13.276354+0000","last_became_active":"2023-09-23T17:45:52.766032+0000","last_became_peered":"2023-09-23T17:45:52.766032+0000","last_unstale":"2023-09-23T17:46:13.276354+0000","last_undegraded":"2023-09-23T17:46:13.276354+0000","last_fullsized":"2023-09-23T17:46:13.276354+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:43.026808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270162+0000","last_change":"2023-09-23T17:45:51.127749+0000","last_active":"2023-09-23T17:46:13.270162+0000","last_peered":"2023-09-23T17:46:13.270162+0000","last_clean":"2023-09-23T17:46:13.270162+0000","last_became_active":"2023-09-23T17:45:51.127444+0000","last_became_peered":"2023-09-23T17:45:51.127444+0000","last_unstale":"2023-09-23T17:46:13.270162+0000","last_undegraded":"2023-09-23T17:46:13.270162+0000","last_fullsized":"2023-09-23T17:46:13.270162+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:26:21.160865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276304+0000","last_change":"2023-09-23T17:45:52.767264+0000","last_active":"2023-09-23T17:46:13.276304+0000","last_peered":"2023-09-23T17:46:13.276304+0000","last_clean":"2023-09-23T17:46:13.276304+0000","last_became_active":"2023-09-23T17:45:52.766896+0000","last_became_peered":"2023-09-23T17:45:52.766896+0000","last_unstale":"2023-09-23T17:46:13.276304+0000","last_undegraded":"2023-09-23T17:46:13.276304+0000","last_fullsized":"2023-09-23T17:46:13.276304+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:59.970789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490645+0000","last_change":"2023-09-23T17:45:51.136115+0000","last_active":"2023-09-23T17:46:13.490645+0000","last_peered":"2023-09-23T17:46:13.490645+0000","last_clean":"2023-09-23T17:46:13.490645+0000","last_became_active":"2023-09-23T17:45:51.132320+0000","last_became_peered":"2023-09-23T17:45:51.132320+0000","last_unstale":"2023-09-23T17:46:13.490645+0000","last_undegraded":"2023-09-23T17:46:13.490645+0000","last_fullsized":"2023-09-23T17:46:13.490645+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:29.721556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276036+0000","last_change":"2023-09-23T17:45:51.299714+0000","last_active":"2023-09-23T17:46:13.276036+0000","last_peered":"2023-09-23T17:46:13.276036+0000","last_clean":"2023-09-23T17:46:13.276036+0000","last_became_active":"2023-09-23T17:45:51.299398+0000","last_became_peered":"2023-09-23T17:45:51.299398+0000","last_unstale":"2023-09-23T17:46:13.276036+0000","last_undegraded":"2023-09-23T17:46:13.276036+0000","last_fullsized":"2023-09-23T17:46:13.276036+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:55.687196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276302+0000","last_change":"2023-09-23T17:45:51.138521+0000","last_active":"2023-09-23T17:46:13.276302+0000","last_peered":"2023-09-23T17:46:13.276302+0000","last_clean":"2023-09-23T17:46:13.276302+0000","last_became_active":"2023-09-23T17:45:51.138165+0000","last_became_peered":"2023-09-23T17:45:51.138165+0000","last_unstale":"2023-09-23T17:46:13.276302+0000","last_undegraded":"2023-09-23T17:46:13.276302+0000","last_fullsized":"2023-09-23T17:46:13.276302+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:34.588391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490603+0000","last_change":"2023-09-23T17:45:52.379529+0000","last_active":"2023-09-23T17:46:13.490603+0000","last_peered":"2023-09-23T17:46:13.490603+0000","last_clean":"2023-09-23T17:46:13.490603+0000","last_became_active":"2023-09-23T17:45:52.379149+0000","last_became_peered":"2023-09-23T17:45:52.379149+0000","last_unstale":"2023-09-23T17:46:13.490603+0000","last_undegraded":"2023-09-23T17:46:13.490603+0000","last_fullsized":"2023-09-23T17:46:13.490603+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:23.171926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.272989+0000","last_change":"2023-09-23T17:45:51.133941+0000","last_active":"2023-09-23T17:46:13.272989+0000","last_peered":"2023-09-23T17:46:13.272989+0000","last_clean":"2023-09-23T17:46:13.272989+0000","last_became_active":"2023-09-23T17:45:51.133450+0000","last_became_peered":"2023-09-23T17:45:51.133450+0000","last_unstale":"2023-09-23T17:46:13.272989+0000","last_undegraded":"2023-09-23T17:46:13.272989+0000","last_fullsized":"2023-09-23T17:46:13.272989+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:45.093055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277166+0000","last_change":"2023-09-23T17:45:52.158131+0000","last_active":"2023-09-23T17:46:13.277166+0000","last_peered":"2023-09-23T17:46:13.277166+0000","last_clean":"2023-09-23T17:46:13.277166+0000","last_became_active":"2023-09-23T17:45:52.157837+0000","last_became_peered":"2023-09-23T17:45:52.157837+0000","last_unstale":"2023-09-23T17:46:13.277166+0000","last_undegraded":"2023-09-23T17:46:13.277166+0000","last_fullsized":"2023-09-23T17:46:13.277166+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:31.428198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273484+0000","last_change":"2023-09-23T17:45:51.128538+0000","last_active":"2023-09-23T17:46:13.273484+0000","last_peered":"2023-09-23T17:46:13.273484+0000","last_clean":"2023-09-23T17:46:13.273484+0000","last_became_active":"2023-09-23T17:45:51.128253+0000","last_became_peered":"2023-09-23T17:45:51.128253+0000","last_unstale":"2023-09-23T17:46:13.273484+0000","last_undegraded":"2023-09-23T17:46:13.273484+0000","last_fullsized":"2023-09-23T17:46:13.273484+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:07.870661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266712,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6612,"kb_used_data":1292,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738540,"statfs":{"total":95995035648,"available":95988264960,"internally_reserved":0,"allocated":1323008,"data_stored":900662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.8319999999999999,"5min":2.8319999999999999,"15min":2.8319999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.497}]},{"osd":1,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.55700000000000005}]},{"osd":3,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.73,"5min":2.73,"15min":2.73},"last":0.61399999999999999}]},{"osd":4,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.628},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.68899999999999995}]}]},{"osd":6,"up_from":45,"seq":193273528348,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6664,"kb_used_data":1280,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93738488,"statfs":{"total":95995035648,"available":95988211712,"internally_reserved":0,"allocated":1310720,"data_stored":900516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.50800000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.47199999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":3.9780000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":3.9500000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.52000000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:45:44 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.42799999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6776,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738376,"statfs":{"total":95995035648,"available":95988097024,"internally_reserved":0,"allocated":704512,"data_stored":308865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:46 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.379,"15min":0.36399999999999999},"min":{"1min":0.28599999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58899999999999997,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.318},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38300000000000001,"15min":0.36499999999999999},"min":{"1min":0.248,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.52100000000000002,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.28699999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.441,"15min":0.42399999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.4260000000000002,"5min":2.4260000000000002,"15min":2.4260000000000002},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.47199999999999998,"15min":0.442},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":5.1070000000000002,"5min":5.1070000000000002,"15min":5.1070000000000002},"last":0.30399999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.51800000000000002,"15min":0.52200000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.69899999999999995,"5min":0.871,"15min":0.871},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.496,"15min":0.499},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.64600000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 17:45:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.65400000000000003,"15min":0.66300000000000003},"min":{"1min":0.44700000000000001,"5min":0.442,"15min":0.442},"max":{"1min":0.81399999999999995,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.75,"5min":0.68799999999999994,"15min":0.67800000000000005},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":3.548,"5min":3.548,"15min":3.548},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Sep 23 17:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.65700000000000003,"15min":0.63400000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.5819999999999999,"5min":3.5819999999999999,"15min":3.5819999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.62,"15min":0.61699999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.395,"5min":2.395,"15min":2.395},"last":0.63200000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.66200000000000003}]},{"osd":7,"last update":"Sat Sep 23 17:45:43 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":3.6179999999999999,"5min":3.6179999999999999,"15min":3.6179999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.56200000000000006}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7428,"kb_used_data":1276,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737724,"statfs":{"total":95995035648,"available":95987429376,"internally_reserved":0,"allocated":1306624,"data_stored":899809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.40699999999999997,"15min":0.38700000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":2.1379999999999999,"5min":2.1379999999999999,"15min":2.1379999999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.40699999999999997,"15min":0.40999999999999998},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.54500000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.42399999999999999,"15min":0.43099999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56200000000000006,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.316},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.433,"15min":0.436},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.55500000000000005,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.432}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.49399999999999999,"15min":0.504},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.60299999999999998,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51900000000000002,"15min":0.52600000000000002},"min":{"1min":0.37,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.64700000000000002,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:45:46 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.63,"15min":0.63900000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.80700000000000005,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.2090000000000001,"5min":2.2090000000000001,"15min":2.2090000000000001},"last":0.64400000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.60099999999999998,"15min":0.59599999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.181,"5min":2.181,"15min":2.181},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.64600000000000002,"15min":0.64200000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":2.1150000000000002,"5min":2.1150000000000002,"15min":2.1150000000000002},"last":0.65300000000000002}]},{"osd":6,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.71299999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:45:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.628},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.57899999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6656,"kb_used_data":696,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738496,"statfs":{"total":95995035648,"available":95988219904,"internally_reserved":0,"allocated":712704,"data_stored":309453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.27100000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42999999999999999,"15min":0.42599999999999999},"min":{"1min":0.27900000000000003,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.26800000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.432,"15min":0.42799999999999999},"min":{"1min":0.26600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.26800000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.308}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.53600000000000003,"15min":0.54500000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.52200000000000002,"15min":0.53300000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.72899999999999998,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.33600000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.625,"15min":0.61399999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.1320000000000001,"5min":2.1320000000000001,"15min":2.1320000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.64900000000000002,"15min":0.63900000000000001},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":2.1789999999999998,"5min":2.1789999999999998,"15min":2.1789999999999998},"last":0.55700000000000005}]},{"osd":5,"last update":"Sat Sep 23 17:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.63300000000000001,"15min":0.623},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.1480000000000001,"5min":2.1480000000000001,"15min":2.1480000000000001},"last":0.498},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.67000000000000004,"15min":0.65500000000000003},"min":{"1min":0.442,"5min":0.434,"15min":0.434},"max":{"1min":2.3279999999999998,"5min":2.3279999999999998,"15min":2.3279999999999998},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.68000000000000005}]},{"osd":7,"last update":"Sat Sep 23 17:45:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":2.1080000000000001,"5min":2.1080000000000001,"15min":2.1080000000000001},"last":0.624},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":2.2949999999999999,"5min":2.2949999999999999,"15min":2.2949999999999999},"last":0.65500000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182439,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6140,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739012,"statfs":{"total":95995035648,"available":95988748288,"internally_reserved":0,"allocated":708608,"data_stored":308955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.58499999999999996,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.312},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.42199999999999999,"15min":0.42399999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.32700000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.43099999999999999,"15min":0.435},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.45200000000000001,"15min":0.45400000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.26300000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.48299999999999998,"15min":0.48799999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.441,"5min":0.47299999999999998,"15min":0.47899999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.64000000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.54600000000000004}]},{"osd":4,"last update":"Sat Sep 23 17:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.59699999999999998,"15min":0.60699999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.76700000000000002,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.77700000000000002,"15min":0.81299999999999994},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.83699999999999997,"5min":5.2069999999999999,"15min":5.2069999999999999},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.59899999999999998,"15min":0.60899999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.78200000000000003,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.64700000000000002,"15min":0.65700000000000003},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.498}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6224,"kb_used_data":712,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738928,"statfs":{"total":95995035648,"available":95988662272,"internally_reserved":0,"allocated":729088,"data_stored":310547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54900000000000004,"15min":0.54500000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.503,"5min":0.53200000000000003,"15min":0.53700000000000003},"min":{"1min":0.314,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.39700000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53200000000000003,"15min":0.53100000000000003},"min":{"1min":0.36099999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.29499999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.58099999999999996,"15min":0.59299999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.3029999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.55900000000000005,"15min":0.57299999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.70899999999999996,"5min":1.073,"15min":1.073},"last":0.36799999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56999999999999995,"15min":0.57399999999999995},"min":{"1min":0.38200000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.318},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.54500000000000004,"15min":0.54200000000000004},"min":{"1min":0.41699999999999998,"5min":0.38,"15min":0.38},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.38600000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.40000000000000002,"15min":0.39200000000000002},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.141,"5min":1.141,"15min":1.141},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.38300000000000001,"15min":0.377},"min":{"1min":0.315,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.27900000000000003}]},{"osd":6,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.47999999999999998}]},{"osd":7,"last update":"Sat Sep 23 17:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.1839999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":0.442},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.42199999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757279,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6076,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739076,"statfs":{"total":95995035648,"available":95988813824,"internally_reserved":0,"allocated":708608,"data_stored":309166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.56699999999999995,"15min":0.58299999999999996},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78200000000000003,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.53700000000000003,"15min":0.54500000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":0.48899999999999999}]},{"osd":1,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.54400000000000004,"15min":0.56399999999999995},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.59699999999999998,"5min":1.5680000000000001,"15min":1.5680000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.51000000000000001,"15min":0.52000000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.61299999999999999,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.65400000000000003}]},{"osd":2,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.58799999999999997,"15min":0.60099999999999998},"min":{"1min":0.40200000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.94199999999999995,"5min":1.581,"15min":1.581},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.61399999999999999,"15min":0.60799999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":4.5739999999999998,"5min":4.5739999999999998,"15min":4.5739999999999998},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.59199999999999997,"15min":0.57799999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.6799999999999997,"5min":4.6799999999999997,"15min":4.6799999999999997},"last":0.496},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.56499999999999995,"15min":0.58099999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.63400000000000001,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.59599999999999997}]},{"osd":4,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.502,"15min":0.504},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.65400000000000003,"5min":1.738,"15min":1.738},"last":0.36899999999999999}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.778,"5min":1.778,"15min":1.778},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":0.38700000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.44600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:46:25.282 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:46:25.376 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 bash[137914]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T17:46:25.666 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T17:46:25.667 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 systemd[1]: Stopped Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:46:25.667 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 systemd[1]: Starting Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:46:25.888 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:25.928 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 podman[138035]: 2023-09-23T17:46:25.928 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 bash[138035]: 3bad347644a355f59fcbf5ebddb71d3864bdde2e75d947204450d4481e2d52a2 2023-09-23T17:46:25.928 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.904Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T17:46:25.928 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.904Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T17:46:25.928 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.904Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi099 (none))" 2023-09-23T17:46:25.928 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.904Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.904Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.906Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.906Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.908Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.908Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.910Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.910Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.851µs 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.910Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.911Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.911Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-09-23T17:46:25.929 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.911Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=20.714µs wal_replay_duration=572.244µs wbl_replay_duration=127ns total_replay_duration=611.09µs 2023-09-23T17:46:25.930 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.912Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T17:46:25.930 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.912Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T17:46:25.930 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.912Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T17:46:25.930 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.928Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.69748ms db_storage=802ns remote_storage=1.129µs web_handler=392ns query_engine=823ns scrape=172.679µs scrape_sd=22.232µs notify=37.075µs notify_sd=5.665µs rules=15.035397ms tracing=5.713µs 2023-09-23T17:46:25.930 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.928Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T17:46:25.930 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:46:25.928Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T17:46:26.262 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:25 smithi055 ceph-mon[95848]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:26.263 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:25 smithi055 ceph-mon[95848]: from='client.24665 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:46:26.263 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:25 smithi055 ceph-mon[109912]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:26.263 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:25 smithi055 ceph-mon[109912]: from='client.24665 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:46:26.263 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STOPPING 2023-09-23T17:46:26.264 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T17:46:26.264 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STOPPED 2023-09-23T17:46:26.264 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STARTING 2023-09-23T17:46:26.264 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Serving on http://:::9283 2023-09-23T17:46:26.264 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STARTED 2023-09-23T17:46:26.264 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STOPPING 2023-09-23T17:46:26.264 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T17:46:26.264 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STOPPED 2023-09-23T17:46:26.264 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STARTING 2023-09-23T17:46:26.265 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Serving on http://:::9283 2023-09-23T17:46:26.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-mon[100270]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:25 smithi099 ceph-mon[100270]: from='client.24665 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:46:26.327 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:46:25 smithi099 systemd[1]: Started Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:46:26.603 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STARTED 2023-09-23T17:46:26.603 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STOPPING 2023-09-23T17:46:26.604 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T17:46:26.604 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STOPPED 2023-09-23T17:46:26.604 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STARTING 2023-09-23T17:46:26.604 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Serving on http://:::9283 2023-09-23T17:46:26.604 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:26] ENGINE Bus STARTED 2023-09-23T17:46:27.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:46:27.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:46:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055:9093"}]: dispatch 2023-09-23T17:46:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055:9093"}]: dispatch 2023-09-23T17:46:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:46:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:46:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T17:46:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T17:46:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:46:27.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:46:27.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T17:46:27.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T17:46:27.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:27.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:46:27.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:46:27.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055:9093"}]: dispatch 2023-09-23T17:46:27.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055:9093"}]: dispatch 2023-09-23T17:46:27.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:46:27.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:46:27.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T17:46:27.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T17:46:27.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:46:27.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:46:27.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T17:46:27.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T17:46:27.185 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.185 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:26 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:27.304 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:46:27.306 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:46:27.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:46:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:46:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055:9093"}]: dispatch 2023-09-23T17:46:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055:9093"}]: dispatch 2023-09-23T17:46:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:46:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:46:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T17:46:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi099.front.sepia.ceph.com:9095"}]: dispatch 2023-09-23T17:46:27.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:46:27.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:46:27.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T17:46:27.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi099.front.sepia.ceph.com:3000"}]: dispatch 2023-09-23T17:46:27.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:27.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:26 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:27.749 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":49,"stamp":"2023-09-23T17:46:26.322485+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52576,"kb_used_data":7328,"kb_used_omap":0,"kb_used_meta":45184,"kb_avail":749908640,"statfs":{"total":767960285184,"available":767906447360,"internally_reserved":0,"allocated":7503872,"data_stored":4247973,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46268416},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002786"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273285+0000","last_change":"2023-09-23T17:45:51.300186+0000","last_active":"2023-09-23T17:46:13.273285+0000","last_peered":"2023-09-23T17:46:13.273285+0000","last_clean":"2023-09-23T17:46:13.273285+0000","last_became_active":"2023-09-23T17:45:51.299890+0000","last_became_peered":"2023-09-23T17:45:51.299890+0000","last_unstale":"2023-09-23T17:46:13.273285+0000","last_undegraded":"2023-09-23T17:46:13.273285+0000","last_fullsized":"2023-09-23T17:46:13.273285+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:44.957690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":18,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490625+0000","last_change":"2023-09-23T17:45:51.135920+0000","last_active":"2023-09-23T17:46:13.490625+0000","last_peered":"2023-09-23T17:46:13.490625+0000","last_clean":"2023-09-23T17:46:13.490625+0000","last_became_active":"2023-09-23T17:45:51.132206+0000","last_became_peered":"2023-09-23T17:45:51.132206+0000","last_unstale":"2023-09-23T17:46:13.490625+0000","last_undegraded":"2023-09-23T17:46:13.490625+0000","last_fullsized":"2023-09-23T17:46:13.490625+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:12.704099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276259+0000","last_change":"2023-09-23T17:45:52.157852+0000","last_active":"2023-09-23T17:46:13.276259+0000","last_peered":"2023-09-23T17:46:13.276259+0000","last_clean":"2023-09-23T17:46:13.276259+0000","last_became_active":"2023-09-23T17:45:52.157517+0000","last_became_peered":"2023-09-23T17:45:52.157517+0000","last_unstale":"2023-09-23T17:46:13.276259+0000","last_undegraded":"2023-09-23T17:46:13.276259+0000","last_fullsized":"2023-09-23T17:46:13.276259+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:16.420615+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277334+0000","last_change":"2023-09-23T17:45:51.132569+0000","last_active":"2023-09-23T17:46:13.277334+0000","last_peered":"2023-09-23T17:46:13.277334+0000","last_clean":"2023-09-23T17:46:13.277334+0000","last_became_active":"2023-09-23T17:45:51.132269+0000","last_became_peered":"2023-09-23T17:45:51.132269+0000","last_unstale":"2023-09-23T17:46:13.277334+0000","last_undegraded":"2023-09-23T17:46:13.277334+0000","last_fullsized":"2023-09-23T17:46:13.277334+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:05.731777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273575+0000","last_change":"2023-09-23T17:45:51.299474+0000","last_active":"2023-09-23T17:46:13.273575+0000","last_peered":"2023-09-23T17:46:13.273575+0000","last_clean":"2023-09-23T17:46:13.273575+0000","last_became_active":"2023-09-23T17:45:51.299180+0000","last_became_peered":"2023-09-23T17:45:51.299180+0000","last_unstale":"2023-09-23T17:46:13.273575+0000","last_undegraded":"2023-09-23T17:46:13.273575+0000","last_fullsized":"2023-09-23T17:46:13.273575+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:54:15.507654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270240+0000","last_change":"2023-09-23T17:45:51.121328+0000","last_active":"2023-09-23T17:46:13.270240+0000","last_peered":"2023-09-23T17:46:13.270240+0000","last_clean":"2023-09-23T17:46:13.270240+0000","last_became_active":"2023-09-23T17:45:51.121034+0000","last_became_peered":"2023-09-23T17:45:51.121034+0000","last_unstale":"2023-09-23T17:46:13.270240+0000","last_undegraded":"2023-09-23T17:46:13.270240+0000","last_fullsized":"2023-09-23T17:46:13.270240+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:13.256406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273805+0000","last_change":"2023-09-23T17:45:51.301129+0000","last_active":"2023-09-23T17:46:13.273805+0000","last_peered":"2023-09-23T17:46:13.273805+0000","last_clean":"2023-09-23T17:46:13.273805+0000","last_became_active":"2023-09-23T17:45:51.300499+0000","last_became_peered":"2023-09-23T17:45:51.300499+0000","last_unstale":"2023-09-23T17:46:13.273805+0000","last_undegraded":"2023-09-23T17:46:13.273805+0000","last_fullsized":"2023-09-23T17:46:13.273805+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:53.744384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270091+0000","last_change":"2023-09-23T17:45:51.136076+0000","last_active":"2023-09-23T17:46:13.270091+0000","last_peered":"2023-09-23T17:46:13.270091+0000","last_clean":"2023-09-23T17:46:13.270091+0000","last_became_active":"2023-09-23T17:45:51.135752+0000","last_became_peered":"2023-09-23T17:45:51.135752+0000","last_unstale":"2023-09-23T17:46:13.270091+0000","last_undegraded":"2023-09-23T17:46:13.270091+0000","last_fullsized":"2023-09-23T17:46:13.270091+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:45.068329+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.271298+0000","last_change":"2023-09-23T17:45:52.158937+0000","last_active":"2023-09-23T17:46:13.271298+0000","last_peered":"2023-09-23T17:46:13.271298+0000","last_clean":"2023-09-23T17:46:13.271298+0000","last_became_active":"2023-09-23T17:45:52.158673+0000","last_became_peered":"2023-09-23T17:45:52.158673+0000","last_unstale":"2023-09-23T17:46:13.271298+0000","last_undegraded":"2023-09-23T17:46:13.271298+0000","last_fullsized":"2023-09-23T17:46:13.271298+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:17.880952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490556+0000","last_change":"2023-09-23T17:45:51.136229+0000","last_active":"2023-09-23T17:46:13.490556+0000","last_peered":"2023-09-23T17:46:13.490556+0000","last_clean":"2023-09-23T17:46:13.490556+0000","last_became_active":"2023-09-23T17:45:51.135863+0000","last_became_peered":"2023-09-23T17:45:51.135863+0000","last_unstale":"2023-09-23T17:46:13.490556+0000","last_undegraded":"2023-09-23T17:46:13.490556+0000","last_fullsized":"2023-09-23T17:46:13.490556+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:50.649397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273531+0000","last_change":"2023-09-23T17:45:51.148003+0000","last_active":"2023-09-23T17:46:13.273531+0000","last_peered":"2023-09-23T17:46:13.273531+0000","last_clean":"2023-09-23T17:46:13.273531+0000","last_became_active":"2023-09-23T17:45:51.147815+0000","last_became_peered":"2023-09-23T17:45:51.147815+0000","last_unstale":"2023-09-23T17:46:13.273531+0000","last_undegraded":"2023-09-23T17:46:13.273531+0000","last_fullsized":"2023-09-23T17:46:13.273531+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:32.374670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490625+0000","last_change":"2023-09-23T17:45:51.136857+0000","last_active":"2023-09-23T17:46:13.490625+0000","last_peered":"2023-09-23T17:46:13.490625+0000","last_clean":"2023-09-23T17:46:13.490625+0000","last_became_active":"2023-09-23T17:45:51.136547+0000","last_became_peered":"2023-09-23T17:45:51.136547+0000","last_unstale":"2023-09-23T17:46:13.490625+0000","last_undegraded":"2023-09-23T17:46:13.490625+0000","last_fullsized":"2023-09-23T17:46:13.490625+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:27.945281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276139+0000","last_change":"2023-09-23T17:45:12.029937+0000","last_active":"2023-09-23T17:46:13.276139+0000","last_peered":"2023-09-23T17:46:13.276139+0000","last_clean":"2023-09-23T17:46:13.276139+0000","last_became_active":"2023-09-23T17:45:12.029619+0000","last_became_peered":"2023-09-23T17:45:12.029619+0000","last_unstale":"2023-09-23T17:46:13.276139+0000","last_undegraded":"2023-09-23T17:46:13.276139+0000","last_fullsized":"2023-09-23T17:46:13.276139+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:49.407425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276173+0000","last_change":"2023-09-23T17:45:52.379329+0000","last_active":"2023-09-23T17:46:13.276173+0000","last_peered":"2023-09-23T17:46:13.276173+0000","last_clean":"2023-09-23T17:46:13.276173+0000","last_became_active":"2023-09-23T17:45:52.378998+0000","last_became_peered":"2023-09-23T17:45:52.378998+0000","last_unstale":"2023-09-23T17:46:13.276173+0000","last_undegraded":"2023-09-23T17:46:13.276173+0000","last_fullsized":"2023-09-23T17:46:13.276173+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:01:56.353879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273852+0000","last_change":"2023-09-23T17:45:12.288804+0000","last_active":"2023-09-23T17:46:13.273852+0000","last_peered":"2023-09-23T17:46:13.273852+0000","last_clean":"2023-09-23T17:46:13.273852+0000","last_became_active":"2023-09-23T17:45:12.288496+0000","last_became_peered":"2023-09-23T17:45:12.288496+0000","last_unstale":"2023-09-23T17:46:13.273852+0000","last_undegraded":"2023-09-23T17:46:13.273852+0000","last_fullsized":"2023-09-23T17:46:13.273852+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:50.938577+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277302+0000","last_change":"2023-09-23T17:45:51.139906+0000","last_active":"2023-09-23T17:46:13.277302+0000","last_peered":"2023-09-23T17:46:13.277302+0000","last_clean":"2023-09-23T17:46:13.277302+0000","last_became_active":"2023-09-23T17:45:51.139637+0000","last_became_peered":"2023-09-23T17:45:51.139637+0000","last_unstale":"2023-09-23T17:46:13.277302+0000","last_undegraded":"2023-09-23T17:46:13.277302+0000","last_fullsized":"2023-09-23T17:46:13.277302+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:46.457695+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490486+0000","last_change":"2023-09-23T17:45:52.766994+0000","last_active":"2023-09-23T17:46:13.490486+0000","last_peered":"2023-09-23T17:46:13.490486+0000","last_clean":"2023-09-23T17:46:13.490486+0000","last_became_active":"2023-09-23T17:45:52.766698+0000","last_became_peered":"2023-09-23T17:45:52.766698+0000","last_unstale":"2023-09-23T17:46:13.490486+0000","last_undegraded":"2023-09-23T17:46:13.490486+0000","last_fullsized":"2023-09-23T17:46:13.490486+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:44.996827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273581+0000","last_change":"2023-09-23T17:45:51.136259+0000","last_active":"2023-09-23T17:46:13.273581+0000","last_peered":"2023-09-23T17:46:13.273581+0000","last_clean":"2023-09-23T17:46:13.273581+0000","last_became_active":"2023-09-23T17:45:51.135902+0000","last_became_peered":"2023-09-23T17:45:51.135902+0000","last_unstale":"2023-09-23T17:46:13.273581+0000","last_undegraded":"2023-09-23T17:46:13.273581+0000","last_fullsized":"2023-09-23T17:46:13.273581+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:18.046084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278159+0000","last_change":"2023-09-23T17:45:12.291380+0000","last_active":"2023-09-23T17:46:13.278159+0000","last_peered":"2023-09-23T17:46:13.278159+0000","last_clean":"2023-09-23T17:46:13.278159+0000","last_became_active":"2023-09-23T17:45:12.290977+0000","last_became_peered":"2023-09-23T17:45:12.290977+0000","last_unstale":"2023-09-23T17:46:13.278159+0000","last_undegraded":"2023-09-23T17:46:13.278159+0000","last_fullsized":"2023-09-23T17:46:13.278159+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:31.931088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490686+0000","last_change":"2023-09-23T17:45:52.158826+0000","last_active":"2023-09-23T17:46:13.490686+0000","last_peered":"2023-09-23T17:46:13.490686+0000","last_clean":"2023-09-23T17:46:13.490686+0000","last_became_active":"2023-09-23T17:45:52.158524+0000","last_became_peered":"2023-09-23T17:45:52.158524+0000","last_unstale":"2023-09-23T17:46:13.490686+0000","last_undegraded":"2023-09-23T17:46:13.490686+0000","last_fullsized":"2023-09-23T17:46:13.490686+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:56.901906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490373+0000","last_change":"2023-09-23T17:45:51.134487+0000","last_active":"2023-09-23T17:46:13.490373+0000","last_peered":"2023-09-23T17:46:13.490373+0000","last_clean":"2023-09-23T17:46:13.490373+0000","last_became_active":"2023-09-23T17:45:51.132072+0000","last_became_peered":"2023-09-23T17:45:51.132072+0000","last_unstale":"2023-09-23T17:46:13.490373+0000","last_undegraded":"2023-09-23T17:46:13.490373+0000","last_fullsized":"2023-09-23T17:46:13.490373+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:26.916208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277721+0000","last_change":"2023-09-23T17:45:12.030797+0000","last_active":"2023-09-23T17:46:13.277721+0000","last_peered":"2023-09-23T17:46:13.277721+0000","last_clean":"2023-09-23T17:46:13.277721+0000","last_became_active":"2023-09-23T17:45:12.030530+0000","last_became_peered":"2023-09-23T17:45:12.030530+0000","last_unstale":"2023-09-23T17:46:13.277721+0000","last_undegraded":"2023-09-23T17:46:13.277721+0000","last_fullsized":"2023-09-23T17:46:13.277721+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:58:27.428598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276016+0000","last_change":"2023-09-23T17:45:51.297621+0000","last_active":"2023-09-23T17:46:13.276016+0000","last_peered":"2023-09-23T17:46:13.276016+0000","last_clean":"2023-09-23T17:46:13.276016+0000","last_became_active":"2023-09-23T17:45:51.297322+0000","last_became_peered":"2023-09-23T17:45:51.297322+0000","last_unstale":"2023-09-23T17:46:13.276016+0000","last_undegraded":"2023-09-23T17:46:13.276016+0000","last_fullsized":"2023-09-23T17:46:13.276016+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:14.268027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276488+0000","last_change":"2023-09-23T17:45:51.125208+0000","last_active":"2023-09-23T17:46:13.276488+0000","last_peered":"2023-09-23T17:46:13.276488+0000","last_clean":"2023-09-23T17:46:13.276488+0000","last_became_active":"2023-09-23T17:45:51.124806+0000","last_became_peered":"2023-09-23T17:45:51.124806+0000","last_unstale":"2023-09-23T17:46:13.276488+0000","last_undegraded":"2023-09-23T17:46:13.276488+0000","last_fullsized":"2023-09-23T17:46:13.276488+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:47.770626+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"60'128","reported_seq":278,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277723+0000","last_change":"2023-09-23T17:44:46.990708+0000","last_active":"2023-09-23T17:46:13.277723+0000","last_peered":"2023-09-23T17:46:13.277723+0000","last_clean":"2023-09-23T17:46:13.277723+0000","last_became_active":"2023-09-23T17:44:46.990139+0000","last_became_peered":"2023-09-23T17:44:46.990139+0000","last_unstale":"2023-09-23T17:46:13.277723+0000","last_undegraded":"2023-09-23T17:46:13.277723+0000","last_fullsized":"2023-09-23T17:46:13.277723+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:51.013639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270011+0000","last_change":"2023-09-23T17:45:12.031891+0000","last_active":"2023-09-23T17:46:13.270011+0000","last_peered":"2023-09-23T17:46:13.270011+0000","last_clean":"2023-09-23T17:46:13.270011+0000","last_became_active":"2023-09-23T17:45:12.031569+0000","last_became_peered":"2023-09-23T17:45:12.031569+0000","last_unstale":"2023-09-23T17:46:13.270011+0000","last_undegraded":"2023-09-23T17:46:13.270011+0000","last_fullsized":"2023-09-23T17:46:13.270011+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:15:08.345244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273943+0000","last_change":"2023-09-23T17:45:52.765915+0000","last_active":"2023-09-23T17:46:13.273943+0000","last_peered":"2023-09-23T17:46:13.273943+0000","last_clean":"2023-09-23T17:46:13.273943+0000","last_became_active":"2023-09-23T17:45:52.765499+0000","last_became_peered":"2023-09-23T17:45:52.765499+0000","last_unstale":"2023-09-23T17:46:13.273943+0000","last_undegraded":"2023-09-23T17:46:13.273943+0000","last_fullsized":"2023-09-23T17:46:13.273943+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:50.326906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.272992+0000","last_change":"2023-09-23T17:45:51.134996+0000","last_active":"2023-09-23T17:46:13.272992+0000","last_peered":"2023-09-23T17:46:13.272992+0000","last_clean":"2023-09-23T17:46:13.272992+0000","last_became_active":"2023-09-23T17:45:51.134548+0000","last_became_peered":"2023-09-23T17:45:51.134548+0000","last_unstale":"2023-09-23T17:46:13.272992+0000","last_undegraded":"2023-09-23T17:46:13.272992+0000","last_fullsized":"2023-09-23T17:46:13.272992+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:03.600399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.271408+0000","last_change":"2023-09-23T17:45:12.621717+0000","last_active":"2023-09-23T17:46:13.271408+0000","last_peered":"2023-09-23T17:46:13.271408+0000","last_clean":"2023-09-23T17:46:13.271408+0000","last_became_active":"2023-09-23T17:45:12.621356+0000","last_became_peered":"2023-09-23T17:45:12.621356+0000","last_unstale":"2023-09-23T17:46:13.271408+0000","last_undegraded":"2023-09-23T17:46:13.271408+0000","last_fullsized":"2023-09-23T17:46:13.271408+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":60,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:00:35.490791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273539+0000","last_change":"2023-09-23T17:45:52.159430+0000","last_active":"2023-09-23T17:46:13.273539+0000","last_peered":"2023-09-23T17:46:13.273539+0000","last_clean":"2023-09-23T17:46:13.273539+0000","last_became_active":"2023-09-23T17:45:52.159106+0000","last_became_peered":"2023-09-23T17:45:52.159106+0000","last_unstale":"2023-09-23T17:46:13.273539+0000","last_undegraded":"2023-09-23T17:46:13.273539+0000","last_fullsized":"2023-09-23T17:46:13.273539+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:48:56.492342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273738+0000","last_change":"2023-09-23T17:45:51.137475+0000","last_active":"2023-09-23T17:46:13.273738+0000","last_peered":"2023-09-23T17:46:13.273738+0000","last_clean":"2023-09-23T17:46:13.273738+0000","last_became_active":"2023-09-23T17:45:51.137123+0000","last_became_peered":"2023-09-23T17:45:51.137123+0000","last_unstale":"2023-09-23T17:46:13.273738+0000","last_undegraded":"2023-09-23T17:46:13.273738+0000","last_fullsized":"2023-09-23T17:46:13.273738+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:07.986191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.269916+0000","last_change":"2023-09-23T17:45:12.031241+0000","last_active":"2023-09-23T17:46:13.269916+0000","last_peered":"2023-09-23T17:46:13.269916+0000","last_clean":"2023-09-23T17:46:13.269916+0000","last_became_active":"2023-09-23T17:45:12.030905+0000","last_became_peered":"2023-09-23T17:45:12.030905+0000","last_unstale":"2023-09-23T17:46:13.269916+0000","last_undegraded":"2023-09-23T17:46:13.269916+0000","last_fullsized":"2023-09-23T17:46:13.269916+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:45.853453+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273046+0000","last_change":"2023-09-23T17:45:52.767267+0000","last_active":"2023-09-23T17:46:13.273046+0000","last_peered":"2023-09-23T17:46:13.273046+0000","last_clean":"2023-09-23T17:46:13.273046+0000","last_became_active":"2023-09-23T17:45:52.766834+0000","last_became_peered":"2023-09-23T17:45:52.766834+0000","last_unstale":"2023-09-23T17:46:13.273046+0000","last_undegraded":"2023-09-23T17:46:13.273046+0000","last_fullsized":"2023-09-23T17:46:13.273046+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:43.240389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270228+0000","last_change":"2023-09-23T17:45:51.129058+0000","last_active":"2023-09-23T17:46:13.270228+0000","last_peered":"2023-09-23T17:46:13.270228+0000","last_clean":"2023-09-23T17:46:13.270228+0000","last_became_active":"2023-09-23T17:45:51.128594+0000","last_became_peered":"2023-09-23T17:45:51.128594+0000","last_unstale":"2023-09-23T17:46:13.270228+0000","last_undegraded":"2023-09-23T17:46:13.270228+0000","last_fullsized":"2023-09-23T17:46:13.270228+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:07.796675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":19,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273501+0000","last_change":"2023-09-23T17:45:12.030639+0000","last_active":"2023-09-23T17:46:13.273501+0000","last_peered":"2023-09-23T17:46:13.273501+0000","last_clean":"2023-09-23T17:46:13.273501+0000","last_became_active":"2023-09-23T17:45:12.030334+0000","last_became_peered":"2023-09-23T17:45:12.030334+0000","last_unstale":"2023-09-23T17:46:13.273501+0000","last_undegraded":"2023-09-23T17:46:13.273501+0000","last_fullsized":"2023-09-23T17:46:13.273501+0000","mapping_epoch":58,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":59,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:35.756901+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T17:46:14.510791+0000","last_change":"2023-09-23T17:46:14.510791+0000","last_active":"2023-09-23T17:46:14.510791+0000","last_peered":"2023-09-23T17:46:14.510791+0000","last_clean":"2023-09-23T17:46:14.510791+0000","last_became_active":"2023-09-23T17:46:14.510517+0000","last_became_peered":"2023-09-23T17:46:14.510517+0000","last_unstale":"2023-09-23T17:46:14.510791+0000","last_undegraded":"2023-09-23T17:46:14.510791+0000","last_fullsized":"2023-09-23T17:46:14.510791+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:21.618776+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276196+0000","last_change":"2023-09-23T17:45:51.136550+0000","last_active":"2023-09-23T17:46:13.276196+0000","last_peered":"2023-09-23T17:46:13.276196+0000","last_clean":"2023-09-23T17:46:13.276196+0000","last_became_active":"2023-09-23T17:45:51.136224+0000","last_became_peered":"2023-09-23T17:45:51.136224+0000","last_unstale":"2023-09-23T17:46:13.276196+0000","last_undegraded":"2023-09-23T17:46:13.276196+0000","last_fullsized":"2023-09-23T17:46:13.276196+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:13.719399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.274128+0000","last_change":"2023-09-23T17:45:51.300454+0000","last_active":"2023-09-23T17:46:13.274128+0000","last_peered":"2023-09-23T17:46:13.274128+0000","last_clean":"2023-09-23T17:46:13.274128+0000","last_became_active":"2023-09-23T17:45:51.300252+0000","last_became_peered":"2023-09-23T17:45:51.300252+0000","last_unstale":"2023-09-23T17:46:13.274128+0000","last_undegraded":"2023-09-23T17:46:13.274128+0000","last_fullsized":"2023-09-23T17:46:13.274128+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:16.235580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273541+0000","last_change":"2023-09-23T17:45:51.129274+0000","last_active":"2023-09-23T17:46:13.273541+0000","last_peered":"2023-09-23T17:46:13.273541+0000","last_clean":"2023-09-23T17:46:13.273541+0000","last_became_active":"2023-09-23T17:45:51.128820+0000","last_became_peered":"2023-09-23T17:45:51.128820+0000","last_unstale":"2023-09-23T17:46:13.273541+0000","last_undegraded":"2023-09-23T17:46:13.273541+0000","last_fullsized":"2023-09-23T17:46:13.273541+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:37:36.246847+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":23,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T17:46:14.281164+0000","last_change":"2023-09-23T17:46:14.281164+0000","last_active":"2023-09-23T17:46:14.281164+0000","last_peered":"2023-09-23T17:46:14.281164+0000","last_clean":"2023-09-23T17:46:14.281164+0000","last_became_active":"2023-09-23T17:46:14.280823+0000","last_became_peered":"2023-09-23T17:46:14.280823+0000","last_unstale":"2023-09-23T17:46:14.281164+0000","last_undegraded":"2023-09-23T17:46:14.281164+0000","last_fullsized":"2023-09-23T17:46:14.281164+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:16:42.311620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276423+0000","last_change":"2023-09-23T17:45:51.136208+0000","last_active":"2023-09-23T17:46:13.276423+0000","last_peered":"2023-09-23T17:46:13.276423+0000","last_clean":"2023-09-23T17:46:13.276423+0000","last_became_active":"2023-09-23T17:45:51.135905+0000","last_became_peered":"2023-09-23T17:45:51.135905+0000","last_unstale":"2023-09-23T17:46:13.276423+0000","last_undegraded":"2023-09-23T17:46:13.276423+0000","last_fullsized":"2023-09-23T17:46:13.276423+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:16.208671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273773+0000","last_change":"2023-09-23T17:45:51.299058+0000","last_active":"2023-09-23T17:46:13.273773+0000","last_peered":"2023-09-23T17:46:13.273773+0000","last_clean":"2023-09-23T17:46:13.273773+0000","last_became_active":"2023-09-23T17:45:51.298721+0000","last_became_peered":"2023-09-23T17:45:51.298721+0000","last_unstale":"2023-09-23T17:46:13.273773+0000","last_undegraded":"2023-09-23T17:46:13.273773+0000","last_fullsized":"2023-09-23T17:46:13.273773+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:36:25.490320+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276623+0000","last_change":"2023-09-23T17:45:51.138661+0000","last_active":"2023-09-23T17:46:13.276623+0000","last_peered":"2023-09-23T17:46:13.276623+0000","last_clean":"2023-09-23T17:46:13.276623+0000","last_became_active":"2023-09-23T17:45:51.138368+0000","last_became_peered":"2023-09-23T17:45:51.138368+0000","last_unstale":"2023-09-23T17:46:13.276623+0000","last_undegraded":"2023-09-23T17:46:13.276623+0000","last_fullsized":"2023-09-23T17:46:13.276623+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:53.073009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490403+0000","last_change":"2023-09-23T17:45:52.156070+0000","last_active":"2023-09-23T17:46:13.490403+0000","last_peered":"2023-09-23T17:46:13.490403+0000","last_clean":"2023-09-23T17:46:13.490403+0000","last_became_active":"2023-09-23T17:45:52.155769+0000","last_became_peered":"2023-09-23T17:45:52.155769+0000","last_unstale":"2023-09-23T17:46:13.490403+0000","last_undegraded":"2023-09-23T17:46:13.490403+0000","last_fullsized":"2023-09-23T17:46:13.490403+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:48.021708+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270041+0000","last_change":"2023-09-23T17:45:51.128931+0000","last_active":"2023-09-23T17:46:13.270041+0000","last_peered":"2023-09-23T17:46:13.270041+0000","last_clean":"2023-09-23T17:46:13.270041+0000","last_became_active":"2023-09-23T17:45:51.127293+0000","last_became_peered":"2023-09-23T17:45:51.127293+0000","last_unstale":"2023-09-23T17:46:13.270041+0000","last_undegraded":"2023-09-23T17:46:13.270041+0000","last_fullsized":"2023-09-23T17:46:13.270041+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:44.738332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":23,"reported_epoch":69,"state":"active+clean","last_fresh":"2023-09-23T17:46:15.277248+0000","last_change":"2023-09-23T17:46:15.277248+0000","last_active":"2023-09-23T17:46:15.277248+0000","last_peered":"2023-09-23T17:46:15.277248+0000","last_clean":"2023-09-23T17:46:15.277248+0000","last_became_active":"2023-09-23T17:46:15.276843+0000","last_became_peered":"2023-09-23T17:46:15.276843+0000","last_unstale":"2023-09-23T17:46:15.277248+0000","last_undegraded":"2023-09-23T17:46:15.277248+0000","last_fullsized":"2023-09-23T17:46:15.277248+0000","mapping_epoch":68,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":69,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:26.911992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278212+0000","last_change":"2023-09-23T17:45:51.129514+0000","last_active":"2023-09-23T17:46:13.278212+0000","last_peered":"2023-09-23T17:46:13.278212+0000","last_clean":"2023-09-23T17:46:13.278212+0000","last_became_active":"2023-09-23T17:45:51.129154+0000","last_became_peered":"2023-09-23T17:45:51.129154+0000","last_unstale":"2023-09-23T17:46:13.278212+0000","last_undegraded":"2023-09-23T17:46:13.278212+0000","last_fullsized":"2023-09-23T17:46:13.278212+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:58.292683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273407+0000","last_change":"2023-09-23T17:45:51.301969+0000","last_active":"2023-09-23T17:46:13.273407+0000","last_peered":"2023-09-23T17:46:13.273407+0000","last_clean":"2023-09-23T17:46:13.273407+0000","last_became_active":"2023-09-23T17:45:51.301698+0000","last_became_peered":"2023-09-23T17:45:51.301698+0000","last_unstale":"2023-09-23T17:46:13.273407+0000","last_undegraded":"2023-09-23T17:46:13.273407+0000","last_fullsized":"2023-09-23T17:46:13.273407+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:41.747178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278309+0000","last_change":"2023-09-23T17:45:51.134783+0000","last_active":"2023-09-23T17:46:13.278309+0000","last_peered":"2023-09-23T17:46:13.278309+0000","last_clean":"2023-09-23T17:46:13.278309+0000","last_became_active":"2023-09-23T17:45:51.134525+0000","last_became_peered":"2023-09-23T17:45:51.134525+0000","last_unstale":"2023-09-23T17:46:13.278309+0000","last_undegraded":"2023-09-23T17:46:13.278309+0000","last_fullsized":"2023-09-23T17:46:13.278309+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:16.716194+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.269862+0000","last_change":"2023-09-23T17:45:51.297370+0000","last_active":"2023-09-23T17:46:13.269862+0000","last_peered":"2023-09-23T17:46:13.269862+0000","last_clean":"2023-09-23T17:46:13.269862+0000","last_became_active":"2023-09-23T17:45:51.297055+0000","last_became_peered":"2023-09-23T17:45:51.297055+0000","last_unstale":"2023-09-23T17:46:13.269862+0000","last_undegraded":"2023-09-23T17:46:13.269862+0000","last_fullsized":"2023-09-23T17:46:13.269862+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:52.209267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278310+0000","last_change":"2023-09-23T17:45:51.138022+0000","last_active":"2023-09-23T17:46:13.278310+0000","last_peered":"2023-09-23T17:46:13.278310+0000","last_clean":"2023-09-23T17:46:13.278310+0000","last_became_active":"2023-09-23T17:45:51.137698+0000","last_became_peered":"2023-09-23T17:45:51.137698+0000","last_unstale":"2023-09-23T17:46:13.278310+0000","last_undegraded":"2023-09-23T17:46:13.278310+0000","last_fullsized":"2023-09-23T17:46:13.278310+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:53.900270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277244+0000","last_change":"2023-09-23T17:45:52.379176+0000","last_active":"2023-09-23T17:46:13.277244+0000","last_peered":"2023-09-23T17:46:13.277244+0000","last_clean":"2023-09-23T17:46:13.277244+0000","last_became_active":"2023-09-23T17:45:52.378904+0000","last_became_peered":"2023-09-23T17:45:52.378904+0000","last_unstale":"2023-09-23T17:46:13.277244+0000","last_undegraded":"2023-09-23T17:46:13.277244+0000","last_fullsized":"2023-09-23T17:46:13.277244+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:18.724594+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":22,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.275671+0000","last_change":"2023-09-23T17:45:51.136689+0000","last_active":"2023-09-23T17:46:13.275671+0000","last_peered":"2023-09-23T17:46:13.275671+0000","last_clean":"2023-09-23T17:46:13.275671+0000","last_became_active":"2023-09-23T17:45:51.136395+0000","last_became_peered":"2023-09-23T17:45:51.136395+0000","last_unstale":"2023-09-23T17:46:13.275671+0000","last_undegraded":"2023-09-23T17:46:13.275671+0000","last_fullsized":"2023-09-23T17:46:13.275671+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:08:37.621501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490499+0000","last_change":"2023-09-23T17:45:52.379468+0000","last_active":"2023-09-23T17:46:13.490499+0000","last_peered":"2023-09-23T17:46:13.490499+0000","last_clean":"2023-09-23T17:46:13.490499+0000","last_became_active":"2023-09-23T17:45:52.379118+0000","last_became_peered":"2023-09-23T17:45:52.379118+0000","last_unstale":"2023-09-23T17:46:13.490499+0000","last_undegraded":"2023-09-23T17:46:13.490499+0000","last_fullsized":"2023-09-23T17:46:13.490499+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:55.577646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277779+0000","last_change":"2023-09-23T17:45:51.127190+0000","last_active":"2023-09-23T17:46:13.277779+0000","last_peered":"2023-09-23T17:46:13.277779+0000","last_clean":"2023-09-23T17:46:13.277779+0000","last_became_active":"2023-09-23T17:45:51.126903+0000","last_became_peered":"2023-09-23T17:45:51.126903+0000","last_unstale":"2023-09-23T17:46:13.277779+0000","last_undegraded":"2023-09-23T17:46:13.277779+0000","last_fullsized":"2023-09-23T17:46:13.277779+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:55.349259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270103+0000","last_change":"2023-09-23T17:45:51.298930+0000","last_active":"2023-09-23T17:46:13.270103+0000","last_peered":"2023-09-23T17:46:13.270103+0000","last_clean":"2023-09-23T17:46:13.270103+0000","last_became_active":"2023-09-23T17:45:51.298621+0000","last_became_peered":"2023-09-23T17:45:51.298621+0000","last_unstale":"2023-09-23T17:46:13.270103+0000","last_undegraded":"2023-09-23T17:46:13.270103+0000","last_fullsized":"2023-09-23T17:46:13.270103+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:08:41.683535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490407+0000","last_change":"2023-09-23T17:45:51.134974+0000","last_active":"2023-09-23T17:46:13.490407+0000","last_peered":"2023-09-23T17:46:13.490407+0000","last_clean":"2023-09-23T17:46:13.490407+0000","last_became_active":"2023-09-23T17:45:51.134625+0000","last_became_peered":"2023-09-23T17:45:51.134625+0000","last_unstale":"2023-09-23T17:46:13.490407+0000","last_undegraded":"2023-09-23T17:46:13.490407+0000","last_fullsized":"2023-09-23T17:46:13.490407+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:31:15.544172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276941+0000","last_change":"2023-09-23T17:45:52.379718+0000","last_active":"2023-09-23T17:46:13.276941+0000","last_peered":"2023-09-23T17:46:13.276941+0000","last_clean":"2023-09-23T17:46:13.276941+0000","last_became_active":"2023-09-23T17:45:52.379447+0000","last_became_peered":"2023-09-23T17:45:52.379447+0000","last_unstale":"2023-09-23T17:46:13.276941+0000","last_undegraded":"2023-09-23T17:46:13.276941+0000","last_fullsized":"2023-09-23T17:46:13.276941+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:45.018672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":16,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278060+0000","last_change":"2023-09-23T17:45:51.130546+0000","last_active":"2023-09-23T17:46:13.278060+0000","last_peered":"2023-09-23T17:46:13.278060+0000","last_clean":"2023-09-23T17:46:13.278060+0000","last_became_active":"2023-09-23T17:45:51.130269+0000","last_became_peered":"2023-09-23T17:45:51.130269+0000","last_unstale":"2023-09-23T17:46:13.278060+0000","last_undegraded":"2023-09-23T17:46:13.278060+0000","last_fullsized":"2023-09-23T17:46:13.278060+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:24.403510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273695+0000","last_change":"2023-09-23T17:45:51.301314+0000","last_active":"2023-09-23T17:46:13.273695+0000","last_peered":"2023-09-23T17:46:13.273695+0000","last_clean":"2023-09-23T17:46:13.273695+0000","last_became_active":"2023-09-23T17:45:51.300858+0000","last_became_peered":"2023-09-23T17:45:51.300858+0000","last_unstale":"2023-09-23T17:46:13.273695+0000","last_undegraded":"2023-09-23T17:46:13.273695+0000","last_fullsized":"2023-09-23T17:46:13.273695+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:50.761311+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"67'6","reported_seq":20,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273852+0000","last_change":"2023-09-23T17:45:51.124742+0000","last_active":"2023-09-23T17:46:13.273852+0000","last_peered":"2023-09-23T17:46:13.273852+0000","last_clean":"2023-09-23T17:46:13.273852+0000","last_became_active":"2023-09-23T17:45:51.124342+0000","last_became_peered":"2023-09-23T17:45:51.124342+0000","last_unstale":"2023-09-23T17:46:13.273852+0000","last_undegraded":"2023-09-23T17:46:13.273852+0000","last_fullsized":"2023-09-23T17:46:13.273852+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:24.351549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.272958+0000","last_change":"2023-09-23T17:45:52.766448+0000","last_active":"2023-09-23T17:46:13.272958+0000","last_peered":"2023-09-23T17:46:13.272958+0000","last_clean":"2023-09-23T17:46:13.272958+0000","last_became_active":"2023-09-23T17:45:52.766105+0000","last_became_peered":"2023-09-23T17:45:52.766105+0000","last_unstale":"2023-09-23T17:46:13.272958+0000","last_undegraded":"2023-09-23T17:46:13.272958+0000","last_fullsized":"2023-09-23T17:46:13.272958+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:46:57.814977+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":17,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.278125+0000","last_change":"2023-09-23T17:45:51.129380+0000","last_active":"2023-09-23T17:46:13.278125+0000","last_peered":"2023-09-23T17:46:13.278125+0000","last_clean":"2023-09-23T17:46:13.278125+0000","last_became_active":"2023-09-23T17:45:51.128769+0000","last_became_peered":"2023-09-23T17:45:51.128769+0000","last_unstale":"2023-09-23T17:46:13.278125+0000","last_undegraded":"2023-09-23T17:46:13.278125+0000","last_fullsized":"2023-09-23T17:46:13.278125+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:26.573582+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276354+0000","last_change":"2023-09-23T17:45:52.766380+0000","last_active":"2023-09-23T17:46:13.276354+0000","last_peered":"2023-09-23T17:46:13.276354+0000","last_clean":"2023-09-23T17:46:13.276354+0000","last_became_active":"2023-09-23T17:45:52.766032+0000","last_became_peered":"2023-09-23T17:45:52.766032+0000","last_unstale":"2023-09-23T17:46:13.276354+0000","last_undegraded":"2023-09-23T17:46:13.276354+0000","last_fullsized":"2023-09-23T17:46:13.276354+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:43.026808+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.270162+0000","last_change":"2023-09-23T17:45:51.127749+0000","last_active":"2023-09-23T17:46:13.270162+0000","last_peered":"2023-09-23T17:46:13.270162+0000","last_clean":"2023-09-23T17:46:13.270162+0000","last_became_active":"2023-09-23T17:45:51.127444+0000","last_became_peered":"2023-09-23T17:45:51.127444+0000","last_unstale":"2023-09-23T17:46:13.270162+0000","last_undegraded":"2023-09-23T17:46:13.270162+0000","last_fullsized":"2023-09-23T17:46:13.270162+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:26:21.160865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276304+0000","last_change":"2023-09-23T17:45:52.767264+0000","last_active":"2023-09-23T17:46:13.276304+0000","last_peered":"2023-09-23T17:46:13.276304+0000","last_clean":"2023-09-23T17:46:13.276304+0000","last_became_active":"2023-09-23T17:45:52.766896+0000","last_became_peered":"2023-09-23T17:45:52.766896+0000","last_unstale":"2023-09-23T17:46:13.276304+0000","last_undegraded":"2023-09-23T17:46:13.276304+0000","last_fullsized":"2023-09-23T17:46:13.276304+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:59.970789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490645+0000","last_change":"2023-09-23T17:45:51.136115+0000","last_active":"2023-09-23T17:46:13.490645+0000","last_peered":"2023-09-23T17:46:13.490645+0000","last_clean":"2023-09-23T17:46:13.490645+0000","last_became_active":"2023-09-23T17:45:51.132320+0000","last_became_peered":"2023-09-23T17:45:51.132320+0000","last_unstale":"2023-09-23T17:46:13.490645+0000","last_undegraded":"2023-09-23T17:46:13.490645+0000","last_fullsized":"2023-09-23T17:46:13.490645+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:29.721556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276036+0000","last_change":"2023-09-23T17:45:51.299714+0000","last_active":"2023-09-23T17:46:13.276036+0000","last_peered":"2023-09-23T17:46:13.276036+0000","last_clean":"2023-09-23T17:46:13.276036+0000","last_became_active":"2023-09-23T17:45:51.299398+0000","last_became_peered":"2023-09-23T17:45:51.299398+0000","last_unstale":"2023-09-23T17:46:13.276036+0000","last_undegraded":"2023-09-23T17:46:13.276036+0000","last_fullsized":"2023-09-23T17:46:13.276036+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:55.687196+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.276302+0000","last_change":"2023-09-23T17:45:51.138521+0000","last_active":"2023-09-23T17:46:13.276302+0000","last_peered":"2023-09-23T17:46:13.276302+0000","last_clean":"2023-09-23T17:46:13.276302+0000","last_became_active":"2023-09-23T17:45:51.138165+0000","last_became_peered":"2023-09-23T17:45:51.138165+0000","last_unstale":"2023-09-23T17:46:13.276302+0000","last_undegraded":"2023-09-23T17:46:13.276302+0000","last_fullsized":"2023-09-23T17:46:13.276302+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:34.588391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.490603+0000","last_change":"2023-09-23T17:45:52.379529+0000","last_active":"2023-09-23T17:46:13.490603+0000","last_peered":"2023-09-23T17:46:13.490603+0000","last_clean":"2023-09-23T17:46:13.490603+0000","last_became_active":"2023-09-23T17:45:52.379149+0000","last_became_peered":"2023-09-23T17:45:52.379149+0000","last_unstale":"2023-09-23T17:46:13.490603+0000","last_undegraded":"2023-09-23T17:46:13.490603+0000","last_fullsized":"2023-09-23T17:46:13.490603+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:23.171926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":14,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.272989+0000","last_change":"2023-09-23T17:45:51.133941+0000","last_active":"2023-09-23T17:46:13.272989+0000","last_peered":"2023-09-23T17:46:13.272989+0000","last_clean":"2023-09-23T17:46:13.272989+0000","last_became_active":"2023-09-23T17:45:51.133450+0000","last_became_peered":"2023-09-23T17:45:51.133450+0000","last_unstale":"2023-09-23T17:46:13.272989+0000","last_undegraded":"2023-09-23T17:46:13.272989+0000","last_fullsized":"2023-09-23T17:46:13.272989+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:45.093055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":13,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.277166+0000","last_change":"2023-09-23T17:45:52.158131+0000","last_active":"2023-09-23T17:46:13.277166+0000","last_peered":"2023-09-23T17:46:13.277166+0000","last_clean":"2023-09-23T17:46:13.277166+0000","last_became_active":"2023-09-23T17:45:52.157837+0000","last_became_peered":"2023-09-23T17:45:52.157837+0000","last_unstale":"2023-09-23T17:46:13.277166+0000","last_undegraded":"2023-09-23T17:46:13.277166+0000","last_fullsized":"2023-09-23T17:46:13.277166+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":66,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:31.428198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":15,"reported_epoch":68,"state":"active+clean","last_fresh":"2023-09-23T17:46:13.273484+0000","last_change":"2023-09-23T17:45:51.128538+0000","last_active":"2023-09-23T17:46:13.273484+0000","last_peered":"2023-09-23T17:46:13.273484+0000","last_clean":"2023-09-23T17:46:13.273484+0000","last_became_active":"2023-09-23T17:45:51.128253+0000","last_became_peered":"2023-09-23T17:45:51.128253+0000","last_unstale":"2023-09-23T17:46:13.273484+0000","last_undegraded":"2023-09-23T17:46:13.273484+0000","last_fullsized":"2023-09-23T17:46:13.273484+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:29:07.870661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266712,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6612,"kb_used_data":1292,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738540,"statfs":{"total":95995035648,"available":95988264960,"internally_reserved":0,"allocated":1323008,"data_stored":900662,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":2.8319999999999999,"5min":2.8319999999999999,"15min":2.8319999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.49199999999999999,"15min":0.49199999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.497}]},{"osd":1,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.51100000000000001,"15min":0.51100000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":1.2569999999999999,"5min":1.2569999999999999,"15min":1.2569999999999999},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"last":0.52900000000000003},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.55700000000000005}]},{"osd":3,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.65200000000000002},{"interface":"front","average":{"1min":0.63700000000000001,"5min":0.63700000000000001,"15min":0.63700000000000001},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":2.73,"5min":2.73,"15min":2.73},"last":0.61399999999999999}]},{"osd":4,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.41399999999999998,"15min":0.41399999999999998},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.57799999999999996},{"interface":"front","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.59499999999999997}]},{"osd":5,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.50800000000000001,"15min":0.50800000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.628},{"interface":"front","average":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.746,"5min":0.746,"15min":0.746},"last":0.68899999999999995}]}]},{"osd":6,"up_from":45,"seq":193273528348,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6664,"kb_used_data":1280,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93738488,"statfs":{"total":95995035648,"available":95988211712,"internally_reserved":0,"allocated":1310720,"data_stored":900516,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"last":0.53800000000000003},{"interface":"front","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.50800000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.84499999999999997,"5min":0.84499999999999997,"15min":0.84499999999999997},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.78700000000000003,"5min":0.78700000000000003,"15min":0.78700000000000003},"last":0.47199999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.80100000000000005,"5min":0.80100000000000005,"15min":0.80100000000000005},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.51900000000000002,"15min":0.51900000000000002},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"last":3.9780000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.56299999999999994,"15min":0.56299999999999994},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.0589999999999999,"5min":1.0589999999999999,"15min":1.0589999999999999},"last":3.9500000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.52000000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.443,"5min":0.443,"15min":0.443},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"last":0.36499999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:45:23 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":0.49099999999999999,"15min":0.49099999999999999},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.68200000000000005,"5min":0.68200000000000005,"15min":0.68200000000000005},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.74399999999999999,"5min":0.74399999999999999,"15min":0.74399999999999999},"last":0.44600000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:45:44 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.41099999999999998},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.42799999999999999}]}]},{"osd":1,"up_from":13,"seq":55834574895,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6776,"kb_used_data":688,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738376,"statfs":{"total":95995035648,"available":95988097024,"internally_reserved":0,"allocated":704512,"data_stored":308865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:46 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.379,"15min":0.36399999999999999},"min":{"1min":0.28599999999999998,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.58899999999999997,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.318},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38300000000000001,"15min":0.36499999999999999},"min":{"1min":0.248,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":0.52100000000000002,"5min":0.66500000000000004,"15min":0.66500000000000004},"last":0.28699999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.46899999999999997,"5min":0.441,"15min":0.42399999999999999},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":2.4260000000000002,"5min":2.4260000000000002,"15min":2.4260000000000002},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.47199999999999998,"15min":0.442},"min":{"1min":0.219,"5min":0.219,"15min":0.219},"max":{"1min":5.1070000000000002,"5min":5.1070000000000002,"15min":5.1070000000000002},"last":0.30399999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.51800000000000002,"15min":0.52200000000000002},"min":{"1min":0.311,"5min":0.311,"15min":0.311},"max":{"1min":0.69899999999999995,"5min":0.871,"15min":0.871},"last":0.27000000000000002},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.496,"15min":0.499},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":0.64600000000000002,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.36899999999999999}]},{"osd":4,"last update":"Sat Sep 23 17:45:46 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.65400000000000003,"15min":0.66300000000000003},"min":{"1min":0.44700000000000001,"5min":0.442,"15min":0.442},"max":{"1min":0.81399999999999995,"5min":0.99099999999999999,"15min":0.99099999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.75,"5min":0.68799999999999994,"15min":0.67800000000000005},"min":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"max":{"1min":3.548,"5min":3.548,"15min":3.548},"last":0.58499999999999996}]},{"osd":5,"last update":"Sat Sep 23 17:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.79600000000000004,"5min":0.65700000000000003,"15min":0.63400000000000001},"min":{"1min":0.34399999999999997,"5min":0.34399999999999997,"15min":0.34399999999999997},"max":{"1min":3.5819999999999999,"5min":3.5819999999999999,"15min":3.5819999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.62,"15min":0.61699999999999999},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":2.395,"5min":2.395,"15min":2.395},"last":0.63200000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:45:25 2023","interfaces":[{"interface":"back","average":{"1min":0.67900000000000005,"5min":0.67900000000000005,"15min":0.67900000000000005},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":1.036,"5min":1.036,"15min":1.036},"last":0.61199999999999999},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.65000000000000002,"15min":0.65000000000000002},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.66200000000000003}]},{"osd":7,"last update":"Sat Sep 23 17:45:43 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.79000000000000004,"15min":0.79000000000000004},"min":{"1min":0.495,"5min":0.495,"15min":0.495},"max":{"1min":3.6179999999999999,"5min":3.6179999999999999,"15min":3.6179999999999999},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.56200000000000006}]}]},{"osd":0,"up_from":8,"seq":34359738419,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7428,"kb_used_data":1276,"kb_used_omap":0,"kb_used_meta":6144,"kb_avail":93737724,"statfs":{"total":95995035648,"available":95987429376,"internally_reserved":0,"allocated":1306624,"data_stored":899809,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6291456},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:45:55 2023","interfaces":[{"interface":"back","average":{"1min":0.48499999999999999,"5min":0.40699999999999997,"15min":0.38700000000000001},"min":{"1min":0.222,"5min":0.222,"15min":0.222},"max":{"1min":2.1379999999999999,"5min":2.1379999999999999,"15min":2.1379999999999999},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.38700000000000001,"5min":0.40699999999999997,"15min":0.40999999999999998},"min":{"1min":0.26900000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.54500000000000004,"5min":0.74099999999999999,"15min":0.74099999999999999},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.40100000000000002,"5min":0.42399999999999999,"15min":0.43099999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":0.56200000000000006,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.316},{"interface":"front","average":{"1min":0.39600000000000002,"5min":0.433,"15min":0.436},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.55500000000000005,"5min":0.61599999999999999,"15min":0.61599999999999999},"last":0.432}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.434,"5min":0.49399999999999999,"15min":0.504},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":0.60299999999999998,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.47899999999999998,"5min":0.51900000000000002,"15min":0.52600000000000002},"min":{"1min":0.37,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.64700000000000002,"5min":0.68899999999999995,"15min":0.68899999999999995},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:45:46 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.63,"15min":0.63900000000000001},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.80700000000000005,"5min":0.81299999999999994,"15min":0.81299999999999994},"last":0.67800000000000005},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.2090000000000001,"5min":2.2090000000000001,"15min":2.2090000000000001},"last":0.64400000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:46:06 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.60099999999999998,"15min":0.59599999999999997},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":2.181,"5min":2.181,"15min":2.181},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.67400000000000004,"5min":0.64600000000000002,"15min":0.64200000000000002},"min":{"1min":0.44800000000000001,"5min":0.44800000000000001,"15min":0.44800000000000001},"max":{"1min":2.1150000000000002,"5min":2.1150000000000002,"15min":2.1150000000000002},"last":0.65300000000000002}]},{"osd":6,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.65100000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"min":{"1min":0.45000000000000001,"5min":0.45000000000000001,"15min":0.45000000000000001},"max":{"1min":0.84899999999999998,"5min":0.84899999999999998,"15min":0.84899999999999998},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"max":{"1min":0.76600000000000001,"5min":0.76600000000000001,"15min":0.76600000000000001},"last":0.71299999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:45:44 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"min":{"1min":0.45600000000000002,"5min":0.45600000000000002,"15min":0.45600000000000002},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.628},{"interface":"front","average":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"min":{"1min":0.42499999999999999,"5min":0.42499999999999999,"15min":0.42499999999999999},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.57899999999999996}]}]},{"osd":2,"up_from":18,"seq":77309411371,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6656,"kb_used_data":696,"kb_used_omap":0,"kb_used_meta":5952,"kb_avail":93738496,"statfs":{"total":95995035648,"available":95988219904,"internally_reserved":0,"allocated":712704,"data_stored":309453,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6094848},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.42499999999999999,"5min":0.40000000000000002,"15min":0.39800000000000002},"min":{"1min":0.27100000000000002,"5min":0.249,"15min":0.249},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.24299999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.42999999999999999,"15min":0.42599999999999999},"min":{"1min":0.27900000000000003,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.67700000000000005,"5min":0.67700000000000005,"15min":0.67700000000000005},"last":0.26800000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.432,"15min":0.42799999999999999},"min":{"1min":0.26600000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.98199999999999998,"5min":0.98199999999999998,"15min":0.98199999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.436,"5min":0.45500000000000002,"15min":0.45700000000000002},"min":{"1min":0.26800000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.70999999999999996,"5min":0.70999999999999996,"15min":0.70999999999999996},"last":0.308}]},{"osd":3,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.53600000000000003,"15min":0.54500000000000004},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.79100000000000004,"5min":0.79100000000000004,"15min":0.79100000000000004},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.52200000000000002,"15min":0.53300000000000003},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.72899999999999998,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.33600000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:45:51 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.625,"15min":0.61399999999999999},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":2.1320000000000001,"5min":2.1320000000000001,"15min":2.1320000000000001},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.64900000000000002,"15min":0.63900000000000001},"min":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"max":{"1min":2.1789999999999998,"5min":2.1789999999999998,"15min":2.1789999999999998},"last":0.55700000000000005}]},{"osd":5,"last update":"Sat Sep 23 17:46:11 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.63300000000000001,"15min":0.623},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":2.1480000000000001,"5min":2.1480000000000001,"15min":2.1480000000000001},"last":0.498},{"interface":"front","average":{"1min":0.76100000000000001,"5min":0.67000000000000004,"15min":0.65500000000000003},"min":{"1min":0.442,"5min":0.434,"15min":0.434},"max":{"1min":2.3279999999999998,"5min":2.3279999999999998,"15min":2.3279999999999998},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.94999999999999996,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.45200000000000001,"5min":0.45200000000000001,"15min":0.45200000000000001},"max":{"1min":1.0780000000000001,"5min":1.0780000000000001,"15min":1.0780000000000001},"last":0.68000000000000005}]},{"osd":7,"last update":"Sat Sep 23 17:45:44 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.73199999999999998,"15min":0.73199999999999998},"min":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"max":{"1min":2.1080000000000001,"5min":2.1080000000000001,"15min":2.1080000000000001},"last":0.624},{"interface":"front","average":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"min":{"1min":0.48499999999999999,"5min":0.48499999999999999,"15min":0.48499999999999999},"max":{"1min":2.2949999999999999,"5min":2.2949999999999999,"15min":2.2949999999999999},"last":0.65500000000000003}]}]},{"osd":3,"up_from":25,"seq":107374182439,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6140,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739012,"statfs":{"total":95995035648,"available":95988748288,"internally_reserved":0,"allocated":708608,"data_stored":308955,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.35499999999999998,"5min":0.40600000000000003,"15min":0.41399999999999998},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":0.58499999999999996,"5min":0.63500000000000001,"15min":0.63500000000000001},"last":0.312},{"interface":"front","average":{"1min":0.40699999999999997,"5min":0.42199999999999999,"15min":0.42399999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":0.58799999999999997,"5min":0.58799999999999997,"15min":0.58799999999999997},"last":0.32700000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.40400000000000003,"5min":0.43099999999999999,"15min":0.435},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.44400000000000001,"5min":0.45200000000000001,"15min":0.45400000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.26300000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:45:33 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.48299999999999998,"15min":0.48799999999999999},"min":{"1min":0.34100000000000003,"5min":0.34100000000000003,"15min":0.34100000000000003},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.441,"5min":0.47299999999999998,"15min":0.47899999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.64000000000000001,"5min":0.64800000000000002,"15min":0.64800000000000002},"last":0.54600000000000004}]},{"osd":4,"last update":"Sat Sep 23 17:45:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53400000000000003,"5min":0.59699999999999998,"15min":0.60699999999999998},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.76700000000000002,"5min":0.78900000000000003,"15min":0.78900000000000003},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.56499999999999995,"5min":0.77700000000000002,"15min":0.81299999999999994},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.83699999999999997,"5min":5.2069999999999999,"15min":5.2069999999999999},"last":0.44400000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:46:07 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.59899999999999998,"15min":0.60899999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.78200000000000003,"5min":0.85899999999999999,"15min":0.85899999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.64700000000000002,"15min":0.65700000000000003},"min":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"max":{"1min":0.879,"5min":0.879,"15min":0.879},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":0.63200000000000001,"15min":0.63200000000000001},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.88800000000000001,"5min":0.88800000000000001,"15min":0.88800000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":0.66300000000000003,"5min":0.66300000000000003,"15min":0.66300000000000003},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.63100000000000001,"5min":0.63100000000000001,"15min":0.63100000000000001},"min":{"1min":0.49299999999999999,"5min":0.49299999999999999,"15min":0.49299999999999999},"max":{"1min":0.872,"5min":0.872,"15min":0.872},"last":0.498}]}]},{"osd":4,"up_from":30,"seq":128849018913,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6224,"kb_used_data":712,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738928,"statfs":{"total":95995035648,"available":95988662272,"internally_reserved":0,"allocated":729088,"data_stored":310547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.54900000000000004,"15min":0.54500000000000004},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.3759999999999999,"5min":1.3759999999999999,"15min":1.3759999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.503,"5min":0.53200000000000003,"15min":0.53700000000000003},"min":{"1min":0.314,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":1.2270000000000001,"5min":1.2270000000000001,"15min":1.2270000000000001},"last":0.39700000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.53600000000000003,"15min":0.54000000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":1.248,"5min":1.248,"15min":1.248},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53200000000000003,"15min":0.53100000000000003},"min":{"1min":0.36099999999999999,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":1.333,"5min":1.333,"15min":1.333},"last":0.29499999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.51100000000000001,"5min":0.58099999999999996,"15min":0.59299999999999997},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":1.3029999999999999,"5min":1.5129999999999999,"15min":1.5129999999999999},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.55900000000000005,"15min":0.57299999999999995},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.70899999999999996,"5min":1.073,"15min":1.073},"last":0.36799999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":0.56999999999999995,"15min":0.57399999999999995},"min":{"1min":0.38200000000000001,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.393,"5min":1.393,"15min":1.393},"last":0.318},{"interface":"front","average":{"1min":0.56200000000000006,"5min":0.54500000000000004,"15min":0.54200000000000004},"min":{"1min":0.41699999999999998,"5min":0.38,"15min":0.38},"max":{"1min":1.2030000000000001,"5min":1.2030000000000001,"15min":1.2030000000000001},"last":0.38600000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.40000000000000002,"15min":0.39200000000000002},"min":{"1min":0.30299999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":1.141,"5min":1.141,"15min":1.141},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.38300000000000001,"15min":0.377},"min":{"1min":0.315,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.27900000000000003}]},{"osd":6,"last update":"Sat Sep 23 17:45:27 2023","interfaces":[{"interface":"back","average":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.497,"5min":0.497,"15min":0.497},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.47999999999999998}]},{"osd":7,"last update":"Sat Sep 23 17:45:48 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54600000000000004,"15min":0.54600000000000004},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":1.1839999999999999,"5min":1.1839999999999999,"15min":1.1839999999999999},"last":0.442},{"interface":"front","average":{"1min":0.501,"5min":0.501,"15min":0.501},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.42199999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757279,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6076,"kb_used_data":692,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739076,"statfs":{"total":95995035648,"available":95988813824,"internally_reserved":0,"allocated":708608,"data_stored":309166,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.56699999999999995,"15min":0.58299999999999996},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.78200000000000003,"5min":1.5960000000000001,"15min":1.5960000000000001},"last":0.66900000000000004},{"interface":"front","average":{"1min":0.48899999999999999,"5min":0.53700000000000003,"15min":0.54500000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":0.48899999999999999}]},{"osd":1,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.42299999999999999,"5min":0.54400000000000004,"15min":0.56399999999999995},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.59699999999999998,"5min":1.5680000000000001,"15min":1.5680000000000001},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.44600000000000001,"5min":0.51000000000000001,"15min":0.52000000000000002},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":0.61299999999999999,"5min":0.75600000000000001,"15min":0.75600000000000001},"last":0.65400000000000003}]},{"osd":2,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.58799999999999997,"15min":0.60099999999999998},"min":{"1min":0.40200000000000002,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":0.94199999999999995,"5min":1.581,"15min":1.581},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.64600000000000002,"5min":0.61399999999999999,"15min":0.60799999999999998},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":4.5739999999999998,"5min":4.5739999999999998,"15min":4.5739999999999998},"last":0.505}]},{"osd":3,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.59199999999999997,"15min":0.57799999999999996},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":4.6799999999999997,"5min":4.6799999999999997,"15min":4.6799999999999997},"last":0.496},{"interface":"front","average":{"1min":0.47099999999999997,"5min":0.56499999999999995,"15min":0.58099999999999996},"min":{"1min":0.35999999999999999,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.63400000000000001,"5min":1.6279999999999999,"15min":1.6279999999999999},"last":0.59599999999999997}]},{"osd":4,"last update":"Sat Sep 23 17:46:08 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.51600000000000001},{"interface":"front","average":{"1min":0.49099999999999999,"5min":0.502,"15min":0.504},"min":{"1min":0.30499999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.65400000000000003,"5min":1.738,"15min":1.738},"last":0.36899999999999999}]},{"osd":6,"last update":"Sat Sep 23 17:45:24 2023","interfaces":[{"interface":"back","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.778,"5min":1.778,"15min":1.778},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.59499999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.7629999999999999,"5min":1.7629999999999999,"15min":1.7629999999999999},"last":0.38700000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:45:45 2023","interfaces":[{"interface":"back","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.44600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:46:27.754 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T17:46:27.754 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T17:46:27.755 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T17:46:27.755 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T17:46:28.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:28 smithi099 ceph-mon[100270]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:28.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:28 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:28.418 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:28.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:28 smithi055 ceph-mon[95848]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:28 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:28.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:28 smithi055 ceph-mon[109912]: pgmap v49: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:28.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:28 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:29 smithi055 ceph-mon[95848]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:46:29.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:29 smithi055 ceph-mon[109912]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:46:29.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:29 smithi099 ceph-mon[100270]: from='client.14790 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:46:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:46:29.749 INFO:teuthology.orchestra.run.smithi055.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T17:46:30.178 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T17:46:30.178 INFO:teuthology.run_tasks:Running task print... 2023-09-23T17:46:30.188 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-09-23T17:46:30.188 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T17:46:30.230 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T17:46:30.230 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T17:46:30.231 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T17:46:30.235 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T17:46:30.235 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T17:46:30.235 DEBUG:teuthology.orchestra.run.smithi055:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T17:46:30.236 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T17:46:30.236 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-09-23T17:46:30.236 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi055.front.sepia.ceph.com 2023-09-23T17:46:30.236 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-09-23T17:46:30.254 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T17:46:30.255 INFO:teuthology.orchestra.run.smithi055.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T17:46:30.255 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T17:46:30.314 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T17:46:30.314 DEBUG:teuthology.orchestra.run.smithi055:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T17:46:30.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:30 smithi099 ceph-mon[100270]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:30.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:30 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2244061751' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T17:46:30.373 INFO:tasks.workunit:timeout=3h 2023-09-23T17:46:30.373 INFO:tasks.workunit:cleanup=True 2023-09-23T17:46:30.373 DEBUG:teuthology.orchestra.run.smithi055:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T17:46:30.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:30 smithi055 ceph-mon[109912]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:30.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:30 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2244061751' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T17:46:30.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:30 smithi055 ceph-mon[95848]: pgmap v50: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:30.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:30 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2244061751' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T17:46:30.431 INFO:tasks.workunit.client.0.smithi055.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T17:46:30.954 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:31.154 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:46:30] "GET /metrics HTTP/1.1" 200 238769 "" "Prometheus/2.43.0" 2023-09-23T17:46:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:31 smithi099 ceph-mon[100270]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:31 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:31 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:31 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:31 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.928 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:31 smithi055 ceph-mon[109912]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:31.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:31 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:31 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:31 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:31 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:31 smithi055 ceph-mon[95848]: pgmap v51: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:31.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:31 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:31 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:31 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:31.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:31 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:32.179 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:46:32.065Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001493256s 2023-09-23T17:46:32.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:32.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:32.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:32.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:32.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3625128642' entity='client.admin' 2023-09-23T17:46:32.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:32] ENGINE Bus STOPPING 2023-09-23T17:46:32.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:32] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T17:46:32.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:32] ENGINE Bus STOPPED 2023-09-23T17:46:32.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:32] ENGINE Bus STARTING 2023-09-23T17:46:32.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:32] ENGINE Serving on http://:::9283 2023-09-23T17:46:32.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:46:32] ENGINE Bus STARTED 2023-09-23T17:46:32.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:32.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:32.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:32.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:32.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:32 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3625128642' entity='client.admin' 2023-09-23T17:46:32.805 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph config set global log_to_journald false --force' 2023-09-23T17:46:32.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:32 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:32.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:32 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:32 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:32 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:32 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3625128642' entity='client.admin' 2023-09-23T17:46:33.529 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:34.150 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:33 smithi055 ceph-mon[95848]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:34.151 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:33 smithi055 ceph-mon[109912]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:34.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:33 smithi099 ceph-mon[100270]: pgmap v52: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:34.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:46:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:46:34] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:46:34.992 DEBUG:teuthology.orchestra.run.smithi055:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T17:46:35.030 INFO:tasks.workunit.client.0.smithi055.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T17:46:35.032 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T17:46:35.032 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T17:46:35.777 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T17:46:35.902 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T17:46:35.950 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T17:46:35.952 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T17:46:35.952 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T17:46:35.960 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:35 smithi055 ceph-mon[95848]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:35.960 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:35 smithi055 ceph-mon[109912]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:35.993 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T17:46:35.997 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T17:46:35.997 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T17:46:36.056 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T17:46:36.058 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T17:46:36.058 DEBUG:teuthology.orchestra.run.smithi055:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T17:46:36.119 INFO:tasks.workunit.client.0.smithi055.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T17:46:36.119 INFO:tasks.workunit.client.0.smithi055.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T17:46:36.119 INFO:tasks.workunit.client.0.smithi055.stderr:+ TMPDIR=/tmp/rbd_import_export_159972 2023-09-23T17:46:36.119 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm -rf /tmp/rbd_import_export_159972 2023-09-23T17:46:36.120 INFO:tasks.workunit.client.0.smithi055.stderr:+ mkdir /tmp/rbd_import_export_159972 2023-09-23T17:46:36.121 INFO:tasks.workunit.client.0.smithi055.stderr:+ trap 'rm -rf /tmp/rbd_import_export_159972' INT TERM EXIT 2023-09-23T17:46:36.121 INFO:tasks.workunit.client.0.smithi055.stderr:+ mkdir foo.159972 2023-09-23T17:46:36.121 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import foo.159972 foo.dir 2023-09-23T17:46:36.169 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: cannot import a directory 2023-09-23T17:46:36.169 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 0% complete...failed. 2023-09-23T17:46:36.170 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: import failed: (21) Is a directory 2023-09-23T17:46:36.172 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T17:46:36.172 INFO:tasks.workunit.client.0.smithi055.stderr:+ rmdir foo.159972 2023-09-23T17:46:36.186 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_159972/img bs=1k count=1 seek=10 2023-09-23T17:46:36.187 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records in 2023-09-23T17:46:36.187 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records out 2023-09-23T17:46:36.188 INFO:tasks.workunit.client.0.smithi055.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.3233e-05 s, 23.7 MB/s 2023-09-23T17:46:36.188 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_159972/img bs=1k count=10 seek=100 2023-09-23T17:46:36.188 INFO:tasks.workunit.client.0.smithi055.stderr:10+0 records in 2023-09-23T17:46:36.188 INFO:tasks.workunit.client.0.smithi055.stderr:10+0 records out 2023-09-23T17:46:36.188 INFO:tasks.workunit.client.0.smithi055.stderr:10240 bytes (10 kB, 10 KiB) copied, 6.1728e-05 s, 166 MB/s 2023-09-23T17:46:36.188 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_159972/img bs=1k count=100 seek=1000 2023-09-23T17:46:36.189 INFO:tasks.workunit.client.0.smithi055.stderr:70+1 records in 2023-09-23T17:46:36.189 INFO:tasks.workunit.client.0.smithi055.stderr:70+1 records out 2023-09-23T17:46:36.189 INFO:tasks.workunit.client.0.smithi055.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000197575 s, 365 MB/s 2023-09-23T17:46:36.189 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_159972/img bs=1k seek=10000 2023-09-23T17:46:36.190 INFO:tasks.workunit.client.0.smithi055.stderr:139+1 records in 2023-09-23T17:46:36.190 INFO:tasks.workunit.client.0.smithi055.stderr:139+1 records out 2023-09-23T17:46:36.190 INFO:tasks.workunit.client.0.smithi055.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.000354006 s, 405 MB/s 2023-09-23T17:46:36.190 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_159972/img bs=1k seek=100000 2023-09-23T17:46:36.211 INFO:tasks.workunit.client.0.smithi055.stderr:70+1 records in 2023-09-23T17:46:36.211 INFO:tasks.workunit.client.0.smithi055.stderr:70+1 records out 2023-09-23T17:46:36.211 INFO:tasks.workunit.client.0.smithi055.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.0204193 s, 3.5 MB/s 2023-09-23T17:46:36.211 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_159972/img bs=1k seek=1000000 2023-09-23T17:46:36.212 INFO:tasks.workunit.client.0.smithi055.stderr:189+1 records in 2023-09-23T17:46:36.212 INFO:tasks.workunit.client.0.smithi055.stderr:189+1 records out 2023-09-23T17:46:36.212 INFO:tasks.workunit.client.0.smithi055.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000473921 s, 409 MB/s 2023-09-23T17:46:36.212 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm testimg 2023-09-23T17:46:36.248 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 0% complete...failed. 2023-09-23T17:46:36.249 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T17:46:36.253 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T17:46:36.253 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format /tmp/rbd_import_export_159972/img testimg 2023-09-23T17:46:36.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:35 smithi099 ceph-mon[100270]: pgmap v53: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:36.359 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-09-23T17:46:37.073 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:37.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:37 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:37 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:37 smithi099 ceph-mon[100270]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:37 smithi099 ceph-mon[100270]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T17:46:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:37 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:37 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:37 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:37 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:37 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.621 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.621 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.621 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[95848]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:37.621 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[95848]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T17:46:37.621 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.621 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.621 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:37.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:37.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[109912]: pgmap v54: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:46:37.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[109912]: osdmap e70: 8 total, 8 up, 8 in 2023-09-23T17:46:37.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:37.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:37.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:37.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:37 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:38 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:38 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:38 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:38 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:38 smithi055 ceph-mon[109912]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T17:46:38.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:38 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:38.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:38 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:38 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:38 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:38 smithi055 ceph-mon[95848]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T17:46:38.490 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T17:46:38.490 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T17:46:38.493 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg /tmp/rbd_import_export_159972/img2 2023-09-23T17:46:38.540 INFO:teuthology.orchestra.run.smithi055.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:46:38.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:38 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:38.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:38 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:38.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:38 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:38.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:38 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:38.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:38 smithi099 ceph-mon[100270]: osdmap e71: 8 total, 8 up, 8 in 2023-09-23T17:46:38.782 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:38.785 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg - 2023-09-23T17:46:38.936 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T17:46:39.542 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:39.545 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm testimg 2023-09-23T17:46:39.601 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:39.647 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T17:46:39.650 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp /tmp/rbd_import_export_159972/img /tmp/rbd_import_export_159972/img2 2023-09-23T17:46:39.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[109912]: pgmap v57: 73 pgs: 73 active+clean; 5.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 39 KiB/s wr, 9 op/s 2023-09-23T17:46:39.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[109912]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:46:39.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[109912]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:46:39.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:39.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:39.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:39.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:39.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:39.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[109912]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:46:39.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[95848]: pgmap v57: 73 pgs: 73 active+clean; 5.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 39 KiB/s wr, 9 op/s 2023-09-23T17:46:39.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[95848]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:46:39.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[95848]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:46:39.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:39.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:39.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:39.806 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:39.806 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:39.806 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:39 smithi055 ceph-mon[95848]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:46:39.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:39 smithi099 ceph-mon[100270]: pgmap v57: 73 pgs: 73 active+clean; 5.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 5.2 KiB/s rd, 39 KiB/s wr, 9 op/s 2023-09-23T17:46:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:39 smithi099 ceph-mon[100270]: from='client.24713 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:46:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:39 smithi099 ceph-mon[100270]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:46:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:39 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:39 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:46:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:39 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:46:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:39 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:46:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:39 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:46:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:39 smithi099 ceph-mon[100270]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:bb0daeff97f020f686596fca6eba20c310e26fb7 2023-09-23T17:46:40.390 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp /tmp/rbd_import_export_159972/img /tmp/rbd_import_export_159972/img3 2023-09-23T17:46:40.954 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/img2 /tmp/rbd_import_export_159972/img3 2023-09-23T17:46:41.163 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format - testimg 2023-09-23T17:46:41.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:46:40] "GET /metrics HTTP/1.1" 200 238769 "" "Prometheus/2.43.0" 2023-09-23T17:46:41.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:40 smithi055 ceph-mon[95848]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T17:46:41.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:40 smithi055 ceph-mon[109912]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T17:46:41.275 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (19s) 4s ago 77s 18.5M - 0.25.0 c8568f914cd2 2593820a0ba0 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi099 *:3000 running (58s) 4s ago 58s 88.2M - 9.4.7 2c41d148cca3 59417245e1d8 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi055.exjftp smithi055 running (42s) 4s ago 42s 15.4M - 17.2.6-1123-g61431f6b 0db14aa01094 7a2127020d8b 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi099.djovrl smithi099 running (40s) 4s ago 40s 15.9M - 17.2.6-1123-g61431f6b 0db14aa01094 5993fe8cebce 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi099 *:8443 running (4m) 4s ago 4m 439M - 17.2.6-1123-g61431f6b 0db14aa01094 6d7959b6f2fc 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:9283 running (6m) 4s ago 6m 495M - 17.2.6-1123-g61431f6b 0db14aa01094 61647c07eed6 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (6m) 4s ago 6m 76.7M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 3ea34fb03ea8 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi099 running (5m) 4s ago 5m 55.0M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 5f0fb2e87ea1 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (4m) 4s ago 4m 55.4M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 549a518b554d 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (54s) 4s ago 53s 15.8M - 1.5.0 0da6a335fe13 d17a99f5c12e 2023-09-23T17:46:41.276 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi099 *:9100 running (50s) 4s ago 49s 3871k - 1.5.0 0da6a335fe13 1dcf070a4b25 2023-09-23T17:46:41.277 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (4m) 4s ago 4m 59.0M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 15403d138a93 2023-09-23T17:46:41.277 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (4m) 4s ago 4m 59.4M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 ee80d8f5fddd 2023-09-23T17:46:41.277 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (3m) 4s ago 3m 52.5M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 7045fb31bd32 2023-09-23T17:46:41.277 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (3m) 4s ago 3m 53.3M 4037M 17.2.6-1123-g61431f6b 0db14aa01094 33f748e0cdcf 2023-09-23T17:46:41.277 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi099 running (3m) 4s ago 3m 50.3M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 ecb0e820fd90 2023-09-23T17:46:41.277 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi099 running (2m) 4s ago 2m 51.6M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 7e6bf34cfdb0 2023-09-23T17:46:41.277 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi099 running (2m) 4s ago 2m 52.1M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 827ec31827bb 2023-09-23T17:46:41.277 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi099 running (2m) 4s ago 2m 51.4M 4036M 17.2.6-1123-g61431f6b 0db14aa01094 dcef78fdede8 2023-09-23T17:46:41.277 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi099 *:9095 running (15s) 4s ago 98s 31.6M - 2.43.0 a07b618ecd1d 3bad347644a3 2023-09-23T17:46:41.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:40 smithi099 ceph-mon[100270]: osdmap e72: 8 total, 8 up, 8 in 2023-09-23T17:46:41.588 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T17:46:41.589 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T17:46:41.592 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg /tmp/rbd_import_export_159972/img2 2023-09-23T17:46:41.698 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for minority of mons to upgrade' 2023-09-23T17:46:41.928 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:41.932 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg - 2023-09-23T17:46:42.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:41 smithi099 ceph-mon[100270]: pgmap v59: 73 pgs: 73 active+clean; 5.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 52 KiB/s wr, 12 op/s 2023-09-23T17:46:42.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:41 smithi099 ceph-mon[100270]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:46:42.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:41 smithi055 ceph-mon[109912]: pgmap v59: 73 pgs: 73 active+clean; 5.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 52 KiB/s wr, 12 op/s 2023-09-23T17:46:42.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:41 smithi055 ceph-mon[109912]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:46:42.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:41 smithi055 ceph-mon[95848]: pgmap v59: 73 pgs: 73 active+clean; 5.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 52 KiB/s wr, 12 op/s 2023-09-23T17:46:42.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:41 smithi055 ceph-mon[95848]: from='client.24737 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:46:42.450 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:42.752 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:42.756 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm testimg 2023-09-23T17:46:42.849 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T17:46:42.852 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp /tmp/rbd_import_export_159972/img /tmp/rbd_import_export_159972/img2 2023-09-23T17:46:43.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:42 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:43.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:42 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:43.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:42 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:43.577 INFO:teuthology.orchestra.run.smithi055.stdout:wait for minority of mons to upgrade 2023-09-23T17:46:43.607 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp /tmp/rbd_import_export_159972/img /tmp/rbd_import_export_159972/img3 2023-09-23T17:46:43.922 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-09-23T17:46:44.171 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/img /tmp/rbd_import_export_159972/img2 /tmp/rbd_import_export_159972/img3 2023-09-23T17:46:44.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:43 smithi055 ceph-mon[109912]: pgmap v60: 73 pgs: 73 active+clean; 5.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 52 KiB/s wr, 12 op/s 2023-09-23T17:46:44.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:43 smithi055 ceph-mon[95848]: pgmap v60: 73 pgs: 73 active+clean; 5.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 52 KiB/s wr, 12 op/s 2023-09-23T17:46:44.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:43 smithi099 ceph-mon[100270]: pgmap v60: 73 pgs: 73 active+clean; 5.2 MiB data, 52 MiB used, 715 GiB / 715 GiB avail; 7.0 KiB/s rd, 52 KiB/s wr, 12 op/s 2023-09-23T17:46:44.438 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd help export 2023-09-23T17:46:44.438 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -q export-format 2023-09-23T17:46:44.469 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_159972/img bs=1k count=10 seek=100 2023-09-23T17:46:44.469 INFO:tasks.workunit.client.0.smithi055.stderr:10+0 records in 2023-09-23T17:46:44.470 INFO:tasks.workunit.client.0.smithi055.stderr:10+0 records out 2023-09-23T17:46:44.470 INFO:tasks.workunit.client.0.smithi055.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.0304e-05 s, 146 MB/s 2023-09-23T17:46:44.470 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format /tmp/rbd_import_export_159972/img testimg 2023-09-23T17:46:44.540 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T17:46:44.541 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T17:46:44.543 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap create testimg@snap 2023-09-23T17:46:44.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:46:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:46:44] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:46:44.632 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:46:45.004 INFO:tasks.workunit.client.0.smithi055.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T17:46:45.011 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T17:46:45.064 INFO:tasks.workunit.client.0.smithi055.stderr:++ rbd image-meta list testimg 2023-09-23T17:46:45.110 INFO:tasks.workunit.client.0.smithi055.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T17:46:45.110 INFO:tasks.workunit.client.0.smithi055.stderr: 2023-09-23T17:46:45.111 INFO:tasks.workunit.client.0.smithi055.stderr:Key Value 2023-09-23T17:46:45.111 INFO:tasks.workunit.client.0.smithi055.stderr:key1 value1' 2023-09-23T17:46:45.111 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_159972/img_v2 2023-09-23T17:46:45.155 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:45.158 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_159972/img_v2 testimg_import 2023-09-23T17:46:46.022 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T17:46:46.025 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import 2023-09-23T17:46:46.063 INFO:tasks.workunit.client.0.smithi055.stdout:rbd image 'testimg_import': 2023-09-23T17:46:46.064 INFO:tasks.workunit.client.0.smithi055.stdout: size 110 KiB in 1 objects 2023-09-23T17:46:46.064 INFO:tasks.workunit.client.0.smithi055.stdout: order 22 (4 MiB objects) 2023-09-23T17:46:46.064 INFO:tasks.workunit.client.0.smithi055.stdout: snapshot_count: 1 2023-09-23T17:46:46.064 INFO:tasks.workunit.client.0.smithi055.stdout: id: 3a441e0595fd 2023-09-23T17:46:46.064 INFO:tasks.workunit.client.0.smithi055.stdout: block_name_prefix: rbd_data.3a441e0595fd 2023-09-23T17:46:46.064 INFO:tasks.workunit.client.0.smithi055.stdout: format: 2 2023-09-23T17:46:46.064 INFO:tasks.workunit.client.0.smithi055.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T17:46:46.064 INFO:tasks.workunit.client.0.smithi055.stdout: op_features: 2023-09-23T17:46:46.064 INFO:tasks.workunit.client.0.smithi055.stdout: flags: 2023-09-23T17:46:46.064 INFO:tasks.workunit.client.0.smithi055.stdout: create_timestamp: Sat Sep 23 17:46:45 2023 2023-09-23T17:46:46.065 INFO:tasks.workunit.client.0.smithi055.stdout: access_timestamp: Sat Sep 23 17:46:45 2023 2023-09-23T17:46:46.065 INFO:tasks.workunit.client.0.smithi055.stdout: modify_timestamp: Sat Sep 23 17:46:45 2023 2023-09-23T17:46:46.067 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import@snap 2023-09-23T17:46:46.106 INFO:tasks.workunit.client.0.smithi055.stdout:rbd image 'testimg_import': 2023-09-23T17:46:46.106 INFO:tasks.workunit.client.0.smithi055.stdout: size 110 KiB in 1 objects 2023-09-23T17:46:46.106 INFO:tasks.workunit.client.0.smithi055.stdout: order 22 (4 MiB objects) 2023-09-23T17:46:46.106 INFO:tasks.workunit.client.0.smithi055.stdout: snapshot_count: 1 2023-09-23T17:46:46.106 INFO:tasks.workunit.client.0.smithi055.stdout: id: 3a441e0595fd 2023-09-23T17:46:46.107 INFO:tasks.workunit.client.0.smithi055.stdout: block_name_prefix: rbd_data.3a441e0595fd 2023-09-23T17:46:46.107 INFO:tasks.workunit.client.0.smithi055.stdout: format: 2 2023-09-23T17:46:46.107 INFO:tasks.workunit.client.0.smithi055.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T17:46:46.107 INFO:tasks.workunit.client.0.smithi055.stdout: op_features: 2023-09-23T17:46:46.107 INFO:tasks.workunit.client.0.smithi055.stdout: flags: 2023-09-23T17:46:46.107 INFO:tasks.workunit.client.0.smithi055.stdout: create_timestamp: Sat Sep 23 17:46:45 2023 2023-09-23T17:46:46.107 INFO:tasks.workunit.client.0.smithi055.stdout: access_timestamp: Sat Sep 23 17:46:45 2023 2023-09-23T17:46:46.107 INFO:tasks.workunit.client.0.smithi055.stdout: modify_timestamp: Sat Sep 23 17:46:45 2023 2023-09-23T17:46:46.107 INFO:tasks.workunit.client.0.smithi055.stdout: protected: False 2023-09-23T17:46:46.110 INFO:tasks.workunit.client.0.smithi055.stderr:++ rbd image-meta list testimg_import 2023-09-23T17:46:46.158 INFO:tasks.workunit.client.0.smithi055.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T17:46:46.158 INFO:tasks.workunit.client.0.smithi055.stderr: 2023-09-23T17:46:46.158 INFO:tasks.workunit.client.0.smithi055.stderr:Key Value 2023-09-23T17:46:46.158 INFO:tasks.workunit.client.0.smithi055.stderr:key1 value1' 2023-09-23T17:46:46.158 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T17:46:46.158 INFO:tasks.workunit.client.0.smithi055.stderr: 2023-09-23T17:46:46.159 INFO:tasks.workunit.client.0.smithi055.stderr:Key Value 2023-09-23T17:46:46.159 INFO:tasks.workunit.client.0.smithi055.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T17:46:46.159 INFO:tasks.workunit.client.0.smithi055.stderr: 2023-09-23T17:46:46.159 INFO:tasks.workunit.client.0.smithi055.stderr:Key Value 2023-09-23T17:46:46.159 INFO:tasks.workunit.client.0.smithi055.stderr:key1 value1' ']' 2023-09-23T17:46:46.159 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg_import /tmp/rbd_import_export_159972/img_import 2023-09-23T17:46:46.195 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete...done. 2023-09-23T17:46:46.198 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159972/img /tmp/rbd_import_export_159972/img_import 2023-09-23T17:46:46.198 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_159972/img /tmp/rbd_import_export_159972/img_import 2023-09-23T17:46:46.199 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/img --format %b 2023-09-23T17:46:46.200 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=24 2023-09-23T17:46:46.201 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/img_import --format %b 2023-09-23T17:46:46.202 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=224 2023-09-23T17:46:46.202 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=200 2023-09-23T17:46:46.202 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=200 2023-09-23T17:46:46.202 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T17:46:46.202 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_159972/img_snap 2023-09-23T17:46:46.239 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete...done. 2023-09-23T17:46:46.242 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_159972/img_snap_import 2023-09-23T17:46:46.286 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete...done. 2023-09-23T17:46:46.289 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159972/img_snap /tmp/rbd_import_export_159972/img_snap_import 2023-09-23T17:46:46.290 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_159972/img_snap /tmp/rbd_import_export_159972/img_snap_import 2023-09-23T17:46:46.290 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/img_snap --format %b 2023-09-23T17:46:46.291 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=224 2023-09-23T17:46:46.292 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/img_snap_import --format %b 2023-09-23T17:46:46.293 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=224 2023-09-23T17:46:46.293 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T17:46:46.293 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T17:46:46.293 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T17:46:46.293 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/img_v2 2023-09-23T17:46:46.294 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/img_import 2023-09-23T17:46:46.294 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/img_snap 2023-09-23T17:46:46.295 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/img_snap_import 2023-09-23T17:46:46.296 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T17:46:46.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:45 smithi099 ceph-mon[100270]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 343 KiB/s rd, 141 KiB/s wr, 109 op/s 2023-09-23T17:46:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:45 smithi099 ceph-mon[100270]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T17:46:46.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:45 smithi055 ceph-mon[109912]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 343 KiB/s rd, 141 KiB/s wr, 109 op/s 2023-09-23T17:46:46.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:45 smithi055 ceph-mon[109912]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T17:46:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:45 smithi055 ceph-mon[95848]: pgmap v61: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 343 KiB/s rd, 141 KiB/s wr, 109 op/s 2023-09-23T17:46:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:45 smithi055 ceph-mon[95848]: osdmap e73: 8 total, 8 up, 8 in 2023-09-23T17:46:47.030 INFO:tasks.workunit.client.0.smithi055.stderr: Removing snap: 100% complete...done. 2023-09-23T17:46:47.037 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg_import 2023-09-23T17:46:47.130 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T17:46:47.133 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap rm testimg@snap 2023-09-23T17:46:47.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:47 smithi099 ceph-mon[100270]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T17:46:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:47 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2244691985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:47.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:47 smithi055 ceph-mon[109912]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T17:46:47.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:47 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2244691985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:47 smithi055 ceph-mon[95848]: osdmap e74: 8 total, 8 up, 8 in 2023-09-23T17:46:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:47 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2244691985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:48.023 INFO:tasks.workunit.client.0.smithi055.stderr: Removing snap: 100% complete...done. 2023-09-23T17:46:48.030 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm testimg 2023-09-23T17:46:48.124 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T17:46:48.126 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --order 20 /tmp/rbd_import_export_159972/img testimg 2023-09-23T17:46:48.184 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T17:46:48.185 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T17:46:48.188 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_159972/img_v2 2023-09-23T17:46:48.228 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:48.231 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_159972/img_v2 testimg_import 2023-09-23T17:46:48.294 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T17:46:48.298 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import 2023-09-23T17:46:48.298 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep order 2023-09-23T17:46:48.298 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 20 2023-09-23T17:46:48.298 INFO:tasks.workunit.client.0.smithi055.stderr:+ awk '{print $2}' 2023-09-23T17:46:48.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:48 smithi099 ceph-mon[100270]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 389 KiB/s rd, 114 KiB/s wr, 114 op/s 2023-09-23T17:46:48.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:48 smithi099 ceph-mon[100270]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T17:46:48.338 INFO:tasks.workunit.client.0.smithi055.stdout:20 2023-09-23T17:46:48.338 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/img_v2 2023-09-23T17:46:48.338 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg_import 2023-09-23T17:46:48.426 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T17:46:48.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:48 smithi055 ceph-mon[95848]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 389 KiB/s rd, 114 KiB/s wr, 114 op/s 2023-09-23T17:46:48.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:48 smithi055 ceph-mon[95848]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T17:46:48.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:48 smithi055 ceph-mon[109912]: pgmap v64: 73 pgs: 73 active+clean; 579 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 389 KiB/s rd, 114 KiB/s wr, 114 op/s 2023-09-23T17:46:48.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:48 smithi055 ceph-mon[109912]: osdmap e75: 8 total, 8 up, 8 in 2023-09-23T17:46:48.431 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg 2023-09-23T17:46:48.524 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T17:46:48.527 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_159972/img testimg 2023-09-23T17:46:48.574 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T17:46:48.577 INFO:tasks.workunit.client.0.smithi055.stderr:++ rbd info testimg 2023-09-23T17:46:48.578 INFO:tasks.workunit.client.0.smithi055.stderr:++ grep features 2023-09-23T17:46:48.616 INFO:tasks.workunit.client.0.smithi055.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T17:46:48.616 INFO:tasks.workunit.client.0.smithi055.stderr: op_features: ' 2023-09-23T17:46:48.616 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_159972/img_v2 2023-09-23T17:46:48.656 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:48.659 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_159972/img_v2 testimg_import 2023-09-23T17:46:48.717 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T17:46:48.720 INFO:tasks.workunit.client.0.smithi055.stderr:++ rbd info testimg_import 2023-09-23T17:46:48.720 INFO:tasks.workunit.client.0.smithi055.stderr:++ grep features 2023-09-23T17:46:48.758 INFO:tasks.workunit.client.0.smithi055.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T17:46:48.758 INFO:tasks.workunit.client.0.smithi055.stderr: op_features: ' 2023-09-23T17:46:48.759 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' ' features: layering 2023-09-23T17:46:48.759 INFO:tasks.workunit.client.0.smithi055.stderr: op_features: ' '!=' ' features: layering 2023-09-23T17:46:48.759 INFO:tasks.workunit.client.0.smithi055.stderr: op_features: ' ']' 2023-09-23T17:46:48.759 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/img_v2 2023-09-23T17:46:48.759 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg_import 2023-09-23T17:46:48.826 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T17:46:48.829 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg 2023-09-23T17:46:48.901 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T17:46:48.904 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_159972/img testimg 2023-09-23T17:46:48.980 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T17:46:48.983 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_159972/img_v2 2023-09-23T17:46:49.024 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:49.028 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_159972/img_v2 testimg_import 2023-09-23T17:46:49.093 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T17:46:49.097 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import 2023-09-23T17:46:49.097 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 'stripe unit' 2023-09-23T17:46:49.097 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T17:46:49.137 INFO:tasks.workunit.client.0.smithi055.stdout: stripe unit: 4 KiB 2023-09-23T17:46:49.138 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import 2023-09-23T17:46:49.138 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 'stripe count' 2023-09-23T17:46:49.138 INFO:tasks.workunit.client.0.smithi055.stderr:+ awk '{print $3}' 2023-09-23T17:46:49.138 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 1000 2023-09-23T17:46:49.176 INFO:tasks.workunit.client.0.smithi055.stdout:1000 2023-09-23T17:46:49.176 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/img_v2 2023-09-23T17:46:49.176 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg_import 2023-09-23T17:46:49.267 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T17:46:49.271 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg 2023-09-23T17:46:49.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:49 smithi099 ceph-mon[100270]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T17:46:49.366 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T17:46:49.369 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_159972/img testimg 2023-09-23T17:46:49.425 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T17:46:49.428 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap create testimg@snap1 2023-09-23T17:46:49.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:49 smithi055 ceph-mon[95848]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T17:46:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:49 smithi055 ceph-mon[109912]: osdmap e76: 8 total, 8 up, 8 in 2023-09-23T17:46:49.973 INFO:tasks.workunit.client.0.smithi055.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T17:46:49.980 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap create testimg@snap2 2023-09-23T17:46:50.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:50 smithi099 ceph-mon[100270]: pgmap v67: 73 pgs: 73 active+clean; 799 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 233 KiB/s rd, 25 KiB/s wr, 302 op/s 2023-09-23T17:46:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:50 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3056039766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:50 smithi099 ceph-mon[100270]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T17:46:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:50 smithi055 ceph-mon[109912]: pgmap v67: 73 pgs: 73 active+clean; 799 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 233 KiB/s rd, 25 KiB/s wr, 302 op/s 2023-09-23T17:46:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:50 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3056039766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:50 smithi055 ceph-mon[109912]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T17:46:50.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:50 smithi055 ceph-mon[95848]: pgmap v67: 73 pgs: 73 active+clean; 799 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 233 KiB/s rd, 25 KiB/s wr, 302 op/s 2023-09-23T17:46:50.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:50 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3056039766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:50.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:50 smithi055 ceph-mon[95848]: osdmap e77: 8 total, 8 up, 8 in 2023-09-23T17:46:50.993 INFO:tasks.workunit.client.0.smithi055.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T17:46:50.999 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap protect testimg@snap2 2023-09-23T17:46:51.043 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_159972/snap_protect 2023-09-23T17:46:51.085 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:51.088 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_159972/snap_protect testimg_import 2023-09-23T17:46:51.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:46:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:46:50] "GET /metrics HTTP/1.1" 200 238984 "" "Prometheus/2.43.0" 2023-09-23T17:46:52.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:51 smithi099 ceph-mon[100270]: pgmap v69: 73 pgs: 73 active+clean; 799 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 215 KiB/s rd, 24 KiB/s wr, 279 op/s 2023-09-23T17:46:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:51 smithi099 ceph-mon[100270]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T17:46:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:51 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/472147248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:51 smithi055 ceph-mon[109912]: pgmap v69: 73 pgs: 73 active+clean; 799 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 215 KiB/s rd, 24 KiB/s wr, 279 op/s 2023-09-23T17:46:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:51 smithi055 ceph-mon[109912]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T17:46:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:51 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/472147248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:51 smithi055 ceph-mon[95848]: pgmap v69: 73 pgs: 73 active+clean; 799 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 215 KiB/s rd, 24 KiB/s wr, 279 op/s 2023-09-23T17:46:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:51 smithi055 ceph-mon[95848]: osdmap e78: 8 total, 8 up, 8 in 2023-09-23T17:46:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:51 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/472147248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:53.094 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T17:46:53.098 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import@snap1 2023-09-23T17:46:53.098 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 'protected: False' 2023-09-23T17:46:53.147 INFO:tasks.workunit.client.0.smithi055.stdout: protected: False 2023-09-23T17:46:53.147 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import@snap2 2023-09-23T17:46:53.148 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 'protected: True' 2023-09-23T17:46:53.186 INFO:tasks.workunit.client.0.smithi055.stdout: protected: True 2023-09-23T17:46:53.186 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/snap_protect 2023-09-23T17:46:53.187 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T17:46:53.232 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T17:46:53.279 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap purge testimg 2023-09-23T17:46:53.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:53 smithi099 ceph-mon[100270]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T17:46:53.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:53 smithi055 ceph-mon[109912]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T17:46:53.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:53 smithi055 ceph-mon[95848]: osdmap e79: 8 total, 8 up, 8 in 2023-09-23T17:46:54.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:54 smithi099 ceph-mon[100270]: pgmap v72: 73 pgs: 73 active+clean; 799 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 216 KiB/s rd, 24 KiB/s wr, 280 op/s 2023-09-23T17:46:54.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:54 smithi099 ceph-mon[100270]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T17:46:54.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:54 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/4188086820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:54.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:54 smithi055 ceph-mon[95848]: pgmap v72: 73 pgs: 73 active+clean; 799 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 216 KiB/s rd, 24 KiB/s wr, 280 op/s 2023-09-23T17:46:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:54 smithi055 ceph-mon[95848]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T17:46:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:54 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/4188086820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:54 smithi055 ceph-mon[109912]: pgmap v72: 73 pgs: 73 active+clean; 799 KiB data, 54 MiB used, 715 GiB / 715 GiB avail; 216 KiB/s rd, 24 KiB/s wr, 280 op/s 2023-09-23T17:46:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:54 smithi055 ceph-mon[109912]: osdmap e80: 8 total, 8 up, 8 in 2023-09-23T17:46:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:54 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/4188086820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:54.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:46:54 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:46:54] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:46:55.087 INFO:tasks.workunit.client.0.smithi055.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T17:46:55.096 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap purge testimg_import 2023-09-23T17:46:55.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:55 smithi055 ceph-mon[95848]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T17:46:55.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:55 smithi055 ceph-mon[109912]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T17:46:55.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:55 smithi099 ceph-mon[100270]: osdmap e81: 8 total, 8 up, 8 in 2023-09-23T17:46:56.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:56 smithi055 ceph-mon[95848]: pgmap v75: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 378 KiB/s rd, 40 KiB/s wr, 502 op/s 2023-09-23T17:46:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:56 smithi055 ceph-mon[95848]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T17:46:56.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:56 smithi055 ceph-mon[109912]: pgmap v75: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 378 KiB/s rd, 40 KiB/s wr, 502 op/s 2023-09-23T17:46:56.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:56 smithi055 ceph-mon[109912]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T17:46:56.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:56 smithi099 ceph-mon[100270]: pgmap v75: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 378 KiB/s rd, 40 KiB/s wr, 502 op/s 2023-09-23T17:46:56.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:56 smithi099 ceph-mon[100270]: osdmap e82: 8 total, 8 up, 8 in 2023-09-23T17:46:57.118 INFO:tasks.workunit.client.0.smithi055.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T17:46:57.125 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg 2023-09-23T17:46:57.224 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T17:46:57.227 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg_import 2023-09-23T17:46:57.317 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T17:46:57.320 INFO:tasks.workunit.client.0.smithi055.stderr:+ tiered=0 2023-09-23T17:46:57.321 INFO:tasks.workunit.client.0.smithi055.stderr:+ ceph osd dump 2023-09-23T17:46:57.321 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep '^pool' 2023-09-23T17:46:57.321 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep ''\''rbd'\''' 2023-09-23T17:46:57.321 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep tier 2023-09-23T17:46:57.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:57 smithi055 ceph-mon[109912]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T17:46:57.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:57 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2242763653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:57 smithi055 ceph-mon[95848]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T17:46:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:57 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2242763653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:57.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:57 smithi099 ceph-mon[100270]: osdmap e83: 8 total, 8 up, 8 in 2023-09-23T17:46:57.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:57 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2242763653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:57.811 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_159972/sparse1 2023-09-23T17:46:57.817 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records in 2023-09-23T17:46:57.817 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records out 2023-09-23T17:46:57.817 INFO:tasks.workunit.client.0.smithi055.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00520598 s, 201 MB/s 2023-09-23T17:46:57.817 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_159972/sparse2 2023-09-23T17:46:57.823 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records in 2023-09-23T17:46:57.823 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records out 2023-09-23T17:46:57.823 INFO:tasks.workunit.client.0.smithi055.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00500925 s, 209 MB/s 2023-09-23T17:46:57.823 INFO:tasks.workunit.client.0.smithi055.stderr:+ truncate /tmp/rbd_import_export_159972/sparse2 -s 2M 2023-09-23T17:46:57.841 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse1 2023-09-23T17:46:57.882 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 0% complete...failed. 2023-09-23T17:46:57.882 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T17:46:57.885 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T17:46:57.885 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_159972/sparse1 2023-09-23T17:46:57.952 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T17:46:57.952 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T17:46:57.953 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T17:46:57.958 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd ls -l 2023-09-23T17:46:57.958 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep sparse1 2023-09-23T17:46:57.959 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T17:46:57.998 INFO:tasks.workunit.client.0.smithi055.stdout:sparse1 2 MiB 2 2023-09-23T17:46:57.999 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse1 2023-09-23T17:46:57.999 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse1 2023-09-23T17:46:57.999 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse1 2023-09-23T17:46:57.999 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T17:46:58.000 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T17:46:58.039 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.620c61db2bce 2023-09-23T17:46:58.040 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.620c61db2bce 2023-09-23T17:46:58.040 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse1 2023-09-23T17:46:58.040 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse1 2023-09-23T17:46:58.040 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.620c61db2bce\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T17:46:58.041 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T17:46:58.042 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse1 2023-09-23T17:46:58.042 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T17:46:58.042 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T17:46:58.084 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T17:46:58.084 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T17:46:58.084 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T17:46:58.084 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T17:46:58.084 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T17:46:58.131 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects=1 2023-09-23T17:46:58.132 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 1 2023-09-23T17:46:58.132 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T17:46:58.132 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export sparse1 /tmp/rbd_import_export_159972/sparse1.out 2023-09-23T17:46:58.171 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:58.175 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159972/sparse1 /tmp/rbd_import_export_159972/sparse1.out 2023-09-23T17:46:58.175 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_159972/sparse1 /tmp/rbd_import_export_159972/sparse1.out 2023-09-23T17:46:58.177 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/sparse1 --format %b 2023-09-23T17:46:58.178 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=2048 2023-09-23T17:46:58.179 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/sparse1.out --format %b 2023-09-23T17:46:58.180 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=2048 2023-09-23T17:46:58.180 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T17:46:58.180 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T17:46:58.180 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T17:46:58.180 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/sparse1.out 2023-09-23T17:46:58.181 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse1 2023-09-23T17:46:58.283 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T17:46:58.287 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse2 2023-09-23T17:46:58.329 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 0% complete...failed. 2023-09-23T17:46:58.330 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T17:46:58.333 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T17:46:58.333 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_159972/sparse2 2023-09-23T17:46:58.406 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T17:46:58.407 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T17:46:58.407 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T17:46:58.410 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd ls -l 2023-09-23T17:46:58.411 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep sparse2 2023-09-23T17:46:58.411 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T17:46:58.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:58 smithi055 ceph-mon[109912]: pgmap v78: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 378 KiB/s rd, 40 KiB/s wr, 502 op/s 2023-09-23T17:46:58.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:58 smithi055 ceph-mon[109912]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T17:46:58.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:58 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:58.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:58 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2875343289' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T17:46:58.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:58 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2875343289' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T17:46:58.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:58 smithi055 ceph-mon[95848]: pgmap v78: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 378 KiB/s rd, 40 KiB/s wr, 502 op/s 2023-09-23T17:46:58.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:58 smithi055 ceph-mon[95848]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T17:46:58.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:58 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:58.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:58 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2875343289' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T17:46:58.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:58 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2875343289' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T17:46:58.463 INFO:tasks.workunit.client.0.smithi055.stdout:sparse2 2 MiB 2 2023-09-23T17:46:58.463 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse2 2023-09-23T17:46:58.464 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse2 2023-09-23T17:46:58.464 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse2 2023-09-23T17:46:58.464 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T17:46:58.470 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T17:46:58.517 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.623ccd40055e 2023-09-23T17:46:58.518 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.623ccd40055e 2023-09-23T17:46:58.518 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse2 2023-09-23T17:46:58.518 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.623ccd40055e\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T17:46:58.518 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse2 2023-09-23T17:46:58.519 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T17:46:58.519 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse2 2023-09-23T17:46:58.519 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T17:46:58.520 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T17:46:58.574 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T17:46:58.574 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T17:46:58.574 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T17:46:58.574 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T17:46:58.574 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T17:46:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:58 smithi099 ceph-mon[100270]: pgmap v78: 73 pgs: 73 active+clean; 799 KiB data, 56 MiB used, 715 GiB / 715 GiB avail; 378 KiB/s rd, 40 KiB/s wr, 502 op/s 2023-09-23T17:46:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:58 smithi099 ceph-mon[100270]: osdmap e84: 8 total, 8 up, 8 in 2023-09-23T17:46:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:58 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:46:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:58 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2875343289' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T17:46:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:58 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2875343289' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T17:46:58.615 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects=0 2023-09-23T17:46:58.615 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 0 2023-09-23T17:46:58.615 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T17:46:58.615 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export sparse2 /tmp/rbd_import_export_159972/sparse2.out 2023-09-23T17:46:58.668 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:58.671 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159972/sparse2 /tmp/rbd_import_export_159972/sparse2.out 2023-09-23T17:46:58.671 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_159972/sparse2 /tmp/rbd_import_export_159972/sparse2.out 2023-09-23T17:46:58.676 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/sparse2 --format %b 2023-09-23T17:46:58.678 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=2048 2023-09-23T17:46:58.678 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/sparse2.out --format %b 2023-09-23T17:46:58.679 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=2048 2023-09-23T17:46:58.680 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T17:46:58.680 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T17:46:58.680 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T17:46:58.680 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/sparse2.out 2023-09-23T17:46:58.680 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse2 2023-09-23T17:46:58.800 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T17:46:58.805 INFO:tasks.workunit.client.0.smithi055.stderr:+ truncate /tmp/rbd_import_export_159972/sparse1 -s 10M 2023-09-23T17:46:58.806 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T17:46:58.918 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T17:46:58.919 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T17:46:58.919 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T17:46:58.923 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd ls -l 2023-09-23T17:46:58.923 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep sparse1 2023-09-23T17:46:58.923 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T17:46:58.967 INFO:tasks.workunit.client.0.smithi055.stdout:sparse1 10 MiB 2 2023-09-23T17:46:58.967 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse1 2023-09-23T17:46:58.967 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse1 2023-09-23T17:46:58.968 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse1 2023-09-23T17:46:58.968 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T17:46:58.968 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T17:46:59.007 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.3bd0a826ffe7 2023-09-23T17:46:59.008 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.3bd0a826ffe7 2023-09-23T17:46:59.008 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse1 2023-09-23T17:46:59.008 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.3bd0a826ffe7\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T17:46:59.009 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse1 2023-09-23T17:46:59.009 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T17:46:59.012 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse1 2023-09-23T17:46:59.012 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T17:46:59.012 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T17:46:59.050 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T17:46:59.051 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T17:46:59.051 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T17:46:59.051 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T17:46:59.051 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T17:46:59.084 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects=1 2023-09-23T17:46:59.084 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 1 2023-09-23T17:46:59.084 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T17:46:59.084 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export sparse1 /tmp/rbd_import_export_159972/sparse1.out 2023-09-23T17:46:59.127 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:59.130 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159972/sparse1 /tmp/rbd_import_export_159972/sparse1.out 2023-09-23T17:46:59.131 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_159972/sparse1 /tmp/rbd_import_export_159972/sparse1.out 2023-09-23T17:46:59.139 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/sparse1 --format %b 2023-09-23T17:46:59.140 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=2048 2023-09-23T17:46:59.140 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/sparse1.out --format %b 2023-09-23T17:46:59.141 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=2048 2023-09-23T17:46:59.141 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T17:46:59.141 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T17:46:59.141 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T17:46:59.141 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/sparse1.out 2023-09-23T17:46:59.142 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse1 2023-09-23T17:46:59.233 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T17:46:59.236 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_159972/sparse2 oflag=append conv=notrunc 2023-09-23T17:46:59.247 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records in 2023-09-23T17:46:59.247 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records out 2023-09-23T17:46:59.247 INFO:tasks.workunit.client.0.smithi055.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0102007 s, 206 MB/s 2023-09-23T17:46:59.247 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T17:46:59.332 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T17:46:59.333 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T17:46:59.333 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T17:46:59.336 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd ls -l 2023-09-23T17:46:59.336 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep sparse2 2023-09-23T17:46:59.336 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T17:46:59.372 INFO:tasks.workunit.client.0.smithi055.stdout:sparse2 4 MiB 2 2023-09-23T17:46:59.373 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse2 2023-09-23T17:46:59.373 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse2 2023-09-23T17:46:59.373 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse2 2023-09-23T17:46:59.373 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T17:46:59.374 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T17:46:59.424 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.3bfae14e8a51 2023-09-23T17:46:59.424 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.3bfae14e8a51 2023-09-23T17:46:59.424 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse2 2023-09-23T17:46:59.425 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse2 2023-09-23T17:46:59.425 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.3bfae14e8a51\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T17:46:59.425 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T17:46:59.425 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse2 2023-09-23T17:46:59.426 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T17:46:59.426 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T17:46:59.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:46:59 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3719956318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:59.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:46:59 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3719956318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:59.463 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T17:46:59.463 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T17:46:59.463 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T17:46:59.463 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T17:46:59.463 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T17:46:59.497 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects='0 2023-09-23T17:46:59.497 INFO:tasks.workunit.client.0.smithi055.stderr:2 2023-09-23T17:46:59.497 INFO:tasks.workunit.client.0.smithi055.stderr:3' 2023-09-23T17:46:59.497 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 0 2 3 2023-09-23T17:46:59.497 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T17:46:59.497 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export sparse2 /tmp/rbd_import_export_159972/sparse2.out 2023-09-23T17:46:59.538 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T17:46:59.541 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_159972/sparse2 /tmp/rbd_import_export_159972/sparse2.out 2023-09-23T17:46:59.541 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_159972/sparse2 /tmp/rbd_import_export_159972/sparse2.out 2023-09-23T17:46:59.544 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/sparse2 --format %b 2023-09-23T17:46:59.545 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=6144 2023-09-23T17:46:59.546 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/sparse2.out --format %b 2023-09-23T17:46:59.547 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=6144 2023-09-23T17:46:59.547 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T17:46:59.547 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T17:46:59.547 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T17:46:59.547 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/sparse2.out 2023-09-23T17:46:59.548 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse2 2023-09-23T17:46:59.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:46:59 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3719956318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:46:59.647 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T17:46:59.651 INFO:tasks.workunit.client.0.smithi055.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T17:46:59.651 INFO:tasks.workunit.client.0.smithi055.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T17:46:59.651 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_159972/sparse1 sparse 2023-09-23T17:46:59.719 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T17:46:59.719 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T17:46:59.720 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T17:46:59.723 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse 2023-09-23T17:46:59.723 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse 2023-09-23T17:46:59.723 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse 2023-09-23T17:46:59.723 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T17:46:59.724 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T17:46:59.761 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.3c27c5894f3b 2023-09-23T17:46:59.762 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.3c27c5894f3b 2023-09-23T17:46:59.762 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse 2023-09-23T17:46:59.763 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.3c27c5894f3b\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T17:46:59.763 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse 2023-09-23T17:46:59.763 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T17:46:59.765 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse 2023-09-23T17:46:59.765 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T17:46:59.765 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T17:46:59.800 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T17:46:59.800 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T17:46:59.800 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T17:46:59.800 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T17:46:59.800 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T17:46:59.835 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects=1 2023-09-23T17:46:59.835 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 1 2023-09-23T17:46:59.835 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T17:46:59.835 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse 2023-09-23T17:46:59.921 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T17:46:59.925 INFO:tasks.workunit.client.0.smithi055.stdout:zeros import through stdin to sparse image 2023-09-23T17:46:59.925 INFO:tasks.workunit.client.0.smithi055.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T17:46:59.925 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T17:46:59.925 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format - sparse 2023-09-23T17:46:59.989 INFO:tasks.workunit.client.0.smithi055.stderr:4+0 records in 2023-09-23T17:46:59.989 INFO:tasks.workunit.client.0.smithi055.stderr:4+0 records out 2023-09-23T17:46:59.989 INFO:tasks.workunit.client.0.smithi055.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0631048 s, 66.5 MB/s 2023-09-23T17:46:59.999 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T17:46:59.999 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T17:47:00.002 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse 2023-09-23T17:47:00.002 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse 2023-09-23T17:47:00.003 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse 2023-09-23T17:47:00.003 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T17:47:00.003 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T17:47:00.040 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.3c45cb18b650 2023-09-23T17:47:00.040 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.3c45cb18b650 2023-09-23T17:47:00.041 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse 2023-09-23T17:47:00.041 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.3c45cb18b650\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T17:47:00.041 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse 2023-09-23T17:47:00.041 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T17:47:00.042 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse 2023-09-23T17:47:00.042 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T17:47:00.042 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T17:47:00.080 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T17:47:00.081 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T17:47:00.081 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T17:47:00.081 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T17:47:00.081 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T17:47:00.115 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects= 2023-09-23T17:47:00.115 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 2023-09-23T17:47:00.116 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T17:47:00.116 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse 2023-09-23T17:47:00.204 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T17:47:00.207 INFO:tasks.workunit.client.0.smithi055.stdout:zeros export to sparse file 2023-09-23T17:47:00.207 INFO:tasks.workunit.client.0.smithi055.stderr:+ echo 'zeros export to sparse file' 2023-09-23T17:47:00.207 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T17:47:00.249 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T17:47:00.252 INFO:tasks.workunit.client.0.smithi055.stderr:++ rbd info sparse 2023-09-23T17:47:00.252 INFO:tasks.workunit.client.0.smithi055.stderr:++ grep block_name_prefix 2023-09-23T17:47:00.252 INFO:tasks.workunit.client.0.smithi055.stderr:++ awk '{print $NF;}' 2023-09-23T17:47:00.289 INFO:tasks.workunit.client.0.smithi055.stderr:+ prefix=rbd_data.3c66defd4c9f 2023-09-23T17:47:00.289 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T17:47:00.289 INFO:tasks.workunit.client.0.smithi055.stderr:++ get_image_data_pool sparse 2023-09-23T17:47:00.289 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse 2023-09-23T17:47:00.290 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse 2023-09-23T17:47:00.290 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep 'data_pool: ' 2023-09-23T17:47:00.290 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T17:47:00.328 INFO:tasks.workunit.client.0.smithi055.stderr:++ data_pool= 2023-09-23T17:47:00.328 INFO:tasks.workunit.client.0.smithi055.stderr:++ '[' -z ']' 2023-09-23T17:47:00.328 INFO:tasks.workunit.client.0.smithi055.stderr:++ data_pool=rbd 2023-09-23T17:47:00.328 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo rbd 2023-09-23T17:47:00.328 INFO:tasks.workunit.client.0.smithi055.stderr:+ rados -p rbd put rbd_data.3c66defd4c9f.000000000000 - 2023-09-23T17:47:00.360 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records in 2023-09-23T17:47:00.360 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records out 2023-09-23T17:47:00.360 INFO:tasks.workunit.client.0.smithi055.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0704338 s, 59.5 MB/s 2023-09-23T17:47:00.376 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse 2023-09-23T17:47:00.376 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse 2023-09-23T17:47:00.376 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse 2023-09-23T17:47:00.376 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T17:47:00.377 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T17:47:00.412 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.3c66defd4c9f 2023-09-23T17:47:00.413 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.3c66defd4c9f 2023-09-23T17:47:00.413 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse 2023-09-23T17:47:00.413 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.3c66defd4c9f\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T17:47:00.413 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse 2023-09-23T17:47:00.413 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T17:47:00.414 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse 2023-09-23T17:47:00.414 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T17:47:00.414 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T17:47:00.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:00 smithi055 ceph-mon[95848]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 390 KiB/s rd, 252 KiB/s wr, 203 op/s 2023-09-23T17:47:00.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:00 smithi055 ceph-mon[95848]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T17:47:00.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:00 smithi055 ceph-mon[109912]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 390 KiB/s rd, 252 KiB/s wr, 203 op/s 2023-09-23T17:47:00.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:00 smithi055 ceph-mon[109912]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T17:47:00.461 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T17:47:00.461 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T17:47:00.461 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T17:47:00.461 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T17:47:00.461 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T17:47:00.495 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects=0 2023-09-23T17:47:00.496 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 0 2023-09-23T17:47:00.496 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T17:47:00.496 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/sparse 2023-09-23T17:47:00.496 INFO:tasks.workunit.client.0.smithi055.stderr:rm: cannot remove '/tmp/rbd_import_export_159972/sparse': No such file or directory 2023-09-23T17:47:00.497 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T17:47:00.497 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export sparse /tmp/rbd_import_export_159972/sparse 2023-09-23T17:47:00.535 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete...done. 2023-09-23T17:47:00.539 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_159972/sparse --format=%b 2023-09-23T17:47:00.540 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 = 0 ']' 2023-09-23T17:47:00.540 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse 2023-09-23T17:47:00.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:00 smithi099 ceph-mon[100270]: pgmap v80: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 390 KiB/s rd, 252 KiB/s wr, 203 op/s 2023-09-23T17:47:00.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:00 smithi099 ceph-mon[100270]: osdmap e85: 8 total, 8 up, 8 in 2023-09-23T17:47:00.628 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T17:47:00.631 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_159972/sparse /tmp/rbd_import_export_159972/sparse1 /tmp/rbd_import_export_159972/sparse2 /tmp/rbd_import_export_159972/sparse3 2023-09-23T17:47:00.633 INFO:tasks.workunit.client.0.smithi055.stderr:rm: cannot remove '/tmp/rbd_import_export_159972/sparse3': No such file or directory 2023-09-23T17:47:00.633 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T17:47:00.634 INFO:tasks.workunit.client.0.smithi055.stderr:+ echo OK 2023-09-23T17:47:00.634 INFO:tasks.workunit.client.0.smithi055.stdout:OK 2023-09-23T17:47:00.635 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm -rf /tmp/rbd_import_export_159972 2023-09-23T17:47:00.635 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T17:47:00.635 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T17:47:00.669 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T17:47:00.669 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T17:47:00.936 DEBUG:teuthology.parallel:result is None 2023-09-23T17:47:00.936 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T17:47:00.969 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T17:47:00.970 DEBUG:teuthology.orchestra.run.smithi055:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T17:47:01.033 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T17:47:01.034 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T17:47:01.034 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T17:47:01.034 DEBUG:teuthology.parallel:result is None 2023-09-23T17:47:01.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:00] "GET /metrics HTTP/1.1" 200 239472 "" "Prometheus/2.43.0" 2023-09-23T17:47:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:02 smithi055 ceph-mon[109912]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 316 KiB/s rd, 204 KiB/s wr, 164 op/s 2023-09-23T17:47:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:02 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3008505839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:02 smithi055 ceph-mon[95848]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 316 KiB/s rd, 204 KiB/s wr, 164 op/s 2023-09-23T17:47:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:02 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3008505839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:02 smithi099 ceph-mon[100270]: pgmap v82: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 316 KiB/s rd, 204 KiB/s wr, 164 op/s 2023-09-23T17:47:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:02 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3008505839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:04.373 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:04 smithi055 ceph-mon[95848]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 266 KiB/s rd, 172 KiB/s wr, 138 op/s 2023-09-23T17:47:04.373 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:04 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2151559886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:04.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:04 smithi055 ceph-mon[109912]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 266 KiB/s rd, 172 KiB/s wr, 138 op/s 2023-09-23T17:47:04.374 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:04 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2151559886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:04.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:04 smithi099 ceph-mon[100270]: pgmap v83: 73 pgs: 73 active+clean; 579 KiB data, 58 MiB used, 715 GiB / 715 GiB avail; 266 KiB/s rd, 172 KiB/s wr, 138 op/s 2023-09-23T17:47:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:04 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2151559886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:04.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:04] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:47:06.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:05 smithi099 ceph-mon[100270]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 352 op/s 2023-09-23T17:47:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:05 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:05 smithi099 ceph-mon[100270]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T17:47:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:05 smithi099 ceph-mon[100270]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T17:47:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:05 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:05 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:05 smithi099 ceph-mon[100270]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T17:47:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:05 smithi099 ceph-mon[100270]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi099 2023-09-23T17:47:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[109912]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 352 op/s 2023-09-23T17:47:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[109912]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T17:47:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[109912]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T17:47:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[109912]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T17:47:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[109912]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi099 2023-09-23T17:47:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[95848]: pgmap v84: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 1.0 MiB/s rd, 1.4 MiB/s wr, 352 op/s 2023-09-23T17:47:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[95848]: Upgrade: Target is version 18.2.0-460-gbb0daeff (unknown) 2023-09-23T17:47:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[95848]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d, digests ['quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d'] 2023-09-23T17:47:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[95848]: Upgrade: Need to upgrade myself (mgr.y) 2023-09-23T17:47:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:05 smithi055 ceph-mon[95848]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d on smithi099 2023-09-23T17:47:07.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:06 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1326791555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:06 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1326791555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:07.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:06 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1326791555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:08.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:07 smithi099 ceph-mon[100270]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 894 KiB/s rd, 1.2 MiB/s wr, 305 op/s 2023-09-23T17:47:08.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:07 smithi055 ceph-mon[95848]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 894 KiB/s rd, 1.2 MiB/s wr, 305 op/s 2023-09-23T17:47:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:07 smithi055 ceph-mon[109912]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 894 KiB/s rd, 1.2 MiB/s wr, 305 op/s 2023-09-23T17:47:10.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:09 smithi099 ceph-mon[100270]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 658 KiB/s rd, 1.0 MiB/s wr, 195 op/s 2023-09-23T17:47:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:09 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2967793832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:09 smithi055 ceph-mon[95848]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 658 KiB/s rd, 1.0 MiB/s wr, 195 op/s 2023-09-23T17:47:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:09 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2967793832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:10.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:09 smithi055 ceph-mon[109912]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 658 KiB/s rd, 1.0 MiB/s wr, 195 op/s 2023-09-23T17:47:10.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:09 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2967793832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:11.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:10] "GET /metrics HTTP/1.1" 200 239472 "" "Prometheus/2.43.0" 2023-09-23T17:47:12.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:11 smithi099 ceph-mon[100270]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 635 KiB/s rd, 995 KiB/s wr, 188 op/s 2023-09-23T17:47:12.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:11 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3450774273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:12.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:11 smithi055 ceph-mon[95848]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 635 KiB/s rd, 995 KiB/s wr, 188 op/s 2023-09-23T17:47:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:11 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3450774273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:11 smithi055 ceph-mon[109912]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 635 KiB/s rd, 995 KiB/s wr, 188 op/s 2023-09-23T17:47:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:11 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3450774273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:13.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:12 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:12 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:12 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:14.321 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:13 smithi099 ceph-mon[100270]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 549 KiB/s rd, 860 KiB/s wr, 162 op/s 2023-09-23T17:47:14.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:13 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3077555328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:14.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:13 smithi055 ceph-mon[95848]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 549 KiB/s rd, 860 KiB/s wr, 162 op/s 2023-09-23T17:47:14.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:13 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3077555328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:14.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:13 smithi055 ceph-mon[109912]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 549 KiB/s rd, 860 KiB/s wr, 162 op/s 2023-09-23T17:47:14.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:13 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3077555328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:14.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:14] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:47:16.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:15 smithi099 ceph-mon[100270]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 549 KiB/s rd, 860 KiB/s wr, 162 op/s 2023-09-23T17:47:16.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:15 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2950272276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:16.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:15 smithi055 ceph-mon[95848]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 549 KiB/s rd, 860 KiB/s wr, 162 op/s 2023-09-23T17:47:16.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:15 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2950272276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:16.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:15 smithi055 ceph-mon[109912]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail; 549 KiB/s rd, 860 KiB/s wr, 162 op/s 2023-09-23T17:47:16.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:15 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2950272276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:18.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:17 smithi099 ceph-mon[100270]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:17 smithi055 ceph-mon[95848]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:18.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:18 smithi055 ceph-mon[109912]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:19.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:19 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1685925620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:19 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1685925620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:19.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:19 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1685925620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:20.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:20 smithi099 ceph-mon[100270]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:20.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:20 smithi055 ceph-mon[95848]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:20.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:20 smithi055 ceph-mon[109912]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:21.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:21 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2022949962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:21.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:20] "GET /metrics HTTP/1.1" 200 239507 "" "Prometheus/2.43.0" 2023-09-23T17:47:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:21 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2022949962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:21 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2022949962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:22.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:22 smithi099 ceph-mon[100270]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:22 smithi055 ceph-mon[109912]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:22 smithi055 ceph-mon[95848]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:24.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:24 smithi099 ceph-mon[100270]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:24.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:24 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1413021277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:24.323 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[102376]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:24] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:47:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:24 smithi055 ceph-mon[95848]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:24 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1413021277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:24.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:24 smithi055 ceph-mon[109912]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:24.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:24 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1413021277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:26.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:26 smithi099 ceph-mon[100270]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:26 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/493474780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:26.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:26 smithi055 ceph-mon[95848]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:26.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:26 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/493474780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:26.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:26 smithi055 ceph-mon[109912]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:26.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:26 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/493474780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:28.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:28 smithi099 ceph-mon[100270]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:28 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:28 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/77935650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:28 smithi055 ceph-mon[95848]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:28 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:28 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/77935650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:28 smithi055 ceph-mon[109912]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:28 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:28 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/77935650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:30.209 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:30 smithi099 ceph-mon[100270]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:30.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:30 smithi055 ceph-mon[95848]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:30.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:30 smithi055 ceph-mon[109912]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:31.335 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:31 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2344593814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:31.335 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:31 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:31.335 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:31 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:47:31.335 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:31 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:47:31.335 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:31 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:47:31.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:30] "GET /metrics HTTP/1.1" 200 239521 "" "Prometheus/2.43.0" 2023-09-23T17:47:31.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:31 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2344593814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:31.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:31 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:31.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:31 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:47:31.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:31 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:47:31.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:31 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:47:31.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:31 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2344593814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:31.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:31 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:31.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:31 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:47:31.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:31 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:47:31.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:31 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:47:32.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:31 smithi099 systemd[1]: Stopping Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:47:32.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:32 smithi055 ceph-mon[109912]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:32.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:32 smithi055 ceph-mon[109912]: Upgrade: Updating mgr.x 2023-09-23T17:47:32.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:32 smithi055 ceph-mon[109912]: Deploying daemon mgr.x on smithi099 2023-09-23T17:47:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:32 smithi055 ceph-mon[95848]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:32 smithi055 ceph-mon[95848]: Upgrade: Updating mgr.x 2023-09-23T17:47:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:32 smithi055 ceph-mon[95848]: Deploying daemon mgr.x on smithi099 2023-09-23T17:47:32.471 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:32 smithi099 ceph-mon[100270]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:32.471 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:32 smithi099 ceph-mon[100270]: Upgrade: Updating mgr.x 2023-09-23T17:47:32.472 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:32 smithi099 ceph-mon[100270]: Deploying daemon mgr.x on smithi099 2023-09-23T17:47:32.472 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:32 smithi099 bash[139933]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T17:47:32.472 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:32 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T17:47:32.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:32 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T17:47:32.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:32 smithi099 systemd[1]: Stopped Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:47:32.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:32 smithi099 systemd[1]: Starting Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:47:33.180 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:33 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3988187278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:33.181 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:32 smithi099 podman[140081]: 2023-09-23T17:47:33.181 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:33 smithi099 ceph-mgr[140121]: -- 172.21.15.99:0/2932739693 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x562de96a51e0 con 0x562de967a800 2023-09-23T17:47:33.181 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:33 smithi099 bash[140081]: fb4d1a168e3e5482b764dc50920ff93da6f6978a9ec306c33fc13b7999c6f3a3 2023-09-23T17:47:33.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:33 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3988187278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:33.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:33 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3988187278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:33.433 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:33 smithi099 systemd[1]: Started Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:47:33.434 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:33.256+0000 7f2985e1a200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:47:33.434 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:33.339+0000 7f2985e1a200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:47:33.434 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:33.432+0000 7f2985e1a200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:47:33.725 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:33.545+0000 7f2985e1a200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:47:33.725 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:33.724+0000 7f2985e1a200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:47:34.460 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:34 smithi099 ceph-mon[100270]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:34.461 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:34 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:34.461 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:34 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:34.461 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:34 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:34.461 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:34.124+0000 7f2985e1a200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:47:34.461 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:34.263+0000 7f2985e1a200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:47:34.461 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:34.355+0000 7f2985e1a200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:47:34.461 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:34.460+0000 7f2985e1a200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:47:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:34 smithi055 ceph-mon[95848]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:34 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:34 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:34 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:34.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:34 smithi055 ceph-mon[109912]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:34.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:34 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:34.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:34 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:34.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:34 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:35.326 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:34.908+0000 7f2985e1a200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:47:35.964 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:35 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:35.621+0000 7f2985e1a200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:47:35.964 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:35 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:35.705+0000 7f2985e1a200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:47:36.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:35 smithi099 ceph-mon[100270]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:36.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:35 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1033384336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:36.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:35 smithi055 ceph-mon[109912]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:36.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:35 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1033384336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:36.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:35 smithi055 ceph-mon[95848]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:36.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:35 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1033384336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:36.805 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:36 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:36.529+0000 7f2985e1a200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:47:36.805 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:36 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:36.622+0000 7f2985e1a200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:47:37.076 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:36 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:36.805+0000 7f2985e1a200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:47:37.489 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:37 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:37.090+0000 7f2985e1a200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:47:37.489 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:37 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:37.331+0000 7f2985e1a200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:47:37.489 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:37 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:37.489+0000 7f2985e1a200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:47:38.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:37 smithi099 ceph-mon[100270]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:37 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3623094023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:38.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:37 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:37.782+0000 7f2985e1a200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:47:38.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:37 smithi055 ceph-mon[95848]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:38.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:37 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3623094023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:37 smithi055 ceph-mon[109912]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:37 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3623094023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:38.826 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:38.534+0000 7f2985e1a200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:47:38.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:38.675+0000 7f2985e1a200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:47:39.236 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:38.878+0000 7f2985e1a200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:47:39.236 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:39.002+0000 7f2985e1a200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:47:39.236 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:39.235+0000 7f2985e1a200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:47:39.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:39 smithi099 ceph-mon[100270]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:39.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:39 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:39.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:39 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:39.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:39 smithi055 ceph-mon[95848]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:39.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:39 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:39.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:39 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:39.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:39 smithi055 ceph-mon[109912]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:39.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:39 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:39.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:39 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:40.479 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:40.150+0000 7f2985e1a200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:47:40.479 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:47:40] ENGINE Bus STARTING 2023-09-23T17:47:40.479 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: CherryPy Checker: 2023-09-23T17:47:40.479 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: The Application mounted at '' has an empty config. 2023-09-23T17:47:40.479 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:40.479 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:47:40] ENGINE Serving on http://:::9283 2023-09-23T17:47:40.479 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:47:40] ENGINE Bus STARTED 2023-09-23T17:47:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3090655303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-mon[100270]: Standby manager daemon x restarted 2023-09-23T17:47:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-mon[100270]: Standby manager daemon x started 2023-09-23T17:47:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:47:40.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:47:40.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:47:40.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:40 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:47:40.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:40.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:40.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3090655303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:40.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[95848]: Standby manager daemon x restarted 2023-09-23T17:47:40.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[95848]: Standby manager daemon x started 2023-09-23T17:47:40.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:47:40.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:47:40.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:47:40.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:47:40.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:40.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:40.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3090655303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:40.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[109912]: Standby manager daemon x restarted 2023-09-23T17:47:40.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T17:47:40.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:47:40.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:47:40.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:47:40.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:47:41.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:40] "GET /metrics HTTP/1.1" 200 239521 "" "Prometheus/2.43.0" 2023-09-23T17:47:42.162 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:41 smithi099 ceph-mon[100270]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:42.163 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:41 smithi099 ceph-mon[100270]: mgrmap e21: y(active, since 2m), standbys: x 2023-09-23T17:47:42.391 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:41 smithi055 ceph-mon[95848]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:42.391 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:41 smithi055 ceph-mon[95848]: mgrmap e21: y(active, since 2m), standbys: x 2023-09-23T17:47:42.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:41 smithi055 ceph-mon[109912]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:42.392 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:41 smithi055 ceph-mon[109912]: mgrmap e21: y(active, since 2m), standbys: x 2023-09-23T17:47:42.392 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:42 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:42.390+0000 7f1685758700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T17:47:42.428 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:47:42] ENGINE Bus STOPPING 2023-09-23T17:47:42.428 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:47:42] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T17:47:42.428 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:47:42] ENGINE Bus STOPPED 2023-09-23T17:47:42.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:42 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ignoring --setuser ceph since I am not root 2023-09-23T17:47:42.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:42 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ignoring --setgroup ceph since I am not root 2023-09-23T17:47:42.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:42 smithi055 ceph-mgr[96116]: -- 172.21.15.55:0/113645596 <== mon.0 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x563db7e909c0 con 0x563db7e66c00 2023-09-23T17:47:42.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:42 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:42.586+0000 7f6e544221c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:47:42.826 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:47:42] ENGINE Bus STARTING 2023-09-23T17:47:42.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:42.555+0000 7f28e32b6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:47:42.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:42.555+0000 7f28e32b6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:47:42.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:42.555+0000 7f28e32b6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:47:42.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:42.555+0000 7f28e02b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:47:42.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:42.555+0000 7f28e02b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:47:42.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:42.555+0000 7f28e02b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:47:42.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:42.555+0000 7f28e02b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:47:42.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:42.555+0000 7f28e02b0700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:47:42.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:42.556+0000 7f28e32b6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:47:42.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:47:42.556+0000 7f28e32b6700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:47:42.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:47:42] ENGINE Serving on http://:::9283 2023-09-23T17:47:42.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:47:42] ENGINE Bus STARTED 2023-09-23T17:47:43.113 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:42 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:42.683+0000 7f6e544221c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:47:43.114 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:42 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:42.793+0000 7f6e544221c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:47:43.114 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:42 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:42.907+0000 7f6e544221c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:47:43.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:43.113+0000 7f6e544221c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:47:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:47:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:47:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T17:47:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:47:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:47:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:47:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:47:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:47:43.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:47:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:47:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:47:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:47:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:47:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:47:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:47:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:47:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:47:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T17:47:43.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:47:43.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T17:47:43.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: mgrmap e22: x(active, starting, since 0.0148033s) 2023-09-23T17:47:43.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:47:43.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:47:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:47:43.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:47:43.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:43.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:43.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T17:47:43.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:43.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:47:43.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:47:43.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:47:43.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:47:43.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:47:43.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:47:43.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:47:43.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:47:43.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:47:43.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:47:43.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:47:43.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:47:43.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:47:43.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:47:43.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T17:47:43.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:47:43.436 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T17:47:43.436 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: mgrmap e22: x(active, starting, since 0.0148033s) 2023-09-23T17:47:43.436 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:47:43.436 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:47:43.436 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: Manager daemon x is now available 2023-09-23T17:47:43.436 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3345049221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:43.436 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:43.436 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:43.436 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:43.437 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:43.437 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:47:43.437 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:47:43.437 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T17:47:43.437 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T17:47:43.437 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:43.437 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:43.437 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: Deploying cephadm binary to smithi099 2023-09-23T17:47:43.437 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[109912]: Deploying cephadm binary to smithi055 2023-09-23T17:47:43.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: Manager daemon x is now available 2023-09-23T17:47:43.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3345049221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:43.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:43.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:43.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:43.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:43.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:47:43.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:47:43.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T17:47:43.438 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T17:47:43.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:43.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:43.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: Deploying cephadm binary to smithi099 2023-09-23T17:47:43.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-mon[95848]: Deploying cephadm binary to smithi055 2023-09-23T17:47:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:47:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:47:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' 2023-09-23T17:47:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-09-23T17:47:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:47:43.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: osdmap e86: 8 total, 8 up, 8 in 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.24338 172.21.15.55:0/3026792495' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: mgrmap e22: x(active, starting, since 0.0148033s) 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: Manager daemon x is now available 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3345049221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:43.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:43.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:47:43.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:43.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:47:43.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:47:43.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T17:47:43.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/x/trash_purge_schedule"}]: dispatch 2023-09-23T17:47:43.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:43.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:43.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: Deploying cephadm binary to smithi099 2023-09-23T17:47:43.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:43 smithi099 ceph-mon[100270]: Deploying cephadm binary to smithi055 2023-09-23T17:47:43.773 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:43.515+0000 7f6e544221c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:47:43.773 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:43.622+0000 7f6e544221c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:47:44.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:43.772+0000 7f6e544221c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:47:44.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:43.886+0000 7f6e544221c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:47:44.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:44.398+0000 7f6e544221c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:47:44.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[95848]: mgrmap e23: x(active, since 1.01509s) 2023-09-23T17:47:44.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[95848]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:44.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[95848]: [23/Sep/2023:17:47:43] ENGINE Bus STARTING 2023-09-23T17:47:44.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[95848]: [23/Sep/2023:17:47:43] ENGINE Serving on http://172.21.15.99:8765 2023-09-23T17:47:44.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[95848]: [23/Sep/2023:17:47:43] ENGINE Serving on https://172.21.15.99:7150 2023-09-23T17:47:44.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[95848]: [23/Sep/2023:17:47:43] ENGINE Bus STARTED 2023-09-23T17:47:44.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[109912]: mgrmap e23: x(active, since 1.01509s) 2023-09-23T17:47:44.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[109912]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:44.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[109912]: [23/Sep/2023:17:47:43] ENGINE Bus STARTING 2023-09-23T17:47:44.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[109912]: [23/Sep/2023:17:47:43] ENGINE Serving on http://172.21.15.99:8765 2023-09-23T17:47:44.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[109912]: [23/Sep/2023:17:47:43] ENGINE Serving on https://172.21.15.99:7150 2023-09-23T17:47:44.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:44 smithi055 ceph-mon[109912]: [23/Sep/2023:17:47:43] ENGINE Bus STARTED 2023-09-23T17:47:44.826 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:44] "GET /metrics HTTP/1.1" 200 31004 "" "Prometheus/2.43.0" 2023-09-23T17:47:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:44 smithi099 ceph-mon[100270]: mgrmap e23: x(active, since 1.01509s) 2023-09-23T17:47:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:44 smithi099 ceph-mon[100270]: pgmap v3: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:44 smithi099 ceph-mon[100270]: [23/Sep/2023:17:47:43] ENGINE Bus STARTING 2023-09-23T17:47:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:44 smithi099 ceph-mon[100270]: [23/Sep/2023:17:47:43] ENGINE Serving on http://172.21.15.99:8765 2023-09-23T17:47:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:44 smithi099 ceph-mon[100270]: [23/Sep/2023:17:47:43] ENGINE Serving on https://172.21.15.99:7150 2023-09-23T17:47:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:44 smithi099 ceph-mon[100270]: [23/Sep/2023:17:47:43] ENGINE Bus STARTED 2023-09-23T17:47:45.408 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:45 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:45.114+0000 7f6e544221c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:47:45.408 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:45 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:45.309+0000 7f6e544221c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:47:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:45 smithi055 ceph-mon[95848]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:45 smithi055 ceph-mon[95848]: mgrmap e24: x(active, since 2s) 2023-09-23T17:47:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:45 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/4205085407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:45.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:45 smithi055 ceph-mon[109912]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:45.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:45 smithi055 ceph-mon[109912]: mgrmap e24: x(active, since 2s) 2023-09-23T17:47:45.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:45 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/4205085407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:45 smithi099 ceph-mon[100270]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:45 smithi099 ceph-mon[100270]: mgrmap e24: x(active, since 2s) 2023-09-23T17:47:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:45 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/4205085407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:46.675 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:46 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:46.330+0000 7f6e544221c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:47:46.675 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:46 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:46.443+0000 7f6e544221c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:47:46.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:46 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:46.674+0000 7f6e544221c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:47:47.414 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:47.042+0000 7f6e544221c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:47:47.415 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:47.155+0000 7f6e544221c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:47:47.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:47.505+0000 7f6e544221c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:47:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:47 smithi055 ceph-mon[95848]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:47 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2069313803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:47 smithi055 ceph-mon[109912]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:47 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2069313803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:47.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:47 smithi099 ceph-mon[100270]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:47 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2069313803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:48.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:47.849+0000 7f6e544221c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:47:49.076 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:47:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:47:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.55:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:47:49.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:47:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:47:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephPGImbalance index=15 msg="Evaluating rule failed" rule="alert: CephPGImbalance\nexpr: abs(((ceph_osd_numpg > 0) - on (job) group_left () avg by (job) (ceph_osd_numpg\n > 0)) / on (job) group_left () avg by (job) (ceph_osd_numpg > 0)) * on (ceph_daemon)\n group_left (hostname) ceph_osd_metadata > 0.3\nfor: 5m\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.5\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} deviates by\n more than 30% from average PG count.\n summary: PGs are not balanced across OSDs\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"172.21.15.55:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:47:49.127 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:48 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:48.708+0000 7f6e544221c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:47:49.127 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:48 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:48.872+0000 7f6e544221c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:47:49.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:49.126+0000 7f6e544221c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:47:49.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:49.270+0000 7f6e544221c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:47:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi055 to 2501M 2023-09-23T17:47:49.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:49.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi055 to 2501M 2023-09-23T17:47:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:49.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi055 to 2501M 2023-09-23T17:47:49.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:49 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:49.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:49 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:49.553+0000 7f6e544221c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:47:50.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:50 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2185686997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:50 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:50 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:50.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2185686997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:50.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:50.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:50.612 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2185686997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:50.612 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:50.612 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:50.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:50.611+0000 7f6e544221c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:47:50.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:47:50] ENGINE Bus STARTING 2023-09-23T17:47:50.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: CherryPy Checker: 2023-09-23T17:47:50.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: The Application mounted at '' has an empty config. 2023-09-23T17:47:50.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: 2023-09-23T17:47:50.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:47:50] ENGINE Serving on http://:::9283 2023-09-23T17:47:50.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: [23/Sep/2023:17:47:50] ENGINE Bus STARTED 2023-09-23T17:47:51.201 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:47:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:47:51.455 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:51 smithi099 ceph-mon[100270]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:51.455 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:51 smithi099 ceph-mon[100270]: Standby manager daemon y started 2023-09-23T17:47:51.455 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:51 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T17:47:51.455 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:51 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:47:51.455 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:51 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T17:47:51.455 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:51 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:47:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[95848]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[95848]: Standby manager daemon y started 2023-09-23T17:47:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T17:47:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:47:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T17:47:51.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:47:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[109912]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[109912]: Standby manager daemon y started 2023-09-23T17:47:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T17:47:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:47:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T17:47:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:51 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.55:0/3062357337' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:47:52.547 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:52 smithi099 ceph-mon[100270]: mgrmap e25: x(active, since 8s), standbys: y 2023-09-23T17:47:52.547 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:52 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:47:52.547 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:52 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3757114103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:52.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:52 smithi055 ceph-mon[95848]: mgrmap e25: x(active, since 8s), standbys: y 2023-09-23T17:47:52.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:52 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:47:52.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:52 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3757114103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:52 smithi055 ceph-mon[109912]: mgrmap e25: x(active, since 8s), standbys: y 2023-09-23T17:47:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:52 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:47:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:52 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3757114103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: Adjusting osd_memory_target on smithi099 to 2500M 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:47:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:54.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: Adjusting osd_memory_target on smithi099 to 2500M 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:47:54.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:47:54.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:47:54.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:47:54.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:53 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:54.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-09-23T17:47:54.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: Adjusting osd_memory_target on smithi099 to 2500M 2023-09-23T17:47:54.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:47:54.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:47:54.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:47:54.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:47:54.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:47:54.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:47:54.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:53 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:54.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:47:54] "GET /metrics HTTP/1.1" 200 31004 "" "Prometheus/2.43.0" 2023-09-23T17:47:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:47:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:47:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:47:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:47:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T17:47:55.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: Reconfiguring daemon alertmanager.a on smithi055 2023-09-23T17:47:55.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/492068805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:47:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:47:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:47:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:47:55.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T17:47:55.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: Reconfiguring daemon alertmanager.a on smithi055 2023-09-23T17:47:55.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:54 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/492068805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:47:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:47:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:47:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:47:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: Reconfiguring alertmanager.a (dependencies changed)... 2023-09-23T17:47:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: Reconfiguring daemon alertmanager.a on smithi055 2023-09-23T17:47:55.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:54 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/492068805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:55.930 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:55 smithi055 systemd[1]: Stopping Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:47:56.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:55 smithi099 ceph-mon[100270]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:56.426 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:56 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[157537]: ts=2023-09-23T17:47:56.120Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T17:47:56.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:55 smithi055 ceph-mon[95848]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:56.427 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:55 smithi055 ceph-mon[109912]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:56.680 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:56 smithi055 bash[166410]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T17:47:56.680 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:56 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T17:47:56.681 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:56 smithi055 systemd[1]: Stopped Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:47:56.681 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:56 smithi055 systemd[1]: Starting Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:47:57.251 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:56 smithi055 podman[166553]: 2023-09-23T17:47:57.252 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:47:57.221Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-09-23T17:47:57.252 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:47:57.221Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-09-23T17:47:57.252 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:47:57.222Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.55 port=9094 2023-09-23T17:47:57.252 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:47:57.223Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-09-23T17:47:57.252 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:57 smithi055 bash[166553]: 5062112ad547ca3ac7857d2746f8c67921dac415243532ac8304da58ef456982 2023-09-23T17:47:57.680 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:57 smithi055 systemd[1]: Started Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:47:57.680 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:47:57.252Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T17:47:57.680 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:47:57.253Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-09-23T17:47:57.680 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:47:57.256Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-09-23T17:47:57.680 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:47:57.256Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-09-23T17:47:57.979 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:57 smithi055 systemd[1]: Stopping Ceph node-exporter.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:47:58.268 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-mon[95848]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:58.268 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2736810064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:58.268 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:58.268 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:58.268 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:58.268 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-mon[109912]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:58.268 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2736810064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:58.268 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:58.269 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:58.269 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:57 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:58.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:57 smithi099 ceph-mon[100270]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:47:58.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:57 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2736810064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:47:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:57 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:57 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:47:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:57 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:47:58.533 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:58 smithi055 bash[166728]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T17:47:58.533 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:58 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T17:47:58.792 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:58 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T17:47:58.792 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:58 smithi055 systemd[1]: Stopped Ceph node-exporter.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:47:58.792 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:58 smithi055 systemd[1]: Starting Ceph node-exporter.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:47:59.053 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:58 smithi055 podman[166847]: 2023-09-23T17:47:59.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:58 smithi055 ceph-mon[109912]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T17:47:59.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:58 smithi055 ceph-mon[109912]: Reconfiguring daemon node-exporter.a on smithi055 2023-09-23T17:47:59.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:58 smithi055 ceph-mon[95848]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T17:47:59.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:58 smithi055 ceph-mon[95848]: Reconfiguring daemon node-exporter.a on smithi055 2023-09-23T17:47:59.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:58 smithi099 ceph-mon[100270]: Reconfiguring node-exporter.a (dependencies changed)... 2023-09-23T17:47:59.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:58 smithi099 ceph-mon[100270]: Reconfiguring daemon node-exporter.a on smithi055 2023-09-23T17:47:59.429 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:47:59.224Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000291592s 2023-09-23T17:47:59.430 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.053Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T17:47:59.430 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.053Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T17:47:59.430 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.054Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T17:47:59.430 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.054Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.054Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.054Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T17:47:59.431 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T17:47:59.432 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T17:47:59.433 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T17:47:59.434 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.055Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T17:47:59.435 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.056Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T17:47:59.435 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a[166862]: ts=2023-09-23T17:47:59.056Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T17:47:59.435 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 bash[166847]: 6f05e33b3606f49d5cd68f5fa3b6573f3648211a7e4f75da12358c9a2abea1c7 2023-09-23T17:47:59.435 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 17:47:59 smithi055 systemd[1]: Started Ceph node-exporter.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:59 smithi099 ceph-mon[100270]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:59 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:59 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:47:59 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2218583942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:00.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-mon[95848]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:00.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:00.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:00.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2218583942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:00.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-mon[109912]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:00.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:47:59 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2218583942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:01.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:01 smithi099 ceph-mon[100270]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T17:48:01.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:01 smithi099 ceph-mon[100270]: Reconfiguring daemon prometheus.a on smithi099 2023-09-23T17:48:01.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[96092]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:48:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:48:01.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:01 smithi055 ceph-mon[95848]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T17:48:01.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:01 smithi055 ceph-mon[95848]: Reconfiguring daemon prometheus.a on smithi099 2023-09-23T17:48:01.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:01 smithi055 ceph-mon[109912]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T17:48:01.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:01 smithi055 ceph-mon[109912]: Reconfiguring daemon prometheus.a on smithi099 2023-09-23T17:48:02.290 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:02 smithi099 ceph-mon[100270]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:02.291 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:02 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2877455721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:02.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:02 smithi055 ceph-mon[95848]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:02.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:02 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2877455721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:02.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:02 smithi055 ceph-mon[109912]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:02.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:02 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2877455721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:04.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:04 smithi099 ceph-mon[100270]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:04 smithi055 ceph-mon[109912]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:04.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:04 smithi055 ceph-mon[95848]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:04.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:48:04] "GET /metrics HTTP/1.1" 200 33906 "" "Prometheus/2.43.0" 2023-09-23T17:48:05.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:05 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/962478656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:05.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:05 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/962478656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:05.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:05 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/962478656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:06.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:06 smithi099 ceph-mon[100270]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:06.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:06 smithi055 ceph-mon[109912]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:06.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:06 smithi055 ceph-mon[95848]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:07.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:07 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1891166834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:07.430 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:48:07 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:48:07.227Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.00405135s 2023-09-23T17:48:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:07 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1891166834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:07.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:07 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1891166834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:08.037 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:07 smithi099 systemd[1]: Stopping Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:08.300 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-mon[100270]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:08.300 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.085Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T17:48:08.301 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[138050]: ts=2023-09-23T17:48:08.086Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T17:48:08.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:08 smithi055 ceph-mon[95848]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:08.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:08 smithi055 ceph-mon[109912]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:08.576 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 bash[151055]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T17:48:08.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T17:48:08.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 systemd[1]: Stopped Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:08.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 systemd[1]: Starting Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:09.058 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:08 smithi099 podman[151175]: 2023-09-23T17:48:09.327 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.058Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T17:48:09.327 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 bash[151175]: 10b04d372ef3793adeac232e017b1692d1cc2e2b789b876299f707bc12a2e67a 2023-09-23T17:48:09.327 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.058Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T17:48:09.327 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.058Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi099 (none))" 2023-09-23T17:48:09.327 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.058Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T17:48:09.327 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.058Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T17:48:09.327 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.060Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.060Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.061Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.061Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.062Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.062Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.411µs 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.062Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.063Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=2 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.077Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=2 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.077Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=2 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.077Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=18.661µs wal_replay_duration=14.858889ms wbl_replay_duration=86ns total_replay_duration=14.894001ms 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.079Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T17:48:09.328 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.079Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T17:48:09.329 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.079Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T17:48:09.329 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.095Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.715668ms db_storage=738ns remote_storage=1.149µs web_handler=451ns query_engine=612ns scrape=234.565µs scrape_sd=106.26µs notify=10.211µs notify_sd=9.142µs rules=15.115487ms tracing=5.443µs 2023-09-23T17:48:09.329 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.095Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T17:48:09.329 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:09.095Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T17:48:09.329 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:09 smithi099 systemd[1]: Started Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:10 smithi099 ceph-mon[100270]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:10 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2662656988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:10 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:10 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:10 smithi099 ceph-mon[100270]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T17:48:10.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:10 smithi055 ceph-mon[95848]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:10.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:10 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2662656988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:10.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:10 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:10.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:10 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:10.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:10 smithi055 ceph-mon[95848]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T17:48:10.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:10 smithi055 ceph-mon[109912]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:10.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:10 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2662656988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:10.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:10 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:10.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:10 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:10.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:10 smithi055 ceph-mon[109912]: Reconfiguring grafana.a (dependencies changed)... 2023-09-23T17:48:11.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:11 smithi099 ceph-mon[100270]: Reconfiguring daemon grafana.a on smithi099 2023-09-23T17:48:11.327 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:11 smithi099 systemd[1]: Stopping Ceph grafana.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:11.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:11 smithi055 ceph-mon[95848]: Reconfiguring daemon grafana.a on smithi099 2023-09-23T17:48:11.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:11 smithi055 ceph-mon[109912]: Reconfiguring daemon grafana.a on smithi099 2023-09-23T17:48:11.744 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:11 smithi099 bash[151475]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T17:48:11.744 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:11 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T17:48:12.034 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:11 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T17:48:12.035 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:11 smithi099 systemd[1]: Stopped Ceph grafana.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:12.035 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:11 smithi099 systemd[1]: Starting Ceph grafana.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:12.327 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 podman[151597]: 2023-09-23T17:48:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-mon[100270]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1231989817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:12 smithi055 ceph-mon[95848]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:12 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1231989817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:12 smithi055 ceph-mon[109912]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:12 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1231989817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:12.827 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 bash[151597]: b140d15d52f251bf9bd676ecbfadefeda624261ac1f19b42b316cfa3bd6306ae 2023-09-23T17:48:12.827 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 systemd[1]: Started Ceph grafana.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:12.827 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619099723Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-09-23T17:48:12.827 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619272713Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-09-23T17:48:12.827 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619290359Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-09-23T17:48:12.827 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619297868Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-09-23T17:48:12.827 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619303956Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-09-23T17:48:12.827 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619310021Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619315776Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619321645Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619327576Z level=info msg="Path Home" path=/usr/share/grafana 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619333388Z level=info msg="Path Data" path=/var/lib/grafana 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619338855Z level=info msg="Path Logs" path=/var/log/grafana 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619344255Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619349993Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=settings t=2023-09-23T17:48:12.619355755Z level=info msg="App mode production" 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=sqlstore t=2023-09-23T17:48:12.619402766Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=sqlstore t=2023-09-23T17:48:12.619419114Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=migrator t=2023-09-23T17:48:12.633767873Z level=info msg="Starting DB migrations" 2023-09-23T17:48:12.828 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=migrator t=2023-09-23T17:48:12.638938096Z level=info msg="migrations completed" performed=0 skipped=481 duration=529.707µs 2023-09-23T17:48:12.829 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=sqlstore t=2023-09-23T17:48:12.639317562Z level=info msg="Created default organization" 2023-09-23T17:48:12.829 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=plugin.loader t=2023-09-23T17:48:12.698395888Z level=info msg="Plugin registered" pluginID=input 2023-09-23T17:48:12.829 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=plugin.loader t=2023-09-23T17:48:12.726211317Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-09-23T17:48:12.829 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=plugin.loader t=2023-09-23T17:48:12.726245469Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-09-23T17:48:12.829 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=secrets t=2023-09-23T17:48:12.726282157Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-09-23T17:48:12.829 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=query_data t=2023-09-23T17:48:12.727980759Z level=info msg="Query Service initialization" 2023-09-23T17:48:12.829 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=live.push_http t=2023-09-23T17:48:12.730408513Z level=info msg="Live Push Gateway initialization" 2023-09-23T17:48:13.326 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:12 smithi099 systemd[1]: Stopping Ceph node-exporter.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:13 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:13 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:13 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:13 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:13 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:13 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:13.658 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=infra.usagestats.collector t=2023-09-23T17:48:13.477324217Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-09-23T17:48:13.658 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=provisioning.datasources t=2023-09-23T17:48:13.565235901Z level=info msg="deleted correlations based on configuration" ds_name=Dashboard1 2023-09-23T17:48:13.658 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=provisioning.datasources t=2023-09-23T17:48:13.565272851Z level=info msg="deleted datasource based on configuration" name=Dashboard1 2023-09-23T17:48:13.658 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=provisioning.datasources t=2023-09-23T17:48:13.575564213Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-09-23T17:48:13.659 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:13 smithi099 bash[151773]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T17:48:13.659 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:13 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T17:48:14.005 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:13 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T17:48:14.005 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:13 smithi099 systemd[1]: Stopped Ceph node-exporter.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:14.005 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:13 smithi099 systemd[1]: Starting Ceph node-exporter.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:14.006 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=provisioning.alerting t=2023-09-23T17:48:13.658354259Z level=info msg="starting to provision alerting" 2023-09-23T17:48:14.006 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=provisioning.alerting t=2023-09-23T17:48:13.658381871Z level=info msg="finished to provision alerting" 2023-09-23T17:48:14.006 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=grafanaStorageLogger t=2023-09-23T17:48:13.658560671Z level=info msg="storage starting" 2023-09-23T17:48:14.006 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=ngalert.state.manager t=2023-09-23T17:48:13.658583406Z level=info msg="Warming state cache for startup" 2023-09-23T17:48:14.006 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=http.server t=2023-09-23T17:48:13.662724413Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-09-23T17:48:14.006 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=ngalert.state.manager t=2023-09-23T17:48:13.669594903Z level=info msg="State cache has been initialized" states=0 duration=11.007449ms 2023-09-23T17:48:14.006 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=ticker t=2023-09-23T17:48:13.669691417Z level=info msg=starting first_tick=2023-09-23T17:48:20Z 2023-09-23T17:48:14.006 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:48:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=ngalert.multiorg.alertmanager t=2023-09-23T17:48:13.66970665Z level=info msg="starting MultiOrg Alertmanager" 2023-09-23T17:48:14.302 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-mon[100270]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:14.302 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-mon[100270]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T17:48:14.303 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-mon[100270]: Reconfiguring daemon node-exporter.b on smithi099 2023-09-23T17:48:14.303 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2039951105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:14.303 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 podman[151897]: 2023-09-23T17:48:14.303 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.209Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-09-23T17:48:14.304 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.209Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-09-23T17:48:14.304 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.209Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-09-23T17:48:14.304 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.209Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-09-23T17:48:14.304 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-09-23T17:48:14.304 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-09-23T17:48:14.304 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-09-23T17:48:14.304 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=arp 2023-09-23T17:48:14.304 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=bcache 2023-09-23T17:48:14.304 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=bonding 2023-09-23T17:48:14.305 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=btrfs 2023-09-23T17:48:14.305 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=conntrack 2023-09-23T17:48:14.305 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=cpu 2023-09-23T17:48:14.305 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-09-23T17:48:14.305 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=diskstats 2023-09-23T17:48:14.305 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=dmi 2023-09-23T17:48:14.305 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=edac 2023-09-23T17:48:14.305 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=entropy 2023-09-23T17:48:14.305 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-09-23T17:48:14.306 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=filefd 2023-09-23T17:48:14.306 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=filesystem 2023-09-23T17:48:14.306 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=hwmon 2023-09-23T17:48:14.306 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=infiniband 2023-09-23T17:48:14.306 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=ipvs 2023-09-23T17:48:14.306 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=loadavg 2023-09-23T17:48:14.306 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=mdadm 2023-09-23T17:48:14.306 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=meminfo 2023-09-23T17:48:14.306 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=netclass 2023-09-23T17:48:14.306 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=netdev 2023-09-23T17:48:14.307 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=netstat 2023-09-23T17:48:14.307 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=nfs 2023-09-23T17:48:14.307 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=nfsd 2023-09-23T17:48:14.307 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=nvme 2023-09-23T17:48:14.307 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=os 2023-09-23T17:48:14.307 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-09-23T17:48:14.307 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=pressure 2023-09-23T17:48:14.307 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=rapl 2023-09-23T17:48:14.307 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=schedstat 2023-09-23T17:48:14.307 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=selinux 2023-09-23T17:48:14.308 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=sockstat 2023-09-23T17:48:14.308 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=softnet 2023-09-23T17:48:14.308 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=stat 2023-09-23T17:48:14.308 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=tapestats 2023-09-23T17:48:14.308 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=textfile 2023-09-23T17:48:14.308 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-09-23T17:48:14.308 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=time 2023-09-23T17:48:14.308 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-09-23T17:48:14.308 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=uname 2023-09-23T17:48:14.308 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=vmstat 2023-09-23T17:48:14.309 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=xfs 2023-09-23T17:48:14.309 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=node_exporter.go:117 level=info collector=zfs 2023-09-23T17:48:14.309 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.210Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-09-23T17:48:14.309 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b[151912]: ts=2023-09-23T17:48:14.211Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-09-23T17:48:14.309 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 bash[151897]: 3f7dcfc22edfd1e2b152643e52449bdbba08f2d343c461e3745980aff4f8d618 2023-09-23T17:48:14.309 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 17:48:14 smithi099 systemd[1]: Started Ceph node-exporter.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:14.309 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:48:14] ENGINE Bus STOPPING 2023-09-23T17:48:14.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:14 smithi055 ceph-mon[95848]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:14.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:14 smithi055 ceph-mon[95848]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T17:48:14.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:14 smithi055 ceph-mon[95848]: Reconfiguring daemon node-exporter.b on smithi099 2023-09-23T17:48:14.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:14 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2039951105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:14.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:14 smithi055 ceph-mon[109912]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:14.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:14 smithi055 ceph-mon[109912]: Reconfiguring node-exporter.b (dependencies changed)... 2023-09-23T17:48:14.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:14 smithi055 ceph-mon[109912]: Reconfiguring daemon node-exporter.b on smithi099 2023-09-23T17:48:14.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:14 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2039951105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:14.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:48:14] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T17:48:14.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:48:14] ENGINE Bus STOPPED 2023-09-23T17:48:14.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:48:14] ENGINE Bus STARTING 2023-09-23T17:48:14.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:48:14] ENGINE Serving on http://:::9283 2023-09-23T17:48:14.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:48:14] ENGINE Bus STARTED 2023-09-23T17:48:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:48:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:48:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T17:48:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T17:48:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:15.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:15.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:48:15.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:48:15.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:15.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T17:48:15.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T17:48:15.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:15 smithi099 ceph-mon[100270]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:15.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:15.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:15.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:48:15.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:48:15.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T17:48:15.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T17:48:15.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:15.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:15.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:15.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:48:15.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:48:15.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:15.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T17:48:15.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T17:48:15.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[95848]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:15.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:15.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:15.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:48:15.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi055.front.sepia.ceph.com:9093"}]: dispatch 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: Upgrade: Need to upgrade myself (mgr.x) 2023-09-23T17:48:15.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:15 smithi055 ceph-mon[109912]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:16.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[109912]: Upgrade: Updating mgr.y 2023-09-23T17:48:16.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:16.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:16.897 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:16.897 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:48:16.897 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:16.897 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[109912]: Deploying daemon mgr.y on smithi055 2023-09-23T17:48:16.897 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1877457317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:16.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[95848]: Upgrade: Updating mgr.y 2023-09-23T17:48:16.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:16.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:16.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:16.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:48:16.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:16.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[95848]: Deploying daemon mgr.y on smithi055 2023-09-23T17:48:16.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:16 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1877457317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:16 smithi099 ceph-mon[100270]: Upgrade: Updating mgr.y 2023-09-23T17:48:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:16 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:16 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:16 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:16 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:48:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:16 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:16 smithi099 ceph-mon[100270]: Deploying daemon mgr.y on smithi055 2023-09-23T17:48:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:16 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1877457317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:17.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:16 smithi055 systemd[1]: Stopping Ceph mgr.y for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:17.648 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:17 smithi055 bash[167458]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T17:48:17.648 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:17 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.y.service: Main process exited, code=exited, status=143/n/a 2023-09-23T17:48:17.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:17 smithi055 ceph-mon[109912]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:17.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:17 smithi055 ceph-mon[95848]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:17.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:17 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.y.service: Failed with result 'exit-code'. 2023-09-23T17:48:17.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:17 smithi055 systemd[1]: Stopped Ceph mgr.y for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:17.931 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:17 smithi055 systemd[1]: Starting Ceph mgr.y for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:18.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:17 smithi099 ceph-mon[100270]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:18.370 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:18 smithi055 podman[167607]: 2023-09-23T17:48:18.370 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:18 smithi055 bash[167607]: fc64ae873ea26ce1b29e30ba92b67d15f6123d965f8d817ffc693124b6f274d2 2023-09-23T17:48:18.370 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:18 smithi055 ceph-mgr[167647]: -- 172.21.15.55:0/3549444944 <== mon.2 v2:172.21.15.55:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560d82b3f1e0 con 0x560d82b14c00 2023-09-23T17:48:18.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:48:18] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.43.0" 2023-09-23T17:48:18.646 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:18 smithi055 systemd[1]: Started Ceph mgr.y for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:18.647 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:18 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:18.430+0000 7f2b266aa200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:48:18.647 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:18 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:18.533+0000 7f2b266aa200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:48:18.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:18 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:18.646+0000 7f2b266aa200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:48:18.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:18 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:18.765+0000 7f2b266aa200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:48:19.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:18.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:48:19.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:18 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:18.983+0000 7f2b266aa200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:48:19.762 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:19.488+0000 7f2b266aa200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:48:19.763 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:19.649+0000 7f2b266aa200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:48:19.763 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:19.761+0000 7f2b266aa200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:48:19.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-mon[95848]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:19.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:19.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:19.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:19.763 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1365040921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:19.764 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-mon[109912]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:19.764 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:19.764 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:19.764 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:19.764 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1365040921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:19 smithi099 ceph-mon[100270]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:19 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:19 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:19 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:19 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1365040921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:20.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:19 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:19.890+0000 7f2b266aa200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:48:20.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:20.428+0000 7f2b266aa200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:48:21.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:21 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:21.290+0000 7f2b266aa200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:48:21.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:21 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:21.395+0000 7f2b266aa200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:48:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:21 smithi099 ceph-mon[100270]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:21 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/971250150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:21 smithi055 ceph-mon[109912]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:21 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/971250150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:21 smithi055 ceph-mon[95848]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:21 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/971250150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:22.815 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:22.455+0000 7f2b266aa200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:48:22.815 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:22.574+0000 7f2b266aa200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:48:23.179 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:22.814+0000 7f2b266aa200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:48:23.456 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:23.186+0000 7f2b266aa200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:48:23.861 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:23.456+0000 7f2b266aa200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:48:23.861 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:23.652+0000 7f2b266aa200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:48:24.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:24.010+0000 7f2b266aa200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:48:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[95848]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1241587002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:24.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[109912]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1241587002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:23 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:23 smithi099 ceph-mon[100270]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:23 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:23 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:23 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:23 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:23 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1241587002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:23 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:23 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:25.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:24.906+0000 7f2b266aa200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:48:25.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:25.074+0000 7f2b266aa200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:48:25.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:25.337+0000 7f2b266aa200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:48:25.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:25.484+0000 7f2b266aa200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:48:26.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:25 smithi055 ceph-mon[95848]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:26.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:25.775+0000 7f2b266aa200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:48:26.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:25 smithi055 ceph-mon[109912]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:26.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:25 smithi099 ceph-mon[100270]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 79 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:27.124 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:26.887+0000 7f2b266aa200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:48:27.124 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:48:26] ENGINE Bus STARTING 2023-09-23T17:48:27.124 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: CherryPy Checker: 2023-09-23T17:48:27.124 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: The Application mounted at '' has an empty config. 2023-09-23T17:48:27.124 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.124 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:48:27] ENGINE Serving on http://:::9283 2023-09-23T17:48:27.124 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:48:27] ENGINE Bus STARTED 2023-09-23T17:48:27.124 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:48:27] ENGINE Bus STOPPING 2023-09-23T17:48:27.125 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:48:27] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T17:48:27.125 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:48:27] ENGINE Bus STOPPED 2023-09-23T17:48:27.125 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:48:27] ENGINE Bus STARTING 2023-09-23T17:48:27.125 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2186982006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.25483 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T17:48:27.126 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: mgrmap e26: y(active, starting, since 0.0150257s) 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: Active manager daemon y restarted 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: Activating manager daemon y 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: mgrmap e27: y(active, starting, since 0.00531924s) 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:48:27.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:48:27.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T17:48:27.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:48:27.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2186982006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:27.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:27.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:27.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:27.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:48:27.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.25483 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: mgrmap e26: y(active, starting, since 0.0150257s) 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: Active manager daemon y restarted 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: Activating manager daemon y 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: mgrmap e27: y(active, starting, since 0.00531924s) 2023-09-23T17:48:27.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:48:27.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:48:27.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:48:27.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:48:27.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:48:27.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:48:27.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:48:27.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T17:48:27.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:26 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:48:27.185 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:26.843+0000 7f29794c8700 -1 mgr handle_mgr_map I was active but no longer am 2023-09-23T17:48:27.185 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: ignoring --setuser ceph since I am not root 2023-09-23T17:48:27.185 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: ignoring --setgroup ceph since I am not root 2023-09-23T17:48:27.185 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mgr[140121]: -- 172.21.15.99:0/1666348076 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55e4eea871e0 con 0x55e4eea5c800 2023-09-23T17:48:27.185 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:27.009+0000 7fe4763b2200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:48:27.185 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:27.088+0000 7fe4763b2200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:48:27.186 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:27.185+0000 7fe4763b2200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:48:27.186 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2186982006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:27.186 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:27.186 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:27.186 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:27.186 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:48:27.186 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.25483 172.21.15.99:0/187635561' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd=[{"prefix": "mgr fail", "who": "x"}]: dispatch 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: osdmap e87: 8 total, 8 up, 8 in 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.25483 ' entity='mgr.x' cmd='[{"prefix": "mgr fail", "who": "x"}]': finished 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: mgrmap e26: y(active, starting, since 0.0150257s) 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: Active manager daemon y restarted 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: Activating manager daemon y 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: mgrmap e27: y(active, starting, since 0.00531924s) 2023-09-23T17:48:27.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:48:27.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:48:27.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:48:27.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:48:27.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:48:27.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:48:27.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:48:27.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-09-23T17:48:27.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:26 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:48:27.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.191+0000 7f2a83b1c700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:48:27.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.191+0000 7f2a83b1c700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:48:27.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.192+0000 7f2a83b1c700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:48:27.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.192+0000 7f2a83b1c700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:48:27.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.192+0000 7f2a83b1c700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:48:27.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.192+0000 7f2a80b16700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:48:27.431 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.192+0000 7f2a80b16700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:48:27.431 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.192+0000 7f2a80b16700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:48:27.431 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.192+0000 7f2a80b16700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:48:27.431 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:27.192+0000 7f2a80b16700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:48:27.431 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:48:27] ENGINE Serving on http://:::9283 2023-09-23T17:48:27.431 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:48:27] ENGINE Bus STARTED 2023-09-23T17:48:27.455 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:27.286+0000 7fe4763b2200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:48:27.456 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:27.455+0000 7fe4763b2200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:48:28.177 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:27.838+0000 7fe4763b2200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:48:28.177 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:27.978+0000 7fe4763b2200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:48:28.177 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:28.069+0000 7fe4763b2200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:48:28.177 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-mon[100270]: Manager daemon y is now available 2023-09-23T17:48:28.177 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:28.178 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:28.178 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:28.178 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:48:28.178 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:48:28.178 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:48:28.178 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:27 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:48:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[95848]: Manager daemon y is now available 2023-09-23T17:48:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:48:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:48:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:48:28.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:48:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[109912]: Manager daemon y is now available 2023-09-23T17:48:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:28.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:48:28.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:48:28.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:48:28.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:27 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:48:28.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:28.176+0000 7fe4763b2200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:48:29.023 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:28.625+0000 7fe4763b2200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:48:29.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:29 smithi099 ceph-mon[100270]: mgrmap e28: y(active, since 1.1079s) 2023-09-23T17:48:29.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:29 smithi099 ceph-mon[100270]: [23/Sep/2023:17:48:28] ENGINE Bus STARTING 2023-09-23T17:48:29.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:29 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2352210920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:29 smithi055 ceph-mon[109912]: mgrmap e28: y(active, since 1.1079s) 2023-09-23T17:48:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:29 smithi055 ceph-mon[109912]: [23/Sep/2023:17:48:28] ENGINE Bus STARTING 2023-09-23T17:48:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:29 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2352210920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:29.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:29 smithi055 ceph-mon[95848]: mgrmap e28: y(active, since 1.1079s) 2023-09-23T17:48:29.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:29 smithi055 ceph-mon[95848]: [23/Sep/2023:17:48:28] ENGINE Bus STARTING 2023-09-23T17:48:29.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:29 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2352210920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:29.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:29 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:29.329+0000 7fe4763b2200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:48:29.827 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:29 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:29.414+0000 7fe4763b2200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:48:30.324 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:30.231+0000 7fe4763b2200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:48:30.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:30 smithi099 ceph-mon[100270]: [23/Sep/2023:17:48:28] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:48:30.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:30 smithi099 ceph-mon[100270]: [23/Sep/2023:17:48:28] ENGINE Serving on http://172.21.15.55:8765 2023-09-23T17:48:30.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:30 smithi099 ceph-mon[100270]: [23/Sep/2023:17:48:28] ENGINE Bus STARTED 2023-09-23T17:48:30.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:30 smithi099 ceph-mon[100270]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:30.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:30 smithi099 ceph-mon[100270]: mgrmap e29: y(active, since 2s) 2023-09-23T17:48:30.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:30 smithi055 ceph-mon[95848]: [23/Sep/2023:17:48:28] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:48:30.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:30 smithi055 ceph-mon[95848]: [23/Sep/2023:17:48:28] ENGINE Serving on http://172.21.15.55:8765 2023-09-23T17:48:30.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:30 smithi055 ceph-mon[95848]: [23/Sep/2023:17:48:28] ENGINE Bus STARTED 2023-09-23T17:48:30.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:30 smithi055 ceph-mon[95848]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:30.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:30 smithi055 ceph-mon[95848]: mgrmap e29: y(active, since 2s) 2023-09-23T17:48:30.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:30 smithi055 ceph-mon[109912]: [23/Sep/2023:17:48:28] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:48:30.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:30 smithi055 ceph-mon[109912]: [23/Sep/2023:17:48:28] ENGINE Serving on http://172.21.15.55:8765 2023-09-23T17:48:30.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:30 smithi055 ceph-mon[109912]: [23/Sep/2023:17:48:28] ENGINE Bus STARTED 2023-09-23T17:48:30.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:30 smithi055 ceph-mon[109912]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:30.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:30 smithi055 ceph-mon[109912]: mgrmap e29: y(active, since 2s) 2023-09-23T17:48:30.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:30.324+0000 7fe4763b2200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:48:30.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:30.526+0000 7fe4763b2200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:48:31.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:31 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3111179824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:31.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:30.826+0000 7fe4763b2200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:48:31.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:31 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:31.068+0000 7fe4763b2200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:48:31.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:31 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3111179824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:31.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:31 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3111179824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:31.517 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:31 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:31.233+0000 7fe4763b2200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:48:31.826 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:31 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:31.517+0000 7fe4763b2200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:48:32.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:32 smithi099 ceph-mon[100270]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:32.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:32 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:32.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:32 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:32.327 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:32.265+0000 7fe4763b2200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:48:32.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:32 smithi055 ceph-mon[95848]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:32 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:32 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:32.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:32 smithi055 ceph-mon[109912]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:32.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:32 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:32.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:32 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:32.739 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:32.408+0000 7fe4763b2200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:48:32.739 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:32.617+0000 7fe4763b2200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:48:33.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:32.739+0000 7fe4763b2200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:48:33.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:32.971+0000 7fe4763b2200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:48:33.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-mon[100270]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[95848]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:33.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.673 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[109912]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:33.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:33.674 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:33 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:34.322 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:33.856+0000 7fe4763b2200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:48:34.323 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:48:33] ENGINE Bus STARTING 2023-09-23T17:48:34.323 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: CherryPy Checker: 2023-09-23T17:48:34.323 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: The Application mounted at '' has an empty config. 2023-09-23T17:48:34.323 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: 2023-09-23T17:48:34.323 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:48:33] ENGINE Serving on http://:::9283 2023-09-23T17:48:34.323 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: [23/Sep/2023:17:48:33] ENGINE Bus STARTED 2023-09-23T17:48:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/943506804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[95848]: Standby manager daemon x started 2023-09-23T17:48:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:48:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:48:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:48:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:48:34.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/943506804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:34.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T17:48:34.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:48:34.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:48:34.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:48:34.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:34 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:48:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:34 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/943506804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:34 smithi099 ceph-mon[100270]: Standby manager daemon x started 2023-09-23T17:48:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:34 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:48:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:34 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:48:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:34 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:48:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:34 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/593949320' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:48:35.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[109912]: mgrmap e30: y(active, since 7s), standbys: x 2023-09-23T17:48:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:48:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[109912]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:48:35.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[95848]: mgrmap e30: y(active, since 7s), standbys: x 2023-09-23T17:48:35.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:48:35.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[95848]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:35.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:35.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:35.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:35.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:35 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:48:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:35 smithi099 ceph-mon[100270]: mgrmap e30: y(active, since 7s), standbys: x 2023-09-23T17:48:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:35 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:48:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:35 smithi099 ceph-mon[100270]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:35 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:35 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:35 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:36.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:35 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:48:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[95848]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:48:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[95848]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:48:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2016523100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[95848]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:48:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[95848]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:48:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[95848]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:48:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[95848]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:48:36.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[109912]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:48:36.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[109912]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:48:36.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2016523100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:36.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[109912]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:48:36.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[109912]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:48:36.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[109912]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:48:36.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:36 smithi055 ceph-mon[109912]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:48:37.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:36 smithi099 ceph-mon[100270]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:48:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:36 smithi099 ceph-mon[100270]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:48:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:36 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2016523100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:36 smithi099 ceph-mon[100270]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:48:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:36 smithi099 ceph-mon[100270]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:48:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:36 smithi099 ceph-mon[100270]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:48:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:36 smithi099 ceph-mon[100270]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:48:37.912 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:37 smithi099 ceph-mon[100270]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:48:37.912 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:37 smithi099 ceph-mon[100270]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:48:37.912 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:37 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.913 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:37 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.913 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:37 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.913 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:37 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.913 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:37 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.913 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:37 smithi099 ceph-mon[100270]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T17:48:37.913 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:37 smithi099 ceph-mon[100270]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:37.913 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:37 smithi099 ceph-mon[100270]: Reconfiguring daemon prometheus.a on smithi099 2023-09-23T17:48:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[109912]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:48:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[109912]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:48:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[109912]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T17:48:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[109912]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[109912]: Reconfiguring daemon prometheus.a on smithi099 2023-09-23T17:48:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[95848]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:48:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[95848]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:48:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[95848]: Reconfiguring prometheus.a (dependencies changed)... 2023-09-23T17:48:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[95848]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:37.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:37 smithi055 ceph-mon[95848]: Reconfiguring daemon prometheus.a on smithi099 2023-09-23T17:48:38.463 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[140097]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:48:38] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-09-23T17:48:38.728 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:38 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3558298289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:38 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3558298289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:38.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:38 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3558298289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:39.317 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:38 smithi099 systemd[1]: Stopping Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:39.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.317Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T17:48:39.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.317Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T17:48:39.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.317Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T17:48:39.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.317Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T17:48:39.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.317Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T17:48:39.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.317Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T17:48:39.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.317Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T17:48:39.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.317Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T17:48:39.577 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.317Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T17:48:39.578 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.318Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T17:48:39.578 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.318Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T17:48:39.578 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[151190]: ts=2023-09-23T17:48:39.318Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T17:48:39.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:39 smithi099 ceph-mon[100270]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:39.837 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 bash[154595]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T17:48:39.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:39 smithi055 ceph-mon[95848]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:39.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:39 smithi055 ceph-mon[109912]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:40.126 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@prometheus.a.service: Succeeded. 2023-09-23T17:48:40.127 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 systemd[1]: Stopped Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:40.127 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:39 smithi099 systemd[1]: Starting Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:40.378 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 podman[154717]: 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 bash[154717]: 12edf856847a83922b72e9a9bfe8f20a959fb2002da0d7bc64735fe236d1f1e2 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.357Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.357Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.357Z caller=main.go:570 level=info host_details="(Linux 4.18.0-372.9.1.el8.x86_64 #1 SMP Fri Apr 15 22:12:19 EDT 2022 x86_64 smithi099 (none))" 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.357Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.357Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.358Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.359Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.359Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.359Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-09-23T17:48:40.379 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.362Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-09-23T17:48:40.380 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.363Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=1.77µs 2023-09-23T17:48:40.380 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.363Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-09-23T17:48:40.380 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.363Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=3 2023-09-23T17:48:40.380 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.375Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=3 2023-09-23T17:48:40.380 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.378Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=2 maxSegment=3 2023-09-23T17:48:40.380 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.379Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=3 maxSegment=3 2023-09-23T17:48:40.380 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.379Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=22.518µs wal_replay_duration=16.178522ms wbl_replay_duration=120ns total_replay_duration=16.219935ms 2023-09-23T17:48:40.826 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.381Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-09-23T17:48:40.827 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.381Z caller=main.go:1029 level=info msg="TSDB started" 2023-09-23T17:48:40.827 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.381Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-09-23T17:48:40.827 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.395Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=13.306844ms db_storage=829ns remote_storage=1.104µs web_handler=390ns query_engine=709ns scrape=175.522µs scrape_sd=78.14µs notify=15.98µs notify_sd=15.736µs rules=12.806136ms tracing=5.43µs 2023-09-23T17:48:40.827 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.395Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-09-23T17:48:40.827 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:40.395Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-09-23T17:48:40.827 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:40 smithi099 systemd[1]: Started Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:41.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:41 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:41.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:41 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:41.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:41 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:41.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:41 smithi099 ceph-mon[100270]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:41.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:41 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:41.703 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:41 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/4193333818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:41.703 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:41 smithi099 ceph-mon[100270]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:41.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:41.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:41.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:41.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[95848]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:41.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:41.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/4193333818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:41.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[95848]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:41.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:41.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:41.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:41.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[109912]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-09-23T17:48:41.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:41.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/4193333818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:41.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:41 smithi055 ceph-mon[109912]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:42.740 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:42 smithi099 ceph-mon[100270]: Upgrade: Updating mgr.x 2023-09-23T17:48:42.740 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:42 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:42.740 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:42 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:42.740 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:42 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:48:42.740 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:42 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:42.741 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:42 smithi099 ceph-mon[100270]: Deploying daemon mgr.x on smithi099 2023-09-23T17:48:42.741 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:42 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[95848]: Upgrade: Updating mgr.x 2023-09-23T17:48:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:48:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[95848]: Deploying daemon mgr.x on smithi099 2023-09-23T17:48:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:42.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[109912]: Upgrade: Updating mgr.x 2023-09-23T17:48:42.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:42.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:48:42.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:48:42.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:42.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[109912]: Deploying daemon mgr.x on smithi099 2023-09-23T17:48:42.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:42 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:43.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:42 smithi099 systemd[1]: Stopping Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:43.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:43 smithi099 ceph-mon[100270]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:43.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:43 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/1928736101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:43.820 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:43 smithi099 bash[155129]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T17:48:43.820 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:43 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-09-23T17:48:43.820 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:43 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.x.service: Failed with result 'exit-code'. 2023-09-23T17:48:43.820 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:43 smithi099 systemd[1]: Stopped Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:43.820 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:43 smithi099 systemd[1]: Starting Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:43.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:43 smithi055 ceph-mon[95848]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:43.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:43 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/1928736101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:43.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:43 smithi055 ceph-mon[109912]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:43.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:43 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/1928736101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:44.376 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:44 smithi099 podman[155279]: 2023-09-23T17:48:44.684 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:44 smithi099 bash[155279]: 32a5f5ae4a5535a6319fbecf7cf231c75fe7039b9ccf4125f7d8ff0688d6d419 2023-09-23T17:48:44.684 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:44 smithi099 ceph-mgr[155319]: -- 172.21.15.99:0/206211840 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5651a0a571e0 con 0x5651a0a2c800 2023-09-23T17:48:44.685 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:44 smithi099 systemd[1]: Started Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:44.685 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:44.500+0000 7fea2c1a3200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:48:44.685 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:44.590+0000 7fea2c1a3200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:48:44.961 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:44.683+0000 7fea2c1a3200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:48:44.961 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:44.784+0000 7fea2c1a3200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:48:44.962 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:44 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:44.960+0000 7fea2c1a3200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:48:45.701 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:45.358+0000 7fea2c1a3200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:48:45.701 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:45.496+0000 7fea2c1a3200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:48:45.701 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:45.591+0000 7fea2c1a3200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:48:45.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:45 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:45.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:45 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:45.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:45 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:45.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:45 smithi099 ceph-mon[100270]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:45.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:45 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2752504754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:45 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:45 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:45 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:45 smithi055 ceph-mon[95848]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:45 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2752504754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:45 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:45 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:45 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:45 smithi055 ceph-mon[109912]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:45 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2752504754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:46.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:45.699+0000 7fea2c1a3200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:48:46.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:46 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:46.155+0000 7fea2c1a3200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:48:47.326 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:46 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:46.862+0000 7fea2c1a3200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:48:47.327 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:46 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:46.956+0000 7fea2c1a3200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:48:48.069 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:47 smithi099 ceph-mon[100270]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:48.070 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:47 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/442741667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:48.070 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:47 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:47.777+0000 7fea2c1a3200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:48:48.070 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:47 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:47.877+0000 7fea2c1a3200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:48:48.326 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:48.067+0000 7fea2c1a3200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:48:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:47 smithi055 ceph-mon[95848]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:47 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/442741667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:48.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:47 smithi055 ceph-mon[109912]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:48.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:47 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/442741667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:48.766 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:48.372+0000 7fea2c1a3200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:48:48.767 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:48.610+0000 7fea2c1a3200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:48:48.767 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:48.765+0000 7fea2c1a3200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:48:49.326 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:49 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:49.075+0000 7fea2c1a3200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:48:50.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:49 smithi099 ceph-mon[100270]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:50.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:49 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:50.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:49 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:50.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:49 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:49.822+0000 7fea2c1a3200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:48:50.077 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:49 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:49.965+0000 7fea2c1a3200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:48:50.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:49 smithi055 ceph-mon[95848]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:50.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:49 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:50.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:49 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:50.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:50 smithi055 ceph-mon[109912]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:50.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:50 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:50.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:50 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:50.530 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:50 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:50.170+0000 7fea2c1a3200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:48:50.531 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:50 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:50.294+0000 7fea2c1a3200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:48:50.531 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:50 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:50.529+0000 7fea2c1a3200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:48:50.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:48:50] "GET /metrics HTTP/1.1" 200 33903 "" "Prometheus/2.43.0" 2023-09-23T17:48:51.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:51 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:51.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:51 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:51.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:51 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/2393318584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:51.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:51.458+0000 7fea2c1a3200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:48:51.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: [23/Sep/2023:17:48:51] ENGINE Bus STARTING 2023-09-23T17:48:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:51 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:51 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:51 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/2393318584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:51.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:51 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:51.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:51 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:51.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:51 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/2393318584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:51.831 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: CherryPy Checker: 2023-09-23T17:48:51.831 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: The Application mounted at '' has an empty config. 2023-09-23T17:48:51.831 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:51.831 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: [23/Sep/2023:17:48:51] ENGINE Serving on http://:::9283 2023-09-23T17:48:51.832 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:51 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: [23/Sep/2023:17:48:51] ENGINE Bus STARTED 2023-09-23T17:48:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:52 smithi099 ceph-mon[100270]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:52 smithi099 ceph-mon[100270]: Standby manager daemon x restarted 2023-09-23T17:48:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:52 smithi099 ceph-mon[100270]: Standby manager daemon x started 2023-09-23T17:48:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:52 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:48:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:52 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:48:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:52 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:48:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:52 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:48:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[109912]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[109912]: Standby manager daemon x restarted 2023-09-23T17:48:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T17:48:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:48:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:48:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:48:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:48:52.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[95848]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:52.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[95848]: Standby manager daemon x restarted 2023-09-23T17:48:52.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[95848]: Standby manager daemon x started 2023-09-23T17:48:52.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:48:52.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:48:52.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:48:52.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:52 smithi055 ceph-mon[95848]: from='mgr.? 172.21.15.99:0/4253652834' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:48:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: mgrmap e31: y(active, since 25s), standbys: x 2023-09-23T17:48:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3085013512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:48:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:48:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T17:48:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T17:48:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T17:48:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T17:48:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T17:48:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T17:48:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T17:48:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:53 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T17:48:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: mgrmap e31: y(active, since 25s), standbys: x 2023-09-23T17:48:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/3085013512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:48:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:48:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T17:48:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T17:48:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T17:48:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T17:48:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T17:48:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T17:48:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T17:48:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T17:48:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: mgrmap e31: y(active, since 25s), standbys: x 2023-09-23T17:48:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3085013512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:48:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:48:53.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:48:53.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:53.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T17:48:53.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]: dispatch 2023-09-23T17:48:53.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.x"}]': finished 2023-09-23T17:48:53.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T17:48:53.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]: dispatch 2023-09-23T17:48:53.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mgr.y"}]': finished 2023-09-23T17:48:53.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T17:48:53.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:53 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["a"]}]: dispatch 2023-09-23T17:48:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:54 smithi055 ceph-mon[109912]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:54 smithi055 ceph-mon[109912]: Upgrade: Setting container_image for all mgr 2023-09-23T17:48:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:54 smithi055 ceph-mon[109912]: Upgrade: It appears safe to stop mon.a 2023-09-23T17:48:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:54 smithi055 ceph-mon[95848]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:54 smithi055 ceph-mon[95848]: Upgrade: Setting container_image for all mgr 2023-09-23T17:48:54.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:54 smithi055 ceph-mon[95848]: Upgrade: It appears safe to stop mon.a 2023-09-23T17:48:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:54 smithi099 ceph-mon[100270]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:54 smithi099 ceph-mon[100270]: Upgrade: Setting container_image for all mgr 2023-09-23T17:48:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:54 smithi099 ceph-mon[100270]: Upgrade: It appears safe to stop mon.a 2023-09-23T17:48:55.867 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[109912]: Upgrade: Updating mon.a 2023-09-23T17:48:55.867 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:55.867 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:48:55.867 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:48:55.868 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:55.868 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[109912]: Deploying daemon mon.a on smithi055 2023-09-23T17:48:55.868 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[109912]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:55.868 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/637131030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:55.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[95848]: Upgrade: Updating mon.a 2023-09-23T17:48:55.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[95848]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:55.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:48:55.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:48:55.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[95848]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:55.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[95848]: Deploying daemon mon.a on smithi055 2023-09-23T17:48:55.869 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[95848]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:55.869 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:55 smithi055 ceph-mon[95848]: from='client.? 172.21.15.55:0/637131030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:55.869 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:55 smithi055 systemd[1]: Stopping Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:55 smithi099 ceph-mon[100270]: Upgrade: Updating mon.a 2023-09-23T17:48:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:55 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='mgr.y' 2023-09-23T17:48:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:55 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:48:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:55 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:48:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:55 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:48:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:55 smithi099 ceph-mon[100270]: Deploying daemon mon.a on smithi055 2023-09-23T17:48:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:55 smithi099 ceph-mon[100270]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:55 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/637131030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:56.169 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:56 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[95824]: 2023-09-23T17:48:56.167+0000 7f17646f5700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:48:56.169 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:56 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[95824]: 2023-09-23T17:48:56.167+0000 7f17646f5700 -1 mon.a@0(leader) e3 *** Got Signal Terminated *** 2023-09-23T17:48:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:56 smithi055 bash[179460]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T17:48:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:56 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.a.service: Succeeded. 2023-09-23T17:48:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:56 smithi055 systemd[1]: Stopped Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:56 smithi055 systemd[1]: Starting Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:48:57.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 podman[179607]: 2023-09-23T17:48:57.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T17:48:57.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T17:48:57.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: pidfile_write: ignore empty --pid-file 2023-09-23T17:48:57.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 bash[179607]: 1422ade2193a5ce112812b8f56cc172821d23edcfcc5f9d206af7832d0fd4c1c 2023-09-23T17:48:57.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: load: jerasure load: lrc 2023-09-23T17:48:57.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: RocksDB version: 7.9.2 2023-09-23T17:48:57.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Git sha 0 2023-09-23T17:48:57.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: DB SUMMARY 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: DB Session ID: FZSAVHHH6BG8H8KOR5FS 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: CURRENT file: CURRENT 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: MANIFEST file: MANIFEST-000015 size: 785 Bytes 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000025.log size: 2548839 ; 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.error_if_exists: 0 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.create_if_missing: 0 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.paranoid_checks: 1 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T17:48:57.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.env: 0x564877551720 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.info_log: 0x56487910d420 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.statistics: (nil) 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.use_fsync: 0 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_log_file_size: 0 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T17:48:57.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.allow_fallocate: 1 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.use_direct_reads: 0 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.db_log_dir: 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.wal_dir: 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T17:48:57.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.write_buffer_manager: 0x56487839e5a0 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T17:48:57.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.unordered_write: 0 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.row_cache: None 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.wal_filter: None 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.two_write_queues: 0 2023-09-23T17:48:57.344 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T17:48:57.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.wal_compression: 0 2023-09-23T17:48:57.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.atomic_flush: 0 2023-09-23T17:48:57.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T17:48:57.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T17:48:57.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T17:48:57.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.log_readahead_size: 0 2023-09-23T17:48:57.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T17:48:57.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T17:48:57.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T17:48:57.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T17:48:57.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_background_jobs: 2 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_background_compactions: -1 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_subcompactions: 1 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T17:48:57.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_open_files: -1 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_background_flushes: -1 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Compression algorithms supported: 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: kZSTD supported: 0 2023-09-23T17:48:57.348 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: kXpressCompression supported: 0 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: kZlibCompression supported: 1 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: kSnappyCompression supported: 1 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: kLZ4Compression supported: 1 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: kBZip2Compression supported: 0 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.merge_operator: 2023-09-23T17:48:57.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_filter: None 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x56487910d520) 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout: cache_index_and_filter_blocks: 1 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout: pin_top_level_index_and_filter: 1 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_type: 0 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout: data_block_index_type: 0 2023-09-23T17:48:57.350 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_shortening: 1 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: checksum: 4 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: no_block_cache: 0 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache: 0x56487841f350 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_name: BinnedLRUCache 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_options: 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: capacity : 536870912 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: num_shard_bits : 4 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: strict_capacity_limit : 0 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: high_pri_pool_ratio: 0.000 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_cache_compressed: (nil) 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: persistent_cache: (nil) 2023-09-23T17:48:57.351 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_size: 4096 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_size_deviation: 10 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_restart_interval: 16 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: index_block_restart_interval: 1 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: metadata_block_size: 4096 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: partition_filters: 0 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: use_delta_encoding: 1 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: filter_policy: bloomfilter 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: whole_key_filtering: 1 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: verify_compression: 0 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: read_amp_bytes_per_bit: 0 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: format_version: 5 2023-09-23T17:48:57.352 INFO:journalctl@ceph.mon.a.smithi055.stdout: enable_index_compression: 1 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout: block_align: 0 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout: max_auto_readahead_size: 262144 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout: prepopulate_block_cache: 0 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout: initial_auto_readahead_size: 8192 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compression: NoCompression 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.num_levels: 7 2023-09-23T17:48:57.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T17:48:57.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T17:48:57.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T17:48:57.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T17:48:57.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.table_properties_collectors: 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.inplace_update_support: 0 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.bloom_locality: 0 2023-09-23T17:48:57.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.max_successive_merges: 0 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.ttl: 2592000 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.enable_blob_files: false 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.min_blob_size: 0 2023-09-23T17:48:57.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000015 succeeded,manifest_file_number is 15, next_file_number is 29, last_sequence is 9993, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T17:48:57.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: e3c8ea17-9219-4534-b4c8-de259f78cf01 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695491337264649, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695491337272239, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 2302295, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 9994, "largest_seqno": 11303, "table_properties": {"data_size": 2296002, "index_size": 3802, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 1477, "raw_key_size": 13441, "raw_average_key_size": 23, "raw_value_size": 2283547, "raw_average_value_size": 3950, "num_data_blocks": 176, "num_entries": 578, "num_filter_entries": 578, "num_deletions": 2, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695491337, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "e3c8ea17-9219-4534-b4c8-de259f78cf01", "db_session_id": "FZSAVHHH6BG8H8KOR5FS", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695491337272333, "job": 1, "event": "recovery_finished"} 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5648784bc000 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: DB pointer 0x5648784a8000 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T17:48:57.361 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** DB Stats ** 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T17:48:57.362 INFO:journalctl@ceph.mon.a.smithi055.stdout: L0 1/0 2.20 MB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 598.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: L6 1/0 6.47 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: Sum 2/0 8.67 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 598.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 598.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 598.1 0.00 0.00 1 0.004 0 0 0.0 0.0 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:48:57.363 INFO:journalctl@ceph.mon.a.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: Flush(GB): cumulative 0.002, interval 0.002 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: Cumulative compaction: 0.00 GB write, 207.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: Interval compaction: 0.00 GB write, 207.66 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: Block cache BinnedLRUCache@0x56487841f350#7 capacity: 512.00 MB usage: 5.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.1e-05 secs_since: 0 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.53 KB,0.000292063%) IndexBlock(1,4.08 KB,0.000777841%) Misc(1,0.00 KB,0%) 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:48:57.364 INFO:journalctl@ceph.mon.a.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: starting mon.a rank 0 at public addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] at bind addrs [v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mon.a@-1(???) e3 preinit fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mon.a@-1(???).mds e9 new map 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mon.a@-1(???).mds e9 print_map 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout: e9 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout: legacy client fscid: 1 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout: Filesystem 'foo' (1) 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout: fs_name foo 2023-09-23T17:48:57.365 INFO:journalctl@ceph.mon.a.smithi055.stdout: epoch 8 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: created 2023-09-23T17:45:51.128752+0000 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: modified 2023-09-23T17:46:03.386449+0000 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: tableserver 0 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: root 0 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: session_timeout 60 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: session_autoclose 300 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: max_file_size 1099511627776 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: required_client_features {} 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: last_failure 0 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: last_failure_osd_epoch 0 2023-09-23T17:48:57.366 INFO:journalctl@ceph.mon.a.smithi055.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: max_mds 1 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: in 0 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: up {0=14658} 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: failed 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: damaged 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: stopped 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: data_pools [4] 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: metadata_pool 3 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: inline_data disabled 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: balancer 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: bal_rank_mask -1 2023-09-23T17:48:57.367 INFO:journalctl@ceph.mon.a.smithi055.stdout: standby_count_wanted 1 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout: [mds.foo.smithi055.exjftp{0:14658} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout: Standby daemons: 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout: 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout: [mds.foo.smithi099.djovrl{-1:24577} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.99:6832/2997750460,v1:172.21.15.99:6833/2997750460] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mon.a@-1(???).osd e88 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mon.a@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mon.a@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mon.a@-1(???).osd e88 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mon.a@-1(???).paxosservice(auth 1..18) refresh upgraded, format 0 -> 3 2023-09-23T17:48:57.368 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 systemd[1]: Started Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:48:58.223 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ignoring --setuser ceph since I am not root 2023-09-23T17:48:58.223 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ignoring --setgroup ceph since I am not root 2023-09-23T17:48:58.223 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mgr[167647]: -- 172.21.15.55:0/4140131170 <== mon.0 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5574b054b1e0 con 0x5574b0520800 2023-09-23T17:48:58.223 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:58 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:58.126+0000 7f3fc0534200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:48:58.223 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:58 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:58.223+0000 7f3fc0534200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:48:58.223 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4050651072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mon.a calling monitor election 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mgrmap e31: y(active, since 31s), standbys: x 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: mgrmap e32: y(active, since 31s), standbys: x 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: overall HEALTH_OK 2023-09-23T17:48:58.224 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[179668]: from='mgr.15759 ' entity='' 2023-09-23T17:48:58.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:58.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:58.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/4050651072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:58.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: mon.a calling monitor election 2023-09-23T17:48:58.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:48:58.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:48:58.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:48:58.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T17:48:58.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: mgrmap e31: y(active, since 31s), standbys: x 2023-09-23T17:48:58.225 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: mgrmap e32: y(active, since 31s), standbys: x 2023-09-23T17:48:58.226 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: overall HEALTH_OK 2023-09-23T17:48:58.226 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:57 smithi055 ceph-mon[109912]: from='mgr.15759 ' entity='' 2023-09-23T17:48:58.267 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: ignoring --setuser ceph since I am not root 2023-09-23T17:48:58.267 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: ignoring --setgroup ceph since I am not root 2023-09-23T17:48:58.267 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mgr[155319]: -- 172.21.15.99:0/3094656140 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55d8ac3951e0 con 0x55d8ac36b000 2023-09-23T17:48:58.267 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:58.097+0000 7f066d1d5200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-09-23T17:48:58.267 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:58.177+0000 7f066d1d5200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-09-23T17:48:58.267 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:58.267+0000 7f066d1d5200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 80 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: from='mgr.15759 172.21.15.55:0/2701181610' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/4050651072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: mon.a calling monitor election 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: osdmap e88: 8 total, 8 up, 8 in 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: mgrmap e31: y(active, since 31s), standbys: x 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: mgrmap e32: y(active, since 31s), standbys: x 2023-09-23T17:48:58.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: overall HEALTH_OK 2023-09-23T17:48:58.269 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:57 smithi099 ceph-mon[100270]: from='mgr.15759 ' entity='' 2023-09-23T17:48:58.529 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:58.360+0000 7f066d1d5200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:48:58.653 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:58 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:58.333+0000 7f3fc0534200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-09-23T17:48:58.653 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:58 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:58.445+0000 7f3fc0534200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-09-23T17:48:58.826 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:58.529+0000 7f066d1d5200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:48:58.827 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:48:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:48:58.659Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:48:58.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:58 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:58.653+0000 7f3fc0534200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-09-23T17:48:59.209 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:58.895+0000 7f066d1d5200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:48:59.209 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:59 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:59.026+0000 7f066d1d5200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:48:59.209 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:59 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:59.114+0000 7f066d1d5200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:48:59.209 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:59 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:59.209+0000 7f066d1d5200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:48:59.391 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:59.127+0000 7f3fc0534200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-09-23T17:48:59.391 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:59.282+0000 7f3fc0534200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-09-23T17:48:59.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:59.391+0000 7f3fc0534200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-09-23T17:48:59.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:48:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:48:59.508+0000 7f3fc0534200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-09-23T17:48:59.926 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 1 2023-09-23T17:48:59.968 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:48:59 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:48:59.629+0000 7f066d1d5200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:49:00.268 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:48:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3344046522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:49:00.269 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:48:59 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3344046522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:49:00.269 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:00.039+0000 7f3fc0534200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-09-23T17:49:00.270 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T17:49:00.282 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:48:59 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3344046522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:49:00.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:00 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:00.282+0000 7f066d1d5200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:49:00.577 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:00 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:00.366+0000 7f066d1d5200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:49:00.884 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:49:01.110 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:00.850+0000 7f3fc0534200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-09-23T17:49:01.110 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:00.968+0000 7f3fc0534200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-09-23T17:49:01.400 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:01 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:01.134+0000 7f066d1d5200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:49:01.401 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:01 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:01.226+0000 7f066d1d5200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:49:01.681 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:01 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:01.400+0000 7f066d1d5200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:49:02.057 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:01 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:01.681+0000 7f066d1d5200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:49:02.058 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:01 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:01.908+0000 7f066d1d5200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:49:02.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:02 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:02.002+0000 7f3fc0534200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:49:02.180 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:02 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:02.123+0000 7f3fc0534200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-09-23T17:49:02.326 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:02.057+0000 7f066d1d5200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:49:02.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:02 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:02.325+0000 7f066d1d5200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:49:02.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:02 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:02.351+0000 7f3fc0534200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-09-23T17:49:02.968 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:02 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:02.711+0000 7f3fc0534200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-09-23T17:49:03.326 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:03 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:03.030+0000 7f066d1d5200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:49:03.327 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:03 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:03.165+0000 7f066d1d5200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:49:03.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:02 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:02.969+0000 7f3fc0534200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-09-23T17:49:03.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:03 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:03.157+0000 7f3fc0534200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-09-23T17:49:03.706 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:03 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:03.364+0000 7f066d1d5200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:49:03.706 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:03 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:03.482+0000 7f066d1d5200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:49:03.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:03 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:03.498+0000 7f3fc0534200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-09-23T17:49:03.930 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:49:03 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:49:03.511Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi099.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.99:8443: connect: connection refused" 2023-09-23T17:49:03.930 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:49:03 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:49:03.511Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi055.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.55:8443: connect: connection refused" 2023-09-23T17:49:04.659 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:04.344+0000 7f3fc0534200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-09-23T17:49:04.659 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:04.508+0000 7f3fc0534200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-09-23T17:49:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-mon[100270]: Standby manager daemon x restarted 2023-09-23T17:49:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-mon[100270]: Standby manager daemon x started 2023-09-23T17:49:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:49:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:49:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:49:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-mon[100270]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:49:04.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:03 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:03.705+0000 7f066d1d5200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:49:04.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:04.573+0000 7f066d1d5200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:49:04.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: [23/Sep/2023:17:49:04] ENGINE Bus STARTING 2023-09-23T17:49:04.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: CherryPy Checker: 2023-09-23T17:49:04.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: The Application mounted at '' has an empty config. 2023-09-23T17:49:04.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: 2023-09-23T17:49:04.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: [23/Sep/2023:17:49:04] ENGINE Serving on http://:::9283 2023-09-23T17:49:04.828 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 17:49:04 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x[155295]: [23/Sep/2023:17:49:04] ENGINE Bus STARTED 2023-09-23T17:49:04.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:04.766+0000 7f3fc0534200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-09-23T17:49:04.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:04.907+0000 7f3fc0534200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-09-23T17:49:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[179668]: Standby manager daemon x restarted 2023-09-23T17:49:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[179668]: Standby manager daemon x started 2023-09-23T17:49:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[179668]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:49:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[179668]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:49:04.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[179668]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:49:04.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[179668]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:49:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[109912]: Standby manager daemon x restarted 2023-09-23T17:49:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[109912]: Standby manager daemon x started 2023-09-23T17:49:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-09-23T17:49:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-09-23T17:49:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-09-23T17:49:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:04 smithi055 ceph-mon[109912]: from='mgr.? 172.21.15.99:0/1032342394' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-09-23T17:49:05.669 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:05.189+0000 7f3fc0534200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-09-23T17:49:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:05 smithi055 ceph-mon[179668]: mgrmap e33: y(active, since 37s), standbys: x 2023-09-23T17:49:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:05 smithi055 ceph-mon[109912]: mgrmap e33: y(active, since 37s), standbys: x 2023-09-23T17:49:05.930 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:49:05 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:49:05.836Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[0] msg="Notify success" attempts=4 2023-09-23T17:49:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:05 smithi099 ceph-mon[100270]: mgrmap e33: y(active, since 37s), standbys: x 2023-09-23T17:49:06.517 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-c[109888]: 2023-09-23T17:49:06.286+0000 7efebc2c6700 -1 mon.c@2(peon).osd e89 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T17:49:06.517 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.260+0000 7f3fc0534200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-09-23T17:49:06.517 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:49:06] ENGINE Bus STARTING 2023-09-23T17:49:06.517 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.459+0000 7f3f1d025700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:49:06.518 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.459+0000 7f3f1d025700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:49:06.518 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.459+0000 7f3f1d025700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:49:06.518 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.459+0000 7f3f1d025700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:49:06.518 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.459+0000 7f3f1d025700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:49:06.518 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.460+0000 7f3f1b021700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:49:06.518 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.460+0000 7f3f1b021700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:49:06.518 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.460+0000 7f3f1b021700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:49:06.518 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.460+0000 7f3f1b021700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:49:06.518 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.460+0000 7f3f1b021700 -1 client.0 error registering admin socket command: (17) File exists 2023-09-23T17:49:06.519 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: CherryPy Checker: 2023-09-23T17:49:06.519 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: The Application mounted at '' has an empty config. 2023-09-23T17:49:06.519 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:06.519 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:49:06] ENGINE Serving on http://:::9283 2023-09-23T17:49:06.519 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:17:49:06] ENGINE Bus STARTED 2023-09-23T17:49:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-b[100222]: 2023-09-23T17:49:06.282+0000 7f2999cee700 -1 mon.b@1(peon).osd e89 update_from_paxos full map CRC mismatch, resetting to canonical 2023-09-23T17:49:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: Active manager daemon y restarted 2023-09-23T17:49:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: Activating manager daemon y 2023-09-23T17:49:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T17:49:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: mgrmap e34: y(active, starting, since 0.00896441s), standbys: x 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: failed to encode map e89 with expected crc 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: failed to encode map e89 with expected crc 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:49:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: Manager daemon y is now available 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:49:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:49:06.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:06.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:49:06.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:49:06.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: Active manager daemon y restarted 2023-09-23T17:49:06.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: Activating manager daemon y 2023-09-23T17:49:06.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T17:49:06.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: mgrmap e34: y(active, starting, since 0.00896441s), standbys: x 2023-09-23T17:49:06.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: failed to encode map e89 with expected crc 2023-09-23T17:49:06.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: failed to encode map e89 with expected crc 2023-09-23T17:49:06.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:49:06.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:49:06.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:49:06.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:49:06.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:49:06.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:49:06.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:49:06.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:49:06.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:49:06.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:49:06.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:49:06.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:49:06.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:49:06.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:49:06.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:49:06.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:49:06.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:49:06.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:49:06.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: Manager daemon y is now available 2023-09-23T17:49:06.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:49:06.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:49:06.937 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:06.937 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:49:06.937 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:49:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: Active manager daemon y restarted 2023-09-23T17:49:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: Activating manager daemon y 2023-09-23T17:49:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T17:49:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: mgrmap e34: y(active, starting, since 0.00896441s), standbys: x 2023-09-23T17:49:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: failed to encode map e89 with expected crc 2023-09-23T17:49:07.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: failed to encode map e89 with expected crc 2023-09-23T17:49:07.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:49:07.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:49:07.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:49:07.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi055.exjftp"}]: dispatch 2023-09-23T17:49:07.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi099.djovrl"}]: dispatch 2023-09-23T17:49:07.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-09-23T17:49:07.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-09-23T17:49:07.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:49:07.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: Manager daemon y is now available 2023-09-23T17:49:07.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:49:07.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:49:07.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:07.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-09-23T17:49:07.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-09-23T17:49:07.319 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T17:49:07.319 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (70s) - 3m 15.6M - 0.25.0 c8568f914cd2 5062112ad547 2023-09-23T17:49:07.320 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi099 *:3000 running (55s) - 3m 83.7M - 9.4.7 2c41d148cca3 b140d15d52f2 2023-09-23T17:49:07.320 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi055.exjftp smithi055 running (3m) - 3m 16.8M - 17.2.6-1123-g61431f6b 0db14aa01094 7a2127020d8b 2023-09-23T17:49:07.320 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi099.djovrl smithi099 running (3m) - 3m 17.0M - 17.2.6-1123-g61431f6b 0db14aa01094 5993fe8cebce 2023-09-23T17:49:07.320 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi099 *:8443,9283,8765 running (23s) - 7m 55.7M - 18.2.0-460-gbb0daeff e32c2a64c126 32a5f5ae4a55 2023-09-23T17:49:07.320 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (49s) - 8m 475M - 18.2.0-460-gbb0daeff e32c2a64c126 fc64ae873ea2 2023-09-23T17:49:07.320 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (8m) - 8m 95.2M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 3ea34fb03ea8 2023-09-23T17:49:07.320 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi099 running (7m) - 7m 73.2M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 5f0fb2e87ea1 2023-09-23T17:49:07.320 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (7m) - 7m 65.1M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 549a518b554d 2023-09-23T17:49:07.320 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (68s) - 3m 9068k - 1.5.0 0da6a335fe13 6f05e33b3606 2023-09-23T17:49:07.321 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi099 *:9100 running (53s) - 3m 9026k - 1.5.0 0da6a335fe13 3f7dcfc22edf 2023-09-23T17:49:07.321 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (6m) - 6m 75.7M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 15403d138a93 2023-09-23T17:49:07.321 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (6m) - 6m 88.6M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 ee80d8f5fddd 2023-09-23T17:49:07.321 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (6m) - 6m 72.3M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 7045fb31bd32 2023-09-23T17:49:07.321 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (5m) - 5m 64.5M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 33f748e0cdcf 2023-09-23T17:49:07.321 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi099 running (5m) - 5m 64.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ecb0e820fd90 2023-09-23T17:49:07.321 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi099 running (5m) - 5m 97.5M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 7e6bf34cfdb0 2023-09-23T17:49:07.321 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi099 running (4m) - 4m 72.9M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 827ec31827bb 2023-09-23T17:49:07.321 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi099 running (4m) - 4m 81.1M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 dcef78fdede8 2023-09-23T17:49:07.321 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi099 *:9095 running (27s) - 4m 39.0M - 2.43.0 a07b618ecd1d 12edf856847a 2023-09-23T17:49:07.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:49:07.303+0000 7f3f79c94700 -1 mgr.server handle_report got status from non-daemon mon.a 2023-09-23T17:49:07.767 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T17:49:08.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:07 smithi099 ceph-mon[100270]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:07 smithi099 ceph-mon[100270]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:07 smithi099 ceph-mon[100270]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:07 smithi099 ceph-mon[100270]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:07 smithi099 ceph-mon[100270]: mgrmap e35: y(active, since 1.023s), standbys: x 2023-09-23T17:49:08.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-mon[179668]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-mon[179668]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-mon[179668]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-mon[179668]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-mon[179668]: mgrmap e35: y(active, since 1.023s), standbys: x 2023-09-23T17:49:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-mon[109912]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-mon[109912]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-mon[109912]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-mon[109912]: failed to encode map e89 with expected crc 2023-09-23T17:49:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:07 smithi055 ceph-mon[109912]: mgrmap e35: y(active, since 1.023s), standbys: x 2023-09-23T17:49:08.480 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:49:08.834 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 17:49:08 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T17:49:08.566Z caller=notify.go:743 level=info component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify success" attempts=5 2023-09-23T17:49:08.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:08 smithi055 ceph-mon[179668]: [23/Sep/2023:17:49:07] ENGINE Bus STARTING 2023-09-23T17:49:08.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:08 smithi055 ceph-mon[179668]: [23/Sep/2023:17:49:07] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:49:08.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:08 smithi055 ceph-mon[179668]: [23/Sep/2023:17:49:07] ENGINE Serving on http://172.21.15.55:8765 2023-09-23T17:49:08.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:08 smithi055 ceph-mon[179668]: [23/Sep/2023:17:49:07] ENGINE Bus STARTED 2023-09-23T17:49:08.835 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:08 smithi055 ceph-mon[109912]: [23/Sep/2023:17:49:07] ENGINE Bus STARTING 2023-09-23T17:49:08.835 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:08 smithi055 ceph-mon[109912]: [23/Sep/2023:17:49:07] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:49:08.835 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:08 smithi055 ceph-mon[109912]: [23/Sep/2023:17:49:07] ENGINE Serving on http://172.21.15.55:8765 2023-09-23T17:49:08.836 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:08 smithi055 ceph-mon[109912]: [23/Sep/2023:17:49:07] ENGINE Bus STARTED 2023-09-23T17:49:09.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:08 smithi099 ceph-mon[100270]: [23/Sep/2023:17:49:07] ENGINE Bus STARTING 2023-09-23T17:49:09.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:08 smithi099 ceph-mon[100270]: [23/Sep/2023:17:49:07] ENGINE Serving on https://172.21.15.55:7150 2023-09-23T17:49:09.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:08 smithi099 ceph-mon[100270]: [23/Sep/2023:17:49:07] ENGINE Serving on http://172.21.15.55:8765 2023-09-23T17:49:09.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:08 smithi099 ceph-mon[100270]: [23/Sep/2023:17:49:07] ENGINE Bus STARTED 2023-09-23T17:49:10.057 INFO:teuthology.orchestra.run.smithi055.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:49:10.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:09 smithi099 ceph-mon[100270]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:10.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:09 smithi099 ceph-mon[100270]: mgrmap e36: y(active, since 2s), standbys: x 2023-09-23T17:49:10.081 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:09 smithi055 ceph-mon[109912]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:10.081 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:09 smithi055 ceph-mon[109912]: mgrmap e36: y(active, since 2s), standbys: x 2023-09-23T17:49:10.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:09 smithi055 ceph-mon[179668]: pgmap v4: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:10.082 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:09 smithi055 ceph-mon[179668]: mgrmap e36: y(active, since 2s), standbys: x 2023-09-23T17:49:10.639 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:49:10] "GET /metrics HTTP/1.1" 200 31219 "" "Prometheus/2.43.0" 2023-09-23T17:49:10.640 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T17:49:11.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:11 smithi099 ceph-mon[100270]: from='client.25844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:49:11.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:11 smithi099 ceph-mon[100270]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:49:11.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:11 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:11.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:11 smithi099 ceph-mon[100270]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:11.352 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:49:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:11 smithi055 ceph-mon[179668]: from='client.25844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:49:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:11 smithi055 ceph-mon[179668]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:49:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:11.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:11 smithi055 ceph-mon[179668]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:11.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:11 smithi055 ceph-mon[109912]: from='client.25844 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:49:11.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:11 smithi055 ceph-mon[109912]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:49:11.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:11 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:11.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:11 smithi055 ceph-mon[109912]: pgmap v5: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:12 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:12 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:12 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:12 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:12 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:12.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:12 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:13 smithi099 ceph-mon[100270]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:13 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:14.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:13 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:14.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:13 smithi055 ceph-mon[179668]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:14.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:14.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:14.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:13 smithi055 ceph-mon[109912]: pgmap v6: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:14.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:13 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:14.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:13 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:15 smithi099 ceph-mon[100270]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:15 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:15 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:15 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:49:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:15 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:49:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:15 smithi055 ceph-mon[179668]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:49:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:49:16.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:15 smithi055 ceph-mon[109912]: pgmap v7: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:16.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:15 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:16.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:15 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:16.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:15 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:49:16.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:15 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:49:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:49:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:49:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:49:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:49:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:49:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:49:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:49:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:49:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:49:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:49:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:49:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:16 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:49:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:49:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:49:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:49:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:49:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:49:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:49:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: Updating smithi055:/etc/ceph/ceph.conf 2023-09-23T17:49:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: Updating smithi099:/etc/ceph/ceph.conf 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.conf 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: Updating smithi055:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: Updating smithi099:/etc/ceph/ceph.client.admin.keyring 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: Updating smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: Updating smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/config/ceph.client.admin.keyring 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:17.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:49:17.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:49:17.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:49:17.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:16 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:49:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:17 smithi099 ceph-mon[100270]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:18.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:17 smithi055 ceph-mon[179668]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:18.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:17 smithi055 ceph-mon[109912]: pgmap v8: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:19.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:49:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:49:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:49:20.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:19 smithi099 ceph-mon[100270]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:19 smithi055 ceph-mon[109912]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:19 smithi055 ceph-mon[179668]: pgmap v9: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:20.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:49:20] "GET /metrics HTTP/1.1" 200 31219 "" "Prometheus/2.43.0" 2023-09-23T17:49:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:21 smithi099 ceph-mon[100270]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:21 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:22.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:21 smithi055 ceph-mon[179668]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:22.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:22.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:21 smithi055 ceph-mon[109912]: pgmap v10: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:22.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:21 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:23 smithi099 ceph-mon[100270]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:23.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:23 smithi055 ceph-mon[179668]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:23 smithi055 ceph-mon[109912]: pgmap v11: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:25.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:25 smithi055 ceph-mon[179668]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:25 smithi055 ceph-mon[109912]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:25 smithi099 ceph-mon[100270]: pgmap v12: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:27.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:27 smithi055 ceph-mon[179668]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:27.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:27 smithi055 ceph-mon[109912]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:27 smithi099 ceph-mon[100270]: pgmap v13: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:29.076 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:49:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:49:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:49:29.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:29 smithi055 ceph-mon[109912]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:29.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:29 smithi055 ceph-mon[179668]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:29.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:29 smithi099 ceph-mon[100270]: pgmap v14: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:30.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:49:30] "GET /metrics HTTP/1.1" 200 33899 "" "Prometheus/2.43.0" 2023-09-23T17:49:31.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:31 smithi055 ceph-mon[179668]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:31.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:31 smithi055 ceph-mon[109912]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:31 smithi099 ceph-mon[100270]: pgmap v15: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:33 smithi099 ceph-mon[100270]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:33 smithi055 ceph-mon[109912]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:33 smithi055 ceph-mon[179668]: pgmap v16: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:35.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:35 smithi055 ceph-mon[179668]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:35.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:35 smithi055 ceph-mon[109912]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:35.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:35 smithi099 ceph-mon[100270]: pgmap v17: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:37 smithi055 ceph-mon[179668]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:37 smithi055 ceph-mon[109912]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:37 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:37 smithi099 ceph-mon[100270]: pgmap v18: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:37 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:39.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:49:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:49:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:49:39.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:39 smithi055 ceph-mon[109912]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:39.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:39 smithi055 ceph-mon[179668]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:39 smithi099 ceph-mon[100270]: pgmap v19: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:40.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:49:40] "GET /metrics HTTP/1.1" 200 33894 "" "Prometheus/2.43.0" 2023-09-23T17:49:41.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:41 smithi055 ceph-mon[179668]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:41 smithi055 ceph-mon[109912]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:41 smithi099 ceph-mon[100270]: pgmap v20: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:43 smithi099 ceph-mon[100270]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:43 smithi055 ceph-mon[109912]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:43 smithi055 ceph-mon[179668]: pgmap v21: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:45.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:45 smithi055 ceph-mon[109912]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:45 smithi055 ceph-mon[179668]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:45 smithi099 ceph-mon[100270]: pgmap v22: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:47 smithi055 ceph-mon[179668]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:47.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:47 smithi055 ceph-mon[109912]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:47 smithi099 ceph-mon[100270]: pgmap v23: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:49.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:49:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:49:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:49:49.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:49 smithi055 ceph-mon[179668]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:49.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:49 smithi055 ceph-mon[109912]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:49.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:49 smithi099 ceph-mon[100270]: pgmap v24: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:50.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:49:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:49:50] "GET /metrics HTTP/1.1" 200 33894 "" "Prometheus/2.43.0" 2023-09-23T17:49:51.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:51 smithi055 ceph-mon[109912]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:51.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:51 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:51 smithi055 ceph-mon[179668]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:51 smithi099 ceph-mon[100270]: pgmap v25: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:51 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:49:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:53 smithi099 ceph-mon[100270]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:53 smithi055 ceph-mon[179668]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:53 smithi055 ceph-mon[109912]: pgmap v26: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:55.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:55 smithi055 ceph-mon[179668]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:55.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:55 smithi055 ceph-mon[109912]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:55.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:55 smithi099 ceph-mon[100270]: pgmap v27: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:57 smithi055 ceph-mon[109912]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:57.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:57 smithi055 ceph-mon[179668]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:57 smithi099 ceph-mon[100270]: pgmap v28: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:59.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:49:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:49:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:49:59.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:49:59 smithi055 ceph-mon[179668]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:59.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:49:59 smithi055 ceph-mon[109912]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:49:59.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:49:59 smithi099 ceph-mon[100270]: pgmap v29: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:00.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:50:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:50:00] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2023-09-23T17:50:00.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:00 smithi055 ceph-mon[109912]: overall HEALTH_OK 2023-09-23T17:50:00.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:00 smithi055 ceph-mon[179668]: overall HEALTH_OK 2023-09-23T17:50:00.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:00 smithi099 ceph-mon[100270]: overall HEALTH_OK 2023-09-23T17:50:01.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:01 smithi055 ceph-mon[179668]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:01.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:01 smithi055 ceph-mon[109912]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:01.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:01 smithi099 ceph-mon[100270]: pgmap v30: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:03.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:03 smithi099 ceph-mon[100270]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:03 smithi055 ceph-mon[109912]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:03 smithi055 ceph-mon[179668]: pgmap v31: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:05.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:05 smithi055 ceph-mon[179668]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:05 smithi055 ceph-mon[109912]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:05 smithi099 ceph-mon[100270]: pgmap v32: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:06 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:06.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:07 smithi055 ceph-mon[179668]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:07 smithi055 ceph-mon[109912]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:07 smithi099 ceph-mon[100270]: pgmap v33: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:09.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:50:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:50:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:50:09.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:09 smithi055 ceph-mon[179668]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:09.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:09 smithi055 ceph-mon[109912]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:09 smithi099 ceph-mon[100270]: pgmap v34: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:10.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:50:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:50:10] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.43.0" 2023-09-23T17:50:11.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:11 smithi055 ceph-mon[109912]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:11.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:11 smithi055 ceph-mon[179668]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:11 smithi099 ceph-mon[100270]: pgmap v35: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:12.958 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T17:50:13.571 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:50:13.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:13 smithi099 ceph-mon[100270]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:13.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:13 smithi055 ceph-mon[179668]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:13.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:13 smithi055 ceph-mon[109912]: pgmap v36: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:15.011 INFO:teuthology.orchestra.run.smithi055.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:50:15.418 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for majority of mons to upgrade' 2023-09-23T17:50:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:15 smithi099 ceph-mon[100270]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:15 smithi099 ceph-mon[100270]: from='client.25850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:50:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:15 smithi099 ceph-mon[100270]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:50:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:15 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:15 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:50:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:15 smithi055 ceph-mon[179668]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:15 smithi055 ceph-mon[179668]: from='client.25850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:50:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:15 smithi055 ceph-mon[179668]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:50:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:50:15.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:15 smithi055 ceph-mon[109912]: pgmap v37: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:15 smithi055 ceph-mon[109912]: from='client.25850 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:50:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:15 smithi055 ceph-mon[109912]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:50:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:15 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:15 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:50:16.047 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:50:17.394 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:17 smithi055 ceph-mon[179668]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:17.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:17 smithi055 ceph-mon[109912]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:17.682 INFO:teuthology.orchestra.run.smithi055.stdout:wait for majority of mons to upgrade 2023-09-23T17:50:17.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:17 smithi099 ceph-mon[100270]: pgmap v38: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:18.099 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | egrep '"'"': [23]'"'"' ; do sleep 2 ; done' 2023-09-23T17:50:18.711 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:50:19.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:50:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:50:18.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:50:19.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:19 smithi055 ceph-mon[179668]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:19.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:19 smithi055 ceph-mon[109912]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:19.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:19 smithi099 ceph-mon[100270]: pgmap v39: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:20.674 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:50:20] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.43.0" 2023-09-23T17:50:20.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:20.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:20.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:20.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:20.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3451973131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:50:20.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:20.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:20.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:20.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:20.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:20 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/3451973131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:50:21.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:20 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:21.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:20 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:21.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:20 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:21.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:20 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:21.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:20 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/3451973131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:50:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:21 smithi099 ceph-mon[100270]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:21 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:21 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:50:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:21 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:21 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:50:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:21 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:50:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:21 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T17:50:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:21 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T17:50:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:21 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:22.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[109912]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:22.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:22.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:50:22.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:22.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:50:22.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:50:22.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T17:50:22.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T17:50:22.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:22.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[179668]: pgmap v40: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:22.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:22.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:50:22.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:22.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:50:22.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:50:22.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T17:50:22.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["c"]}]: dispatch 2023-09-23T17:50:22.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:23.031 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:22 smithi055 ceph-mon[179668]: Upgrade: It appears safe to stop mon.c 2023-09-23T17:50:23.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:23.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:50:23.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:50:23.032 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:23.032 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:22 smithi055 ceph-mon[109912]: Upgrade: It appears safe to stop mon.c 2023-09-23T17:50:23.032 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:22 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:23.032 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:22 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:50:23.032 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:22 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:50:23.032 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:22 smithi055 ceph-mon[109912]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:22 smithi099 ceph-mon[100270]: Upgrade: It appears safe to stop mon.c 2023-09-23T17:50:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:22 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:22 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:50:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:22 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:50:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:22 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:23.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:23 smithi055 ceph-mon[179668]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:23.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:23 smithi055 ceph-mon[179668]: Upgrade: Updating mon.c 2023-09-23T17:50:23.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:23 smithi055 ceph-mon[179668]: Deploying daemon mon.c on smithi055 2023-09-23T17:50:23.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:23 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/57664745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:50:23.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:23 smithi055 systemd[1]: Stopping Ceph mon.c for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:50:23.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:23 smithi055 ceph-mon[109912]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:23.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:23 smithi055 ceph-mon[109912]: Upgrade: Updating mon.c 2023-09-23T17:50:23.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:23 smithi055 ceph-mon[109912]: Deploying daemon mon.c on smithi055 2023-09-23T17:50:23.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:23 smithi055 ceph-mon[109912]: from='client.? 172.21.15.55:0/57664745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:50:24.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:23 smithi099 ceph-mon[100270]: pgmap v41: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:23 smithi099 ceph-mon[100270]: Upgrade: Updating mon.c 2023-09-23T17:50:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:23 smithi099 ceph-mon[100270]: Deploying daemon mon.c on smithi055 2023-09-23T17:50:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:23 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/57664745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:50:24.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-c[109888]: 2023-09-23T17:50:23.992+0000 7efec1ad1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:50:24.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-c[109888]: 2023-09-23T17:50:23.992+0000 7efec1ad1700 -1 mon.c@2(peon) e3 *** Got Signal Terminated *** 2023-09-23T17:50:24.811 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:24 smithi055 bash[183951]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T17:50:25.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:24 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T17:50:25.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:24 smithi055 systemd[1]: Stopped Ceph mon.c for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:50:25.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:24 smithi055 systemd[1]: Starting Ceph mon.c for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:50:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 podman[184131]: 2023-09-23T17:50:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 bash[184131]: e1bc5b196d92f277f4450c8a3e39e05db98139ffe9abbaad2c7dd6589a1a9c08 2023-09-23T17:50:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T17:50:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T17:50:25.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: pidfile_write: ignore empty --pid-file 2023-09-23T17:50:25.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: load: jerasure load: lrc 2023-09-23T17:50:25.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: RocksDB version: 7.9.2 2023-09-23T17:50:25.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Git sha 0 2023-09-23T17:50:25.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T17:50:25.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: DB SUMMARY 2023-09-23T17:50:25.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: DB Session ID: TCW2TVGSUFW064G7FYSN 2023-09-23T17:50:25.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: CURRENT file: CURRENT 2023-09-23T17:50:25.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T17:50:25.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: MANIFEST file: MANIFEST-000009 size: 897 Bytes 2023-09-23T17:50:25.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 1, files: 000024.sst 2023-09-23T17:50:25.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000022.log size: 479168 ; 2023-09-23T17:50:25.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.error_if_exists: 0 2023-09-23T17:50:25.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.create_if_missing: 0 2023-09-23T17:50:25.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.paranoid_checks: 1 2023-09-23T17:50:25.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T17:50:25.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T17:50:25.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T17:50:25.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.env: 0x562ba8c39720 2023-09-23T17:50:25.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.info_log: 0x562bab8b7420 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.statistics: (nil) 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.use_fsync: 0 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_log_file_size: 0 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.allow_fallocate: 1 2023-09-23T17:50:25.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T17:50:25.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T17:50:25.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.use_direct_reads: 0 2023-09-23T17:50:25.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T17:50:25.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T17:50:25.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.db_log_dir: 2023-09-23T17:50:25.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.wal_dir: 2023-09-23T17:50:25.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T17:50:25.686 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T17:50:25.686 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T17:50:25.686 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T17:50:25.686 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T17:50:25.686 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T17:50:25.686 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T17:50:25.686 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T17:50:25.686 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.write_buffer_manager: 0x562baab485a0 2023-09-23T17:50:25.686 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.unordered_write: 0 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T17:50:25.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T17:50:25.688 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T17:50:25.688 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.row_cache: None 2023-09-23T17:50:25.688 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.wal_filter: None 2023-09-23T17:50:25.688 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T17:50:25.688 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T17:50:25.688 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.two_write_queues: 0 2023-09-23T17:50:25.688 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T17:50:25.688 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.wal_compression: 0 2023-09-23T17:50:25.688 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.atomic_flush: 0 2023-09-23T17:50:25.688 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.log_readahead_size: 0 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T17:50:25.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_background_jobs: 2 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_background_compactions: -1 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_subcompactions: 1 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T17:50:25.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_open_files: -1 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_background_flushes: -1 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Compression algorithms supported: 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: kZSTD supported: 0 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: kXpressCompression supported: 0 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T17:50:25.691 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: kZlibCompression supported: 1 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: kSnappyCompression supported: 1 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: kLZ4Compression supported: 1 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: kBZip2Compression supported: 0 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.merge_operator: 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_filter: None 2023-09-23T17:50:25.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x562bab8b7520) 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout: cache_index_and_filter_blocks: 1 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout: pin_top_level_index_and_filter: 1 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_type: 0 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout: data_block_index_type: 0 2023-09-23T17:50:25.693 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_shortening: 1 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: checksum: 4 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: no_block_cache: 0 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache: 0x562baabc91f0 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_name: BinnedLRUCache 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_options: 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: capacity : 536870912 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: num_shard_bits : 4 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: strict_capacity_limit : 0 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: high_pri_pool_ratio: 0.000 2023-09-23T17:50:25.694 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_cache_compressed: (nil) 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: persistent_cache: (nil) 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_size: 4096 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_size_deviation: 10 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_restart_interval: 16 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: index_block_restart_interval: 1 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: metadata_block_size: 4096 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: partition_filters: 0 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: use_delta_encoding: 1 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: filter_policy: bloomfilter 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: whole_key_filtering: 1 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: verify_compression: 0 2023-09-23T17:50:25.695 INFO:journalctl@ceph.mon.c.smithi055.stdout: read_amp_bytes_per_bit: 0 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout: format_version: 5 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout: enable_index_compression: 1 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout: block_align: 0 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout: max_auto_readahead_size: 262144 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout: prepopulate_block_cache: 0 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout: initial_auto_readahead_size: 8192 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compression: NoCompression 2023-09-23T17:50:25.696 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.num_levels: 7 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T17:50:25.697 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T17:50:25.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T17:50:25.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T17:50:25.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T17:50:25.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T17:50:25.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T17:50:25.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T17:50:25.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T17:50:25.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T17:50:25.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T17:50:25.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T17:50:25.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T17:50:25.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T17:50:25.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.table_properties_collectors: 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.inplace_update_support: 0 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.bloom_locality: 0 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.max_successive_merges: 0 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T17:50:25.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.ttl: 2592000 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.enable_blob_files: false 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.min_blob_size: 0 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T17:50:25.703 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 24.sst 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 26, last_sequence is 12915, log_number is 22,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 22 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: 778fb6bd-2907-4c6a-b25c-fd01e770e8f6 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695491425461026, "job": 1, "event": "recovery_started", "wal_files": [22]} 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #22 mode 2 2023-09-23T17:50:25.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695491425463062, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 27, "file_size": 313833, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 12920, "largest_seqno": 13335, "table_properties": {"data_size": 311396, "index_size": 1039, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 389, "raw_key_size": 3532, "raw_average_key_size": 23, "raw_value_size": 307960, "raw_average_value_size": 2012, "num_data_blocks": 45, "num_entries": 153, "num_filter_entries": 153, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695491425, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "778fb6bd-2907-4c6a-b25c-fd01e770e8f6", "db_session_id": "TCW2TVGSUFW064G7FYSN", "orig_file_number": 27, "seqno_to_time_mapping": "N/A"}} 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695491425463135, "job": 1, "event": "recovery_finished"} 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/version_set.cc:5047] Creating manifest 29 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000022.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x562baac66000 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: DB pointer 0x562baac52000 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** DB Stats ** 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:50:25.705 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: L0 1/0 306.48 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 265.8 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T17:50:25.706 INFO:journalctl@ceph.mon.c.smithi055.stdout: L6 1/0 8.70 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: Sum 2/0 9.00 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 265.8 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 265.8 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** Compaction Stats [default] ** 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 265.8 0.00 0.00 1 0.001 0 0 0.0 0.0 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:50:25.707 INFO:journalctl@ceph.mon.c.smithi055.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: Cumulative compaction: 0.00 GB write, 45.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: Interval compaction: 0.00 GB write, 45.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: Block cache BinnedLRUCache@0x562baabc91f0#7 capacity: 512.00 MB usage: 1.64 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.5e-05 secs_since: 0 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,0.44 KB,8.34465e-05%) IndexBlock(1,1.20 KB,0.000229478%) Misc(1,0.00 KB,0%) 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:50:25.708 INFO:journalctl@ceph.mon.c.smithi055.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 systemd[1]: Started Ceph mon.c for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: starting mon.c rank 2 at public addrs [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] at bind addrs [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: mon.c@-1(???) e3 preinit fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: mon.c@-1(???).mds e9 new map 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: mon.c@-1(???).mds e9 print_map 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout: e9 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout: legacy client fscid: 1 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout: Filesystem 'foo' (1) 2023-09-23T17:50:25.709 INFO:journalctl@ceph.mon.c.smithi055.stdout: fs_name foo 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: epoch 8 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: created 2023-09-23T17:45:51.128752+0000 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: modified 2023-09-23T17:46:03.386449+0000 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: tableserver 0 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: root 0 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: session_timeout 60 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: session_autoclose 300 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: max_file_size 1099511627776 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: required_client_features {} 2023-09-23T17:50:25.710 INFO:journalctl@ceph.mon.c.smithi055.stdout: last_failure 0 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: last_failure_osd_epoch 0 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: max_mds 1 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: in 0 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: up {0=14658} 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: failed 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: damaged 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: stopped 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: data_pools [4] 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: metadata_pool 3 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: inline_data disabled 2023-09-23T17:50:25.711 INFO:journalctl@ceph.mon.c.smithi055.stdout: balancer 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout: bal_rank_mask -1 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout: standby_count_wanted 1 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout: [mds.foo.smithi055.exjftp{0:14658} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout: Standby daemons: 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout: 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout: [mds.foo.smithi099.djovrl{-1:24577} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.99:6832/2997750460,v1:172.21.15.99:6833/2997750460] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: mon.c@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: mon.c@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: mon.c@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T17:50:25.712 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: mon.c@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: mon.c@-1(???).paxosservice(auth 1..19) refresh upgraded, format 0 -> 3 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta expand map: {default=false} 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta from 'false' to 'false' 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta expanded map: {default=false} 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta expand map: {default=info} 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta from 'info' to 'info' 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta expanded map: {default=info} 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta expand map: {default=daemon} 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta from 'daemon' to 'daemon' 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta expanded map: {default=daemon} 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta expand map: {default=debug} 2023-09-23T17:50:25.713 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta from 'debug' to 'debug' 2023-09-23T17:50:25.714 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:25 smithi055 ceph-mon[184174]: expand_channel_meta expanded map: {default=debug} 2023-09-23T17:50:29.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:50:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:50:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:50:30.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:50:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:50:30] "GET /metrics HTTP/1.1" 200 33896 "" "Prometheus/2.43.0" 2023-09-23T17:50:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:31 smithi099 ceph-mon[100270]: mon.c calling monitor election 2023-09-23T17:50:31.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:31 smithi099 ceph-mon[100270]: mon.b calling monitor election 2023-09-23T17:50:31.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:31 smithi099 ceph-mon[100270]: mon.a calling monitor election 2023-09-23T17:50:31.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:31 smithi099 ceph-mon[100270]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:50:31.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:31 smithi099 ceph-mon[100270]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:50:31.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:31 smithi099 ceph-mon[100270]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:50:31.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:31 smithi099 ceph-mon[100270]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T17:50:31.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:31 smithi099 ceph-mon[100270]: mgrmap e36: y(active, since 85s), standbys: x 2023-09-23T17:50:31.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:31 smithi099 ceph-mon[100270]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T17:50:31.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:31 smithi099 ceph-mon[100270]: Cluster is now healthy 2023-09-23T17:50:31.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[179668]: mon.c calling monitor election 2023-09-23T17:50:31.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[179668]: mon.b calling monitor election 2023-09-23T17:50:31.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[179668]: mon.a calling monitor election 2023-09-23T17:50:31.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[179668]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:50:31.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[179668]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:50:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[179668]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:50:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[179668]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T17:50:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[179668]: mgrmap e36: y(active, since 85s), standbys: x 2023-09-23T17:50:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[179668]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T17:50:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T17:50:31.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[184174]: mon.c calling monitor election 2023-09-23T17:50:31.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[184174]: mon.b calling monitor election 2023-09-23T17:50:31.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[184174]: mon.a calling monitor election 2023-09-23T17:50:31.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[184174]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:50:31.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[184174]: monmap e3: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:50:31.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[184174]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:50:31.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[184174]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T17:50:31.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[184174]: mgrmap e36: y(active, since 85s), standbys: x 2023-09-23T17:50:31.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[184174]: Health check cleared: MON_DOWN (was: 1/3 mons down, quorum a,b) 2023-09-23T17:50:31.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:31 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T17:50:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:32 smithi099 ceph-mon[100270]: overall HEALTH_OK 2023-09-23T17:50:32.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:32 smithi055 ceph-mon[179668]: overall HEALTH_OK 2023-09-23T17:50:32.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:32 smithi055 ceph-mon[184174]: overall HEALTH_OK 2023-09-23T17:50:33.402 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T17:50:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:33 smithi055 ceph-mon[179668]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/633309954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:50:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:33 smithi055 ceph-mon[184174]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/633309954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:50:33.799 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T17:50:33.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:33 smithi099 ceph-mon[100270]: pgmap v46: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:33 smithi099 ceph-mon[100270]: from='client.? 172.21.15.55:0/633309954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:50:34.422 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:50:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:35 smithi099 ceph-mon[100270]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:35 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:35 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:35.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:50:35 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:50:35.474+0000 7f3f79c94700 -1 mgr.server handle_report got status from non-daemon mon.c 2023-09-23T17:50:35.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:35 smithi055 ceph-mon[179668]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:35.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:35 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:35.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:35 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:35 smithi055 ceph-mon[184174]: pgmap v47: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:35.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:35 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:35.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:35 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:35.963 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T17:50:35.963 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (2m) - 5m 28.5M - 0.25.0 c8568f914cd2 5062112ad547 2023-09-23T17:50:35.963 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi099 *:3000 running (2m) - 4m 86.3M - 9.4.7 2c41d148cca3 b140d15d52f2 2023-09-23T17:50:35.963 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi055.exjftp smithi055 running (4m) - 4m 18.0M - 17.2.6-1123-g61431f6b 0db14aa01094 7a2127020d8b 2023-09-23T17:50:35.963 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi099.djovrl smithi099 running (4m) - 4m 17.6M - 17.2.6-1123-g61431f6b 0db14aa01094 5993fe8cebce 2023-09-23T17:50:35.963 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi099 *:8443,9283,8765 running (111s) - 8m 447M - 18.2.0-460-gbb0daeff e32c2a64c126 32a5f5ae4a55 2023-09-23T17:50:35.963 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (2m) - 10m 502M - 18.2.0-460-gbb0daeff e32c2a64c126 fc64ae873ea2 2023-09-23T17:50:35.964 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (99s) - 10m 56.8M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 1422ade2193a 2023-09-23T17:50:35.964 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi099 running (8m) - 8m 78.3M 2048M 17.2.6-1123-g61431f6b 0db14aa01094 5f0fb2e87ea1 2023-09-23T17:50:35.964 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (10s) - 8m 19.0M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e1bc5b196d92 2023-09-23T17:50:35.964 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (2m) - 4m 18.3M - 1.5.0 0da6a335fe13 6f05e33b3606 2023-09-23T17:50:35.964 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi099 *:9100 running (2m) - 4m 10.0M - 1.5.0 0da6a335fe13 3f7dcfc22edf 2023-09-23T17:50:35.964 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (8m) - 8m 81.4M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 15403d138a93 2023-09-23T17:50:35.964 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (7m) - 7m 94.3M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 ee80d8f5fddd 2023-09-23T17:50:35.964 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (7m) - 7m 76.2M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 7045fb31bd32 2023-09-23T17:50:35.964 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (7m) - 7m 70.0M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 33f748e0cdcf 2023-09-23T17:50:35.964 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi099 running (6m) - 6m 69.8M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ecb0e820fd90 2023-09-23T17:50:35.965 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi099 running (6m) - 6m 101M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 7e6bf34cfdb0 2023-09-23T17:50:35.965 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi099 running (6m) - 6m 77.0M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 827ec31827bb 2023-09-23T17:50:35.965 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi099 running (5m) - 5m 86.2M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 dcef78fdede8 2023-09-23T17:50:35.965 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi099 *:9095 running (115s) - 5m 50.4M - 2.43.0 a07b618ecd1d 12edf856847a 2023-09-23T17:50:36.273 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T17:50:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:36 smithi099 ceph-mon[100270]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:50:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:36 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:36 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:36 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:36.900 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:50:36.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:36 smithi055 ceph-mon[179668]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:50:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:36 smithi055 ceph-mon[184174]: from='client.34200 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:50:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:37 smithi055 ceph-mon[179668]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:37 smithi055 ceph-mon[184174]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:37 smithi099 ceph-mon[100270]: pgmap v48: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:38.680 INFO:teuthology.orchestra.run.smithi055.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:50:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:38 smithi055 ceph-mon[179668]: mgrmap e37: y(active, since 91s), standbys: x 2023-09-23T17:50:38.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:38 smithi055 ceph-mon[184174]: mgrmap e37: y(active, since 91s), standbys: x 2023-09-23T17:50:38.992 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T17:50:39.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:38 smithi099 ceph-mon[100270]: mgrmap e37: y(active, since 91s), standbys: x 2023-09-23T17:50:39.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:50:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:50:38.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:50:39.657 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:50:39.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:39.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:50:39.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:50:39.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:39.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:39.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:39.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:39.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:50:39.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:39.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:50:39.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:39.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:50:39.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:39.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:39.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:50:39.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:50:39.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:39.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:39.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:39.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:39.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:50:39.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:39.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:50:39.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:39.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:50:39.997 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:40.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: pgmap v49: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:40.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='client.44108 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:50:40.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:50:40.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:40.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:40.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:40.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:40.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:50:40.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:40.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:50:40.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:50:40.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:50:40.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:39 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:50:40.639 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:50:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:50:40] "GET /metrics HTTP/1.1" 200 33894 "" "Prometheus/2.43.0" 2023-09-23T17:50:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:41 smithi099 ceph-mon[100270]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:42.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:41 smithi055 ceph-mon[179668]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:42.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:41 smithi055 ceph-mon[184174]: pgmap v50: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:43 smithi055 ceph-mon[179668]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:43 smithi055 ceph-mon[184174]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:43 smithi099 ceph-mon[100270]: pgmap v51: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:45.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:45 smithi055 ceph-mon[184174]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:45 smithi055 ceph-mon[179668]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:45 smithi099 ceph-mon[100270]: pgmap v52: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:47 smithi055 ceph-mon[179668]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:47 smithi055 ceph-mon[184174]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:47 smithi099 ceph-mon[100270]: pgmap v53: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:49.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:50:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:50:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:50:49.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:49 smithi055 ceph-mon[179668]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:49.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:49 smithi055 ceph-mon[184174]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:49.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:49 smithi099 ceph-mon[100270]: pgmap v54: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 255 B/s wr, 0 op/s 2023-09-23T17:50:50.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:50:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:50:50] "GET /metrics HTTP/1.1" 200 33894 "" "Prometheus/2.43.0" 2023-09-23T17:50:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:51 smithi055 ceph-mon[179668]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:51.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:51 smithi055 ceph-mon[184174]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:51 smithi099 ceph-mon[100270]: pgmap v55: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:52.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:52 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:50:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:53 smithi055 ceph-mon[179668]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:53 smithi055 ceph-mon[184174]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:53.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:53 smithi099 ceph-mon[100270]: pgmap v56: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:55.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:55 smithi055 ceph-mon[184174]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:55.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:55 smithi055 ceph-mon[179668]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:55.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:55 smithi099 ceph-mon[100270]: pgmap v57: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:57.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:57 smithi055 ceph-mon[179668]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:57 smithi055 ceph-mon[184174]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:57 smithi099 ceph-mon[100270]: pgmap v58: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:59.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:50:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:50:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:50:59.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:50:59 smithi055 ceph-mon[179668]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:59.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:50:59 smithi055 ceph-mon[184174]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:50:59.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:50:59 smithi099 ceph-mon[100270]: pgmap v59: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:00.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:51:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:51:00] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.43.0" 2023-09-23T17:51:01.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:01 smithi055 ceph-mon[184174]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:01.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:01 smithi055 ceph-mon[179668]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:01.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:01 smithi099 ceph-mon[100270]: pgmap v60: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:03.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:03 smithi099 ceph-mon[100270]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:03 smithi055 ceph-mon[184174]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:03 smithi055 ceph-mon[179668]: pgmap v61: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:05.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:05 smithi055 ceph-mon[179668]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:05 smithi055 ceph-mon[184174]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:05.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:05 smithi099 ceph-mon[100270]: pgmap v62: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:06.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:06 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:07 smithi055 ceph-mon[179668]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:07 smithi055 ceph-mon[184174]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:07 smithi099 ceph-mon[100270]: pgmap v63: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:09.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:51:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:51:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:51:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:09 smithi099 ceph-mon[100270]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:09.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:09 smithi055 ceph-mon[179668]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:09 smithi055 ceph-mon[184174]: pgmap v64: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:10.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:51:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:51:10] "GET /metrics HTTP/1.1" 200 33891 "" "Prometheus/2.43.0" 2023-09-23T17:51:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:11 smithi099 ceph-mon[100270]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:11.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:11 smithi055 ceph-mon[184174]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:11.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:11 smithi055 ceph-mon[179668]: pgmap v65: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:13 smithi099 ceph-mon[100270]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:13.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:13 smithi055 ceph-mon[184174]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:13.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:13 smithi055 ceph-mon[179668]: pgmap v66: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:15 smithi055 ceph-mon[179668]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:15.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:15 smithi055 ceph-mon[184174]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:15.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:15 smithi099 ceph-mon[100270]: pgmap v67: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:17.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:17 smithi055 ceph-mon[179668]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:17.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:17 smithi055 ceph-mon[184174]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:17 smithi099 ceph-mon[100270]: pgmap v68: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:19.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:51:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:51:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:51:19.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:19 smithi055 ceph-mon[179668]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:19 smithi055 ceph-mon[184174]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:19 smithi099 ceph-mon[100270]: pgmap v69: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:20.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:51:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:51:20] "GET /metrics HTTP/1.1" 200 33891 "" "Prometheus/2.43.0" 2023-09-23T17:51:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:21 smithi099 ceph-mon[100270]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:21 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:21.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:21 smithi055 ceph-mon[179668]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:21.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:21.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:21 smithi055 ceph-mon[184174]: pgmap v70: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:21.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:23 smithi055 ceph-mon[184174]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:23.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:23 smithi055 ceph-mon[179668]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:23 smithi099 ceph-mon[100270]: pgmap v71: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:25.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:25 smithi055 ceph-mon[179668]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:25 smithi055 ceph-mon[184174]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:25 smithi099 ceph-mon[100270]: pgmap v72: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:27.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:27 smithi055 ceph-mon[179668]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:27.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:27 smithi055 ceph-mon[184174]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:27 smithi099 ceph-mon[100270]: pgmap v73: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:29.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:51:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:51:28.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:51:29.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:29 smithi055 ceph-mon[179668]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:29.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:29 smithi055 ceph-mon[184174]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:29 smithi099 ceph-mon[100270]: pgmap v74: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:30.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:51:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:51:30] "GET /metrics HTTP/1.1" 200 33904 "" "Prometheus/2.43.0" 2023-09-23T17:51:31.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:31 smithi055 ceph-mon[179668]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:31.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:31 smithi055 ceph-mon[184174]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:31 smithi099 ceph-mon[100270]: pgmap v75: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:33 smithi055 ceph-mon[179668]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:33 smithi055 ceph-mon[184174]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:33 smithi099 ceph-mon[100270]: pgmap v76: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:35.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:35 smithi055 ceph-mon[184174]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:35.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:35 smithi055 ceph-mon[179668]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:35 smithi099 ceph-mon[100270]: pgmap v77: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:36.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:36 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:37 smithi055 ceph-mon[184174]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:37.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:37 smithi055 ceph-mon[179668]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:37.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:37 smithi099 ceph-mon[100270]: pgmap v78: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:39.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:51:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:51:38.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:51:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:39 smithi099 ceph-mon[100270]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:39.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:39 smithi055 ceph-mon[179668]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:39.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:39 smithi055 ceph-mon[184174]: pgmap v79: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:40.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:51:40] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2023-09-23T17:51:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:51:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:51:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:40 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:40 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:51:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:40 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:40 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:40 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:40 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:41.056 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T17:51:41.678 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:51:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:41 smithi099 ceph-mon[100270]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:41.871 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:41 smithi055 ceph-mon[184174]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:41.871 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:41 smithi055 ceph-mon[179668]: pgmap v80: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:43.245 INFO:teuthology.orchestra.run.smithi055.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:51:43.635 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:43 smithi055 ceph-mon[184174]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:43.635 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:43 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:43.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:43 smithi055 ceph-mon[179668]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:43.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:43 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:43.637 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for all mons to upgrade' 2023-09-23T17:51:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:43 smithi099 ceph-mon[100270]: pgmap v81: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:43 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:44.273 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:51:44.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:44 smithi055 ceph-mon[179668]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:51:44.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:44 smithi055 ceph-mon[179668]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:51:44.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:44 smithi055 ceph-mon[184174]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:51:44.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:44 smithi055 ceph-mon[184174]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:51:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:44 smithi099 ceph-mon[100270]: from='client.44114 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:51:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:44 smithi099 ceph-mon[100270]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:51:45.463 INFO:teuthology.orchestra.run.smithi055.stdout:wait for all mons to upgrade 2023-09-23T17:51:45.782 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph mon versions | grep $sha1 | grep '"'"': 3'"'"' ; do sleep 2 ; done' 2023-09-23T17:51:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:51:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:51:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:51:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:51:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T17:51:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T17:51:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:51:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:51:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:51:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:51:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T17:51:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T17:51:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: pgmap v82: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:46.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:46.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:46.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:51:46.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:51:46.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:46.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:51:46.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:51:46.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-09-23T17:51:46.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:45 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon ok-to-stop", "ids": ["b"]}]: dispatch 2023-09-23T17:51:46.388 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:51:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:46 smithi099 ceph-mon[100270]: Upgrade: It appears safe to stop mon.b 2023-09-23T17:51:46.887 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:46 smithi055 ceph-mon[179668]: Upgrade: It appears safe to stop mon.b 2023-09-23T17:51:46.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:46 smithi055 ceph-mon[184174]: Upgrade: It appears safe to stop mon.b 2023-09-23T17:51:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:47 smithi099 ceph-mon[100270]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:47 smithi099 ceph-mon[100270]: Upgrade: Updating mon.b 2023-09-23T17:51:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:47 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:47 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:51:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:47 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:51:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:47 smithi099 ceph-mon[100270]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:51:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:47 smithi099 ceph-mon[100270]: Deploying daemon mon.b on smithi099 2023-09-23T17:51:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[179668]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[179668]: Upgrade: Updating mon.b 2023-09-23T17:51:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:51:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:51:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:51:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[179668]: Deploying daemon mon.b on smithi099 2023-09-23T17:51:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[184174]: pgmap v83: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[184174]: Upgrade: Updating mon.b 2023-09-23T17:51:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:51:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:51:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:51:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:47 smithi055 ceph-mon[184174]: Deploying daemon mon.b on smithi099 2023-09-23T17:51:48.521 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:48 smithi099 systemd[1]: Stopping Ceph mon.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:51:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-b[100222]: 2023-09-23T17:51:48.518+0000 7f299f4f9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:51:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-b[100222]: 2023-09-23T17:51:48.518+0000 7f299f4f9700 -1 mon.b@1(peon) e3 *** Got Signal Terminated *** 2023-09-23T17:51:48.827 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:51:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:51:48.660Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:51:49.176 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:48 smithi099 bash[166888]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T17:51:49.177 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T17:51:49.177 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 systemd[1]: Stopped Ceph mon.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:51:49.474 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 systemd[1]: Starting Ceph mon.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:51:49.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 podman[167036]: 2023-09-23T17:51:49.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: set uid:gid to 167:167 (ceph:ceph) 2023-09-23T17:51:49.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable), process ceph-mon, pid 7 2023-09-23T17:51:49.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 bash[167036]: 8b31f1fe680609d53494e214b425cda12aa0339d1b7b7e0df1e2c3a7fc34277c 2023-09-23T17:51:49.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: pidfile_write: ignore empty --pid-file 2023-09-23T17:51:49.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: load: jerasure load: lrc 2023-09-23T17:51:49.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: RocksDB version: 7.9.2 2023-09-23T17:51:49.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Git sha 0 2023-09-23T17:51:49.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Compile date 2023-09-22 15:19:43 2023-09-23T17:51:49.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: DB SUMMARY 2023-09-23T17:51:49.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: DB Session ID: 4ZZF6XI82T8WLH7CUK8L 2023-09-23T17:51:49.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: CURRENT file: CURRENT 2023-09-23T17:51:49.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: IDENTITY file: IDENTITY 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: MANIFEST file: MANIFEST-000009 size: 1097 Bytes 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 1, files: 000027.sst 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000025.log size: 1612162 ; 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.error_if_exists: 0 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.create_if_missing: 0 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.paranoid_checks: 1 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.flush_verify_memtable_count: 1 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.verify_sst_unique_id_in_manifest: 1 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.env: 0x561870c74720 2023-09-23T17:51:49.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.fs: PosixFileSystem 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.info_log: 0x561873609420 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_file_opening_threads: 16 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.statistics: (nil) 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.use_fsync: 0 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_log_file_size: 0 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.log_file_time_to_roll: 0 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.keep_log_file_num: 1000 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.recycle_log_file_num: 0 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.allow_fallocate: 1 2023-09-23T17:51:49.836 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.allow_mmap_reads: 0 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.allow_mmap_writes: 0 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.use_direct_reads: 0 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.create_missing_column_families: 0 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.db_log_dir: 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.wal_dir: 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.table_cache_numshardbits: 6 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.WAL_ttl_seconds: 0 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.WAL_size_limit_MB: 0 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-09-23T17:51:49.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.is_fd_close_on_exec: 1 2023-09-23T17:51:49.838 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.advise_random_on_open: 1 2023-09-23T17:51:49.838 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.db_write_buffer_size: 0 2023-09-23T17:51:49.838 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.write_buffer_manager: 0x56187289a5a0 2023-09-23T17:51:49.838 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-09-23T17:51:49.838 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-09-23T17:51:49.838 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.use_adaptive_mutex: 0 2023-09-23T17:51:49.838 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.rate_limiter: (nil) 2023-09-23T17:51:49.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-09-23T17:51:49.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.wal_recovery_mode: 2 2023-09-23T17:51:49.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.enable_thread_tracking: 0 2023-09-23T17:51:49.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.enable_pipelined_write: 0 2023-09-23T17:51:49.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.unordered_write: 0 2023-09-23T17:51:49.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-09-23T17:51:49.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-09-23T17:51:49.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-09-23T17:51:49.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-09-23T17:51:49.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.row_cache: None 2023-09-23T17:51:49.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.wal_filter: None 2023-09-23T17:51:49.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-09-23T17:51:49.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.allow_ingest_behind: 0 2023-09-23T17:51:49.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.two_write_queues: 0 2023-09-23T17:51:49.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.manual_wal_flush: 0 2023-09-23T17:51:49.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.wal_compression: 0 2023-09-23T17:51:49.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.atomic_flush: 0 2023-09-23T17:51:49.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-09-23T17:51:49.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.persist_stats_to_disk: 0 2023-09-23T17:51:49.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.write_dbid_to_manifest: 0 2023-09-23T17:51:49.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.log_readahead_size: 0 2023-09-23T17:51:49.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-09-23T17:51:49.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.best_efforts_recovery: 0 2023-09-23T17:51:49.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-09-23T17:51:49.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-09-23T17:51:49.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.allow_data_in_errors: 0 2023-09-23T17:51:49.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.db_host_id: __hostname__ 2023-09-23T17:51:49.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.enforce_single_del_contracts: true 2023-09-23T17:51:49.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_background_jobs: 2 2023-09-23T17:51:49.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_background_compactions: -1 2023-09-23T17:51:49.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_subcompactions: 1 2023-09-23T17:51:49.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-09-23T17:51:49.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-09-23T17:51:49.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.delayed_write_rate : 16777216 2023-09-23T17:51:49.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_total_wal_size: 0 2023-09-23T17:51:49.845 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-09-23T17:51:49.845 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.stats_dump_period_sec: 600 2023-09-23T17:51:49.845 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.stats_persist_period_sec: 600 2023-09-23T17:51:49.845 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-09-23T17:51:49.845 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_open_files: -1 2023-09-23T17:51:49.845 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bytes_per_sync: 0 2023-09-23T17:51:49.845 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.wal_bytes_per_sync: 0 2023-09-23T17:51:49.846 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.strict_bytes_per_sync: 0 2023-09-23T17:51:49.846 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_readahead_size: 0 2023-09-23T17:51:49.846 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_background_flushes: -1 2023-09-23T17:51:49.846 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Compression algorithms supported: 2023-09-23T17:51:49.846 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-09-23T17:51:49.846 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: kZSTD supported: 0 2023-09-23T17:51:49.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: kXpressCompression supported: 0 2023-09-23T17:51:49.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: kLZ4HCCompression supported: 1 2023-09-23T17:51:49.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: kZlibCompression supported: 1 2023-09-23T17:51:49.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: kSnappyCompression supported: 1 2023-09-23T17:51:49.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: kLZ4Compression supported: 1 2023-09-23T17:51:49.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: kBZip2Compression supported: 0 2023-09-23T17:51:49.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Fast CRC32 supported: Supported on x86 2023-09-23T17:51:49.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: DMutex implementation: pthread_mutex_t 2023-09-23T17:51:49.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/version_set.cc:5527] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 2023-09-23T17:51:49.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/column_family.cc:630] --------------- Options for column family [default]: 2023-09-23T17:51:49.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-09-23T17:51:49.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.merge_operator: 2023-09-23T17:51:49.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_filter: None 2023-09-23T17:51:49.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_filter_factory: None 2023-09-23T17:51:49.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.sst_partitioner_factory: None 2023-09-23T17:51:49.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.memtable_factory: SkipListFactory 2023-09-23T17:51:49.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.table_factory: BlockBasedTable 2023-09-23T17:51:49.849 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x561873609520) 2023-09-23T17:51:49.850 INFO:journalctl@ceph.mon.b.smithi099.stdout: cache_index_and_filter_blocks: 1 2023-09-23T17:51:49.850 INFO:journalctl@ceph.mon.b.smithi099.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-09-23T17:51:49.850 INFO:journalctl@ceph.mon.b.smithi099.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-09-23T17:51:49.850 INFO:journalctl@ceph.mon.b.smithi099.stdout: pin_top_level_index_and_filter: 1 2023-09-23T17:51:49.850 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_type: 0 2023-09-23T17:51:49.851 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_block_index_type: 0 2023-09-23T17:51:49.851 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_shortening: 1 2023-09-23T17:51:49.851 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_block_hash_table_util_ratio: 0.750000 2023-09-23T17:51:49.851 INFO:journalctl@ceph.mon.b.smithi099.stdout: checksum: 4 2023-09-23T17:51:49.851 INFO:journalctl@ceph.mon.b.smithi099.stdout: no_block_cache: 0 2023-09-23T17:51:49.851 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache: 0x56187291b350 2023-09-23T17:51:49.852 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_name: BinnedLRUCache 2023-09-23T17:51:49.852 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_options: 2023-09-23T17:51:49.852 INFO:journalctl@ceph.mon.b.smithi099.stdout: capacity : 536870912 2023-09-23T17:51:49.852 INFO:journalctl@ceph.mon.b.smithi099.stdout: num_shard_bits : 4 2023-09-23T17:51:49.852 INFO:journalctl@ceph.mon.b.smithi099.stdout: strict_capacity_limit : 0 2023-09-23T17:51:49.852 INFO:journalctl@ceph.mon.b.smithi099.stdout: high_pri_pool_ratio: 0.000 2023-09-23T17:51:49.853 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_cache_compressed: (nil) 2023-09-23T17:51:49.853 INFO:journalctl@ceph.mon.b.smithi099.stdout: persistent_cache: (nil) 2023-09-23T17:51:49.853 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_size: 4096 2023-09-23T17:51:49.853 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_size_deviation: 10 2023-09-23T17:51:49.853 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_restart_interval: 16 2023-09-23T17:51:49.853 INFO:journalctl@ceph.mon.b.smithi099.stdout: index_block_restart_interval: 1 2023-09-23T17:51:49.853 INFO:journalctl@ceph.mon.b.smithi099.stdout: metadata_block_size: 4096 2023-09-23T17:51:49.853 INFO:journalctl@ceph.mon.b.smithi099.stdout: partition_filters: 0 2023-09-23T17:51:49.854 INFO:journalctl@ceph.mon.b.smithi099.stdout: use_delta_encoding: 1 2023-09-23T17:51:49.854 INFO:journalctl@ceph.mon.b.smithi099.stdout: filter_policy: bloomfilter 2023-09-23T17:51:49.854 INFO:journalctl@ceph.mon.b.smithi099.stdout: whole_key_filtering: 1 2023-09-23T17:51:49.854 INFO:journalctl@ceph.mon.b.smithi099.stdout: verify_compression: 0 2023-09-23T17:51:49.854 INFO:journalctl@ceph.mon.b.smithi099.stdout: read_amp_bytes_per_bit: 0 2023-09-23T17:51:49.854 INFO:journalctl@ceph.mon.b.smithi099.stdout: format_version: 5 2023-09-23T17:51:49.854 INFO:journalctl@ceph.mon.b.smithi099.stdout: enable_index_compression: 1 2023-09-23T17:51:49.855 INFO:journalctl@ceph.mon.b.smithi099.stdout: block_align: 0 2023-09-23T17:51:49.855 INFO:journalctl@ceph.mon.b.smithi099.stdout: max_auto_readahead_size: 262144 2023-09-23T17:51:49.855 INFO:journalctl@ceph.mon.b.smithi099.stdout: prepopulate_block_cache: 0 2023-09-23T17:51:49.855 INFO:journalctl@ceph.mon.b.smithi099.stdout: initial_auto_readahead_size: 8192 2023-09-23T17:51:49.855 INFO:journalctl@ceph.mon.b.smithi099.stdout: num_file_reads_for_auto_readahead: 2 2023-09-23T17:51:49.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.write_buffer_size: 33554432 2023-09-23T17:51:49.855 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_write_buffer_number: 2 2023-09-23T17:51:49.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compression: NoCompression 2023-09-23T17:51:49.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bottommost_compression: Disabled 2023-09-23T17:51:49.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.prefix_extractor: nullptr 2023-09-23T17:51:49.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-09-23T17:51:49.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.num_levels: 7 2023-09-23T17:51:49.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-09-23T17:51:49.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-09-23T17:51:49.856 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-09-23T17:51:49.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-09-23T17:51:49.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-09-23T17:51:49.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-09-23T17:51:49.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-09-23T17:51:49.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:51:49.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-09-23T17:51:49.857 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-09-23T17:51:49.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bottommost_compression_opts.max_dict_buffer_bytes: 0 2023-09-23T17:51:49.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bottommost_compression_opts.use_zstd_dict_trainer: true 2023-09-23T17:51:49.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compression_opts.window_bits: -14 2023-09-23T17:51:49.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compression_opts.level: 32767 2023-09-23T17:51:49.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compression_opts.strategy: 0 2023-09-23T17:51:49.858 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-09-23T17:51:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-09-23T17:51:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compression_opts.use_zstd_dict_trainer: true 2023-09-23T17:51:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-09-23T17:51:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compression_opts.enabled: false 2023-09-23T17:51:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compression_opts.max_dict_buffer_bytes: 0 2023-09-23T17:51:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-09-23T17:51:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-09-23T17:51:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-09-23T17:51:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.target_file_size_base: 67108864 2023-09-23T17:51:49.859 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.target_file_size_multiplier: 1 2023-09-23T17:51:49.860 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-09-23T17:51:49.860 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-09-23T17:51:49.860 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-09-23T17:51:49.860 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-09-23T17:51:49.860 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-09-23T17:51:49.860 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-09-23T17:51:49.860 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-09-23T17:51:49.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-09-23T17:51:49.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-09-23T17:51:49.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-09-23T17:51:49.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-09-23T17:51:49.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-09-23T17:51:49.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.ignore_max_compaction_bytes_for_input: true 2023-09-23T17:51:49.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.arena_block_size: 1048576 2023-09-23T17:51:49.861 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-09-23T17:51:49.862 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-09-23T17:51:49.862 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.disable_auto_compactions: 0 2023-09-23T17:51:49.862 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-09-23T17:51:49.862 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-09-23T17:51:49.862 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-09-23T17:51:49.862 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-09-23T17:51:49.863 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-09-23T17:51:49.863 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-09-23T17:51:49.863 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-09-23T17:51:49.863 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-09-23T17:51:49.863 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-09-23T17:51:49.863 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-09-23T17:51:49.863 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.table_properties_collectors: 2023-09-23T17:51:49.863 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.inplace_update_support: 0 2023-09-23T17:51:49.863 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.inplace_update_num_locks: 10000 2023-09-23T17:51:49.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-09-23T17:51:49.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-09-23T17:51:49.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.memtable_huge_page_size: 0 2023-09-23T17:51:49.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.bloom_locality: 0 2023-09-23T17:51:49.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.max_successive_merges: 0 2023-09-23T17:51:49.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.optimize_filters_for_hits: 0 2023-09-23T17:51:49.864 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.paranoid_file_checks: 0 2023-09-23T17:51:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.force_consistency_checks: 1 2023-09-23T17:51:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.report_bg_io_stats: 0 2023-09-23T17:51:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.ttl: 2592000 2023-09-23T17:51:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.periodic_compaction_seconds: 0 2023-09-23T17:51:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.preclude_last_level_data_seconds: 0 2023-09-23T17:51:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.preserve_internal_time_seconds: 0 2023-09-23T17:51:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.enable_blob_files: false 2023-09-23T17:51:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.min_blob_size: 0 2023-09-23T17:51:49.865 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.blob_file_size: 268435456 2023-09-23T17:51:49.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.blob_compression_type: NoCompression 2023-09-23T17:51:49.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.enable_blob_garbage_collection: false 2023-09-23T17:51:49.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-09-23T17:51:49.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.blob_garbage_collection_force_threshold: 1.000000 2023-09-23T17:51:49.866 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.blob_compaction_readahead_size: 0 2023-09-23T17:51:49.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.blob_file_starting_level: 0 2023-09-23T17:51:49.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: Options.experimental_mempurge_threshold: 0.000000 2023-09-23T17:51:49.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [table/block_based/block_based_table_reader.cc:721] At least one SST file opened without unique ID to verify: 27.sst 2023-09-23T17:51:49.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T17:51:49.867 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/version_set.cc:5566] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 29, last_sequence is 13786, log_number is 25,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/version_set.cc:5581] Column family [default] (ID 0), log number is 25 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/db_impl/db_impl_open.cc:539] DB ID: c77aa3da-3c8d-4a63-a689-12e97feda96e 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695491509657540, "job": 1, "event": "recovery_started", "wal_files": [25]} 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/db_impl/db_impl_open.cc:1043] Recovering log #25 mode 2 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 systemd[1]: Started Ceph mon.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695491509663401, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 30, "file_size": 998300, "file_checksum": "", "file_checksum_func_name": "Unknown", "smallest_seqno": 13781, "largest_seqno": 14856, "table_properties": {"data_size": 993870, "index_size": 2520, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 901, "raw_key_size": 7508, "raw_average_key_size": 21, "raw_value_size": 986154, "raw_average_value_size": 2817, "num_data_blocks": 115, "num_entries": 350, "num_filter_entries": 350, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "bloomfilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; max_dict_buffer_bytes=0; use_zstd_dict_trainer=1; ", "creation_time": 1695491509, "oldest_key_time": 0, "file_creation_time": 0, "slow_compression_estimated_data_size": 0, "fast_compression_estimated_data_size": 0, "db_id": "c77aa3da-3c8d-4a63-a689-12e97feda96e", "db_session_id": "4ZZF6XI82T8WLH7CUK8L", "orig_file_number": 30, "seqno_to_time_mapping": "N/A"}} 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: EVENT_LOG_v1 {"time_micros": 1695491509663523, "job": 1, "event": "recovery_finished"} 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/version_set.cc:5047] Creating manifest 32 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/version_set.cc:4390] More existing levels in DB than needed. max_bytes_for_level_multiplier may not be guaranteed. 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [file/delete_scheduler.cc:74] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000025.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-09-23T17:51:49.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/db_impl/db_impl_open.cc:1987] SstFileManager instance 0x5618729b8000 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: DB pointer 0x5618729a4000 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/db_impl/db_impl.cc:1109] ------- DUMPING STATS ------- 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: rocksdb: [db/db_impl/db_impl.cc:1111] 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** DB Stats ** 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-09-23T17:51:49.869 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:51:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2023-09-23T17:51:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T17:51:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ 2023-09-23T17:51:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout: L0 1/0 974.90 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 412.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T17:51:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout: L6 1/0 8.02 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 0.0 0.0 2023-09-23T17:51:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout: Sum 2/0 8.98 MB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 412.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T17:51:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 412.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T17:51:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:51:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** Compaction Stats [default] ** 2023-09-23T17:51:49.870 INFO:journalctl@ceph.mon.b.smithi099.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop Rblob(GB) Wblob(GB) 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 412.0 0.00 0.00 1 0.002 0 0 0.0 0.0 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: Blob file count: 0, total size: 0.0 GB, garbage size: 0.0 GB, space amp: 0.0 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: Flush(GB): cumulative 0.001, interval 0.001 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: AddFile(Keys): cumulative 0, interval 0 2023-09-23T17:51:49.871 INFO:journalctl@ceph.mon.b.smithi099.stdout: Cumulative compaction: 0.00 GB write, 109.92 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:51:49.872 INFO:journalctl@ceph.mon.b.smithi099.stdout: Interval compaction: 0.00 GB write, 109.92 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-09-23T17:51:49.872 INFO:journalctl@ceph.mon.b.smithi099.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-09-23T17:51:49.872 INFO:journalctl@ceph.mon.b.smithi099.stdout: Block cache BinnedLRUCache@0x56187291b350#7 capacity: 512.00 MB usage: 3.61 KB table_size: 0 occupancy: 18446744073709551615 collections: 1 last_copies: 0 last_secs: 1.2e-05 secs_since: 0 2023-09-23T17:51:49.872 INFO:journalctl@ceph.mon.b.smithi099.stdout: Block cache entry stats(count,size,portion): FilterBlock(1,1.03 KB,0.000196695%) IndexBlock(1,2.58 KB,0.000491738%) Misc(1,0.00 KB,0%) 2023-09-23T17:51:49.872 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:51:49.872 INFO:journalctl@ceph.mon.b.smithi099.stdout: ** File Read Latency Histogram By Level [default] ** 2023-09-23T17:51:49.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: starting mon.b rank 1 at public addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] at bind addrs [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:51:49.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: mon.b@-1(???) e3 preinit fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:51:49.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: mon.b@-1(???).mds e9 new map 2023-09-23T17:51:49.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: mon.b@-1(???).mds e9 print_map 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: e9 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: legacy client fscid: 1 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: Filesystem 'foo' (1) 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: fs_name foo 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: epoch 8 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: flags 12 joinable allow_snaps allow_multimds_snaps 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: created 2023-09-23T17:45:51.128752+0000 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: modified 2023-09-23T17:46:03.386449+0000 2023-09-23T17:51:49.873 INFO:journalctl@ceph.mon.b.smithi099.stdout: tableserver 0 2023-09-23T17:51:49.874 INFO:journalctl@ceph.mon.b.smithi099.stdout: root 0 2023-09-23T17:51:49.874 INFO:journalctl@ceph.mon.b.smithi099.stdout: session_timeout 60 2023-09-23T17:51:49.874 INFO:journalctl@ceph.mon.b.smithi099.stdout: session_autoclose 300 2023-09-23T17:51:49.874 INFO:journalctl@ceph.mon.b.smithi099.stdout: max_file_size 1099511627776 2023-09-23T17:51:49.874 INFO:journalctl@ceph.mon.b.smithi099.stdout: required_client_features {} 2023-09-23T17:51:49.874 INFO:journalctl@ceph.mon.b.smithi099.stdout: last_failure 0 2023-09-23T17:51:49.874 INFO:journalctl@ceph.mon.b.smithi099.stdout: last_failure_osd_epoch 0 2023-09-23T17:51:49.874 INFO:journalctl@ceph.mon.b.smithi099.stdout: compat compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,7=mds uses inline data,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-09-23T17:51:49.874 INFO:journalctl@ceph.mon.b.smithi099.stdout: max_mds 1 2023-09-23T17:51:49.875 INFO:journalctl@ceph.mon.b.smithi099.stdout: in 0 2023-09-23T17:51:49.875 INFO:journalctl@ceph.mon.b.smithi099.stdout: up {0=14658} 2023-09-23T17:51:49.875 INFO:journalctl@ceph.mon.b.smithi099.stdout: failed 2023-09-23T17:51:49.875 INFO:journalctl@ceph.mon.b.smithi099.stdout: damaged 2023-09-23T17:51:49.875 INFO:journalctl@ceph.mon.b.smithi099.stdout: stopped 2023-09-23T17:51:49.875 INFO:journalctl@ceph.mon.b.smithi099.stdout: data_pools [4] 2023-09-23T17:51:49.875 INFO:journalctl@ceph.mon.b.smithi099.stdout: metadata_pool 3 2023-09-23T17:51:49.875 INFO:journalctl@ceph.mon.b.smithi099.stdout: inline_data disabled 2023-09-23T17:51:49.875 INFO:journalctl@ceph.mon.b.smithi099.stdout: balancer 2023-09-23T17:51:49.876 INFO:journalctl@ceph.mon.b.smithi099.stdout: bal_rank_mask -1 2023-09-23T17:51:49.876 INFO:journalctl@ceph.mon.b.smithi099.stdout: standby_count_wanted 1 2023-09-23T17:51:49.876 INFO:journalctl@ceph.mon.b.smithi099.stdout: [mds.foo.smithi055.exjftp{0:14658} state up:active seq 3 join_fscid=1 addr [v2:172.21.15.55:6834/1594202073,v1:172.21.15.55:6835/1594202073] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T17:51:49.876 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:51:49.876 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:51:49.876 INFO:journalctl@ceph.mon.b.smithi099.stdout: Standby daemons: 2023-09-23T17:51:49.876 INFO:journalctl@ceph.mon.b.smithi099.stdout: 2023-09-23T17:51:49.876 INFO:journalctl@ceph.mon.b.smithi099.stdout: [mds.foo.smithi099.djovrl{-1:24577} state up:standby seq 2 join_fscid=1 addr [v2:172.21.15.99:6832/2997750460,v1:172.21.15.99:6833/2997750460] compat {c=[1],r=[1],i=[7ff]}] 2023-09-23T17:51:49.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: mon.b@-1(???).osd e89 crush map has features 3314933000854323200, adjusting msgr requires 2023-09-23T17:51:49.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: mon.b@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T17:51:49.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: mon.b@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T17:51:49.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: mon.b@-1(???).osd e89 crush map has features 432629239337189376, adjusting msgr requires 2023-09-23T17:51:49.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:49 smithi099 ceph-mon[167076]: mon.b@-1(???).paxosservice(auth 1..20) refresh upgraded, format 0 -> 3 2023-09-23T17:51:50.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:51:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:51:50] "GET /metrics HTTP/1.1" 200 33897 "" "Prometheus/2.43.0" 2023-09-23T17:51:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/247303190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:51:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: overall HEALTH_OK 2023-09-23T17:51:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:51:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: mon.b calling monitor election 2023-09-23T17:51:51.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: mon.c calling monitor election 2023-09-23T17:51:51.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: mon.a calling monitor election 2023-09-23T17:51:51.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:51:51.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:51:51.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:51:51.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: monmap e4: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:51:51.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:51:51.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T17:51:51.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: mgrmap e37: y(active, since 2m), standbys: x 2023-09-23T17:51:51.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: overall HEALTH_OK 2023-09-23T17:51:51.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:51.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:51.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:51:51.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:51.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/247303190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: overall HEALTH_OK 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: mon.b calling monitor election 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: mon.c calling monitor election 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: mon.a calling monitor election 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: monmap e4: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:51:51.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T17:51:51.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: mgrmap e37: y(active, since 2m), standbys: x 2023-09-23T17:51:51.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: overall HEALTH_OK 2023-09-23T17:51:51.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:51.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:51.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:51:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: pgmap v85: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/247303190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:51:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: overall HEALTH_OK 2023-09-23T17:51:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-09-23T17:51:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: mon.b calling monitor election 2023-09-23T17:51:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: mon.c calling monitor election 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: mon.a calling monitor election 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: monmap e4: 3 mons at {a=[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0],b=[v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0],c=[v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]} removed_ranks: {} 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: fsmap foo:1 {0=foo.smithi055.exjftp=up:active} 1 up:standby 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: osdmap e89: 8 total, 8 up, 8 in 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: mgrmap e37: y(active, since 2m), standbys: x 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: overall HEALTH_OK 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:51.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:51:52.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:52.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:52.774 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T17:51:52.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:51:53.046 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T17:51:53.362 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:53 smithi055 ceph-mon[179668]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:53.652 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:51:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/481956943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:51:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:53 smithi055 ceph-mon[184174]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/481956943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:51:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:53 smithi099 ceph-mon[167076]: pgmap v86: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/481956943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-09-23T17:51:54.929 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:51:54 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:51:54.672+0000 7f3f79c94700 -1 mgr.server handle_report got status from non-daemon mon.b 2023-09-23T17:51:55.024 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T17:51:55.024 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (3m) - 6m 28.5M - 0.25.0 c8568f914cd2 5062112ad547 2023-09-23T17:51:55.024 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi099 *:3000 running (3m) - 6m 88.4M - 9.4.7 2c41d148cca3 b140d15d52f2 2023-09-23T17:51:55.025 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi055.exjftp smithi055 running (5m) - 5m 18.5M - 17.2.6-1123-g61431f6b 0db14aa01094 7a2127020d8b 2023-09-23T17:51:55.025 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi099.djovrl smithi099 running (5m) - 5m 17.8M - 17.2.6-1123-g61431f6b 0db14aa01094 5993fe8cebce 2023-09-23T17:51:55.025 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi099 *:8443,9283,8765 running (3m) - 9m 448M - 18.2.0-460-gbb0daeff e32c2a64c126 32a5f5ae4a55 2023-09-23T17:51:55.025 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (3m) - 11m 504M - 18.2.0-460-gbb0daeff e32c2a64c126 fc64ae873ea2 2023-09-23T17:51:55.025 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (2m) - 11m 61.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 1422ade2193a 2023-09-23T17:51:55.025 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi099 starting - - - 2048M 2023-09-23T17:51:55.025 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (89s) - 10m 45.7M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e1bc5b196d92 2023-09-23T17:51:55.025 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (3m) - 6m 18.7M - 1.5.0 0da6a335fe13 6f05e33b3606 2023-09-23T17:51:55.025 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi099 *:9100 running (3m) - 6m 17.4M - 1.5.0 0da6a335fe13 3f7dcfc22edf 2023-09-23T17:51:55.026 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (9m) - 9m 84.3M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 15403d138a93 2023-09-23T17:51:55.026 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (9m) - 9m 97.4M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 ee80d8f5fddd 2023-09-23T17:51:55.026 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (8m) - 8m 78.2M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 7045fb31bd32 2023-09-23T17:51:55.026 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (8m) - 8m 73.1M 2501M 17.2.6-1123-g61431f6b 0db14aa01094 33f748e0cdcf 2023-09-23T17:51:55.026 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi099 running (8m) - 8m 73.7M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ecb0e820fd90 2023-09-23T17:51:55.026 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi099 running (7m) - 7m 103M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 7e6bf34cfdb0 2023-09-23T17:51:55.026 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi099 running (7m) - 7m 80.1M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 827ec31827bb 2023-09-23T17:51:55.026 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi099 running (7m) - 7m 89.7M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 dcef78fdede8 2023-09-23T17:51:55.026 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi099 *:9095 running (3m) - 6m 55.5M - 2.43.0 a07b618ecd1d 12edf856847a 2023-09-23T17:51:55.399 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T17:51:55.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:55 smithi055 ceph-mon[179668]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:55.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:55 smithi055 ceph-mon[179668]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:51:55.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:55 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:55.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:55 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:55.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:55 smithi055 ceph-mon[184174]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:55.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:55 smithi055 ceph-mon[184174]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:51:55.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:55 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:55.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:55 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:56.005 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:51:56.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:55 smithi099 ceph-mon[167076]: pgmap v87: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:56.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:55 smithi099 ceph-mon[167076]: from='client.34236 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:51:56.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:55 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:56.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:55 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:57.192 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:57.192 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:57.192 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:57 smithi099 ceph-mon[167076]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:57 smithi055 ceph-mon[179668]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:57.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:57.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:57.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:57 smithi055 ceph-mon[184174]: pgmap v88: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:51:57.538 INFO:teuthology.orchestra.run.smithi055.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:51:57.869 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'sleep 60' 2023-09-23T17:51:58.476 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:51:58.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:58 smithi099 ceph-mon[167076]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:51:58.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:58 smithi099 ceph-mon[167076]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:51:58.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:51:58 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:58.828 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:51:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:51:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:51:58.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:58 smithi055 ceph-mon[184174]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:51:58.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:58 smithi055 ceph-mon[184174]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:51:58.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:51:58 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:51:58.859 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:58 smithi055 ceph-mon[179668]: from='client.44144 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:51:58.859 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:58 smithi055 ceph-mon[179668]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:51:58.860 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:51:58 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:00 smithi099 ceph-mon[167076]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:52:00.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:00.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:52:00.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:52:00.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:00.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[184174]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:00.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:00.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:00.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:00.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:52:00.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:00.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:52:00.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:52:00.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[179668]: pgmap v89: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:52:00.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:00.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:52:00.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:52:00.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:00.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:52:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:52:00] "GET /metrics HTTP/1.1" 200 33828 "" "Prometheus/2.43.0" 2023-09-23T17:52:01.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[179668]: Reconfiguring mon.a (monmap changed)... 2023-09-23T17:52:01.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[179668]: Reconfiguring daemon mon.a on smithi055 2023-09-23T17:52:01.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:01.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:01.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:52:01.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:52:01.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:01.123 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[184174]: Reconfiguring mon.a (monmap changed)... 2023-09-23T17:52:01.123 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[184174]: Reconfiguring daemon mon.a on smithi055 2023-09-23T17:52:01.123 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:01.123 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:01.123 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:52:01.124 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:52:01.124 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:01 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:01.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:01 smithi099 ceph-mon[167076]: Reconfiguring mon.a (monmap changed)... 2023-09-23T17:52:01.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:01 smithi099 ceph-mon[167076]: Reconfiguring daemon mon.a on smithi055 2023-09-23T17:52:01.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:01 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:01.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:01 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:01.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:01 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:52:01.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:01 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:52:01.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:01 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:02.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[179668]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T17:52:02.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[179668]: Reconfiguring daemon mgr.y on smithi055 2023-09-23T17:52:02.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[179668]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:02.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:02.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:02.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:52:02.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:52:02.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:02.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[184174]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T17:52:02.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[184174]: Reconfiguring daemon mgr.y on smithi055 2023-09-23T17:52:02.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[184174]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:02.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:02.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:02.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:52:02.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:52:02.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:02.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:02 smithi099 ceph-mon[167076]: Reconfiguring mgr.y (monmap changed)... 2023-09-23T17:52:02.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:02 smithi099 ceph-mon[167076]: Reconfiguring daemon mgr.y on smithi055 2023-09-23T17:52:02.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:02 smithi099 ceph-mon[167076]: pgmap v90: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:02.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:02.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:02.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:52:02.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:52:02.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:03 smithi099 ceph-mon[167076]: Reconfiguring mon.c (monmap changed)... 2023-09-23T17:52:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:03 smithi099 ceph-mon[167076]: Reconfiguring daemon mon.c on smithi055 2023-09-23T17:52:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:03 smithi099 ceph-mon[167076]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:03 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:03 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:03 smithi099 ceph-mon[167076]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T17:52:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:03 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T17:52:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:03 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:03.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:03 smithi099 ceph-mon[167076]: Reconfiguring daemon osd.0 on smithi055 2023-09-23T17:52:03.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[184174]: Reconfiguring mon.c (monmap changed)... 2023-09-23T17:52:03.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[184174]: Reconfiguring daemon mon.c on smithi055 2023-09-23T17:52:03.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[184174]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:03.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:03.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:03.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[184174]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T17:52:03.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T17:52:03.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:03.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[184174]: Reconfiguring daemon osd.0 on smithi055 2023-09-23T17:52:03.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[179668]: Reconfiguring mon.c (monmap changed)... 2023-09-23T17:52:03.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[179668]: Reconfiguring daemon mon.c on smithi055 2023-09-23T17:52:03.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[179668]: pgmap v91: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:03.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:03.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:03.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[179668]: Reconfiguring osd.0 (monmap changed)... 2023-09-23T17:52:03.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T17:52:03.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:03.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:03 smithi055 ceph-mon[179668]: Reconfiguring daemon osd.0 on smithi055 2023-09-23T17:52:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[179668]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T17:52:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T17:52:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[179668]: Reconfiguring daemon osd.1 on smithi055 2023-09-23T17:52:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[184174]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T17:52:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T17:52:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:04 smithi055 ceph-mon[184174]: Reconfiguring daemon osd.1 on smithi055 2023-09-23T17:52:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:04 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:04 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:04 smithi099 ceph-mon[167076]: Reconfiguring osd.1 (monmap changed)... 2023-09-23T17:52:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:04 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T17:52:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:04 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:04 smithi099 ceph-mon[167076]: Reconfiguring daemon osd.1 on smithi055 2023-09-23T17:52:06.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[179668]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:06.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:06.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:06.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[179668]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T17:52:06.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T17:52:06.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:06.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[179668]: Reconfiguring daemon osd.2 on smithi055 2023-09-23T17:52:06.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[184174]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:06.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:06.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:06.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[184174]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T17:52:06.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T17:52:06.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:06.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:05 smithi055 ceph-mon[184174]: Reconfiguring daemon osd.2 on smithi055 2023-09-23T17:52:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:05 smithi099 ceph-mon[167076]: pgmap v92: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:05 smithi099 ceph-mon[167076]: Reconfiguring osd.2 (monmap changed)... 2023-09-23T17:52:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T17:52:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:05 smithi099 ceph-mon[167076]: Reconfiguring daemon osd.2 on smithi055 2023-09-23T17:52:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[179668]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T17:52:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T17:52:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[179668]: Reconfiguring daemon osd.3 on smithi055 2023-09-23T17:52:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[179668]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:07.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:07.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:07.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:07.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[184174]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T17:52:07.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T17:52:07.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:07.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[184174]: Reconfiguring daemon osd.3 on smithi055 2023-09-23T17:52:07.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[184174]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:07.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:07 smithi099 ceph-mon[167076]: Reconfiguring osd.3 (monmap changed)... 2023-09-23T17:52:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T17:52:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:07 smithi099 ceph-mon[167076]: Reconfiguring daemon osd.3 on smithi055 2023-09-23T17:52:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:07 smithi099 ceph-mon[167076]: pgmap v93: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:07.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:08.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:08.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:08.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[179668]: Reconfiguring mds.foo.smithi055.exjftp (monmap changed)... 2023-09-23T17:52:08.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.exjftp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:52:08.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:08.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[179668]: Reconfiguring daemon mds.foo.smithi055.exjftp on smithi055 2023-09-23T17:52:08.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:08.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:08.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[184174]: Reconfiguring mds.foo.smithi055.exjftp (monmap changed)... 2023-09-23T17:52:08.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.exjftp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:52:08.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:08.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:08 smithi055 ceph-mon[184174]: Reconfiguring daemon mds.foo.smithi055.exjftp on smithi055 2023-09-23T17:52:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:08 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:08 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:08 smithi099 ceph-mon[167076]: Reconfiguring mds.foo.smithi055.exjftp (monmap changed)... 2023-09-23T17:52:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:08 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi055.exjftp", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:52:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:08 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:08 smithi099 ceph-mon[167076]: Reconfiguring daemon mds.foo.smithi055.exjftp on smithi055 2023-09-23T17:52:09.014 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:52:08 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:52:08.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:52:09.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[184174]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:09.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:09.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:09.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[184174]: Reconfiguring mon.b (monmap changed)... 2023-09-23T17:52:09.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:52:09.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:52:09.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:09.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[184174]: Reconfiguring daemon mon.b on smithi099 2023-09-23T17:52:09.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[179668]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:09.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:09.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:09.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[179668]: Reconfiguring mon.b (monmap changed)... 2023-09-23T17:52:09.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:52:09.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:52:09.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:09.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:09 smithi055 ceph-mon[179668]: Reconfiguring daemon mon.b on smithi099 2023-09-23T17:52:09.685 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:09 smithi099 ceph-mon[167076]: pgmap v94: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:09.685 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:09 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:09.685 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:09 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:09.686 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:09 smithi099 ceph-mon[167076]: Reconfiguring mon.b (monmap changed)... 2023-09-23T17:52:09.686 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:09 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-09-23T17:52:09.686 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:09 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-09-23T17:52:09.686 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:09 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:09.686 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:09 smithi099 ceph-mon[167076]: Reconfiguring daemon mon.b on smithi099 2023-09-23T17:52:10.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:52:10] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.43.0" 2023-09-23T17:52:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:10 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:10 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:10 smithi099 ceph-mon[167076]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T17:52:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:10 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:52:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:10 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:52:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:10 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:10 smithi099 ceph-mon[167076]: Reconfiguring daemon mgr.x on smithi099 2023-09-23T17:52:11.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:11.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:11.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[179668]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T17:52:11.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:52:11.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:52:11.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:11.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[179668]: Reconfiguring daemon mgr.x on smithi099 2023-09-23T17:52:11.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:11.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:11.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[184174]: Reconfiguring mgr.x (monmap changed)... 2023-09-23T17:52:11.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-09-23T17:52:11.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-09-23T17:52:11.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:11.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:10 smithi055 ceph-mon[184174]: Reconfiguring daemon mgr.x on smithi099 2023-09-23T17:52:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:11 smithi099 ceph-mon[167076]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:11 smithi099 ceph-mon[167076]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T17:52:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T17:52:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:11 smithi099 ceph-mon[167076]: Reconfiguring daemon osd.4 on smithi099 2023-09-23T17:52:12.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[179668]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:12.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:12.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:12.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[179668]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T17:52:12.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T17:52:12.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:12.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[179668]: Reconfiguring daemon osd.4 on smithi099 2023-09-23T17:52:12.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[184174]: pgmap v95: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:12.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:12.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:12.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[184174]: Reconfiguring osd.4 (monmap changed)... 2023-09-23T17:52:12.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-09-23T17:52:12.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:12.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:11 smithi055 ceph-mon[184174]: Reconfiguring daemon osd.4 on smithi099 2023-09-23T17:52:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:13 smithi099 ceph-mon[167076]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T17:52:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T17:52:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:13 smithi099 ceph-mon[167076]: Reconfiguring daemon osd.5 on smithi099 2023-09-23T17:52:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:13 smithi099 ceph-mon[167076]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[179668]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T17:52:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T17:52:13.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[179668]: Reconfiguring daemon osd.5 on smithi099 2023-09-23T17:52:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[179668]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:13.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:13.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:13.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[184174]: Reconfiguring osd.5 (monmap changed)... 2023-09-23T17:52:13.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-09-23T17:52:13.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:13.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[184174]: Reconfiguring daemon osd.5 on smithi099 2023-09-23T17:52:13.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:13 smithi055 ceph-mon[184174]: pgmap v96: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:14 smithi099 ceph-mon[167076]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T17:52:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T17:52:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:14 smithi099 ceph-mon[167076]: Reconfiguring daemon osd.6 on smithi099 2023-09-23T17:52:14.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:14.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:14.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[179668]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T17:52:14.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T17:52:14.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:14.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[179668]: Reconfiguring daemon osd.6 on smithi099 2023-09-23T17:52:14.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:14.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:14.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[184174]: Reconfiguring osd.6 (monmap changed)... 2023-09-23T17:52:14.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-09-23T17:52:14.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:14.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:14 smithi055 ceph-mon[184174]: Reconfiguring daemon osd.6 on smithi099 2023-09-23T17:52:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:15 smithi099 ceph-mon[167076]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:15 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:15 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:15 smithi099 ceph-mon[167076]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T17:52:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:15 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T17:52:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:15 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:15.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:15 smithi099 ceph-mon[167076]: Reconfiguring daemon osd.7 on smithi099 2023-09-23T17:52:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[179668]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[179668]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T17:52:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T17:52:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[179668]: Reconfiguring daemon osd.7 on smithi099 2023-09-23T17:52:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[184174]: pgmap v97: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[184174]: Reconfiguring osd.7 (monmap changed)... 2023-09-23T17:52:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-09-23T17:52:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:15.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:15 smithi055 ceph-mon[184174]: Reconfiguring daemon osd.7 on smithi099 2023-09-23T17:52:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:16 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:16 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:16 smithi099 ceph-mon[167076]: Reconfiguring mds.foo.smithi099.djovrl (monmap changed)... 2023-09-23T17:52:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:16 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.djovrl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:52:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:16 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:16 smithi099 ceph-mon[167076]: Reconfiguring daemon mds.foo.smithi099.djovrl on smithi099 2023-09-23T17:52:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[179668]: Reconfiguring mds.foo.smithi099.djovrl (monmap changed)... 2023-09-23T17:52:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.djovrl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:52:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[179668]: Reconfiguring daemon mds.foo.smithi099.djovrl on smithi099 2023-09-23T17:52:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:16.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:16.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[184174]: Reconfiguring mds.foo.smithi099.djovrl (monmap changed)... 2023-09-23T17:52:16.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi099.djovrl", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-09-23T17:52:16.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:16.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:16 smithi055 ceph-mon[184174]: Reconfiguring daemon mds.foo.smithi099.djovrl on smithi099 2023-09-23T17:52:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:17 smithi099 ceph-mon[167076]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:52:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:52:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[179668]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:52:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:52:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:18.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[184174]: pgmap v98: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:18.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:18.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:18.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:52:18.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:52:18.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:52:18.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:52:19.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:52:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:52:18.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:52:20.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:19 smithi099 ceph-mon[167076]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:20.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:19 smithi055 ceph-mon[184174]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:19 smithi055 ceph-mon[179668]: pgmap v99: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:20.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:52:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:52:20] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.43.0" 2023-09-23T17:52:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:21 smithi099 ceph-mon[167076]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:22.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:21 smithi055 ceph-mon[179668]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:22.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:22.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:21 smithi055 ceph-mon[184174]: pgmap v100: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:22.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:23.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:23 smithi055 ceph-mon[179668]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:23 smithi055 ceph-mon[184174]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:23 smithi099 ceph-mon[167076]: pgmap v101: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:25.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:25 smithi055 ceph-mon[179668]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:25 smithi055 ceph-mon[184174]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:25 smithi099 ceph-mon[167076]: pgmap v102: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:27.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:27 smithi055 ceph-mon[179668]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:27.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:27 smithi055 ceph-mon[184174]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:27.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:27 smithi099 ceph-mon[167076]: pgmap v103: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:29.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:52:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:52:28.662Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:52:29.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:29 smithi055 ceph-mon[179668]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:29.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:29 smithi055 ceph-mon[184174]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:29 smithi099 ceph-mon[167076]: pgmap v104: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:30.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:52:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:52:30] "GET /metrics HTTP/1.1" 200 33898 "" "Prometheus/2.43.0" 2023-09-23T17:52:31.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:31 smithi055 ceph-mon[179668]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:31.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:31 smithi055 ceph-mon[184174]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:31 smithi099 ceph-mon[167076]: pgmap v105: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:33 smithi055 ceph-mon[179668]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:33 smithi055 ceph-mon[184174]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:33 smithi099 ceph-mon[167076]: pgmap v106: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:35.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:35 smithi055 ceph-mon[179668]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:35.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:35 smithi055 ceph-mon[184174]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:35 smithi099 ceph-mon[167076]: pgmap v107: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:37 smithi099 ceph-mon[167076]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:37.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:37 smithi055 ceph-mon[179668]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:37 smithi055 ceph-mon[184174]: pgmap v108: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:39.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:52:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:52:38.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:52:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:39 smithi099 ceph-mon[167076]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:39.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:39 smithi055 ceph-mon[184174]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:39.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:39 smithi055 ceph-mon[179668]: pgmap v109: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:40.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:52:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:52:40] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.43.0" 2023-09-23T17:52:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:41 smithi099 ceph-mon[167076]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:41.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:41 smithi055 ceph-mon[179668]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:41.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:41 smithi055 ceph-mon[184174]: pgmap v110: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:43 smithi055 ceph-mon[179668]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:43 smithi055 ceph-mon[184174]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:43 smithi099 ceph-mon[167076]: pgmap v111: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:45 smithi055 ceph-mon[179668]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:45.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:45 smithi055 ceph-mon[184174]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:45 smithi099 ceph-mon[167076]: pgmap v112: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:47 smithi099 ceph-mon[167076]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:47 smithi055 ceph-mon[179668]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:47.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:47 smithi055 ceph-mon[184174]: pgmap v113: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:49.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:52:48 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:52:48.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:52:49.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:49 smithi099 ceph-mon[167076]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:49 smithi055 ceph-mon[179668]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:49.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:49 smithi055 ceph-mon[184174]: pgmap v114: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:50.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:52:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:52:50] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.43.0" 2023-09-23T17:52:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:51 smithi099 ceph-mon[167076]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:51.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:51 smithi055 ceph-mon[179668]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:51.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:51.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:51 smithi055 ceph-mon[184174]: pgmap v115: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:51.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:52:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:53 smithi055 ceph-mon[179668]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:53.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:53 smithi055 ceph-mon[184174]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:53 smithi099 ceph-mon[167076]: pgmap v116: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:55.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:55 smithi055 ceph-mon[184174]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:55.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:55 smithi055 ceph-mon[179668]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:55.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:55 smithi099 ceph-mon[167076]: pgmap v117: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:57.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:57 smithi055 ceph-mon[179668]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:57 smithi055 ceph-mon[184174]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:57 smithi099 ceph-mon[167076]: pgmap v118: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:59.077 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 17:52:58 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T17:52:58.661Z caller=manager.go:640 level=warn component="rule manager" file=/etc/prometheus/alerting/ceph_alerts.yml group=osd name=CephOSDFlapping index=13 msg="Evaluating rule failed" rule="alert: CephOSDFlapping\nexpr: (rate(ceph_osd_up[5m]) * on (ceph_daemon) group_left (hostname) ceph_osd_metadata)\n * 60 > 1\nlabels:\n oid: 1.3.6.1.4.1.50495.1.2.1.4.4\n severity: warning\n type: ceph_default\nannotations:\n description: OSD {{ $labels.ceph_daemon }} on {{ $labels.hostname }} was marked\n down and back up {{ $value | humanize }} times once a minute for 5 minutes. This\n may indicate a network issue (latency, packet loss, MTU mismatch) on the cluster\n network, or the public network if no cluster network is deployed. Check the network\n stats on the listed host(s).\n documentation: https://docs.ceph.com/en/latest/rados/troubleshooting/troubleshooting-osd#flapping-osds\n summary: Network issues are causing OSDs to flap (mark each other down)\n" err="found duplicate series for the match group {ceph_daemon=\"osd.0\"} on the right hand-side of the operation: [{__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi099.front.sepia.ceph.com:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}, {__name__=\"ceph_osd_metadata\", ceph_daemon=\"osd.0\", ceph_version=\"ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)\", cluster_addr=\"172.21.15.55\", device_class=\"ssd\", hostname=\"smithi055\", instance=\"smithi055:9283\", job=\"ceph\", objectstore=\"bluestore\", public_addr=\"172.21.15.55\"}];many-to-many matching not allowed: matching labels must be unique on one side" 2023-09-23T17:52:59.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:52:59 smithi055 ceph-mon[179668]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:59.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:52:59 smithi055 ceph-mon[184174]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:59.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:52:59 smithi099 ceph-mon[167076]: pgmap v119: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:52:59.867 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade resume' 2023-09-23T17:53:00.474 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:53:00.669 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:53:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:53:00] "GET /metrics HTTP/1.1" 200 33895 "" "Prometheus/2.43.0" 2023-09-23T17:53:01.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:01 smithi099 ceph-mon[167076]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:01.892 INFO:teuthology.orchestra.run.smithi055.stdout:Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:53:01.917 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:01 smithi055 ceph-mon[179668]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:01.917 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:01 smithi055 ceph-mon[184174]: pgmap v120: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:02.230 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'echo wait for half of osds to upgrade' 2023-09-23T17:53:02.901 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:53:03.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:02 smithi055 ceph-mon[184174]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:53:03.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:02 smithi055 ceph-mon[184174]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:53:03.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:03.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:02 smithi055 ceph-mon[179668]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:53:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:02 smithi055 ceph-mon[179668]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:53:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:03.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:02 smithi099 ceph-mon[167076]: from='client.34248 -' entity='client.admin' cmd=[{"prefix": "orch upgrade resume", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:53:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:02 smithi099 ceph-mon[167076]: Upgrade: Resumed upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:53:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:03.905 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:03 smithi055 ceph-mon[179668]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:04.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:03 smithi055 ceph-mon[184174]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:04.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:03 smithi099 ceph-mon[167076]: pgmap v121: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:04.364 INFO:teuthology.orchestra.run.smithi055.stdout:wait for half of osds to upgrade 2023-09-23T17:53:04.753 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'while ! ceph osd versions | grep $sha1 | egrep '"'"': [45678]'"'"'; do sleep 2 ; done' 2023-09-23T17:53:05.421 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:53:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:05 smithi055 ceph-mon[184174]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:06.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:05 smithi055 ceph-mon[179668]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:05 smithi099 ceph-mon[167076]: pgmap v122: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:06.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:07.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:07.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:07.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:07.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3514757648' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:53:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:08.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T17:53:08.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:08.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3514757648' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:08.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:08.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:53:08.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:08.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:08.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:08.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T17:53:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T17:53:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T17:53:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T17:53:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T17:53:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T17:53:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T17:53:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: pgmap v123: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3514757648' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:53:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]: dispatch 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.a"}]': finished 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]: dispatch 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.b"}]': finished 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]: dispatch 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "config rm", "name": "container_image", "who": "mon.c"}]': finished 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T17:53:09.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:08 smithi055 ceph-mon[179668]: Upgrade: Setting container_image for all mon 2023-09-23T17:53:09.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:08 smithi055 ceph-mon[179668]: Upgrade: Setting container_image for all crash 2023-09-23T17:53:09.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:08 smithi055 ceph-mon[179668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T17:53:09.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:08 smithi055 ceph-mon[179668]: Upgrade: osd.0 is safe to restart 2023-09-23T17:53:09.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:08 smithi055 ceph-mon[184174]: Upgrade: Setting container_image for all mon 2023-09-23T17:53:09.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:08 smithi055 ceph-mon[184174]: Upgrade: Setting container_image for all crash 2023-09-23T17:53:09.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:08 smithi055 ceph-mon[184174]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T17:53:09.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:08 smithi055 ceph-mon[184174]: Upgrade: osd.0 is safe to restart 2023-09-23T17:53:09.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:08 smithi099 ceph-mon[167076]: Upgrade: Setting container_image for all mon 2023-09-23T17:53:09.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:08 smithi099 ceph-mon[167076]: Upgrade: Setting container_image for all crash 2023-09-23T17:53:09.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:08 smithi099 ceph-mon[167076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["0"], "max": 16}]: dispatch 2023-09-23T17:53:09.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:08 smithi099 ceph-mon[167076]: Upgrade: osd.0 is safe to restart 2023-09-23T17:53:10.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[179668]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:10.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[179668]: Upgrade: Updating osd.0 2023-09-23T17:53:10.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:10.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T17:53:10.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:10.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[179668]: Deploying daemon osd.0 on smithi055 2023-09-23T17:53:10.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2491720023' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:10.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[184174]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:10.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[184174]: Upgrade: Updating osd.0 2023-09-23T17:53:10.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:10.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T17:53:10.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:10.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[184174]: Deploying daemon osd.0 on smithi055 2023-09-23T17:53:10.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2491720023' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:09 smithi099 ceph-mon[167076]: pgmap v124: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:09 smithi099 ceph-mon[167076]: Upgrade: Updating osd.0 2023-09-23T17:53:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:09 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:09 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-09-23T17:53:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:09 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:09 smithi099 ceph-mon[167076]: Deploying daemon osd.0 on smithi055 2023-09-23T17:53:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2491720023' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:10.564 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:53:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:53:10] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.43.0" 2023-09-23T17:53:11.430 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:11 smithi055 systemd[1]: Stopping Ceph osd.0 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:53:11.930 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:11 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[114371]: 2023-09-23T17:53:11.488+0000 7f5920cf5700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:53:11.930 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:11 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[114371]: 2023-09-23T17:53:11.488+0000 7f5920cf5700 -1 osd.0 89 *** Got signal Terminated *** 2023-09-23T17:53:11.930 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:11 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[114371]: 2023-09-23T17:53:11.488+0000 7f5920cf5700 -1 osd.0 89 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T17:53:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:11 smithi099 ceph-mon[167076]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:11 smithi099 ceph-mon[167076]: osd.0 marked itself down and dead 2023-09-23T17:53:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:11 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/996733272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:11 smithi055 ceph-mon[179668]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:11 smithi055 ceph-mon[179668]: osd.0 marked itself down and dead 2023-09-23T17:53:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:11 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/996733272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:11 smithi055 ceph-mon[184174]: pgmap v125: 73 pgs: 73 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:12.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:11 smithi055 ceph-mon[184174]: osd.0 marked itself down and dead 2023-09-23T17:53:12.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:11 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/996733272' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:12.929 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:12 smithi055 bash[194353]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T17:53:13.226 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:12 smithi055 ceph-mon[179668]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:53:13.227 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:12 smithi055 ceph-mon[179668]: failed to encode map e90 with expected crc 2023-09-23T17:53:13.227 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:12 smithi055 ceph-mon[179668]: failed to encode map e90 with expected crc 2023-09-23T17:53:13.227 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:12 smithi055 ceph-mon[179668]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T17:53:13.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:12 smithi055 ceph-mon[184174]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:53:13.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:12 smithi055 ceph-mon[184174]: failed to encode map e90 with expected crc 2023-09-23T17:53:13.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:12 smithi055 ceph-mon[184174]: failed to encode map e90 with expected crc 2023-09-23T17:53:13.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:12 smithi055 ceph-mon[184174]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T17:53:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:12 smithi099 ceph-mon[167076]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:53:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:12 smithi099 ceph-mon[167076]: failed to encode map e90 with expected crc 2023-09-23T17:53:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:12 smithi099 ceph-mon[167076]: failed to encode map e90 with expected crc 2023-09-23T17:53:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:12 smithi099 ceph-mon[167076]: osdmap e90: 8 total, 7 up, 8 in 2023-09-23T17:53:14.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:13 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.0.service: Succeeded. 2023-09-23T17:53:14.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:13 smithi055 systemd[1]: Stopped Ceph osd.0 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:53:14.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:14 smithi055 systemd[1]: Starting Ceph osd.0 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:53:14.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: pgmap v127: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T17:53:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 9 pgs inactive, 26 pgs peering (PG_AVAILABILITY) 2023-09-23T17:53:14.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: pgmap v127: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:14.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T17:53:14.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:13 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 9 pgs inactive, 26 pgs peering (PG_AVAILABILITY) 2023-09-23T17:53:14.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: pgmap v127: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:14.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: osdmap e91: 8 total, 7 up, 8 in 2023-09-23T17:53:14.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: failed to encode map e91 with expected crc 2023-09-23T17:53:14.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:13 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 9 pgs inactive, 26 pgs peering (PG_AVAILABILITY) 2023-09-23T17:53:15.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2811086331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:15.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2811086331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:15.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:14 smithi055 bash[194671]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T17:53:15.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:14 smithi055 bash[194671]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T17:53:15.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:14 smithi055 bash[194671]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_4 --path /var/lib/ceph/osd/ceph-0 --no-mon-config 2023-09-23T17:53:15.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:14 smithi055 bash[194671]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_4 /var/lib/ceph/osd/ceph-0/block 2023-09-23T17:53:15.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:14 smithi055 bash[194671]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-0/block 2023-09-23T17:53:15.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:14 smithi055 bash[194671]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-3 2023-09-23T17:53:15.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:14 smithi055 bash[194671]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-0 2023-09-23T17:53:15.181 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:14 smithi055 bash[194671]: --> ceph-volume lvm activate successful for osd ID: 0 2023-09-23T17:53:15.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2811086331' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:15.930 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:15 smithi055 podman[194940]: 2023-09-23T17:53:15.930 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:15 smithi055 bash[194940]: a5bacbf2c1012af5590576ac2d3637d99e3e8e3846d5405d3b605616181fa59e 2023-09-23T17:53:15.930 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:15 smithi055 ceph-osd[194980]: -- 172.21.15.55:0/1839553390 <== mon.2 v2:172.21.15.55:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5561348504e0 con 0x556134821400 2023-09-23T17:53:15.930 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:15 smithi055 systemd[1]: Started Ceph osd.0 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:53:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:15 smithi099 ceph-mon[167076]: pgmap v129: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:15 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:15 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:15 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:16.429 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:16 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[194956]: 2023-09-23T17:53:16.208+0000 7f1c2b970640 -1 Falling back to public interface 2023-09-23T17:53:16.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:15 smithi055 ceph-mon[179668]: pgmap v129: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:16.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:16.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:16.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:16.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:15 smithi055 ceph-mon[184174]: pgmap v129: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:16.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:15 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:16.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:15 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:16.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:15 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:17.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3927791167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:17.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3927791167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3927791167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:17 smithi099 ceph-mon[167076]: pgmap v130: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:18.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:17 smithi055 ceph-mon[179668]: pgmap v130: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:18.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:17 smithi055 ceph-mon[184174]: pgmap v130: 73 pgs: 26 peering, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:19 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2818567447' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:19 smithi099 ceph-mon[167076]: from='osd.0 [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T17:53:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:19 smithi099 ceph-mon[167076]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T17:53:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:19 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2818567447' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:19 smithi055 ceph-mon[179668]: from='osd.0 [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T17:53:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:19 smithi055 ceph-mon[179668]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T17:53:19.430 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:18 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[194956]: 2023-09-23T17:53:18.964+0000 7f1c2b970640 -1 osd.0 89 log_to_monitors true 2023-09-23T17:53:19.431 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 17:53:19 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[194956]: 2023-09-23T17:53:19.040+0000 7f1c1b5da700 -1 osd.0 89 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:53:19.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:19 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2818567447' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:19.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:19 smithi055 ceph-mon[184174]: from='osd.0 [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T17:53:19.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:19 smithi055 ceph-mon[184174]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-09-23T17:53:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: pgmap v131: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T17:53:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T17:53:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 26 pgs peering) 2023-09-23T17:53:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: failed to encode map e92 with expected crc 2023-09-23T17:53:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: failed to encode map e92 with expected crc 2023-09-23T17:53:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T17:53:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T17:53:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: from='osd.0 [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:20.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:20.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:20.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:20 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:20.361 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: pgmap v131: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T17:53:20.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T17:53:20.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 26 pgs peering) 2023-09-23T17:53:20.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: failed to encode map e92 with expected crc 2023-09-23T17:53:20.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: failed to encode map e92 with expected crc 2023-09-23T17:53:20.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T17:53:20.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T17:53:20.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: from='osd.0 [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:20.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:20.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:20.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:20.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: pgmap v131: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 84 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T17:53:20.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 14/84 objects degraded (16.667%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T17:53:20.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 9 pgs inactive, 26 pgs peering) 2023-09-23T17:53:20.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: failed to encode map e92 with expected crc 2023-09-23T17:53:20.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: failed to encode map e92 with expected crc 2023-09-23T17:53:20.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-09-23T17:53:20.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: osdmap e92: 8 total, 7 up, 8 in 2023-09-23T17:53:20.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: from='osd.0 [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:20.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:20.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:20.364 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:20.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:53:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:53:20] "GET /metrics HTTP/1.1" 200 33900 "" "Prometheus/2.43.0" 2023-09-23T17:53:21.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:21 smithi099 ceph-mon[167076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:53:21.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:21 smithi099 ceph-mon[167076]: failed to encode map e93 with expected crc 2023-09-23T17:53:21.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:21 smithi099 ceph-mon[167076]: failed to encode map e93 with expected crc 2023-09-23T17:53:21.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:21 smithi099 ceph-mon[167076]: osd.0 [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049] boot 2023-09-23T17:53:21.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:21 smithi099 ceph-mon[167076]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T17:53:21.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:53:21.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:21.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:21.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2640800160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:21.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:21 smithi099 ceph-mon[167076]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T17:53:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[179668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:53:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[179668]: failed to encode map e93 with expected crc 2023-09-23T17:53:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[179668]: failed to encode map e93 with expected crc 2023-09-23T17:53:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[179668]: osd.0 [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049] boot 2023-09-23T17:53:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[179668]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T17:53:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:53:21.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:21.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:21.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2640800160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:21.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[179668]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T17:53:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[184174]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:53:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[184174]: failed to encode map e93 with expected crc 2023-09-23T17:53:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[184174]: failed to encode map e93 with expected crc 2023-09-23T17:53:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[184174]: osd.0 [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049] boot 2023-09-23T17:53:21.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[184174]: osdmap e93: 8 total, 8 up, 8 in 2023-09-23T17:53:21.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-09-23T17:53:21.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:21.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:21.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2640800160' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:21.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:21 smithi055 ceph-mon[184174]: osdmap e94: 8 total, 8 up, 8 in 2023-09-23T17:53:22.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[184174]: pgmap v134: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T17:53:22.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[184174]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[184174]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[184174]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[184174]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[184174]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[184174]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:22.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:22.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[179668]: pgmap v134: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T17:53:22.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[179668]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[179668]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[179668]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[179668]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[179668]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[179668]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:22.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:22 smithi099 ceph-mon[167076]: pgmap v134: 73 pgs: 9 active+undersized+degraded, 17 active+undersized, 47 active+clean; 4.6 MiB data, 100 MiB used, 715 GiB / 715 GiB avail; 14/84 objects degraded (16.667%) 2023-09-23T17:53:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:22 smithi099 ceph-mon[167076]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:22 smithi099 ceph-mon[167076]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:22 smithi099 ceph-mon[167076]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:22 smithi099 ceph-mon[167076]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:22 smithi099 ceph-mon[167076]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:22 smithi099 ceph-mon[167076]: failed to encode map e94 with expected crc 2023-09-23T17:53:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:23.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:23 smithi055 ceph-mon[179668]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:53:23.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:23 smithi055 ceph-mon[184174]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:53:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:23 smithi099 ceph-mon[167076]: pgmap v136: 73 pgs: 2 active+undersized+degraded, 9 active+undersized, 62 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:53:24.357 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:53:24.217+0000 7f3f79c94700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T17:53:24.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2735803682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:24.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:24.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:24.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:24.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:53:24.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:24.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:24.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:24.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:24.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:24.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:24.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2735803682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:24.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:24.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:24.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:24.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:53:24.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:24.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:24.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:24.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:24.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:24.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2735803682' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:53:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:24.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:25.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:25 smithi055 ceph-mon[179668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:25.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:25 smithi055 ceph-mon[179668]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T17:53:25.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:25 smithi055 ceph-mon[179668]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:25 smithi055 ceph-mon[184174]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:25 smithi055 ceph-mon[184174]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T17:53:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:25 smithi055 ceph-mon[184174]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:25 smithi099 ceph-mon[167076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:25 smithi099 ceph-mon[167076]: Upgrade: unsafe to stop osd(s) at this time (3 PGs are or would become offline) 2023-09-23T17:53:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:25 smithi099 ceph-mon[167076]: pgmap v137: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:26 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T17:53:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:26 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T17:53:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2810760531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:26.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:26 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T17:53:26.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:26 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T17:53:26.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2810760531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:26.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:26 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T17:53:26.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:26 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T17:53:26.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2810760531' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:27.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:27 smithi055 ceph-mon[179668]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:27.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:27 smithi055 ceph-mon[184174]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:27 smithi099 ceph-mon[167076]: pgmap v138: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:28.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:28 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/959399533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:28.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:28 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/959399533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:28.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:28 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/959399533' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:29.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:29 smithi055 ceph-mon[179668]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:29.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:29 smithi055 ceph-mon[184174]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:29.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:29 smithi099 ceph-mon[167076]: pgmap v139: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:30.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:53:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:53:30] "GET /metrics HTTP/1.1" 200 33819 "" "Prometheus/2.43.0" 2023-09-23T17:53:30.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1209600586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1209600586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1209600586' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:31 smithi099 ceph-mon[167076]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:31 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:31 smithi055 ceph-mon[184174]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:31 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:31.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:31 smithi055 ceph-mon[179668]: pgmap v140: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:31.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:31 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:33 smithi055 ceph-mon[179668]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1829272195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:33 smithi055 ceph-mon[184174]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1829272195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:33 smithi099 ceph-mon[167076]: pgmap v141: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:33 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1829272195' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:35 smithi099 ceph-mon[167076]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:35 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2156816110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:35.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:35 smithi055 ceph-mon[179668]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:35.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:35 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2156816110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:35 smithi055 ceph-mon[184174]: pgmap v142: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:35 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2156816110' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:37 smithi055 ceph-mon[179668]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1749975167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:37 smithi055 ceph-mon[184174]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1749975167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:37 smithi099 ceph-mon[167076]: pgmap v143: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1749975167' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:39.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:39 smithi055 ceph-mon[179668]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:39.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:39.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:39 smithi055 ceph-mon[184174]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:39.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:39 smithi099 ceph-mon[167076]: pgmap v144: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:39 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:40.433 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:53:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:53:40] "GET /metrics HTTP/1.1" 200 33959 "" "Prometheus/2.43.0" 2023-09-23T17:53:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:40 smithi099 ceph-mon[167076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:40 smithi099 ceph-mon[167076]: Upgrade: osd.1 is safe to restart 2023-09-23T17:53:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/904291486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:40.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:40 smithi055 ceph-mon[179668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:40.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:40 smithi055 ceph-mon[179668]: Upgrade: osd.1 is safe to restart 2023-09-23T17:53:40.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/904291486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:40.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:40 smithi055 ceph-mon[184174]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["1"], "max": 16}]: dispatch 2023-09-23T17:53:40.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:40 smithi055 ceph-mon[184174]: Upgrade: osd.1 is safe to restart 2023-09-23T17:53:40.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/904291486' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[184174]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[184174]: Upgrade: Updating osd.1 2023-09-23T17:53:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T17:53:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[184174]: Deploying daemon osd.1 on smithi055 2023-09-23T17:53:41.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[179668]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:41.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[179668]: Upgrade: Updating osd.1 2023-09-23T17:53:41.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:41.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T17:53:41.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:41.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:41 smithi055 ceph-mon[179668]: Deploying daemon osd.1 on smithi055 2023-09-23T17:53:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:41 smithi099 ceph-mon[167076]: pgmap v145: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:41 smithi099 ceph-mon[167076]: Upgrade: Updating osd.1 2023-09-23T17:53:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:41 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:41 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-09-23T17:53:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:41 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:41 smithi099 ceph-mon[167076]: Deploying daemon osd.1 on smithi055 2023-09-23T17:53:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:42 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3622644739' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:42.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:42 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3622644739' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:43.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:42 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3622644739' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:43.268 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:42 smithi055 systemd[1]: Stopping Ceph osd.1 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:53:43.626 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[121485]: 2023-09-23T17:53:43.266+0000 7f219fb80700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:53:43.626 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[121485]: 2023-09-23T17:53:43.266+0000 7f219fb80700 -1 osd.1 94 *** Got signal Terminated *** 2023-09-23T17:53:43.626 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[121485]: 2023-09-23T17:53:43.266+0000 7f219fb80700 -1 osd.1 94 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T17:53:43.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:43 smithi055 ceph-mon[179668]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:43.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:43 smithi055 ceph-mon[179668]: osd.1 marked itself down and dead 2023-09-23T17:53:43.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:43 smithi055 ceph-mon[184174]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:43.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:43 smithi055 ceph-mon[184174]: osd.1 marked itself down and dead 2023-09-23T17:53:44.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:43 smithi099 ceph-mon[167076]: pgmap v146: 73 pgs: 73 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:44.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:43 smithi099 ceph-mon[167076]: osd.1 marked itself down and dead 2023-09-23T17:53:44.430 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:44 smithi055 bash[201154]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T17:53:44.911 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:44 smithi055 ceph-mon[179668]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:53:44.913 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:44 smithi055 ceph-mon[179668]: failed to encode map e95 with expected crc 2023-09-23T17:53:44.913 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:44 smithi055 ceph-mon[179668]: osdmap e95: 8 total, 7 up, 8 in 2023-09-23T17:53:44.913 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:44 smithi055 ceph-mon[179668]: failed to encode map e95 with expected crc 2023-09-23T17:53:44.913 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4234108909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:44.914 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:44 smithi055 ceph-mon[184174]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:53:44.914 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:44 smithi055 ceph-mon[184174]: failed to encode map e95 with expected crc 2023-09-23T17:53:44.914 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:44 smithi055 ceph-mon[184174]: osdmap e95: 8 total, 7 up, 8 in 2023-09-23T17:53:44.914 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:44 smithi055 ceph-mon[184174]: failed to encode map e95 with expected crc 2023-09-23T17:53:44.914 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4234108909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:45.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:44 smithi099 ceph-mon[167076]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:53:45.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:44 smithi099 ceph-mon[167076]: failed to encode map e95 with expected crc 2023-09-23T17:53:45.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:44 smithi099 ceph-mon[167076]: osdmap e95: 8 total, 7 up, 8 in 2023-09-23T17:53:45.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:44 smithi099 ceph-mon[167076]: failed to encode map e95 with expected crc 2023-09-23T17:53:45.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4234108909' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:45.930 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:45 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.1.service: Succeeded. 2023-09-23T17:53:45.930 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:45 smithi055 systemd[1]: Stopped Ceph osd.1 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:53:45.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:45 smithi055 systemd[1]: Starting Ceph osd.1 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:53:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[179668]: pgmap v148: 73 pgs: 4 peering, 8 stale+active+clean, 61 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:53:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[179668]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[179668]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[179668]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T17:53:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[179668]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[179668]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[179668]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[179668]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[179668]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[184174]: pgmap v148: 73 pgs: 4 peering, 8 stale+active+clean, 61 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:53:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[184174]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[184174]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[184174]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T17:53:45.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[184174]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[184174]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[184174]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[184174]: failed to encode map e96 with expected crc 2023-09-23T17:53:45.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:45 smithi055 ceph-mon[184174]: failed to encode map e96 with expected crc 2023-09-23T17:53:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:45 smithi099 ceph-mon[167076]: pgmap v148: 73 pgs: 4 peering, 8 stale+active+clean, 61 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:45 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:53:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:45 smithi099 ceph-mon[167076]: failed to encode map e96 with expected crc 2023-09-23T17:53:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:45 smithi099 ceph-mon[167076]: failed to encode map e96 with expected crc 2023-09-23T17:53:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:45 smithi099 ceph-mon[167076]: osdmap e96: 8 total, 7 up, 8 in 2023-09-23T17:53:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:45 smithi099 ceph-mon[167076]: failed to encode map e96 with expected crc 2023-09-23T17:53:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:45 smithi099 ceph-mon[167076]: failed to encode map e96 with expected crc 2023-09-23T17:53:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:45 smithi099 ceph-mon[167076]: failed to encode map e96 with expected crc 2023-09-23T17:53:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:45 smithi099 ceph-mon[167076]: failed to encode map e96 with expected crc 2023-09-23T17:53:46.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:45 smithi099 ceph-mon[167076]: failed to encode map e96 with expected crc 2023-09-23T17:53:46.930 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:46 smithi055 bash[201447]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T17:53:46.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:46 smithi055 bash[201447]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T17:53:46.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:46 smithi055 bash[201447]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T17:53:46.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:46 smithi055 bash[201447]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T17:53:46.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:46 smithi055 bash[201447]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T17:53:46.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:46 smithi055 bash[201447]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T17:53:46.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:46 smithi055 bash[201447]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T17:53:46.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:46 smithi055 bash[201447]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T17:53:47.650 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:47 smithi055 podman[201737]: 2023-09-23T17:53:47.650 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:47 smithi055 bash[201737]: 36ea52c16903fa134b2eb8492d65257c46eadff236dfeddb031f629d452e5846 2023-09-23T17:53:47.650 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-osd[201777]: -- 172.21.15.55:0/1649980822 <== mon.2 v2:172.21.15.55:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ccc19344e0 con 0x55ccc1904c00 2023-09-23T17:53:47.650 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:47 smithi055 systemd[1]: Started Ceph osd.1 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:53:47.930 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[201753]: 2023-09-23T17:53:47.852+0000 7fc9a82a1640 -1 Falling back to public interface 2023-09-23T17:53:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-mon[179668]: pgmap v150: 73 pgs: 4 peering, 8 stale+active+clean, 61 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2081414245' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:47.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-mon[184174]: pgmap v150: 73 pgs: 4 peering, 8 stale+active+clean, 61 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2081414245' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:47 smithi099 ceph-mon[167076]: pgmap v150: 73 pgs: 4 peering, 8 stale+active+clean, 61 active+clean; 4.6 MiB data, 101 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2081414245' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:48.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:49.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:48 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T17:53:49.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:48 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T17:53:49.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:48 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T17:53:50.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:49 smithi099 ceph-mon[167076]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 4 peering, 25 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:53:50.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/712020859' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:50.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:49 smithi055 ceph-mon[179668]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 4 peering, 25 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:53:50.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/712020859' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:50.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:49 smithi055 ceph-mon[184174]: pgmap v151: 73 pgs: 3 active+undersized+degraded, 4 peering, 25 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:53:50.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/712020859' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:50.624 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:53:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:53:50] "GET /metrics HTTP/1.1" 200 33959 "" "Prometheus/2.43.0" 2023-09-23T17:53:50.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:50 smithi055 ceph-mon[184174]: from='osd.1 [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T17:53:50.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:50 smithi055 ceph-mon[184174]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T17:53:50.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:50 smithi055 ceph-mon[179668]: from='osd.1 [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T17:53:50.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:50 smithi055 ceph-mon[179668]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T17:53:50.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[201753]: 2023-09-23T17:53:50.624+0000 7fc9a82a1640 -1 osd.1 94 log_to_monitors true 2023-09-23T17:53:50.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 17:53:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[201753]: 2023-09-23T17:53:50.747+0000 7fc997f0b700 -1 osd.1 94 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:53:51.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:50 smithi099 ceph-mon[167076]: from='osd.1 [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T17:53:51.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:50 smithi099 ceph-mon[167076]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T17:53:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: pgmap v152: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:53:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T17:53:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: failed to encode map e97 with expected crc 2023-09-23T17:53:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: failed to encode map e97 with expected crc 2023-09-23T17:53:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T17:53:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T17:53:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: from='osd.1 [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2730577878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: pgmap v152: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:53:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T17:53:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: failed to encode map e97 with expected crc 2023-09-23T17:53:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: failed to encode map e97 with expected crc 2023-09-23T17:53:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T17:53:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T17:53:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: from='osd.1 [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2730577878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: pgmap v152: 73 pgs: 5 active+undersized+degraded, 27 active+undersized, 41 active+clean; 4.6 MiB data, 105 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:53:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T17:53:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: failed to encode map e97 with expected crc 2023-09-23T17:53:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: failed to encode map e97 with expected crc 2023-09-23T17:53:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: from='osd.1 ' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T17:53:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: osdmap e97: 8 total, 7 up, 8 in 2023-09-23T17:53:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: from='osd.1 [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: from='osd.1 ' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:53:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:53:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2730577878' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: failed to encode map e97 with expected crc 2023-09-23T17:53:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: failed to encode map e97 with expected crc 2023-09-23T17:53:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: failed to encode map e97 with expected crc 2023-09-23T17:53:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:53:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: failed to encode map e98 with expected crc 2023-09-23T17:53:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: osd.1 [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277] boot 2023-09-23T17:53:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T17:53:53.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:53:53.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:52 smithi099 ceph-mon[167076]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T17:53:53.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: failed to encode map e97 with expected crc 2023-09-23T17:53:53.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: failed to encode map e97 with expected crc 2023-09-23T17:53:53.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: failed to encode map e97 with expected crc 2023-09-23T17:53:53.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:53:53.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: failed to encode map e98 with expected crc 2023-09-23T17:53:53.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: osd.1 [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277] boot 2023-09-23T17:53:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T17:53:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:53:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[179668]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T17:53:53.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: failed to encode map e97 with expected crc 2023-09-23T17:53:53.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: failed to encode map e97 with expected crc 2023-09-23T17:53:53.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: failed to encode map e97 with expected crc 2023-09-23T17:53:53.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:53:53.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: failed to encode map e98 with expected crc 2023-09-23T17:53:53.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: osd.1 [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277] boot 2023-09-23T17:53:53.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: osdmap e98: 8 total, 8 up, 8 in 2023-09-23T17:53:53.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T17:53:53.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:53.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:52 smithi055 ceph-mon[184174]: osdmap e99: 8 total, 8 up, 8 in 2023-09-23T17:53:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: pgmap v155: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T17:53:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:53 smithi099 ceph-mon[167076]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: pgmap v155: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T17:53:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[179668]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: pgmap v155: 73 pgs: 20 peering, 2 active+undersized+degraded, 10 active+undersized, 41 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: failed to encode map e99 with expected crc 2023-09-23T17:53:54.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:53 smithi055 ceph-mon[184174]: failed to encode map e99 with expected crc 2023-09-23T17:53:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/840151138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:55.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/840151138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:55.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/840151138' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:55.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:53:55.821+0000 7f3f79c94700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T17:53:55.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[184174]: pgmap v157: 73 pgs: 20 peering, 1 active+undersized+degraded, 8 active+undersized, 44 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T17:53:55.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T17:53:55.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:55.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:55.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:55.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:53:55.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:55.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[179668]: pgmap v157: 73 pgs: 20 peering, 1 active+undersized+degraded, 8 active+undersized, 44 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T17:53:55.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T17:53:55.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:55.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:55.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:55.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:53:55.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:55 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:55 smithi099 ceph-mon[167076]: pgmap v157: 73 pgs: 20 peering, 1 active+undersized+degraded, 8 active+undersized, 44 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail; 1/84 objects degraded (1.190%) 2023-09-23T17:53:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:55 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded (PG_DEGRADED) 2023-09-23T17:53:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:55 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:55 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:55 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:53:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:55 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:53:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:55 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:53:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:56 smithi099 ceph-mon[167076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:53:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:56 smithi099 ceph-mon[167076]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T17:53:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2136015505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:57.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:57.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:57.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:57.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:57.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:53:57.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[179668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:53:57.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[179668]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T17:53:57.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2136015505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:57.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:57.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:53:57.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:57.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:57.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:53:57.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:53:57.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[184174]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:53:57.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[184174]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T17:53:57.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2136015505' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:57.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:53:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:57 smithi099 ceph-mon[167076]: pgmap v158: 73 pgs: 20 peering, 53 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:57 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T17:53:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:57 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T17:53:58.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:57 smithi055 ceph-mon[184174]: pgmap v158: 73 pgs: 20 peering, 53 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:57 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T17:53:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:57 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T17:53:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:57 smithi055 ceph-mon[179668]: pgmap v158: 73 pgs: 20 peering, 53 active+clean; 4.6 MiB data, 121 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:53:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:57 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1/84 objects degraded (1.190%), 1 pg degraded) 2023-09-23T17:53:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:57 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T17:53:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:58 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2949441424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:59.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:58 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2949441424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:53:59.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:58 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2949441424' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:53:59 smithi099 ceph-mon[167076]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:00.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:53:59 smithi055 ceph-mon[179668]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:00.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:53:59 smithi055 ceph-mon[184174]: pgmap v159: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:00.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:54:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:54:00] "GET /metrics HTTP/1.1" 200 33878 "" "Prometheus/2.43.0" 2023-09-23T17:54:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1062299480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1062299480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:01.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1062299480' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:01 smithi099 ceph-mon[167076]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:02.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:01 smithi055 ceph-mon[184174]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:02.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:01 smithi055 ceph-mon[179668]: pgmap v160: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:03 smithi055 ceph-mon[184174]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1126627844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:03 smithi055 ceph-mon[179668]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1126627844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:03.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:03 smithi099 ceph-mon[167076]: pgmap v161: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:03.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1126627844' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:05 smithi099 ceph-mon[167076]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/972600906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:05 smithi055 ceph-mon[179668]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/972600906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:05 smithi055 ceph-mon[184174]: pgmap v162: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/972600906' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:07 smithi055 ceph-mon[179668]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:07 smithi055 ceph-mon[184174]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:07 smithi099 ceph-mon[167076]: pgmap v163: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4042772268' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4042772268' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:08.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4042772268' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:09.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:09 smithi055 ceph-mon[179668]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:09.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:09 smithi055 ceph-mon[184174]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:09.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:09 smithi099 ceph-mon[167076]: pgmap v164: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:10.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:54:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:54:10] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.43.0" 2023-09-23T17:54:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2462405075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2462405075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:10.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2462405075' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:11.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:11 smithi055 ceph-mon[184174]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:11.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:54:11.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:11 smithi055 ceph-mon[184174]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:54:11.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:11 smithi055 ceph-mon[184174]: Upgrade: osd.2 is safe to restart 2023-09-23T17:54:11.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:11 smithi055 ceph-mon[179668]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:11.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:54:11.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:11 smithi055 ceph-mon[179668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:54:11.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:11 smithi055 ceph-mon[179668]: Upgrade: osd.2 is safe to restart 2023-09-23T17:54:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:11 smithi099 ceph-mon[167076]: pgmap v165: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:54:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:11 smithi099 ceph-mon[167076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["2"], "max": 16}]: dispatch 2023-09-23T17:54:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:11 smithi099 ceph-mon[167076]: Upgrade: osd.2 is safe to restart 2023-09-23T17:54:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[184174]: Upgrade: Updating osd.2 2023-09-23T17:54:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T17:54:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:54:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[184174]: Deploying daemon osd.2 on smithi055 2023-09-23T17:54:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2793964915' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:13.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[184174]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[179668]: Upgrade: Updating osd.2 2023-09-23T17:54:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T17:54:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:54:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[179668]: Deploying daemon osd.2 on smithi055 2023-09-23T17:54:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2793964915' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:13.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:13 smithi055 ceph-mon[179668]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:13 smithi099 ceph-mon[167076]: Upgrade: Updating osd.2 2023-09-23T17:54:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-09-23T17:54:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:54:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:13 smithi099 ceph-mon[167076]: Deploying daemon osd.2 on smithi055 2023-09-23T17:54:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:13 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2793964915' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:13 smithi099 ceph-mon[167076]: pgmap v166: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:14.431 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:14 smithi055 systemd[1]: Stopping Ceph osd.2 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:54:15.180 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:14 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[129328]: 2023-09-23T17:54:14.713+0000 7efd7fca9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:54:15.180 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:14 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[129328]: 2023-09-23T17:54:14.713+0000 7efd7fca9700 -1 osd.2 99 *** Got signal Terminated *** 2023-09-23T17:54:15.180 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:14 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[129328]: 2023-09-23T17:54:14.713+0000 7efd7fca9700 -1 osd.2 99 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T17:54:15.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:15 smithi099 ceph-mon[167076]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:15.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2989479720' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:15.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:15 smithi099 ceph-mon[167076]: osd.2 marked itself down and dead 2023-09-23T17:54:15.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:15 smithi055 ceph-mon[184174]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:15.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2989479720' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:15.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:15 smithi055 ceph-mon[184174]: osd.2 marked itself down and dead 2023-09-23T17:54:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:15 smithi055 ceph-mon[179668]: pgmap v167: 73 pgs: 73 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2989479720' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:15 smithi055 ceph-mon[179668]: osd.2 marked itself down and dead 2023-09-23T17:54:16.430 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:16 smithi055 bash[207986]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T17:54:16.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:16 smithi055 ceph-mon[184174]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:54:16.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:16 smithi055 ceph-mon[184174]: failed to encode map e100 with expected crc 2023-09-23T17:54:16.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:16 smithi055 ceph-mon[184174]: failed to encode map e100 with expected crc 2023-09-23T17:54:16.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:16 smithi055 ceph-mon[184174]: osdmap e100: 8 total, 7 up, 8 in 2023-09-23T17:54:16.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:16 smithi055 ceph-mon[184174]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T17:54:16.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:16 smithi055 ceph-mon[179668]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:54:16.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:16 smithi055 ceph-mon[179668]: failed to encode map e100 with expected crc 2023-09-23T17:54:16.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:16 smithi055 ceph-mon[179668]: failed to encode map e100 with expected crc 2023-09-23T17:54:16.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:16 smithi055 ceph-mon[179668]: osdmap e100: 8 total, 7 up, 8 in 2023-09-23T17:54:16.805 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:16 smithi055 ceph-mon[179668]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T17:54:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:16 smithi099 ceph-mon[167076]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:54:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:16 smithi099 ceph-mon[167076]: failed to encode map e100 with expected crc 2023-09-23T17:54:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:16 smithi099 ceph-mon[167076]: failed to encode map e100 with expected crc 2023-09-23T17:54:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:16 smithi099 ceph-mon[167076]: osdmap e100: 8 total, 7 up, 8 in 2023-09-23T17:54:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:16 smithi099 ceph-mon[167076]: osdmap e101: 8 total, 7 up, 8 in 2023-09-23T17:54:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:17 smithi099 ceph-mon[167076]: pgmap v169: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:17 smithi099 ceph-mon[167076]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:17 smithi099 ceph-mon[167076]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:17 smithi099 ceph-mon[167076]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:17 smithi099 ceph-mon[167076]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:17 smithi099 ceph-mon[167076]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:17 smithi099 ceph-mon[167076]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:17 smithi099 ceph-mon[167076]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3802985607' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:17.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[179668]: pgmap v169: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[179668]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[179668]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[179668]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[179668]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[179668]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[179668]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[179668]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3802985607' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[184174]: pgmap v169: 73 pgs: 5 peering, 2 stale+active+clean, 66 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[184174]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[184174]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[184174]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[184174]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[184174]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[184174]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[184174]: failed to encode map e101 with expected crc 2023-09-23T17:54:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3802985607' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:17.933 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:17 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.2.service: Succeeded. 2023-09-23T17:54:17.933 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:17 smithi055 systemd[1]: Stopped Ceph osd.2 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:54:17.933 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:17 smithi055 systemd[1]: Starting Ceph osd.2 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:54:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:18 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:18.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:18 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:18.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:18 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 4/84 objects degraded (4.762%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:18.930 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:18 smithi055 bash[208295]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T17:54:18.930 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:18 smithi055 bash[208295]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T17:54:18.930 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:18 smithi055 bash[208295]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T17:54:18.931 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:18 smithi055 bash[208295]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T17:54:18.931 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:18 smithi055 bash[208295]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T17:54:18.931 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:18 smithi055 bash[208295]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T17:54:18.931 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:18 smithi055 bash[208295]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T17:54:18.931 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:18 smithi055 bash[208295]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T17:54:19.480 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:19 smithi055 podman[208563]: 2023-09-23T17:54:19.480 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:19 smithi055 ceph-osd[208627]: -- 172.21.15.55:0/1667736788 <== mon.0 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x5613706924e0 con 0x561370662c00 2023-09-23T17:54:19.480 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:19 smithi055 bash[208563]: 3ba01e3467aff716e2eae4152b850fed5b7e117d3f7eb4e33a2671f640306383 2023-09-23T17:54:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:19 smithi099 ceph-mon[167076]: pgmap v171: 73 pgs: 3 active+undersized+degraded, 5 peering, 9 active+undersized, 56 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T17:54:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:19 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3177114120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:19 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:19.918 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:19 smithi055 ceph-mon[179668]: pgmap v171: 73 pgs: 3 active+undersized+degraded, 5 peering, 9 active+undersized, 56 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T17:54:19.918 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:19 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3177114120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:19.918 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:19 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:19.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:19 smithi055 ceph-mon[184174]: pgmap v171: 73 pgs: 3 active+undersized+degraded, 5 peering, 9 active+undersized, 56 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 4/84 objects degraded (4.762%) 2023-09-23T17:54:19.919 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:19 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3177114120' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:19.919 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:19 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:19.919 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:19 smithi055 systemd[1]: Started Ceph osd.2 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:54:20.179 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:19 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[208602]: 2023-09-23T17:54:19.916+0000 7fc9dc353640 -1 Falling back to public interface 2023-09-23T17:54:20.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:54:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:54:20] "GET /metrics HTTP/1.1" 200 34021 "" "Prometheus/2.43.0" 2023-09-23T17:54:20.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:20 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:20.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:20 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:54:20.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:20 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:20.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:20 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:54:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:20 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:20 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:54:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:21 smithi099 ceph-mon[167076]: pgmap v172: 73 pgs: 4 active+undersized+degraded, 5 peering, 10 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:21.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:21 smithi055 ceph-mon[184174]: pgmap v172: 73 pgs: 4 active+undersized+degraded, 5 peering, 10 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:21.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:21.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:21.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:21 smithi055 ceph-mon[179668]: pgmap v172: 73 pgs: 4 active+undersized+degraded, 5 peering, 10 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:21.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:21.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1833523389' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:22.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1833523389' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:22.930 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:22 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[208602]: 2023-09-23T17:54:22.701+0000 7fc9dc353640 -1 osd.2 99 log_to_monitors true 2023-09-23T17:54:22.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1833523389' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:23 smithi099 ceph-mon[167076]: pgmap v173: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:23 smithi099 ceph-mon[167076]: from='osd.2 [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:54:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:23 smithi099 ceph-mon[167076]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:54:23.929 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 17:54:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[208602]: 2023-09-23T17:54:23.606+0000 7fc9cc7be700 -1 osd.2 99 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:54:23.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:23 smithi055 ceph-mon[179668]: pgmap v173: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:23.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:23 smithi055 ceph-mon[179668]: from='osd.2 [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:54:23.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:23 smithi055 ceph-mon[179668]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:54:23.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:23 smithi055 ceph-mon[184174]: pgmap v173: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 122 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:23.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:23 smithi055 ceph-mon[184174]: from='osd.2 [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:54:23.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:23 smithi055 ceph-mon[184174]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T17:54:24.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: failed to encode map e102 with expected crc 2023-09-23T17:54:24.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T17:54:24.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T17:54:24.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: from='osd.2 [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:54:24.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:54:24.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: failed to encode map e102 with expected crc 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: failed to encode map e102 with expected crc 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: failed to encode map e102 with expected crc 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: failed to encode map e102 with expected crc 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/329858128' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: failed to encode map e102 with expected crc 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T17:54:24.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: from='osd.2 [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: failed to encode map e102 with expected crc 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: failed to encode map e102 with expected crc 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: failed to encode map e102 with expected crc 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: failed to encode map e102 with expected crc 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/329858128' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:24.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: failed to encode map e102 with expected crc 2023-09-23T17:54:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T17:54:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: osdmap e102: 8 total, 7 up, 8 in 2023-09-23T17:54:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: from='osd.2 [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:54:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:54:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: failed to encode map e102 with expected crc 2023-09-23T17:54:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: failed to encode map e102 with expected crc 2023-09-23T17:54:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: failed to encode map e102 with expected crc 2023-09-23T17:54:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: failed to encode map e102 with expected crc 2023-09-23T17:54:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/329858128' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:25.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:25.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[179668]: pgmap v175: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:25.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[179668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:54:25.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[179668]: failed to encode map e103 with expected crc 2023-09-23T17:54:25.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[179668]: failed to encode map e103 with expected crc 2023-09-23T17:54:25.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[179668]: osd.2 [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182] boot 2023-09-23T17:54:25.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[179668]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T17:54:25.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:54:25.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[184174]: pgmap v175: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:25.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[184174]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:54:25.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[184174]: failed to encode map e103 with expected crc 2023-09-23T17:54:25.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[184174]: failed to encode map e103 with expected crc 2023-09-23T17:54:25.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[184174]: osd.2 [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182] boot 2023-09-23T17:54:25.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[184174]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T17:54:25.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:25 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:54:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:25 smithi099 ceph-mon[167076]: pgmap v175: 73 pgs: 4 active+undersized+degraded, 15 active+undersized, 54 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:25 smithi099 ceph-mon[167076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:54:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:25 smithi099 ceph-mon[167076]: failed to encode map e103 with expected crc 2023-09-23T17:54:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:25 smithi099 ceph-mon[167076]: failed to encode map e103 with expected crc 2023-09-23T17:54:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:25 smithi099 ceph-mon[167076]: osd.2 [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182] boot 2023-09-23T17:54:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:25 smithi099 ceph-mon[167076]: osdmap e103: 8 total, 8 up, 8 in 2023-09-23T17:54:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:25 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T17:54:26.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[179668]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[179668]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T17:54:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[179668]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[179668]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[179668]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[179668]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[179668]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[179668]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/461043201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:26.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[184174]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[184174]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T17:54:26.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[184174]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[184174]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[184174]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[184174]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[184174]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[184174]: failed to encode map e104 with expected crc 2023-09-23T17:54:26.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/461043201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:26.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:26 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:26 smithi099 ceph-mon[167076]: failed to encode map e104 with expected crc 2023-09-23T17:54:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:26 smithi099 ceph-mon[167076]: osdmap e104: 8 total, 8 up, 8 in 2023-09-23T17:54:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:26 smithi099 ceph-mon[167076]: failed to encode map e104 with expected crc 2023-09-23T17:54:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:26 smithi099 ceph-mon[167076]: failed to encode map e104 with expected crc 2023-09-23T17:54:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:26 smithi099 ceph-mon[167076]: failed to encode map e104 with expected crc 2023-09-23T17:54:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:26 smithi099 ceph-mon[167076]: failed to encode map e104 with expected crc 2023-09-23T17:54:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:26 smithi099 ceph-mon[167076]: failed to encode map e104 with expected crc 2023-09-23T17:54:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:26 smithi099 ceph-mon[167076]: failed to encode map e104 with expected crc 2023-09-23T17:54:27.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/461043201' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:27.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:26 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:27.930 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:54:27.759+0000 7f3f79c94700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T17:54:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[179668]: pgmap v178: 73 pgs: 4 active+undersized+degraded, 11 active+undersized, 58 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:54:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:54:27.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:27.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[184174]: pgmap v178: 73 pgs: 4 active+undersized+degraded, 11 active+undersized, 58 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:27.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:27.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:27.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:27.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:54:27.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:54:27.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:27 smithi099 ceph-mon[167076]: pgmap v178: 73 pgs: 4 active+undersized+degraded, 11 active+undersized, 58 active+clean; 4.6 MiB data, 138 MiB used, 715 GiB / 715 GiB avail; 6/84 objects degraded (7.143%) 2023-09-23T17:54:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:27 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 6/84 objects degraded (7.143%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:54:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:54:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:54:28.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:54:28.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:54:28.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:54:28.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:28.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[179668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:28.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[179668]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T17:54:28.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:54:28.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:54:28.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:54:28.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:54:28.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:28.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[184174]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:28.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:28 smithi055 ceph-mon[184174]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T17:54:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:28 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:54:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:28 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:54:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:28 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:54:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:28 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:54:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:28 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:28 smithi099 ceph-mon[167076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:28 smithi099 ceph-mon[167076]: Upgrade: unsafe to stop osd(s) at this time (6 PGs are or would become offline) 2023-09-23T17:54:29.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:29 smithi055 ceph-mon[179668]: pgmap v179: 73 pgs: 2 active+undersized+degraded, 3 active+undersized, 68 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:54:29.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4223548500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:29.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:29 smithi055 ceph-mon[184174]: pgmap v179: 73 pgs: 2 active+undersized+degraded, 3 active+undersized, 68 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:54:29.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4223548500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:30.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:29 smithi099 ceph-mon[167076]: pgmap v179: 73 pgs: 2 active+undersized+degraded, 3 active+undersized, 68 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:54:30.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4223548500' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:30.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:54:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:54:30] "GET /metrics HTTP/1.1" 200 34026 "" "Prometheus/2.43.0" 2023-09-23T17:54:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:30 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T17:54:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:30 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T17:54:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:30 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T17:54:31.187 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:30 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T17:54:31.187 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:30 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded) 2023-09-23T17:54:31.187 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:30 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T17:54:32.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:31 smithi099 ceph-mon[167076]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:32.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1615624995' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:32.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:31 smithi055 ceph-mon[179668]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:32.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1615624995' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:32.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:31 smithi055 ceph-mon[184174]: pgmap v180: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:32.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1615624995' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:33 smithi055 ceph-mon[179668]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:33 smithi055 ceph-mon[184174]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:33 smithi099 ceph-mon[167076]: pgmap v181: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2262652399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:34.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2262652399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:34.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2262652399' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:35.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:35 smithi055 ceph-mon[179668]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:35.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:35 smithi055 ceph-mon[184174]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:35 smithi099 ceph-mon[167076]: pgmap v182: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/988083615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:36.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/988083615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/988083615' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:37 smithi055 ceph-mon[179668]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:37 smithi055 ceph-mon[184174]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:37 smithi099 ceph-mon[167076]: pgmap v183: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:38.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1606778249' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:38.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1606778249' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1606778249' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:39.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:39 smithi055 ceph-mon[179668]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:39.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:39 smithi055 ceph-mon[184174]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:39 smithi099 ceph-mon[167076]: pgmap v184: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:40.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:54:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:54:40] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.43.0" 2023-09-23T17:54:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3506026963' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3506026963' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3506026963' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:41 smithi099 ceph-mon[167076]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:41.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:41 smithi055 ceph-mon[179668]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:41.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:41 smithi055 ceph-mon[184174]: pgmap v185: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:43 smithi055 ceph-mon[179668]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/756706158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:43 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:43 smithi055 ceph-mon[179668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:43 smithi055 ceph-mon[179668]: Upgrade: osd.3 is safe to restart 2023-09-23T17:54:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:43 smithi055 ceph-mon[184174]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/756706158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:43 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:43 smithi055 ceph-mon[184174]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:43 smithi055 ceph-mon[184174]: Upgrade: osd.3 is safe to restart 2023-09-23T17:54:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:43 smithi099 ceph-mon[167076]: pgmap v186: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/756706158' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:43 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:43 smithi099 ceph-mon[167076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["3"], "max": 16}]: dispatch 2023-09-23T17:54:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:43 smithi099 ceph-mon[167076]: Upgrade: osd.3 is safe to restart 2023-09-23T17:54:45.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[184174]: Upgrade: Updating osd.3 2023-09-23T17:54:45.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:45.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T17:54:45.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:54:45.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[184174]: Deploying daemon osd.3 on smithi055 2023-09-23T17:54:45.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/822580203' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:45.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[179668]: Upgrade: Updating osd.3 2023-09-23T17:54:45.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:45.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T17:54:45.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:54:45.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[179668]: Deploying daemon osd.3 on smithi055 2023-09-23T17:54:45.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/822580203' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:45 smithi099 ceph-mon[167076]: Upgrade: Updating osd.3 2023-09-23T17:54:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:45 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:45 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-09-23T17:54:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:45 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:54:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:45 smithi099 ceph-mon[167076]: Deploying daemon osd.3 on smithi055 2023-09-23T17:54:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/822580203' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:46 smithi099 ceph-mon[167076]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:46.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:46 smithi055 ceph-mon[179668]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:46.350 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:46 smithi055 ceph-mon[184174]: pgmap v187: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:46.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:46 smithi055 systemd[1]: Stopping Ceph osd.3 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:54:47.072 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:46 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[137230]: 2023-09-23T17:54:46.771+0000 7f1f68fb7700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:54:47.072 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:46 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[137230]: 2023-09-23T17:54:46.771+0000 7f1f68fb7700 -1 osd.3 104 *** Got signal Terminated *** 2023-09-23T17:54:47.072 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:46 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[137230]: 2023-09-23T17:54:46.771+0000 7f1f68fb7700 -1 osd.3 104 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T17:54:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:47 smithi099 ceph-mon[167076]: osd.3 marked itself down and dead 2023-09-23T17:54:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:47 smithi055 ceph-mon[179668]: osd.3 marked itself down and dead 2023-09-23T17:54:47.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:47 smithi055 ceph-mon[184174]: osd.3 marked itself down and dead 2023-09-23T17:54:47.839 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:47 smithi055 bash[214028]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T17:54:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:48 smithi099 ceph-mon[167076]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:48 smithi099 ceph-mon[167076]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:54:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:48 smithi099 ceph-mon[167076]: failed to encode map e105 with expected crc 2023-09-23T17:54:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:48 smithi099 ceph-mon[167076]: failed to encode map e105 with expected crc 2023-09-23T17:54:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:48 smithi099 ceph-mon[167076]: osdmap e105: 8 total, 7 up, 8 in 2023-09-23T17:54:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:48 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2464508619' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[179668]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:48.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[179668]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:54:48.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[179668]: failed to encode map e105 with expected crc 2023-09-23T17:54:48.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[179668]: failed to encode map e105 with expected crc 2023-09-23T17:54:48.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[179668]: osdmap e105: 8 total, 7 up, 8 in 2023-09-23T17:54:48.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2464508619' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[184174]: pgmap v188: 73 pgs: 73 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[184174]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:54:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[184174]: failed to encode map e105 with expected crc 2023-09-23T17:54:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[184174]: failed to encode map e105 with expected crc 2023-09-23T17:54:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[184174]: osdmap e105: 8 total, 7 up, 8 in 2023-09-23T17:54:48.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:48 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2464508619' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:49.347 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:49 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T17:54:49.348 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:49 smithi055 systemd[1]: Stopped Ceph osd.3 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:54:49.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[184174]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[184174]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[184174]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T17:54:49.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[184174]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[184174]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[184174]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.348 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[184174]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.349 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[184174]: pgmap v191: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:49.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[179668]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[179668]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[179668]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T17:54:49.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[179668]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[179668]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[179668]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[179668]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:49 smithi055 ceph-mon[179668]: pgmap v191: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:49 smithi099 ceph-mon[167076]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:49 smithi099 ceph-mon[167076]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:49 smithi099 ceph-mon[167076]: osdmap e106: 8 total, 7 up, 8 in 2023-09-23T17:54:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:49 smithi099 ceph-mon[167076]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:49 smithi099 ceph-mon[167076]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:49 smithi099 ceph-mon[167076]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:49 smithi099 ceph-mon[167076]: failed to encode map e106 with expected crc 2023-09-23T17:54:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:49 smithi099 ceph-mon[167076]: pgmap v191: 73 pgs: 14 peering, 7 stale+active+clean, 52 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:54:49.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:49 smithi055 systemd[1]: Starting Ceph osd.3 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:54:50.356 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:50 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T17:54:50.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:50 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/944969494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:50.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:50 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T17:54:50.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:50 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/944969494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:50.357 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 bash[214348]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T17:54:50.357 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 bash[214348]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T17:54:50.357 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 bash[214348]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T17:54:50.358 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 bash[214348]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T17:54:50.358 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 bash[214348]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T17:54:50.358 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 bash[214348]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T17:54:50.358 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 bash[214348]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T17:54:50.358 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 bash[214348]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T17:54:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:50 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 4 pgs inactive, 6 pgs peering (PG_AVAILABILITY) 2023-09-23T17:54:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:50 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/944969494' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:50.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:54:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:54:50] "GET /metrics HTTP/1.1" 200 34016 "" "Prometheus/2.43.0" 2023-09-23T17:54:51.180 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 podman[214614]: 2023-09-23T17:54:51.180 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 bash[214614]: 6e67ef89cba71198c450f0f01e3eac68a21cb08dbe0504e7b96fb69af3953780 2023-09-23T17:54:51.180 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 ceph-osd[214654]: -- 172.21.15.55:0/4285739303 <== mon.2 v2:172.21.15.55:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x56264b8a44e0 con 0x56264b874c00 2023-09-23T17:54:51.180 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:50 smithi055 systemd[1]: Started Ceph osd.3 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:54:51.930 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[214630]: 2023-09-23T17:54:51.461+0000 7f02ec5f8640 -1 Falling back to public interface 2023-09-23T17:54:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:51 smithi099 ceph-mon[167076]: pgmap v192: 73 pgs: 2 active+undersized+degraded, 14 peering, 4 stale+active+clean, 9 active+undersized, 44 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T17:54:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:54:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:51 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:52.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[179668]: pgmap v192: 73 pgs: 2 active+undersized+degraded, 14 peering, 4 stale+active+clean, 9 active+undersized, 44 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T17:54:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:54:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[184174]: pgmap v192: 73 pgs: 2 active+undersized+degraded, 14 peering, 4 stale+active+clean, 9 active+undersized, 44 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 2/84 objects degraded (2.381%) 2023-09-23T17:54:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:54:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 2/84 objects degraded (2.381%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:54:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2911225763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:53.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2911225763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:53.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2911225763' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:53 smithi099 ceph-mon[167076]: pgmap v193: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:54:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:53 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-09-23T17:54:54.430 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:54 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[214630]: 2023-09-23T17:54:54.223+0000 7f02ec5f8640 -1 osd.3 104 log_to_monitors true 2023-09-23T17:54:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:53 smithi055 ceph-mon[179668]: pgmap v193: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:54:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:53 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-09-23T17:54:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:53 smithi055 ceph-mon[184174]: pgmap v193: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:54:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:53 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 4 pgs inactive, 6 pgs peering) 2023-09-23T17:54:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:54 smithi099 ceph-mon[167076]: from='osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T17:54:55.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3021033569' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:55.430 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:54:55 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[214630]: 2023-09-23T17:54:55.032+0000 7f02dba61700 -1 osd.3 104 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:54:55.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:54 smithi055 ceph-mon[179668]: from='osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T17:54:55.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3021033569' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:55.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:54 smithi055 ceph-mon[184174]: from='osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T17:54:55.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3021033569' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:56 smithi099 ceph-mon[167076]: pgmap v194: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:54:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:56 smithi099 ceph-mon[167076]: failed to encode map e107 with expected crc 2023-09-23T17:54:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:56 smithi099 ceph-mon[167076]: from='osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T17:54:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:56 smithi099 ceph-mon[167076]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T17:54:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:56 smithi099 ceph-mon[167076]: from='osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:54:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[179668]: pgmap v194: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:54:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[179668]: failed to encode map e107 with expected crc 2023-09-23T17:54:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[179668]: from='osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T17:54:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[179668]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T17:54:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[179668]: from='osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:54:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[184174]: pgmap v194: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 139 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:54:56.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[184174]: failed to encode map e107 with expected crc 2023-09-23T17:54:56.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[184174]: from='osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856]' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T17:54:56.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[184174]: osdmap e107: 8 total, 7 up, 8 in 2023-09-23T17:54:56.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[184174]: from='osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T17:54:56.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:54:56.769 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T17:54:57.225 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch upgrade pause' 2023-09-23T17:54:57.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:57 smithi099 ceph-mon[167076]: failed to encode map e107 with expected crc 2023-09-23T17:54:57.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:57 smithi099 ceph-mon[167076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:54:57.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:57 smithi099 ceph-mon[167076]: osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856] boot 2023-09-23T17:54:57.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:57 smithi099 ceph-mon[167076]: failed to encode map e108 with expected crc 2023-09-23T17:54:57.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:57 smithi099 ceph-mon[167076]: osdmap e108: 8 total, 8 up, 8 in 2023-09-23T17:54:57.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:54:57.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1757226641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:57.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[179668]: failed to encode map e107 with expected crc 2023-09-23T17:54:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[179668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:54:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[179668]: osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856] boot 2023-09-23T17:54:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[179668]: failed to encode map e108 with expected crc 2023-09-23T17:54:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[179668]: osdmap e108: 8 total, 8 up, 8 in 2023-09-23T17:54:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:54:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1757226641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:57.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[184174]: failed to encode map e107 with expected crc 2023-09-23T17:54:57.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[184174]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T17:54:57.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[184174]: osd.3 [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856] boot 2023-09-23T17:54:57.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[184174]: failed to encode map e108 with expected crc 2023-09-23T17:54:57.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[184174]: osdmap e108: 8 total, 8 up, 8 in 2023-09-23T17:54:57.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T17:54:57.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1757226641' entity='client.admin' cmd=[{"prefix": "osd versions"}]: dispatch 2023-09-23T17:54:58.055 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:54:58.322 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: pgmap v197: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:54:58.322 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T17:54:58.323 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.323 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.323 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.323 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.323 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.323 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.323 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.323 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.323 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.323 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: pgmap v197: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.324 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.325 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: pgmap v197: 73 pgs: 4 active+undersized+degraded, 27 active+undersized, 42 active+clean; 4.6 MiB data, 160 MiB used, 715 GiB / 715 GiB avail; 5/84 objects degraded (5.952%) 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: osdmap e109: 8 total, 8 up, 8 in 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: failed to encode map e109 with expected crc 2023-09-23T17:54:58.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 5/84 objects degraded (5.952%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T17:54:59.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:54:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T17:54:59.339+0000 7f3f79c94700 -1 mgr.server reply reply (16) Device or resource busy unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T17:54:59.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:54:59 smithi055 ceph-mon[179668]: pgmap v199: 73 pgs: 3 active+undersized+degraded, 20 active+undersized, 50 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:54:59.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:54:59 smithi055 ceph-mon[184174]: pgmap v199: 73 pgs: 3 active+undersized+degraded, 20 active+undersized, 50 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:54:59.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:54:59 smithi099 ceph-mon[167076]: pgmap v199: 73 pgs: 3 active+undersized+degraded, 20 active+undersized, 50 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T17:54:59.647 INFO:teuthology.orchestra.run.smithi055.stdout:Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:55:00.003 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T17:55:00.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T17:55:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:55:00.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:55:00.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:55:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:55:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:55:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:00.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:00.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T17:55:00.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T17:55:00.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T17:55:00.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:55:00.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:55:00.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:55:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:55:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:55:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T17:55:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mon.0 -' entity='mon.' cmd=[{"prefix": "osd ok-to-stop", "ids": ["4"], "max": 16}]: dispatch 2023-09-23T17:55:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: Upgrade: unsafe to stop osd(s) at this time (5 PGs are or would become offline) 2023-09-23T17:55:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='client.34563 -' entity='client.admin' cmd=[{"prefix": "orch upgrade pause", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:55:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: Upgrade: Paused upgrade to quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d 2023-09-23T17:55:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:00.611 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:00.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:55:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:55:00] "GET /metrics HTTP/1.1" 200 34092 "" "Prometheus/2.43.0" 2023-09-23T17:55:01.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:01 smithi055 ceph-mon[179668]: pgmap v200: 73 pgs: 2 active+undersized, 71 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:01.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:01 smithi055 ceph-mon[184174]: pgmap v200: 73 pgs: 2 active+undersized, 71 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:01 smithi099 ceph-mon[167076]: pgmap v200: 73 pgs: 2 active+undersized, 71 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:02.059 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T17:55:02.059 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (7m) - 9m 28.5M - 0.25.0 c8568f914cd2 5062112ad547 2023-09-23T17:55:02.059 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi099 *:3000 running (6m) - 9m 88.5M - 9.4.7 2c41d148cca3 b140d15d52f2 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi055.exjftp smithi055 running (9m) - 9m 19.3M - 17.2.6-1123-g61431f6b 0db14aa01094 7a2127020d8b 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi099.djovrl smithi099 running (9m) - 9m 18.7M - 17.2.6-1123-g61431f6b 0db14aa01094 5993fe8cebce 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi099 *:8443,9283,8765 running (6m) - 13m 450M - 18.2.0-460-gbb0daeff e32c2a64c126 32a5f5ae4a55 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (6m) - 14m 514M - 18.2.0-460-gbb0daeff e32c2a64c126 fc64ae873ea2 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (6m) - 14m 84.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 1422ade2193a 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi099 running (3m) - 13m 45.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 8b31f1fe6806 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (4m) - 13m 57.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e1bc5b196d92 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (7m) - 9m 18.8M - 1.5.0 0da6a335fe13 6f05e33b3606 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi099 *:9100 running (6m) - 9m 17.7M - 1.5.0 0da6a335fe13 3f7dcfc22edf 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (106s) - 12m 58.9M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 a5bacbf2c101 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (74s) - 12m 58.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 36ea52c16903 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (42s) - 12m 48.8M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 3ba01e3467af 2023-09-23T17:55:02.060 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (11s) - 11m 12.0M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 6e67ef89cba7 2023-09-23T17:55:02.061 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi099 running (11m) - 11m 90.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ecb0e820fd90 2023-09-23T17:55:02.061 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi099 running (11m) - 11m 114M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 7e6bf34cfdb0 2023-09-23T17:55:02.061 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi099 running (10m) - 10m 91.5M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 827ec31827bb 2023-09-23T17:55:02.061 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi099 running (10m) - 10m 102M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 dcef78fdede8 2023-09-23T17:55:02.061 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi099 *:9095 running (6m) - 9m 62.7M - 2.43.0 a07b618ecd1d 12edf856847a 2023-09-23T17:55:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:02 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded) 2023-09-23T17:55:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:02 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T17:55:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:02 smithi055 ceph-mon[184174]: from='client.44474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:55:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:02 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded) 2023-09-23T17:55:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:02 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T17:55:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:02.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:02 smithi055 ceph-mon[179668]: from='client.44474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:55:02.461 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph orch ps' 2023-09-23T17:55:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:02 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 3/84 objects degraded (3.571%), 3 pgs degraded) 2023-09-23T17:55:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:02 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T17:55:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:02 smithi099 ceph-mon[167076]: from='client.44474 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:55:03.069 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:03.581 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:03 smithi055 ceph-mon[179668]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:03.581 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:03 smithi055 ceph-mon[184174]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:03.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:03 smithi099 ceph-mon[167076]: pgmap v201: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:04.526 INFO:teuthology.orchestra.run.smithi055.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-09-23T17:55:04.526 INFO:teuthology.orchestra.run.smithi055.stdout:alertmanager.a smithi055 *:9093,9094 running (7m) - 9m 28.5M - 0.25.0 c8568f914cd2 5062112ad547 2023-09-23T17:55:04.526 INFO:teuthology.orchestra.run.smithi055.stdout:grafana.a smithi099 *:3000 running (6m) - 9m 88.5M - 9.4.7 2c41d148cca3 b140d15d52f2 2023-09-23T17:55:04.526 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi055.exjftp smithi055 running (9m) - 9m 19.3M - 17.2.6-1123-g61431f6b 0db14aa01094 7a2127020d8b 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:mds.foo.smithi099.djovrl smithi099 running (9m) - 9m 18.7M - 17.2.6-1123-g61431f6b 0db14aa01094 5993fe8cebce 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.x smithi099 *:8443,9283,8765 running (6m) - 13m 450M - 18.2.0-460-gbb0daeff e32c2a64c126 32a5f5ae4a55 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:mgr.y smithi055 *:8443,9283,8765 running (6m) - 14m 514M - 18.2.0-460-gbb0daeff e32c2a64c126 fc64ae873ea2 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a smithi055 running (6m) - 14m 84.3M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 1422ade2193a 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b smithi099 running (3m) - 13m 45.5M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 8b31f1fe6806 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c smithi055 running (4m) - 13m 57.9M 2048M 18.2.0-460-gbb0daeff e32c2a64c126 e1bc5b196d92 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.a smithi055 *:9100 running (7m) - 9m 18.8M - 1.5.0 0da6a335fe13 6f05e33b3606 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:node-exporter.b smithi099 *:9100 running (6m) - 9m 17.7M - 1.5.0 0da6a335fe13 3f7dcfc22edf 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 smithi055 running (109s) - 12m 58.9M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 a5bacbf2c101 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 smithi055 running (77s) - 12m 58.6M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 36ea52c16903 2023-09-23T17:55:04.527 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 smithi055 running (45s) - 12m 48.8M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 3ba01e3467af 2023-09-23T17:55:04.528 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 smithi055 running (13s) - 11m 12.0M 2501M 18.2.0-460-gbb0daeff e32c2a64c126 6e67ef89cba7 2023-09-23T17:55:04.528 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 smithi099 running (11m) - 11m 90.6M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 ecb0e820fd90 2023-09-23T17:55:04.528 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 smithi099 running (11m) - 11m 114M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 7e6bf34cfdb0 2023-09-23T17:55:04.528 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 smithi099 running (10m) - 10m 91.5M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 827ec31827bb 2023-09-23T17:55:04.528 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 smithi099 running (10m) - 10m 102M 2500M 17.2.6-1123-g61431f6b 0db14aa01094 dcef78fdede8 2023-09-23T17:55:04.528 INFO:teuthology.orchestra.run.smithi055.stdout:prometheus.a smithi099 *:9095 running (6m) - 10m 62.7M - 2.43.0 a07b618ecd1d 12edf856847a 2023-09-23T17:55:04.917 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -e sha1=bb0daeff97f020f686596fca6eba20c310e26fb7 -- bash -c 'ceph versions' 2023-09-23T17:55:05.525 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:05.754 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:05 smithi055 ceph-mon[184174]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:05.754 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:05 smithi055 ceph-mon[184174]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:55:05.754 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:05 smithi055 ceph-mon[179668]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:05.754 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:05 smithi055 ceph-mon[179668]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:55:05.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:05 smithi099 ceph-mon[167076]: pgmap v202: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:05 smithi099 ceph-mon[167076]: from='client.34575 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T17:55:07.015 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: "mon": { 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 3 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr": { 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 2 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: "osd": { 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 4, 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 4 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:55:07.016 INFO:teuthology.orchestra.run.smithi055.stdout: "mds": { 2023-09-23T17:55:07.017 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 2 2023-09-23T17:55:07.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:55:07.017 INFO:teuthology.orchestra.run.smithi055.stdout: "overall": { 2023-09-23T17:55:07.017 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 17.2.6-1123-g61431f6b (61431f6ba2752649b506867ec856b4a6236f90dd) quincy (stable)": 6, 2023-09-23T17:55:07.017 INFO:teuthology.orchestra.run.smithi055.stdout: "ceph version 18.2.0-460-gbb0daeff (bb0daeff97f020f686596fca6eba20c310e26fb7) reef (stable)": 9 2023-09-23T17:55:07.017 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:55:07.017 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:55:07.388 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:07.389 INFO:teuthology.run_tasks:Running task print... 2023-09-23T17:55:07.398 INFO:teuthology.task.print:**** done upgrading first half of cluster **** 2023-09-23T17:55:07.398 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-09-23T17:55:07.420 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-09-23T17:55:07.421 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-09-23T17:55:07.421 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph mgr dump --format=json 2023-09-23T17:55:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:07 smithi055 ceph-mon[179668]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:07 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/23251796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:07 smithi055 ceph-mon[184174]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:07 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/23251796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:07 smithi099 ceph-mon[167076]: pgmap v203: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:07 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/23251796' entity='client.admin' cmd=[{"prefix": "versions"}]: dispatch 2023-09-23T17:55:08.028 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:09.491 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:55:09.812 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:09 smithi055 ceph-mon[184174]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:09.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:09 smithi055 ceph-mon[179668]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:09.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:09 smithi099 ceph-mon[167076]: pgmap v204: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:09.876 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":37,"active_gid":25793,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6800","nonce":295474777},{"type":"v1","addr":"172.21.15.55:6801","nonce":295474777}]},"active_addr":"172.21.15.55:6801/295474777","active_change":"2023-09-23T17:49:06.276317+0000","active_mgr_features":4540138322906710015,"available":true,"standbys":[{"gid":25768,"name":"x","mgr_features":4540138322906710015,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2359","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"6","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph:v18","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_elasticsearch":{"name":"container_image_elasticsearch","type":"str","level":"advanced","flags":0,"default_value":"quay.io/omrizeneva/elasticsearch:6.8.23","min":"","max":"","enum_allowed":[],"desc":"elasticsearch container image","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_agent":{"name":"container_image_jaeger_agent","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-agent:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger agent container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_collector":{"name":"container_image_jaeger_collector","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-collector:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger collector container image","long_desc":"","tags":[],"see_also":[]},"container_image_jaeger_query":{"name":"container_image_jaeger_query","type":"str","level":"advanced","flags":0,"default_value":"quay.io/jaegertracing/jaeger-query:1.29","min":"","max":"","enum_allowed":[],"desc":"Jaeger query container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.2.4","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_nvmeof":{"name":"container_image_nvmeof","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/nvmeof:0.0.2","min":"","max":"","enum_allowed":[],"desc":"Nvme-of container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"inventory_list_all":{"name":"inventory_list_all","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Whether ceph-volume inventory should report more devices (mostly mappers (LVs / mpaths), partitions...)","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"secure_monitoring_stack":{"name":"secure_monitoring_stack","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable TLS security for all the monitoring stack daemons","long_desc":"","tags":[],"see_also":[]},"service_discovery_port":{"name":"service_discovery_port","type":"int","level":"advanced","flags":0,"default_value":"8765","min":"","max":"","enum_allowed":[],"desc":"cephadm service discovery port","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"fail_fs":{"name":"fail_fs","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Fail filesystem for rapid multi-rank mds upgrade","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"exclude_perf_counters":{"name":"exclude_perf_counters","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Do not include perf-counters in the metrics output","long_desc":"Gathering perf-counters from a single Prometheus exporter can degrade ceph-mgr performance, especially in large clusters. Instead, Ceph-exporter daemons are now used by default for perf-counter gathering. This should only be disabled when no ceph-exporters are deployed.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard_description":{"name":"leaderboard_description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"reef":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":89,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":2275336764}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":1475246835}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":3750083392}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.55:0","nonce":1645518004}]}]} 2023-09-23T17:55:09.880 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-09-23T17:55:09.880 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T17:55:09.880 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T17:55:10.486 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:10.653 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:55:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:55:10] "GET /metrics HTTP/1.1" 200 34083 "" "Prometheus/2.43.0" 2023-09-23T17:55:10.654 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1840699461' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T17:55:10.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1840699461' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T17:55:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1840699461' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-09-23T17:55:11.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:11 smithi055 ceph-mon[179668]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:11.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:11 smithi055 ceph-mon[184174]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:11.789 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:55:11.790 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":109,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","created":"2023-09-23T17:40:13.381005+0000","modified":"2023-09-23T17:54:57.011846+0000","last_up_change":"2023-09-23T17:54:56.008824+0000","last_in_change":"2023-09-23T17:44:27.554515+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T17:43:05.158393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T17:45:10.969164+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":84,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T17:45:49.622328+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T17:45:50.526600+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"99e7cae6-389e-461a-8a86-a5789673908b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":89,"up_from":93,"up_thru":108,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6803","nonce":3605839049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6805","nonce":3605839049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6809","nonce":3605839049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6807","nonce":3605839049}]},"public_addr":"172.21.15.55:6803/3605839049","cluster_addr":"172.21.15.55:6805/3605839049","heartbeat_back_addr":"172.21.15.55:6809/3605839049","heartbeat_front_addr":"172.21.15.55:6807/3605839049","state":["exists","up"]},{"osd":1,"uuid":"bfb0da01-7a06-430b-b973-209f22ac30d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":94,"up_from":98,"up_thru":108,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":4278491277},{"type":"v1","addr":"172.21.15.55:6811","nonce":4278491277}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":4278491277},{"type":"v1","addr":"172.21.15.55:6813","nonce":4278491277}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":4278491277},{"type":"v1","addr":"172.21.15.55:6817","nonce":4278491277}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":4278491277},{"type":"v1","addr":"172.21.15.55:6815","nonce":4278491277}]},"public_addr":"172.21.15.55:6811/4278491277","cluster_addr":"172.21.15.55:6813/4278491277","heartbeat_back_addr":"172.21.15.55:6817/4278491277","heartbeat_front_addr":"172.21.15.55:6815/4278491277","state":["exists","up"]},{"osd":2,"uuid":"f766d604-db54-4a09-9993-fe1a3ce0048e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":99,"up_from":103,"up_thru":108,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1645904182},{"type":"v1","addr":"172.21.15.55:6819","nonce":1645904182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1645904182},{"type":"v1","addr":"172.21.15.55:6821","nonce":1645904182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1645904182},{"type":"v1","addr":"172.21.15.55:6825","nonce":1645904182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1645904182},{"type":"v1","addr":"172.21.15.55:6823","nonce":1645904182}]},"public_addr":"172.21.15.55:6819/1645904182","cluster_addr":"172.21.15.55:6821/1645904182","heartbeat_back_addr":"172.21.15.55:6825/1645904182","heartbeat_front_addr":"172.21.15.55:6823/1645904182","state":["exists","up"]},{"osd":3,"uuid":"9f7bc62d-c839-4d05-89c7-98e5abd0a121","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":104,"up_from":108,"up_thru":108,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":3148055856},{"type":"v1","addr":"172.21.15.55:6827","nonce":3148055856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":3148055856},{"type":"v1","addr":"172.21.15.55:6829","nonce":3148055856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":3148055856},{"type":"v1","addr":"172.21.15.55:6833","nonce":3148055856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":3148055856},{"type":"v1","addr":"172.21.15.55:6831","nonce":3148055856}]},"public_addr":"172.21.15.55:6827/3148055856","cluster_addr":"172.21.15.55:6829/3148055856","heartbeat_back_addr":"172.21.15.55:6833/3148055856","heartbeat_front_addr":"172.21.15.55:6831/3148055856","state":["exists","up"]},{"osd":4,"uuid":"a9c0950d-2618-4a47-ad13-035e2bd227f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6801","nonce":2570729307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6803","nonce":2570729307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6807","nonce":2570729307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6805","nonce":2570729307}]},"public_addr":"172.21.15.99:6801/2570729307","cluster_addr":"172.21.15.99:6803/2570729307","heartbeat_back_addr":"172.21.15.99:6807/2570729307","heartbeat_front_addr":"172.21.15.99:6805/2570729307","state":["exists","up"]},{"osd":5,"uuid":"64ac7fc4-adc3-4e7f-8512-a458d0092f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6809","nonce":3690413267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6811","nonce":3690413267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6815","nonce":3690413267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6813","nonce":3690413267}]},"public_addr":"172.21.15.99:6809/3690413267","cluster_addr":"172.21.15.99:6811/3690413267","heartbeat_back_addr":"172.21.15.99:6815/3690413267","heartbeat_front_addr":"172.21.15.99:6813/3690413267","state":["exists","up"]},{"osd":6,"uuid":"9be63962-c98b-4df9-8781-270556e51119","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":105,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6817","nonce":2102700865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6819","nonce":2102700865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6823","nonce":2102700865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6821","nonce":2102700865}]},"public_addr":"172.21.15.99:6817/2102700865","cluster_addr":"172.21.15.99:6819/2102700865","heartbeat_back_addr":"172.21.15.99:6823/2102700865","heartbeat_front_addr":"172.21.15.99:6821/2102700865","state":["exists","up"]},{"osd":7,"uuid":"ea011d8c-d9f8-4619-9ac0-575813be808a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6825","nonce":453077889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6827","nonce":453077889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6831","nonce":453077889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6829","nonce":453077889}]},"public_addr":"172.21.15.99:6825/453077889","cluster_addr":"172.21.15.99:6827/453077889","heartbeat_back_addr":"172.21.15.99:6831/453077889","heartbeat_front_addr":"172.21.15.99:6829/453077889","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:22.099373+0000","dead_epoch":89},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:42.018683+0000","dead_epoch":94},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:01.856304+0000","dead_epoch":99},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:22.152206+0000","dead_epoch":104},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:41.385241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:01.725749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:21.176273+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:41.516585+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/1552377843":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/1663054712":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/4290529019":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/2173356416":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.99:0/3564577630":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/591596032":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1826983981":"2023-09-24T17:40:48.304146+0000","172.21.15.99:6835/2823676616":"2023-09-24T17:48:26.829766+0000","172.21.15.99:0/2950334811":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/880894918":"2023-09-24T17:47:42.374356+0000","172.21.15.99:0/1490815752":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/1355725764":"2023-09-24T17:49:06.276090+0000","172.21.15.55:6801/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6800/32209804":"2023-09-24T17:47:42.374356+0000","172.21.15.99:0/2680034432":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/3452781072":"2023-09-24T17:49:06.276090+0000","172.21.15.99:0/3129059336":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6801/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/3953275940":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/3757490612":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/658967358":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/573100440":"2023-09-24T17:47:42.374356+0000","172.21.15.55:6800/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/2880364962":"2023-09-24T17:47:42.374356+0000","172.21.15.55:0/218576037":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/238250704":"2023-09-24T17:41:12.624150+0000","172.21.15.99:6834/2823676616":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6801/2816080264":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/880528732":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/2064251389":"2023-09-24T17:47:42.374356+0000","172.21.15.55:6801/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/3781541742":"2023-09-24T17:49:06.276090+0000","172.21.15.99:0/1538541481":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6800/2816080264":"2023-09-24T17:49:06.276090+0000","172.21.15.55:6801/32209804":"2023-09-24T17:47:42.374356+0000","172.21.15.55:0/2453190559":"2023-09-24T17:49:06.276090+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T17:55:11.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:11 smithi099 ceph-mon[167076]: pgmap v205: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:12.145 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T17:55:12.145 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T17:55:12.755 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:12.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:12 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:12.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:12 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/498236187' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:55:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:12 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/498236187' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:55:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:12 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:12 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/498236187' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:55:13.508 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:13 smithi055 ceph-mon[179668]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:13.760 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:13 smithi055 ceph-mon[184174]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:13.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:13 smithi099 ceph-mon[167076]: pgmap v206: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:14.146 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:55:14.146 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":109,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","created":"2023-09-23T17:40:13.381005+0000","modified":"2023-09-23T17:54:57.011846+0000","last_up_change":"2023-09-23T17:54:56.008824+0000","last_in_change":"2023-09-23T17:44:27.554515+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":22,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T17:43:05.158393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"21","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":7.8899998664855957,"score_stable":7.8899998664855957,"optimal_score":0.37999999523162842,"raw_score_acting":3,"raw_score_stable":3,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T17:45:10.969164+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"84","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":15,"snap_epoch":84,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T17:45:49.622328+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T17:45:50.526600+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"99e7cae6-389e-461a-8a86-a5789673908b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":89,"up_from":93,"up_thru":108,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6803","nonce":3605839049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6805","nonce":3605839049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6809","nonce":3605839049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6807","nonce":3605839049}]},"public_addr":"172.21.15.55:6803/3605839049","cluster_addr":"172.21.15.55:6805/3605839049","heartbeat_back_addr":"172.21.15.55:6809/3605839049","heartbeat_front_addr":"172.21.15.55:6807/3605839049","state":["exists","up"]},{"osd":1,"uuid":"bfb0da01-7a06-430b-b973-209f22ac30d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":13,"last_clean_end":94,"up_from":98,"up_thru":108,"down_at":95,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":4278491277},{"type":"v1","addr":"172.21.15.55:6811","nonce":4278491277}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":4278491277},{"type":"v1","addr":"172.21.15.55:6813","nonce":4278491277}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":4278491277},{"type":"v1","addr":"172.21.15.55:6817","nonce":4278491277}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":4278491277},{"type":"v1","addr":"172.21.15.55:6815","nonce":4278491277}]},"public_addr":"172.21.15.55:6811/4278491277","cluster_addr":"172.21.15.55:6813/4278491277","heartbeat_back_addr":"172.21.15.55:6817/4278491277","heartbeat_front_addr":"172.21.15.55:6815/4278491277","state":["exists","up"]},{"osd":2,"uuid":"f766d604-db54-4a09-9993-fe1a3ce0048e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":18,"last_clean_end":99,"up_from":103,"up_thru":108,"down_at":100,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1645904182},{"type":"v1","addr":"172.21.15.55:6819","nonce":1645904182}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1645904182},{"type":"v1","addr":"172.21.15.55:6821","nonce":1645904182}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1645904182},{"type":"v1","addr":"172.21.15.55:6825","nonce":1645904182}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1645904182},{"type":"v1","addr":"172.21.15.55:6823","nonce":1645904182}]},"public_addr":"172.21.15.55:6819/1645904182","cluster_addr":"172.21.15.55:6821/1645904182","heartbeat_back_addr":"172.21.15.55:6825/1645904182","heartbeat_front_addr":"172.21.15.55:6823/1645904182","state":["exists","up"]},{"osd":3,"uuid":"9f7bc62d-c839-4d05-89c7-98e5abd0a121","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":25,"last_clean_end":104,"up_from":108,"up_thru":108,"down_at":105,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":3148055856},{"type":"v1","addr":"172.21.15.55:6827","nonce":3148055856}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":3148055856},{"type":"v1","addr":"172.21.15.55:6829","nonce":3148055856}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":3148055856},{"type":"v1","addr":"172.21.15.55:6833","nonce":3148055856}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":3148055856},{"type":"v1","addr":"172.21.15.55:6831","nonce":3148055856}]},"public_addr":"172.21.15.55:6827/3148055856","cluster_addr":"172.21.15.55:6829/3148055856","heartbeat_back_addr":"172.21.15.55:6833/3148055856","heartbeat_front_addr":"172.21.15.55:6831/3148055856","state":["exists","up"]},{"osd":4,"uuid":"a9c0950d-2618-4a47-ad13-035e2bd227f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6801","nonce":2570729307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6803","nonce":2570729307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6807","nonce":2570729307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6805","nonce":2570729307}]},"public_addr":"172.21.15.99:6801/2570729307","cluster_addr":"172.21.15.99:6803/2570729307","heartbeat_back_addr":"172.21.15.99:6807/2570729307","heartbeat_front_addr":"172.21.15.99:6805/2570729307","state":["exists","up"]},{"osd":5,"uuid":"64ac7fc4-adc3-4e7f-8512-a458d0092f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":38,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6809","nonce":3690413267}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6811","nonce":3690413267}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6815","nonce":3690413267}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":3690413267},{"type":"v1","addr":"172.21.15.99:6813","nonce":3690413267}]},"public_addr":"172.21.15.99:6809/3690413267","cluster_addr":"172.21.15.99:6811/3690413267","heartbeat_back_addr":"172.21.15.99:6815/3690413267","heartbeat_front_addr":"172.21.15.99:6813/3690413267","state":["exists","up"]},{"osd":6,"uuid":"9be63962-c98b-4df9-8781-270556e51119","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":45,"up_thru":105,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6817","nonce":2102700865}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6819","nonce":2102700865}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6823","nonce":2102700865}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":2102700865},{"type":"v1","addr":"172.21.15.99:6821","nonce":2102700865}]},"public_addr":"172.21.15.99:6817/2102700865","cluster_addr":"172.21.15.99:6819/2102700865","heartbeat_back_addr":"172.21.15.99:6823/2102700865","heartbeat_front_addr":"172.21.15.99:6821/2102700865","state":["exists","up"]},{"osd":7,"uuid":"ea011d8c-d9f8-4619-9ac0-575813be808a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":53,"up_thru":108,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6825","nonce":453077889}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6827","nonce":453077889}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6831","nonce":453077889}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":453077889},{"type":"v1","addr":"172.21.15.99:6829","nonce":453077889}]},"public_addr":"172.21.15.99:6825/453077889","cluster_addr":"172.21.15.99:6827/453077889","heartbeat_back_addr":"172.21.15.99:6831/453077889","heartbeat_front_addr":"172.21.15.99:6829/453077889","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:22.099373+0000","dead_epoch":89},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:42.018683+0000","dead_epoch":94},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:01.856304+0000","dead_epoch":99},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:22.152206+0000","dead_epoch":104},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:41.385241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:01.725749+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:21.176273+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:41.516585+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.a","mappings":[{"from":1,"to":2}]},{"pgid":"4.d","mappings":[{"from":1,"to":5}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/1552377843":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/1663054712":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/4290529019":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/2173356416":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.99:0/3564577630":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/591596032":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1826983981":"2023-09-24T17:40:48.304146+0000","172.21.15.99:6835/2823676616":"2023-09-24T17:48:26.829766+0000","172.21.15.99:0/2950334811":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/880894918":"2023-09-24T17:47:42.374356+0000","172.21.15.99:0/1490815752":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/1355725764":"2023-09-24T17:49:06.276090+0000","172.21.15.55:6801/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6800/32209804":"2023-09-24T17:47:42.374356+0000","172.21.15.99:0/2680034432":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/3452781072":"2023-09-24T17:49:06.276090+0000","172.21.15.99:0/3129059336":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6801/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/3953275940":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/3757490612":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/658967358":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/573100440":"2023-09-24T17:47:42.374356+0000","172.21.15.55:6800/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/2880364962":"2023-09-24T17:47:42.374356+0000","172.21.15.55:0/218576037":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/238250704":"2023-09-24T17:41:12.624150+0000","172.21.15.99:6834/2823676616":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6801/2816080264":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/880528732":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/2064251389":"2023-09-24T17:47:42.374356+0000","172.21.15.55:6801/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/3781541742":"2023-09-24T17:49:06.276090+0000","172.21.15.99:0/1538541481":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6800/2816080264":"2023-09-24T17:49:06.276090+0000","172.21.15.55:6801/32209804":"2023-09-24T17:47:42.374356+0000","172.21.15.55:0/2453190559":"2023-09-24T17:49:06.276090+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T17:55:14.517 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T17:55:14.518 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T17:55:14.518 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T17:55:14.519 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T17:55:14.520 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T17:55:14.520 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T17:55:14.521 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T17:55:14.522 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T17:55:14.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/80868096' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:55:14.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:55:14.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:55:14.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:55:14.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:14.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:55:14.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:55:14.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:55:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/80868096' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:55:14.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:55:14.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:55:14.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:55:14.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:14.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:55:14.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:55:14.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:55:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/80868096' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T17:55:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:55:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:55:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:55:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:55:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:55:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:55:15.450 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:15.465 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:15.561 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:15.630 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:15.775 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:15.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:15 smithi055 ceph-mon[179668]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:15.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:15.811 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:15 smithi055 ceph-mon[184174]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:15.811 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:15 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:15.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:15 smithi099 ceph-mon[167076]: pgmap v207: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:15.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:15 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:55:15.885 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:16.489 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:17.429 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:17.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:17 smithi099 ceph-mon[167076]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:17.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:17 smithi055 ceph-mon[184174]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:17.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:17 smithi055 ceph-mon[179668]: pgmap v208: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:19.606 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:19 smithi055 ceph-mon[184174]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:19.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:19 smithi055 ceph-mon[179668]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:19 smithi099 ceph-mon[167076]: pgmap v209: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:19.956 INFO:teuthology.orchestra.run.smithi055.stdout:442381631501 2023-09-23T17:55:19.956 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T17:55:20.524 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:55:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:55:20] "GET /metrics HTTP/1.1" 200 34083 "" "Prometheus/2.43.0" 2023-09-23T17:55:21.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:21 smithi099 ceph-mon[167076]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:21.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:21 smithi055 ceph-mon[179668]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:21.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:21.878 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:21 smithi055 ceph-mon[184174]: pgmap v210: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:21.878 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:21.951 INFO:teuthology.orchestra.run.smithi055.stdout:193273528454 2023-09-23T17:55:21.951 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T17:55:22.000 INFO:teuthology.orchestra.run.smithi055.stdout:463856467975 2023-09-23T17:55:22.000 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T17:55:22.217 INFO:teuthology.orchestra.run.smithi055.stdout:163208757385 2023-09-23T17:55:22.217 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T17:55:22.247 INFO:teuthology.orchestra.run.smithi055.stdout:227633266818 2023-09-23T17:55:22.248 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T17:55:22.641 INFO:teuthology.orchestra.run.smithi055.stdout:128849019019 2023-09-23T17:55:22.641 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T17:55:22.671 INFO:teuthology.orchestra.run.smithi055.stdout:399431958554 2023-09-23T17:55:22.671 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T17:55:22.699 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:22.841 INFO:teuthology.orchestra.run.smithi055.stdout:420906795028 2023-09-23T17:55:22.842 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T17:55:23.082 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:23.400 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:23.401 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:23.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:23 smithi055 ceph-mon[179668]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:23.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:23 smithi055 ceph-mon[184174]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:23 smithi099 ceph-mon[167076]: pgmap v211: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:24.428 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:24.787 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:24.830 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:24.972 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:25 smithi099 ceph-mon[167076]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:25.838 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:25 smithi055 ceph-mon[179668]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:25.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:25 smithi055 ceph-mon[184174]: pgmap v212: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:25.901 INFO:teuthology.orchestra.run.smithi055.stdout:442381631502 2023-09-23T17:55:26.670 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/881643177' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T17:55:26.670 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/881643177' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T17:55:26.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/881643177' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T17:55:27.373 INFO:teuthology.orchestra.run.smithi055.stdout:193273528455 2023-09-23T17:55:27.523 INFO:teuthology.orchestra.run.smithi055.stdout:163208757386 2023-09-23T17:55:27.557 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:27 smithi055 ceph-mon[179668]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:27.557 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/749539245' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T17:55:27.557 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/776294238' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T17:55:27.557 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:27 smithi055 ceph-mon[184174]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:27.557 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/749539245' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T17:55:27.557 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/776294238' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T17:55:27.635 INFO:tasks.cephadm.ceph_manager.ceph:need seq 442381631501 got 442381631502 for osd.2 2023-09-23T17:55:27.635 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:27 smithi099 ceph-mon[167076]: pgmap v213: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/749539245' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T17:55:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/776294238' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T17:55:28.298 INFO:tasks.cephadm.ceph_manager.ceph:need seq 163208757385 got 163208757386 for osd.5 2023-09-23T17:55:28.298 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:28.439 INFO:teuthology.orchestra.run.smithi055.stdout:463856467976 2023-09-23T17:55:28.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:28 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/752576713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T17:55:28.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:28 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/752576713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T17:55:28.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:28 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/752576713' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T17:55:29.198 INFO:tasks.cephadm.ceph_manager.ceph:need seq 193273528454 got 193273528455 for osd.6 2023-09-23T17:55:29.198 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:29.478 INFO:teuthology.orchestra.run.smithi055.stdout:399431958555 2023-09-23T17:55:29.505 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:29 smithi055 ceph-mon[184174]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:29.505 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2447834161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T17:55:29.673 INFO:tasks.cephadm.ceph_manager.ceph:need seq 463856467975 got 463856467976 for osd.3 2023-09-23T17:55:29.673 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:29.680 INFO:teuthology.orchestra.run.smithi055.stdout:227633266820 2023-09-23T17:55:29.746 INFO:teuthology.orchestra.run.smithi055.stdout:420906795029 2023-09-23T17:55:29.774 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:29 smithi055 ceph-mon[179668]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:29.774 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2447834161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T17:55:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:29 smithi099 ceph-mon[167076]: pgmap v214: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2447834161' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T17:55:30.135 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958554 got 399431958555 for osd.0 2023-09-23T17:55:30.135 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:30.228 INFO:teuthology.orchestra.run.smithi055.stdout:128849019021 2023-09-23T17:55:30.336 INFO:tasks.cephadm.ceph_manager.ceph:need seq 227633266818 got 227633266820 for osd.7 2023-09-23T17:55:30.336 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:30.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:55:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:55:30] "GET /metrics HTTP/1.1" 200 34082 "" "Prometheus/2.43.0" 2023-09-23T17:55:30.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3820171340' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T17:55:30.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1290346296' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T17:55:30.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2887895749' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T17:55:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3820171340' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T17:55:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1290346296' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T17:55:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2887895749' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T17:55:30.738 INFO:tasks.cephadm.ceph_manager.ceph:need seq 420906795028 got 420906795029 for osd.1 2023-09-23T17:55:30.739 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3820171340' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T17:55:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1290346296' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T17:55:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2887895749' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T17:55:30.841 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019019 got 128849019021 for osd.4 2023-09-23T17:55:30.842 DEBUG:teuthology.parallel:result is None 2023-09-23T17:55:30.842 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T17:55:30.842 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:55:31.449 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:31.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:31 smithi099 ceph-mon[167076]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:31.855 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:31 smithi055 ceph-mon[184174]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:31.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:31 smithi055 ceph-mon[179668]: pgmap v215: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:32.900 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:55:32.901 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:55:33.311 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":216,"stamp":"2023-09-23T17:55:32.405263+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":168208,"kb_used_data":22928,"kb_used_omap":29,"kb_used_meta":145186,"kb_avail":749793008,"statfs":{"total":767960285184,"available":767788040192,"internally_reserved":0,"allocated":23478272,"data_stored":19282825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30208,"internal_metadata":148670976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003088"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.020803+0000","last_change":"2023-09-23T17:53:52.736718+0000","last_active":"2023-09-23T17:54:56.020803+0000","last_peered":"2023-09-23T17:54:56.020803+0000","last_clean":"2023-09-23T17:54:56.020803+0000","last_became_active":"2023-09-23T17:53:52.736314+0000","last_became_peered":"2023-09-23T17:53:52.736314+0000","last_unstale":"2023-09-23T17:54:56.020803+0000","last_undegraded":"2023-09-23T17:54:56.020803+0000","last_fullsized":"2023-09-23T17:54:56.020803+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:26.017288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026023+0000","last_change":"2023-09-23T17:54:25.601065+0000","last_active":"2023-09-23T17:54:57.026023+0000","last_peered":"2023-09-23T17:54:57.026023+0000","last_clean":"2023-09-23T17:54:57.026023+0000","last_became_active":"2023-09-23T17:54:25.600727+0000","last_became_peered":"2023-09-23T17:54:25.600727+0000","last_unstale":"2023-09-23T17:54:57.026023+0000","last_undegraded":"2023-09-23T17:54:57.026023+0000","last_fullsized":"2023-09-23T17:54:57.026023+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:20.919498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034722+0000","last_change":"2023-09-23T17:54:57.034722+0000","last_active":"2023-09-23T17:54:57.034722+0000","last_peered":"2023-09-23T17:54:57.034722+0000","last_clean":"2023-09-23T17:54:57.034722+0000","last_became_active":"2023-09-23T17:54:57.034289+0000","last_became_peered":"2023-09-23T17:54:57.034289+0000","last_unstale":"2023-09-23T17:54:57.034722+0000","last_undegraded":"2023-09-23T17:54:57.034722+0000","last_fullsized":"2023-09-23T17:54:57.034722+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:03.041518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.032421+0000","last_change":"2023-09-23T17:54:57.032421+0000","last_active":"2023-09-23T17:54:57.032421+0000","last_peered":"2023-09-23T17:54:57.032421+0000","last_clean":"2023-09-23T17:54:57.032421+0000","last_became_active":"2023-09-23T17:54:57.032020+0000","last_became_peered":"2023-09-23T17:54:57.032020+0000","last_unstale":"2023-09-23T17:54:57.032421+0000","last_undegraded":"2023-09-23T17:54:57.032421+0000","last_fullsized":"2023-09-23T17:54:57.032421+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:14.716268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039895+0000","last_change":"2023-09-23T17:54:57.039895+0000","last_active":"2023-09-23T17:54:57.039895+0000","last_peered":"2023-09-23T17:54:57.039895+0000","last_clean":"2023-09-23T17:54:57.039895+0000","last_became_active":"2023-09-23T17:54:57.039581+0000","last_became_peered":"2023-09-23T17:54:57.039581+0000","last_unstale":"2023-09-23T17:54:57.039895+0000","last_undegraded":"2023-09-23T17:54:57.039895+0000","last_fullsized":"2023-09-23T17:54:57.039895+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:52.622668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029112+0000","last_change":"2023-09-23T17:45:51.121328+0000","last_active":"2023-09-23T17:54:56.029112+0000","last_peered":"2023-09-23T17:54:56.029112+0000","last_clean":"2023-09-23T17:54:56.029112+0000","last_became_active":"2023-09-23T17:45:51.121034+0000","last_became_peered":"2023-09-23T17:45:51.121034+0000","last_unstale":"2023-09-23T17:54:56.029112+0000","last_undegraded":"2023-09-23T17:54:56.029112+0000","last_fullsized":"2023-09-23T17:54:56.029112+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:13.256406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041535+0000","last_change":"2023-09-23T17:54:57.041535+0000","last_active":"2023-09-23T17:54:57.041535+0000","last_peered":"2023-09-23T17:54:57.041535+0000","last_clean":"2023-09-23T17:54:57.041535+0000","last_became_active":"2023-09-23T17:54:57.040909+0000","last_became_peered":"2023-09-23T17:54:57.040909+0000","last_unstale":"2023-09-23T17:54:57.041535+0000","last_undegraded":"2023-09-23T17:54:57.041535+0000","last_fullsized":"2023-09-23T17:54:57.041535+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:57.504646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.030937+0000","last_change":"2023-09-23T17:53:52.732284+0000","last_active":"2023-09-23T17:54:56.030937+0000","last_peered":"2023-09-23T17:54:56.030937+0000","last_clean":"2023-09-23T17:54:56.030937+0000","last_became_active":"2023-09-23T17:53:52.731928+0000","last_became_peered":"2023-09-23T17:53:52.731928+0000","last_unstale":"2023-09-23T17:54:56.030937+0000","last_undegraded":"2023-09-23T17:54:56.030937+0000","last_fullsized":"2023-09-23T17:54:56.030937+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:35.936955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034729+0000","last_change":"2023-09-23T17:54:57.034729+0000","last_active":"2023-09-23T17:54:57.034729+0000","last_peered":"2023-09-23T17:54:57.034729+0000","last_clean":"2023-09-23T17:54:57.034729+0000","last_became_active":"2023-09-23T17:54:57.034381+0000","last_became_peered":"2023-09-23T17:54:57.034381+0000","last_unstale":"2023-09-23T17:54:57.034729+0000","last_undegraded":"2023-09-23T17:54:57.034729+0000","last_fullsized":"2023-09-23T17:54:57.034729+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:19.369673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":39,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026132+0000","last_change":"2023-09-23T17:53:21.039717+0000","last_active":"2023-09-23T17:54:56.026132+0000","last_peered":"2023-09-23T17:54:56.026132+0000","last_clean":"2023-09-23T17:54:56.026132+0000","last_became_active":"2023-09-23T17:53:21.038793+0000","last_became_peered":"2023-09-23T17:53:21.038793+0000","last_unstale":"2023-09-23T17:54:56.026132+0000","last_undegraded":"2023-09-23T17:54:56.026132+0000","last_fullsized":"2023-09-23T17:54:56.026132+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:02.116246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025537+0000","last_change":"2023-09-23T17:45:51.148003+0000","last_active":"2023-09-23T17:54:56.025537+0000","last_peered":"2023-09-23T17:54:56.025537+0000","last_clean":"2023-09-23T17:54:56.025537+0000","last_became_active":"2023-09-23T17:45:51.147815+0000","last_became_peered":"2023-09-23T17:45:51.147815+0000","last_unstale":"2023-09-23T17:54:56.025537+0000","last_undegraded":"2023-09-23T17:54:56.025537+0000","last_fullsized":"2023-09-23T17:54:56.025537+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:32.374670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026943+0000","last_change":"2023-09-23T17:53:52.732070+0000","last_active":"2023-09-23T17:54:57.026943+0000","last_peered":"2023-09-23T17:54:57.026943+0000","last_clean":"2023-09-23T17:54:57.026943+0000","last_became_active":"2023-09-23T17:53:52.731739+0000","last_became_peered":"2023-09-23T17:53:52.731739+0000","last_unstale":"2023-09-23T17:54:57.026943+0000","last_undegraded":"2023-09-23T17:54:57.026943+0000","last_fullsized":"2023-09-23T17:54:57.026943+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:03.726754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":333,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.024215+0000","last_change":"2023-09-23T17:54:25.600838+0000","last_active":"2023-09-23T17:54:56.024215+0000","last_peered":"2023-09-23T17:54:56.024215+0000","last_clean":"2023-09-23T17:54:56.024215+0000","last_became_active":"2023-09-23T17:54:25.600305+0000","last_became_peered":"2023-09-23T17:54:25.600305+0000","last_unstale":"2023-09-23T17:54:56.024215+0000","last_undegraded":"2023-09-23T17:54:56.024215+0000","last_fullsized":"2023-09-23T17:54:56.024215+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:36.605455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":1405,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040450+0000","last_change":"2023-09-23T17:54:57.040450+0000","last_active":"2023-09-23T17:54:57.040450+0000","last_peered":"2023-09-23T17:54:57.040450+0000","last_clean":"2023-09-23T17:54:57.040450+0000","last_became_active":"2023-09-23T17:54:57.040134+0000","last_became_peered":"2023-09-23T17:54:57.040134+0000","last_unstale":"2023-09-23T17:54:57.040450+0000","last_undegraded":"2023-09-23T17:54:57.040450+0000","last_fullsized":"2023-09-23T17:54:57.040450+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:07.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":286,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.032152+0000","last_change":"2023-09-23T17:53:52.734869+0000","last_active":"2023-09-23T17:54:56.032152+0000","last_peered":"2023-09-23T17:54:56.032152+0000","last_clean":"2023-09-23T17:54:56.032152+0000","last_became_active":"2023-09-23T17:53:52.734387+0000","last_became_peered":"2023-09-23T17:53:52.734387+0000","last_unstale":"2023-09-23T17:54:56.032152+0000","last_undegraded":"2023-09-23T17:54:56.032152+0000","last_fullsized":"2023-09-23T17:54:56.032152+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:15.397601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041796+0000","last_change":"2023-09-23T17:54:57.041796+0000","last_active":"2023-09-23T17:54:57.041796+0000","last_peered":"2023-09-23T17:54:57.041796+0000","last_clean":"2023-09-23T17:54:57.041796+0000","last_became_active":"2023-09-23T17:54:57.041245+0000","last_became_peered":"2023-09-23T17:54:57.041245+0000","last_unstale":"2023-09-23T17:54:57.041796+0000","last_undegraded":"2023-09-23T17:54:57.041796+0000","last_fullsized":"2023-09-23T17:54:57.041796+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:26.340510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026049+0000","last_change":"2023-09-23T17:53:21.039195+0000","last_active":"2023-09-23T17:54:57.026049+0000","last_peered":"2023-09-23T17:54:57.026049+0000","last_clean":"2023-09-23T17:54:57.026049+0000","last_became_active":"2023-09-23T17:53:21.038654+0000","last_became_peered":"2023-09-23T17:53:21.038654+0000","last_unstale":"2023-09-23T17:54:57.026049+0000","last_undegraded":"2023-09-23T17:54:57.026049+0000","last_fullsized":"2023-09-23T17:54:57.026049+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:34.466980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033621+0000","last_change":"2023-09-23T17:54:25.604720+0000","last_active":"2023-09-23T17:54:56.033621+0000","last_peered":"2023-09-23T17:54:56.033621+0000","last_clean":"2023-09-23T17:54:56.033621+0000","last_became_active":"2023-09-23T17:54:25.604386+0000","last_became_peered":"2023-09-23T17:54:25.604386+0000","last_unstale":"2023-09-23T17:54:56.033621+0000","last_undegraded":"2023-09-23T17:54:56.033621+0000","last_fullsized":"2023-09-23T17:54:56.033621+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:56.869273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":321,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028220+0000","last_change":"2023-09-23T17:53:21.035484+0000","last_active":"2023-09-23T17:54:56.028220+0000","last_peered":"2023-09-23T17:54:56.028220+0000","last_clean":"2023-09-23T17:54:56.028220+0000","last_became_active":"2023-09-23T17:53:21.035035+0000","last_became_peered":"2023-09-23T17:53:21.035035+0000","last_unstale":"2023-09-23T17:54:56.028220+0000","last_undegraded":"2023-09-23T17:54:56.028220+0000","last_fullsized":"2023-09-23T17:54:56.028220+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:05.145111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026968+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T17:54:57.026968+0000","last_peered":"2023-09-23T17:54:57.026968+0000","last_clean":"2023-09-23T17:54:57.026968+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T17:54:57.026968+0000","last_undegraded":"2023-09-23T17:54:57.026968+0000","last_fullsized":"2023-09-23T17:54:57.026968+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039608+0000","last_change":"2023-09-23T17:54:57.036095+0000","last_active":"2023-09-23T17:54:57.039608+0000","last_peered":"2023-09-23T17:54:57.039608+0000","last_clean":"2023-09-23T17:54:57.039608+0000","last_became_active":"2023-09-23T17:54:57.035606+0000","last_became_peered":"2023-09-23T17:54:57.035606+0000","last_unstale":"2023-09-23T17:54:57.039608+0000","last_undegraded":"2023-09-23T17:54:57.039608+0000","last_fullsized":"2023-09-23T17:54:57.039608+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:10.936020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":307,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025278+0000","last_change":"2023-09-23T17:53:52.734324+0000","last_active":"2023-09-23T17:54:56.025278+0000","last_peered":"2023-09-23T17:54:56.025278+0000","last_clean":"2023-09-23T17:54:56.025278+0000","last_became_active":"2023-09-23T17:53:52.733825+0000","last_became_peered":"2023-09-23T17:53:52.733825+0000","last_unstale":"2023-09-23T17:54:56.025278+0000","last_undegraded":"2023-09-23T17:54:56.025278+0000","last_fullsized":"2023-09-23T17:54:56.025278+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:34.742687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.027204+0000","last_change":"2023-09-23T17:45:51.297621+0000","last_active":"2023-09-23T17:54:56.027204+0000","last_peered":"2023-09-23T17:54:56.027204+0000","last_clean":"2023-09-23T17:54:56.027204+0000","last_became_active":"2023-09-23T17:45:51.297322+0000","last_became_peered":"2023-09-23T17:45:51.297322+0000","last_unstale":"2023-09-23T17:54:56.027204+0000","last_undegraded":"2023-09-23T17:54:56.027204+0000","last_fullsized":"2023-09-23T17:54:56.027204+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:14.268027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036074+0000","last_change":"2023-09-23T17:54:57.036074+0000","last_active":"2023-09-23T17:54:57.036074+0000","last_peered":"2023-09-23T17:54:57.036074+0000","last_clean":"2023-09-23T17:54:57.036074+0000","last_became_active":"2023-09-23T17:54:57.035801+0000","last_became_peered":"2023-09-23T17:54:57.035801+0000","last_unstale":"2023-09-23T17:54:57.036074+0000","last_undegraded":"2023-09-23T17:54:57.036074+0000","last_fullsized":"2023-09-23T17:54:57.036074+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:11.087766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":452,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025187+0000","last_change":"2023-09-23T17:53:21.035503+0000","last_active":"2023-09-23T17:54:56.025187+0000","last_peered":"2023-09-23T17:54:56.025187+0000","last_clean":"2023-09-23T17:54:56.025187+0000","last_became_active":"2023-09-23T17:53:21.035119+0000","last_became_peered":"2023-09-23T17:53:21.035119+0000","last_unstale":"2023-09-23T17:54:56.025187+0000","last_undegraded":"2023-09-23T17:54:56.025187+0000","last_fullsized":"2023-09-23T17:54:56.025187+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:32.369675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1023,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029395+0000","last_change":"2023-09-23T17:54:25.601298+0000","last_active":"2023-09-23T17:54:56.029395+0000","last_peered":"2023-09-23T17:54:56.029395+0000","last_clean":"2023-09-23T17:54:56.029395+0000","last_became_active":"2023-09-23T17:54:25.600854+0000","last_became_peered":"2023-09-23T17:54:25.600854+0000","last_unstale":"2023-09-23T17:54:56.029395+0000","last_undegraded":"2023-09-23T17:54:56.029395+0000","last_fullsized":"2023-09-23T17:54:56.029395+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:56.978955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028625+0000","last_change":"2023-09-23T17:53:52.728795+0000","last_active":"2023-09-23T17:54:56.028625+0000","last_peered":"2023-09-23T17:54:56.028625+0000","last_clean":"2023-09-23T17:54:56.028625+0000","last_became_active":"2023-09-23T17:53:52.728423+0000","last_became_peered":"2023-09-23T17:53:52.728423+0000","last_unstale":"2023-09-23T17:54:56.028625+0000","last_undegraded":"2023-09-23T17:54:56.028625+0000","last_fullsized":"2023-09-23T17:54:56.028625+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.759322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033868+0000","last_change":"2023-09-23T17:54:25.603824+0000","last_active":"2023-09-23T17:54:56.033868+0000","last_peered":"2023-09-23T17:54:56.033868+0000","last_clean":"2023-09-23T17:54:56.033868+0000","last_became_active":"2023-09-23T17:54:25.603526+0000","last_became_peered":"2023-09-23T17:54:25.603526+0000","last_unstale":"2023-09-23T17:54:56.033868+0000","last_undegraded":"2023-09-23T17:54:56.033868+0000","last_fullsized":"2023-09-23T17:54:56.033868+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:29.842346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":372,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.037634+0000","last_change":"2023-09-23T17:54:57.037634+0000","last_active":"2023-09-23T17:54:57.037634+0000","last_peered":"2023-09-23T17:54:57.037634+0000","last_clean":"2023-09-23T17:54:57.037634+0000","last_became_active":"2023-09-23T17:54:57.037115+0000","last_became_peered":"2023-09-23T17:54:57.037115+0000","last_unstale":"2023-09-23T17:54:57.037634+0000","last_undegraded":"2023-09-23T17:54:57.037634+0000","last_fullsized":"2023-09-23T17:54:57.037634+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:52:58.348982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035632+0000","last_change":"2023-09-23T17:54:57.035632+0000","last_active":"2023-09-23T17:54:57.035632+0000","last_peered":"2023-09-23T17:54:57.035632+0000","last_clean":"2023-09-23T17:54:57.035632+0000","last_became_active":"2023-09-23T17:54:57.035338+0000","last_became_peered":"2023-09-23T17:54:57.035338+0000","last_unstale":"2023-09-23T17:54:57.035632+0000","last_undegraded":"2023-09-23T17:54:57.035632+0000","last_fullsized":"2023-09-23T17:54:57.035632+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:18.773130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022511+0000","last_change":"2023-09-23T17:53:21.036597+0000","last_active":"2023-09-23T17:54:56.022511+0000","last_peered":"2023-09-23T17:54:56.022511+0000","last_clean":"2023-09-23T17:54:56.022511+0000","last_became_active":"2023-09-23T17:53:21.036282+0000","last_became_peered":"2023-09-23T17:53:21.036282+0000","last_unstale":"2023-09-23T17:54:56.022511+0000","last_undegraded":"2023-09-23T17:54:56.022511+0000","last_fullsized":"2023-09-23T17:54:56.022511+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:28.486975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1153,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029744+0000","last_change":"2023-09-23T17:53:52.733510+0000","last_active":"2023-09-23T17:54:56.029744+0000","last_peered":"2023-09-23T17:54:56.029744+0000","last_clean":"2023-09-23T17:54:56.029744+0000","last_became_active":"2023-09-23T17:53:52.733030+0000","last_became_peered":"2023-09-23T17:53:52.733030+0000","last_unstale":"2023-09-23T17:54:56.029744+0000","last_undegraded":"2023-09-23T17:54:56.029744+0000","last_fullsized":"2023-09-23T17:54:56.029744+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:59.009283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1894,"num_read_kb":2706,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029081+0000","last_change":"2023-09-23T17:53:52.735673+0000","last_active":"2023-09-23T17:54:56.029081+0000","last_peered":"2023-09-23T17:54:56.029081+0000","last_clean":"2023-09-23T17:54:56.029081+0000","last_became_active":"2023-09-23T17:53:52.734890+0000","last_became_peered":"2023-09-23T17:53:52.734890+0000","last_unstale":"2023-09-23T17:54:56.029081+0000","last_undegraded":"2023-09-23T17:54:56.029081+0000","last_fullsized":"2023-09-23T17:54:56.029081+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:22.338546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040788+0000","last_change":"2023-09-23T17:54:57.040788+0000","last_active":"2023-09-23T17:54:57.040788+0000","last_peered":"2023-09-23T17:54:57.040788+0000","last_clean":"2023-09-23T17:54:57.040788+0000","last_became_active":"2023-09-23T17:54:57.040271+0000","last_became_peered":"2023-09-23T17:54:57.040271+0000","last_unstale":"2023-09-23T17:54:57.040788+0000","last_undegraded":"2023-09-23T17:54:57.040788+0000","last_fullsized":"2023-09-23T17:54:57.040788+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:37.176032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":528,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033496+0000","last_change":"2023-09-23T17:53:52.730761+0000","last_active":"2023-09-23T17:54:56.033496+0000","last_peered":"2023-09-23T17:54:56.033496+0000","last_clean":"2023-09-23T17:54:56.033496+0000","last_became_active":"2023-09-23T17:53:52.730121+0000","last_became_peered":"2023-09-23T17:53:52.730121+0000","last_unstale":"2023-09-23T17:54:56.033496+0000","last_undegraded":"2023-09-23T17:54:56.033496+0000","last_fullsized":"2023-09-23T17:54:56.033496+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:02.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035100+0000","last_change":"2023-09-23T17:54:57.035100+0000","last_active":"2023-09-23T17:54:57.035100+0000","last_peered":"2023-09-23T17:54:57.035100+0000","last_clean":"2023-09-23T17:54:57.035100+0000","last_became_active":"2023-09-23T17:54:57.034733+0000","last_became_peered":"2023-09-23T17:54:57.034733+0000","last_unstale":"2023-09-23T17:54:57.035100+0000","last_undegraded":"2023-09-23T17:54:57.035100+0000","last_fullsized":"2023-09-23T17:54:57.035100+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:27.841590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038911+0000","last_change":"2023-09-23T17:54:57.038911+0000","last_active":"2023-09-23T17:54:57.038911+0000","last_peered":"2023-09-23T17:54:57.038911+0000","last_clean":"2023-09-23T17:54:57.038911+0000","last_became_active":"2023-09-23T17:54:57.038607+0000","last_became_peered":"2023-09-23T17:54:57.038607+0000","last_unstale":"2023-09-23T17:54:57.038911+0000","last_undegraded":"2023-09-23T17:54:57.038911+0000","last_fullsized":"2023-09-23T17:54:57.038911+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:20.373213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021390+0000","last_change":"2023-09-23T17:53:52.736573+0000","last_active":"2023-09-23T17:54:56.021390+0000","last_peered":"2023-09-23T17:54:56.021390+0000","last_clean":"2023-09-23T17:54:56.021390+0000","last_became_active":"2023-09-23T17:53:52.736320+0000","last_became_peered":"2023-09-23T17:53:52.736320+0000","last_unstale":"2023-09-23T17:54:56.021390+0000","last_undegraded":"2023-09-23T17:54:56.021390+0000","last_fullsized":"2023-09-23T17:54:56.021390+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:05.760412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025639+0000","last_change":"2023-09-23T17:54:25.602420+0000","last_active":"2023-09-23T17:54:56.025639+0000","last_peered":"2023-09-23T17:54:56.025639+0000","last_clean":"2023-09-23T17:54:56.025639+0000","last_became_active":"2023-09-23T17:54:25.601988+0000","last_became_peered":"2023-09-23T17:54:25.601988+0000","last_unstale":"2023-09-23T17:54:56.025639+0000","last_undegraded":"2023-09-23T17:54:56.025639+0000","last_fullsized":"2023-09-23T17:54:56.025639+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:17.703211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":79,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022486+0000","last_change":"2023-09-23T17:54:25.603149+0000","last_active":"2023-09-23T17:54:56.022486+0000","last_peered":"2023-09-23T17:54:56.022486+0000","last_clean":"2023-09-23T17:54:56.022486+0000","last_became_active":"2023-09-23T17:54:25.602846+0000","last_became_peered":"2023-09-23T17:54:25.602846+0000","last_unstale":"2023-09-23T17:54:56.022486+0000","last_undegraded":"2023-09-23T17:54:56.022486+0000","last_fullsized":"2023-09-23T17:54:56.022486+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:06.955023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029512+0000","last_change":"2023-09-23T17:53:52.732276+0000","last_active":"2023-09-23T17:54:56.029512+0000","last_peered":"2023-09-23T17:54:56.029512+0000","last_clean":"2023-09-23T17:54:56.029512+0000","last_became_active":"2023-09-23T17:53:52.731952+0000","last_became_peered":"2023-09-23T17:53:52.731952+0000","last_unstale":"2023-09-23T17:54:56.029512+0000","last_undegraded":"2023-09-23T17:54:56.029512+0000","last_fullsized":"2023-09-23T17:54:56.029512+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:06:07.529125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036113+0000","last_change":"2023-09-23T17:54:57.036113+0000","last_active":"2023-09-23T17:54:57.036113+0000","last_peered":"2023-09-23T17:54:57.036113+0000","last_clean":"2023-09-23T17:54:57.036113+0000","last_became_active":"2023-09-23T17:54:57.035704+0000","last_became_peered":"2023-09-23T17:54:57.035704+0000","last_unstale":"2023-09-23T17:54:57.036113+0000","last_undegraded":"2023-09-23T17:54:57.036113+0000","last_fullsized":"2023-09-23T17:54:57.036113+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:10.397866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035203+0000","last_change":"2023-09-23T17:54:57.035203+0000","last_active":"2023-09-23T17:54:57.035203+0000","last_peered":"2023-09-23T17:54:57.035203+0000","last_clean":"2023-09-23T17:54:57.035203+0000","last_became_active":"2023-09-23T17:54:57.034854+0000","last_became_peered":"2023-09-23T17:54:57.034854+0000","last_unstale":"2023-09-23T17:54:57.035203+0000","last_undegraded":"2023-09-23T17:54:57.035203+0000","last_fullsized":"2023-09-23T17:54:57.035203+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:34.657128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025997+0000","last_change":"2023-09-23T17:53:52.731322+0000","last_active":"2023-09-23T17:54:56.025997+0000","last_peered":"2023-09-23T17:54:56.025997+0000","last_clean":"2023-09-23T17:54:56.025997+0000","last_became_active":"2023-09-23T17:53:52.730995+0000","last_became_peered":"2023-09-23T17:53:52.730995+0000","last_unstale":"2023-09-23T17:54:56.025997+0000","last_undegraded":"2023-09-23T17:54:56.025997+0000","last_fullsized":"2023-09-23T17:54:56.025997+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:55.121531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040294+0000","last_change":"2023-09-23T17:54:57.040294+0000","last_active":"2023-09-23T17:54:57.040294+0000","last_peered":"2023-09-23T17:54:57.040294+0000","last_clean":"2023-09-23T17:54:57.040294+0000","last_became_active":"2023-09-23T17:54:57.039943+0000","last_became_peered":"2023-09-23T17:54:57.039943+0000","last_unstale":"2023-09-23T17:54:57.040294+0000","last_undegraded":"2023-09-23T17:54:57.040294+0000","last_fullsized":"2023-09-23T17:54:57.040294+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:38.474814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":80,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029374+0000","last_change":"2023-09-23T17:54:25.601375+0000","last_active":"2023-09-23T17:54:56.029374+0000","last_peered":"2023-09-23T17:54:56.029374+0000","last_clean":"2023-09-23T17:54:56.029374+0000","last_became_active":"2023-09-23T17:54:25.601051+0000","last_became_peered":"2023-09-23T17:54:25.601051+0000","last_unstale":"2023-09-23T17:54:56.029374+0000","last_undegraded":"2023-09-23T17:54:56.029374+0000","last_fullsized":"2023-09-23T17:54:56.029374+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:24.772636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028296+0000","last_change":"2023-09-23T17:45:51.129514+0000","last_active":"2023-09-23T17:54:56.028296+0000","last_peered":"2023-09-23T17:54:56.028296+0000","last_clean":"2023-09-23T17:54:56.028296+0000","last_became_active":"2023-09-23T17:45:51.129154+0000","last_became_peered":"2023-09-23T17:45:51.129154+0000","last_unstale":"2023-09-23T17:54:56.028296+0000","last_undegraded":"2023-09-23T17:54:56.028296+0000","last_fullsized":"2023-09-23T17:54:56.028296+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:58.292683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040014+0000","last_change":"2023-09-23T17:54:57.040014+0000","last_active":"2023-09-23T17:54:57.040014+0000","last_peered":"2023-09-23T17:54:57.040014+0000","last_clean":"2023-09-23T17:54:57.040014+0000","last_became_active":"2023-09-23T17:54:57.039661+0000","last_became_peered":"2023-09-23T17:54:57.039661+0000","last_unstale":"2023-09-23T17:54:57.040014+0000","last_undegraded":"2023-09-23T17:54:57.040014+0000","last_fullsized":"2023-09-23T17:54:57.040014+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:01.077579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026256+0000","last_change":"2023-09-23T17:53:52.732018+0000","last_active":"2023-09-23T17:54:56.026256+0000","last_peered":"2023-09-23T17:54:56.026256+0000","last_clean":"2023-09-23T17:54:56.026256+0000","last_became_active":"2023-09-23T17:53:52.731641+0000","last_became_peered":"2023-09-23T17:53:52.731641+0000","last_unstale":"2023-09-23T17:54:56.026256+0000","last_undegraded":"2023-09-23T17:54:56.026256+0000","last_fullsized":"2023-09-23T17:54:56.026256+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:37.290885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029040+0000","last_change":"2023-09-23T17:45:51.297370+0000","last_active":"2023-09-23T17:54:56.029040+0000","last_peered":"2023-09-23T17:54:56.029040+0000","last_clean":"2023-09-23T17:54:56.029040+0000","last_became_active":"2023-09-23T17:45:51.297055+0000","last_became_peered":"2023-09-23T17:45:51.297055+0000","last_unstale":"2023-09-23T17:54:56.029040+0000","last_undegraded":"2023-09-23T17:54:56.029040+0000","last_fullsized":"2023-09-23T17:54:56.029040+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:52.209267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026812+0000","last_change":"2023-09-23T17:53:21.041393+0000","last_active":"2023-09-23T17:54:56.026812+0000","last_peered":"2023-09-23T17:54:56.026812+0000","last_clean":"2023-09-23T17:54:56.026812+0000","last_became_active":"2023-09-23T17:53:21.040995+0000","last_became_peered":"2023-09-23T17:53:21.040995+0000","last_unstale":"2023-09-23T17:54:56.026812+0000","last_undegraded":"2023-09-23T17:54:56.026812+0000","last_fullsized":"2023-09-23T17:54:56.026812+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:12.660494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041584+0000","last_change":"2023-09-23T17:54:57.041584+0000","last_active":"2023-09-23T17:54:57.041584+0000","last_peered":"2023-09-23T17:54:57.041584+0000","last_clean":"2023-09-23T17:54:57.041584+0000","last_became_active":"2023-09-23T17:54:57.040808+0000","last_became_peered":"2023-09-23T17:54:57.040808+0000","last_unstale":"2023-09-23T17:54:57.041584+0000","last_undegraded":"2023-09-23T17:54:57.041584+0000","last_fullsized":"2023-09-23T17:54:57.041584+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:01.079083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":79,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026234+0000","last_change":"2023-09-23T17:53:21.034197+0000","last_active":"2023-09-23T17:54:56.026234+0000","last_peered":"2023-09-23T17:54:56.026234+0000","last_clean":"2023-09-23T17:54:56.026234+0000","last_became_active":"2023-09-23T17:53:21.033784+0000","last_became_peered":"2023-09-23T17:53:21.033784+0000","last_unstale":"2023-09-23T17:54:56.026234+0000","last_undegraded":"2023-09-23T17:54:56.026234+0000","last_fullsized":"2023-09-23T17:54:56.026234+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:33.645714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039517+0000","last_change":"2023-09-23T17:54:57.039517+0000","last_active":"2023-09-23T17:54:57.039517+0000","last_peered":"2023-09-23T17:54:57.039517+0000","last_clean":"2023-09-23T17:54:57.039517+0000","last_became_active":"2023-09-23T17:54:57.039236+0000","last_became_peered":"2023-09-23T17:54:57.039236+0000","last_unstale":"2023-09-23T17:54:57.039517+0000","last_undegraded":"2023-09-23T17:54:57.039517+0000","last_fullsized":"2023-09-23T17:54:57.039517+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.126485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025164+0000","last_change":"2023-09-23T17:45:51.127190+0000","last_active":"2023-09-23T17:54:56.025164+0000","last_peered":"2023-09-23T17:54:56.025164+0000","last_clean":"2023-09-23T17:54:56.025164+0000","last_became_active":"2023-09-23T17:45:51.126903+0000","last_became_peered":"2023-09-23T17:45:51.126903+0000","last_unstale":"2023-09-23T17:54:56.025164+0000","last_undegraded":"2023-09-23T17:54:56.025164+0000","last_fullsized":"2023-09-23T17:54:56.025164+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:55.349259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.042234+0000","last_change":"2023-09-23T17:54:57.042234+0000","last_active":"2023-09-23T17:54:57.042234+0000","last_peered":"2023-09-23T17:54:57.042234+0000","last_clean":"2023-09-23T17:54:57.042234+0000","last_became_active":"2023-09-23T17:54:57.041918+0000","last_became_peered":"2023-09-23T17:54:57.041918+0000","last_unstale":"2023-09-23T17:54:57.042234+0000","last_undegraded":"2023-09-23T17:54:57.042234+0000","last_fullsized":"2023-09-23T17:54:57.042234+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:44.128871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":50,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038461+0000","last_change":"2023-09-23T17:54:57.038461+0000","last_active":"2023-09-23T17:54:57.038461+0000","last_peered":"2023-09-23T17:54:57.038461+0000","last_clean":"2023-09-23T17:54:57.038461+0000","last_became_active":"2023-09-23T17:54:57.038210+0000","last_became_peered":"2023-09-23T17:54:57.038210+0000","last_unstale":"2023-09-23T17:54:57.038461+0000","last_undegraded":"2023-09-23T17:54:57.038461+0000","last_fullsized":"2023-09-23T17:54:57.038461+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:50.077733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040422+0000","last_change":"2023-09-23T17:54:57.040422+0000","last_active":"2023-09-23T17:54:57.040422+0000","last_peered":"2023-09-23T17:54:57.040422+0000","last_clean":"2023-09-23T17:54:57.040422+0000","last_became_active":"2023-09-23T17:54:57.040085+0000","last_became_peered":"2023-09-23T17:54:57.040085+0000","last_unstale":"2023-09-23T17:54:57.040422+0000","last_undegraded":"2023-09-23T17:54:57.040422+0000","last_fullsized":"2023-09-23T17:54:57.040422+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:47.846454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025676+0000","last_change":"2023-09-23T17:54:25.602669+0000","last_active":"2023-09-23T17:54:56.025676+0000","last_peered":"2023-09-23T17:54:56.025676+0000","last_clean":"2023-09-23T17:54:56.025676+0000","last_became_active":"2023-09-23T17:54:25.602338+0000","last_became_peered":"2023-09-23T17:54:25.602338+0000","last_unstale":"2023-09-23T17:54:56.025676+0000","last_undegraded":"2023-09-23T17:54:56.025676+0000","last_fullsized":"2023-09-23T17:54:56.025676+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:02.816165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022313+0000","last_change":"2023-09-23T17:53:52.737008+0000","last_active":"2023-09-23T17:54:56.022313+0000","last_peered":"2023-09-23T17:54:56.022313+0000","last_clean":"2023-09-23T17:54:56.022313+0000","last_became_active":"2023-09-23T17:53:52.736702+0000","last_became_peered":"2023-09-23T17:53:52.736702+0000","last_unstale":"2023-09-23T17:54:56.022313+0000","last_undegraded":"2023-09-23T17:54:56.022313+0000","last_fullsized":"2023-09-23T17:54:56.022313+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:14.384950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021344+0000","last_change":"2023-09-23T17:45:51.124742+0000","last_active":"2023-09-23T17:54:56.021344+0000","last_peered":"2023-09-23T17:54:56.021344+0000","last_clean":"2023-09-23T17:54:56.021344+0000","last_became_active":"2023-09-23T17:45:51.124342+0000","last_became_peered":"2023-09-23T17:45:51.124342+0000","last_unstale":"2023-09-23T17:54:56.021344+0000","last_undegraded":"2023-09-23T17:54:56.021344+0000","last_fullsized":"2023-09-23T17:54:56.021344+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:24.351549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.032458+0000","last_change":"2023-09-23T17:54:25.603148+0000","last_active":"2023-09-23T17:54:56.032458+0000","last_peered":"2023-09-23T17:54:56.032458+0000","last_clean":"2023-09-23T17:54:56.032458+0000","last_became_active":"2023-09-23T17:54:25.602859+0000","last_became_peered":"2023-09-23T17:54:25.602859+0000","last_unstale":"2023-09-23T17:54:56.032458+0000","last_undegraded":"2023-09-23T17:54:56.032458+0000","last_fullsized":"2023-09-23T17:54:56.032458+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:19.802583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.043487+0000","last_change":"2023-09-23T17:54:57.043487+0000","last_active":"2023-09-23T17:54:57.043487+0000","last_peered":"2023-09-23T17:54:57.043487+0000","last_clean":"2023-09-23T17:54:57.043487+0000","last_became_active":"2023-09-23T17:54:57.043064+0000","last_became_peered":"2023-09-23T17:54:57.043064+0000","last_unstale":"2023-09-23T17:54:57.043487+0000","last_undegraded":"2023-09-23T17:54:57.043487+0000","last_fullsized":"2023-09-23T17:54:57.043487+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:34:21.634412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038085+0000","last_change":"2023-09-23T17:54:57.038085+0000","last_active":"2023-09-23T17:54:57.038085+0000","last_peered":"2023-09-23T17:54:57.038085+0000","last_clean":"2023-09-23T17:54:57.038085+0000","last_became_active":"2023-09-23T17:54:57.037551+0000","last_became_peered":"2023-09-23T17:54:57.037551+0000","last_unstale":"2023-09-23T17:54:57.038085+0000","last_undegraded":"2023-09-23T17:54:57.038085+0000","last_fullsized":"2023-09-23T17:54:57.038085+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:20:29.582747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029706+0000","last_change":"2023-09-23T17:53:52.732689+0000","last_active":"2023-09-23T17:54:56.029706+0000","last_peered":"2023-09-23T17:54:56.029706+0000","last_clean":"2023-09-23T17:54:56.029706+0000","last_became_active":"2023-09-23T17:53:52.732382+0000","last_became_peered":"2023-09-23T17:53:52.732382+0000","last_unstale":"2023-09-23T17:54:56.029706+0000","last_undegraded":"2023-09-23T17:54:56.029706+0000","last_fullsized":"2023-09-23T17:54:56.029706+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:34.408802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034454+0000","last_change":"2023-09-23T17:54:57.034454+0000","last_active":"2023-09-23T17:54:57.034454+0000","last_peered":"2023-09-23T17:54:57.034454+0000","last_clean":"2023-09-23T17:54:57.034454+0000","last_became_active":"2023-09-23T17:54:57.034128+0000","last_became_peered":"2023-09-23T17:54:57.034128+0000","last_unstale":"2023-09-23T17:54:57.034454+0000","last_undegraded":"2023-09-23T17:54:57.034454+0000","last_fullsized":"2023-09-23T17:54:57.034454+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:39.315294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":50,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036930+0000","last_change":"2023-09-23T17:54:57.036930+0000","last_active":"2023-09-23T17:54:57.036930+0000","last_peered":"2023-09-23T17:54:57.036930+0000","last_clean":"2023-09-23T17:54:57.036930+0000","last_became_active":"2023-09-23T17:54:57.036631+0000","last_became_peered":"2023-09-23T17:54:57.036631+0000","last_unstale":"2023-09-23T17:54:57.036930+0000","last_undegraded":"2023-09-23T17:54:57.036930+0000","last_fullsized":"2023-09-23T17:54:57.036930+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:08.367357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.024125+0000","last_change":"2023-09-23T17:53:52.732305+0000","last_active":"2023-09-23T17:54:56.024125+0000","last_peered":"2023-09-23T17:54:56.024125+0000","last_clean":"2023-09-23T17:54:56.024125+0000","last_became_active":"2023-09-23T17:53:52.731894+0000","last_became_peered":"2023-09-23T17:53:52.731894+0000","last_unstale":"2023-09-23T17:54:56.024125+0000","last_undegraded":"2023-09-23T17:54:56.024125+0000","last_fullsized":"2023-09-23T17:54:56.024125+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:30.224762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.031226+0000","last_change":"2023-09-23T17:54:57.031226+0000","last_active":"2023-09-23T17:54:57.031226+0000","last_peered":"2023-09-23T17:54:57.031226+0000","last_clean":"2023-09-23T17:54:57.031226+0000","last_became_active":"2023-09-23T17:54:57.030880+0000","last_became_peered":"2023-09-23T17:54:57.030880+0000","last_unstale":"2023-09-23T17:54:57.031226+0000","last_undegraded":"2023-09-23T17:54:57.031226+0000","last_fullsized":"2023-09-23T17:54:57.031226+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:49.836129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040141+0000","last_change":"2023-09-23T17:54:57.039438+0000","last_active":"2023-09-23T17:54:57.040141+0000","last_peered":"2023-09-23T17:54:57.040141+0000","last_clean":"2023-09-23T17:54:57.040141+0000","last_became_active":"2023-09-23T17:54:57.039163+0000","last_became_peered":"2023-09-23T17:54:57.039163+0000","last_unstale":"2023-09-23T17:54:57.040141+0000","last_undegraded":"2023-09-23T17:54:57.040141+0000","last_fullsized":"2023-09-23T17:54:57.040141+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.265833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034381+0000","last_change":"2023-09-23T17:54:57.034381+0000","last_active":"2023-09-23T17:54:57.034381+0000","last_peered":"2023-09-23T17:54:57.034381+0000","last_clean":"2023-09-23T17:54:57.034381+0000","last_became_active":"2023-09-23T17:54:57.034043+0000","last_became_peered":"2023-09-23T17:54:57.034043+0000","last_unstale":"2023-09-23T17:54:57.034381+0000","last_undegraded":"2023-09-23T17:54:57.034381+0000","last_fullsized":"2023-09-23T17:54:57.034381+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:40.217762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036156+0000","last_change":"2023-09-23T17:54:57.036156+0000","last_active":"2023-09-23T17:54:57.036156+0000","last_peered":"2023-09-23T17:54:57.036156+0000","last_clean":"2023-09-23T17:54:57.036156+0000","last_became_active":"2023-09-23T17:54:57.035478+0000","last_became_peered":"2023-09-23T17:54:57.035478+0000","last_unstale":"2023-09-23T17:54:57.036156+0000","last_undegraded":"2023-09-23T17:54:57.036156+0000","last_fullsized":"2023-09-23T17:54:57.036156+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:53.126638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":88,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021007+0000","last_change":"2023-09-23T17:54:25.605945+0000","last_active":"2023-09-23T17:54:56.021007+0000","last_peered":"2023-09-23T17:54:56.021007+0000","last_clean":"2023-09-23T17:54:56.021007+0000","last_became_active":"2023-09-23T17:54:25.605583+0000","last_became_peered":"2023-09-23T17:54:25.605583+0000","last_unstale":"2023-09-23T17:54:56.021007+0000","last_undegraded":"2023-09-23T17:54:56.021007+0000","last_fullsized":"2023-09-23T17:54:56.021007+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:14.076789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266821,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":13024,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":11328,"kb_avail":93732128,"statfs":{"total":95995035648,"available":95981699072,"internally_reserved":0,"allocated":1728512,"data_stored":1206046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11599872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55500000000000005,"15min":0.55200000000000005},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81200000000000006,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57199999999999995,"15min":0.56899999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.56799999999999995}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.52100000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 17:54:01 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.58099999999999996,"15min":0.54400000000000004},"min":{"1min":0.436,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.79400000000000004,"5min":1.0409999999999999,"15min":1.9370000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.60399999999999998,"15min":0.63200000000000001},"min":{"1min":0.42399999999999999,"5min":0.36799999999999999,"15min":0.35899999999999999},"max":{"1min":0.75800000000000001,"5min":0.91300000000000003,"15min":8.8499999999999996},"last":0.57899999999999996}]},{"osd":4,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.49399999999999999,"15min":0.44600000000000001},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52300000000000002,"15min":0.47499999999999998},"min":{"1min":0.34699999999999998,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.438}]},{"osd":5,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53000000000000003,"15min":0.48999999999999999},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.28699999999999998},"max":{"1min":0.72799999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.57999999999999996,"15min":0.51400000000000001},"min":{"1min":0.40100000000000002,"5min":0.30099999999999999,"15min":0.29099999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.66200000000000003}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.55300000000000005}]}]},{"osd":6,"up_from":45,"seq":193273528457,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8344,"kb_used_data":1680,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93736808,"statfs":{"total":95995035648,"available":95986491392,"internally_reserved":0,"allocated":1720320,"data_stored":1205919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.81100000000000005,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.78300000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56399999999999995}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.57899999999999996}]},{"osd":2,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.61199999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.38800000000000001,"5min":0.36799999999999999,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":0.80700000000000005,"15min":4.5599999999999996},"last":0.502},{"interface":"front","average":{"1min":0.625,"5min":0.58499999999999996,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34999999999999998,"15min":0.30399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":1.6850000000000001},"last":0.54500000000000004}]},{"osd":4,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.47099999999999997,"15min":0.45200000000000001},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":0.748,"5min":0.76400000000000001,"15min":0.82899999999999996},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52800000000000002,"15min":0.49199999999999999},"min":{"1min":0.34999999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.53000000000000003}]},{"osd":5,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.53400000000000003,"15min":0.496},"min":{"1min":0.33700000000000002,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":0.77700000000000002,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56599999999999995,"15min":0.52200000000000002},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.33200000000000002},"max":{"1min":0.79700000000000004,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:55:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.58999999999999997,"15min":0.53700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":0.83299999999999996,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.57499999999999996,"15min":0.51900000000000002},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":0.81299999999999994,"5min":0.82299999999999995,"15min":1.591},"last":0.63100000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795030,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28716,"kb_used_data":1124,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93716436,"statfs":{"total":95995035648,"available":95965630464,"internally_reserved":0,"allocated":1150976,"data_stored":617993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":28238525},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.976,"5min":1.976,"15min":1.976},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.40699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.313}]},{"osd":4,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.79600000000000004}]},{"osd":5,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1,"5min":1,"15min":1},"last":0.80700000000000005}]},{"osd":6,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.68700000000000006}]},{"osd":7,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.67500000000000004}]}]},{"osd":0,"up_from":93,"seq":399431958556,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33408,"kb_used_data":5816,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93711744,"statfs":{"total":95995035648,"available":95960825856,"internally_reserved":0,"allocated":5955584,"data_stored":5404139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":28238564},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:54:51 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.309},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.40100000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"last":0.34100000000000003}]},{"osd":3,"last update":"Sat Sep 23 17:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.433}]},{"osd":4,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82199999999999995,"5min":0.872,"15min":0.872},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61699999999999999,"15min":0.621},"min":{"1min":0.44700000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77700000000000002,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.45700000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.624,"15min":0.63},"min":{"1min":0.41799999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.73599999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.63900000000000001,"15min":0.64300000000000002},"min":{"1min":0.46600000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.46600000000000003}]},{"osd":6,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.42899999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65200000000000002},"min":{"1min":0.52700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.81100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.626}]},{"osd":7,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65900000000000003,"15min":0.66300000000000003},"min":{"1min":0.51600000000000001,"5min":0.435,"15min":0.435},"max":{"1min":0.84299999999999997,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.66600000000000004,"15min":0.67000000000000004},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86599999999999999,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.67700000000000005}]}]},{"osd":2,"up_from":103,"seq":442381631503,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32744,"kb_used_data":5216,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93712408,"statfs":{"total":95995035648,"available":95961505792,"internally_reserved":0,"allocated":5341184,"data_stored":4811538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":28168567},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.248},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.35399999999999998}]},{"osd":1,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.308}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29099999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.49199999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.441},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.51800000000000002}]}]},{"osd":3,"up_from":108,"seq":463856467977,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32604,"kb_used_data":5204,"kb_used_omap":3,"kb_used_meta":27388,"kb_avail":93712548,"statfs":{"total":95995035648,"available":95961649152,"internally_reserved":0,"allocated":5328896,"data_stored":4809888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":28046056},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34399999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38200000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.496}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61399999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.504}]}]},{"osd":4,"up_from":30,"seq":128849019022,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7644,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6528,"kb_avail":93737508,"statfs":{"total":95995035648,"available":95987208192,"internally_reserved":0,"allocated":1134592,"data_stored":615931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6684672},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.75600000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.53800000000000003,"15min":0.52900000000000003},"min":{"1min":0.42699999999999999,"5min":0.376,"15min":0.376},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.59799999999999998}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.44900000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.495},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.623}]},{"osd":3,"last update":"Sat Sep 23 17:54:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.42599999999999999,"5min":0.34399999999999997,"15min":0.318},"max":{"1min":0.73999999999999999,"5min":0.77100000000000002,"15min":5.7119999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.59199999999999997,"15min":0.57999999999999996},"min":{"1min":0.41599999999999998,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":0.70999999999999996,"5min":0.85499999999999998,"15min":5.9429999999999996},"last":0.54800000000000004}]},{"osd":5,"last update":"Sat Sep 23 17:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48399999999999999,"15min":0.438},"min":{"1min":0.379,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":0.621,"5min":0.753,"15min":1.141},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.51700000000000002,"15min":0.44700000000000001},"min":{"1min":0.41499999999999998,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":0.749,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:54:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.54700000000000004,"15min":0.501},"min":{"1min":0.36099999999999999,"5min":0.33200000000000002,"15min":0.28899999999999998},"max":{"1min":0.68400000000000005,"5min":0.77400000000000002,"15min":1.276},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58799999999999997,"15min":0.53600000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28999999999999998},"max":{"1min":0.77100000000000002,"5min":0.80900000000000005,"15min":1.1180000000000001},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Sep 23 17:55:06 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.57699999999999996,"15min":0.54700000000000004},"min":{"1min":0.40600000000000003,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":1.1839999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.57699999999999996,"15min":0.53000000000000003},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.65200000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757388,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11724,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":10624,"kb_avail":93733428,"statfs":{"total":95995035648,"available":95983030272,"internally_reserved":0,"allocated":1118208,"data_stored":611371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":10878976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.57199999999999995,"15min":0.57599999999999996},"min":{"1min":0.39800000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.77100000000000002,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.74199999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.63600000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.4279999999999999,"5min":4.4279999999999999,"15min":4.4279999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.53600000000000003}]},{"osd":2,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.81499999999999995}]},{"osd":3,"last update":"Sat Sep 23 17:54:27 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61199999999999999,"15min":0.58099999999999996},"min":{"1min":0.42499999999999999,"5min":0.41999999999999998,"15min":0.32100000000000001},"max":{"1min":0.76700000000000002,"5min":0.85799999999999998,"15min":4.6799999999999997},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.61199999999999999,"15min":0.57699999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.33100000000000002},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.631},"last":0.67600000000000005}]},{"osd":4,"last update":"Sat Sep 23 17:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.48899999999999999,"15min":0.44800000000000001},"min":{"1min":0.34300000000000003,"5min":0.25,"15min":0.25},"max":{"1min":0.70799999999999996,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.54400000000000004,"15min":0.51400000000000001},"min":{"1min":0.435,"5min":0.33100000000000002,"15min":0.28899999999999998},"max":{"1min":0.81399999999999995,"5min":0.83199999999999996,"15min":1.738},"last":0.67100000000000004}]},{"osd":6,"last update":"Sat Sep 23 17:54:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.55300000000000005,"15min":0.53400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":1.778},"last":0.624},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.61599999999999999,"15min":0.58299999999999996},"min":{"1min":0.38100000000000001,"5min":0.375,"15min":0.33200000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":1.7629999999999999},"last":0.64700000000000002}]},{"osd":7,"last update":"Sat Sep 23 17:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.56299999999999994,"15min":0.52000000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.83999999999999997,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.58799999999999997,"15min":0.54900000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.79600000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.76600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:55:33.317 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:55:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:33 smithi055 ceph-mon[179668]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:33 smithi055 ceph-mon[179668]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:33 smithi055 ceph-mon[184174]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:33 smithi055 ceph-mon[184174]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:33 smithi099 ceph-mon[167076]: pgmap v216: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:33 smithi099 ceph-mon[167076]: from='client.44606 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:33.925 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:35.341 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:55:35.343 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:55:35.658 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:35 smithi055 ceph-mon[179668]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:35.658 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:35 smithi055 ceph-mon[184174]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:35.682 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":217,"stamp":"2023-09-23T17:55:34.405859+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":168208,"kb_used_data":22928,"kb_used_omap":29,"kb_used_meta":145186,"kb_avail":749793008,"statfs":{"total":767960285184,"available":767788040192,"internally_reserved":0,"allocated":23478272,"data_stored":19282825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30208,"internal_metadata":148670976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003250"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.020803+0000","last_change":"2023-09-23T17:53:52.736718+0000","last_active":"2023-09-23T17:54:56.020803+0000","last_peered":"2023-09-23T17:54:56.020803+0000","last_clean":"2023-09-23T17:54:56.020803+0000","last_became_active":"2023-09-23T17:53:52.736314+0000","last_became_peered":"2023-09-23T17:53:52.736314+0000","last_unstale":"2023-09-23T17:54:56.020803+0000","last_undegraded":"2023-09-23T17:54:56.020803+0000","last_fullsized":"2023-09-23T17:54:56.020803+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:26.017288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026023+0000","last_change":"2023-09-23T17:54:25.601065+0000","last_active":"2023-09-23T17:54:57.026023+0000","last_peered":"2023-09-23T17:54:57.026023+0000","last_clean":"2023-09-23T17:54:57.026023+0000","last_became_active":"2023-09-23T17:54:25.600727+0000","last_became_peered":"2023-09-23T17:54:25.600727+0000","last_unstale":"2023-09-23T17:54:57.026023+0000","last_undegraded":"2023-09-23T17:54:57.026023+0000","last_fullsized":"2023-09-23T17:54:57.026023+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:20.919498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034722+0000","last_change":"2023-09-23T17:54:57.034722+0000","last_active":"2023-09-23T17:54:57.034722+0000","last_peered":"2023-09-23T17:54:57.034722+0000","last_clean":"2023-09-23T17:54:57.034722+0000","last_became_active":"2023-09-23T17:54:57.034289+0000","last_became_peered":"2023-09-23T17:54:57.034289+0000","last_unstale":"2023-09-23T17:54:57.034722+0000","last_undegraded":"2023-09-23T17:54:57.034722+0000","last_fullsized":"2023-09-23T17:54:57.034722+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:03.041518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.032421+0000","last_change":"2023-09-23T17:54:57.032421+0000","last_active":"2023-09-23T17:54:57.032421+0000","last_peered":"2023-09-23T17:54:57.032421+0000","last_clean":"2023-09-23T17:54:57.032421+0000","last_became_active":"2023-09-23T17:54:57.032020+0000","last_became_peered":"2023-09-23T17:54:57.032020+0000","last_unstale":"2023-09-23T17:54:57.032421+0000","last_undegraded":"2023-09-23T17:54:57.032421+0000","last_fullsized":"2023-09-23T17:54:57.032421+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:14.716268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039895+0000","last_change":"2023-09-23T17:54:57.039895+0000","last_active":"2023-09-23T17:54:57.039895+0000","last_peered":"2023-09-23T17:54:57.039895+0000","last_clean":"2023-09-23T17:54:57.039895+0000","last_became_active":"2023-09-23T17:54:57.039581+0000","last_became_peered":"2023-09-23T17:54:57.039581+0000","last_unstale":"2023-09-23T17:54:57.039895+0000","last_undegraded":"2023-09-23T17:54:57.039895+0000","last_fullsized":"2023-09-23T17:54:57.039895+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:52.622668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029112+0000","last_change":"2023-09-23T17:45:51.121328+0000","last_active":"2023-09-23T17:54:56.029112+0000","last_peered":"2023-09-23T17:54:56.029112+0000","last_clean":"2023-09-23T17:54:56.029112+0000","last_became_active":"2023-09-23T17:45:51.121034+0000","last_became_peered":"2023-09-23T17:45:51.121034+0000","last_unstale":"2023-09-23T17:54:56.029112+0000","last_undegraded":"2023-09-23T17:54:56.029112+0000","last_fullsized":"2023-09-23T17:54:56.029112+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:13.256406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041535+0000","last_change":"2023-09-23T17:54:57.041535+0000","last_active":"2023-09-23T17:54:57.041535+0000","last_peered":"2023-09-23T17:54:57.041535+0000","last_clean":"2023-09-23T17:54:57.041535+0000","last_became_active":"2023-09-23T17:54:57.040909+0000","last_became_peered":"2023-09-23T17:54:57.040909+0000","last_unstale":"2023-09-23T17:54:57.041535+0000","last_undegraded":"2023-09-23T17:54:57.041535+0000","last_fullsized":"2023-09-23T17:54:57.041535+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:57.504646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.030937+0000","last_change":"2023-09-23T17:53:52.732284+0000","last_active":"2023-09-23T17:54:56.030937+0000","last_peered":"2023-09-23T17:54:56.030937+0000","last_clean":"2023-09-23T17:54:56.030937+0000","last_became_active":"2023-09-23T17:53:52.731928+0000","last_became_peered":"2023-09-23T17:53:52.731928+0000","last_unstale":"2023-09-23T17:54:56.030937+0000","last_undegraded":"2023-09-23T17:54:56.030937+0000","last_fullsized":"2023-09-23T17:54:56.030937+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:35.936955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034729+0000","last_change":"2023-09-23T17:54:57.034729+0000","last_active":"2023-09-23T17:54:57.034729+0000","last_peered":"2023-09-23T17:54:57.034729+0000","last_clean":"2023-09-23T17:54:57.034729+0000","last_became_active":"2023-09-23T17:54:57.034381+0000","last_became_peered":"2023-09-23T17:54:57.034381+0000","last_unstale":"2023-09-23T17:54:57.034729+0000","last_undegraded":"2023-09-23T17:54:57.034729+0000","last_fullsized":"2023-09-23T17:54:57.034729+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:19.369673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":39,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026132+0000","last_change":"2023-09-23T17:53:21.039717+0000","last_active":"2023-09-23T17:54:56.026132+0000","last_peered":"2023-09-23T17:54:56.026132+0000","last_clean":"2023-09-23T17:54:56.026132+0000","last_became_active":"2023-09-23T17:53:21.038793+0000","last_became_peered":"2023-09-23T17:53:21.038793+0000","last_unstale":"2023-09-23T17:54:56.026132+0000","last_undegraded":"2023-09-23T17:54:56.026132+0000","last_fullsized":"2023-09-23T17:54:56.026132+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:02.116246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025537+0000","last_change":"2023-09-23T17:45:51.148003+0000","last_active":"2023-09-23T17:54:56.025537+0000","last_peered":"2023-09-23T17:54:56.025537+0000","last_clean":"2023-09-23T17:54:56.025537+0000","last_became_active":"2023-09-23T17:45:51.147815+0000","last_became_peered":"2023-09-23T17:45:51.147815+0000","last_unstale":"2023-09-23T17:54:56.025537+0000","last_undegraded":"2023-09-23T17:54:56.025537+0000","last_fullsized":"2023-09-23T17:54:56.025537+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:32.374670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026943+0000","last_change":"2023-09-23T17:53:52.732070+0000","last_active":"2023-09-23T17:54:57.026943+0000","last_peered":"2023-09-23T17:54:57.026943+0000","last_clean":"2023-09-23T17:54:57.026943+0000","last_became_active":"2023-09-23T17:53:52.731739+0000","last_became_peered":"2023-09-23T17:53:52.731739+0000","last_unstale":"2023-09-23T17:54:57.026943+0000","last_undegraded":"2023-09-23T17:54:57.026943+0000","last_fullsized":"2023-09-23T17:54:57.026943+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:03.726754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":333,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.024215+0000","last_change":"2023-09-23T17:54:25.600838+0000","last_active":"2023-09-23T17:54:56.024215+0000","last_peered":"2023-09-23T17:54:56.024215+0000","last_clean":"2023-09-23T17:54:56.024215+0000","last_became_active":"2023-09-23T17:54:25.600305+0000","last_became_peered":"2023-09-23T17:54:25.600305+0000","last_unstale":"2023-09-23T17:54:56.024215+0000","last_undegraded":"2023-09-23T17:54:56.024215+0000","last_fullsized":"2023-09-23T17:54:56.024215+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:36.605455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":1405,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040450+0000","last_change":"2023-09-23T17:54:57.040450+0000","last_active":"2023-09-23T17:54:57.040450+0000","last_peered":"2023-09-23T17:54:57.040450+0000","last_clean":"2023-09-23T17:54:57.040450+0000","last_became_active":"2023-09-23T17:54:57.040134+0000","last_became_peered":"2023-09-23T17:54:57.040134+0000","last_unstale":"2023-09-23T17:54:57.040450+0000","last_undegraded":"2023-09-23T17:54:57.040450+0000","last_fullsized":"2023-09-23T17:54:57.040450+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:07.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":286,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.032152+0000","last_change":"2023-09-23T17:53:52.734869+0000","last_active":"2023-09-23T17:54:56.032152+0000","last_peered":"2023-09-23T17:54:56.032152+0000","last_clean":"2023-09-23T17:54:56.032152+0000","last_became_active":"2023-09-23T17:53:52.734387+0000","last_became_peered":"2023-09-23T17:53:52.734387+0000","last_unstale":"2023-09-23T17:54:56.032152+0000","last_undegraded":"2023-09-23T17:54:56.032152+0000","last_fullsized":"2023-09-23T17:54:56.032152+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:15.397601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041796+0000","last_change":"2023-09-23T17:54:57.041796+0000","last_active":"2023-09-23T17:54:57.041796+0000","last_peered":"2023-09-23T17:54:57.041796+0000","last_clean":"2023-09-23T17:54:57.041796+0000","last_became_active":"2023-09-23T17:54:57.041245+0000","last_became_peered":"2023-09-23T17:54:57.041245+0000","last_unstale":"2023-09-23T17:54:57.041796+0000","last_undegraded":"2023-09-23T17:54:57.041796+0000","last_fullsized":"2023-09-23T17:54:57.041796+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:26.340510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026049+0000","last_change":"2023-09-23T17:53:21.039195+0000","last_active":"2023-09-23T17:54:57.026049+0000","last_peered":"2023-09-23T17:54:57.026049+0000","last_clean":"2023-09-23T17:54:57.026049+0000","last_became_active":"2023-09-23T17:53:21.038654+0000","last_became_peered":"2023-09-23T17:53:21.038654+0000","last_unstale":"2023-09-23T17:54:57.026049+0000","last_undegraded":"2023-09-23T17:54:57.026049+0000","last_fullsized":"2023-09-23T17:54:57.026049+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:34.466980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033621+0000","last_change":"2023-09-23T17:54:25.604720+0000","last_active":"2023-09-23T17:54:56.033621+0000","last_peered":"2023-09-23T17:54:56.033621+0000","last_clean":"2023-09-23T17:54:56.033621+0000","last_became_active":"2023-09-23T17:54:25.604386+0000","last_became_peered":"2023-09-23T17:54:25.604386+0000","last_unstale":"2023-09-23T17:54:56.033621+0000","last_undegraded":"2023-09-23T17:54:56.033621+0000","last_fullsized":"2023-09-23T17:54:56.033621+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:56.869273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":321,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028220+0000","last_change":"2023-09-23T17:53:21.035484+0000","last_active":"2023-09-23T17:54:56.028220+0000","last_peered":"2023-09-23T17:54:56.028220+0000","last_clean":"2023-09-23T17:54:56.028220+0000","last_became_active":"2023-09-23T17:53:21.035035+0000","last_became_peered":"2023-09-23T17:53:21.035035+0000","last_unstale":"2023-09-23T17:54:56.028220+0000","last_undegraded":"2023-09-23T17:54:56.028220+0000","last_fullsized":"2023-09-23T17:54:56.028220+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:05.145111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026968+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T17:54:57.026968+0000","last_peered":"2023-09-23T17:54:57.026968+0000","last_clean":"2023-09-23T17:54:57.026968+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T17:54:57.026968+0000","last_undegraded":"2023-09-23T17:54:57.026968+0000","last_fullsized":"2023-09-23T17:54:57.026968+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039608+0000","last_change":"2023-09-23T17:54:57.036095+0000","last_active":"2023-09-23T17:54:57.039608+0000","last_peered":"2023-09-23T17:54:57.039608+0000","last_clean":"2023-09-23T17:54:57.039608+0000","last_became_active":"2023-09-23T17:54:57.035606+0000","last_became_peered":"2023-09-23T17:54:57.035606+0000","last_unstale":"2023-09-23T17:54:57.039608+0000","last_undegraded":"2023-09-23T17:54:57.039608+0000","last_fullsized":"2023-09-23T17:54:57.039608+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:10.936020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":307,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025278+0000","last_change":"2023-09-23T17:53:52.734324+0000","last_active":"2023-09-23T17:54:56.025278+0000","last_peered":"2023-09-23T17:54:56.025278+0000","last_clean":"2023-09-23T17:54:56.025278+0000","last_became_active":"2023-09-23T17:53:52.733825+0000","last_became_peered":"2023-09-23T17:53:52.733825+0000","last_unstale":"2023-09-23T17:54:56.025278+0000","last_undegraded":"2023-09-23T17:54:56.025278+0000","last_fullsized":"2023-09-23T17:54:56.025278+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:34.742687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.027204+0000","last_change":"2023-09-23T17:45:51.297621+0000","last_active":"2023-09-23T17:54:56.027204+0000","last_peered":"2023-09-23T17:54:56.027204+0000","last_clean":"2023-09-23T17:54:56.027204+0000","last_became_active":"2023-09-23T17:45:51.297322+0000","last_became_peered":"2023-09-23T17:45:51.297322+0000","last_unstale":"2023-09-23T17:54:56.027204+0000","last_undegraded":"2023-09-23T17:54:56.027204+0000","last_fullsized":"2023-09-23T17:54:56.027204+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:14.268027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036074+0000","last_change":"2023-09-23T17:54:57.036074+0000","last_active":"2023-09-23T17:54:57.036074+0000","last_peered":"2023-09-23T17:54:57.036074+0000","last_clean":"2023-09-23T17:54:57.036074+0000","last_became_active":"2023-09-23T17:54:57.035801+0000","last_became_peered":"2023-09-23T17:54:57.035801+0000","last_unstale":"2023-09-23T17:54:57.036074+0000","last_undegraded":"2023-09-23T17:54:57.036074+0000","last_fullsized":"2023-09-23T17:54:57.036074+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:11.087766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":452,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025187+0000","last_change":"2023-09-23T17:53:21.035503+0000","last_active":"2023-09-23T17:54:56.025187+0000","last_peered":"2023-09-23T17:54:56.025187+0000","last_clean":"2023-09-23T17:54:56.025187+0000","last_became_active":"2023-09-23T17:53:21.035119+0000","last_became_peered":"2023-09-23T17:53:21.035119+0000","last_unstale":"2023-09-23T17:54:56.025187+0000","last_undegraded":"2023-09-23T17:54:56.025187+0000","last_fullsized":"2023-09-23T17:54:56.025187+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:32.369675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1023,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029395+0000","last_change":"2023-09-23T17:54:25.601298+0000","last_active":"2023-09-23T17:54:56.029395+0000","last_peered":"2023-09-23T17:54:56.029395+0000","last_clean":"2023-09-23T17:54:56.029395+0000","last_became_active":"2023-09-23T17:54:25.600854+0000","last_became_peered":"2023-09-23T17:54:25.600854+0000","last_unstale":"2023-09-23T17:54:56.029395+0000","last_undegraded":"2023-09-23T17:54:56.029395+0000","last_fullsized":"2023-09-23T17:54:56.029395+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:56.978955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028625+0000","last_change":"2023-09-23T17:53:52.728795+0000","last_active":"2023-09-23T17:54:56.028625+0000","last_peered":"2023-09-23T17:54:56.028625+0000","last_clean":"2023-09-23T17:54:56.028625+0000","last_became_active":"2023-09-23T17:53:52.728423+0000","last_became_peered":"2023-09-23T17:53:52.728423+0000","last_unstale":"2023-09-23T17:54:56.028625+0000","last_undegraded":"2023-09-23T17:54:56.028625+0000","last_fullsized":"2023-09-23T17:54:56.028625+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.759322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033868+0000","last_change":"2023-09-23T17:54:25.603824+0000","last_active":"2023-09-23T17:54:56.033868+0000","last_peered":"2023-09-23T17:54:56.033868+0000","last_clean":"2023-09-23T17:54:56.033868+0000","last_became_active":"2023-09-23T17:54:25.603526+0000","last_became_peered":"2023-09-23T17:54:25.603526+0000","last_unstale":"2023-09-23T17:54:56.033868+0000","last_undegraded":"2023-09-23T17:54:56.033868+0000","last_fullsized":"2023-09-23T17:54:56.033868+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:29.842346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":372,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.037634+0000","last_change":"2023-09-23T17:54:57.037634+0000","last_active":"2023-09-23T17:54:57.037634+0000","last_peered":"2023-09-23T17:54:57.037634+0000","last_clean":"2023-09-23T17:54:57.037634+0000","last_became_active":"2023-09-23T17:54:57.037115+0000","last_became_peered":"2023-09-23T17:54:57.037115+0000","last_unstale":"2023-09-23T17:54:57.037634+0000","last_undegraded":"2023-09-23T17:54:57.037634+0000","last_fullsized":"2023-09-23T17:54:57.037634+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:52:58.348982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035632+0000","last_change":"2023-09-23T17:54:57.035632+0000","last_active":"2023-09-23T17:54:57.035632+0000","last_peered":"2023-09-23T17:54:57.035632+0000","last_clean":"2023-09-23T17:54:57.035632+0000","last_became_active":"2023-09-23T17:54:57.035338+0000","last_became_peered":"2023-09-23T17:54:57.035338+0000","last_unstale":"2023-09-23T17:54:57.035632+0000","last_undegraded":"2023-09-23T17:54:57.035632+0000","last_fullsized":"2023-09-23T17:54:57.035632+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:18.773130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022511+0000","last_change":"2023-09-23T17:53:21.036597+0000","last_active":"2023-09-23T17:54:56.022511+0000","last_peered":"2023-09-23T17:54:56.022511+0000","last_clean":"2023-09-23T17:54:56.022511+0000","last_became_active":"2023-09-23T17:53:21.036282+0000","last_became_peered":"2023-09-23T17:53:21.036282+0000","last_unstale":"2023-09-23T17:54:56.022511+0000","last_undegraded":"2023-09-23T17:54:56.022511+0000","last_fullsized":"2023-09-23T17:54:56.022511+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:28.486975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1153,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029744+0000","last_change":"2023-09-23T17:53:52.733510+0000","last_active":"2023-09-23T17:54:56.029744+0000","last_peered":"2023-09-23T17:54:56.029744+0000","last_clean":"2023-09-23T17:54:56.029744+0000","last_became_active":"2023-09-23T17:53:52.733030+0000","last_became_peered":"2023-09-23T17:53:52.733030+0000","last_unstale":"2023-09-23T17:54:56.029744+0000","last_undegraded":"2023-09-23T17:54:56.029744+0000","last_fullsized":"2023-09-23T17:54:56.029744+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:59.009283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1894,"num_read_kb":2706,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029081+0000","last_change":"2023-09-23T17:53:52.735673+0000","last_active":"2023-09-23T17:54:56.029081+0000","last_peered":"2023-09-23T17:54:56.029081+0000","last_clean":"2023-09-23T17:54:56.029081+0000","last_became_active":"2023-09-23T17:53:52.734890+0000","last_became_peered":"2023-09-23T17:53:52.734890+0000","last_unstale":"2023-09-23T17:54:56.029081+0000","last_undegraded":"2023-09-23T17:54:56.029081+0000","last_fullsized":"2023-09-23T17:54:56.029081+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:22.338546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040788+0000","last_change":"2023-09-23T17:54:57.040788+0000","last_active":"2023-09-23T17:54:57.040788+0000","last_peered":"2023-09-23T17:54:57.040788+0000","last_clean":"2023-09-23T17:54:57.040788+0000","last_became_active":"2023-09-23T17:54:57.040271+0000","last_became_peered":"2023-09-23T17:54:57.040271+0000","last_unstale":"2023-09-23T17:54:57.040788+0000","last_undegraded":"2023-09-23T17:54:57.040788+0000","last_fullsized":"2023-09-23T17:54:57.040788+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:37.176032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":528,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033496+0000","last_change":"2023-09-23T17:53:52.730761+0000","last_active":"2023-09-23T17:54:56.033496+0000","last_peered":"2023-09-23T17:54:56.033496+0000","last_clean":"2023-09-23T17:54:56.033496+0000","last_became_active":"2023-09-23T17:53:52.730121+0000","last_became_peered":"2023-09-23T17:53:52.730121+0000","last_unstale":"2023-09-23T17:54:56.033496+0000","last_undegraded":"2023-09-23T17:54:56.033496+0000","last_fullsized":"2023-09-23T17:54:56.033496+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:02.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035100+0000","last_change":"2023-09-23T17:54:57.035100+0000","last_active":"2023-09-23T17:54:57.035100+0000","last_peered":"2023-09-23T17:54:57.035100+0000","last_clean":"2023-09-23T17:54:57.035100+0000","last_became_active":"2023-09-23T17:54:57.034733+0000","last_became_peered":"2023-09-23T17:54:57.034733+0000","last_unstale":"2023-09-23T17:54:57.035100+0000","last_undegraded":"2023-09-23T17:54:57.035100+0000","last_fullsized":"2023-09-23T17:54:57.035100+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:27.841590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038911+0000","last_change":"2023-09-23T17:54:57.038911+0000","last_active":"2023-09-23T17:54:57.038911+0000","last_peered":"2023-09-23T17:54:57.038911+0000","last_clean":"2023-09-23T17:54:57.038911+0000","last_became_active":"2023-09-23T17:54:57.038607+0000","last_became_peered":"2023-09-23T17:54:57.038607+0000","last_unstale":"2023-09-23T17:54:57.038911+0000","last_undegraded":"2023-09-23T17:54:57.038911+0000","last_fullsized":"2023-09-23T17:54:57.038911+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:20.373213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021390+0000","last_change":"2023-09-23T17:53:52.736573+0000","last_active":"2023-09-23T17:54:56.021390+0000","last_peered":"2023-09-23T17:54:56.021390+0000","last_clean":"2023-09-23T17:54:56.021390+0000","last_became_active":"2023-09-23T17:53:52.736320+0000","last_became_peered":"2023-09-23T17:53:52.736320+0000","last_unstale":"2023-09-23T17:54:56.021390+0000","last_undegraded":"2023-09-23T17:54:56.021390+0000","last_fullsized":"2023-09-23T17:54:56.021390+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:05.760412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025639+0000","last_change":"2023-09-23T17:54:25.602420+0000","last_active":"2023-09-23T17:54:56.025639+0000","last_peered":"2023-09-23T17:54:56.025639+0000","last_clean":"2023-09-23T17:54:56.025639+0000","last_became_active":"2023-09-23T17:54:25.601988+0000","last_became_peered":"2023-09-23T17:54:25.601988+0000","last_unstale":"2023-09-23T17:54:56.025639+0000","last_undegraded":"2023-09-23T17:54:56.025639+0000","last_fullsized":"2023-09-23T17:54:56.025639+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:17.703211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":79,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022486+0000","last_change":"2023-09-23T17:54:25.603149+0000","last_active":"2023-09-23T17:54:56.022486+0000","last_peered":"2023-09-23T17:54:56.022486+0000","last_clean":"2023-09-23T17:54:56.022486+0000","last_became_active":"2023-09-23T17:54:25.602846+0000","last_became_peered":"2023-09-23T17:54:25.602846+0000","last_unstale":"2023-09-23T17:54:56.022486+0000","last_undegraded":"2023-09-23T17:54:56.022486+0000","last_fullsized":"2023-09-23T17:54:56.022486+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:06.955023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029512+0000","last_change":"2023-09-23T17:53:52.732276+0000","last_active":"2023-09-23T17:54:56.029512+0000","last_peered":"2023-09-23T17:54:56.029512+0000","last_clean":"2023-09-23T17:54:56.029512+0000","last_became_active":"2023-09-23T17:53:52.731952+0000","last_became_peered":"2023-09-23T17:53:52.731952+0000","last_unstale":"2023-09-23T17:54:56.029512+0000","last_undegraded":"2023-09-23T17:54:56.029512+0000","last_fullsized":"2023-09-23T17:54:56.029512+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:06:07.529125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036113+0000","last_change":"2023-09-23T17:54:57.036113+0000","last_active":"2023-09-23T17:54:57.036113+0000","last_peered":"2023-09-23T17:54:57.036113+0000","last_clean":"2023-09-23T17:54:57.036113+0000","last_became_active":"2023-09-23T17:54:57.035704+0000","last_became_peered":"2023-09-23T17:54:57.035704+0000","last_unstale":"2023-09-23T17:54:57.036113+0000","last_undegraded":"2023-09-23T17:54:57.036113+0000","last_fullsized":"2023-09-23T17:54:57.036113+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:10.397866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035203+0000","last_change":"2023-09-23T17:54:57.035203+0000","last_active":"2023-09-23T17:54:57.035203+0000","last_peered":"2023-09-23T17:54:57.035203+0000","last_clean":"2023-09-23T17:54:57.035203+0000","last_became_active":"2023-09-23T17:54:57.034854+0000","last_became_peered":"2023-09-23T17:54:57.034854+0000","last_unstale":"2023-09-23T17:54:57.035203+0000","last_undegraded":"2023-09-23T17:54:57.035203+0000","last_fullsized":"2023-09-23T17:54:57.035203+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:34.657128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026231+0000","last_change":"2023-09-23T17:53:52.731322+0000","last_active":"2023-09-23T17:54:57.026231+0000","last_peered":"2023-09-23T17:54:57.026231+0000","last_clean":"2023-09-23T17:54:57.026231+0000","last_became_active":"2023-09-23T17:53:52.730995+0000","last_became_peered":"2023-09-23T17:53:52.730995+0000","last_unstale":"2023-09-23T17:54:57.026231+0000","last_undegraded":"2023-09-23T17:54:57.026231+0000","last_fullsized":"2023-09-23T17:54:57.026231+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:55.121531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040294+0000","last_change":"2023-09-23T17:54:57.040294+0000","last_active":"2023-09-23T17:54:57.040294+0000","last_peered":"2023-09-23T17:54:57.040294+0000","last_clean":"2023-09-23T17:54:57.040294+0000","last_became_active":"2023-09-23T17:54:57.039943+0000","last_became_peered":"2023-09-23T17:54:57.039943+0000","last_unstale":"2023-09-23T17:54:57.040294+0000","last_undegraded":"2023-09-23T17:54:57.040294+0000","last_fullsized":"2023-09-23T17:54:57.040294+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:38.474814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":80,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029374+0000","last_change":"2023-09-23T17:54:25.601375+0000","last_active":"2023-09-23T17:54:56.029374+0000","last_peered":"2023-09-23T17:54:56.029374+0000","last_clean":"2023-09-23T17:54:56.029374+0000","last_became_active":"2023-09-23T17:54:25.601051+0000","last_became_peered":"2023-09-23T17:54:25.601051+0000","last_unstale":"2023-09-23T17:54:56.029374+0000","last_undegraded":"2023-09-23T17:54:56.029374+0000","last_fullsized":"2023-09-23T17:54:56.029374+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:24.772636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028296+0000","last_change":"2023-09-23T17:45:51.129514+0000","last_active":"2023-09-23T17:54:56.028296+0000","last_peered":"2023-09-23T17:54:56.028296+0000","last_clean":"2023-09-23T17:54:56.028296+0000","last_became_active":"2023-09-23T17:45:51.129154+0000","last_became_peered":"2023-09-23T17:45:51.129154+0000","last_unstale":"2023-09-23T17:54:56.028296+0000","last_undegraded":"2023-09-23T17:54:56.028296+0000","last_fullsized":"2023-09-23T17:54:56.028296+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:58.292683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040014+0000","last_change":"2023-09-23T17:54:57.040014+0000","last_active":"2023-09-23T17:54:57.040014+0000","last_peered":"2023-09-23T17:54:57.040014+0000","last_clean":"2023-09-23T17:54:57.040014+0000","last_became_active":"2023-09-23T17:54:57.039661+0000","last_became_peered":"2023-09-23T17:54:57.039661+0000","last_unstale":"2023-09-23T17:54:57.040014+0000","last_undegraded":"2023-09-23T17:54:57.040014+0000","last_fullsized":"2023-09-23T17:54:57.040014+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:01.077579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026256+0000","last_change":"2023-09-23T17:53:52.732018+0000","last_active":"2023-09-23T17:54:56.026256+0000","last_peered":"2023-09-23T17:54:56.026256+0000","last_clean":"2023-09-23T17:54:56.026256+0000","last_became_active":"2023-09-23T17:53:52.731641+0000","last_became_peered":"2023-09-23T17:53:52.731641+0000","last_unstale":"2023-09-23T17:54:56.026256+0000","last_undegraded":"2023-09-23T17:54:56.026256+0000","last_fullsized":"2023-09-23T17:54:56.026256+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:37.290885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029040+0000","last_change":"2023-09-23T17:45:51.297370+0000","last_active":"2023-09-23T17:54:56.029040+0000","last_peered":"2023-09-23T17:54:56.029040+0000","last_clean":"2023-09-23T17:54:56.029040+0000","last_became_active":"2023-09-23T17:45:51.297055+0000","last_became_peered":"2023-09-23T17:45:51.297055+0000","last_unstale":"2023-09-23T17:54:56.029040+0000","last_undegraded":"2023-09-23T17:54:56.029040+0000","last_fullsized":"2023-09-23T17:54:56.029040+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:52.209267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026812+0000","last_change":"2023-09-23T17:53:21.041393+0000","last_active":"2023-09-23T17:54:56.026812+0000","last_peered":"2023-09-23T17:54:56.026812+0000","last_clean":"2023-09-23T17:54:56.026812+0000","last_became_active":"2023-09-23T17:53:21.040995+0000","last_became_peered":"2023-09-23T17:53:21.040995+0000","last_unstale":"2023-09-23T17:54:56.026812+0000","last_undegraded":"2023-09-23T17:54:56.026812+0000","last_fullsized":"2023-09-23T17:54:56.026812+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:12.660494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041584+0000","last_change":"2023-09-23T17:54:57.041584+0000","last_active":"2023-09-23T17:54:57.041584+0000","last_peered":"2023-09-23T17:54:57.041584+0000","last_clean":"2023-09-23T17:54:57.041584+0000","last_became_active":"2023-09-23T17:54:57.040808+0000","last_became_peered":"2023-09-23T17:54:57.040808+0000","last_unstale":"2023-09-23T17:54:57.041584+0000","last_undegraded":"2023-09-23T17:54:57.041584+0000","last_fullsized":"2023-09-23T17:54:57.041584+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:01.079083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":79,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026234+0000","last_change":"2023-09-23T17:53:21.034197+0000","last_active":"2023-09-23T17:54:56.026234+0000","last_peered":"2023-09-23T17:54:56.026234+0000","last_clean":"2023-09-23T17:54:56.026234+0000","last_became_active":"2023-09-23T17:53:21.033784+0000","last_became_peered":"2023-09-23T17:53:21.033784+0000","last_unstale":"2023-09-23T17:54:56.026234+0000","last_undegraded":"2023-09-23T17:54:56.026234+0000","last_fullsized":"2023-09-23T17:54:56.026234+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:33.645714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039517+0000","last_change":"2023-09-23T17:54:57.039517+0000","last_active":"2023-09-23T17:54:57.039517+0000","last_peered":"2023-09-23T17:54:57.039517+0000","last_clean":"2023-09-23T17:54:57.039517+0000","last_became_active":"2023-09-23T17:54:57.039236+0000","last_became_peered":"2023-09-23T17:54:57.039236+0000","last_unstale":"2023-09-23T17:54:57.039517+0000","last_undegraded":"2023-09-23T17:54:57.039517+0000","last_fullsized":"2023-09-23T17:54:57.039517+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.126485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025164+0000","last_change":"2023-09-23T17:45:51.127190+0000","last_active":"2023-09-23T17:54:56.025164+0000","last_peered":"2023-09-23T17:54:56.025164+0000","last_clean":"2023-09-23T17:54:56.025164+0000","last_became_active":"2023-09-23T17:45:51.126903+0000","last_became_peered":"2023-09-23T17:45:51.126903+0000","last_unstale":"2023-09-23T17:54:56.025164+0000","last_undegraded":"2023-09-23T17:54:56.025164+0000","last_fullsized":"2023-09-23T17:54:56.025164+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:55.349259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.042234+0000","last_change":"2023-09-23T17:54:57.042234+0000","last_active":"2023-09-23T17:54:57.042234+0000","last_peered":"2023-09-23T17:54:57.042234+0000","last_clean":"2023-09-23T17:54:57.042234+0000","last_became_active":"2023-09-23T17:54:57.041918+0000","last_became_peered":"2023-09-23T17:54:57.041918+0000","last_unstale":"2023-09-23T17:54:57.042234+0000","last_undegraded":"2023-09-23T17:54:57.042234+0000","last_fullsized":"2023-09-23T17:54:57.042234+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:44.128871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":50,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038461+0000","last_change":"2023-09-23T17:54:57.038461+0000","last_active":"2023-09-23T17:54:57.038461+0000","last_peered":"2023-09-23T17:54:57.038461+0000","last_clean":"2023-09-23T17:54:57.038461+0000","last_became_active":"2023-09-23T17:54:57.038210+0000","last_became_peered":"2023-09-23T17:54:57.038210+0000","last_unstale":"2023-09-23T17:54:57.038461+0000","last_undegraded":"2023-09-23T17:54:57.038461+0000","last_fullsized":"2023-09-23T17:54:57.038461+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:50.077733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040422+0000","last_change":"2023-09-23T17:54:57.040422+0000","last_active":"2023-09-23T17:54:57.040422+0000","last_peered":"2023-09-23T17:54:57.040422+0000","last_clean":"2023-09-23T17:54:57.040422+0000","last_became_active":"2023-09-23T17:54:57.040085+0000","last_became_peered":"2023-09-23T17:54:57.040085+0000","last_unstale":"2023-09-23T17:54:57.040422+0000","last_undegraded":"2023-09-23T17:54:57.040422+0000","last_fullsized":"2023-09-23T17:54:57.040422+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:47.846454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025676+0000","last_change":"2023-09-23T17:54:25.602669+0000","last_active":"2023-09-23T17:54:56.025676+0000","last_peered":"2023-09-23T17:54:56.025676+0000","last_clean":"2023-09-23T17:54:56.025676+0000","last_became_active":"2023-09-23T17:54:25.602338+0000","last_became_peered":"2023-09-23T17:54:25.602338+0000","last_unstale":"2023-09-23T17:54:56.025676+0000","last_undegraded":"2023-09-23T17:54:56.025676+0000","last_fullsized":"2023-09-23T17:54:56.025676+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:02.816165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022313+0000","last_change":"2023-09-23T17:53:52.737008+0000","last_active":"2023-09-23T17:54:56.022313+0000","last_peered":"2023-09-23T17:54:56.022313+0000","last_clean":"2023-09-23T17:54:56.022313+0000","last_became_active":"2023-09-23T17:53:52.736702+0000","last_became_peered":"2023-09-23T17:53:52.736702+0000","last_unstale":"2023-09-23T17:54:56.022313+0000","last_undegraded":"2023-09-23T17:54:56.022313+0000","last_fullsized":"2023-09-23T17:54:56.022313+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:14.384950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021344+0000","last_change":"2023-09-23T17:45:51.124742+0000","last_active":"2023-09-23T17:54:56.021344+0000","last_peered":"2023-09-23T17:54:56.021344+0000","last_clean":"2023-09-23T17:54:56.021344+0000","last_became_active":"2023-09-23T17:45:51.124342+0000","last_became_peered":"2023-09-23T17:45:51.124342+0000","last_unstale":"2023-09-23T17:54:56.021344+0000","last_undegraded":"2023-09-23T17:54:56.021344+0000","last_fullsized":"2023-09-23T17:54:56.021344+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:24.351549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.032458+0000","last_change":"2023-09-23T17:54:25.603148+0000","last_active":"2023-09-23T17:54:56.032458+0000","last_peered":"2023-09-23T17:54:56.032458+0000","last_clean":"2023-09-23T17:54:56.032458+0000","last_became_active":"2023-09-23T17:54:25.602859+0000","last_became_peered":"2023-09-23T17:54:25.602859+0000","last_unstale":"2023-09-23T17:54:56.032458+0000","last_undegraded":"2023-09-23T17:54:56.032458+0000","last_fullsized":"2023-09-23T17:54:56.032458+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:19.802583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.043487+0000","last_change":"2023-09-23T17:54:57.043487+0000","last_active":"2023-09-23T17:54:57.043487+0000","last_peered":"2023-09-23T17:54:57.043487+0000","last_clean":"2023-09-23T17:54:57.043487+0000","last_became_active":"2023-09-23T17:54:57.043064+0000","last_became_peered":"2023-09-23T17:54:57.043064+0000","last_unstale":"2023-09-23T17:54:57.043487+0000","last_undegraded":"2023-09-23T17:54:57.043487+0000","last_fullsized":"2023-09-23T17:54:57.043487+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:34:21.634412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038085+0000","last_change":"2023-09-23T17:54:57.038085+0000","last_active":"2023-09-23T17:54:57.038085+0000","last_peered":"2023-09-23T17:54:57.038085+0000","last_clean":"2023-09-23T17:54:57.038085+0000","last_became_active":"2023-09-23T17:54:57.037551+0000","last_became_peered":"2023-09-23T17:54:57.037551+0000","last_unstale":"2023-09-23T17:54:57.038085+0000","last_undegraded":"2023-09-23T17:54:57.038085+0000","last_fullsized":"2023-09-23T17:54:57.038085+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:20:29.582747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029706+0000","last_change":"2023-09-23T17:53:52.732689+0000","last_active":"2023-09-23T17:54:56.029706+0000","last_peered":"2023-09-23T17:54:56.029706+0000","last_clean":"2023-09-23T17:54:56.029706+0000","last_became_active":"2023-09-23T17:53:52.732382+0000","last_became_peered":"2023-09-23T17:53:52.732382+0000","last_unstale":"2023-09-23T17:54:56.029706+0000","last_undegraded":"2023-09-23T17:54:56.029706+0000","last_fullsized":"2023-09-23T17:54:56.029706+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:34.408802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034454+0000","last_change":"2023-09-23T17:54:57.034454+0000","last_active":"2023-09-23T17:54:57.034454+0000","last_peered":"2023-09-23T17:54:57.034454+0000","last_clean":"2023-09-23T17:54:57.034454+0000","last_became_active":"2023-09-23T17:54:57.034128+0000","last_became_peered":"2023-09-23T17:54:57.034128+0000","last_unstale":"2023-09-23T17:54:57.034454+0000","last_undegraded":"2023-09-23T17:54:57.034454+0000","last_fullsized":"2023-09-23T17:54:57.034454+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:39.315294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":50,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036930+0000","last_change":"2023-09-23T17:54:57.036930+0000","last_active":"2023-09-23T17:54:57.036930+0000","last_peered":"2023-09-23T17:54:57.036930+0000","last_clean":"2023-09-23T17:54:57.036930+0000","last_became_active":"2023-09-23T17:54:57.036631+0000","last_became_peered":"2023-09-23T17:54:57.036631+0000","last_unstale":"2023-09-23T17:54:57.036930+0000","last_undegraded":"2023-09-23T17:54:57.036930+0000","last_fullsized":"2023-09-23T17:54:57.036930+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:08.367357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.024125+0000","last_change":"2023-09-23T17:53:52.732305+0000","last_active":"2023-09-23T17:54:56.024125+0000","last_peered":"2023-09-23T17:54:56.024125+0000","last_clean":"2023-09-23T17:54:56.024125+0000","last_became_active":"2023-09-23T17:53:52.731894+0000","last_became_peered":"2023-09-23T17:53:52.731894+0000","last_unstale":"2023-09-23T17:54:56.024125+0000","last_undegraded":"2023-09-23T17:54:56.024125+0000","last_fullsized":"2023-09-23T17:54:56.024125+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:30.224762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.031226+0000","last_change":"2023-09-23T17:54:57.031226+0000","last_active":"2023-09-23T17:54:57.031226+0000","last_peered":"2023-09-23T17:54:57.031226+0000","last_clean":"2023-09-23T17:54:57.031226+0000","last_became_active":"2023-09-23T17:54:57.030880+0000","last_became_peered":"2023-09-23T17:54:57.030880+0000","last_unstale":"2023-09-23T17:54:57.031226+0000","last_undegraded":"2023-09-23T17:54:57.031226+0000","last_fullsized":"2023-09-23T17:54:57.031226+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:49.836129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040141+0000","last_change":"2023-09-23T17:54:57.039438+0000","last_active":"2023-09-23T17:54:57.040141+0000","last_peered":"2023-09-23T17:54:57.040141+0000","last_clean":"2023-09-23T17:54:57.040141+0000","last_became_active":"2023-09-23T17:54:57.039163+0000","last_became_peered":"2023-09-23T17:54:57.039163+0000","last_unstale":"2023-09-23T17:54:57.040141+0000","last_undegraded":"2023-09-23T17:54:57.040141+0000","last_fullsized":"2023-09-23T17:54:57.040141+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.265833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034381+0000","last_change":"2023-09-23T17:54:57.034381+0000","last_active":"2023-09-23T17:54:57.034381+0000","last_peered":"2023-09-23T17:54:57.034381+0000","last_clean":"2023-09-23T17:54:57.034381+0000","last_became_active":"2023-09-23T17:54:57.034043+0000","last_became_peered":"2023-09-23T17:54:57.034043+0000","last_unstale":"2023-09-23T17:54:57.034381+0000","last_undegraded":"2023-09-23T17:54:57.034381+0000","last_fullsized":"2023-09-23T17:54:57.034381+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:40.217762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036156+0000","last_change":"2023-09-23T17:54:57.036156+0000","last_active":"2023-09-23T17:54:57.036156+0000","last_peered":"2023-09-23T17:54:57.036156+0000","last_clean":"2023-09-23T17:54:57.036156+0000","last_became_active":"2023-09-23T17:54:57.035478+0000","last_became_peered":"2023-09-23T17:54:57.035478+0000","last_unstale":"2023-09-23T17:54:57.036156+0000","last_undegraded":"2023-09-23T17:54:57.036156+0000","last_fullsized":"2023-09-23T17:54:57.036156+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:53.126638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":88,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021007+0000","last_change":"2023-09-23T17:54:25.605945+0000","last_active":"2023-09-23T17:54:56.021007+0000","last_peered":"2023-09-23T17:54:56.021007+0000","last_clean":"2023-09-23T17:54:56.021007+0000","last_became_active":"2023-09-23T17:54:25.605583+0000","last_became_peered":"2023-09-23T17:54:25.605583+0000","last_unstale":"2023-09-23T17:54:56.021007+0000","last_undegraded":"2023-09-23T17:54:56.021007+0000","last_fullsized":"2023-09-23T17:54:56.021007+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:14.076789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266821,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":13024,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":11328,"kb_avail":93732128,"statfs":{"total":95995035648,"available":95981699072,"internally_reserved":0,"allocated":1728512,"data_stored":1206046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11599872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55500000000000005,"15min":0.55200000000000005},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81200000000000006,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.48899999999999999},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57199999999999995,"15min":0.56899999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.56799999999999995}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.52100000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 17:54:01 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.58099999999999996,"15min":0.54400000000000004},"min":{"1min":0.436,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.79400000000000004,"5min":1.0409999999999999,"15min":1.9370000000000001},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.60399999999999998,"15min":0.63200000000000001},"min":{"1min":0.42399999999999999,"5min":0.36799999999999999,"15min":0.35899999999999999},"max":{"1min":0.75800000000000001,"5min":0.91300000000000003,"15min":8.8499999999999996},"last":0.57899999999999996}]},{"osd":4,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.49399999999999999,"15min":0.44600000000000001},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52300000000000002,"15min":0.47499999999999998},"min":{"1min":0.34699999999999998,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.438}]},{"osd":5,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53000000000000003,"15min":0.48999999999999999},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.28699999999999998},"max":{"1min":0.72799999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.57999999999999996,"15min":0.51400000000000001},"min":{"1min":0.40100000000000002,"5min":0.30099999999999999,"15min":0.29099999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.66200000000000003}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.55300000000000005}]}]},{"osd":6,"up_from":45,"seq":193273528457,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8344,"kb_used_data":1680,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93736808,"statfs":{"total":95995035648,"available":95986491392,"internally_reserved":0,"allocated":1720320,"data_stored":1205919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.81100000000000005,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.78300000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.56399999999999995}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.57899999999999996}]},{"osd":2,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.61199999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.38800000000000001,"5min":0.36799999999999999,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":0.80700000000000005,"15min":4.5599999999999996},"last":0.502},{"interface":"front","average":{"1min":0.625,"5min":0.58499999999999996,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34999999999999998,"15min":0.30399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":1.6850000000000001},"last":0.54500000000000004}]},{"osd":4,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.47099999999999997,"15min":0.45200000000000001},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":0.748,"5min":0.76400000000000001,"15min":0.82899999999999996},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52800000000000002,"15min":0.49199999999999999},"min":{"1min":0.34999999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.53000000000000003}]},{"osd":5,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.53400000000000003,"15min":0.496},"min":{"1min":0.33700000000000002,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":0.77700000000000002,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56599999999999995,"15min":0.52200000000000002},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.33200000000000002},"max":{"1min":0.79700000000000004,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.59499999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:55:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.58999999999999997,"15min":0.53700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":0.83299999999999996,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.57499999999999996,"15min":0.51900000000000002},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":0.81299999999999994,"5min":0.82299999999999995,"15min":1.591},"last":0.63100000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795030,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28716,"kb_used_data":1124,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93716436,"statfs":{"total":95995035648,"available":95965630464,"internally_reserved":0,"allocated":1150976,"data_stored":617993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":28238525},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.41899999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.976,"5min":1.976,"15min":1.976},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.40699999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.313}]},{"osd":4,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.83399999999999996},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.79600000000000004}]},{"osd":5,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1,"5min":1,"15min":1},"last":0.80700000000000005}]},{"osd":6,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.81799999999999995},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.68700000000000006}]},{"osd":7,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.67500000000000004}]}]},{"osd":0,"up_from":93,"seq":399431958557,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33408,"kb_used_data":5816,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93711744,"statfs":{"total":95995035648,"available":95960825856,"internally_reserved":0,"allocated":5955584,"data_stored":5404139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":28238564},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:54:51 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.374}]},{"osd":2,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"last":0.33200000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.39400000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82199999999999995,"5min":0.872,"15min":0.872},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61699999999999999,"15min":0.621},"min":{"1min":0.44700000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77700000000000002,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.624,"15min":0.63},"min":{"1min":0.41799999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.73599999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.63900000000000001,"15min":0.64300000000000002},"min":{"1min":0.46600000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.53000000000000003}]},{"osd":6,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.42899999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.504},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65200000000000002},"min":{"1min":0.52700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.81100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65900000000000003,"15min":0.66300000000000003},"min":{"1min":0.51600000000000001,"5min":0.435,"15min":0.435},"max":{"1min":0.84299999999999997,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.66600000000000004,"15min":0.67000000000000004},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86599999999999999,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.56499999999999995}]}]},{"osd":2,"up_from":103,"seq":442381631504,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32744,"kb_used_data":5216,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93712408,"statfs":{"total":95995035648,"available":95961505792,"internally_reserved":0,"allocated":5341184,"data_stored":4811538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":28168567},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.30599999999999999}]},{"osd":1,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.35099999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.624}]},{"osd":7,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.57099999999999995}]}]},{"osd":3,"up_from":108,"seq":463856467977,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32604,"kb_used_data":5204,"kb_used_omap":3,"kb_used_meta":27388,"kb_avail":93712548,"statfs":{"total":95995035648,"available":95961649152,"internally_reserved":0,"allocated":5328896,"data_stored":4809888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":28046056},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34399999999999997}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26900000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38200000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.496}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61399999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56499999999999995}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.504}]}]},{"osd":4,"up_from":30,"seq":128849019022,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7644,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6528,"kb_avail":93737508,"statfs":{"total":95995035648,"available":95987208192,"internally_reserved":0,"allocated":1134592,"data_stored":615931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6684672},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.75600000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.53800000000000003,"15min":0.52900000000000003},"min":{"1min":0.42699999999999999,"5min":0.376,"15min":0.376},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.59799999999999998}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.44900000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.495},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.623}]},{"osd":3,"last update":"Sat Sep 23 17:54:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.42599999999999999,"5min":0.34399999999999997,"15min":0.318},"max":{"1min":0.73999999999999999,"5min":0.77100000000000002,"15min":5.7119999999999997},"last":0.56799999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.59199999999999997,"15min":0.57999999999999996},"min":{"1min":0.41599999999999998,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":0.70999999999999996,"5min":0.85499999999999998,"15min":5.9429999999999996},"last":0.54800000000000004}]},{"osd":5,"last update":"Sat Sep 23 17:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48399999999999999,"15min":0.438},"min":{"1min":0.379,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":0.621,"5min":0.753,"15min":1.141},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.51700000000000002,"15min":0.44700000000000001},"min":{"1min":0.41499999999999998,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":0.749,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:54:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.54700000000000004,"15min":0.501},"min":{"1min":0.36099999999999999,"5min":0.33200000000000002,"15min":0.28899999999999998},"max":{"1min":0.68400000000000005,"5min":0.77400000000000002,"15min":1.276},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58799999999999997,"15min":0.53600000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28999999999999998},"max":{"1min":0.77100000000000002,"5min":0.80900000000000005,"15min":1.1180000000000001},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Sep 23 17:55:06 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.57699999999999996,"15min":0.54700000000000004},"min":{"1min":0.40600000000000003,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":1.1839999999999999},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.57699999999999996,"15min":0.53000000000000003},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.65200000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757388,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11724,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":10624,"kb_avail":93733428,"statfs":{"total":95995035648,"available":95983030272,"internally_reserved":0,"allocated":1118208,"data_stored":611371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":10878976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.57199999999999995,"15min":0.57599999999999996},"min":{"1min":0.39800000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.77100000000000002,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.73599999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.74199999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.63600000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.4279999999999999,"5min":4.4279999999999999,"15min":4.4279999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.53600000000000003}]},{"osd":2,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.81499999999999995}]},{"osd":3,"last update":"Sat Sep 23 17:54:27 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61199999999999999,"15min":0.58099999999999996},"min":{"1min":0.42499999999999999,"5min":0.41999999999999998,"15min":0.32100000000000001},"max":{"1min":0.76700000000000002,"5min":0.85799999999999998,"15min":4.6799999999999997},"last":0.77900000000000003},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.61199999999999999,"15min":0.57699999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.33100000000000002},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.631},"last":0.67600000000000005}]},{"osd":4,"last update":"Sat Sep 23 17:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.48899999999999999,"15min":0.44800000000000001},"min":{"1min":0.34300000000000003,"5min":0.25,"15min":0.25},"max":{"1min":0.70799999999999996,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.70799999999999996},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.54400000000000004,"15min":0.51400000000000001},"min":{"1min":0.435,"5min":0.33100000000000002,"15min":0.28899999999999998},"max":{"1min":0.81399999999999995,"5min":0.83199999999999996,"15min":1.738},"last":0.67100000000000004}]},{"osd":6,"last update":"Sat Sep 23 17:54:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.55300000000000005,"15min":0.53400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":1.778},"last":0.624},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.61599999999999999,"15min":0.58299999999999996},"min":{"1min":0.38100000000000001,"5min":0.375,"15min":0.33200000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":1.7629999999999999},"last":0.64700000000000002}]},{"osd":7,"last update":"Sat Sep 23 17:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.56299999999999994,"15min":0.52000000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.83999999999999997,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.83999999999999997},{"interface":"front","average":{"1min":0.62,"5min":0.58799999999999997,"15min":0.54900000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.79600000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.76600000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:55:35.688 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T17:55:35.688 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-09-23T17:55:35.688 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-09-23T17:55:35.688 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph health --format=json 2023-09-23T17:55:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:35 smithi099 ceph-mon[167076]: pgmap v217: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:36.291 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:36.760 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:36 smithi055 ceph-mon[179668]: from='client.34707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:36.760 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:36.760 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:36 smithi055 ceph-mon[184174]: from='client.34707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:36.761 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:36.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:36 smithi099 ceph-mon[167076]: from='client.34707 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:37.646 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:55:37.647 INFO:teuthology.orchestra.run.smithi055.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-09-23T17:55:37.794 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:37 smithi055 ceph-mon[184174]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:37.794 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:37 smithi055 ceph-mon[179668]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:37.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:37 smithi099 ceph-mon[167076]: pgmap v218: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:38.049 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-09-23T17:55:38.049 INFO:teuthology.run_tasks:Running task print... 2023-09-23T17:55:38.058 INFO:teuthology.task.print:**** applying stress + thrashing to mixed-version cluster **** 2023-09-23T17:55:38.059 INFO:teuthology.run_tasks:Running task parallel... 2023-09-23T17:55:38.076 INFO:teuthology.task.parallel:starting parallel... 2023-09-23T17:55:38.076 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-09-23T17:55:38.076 INFO:teuthology.task.sequential:In sequential, running task thrashosds... 2023-09-23T17:55:38.078 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T17:55:38.078 INFO:tasks.thrashosds:overrides is {'admin_socket': {'branch': 'reef'}, 'ceph': {'conf': {'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}, 'osd': {'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}}, 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}, 'ceph-deploy': {'conf': {'client': {'log file': '/var/log/ceph/ceph-$name.$pid.log'}, 'mon': {}}}, 'install': {'ceph': {'flavor': 'default', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}}, 'selinux': {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']}, 'workunit': {'branch': 'reef', 'sha1': 'bb0daeff97f020f686596fca6eba20c310e26fb7'}} 2023-09-23T17:55:38.078 INFO:tasks.thrashosds:config is {'aggressive_pg_num_changes': False, 'chance_force_recovery': 0, 'chance_pgnum_grow': 1, 'chance_pgpnum_fix': 1, 'chance_thrash_cluster_full': 0, 'chance_thrash_pg_upmap': 0, 'chance_thrash_pg_upmap_items': 0, 'disable_objectstore_tool_tests': True, 'timeout': 1200, 'sighup_delay': 0.1, 'optrack_toggle_delay': 2.0, 'dump_ops_enable': 'true', 'noscrub_toggle_delay': 2.0, 'random_eio': 0.0} 2023-09-23T17:55:38.078 INFO:tasks.thrashosds:Beginning thrashosds... 2023-09-23T17:55:38.078 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-09-23T17:55:38.079 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:55:38.682 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:38.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/900637964' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T17:55:38.874 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/900637964' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T17:55:38.874 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/900637964' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-09-23T17:55:39.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:39 smithi099 ceph-mon[167076]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:39.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:39 smithi055 ceph-mon[179668]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:39.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:39 smithi055 ceph-mon[184174]: pgmap v219: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:39.939 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:55:39.941 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:55:40.328 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":219,"stamp":"2023-09-23T17:55:38.406567+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":168208,"kb_used_data":22928,"kb_used_omap":29,"kb_used_meta":145186,"kb_avail":749793008,"statfs":{"total":767960285184,"available":767788040192,"internally_reserved":0,"allocated":23478272,"data_stored":19282825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30208,"internal_metadata":148670976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002959"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.020803+0000","last_change":"2023-09-23T17:53:52.736718+0000","last_active":"2023-09-23T17:54:56.020803+0000","last_peered":"2023-09-23T17:54:56.020803+0000","last_clean":"2023-09-23T17:54:56.020803+0000","last_became_active":"2023-09-23T17:53:52.736314+0000","last_became_peered":"2023-09-23T17:53:52.736314+0000","last_unstale":"2023-09-23T17:54:56.020803+0000","last_undegraded":"2023-09-23T17:54:56.020803+0000","last_fullsized":"2023-09-23T17:54:56.020803+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:26.017288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026023+0000","last_change":"2023-09-23T17:54:25.601065+0000","last_active":"2023-09-23T17:54:57.026023+0000","last_peered":"2023-09-23T17:54:57.026023+0000","last_clean":"2023-09-23T17:54:57.026023+0000","last_became_active":"2023-09-23T17:54:25.600727+0000","last_became_peered":"2023-09-23T17:54:25.600727+0000","last_unstale":"2023-09-23T17:54:57.026023+0000","last_undegraded":"2023-09-23T17:54:57.026023+0000","last_fullsized":"2023-09-23T17:54:57.026023+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:20.919498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034722+0000","last_change":"2023-09-23T17:54:57.034722+0000","last_active":"2023-09-23T17:54:57.034722+0000","last_peered":"2023-09-23T17:54:57.034722+0000","last_clean":"2023-09-23T17:54:57.034722+0000","last_became_active":"2023-09-23T17:54:57.034289+0000","last_became_peered":"2023-09-23T17:54:57.034289+0000","last_unstale":"2023-09-23T17:54:57.034722+0000","last_undegraded":"2023-09-23T17:54:57.034722+0000","last_fullsized":"2023-09-23T17:54:57.034722+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:03.041518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.032421+0000","last_change":"2023-09-23T17:54:57.032421+0000","last_active":"2023-09-23T17:54:57.032421+0000","last_peered":"2023-09-23T17:54:57.032421+0000","last_clean":"2023-09-23T17:54:57.032421+0000","last_became_active":"2023-09-23T17:54:57.032020+0000","last_became_peered":"2023-09-23T17:54:57.032020+0000","last_unstale":"2023-09-23T17:54:57.032421+0000","last_undegraded":"2023-09-23T17:54:57.032421+0000","last_fullsized":"2023-09-23T17:54:57.032421+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:14.716268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039895+0000","last_change":"2023-09-23T17:54:57.039895+0000","last_active":"2023-09-23T17:54:57.039895+0000","last_peered":"2023-09-23T17:54:57.039895+0000","last_clean":"2023-09-23T17:54:57.039895+0000","last_became_active":"2023-09-23T17:54:57.039581+0000","last_became_peered":"2023-09-23T17:54:57.039581+0000","last_unstale":"2023-09-23T17:54:57.039895+0000","last_undegraded":"2023-09-23T17:54:57.039895+0000","last_fullsized":"2023-09-23T17:54:57.039895+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:52.622668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029112+0000","last_change":"2023-09-23T17:45:51.121328+0000","last_active":"2023-09-23T17:54:56.029112+0000","last_peered":"2023-09-23T17:54:56.029112+0000","last_clean":"2023-09-23T17:54:56.029112+0000","last_became_active":"2023-09-23T17:45:51.121034+0000","last_became_peered":"2023-09-23T17:45:51.121034+0000","last_unstale":"2023-09-23T17:54:56.029112+0000","last_undegraded":"2023-09-23T17:54:56.029112+0000","last_fullsized":"2023-09-23T17:54:56.029112+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:13.256406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041535+0000","last_change":"2023-09-23T17:54:57.041535+0000","last_active":"2023-09-23T17:54:57.041535+0000","last_peered":"2023-09-23T17:54:57.041535+0000","last_clean":"2023-09-23T17:54:57.041535+0000","last_became_active":"2023-09-23T17:54:57.040909+0000","last_became_peered":"2023-09-23T17:54:57.040909+0000","last_unstale":"2023-09-23T17:54:57.041535+0000","last_undegraded":"2023-09-23T17:54:57.041535+0000","last_fullsized":"2023-09-23T17:54:57.041535+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:57.504646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.030937+0000","last_change":"2023-09-23T17:53:52.732284+0000","last_active":"2023-09-23T17:54:56.030937+0000","last_peered":"2023-09-23T17:54:56.030937+0000","last_clean":"2023-09-23T17:54:56.030937+0000","last_became_active":"2023-09-23T17:53:52.731928+0000","last_became_peered":"2023-09-23T17:53:52.731928+0000","last_unstale":"2023-09-23T17:54:56.030937+0000","last_undegraded":"2023-09-23T17:54:56.030937+0000","last_fullsized":"2023-09-23T17:54:56.030937+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:35.936955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034729+0000","last_change":"2023-09-23T17:54:57.034729+0000","last_active":"2023-09-23T17:54:57.034729+0000","last_peered":"2023-09-23T17:54:57.034729+0000","last_clean":"2023-09-23T17:54:57.034729+0000","last_became_active":"2023-09-23T17:54:57.034381+0000","last_became_peered":"2023-09-23T17:54:57.034381+0000","last_unstale":"2023-09-23T17:54:57.034729+0000","last_undegraded":"2023-09-23T17:54:57.034729+0000","last_fullsized":"2023-09-23T17:54:57.034729+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:19.369673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":39,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026132+0000","last_change":"2023-09-23T17:53:21.039717+0000","last_active":"2023-09-23T17:54:56.026132+0000","last_peered":"2023-09-23T17:54:56.026132+0000","last_clean":"2023-09-23T17:54:56.026132+0000","last_became_active":"2023-09-23T17:53:21.038793+0000","last_became_peered":"2023-09-23T17:53:21.038793+0000","last_unstale":"2023-09-23T17:54:56.026132+0000","last_undegraded":"2023-09-23T17:54:56.026132+0000","last_fullsized":"2023-09-23T17:54:56.026132+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:02.116246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025537+0000","last_change":"2023-09-23T17:45:51.148003+0000","last_active":"2023-09-23T17:54:56.025537+0000","last_peered":"2023-09-23T17:54:56.025537+0000","last_clean":"2023-09-23T17:54:56.025537+0000","last_became_active":"2023-09-23T17:45:51.147815+0000","last_became_peered":"2023-09-23T17:45:51.147815+0000","last_unstale":"2023-09-23T17:54:56.025537+0000","last_undegraded":"2023-09-23T17:54:56.025537+0000","last_fullsized":"2023-09-23T17:54:56.025537+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:32.374670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026943+0000","last_change":"2023-09-23T17:53:52.732070+0000","last_active":"2023-09-23T17:54:57.026943+0000","last_peered":"2023-09-23T17:54:57.026943+0000","last_clean":"2023-09-23T17:54:57.026943+0000","last_became_active":"2023-09-23T17:53:52.731739+0000","last_became_peered":"2023-09-23T17:53:52.731739+0000","last_unstale":"2023-09-23T17:54:57.026943+0000","last_undegraded":"2023-09-23T17:54:57.026943+0000","last_fullsized":"2023-09-23T17:54:57.026943+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:03.726754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":333,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.024215+0000","last_change":"2023-09-23T17:54:25.600838+0000","last_active":"2023-09-23T17:54:56.024215+0000","last_peered":"2023-09-23T17:54:56.024215+0000","last_clean":"2023-09-23T17:54:56.024215+0000","last_became_active":"2023-09-23T17:54:25.600305+0000","last_became_peered":"2023-09-23T17:54:25.600305+0000","last_unstale":"2023-09-23T17:54:56.024215+0000","last_undegraded":"2023-09-23T17:54:56.024215+0000","last_fullsized":"2023-09-23T17:54:56.024215+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:36.605455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":1405,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040450+0000","last_change":"2023-09-23T17:54:57.040450+0000","last_active":"2023-09-23T17:54:57.040450+0000","last_peered":"2023-09-23T17:54:57.040450+0000","last_clean":"2023-09-23T17:54:57.040450+0000","last_became_active":"2023-09-23T17:54:57.040134+0000","last_became_peered":"2023-09-23T17:54:57.040134+0000","last_unstale":"2023-09-23T17:54:57.040450+0000","last_undegraded":"2023-09-23T17:54:57.040450+0000","last_fullsized":"2023-09-23T17:54:57.040450+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:07.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":286,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.032152+0000","last_change":"2023-09-23T17:53:52.734869+0000","last_active":"2023-09-23T17:54:56.032152+0000","last_peered":"2023-09-23T17:54:56.032152+0000","last_clean":"2023-09-23T17:54:56.032152+0000","last_became_active":"2023-09-23T17:53:52.734387+0000","last_became_peered":"2023-09-23T17:53:52.734387+0000","last_unstale":"2023-09-23T17:54:56.032152+0000","last_undegraded":"2023-09-23T17:54:56.032152+0000","last_fullsized":"2023-09-23T17:54:56.032152+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:15.397601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041796+0000","last_change":"2023-09-23T17:54:57.041796+0000","last_active":"2023-09-23T17:54:57.041796+0000","last_peered":"2023-09-23T17:54:57.041796+0000","last_clean":"2023-09-23T17:54:57.041796+0000","last_became_active":"2023-09-23T17:54:57.041245+0000","last_became_peered":"2023-09-23T17:54:57.041245+0000","last_unstale":"2023-09-23T17:54:57.041796+0000","last_undegraded":"2023-09-23T17:54:57.041796+0000","last_fullsized":"2023-09-23T17:54:57.041796+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:26.340510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026049+0000","last_change":"2023-09-23T17:53:21.039195+0000","last_active":"2023-09-23T17:54:57.026049+0000","last_peered":"2023-09-23T17:54:57.026049+0000","last_clean":"2023-09-23T17:54:57.026049+0000","last_became_active":"2023-09-23T17:53:21.038654+0000","last_became_peered":"2023-09-23T17:53:21.038654+0000","last_unstale":"2023-09-23T17:54:57.026049+0000","last_undegraded":"2023-09-23T17:54:57.026049+0000","last_fullsized":"2023-09-23T17:54:57.026049+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:34.466980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033621+0000","last_change":"2023-09-23T17:54:25.604720+0000","last_active":"2023-09-23T17:54:56.033621+0000","last_peered":"2023-09-23T17:54:56.033621+0000","last_clean":"2023-09-23T17:54:56.033621+0000","last_became_active":"2023-09-23T17:54:25.604386+0000","last_became_peered":"2023-09-23T17:54:25.604386+0000","last_unstale":"2023-09-23T17:54:56.033621+0000","last_undegraded":"2023-09-23T17:54:56.033621+0000","last_fullsized":"2023-09-23T17:54:56.033621+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:56.869273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":321,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028220+0000","last_change":"2023-09-23T17:53:21.035484+0000","last_active":"2023-09-23T17:54:56.028220+0000","last_peered":"2023-09-23T17:54:56.028220+0000","last_clean":"2023-09-23T17:54:56.028220+0000","last_became_active":"2023-09-23T17:53:21.035035+0000","last_became_peered":"2023-09-23T17:53:21.035035+0000","last_unstale":"2023-09-23T17:54:56.028220+0000","last_undegraded":"2023-09-23T17:54:56.028220+0000","last_fullsized":"2023-09-23T17:54:56.028220+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:05.145111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026968+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T17:54:57.026968+0000","last_peered":"2023-09-23T17:54:57.026968+0000","last_clean":"2023-09-23T17:54:57.026968+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T17:54:57.026968+0000","last_undegraded":"2023-09-23T17:54:57.026968+0000","last_fullsized":"2023-09-23T17:54:57.026968+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039608+0000","last_change":"2023-09-23T17:54:57.036095+0000","last_active":"2023-09-23T17:54:57.039608+0000","last_peered":"2023-09-23T17:54:57.039608+0000","last_clean":"2023-09-23T17:54:57.039608+0000","last_became_active":"2023-09-23T17:54:57.035606+0000","last_became_peered":"2023-09-23T17:54:57.035606+0000","last_unstale":"2023-09-23T17:54:57.039608+0000","last_undegraded":"2023-09-23T17:54:57.039608+0000","last_fullsized":"2023-09-23T17:54:57.039608+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:10.936020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":307,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025278+0000","last_change":"2023-09-23T17:53:52.734324+0000","last_active":"2023-09-23T17:54:56.025278+0000","last_peered":"2023-09-23T17:54:56.025278+0000","last_clean":"2023-09-23T17:54:56.025278+0000","last_became_active":"2023-09-23T17:53:52.733825+0000","last_became_peered":"2023-09-23T17:53:52.733825+0000","last_unstale":"2023-09-23T17:54:56.025278+0000","last_undegraded":"2023-09-23T17:54:56.025278+0000","last_fullsized":"2023-09-23T17:54:56.025278+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:34.742687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.027204+0000","last_change":"2023-09-23T17:45:51.297621+0000","last_active":"2023-09-23T17:54:56.027204+0000","last_peered":"2023-09-23T17:54:56.027204+0000","last_clean":"2023-09-23T17:54:56.027204+0000","last_became_active":"2023-09-23T17:45:51.297322+0000","last_became_peered":"2023-09-23T17:45:51.297322+0000","last_unstale":"2023-09-23T17:54:56.027204+0000","last_undegraded":"2023-09-23T17:54:56.027204+0000","last_fullsized":"2023-09-23T17:54:56.027204+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:14.268027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036074+0000","last_change":"2023-09-23T17:54:57.036074+0000","last_active":"2023-09-23T17:54:57.036074+0000","last_peered":"2023-09-23T17:54:57.036074+0000","last_clean":"2023-09-23T17:54:57.036074+0000","last_became_active":"2023-09-23T17:54:57.035801+0000","last_became_peered":"2023-09-23T17:54:57.035801+0000","last_unstale":"2023-09-23T17:54:57.036074+0000","last_undegraded":"2023-09-23T17:54:57.036074+0000","last_fullsized":"2023-09-23T17:54:57.036074+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:11.087766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":452,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025187+0000","last_change":"2023-09-23T17:53:21.035503+0000","last_active":"2023-09-23T17:54:56.025187+0000","last_peered":"2023-09-23T17:54:56.025187+0000","last_clean":"2023-09-23T17:54:56.025187+0000","last_became_active":"2023-09-23T17:53:21.035119+0000","last_became_peered":"2023-09-23T17:53:21.035119+0000","last_unstale":"2023-09-23T17:54:56.025187+0000","last_undegraded":"2023-09-23T17:54:56.025187+0000","last_fullsized":"2023-09-23T17:54:56.025187+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:32.369675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1023,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029395+0000","last_change":"2023-09-23T17:54:25.601298+0000","last_active":"2023-09-23T17:54:56.029395+0000","last_peered":"2023-09-23T17:54:56.029395+0000","last_clean":"2023-09-23T17:54:56.029395+0000","last_became_active":"2023-09-23T17:54:25.600854+0000","last_became_peered":"2023-09-23T17:54:25.600854+0000","last_unstale":"2023-09-23T17:54:56.029395+0000","last_undegraded":"2023-09-23T17:54:56.029395+0000","last_fullsized":"2023-09-23T17:54:56.029395+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:56.978955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028625+0000","last_change":"2023-09-23T17:53:52.728795+0000","last_active":"2023-09-23T17:54:56.028625+0000","last_peered":"2023-09-23T17:54:56.028625+0000","last_clean":"2023-09-23T17:54:56.028625+0000","last_became_active":"2023-09-23T17:53:52.728423+0000","last_became_peered":"2023-09-23T17:53:52.728423+0000","last_unstale":"2023-09-23T17:54:56.028625+0000","last_undegraded":"2023-09-23T17:54:56.028625+0000","last_fullsized":"2023-09-23T17:54:56.028625+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.759322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033868+0000","last_change":"2023-09-23T17:54:25.603824+0000","last_active":"2023-09-23T17:54:56.033868+0000","last_peered":"2023-09-23T17:54:56.033868+0000","last_clean":"2023-09-23T17:54:56.033868+0000","last_became_active":"2023-09-23T17:54:25.603526+0000","last_became_peered":"2023-09-23T17:54:25.603526+0000","last_unstale":"2023-09-23T17:54:56.033868+0000","last_undegraded":"2023-09-23T17:54:56.033868+0000","last_fullsized":"2023-09-23T17:54:56.033868+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:29.842346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":372,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.037634+0000","last_change":"2023-09-23T17:54:57.037634+0000","last_active":"2023-09-23T17:54:57.037634+0000","last_peered":"2023-09-23T17:54:57.037634+0000","last_clean":"2023-09-23T17:54:57.037634+0000","last_became_active":"2023-09-23T17:54:57.037115+0000","last_became_peered":"2023-09-23T17:54:57.037115+0000","last_unstale":"2023-09-23T17:54:57.037634+0000","last_undegraded":"2023-09-23T17:54:57.037634+0000","last_fullsized":"2023-09-23T17:54:57.037634+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:52:58.348982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035632+0000","last_change":"2023-09-23T17:54:57.035632+0000","last_active":"2023-09-23T17:54:57.035632+0000","last_peered":"2023-09-23T17:54:57.035632+0000","last_clean":"2023-09-23T17:54:57.035632+0000","last_became_active":"2023-09-23T17:54:57.035338+0000","last_became_peered":"2023-09-23T17:54:57.035338+0000","last_unstale":"2023-09-23T17:54:57.035632+0000","last_undegraded":"2023-09-23T17:54:57.035632+0000","last_fullsized":"2023-09-23T17:54:57.035632+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:18.773130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022511+0000","last_change":"2023-09-23T17:53:21.036597+0000","last_active":"2023-09-23T17:54:56.022511+0000","last_peered":"2023-09-23T17:54:56.022511+0000","last_clean":"2023-09-23T17:54:56.022511+0000","last_became_active":"2023-09-23T17:53:21.036282+0000","last_became_peered":"2023-09-23T17:53:21.036282+0000","last_unstale":"2023-09-23T17:54:56.022511+0000","last_undegraded":"2023-09-23T17:54:56.022511+0000","last_fullsized":"2023-09-23T17:54:56.022511+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:28.486975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1153,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029744+0000","last_change":"2023-09-23T17:53:52.733510+0000","last_active":"2023-09-23T17:54:56.029744+0000","last_peered":"2023-09-23T17:54:56.029744+0000","last_clean":"2023-09-23T17:54:56.029744+0000","last_became_active":"2023-09-23T17:53:52.733030+0000","last_became_peered":"2023-09-23T17:53:52.733030+0000","last_unstale":"2023-09-23T17:54:56.029744+0000","last_undegraded":"2023-09-23T17:54:56.029744+0000","last_fullsized":"2023-09-23T17:54:56.029744+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:59.009283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1894,"num_read_kb":2706,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029081+0000","last_change":"2023-09-23T17:53:52.735673+0000","last_active":"2023-09-23T17:54:56.029081+0000","last_peered":"2023-09-23T17:54:56.029081+0000","last_clean":"2023-09-23T17:54:56.029081+0000","last_became_active":"2023-09-23T17:53:52.734890+0000","last_became_peered":"2023-09-23T17:53:52.734890+0000","last_unstale":"2023-09-23T17:54:56.029081+0000","last_undegraded":"2023-09-23T17:54:56.029081+0000","last_fullsized":"2023-09-23T17:54:56.029081+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:22.338546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040788+0000","last_change":"2023-09-23T17:54:57.040788+0000","last_active":"2023-09-23T17:54:57.040788+0000","last_peered":"2023-09-23T17:54:57.040788+0000","last_clean":"2023-09-23T17:54:57.040788+0000","last_became_active":"2023-09-23T17:54:57.040271+0000","last_became_peered":"2023-09-23T17:54:57.040271+0000","last_unstale":"2023-09-23T17:54:57.040788+0000","last_undegraded":"2023-09-23T17:54:57.040788+0000","last_fullsized":"2023-09-23T17:54:57.040788+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:37.176032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":528,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033496+0000","last_change":"2023-09-23T17:53:52.730761+0000","last_active":"2023-09-23T17:54:56.033496+0000","last_peered":"2023-09-23T17:54:56.033496+0000","last_clean":"2023-09-23T17:54:56.033496+0000","last_became_active":"2023-09-23T17:53:52.730121+0000","last_became_peered":"2023-09-23T17:53:52.730121+0000","last_unstale":"2023-09-23T17:54:56.033496+0000","last_undegraded":"2023-09-23T17:54:56.033496+0000","last_fullsized":"2023-09-23T17:54:56.033496+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:02.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035100+0000","last_change":"2023-09-23T17:54:57.035100+0000","last_active":"2023-09-23T17:54:57.035100+0000","last_peered":"2023-09-23T17:54:57.035100+0000","last_clean":"2023-09-23T17:54:57.035100+0000","last_became_active":"2023-09-23T17:54:57.034733+0000","last_became_peered":"2023-09-23T17:54:57.034733+0000","last_unstale":"2023-09-23T17:54:57.035100+0000","last_undegraded":"2023-09-23T17:54:57.035100+0000","last_fullsized":"2023-09-23T17:54:57.035100+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:27.841590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038911+0000","last_change":"2023-09-23T17:54:57.038911+0000","last_active":"2023-09-23T17:54:57.038911+0000","last_peered":"2023-09-23T17:54:57.038911+0000","last_clean":"2023-09-23T17:54:57.038911+0000","last_became_active":"2023-09-23T17:54:57.038607+0000","last_became_peered":"2023-09-23T17:54:57.038607+0000","last_unstale":"2023-09-23T17:54:57.038911+0000","last_undegraded":"2023-09-23T17:54:57.038911+0000","last_fullsized":"2023-09-23T17:54:57.038911+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:20.373213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021390+0000","last_change":"2023-09-23T17:53:52.736573+0000","last_active":"2023-09-23T17:54:56.021390+0000","last_peered":"2023-09-23T17:54:56.021390+0000","last_clean":"2023-09-23T17:54:56.021390+0000","last_became_active":"2023-09-23T17:53:52.736320+0000","last_became_peered":"2023-09-23T17:53:52.736320+0000","last_unstale":"2023-09-23T17:54:56.021390+0000","last_undegraded":"2023-09-23T17:54:56.021390+0000","last_fullsized":"2023-09-23T17:54:56.021390+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:05.760412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025639+0000","last_change":"2023-09-23T17:54:25.602420+0000","last_active":"2023-09-23T17:54:56.025639+0000","last_peered":"2023-09-23T17:54:56.025639+0000","last_clean":"2023-09-23T17:54:56.025639+0000","last_became_active":"2023-09-23T17:54:25.601988+0000","last_became_peered":"2023-09-23T17:54:25.601988+0000","last_unstale":"2023-09-23T17:54:56.025639+0000","last_undegraded":"2023-09-23T17:54:56.025639+0000","last_fullsized":"2023-09-23T17:54:56.025639+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:17.703211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":79,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022486+0000","last_change":"2023-09-23T17:54:25.603149+0000","last_active":"2023-09-23T17:54:56.022486+0000","last_peered":"2023-09-23T17:54:56.022486+0000","last_clean":"2023-09-23T17:54:56.022486+0000","last_became_active":"2023-09-23T17:54:25.602846+0000","last_became_peered":"2023-09-23T17:54:25.602846+0000","last_unstale":"2023-09-23T17:54:56.022486+0000","last_undegraded":"2023-09-23T17:54:56.022486+0000","last_fullsized":"2023-09-23T17:54:56.022486+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:06.955023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029512+0000","last_change":"2023-09-23T17:53:52.732276+0000","last_active":"2023-09-23T17:54:56.029512+0000","last_peered":"2023-09-23T17:54:56.029512+0000","last_clean":"2023-09-23T17:54:56.029512+0000","last_became_active":"2023-09-23T17:53:52.731952+0000","last_became_peered":"2023-09-23T17:53:52.731952+0000","last_unstale":"2023-09-23T17:54:56.029512+0000","last_undegraded":"2023-09-23T17:54:56.029512+0000","last_fullsized":"2023-09-23T17:54:56.029512+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:06:07.529125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036113+0000","last_change":"2023-09-23T17:54:57.036113+0000","last_active":"2023-09-23T17:54:57.036113+0000","last_peered":"2023-09-23T17:54:57.036113+0000","last_clean":"2023-09-23T17:54:57.036113+0000","last_became_active":"2023-09-23T17:54:57.035704+0000","last_became_peered":"2023-09-23T17:54:57.035704+0000","last_unstale":"2023-09-23T17:54:57.036113+0000","last_undegraded":"2023-09-23T17:54:57.036113+0000","last_fullsized":"2023-09-23T17:54:57.036113+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:10.397866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035203+0000","last_change":"2023-09-23T17:54:57.035203+0000","last_active":"2023-09-23T17:54:57.035203+0000","last_peered":"2023-09-23T17:54:57.035203+0000","last_clean":"2023-09-23T17:54:57.035203+0000","last_became_active":"2023-09-23T17:54:57.034854+0000","last_became_peered":"2023-09-23T17:54:57.034854+0000","last_unstale":"2023-09-23T17:54:57.035203+0000","last_undegraded":"2023-09-23T17:54:57.035203+0000","last_fullsized":"2023-09-23T17:54:57.035203+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:34.657128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026231+0000","last_change":"2023-09-23T17:53:52.731322+0000","last_active":"2023-09-23T17:54:57.026231+0000","last_peered":"2023-09-23T17:54:57.026231+0000","last_clean":"2023-09-23T17:54:57.026231+0000","last_became_active":"2023-09-23T17:53:52.730995+0000","last_became_peered":"2023-09-23T17:53:52.730995+0000","last_unstale":"2023-09-23T17:54:57.026231+0000","last_undegraded":"2023-09-23T17:54:57.026231+0000","last_fullsized":"2023-09-23T17:54:57.026231+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:55.121531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040294+0000","last_change":"2023-09-23T17:54:57.040294+0000","last_active":"2023-09-23T17:54:57.040294+0000","last_peered":"2023-09-23T17:54:57.040294+0000","last_clean":"2023-09-23T17:54:57.040294+0000","last_became_active":"2023-09-23T17:54:57.039943+0000","last_became_peered":"2023-09-23T17:54:57.039943+0000","last_unstale":"2023-09-23T17:54:57.040294+0000","last_undegraded":"2023-09-23T17:54:57.040294+0000","last_fullsized":"2023-09-23T17:54:57.040294+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:38.474814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":80,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029374+0000","last_change":"2023-09-23T17:54:25.601375+0000","last_active":"2023-09-23T17:54:56.029374+0000","last_peered":"2023-09-23T17:54:56.029374+0000","last_clean":"2023-09-23T17:54:56.029374+0000","last_became_active":"2023-09-23T17:54:25.601051+0000","last_became_peered":"2023-09-23T17:54:25.601051+0000","last_unstale":"2023-09-23T17:54:56.029374+0000","last_undegraded":"2023-09-23T17:54:56.029374+0000","last_fullsized":"2023-09-23T17:54:56.029374+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:24.772636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028296+0000","last_change":"2023-09-23T17:45:51.129514+0000","last_active":"2023-09-23T17:54:56.028296+0000","last_peered":"2023-09-23T17:54:56.028296+0000","last_clean":"2023-09-23T17:54:56.028296+0000","last_became_active":"2023-09-23T17:45:51.129154+0000","last_became_peered":"2023-09-23T17:45:51.129154+0000","last_unstale":"2023-09-23T17:54:56.028296+0000","last_undegraded":"2023-09-23T17:54:56.028296+0000","last_fullsized":"2023-09-23T17:54:56.028296+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:58.292683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040014+0000","last_change":"2023-09-23T17:54:57.040014+0000","last_active":"2023-09-23T17:54:57.040014+0000","last_peered":"2023-09-23T17:54:57.040014+0000","last_clean":"2023-09-23T17:54:57.040014+0000","last_became_active":"2023-09-23T17:54:57.039661+0000","last_became_peered":"2023-09-23T17:54:57.039661+0000","last_unstale":"2023-09-23T17:54:57.040014+0000","last_undegraded":"2023-09-23T17:54:57.040014+0000","last_fullsized":"2023-09-23T17:54:57.040014+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:01.077579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026256+0000","last_change":"2023-09-23T17:53:52.732018+0000","last_active":"2023-09-23T17:54:56.026256+0000","last_peered":"2023-09-23T17:54:56.026256+0000","last_clean":"2023-09-23T17:54:56.026256+0000","last_became_active":"2023-09-23T17:53:52.731641+0000","last_became_peered":"2023-09-23T17:53:52.731641+0000","last_unstale":"2023-09-23T17:54:56.026256+0000","last_undegraded":"2023-09-23T17:54:56.026256+0000","last_fullsized":"2023-09-23T17:54:56.026256+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:37.290885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029040+0000","last_change":"2023-09-23T17:45:51.297370+0000","last_active":"2023-09-23T17:54:56.029040+0000","last_peered":"2023-09-23T17:54:56.029040+0000","last_clean":"2023-09-23T17:54:56.029040+0000","last_became_active":"2023-09-23T17:45:51.297055+0000","last_became_peered":"2023-09-23T17:45:51.297055+0000","last_unstale":"2023-09-23T17:54:56.029040+0000","last_undegraded":"2023-09-23T17:54:56.029040+0000","last_fullsized":"2023-09-23T17:54:56.029040+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:52.209267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026812+0000","last_change":"2023-09-23T17:53:21.041393+0000","last_active":"2023-09-23T17:54:56.026812+0000","last_peered":"2023-09-23T17:54:56.026812+0000","last_clean":"2023-09-23T17:54:56.026812+0000","last_became_active":"2023-09-23T17:53:21.040995+0000","last_became_peered":"2023-09-23T17:53:21.040995+0000","last_unstale":"2023-09-23T17:54:56.026812+0000","last_undegraded":"2023-09-23T17:54:56.026812+0000","last_fullsized":"2023-09-23T17:54:56.026812+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:12.660494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041584+0000","last_change":"2023-09-23T17:54:57.041584+0000","last_active":"2023-09-23T17:54:57.041584+0000","last_peered":"2023-09-23T17:54:57.041584+0000","last_clean":"2023-09-23T17:54:57.041584+0000","last_became_active":"2023-09-23T17:54:57.040808+0000","last_became_peered":"2023-09-23T17:54:57.040808+0000","last_unstale":"2023-09-23T17:54:57.041584+0000","last_undegraded":"2023-09-23T17:54:57.041584+0000","last_fullsized":"2023-09-23T17:54:57.041584+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:01.079083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":79,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026234+0000","last_change":"2023-09-23T17:53:21.034197+0000","last_active":"2023-09-23T17:54:56.026234+0000","last_peered":"2023-09-23T17:54:56.026234+0000","last_clean":"2023-09-23T17:54:56.026234+0000","last_became_active":"2023-09-23T17:53:21.033784+0000","last_became_peered":"2023-09-23T17:53:21.033784+0000","last_unstale":"2023-09-23T17:54:56.026234+0000","last_undegraded":"2023-09-23T17:54:56.026234+0000","last_fullsized":"2023-09-23T17:54:56.026234+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:33.645714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039517+0000","last_change":"2023-09-23T17:54:57.039517+0000","last_active":"2023-09-23T17:54:57.039517+0000","last_peered":"2023-09-23T17:54:57.039517+0000","last_clean":"2023-09-23T17:54:57.039517+0000","last_became_active":"2023-09-23T17:54:57.039236+0000","last_became_peered":"2023-09-23T17:54:57.039236+0000","last_unstale":"2023-09-23T17:54:57.039517+0000","last_undegraded":"2023-09-23T17:54:57.039517+0000","last_fullsized":"2023-09-23T17:54:57.039517+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.126485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025164+0000","last_change":"2023-09-23T17:45:51.127190+0000","last_active":"2023-09-23T17:54:56.025164+0000","last_peered":"2023-09-23T17:54:56.025164+0000","last_clean":"2023-09-23T17:54:56.025164+0000","last_became_active":"2023-09-23T17:45:51.126903+0000","last_became_peered":"2023-09-23T17:45:51.126903+0000","last_unstale":"2023-09-23T17:54:56.025164+0000","last_undegraded":"2023-09-23T17:54:56.025164+0000","last_fullsized":"2023-09-23T17:54:56.025164+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:55.349259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.042234+0000","last_change":"2023-09-23T17:54:57.042234+0000","last_active":"2023-09-23T17:54:57.042234+0000","last_peered":"2023-09-23T17:54:57.042234+0000","last_clean":"2023-09-23T17:54:57.042234+0000","last_became_active":"2023-09-23T17:54:57.041918+0000","last_became_peered":"2023-09-23T17:54:57.041918+0000","last_unstale":"2023-09-23T17:54:57.042234+0000","last_undegraded":"2023-09-23T17:54:57.042234+0000","last_fullsized":"2023-09-23T17:54:57.042234+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:44.128871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":50,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038461+0000","last_change":"2023-09-23T17:54:57.038461+0000","last_active":"2023-09-23T17:54:57.038461+0000","last_peered":"2023-09-23T17:54:57.038461+0000","last_clean":"2023-09-23T17:54:57.038461+0000","last_became_active":"2023-09-23T17:54:57.038210+0000","last_became_peered":"2023-09-23T17:54:57.038210+0000","last_unstale":"2023-09-23T17:54:57.038461+0000","last_undegraded":"2023-09-23T17:54:57.038461+0000","last_fullsized":"2023-09-23T17:54:57.038461+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:50.077733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040422+0000","last_change":"2023-09-23T17:54:57.040422+0000","last_active":"2023-09-23T17:54:57.040422+0000","last_peered":"2023-09-23T17:54:57.040422+0000","last_clean":"2023-09-23T17:54:57.040422+0000","last_became_active":"2023-09-23T17:54:57.040085+0000","last_became_peered":"2023-09-23T17:54:57.040085+0000","last_unstale":"2023-09-23T17:54:57.040422+0000","last_undegraded":"2023-09-23T17:54:57.040422+0000","last_fullsized":"2023-09-23T17:54:57.040422+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:47.846454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025676+0000","last_change":"2023-09-23T17:54:25.602669+0000","last_active":"2023-09-23T17:54:56.025676+0000","last_peered":"2023-09-23T17:54:56.025676+0000","last_clean":"2023-09-23T17:54:56.025676+0000","last_became_active":"2023-09-23T17:54:25.602338+0000","last_became_peered":"2023-09-23T17:54:25.602338+0000","last_unstale":"2023-09-23T17:54:56.025676+0000","last_undegraded":"2023-09-23T17:54:56.025676+0000","last_fullsized":"2023-09-23T17:54:56.025676+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:02.816165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022313+0000","last_change":"2023-09-23T17:53:52.737008+0000","last_active":"2023-09-23T17:54:56.022313+0000","last_peered":"2023-09-23T17:54:56.022313+0000","last_clean":"2023-09-23T17:54:56.022313+0000","last_became_active":"2023-09-23T17:53:52.736702+0000","last_became_peered":"2023-09-23T17:53:52.736702+0000","last_unstale":"2023-09-23T17:54:56.022313+0000","last_undegraded":"2023-09-23T17:54:56.022313+0000","last_fullsized":"2023-09-23T17:54:56.022313+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:14.384950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021344+0000","last_change":"2023-09-23T17:45:51.124742+0000","last_active":"2023-09-23T17:54:56.021344+0000","last_peered":"2023-09-23T17:54:56.021344+0000","last_clean":"2023-09-23T17:54:56.021344+0000","last_became_active":"2023-09-23T17:45:51.124342+0000","last_became_peered":"2023-09-23T17:45:51.124342+0000","last_unstale":"2023-09-23T17:54:56.021344+0000","last_undegraded":"2023-09-23T17:54:56.021344+0000","last_fullsized":"2023-09-23T17:54:56.021344+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:24.351549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.032458+0000","last_change":"2023-09-23T17:54:25.603148+0000","last_active":"2023-09-23T17:54:56.032458+0000","last_peered":"2023-09-23T17:54:56.032458+0000","last_clean":"2023-09-23T17:54:56.032458+0000","last_became_active":"2023-09-23T17:54:25.602859+0000","last_became_peered":"2023-09-23T17:54:25.602859+0000","last_unstale":"2023-09-23T17:54:56.032458+0000","last_undegraded":"2023-09-23T17:54:56.032458+0000","last_fullsized":"2023-09-23T17:54:56.032458+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:19.802583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.043487+0000","last_change":"2023-09-23T17:54:57.043487+0000","last_active":"2023-09-23T17:54:57.043487+0000","last_peered":"2023-09-23T17:54:57.043487+0000","last_clean":"2023-09-23T17:54:57.043487+0000","last_became_active":"2023-09-23T17:54:57.043064+0000","last_became_peered":"2023-09-23T17:54:57.043064+0000","last_unstale":"2023-09-23T17:54:57.043487+0000","last_undegraded":"2023-09-23T17:54:57.043487+0000","last_fullsized":"2023-09-23T17:54:57.043487+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:34:21.634412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038085+0000","last_change":"2023-09-23T17:54:57.038085+0000","last_active":"2023-09-23T17:54:57.038085+0000","last_peered":"2023-09-23T17:54:57.038085+0000","last_clean":"2023-09-23T17:54:57.038085+0000","last_became_active":"2023-09-23T17:54:57.037551+0000","last_became_peered":"2023-09-23T17:54:57.037551+0000","last_unstale":"2023-09-23T17:54:57.038085+0000","last_undegraded":"2023-09-23T17:54:57.038085+0000","last_fullsized":"2023-09-23T17:54:57.038085+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:20:29.582747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029706+0000","last_change":"2023-09-23T17:53:52.732689+0000","last_active":"2023-09-23T17:54:56.029706+0000","last_peered":"2023-09-23T17:54:56.029706+0000","last_clean":"2023-09-23T17:54:56.029706+0000","last_became_active":"2023-09-23T17:53:52.732382+0000","last_became_peered":"2023-09-23T17:53:52.732382+0000","last_unstale":"2023-09-23T17:54:56.029706+0000","last_undegraded":"2023-09-23T17:54:56.029706+0000","last_fullsized":"2023-09-23T17:54:56.029706+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:34.408802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034454+0000","last_change":"2023-09-23T17:54:57.034454+0000","last_active":"2023-09-23T17:54:57.034454+0000","last_peered":"2023-09-23T17:54:57.034454+0000","last_clean":"2023-09-23T17:54:57.034454+0000","last_became_active":"2023-09-23T17:54:57.034128+0000","last_became_peered":"2023-09-23T17:54:57.034128+0000","last_unstale":"2023-09-23T17:54:57.034454+0000","last_undegraded":"2023-09-23T17:54:57.034454+0000","last_fullsized":"2023-09-23T17:54:57.034454+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:39.315294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":50,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036930+0000","last_change":"2023-09-23T17:54:57.036930+0000","last_active":"2023-09-23T17:54:57.036930+0000","last_peered":"2023-09-23T17:54:57.036930+0000","last_clean":"2023-09-23T17:54:57.036930+0000","last_became_active":"2023-09-23T17:54:57.036631+0000","last_became_peered":"2023-09-23T17:54:57.036631+0000","last_unstale":"2023-09-23T17:54:57.036930+0000","last_undegraded":"2023-09-23T17:54:57.036930+0000","last_fullsized":"2023-09-23T17:54:57.036930+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:08.367357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.024125+0000","last_change":"2023-09-23T17:53:52.732305+0000","last_active":"2023-09-23T17:54:56.024125+0000","last_peered":"2023-09-23T17:54:56.024125+0000","last_clean":"2023-09-23T17:54:56.024125+0000","last_became_active":"2023-09-23T17:53:52.731894+0000","last_became_peered":"2023-09-23T17:53:52.731894+0000","last_unstale":"2023-09-23T17:54:56.024125+0000","last_undegraded":"2023-09-23T17:54:56.024125+0000","last_fullsized":"2023-09-23T17:54:56.024125+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:30.224762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.031226+0000","last_change":"2023-09-23T17:54:57.031226+0000","last_active":"2023-09-23T17:54:57.031226+0000","last_peered":"2023-09-23T17:54:57.031226+0000","last_clean":"2023-09-23T17:54:57.031226+0000","last_became_active":"2023-09-23T17:54:57.030880+0000","last_became_peered":"2023-09-23T17:54:57.030880+0000","last_unstale":"2023-09-23T17:54:57.031226+0000","last_undegraded":"2023-09-23T17:54:57.031226+0000","last_fullsized":"2023-09-23T17:54:57.031226+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:49.836129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040141+0000","last_change":"2023-09-23T17:54:57.039438+0000","last_active":"2023-09-23T17:54:57.040141+0000","last_peered":"2023-09-23T17:54:57.040141+0000","last_clean":"2023-09-23T17:54:57.040141+0000","last_became_active":"2023-09-23T17:54:57.039163+0000","last_became_peered":"2023-09-23T17:54:57.039163+0000","last_unstale":"2023-09-23T17:54:57.040141+0000","last_undegraded":"2023-09-23T17:54:57.040141+0000","last_fullsized":"2023-09-23T17:54:57.040141+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.265833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034381+0000","last_change":"2023-09-23T17:54:57.034381+0000","last_active":"2023-09-23T17:54:57.034381+0000","last_peered":"2023-09-23T17:54:57.034381+0000","last_clean":"2023-09-23T17:54:57.034381+0000","last_became_active":"2023-09-23T17:54:57.034043+0000","last_became_peered":"2023-09-23T17:54:57.034043+0000","last_unstale":"2023-09-23T17:54:57.034381+0000","last_undegraded":"2023-09-23T17:54:57.034381+0000","last_fullsized":"2023-09-23T17:54:57.034381+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:40.217762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036156+0000","last_change":"2023-09-23T17:54:57.036156+0000","last_active":"2023-09-23T17:54:57.036156+0000","last_peered":"2023-09-23T17:54:57.036156+0000","last_clean":"2023-09-23T17:54:57.036156+0000","last_became_active":"2023-09-23T17:54:57.035478+0000","last_became_peered":"2023-09-23T17:54:57.035478+0000","last_unstale":"2023-09-23T17:54:57.036156+0000","last_undegraded":"2023-09-23T17:54:57.036156+0000","last_fullsized":"2023-09-23T17:54:57.036156+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:53.126638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":88,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021007+0000","last_change":"2023-09-23T17:54:25.605945+0000","last_active":"2023-09-23T17:54:56.021007+0000","last_peered":"2023-09-23T17:54:56.021007+0000","last_clean":"2023-09-23T17:54:56.021007+0000","last_became_active":"2023-09-23T17:54:25.605583+0000","last_became_peered":"2023-09-23T17:54:25.605583+0000","last_unstale":"2023-09-23T17:54:56.021007+0000","last_undegraded":"2023-09-23T17:54:56.021007+0000","last_fullsized":"2023-09-23T17:54:56.021007+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:14.076789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266822,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":13024,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":11328,"kb_avail":93732128,"statfs":{"total":95995035648,"available":95981699072,"internally_reserved":0,"allocated":1728512,"data_stored":1206046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11599872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55500000000000005,"15min":0.55200000000000005},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81200000000000006,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57199999999999995,"15min":0.56899999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.59499999999999997}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.496},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.45200000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:54:01 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.58099999999999996,"15min":0.54400000000000004},"min":{"1min":0.436,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.79400000000000004,"5min":1.0409999999999999,"15min":1.9370000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.60399999999999998,"15min":0.63200000000000001},"min":{"1min":0.42399999999999999,"5min":0.36799999999999999,"15min":0.35899999999999999},"max":{"1min":0.75800000000000001,"5min":0.91300000000000003,"15min":8.8499999999999996},"last":0.52400000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.49399999999999999,"15min":0.44600000000000001},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52300000000000002,"15min":0.47499999999999998},"min":{"1min":0.34699999999999998,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.60499999999999998}]},{"osd":5,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53000000000000003,"15min":0.48999999999999999},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.28699999999999998},"max":{"1min":0.72799999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.627},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.57999999999999996,"15min":0.51400000000000001},"min":{"1min":0.40100000000000002,"5min":0.30099999999999999,"15min":0.29099999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.69099999999999995}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.64300000000000002}]}]},{"osd":6,"up_from":45,"seq":193273528458,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8344,"kb_used_data":1680,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93736808,"statfs":{"total":95995035648,"available":95986491392,"internally_reserved":0,"allocated":1720320,"data_stored":1205919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.81100000000000005,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.78300000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.495}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.47099999999999997}]},{"osd":2,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.40600000000000003}]},{"osd":3,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.38800000000000001,"5min":0.36799999999999999,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":0.80700000000000005,"15min":4.5599999999999996},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.625,"5min":0.58499999999999996,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34999999999999998,"15min":0.30399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":1.6850000000000001},"last":0.67000000000000004}]},{"osd":4,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.47099999999999997,"15min":0.45200000000000001},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":0.748,"5min":0.76400000000000001,"15min":0.82899999999999996},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52800000000000002,"15min":0.49199999999999999},"min":{"1min":0.34999999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.65400000000000003}]},{"osd":5,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.53400000000000003,"15min":0.496},"min":{"1min":0.33700000000000002,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":0.77700000000000002,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56599999999999995,"15min":0.52200000000000002},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.33200000000000002},"max":{"1min":0.79700000000000004,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Sep 23 17:55:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.58999999999999997,"15min":0.53700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":0.83299999999999996,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.57499999999999996,"15min":0.51900000000000002},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":0.81299999999999994,"5min":0.82299999999999995,"15min":1.591},"last":0.622}]}]},{"osd":1,"up_from":98,"seq":420906795031,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28716,"kb_used_data":1124,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93716436,"statfs":{"total":95995035648,"available":95965630464,"internally_reserved":0,"allocated":1150976,"data_stored":617993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":28238525},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.308},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.976,"5min":1.976,"15min":1.976},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.377}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41099999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.77200000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1,"5min":1,"15min":1},"last":0.69299999999999995}]},{"osd":6,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.71699999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.74099999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958557,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33408,"kb_used_data":5816,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93711744,"statfs":{"total":95995035648,"available":95960825856,"internally_reserved":0,"allocated":5955584,"data_stored":5404139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":28238564},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:54:51 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.374}]},{"osd":2,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"last":0.33200000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.39400000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82199999999999995,"5min":0.872,"15min":0.872},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61699999999999999,"15min":0.621},"min":{"1min":0.44700000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77700000000000002,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.624,"15min":0.63},"min":{"1min":0.41799999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.73599999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.63900000000000001,"15min":0.64300000000000002},"min":{"1min":0.46600000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.53000000000000003}]},{"osd":6,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.42899999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.504},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65200000000000002},"min":{"1min":0.52700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.81100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.59599999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65900000000000003,"15min":0.66300000000000003},"min":{"1min":0.51600000000000001,"5min":0.435,"15min":0.435},"max":{"1min":0.84299999999999997,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.66600000000000004,"15min":0.67000000000000004},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86599999999999999,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.56499999999999995}]}]},{"osd":2,"up_from":103,"seq":442381631504,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32744,"kb_used_data":5216,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93712408,"statfs":{"total":95995035648,"available":95961505792,"internally_reserved":0,"allocated":5341184,"data_stored":4811538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":28168567},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.30599999999999999}]},{"osd":1,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.35099999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.59999999999999998}]},{"osd":5,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.58099999999999996}]},{"osd":6,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.624}]},{"osd":7,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.57099999999999995}]}]},{"osd":3,"up_from":108,"seq":463856467978,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32604,"kb_used_data":5204,"kb_used_omap":3,"kb_used_meta":27388,"kb_avail":93712548,"statfs":{"total":95995035648,"available":95961649152,"internally_reserved":0,"allocated":5328896,"data_stored":4809888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":28046056},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32900000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.372}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.31}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51400000000000001}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.52000000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.61399999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.504}]}]},{"osd":4,"up_from":30,"seq":128849019023,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7644,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6528,"kb_avail":93737508,"statfs":{"total":95995035648,"available":95987208192,"internally_reserved":0,"allocated":1134592,"data_stored":615931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6684672},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.75600000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.53800000000000003,"15min":0.52900000000000003},"min":{"1min":0.42699999999999999,"5min":0.376,"15min":0.376},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.59999999999999998}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.64800000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:54:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.42599999999999999,"5min":0.34399999999999997,"15min":0.318},"max":{"1min":0.73999999999999999,"5min":0.77100000000000002,"15min":5.7119999999999997},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.59199999999999997,"15min":0.57999999999999996},"min":{"1min":0.41599999999999998,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":0.70999999999999996,"5min":0.85499999999999998,"15min":5.9429999999999996},"last":0.56999999999999995}]},{"osd":5,"last update":"Sat Sep 23 17:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48399999999999999,"15min":0.438},"min":{"1min":0.379,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":0.621,"5min":0.753,"15min":1.141},"last":0.441},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.51700000000000002,"15min":0.44700000000000001},"min":{"1min":0.41499999999999998,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":0.749,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.47799999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:54:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.54700000000000004,"15min":0.501},"min":{"1min":0.36099999999999999,"5min":0.33200000000000002,"15min":0.28899999999999998},"max":{"1min":0.68400000000000005,"5min":0.77400000000000002,"15min":1.276},"last":0.504},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58799999999999997,"15min":0.53600000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28999999999999998},"max":{"1min":0.77100000000000002,"5min":0.80900000000000005,"15min":1.1180000000000001},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 23 17:55:06 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.57699999999999996,"15min":0.54700000000000004},"min":{"1min":0.40600000000000003,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":1.1839999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.57699999999999996,"15min":0.53000000000000003},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.55700000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757389,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11724,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":10624,"kb_avail":93733428,"statfs":{"total":95995035648,"available":95983030272,"internally_reserved":0,"allocated":1118208,"data_stored":611371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":10878976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.57199999999999995,"15min":0.57599999999999996},"min":{"1min":0.39800000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.77100000000000002,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.74199999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.69399999999999995}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.4279999999999999,"5min":4.4279999999999999,"15min":4.4279999999999999},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.54100000000000004}]},{"osd":2,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.749}]},{"osd":3,"last update":"Sat Sep 23 17:54:27 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61199999999999999,"15min":0.58099999999999996},"min":{"1min":0.42499999999999999,"5min":0.41999999999999998,"15min":0.32100000000000001},"max":{"1min":0.76700000000000002,"5min":0.85799999999999998,"15min":4.6799999999999997},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.61199999999999999,"15min":0.57699999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.33100000000000002},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.631},"last":0.80900000000000005}]},{"osd":4,"last update":"Sat Sep 23 17:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.48899999999999999,"15min":0.44800000000000001},"min":{"1min":0.34300000000000003,"5min":0.25,"15min":0.25},"max":{"1min":0.70799999999999996,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.54400000000000004,"15min":0.51400000000000001},"min":{"1min":0.435,"5min":0.33100000000000002,"15min":0.28899999999999998},"max":{"1min":0.81399999999999995,"5min":0.83199999999999996,"15min":1.738},"last":0.67200000000000004}]},{"osd":6,"last update":"Sat Sep 23 17:54:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.55300000000000005,"15min":0.53400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":1.778},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.61599999999999999,"15min":0.58299999999999996},"min":{"1min":0.38100000000000001,"5min":0.375,"15min":0.33200000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":1.7629999999999999},"last":0.72799999999999998}]},{"osd":7,"last update":"Sat Sep 23 17:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.56299999999999994,"15min":0.52000000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.83999999999999997,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.62,"5min":0.58799999999999997,"15min":0.54900000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.79600000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.76700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:55:40.334 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:55:40.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:55:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:55:40] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.43.0" 2023-09-23T17:55:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:40 smithi055 ceph-mon[179668]: from='client.34719 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:40 smithi055 ceph-mon[184174]: from='client.34719 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:40 smithi099 ceph-mon[167076]: from='client.34719 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:40.974 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:41 smithi099 ceph-mon[167076]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:41.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:41 smithi055 ceph-mon[179668]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:41.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:41 smithi055 ceph-mon[184174]: pgmap v220: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:42.381 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:55:42.382 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:55:42.834 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":220,"stamp":"2023-09-23T17:55:40.406997+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4790195,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":6495,"num_read_kb":14136,"num_write":1146,"num_write_kb":15141,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1192,"ondisk_log_size":1192,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":219,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":168208,"kb_used_data":22928,"kb_used_omap":29,"kb_used_meta":145186,"kb_avail":749793008,"statfs":{"total":767960285184,"available":767788040192,"internally_reserved":0,"allocated":23478272,"data_stored":19282825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30208,"internal_metadata":148670976},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002871"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.020803+0000","last_change":"2023-09-23T17:53:52.736718+0000","last_active":"2023-09-23T17:54:56.020803+0000","last_peered":"2023-09-23T17:54:56.020803+0000","last_clean":"2023-09-23T17:54:56.020803+0000","last_became_active":"2023-09-23T17:53:52.736314+0000","last_became_peered":"2023-09-23T17:53:52.736314+0000","last_unstale":"2023-09-23T17:54:56.020803+0000","last_undegraded":"2023-09-23T17:54:56.020803+0000","last_fullsized":"2023-09-23T17:54:56.020803+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:26.017288+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026023+0000","last_change":"2023-09-23T17:54:25.601065+0000","last_active":"2023-09-23T17:54:57.026023+0000","last_peered":"2023-09-23T17:54:57.026023+0000","last_clean":"2023-09-23T17:54:57.026023+0000","last_became_active":"2023-09-23T17:54:25.600727+0000","last_became_peered":"2023-09-23T17:54:25.600727+0000","last_unstale":"2023-09-23T17:54:57.026023+0000","last_undegraded":"2023-09-23T17:54:57.026023+0000","last_fullsized":"2023-09-23T17:54:57.026023+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:20.919498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034722+0000","last_change":"2023-09-23T17:54:57.034722+0000","last_active":"2023-09-23T17:54:57.034722+0000","last_peered":"2023-09-23T17:54:57.034722+0000","last_clean":"2023-09-23T17:54:57.034722+0000","last_became_active":"2023-09-23T17:54:57.034289+0000","last_became_peered":"2023-09-23T17:54:57.034289+0000","last_unstale":"2023-09-23T17:54:57.034722+0000","last_undegraded":"2023-09-23T17:54:57.034722+0000","last_fullsized":"2023-09-23T17:54:57.034722+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:03.041518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":77,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.032421+0000","last_change":"2023-09-23T17:54:57.032421+0000","last_active":"2023-09-23T17:54:57.032421+0000","last_peered":"2023-09-23T17:54:57.032421+0000","last_clean":"2023-09-23T17:54:57.032421+0000","last_became_active":"2023-09-23T17:54:57.032020+0000","last_became_peered":"2023-09-23T17:54:57.032020+0000","last_unstale":"2023-09-23T17:54:57.032421+0000","last_undegraded":"2023-09-23T17:54:57.032421+0000","last_fullsized":"2023-09-23T17:54:57.032421+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:14.716268+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039895+0000","last_change":"2023-09-23T17:54:57.039895+0000","last_active":"2023-09-23T17:54:57.039895+0000","last_peered":"2023-09-23T17:54:57.039895+0000","last_clean":"2023-09-23T17:54:57.039895+0000","last_became_active":"2023-09-23T17:54:57.039581+0000","last_became_peered":"2023-09-23T17:54:57.039581+0000","last_unstale":"2023-09-23T17:54:57.039895+0000","last_undegraded":"2023-09-23T17:54:57.039895+0000","last_fullsized":"2023-09-23T17:54:57.039895+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:52.622668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029112+0000","last_change":"2023-09-23T17:45:51.121328+0000","last_active":"2023-09-23T17:54:56.029112+0000","last_peered":"2023-09-23T17:54:56.029112+0000","last_clean":"2023-09-23T17:54:56.029112+0000","last_became_active":"2023-09-23T17:45:51.121034+0000","last_became_peered":"2023-09-23T17:45:51.121034+0000","last_unstale":"2023-09-23T17:54:56.029112+0000","last_undegraded":"2023-09-23T17:54:56.029112+0000","last_fullsized":"2023-09-23T17:54:56.029112+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:13.256406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041535+0000","last_change":"2023-09-23T17:54:57.041535+0000","last_active":"2023-09-23T17:54:57.041535+0000","last_peered":"2023-09-23T17:54:57.041535+0000","last_clean":"2023-09-23T17:54:57.041535+0000","last_became_active":"2023-09-23T17:54:57.040909+0000","last_became_peered":"2023-09-23T17:54:57.040909+0000","last_unstale":"2023-09-23T17:54:57.041535+0000","last_undegraded":"2023-09-23T17:54:57.041535+0000","last_fullsized":"2023-09-23T17:54:57.041535+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:57.504646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.030937+0000","last_change":"2023-09-23T17:53:52.732284+0000","last_active":"2023-09-23T17:54:56.030937+0000","last_peered":"2023-09-23T17:54:56.030937+0000","last_clean":"2023-09-23T17:54:56.030937+0000","last_became_active":"2023-09-23T17:53:52.731928+0000","last_became_peered":"2023-09-23T17:53:52.731928+0000","last_unstale":"2023-09-23T17:54:56.030937+0000","last_undegraded":"2023-09-23T17:54:56.030937+0000","last_fullsized":"2023-09-23T17:54:56.030937+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:35.936955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":92,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034729+0000","last_change":"2023-09-23T17:54:57.034729+0000","last_active":"2023-09-23T17:54:57.034729+0000","last_peered":"2023-09-23T17:54:57.034729+0000","last_clean":"2023-09-23T17:54:57.034729+0000","last_became_active":"2023-09-23T17:54:57.034381+0000","last_became_peered":"2023-09-23T17:54:57.034381+0000","last_unstale":"2023-09-23T17:54:57.034729+0000","last_undegraded":"2023-09-23T17:54:57.034729+0000","last_fullsized":"2023-09-23T17:54:57.034729+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:19.369673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":39,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026132+0000","last_change":"2023-09-23T17:53:21.039717+0000","last_active":"2023-09-23T17:54:56.026132+0000","last_peered":"2023-09-23T17:54:56.026132+0000","last_clean":"2023-09-23T17:54:56.026132+0000","last_became_active":"2023-09-23T17:53:21.038793+0000","last_became_peered":"2023-09-23T17:53:21.038793+0000","last_unstale":"2023-09-23T17:54:56.026132+0000","last_undegraded":"2023-09-23T17:54:56.026132+0000","last_fullsized":"2023-09-23T17:54:56.026132+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:02.116246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":52,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025537+0000","last_change":"2023-09-23T17:45:51.148003+0000","last_active":"2023-09-23T17:54:56.025537+0000","last_peered":"2023-09-23T17:54:56.025537+0000","last_clean":"2023-09-23T17:54:56.025537+0000","last_became_active":"2023-09-23T17:45:51.147815+0000","last_became_peered":"2023-09-23T17:45:51.147815+0000","last_unstale":"2023-09-23T17:54:56.025537+0000","last_undegraded":"2023-09-23T17:54:56.025537+0000","last_fullsized":"2023-09-23T17:54:56.025537+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:32.374670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":55,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026943+0000","last_change":"2023-09-23T17:53:52.732070+0000","last_active":"2023-09-23T17:54:57.026943+0000","last_peered":"2023-09-23T17:54:57.026943+0000","last_clean":"2023-09-23T17:54:57.026943+0000","last_became_active":"2023-09-23T17:53:52.731739+0000","last_became_peered":"2023-09-23T17:53:52.731739+0000","last_unstale":"2023-09-23T17:54:57.026943+0000","last_undegraded":"2023-09-23T17:54:57.026943+0000","last_fullsized":"2023-09-23T17:54:57.026943+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:03.726754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":333,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.024215+0000","last_change":"2023-09-23T17:54:25.600838+0000","last_active":"2023-09-23T17:54:56.024215+0000","last_peered":"2023-09-23T17:54:56.024215+0000","last_clean":"2023-09-23T17:54:56.024215+0000","last_became_active":"2023-09-23T17:54:25.600305+0000","last_became_peered":"2023-09-23T17:54:25.600305+0000","last_unstale":"2023-09-23T17:54:56.024215+0000","last_undegraded":"2023-09-23T17:54:56.024215+0000","last_fullsized":"2023-09-23T17:54:56.024215+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:36.605455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":430,"num_read_kb":1405,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040450+0000","last_change":"2023-09-23T17:54:57.040450+0000","last_active":"2023-09-23T17:54:57.040450+0000","last_peered":"2023-09-23T17:54:57.040450+0000","last_clean":"2023-09-23T17:54:57.040450+0000","last_became_active":"2023-09-23T17:54:57.040134+0000","last_became_peered":"2023-09-23T17:54:57.040134+0000","last_unstale":"2023-09-23T17:54:57.040450+0000","last_undegraded":"2023-09-23T17:54:57.040450+0000","last_fullsized":"2023-09-23T17:54:57.040450+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:07.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":286,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.032152+0000","last_change":"2023-09-23T17:53:52.734869+0000","last_active":"2023-09-23T17:54:56.032152+0000","last_peered":"2023-09-23T17:54:56.032152+0000","last_clean":"2023-09-23T17:54:56.032152+0000","last_became_active":"2023-09-23T17:53:52.734387+0000","last_became_peered":"2023-09-23T17:53:52.734387+0000","last_unstale":"2023-09-23T17:54:56.032152+0000","last_undegraded":"2023-09-23T17:54:56.032152+0000","last_fullsized":"2023-09-23T17:54:56.032152+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:15.397601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041796+0000","last_change":"2023-09-23T17:54:57.041796+0000","last_active":"2023-09-23T17:54:57.041796+0000","last_peered":"2023-09-23T17:54:57.041796+0000","last_clean":"2023-09-23T17:54:57.041796+0000","last_became_active":"2023-09-23T17:54:57.041245+0000","last_became_peered":"2023-09-23T17:54:57.041245+0000","last_unstale":"2023-09-23T17:54:57.041796+0000","last_undegraded":"2023-09-23T17:54:57.041796+0000","last_fullsized":"2023-09-23T17:54:57.041796+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:26.340510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":37,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026049+0000","last_change":"2023-09-23T17:53:21.039195+0000","last_active":"2023-09-23T17:54:57.026049+0000","last_peered":"2023-09-23T17:54:57.026049+0000","last_clean":"2023-09-23T17:54:57.026049+0000","last_became_active":"2023-09-23T17:53:21.038654+0000","last_became_peered":"2023-09-23T17:53:21.038654+0000","last_unstale":"2023-09-23T17:54:57.026049+0000","last_undegraded":"2023-09-23T17:54:57.026049+0000","last_fullsized":"2023-09-23T17:54:57.026049+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:34.466980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033621+0000","last_change":"2023-09-23T17:54:25.604720+0000","last_active":"2023-09-23T17:54:56.033621+0000","last_peered":"2023-09-23T17:54:56.033621+0000","last_clean":"2023-09-23T17:54:56.033621+0000","last_became_active":"2023-09-23T17:54:25.604386+0000","last_became_peered":"2023-09-23T17:54:25.604386+0000","last_unstale":"2023-09-23T17:54:56.033621+0000","last_undegraded":"2023-09-23T17:54:56.033621+0000","last_fullsized":"2023-09-23T17:54:56.033621+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:56.869273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":321,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028220+0000","last_change":"2023-09-23T17:53:21.035484+0000","last_active":"2023-09-23T17:54:56.028220+0000","last_peered":"2023-09-23T17:54:56.028220+0000","last_clean":"2023-09-23T17:54:56.028220+0000","last_became_active":"2023-09-23T17:53:21.035035+0000","last_became_peered":"2023-09-23T17:53:21.035035+0000","last_unstale":"2023-09-23T17:54:56.028220+0000","last_undegraded":"2023-09-23T17:54:56.028220+0000","last_fullsized":"2023-09-23T17:54:56.028220+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:05.145111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":67,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026968+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T17:54:57.026968+0000","last_peered":"2023-09-23T17:54:57.026968+0000","last_clean":"2023-09-23T17:54:57.026968+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T17:54:57.026968+0000","last_undegraded":"2023-09-23T17:54:57.026968+0000","last_fullsized":"2023-09-23T17:54:57.026968+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039608+0000","last_change":"2023-09-23T17:54:57.036095+0000","last_active":"2023-09-23T17:54:57.039608+0000","last_peered":"2023-09-23T17:54:57.039608+0000","last_clean":"2023-09-23T17:54:57.039608+0000","last_became_active":"2023-09-23T17:54:57.035606+0000","last_became_peered":"2023-09-23T17:54:57.035606+0000","last_unstale":"2023-09-23T17:54:57.039608+0000","last_undegraded":"2023-09-23T17:54:57.039608+0000","last_fullsized":"2023-09-23T17:54:57.039608+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:10.936020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":307,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025278+0000","last_change":"2023-09-23T17:53:52.734324+0000","last_active":"2023-09-23T17:54:56.025278+0000","last_peered":"2023-09-23T17:54:56.025278+0000","last_clean":"2023-09-23T17:54:56.025278+0000","last_became_active":"2023-09-23T17:53:52.733825+0000","last_became_peered":"2023-09-23T17:53:52.733825+0000","last_unstale":"2023-09-23T17:54:56.025278+0000","last_undegraded":"2023-09-23T17:54:56.025278+0000","last_fullsized":"2023-09-23T17:54:56.025278+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:34.742687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":53,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.027204+0000","last_change":"2023-09-23T17:45:51.297621+0000","last_active":"2023-09-23T17:54:56.027204+0000","last_peered":"2023-09-23T17:54:56.027204+0000","last_clean":"2023-09-23T17:54:56.027204+0000","last_became_active":"2023-09-23T17:45:51.297322+0000","last_became_peered":"2023-09-23T17:45:51.297322+0000","last_unstale":"2023-09-23T17:54:56.027204+0000","last_undegraded":"2023-09-23T17:54:56.027204+0000","last_fullsized":"2023-09-23T17:54:56.027204+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:15:14.268027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":22,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036074+0000","last_change":"2023-09-23T17:54:57.036074+0000","last_active":"2023-09-23T17:54:57.036074+0000","last_peered":"2023-09-23T17:54:57.036074+0000","last_clean":"2023-09-23T17:54:57.036074+0000","last_became_active":"2023-09-23T17:54:57.035801+0000","last_became_peered":"2023-09-23T17:54:57.035801+0000","last_unstale":"2023-09-23T17:54:57.036074+0000","last_undegraded":"2023-09-23T17:54:57.036074+0000","last_fullsized":"2023-09-23T17:54:57.036074+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:11.087766+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":452,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025187+0000","last_change":"2023-09-23T17:53:21.035503+0000","last_active":"2023-09-23T17:54:56.025187+0000","last_peered":"2023-09-23T17:54:56.025187+0000","last_clean":"2023-09-23T17:54:56.025187+0000","last_became_active":"2023-09-23T17:53:21.035119+0000","last_became_peered":"2023-09-23T17:53:21.035119+0000","last_unstale":"2023-09-23T17:54:56.025187+0000","last_undegraded":"2023-09-23T17:54:56.025187+0000","last_fullsized":"2023-09-23T17:54:56.025187+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:32.369675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1023,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029395+0000","last_change":"2023-09-23T17:54:25.601298+0000","last_active":"2023-09-23T17:54:56.029395+0000","last_peered":"2023-09-23T17:54:56.029395+0000","last_clean":"2023-09-23T17:54:56.029395+0000","last_became_active":"2023-09-23T17:54:25.600854+0000","last_became_peered":"2023-09-23T17:54:25.600854+0000","last_unstale":"2023-09-23T17:54:56.029395+0000","last_undegraded":"2023-09-23T17:54:56.029395+0000","last_fullsized":"2023-09-23T17:54:56.029395+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:56.978955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028625+0000","last_change":"2023-09-23T17:53:52.728795+0000","last_active":"2023-09-23T17:54:56.028625+0000","last_peered":"2023-09-23T17:54:56.028625+0000","last_clean":"2023-09-23T17:54:56.028625+0000","last_became_active":"2023-09-23T17:53:52.728423+0000","last_became_peered":"2023-09-23T17:53:52.728423+0000","last_unstale":"2023-09-23T17:54:56.028625+0000","last_undegraded":"2023-09-23T17:54:56.028625+0000","last_fullsized":"2023-09-23T17:54:56.028625+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.759322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":46,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033868+0000","last_change":"2023-09-23T17:54:25.603824+0000","last_active":"2023-09-23T17:54:56.033868+0000","last_peered":"2023-09-23T17:54:56.033868+0000","last_clean":"2023-09-23T17:54:56.033868+0000","last_became_active":"2023-09-23T17:54:25.603526+0000","last_became_peered":"2023-09-23T17:54:25.603526+0000","last_unstale":"2023-09-23T17:54:56.033868+0000","last_undegraded":"2023-09-23T17:54:56.033868+0000","last_fullsized":"2023-09-23T17:54:56.033868+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:29.842346+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":372,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.037634+0000","last_change":"2023-09-23T17:54:57.037634+0000","last_active":"2023-09-23T17:54:57.037634+0000","last_peered":"2023-09-23T17:54:57.037634+0000","last_clean":"2023-09-23T17:54:57.037634+0000","last_became_active":"2023-09-23T17:54:57.037115+0000","last_became_peered":"2023-09-23T17:54:57.037115+0000","last_unstale":"2023-09-23T17:54:57.037634+0000","last_undegraded":"2023-09-23T17:54:57.037634+0000","last_fullsized":"2023-09-23T17:54:57.037634+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:52:58.348982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":62,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035632+0000","last_change":"2023-09-23T17:54:57.035632+0000","last_active":"2023-09-23T17:54:57.035632+0000","last_peered":"2023-09-23T17:54:57.035632+0000","last_clean":"2023-09-23T17:54:57.035632+0000","last_became_active":"2023-09-23T17:54:57.035338+0000","last_became_peered":"2023-09-23T17:54:57.035338+0000","last_unstale":"2023-09-23T17:54:57.035632+0000","last_undegraded":"2023-09-23T17:54:57.035632+0000","last_fullsized":"2023-09-23T17:54:57.035632+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:18.773130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022511+0000","last_change":"2023-09-23T17:53:21.036597+0000","last_active":"2023-09-23T17:54:56.022511+0000","last_peered":"2023-09-23T17:54:56.022511+0000","last_clean":"2023-09-23T17:54:56.022511+0000","last_became_active":"2023-09-23T17:53:21.036282+0000","last_became_peered":"2023-09-23T17:53:21.036282+0000","last_unstale":"2023-09-23T17:54:56.022511+0000","last_undegraded":"2023-09-23T17:54:56.022511+0000","last_fullsized":"2023-09-23T17:54:56.022511+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:28.486975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1153,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029744+0000","last_change":"2023-09-23T17:53:52.733510+0000","last_active":"2023-09-23T17:54:56.029744+0000","last_peered":"2023-09-23T17:54:56.029744+0000","last_clean":"2023-09-23T17:54:56.029744+0000","last_became_active":"2023-09-23T17:53:52.733030+0000","last_became_peered":"2023-09-23T17:53:52.733030+0000","last_unstale":"2023-09-23T17:54:56.029744+0000","last_undegraded":"2023-09-23T17:54:56.029744+0000","last_fullsized":"2023-09-23T17:54:56.029744+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:59.009283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1894,"num_read_kb":2706,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":32,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029081+0000","last_change":"2023-09-23T17:53:52.735673+0000","last_active":"2023-09-23T17:54:56.029081+0000","last_peered":"2023-09-23T17:54:56.029081+0000","last_clean":"2023-09-23T17:54:56.029081+0000","last_became_active":"2023-09-23T17:53:52.734890+0000","last_became_peered":"2023-09-23T17:53:52.734890+0000","last_unstale":"2023-09-23T17:54:56.029081+0000","last_undegraded":"2023-09-23T17:54:56.029081+0000","last_fullsized":"2023-09-23T17:54:56.029081+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:22.338546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040788+0000","last_change":"2023-09-23T17:54:57.040788+0000","last_active":"2023-09-23T17:54:57.040788+0000","last_peered":"2023-09-23T17:54:57.040788+0000","last_clean":"2023-09-23T17:54:57.040788+0000","last_became_active":"2023-09-23T17:54:57.040271+0000","last_became_peered":"2023-09-23T17:54:57.040271+0000","last_unstale":"2023-09-23T17:54:57.040788+0000","last_undegraded":"2023-09-23T17:54:57.040788+0000","last_fullsized":"2023-09-23T17:54:57.040788+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:37.176032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":528,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.033496+0000","last_change":"2023-09-23T17:53:52.730761+0000","last_active":"2023-09-23T17:54:56.033496+0000","last_peered":"2023-09-23T17:54:56.033496+0000","last_clean":"2023-09-23T17:54:56.033496+0000","last_became_active":"2023-09-23T17:53:52.730121+0000","last_became_peered":"2023-09-23T17:53:52.730121+0000","last_unstale":"2023-09-23T17:54:56.033496+0000","last_undegraded":"2023-09-23T17:54:56.033496+0000","last_fullsized":"2023-09-23T17:54:56.033496+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:02.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":43,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035100+0000","last_change":"2023-09-23T17:54:57.035100+0000","last_active":"2023-09-23T17:54:57.035100+0000","last_peered":"2023-09-23T17:54:57.035100+0000","last_clean":"2023-09-23T17:54:57.035100+0000","last_became_active":"2023-09-23T17:54:57.034733+0000","last_became_peered":"2023-09-23T17:54:57.034733+0000","last_unstale":"2023-09-23T17:54:57.035100+0000","last_undegraded":"2023-09-23T17:54:57.035100+0000","last_fullsized":"2023-09-23T17:54:57.035100+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:27.841590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":72,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038911+0000","last_change":"2023-09-23T17:54:57.038911+0000","last_active":"2023-09-23T17:54:57.038911+0000","last_peered":"2023-09-23T17:54:57.038911+0000","last_clean":"2023-09-23T17:54:57.038911+0000","last_became_active":"2023-09-23T17:54:57.038607+0000","last_became_peered":"2023-09-23T17:54:57.038607+0000","last_unstale":"2023-09-23T17:54:57.038911+0000","last_undegraded":"2023-09-23T17:54:57.038911+0000","last_fullsized":"2023-09-23T17:54:57.038911+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:20.373213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021390+0000","last_change":"2023-09-23T17:53:52.736573+0000","last_active":"2023-09-23T17:54:56.021390+0000","last_peered":"2023-09-23T17:54:56.021390+0000","last_clean":"2023-09-23T17:54:56.021390+0000","last_became_active":"2023-09-23T17:53:52.736320+0000","last_became_peered":"2023-09-23T17:53:52.736320+0000","last_unstale":"2023-09-23T17:54:56.021390+0000","last_undegraded":"2023-09-23T17:54:56.021390+0000","last_fullsized":"2023-09-23T17:54:56.021390+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:05.760412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025639+0000","last_change":"2023-09-23T17:54:25.602420+0000","last_active":"2023-09-23T17:54:56.025639+0000","last_peered":"2023-09-23T17:54:56.025639+0000","last_clean":"2023-09-23T17:54:56.025639+0000","last_became_active":"2023-09-23T17:54:25.601988+0000","last_became_peered":"2023-09-23T17:54:25.601988+0000","last_unstale":"2023-09-23T17:54:56.025639+0000","last_undegraded":"2023-09-23T17:54:56.025639+0000","last_fullsized":"2023-09-23T17:54:56.025639+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:17.703211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":79,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022486+0000","last_change":"2023-09-23T17:54:25.603149+0000","last_active":"2023-09-23T17:54:56.022486+0000","last_peered":"2023-09-23T17:54:56.022486+0000","last_clean":"2023-09-23T17:54:56.022486+0000","last_became_active":"2023-09-23T17:54:25.602846+0000","last_became_peered":"2023-09-23T17:54:25.602846+0000","last_unstale":"2023-09-23T17:54:56.022486+0000","last_undegraded":"2023-09-23T17:54:56.022486+0000","last_fullsized":"2023-09-23T17:54:56.022486+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:06.955023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":71,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029512+0000","last_change":"2023-09-23T17:53:52.732276+0000","last_active":"2023-09-23T17:54:56.029512+0000","last_peered":"2023-09-23T17:54:56.029512+0000","last_clean":"2023-09-23T17:54:56.029512+0000","last_became_active":"2023-09-23T17:53:52.731952+0000","last_became_peered":"2023-09-23T17:53:52.731952+0000","last_unstale":"2023-09-23T17:54:56.029512+0000","last_undegraded":"2023-09-23T17:54:56.029512+0000","last_fullsized":"2023-09-23T17:54:56.029512+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:06:07.529125+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036113+0000","last_change":"2023-09-23T17:54:57.036113+0000","last_active":"2023-09-23T17:54:57.036113+0000","last_peered":"2023-09-23T17:54:57.036113+0000","last_clean":"2023-09-23T17:54:57.036113+0000","last_became_active":"2023-09-23T17:54:57.035704+0000","last_became_peered":"2023-09-23T17:54:57.035704+0000","last_unstale":"2023-09-23T17:54:57.036113+0000","last_undegraded":"2023-09-23T17:54:57.036113+0000","last_fullsized":"2023-09-23T17:54:57.036113+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:10.397866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":70,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.035203+0000","last_change":"2023-09-23T17:54:57.035203+0000","last_active":"2023-09-23T17:54:57.035203+0000","last_peered":"2023-09-23T17:54:57.035203+0000","last_clean":"2023-09-23T17:54:57.035203+0000","last_became_active":"2023-09-23T17:54:57.034854+0000","last_became_peered":"2023-09-23T17:54:57.034854+0000","last_unstale":"2023-09-23T17:54:57.035203+0000","last_undegraded":"2023-09-23T17:54:57.035203+0000","last_fullsized":"2023-09-23T17:54:57.035203+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:34.657128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.026231+0000","last_change":"2023-09-23T17:53:52.731322+0000","last_active":"2023-09-23T17:54:57.026231+0000","last_peered":"2023-09-23T17:54:57.026231+0000","last_clean":"2023-09-23T17:54:57.026231+0000","last_became_active":"2023-09-23T17:53:52.730995+0000","last_became_peered":"2023-09-23T17:53:52.730995+0000","last_unstale":"2023-09-23T17:54:57.026231+0000","last_undegraded":"2023-09-23T17:54:57.026231+0000","last_fullsized":"2023-09-23T17:54:57.026231+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:55.121531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":69,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040294+0000","last_change":"2023-09-23T17:54:57.040294+0000","last_active":"2023-09-23T17:54:57.040294+0000","last_peered":"2023-09-23T17:54:57.040294+0000","last_clean":"2023-09-23T17:54:57.040294+0000","last_became_active":"2023-09-23T17:54:57.039943+0000","last_became_peered":"2023-09-23T17:54:57.039943+0000","last_unstale":"2023-09-23T17:54:57.040294+0000","last_undegraded":"2023-09-23T17:54:57.040294+0000","last_fullsized":"2023-09-23T17:54:57.040294+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:38.474814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":80,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029374+0000","last_change":"2023-09-23T17:54:25.601375+0000","last_active":"2023-09-23T17:54:56.029374+0000","last_peered":"2023-09-23T17:54:56.029374+0000","last_clean":"2023-09-23T17:54:56.029374+0000","last_became_active":"2023-09-23T17:54:25.601051+0000","last_became_peered":"2023-09-23T17:54:25.601051+0000","last_unstale":"2023-09-23T17:54:56.029374+0000","last_undegraded":"2023-09-23T17:54:56.029374+0000","last_fullsized":"2023-09-23T17:54:56.029374+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:19:24.772636+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":55,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.028296+0000","last_change":"2023-09-23T17:45:51.129514+0000","last_active":"2023-09-23T17:54:56.028296+0000","last_peered":"2023-09-23T17:54:56.028296+0000","last_clean":"2023-09-23T17:54:56.028296+0000","last_became_active":"2023-09-23T17:45:51.129154+0000","last_became_peered":"2023-09-23T17:45:51.129154+0000","last_unstale":"2023-09-23T17:54:56.028296+0000","last_undegraded":"2023-09-23T17:54:56.028296+0000","last_fullsized":"2023-09-23T17:54:56.028296+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:58.292683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":86,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040014+0000","last_change":"2023-09-23T17:54:57.040014+0000","last_active":"2023-09-23T17:54:57.040014+0000","last_peered":"2023-09-23T17:54:57.040014+0000","last_clean":"2023-09-23T17:54:57.040014+0000","last_became_active":"2023-09-23T17:54:57.039661+0000","last_became_peered":"2023-09-23T17:54:57.039661+0000","last_unstale":"2023-09-23T17:54:57.040014+0000","last_undegraded":"2023-09-23T17:54:57.040014+0000","last_fullsized":"2023-09-23T17:54:57.040014+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:01.077579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026256+0000","last_change":"2023-09-23T17:53:52.732018+0000","last_active":"2023-09-23T17:54:56.026256+0000","last_peered":"2023-09-23T17:54:56.026256+0000","last_clean":"2023-09-23T17:54:56.026256+0000","last_became_active":"2023-09-23T17:53:52.731641+0000","last_became_peered":"2023-09-23T17:53:52.731641+0000","last_unstale":"2023-09-23T17:54:56.026256+0000","last_undegraded":"2023-09-23T17:54:56.026256+0000","last_fullsized":"2023-09-23T17:54:56.026256+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:37.290885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":51,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029040+0000","last_change":"2023-09-23T17:45:51.297370+0000","last_active":"2023-09-23T17:54:56.029040+0000","last_peered":"2023-09-23T17:54:56.029040+0000","last_clean":"2023-09-23T17:54:56.029040+0000","last_became_active":"2023-09-23T17:45:51.297055+0000","last_became_peered":"2023-09-23T17:45:51.297055+0000","last_unstale":"2023-09-23T17:54:56.029040+0000","last_undegraded":"2023-09-23T17:54:56.029040+0000","last_fullsized":"2023-09-23T17:54:56.029040+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:52.209267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":72,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026812+0000","last_change":"2023-09-23T17:53:21.041393+0000","last_active":"2023-09-23T17:54:56.026812+0000","last_peered":"2023-09-23T17:54:56.026812+0000","last_clean":"2023-09-23T17:54:56.026812+0000","last_became_active":"2023-09-23T17:53:21.040995+0000","last_became_peered":"2023-09-23T17:53:21.040995+0000","last_unstale":"2023-09-23T17:54:56.026812+0000","last_undegraded":"2023-09-23T17:54:56.026812+0000","last_fullsized":"2023-09-23T17:54:56.026812+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:12.660494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.041584+0000","last_change":"2023-09-23T17:54:57.041584+0000","last_active":"2023-09-23T17:54:57.041584+0000","last_peered":"2023-09-23T17:54:57.041584+0000","last_clean":"2023-09-23T17:54:57.041584+0000","last_became_active":"2023-09-23T17:54:57.040808+0000","last_became_peered":"2023-09-23T17:54:57.040808+0000","last_unstale":"2023-09-23T17:54:57.041584+0000","last_undegraded":"2023-09-23T17:54:57.041584+0000","last_fullsized":"2023-09-23T17:54:57.041584+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:01.079083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":79,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.026234+0000","last_change":"2023-09-23T17:53:21.034197+0000","last_active":"2023-09-23T17:54:56.026234+0000","last_peered":"2023-09-23T17:54:56.026234+0000","last_clean":"2023-09-23T17:54:56.026234+0000","last_became_active":"2023-09-23T17:53:21.033784+0000","last_became_peered":"2023-09-23T17:53:21.033784+0000","last_unstale":"2023-09-23T17:54:56.026234+0000","last_undegraded":"2023-09-23T17:54:56.026234+0000","last_fullsized":"2023-09-23T17:54:56.026234+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:33.645714+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039517+0000","last_change":"2023-09-23T17:54:57.039517+0000","last_active":"2023-09-23T17:54:57.039517+0000","last_peered":"2023-09-23T17:54:57.039517+0000","last_clean":"2023-09-23T17:54:57.039517+0000","last_became_active":"2023-09-23T17:54:57.039236+0000","last_became_peered":"2023-09-23T17:54:57.039236+0000","last_unstale":"2023-09-23T17:54:57.039517+0000","last_undegraded":"2023-09-23T17:54:57.039517+0000","last_fullsized":"2023-09-23T17:54:57.039517+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.126485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025164+0000","last_change":"2023-09-23T17:45:51.127190+0000","last_active":"2023-09-23T17:54:56.025164+0000","last_peered":"2023-09-23T17:54:56.025164+0000","last_clean":"2023-09-23T17:54:56.025164+0000","last_became_active":"2023-09-23T17:45:51.126903+0000","last_became_peered":"2023-09-23T17:45:51.126903+0000","last_unstale":"2023-09-23T17:54:56.025164+0000","last_undegraded":"2023-09-23T17:54:56.025164+0000","last_fullsized":"2023-09-23T17:54:56.025164+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:55.349259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":68,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.042234+0000","last_change":"2023-09-23T17:54:57.042234+0000","last_active":"2023-09-23T17:54:57.042234+0000","last_peered":"2023-09-23T17:54:57.042234+0000","last_clean":"2023-09-23T17:54:57.042234+0000","last_became_active":"2023-09-23T17:54:57.041918+0000","last_became_peered":"2023-09-23T17:54:57.041918+0000","last_unstale":"2023-09-23T17:54:57.042234+0000","last_undegraded":"2023-09-23T17:54:57.042234+0000","last_fullsized":"2023-09-23T17:54:57.042234+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:44.128871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":51,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.039643+0000","last_change":"2023-09-23T17:54:57.038461+0000","last_active":"2023-09-23T17:54:57.039643+0000","last_peered":"2023-09-23T17:54:57.039643+0000","last_clean":"2023-09-23T17:54:57.039643+0000","last_became_active":"2023-09-23T17:54:57.038210+0000","last_became_peered":"2023-09-23T17:54:57.038210+0000","last_unstale":"2023-09-23T17:54:57.039643+0000","last_undegraded":"2023-09-23T17:54:57.039643+0000","last_fullsized":"2023-09-23T17:54:57.039643+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:50.077733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040422+0000","last_change":"2023-09-23T17:54:57.040422+0000","last_active":"2023-09-23T17:54:57.040422+0000","last_peered":"2023-09-23T17:54:57.040422+0000","last_clean":"2023-09-23T17:54:57.040422+0000","last_became_active":"2023-09-23T17:54:57.040085+0000","last_became_peered":"2023-09-23T17:54:57.040085+0000","last_unstale":"2023-09-23T17:54:57.040422+0000","last_undegraded":"2023-09-23T17:54:57.040422+0000","last_fullsized":"2023-09-23T17:54:57.040422+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:47.846454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":73,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.025676+0000","last_change":"2023-09-23T17:54:25.602669+0000","last_active":"2023-09-23T17:54:56.025676+0000","last_peered":"2023-09-23T17:54:56.025676+0000","last_clean":"2023-09-23T17:54:56.025676+0000","last_became_active":"2023-09-23T17:54:25.602338+0000","last_became_peered":"2023-09-23T17:54:25.602338+0000","last_unstale":"2023-09-23T17:54:56.025676+0000","last_undegraded":"2023-09-23T17:54:56.025676+0000","last_fullsized":"2023-09-23T17:54:56.025676+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:02.816165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.022313+0000","last_change":"2023-09-23T17:53:52.737008+0000","last_active":"2023-09-23T17:54:56.022313+0000","last_peered":"2023-09-23T17:54:56.022313+0000","last_clean":"2023-09-23T17:54:56.022313+0000","last_became_active":"2023-09-23T17:53:52.736702+0000","last_became_peered":"2023-09-23T17:53:52.736702+0000","last_unstale":"2023-09-23T17:54:56.022313+0000","last_undegraded":"2023-09-23T17:54:56.022313+0000","last_fullsized":"2023-09-23T17:54:56.022313+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:14.384950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":75,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021344+0000","last_change":"2023-09-23T17:45:51.124742+0000","last_active":"2023-09-23T17:54:56.021344+0000","last_peered":"2023-09-23T17:54:56.021344+0000","last_clean":"2023-09-23T17:54:56.021344+0000","last_became_active":"2023-09-23T17:45:51.124342+0000","last_became_peered":"2023-09-23T17:45:51.124342+0000","last_unstale":"2023-09-23T17:54:56.021344+0000","last_undegraded":"2023-09-23T17:54:56.021344+0000","last_fullsized":"2023-09-23T17:54:56.021344+0000","mapping_epoch":63,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:24.351549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.032458+0000","last_change":"2023-09-23T17:54:25.603148+0000","last_active":"2023-09-23T17:54:56.032458+0000","last_peered":"2023-09-23T17:54:56.032458+0000","last_clean":"2023-09-23T17:54:56.032458+0000","last_became_active":"2023-09-23T17:54:25.602859+0000","last_became_peered":"2023-09-23T17:54:25.602859+0000","last_unstale":"2023-09-23T17:54:56.032458+0000","last_undegraded":"2023-09-23T17:54:56.032458+0000","last_fullsized":"2023-09-23T17:54:56.032458+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:19.802583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":74,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.043487+0000","last_change":"2023-09-23T17:54:57.043487+0000","last_active":"2023-09-23T17:54:57.043487+0000","last_peered":"2023-09-23T17:54:57.043487+0000","last_clean":"2023-09-23T17:54:57.043487+0000","last_became_active":"2023-09-23T17:54:57.043064+0000","last_became_peered":"2023-09-23T17:54:57.043064+0000","last_unstale":"2023-09-23T17:54:57.043487+0000","last_undegraded":"2023-09-23T17:54:57.043487+0000","last_fullsized":"2023-09-23T17:54:57.043487+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:34:21.634412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.038085+0000","last_change":"2023-09-23T17:54:57.038085+0000","last_active":"2023-09-23T17:54:57.038085+0000","last_peered":"2023-09-23T17:54:57.038085+0000","last_clean":"2023-09-23T17:54:57.038085+0000","last_became_active":"2023-09-23T17:54:57.037551+0000","last_became_peered":"2023-09-23T17:54:57.037551+0000","last_unstale":"2023-09-23T17:54:57.038085+0000","last_undegraded":"2023-09-23T17:54:57.038085+0000","last_fullsized":"2023-09-23T17:54:57.038085+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:20:29.582747+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":69,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.029706+0000","last_change":"2023-09-23T17:53:52.732689+0000","last_active":"2023-09-23T17:54:56.029706+0000","last_peered":"2023-09-23T17:54:56.029706+0000","last_clean":"2023-09-23T17:54:56.029706+0000","last_became_active":"2023-09-23T17:53:52.732382+0000","last_became_peered":"2023-09-23T17:53:52.732382+0000","last_unstale":"2023-09-23T17:54:56.029706+0000","last_undegraded":"2023-09-23T17:54:56.029706+0000","last_fullsized":"2023-09-23T17:54:56.029706+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:34.408802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034454+0000","last_change":"2023-09-23T17:54:57.034454+0000","last_active":"2023-09-23T17:54:57.034454+0000","last_peered":"2023-09-23T17:54:57.034454+0000","last_clean":"2023-09-23T17:54:57.034454+0000","last_became_active":"2023-09-23T17:54:57.034128+0000","last_became_peered":"2023-09-23T17:54:57.034128+0000","last_unstale":"2023-09-23T17:54:57.034454+0000","last_undegraded":"2023-09-23T17:54:57.034454+0000","last_fullsized":"2023-09-23T17:54:57.034454+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:39.315294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":50,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036930+0000","last_change":"2023-09-23T17:54:57.036930+0000","last_active":"2023-09-23T17:54:57.036930+0000","last_peered":"2023-09-23T17:54:57.036930+0000","last_clean":"2023-09-23T17:54:57.036930+0000","last_became_active":"2023-09-23T17:54:57.036631+0000","last_became_peered":"2023-09-23T17:54:57.036631+0000","last_unstale":"2023-09-23T17:54:57.036930+0000","last_undegraded":"2023-09-23T17:54:57.036930+0000","last_fullsized":"2023-09-23T17:54:57.036930+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:08.367357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":70,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.024125+0000","last_change":"2023-09-23T17:53:52.732305+0000","last_active":"2023-09-23T17:54:56.024125+0000","last_peered":"2023-09-23T17:54:56.024125+0000","last_clean":"2023-09-23T17:54:56.024125+0000","last_became_active":"2023-09-23T17:53:52.731894+0000","last_became_peered":"2023-09-23T17:53:52.731894+0000","last_unstale":"2023-09-23T17:54:56.024125+0000","last_undegraded":"2023-09-23T17:54:56.024125+0000","last_fullsized":"2023-09-23T17:54:56.024125+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:30.224762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":89,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.031226+0000","last_change":"2023-09-23T17:54:57.031226+0000","last_active":"2023-09-23T17:54:57.031226+0000","last_peered":"2023-09-23T17:54:57.031226+0000","last_clean":"2023-09-23T17:54:57.031226+0000","last_became_active":"2023-09-23T17:54:57.030880+0000","last_became_peered":"2023-09-23T17:54:57.030880+0000","last_unstale":"2023-09-23T17:54:57.031226+0000","last_undegraded":"2023-09-23T17:54:57.031226+0000","last_fullsized":"2023-09-23T17:54:57.031226+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:49.836129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":52,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.040141+0000","last_change":"2023-09-23T17:54:57.039438+0000","last_active":"2023-09-23T17:54:57.040141+0000","last_peered":"2023-09-23T17:54:57.040141+0000","last_clean":"2023-09-23T17:54:57.040141+0000","last_became_active":"2023-09-23T17:54:57.039163+0000","last_became_peered":"2023-09-23T17:54:57.039163+0000","last_unstale":"2023-09-23T17:54:57.040141+0000","last_undegraded":"2023-09-23T17:54:57.040141+0000","last_fullsized":"2023-09-23T17:54:57.040141+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.265833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":46,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.034381+0000","last_change":"2023-09-23T17:54:57.034381+0000","last_active":"2023-09-23T17:54:57.034381+0000","last_peered":"2023-09-23T17:54:57.034381+0000","last_clean":"2023-09-23T17:54:57.034381+0000","last_became_active":"2023-09-23T17:54:57.034043+0000","last_became_peered":"2023-09-23T17:54:57.034043+0000","last_unstale":"2023-09-23T17:54:57.034381+0000","last_undegraded":"2023-09-23T17:54:57.034381+0000","last_fullsized":"2023-09-23T17:54:57.034381+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:40.217762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":71,"reported_epoch":109,"state":"active+clean","last_fresh":"2023-09-23T17:54:57.036156+0000","last_change":"2023-09-23T17:54:57.036156+0000","last_active":"2023-09-23T17:54:57.036156+0000","last_peered":"2023-09-23T17:54:57.036156+0000","last_clean":"2023-09-23T17:54:57.036156+0000","last_became_active":"2023-09-23T17:54:57.035478+0000","last_became_peered":"2023-09-23T17:54:57.035478+0000","last_unstale":"2023-09-23T17:54:57.036156+0000","last_undegraded":"2023-09-23T17:54:57.036156+0000","last_fullsized":"2023-09-23T17:54:57.036156+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:53.126638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":88,"reported_epoch":108,"state":"active+clean","last_fresh":"2023-09-23T17:54:56.021007+0000","last_change":"2023-09-23T17:54:25.605945+0000","last_active":"2023-09-23T17:54:56.021007+0000","last_peered":"2023-09-23T17:54:56.021007+0000","last_clean":"2023-09-23T17:54:56.021007+0000","last_became_active":"2023-09-23T17:54:25.605583+0000","last_became_peered":"2023-09-23T17:54:25.605583+0000","last_unstale":"2023-09-23T17:54:56.021007+0000","last_undegraded":"2023-09-23T17:54:56.021007+0000","last_fullsized":"2023-09-23T17:54:56.021007+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:14.076789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5995,"num_read_kb":13212,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266822,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":13024,"kb_used_data":1688,"kb_used_omap":0,"kb_used_meta":11328,"kb_avail":93732128,"statfs":{"total":95995035648,"available":95981699072,"internally_reserved":0,"allocated":1728512,"data_stored":1206046,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11599872},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55500000000000005,"15min":0.55200000000000005},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81200000000000006,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.66700000000000004},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57199999999999995,"15min":0.56899999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.59499999999999997}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.496},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.45200000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:54:01 2023","interfaces":[{"interface":"back","average":{"1min":0.55400000000000005,"5min":0.58099999999999996,"15min":0.54400000000000004},"min":{"1min":0.436,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.79400000000000004,"5min":1.0409999999999999,"15min":1.9370000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.60399999999999998,"15min":0.63200000000000001},"min":{"1min":0.42399999999999999,"5min":0.36799999999999999,"15min":0.35899999999999999},"max":{"1min":0.75800000000000001,"5min":0.91300000000000003,"15min":8.8499999999999996},"last":0.52400000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.50900000000000001,"5min":0.49399999999999999,"15min":0.44600000000000001},"min":{"1min":0.312,"5min":0.252,"15min":0.252},"max":{"1min":0.80300000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.52100000000000002,"5min":0.52300000000000002,"15min":0.47499999999999998},"min":{"1min":0.34699999999999998,"5min":0.30599999999999999,"15min":0.30399999999999999},"max":{"1min":0.81799999999999995,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.60499999999999998}]},{"osd":5,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.52200000000000002,"5min":0.53000000000000003,"15min":0.48999999999999999},"min":{"1min":0.33200000000000002,"5min":0.29599999999999999,"15min":0.28699999999999998},"max":{"1min":0.72799999999999998,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.627},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.57999999999999996,"15min":0.51400000000000001},"min":{"1min":0.40100000000000002,"5min":0.30099999999999999,"15min":0.29099999999999998},"max":{"1min":0.83899999999999997,"5min":0.83899999999999997,"15min":0.83899999999999997},"last":0.69099999999999995}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":0.75900000000000001},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.64300000000000002}]}]},{"osd":6,"up_from":45,"seq":193273528458,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":8344,"kb_used_data":1680,"kb_used_omap":0,"kb_used_meta":6656,"kb_avail":93736808,"statfs":{"total":95995035648,"available":95986491392,"internally_reserved":0,"allocated":1720320,"data_stored":1205919,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6815744},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.81100000000000005,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.51300000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.78300000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.495}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.45200000000000001},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.47099999999999997}]},{"osd":2,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.54200000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":0.40600000000000003}]},{"osd":3,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.38800000000000001,"5min":0.36799999999999999,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":0.80700000000000005,"15min":4.5599999999999996},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.625,"5min":0.58499999999999996,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34999999999999998,"15min":0.30399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":1.6850000000000001},"last":0.67000000000000004}]},{"osd":4,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.47099999999999997,"15min":0.45200000000000001},"min":{"1min":0.29999999999999999,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":0.748,"5min":0.76400000000000001,"15min":0.82899999999999996},"last":0.64200000000000002},{"interface":"front","average":{"1min":0.51900000000000002,"5min":0.52800000000000002,"15min":0.49199999999999999},"min":{"1min":0.34999999999999998,"5min":0.28899999999999998,"15min":0.27800000000000002},"max":{"1min":0.86599999999999999,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.65400000000000003}]},{"osd":5,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.53400000000000003,"15min":0.496},"min":{"1min":0.33700000000000002,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":0.77700000000000002,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.56599999999999995,"15min":0.52200000000000002},"min":{"1min":0.36699999999999999,"5min":0.33300000000000002,"15min":0.33200000000000002},"max":{"1min":0.79700000000000004,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.58399999999999996}]},{"osd":7,"last update":"Sat Sep 23 17:55:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.58999999999999997,"15min":0.53700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":0.83299999999999996,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.57499999999999996,"15min":0.51900000000000002},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":0.81299999999999994,"5min":0.82299999999999995,"15min":1.591},"last":0.622}]}]},{"osd":1,"up_from":98,"seq":420906795031,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":28716,"kb_used_data":1124,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93716436,"statfs":{"total":95995035648,"available":95965630464,"internally_reserved":0,"allocated":1150976,"data_stored":617993,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":28238525},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"last":0.308},{"interface":"front","average":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.976,"5min":1.976,"15min":1.976},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":0.377}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41099999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":0.68300000000000005,"15min":0.68300000000000005},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":0.96799999999999997,"5min":0.96799999999999997,"15min":0.96799999999999997},"last":0.75600000000000001},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.42899999999999999,"5min":0.42899999999999999,"15min":0.42899999999999999},"max":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"last":0.77200000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.71899999999999997,"5min":0.71899999999999997,"15min":0.71899999999999997},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.006,"5min":1.006,"15min":1.006},"last":0.66300000000000003},{"interface":"front","average":{"1min":0.70699999999999996,"5min":0.70699999999999996,"15min":0.70699999999999996},"min":{"1min":0.48799999999999999,"5min":0.48799999999999999,"15min":0.48799999999999999},"max":{"1min":1,"5min":1,"15min":1},"last":0.69299999999999995}]},{"osd":6,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":1.0449999999999999,"5min":1.0449999999999999,"15min":1.0449999999999999},"last":0.72699999999999998},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.1000000000000001,"5min":1.1000000000000001,"15min":1.1000000000000001},"last":0.71699999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:54:52 2023","interfaces":[{"interface":"back","average":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.91700000000000004,"5min":0.91700000000000004,"15min":0.91700000000000004},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":1.0269999999999999,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.74099999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958558,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":33408,"kb_used_data":5816,"kb_used_omap":7,"kb_used_meta":27576,"kb_avail":93711744,"statfs":{"total":95995035648,"available":95960825856,"internally_reserved":0,"allocated":5955584,"data_stored":5404139,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":28238564},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:54:51 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.37,"15min":0.37},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.70399999999999996,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 17:54:23 2023","interfaces":[{"interface":"back","average":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.47799999999999998,"15min":0.47799999999999998},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":0.626,"5min":0.626,"15min":0.626},"last":0.40500000000000003}]},{"osd":4,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82199999999999995,"5min":0.872,"15min":0.872},"last":0.69599999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61699999999999999,"15min":0.621},"min":{"1min":0.44700000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77700000000000002,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":0.64600000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.624,"15min":0.63},"min":{"1min":0.41799999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.73599999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":0.61099999999999999},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.63900000000000001,"15min":0.64300000000000002},"min":{"1min":0.46600000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.67900000000000005}]},{"osd":6,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.42899999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65200000000000002},"min":{"1min":0.52700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.81100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.66300000000000003}]},{"osd":7,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65900000000000003,"15min":0.66300000000000003},"min":{"1min":0.51600000000000001,"5min":0.435,"15min":0.435},"max":{"1min":0.84299999999999997,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.59799999999999998},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.66600000000000004,"15min":0.67000000000000004},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86599999999999999,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":0.71799999999999997}]}]},{"osd":2,"up_from":103,"seq":442381631505,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32744,"kb_used_data":5216,"kb_used_omap":11,"kb_used_meta":27508,"kb_avail":93712408,"statfs":{"total":95995035648,"available":95961505792,"internally_reserved":0,"allocated":5341184,"data_stored":4811538,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":28168567},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.52100000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.47099999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42299999999999999}]},{"osd":4,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.62}]},{"osd":5,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.60699999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.69699999999999995},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.65500000000000003}]},{"osd":7,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.63400000000000001},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.67800000000000005}]}]},{"osd":3,"up_from":108,"seq":463856467979,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":32604,"kb_used_data":5204,"kb_used_omap":3,"kb_used_meta":27388,"kb_avail":93712548,"statfs":{"total":95995035648,"available":95961649152,"internally_reserved":0,"allocated":5328896,"data_stored":4809888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":28046056},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53600000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55200000000000005}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.75}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.95299999999999996}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.91000000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.76600000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019023,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7644,"kb_used_data":1108,"kb_used_omap":0,"kb_used_meta":6528,"kb_avail":93737508,"statfs":{"total":95995035648,"available":95987208192,"internally_reserved":0,"allocated":1134592,"data_stored":615931,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6684672},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.75600000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.67900000000000005},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.53800000000000003,"15min":0.52900000000000003},"min":{"1min":0.42699999999999999,"5min":0.376,"15min":0.376},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":0.59999999999999998}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.66400000000000003,"5min":0.66400000000000003,"15min":0.66400000000000003},"last":0.65900000000000003},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.82699999999999996,"5min":0.82699999999999996,"15min":0.82699999999999996},"last":0.61199999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.64800000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:54:03 2023","interfaces":[{"interface":"back","average":{"1min":0.57799999999999996,"5min":0.57399999999999995,"15min":0.57999999999999996},"min":{"1min":0.42599999999999999,"5min":0.34399999999999997,"15min":0.318},"max":{"1min":0.73999999999999999,"5min":0.77100000000000002,"15min":5.7119999999999997},"last":0.69899999999999995},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.59199999999999997,"15min":0.57999999999999996},"min":{"1min":0.41599999999999998,"5min":0.34999999999999998,"15min":0.307},"max":{"1min":0.70999999999999996,"5min":0.85499999999999998,"15min":5.9429999999999996},"last":0.56999999999999995}]},{"osd":5,"last update":"Sat Sep 23 17:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48399999999999999,"15min":0.438},"min":{"1min":0.379,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":0.621,"5min":0.753,"15min":1.141},"last":0.441},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.51700000000000002,"15min":0.44700000000000001},"min":{"1min":0.41499999999999998,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":0.749,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.47799999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:54:42 2023","interfaces":[{"interface":"back","average":{"1min":0.53500000000000003,"5min":0.54700000000000004,"15min":0.501},"min":{"1min":0.36099999999999999,"5min":0.33200000000000002,"15min":0.28899999999999998},"max":{"1min":0.68400000000000005,"5min":0.77400000000000002,"15min":1.276},"last":0.504},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.58799999999999997,"15min":0.53600000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.28999999999999998},"max":{"1min":0.77100000000000002,"5min":0.80900000000000005,"15min":1.1180000000000001},"last":0.52500000000000002}]},{"osd":7,"last update":"Sat Sep 23 17:55:06 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.57699999999999996,"15min":0.54700000000000004},"min":{"1min":0.40600000000000003,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":1.1839999999999999},"last":0.46400000000000002},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.57699999999999996,"15min":0.53000000000000003},"min":{"1min":0.32800000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.55700000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757389,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":11724,"kb_used_data":1092,"kb_used_omap":0,"kb_used_meta":10624,"kb_avail":93733428,"statfs":{"total":95995035648,"available":95983030272,"internally_reserved":0,"allocated":1118208,"data_stored":611371,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":10878976},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.57199999999999995,"15min":0.57599999999999996},"min":{"1min":0.39800000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.77100000000000002,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.74199999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":0.69399999999999995}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.4279999999999999,"5min":4.4279999999999999,"15min":4.4279999999999999},"last":0.79000000000000004},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":1.45,"5min":1.45,"15min":1.45},"last":0.54100000000000004}]},{"osd":2,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":0.57199999999999995},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.749}]},{"osd":3,"last update":"Sat Sep 23 17:54:27 2023","interfaces":[{"interface":"back","average":{"1min":0.57999999999999996,"5min":0.61199999999999999,"15min":0.58099999999999996},"min":{"1min":0.42499999999999999,"5min":0.41999999999999998,"15min":0.32100000000000001},"max":{"1min":0.76700000000000002,"5min":0.85799999999999998,"15min":4.6799999999999997},"last":0.71499999999999997},{"interface":"front","average":{"1min":0.66000000000000003,"5min":0.61199999999999999,"15min":0.57699999999999996},"min":{"1min":0.374,"5min":0.374,"15min":0.33100000000000002},"max":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.631},"last":0.80900000000000005}]},{"osd":4,"last update":"Sat Sep 23 17:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.48899999999999999,"15min":0.44800000000000001},"min":{"1min":0.34300000000000003,"5min":0.25,"15min":0.25},"max":{"1min":0.70799999999999996,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.58599999999999997},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.54400000000000004,"15min":0.51400000000000001},"min":{"1min":0.435,"5min":0.33100000000000002,"15min":0.28899999999999998},"max":{"1min":0.81399999999999995,"5min":0.83199999999999996,"15min":1.738},"last":0.67200000000000004}]},{"osd":6,"last update":"Sat Sep 23 17:54:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.55300000000000005,"15min":0.53400000000000003},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.85499999999999998,"5min":0.85499999999999998,"15min":1.778},"last":0.60099999999999998},{"interface":"front","average":{"1min":0.65000000000000002,"5min":0.61599999999999999,"15min":0.58299999999999996},"min":{"1min":0.38100000000000001,"5min":0.375,"15min":0.33200000000000002},"max":{"1min":0.90200000000000002,"5min":0.90200000000000002,"15min":1.7629999999999999},"last":0.72799999999999998}]},{"osd":7,"last update":"Sat Sep 23 17:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.56299999999999994,"15min":0.52000000000000002},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":0.83999999999999997,"5min":1.0840000000000001,"15min":1.0840000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.62,"5min":0.58799999999999997,"15min":0.54900000000000004},"min":{"1min":0.313,"5min":0.313,"15min":0.313},"max":{"1min":0.79600000000000004,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":0.76700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:55:42.840 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-09-23T17:55:42.840 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd dump 2023-09-23T17:55:43.445 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:43.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:43 smithi055 ceph-mon[184174]: from='client.34725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:43.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:43 smithi055 ceph-mon[184174]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:43.665 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:43 smithi055 ceph-mon[179668]: from='client.34725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:43.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:43 smithi055 ceph-mon[179668]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:43 smithi099 ceph-mon[167076]: from='client.34725 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:55:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:43 smithi099 ceph-mon[167076]: pgmap v221: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:44.833 INFO:teuthology.orchestra.run.smithi055.stdout:epoch 109 2023-09-23T17:55:44.833 INFO:teuthology.orchestra.run.smithi055.stdout:fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 2023-09-23T17:55:44.833 INFO:teuthology.orchestra.run.smithi055.stdout:created 2023-09-23T17:40:13.381005+0000 2023-09-23T17:55:44.833 INFO:teuthology.orchestra.run.smithi055.stdout:modified 2023-09-23T17:54:57.011846+0000 2023-09-23T17:55:44.833 INFO:teuthology.orchestra.run.smithi055.stdout:flags sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit 2023-09-23T17:55:44.833 INFO:teuthology.orchestra.run.smithi055.stdout:crush_version 22 2023-09-23T17:55:44.833 INFO:teuthology.orchestra.run.smithi055.stdout:full_ratio 0.95 2023-09-23T17:55:44.833 INFO:teuthology.orchestra.run.smithi055.stdout:backfillfull_ratio 0.9 2023-09-23T17:55:44.834 INFO:teuthology.orchestra.run.smithi055.stdout:nearfull_ratio 0.85 2023-09-23T17:55:44.834 INFO:teuthology.orchestra.run.smithi055.stdout:require_min_compat_client luminous 2023-09-23T17:55:44.834 INFO:teuthology.orchestra.run.smithi055.stdout:min_compat_client luminous 2023-09-23T17:55:44.834 INFO:teuthology.orchestra.run.smithi055.stdout:require_osd_release quincy 2023-09-23T17:55:44.834 INFO:teuthology.orchestra.run.smithi055.stdout:stretch_mode_enabled false 2023-09-23T17:55:44.834 INFO:teuthology.orchestra.run.smithi055.stdout:pool 1 '.mgr' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 1 pgp_num 1 autoscale_mode off last_change 21 flags hashpspool stripe_width 0 pg_num_max 32 pg_num_min 1 application mgr read_balance_score 7.89 2023-09-23T17:55:44.834 INFO:teuthology.orchestra.run.smithi055.stdout:pool 2 'rbd' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 8 pgp_num 8 autoscale_mode off last_change 84 flags hashpspool,selfmanaged_snaps stripe_width 0 application rbd read_balance_score 2.00 2023-09-23T17:55:44.834 INFO:teuthology.orchestra.run.smithi055.stdout:pool 3 'cephfs.foo.meta' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 66 flags hashpspool stripe_width 0 pg_autoscale_bias 4 pg_num_min 16 recovery_priority 5 application cephfs read_balance_score 1.50 2023-09-23T17:55:44.834 INFO:teuthology.orchestra.run.smithi055.stdout:pool 4 'cephfs.foo.data' replicated size 3 min_size 2 crush_rule 0 object_hash rjenkins pg_num 32 pgp_num 32 autoscale_mode off last_change 67 flags hashpspool stripe_width 0 application cephfs read_balance_score 2.25 2023-09-23T17:55:44.834 INFO:teuthology.orchestra.run.smithi055.stdout:max_osd 8 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 up in weight 1 up_from 93 up_thru 108 down_at 90 last_clean_interval [8,89) [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049] [v2:172.21.15.55:6804/3605839049,v1:172.21.15.55:6805/3605839049] exists,up 99e7cae6-389e-461a-8a86-a5789673908b 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1 up in weight 1 up_from 98 up_thru 108 down_at 95 last_clean_interval [13,94) [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277] [v2:172.21.15.55:6812/4278491277,v1:172.21.15.55:6813/4278491277] exists,up bfb0da01-7a06-430b-b973-209f22ac30d4 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 up in weight 1 up_from 103 up_thru 108 down_at 100 last_clean_interval [18,99) [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182] [v2:172.21.15.55:6820/1645904182,v1:172.21.15.55:6821/1645904182] exists,up f766d604-db54-4a09-9993-fe1a3ce0048e 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 up in weight 1 up_from 108 up_thru 108 down_at 105 last_clean_interval [25,104) [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856] [v2:172.21.15.55:6828/3148055856,v1:172.21.15.55:6829/3148055856] exists,up 9f7bc62d-c839-4d05-89c7-98e5abd0a121 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 up in weight 1 up_from 30 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307] [v2:172.21.15.99:6802/2570729307,v1:172.21.15.99:6803/2570729307] exists,up a9c0950d-2618-4a47-ad13-035e2bd227f1 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5 up in weight 1 up_from 38 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267] [v2:172.21.15.99:6810/3690413267,v1:172.21.15.99:6811/3690413267] exists,up 64ac7fc4-adc3-4e7f-8512-a458d0092f49 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6 up in weight 1 up_from 45 up_thru 105 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865] [v2:172.21.15.99:6818/2102700865,v1:172.21.15.99:6819/2102700865] exists,up 9be63962-c98b-4df9-8781-270556e51119 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7 up in weight 1 up_from 53 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889] [v2:172.21.15.99:6826/453077889,v1:172.21.15.99:6827/453077889] exists,up ea011d8c-d9f8-4619-9ac0-575813be808a 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:pg_upmap_items 4.a [1,2] 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:pg_upmap_items 4.d [1,5] 2023-09-23T17:55:44.835 INFO:teuthology.orchestra.run.smithi055.stdout:pg_upmap_items 4.f [1,2] 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/1552377843 expires 2023-09-24T17:49:06.276090+0000 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/1663054712 expires 2023-09-24T17:45:12.276860+0000 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/4290529019 expires 2023-09-24T17:45:12.276860+0000 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/2173356416 expires 2023-09-24T17:41:12.624150+0000 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:6800/106254940 expires 2023-09-24T17:41:12.624150+0000 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.99:0/3564577630 expires 2023-09-24T17:48:26.829766+0000 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/591596032 expires 2023-09-24T17:41:12.624150+0000 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/1826983981 expires 2023-09-24T17:40:48.304146+0000 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.99:6835/2823676616 expires 2023-09-24T17:48:26.829766+0000 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.99:0/2950334811 expires 2023-09-24T17:48:26.829766+0000 2023-09-23T17:55:44.836 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/880894918 expires 2023-09-24T17:47:42.374356+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.99:0/1490815752 expires 2023-09-24T17:48:26.829766+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/1355725764 expires 2023-09-24T17:49:06.276090+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:6801/106254940 expires 2023-09-24T17:41:12.624150+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:6800/2971878511 expires 2023-09-24T17:45:12.276860+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:6800/32209804 expires 2023-09-24T17:47:42.374356+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.99:0/2680034432 expires 2023-09-24T17:48:26.829766+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/3452781072 expires 2023-09-24T17:49:06.276090+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.99:0/3129059336 expires 2023-09-24T17:48:26.829766+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:6801/470581910 expires 2023-09-24T17:40:48.304146+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/3953275940 expires 2023-09-24T17:45:12.276860+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/3757490612 expires 2023-09-24T17:49:06.276090+0000 2023-09-23T17:55:44.837 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/658967358 expires 2023-09-24T17:40:48.304146+0000 2023-09-23T17:55:44.838 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/573100440 expires 2023-09-24T17:47:42.374356+0000 2023-09-23T17:55:44.838 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:6800/470581910 expires 2023-09-24T17:40:48.304146+0000 2023-09-23T17:55:44.838 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/2880364962 expires 2023-09-24T17:47:42.374356+0000 2023-09-23T17:55:44.838 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/218576037 expires 2023-09-24T17:45:12.276860+0000 2023-09-23T17:55:44.838 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/238250704 expires 2023-09-24T17:41:12.624150+0000 2023-09-23T17:55:44.838 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.99:6834/2823676616 expires 2023-09-24T17:48:26.829766+0000 2023-09-23T17:55:44.838 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:6801/2816080264 expires 2023-09-24T17:49:06.276090+0000 2023-09-23T17:55:44.838 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/880528732 expires 2023-09-24T17:40:48.304146+0000 2023-09-23T17:55:44.838 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/2064251389 expires 2023-09-24T17:47:42.374356+0000 2023-09-23T17:55:44.838 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:6801/2971878511 expires 2023-09-24T17:45:12.276860+0000 2023-09-23T17:55:44.839 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/3781541742 expires 2023-09-24T17:49:06.276090+0000 2023-09-23T17:55:44.839 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.99:0/1538541481 expires 2023-09-24T17:48:26.829766+0000 2023-09-23T17:55:44.839 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:6800/2816080264 expires 2023-09-24T17:49:06.276090+0000 2023-09-23T17:55:44.839 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:6801/32209804 expires 2023-09-24T17:47:42.374356+0000 2023-09-23T17:55:44.839 INFO:teuthology.orchestra.run.smithi055.stdout:blocklist 172.21.15.55:0/2453190559 expires 2023-09-24T17:49:06.276090+0000 2023-09-23T17:55:45.197 INFO:tasks.cephadm.ceph_manager.ceph:['osd.0 up in weight 1 up_from 93 up_thru 108 down_at 90 last_clean_interval [8,89) [v2:172.21.15.55:6802/3605839049,v1:172.21.15.55:6803/3605839049] [v2:172.21.15.55:6804/3605839049,v1:172.21.15.55:6805/3605839049] exists,up 99e7cae6-389e-461a-8a86-a5789673908b', 'osd.1 up in weight 1 up_from 98 up_thru 108 down_at 95 last_clean_interval [13,94) [v2:172.21.15.55:6810/4278491277,v1:172.21.15.55:6811/4278491277] [v2:172.21.15.55:6812/4278491277,v1:172.21.15.55:6813/4278491277] exists,up bfb0da01-7a06-430b-b973-209f22ac30d4', 'osd.2 up in weight 1 up_from 103 up_thru 108 down_at 100 last_clean_interval [18,99) [v2:172.21.15.55:6818/1645904182,v1:172.21.15.55:6819/1645904182] [v2:172.21.15.55:6820/1645904182,v1:172.21.15.55:6821/1645904182] exists,up f766d604-db54-4a09-9993-fe1a3ce0048e', 'osd.3 up in weight 1 up_from 108 up_thru 108 down_at 105 last_clean_interval [25,104) [v2:172.21.15.55:6826/3148055856,v1:172.21.15.55:6827/3148055856] [v2:172.21.15.55:6828/3148055856,v1:172.21.15.55:6829/3148055856] exists,up 9f7bc62d-c839-4d05-89c7-98e5abd0a121', 'osd.4 up in weight 1 up_from 30 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6800/2570729307,v1:172.21.15.99:6801/2570729307] [v2:172.21.15.99:6802/2570729307,v1:172.21.15.99:6803/2570729307] exists,up a9c0950d-2618-4a47-ad13-035e2bd227f1', 'osd.5 up in weight 1 up_from 38 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6808/3690413267,v1:172.21.15.99:6809/3690413267] [v2:172.21.15.99:6810/3690413267,v1:172.21.15.99:6811/3690413267] exists,up 64ac7fc4-adc3-4e7f-8512-a458d0092f49', 'osd.6 up in weight 1 up_from 45 up_thru 105 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6816/2102700865,v1:172.21.15.99:6817/2102700865] [v2:172.21.15.99:6818/2102700865,v1:172.21.15.99:6819/2102700865] exists,up 9be63962-c98b-4df9-8781-270556e51119', 'osd.7 up in weight 1 up_from 53 up_thru 108 down_at 0 last_clean_interval [0,0) [v2:172.21.15.99:6824/453077889,v1:172.21.15.99:6825/453077889] [v2:172.21.15.99:6826/453077889,v1:172.21.15.99:6827/453077889] exists,up ea011d8c-d9f8-4619-9ac0-575813be808a'] 2023-09-23T17:55:45.198 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon mon.a config show 2023-09-23T17:55:45.803 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:45 smithi099 ceph-mon[167076]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2785414517' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T17:55:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:45 smithi055 ceph-mon[179668]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2785414517' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T17:55:45.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:45 smithi055 ceph-mon[184174]: pgmap v222: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:45.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2785414517' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T17:55:47.010 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:55:47.010 INFO:teuthology.orchestra.run.smithi055.stdout: "name": "mon.a", 2023-09-23T17:55:47.010 INFO:teuthology.orchestra.run.smithi055.stdout: "cluster": "ceph", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "admin_socket": "/var/run/ceph/ceph-mon.a.asok", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "admin_socket_mode": "", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "auth_allow_insecure_global_id_reclaim": "false", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "auth_client_required": "cephx, none", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "auth_cluster_required": "cephx", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "auth_debug": "true", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "auth_expose_insecure_global_id_reclaim": "true", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "auth_mon_ticket_ttl": "660.000000", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "auth_service_required": "cephx", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "auth_service_ticket_ttl": "240.000000", 2023-09-23T17:55:47.011 INFO:teuthology.orchestra.run.smithi055.stdout: "auth_supported": "", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_aio": "true", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_aio_max_queue_depth": "1024", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_aio_poll_ms": "250", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_aio_reap_max": "16", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_async_discard": "false", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_block_size": "4096", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_debug_aio": "false", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_debug_aio_log_age": "5.000000", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_debug_aio_suicide_timeout": "60.000000", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_debug_inflight_ios": "false", 2023-09-23T17:55:47.012 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_enable_discard": "false", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_flock_retry": "3", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_flock_retry_interval": "0.100000", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_inject_crash": "0", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_inject_crash_flush_delay": "2", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_ioring": "false", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_ioring_hipri": "false", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_ioring_sqthread_poll": "false", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_nvme_unbind_from_kernel": "false", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_read_buffer_alignment": "4096", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_read_preallocated_huge_buffers": "", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bdev_type": "", 2023-09-23T17:55:47.013 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_alloc_size": "1048576", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_allocator": "hybrid", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_buffered_io": "true", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_check_for_zeros": "false", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_check_volume_selector_often": "false", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_check_volume_selector_on_umount": "false", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_compact_log_sync": "false", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_failed_shared_alloc_cooldown": "600.000000", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_log_compact_min_ratio": "5.000000", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_log_compact_min_size": "16777216", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_log_replay_check_allocations": "true", 2023-09-23T17:55:47.014 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_max_log_runway": "4194304", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_max_prefetch": "1048576", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_min_flush_size": "524288", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_min_log_runway": "1048576", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_replay_recovery": "false", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_replay_recovery_disable_compact": "false", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_shared_alloc_size": "65536", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluefs_sync_write": "false", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_2q_cache_kin_ratio": "0.500000", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_2q_cache_kout_ratio": "0.500000", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_alloc_stats_dump_interval": "86400.000000", 2023-09-23T17:55:47.015 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_allocation_from_file": "true", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_allocator": "hybrid", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_avl_alloc_bf_free_pct": "4", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_avl_alloc_bf_threshold": "131072", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_avl_alloc_ff_max_search_bytes": "16777216", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_avl_alloc_ff_max_search_count": "100", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_bitmapallocator_blocks_per_zone": "1024", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_bitmapallocator_span_size": "1024", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_blobid_prealloc": "10240", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_block_create": "true", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_block_db_create": "false", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_block_db_path": "", 2023-09-23T17:55:47.016 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_block_db_size": "0", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_block_path": "", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_block_preallocate_file": "false", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_block_size": "107374182400", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_block_wal_create": "false", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_block_wal_path": "", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_block_wal_size": "100663296", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_bluefs": "true", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_bluefs_alloc_failure_dump_interval": "0.000000", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_bluefs_env_mirror": "false", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_bluefs_max_free": "10737418240", 2023-09-23T17:55:47.017 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_age_bin_interval": "1.000000", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_age_bins_data": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_age_bins_kv": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_age_bins_kv_onode": "0 0 0 0 0 0 0 0 0 720", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_age_bins_meta": "1 2 6 24 120 720 0 0 0 0", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_autotune": "true", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_autotune_interval": "5.000000", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_kv_onode_ratio": "0.040000", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_kv_ratio": "0.450000", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_meta_ratio": "0.450000", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_size": "0", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_size_hdd": "1073741824", 2023-09-23T17:55:47.018 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_size_ssd": "3221225472", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_trim_interval": "0.050000", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_trim_max_skip_pinned": "1000", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cache_type": "2q", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_cleaner_sleep_interval": "5.000000", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_clone_cow": "true", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_compression_algorithm": "snappy", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_compression_max_blob_size": "0", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_compression_max_blob_size_hdd": "65536", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_compression_max_blob_size_ssd": "65536", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_compression_min_blob_size": "0", 2023-09-23T17:55:47.019 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_compression_min_blob_size_hdd": "8192", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_compression_min_blob_size_ssd": "65536", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_compression_mode": "none", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_compression_required_ratio": "0.875000", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_csum_type": "crc32c", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_enforce_settings": "default", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_freelist": "false", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_fsck_abort": "false", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_inject_allocation_from_file_failure": "0.000000", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_inject_bug21040": "false", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_inject_csum_err_probability": "0.000000", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_inject_read_err": "false", 2023-09-23T17:55:47.020 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_legacy_omap": "false", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_no_reuse_blocks": "false", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_omit_block_device_write": "false", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_omit_kv_commit": "false", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_permit_any_bdev_label": "false", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_prefill": "0.000000", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_prefragment_max": "1048576", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_random_read_err": "0.000000", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_randomize_serial_transaction": "0", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_small_allocations": "0", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_debug_too_many_blobs_threshold": "24576", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_default_buffered_read": "true", 2023-09-23T17:55:47.021 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_default_buffered_write": "false", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_deferred_batch_ops": "0", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_deferred_batch_ops_hdd": "64", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_deferred_batch_ops_ssd": "16", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_extent_map_inline_shard_prealloc_size": "256", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_extent_map_shard_max_size": "1200", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_extent_map_shard_min_size": "150", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_extent_map_shard_target_size": "500", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_extent_map_shard_target_size_slop": "0.200000", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fail_eio": "false", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_freelist_blocks_per_key": "128", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_error_on_no_per_pg_omap": "false", 2023-09-23T17:55:47.022 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_error_on_no_per_pool_omap": "false", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_error_on_no_per_pool_stats": "false", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_on_mkfs": "true", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_on_mkfs_deep": "false", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_on_mount": "false", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_on_mount_deep": "false", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_on_umount": "false", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_on_umount_deep": "false", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_quick_fix_on_mount": "false", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_quick_fix_threads": "2", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_read_bytes_cap": "67108864", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_fsck_shared_blob_tracker_size": "0.031250", 2023-09-23T17:55:47.023 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_gc_enable_blob_threshold": "0", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_gc_enable_total_threshold": "0", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_hybrid_alloc_mem_cap": "67108864", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_ignore_data_csum": "false", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_kv_sync_util_logging_s": "10.000000", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_kvbackend": "rocksdb", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_log_collection_list_age": "60.000000", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_log_omap_iterator_age": "5.000000", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_log_op_age": "5.000000", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_max_alloc_size": "0", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_max_blob_size": "0", 2023-09-23T17:55:47.024 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_max_blob_size_hdd": "65536", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_max_blob_size_ssd": "65536", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_max_defer_interval": "3.000000", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_max_deferred_txc": "32", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_min_alloc_size": "0", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_min_alloc_size_hdd": "4096", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_min_alloc_size_ssd": "4096", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_nid_prealloc": "1024", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_prefer_deferred_size": "0", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_prefer_deferred_size_hdd": "65536", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_prefer_deferred_size_ssd": "0", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_qfsck_on_mount": "true", 2023-09-23T17:55:47.025 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_retry_disk_reads": "3", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_rocksdb_cf": "true", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_rocksdb_cfs": "m(3) p(3,0-12) O(3,0-13)=block_cache={type=binned_lru} L=min_write_buffer_number_to_merge=32 P=min_write_buffer_number_to_merge=32", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_rocksdb_options": "compression=kNoCompression,max_write_buffer_number=64,min_write_buffer_number_to_merge=6,compaction_style=kCompactionStyleLevel,write_buffer_size=16777216,max_background_jobs=4,level0_file_num_compaction_trigger=8,max_bytes_for_level_base=1073741824,max_bytes_for_level_multiplier=8,compaction_readahead_size=2MB,max_total_wal_size=1073741824,writable_file_max_buffer_size=0", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_rocksdb_options_annex": "", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_spdk_coremask": "0x1", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_spdk_io_sleep": "5", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_spdk_max_io_completion": "0", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_spdk_mem": "512", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_sync_submit_transaction": "false", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_throttle_bytes": "67108864", 2023-09-23T17:55:47.026 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_throttle_cost_per_io": "0", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_throttle_cost_per_io_hdd": "670000", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_throttle_cost_per_io_ssd": "4000", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_throttle_deferred_bytes": "134217728", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_throttle_trace_rate": "0.000000", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_tracing": "false", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_use_optimal_io_size_for_min_alloc_size": "false", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_volume_selection_policy": "use_some_extra", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_volume_selection_reserved": "0", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_volume_selection_reserved_factor": "2.000000", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_warn_on_bluefs_spillover": "true", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_warn_on_legacy_statfs": "true", 2023-09-23T17:55:47.027 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_warn_on_no_per_pg_omap": "false", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_warn_on_no_per_pool_omap": "true", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_warn_on_spurious_read_errors": "true", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "bluestore_zero_block_detection": "false", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "cephadm_path": "/usr/sbin/cephadm", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "cephfs_mirror_action_update_interval": "2", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "cephfs_mirror_directory_scan_interval": "10", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "cephfs_mirror_max_concurrent_directory_syncs": "3", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "cephfs_mirror_max_consecutive_failures_per_directory": "10", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "cephfs_mirror_max_snapshot_sync_per_cycle": "3", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "cephfs_mirror_mount_timeout": "10", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "cephfs_mirror_restart_mirror_on_blocklist_interval": "30", 2023-09-23T17:55:47.028 INFO:teuthology.orchestra.run.smithi055.stdout: "cephfs_mirror_restart_mirror_on_failure_interval": "20", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephfs_mirror_retry_failed_directories_interval": "60", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephsqlite_blocklist_dead_locker": "true", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephsqlite_lock_renewal_interval": "2000", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephsqlite_lock_renewal_timeout": "30000", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephx_cluster_require_signatures": "false", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephx_cluster_require_version": "2", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephx_require_signatures": "false", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephx_require_version": "2", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephx_service_require_signatures": "false", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephx_service_require_version": "2", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "cephx_sign_messages": "true", 2023-09-23T17:55:47.029 INFO:teuthology.orchestra.run.smithi055.stdout: "chdir": "", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_acl_type": "", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_asio_thread_count": "2", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_cache_mid": "0.750000", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_cache_size": "16384", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_caps_release_delay": "5", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_check_pool_perm": "true", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_collect_and_send_global_metrics": "false", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_debug_force_sync_read": "false", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_debug_getattr_caps": "false", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_debug_inject_tick_delay": "0", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_die_on_failed_dentry_invalidate": "true", 2023-09-23T17:55:47.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_die_on_failed_remount": "false", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_dirsize_rbytes": "true", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_force_lazyio": "false", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_fs": "", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_inject_fixed_oldest_tid": "false", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_inject_release_failure": "false", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_max_inline_size": "4096", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_max_retries_on_remount_failure": "5", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_mds_namespace": "", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_metadata": "", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_mount_gid": "-1", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_mount_timeout": "300", 2023-09-23T17:55:47.031 INFO:teuthology.orchestra.run.smithi055.stdout: "client_mount_uid": "-1", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_mountpoint": "/", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_notify_timeout": "10", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_oc": "true", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_oc_max_dirty": "104857600", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_oc_max_dirty_age": "5.000000", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_oc_max_objects": "1000", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_oc_size": "209715200", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_oc_target_dirty": "8388608", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_permissions": "true", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_quota": "true", 2023-09-23T17:55:47.032 INFO:teuthology.orchestra.run.smithi055.stdout: "client_quota_df": "true", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_readahead_max_bytes": "0", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_readahead_max_periods": "4", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_readahead_min": "131072", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_reconnect_stale": "false", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_shutdown_timeout": "30", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_snapdir": ".snap", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_tick_interval": "1", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_trace": "", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_try_dentry_invalidate": "false", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_use_faked_inos": "false", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_use_random_mds": "false", 2023-09-23T17:55:47.033 INFO:teuthology.orchestra.run.smithi055.stdout: "clog_to_graylog": "false", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "clog_to_graylog_host": "127.0.0.1", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "clog_to_graylog_port": "12201", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "clog_to_monitors": "default=true", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "clog_to_syslog": "false", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "clog_to_syslog_facility": "default=daemon audit=local0", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "clog_to_syslog_level": "info", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "cluster_addr": "-", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "cluster_network": "", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "cluster_network_interface": "", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "compressor_zlib_isal": "false", 2023-09-23T17:55:47.034 INFO:teuthology.orchestra.run.smithi055.stdout: "compressor_zlib_level": "5", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "compressor_zlib_winsize": "-15", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "compressor_zstd_level": "1", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "container_image": "quay.ceph.io/ceph-ci/ceph@sha256:50a9ac7e1b843c0c455a7342fc5c680f52d11adad0155cbe9df9c48a9a29f68d", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "crash_dir": "/var/lib/ceph/crash", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "crimson_alien_op_num_threads": "6", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "crimson_alien_thread_cpu_cores": "", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "crimson_osd_obc_lru_size": "10", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "crimson_osd_scheduler_concurrency": "0", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "crimson_seastar_smp": "1", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "crush_location": "", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "crush_location_hook": "", 2023-09-23T17:55:47.035 INFO:teuthology.orchestra.run.smithi055.stdout: "crush_location_hook_timeout": "10", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "daemonize": "false", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "daos_pool": "tank", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "dbstore_config_uri": "file:/var/lib/ceph/radosgw/dbstore-config.db", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "dbstore_db_dir": "/var/lib/ceph/radosgw", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "dbstore_db_name_prefix": "dbstore", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_alienstore": "0/5", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_allow_any_pool_priority": "false", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_asok": "1/5", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_asok_assert_abort": "false", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_asserts_on_shutdown": "true", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_auth": "1/5", 2023-09-23T17:55:47.036 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_bdev": "1/3", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_bluefs": "1/5", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_bluestore": "1/5", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_buffer": "0/1", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_ceph_exporter": "1/5", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_cephfs_mirror": "0/5", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_cephsqlite": "0/5", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_client": "0/5", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_compressor": "1/5", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_context": "0/1", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_crush": "1/1", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_crypto": "1/5", 2023-09-23T17:55:47.037 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_cyanstore": "0/5", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_deliberately_leak_memory": "false", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_disable_randomized_ping": "false", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_dpdk": "1/5", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_eventtrace": "1/5", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_filer": "0/1", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_filestore": "1/3", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_finisher": "1/1", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_fuse": "1/5", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_heartbeat_testing_span": "0", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_heartbeatmap": "1/5", 2023-09-23T17:55:47.038 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_immutable_obj_cache": "0/5", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_javaclient": "1/5", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_journal": "1/3", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_journaler": "0/5", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_kstore": "1/5", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_leveldb": "4/5", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_lockdep": "0/1", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_mclock": "1/5", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_mds": "1/5", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_mds_balancer": "1/5", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_mds_locker": "1/5", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_mds_log": "1/5", 2023-09-23T17:55:47.039 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_mds_log_expire": "1/5", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_mds_migrator": "1/5", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_memstore": "1/5", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_mgr": "2/5", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_mgrc": "1/5", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_mon": "20/20", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_monc": "0/10", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_ms": "1/1", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_none": "0/5", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_objclass": "0/5", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_objectcacher": "0/5", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_objecter": "0/1", 2023-09-23T17:55:47.040 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_optracker": "0/5", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_osd": "1/5", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_paxos": "20/20", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_perfcounter": "1/5", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_prioritycache": "1/5", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rados": "0/5", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rbd": "0/5", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rbd_mirror": "0/5", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rbd_pwl": "0/5", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rbd_replay": "0/5", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_refs": "0/0", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_reserver": "1/1", 2023-09-23T17:55:47.041 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rgw": "1/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rgw_access": "1/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rgw_datacache": "1/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rgw_dbstore": "1/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rgw_flight": "1/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rgw_sync": "1/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_rocksdb": "4/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore": "0/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_backref": "0/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_cache": "0/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_cleaner": "0/5", 2023-09-23T17:55:47.042 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_device": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_epm": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_fixedkv_tree": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_journal": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_lba": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_odata": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_omap": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_onode": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_t": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_seastore_tm": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_striper": "0/1", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_test": "0/5", 2023-09-23T17:55:47.043 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_throttle": "1/1", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_timer": "0/1", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "debug_tp": "0/5", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "device_failure_prediction_mode": "none", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "enable_experimental_unrecoverable_data_corrupting_features": "", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "erasure_code_dir": "/usr/lib64/ceph/erasure-code", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "err_to_graylog": "false", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "err_to_journald": "false", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "err_to_stderr": "true", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "err_to_syslog": "false", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "event_tracing": "false", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "exporter_addr": "0.0.0.0", 2023-09-23T17:55:47.044 INFO:teuthology.orchestra.run.smithi055.stdout: "exporter_http_port": "9926", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "exporter_prio_limit": "5", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "exporter_sock_dir": "/var/run/ceph/", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "exporter_sort_metrics": "true", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "exporter_stats_period": "5", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "fake_statfs_for_testing": "0", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "fatal_signal_handlers": "true", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "filer_max_purge_ops": "10", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "filer_max_truncate_ops": "128", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_apply_finisher_threads": "1", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_blackhole": "false", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_btrfs_clone_range": "true", 2023-09-23T17:55:47.045 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_btrfs_snap": "true", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_caller_concurrency": "10", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_collect_device_partition_information": "true", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_commit_timeout": "600.000000", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_debug_inject_read_err": "false", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_debug_omap_check": "false", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_debug_random_read_err": "0.000000", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_debug_verify_split": "false", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_dump_file": "", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_expected_throughput_bytes": "209715200.000000", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_expected_throughput_ops": "200.000000", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_fadvise": "true", 2023-09-23T17:55:47.046 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_fail_eio": "true", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_fd_cache_shards": "16", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_fd_cache_size": "128", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_fiemap": "false", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_fiemap_threshold": "4096", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_fsync_flushes_journal_data": "false", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_index_retry_probability": "0.000000", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_inject_stall": "0", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_journal_parallel": "false", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_journal_trailing": "false", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_journal_writeahead": "false", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_kill_at": "0", 2023-09-23T17:55:47.047 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_alloc_hint_size": "1048576", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_inline_xattr_size": "0", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_inline_xattr_size_btrfs": "2048", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_inline_xattr_size_other": "512", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_inline_xattr_size_xfs": "65536", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_inline_xattrs": "0", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_inline_xattrs_btrfs": "10", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_inline_xattrs_other": "2", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_inline_xattrs_xfs": "10", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_sync_interval": "5.000000", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_xattr_value_size": "0", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_xattr_value_size_btrfs": "65536", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_xattr_value_size_other": "1024", 2023-09-23T17:55:47.048 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_max_xattr_value_size_xfs": "65536", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_merge_threshold": "-10", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_min_sync_interval": "0.010000", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_odsync_write": "false", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_omap_backend": "rocksdb", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_omap_backend_path": "", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_omap_header_cache_size": "1024", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_ondisk_finisher_threads": "1", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_op_thread_suicide_timeout": "180", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_op_thread_timeout": "60", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_op_threads": "2", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_punch_hole": "false", 2023-09-23T17:55:47.049 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_queue_high_delay_multiple": "0.000000", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_queue_high_delay_multiple_bytes": "0.000000", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_queue_high_delay_multiple_ops": "0.000000", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_queue_high_threshhold": "0.900000", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_queue_low_threshhold": "0.300000", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_queue_max_bytes": "104857600", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_queue_max_delay_multiple": "0.000000", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_queue_max_delay_multiple_bytes": "0.000000", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_queue_max_delay_multiple_ops": "0.000000", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_queue_max_ops": "50", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_rocksdb_options": "max_background_jobs=10,compaction_readahead_size=2097152,compression=kNoCompression", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_seek_data_hole": "false", 2023-09-23T17:55:47.050 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_sloppy_crc": "false", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_sloppy_crc_block_size": "65536", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_splice": "false", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_split_multiple": "2", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_split_rand_factor": "20", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_update_to": "1000", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_btrfs_bytes_hard_limit": "419430400", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_btrfs_bytes_start_flusher": "41943040", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_btrfs_inodes_hard_limit": "5000", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_btrfs_inodes_start_flusher": "500", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_btrfs_ios_hard_limit": "5000", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_btrfs_ios_start_flusher": "500", 2023-09-23T17:55:47.051 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_enable": "true", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_xfs_bytes_hard_limit": "419430400", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_xfs_bytes_start_flusher": "41943040", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_xfs_inodes_hard_limit": "5000", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_xfs_inodes_start_flusher": "500", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_xfs_ios_hard_limit": "5000", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_wbthrottle_xfs_ios_start_flusher": "500", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_xfs_extsize": "false", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "filestore_zfs_snap": "false", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "fio_dir": "/tmp/fio", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "fsid": "2b70c030-5a38-11ee-8db4-212e2dc638e7", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_allow_other": "true", 2023-09-23T17:55:47.052 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_atomic_o_trunc": "true", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_big_writes": "true", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_debug": "false", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_default_permissions": "false", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_disable_pagecache": "false", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_max_write": "0", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_multithreaded": "true", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_require_active_mds": "true", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_set_user_groups": "true", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_splice_move": "true", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_splice_read": "true", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_splice_write": "true", 2023-09-23T17:55:47.053 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_syncfs_on_mksnap": "true", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "fuse_use_invalidate_cb": "true", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "gss_ktab_client_file": "/var/lib/ceph/mon.a/gss_client_mon.a.ktab", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "gss_target_name": "ceph", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "heartbeat_file": "", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "heartbeat_inject_failure": "0", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "heartbeat_interval": "5", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "host": "smithi055", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_client_dedicated_thread_num": "2", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_max_inflight_ops": "128", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_max_size": "1073741824", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_path": "/tmp/ceph_immutable_object_cache", 2023-09-23T17:55:47.054 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_qos_bps_burst": "0", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_qos_bps_burst_seconds": "1", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_qos_bps_limit": "0", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_qos_iops_burst": "0", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_qos_iops_burst_seconds": "1", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_qos_iops_limit": "0", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_qos_schedule_tick_min": "50", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_sock": "/var/run/ceph/immutable_object_cache_sock", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "immutable_object_cache_watermark": "0.900000", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "inject_early_sigterm": "false", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "jaeger_agent_port": "6799", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "jaeger_tracing_enable": "false", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_aio": "true", 2023-09-23T17:55:47.055 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_align_min_size": "65536", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_block_align": "true", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_block_size": "4096", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_dio": "true", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_discard": "false", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_force_aio": "false", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_ignore_corruption": "false", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_max_write_bytes": "10485760", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_max_write_entries": "100", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_replay_from": "0", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_throttle_high_multiple": "0.000000", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_throttle_high_threshhold": "0.900000", 2023-09-23T17:55:47.056 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_throttle_low_threshhold": "0.600000", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_throttle_max_multiple": "0.000000", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_write_header_frequency": "0", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "journal_zero_on_create": "false", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "journaler_prefetch_periods": "10", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "journaler_prezero_periods": "5", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "journaler_write_head_interval": "15", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "key": "", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "keyfile": "", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "keyring": "/var/lib/ceph/mon/ceph-a/keyring", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_backend": "rocksdb", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_default_stripe_size": "65536", 2023-09-23T17:55:47.057 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_fsck_on_mount": "false", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_fsck_on_mount_deep": "true", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_max_bytes": "67108864", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_max_ops": "512", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_nid_prealloc": "1024", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_onode_map_size": "1024", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_rocksdb_options": "compression=kNoCompression", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_sync_submit_transaction": "false", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "kstore_sync_transaction": "false", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "leveldb_block_size": "65536", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "leveldb_bloom_size": "0", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "leveldb_cache_size": "536870912", 2023-09-23T17:55:47.058 INFO:teuthology.orchestra.run.smithi055.stdout: "leveldb_compact_on_mount": "false", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "leveldb_compression": "false", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "leveldb_log": "", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "leveldb_log_to_ceph_log": "true", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "leveldb_max_open_files": "0", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "leveldb_paranoid": "false", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "leveldb_write_buffer_size": "33554432", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "librados_thread_count": "2", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "lockdep": "false", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "lockdep_force_backtrace": "false", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "log_coarse_timestamps": "true", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "log_file": "/var/log/ceph/ceph-mon.a.log", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "log_flush_on_exit": "false", 2023-09-23T17:55:47.059 INFO:teuthology.orchestra.run.smithi055.stdout: "log_graylog_host": "127.0.0.1", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "log_graylog_port": "12201", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "log_max_new": "1000", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "log_max_recent": "10000", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "log_stderr_prefix": "", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "log_stop_at_utilization": "0.970000", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "log_to_file": "true", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "log_to_graylog": "false", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "log_to_journald": "false", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "log_to_stderr": "false", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "log_to_syslog": "false", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "max_rotating_auth_attempts": "10", 2023-09-23T17:55:47.060 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_abort_on_newly_corrupt_dentry": "true", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_action_on_write_error": "1", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_alternate_name_max": "8192", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_asio_thread_count": "2", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_export_pin": "true", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_fragment_dirs": "true", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_fragment_fast_factor": "1.500000", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_fragment_interval": "5", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_fragment_size_max": "100000", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_idle_threshold": "0.000000", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_interval": "10", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_max": "-1", 2023-09-23T17:55:47.061 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_max_until": "-1", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_merge_size": "50", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_midchunk": "0.300000", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_min_rebalance": "0.100000", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_min_start": "0.200000", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_minchunk": "0.001000", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_mode": "0", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_need_max": "1.200000", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_need_min": "0.800000", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_replicate_threshold": "8000.000000", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_sample_interval": "3.000000", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_split_bits": "3", 2023-09-23T17:55:47.062 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_split_rd": "25000.000000", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_split_size": "10000", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_split_wr": "10000.000000", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_target_decay": "10.000000", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_bal_unreplicate_threshold": "0.000000", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_beacon_grace": "15.000000", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_beacon_interval": "4.000000", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_beacon_mon_down_grace": "60", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_cache_memory_limit": "4294967296", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_cache_mid": "0.700000", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_cache_release_free_interval": "10", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_cache_reservation": "0.050000", 2023-09-23T17:55:47.063 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_cache_trim_decay_rate": "1.000000", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_cache_trim_interval": "1", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_cache_trim_threshold": "262144", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_cap_acquisition_throttle_retry_request_timeout": "0.500000", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_cap_revoke_eviction_timeout": "0.000000", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_client_delegate_inos_pct": "50", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_client_prealloc_inos": "1000", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_client_writeable_range_max_inc_objs": "1024", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_connect_bootstrapping": "false", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_damage_table_max_entries": "10000", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_data": "/var/lib/ceph/mds/ceph-a", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_debug_auth_pins": "false", 2023-09-23T17:55:47.064 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_debug_frag": "false", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_debug_scatterstat": "false", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_debug_subtrees": "false", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_decay_halflife": "5.000000", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_default_dir_hash": "2", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_defer_session_stale": "true", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_deny_all_reconnect": "false", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_dir_keys_per_op": "16384", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_dir_max_commit_size": "10", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_dir_max_entries": "0", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_dir_prefetch": "true", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_dirstat_min_interval": "1.000000", 2023-09-23T17:55:47.065 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_dump_cache_after_rejoin": "false", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_dump_cache_on_map": "false", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_dump_cache_threshold_file": "0", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_dump_cache_threshold_formatter": "1073741824", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_early_reply": "true", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_enable_op_tracker": "true", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_enforce_unique_name": "true", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_export_ephemeral_distributed": "true", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_export_ephemeral_distributed_factor": "2.000000", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_export_ephemeral_random": "true", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_export_ephemeral_random_max": "0.010000", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_extraordinary_events_dump_interval": "0", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_forward_all_requests_to_auth": "false", 2023-09-23T17:55:47.066 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_freeze_tree_timeout": "30.000000", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_fscrypt_last_block_max_size": "4096", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_go_bad_corrupt_dentry": "true", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_hack_allow_loading_invalid_metadata": "false", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_health_cache_threshold": "1.500000", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_health_summarize_threshold": "10", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_heartbeat_grace": "15.000000", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_heartbeat_reset_grace": "1000", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_inject_health_dummy": "false", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_inject_journal_corrupt_dentry_first": "0.000000", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_inject_migrator_session_race": "false", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_inject_rename_corrupt_dentry_first": "0.000000", 2023-09-23T17:55:47.067 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_inject_skip_replaying_inotable": "false", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_inject_traceless_reply_probability": "0.000000", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_join_fs": "", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_journal_format": "1", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_create_at": "0", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_export_at": "0", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_import_at": "0", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_journal_at": "0", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_journal_expire_at": "0", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_journal_replay_at": "0", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_link_at": "0", 2023-09-23T17:55:47.068 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_mdstable_at": "0", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_openc_at": "0", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_rename_at": "0", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_kill_skip_replaying_inotable": "false", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_log_events_per_segment": "1024", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_log_max_events": "-1", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_log_max_segments": "128", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_log_pause": "false", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_log_segment_size": "0", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_log_skip_corrupt_events": "false", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_log_warn_factor": "2.000000", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_caps_per_client": "1048576", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_completed_flushes": "100000", 2023-09-23T17:55:47.069 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_completed_requests": "100000", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_export_size": "20971520", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_file_recover": "32", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_purge_files": "64", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_purge_ops": "8192", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_purge_ops_per_pg": "0.500000", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_scrub_ops_in_progress": "5", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_snaps_per_dir": "100", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_max_xattr_pairs_size": "65536", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_metrics_update_interval": "2", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_min_caps_per_client": "100", 2023-09-23T17:55:47.070 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_min_caps_working_set": "10000", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_mon_shutdown_timeout": "5.000000", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_numa_node": "-1", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_oft_prefetch_dirfrags": "false", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_op_complaint_time": "30.000000", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_op_history_duration": "600", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_op_history_size": "20", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_op_log_threshold": "5", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_ping_grace": "15", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_ping_interval": "5", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_purge_queue_busy_flush_period": "1.000000", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_recall_global_max_decay_threshold": "131072", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_recall_max_caps": "30000", 2023-09-23T17:55:47.071 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_recall_max_decay_rate": "1.500000", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_recall_max_decay_threshold": "131072", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_recall_warning_decay_rate": "60.000000", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_recall_warning_threshold": "262144", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_reconnect_timeout": "45.000000", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_replay_interval": "1.000000", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_replay_unsafe_with_closed_session": "false", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_request_load_average_decay_rate": "60.000000", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_root_ino_gid": "0", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_root_ino_uid": "0", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_scatter_nudge_interval": "5.000000", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_session_blocklist_on_evict": "true", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_session_blocklist_on_timeout": "true", 2023-09-23T17:55:47.072 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_session_cache_liveness_decay_rate": "300.000000", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_session_cache_liveness_magnitude": "10", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_session_cap_acquisition_decay_rate": "30.000000", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_session_cap_acquisition_throttle": "100000", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_session_max_caps_throttle_ratio": "1.100000", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_sessionmap_keys_per_op": "1024", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_shutdown_check": "0", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_skip_ino": "0", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_sleep_rank_change": "0.000000", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_snap_max_uid": "4294967294", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_snap_min_uid": "0", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_snap_rstat": "false", 2023-09-23T17:55:47.073 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_standby_replay_damaged": "false", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_symlink_recovery": "true", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_task_status_update_interval": "2.000000", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_thrash_exports": "0", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_thrash_fragments": "0", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_tick_interval": "5.000000", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_valgrind_exit": "false", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_verify_backtrace": "1", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_verify_scatter": "false", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_wipe_ino_prealloc": "false", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mds_wipe_sessions": "false", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "mempool_debug": "false", 2023-09-23T17:55:47.074 INFO:teuthology.orchestra.run.smithi055.stdout: "memstore_debug_omit_block_device_write": "false", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "memstore_device_bytes": "1073741824", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "memstore_page_set": "false", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "memstore_page_size": "65536", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_client_bytes": "134217728", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_client_messages": "512", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_client_service_daemon_unregister_timeout": "1.000000", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_connect_retry_interval": "1.000000", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_data": "/var/lib/ceph/mgr/ceph-a", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_debug_aggressive_pg_num_changes": "false", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_disabled_modules": "", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_initial_modules": "restful iostat nfs", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_max_pg_num_change": "128", 2023-09-23T17:55:47.075 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_mds_bytes": "134217728", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_mds_messages": "128", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_module_path": "/usr/share/ceph/mgr", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_mon_bytes": "134217728", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_mon_messages": "128", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_osd_bytes": "536870912", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_osd_messages": "8192", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_pool": "true", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_service_beacon_grace": "60.000000", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_standby_modules": "true", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_stats_period": "5", 2023-09-23T17:55:47.076 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_stats_threshold": "5", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_tick_period": "2", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mgr_ttl_cache_expire_seconds": "0", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_accept_timeout_factor": "2.000000", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_allow_pool_delete": "true", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_allow_pool_size_one": "false", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_auth_validate_all_caps": "true", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cache_target_full_warn_ratio": "0.660000", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_clean_pg_upmaps_per_chunk": "256", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_bytes": "104857600", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_directed_command_retry": "2", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_hunt_interval": "3.000000", 2023-09-23T17:55:47.077 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_hunt_interval_backoff": "1.500000", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_hunt_interval_max_multiple": "10.000000", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_hunt_interval_min_multiple": "1.000000", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_hunt_parallel": "3", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_log_interval": "1.000000", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_max_log_entries_per_message": "1000", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_ping_interval": "10.000000", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_client_ping_timeout": "30.000000", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_clock_drift_allowed": "1.000000", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_clock_drift_warn_backoff": "5.000000", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_file": "default=/var/log/ceph/ceph.$channel.log cluster=/var/log/ceph/ceph.log", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_file_level": "debug", 2023-09-23T17:55:47.078 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_to_file": "false", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_to_graylog": "false", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_to_graylog_host": "127.0.0.1", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_to_graylog_port": "12201", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_to_journald": "true", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_to_stderr": "false", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_to_syslog": "default=false", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_to_syslog_facility": "daemon", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cluster_log_to_syslog_level": "info", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_compact_on_bootstrap": "false", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_compact_on_start": "false", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_compact_on_trim": "true", 2023-09-23T17:55:47.079 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_con_tracker_persist_interval": "10", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_con_tracker_score_halflife": "43200", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_config_key_max_entry_size": "65536", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_cpu_threads": "4", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_crush_min_required_version": "hammer", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_daemon_bytes": "419430400", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_data": "/var/lib/ceph/mon/ceph-a", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_data_avail_crit": "5", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_data_avail_warn": "5", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_data_size_warn": "16106127360", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_block_osdmap_trim": "false", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_deprecated_as_obsolete": "false", 2023-09-23T17:55:47.080 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_dump_json": "false", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_dump_location": "/var/log/ceph/ceph-mon.a.tdump", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_dump_transactions": "false", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_extra_checks": "false", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_no_initial_persistent_features": "false", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_no_require_bluestore_for_ec_overwrites": "false", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_no_require_quincy": "false", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_no_require_reef": "false", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_debug_unsafe_allow_tier_with_nonempty_snaps": "false", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_delta_reset_interval": "10.000000", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_dns_srv_name": "ceph-mon", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_down_mkfs_grace": "60", 2023-09-23T17:55:47.081 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_election_default_strategy": "3", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_election_timeout": "5.000000", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_elector_ignore_propose_margin": "0.000500", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_elector_ping_divisor": "2", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_elector_ping_timeout": "2.000000", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_enable_op_tracker": "true", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_fake_pool_delete": "false", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_force_quorum_join": "false", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_globalid_prealloc": "10000", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_health_detail_to_clog": "true", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_health_log_update_period": "5", 2023-09-23T17:55:47.082 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_health_max_detail": "50", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_health_to_clog": "true", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_health_to_clog_interval": "600", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_health_to_clog_tick_interval": "60.000000", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_host": "[v2:172.21.15.55:3300/0,v1:172.21.15.55:6789/0] [v2:172.21.15.99:3300/0,v1:172.21.15.99:6789/0] [v2:172.21.15.55:3301/0,v1:172.21.15.55:6790/0]", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_host_override": "", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_initial_members": "", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_inject_pg_merge_bounce_probability": "0.000000", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_inject_sync_get_chunk_delay": "0.000000", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_inject_transaction_delay_max": "10.000000", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_inject_transaction_delay_probability": "0.000000", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_keyvaluedb": "rocksdb", 2023-09-23T17:55:47.083 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_lease": "5.000000", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_lease_ack_timeout_factor": "2.000000", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_lease_renew_interval_factor": "0.600000", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_log_full_interval": "50", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_log_max": "10000", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_log_max_summary": "50", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_max_log_entries_per_event": "4096", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_max_log_epochs": "500", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_max_mdsmap_epochs": "500", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_max_mgrmap_epochs": "500", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_max_osd": "10000", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_max_pg_per_osd": "10000", 2023-09-23T17:55:47.084 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_max_pool_pg_num": "65536", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_max_snap_prune_per_epoch": "100", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_mds_blocklist_interval": "86400.000000", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_mds_force_trim_to": "0", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_mds_skip_sanity": "true", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_memory_autotune": "true", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_memory_target": "2147483648", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_mgr_beacon_grace": "30", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_mgr_blocklist_interval": "86400.000000", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_mgr_digest_period": "5", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_mgr_inactive_grace": "60", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_mgr_mkfs_grace": "240", 2023-09-23T17:55:47.085 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_mgr_proxy_client_bytes_ratio": "0.300000", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_min_osdmap_epochs": "500", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_op_complaint_time": "30", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_op_history_duration": "600", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_op_history_size": "20", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_op_history_slow_op_size": "20", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_op_history_slow_op_threshold": "10", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_op_log_threshold": "5", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_adjust_down_out_interval": "true", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_adjust_heartbeat_grace": "true", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_auto_mark_auto_out_in": "true", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_auto_mark_in": "false", 2023-09-23T17:55:47.086 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_auto_mark_new_in": "true", 2023-09-23T17:55:47.087 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_backfillfull_ratio": "0.900000", 2023-09-23T17:55:47.089 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_blocklist_default_expire": "3600.000000", 2023-09-23T17:55:47.090 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_cache_size": "500", 2023-09-23T17:55:47.090 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_cache_size_min": "134217728", 2023-09-23T17:55:47.090 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_crush_smoke_test": "true", 2023-09-23T17:55:47.090 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_destroyed_out_interval": "600", 2023-09-23T17:55:47.090 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_down_out_interval": "600", 2023-09-23T17:55:47.090 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_down_out_subtree_limit": "rack", 2023-09-23T17:55:47.090 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_err_op_age_ratio": "128.000000", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_force_trim_to": "0", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_full_ratio": "0.950000", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_initial_require_min_compat_client": "luminous", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_laggy_halflife": "3600", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_laggy_max_interval": "300", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_laggy_weight": "0.300000", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_mapping_pgs_per_chunk": "4096", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_max_creating_pgs": "1024", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_max_initial_pgs": "1024", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_min_down_reporters": "2", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_min_in_ratio": "0.750000", 2023-09-23T17:55:47.091 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_min_up_ratio": "0.300000", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_nearfull_ratio": "0.850000", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_prime_pg_temp": "true", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_prime_pg_temp_max_estimate": "0.250000", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_prime_pg_temp_max_time": "0.500000", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_report_timeout": "900", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_reporter_subtree_level": "osd", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_snap_trim_queue_warn_on": "32768", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_warn_num_repaired": "10", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osd_warn_op_age": "32.000000", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osdmap_full_prune_enabled": "true", 2023-09-23T17:55:47.092 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osdmap_full_prune_interval": "10", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osdmap_full_prune_min": "10000", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_osdmap_full_prune_txsize": "100", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_pg_check_down_all_threshold": "0.500000", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_pg_stuck_threshold": "60", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_pg_warn_max_object_skew": "0.000000", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_pg_warn_min_objects": "10000", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_pg_warn_min_per_osd": "0", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_pg_warn_min_pool_objects": "1000", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_pool_quota_crit_threshold": "0", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_pool_quota_warn_threshold": "0", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_probe_timeout": "2.000000", 2023-09-23T17:55:47.093 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_reweight_max_change": "0.050000", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_reweight_max_osds": "4", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_reweight_min_bytes_per_osd": "10", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_reweight_min_pgs_per_osd": "4", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_rocksdb_options": "write_buffer_size=33554432,compression=kNoCompression,level_compaction_dynamic_level_bytes=true", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_scrub_inject_crc_mismatch": "0.000000", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_scrub_inject_missing_keys": "0.000000", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_scrub_interval": "86400", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_scrub_max_keys": "100", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_scrub_timeout": "300", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_session_timeout": "300", 2023-09-23T17:55:47.094 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_smart_report_timeout": "5", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_stat_smooth_intervals": "6", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_stretch_cluster_recovery_ratio": "0.600000", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_stretch_pool_min_size": "2", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_stretch_pool_size": "4", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_stretch_recovery_min_wait": "15.000000", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_subscribe_interval": "86400.000000", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_sync_debug": "false", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_sync_max_payload_keys": "2000", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_sync_max_payload_size": "1048576", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_sync_provider_kill_at": "0", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_sync_requester_kill_at": "0", 2023-09-23T17:55:47.095 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_sync_timeout": "60.000000", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_target_pg_per_osd": "100", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_tick_interval": "5", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_timecheck_interval": "300.000000", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_timecheck_skew_interval": "30.000000", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_use_min_delay_socket": "false", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_older_version_delay": "604800", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_cache_pools_without_hit_sets": "true", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_crush_straw_calc_version_zero": "false", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_degraded_stretch_mode": "true", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_filestore_osds": "true", 2023-09-23T17:55:47.096 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_insecure_global_id_reclaim": "false", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_insecure_global_id_reclaim_allowed": "false", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_legacy_crush_tunables": "false", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_misplaced": "false", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_msgr2_not_enabled": "true", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_older_version": "true", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_osd_down_out_interval_zero": "false", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_pool_no_app": "true", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_pool_no_redundancy": "true", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_pool_pg_num_not_power_of_two": "false", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_slow_ping_ratio": "0.050000", 2023-09-23T17:55:47.097 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_slow_ping_time": "0.000000", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_on_too_few_osds": "false", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_pg_not_deep_scrubbed_ratio": "0.750000", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "mon_warn_pg_not_scrubbed_ratio": "0.500000", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "monmap": "", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "motr_admin_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "motr_admin_fid": "0x7200000000000001:0x0", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "motr_ha_endpoint": "192.168.180.182@tcp:12345:1:1", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "motr_my_endpoint": "192.168.180.182@tcp:12345:4:1", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "motr_my_fid": "0x7200000000000001:0x0", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "motr_profile_fid": "0x7000000000000001:0x0", 2023-09-23T17:55:47.098 INFO:teuthology.orchestra.run.smithi055.stdout: "motr_tracing_enabled": "false", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_op_threads": "3", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_buffer_size": "131072", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_cm": "false", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_device_name": "", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_dscp": "96", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_enable_hugepage": "false", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_gid_idx": "0", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_local_gid": "", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_polling_us": "1000", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_port_num": "1", 2023-09-23T17:55:47.099 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_receive_buffers": "32768", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_receive_queue_len": "4096", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_roce_ver": "1", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_send_buffers": "1024", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_sl": "3", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_support_srq": "true", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_rdma_type": "ib", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_async_reap_threshold": "5", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_bind_before_connect": "false", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_bind_ipv4": "true", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_bind_ipv6": "false", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_bind_msgr1": "true", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_bind_msgr2": "true", 2023-09-23T17:55:47.100 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_bind_port_max": "7568", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_bind_port_min": "6800", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_bind_prefer_ipv4": "false", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_bind_retry_count": "3", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_bind_retry_delay": "5", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_blackhole_client": "false", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_blackhole_mds": "false", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_blackhole_mgr": "false", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_blackhole_mon": "false", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_blackhole_osd": "false", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_client_mode": "crc secure", 2023-09-23T17:55:47.101 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_cluster_mode": "crc secure", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_cluster_type": "", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_compress_secure": "false", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_connection_idle_timeout": "900", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_connection_ready_timeout": "10", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_crc_data": "true", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_crc_header": "true", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_die_on_bad_msg": "false", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_die_on_bug": "true", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_die_on_old_message": "true", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_die_on_skipped_message": "false", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_die_on_unhandled_msg": "false", 2023-09-23T17:55:47.102 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dispatch_throttle_bytes": "104857600", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_coremask": "0xF", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_debug_allow_loopback": "false", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_devs_allowlist": "", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_enable_tso": "true", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_gateway_ipv4_addr": "", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_host_ipv4_addr": "", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_hugepages": "", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_hw_flow_control": "true", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_hw_queue_weight": "1.000000", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_lro": "true", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_memory_channel": "4", 2023-09-23T17:55:47.103 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_netmask_ipv4_addr": "", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_pmd": "", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_port_id": "0", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dpdk_rx_buffer_count_per_core": "8192", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dump_corrupt_message_level": "1", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_dump_on_send": "false", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_initial_backoff": "0.200000", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_inject_delay_max": "1.000000", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_inject_delay_probability": "0.000000", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_inject_delay_type": "", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_inject_internal_delays": "0.000000", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_inject_network_congestion": "0", 2023-09-23T17:55:47.104 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_inject_socket_failures": "0", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_learn_addr_from_peer": "true", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_max_accept_failures": "4", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_max_backoff": "15.000000", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_mon_client_mode": "secure crc", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_mon_cluster_mode": "secure crc", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_mon_service_mode": "secure crc", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_osd_compress_min_size": "1024", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_osd_compress_mode": "none", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_osd_compression_algorithm": "snappy", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_pq_max_tokens_per_priority": "16777216", 2023-09-23T17:55:47.105 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_pq_min_cost": "65536", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_public_type": "", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_service_mode": "crc secure", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_tcp_listen_backlog": "512", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_tcp_nodelay": "true", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_tcp_prefetch_max_size": "4096", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_tcp_rcvbuf": "0", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "ms_type": "async+posix", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "no_config_file": "false", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "objecter_completion_locks_per_session": "32", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "objecter_debug_inject_relock_delay": "false", 2023-09-23T17:55:47.106 INFO:teuthology.orchestra.run.smithi055.stdout: "objecter_inflight_op_bytes": "104857600", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "objecter_inflight_ops": "1024", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "objecter_inject_no_watch_ping": "false", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "objecter_retry_writes_after_first_reply": "false", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "objecter_tick_interval": "5.000000", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "objecter_timeout": "10.000000", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "objectstore_blackhole": "false", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "openssl_engine_opts": "", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_agent_delay_time": "5.000000", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_agent_hist_halflife": "1000", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_agent_max_low_ops": "2", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_agent_max_ops": "4", 2023-09-23T17:55:47.107 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_agent_min_evict_effort": "0.100000", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_agent_quantize_effort": "0.100000", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_agent_slop": "0.020000", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_aggregated_slow_ops_logging": "true", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_allow_recovery_below_min_size": "true", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_asio_thread_count": "2", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_async_recovery_min_cost": "100", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_auto_mark_unfound_lost": "false", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_backfill_retry_interval": "30.000000", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_backfill_scan_max": "512", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_backfill_scan_min": "64", 2023-09-23T17:55:47.108 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_backoff_on_degraded": "false", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_backoff_on_peering": "false", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_backoff_on_unfound": "true", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_beacon_report_interval": "300", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_bench_duration": "30", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_bench_large_size_max_throughput": "104857600", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_bench_max_block_size": "67108864", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_bench_small_size_max_iops": "100", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_blkin_trace_all": "false", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_blocked_scrub_grace_period": "120", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_calc_pg_upmaps_aggressively": "true", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_calc_pg_upmaps_aggressively_fast": "true", 2023-09-23T17:55:47.109 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_calc_pg_upmaps_local_fallback_retries": "100", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_check_for_log_corruption": "false", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_check_max_object_name_len_on_startup": "true", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_class_default_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_class_dir": "/usr/lib64/rados-classes", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_class_load_list": "cephfs hello journal lock log numops otp rbd refcount rgw rgw_gc timeindex user version cas cmpomap queue 2pc_queue fifo", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_class_update_on_start": "true", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_client_message_cap": "256", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_client_message_size_cap": "524288000", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_client_op_priority": "63", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_client_watch_timeout": "30", 2023-09-23T17:55:47.110 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_command_max_records": "256", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_compact_on_start": "false", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_copyfrom_max_chunk": "8388608", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_crush_chooseleaf_type": "1", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_crush_initial_weight": "-1.000000", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_crush_update_on_start": "true", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_crush_update_weight_set": "true", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_data": "/var/lib/ceph/osd/ceph-a", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_crash_on_ignored_backoff": "false", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_deep_scrub_sleep": "0.000000", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_drop_ping_duration": "0", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_drop_ping_probability": "0.000000", 2023-09-23T17:55:47.111 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_feed_pullee": "-1", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_inject_copyfrom_error": "false", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_inject_dispatch_delay_duration": "0.100000", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_inject_dispatch_delay_probability": "0.000000", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_misdirected_ops": "false", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_no_acting_change": "false", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_no_purge_strays": "false", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_op_order": "false", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_pg_log_writeout": "false", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_pretend_recovery_active": "false", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_random_push_read_error": "0.000000", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_reject_backfill_probability": "0.000000", 2023-09-23T17:55:47.112 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_shutdown": "false", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_skip_full_check_in_backfill_reservation": "false", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_skip_full_check_in_recovery": "false", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_verify_cached_snaps": "false", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_verify_missing_on_start": "false", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_verify_snaps": "false", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_debug_verify_stray_on_activate": "false", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_deep_scrub_interval": "604800.000000", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_deep_scrub_keys": "1024", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_deep_scrub_large_omap_object_key_threshold": "200000", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_deep_scrub_large_omap_object_value_sum_threshold": "1073741824", 2023-09-23T17:55:47.113 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_deep_scrub_randomize_ratio": "0.150000", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_deep_scrub_stride": "524288", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_deep_scrub_update_digest_min_age": "7200", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_default_data_pool_replay_window": "45", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_default_notify_timeout": "30", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_delete_sleep": "0.000000", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_delete_sleep_hdd": "5.000000", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_delete_sleep_hybrid": "1.000000", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_delete_sleep_ssd": "1.000000", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_discard_disconnected_ops": "true", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_enable_op_tracker": "true", 2023-09-23T17:55:47.114 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_erasure_code_plugins": "jerasure lrc", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_extblkdev_plugins": "vdo", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_failsafe_full_ratio": "0.970000", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_fast_fail_on_connection_refused": "true", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_fast_info": "true", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_fast_shutdown": "true", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_fast_shutdown_notify_mon": "true", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_fast_shutdown_timeout": "15", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_find_best_info_ignore_history_les": "false", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_force_auth_primary_missing_objects": "100", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_force_recovery_pg_log_entries_factor": "1.300000", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_function_tracing": "false", 2023-09-23T17:55:47.115 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_heartbeat_grace": "20", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_heartbeat_interval": "6", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_heartbeat_min_healthy_ratio": "0.330000", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_heartbeat_min_peers": "10", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_heartbeat_min_size": "2000", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_heartbeat_stale": "600", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_heartbeat_use_min_delay_socket": "false", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_hit_set_max_size": "100000", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_hit_set_min_size": "1000", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_hit_set_namespace": ".ceph-internal", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_ignore_stale_divergent_priors": "false", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_inject_bad_map_crc_probability": "0.000000", 2023-09-23T17:55:47.116 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_inject_failure_on_pg_removal": "false", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_journal": "/var/lib/ceph/osd/ceph-a/journal", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_journal_flush_on_shutdown": "true", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_journal_size": "5120", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_kill_backfill_at": "0", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_loop_before_reset_tphandle": "64", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_map_cache_size": "50", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_map_dedup": "true", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_map_message_max": "40", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_map_message_max_bytes": "10485760", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_map_share_max_epochs": "40", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_attr_name_len": "100", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_attr_size": "0", 2023-09-23T17:55:47.117 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_backfills": "1", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_markdown_count": "5", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_markdown_period": "600", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_object_name_len": "2048", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_object_namespace_len": "256", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_object_size": "134217728", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_omap_bytes_per_request": "1073741824", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_omap_entries_per_request": "1024", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_pg_blocked_by": "16", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_pg_log_entries": "10000", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_pg_per_osd_hard_ratio": "3.000000", 2023-09-23T17:55:47.118 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_pgls": "1024", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_push_cost": "8388608", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_push_objects": "10", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_scrubs": "1", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_snap_prune_intervals_per_epoch": "512", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_trimming_pgs": "2", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_write_op_reply_len": "64", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_max_write_size": "90", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_force_run_benchmark_on_init": "false", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_iops_capacity_threshold_hdd": "500.000000", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_iops_capacity_threshold_ssd": "80000.000000", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_max_capacity_iops_hdd": "315.000000", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_max_capacity_iops_ssd": "21500.000000", 2023-09-23T17:55:47.119 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_max_sequential_bandwidth_hdd": "157286400", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_max_sequential_bandwidth_ssd": "1258291200", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_override_recovery_settings": "false", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_profile": "balanced", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_scheduler_anticipation_timeout": "0.000000", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_scheduler_background_best_effort_lim": "0.000000", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_scheduler_background_best_effort_res": "0.000000", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_scheduler_background_best_effort_wgt": "1", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_scheduler_background_recovery_lim": "0.000000", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_scheduler_background_recovery_res": "0.000000", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_scheduler_background_recovery_wgt": "1", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_scheduler_client_lim": "0.000000", 2023-09-23T17:55:47.120 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_scheduler_client_res": "0.000000", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_scheduler_client_wgt": "1", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mclock_skip_benchmark": "false", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_memory_base": "805306368", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_memory_cache_min": "134217728", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_memory_cache_resize_interval": "1.000000", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_memory_expected_fragmentation": "0.150000", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_memory_target": "4294967296", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_memory_target_autotune": "false", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_memory_target_cgroup_limit_ratio": "0.800000", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_min_pg_log_entries": "250", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_min_recovery_priority": "0", 2023-09-23T17:55:47.121 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mon_heartbeat_interval": "30", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mon_heartbeat_stat_stale": "3600", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mon_report_interval": "5", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mon_report_max_in_flight": "2", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_mon_shutdown_timeout": "5.000000", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_num_cache_shards": "32", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_num_op_tracker_shard": "32", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_numa_auto_affinity": "true", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_numa_node": "-1", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_numa_prefer_iface": "true", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_object_clean_region_max_num_intervals": "10", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_objecter_finishers": "1", 2023-09-23T17:55:47.122 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_objectstore": "bluestore", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_objectstore_fuse": "false", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_objectstore_tracing": "false", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_complaint_time": "30.000000", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_history_duration": "600", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_history_size": "20", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_history_slow_op_size": "20", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_history_slow_op_threshold": "10.000000", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_log_threshold": "5", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_num_shards": "0", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_num_shards_hdd": "5", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_num_shards_ssd": "8", 2023-09-23T17:55:47.123 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_num_threads_per_shard": "0", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_num_threads_per_shard_hdd": "1", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_num_threads_per_shard_ssd": "2", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_pq_max_tokens_per_priority": "4194304", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_pq_min_cost": "65536", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_queue": "mclock_scheduler", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_queue_cut_off": "high", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_thread_suicide_timeout": "150", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_op_thread_timeout": "15", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_open_classes_on_start": "true", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_os_flags": "0", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_peering_op_priority": "255", 2023-09-23T17:55:47.124 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pg_delete_cost": "1048576", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pg_delete_priority": "5", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pg_epoch_max_lag_factor": "2.000000", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pg_epoch_persisted_max_stale": "40", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pg_log_dups_tracked": "3000", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pg_log_trim_max": "10000", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pg_log_trim_min": "100", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pg_max_concurrent_snap_trims": "2", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pg_object_context_cache_count": "64", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pg_stat_report_interval_max": "500", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_cache_max_evict_check_size": "10", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_cache_min_evict_age": "0", 2023-09-23T17:55:47.125 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_cache_min_flush_age": "0", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_cache_target_dirty_high_ratio": "0.600000", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_cache_target_dirty_ratio": "0.400000", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_cache_target_full_ratio": "0.800000", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_crimson": "false", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_crush_rule": "-1", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_ec_fast_read": "false", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_erasure_code_profile": "plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_flag_bulk": "false", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_flag_hashpspool": "true", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_flag_nodelete": "false", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_flag_nopgchange": "false", 2023-09-23T17:55:47.126 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_flag_nosizechange": "false", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_flags": "0", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_hit_set_bloom_fpp": "0.050000", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_min_size": "0", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_pg_autoscale_mode": "off", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_pg_num": "32", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_pgp_num": "0", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_read_lease_ratio": "0.800000", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_size": "3", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_default_type": "replicated", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_erasure_code_stripe_unit": "4096", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_pool_use_gmt_hitset": "true", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_push_per_object_cost": "1000", 2023-09-23T17:55:47.127 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_read_ec_check_for_errors": "false", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recover_clone_overlap": "true", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recover_clone_overlap_limit": "10", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_cost": "20971520", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_delay_start": "0.000000", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_max_active": "0", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_max_active_hdd": "3", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_max_active_ssd": "10", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_max_chunk": "8388608", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_max_omap_entries_per_chunk": "8096", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_max_single_start": "1", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_op_priority": "3", 2023-09-23T17:55:47.128 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_op_warn_multiple": "16", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_priority": "5", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_retry_interval": "30.000000", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_sleep": "0.000000", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_sleep_hdd": "0.100000", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_sleep_hybrid": "0.025000", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_recovery_sleep_ssd": "0.000000", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_repair_during_recovery": "false", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_requested_scrub_priority": "120", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_rocksdb_iterator_bounds_enabled": "true", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_rollback_to_cluster_snap": "", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_auto_repair": "false", 2023-09-23T17:55:47.129 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_auto_repair_num_errors": "5", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_backoff_ratio": "0.660000", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_begin_hour": "0", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_begin_week_day": "0", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_chunk_max": "25", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_chunk_min": "5", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_cost": "52428800", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_during_recovery": "false", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_end_hour": "0", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_end_week_day": "0", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_event_cost": "4096", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_extended_sleep": "0.000000", 2023-09-23T17:55:47.130 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_interval_randomize_ratio": "0.500000", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_invalid_stats": "true", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_load_threshold": "0.500000", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_max_interval": "604800.000000", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_max_preemptions": "5", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_min_interval": "86400.000000", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_priority": "5", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_reservation_timeout": "5000", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_sleep": "0.000000", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_scrub_slow_reservation_response": "2200", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_shallow_scrub_chunk_max": "100", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_shallow_scrub_chunk_min": "50", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_shutdown_pgref_assert": "false", 2023-09-23T17:55:47.131 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_skip_data_digest": "false", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_smart_report_timeout": "5", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_snap_trim_cost": "1048576", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_snap_trim_priority": "5", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_snap_trim_sleep": "0.000000", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_snap_trim_sleep_hdd": "5.000000", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_snap_trim_sleep_hybrid": "2.000000", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_snap_trim_sleep_ssd": "0.000000", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_stats_update_period_not_scrubbing": "120", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_stats_update_period_scrubbing": "15", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_target_pg_log_entries_per_osd": "300000", 2023-09-23T17:55:47.132 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_target_transaction_size": "30", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tier_default_cache_hit_set_count": "4", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tier_default_cache_hit_set_grade_decay_rate": "20", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tier_default_cache_hit_set_period": "1200", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tier_default_cache_hit_set_search_last_n": "1", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tier_default_cache_hit_set_type": "bloom", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tier_default_cache_min_read_recency_for_promote": "1", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tier_default_cache_min_write_recency_for_promote": "1", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tier_default_cache_mode": "writeback", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tier_promote_max_bytes_sec": "5242880", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tier_promote_max_objects_sec": "25", 2023-09-23T17:55:47.133 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_tracing": "false", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_use_stale_snap": "false", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "osd_uuid": "00000000-0000-0000-0000-000000000000", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "osdc_blkin_trace_all": "false", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_kill_at": "0", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_max_join_drift": "10", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_min": "500", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_min_wait": "0.050000", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_propose_interval": "1.000000", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_service_trim_max": "500", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_service_trim_max_multiplier": "20", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_service_trim_min": "250", 2023-09-23T17:55:47.134 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_stash_full_interval": "25", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_trim_max": "500", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "paxos_trim_min": "250", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "perf": "true", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "pid_file": "", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "plugin_crypto_accelerator": "crypto_isal", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "plugin_dir": "/usr/lib64/ceph", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "public_addr": "-", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "public_addrv": "", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "public_bind_addr": "-", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "public_network": "172.21.0.0/20", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "public_network_interface": "", 2023-09-23T17:55:47.135 INFO:teuthology.orchestra.run.smithi055.stdout: "qat_compressor_enabled": "false", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "qat_compressor_session_max_number": "256", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rados_mon_op_timeout": "0", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rados_osd_op_timeout": "0", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rados_tracing": "false", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_atime_update_interval": "60", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_auto_exclusive_lock_until_manual_request": "true", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_balance_parent_reads": "false", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_balance_snap_reads": "false", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_blkin_trace_all": "false", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_blocklist_expire_seconds": "0", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_blocklist_on_break_lock": "true", 2023-09-23T17:55:47.136 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_cache": "true", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_cache_block_writes_upfront": "false", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_cache_max_dirty": "25165824", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_cache_max_dirty_age": "1.000000", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_cache_max_dirty_object": "0", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_cache_policy": "writearound", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_cache_size": "33554432", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_cache_target_dirty": "16777216", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_cache_writethrough_until_flush": "true", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_clone_copy_on_read": "false", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_compression_hint": "none", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_concurrent_management_ops": "10", 2023-09-23T17:55:47.137 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_config_pool_override_update_timestamp": "0", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_default_clone_format": "auto", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_default_data_pool": "", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_default_features": "61", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_default_format": "2", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_default_map_options": "", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_default_order": "22", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_default_pool": "rbd", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_default_snapshot_quiesce_mode": "required", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_default_stripe_count": "0", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_default_stripe_unit": "0", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_disable_zero_copy_writes": "true", 2023-09-23T17:55:47.138 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_discard_granularity_bytes": "65536", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_discard_on_zeroed_write_same": "true", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_enable_alloc_hint": "true", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_invalidate_object_map_on_timeout": "true", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_io_scheduler": "simple", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_io_scheduler_simple_max_delay": "0", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_commit_age": "5.000000", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_max_concurrent_object_sets": "0", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_max_payload_bytes": "16384", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_object_flush_age": "0.000000", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_object_flush_bytes": "1048576", 2023-09-23T17:55:47.139 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_object_flush_interval": "0", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_object_max_in_flight_appends": "0", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_object_writethrough_until_flush": "true", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_order": "24", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_pool": "", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_journal_splay_width": "4", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_localize_parent_reads": "false", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_localize_snap_reads": "false", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_concurrent_image_deletions": "1", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_concurrent_image_syncs": "5", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_delete_retry_interval": "30.000000", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_image_perf_stats_prio": "5", 2023-09-23T17:55:47.140 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_image_policy_migration_throttle": "300", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_image_policy_rebalance_timeout": "0.000000", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_image_policy_type": "simple", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_image_policy_update_throttle_interval": "1.000000", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_image_state_check_interval": "30", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_journal_commit_age": "5.000000", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_journal_poll_age": "5.000000", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_leader_heartbeat_interval": "5", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_leader_max_acquire_attempts_before_break": "3", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_leader_max_missed_heartbeats": "2", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_memory_autotune": "true", 2023-09-23T17:55:47.141 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_memory_base": "805306368", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_memory_cache_autotune_interval": "30.000000", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_memory_cache_min": "134217728", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_memory_cache_resize_interval": "5.000000", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_memory_expected_fragmentation": "0.150000", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_memory_target": "4294967296", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_perf_stats_prio": "5", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_pool_replayers_refresh_interval": "30", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirror_sync_point_update_age": "30.000000", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirroring_delete_delay": "0", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirroring_max_mirroring_snapshots": "5", 2023-09-23T17:55:47.142 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirroring_replay_delay": "0", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mirroring_resync_after_disconnect": "false", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_move_parent_to_trash_on_remove": "false", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_move_to_trash_on_remove": "false", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_move_to_trash_on_remove_expire_seconds": "0", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_mtime_update_interval": "60", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_non_blocking_aio": "true", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_op_thread_timeout": "60", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_op_threads": "1", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_parent_cache_enabled": "false", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_persistent_cache_mode": "disabled", 2023-09-23T17:55:47.143 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_persistent_cache_path": "/tmp", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_persistent_cache_size": "1073741824", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_plugins": "", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_bps_burst": "0", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_bps_burst_seconds": "1", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_bps_limit": "0", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_exclude_ops": "0", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_iops_burst": "0", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_iops_burst_seconds": "1", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_iops_limit": "0", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_read_bps_burst": "0", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_read_bps_burst_seconds": "1", 2023-09-23T17:55:47.144 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_read_bps_limit": "0", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_read_iops_burst": "0", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_read_iops_burst_seconds": "1", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_read_iops_limit": "0", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_schedule_tick_min": "50", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_write_bps_burst": "0", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_write_bps_burst_seconds": "1", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_write_bps_limit": "0", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_write_iops_burst": "0", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_write_iops_burst_seconds": "1", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_qos_write_iops_limit": "0", 2023-09-23T17:55:47.145 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_quiesce_notification_attempts": "10", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_read_from_replica_policy": "default", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_readahead_disable_after_bytes": "52428800", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_readahead_max_bytes": "524288", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_readahead_trigger_requests": "10", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_request_timed_out_seconds": "30", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_skip_partial_discard": "true", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_sparse_read_threshold_bytes": "65536", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_tracing": "false", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_validate_names": "true", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "rbd_validate_pool": "true", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "restapi_base_url": "", 2023-09-23T17:55:47.146 INFO:teuthology.orchestra.run.smithi055.stdout: "restapi_log_level": "", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_acl_grants_max_num": "100", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_admin_entry": "admin", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_allow_notification_secrets_in_cleartext": "false", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_backend_store": "rados", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_barbican_url": "", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_beast_enable_async": "true", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_bucket_default_quota_max_objects": "-1", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_bucket_default_quota_max_size": "-1", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_bucket_index_max_aio": "128", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_bucket_index_transaction_instrumentation": "false", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_bucket_quota_cache_size": "10000", 2023-09-23T17:55:47.147 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_bucket_quota_ttl": "600", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_bucket_sync_spawn_window": "20", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_cache_enabled": "true", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_cache_expiry_interval": "900", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_cache_lru_size": "10000", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_config_store": "rados", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_content_length_compat": "false", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_copy_obj_progress": "true", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_copy_obj_progress_every_bytes": "1048576", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_cors_rules_max_num": "100", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_cross_domain_policy": "", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_default_encryption_key": "", 2023-09-23T17:55:47.148 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_kmip_addr": "", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_kmip_ca_path": "", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_kmip_client_cert": "", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_kmip_client_key": "", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_kmip_kms_key_template": "", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_kmip_password": "", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_kmip_s3_key_template": "$keyid", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_kmip_username": "", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_require_ssl": "true", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_s3_kms_backend": "barbican", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_s3_kms_encryption_keys": "", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_backend": "vault", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_key_template": "%bucket_id", 2023-09-23T17:55:47.149 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_vault_addr": "", 2023-09-23T17:55:47.150 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_vault_auth": "token", 2023-09-23T17:55:47.150 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_vault_namespace": "", 2023-09-23T17:55:47.150 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_vault_prefix": "", 2023-09-23T17:55:47.150 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_vault_secret_engine": "transit", 2023-09-23T17:55:47.150 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_vault_ssl_cacert": "", 2023-09-23T17:55:47.150 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_vault_ssl_clientcert": "", 2023-09-23T17:55:47.150 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_vault_ssl_clientkey": "", 2023-09-23T17:55:47.150 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_vault_token_file": "", 2023-09-23T17:55:47.150 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_sse_s3_vault_verify_ssl": "true", 2023-09-23T17:55:47.150 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_suppress_logs": "true", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_vault_addr": "", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_vault_auth": "token", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_vault_namespace": "", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_vault_prefix": "", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_vault_secret_engine": "transit", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_vault_ssl_cacert": "", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_vault_ssl_clientcert": "", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_vault_ssl_clientkey": "", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_vault_token_file": "", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_crypt_vault_verify_ssl": "true", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_curl_buffersize": "524288", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_curl_low_speed_limit": "1024", 2023-09-23T17:55:47.151 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_curl_low_speed_time": "300", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_curl_tcp_keepalive": "0", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_curl_wait_timeout_ms": "1000", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_d3n_l1_datacache_persistent_path": "/tmp/rgw_datacache/", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_d3n_l1_datacache_size": "1073741824", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_d3n_l1_evict_cache_on_start": "true", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_d3n_l1_eviction_policy": "lru", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_d3n_l1_fadvise": "4", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_d3n_l1_local_datacache_enabled": "false", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_d3n_libaio_aio_num": "64", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_d3n_libaio_aio_threads": "20", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_data": "/var/lib/ceph/radosgw/ceph-a", 2023-09-23T17:55:47.152 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_data_log_changes_size": "1000", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_data_log_num_shards": "128", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_data_log_obj_prefix": "data_log", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_data_log_window": "30", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_data_notify_interval_msec": "0", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_data_sync_poll_interval": "20", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_data_sync_spawn_window": "20", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_debug_inject_olh_cancel_modification_err": "false", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_debug_inject_set_olh_err": "0", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_default_data_log_backing": "fifo", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_default_realm_info_oid": "default.realm", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_default_region_info_oid": "default.region", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_default_zone_info_oid": "default.zone", 2023-09-23T17:55:47.153 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_default_zonegroup_info_oid": "default.zonegroup", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_defer_to_bucket_acls": "", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_delete_multi_obj_max_num": "1000", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_admin_lim": "0.000000", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_admin_res": "100.000000", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_admin_wgt": "100.000000", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_auth_lim": "0.000000", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_auth_res": "200.000000", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_auth_wgt": "100.000000", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_data_lim": "0.000000", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_data_res": "500.000000", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_data_wgt": "500.000000", 2023-09-23T17:55:47.154 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_metadata_lim": "0.000000", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_metadata_res": "500.000000", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dmclock_metadata_wgt": "500.000000", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dns_name": "", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dns_s3website_name": "", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_dynamic_resharding": "true", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_enable_apis": "s3, s3website, swift, swift_auth, admin, sts, iam, notifications", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_enable_gc_threads": "true", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_enable_lc_threads": "true", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_enable_ops_log": "false", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_enable_quota_threads": "true", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_enable_static_website": "false", 2023-09-23T17:55:47.155 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_enable_usage_log": "false", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_enforce_swift_acls": "true", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_exit_timeout_secs": "120", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_expose_bucket": "false", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_extended_http_attrs": "", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_filter": "none", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_frontend_defaults": "beast ssl_certificate=config://rgw/cert/$realm/$zone.crt ssl_private_key=config://rgw/cert/$realm/$zone.key", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_frontends": "beast port=7480", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_gc_max_concurrent_io": "10", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_gc_max_deferred": "50", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_gc_max_deferred_entries_size": "3072", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_gc_max_objs": "32", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_gc_max_queue_size": "134213632", 2023-09-23T17:55:47.156 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_gc_max_trim_chunk": "16", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_gc_obj_min_wait": "7200", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_gc_processor_max_time": "3600", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_gc_processor_period": "3600", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_get_obj_max_req_size": "4194304", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_get_obj_window_size": "16777216", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_healthcheck_disabling_path": "", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ignore_get_invalid_range": "false", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_init_timeout": "300", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_inject_notify_timeout_probability": "0.000000", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_json_config": "/var/lib/ceph/radosgw/config.json", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_accepted_admin_roles": "", 2023-09-23T17:55:47.157 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_accepted_roles": "Member, admin", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_admin_domain": "", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_admin_password": "", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_admin_password_path": "", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_admin_project": "", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_admin_tenant": "", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_admin_token": "", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_admin_token_path": "", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_admin_user": "", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_api_version": "2", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_barbican_domain": "", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_barbican_password": "", 2023-09-23T17:55:47.158 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_barbican_project": "", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_barbican_tenant": "", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_barbican_user": "", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_expired_token_cache_expiration": "3600", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_implicit_tenants": "false", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_service_token_accepted_roles": "admin", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_service_token_enabled": "false", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_token_cache_size": "10000", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_url": "", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_keystone_verify_ssl": "true", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_lc_debug_interval": "-1", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_lc_lock_max_time": "90", 2023-09-23T17:55:47.159 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_lc_max_objs": "32", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_lc_max_rules": "1000", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_lc_max_worker": "3", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_lc_max_wp_worker": "3", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_lc_thread_delay": "0", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ldap_binddn": "uid=admin,cn=users,dc=example,dc=com", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ldap_dnattr": "uid", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ldap_searchdn": "cn=users,cn=accounts,dc=example,dc=com", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ldap_searchfilter": "", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ldap_secret": "/etc/openldap/secret", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ldap_uri": "ldaps://", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_lifecycle_work_time": "00:00-06:00", 2023-09-23T17:55:47.160 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_list_bucket_min_readahead": "1000", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_list_buckets_max_chunk": "1000", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_log_http_headers": "", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_log_nonexistent_bucket": "false", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_log_object_name": "%Y-%m-%d-%H-%i-%n", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_log_object_name_utc": "false", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_luarocks_location": "/tmp/luarocks", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_attr_name_len": "0", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_attr_size": "0", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_attrs_num_in_req": "0", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_chunk_size": "4194304", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_concurrent_requests": "1024", 2023-09-23T17:55:47.161 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_copy_obj_concurrent_io": "10", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_dynamic_shards": "1999", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_listing_results": "1000", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_notify_retries": "10", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_objs_per_shard": "100000", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_put_param_size": "1048576", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_put_size": "5368709120", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_max_slo_entries": "1000", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_md_log_max_shards": "64", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_md_notify_interval_msec": "200", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_meta_sync_poll_interval": "20", 2023-09-23T17:55:47.162 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_meta_sync_spawn_window": "20", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_mime_types_file": "/etc/mime.types", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_mp_lock_max_time": "600", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_multi_obj_del_max_aio": "16", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_multipart_min_part_size": "5242880", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_multipart_part_upload_limit": "10000", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_fhcache_partitions": "3", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_fhcache_size": "2017", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_frontends": "rgw-nfs", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_lru_lane_hiwat": "911", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_lru_lanes": "5", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_max_gc": "300", 2023-09-23T17:55:47.163 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_namespace_expire_secs": "300", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_run_gc_threads": "false", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_run_lc_threads": "false", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_run_quota_threads": "false", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_run_sync_thread": "false", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_s3_fast_attrs": "false", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_nfs_write_completion_interval_s": "10", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_num_async_rados_threads": "32", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_num_control_oids": "8", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_numa_node": "-1", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_obj_stripe_size": "4194304", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_obj_tombstone_cache_size": "1000", 2023-09-23T17:55:47.164 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_objexp_chunk_size": "100", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_objexp_gc_interval": "600", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_objexp_hints_num_shards": "127", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_olh_pending_timeout_sec": "3600", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_op_thread_suicide_timeout": "0", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_op_thread_timeout": "600", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_op_tracing": "false", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_opa_token": "", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_opa_url": "", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_opa_verify_ssl": "true", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ops_log_data_backlog": "5242880", 2023-09-23T17:55:47.165 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ops_log_file_path": "/var/log/ceph/ops-log-ceph-mon.a.log", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ops_log_rados": "false", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_ops_log_socket_path": "", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_override_bucket_index_max_shards": "0", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_pending_bucket_index_op_expiration": "120", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_period_latest_epoch_info_oid": ".latest_epoch", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_period_push_interval": "2.000000", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_period_push_interval_max": "30.000000", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_period_root_pool": ".rgw.root", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_policy_reject_invalid_principals": "true", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_print_continue": "true", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_print_prohibited_content_length": "false", 2023-09-23T17:55:47.166 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_put_obj_max_window_size": "67108864", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_put_obj_min_window_size": "16777216", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_rados_pool_autoscale_bias": "4.000000", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_rados_pool_recovery_priority": "5", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_rados_tracing": "false", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_realm": "", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_realm_id": "", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_realm_root_pool": ".rgw.root", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_region": "", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_region_root_pool": ".rgw.root", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_relaxed_region_enforcement": "false", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_relaxed_s3_bucket_names": "false", 2023-09-23T17:55:47.167 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_remote_addr_param": "REMOTE_ADDR", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_request_uri": "", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_reshard_batch_size": "64", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_reshard_bucket_lock_duration": "360", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_reshard_max_aio": "128", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_reshard_num_logs": "16", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_reshard_thread_interval": "600", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_resolve_cname": "false", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_rest_getusage_op_compat": "false", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_run_sync_thread": "true", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_s3_auth_order": "sts, external, local", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_s3_auth_use_keystone": "false", 2023-09-23T17:55:47.168 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_s3_auth_use_ldap": "false", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_s3_auth_use_rados": "true", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_s3_auth_use_sts": "false", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_s3_client_max_sig_ver": "-1", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_s3_success_create_obj_status": "0", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_safe_max_objects_per_shard": "102400", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_scheduler_type": "throttler", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_script_uri": "", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_service_provider_name": "", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_shard_warning_threshold": "90.000000", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sts_client_id": "", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sts_client_secret": "", 2023-09-23T17:55:47.169 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sts_entry": "sts", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sts_key": "sts", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sts_max_session_duration": "43200", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sts_min_session_duration": "900", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sts_token_introspection_url": "", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_account_in_url": "false", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_auth_entry": "auth", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_auth_url": "", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_custom_header": "", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_enforce_content_length": "false", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_need_stats": "true", 2023-09-23T17:55:47.170 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_tenant_name": "", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_token_expiration": "86400", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_url": "", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_url_prefix": "swift", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_swift_versioning_enabled": "false", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_data_full_inject_err_probability": "0.000000", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_data_inject_err_probability": "0.000000", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_lease_period": "120", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_log_trim_concurrent_buckets": "4", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_log_trim_interval": "1200", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_log_trim_max_buckets": "16", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_log_trim_min_cold_buckets": "4", 2023-09-23T17:55:47.171 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_meta_inject_err_probability": "0.000000", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_obj_etag_verify": "false", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_trace_history_size": "4096", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_trace_per_node_log_size": "32", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_sync_trace_servicemap_update_interval": "10", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_thread_pool_size": "512", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_torrent_comment": "", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_torrent_createby": "", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_torrent_encoding": "", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_torrent_flag": "false", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_torrent_origin": "", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_torrent_sha_unit": "524288", 2023-09-23T17:55:47.172 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_torrent_tracker": "", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_trust_forwarded_https": "false", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_usage_log_flush_threshold": "1024", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_usage_log_tick_interval": "30", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_usage_max_shards": "32", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_usage_max_user_shards": "1", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_use_opa_authz": "false", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_user_default_quota_max_objects": "-1", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_user_default_quota_max_size": "-1", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_user_max_buckets": "1000", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_user_policies_max_num": "100", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_user_quota_bucket_sync_interval": "180", 2023-09-23T17:55:47.173 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_user_quota_sync_idle_users": "false", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_user_quota_sync_interval": "86400", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_user_quota_sync_wait_time": "86400", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_user_unique_email": "true", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_verify_ssl": "true", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_website_routing_rules_max_num": "50", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_zone": "", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_zone_id": "", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_zone_root_pool": ".rgw.root", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_zonegroup": "", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_zonegroup_id": "", 2023-09-23T17:55:47.174 INFO:teuthology.orchestra.run.smithi055.stdout: "rgw_zonegroup_root_pool": ".rgw.root", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rgwlc_auto_session_clear": "true", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rgwlc_skip_bucket_step": "false", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_block_size": "4096", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_bloom_bits_per_key": "20", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_cache_index_and_filter_blocks": "true", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_cache_index_and_filter_blocks_with_high_priority": "false", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_cache_row_ratio": "0.000000", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_cache_shard_bits": "4", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_cache_size": "536870912", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_cache_type": "binned_lru", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_cf_compact_on_deletion": "true", 2023-09-23T17:55:47.175 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_cf_compact_on_deletion_sliding_window": "32768", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_cf_compact_on_deletion_trigger": "16384", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_collect_compaction_stats": "false", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_collect_extended_stats": "false", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_collect_memory_stats": "false", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_delete_range_threshold": "1048576", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_index_type": "binary_search", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_log_to_ceph_log": "true", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_metadata_block_size": "4096", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_partition_filters": "false", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_perf": "false", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rocksdb_pin_l0_filter_and_index_blocks_in_cache": "false", 2023-09-23T17:55:47.176 INFO:teuthology.orchestra.run.smithi055.stdout: "rotating_keys_bootstrap_timeout": "30", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "rotating_keys_renewal_timeout": "10", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "run_dir": "/var/run/ceph", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_block_create": "true", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_cache_lru_size": "67108864", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_cbjournal_size": "5368709120", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_default_max_object_size": "16777216", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_default_object_metadata_reservation": "16777216", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_device_size": "53687091200", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_journal_batch_capacity": "16", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_journal_batch_flush_size": "16777216", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_journal_batch_preferred_fullness": "0.950000", 2023-09-23T17:55:47.177 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_journal_iodepth_limit": "5", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_main_device_type": "SSD", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_max_concurrent_transactions": "8", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_multiple_tiers_default_evict_ratio": "0.600000", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_multiple_tiers_fast_evict_ratio": "0.700000", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_multiple_tiers_stop_evict_ratio": "0.500000", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_obj_data_write_amplification": "1.250000", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "seastore_segment_size": "67108864", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "set_keepcaps": "false", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "setgroup": "ceph", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "setuser": "ceph", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "setuser_match_path": "", 2023-09-23T17:55:47.178 INFO:teuthology.orchestra.run.smithi055.stdout: "target_max_misplaced_ratio": "0.050000", 2023-09-23T17:55:47.179 INFO:teuthology.orchestra.run.smithi055.stdout: "thp": "false", 2023-09-23T17:55:47.179 INFO:teuthology.orchestra.run.smithi055.stdout: "threadpool_default_timeout": "60", 2023-09-23T17:55:47.179 INFO:teuthology.orchestra.run.smithi055.stdout: "threadpool_empty_queue_max_wait": "2", 2023-09-23T17:55:47.179 INFO:teuthology.orchestra.run.smithi055.stdout: "throttler_perf_counter": "true" 2023-09-23T17:55:47.179 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:55:47.365 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=0 2023-09-23T17:55:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:47 smithi055 ceph-mon[179668]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:47 smithi055 ceph-mon[184174]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:47 smithi099 ceph-mon[167076]: pgmap v223: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:47.974 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:49.210 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a: {} 2023-09-23T17:55:49.220 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b: {} 2023-09-23T17:55:49.229 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c: {} 2023-09-23T17:55:49.231 INFO:teuthology.orchestra.run.smithi055.stderr:mon.a: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T17:55:49.231 INFO:teuthology.orchestra.run.smithi055.stderr:mon.b: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T17:55:49.231 INFO:teuthology.orchestra.run.smithi055.stderr:mon.c: mon_osd_down_out_interval = '0' (not observed, change may require restart) 2023-09-23T17:55:49.620 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:49 smithi055 ceph-mon[184174]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:49.620 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3397242506' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T17:55:49.620 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:49 smithi055 ceph-mon[179668]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:49.620 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3397242506' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T17:55:49.621 INFO:tasks.thrashosds.thrasher:Unable to test ceph-objectstore-tool, not available on all OSD nodes 2023-09-23T17:55:49.621 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-09-23T17:55:49.622 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T17:55:49.624 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T17:55:49.624 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-09-23T17:55:49.624 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_0 16 2023-09-23T17:55:49.624 INFO:tasks.thrashosds.thrasher:starting do_thrash 2023-09-23T17:55:49.624 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T17:55:49.625 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:55:49.625 INFO:tasks.thrashosds.thrasher:inject_pause on osd.4 2023-09-23T17:55:49.625 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T17:55:49.625 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T17:55:49.625 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 config set filestore_inject_stall 3 2023-09-23T17:55:49.626 INFO:tasks.thrashosds.thrasher:starting do_sighup with a delay of 0.1 2023-09-23T17:55:49.626 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:55:49.626 INFO:tasks.thrashosds.thrasher:starting do_optrack_toggle with a delay of 2.0 2023-09-23T17:55:49.626 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:55:49.627 INFO:tasks.thrashosds.thrasher:starting do_dump_ops 2023-09-23T17:55:49.627 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T17:55:49.627 INFO:tasks.thrashosds.thrasher:starting do_noscrub_toggle with a delay of 2.0 2023-09-23T17:55:49.627 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T17:55:49.677 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:49.730 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:55:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:49.735 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:55:49.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:49 smithi099 ceph-mon[167076]: pgmap v224: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:49.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3397242506' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T17:55:49.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:55:49.882 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:49.913 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:55:49.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:49.916 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:55:50.017 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:55:50.060 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:50.093 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:55:50.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:50.096 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:55:50.197 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:55:50.222 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:50.252 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:55:50.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:55:50.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:50.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:55:50.277 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:50.307 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:50.359 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:55:50.386 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:50.406 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:50.407 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:50.429 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:55:50.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:50.436 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:55:50.537 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:55:50.565 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:50.645 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:55:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:55:50] "GET /metrics HTTP/1.1" 200 34084 "" "Prometheus/2.43.0" 2023-09-23T17:55:50.793 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:55:50.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:50.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:55:50.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:55:50.925 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:50.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:55:50.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:50.960 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:55:51.061 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:55:51.090 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:51.432 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:55:51.432 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T17:55:51.432 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:55:51.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:51 smithi099 ceph-mon[167076]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:51.803 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:51.826 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:51 smithi055 ceph-mon[179668]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:51.827 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:51.827 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:51 smithi055 ceph-mon[184174]: pgmap v225: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:51.827 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:55:52.197 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:55:52.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:52.202 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:55:52.303 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:55:52.333 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:52.530 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:55:52.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:52.533 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:55:52.634 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:55:52.668 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:52.704 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:55:52.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:52.707 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:55:52.802 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_0' created 2023-09-23T17:55:52.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:55:52.833 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:52.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:55:52.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:52.868 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:55:52.969 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:55:52.992 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1762103689' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T17:55:52.992 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:52 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T17:55:52.995 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:53.027 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:55:53.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:53.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:55:53.130 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:55:53.142 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:55:53.142 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T17:55:53.142 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T17:55:53.142 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:55:53.156 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:53.168 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1762103689' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T17:55:53.168 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:52 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T17:55:53.168 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1762103689' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T17:55:53.168 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:52 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-09-23T17:55:53.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:55:53.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:53.190 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:55:53.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:55:53.317 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:53.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:55:53.533 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:55:53.533 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:55:53.533 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:55:53.534 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:55:53.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:53.534 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:55:53.552 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:55:53.571 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:55:53.581 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-09-23T17:55:53.608 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:55:53.630 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:55:53.633 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:55:53.633 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:55:53.633 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:55:53.633 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T17:55:53.633 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:55:53.633 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:55:53.633 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T17:55:53.633 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T17:55:53.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:55:53.665 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:53.696 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:55:53.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:53.699 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:55:53.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:55:53.828 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T17:55:53.833 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:53.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[184174]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:53.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[184174]: failed to encode map e110 with expected crc 2023-09-23T17:55:53.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T17:55:53.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[184174]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T17:55:53.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[184174]: failed to encode map e110 with expected crc 2023-09-23T17:55:53.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[184174]: failed to encode map e110 with expected crc 2023-09-23T17:55:53.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[184174]: failed to encode map e110 with expected crc 2023-09-23T17:55:53.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3641690319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:55:53.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[179668]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:53.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[179668]: failed to encode map e110 with expected crc 2023-09-23T17:55:53.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T17:55:53.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[179668]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T17:55:53.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[179668]: failed to encode map e110 with expected crc 2023-09-23T17:55:53.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[179668]: failed to encode map e110 with expected crc 2023-09-23T17:55:53.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[179668]: failed to encode map e110 with expected crc 2023-09-23T17:55:53.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3641690319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:55:53.863 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:55:53.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:53.867 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:55:53.967 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:55:53.994 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:54.028 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:55:54.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:54.031 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:55:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:53 smithi099 ceph-mon[167076]: pgmap v226: 73 pgs: 73 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:53 smithi099 ceph-mon[167076]: failed to encode map e110 with expected crc 2023-09-23T17:55:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:53 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-09-23T17:55:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:53 smithi099 ceph-mon[167076]: osdmap e110: 8 total, 8 up, 8 in 2023-09-23T17:55:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:53 smithi099 ceph-mon[167076]: failed to encode map e110 with expected crc 2023-09-23T17:55:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:53 smithi099 ceph-mon[167076]: failed to encode map e110 with expected crc 2023-09-23T17:55:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:53 smithi099 ceph-mon[167076]: failed to encode map e110 with expected crc 2023-09-23T17:55:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3641690319' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:55:54.132 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:55:54.158 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:54.194 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:55:54.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:54.198 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:55:54.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:55:54.326 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:54.359 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:55:54.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:54.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:55:54.375 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:54.462 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:55:54.489 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:54.492 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:54.527 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:55:54.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:54.530 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:55:54.631 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:55:54.659 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:54.743 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:55:54.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:54.746 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:55:54.833 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T17:55:54.847 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:55:54.874 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:54.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:55:54.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:54.909 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:55:55.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[179668]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T17:55:55.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[179668]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[179668]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[179668]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.008 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[179668]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.009 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1445267871' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:55:55.009 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[184174]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T17:55:55.009 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[184174]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.009 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[184174]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.009 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[184174]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.009 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[184174]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.009 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1445267871' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:55:55.010 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:55:55.040 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:54 smithi099 ceph-mon[167076]: osdmap e111: 8 total, 8 up, 8 in 2023-09-23T17:55:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:54 smithi099 ceph-mon[167076]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:54 smithi099 ceph-mon[167076]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:54 smithi099 ceph-mon[167076]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:54 smithi099 ceph-mon[167076]: failed to encode map e111 with expected crc 2023-09-23T17:55:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1445267871' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:55:55.225 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:55:55.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:55.228 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:55:55.329 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:55:55.356 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:55.391 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:55:55.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:55.393 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:55:55.494 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:55:55.521 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:55.607 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:55:55.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:55.611 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:55:55.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:55:55.743 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:55.776 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:55:55.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:55.779 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:55:55.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:55:55.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[179668]: pgmap v229: 89 pgs: 5 creating+peering, 9 unknown, 75 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:55.899 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:55:55.899 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[179668]: failed to encode map e112 with expected crc 2023-09-23T17:55:55.899 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[179668]: failed to encode map e112 with expected crc 2023-09-23T17:55:55.899 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1445267871' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:55:55.899 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[179668]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T17:55:55.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[184174]: pgmap v229: 89 pgs: 5 creating+peering, 9 unknown, 75 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:55.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:55:55.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[184174]: failed to encode map e112 with expected crc 2023-09-23T17:55:55.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[184174]: failed to encode map e112 with expected crc 2023-09-23T17:55:55.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1445267871' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:55:55.900 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:55 smithi055 ceph-mon[184174]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T17:55:55.908 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:55.941 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:55:55.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:55.945 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:55:56.045 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:55:56.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:55 smithi099 ceph-mon[167076]: pgmap v229: 89 pgs: 5 creating+peering, 9 unknown, 75 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:55 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:55:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:55 smithi099 ceph-mon[167076]: failed to encode map e112 with expected crc 2023-09-23T17:55:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:55 smithi099 ceph-mon[167076]: failed to encode map e112 with expected crc 2023-09-23T17:55:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1445267871' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:55:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:55 smithi099 ceph-mon[167076]: osdmap e112: 8 total, 8 up, 8 in 2023-09-23T17:55:56.077 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:56.081 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:55:56.288 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:55:56.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:56.292 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:55:56.393 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:55:56.424 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:56.465 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:55:56.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:56.469 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:55:56.535 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:55:56.535 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:55:56.570 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:55:56.601 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:56.805 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T17:55:56.805 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:55:56.806 INFO:tasks.thrashosds.thrasher:Killing osd 6, live_osds are [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T17:55:56.806 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T17:55:56.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T17:55:56.827 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/510485404' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:55:56.828 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T17:55:56.847 INFO:teuthology.orchestra.run.smithi055.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-09-23T17:55:56.887 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:57.153 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:55:57.155 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/510485404' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:55:57.156 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/510485404' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T17:55:57.156 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:55:56 smithi099 systemd[1]: Stopping Ceph osd.6 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:55:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:57.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:55:57.258 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:55:57.289 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:57.312 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T17:55:57.374 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:55:57.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:57.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:55:57.479 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:55:57.506 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:57.546 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:55:57.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:57.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:55:57.576 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:55:57 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[119847]: 2023-09-23T17:55:57.154+0000 7f9469710700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:55:57.577 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:55:57 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[119847]: 2023-09-23T17:55:57.154+0000 7f9469710700 -1 osd.6 112 *** Got signal Terminated *** 2023-09-23T17:55:57.577 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:55:57 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[119847]: 2023-09-23T17:55:57.154+0000 7f9469710700 -1 osd.6 112 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T17:55:57.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:55:57.682 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:57.717 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:55:57.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:57.719 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:55:57.820 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:55:57.857 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:57.890 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:55:57.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:57.893 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:55:57.893 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:57.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:55:58.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:57 smithi099 ceph-mon[167076]: pgmap v231: 89 pgs: 5 creating+peering, 7 unknown, 77 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:58.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/510485404' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T17:55:58.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:57 smithi099 ceph-mon[167076]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T17:55:58.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:57 smithi099 ceph-mon[167076]: failed to encode map e113 with expected crc 2023-09-23T17:55:58.019 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:57 smithi099 ceph-mon[167076]: osd.6 marked itself down and dead 2023-09-23T17:55:58.022 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:58.053 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:55:58.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:58.055 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:55:58.129 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:55:58.156 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:55:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:57 smithi055 ceph-mon[179668]: pgmap v231: 89 pgs: 5 creating+peering, 7 unknown, 77 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/510485404' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T17:55:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:57 smithi055 ceph-mon[179668]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T17:55:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:57 smithi055 ceph-mon[179668]: failed to encode map e113 with expected crc 2023-09-23T17:55:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:57 smithi055 ceph-mon[179668]: osd.6 marked itself down and dead 2023-09-23T17:55:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:57 smithi055 ceph-mon[184174]: pgmap v231: 89 pgs: 5 creating+peering, 7 unknown, 77 active+clean; 4.6 MiB data, 164 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/510485404' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T17:55:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:57 smithi055 ceph-mon[184174]: osdmap e113: 8 total, 8 up, 8 in 2023-09-23T17:55:58.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:57 smithi055 ceph-mon[184174]: failed to encode map e113 with expected crc 2023-09-23T17:55:58.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:57 smithi055 ceph-mon[184174]: osd.6 marked itself down and dead 2023-09-23T17:55:58.184 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:58.280 DEBUG:teuthology.orchestra.run.smithi055:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_0 bench 90 write' 2023-09-23T17:55:58.283 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T17:55:58.283 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T17:55:58.330 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:hints = 1 2023-09-23T17:55:58.330 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T17:55:58.330 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object prefix: benchmark_data_smithi055_229117 2023-09-23T17:55:58.331 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T17:55:58.331 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 0 0 0 0 0 0 - 0 2023-09-23T17:55:58.512 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:55:58.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:58.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:55:58.616 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:55:58.646 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:58.683 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:55:58.685 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:55:58 smithi099 bash[175681]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6 2023-09-23T17:55:58.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:58.687 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:55:58.787 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:55:58.820 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:58.833 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:55:58.866 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:55:58.867 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:55:58.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:58.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:55:58.907 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:55:58.954 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:55:58.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:55:58.991 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:55:59.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:58 smithi099 ceph-mon[167076]: failed to encode map e113 with expected crc 2023-09-23T17:55:59.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:58 smithi099 ceph-mon[167076]: failed to encode map e113 with expected crc 2023-09-23T17:55:59.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:58 smithi099 ceph-mon[167076]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:55:59.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:58 smithi099 ceph-mon[167076]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:58 smithi099 ceph-mon[167076]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:58 smithi099 ceph-mon[167076]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:58 smithi099 ceph-mon[167076]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:58 smithi099 ceph-mon[167076]: osdmap e114: 8 total, 7 up, 8 in 2023-09-23T17:55:59.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:58 smithi099 ceph-mon[167076]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:58 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1582543214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:55:59.021 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:59.034 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:55:59.077 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:55:59.078 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:55:59.078 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:55:59.078 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:55:59.078 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T17:55:59.078 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:55:59.078 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:55:59.078 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:55:59.078 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T17:55:59.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[179668]: failed to encode map e113 with expected crc 2023-09-23T17:55:59.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[179668]: failed to encode map e113 with expected crc 2023-09-23T17:55:59.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[179668]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:55:59.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[179668]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[179668]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[179668]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.097 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[179668]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.097 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[179668]: osdmap e114: 8 total, 7 up, 8 in 2023-09-23T17:55:59.097 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[179668]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.097 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1582543214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:55:59.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[184174]: failed to encode map e113 with expected crc 2023-09-23T17:55:59.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[184174]: failed to encode map e113 with expected crc 2023-09-23T17:55:59.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[184174]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T17:55:59.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[184174]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[184174]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[184174]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[184174]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[184174]: osdmap e114: 8 total, 7 up, 8 in 2023-09-23T17:55:59.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[184174]: failed to encode map e114 with expected crc 2023-09-23T17:55:59.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:58 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1582543214' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:55:59.124 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:55:59.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:59.128 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:55:59.229 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:55:59.288 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:59.330 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 1 16 1632 1616 100.994 101 0.00579319 0.00986805 2023-09-23T17:55:59.473 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:55:59.474 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:55:59.564 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:55:59.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:59.570 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:55:59.670 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:55:59.714 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:59.771 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:55:59.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:55:59.776 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:55:59.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:55:59.924 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:55:59.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[179668]: pgmap v234: 89 pgs: 6 stale+active+clean, 5 creating+peering, 78 active+clean; 4.6 MiB data, 169 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:59.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[179668]: osdmap e115: 8 total, 7 up, 8 in 2023-09-23T17:55:59.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[179668]: failed to encode map e115 with expected crc 2023-09-23T17:55:59.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[179668]: failed to encode map e115 with expected crc 2023-09-23T17:55:59.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[179668]: failed to encode map e115 with expected crc 2023-09-23T17:55:59.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[179668]: failed to encode map e115 with expected crc 2023-09-23T17:55:59.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[179668]: failed to encode map e115 with expected crc 2023-09-23T17:55:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[184174]: pgmap v234: 89 pgs: 6 stale+active+clean, 5 creating+peering, 78 active+clean; 4.6 MiB data, 169 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:55:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[184174]: osdmap e115: 8 total, 7 up, 8 in 2023-09-23T17:55:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[184174]: failed to encode map e115 with expected crc 2023-09-23T17:55:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[184174]: failed to encode map e115 with expected crc 2023-09-23T17:55:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[184174]: failed to encode map e115 with expected crc 2023-09-23T17:55:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[184174]: failed to encode map e115 with expected crc 2023-09-23T17:55:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:55:59 smithi055 ceph-mon[184174]: failed to encode map e115 with expected crc 2023-09-23T17:56:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:59 smithi099 ceph-mon[167076]: pgmap v234: 89 pgs: 6 stale+active+clean, 5 creating+peering, 78 active+clean; 4.6 MiB data, 169 MiB used, 715 GiB / 715 GiB avail 2023-09-23T17:56:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:59 smithi099 ceph-mon[167076]: osdmap e115: 8 total, 7 up, 8 in 2023-09-23T17:56:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:59 smithi099 ceph-mon[167076]: failed to encode map e115 with expected crc 2023-09-23T17:56:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:59 smithi099 ceph-mon[167076]: failed to encode map e115 with expected crc 2023-09-23T17:56:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:59 smithi099 ceph-mon[167076]: failed to encode map e115 with expected crc 2023-09-23T17:56:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:59 smithi099 ceph-mon[167076]: failed to encode map e115 with expected crc 2023-09-23T17:56:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:55:59 smithi099 ceph-mon[167076]: failed to encode map e115 with expected crc 2023-09-23T17:56:00.103 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:00.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:00.108 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:00.169 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T17:56:00.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:00.250 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T17:56:00.250 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T17:56:00.258 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:00.307 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:00.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:00.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:00.330 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 2 16 5130 5114 159.8 218.625 0.00255696 0.00624624 2023-09-23T17:56:00.376 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:56:00.376 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T17:56:00.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T17:56:00.376 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:56:00.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.376 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:227 5.7 e115/110 5:e557fa96:::benchmark_data_smithi055_229117_object226:head v 115'23, mlcod=115'20)", 2023-09-23T17:56:00.376 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:58.922068+0000", 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.4478978599999999, 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014393681, 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.922068+0000", 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.922068+0000", 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.922067+0000", 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.922101+0000", 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3887999999999998e-05 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.922110+0000", 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9989999999999997e-06 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.922116+0000", 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.699e-06 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.927443+0000", 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053271439999999998 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.927469+0000", 2023-09-23T17:56:00.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5893000000000001e-05 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.936416+0000", 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0089473299999999999 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.936462+0000", 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5204000000000003e-05 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.381 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.382 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.382 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:260 5.2 5:4b629c69:::benchmark_data_smithi055_229117_object259:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.382 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:58.931343+0000", 2023-09-23T17:56:00.382 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.4386225619999999, 2023-09-23T17:56:00.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0090866349999999992, 2023-09-23T17:56:00.382 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.382 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.382 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 260 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.931343+0000", 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.931343+0000", 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.931342+0000", 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.931364+0000", 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.192e-05 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.931379+0000", 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4618e-05 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.931388+0000", 2023-09-23T17:56:00.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1880000000000008e-06 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.931443+0000", 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4889999999999998e-05 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.931575+0000", 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000132707 2023-09-23T17:56:00.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.931728+0000", 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015299700000000001 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.932350+0000", 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00062157399999999995 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.940283+0000", 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0079327979999999996 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.940338+0000", 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5427999999999999e-05 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.940430+0000", 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1586e-05 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:354 5.7 e115/110 5:efec4072:::benchmark_data_smithi055_229117_object353:head v 115'32, mlcod=115'30)", 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:58.961082+0000", 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.408883841, 2023-09-23T17:56:00.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0094514860000000003, 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.961082+0000", 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.961082+0000", 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.961081+0000", 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.961167+0000", 2023-09-23T17:56:00.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5805999999999998e-05 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.961198+0000", 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0846000000000002e-05 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.961206+0000", 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7100000000000007e-06 2023-09-23T17:56:00.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.961268+0000", 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2308e-05 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.961294+0000", 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5933000000000001e-05 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.970494+0000", 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.009200171 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.970533+0000", 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9505999999999997e-05 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:350 5.2 5:48c4cc21:::benchmark_data_smithi055_229117_object349:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:58.962160+0000", 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.407805497, 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0098110049999999994, 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 350 2023-09-23T17:56:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962160+0000", 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962160+0000", 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962159+0000", 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962195+0000", 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6514000000000003e-05 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962210+0000", 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4808e-05 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962218+0000", 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2160000000000006e-06 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962272+0000", 2023-09-23T17:56:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3823000000000003e-05 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962437+0000", 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000165122 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962645+0000", 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020799499999999999 2023-09-23T17:56:00.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.970765+0000", 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.008119734 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.971831+0000", 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001065893 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.971882+0000", 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1848999999999998e-05 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.971971+0000", 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8797999999999999e-05 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:351 5.3 5:c46d0009:::benchmark_data_smithi055_229117_object350:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:58.962231+0000", 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.4077345299999999, 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0098597870000000001, 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.405 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 351 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962231+0000", 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962231+0000", 2023-09-23T17:56:00.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962231+0000", 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962268+0000", 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6934000000000002e-05 2023-09-23T17:56:00.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962277+0000", 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3200000000000006e-06 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962284+0000", 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3139999999999998e-06 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962527+0000", 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024309700000000001 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962641+0000", 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000113267 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962774+0000", 2023-09-23T17:56:00.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013333299999999999 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.970278+0000", 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0075041659999999996 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.971958+0000", 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016801959999999999 2023-09-23T17:56:00.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.972004+0000", 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5747000000000001e-05 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.972091+0000", 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6979999999999997e-05 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.411 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:357 5.2 5:435fc597:::benchmark_data_smithi055_229117_object356:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:58.962298+0000", 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.4076681390000001, 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0098407140000000004, 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.412 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 357 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962298+0000", 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962298+0000", 2023-09-23T17:56:00.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962297+0000", 2023-09-23T17:56:00.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:00.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962351+0000", 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3470999999999997e-05 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962359+0000", 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9409999999999996e-06 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962366+0000", 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3860000000000004e-06 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962927+0000", 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00056132299999999997 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.963039+0000", 2023-09-23T17:56:00.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011150999999999999 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.963176+0000", 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013704900000000001 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.970770+0000", 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0075942789999999998 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.972014+0000", 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0012436389999999999 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.972059+0000", 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4916000000000002e-05 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.972138+0000", 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9802999999999995e-05 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.418 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:359 5.5 5:a27e7e7c:::benchmark_data_smithi055_229117_object358:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:58.962372+0000", 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.407593909, 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0097217029999999999, 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.419 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 359 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962372+0000", 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.420 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962372+0000", 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962371+0000", 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962440+0000", 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8664000000000005e-05 2023-09-23T17:56:00.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962458+0000", 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8485e-05 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962498+0000", 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9716999999999999e-05 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962558+0000", 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9803000000000003e-05 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962659+0000", 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010146599999999999 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:56:00.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.962778+0000", 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011849999999999999 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.965346+0000", 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025684229999999998 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.970297+0000", 2023-09-23T17:56:00.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049505210000000003 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.972000+0000", 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001702989 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.972049+0000", 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9472999999999999e-05 2023-09-23T17:56:00.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:58.972094+0000", 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4356e-05 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:2900 5.5 5:a1410b90:::benchmark_data_smithi055_229117_object2899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:59.659840+0000", 2023-09-23T17:56:00.426 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.71012573400000001, 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010528878, 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 2900 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.659840+0000", 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.659840+0000", 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.659839+0000", 2023-09-23T17:56:00.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.659866+0000", 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6968e-05 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.659877+0000", 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1110999999999999e-05 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.659885+0000", 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1349999999999992e-06 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.659944+0000", 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8870999999999999e-05 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.660070+0000", 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000126048 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.660222+0000", 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015141800000000001 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.661316+0000", 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001093761 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.661765+0000", 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00044985100000000003 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.670277+0000", 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0085120279999999996 2023-09-23T17:56:00.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.670333+0000", 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5485999999999997e-05 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.670369+0000", 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6019000000000002e-05 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.433 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:3265 5.5 5:ac82d579:::benchmark_data_smithi055_229117_object3264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:59.768710+0000", 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.60125582, 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0099926960000000006, 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.434 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 3265 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.768710+0000", 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.768710+0000", 2023-09-23T17:56:00.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.768708+0000", 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.768734+0000", 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5743000000000001e-05 2023-09-23T17:56:00.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.768746+0000", 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2065999999999999e-05 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.768754+0000", 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8250000000000005e-06 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.768849+0000", 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5031000000000005e-05 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.769375+0000", 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00052563700000000002 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.769510+0000", 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000135439 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.770326+0000", 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00081614000000000003 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.774461+0000", 2023-09-23T17:56:00.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004134881 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.778649+0000", 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041879279999999996 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.778681+0000", 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1699000000000003e-05 2023-09-23T17:56:00.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.778703+0000", 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1886999999999999e-05 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:3583 5.2 5:4d70453c:::benchmark_data_smithi055_229117_object3582:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:59.858410+0000", 2023-09-23T17:56:00.441 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.51155585599999998, 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013006676999999999, 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 3583 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.858410+0000", 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.858410+0000", 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.858409+0000", 2023-09-23T17:56:00.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.858432+0000", 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2906000000000001e-05 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.858443+0000", 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.066e-05 2023-09-23T17:56:00.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.858451+0000", 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0499999999999992e-06 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.859069+0000", 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00061848800000000005 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.859221+0000", 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015193699999999999 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.859401+0000", 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017971299999999999 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.860774+0000", 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00137268 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.871257+0000", 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010483128 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.871311+0000", 2023-09-23T17:56:00.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4635999999999997e-05 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.871417+0000", 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010514900000000001 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:4022 5.5 5:a289e35b:::benchmark_data_smithi055_229117_object4021:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.448 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:59.994797+0000", 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.37516834199999999, 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010463194, 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 4022 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.449 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994797+0000", 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994797+0000", 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994795+0000", 2023-09-23T17:56:00.450 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994833+0000", 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7968e-05 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994859+0000", 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5570000000000001e-05 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.451 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994872+0000", 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2897e-05 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.995066+0000", 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000193711 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.452 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.995231+0000", 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000165274 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.995373+0000", 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014234100000000001 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.997380+0000", 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020072340000000001 2023-09-23T17:56:00.453 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.005110+0000", 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0077296559999999997 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.005164+0000", 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3786999999999998e-05 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.454 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.005222+0000", 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7806000000000002e-05 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.005261+0000", 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8976000000000001e-05 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:4026 5.5 5:aa01932f:::benchmark_data_smithi055_229117_object4025:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:55:59.994893+0000", 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.37507253899999998, 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01044872, 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 4026 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.456 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994893+0000", 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994893+0000", 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994893+0000", 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994929+0000", 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6191e-05 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994941+0000", 2023-09-23T17:56:00.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2397e-05 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.994948+0000", 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0639999999999996e-06 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.995828+0000", 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00088001300000000002 2023-09-23T17:56:00.459 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.995958+0000", 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012975000000000001 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.996099+0000", 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000141352 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.460 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:55:59.997410+0000", 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001310748 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.005145+0000", 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0077346910000000001 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.461 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.005269+0000", 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012426700000000001 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.005306+0000", 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7134000000000001e-05 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.005342+0000", 2023-09-23T17:56:00.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5796000000000001e-05 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:4475 5.3 5:c2c032c3:::benchmark_data_smithi055_229117_object4474:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.127233+0000", 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.242732842, 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015033427, 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.463 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 4475 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127233+0000", 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.464 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127233+0000", 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127232+0000", 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127263+0000", 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0658999999999999e-05 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127274+0000", 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1161999999999999e-05 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127281+0000", 2023-09-23T17:56:00.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6990000000000003e-06 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127335+0000", 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3309000000000001e-05 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127459+0000", 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000123783 2023-09-23T17:56:00.467 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127591+0000", 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013275400000000001 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.140454+0000", 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012863031 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.468 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142106+0000", 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001652073 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142161+0000", 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4360000000000001e-05 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.469 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142266+0000", 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000105565 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:4480 5.2 5:4b7c19f8:::benchmark_data_smithi055_229117_object4479:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.127936+0000", 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.242029986, 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014085446999999999, 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.470 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 4480 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127936+0000", 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.471 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127936+0000", 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127935+0000", 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.472 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127968+0000", 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2771999999999997e-05 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127980+0000", 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2429000000000001e-05 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.127988+0000", 2023-09-23T17:56:00.473 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0010000000000001e-06 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128034+0000", 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5599000000000002e-05 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128156+0000", 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012258200000000001 2023-09-23T17:56:00.474 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128304+0000", 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014762 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.140302+0000", 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011998422999999999 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.475 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141814+0000", 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001512167 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141870+0000", 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5996999999999999e-05 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142021+0000", 2023-09-23T17:56:00.476 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000150755 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:4481 5.5 5:a2a5ad28:::benchmark_data_smithi055_229117_object4480:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.128006+0000", 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.24195968500000001, 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014955174, 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.477 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 4481 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128006+0000", 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.478 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128006+0000", 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128005+0000", 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128037+0000", 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1797000000000001e-05 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128047+0000", 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7690000000000008e-06 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128053+0000", 2023-09-23T17:56:00.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8440000000000003e-06 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128263+0000", 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021047199999999999 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128445+0000", 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000182108 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128601+0000", 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000156057 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141387+0000", 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012785698 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.482 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142875+0000", 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014879229999999999 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142883+0000", 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9510000000000006e-06 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142924+0000", 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1451e-05 2023-09-23T17:56:00.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142961+0000", 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6943000000000002e-05 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:4482 5.5 5:a287a610:::benchmark_data_smithi055_229117_object4481:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.128068+0000", 2023-09-23T17:56:00.484 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.241897947, 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016327303000000001, 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 4482 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.485 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128068+0000", 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128068+0000", 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128067+0000", 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:00.486 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128104+0000", 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6183000000000002e-05 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128123+0000", 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9748000000000001e-05 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.487 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128143+0000", 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9219999999999999e-05 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.140699+0000", 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012556914000000001 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.488 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.140836+0000", 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013625599999999999 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.140988+0000", 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015218 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142969+0000", 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019816019999999998 2023-09-23T17:56:00.489 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.143450+0000", 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00048034299999999999 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.144307+0000", 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00085766399999999995 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.490 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.144355+0000", 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7870000000000001e-05 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.144395+0000", 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9805999999999998e-05 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.491 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:4484 5.5 5:af167e2e:::benchmark_data_smithi055_229117_object4483:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.128157+0000", 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.241809205, 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016376043, 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 4484 2023-09-23T17:56:00.492 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128157+0000", 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128157+0000", 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.493 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128156+0000", 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128193+0000", 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7468000000000001e-05 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.494 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128211+0000", 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7164e-05 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128222+0000", 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1677e-05 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.495 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141415+0000", 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013192963 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141532+0000", 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011678700000000001 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141718+0000", 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018589600000000001 2023-09-23T17:56:00.496 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142974+0000", 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0012563279999999999 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.143497+0000", 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00052230400000000002 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.144439+0000", 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00094275900000000004 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.144498+0000", 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8792999999999998e-05 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.144533+0000", 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4477000000000002e-05 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:4478 5.b e115/110 5:d7b25dc2:::benchmark_data_smithi055_229117_object4477:head v 115'260, mlcod=115'258)", 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.128563+0000", 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.24140235199999999, 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012584888000000001, 2023-09-23T17:56:00.499 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128563+0000", 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128563+0000", 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.500 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128557+0000", 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999933 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128595+0000", 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8553999999999997e-05 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.501 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128641+0000", 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5163000000000001e-05 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128670+0000", 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9054e-05 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.502 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128713+0000", 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3779999999999998e-05 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.140394+0000", 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011681079 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141118+0000", 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00072393100000000003 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.503 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141148+0000", 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9907e-05 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:4487 5.5 5:ab147f47:::benchmark_data_smithi055_229117_object4486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.128911+0000", 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.24105443300000001, 2023-09-23T17:56:00.504 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015763263999999999, 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 4487 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.505 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128911+0000", 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128911+0000", 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128910+0000", 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:00.506 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.128934+0000", 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3734999999999999e-05 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.140290+0000", 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011355675000000001 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.507 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.140302+0000", 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2398999999999999e-05 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142127+0000", 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018245920000000001 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142274+0000", 2023-09-23T17:56:00.508 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014746900000000001 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.142446+0000", 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017227 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.143531+0000", 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001084956 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.509 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.510 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:00.510 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.144406+0000", 2023-09-23T17:56:00.510 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:00.511 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00087489100000000004 2023-09-23T17:56:00.511 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.511 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.511 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:00.511 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.144600+0000", 2023-09-23T17:56:00.511 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019362200000000001 2023-09-23T17:56:00.511 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.511 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.511 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.144641+0000", 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0809999999999997e-05 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.144675+0000", 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3835999999999999e-05 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.512 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:4479 5.7 e115/110 5:e1768b15:::benchmark_data_smithi055_229117_object4478:head v 115'304, mlcod=115'301)", 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.129136+0000", 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.240829667, 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014404286000000001, 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.129136+0000", 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.513 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.129136+0000", 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.129135+0000", 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.514 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.129214+0000", 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8999999999999996e-05 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141325+0000", 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012110249 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:00.515 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141340+0000", 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.535e-05 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141645+0000", 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00030544099999999999 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.141665+0000", 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9902e-05 2023-09-23T17:56:00.516 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.143494+0000", 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018281899999999999 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.143540+0000", 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6811000000000003e-05 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.517 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.518 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.518 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:00.518 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:00.518 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:56:00.574 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:56:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:56:00] "GET /metrics HTTP/1.1" 200 34087 "" "Prometheus/2.43.0" 2023-09-23T17:56:00.585 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:00.775 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:00.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:00.781 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:00.859 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T17:56:00.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:00.919 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:00.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:00.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:00.978 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:01.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:01.117 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:01.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:01.168 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:00 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:56:01.168 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:00 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 79/1311 objects degraded (6.026%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:01.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:01.169 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:00 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:56:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:00 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 79/1311 objects degraded (6.026%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:01.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:00 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:56:01.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:00 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 79/1311 objects degraded (6.026%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:01.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:01.308 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:01.332 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 3 16 8240 8224 171.241 194.375 0.00571775 0.00583185 2023-09-23T17:56:01.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:01.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:01.367 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:01.468 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:01.475 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:56:01.528 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:01.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:01.601 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:01.601 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:01.702 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:01.788 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:01.801 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:01.871 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:01.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:01.877 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:01.954 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T17:56:01.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:02.015 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:02.061 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:02.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:02.064 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:02.086 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:01 smithi055 ceph-mon[184174]: pgmap v236: 89 pgs: 3 active+undersized+degraded, 1 peering, 4 stale+active+clean, 4 active+undersized, 77 active+clean; 30 MiB data, 189 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s wr, 73 op/s; 79/1311 objects degraded (6.026%) 2023-09-23T17:56:02.086 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:01 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3968912919' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:02.087 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:01 smithi055 ceph-mon[179668]: pgmap v236: 89 pgs: 3 active+undersized+degraded, 1 peering, 4 stale+active+clean, 4 active+undersized, 77 active+clean; 30 MiB data, 189 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s wr, 73 op/s; 79/1311 objects degraded (6.026%) 2023-09-23T17:56:02.087 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:01 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3968912919' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:02.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:02.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:01 smithi099 ceph-mon[167076]: pgmap v236: 89 pgs: 3 active+undersized+degraded, 1 peering, 4 stale+active+clean, 4 active+undersized, 77 active+clean; 30 MiB data, 189 MiB used, 715 GiB / 715 GiB avail; 4.6 MiB/s wr, 73 op/s; 79/1311 objects degraded (6.026%) 2023-09-23T17:56:02.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:01 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3968912919' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:02.205 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:02.254 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:02.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:02.259 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:02.332 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 4 16 11164 11148 174.111 182.75 0.00322585 0.0057351 2023-09-23T17:56:02.360 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:02.426 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:02.601 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:02.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:02.612 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:02.712 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:02.740 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:02.770 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:02.951 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:02.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:02.956 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:03.057 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:03.110 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:02 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:03.110 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:02 smithi055 ceph-mon[184174]: failed to encode map e116 with expected crc 2023-09-23T17:56:03.110 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3968912919' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:03.110 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:02 smithi055 ceph-mon[184174]: osdmap e116: 8 total, 7 up, 8 in 2023-09-23T17:56:03.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:02 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:03.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:02 smithi055 ceph-mon[179668]: failed to encode map e116 with expected crc 2023-09-23T17:56:03.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3968912919' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:03.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:02 smithi055 ceph-mon[179668]: osdmap e116: 8 total, 7 up, 8 in 2023-09-23T17:56:03.126 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:03.199 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:03.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:03.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:03.306 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:03.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:02 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:02 smithi099 ceph-mon[167076]: failed to encode map e116 with expected crc 2023-09-23T17:56:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3968912919' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:02 smithi099 ceph-mon[167076]: osdmap e116: 8 total, 7 up, 8 in 2023-09-23T17:56:03.332 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 5 16 14010 13994 174.86 177.875 0.00228287 0.00570934 2023-09-23T17:56:03.361 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:03.736 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:03.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:03.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:03.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:03.878 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:03.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:03.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:03.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:04.032 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:04.079 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:56:04.079 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:56:04.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.079 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:16047 5.c 5:37d6c6bb:::benchmark_data_smithi055_229117_object16046:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:04.079 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:04.071220+0000", 2023-09-23T17:56:04.079 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0057179600000000002, 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0057787439999999997, 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 16047 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:04.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.071220+0000", 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.071220+0000", 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.071219+0000", 2023-09-23T17:56:04.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.071250+0000", 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1341999999999999e-05 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.071262+0000", 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1848000000000001e-05 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.071270+0000", 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0600000000000008e-06 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.071673+0000", 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00040305400000000002 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.071827+0000", 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000154227 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.071975+0000", 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014743699999999999 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.073078+0000", 2023-09-23T17:56:04.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001103194 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074120+0000", 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001041557 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:16057 5.c 5:3e542363:::benchmark_data_smithi055_229117_object16056:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:04.085 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:04.074355+0000", 2023-09-23T17:56:04.086 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.002583264, 2023-09-23T17:56:04.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027680460000000001, 2023-09-23T17:56:04.086 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:04.086 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:04.086 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:04.086 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:04.086 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:04.086 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 16057 2023-09-23T17:56:04.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.086 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074355+0000", 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074355+0000", 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.087 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074354+0000", 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074377+0000", 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3062999999999999e-05 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074395+0000", 2023-09-23T17:56:04.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7989999999999999e-05 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074404+0000", 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9709999999999993e-06 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074449+0000", 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5637999999999999e-05 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074588+0000", 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013903599999999999 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074727+0000", 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013881700000000001 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:04.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.075852+0000", 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001124863 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:16048 5.0 e116/110 5:019a2b4e:::benchmark_data_smithi055_229117_object16047:head v 116'1050, mlcod=116'1047)", 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:04.074628+0000", 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0023101990000000002, 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026429790000000002, 2023-09-23T17:56:04.091 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074628+0000", 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074628+0000", 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:04.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:04.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074627+0000", 2023-09-23T17:56:04.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:04.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:04.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074653+0000", 2023-09-23T17:56:04.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5947000000000001e-05 2023-09-23T17:56:04.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:04.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074665+0000", 2023-09-23T17:56:04.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1155000000000001e-05 2023-09-23T17:56:04.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:04.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074672+0000", 2023-09-23T17:56:04.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1049999999999997e-06 2023-09-23T17:56:04.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074734+0000", 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2325999999999999e-05 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.074750+0000", 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5931999999999999e-05 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:04.095 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:04.096 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:04.096 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:04.096 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T17:56:04.096 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 3 2023-09-23T17:56:04.096 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:56:04.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:03 smithi055 ceph-mon[184174]: pgmap v238: 89 pgs: 10 active+undersized+degraded, 1 peering, 20 active+undersized, 58 active+clean; 317 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 1474/15090 objects degraded (9.768%) 2023-09-23T17:56:04.102 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:03 smithi055 ceph-mon[179668]: pgmap v238: 89 pgs: 10 active+undersized+degraded, 1 peering, 20 active+undersized, 58 active+clean; 317 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 1474/15090 objects degraded (9.768%) 2023-09-23T17:56:04.109 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:04.177 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:04.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:04.183 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:04.284 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:04.321 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:03 smithi099 ceph-mon[167076]: pgmap v238: 89 pgs: 10 active+undersized+degraded, 1 peering, 20 active+undersized, 58 active+clean; 317 MiB data, 1.1 GiB used, 714 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 1474/15090 objects degraded (9.768%) 2023-09-23T17:56:04.329 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:04.332 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 6 16 16703 16687 173.767 168.312 0.00439879 0.0057494 2023-09-23T17:56:04.379 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:04.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:04.486 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:04.552 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:04.618 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:04.622 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T17:56:04.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:04.634 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:04.653 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T17:56:04.663 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:56:04.705 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:56:04.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:04.745 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:56:04.777 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:04.786 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:56:04.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:04.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:04.826 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:04.828 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:56:04.871 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:56:04.927 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:04.930 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:56:04.931 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:56:04.931 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:56:04.931 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:56:04.931 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T17:56:04.931 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:56:04.931 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:56:04.931 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:56:04.931 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T17:56:04.965 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:05.008 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:05.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:05.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:05.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:04 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3005850784' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:05.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:05.158 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:05.210 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:05.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:05.212 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:05.251 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T17:56:05.251 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:56:05.251 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T17:56:05.252 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T17:56:05.252 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T17:56:05.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T17:56:05.272 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:04 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3005850784' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:05.273 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:04 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3005850784' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:05.313 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:05.331 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:56:05.332 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:56:05.333 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 7 16 19082 19066 170.159 148.688 0.00680831 0.00587078 2023-09-23T17:56:05.406 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:05.454 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:05.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:05.459 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:05.511 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:05.560 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:05.622 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:05.694 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:05.696 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:05.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:05.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:05.801 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:05.866 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:05.969 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:56:06.037 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:06.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:06.044 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:06.145 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:06.185 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:06.228 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:05 smithi099 ceph-mon[167076]: pgmap v239: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 780 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s; 3316/37314 objects degraded (8.887%) 2023-09-23T17:56:06.228 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:05 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T17:56:06.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:06.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:06.245 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:06.301 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:05 smithi055 ceph-mon[184174]: pgmap v239: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 780 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s; 3316/37314 objects degraded (8.887%) 2023-09-23T17:56:06.301 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:05 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T17:56:06.301 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:05 smithi055 ceph-mon[179668]: pgmap v239: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 780 MiB data, 1.8 GiB used, 713 GiB / 715 GiB avail; 118 MiB/s wr, 1.89k op/s; 3316/37314 objects degraded (8.887%) 2023-09-23T17:56:06.301 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:05 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T17:56:06.333 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 8 16 21431 21415 167.24 146.812 0.00844343 0.00597435 2023-09-23T17:56:06.346 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:06.410 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:06.534 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:06.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:06.637 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:06.686 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:06.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:06.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:06.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:06.860 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:06.928 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:06.995 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:07.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:07.006 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:07.106 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:07.154 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:07.333 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:56:07.334 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 9 16 23723 23707 164.574 143.25 0.00943269 0.00607244 2023-09-23T17:56:07.378 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:56:07.378 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:56:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:07 smithi055 ceph-mon[184174]: pgmap v240: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 890 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s; 3316/42576 objects degraded (7.788%) 2023-09-23T17:56:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:07.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:07.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:07.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:07 smithi055 ceph-mon[179668]: pgmap v240: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 890 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s; 3316/42576 objects degraded (7.788%) 2023-09-23T17:56:07.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:07 smithi099 ceph-mon[167076]: pgmap v240: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 890 MiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 111 MiB/s wr, 1.77k op/s; 3316/42576 objects degraded (7.788%) 2023-09-23T17:56:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:07.926 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T17:56:08.034 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:08.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:08.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:08.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:08.168 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:08.212 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:08.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:08.217 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:08.292 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:08.318 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:08.333 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 10 16 25579 25563 159.716 116 0.0064381 0.00625456 2023-09-23T17:56:08.354 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 3316/42576 objects degraded (7.788%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/337755483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:08 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 3316/42576 objects degraded (7.788%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/337755483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:08 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:08.775 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T17:56:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 3316/42576 objects degraded (7.788%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/337755483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:08 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:09.010 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:09.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:09.015 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:09.116 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:09.188 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:09.293 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:56:09.293 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T17:56:09.293 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:56:09.334 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 11 16 27666 27650 157.054 130.438 0.00733703 0.00636284 2023-09-23T17:56:09.444 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:09 smithi055 ceph-mon[179668]: pgmap v241: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 4864/56871 objects degraded (8.553%) 2023-09-23T17:56:09.444 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:09 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:09.444 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:09 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:56:09.444 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:09 smithi055 ceph-mon[179668]: osdmap e117: 8 total, 7 up, 8 in 2023-09-23T17:56:09.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:09 smithi099 ceph-mon[167076]: pgmap v241: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 4864/56871 objects degraded (8.553%) 2023-09-23T17:56:09.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:09 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:09.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:09 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:56:09.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:09 smithi099 ceph-mon[167076]: osdmap e117: 8 total, 7 up, 8 in 2023-09-23T17:56:09.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:09 smithi055 ceph-mon[184174]: pgmap v241: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 1.2 GiB data, 3.4 GiB used, 712 GiB / 715 GiB avail; 124 MiB/s wr, 1.98k op/s; 4864/56871 objects degraded (8.553%) 2023-09-23T17:56:09.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:09 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:09.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:09 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:56:09.689 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:09 smithi055 ceph-mon[184174]: osdmap e117: 8 total, 7 up, 8 in 2023-09-23T17:56:09.728 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:09.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:09.733 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:09.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:09.835 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:09.904 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:10.140 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:10.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:10.150 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:10.251 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:10.295 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:10.334 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 12 16 29877 29861 155.481 138.188 0.00899221 0.00642593 2023-09-23T17:56:10.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:10.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:10.359 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:10.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:10.497 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:10.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:10.551 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:10 smithi099 ceph-mon[167076]: failed to encode map e117 with expected crc 2023-09-23T17:56:10.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:10.552 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:10.587 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:56:10.623 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:56:10.653 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:10.662 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:56:10.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:56:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:56:10] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.43.0" 2023-09-23T17:56:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:10 smithi055 ceph-mon[184174]: failed to encode map e117 with expected crc 2023-09-23T17:56:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:10 smithi055 ceph-mon[179668]: failed to encode map e117 with expected crc 2023-09-23T17:56:10.706 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:56:10.717 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:10.746 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:56:10.781 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:10.785 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:56:10.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:10.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:10.824 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:56:10.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:56:10.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:56:10.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:56:10.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T17:56:10.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:56:10.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:56:10.825 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:56:10.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T17:56:10.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:10.927 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:10.980 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:10.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:10.986 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:11.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:11.129 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:11.185 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:11.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:11.189 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:11.198 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:56:11.199 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:56:11.289 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:11.325 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:11.335 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 13 16 31908 31892 153.265 126.938 0.00504746 0.00651982 2023-09-23T17:56:11.372 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:11.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:11.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:11.479 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:11.525 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:11.574 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:11.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:11.579 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:11.585 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:56:11.585 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T17:56:11.585 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T17:56:11.585 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:56:11.585 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.585 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:5660 5.c 5:39a0ba1d:::benchmark_data_smithi055_229117_object5659:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:11.586 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.499733+0000", 2023-09-23T17:56:11.586 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.081582321999999, 2023-09-23T17:56:11.586 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014406909000000001, 2023-09-23T17:56:11.586 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.586 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.586 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.586 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.586 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.586 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 5660 2023-09-23T17:56:11.586 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.587 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.587 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.587 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.587 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.499733+0000", 2023-09-23T17:56:11.587 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.587 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.587 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.587 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.587 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.499733+0000", 2023-09-23T17:56:11.587 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.588 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.588 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.588 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.588 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.499731+0000", 2023-09-23T17:56:11.588 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:11.588 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.588 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.588 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.588 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.499750+0000", 2023-09-23T17:56:11.588 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8921000000000001e-05 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.499763+0000", 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2119e-05 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.499768+0000", 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7270000000000004e-06 2023-09-23T17:56:11.589 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.590 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.590 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.590 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.499820+0000", 2023-09-23T17:56:11.590 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1860999999999997e-05 2023-09-23T17:56:11.590 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.590 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.590 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.590 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.499954+0000", 2023-09-23T17:56:11.590 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013424300000000001 2023-09-23T17:56:11.590 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.500099+0000", 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014485400000000001 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.501130+0000", 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001030869 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.591 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.592 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.592 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.501786+0000", 2023-09-23T17:56:11.592 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00065568199999999999 2023-09-23T17:56:11.592 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.592 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.592 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.592 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.514050+0000", 2023-09-23T17:56:11.592 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012263918 2023-09-23T17:56:11.592 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.592 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.593 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.593 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.514100+0000", 2023-09-23T17:56:11.593 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9908999999999999e-05 2023-09-23T17:56:11.593 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.593 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.593 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.593 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.514139+0000", 2023-09-23T17:56:11.593 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9889999999999999e-05 2023-09-23T17:56:11.593 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.593 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.594 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.594 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.594 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.594 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:5916 5.c 5:356105f2:::benchmark_data_smithi055_229117_object5915:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:11.594 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.598909+0000", 2023-09-23T17:56:11.594 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 10.982406166000001, 2023-09-23T17:56:11.594 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012520331000000001, 2023-09-23T17:56:11.594 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.594 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.594 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.595 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.595 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.595 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 5916 2023-09-23T17:56:11.595 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.595 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.595 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.595 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.595 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.598909+0000", 2023-09-23T17:56:11.595 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.595 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.596 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.596 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.596 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.598909+0000", 2023-09-23T17:56:11.596 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.596 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.596 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.596 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.596 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.598907+0000", 2023-09-23T17:56:11.597 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:11.597 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.597 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.597 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.597 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.598931+0000", 2023-09-23T17:56:11.597 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4377000000000001e-05 2023-09-23T17:56:11.597 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.597 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.597 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.598 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.598949+0000", 2023-09-23T17:56:11.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7493999999999999e-05 2023-09-23T17:56:11.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.598 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.598 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.598 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.598966+0000", 2023-09-23T17:56:11.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7025999999999999e-05 2023-09-23T17:56:11.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.599 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.599085+0000", 2023-09-23T17:56:11.599 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118838 2023-09-23T17:56:11.599 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.599 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.599215+0000", 2023-09-23T17:56:11.600 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013002200000000001 2023-09-23T17:56:11.600 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.600 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.600 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.599346+0000", 2023-09-23T17:56:11.600 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013090700000000001 2023-09-23T17:56:11.600 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.600 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.600391+0000", 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001045011 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.600400+0000", 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6590000000000002e-06 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.611333+0000", 2023-09-23T17:56:11.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010932493 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.611387+0000", 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3940999999999997e-05 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.611429+0000", 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2256000000000003e-05 2023-09-23T17:56:11.602 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:5917 5.c 5:335da303:::benchmark_data_smithi055_229117_object5916:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:00.598985+0000", 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 10.982330103000001, 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012582579, 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.603 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 5917 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.598985+0000", 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.604 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.598985+0000", 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.598984+0000", 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.605 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.599009+0000", 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4426999999999998e-05 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.599018+0000", 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6809999999999993e-06 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.599024+0000", 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0270000000000001e-06 2023-09-23T17:56:11.606 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.599776+0000", 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00075126900000000005 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.599895+0000", 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119621 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.600030+0000", 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000135029 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.601245+0000", 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001214674 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.611477+0000", 2023-09-23T17:56:11.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01023202 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.611492+0000", 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4803e-05 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.611538+0000", 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5809000000000001e-05 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.609 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:00.611567+0000", 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9774999999999999e-05 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:9458 5.c 5:3b2591a8:::benchmark_data_smithi055_229117_object9457:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e115)", 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:01.737134+0000", 2023-09-23T17:56:11.610 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 9.8441804259999994, 2023-09-23T17:56:11.611 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01522574, 2023-09-23T17:56:11.611 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.611 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.611 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.611 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.611 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.611 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 9458 2023-09-23T17:56:11.611 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.611 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.611 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.737134+0000", 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.737134+0000", 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.612 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.737133+0000", 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.737157+0000", 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3620000000000001e-05 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.737169+0000", 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2174999999999999e-05 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.737177+0000", 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9759999999999995e-06 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.737229+0000", 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1705e-05 2023-09-23T17:56:11.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.737358+0000", 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012974500000000001 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.737507+0000", 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014830400000000001 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.738914+0000", 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014074809999999999 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.741506+0000", 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025918740000000001 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.752288+0000", 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010781437 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.752330+0000", 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2626999999999997e-05 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:01.752360+0000", 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0037999999999999e-05 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.617 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:11614 5.5 e116/110 5:aeab7b99:::benchmark_data_smithi055_229117_object11613:head v 116'709, mlcod=116'706)", 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:02.496347+0000", 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 9.0849681859999993, 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016097389, 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.496347+0000", 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.496347+0000", 2023-09-23T17:56:11.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.496346+0000", 2023-09-23T17:56:11.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.496368+0000", 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2592999999999999e-05 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.496380+0000", 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1620999999999999e-05 2023-09-23T17:56:11.620 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.621 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.621 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.621 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.496387+0000", 2023-09-23T17:56:11.621 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3209999999999999e-06 2023-09-23T17:56:11.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.621 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.621 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.621 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.503094+0000", 2023-09-23T17:56:11.621 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0067063549999999998 2023-09-23T17:56:11.622 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.622 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.503113+0000", 2023-09-23T17:56:11.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8824000000000001e-05 2023-09-23T17:56:11.622 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.622 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.512399+0000", 2023-09-23T17:56:11.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092860019999999998 2023-09-23T17:56:11.623 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.623 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.623 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.623 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.512444+0000", 2023-09-23T17:56:11.623 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5563999999999999e-05 2023-09-23T17:56:11.623 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.623 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.623 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.623 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.623 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.624 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:11633 5.c 5:372720d0:::benchmark_data_smithi055_229117_object11632:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:11.624 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:02.504095+0000", 2023-09-23T17:56:11.624 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 9.0772203240000007, 2023-09-23T17:56:11.624 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018277750999999998, 2023-09-23T17:56:11.624 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.624 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.624 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.624 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.624 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.624 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 11633 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.504095+0000", 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.504095+0000", 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.625 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.504093+0000", 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.504131+0000", 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7577999999999998e-05 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.626 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.504147+0000", 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6456e-05 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.504156+0000", 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9700000000000005e-06 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.512327+0000", 2023-09-23T17:56:11.627 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0081714040000000002 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.512455+0000", 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000127861 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.512590+0000", 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013493399999999999 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.628 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.513954+0000", 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013634070000000001 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.514715+0000", 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00076180699999999998 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.522277+0000", 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.007561702 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.522330+0000", 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2939999999999998e-05 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.522372+0000", 2023-09-23T17:56:11.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2289000000000001e-05 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:11884 5.c 5:3141f1f6:::benchmark_data_smithi055_229117_object11883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:02.590000+0000", 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 8.9913149719999996, 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012415926000000001, 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.631 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 11884 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.590000+0000", 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.632 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.590000+0000", 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.589998+0000", 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.590023+0000", 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5313e-05 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.590043+0000", 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9264999999999999e-05 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.590052+0000", 2023-09-23T17:56:11.634 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2010000000000005e-06 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.590108+0000", 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6366999999999998e-05 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.590288+0000", 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000179283 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.635 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.590468+0000", 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017991400000000001 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.592035+0000", 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015673880000000001 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.594432+0000", 2023-09-23T17:56:11.636 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023973839999999998 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.602324+0000", 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0078914469999999994 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.602379+0000", 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4982999999999999e-05 2023-09-23T17:56:11.637 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:02.602416+0000", 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7109e-05 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:14273 5.c 5:32ca4629:::benchmark_data_smithi055_229117_object14272:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:03.440806+0000", 2023-09-23T17:56:11.638 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 8.1405093829999995, 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012123816000000001, 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 14273 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.639 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.440806+0000", 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.440806+0000", 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.440804+0000", 2023-09-23T17:56:11.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.440835+0000", 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1597999999999998e-05 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.440856+0000", 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0398999999999999e-05 2023-09-23T17:56:11.641 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.440865+0000", 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7200000000000001e-06 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.440956+0000", 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0158e-05 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.642 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.441112+0000", 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015641799999999999 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.441283+0000", 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000171058 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.643 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.443020+0000", 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0017368450000000001 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.444559+0000", 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015386569999999999 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.452830+0000", 2023-09-23T17:56:11.644 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082714969999999992 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.452890+0000", 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9472000000000003e-05 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:03.452929+0000", 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9759000000000003e-05 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.645 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:15988 5.c 5:36b5e9f4:::benchmark_data_smithi055_229117_object15987:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:04.044108+0000", 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 7.5372066740000001, 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015303165000000001, 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.646 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 15988 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044108+0000", 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044108+0000", 2023-09-23T17:56:11.647 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044106+0000", 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044133+0000", 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7396999999999999e-05 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.648 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044147+0000", 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3084e-05 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044155+0000", 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2649999999999996e-06 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.649 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044352+0000", 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019734900000000001 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044482+0000", 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000129586 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044624+0000", 2023-09-23T17:56:11.650 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014234800000000001 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.049390+0000", 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047662370000000004 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.049398+0000", 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5580000000000003e-06 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.651 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.059317+0000", 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0099188279999999993 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.059374+0000", 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7234000000000001e-05 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.059411+0000", 2023-09-23T17:56:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7444000000000003e-05 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:15990 5.c 5:34a8055d:::benchmark_data_smithi055_229117_object15989:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:04.044217+0000", 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 7.5370976409999999, 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015308212, 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.653 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 15990 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044217+0000", 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.654 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044217+0000", 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044217+0000", 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.655 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044238+0000", 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1458999999999999e-05 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044247+0000", 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0119999999999994e-06 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.044253+0000", 2023-09-23T17:56:11.656 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.879e-06 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.045325+0000", 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0010721909999999999 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.045459+0000", 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013407600000000001 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.657 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.045594+0000", 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013506999999999999 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.049403+0000", 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038087519999999999 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.658 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.049434+0000", 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1069999999999999e-05 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.059451+0000", 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01001699 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.059494+0000", 2023-09-23T17:56:11.659 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2762000000000002e-05 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.059525+0000", 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1476000000000002e-05 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:16007 5.c 5:3d20bc4d:::benchmark_data_smithi055_229117_object16006:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:11.660 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:04.051094+0000", 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 7.5302204560000003, 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013675079999999999, 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 16007 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.661 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.051094+0000", 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.051094+0000", 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.051093+0000", 2023-09-23T17:56:11.662 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.051118+0000", 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5664e-05 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.051137+0000", 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8131e-05 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.051145+0000", 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6799999999999999e-06 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.058265+0000", 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.007120085 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.664 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.058378+0000", 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011291800000000001 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.058534+0000", 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015541700000000001 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.059869+0000", 2023-09-23T17:56:11.665 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001334993 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.060139+0000", 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00027079399999999998 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.064676+0000", 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045367239999999998 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.667 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.667 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.064729+0000", 2023-09-23T17:56:11.667 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2788999999999999e-05 2023-09-23T17:56:11.667 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.667 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.667 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.667 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.064770+0000", 2023-09-23T17:56:11.667 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0562999999999998e-05 2023-09-23T17:56:11.667 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:16242 5.c 5:3f97b8eb:::benchmark_data_smithi055_229117_object16241:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:04.152389+0000", 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 7.4289261949999998, 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013479984, 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.668 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 16242 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.152389+0000", 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.669 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.152389+0000", 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.152387+0000", 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.670 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.152410+0000", 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2744000000000001e-05 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.152424+0000", 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4049000000000001e-05 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.152432+0000", 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2509999999999994e-06 2023-09-23T17:56:11.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.157070+0000", 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0046376480000000003 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.157255+0000", 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018494200000000001 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.157435+0000", 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000180722 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.158730+0000", 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001294461 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.673 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.163020+0000", 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0042905180000000001 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.165767+0000", 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027470350000000001 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.165830+0000", 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2482000000000002e-05 2023-09-23T17:56:11.674 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.675 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.675 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.677 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.165869+0000", 2023-09-23T17:56:11.677 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8878999999999998e-05 2023-09-23T17:56:11.677 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.677 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.677 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.677 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.677 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:16335 5.8 e116/110 5:1270286c:::benchmark_data_smithi055_229117_object16334:head v 116'1008, mlcod=116'1005)", 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:04.203832+0000", 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 7.3774830869999999, 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013234602, 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.203832+0000", 2023-09-23T17:56:11.678 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.203832+0000", 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.203831+0000", 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.679 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.203855+0000", 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3898999999999999e-05 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.203866+0000", 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0888e-05 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.203873+0000", 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2860000000000002e-06 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.203941+0000", 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7225999999999996e-05 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.203960+0000", 2023-09-23T17:56:11.681 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9650999999999999e-05 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.217015+0000", 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013055031999999999 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.217066+0000", 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1206999999999999e-05 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.682 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:17412 5.c 5:387e8647:::benchmark_data_smithi055_229117_object17411:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:04.597612+0000", 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 6.9837032829999997, 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015857954, 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.683 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 17412 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597612+0000", 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597612+0000", 2023-09-23T17:56:11.684 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597611+0000", 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597641+0000", 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0527000000000001e-05 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.685 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597654+0000", 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2468e-05 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597661+0000", 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8589999999999999e-06 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.686 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.598887+0000", 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001226359 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.599018+0000", 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013107 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.599162+0000", 2023-09-23T17:56:11.687 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014349999999999999 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.600702+0000", 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015403179999999999 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.604522+0000", 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003820351 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.688 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.613388+0000", 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0088661079999999993 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.613430+0000", 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1619000000000003e-05 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.689 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.613470+0000", 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9625000000000001e-05 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:17416 5.c 5:309d4cdc:::benchmark_data_smithi055_229117_object17415:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e116)", 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:04.597667+0000", 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 6.9836473960000003, 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015925689999999999, 2023-09-23T17:56:11.690 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 17416 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597667+0000", 2023-09-23T17:56:11.691 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597667+0000", 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597667+0000", 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.692 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597688+0000", 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1172999999999999e-05 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597699+0000", 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.092e-05 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.693 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.597705+0000", 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8710000000000002e-06 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.599823+0000", 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002118577 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.599940+0000", 2023-09-23T17:56:11.694 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011699 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.600083+0000", 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014283000000000001 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.600838+0000", 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00075513499999999999 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.695 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.604546+0000", 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037072239999999998 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.613516+0000", 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0089705300000000009 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.696 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.613558+0000", 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1334000000000001e-05 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:04.613593+0000", 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5669e-05 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:30635 5.c 5:35b3bfe7:::benchmark_data_smithi055_229117_object30634:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:11.697 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:10.731424+0000", 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.84989094200000004, 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016641864999999999, 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 30635 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.698 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.731424+0000", 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.731424+0000", 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.731422+0000", 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.731452+0000", 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.976e-05 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.731467+0000", 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5285e-05 2023-09-23T17:56:11.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.731475+0000", 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1950000000000003e-06 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.731522+0000", 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6653999999999998e-05 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.701 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.731642+0000", 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119863 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.731763+0000", 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000121614 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.734092+0000", 2023-09-23T17:56:11.702 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023290350000000001 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.738473+0000", 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0043809019999999999 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.747969+0000", 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0094961960000000002 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.703 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.748021+0000", 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1733000000000001e-05 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.748066+0000", 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4577999999999999e-05 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:30643 5.c 5:389f3540:::benchmark_data_smithi055_229117_object30642:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:10.735595+0000", 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.845719887, 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012577988, 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.705 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 30643 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.735595+0000", 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.735595+0000", 2023-09-23T17:56:11.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.735593+0000", 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.735627+0000", 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3825000000000002e-05 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.735647+0000", 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0373999999999998e-05 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.735658+0000", 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0487e-05 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.708 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.735762+0000", 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104065 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.735887+0000", 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000125778 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.736033+0000", 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014572399999999999 2023-09-23T17:56:11.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.739168+0000", 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031345919999999998 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.739368+0000", 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020006599999999999 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.710 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.748100+0000", 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00873261 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.748144+0000", 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3121000000000003e-05 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.748173+0000", 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9445999999999999e-05 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:30691 5.c 5:312df1a6:::benchmark_data_smithi055_229117_object30690:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:10.761821+0000", 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.81949344199999996, 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017497759000000002, 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.712 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 30691 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.761821+0000", 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.761821+0000", 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.761819+0000", 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.761848+0000", 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8521999999999999e-05 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.761871+0000", 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2849000000000001e-05 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.761881+0000", 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0617e-05 2023-09-23T17:56:11.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.761954+0000", 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2755000000000004e-05 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.762105+0000", 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015141199999999999 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.716 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.762270+0000", 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000164167 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.765097+0000", 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00282738 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.769172+0000", 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040746209999999996 2023-09-23T17:56:11.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.779225+0000", 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010053484 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.779281+0000", 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6124e-05 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.718 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.779319+0000", 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8039e-05 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:30975 5.c 5:3d1183d0:::benchmark_data_smithi055_229117_object30974:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:10.917700+0000", 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.66361505399999998, 2023-09-23T17:56:11.719 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012475727000000001, 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 30975 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917700+0000", 2023-09-23T17:56:11.720 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917700+0000", 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917698+0000", 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.721 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917726+0000", 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7858999999999999e-05 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917744+0000", 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7521e-05 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917753+0000", 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1810000000000007e-06 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.921251+0000", 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034980549999999999 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.921400+0000", 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014913700000000001 2023-09-23T17:56:11.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.921564+0000", 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016406999999999999 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.925210+0000", 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003645718 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.724 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.925699+0000", 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00048973599999999995 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.930079+0000", 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0043791079999999996 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.930135+0000", 2023-09-23T17:56:11.725 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6041000000000001e-05 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.930176+0000", 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0960000000000001e-05 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.726 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:30976 5.c 5:33f955ff:::benchmark_data_smithi055_229117_object30975:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:10.917766+0000", 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.66354862599999997, 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012539178, 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 30976 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:11.727 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917766+0000", 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917766+0000", 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:11.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917766+0000", 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917791+0000", 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5307999999999999e-05 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917801+0000", 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7699999999999996e-06 2023-09-23T17:56:11.729 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.917807+0000", 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6560000000000003e-06 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.922815+0000", 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050072809999999997 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.730 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.922941+0000", 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000126129 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.923077+0000", 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000136372 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.925219+0000", 2023-09-23T17:56:11.731 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002141596 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.929977+0000", 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047586829999999997 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.930233+0000", 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00025537900000000002 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.732 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.930271+0000", 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7902999999999999e-05 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:10.930305+0000", 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4669000000000003e-05 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.733 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:11.734 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:11.734 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:56:11.734 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:11.741 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T17:56:11.746 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:11 smithi055 ceph-mon[179668]: pgmap v243: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 1.5 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 180 MiB/s wr, 2.88k op/s; 6200/75762 objects degraded (8.184%) 2023-09-23T17:56:11.746 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:11 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3304402905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:11.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:11 smithi055 ceph-mon[184174]: pgmap v243: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 1.5 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 180 MiB/s wr, 2.88k op/s; 6200/75762 objects degraded (8.184%) 2023-09-23T17:56:11.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:11 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3304402905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:11.812 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:11.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:11 smithi099 ceph-mon[167076]: pgmap v243: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 1.5 GiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 180 MiB/s wr, 2.88k op/s; 6200/75762 objects degraded (8.184%) 2023-09-23T17:56:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:11 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3304402905' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:11.950 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:11.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:11.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:11.981 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T17:56:12.060 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:12.143 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:12.219 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:12.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:12.224 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:12.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:12.335 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 14 16 33834 33818 150.915 120.375 0.0123416 0.00662106 2023-09-23T17:56:12.358 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:12.406 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:12.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:12.414 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:12.515 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:12.559 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:12.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:12.610 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:12 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7458/89124 objects degraded (8.368%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:12.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:12.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:12.647 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:12.713 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:12.772 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:12 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7458/89124 objects degraded (8.368%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:12.773 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:12 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7458/89124 objects degraded (8.368%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:12.779 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:12.824 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:12.836 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:12.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:12.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:12.947 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:12.988 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:13.035 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:13.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:13.039 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:13.140 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:13.183 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:13.200 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:56:13.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:13.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:13.241 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:13.336 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 15 16 35694 35678 148.604 116.25 0.00672415 0.00672274 2023-09-23T17:56:13.341 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:13.376 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:13.427 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:13.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:13.432 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:13.532 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:13.612 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:13.811 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:13.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:13.819 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:13.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:13 smithi055 ceph-mon[179668]: pgmap v244: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 1.8 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s; 7458/89124 objects degraded (8.368%) 2023-09-23T17:56:13.868 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:13 smithi055 ceph-mon[184174]: pgmap v244: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 1.8 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s; 7458/89124 objects degraded (8.368%) 2023-09-23T17:56:13.920 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:13.996 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:14.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:13 smithi099 ceph-mon[167076]: pgmap v244: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 1.8 GiB data, 5.2 GiB used, 710 GiB / 715 GiB avail; 154 MiB/s wr, 2.46k op/s; 7458/89124 objects degraded (8.368%) 2023-09-23T17:56:14.165 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:14.170 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:14.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:14.271 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:14.333 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:14.336 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 16 16 37473 37457 146.265 111.188 0.0080824 0.00683152 2023-09-23T17:56:14.376 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:14.420 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:14.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:14.431 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:14.531 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:14.599 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:14.701 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T17:56:14.701 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:56:14.702 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_0 2023-09-23T17:56:14.702 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:56:14.746 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:56:14.746 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:56:14.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.746 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:37900 5.8 5:1e455fc1:::benchmark_data_smithi055_229117_object37899:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:14.747 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:14.631415+0000", 2023-09-23T17:56:14.747 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0066752269999999997, 2023-09-23T17:56:14.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0067178309999999996, 2023-09-23T17:56:14.747 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:14.747 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:14.747 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:14.747 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:14.747 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:14.747 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 37900 2023-09-23T17:56:14.747 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631415+0000", 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631415+0000", 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:14.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631413+0000", 2023-09-23T17:56:14.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:14.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:14.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631442+0000", 2023-09-23T17:56:14.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8691999999999999e-05 2023-09-23T17:56:14.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:14.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631456+0000", 2023-09-23T17:56:14.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4177e-05 2023-09-23T17:56:14.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:14.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631465+0000", 2023-09-23T17:56:14.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6209999999999993e-06 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631607+0000", 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000142387 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631762+0000", 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015491499999999999 2023-09-23T17:56:14.751 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.752 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.752 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:56:14.752 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.632349+0000", 2023-09-23T17:56:14.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00058714300000000004 2023-09-23T17:56:14.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.752 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.752 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:14.752 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.635241+0000", 2023-09-23T17:56:14.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028915970000000001 2023-09-23T17:56:14.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.753 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:14.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.637189+0000", 2023-09-23T17:56:14.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019483 2023-09-23T17:56:14.753 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.753 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:14.753 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.754 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:37902 5.a 5:5c2a96d0:::benchmark_data_smithi055_229117_object37901:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:14.754 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:14.631481+0000", 2023-09-23T17:56:14.754 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0066092779999999997, 2023-09-23T17:56:14.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0067860300000000002, 2023-09-23T17:56:14.754 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:14.754 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:14.754 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:14.754 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:14.754 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:14.754 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 37902 2023-09-23T17:56:14.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.755 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:14.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:14.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631481+0000", 2023-09-23T17:56:14.755 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:14.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631481+0000", 2023-09-23T17:56:14.756 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.756 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.756 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.756 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:14.756 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631480+0000", 2023-09-23T17:56:14.756 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:14.756 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.756 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.756 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:14.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631503+0000", 2023-09-23T17:56:14.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2365e-05 2023-09-23T17:56:14.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:14.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631511+0000", 2023-09-23T17:56:14.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5189999999999992e-06 2023-09-23T17:56:14.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:14.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631518+0000", 2023-09-23T17:56:14.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5420000000000002e-06 2023-09-23T17:56:14.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:14.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631890+0000", 2023-09-23T17:56:14.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00037236499999999999 2023-09-23T17:56:14.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.632031+0000", 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014102599999999999 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.632367+0000", 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000335378 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:14.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.637061+0000", 2023-09-23T17:56:14.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0046943430000000001 2023-09-23T17:56:14.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.760 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.760 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:14.760 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.637069+0000", 2023-09-23T17:56:14.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.96e-06 2023-09-23T17:56:14.760 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.760 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:14.760 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.761 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:37904 5.8 5:1521d050:::benchmark_data_smithi055_229117_object37903:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:14.761 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:14.631531+0000", 2023-09-23T17:56:14.761 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.006559273, 2023-09-23T17:56:14.761 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0068587300000000004, 2023-09-23T17:56:14.761 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:14.761 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:14.761 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:14.761 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:14.761 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:14.762 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 37904 2023-09-23T17:56:14.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.762 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:14.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:14.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631531+0000", 2023-09-23T17:56:14.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631531+0000", 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631531+0000", 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:14.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631552+0000", 2023-09-23T17:56:14.764 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1803000000000001e-05 2023-09-23T17:56:14.764 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.764 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:14.764 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631560+0000", 2023-09-23T17:56:14.764 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3329999999999999e-06 2023-09-23T17:56:14.764 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.764 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:14.764 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631566+0000", 2023-09-23T17:56:14.765 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0970000000000002e-06 2023-09-23T17:56:14.765 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.765 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.765 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:14.765 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.633774+0000", 2023-09-23T17:56:14.765 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022087019999999999 2023-09-23T17:56:14.765 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.765 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.765 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:14.765 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.633899+0000", 2023-09-23T17:56:14.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012452900000000001 2023-09-23T17:56:14.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:56:14.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.634041+0000", 2023-09-23T17:56:14.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014222700000000001 2023-09-23T17:56:14.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:14.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.635713+0000", 2023-09-23T17:56:14.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001671489 2023-09-23T17:56:14.767 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.767 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:14.767 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.767 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:37907 5.b 5:de65b9d1:::benchmark_data_smithi055_229117_object37906:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:14.767 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:14.631937+0000", 2023-09-23T17:56:14.767 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0061536990000000003, 2023-09-23T17:56:14.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0065560849999999997, 2023-09-23T17:56:14.768 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:14.768 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:14.768 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:14.768 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:14.768 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:14.768 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 37907 2023-09-23T17:56:14.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.768 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:14.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:14.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631937+0000", 2023-09-23T17:56:14.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:14.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631937+0000", 2023-09-23T17:56:14.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:14.770 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631936+0000", 2023-09-23T17:56:14.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:14.770 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.770 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:14.770 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631959+0000", 2023-09-23T17:56:14.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3438000000000002e-05 2023-09-23T17:56:14.770 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.770 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:14.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631969+0000", 2023-09-23T17:56:14.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0750000000000004e-06 2023-09-23T17:56:14.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:14.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.631975+0000", 2023-09-23T17:56:14.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9829999999999999e-06 2023-09-23T17:56:14.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:14.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.633495+0000", 2023-09-23T17:56:14.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001519063 2023-09-23T17:56:14.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:14.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.633621+0000", 2023-09-23T17:56:14.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0001261 2023-09-23T17:56:14.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:14.773 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.633780+0000", 2023-09-23T17:56:14.773 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000159407 2023-09-23T17:56:14.773 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.773 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:14.773 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.635691+0000", 2023-09-23T17:56:14.773 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019107060000000001 2023-09-23T17:56:14.773 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.773 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:14.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.637396+0000", 2023-09-23T17:56:14.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0017052980000000001 2023-09-23T17:56:14.774 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.774 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:14.774 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.774 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.774 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:37912 5.d 5:b4b1495f:::benchmark_data_smithi055_229117_object37911:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:14.774 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:14.638362+0000", 2023-09-23T17:56:14.774 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4294967295.9997282, 2023-09-23T17:56:14.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000243212, 2023-09-23T17:56:14.775 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:14.775 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:56:14.775 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:14.775 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:14.775 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:14.775 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 37912 2023-09-23T17:56:14.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.775 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:14.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:14.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638362+0000", 2023-09-23T17:56:14.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:14.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638362+0000", 2023-09-23T17:56:14.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.777 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:14.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638361+0000", 2023-09-23T17:56:14.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:14.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.777 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:14.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638407+0000", 2023-09-23T17:56:14.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6186000000000001e-05 2023-09-23T17:56:14.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:14.778 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638422+0000", 2023-09-23T17:56:14.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5043e-05 2023-09-23T17:56:14.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:14.778 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638432+0000", 2023-09-23T17:56:14.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0029e-05 2023-09-23T17:56:14.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638485+0000", 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3545000000000003e-05 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:37914 5.a 5:585549c9:::benchmark_data_smithi055_229117_object37913:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:14.638450+0000", 2023-09-23T17:56:14.779 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4294967295.9996405, 2023-09-23T17:56:14.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024321400000000001, 2023-09-23T17:56:14.780 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:14.780 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:56:14.780 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:14.780 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:14.780 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:14.780 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 37914 2023-09-23T17:56:14.780 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.780 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:14.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:14.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638450+0000", 2023-09-23T17:56:14.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:14.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638450+0000", 2023-09-23T17:56:14.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.782 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:14.782 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638449+0000", 2023-09-23T17:56:14.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:14.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.782 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:14.782 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638481+0000", 2023-09-23T17:56:14.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1931000000000003e-05 2023-09-23T17:56:14.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638489+0000", 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7200000000000006e-06 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638494+0000", 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8629999999999999e-06 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:14.784 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638580+0000", 2023-09-23T17:56:14.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6448999999999999e-05 2023-09-23T17:56:14.784 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.784 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:14.784 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.784 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:37915 5.6 5:6bed45a8:::benchmark_data_smithi055_229117_object37914:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:14.784 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:14.638506+0000", 2023-09-23T17:56:14.784 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4294967295.9995847, 2023-09-23T17:56:14.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00027079300000000002, 2023-09-23T17:56:14.785 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:14.785 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T17:56:14.785 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:14.785 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:14.785 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:14.785 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 37915 2023-09-23T17:56:14.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.785 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:14.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:14.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638506+0000", 2023-09-23T17:56:14.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:14.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638506+0000", 2023-09-23T17:56:14.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:14.787 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638505+0000", 2023-09-23T17:56:14.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:14.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:14.787 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638528+0000", 2023-09-23T17:56:14.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2751e-05 2023-09-23T17:56:14.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:14.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638534+0000", 2023-09-23T17:56:14.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5869999999999998e-06 2023-09-23T17:56:14.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:14.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638538+0000", 2023-09-23T17:56:14.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.019e-06 2023-09-23T17:56:14.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:14.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638580+0000", 2023-09-23T17:56:14.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1702999999999997e-05 2023-09-23T17:56:14.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:14.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.638720+0000", 2023-09-23T17:56:14.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014044400000000001 2023-09-23T17:56:14.789 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.790 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:14.790 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.790 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:37920 5.d 5:b98684c5:::benchmark_data_smithi055_229117_object37919:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:14.790 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:14.640436+0000", 2023-09-23T17:56:14.790 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4294967295.9976549, 2023-09-23T17:56:14.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0051607739999999999, 2023-09-23T17:56:14.790 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:14.790 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:14.791 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:14.791 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:14.791 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:14.791 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 37920 2023-09-23T17:56:14.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.791 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:14.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:14.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.640436+0000", 2023-09-23T17:56:14.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:14.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.640436+0000", 2023-09-23T17:56:14.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:14.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.640434+0000", 2023-09-23T17:56:14.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:14.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:14.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.640461+0000", 2023-09-23T17:56:14.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6401999999999999e-05 2023-09-23T17:56:14.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:14.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.640471+0000", 2023-09-23T17:56:14.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0244999999999999e-05 2023-09-23T17:56:14.794 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.794 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:14.794 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.640478+0000", 2023-09-23T17:56:14.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.973e-06 2023-09-23T17:56:14.794 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.794 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:14.794 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641860+0000", 2023-09-23T17:56:14.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013815909999999999 2023-09-23T17:56:14.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:14.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.642001+0000", 2023-09-23T17:56:14.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014137399999999999 2023-09-23T17:56:14.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T17:56:14.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.642154+0000", 2023-09-23T17:56:14.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015282199999999999 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.645446+0000", 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0032922210000000001 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.645483+0000", 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6570999999999999e-05 2023-09-23T17:56:14.796 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.797 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:14.797 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.797 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:37922 5.d 5:bc41f4f5:::benchmark_data_smithi055_229117_object37921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:14.797 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:14.641275+0000", 2023-09-23T17:56:14.797 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4294967295.9968152, 2023-09-23T17:56:14.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044205920000000001, 2023-09-23T17:56:14.797 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:14.797 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:14.798 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:14.798 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:14.798 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:14.798 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 37922 2023-09-23T17:56:14.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.798 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:14.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:14.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641275+0000", 2023-09-23T17:56:14.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:14.799 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641275+0000", 2023-09-23T17:56:14.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:14.799 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641274+0000", 2023-09-23T17:56:14.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:14.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:14.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641297+0000", 2023-09-23T17:56:14.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2935999999999999e-05 2023-09-23T17:56:14.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:14.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641310+0000", 2023-09-23T17:56:14.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2544000000000001e-05 2023-09-23T17:56:14.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:14.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641318+0000", 2023-09-23T17:56:14.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0830000000000004e-06 2023-09-23T17:56:14.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:14.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.643680+0000", 2023-09-23T17:56:14.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023625500000000002 2023-09-23T17:56:14.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:14.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.643812+0000", 2023-09-23T17:56:14.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013154599999999999 2023-09-23T17:56:14.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T17:56:14.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.643984+0000", 2023-09-23T17:56:14.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000171695 2023-09-23T17:56:14.803 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.803 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:14.803 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.803 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:37923 5.b 5:d552184a:::benchmark_data_smithi055_229117_object37922:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:14.803 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:14.641325+0000", 2023-09-23T17:56:14.803 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4294967295.9967656, 2023-09-23T17:56:14.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044621729999999998, 2023-09-23T17:56:14.803 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:14.804 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:14.804 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:14.804 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:14.804 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:14.804 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 37923 2023-09-23T17:56:14.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.804 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:14.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:14.804 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641325+0000", 2023-09-23T17:56:14.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:14.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641325+0000", 2023-09-23T17:56:14.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:14.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:14.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641324+0000", 2023-09-23T17:56:14.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:14.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:14.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641345+0000", 2023-09-23T17:56:14.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1089000000000001e-05 2023-09-23T17:56:14.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:14.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641352+0000", 2023-09-23T17:56:14.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.182e-06 2023-09-23T17:56:14.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:14.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.641358+0000", 2023-09-23T17:56:14.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7119999999999997e-06 2023-09-23T17:56:14.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:14.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.645306+0000", 2023-09-23T17:56:14.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039479479999999997 2023-09-23T17:56:14.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:14.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.645400+0000", 2023-09-23T17:56:14.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4296999999999995e-05 2023-09-23T17:56:14.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:14.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:14.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:14.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:14.645524+0000", 2023-09-23T17:56:14.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012395599999999999 2023-09-23T17:56:14.809 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.809 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:14.809 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.809 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:14.809 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T17:56:14.809 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 10 2023-09-23T17:56:14.809 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:56:14.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:14.811 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:14.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:14.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:14.964 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:15.007 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:15.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:15.014 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:15.115 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:15.158 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:15.211 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:15.214 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:15.214 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:15.315 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:15.336 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 17 16 38894 38878 142.885 88.8125 0.0018452 0.00695828 2023-09-23T17:56:15.342 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:15.775 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T17:56:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:15 smithi099 ceph-mon[167076]: pgmap v245: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s; 8658/103986 objects degraded (8.326%) 2023-09-23T17:56:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:15 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:56:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3695536380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:15 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:16.129 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:16.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:15 smithi055 ceph-mon[184174]: pgmap v245: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s; 8658/103986 objects degraded (8.326%) 2023-09-23T17:56:16.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:15 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:56:16.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3695536380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:16.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:15 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:16.136 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:15 smithi055 ceph-mon[179668]: pgmap v245: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.1 GiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 139 MiB/s wr, 2.22k op/s; 8658/103986 objects degraded (8.326%) 2023-09-23T17:56:16.136 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:15 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:56:16.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3695536380' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:16.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:15 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:16.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:16.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:16.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:16.289 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:16.336 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 18 16 40651 40635 141.047 109.812 0.0153222 0.00708396 2023-09-23T17:56:16.348 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:16.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:16.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:16.455 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:16.494 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:16 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:56:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:16 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:16 smithi099 ceph-mon[167076]: osdmap e118: 8 total, 7 up, 8 in 2023-09-23T17:56:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:16 smithi099 ceph-mon[167076]: failed to encode map e118 with expected crc 2023-09-23T17:56:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:16 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:56:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:16 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:16 smithi055 ceph-mon[184174]: osdmap e118: 8 total, 7 up, 8 in 2023-09-23T17:56:17.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:16 smithi055 ceph-mon[184174]: failed to encode map e118 with expected crc 2023-09-23T17:56:17.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:16 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:56:17.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:16 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:17.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:16 smithi055 ceph-mon[179668]: osdmap e118: 8 total, 7 up, 8 in 2023-09-23T17:56:17.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:16 smithi055 ceph-mon[179668]: failed to encode map e118 with expected crc 2023-09-23T17:56:17.267 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:17.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:17.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:17.337 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 19 16 42479 42463 139.63 114.25 0.0063932 0.00715666 2023-09-23T17:56:17.372 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:17.438 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:17.750 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:17.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:17.754 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:17.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:17.912 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:17.965 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:17.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:17.969 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:17.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:17 smithi055 ceph-mon[184174]: pgmap v247: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s; 8658/107553 objects degraded (8.050%) 2023-09-23T17:56:17.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:17 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 8658/107553 objects degraded (8.050%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:17.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:17 smithi055 ceph-mon[179668]: pgmap v247: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s; 8658/107553 objects degraded (8.050%) 2023-09-23T17:56:17.992 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:17 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 8658/107553 objects degraded (8.050%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:18.070 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:17 smithi099 ceph-mon[167076]: pgmap v247: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.2 GiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 132 MiB/s wr, 2.11k op/s; 8658/107553 objects degraded (8.050%) 2023-09-23T17:56:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:17 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 8658/107553 objects degraded (8.050%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:18.136 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:18.143 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T17:56:18.340 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T17:56:18.338514+0000 min lat: 0.00103404 max lat: 0.541726 avg lat: 0.00723588 2023-09-23T17:56:18.340 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T17:56:18.340 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 20 16 44225 44209 138.105 109.125 0.00590631 0.00723588 2023-09-23T17:56:18.439 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:18.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:18.445 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:18.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:18.581 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:18.738 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:18.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:18.745 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:18.752 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:18.846 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:18.920 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:19.149 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:19.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:19.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:19.256 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:19.338 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 21 15 45715 45700 135.961 93.1875 0.0159717 0.0073493 2023-09-23T17:56:19.346 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:19.346 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:19.378 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:56:19.417 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:56:19.461 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:56:19.486 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:19.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:19.494 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:19.505 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:56:19.545 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:56:19.581 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:56:19.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:19.634 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:56:19.634 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:56:19.635 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:56:19.635 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:56:19.635 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T17:56:19.635 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:56:19.635 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:56:19.635 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:56:19.635 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T17:56:19.639 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:19.697 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:19.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:19.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:19.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:19.847 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:19.896 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:19.899 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:19 smithi099 ceph-mon[167076]: pgmap v248: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 9739/117105 objects degraded (8.316%) 2023-09-23T17:56:19.900 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:19 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/27821707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:19.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:19.901 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:19.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:19 smithi055 ceph-mon[184174]: pgmap v248: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 9739/117105 objects degraded (8.316%) 2023-09-23T17:56:19.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:19 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/27821707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:19.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:19 smithi055 ceph-mon[179668]: pgmap v248: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.4 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 9739/117105 objects degraded (8.316%) 2023-09-23T17:56:19.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:19 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/27821707' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:20.002 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:20.071 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:20.075 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T17:56:20.117 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:20.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:20.120 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:20.221 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:20.266 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:20.309 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:20.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:20.313 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:20.323 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:56:20.324 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:56:20.338 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 22 16 47165 47149 133.895 90.5625 0.0182038 0.0074629 2023-09-23T17:56:20.413 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:20.455 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:20.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:20.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:20.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:20.555 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:56:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:56:20] "GET /metrics HTTP/1.1" 200 36510 "" "Prometheus/2.43.0" 2023-09-23T17:56:20.620 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:20.656 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:20.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:20.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:20.711 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:20.811 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:20.884 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:21.121 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:21.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:21.129 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:21.230 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:21.264 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:56:21.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:21.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:21.316 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:21.326 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:56:21.339 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 23 16 48510 48494 131.729 84.0625 0.00810455 0.00758712 2023-09-23T17:56:21.417 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:21.499 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:21.804 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:21 smithi055 ceph-mon[179668]: pgmap v249: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s; 10810/131481 objects degraded (8.222%) 2023-09-23T17:56:21.804 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:21.804 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:21.804 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:21.804 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:21.804 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:21 smithi055 ceph-mon[184174]: pgmap v249: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s; 10810/131481 objects degraded (8.222%) 2023-09-23T17:56:21.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:21.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:21.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:21.805 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:21.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:21.825 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:21.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:21.926 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:21.944 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:21.957 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:21 smithi099 ceph-mon[167076]: pgmap v249: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.7 GiB data, 7.4 GiB used, 708 GiB / 715 GiB avail; 116 MiB/s wr, 1.86k op/s; 10810/131481 objects degraded (8.222%) 2023-09-23T17:56:21.958 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:21.958 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:21.958 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:21.958 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:21.962 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:21.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:22.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:22.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:22.015 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":249,"stamp":"2023-09-23T17:56:20.427532+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2875201459,"num_objects":43827,"num_object_clones":0,"num_object_copies":131481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10810,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43827,"num_whiteouts":0,"num_read":6494,"num_read_kb":13112,"num_write":44945,"num_write_kb":2818277,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":44991,"ondisk_log_size":44991,"up":236,"acting":236,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":264,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":7724792,"kb_used_data":7430968,"kb_used_omap":29,"kb_used_meta":293730,"kb_avail":742236424,"statfs":{"total":767960285184,"available":760050098176,"internally_reserved":0,"allocated":7609311232,"data_stored":7605022310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30208,"internal_metadata":300780032},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,5,9,6,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1217200128,"num_objects":18573,"num_object_clones":0,"num_object_copies":55719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4610,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":18573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":18573,"num_write_kb":1188672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004280"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":97,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.759127+0000","last_change":"2023-09-23T17:54:25.605945+0000","last_active":"2023-09-23T17:56:08.759127+0000","last_peered":"2023-09-23T17:56:08.759127+0000","last_clean":"2023-09-23T17:56:08.759127+0000","last_became_active":"2023-09-23T17:54:25.605583+0000","last_became_peered":"2023-09-23T17:54:25.605583+0000","last_unstale":"2023-09-23T17:56:08.759127+0000","last_undegraded":"2023-09-23T17:56:08.759127+0000","last_fullsized":"2023-09-23T17:56:08.759127+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:14.076789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.758033+0000","last_change":"2023-09-23T17:54:57.036156+0000","last_active":"2023-09-23T17:56:08.758033+0000","last_peered":"2023-09-23T17:56:08.758033+0000","last_clean":"2023-09-23T17:56:08.758033+0000","last_became_active":"2023-09-23T17:54:57.035478+0000","last_became_peered":"2023-09-23T17:54:57.035478+0000","last_unstale":"2023-09-23T17:56:08.758033+0000","last_undegraded":"2023-09-23T17:56:08.758033+0000","last_fullsized":"2023-09-23T17:56:08.758033+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:53.126638+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":55,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.763994+0000","last_change":"2023-09-23T17:54:57.034381+0000","last_active":"2023-09-23T17:56:08.763994+0000","last_peered":"2023-09-23T17:56:08.763994+0000","last_clean":"2023-09-23T17:56:08.763994+0000","last_became_active":"2023-09-23T17:54:57.034043+0000","last_became_peered":"2023-09-23T17:54:57.034043+0000","last_unstale":"2023-09-23T17:56:08.763994+0000","last_undegraded":"2023-09-23T17:56:08.763994+0000","last_fullsized":"2023-09-23T17:56:08.763994+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:46:40.217762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":61,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762269+0000","last_change":"2023-09-23T17:54:57.039438+0000","last_active":"2023-09-23T17:56:08.762269+0000","last_peered":"2023-09-23T17:56:08.762269+0000","last_clean":"2023-09-23T17:56:08.762269+0000","last_became_active":"2023-09-23T17:54:57.039163+0000","last_became_peered":"2023-09-23T17:54:57.039163+0000","last_unstale":"2023-09-23T17:56:08.762269+0000","last_undegraded":"2023-09-23T17:56:08.762269+0000","last_fullsized":"2023-09-23T17:56:08.762269+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.265833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":98,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.758446+0000","last_change":"2023-09-23T17:54:57.031226+0000","last_active":"2023-09-23T17:56:08.758446+0000","last_peered":"2023-09-23T17:56:08.758446+0000","last_clean":"2023-09-23T17:56:08.758446+0000","last_became_active":"2023-09-23T17:54:57.030880+0000","last_became_peered":"2023-09-23T17:54:57.030880+0000","last_unstale":"2023-09-23T17:56:08.758446+0000","last_undegraded":"2023-09-23T17:56:08.758446+0000","last_fullsized":"2023-09-23T17:56:08.758446+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:49.836129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":64,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.761197+0000","last_change":"2023-09-23T17:55:58.864932+0000","last_active":"2023-09-23T17:56:08.761197+0000","last_peered":"2023-09-23T17:56:08.761197+0000","last_clean":"2023-09-23T17:53:21.027423+0000","last_became_active":"2023-09-23T17:55:58.864932+0000","last_became_peered":"2023-09-23T17:55:58.864932+0000","last_unstale":"2023-09-23T17:56:08.761197+0000","last_undegraded":"2023-09-23T17:56:08.761197+0000","last_fullsized":"2023-09-23T17:55:58.862677+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:37.192868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":60,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762664+0000","last_change":"2023-09-23T17:54:57.036930+0000","last_active":"2023-09-23T17:56:08.762664+0000","last_peered":"2023-09-23T17:56:08.762664+0000","last_clean":"2023-09-23T17:56:08.762664+0000","last_became_active":"2023-09-23T17:54:57.036631+0000","last_became_peered":"2023-09-23T17:54:57.036631+0000","last_unstale":"2023-09-23T17:56:08.762664+0000","last_undegraded":"2023-09-23T17:56:08.762664+0000","last_fullsized":"2023-09-23T17:56:08.762664+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:08.367357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":86,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.757986+0000","last_change":"2023-09-23T17:55:58.862654+0000","last_active":"2023-09-23T17:56:08.757986+0000","last_peered":"2023-09-23T17:56:08.757986+0000","last_clean":"2023-09-23T17:55:56.840611+0000","last_became_active":"2023-09-23T17:55:58.862654+0000","last_became_peered":"2023-09-23T17:55:58.862654+0000","last_unstale":"2023-09-23T17:56:08.757986+0000","last_undegraded":"2023-09-23T17:56:08.757986+0000","last_fullsized":"2023-09-23T17:55:58.855351+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:08.275429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.760932+0000","last_change":"2023-09-23T17:53:52.732689+0000","last_active":"2023-09-23T17:56:08.760932+0000","last_peered":"2023-09-23T17:56:08.760932+0000","last_clean":"2023-09-23T17:56:08.760932+0000","last_became_active":"2023-09-23T17:53:52.732382+0000","last_became_peered":"2023-09-23T17:53:52.732382+0000","last_unstale":"2023-09-23T17:56:08.760932+0000","last_undegraded":"2023-09-23T17:56:08.760932+0000","last_fullsized":"2023-09-23T17:56:08.760932+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:34.408802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":38,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.763393+0000","last_change":"2023-09-23T17:55:58.861640+0000","last_active":"2023-09-23T17:56:08.763393+0000","last_peered":"2023-09-23T17:56:08.763393+0000","last_clean":"2023-09-23T17:55:56.840970+0000","last_became_active":"2023-09-23T17:55:58.861640+0000","last_became_peered":"2023-09-23T17:55:58.861640+0000","last_unstale":"2023-09-23T17:56:08.763393+0000","last_undegraded":"2023-09-23T17:56:08.763393+0000","last_fullsized":"2023-09-23T17:55:58.856676+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:53:24.404170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":83,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.763017+0000","last_change":"2023-09-23T17:54:57.043487+0000","last_active":"2023-09-23T17:56:08.763017+0000","last_peered":"2023-09-23T17:56:08.763017+0000","last_clean":"2023-09-23T17:56:08.763017+0000","last_became_active":"2023-09-23T17:54:57.043064+0000","last_became_peered":"2023-09-23T17:54:57.043064+0000","last_unstale":"2023-09-23T17:56:08.763017+0000","last_undegraded":"2023-09-23T17:56:08.763017+0000","last_fullsized":"2023-09-23T17:56:08.763017+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:34:21.634412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":62,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.764785+0000","last_change":"2023-09-23T17:55:58.865485+0000","last_active":"2023-09-23T17:56:08.764785+0000","last_peered":"2023-09-23T17:56:08.764785+0000","last_clean":"2023-09-23T17:55:57.820806+0000","last_became_active":"2023-09-23T17:55:58.865485+0000","last_became_peered":"2023-09-23T17:55:58.865485+0000","last_unstale":"2023-09-23T17:56:08.764785+0000","last_undegraded":"2023-09-23T17:56:08.764785+0000","last_fullsized":"2023-09-23T17:55:58.859630+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:40.150526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":91,"reported_epoch":117,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:08.759398+0000","last_change":"2023-09-23T17:55:58.870521+0000","last_active":"2023-09-23T17:56:08.759398+0000","last_peered":"2023-09-23T17:56:08.759398+0000","last_clean":"2023-09-23T17:55:57.822071+0000","last_became_active":"2023-09-23T17:55:58.870521+0000","last_became_peered":"2023-09-23T17:55:58.870521+0000","last_unstale":"2023-09-23T17:56:08.759398+0000","last_undegraded":"2023-09-23T17:55:58.861842+0000","last_fullsized":"2023-09-23T17:55:58.861743+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:44.493624+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":85,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.759109+0000","last_change":"2023-09-23T17:55:58.867566+0000","last_active":"2023-09-23T17:56:08.759109+0000","last_peered":"2023-09-23T17:56:08.759109+0000","last_clean":"2023-09-23T17:55:57.822027+0000","last_became_active":"2023-09-23T17:55:58.867566+0000","last_became_peered":"2023-09-23T17:55:58.867566+0000","last_unstale":"2023-09-23T17:56:08.759109+0000","last_undegraded":"2023-09-23T17:56:08.759109+0000","last_fullsized":"2023-09-23T17:55:58.861157+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:25.921118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":82,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762322+0000","last_change":"2023-09-23T17:54:25.602669+0000","last_active":"2023-09-23T17:56:08.762322+0000","last_peered":"2023-09-23T17:56:08.762322+0000","last_clean":"2023-09-23T17:56:08.762322+0000","last_became_active":"2023-09-23T17:54:25.602338+0000","last_became_peered":"2023-09-23T17:54:25.602338+0000","last_unstale":"2023-09-23T17:56:08.762322+0000","last_undegraded":"2023-09-23T17:56:08.762322+0000","last_fullsized":"2023-09-23T17:56:08.762322+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:02.816165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.758296+0000","last_change":"2023-09-23T17:54:57.040422+0000","last_active":"2023-09-23T17:56:08.758296+0000","last_peered":"2023-09-23T17:56:08.758296+0000","last_clean":"2023-09-23T17:56:08.758296+0000","last_became_active":"2023-09-23T17:54:57.040085+0000","last_became_peered":"2023-09-23T17:54:57.040085+0000","last_unstale":"2023-09-23T17:56:08.758296+0000","last_undegraded":"2023-09-23T17:56:08.758296+0000","last_fullsized":"2023-09-23T17:56:08.758296+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:47.846454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":60,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.763311+0000","last_change":"2023-09-23T17:54:57.038461+0000","last_active":"2023-09-23T17:56:08.763311+0000","last_peered":"2023-09-23T17:56:08.763311+0000","last_clean":"2023-09-23T17:56:08.763311+0000","last_became_active":"2023-09-23T17:54:57.038210+0000","last_became_peered":"2023-09-23T17:54:57.038210+0000","last_unstale":"2023-09-23T17:56:08.763311+0000","last_undegraded":"2023-09-23T17:56:08.763311+0000","last_fullsized":"2023-09-23T17:56:08.763311+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:50.077733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":77,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.760799+0000","last_change":"2023-09-23T17:54:57.042234+0000","last_active":"2023-09-23T17:56:08.760799+0000","last_peered":"2023-09-23T17:56:08.760799+0000","last_clean":"2023-09-23T17:56:08.760799+0000","last_became_active":"2023-09-23T17:54:57.041918+0000","last_became_peered":"2023-09-23T17:54:57.041918+0000","last_unstale":"2023-09-23T17:56:08.760799+0000","last_undegraded":"2023-09-23T17:56:08.760799+0000","last_fullsized":"2023-09-23T17:56:08.760799+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:44.128871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":70,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.762074+0000","last_change":"2023-09-23T17:55:58.868140+0000","last_active":"2023-09-23T17:56:08.762074+0000","last_peered":"2023-09-23T17:56:08.762074+0000","last_clean":"2023-09-23T17:55:57.822613+0000","last_became_active":"2023-09-23T17:55:58.868140+0000","last_became_peered":"2023-09-23T17:55:58.868140+0000","last_unstale":"2023-09-23T17:56:08.762074+0000","last_undegraded":"2023-09-23T17:56:08.762074+0000","last_fullsized":"2023-09-23T17:55:58.860701+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:48:05.561690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":61,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762168+0000","last_change":"2023-09-23T17:54:57.039517+0000","last_active":"2023-09-23T17:56:08.762168+0000","last_peered":"2023-09-23T17:56:08.762168+0000","last_clean":"2023-09-23T17:56:08.762168+0000","last_became_active":"2023-09-23T17:54:57.039236+0000","last_became_peered":"2023-09-23T17:54:57.039236+0000","last_unstale":"2023-09-23T17:56:08.762168+0000","last_undegraded":"2023-09-23T17:56:08.762168+0000","last_fullsized":"2023-09-23T17:56:08.762168+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.126485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":23,"reported_epoch":117,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:08.762264+0000","last_change":"2023-09-23T17:55:58.867766+0000","last_active":"2023-09-23T17:56:08.762264+0000","last_peered":"2023-09-23T17:56:08.762264+0000","last_clean":"2023-09-23T17:45:53.616299+0000","last_became_active":"2023-09-23T17:55:58.867766+0000","last_became_peered":"2023-09-23T17:55:58.867766+0000","last_unstale":"2023-09-23T17:56:08.762264+0000","last_undegraded":"2023-09-23T17:55:58.863587+0000","last_fullsized":"2023-09-23T17:55:58.863517+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:36.202414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.758123+0000","last_change":"2023-09-23T17:54:57.041584+0000","last_active":"2023-09-23T17:56:08.758123+0000","last_peered":"2023-09-23T17:56:08.758123+0000","last_clean":"2023-09-23T17:56:08.758123+0000","last_became_active":"2023-09-23T17:54:57.040808+0000","last_became_peered":"2023-09-23T17:54:57.040808+0000","last_unstale":"2023-09-23T17:56:08.758123+0000","last_undegraded":"2023-09-23T17:56:08.758123+0000","last_fullsized":"2023-09-23T17:56:08.758123+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:01.079083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"118'2619","reported_seq":2642,"reported_epoch":118,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:17.306042+0000","last_change":"2023-09-23T17:55:58.873244+0000","last_active":"2023-09-23T17:56:17.306042+0000","last_peered":"2023-09-23T17:56:17.306042+0000","last_clean":"2023-09-23T17:55:57.822204+0000","last_became_active":"2023-09-23T17:55:58.867817+0000","last_became_peered":"2023-09-23T17:55:58.867817+0000","last_unstale":"2023-09-23T17:56:17.306042+0000","last_undegraded":"2023-09-23T17:55:58.873063+0000","last_fullsized":"2023-09-23T17:55:58.860307+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2619,"log_dups_size":0,"ondisk_log_size":2619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:31.336911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171638784,"num_objects":2619,"num_object_clones":0,"num_object_copies":7857,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2619,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2619,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2619,"num_write_kb":167616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2619}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":81,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762722+0000","last_change":"2023-09-23T17:53:21.041393+0000","last_active":"2023-09-23T17:56:08.762722+0000","last_peered":"2023-09-23T17:56:08.762722+0000","last_clean":"2023-09-23T17:56:08.762722+0000","last_became_active":"2023-09-23T17:53:21.040995+0000","last_became_peered":"2023-09-23T17:53:21.040995+0000","last_unstale":"2023-09-23T17:56:08.762722+0000","last_undegraded":"2023-09-23T17:56:08.762722+0000","last_fullsized":"2023-09-23T17:56:08.762722+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:12.660494+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":67,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.760527+0000","last_change":"2023-09-23T17:55:58.870416+0000","last_active":"2023-09-23T17:56:08.760527+0000","last_peered":"2023-09-23T17:56:08.760527+0000","last_clean":"2023-09-23T17:55:56.845347+0000","last_became_active":"2023-09-23T17:55:58.870416+0000","last_became_peered":"2023-09-23T17:55:58.870416+0000","last_unstale":"2023-09-23T17:56:08.760527+0000","last_undegraded":"2023-09-23T17:56:08.760527+0000","last_fullsized":"2023-09-23T17:55:58.862810+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:13.418266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"118'2796","reported_seq":2813,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:18.566810+0000","last_change":"2023-09-23T17:55:53.818806+0000","last_active":"2023-09-23T17:56:18.566810+0000","last_peered":"2023-09-23T17:56:18.566810+0000","last_clean":"2023-09-23T17:56:18.566810+0000","last_became_active":"2023-09-23T17:55:53.818291+0000","last_became_peered":"2023-09-23T17:55:53.818291+0000","last_unstale":"2023-09-23T17:56:18.566810+0000","last_undegraded":"2023-09-23T17:56:18.566810+0000","last_fullsized":"2023-09-23T17:56:18.566810+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2796,"log_dups_size":0,"ondisk_log_size":2796,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:58.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183238656,"num_objects":2796,"num_object_clones":0,"num_object_copies":8388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2796,"num_write_kb":178944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":82,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762685+0000","last_change":"2023-09-23T17:53:52.732018+0000","last_active":"2023-09-23T17:56:08.762685+0000","last_peered":"2023-09-23T17:56:08.762685+0000","last_clean":"2023-09-23T17:56:08.762685+0000","last_became_active":"2023-09-23T17:53:52.731641+0000","last_became_peered":"2023-09-23T17:53:52.731641+0000","last_unstale":"2023-09-23T17:56:08.762685+0000","last_undegraded":"2023-09-23T17:56:08.762685+0000","last_fullsized":"2023-09-23T17:56:08.762685+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:37.290885+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":95,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.759230+0000","last_change":"2023-09-23T17:54:57.040014+0000","last_active":"2023-09-23T17:56:08.759230+0000","last_peered":"2023-09-23T17:56:08.759230+0000","last_clean":"2023-09-23T17:56:08.759230+0000","last_became_active":"2023-09-23T17:54:57.039661+0000","last_became_peered":"2023-09-23T17:54:57.039661+0000","last_unstale":"2023-09-23T17:56:08.759230+0000","last_undegraded":"2023-09-23T17:56:08.759230+0000","last_fullsized":"2023-09-23T17:56:08.759230+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:01.077579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"118'2761","reported_seq":2777,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:18.606375+0000","last_change":"2023-09-23T17:55:53.829853+0000","last_active":"2023-09-23T17:56:18.606375+0000","last_peered":"2023-09-23T17:56:18.606375+0000","last_clean":"2023-09-23T17:56:18.606375+0000","last_became_active":"2023-09-23T17:55:53.829573+0000","last_became_peered":"2023-09-23T17:55:53.829573+0000","last_unstale":"2023-09-23T17:56:18.606375+0000","last_undegraded":"2023-09-23T17:56:18.606375+0000","last_fullsized":"2023-09-23T17:56:18.606375+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2761,"log_dups_size":0,"ondisk_log_size":2761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:39.218789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180944896,"num_objects":2761,"num_object_clones":0,"num_object_copies":8283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2761,"num_write_kb":176704,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":71,"reported_epoch":117,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:08.763199+0000","last_change":"2023-09-23T17:55:58.870420+0000","last_active":"2023-09-23T17:56:08.763199+0000","last_peered":"2023-09-23T17:56:08.763199+0000","last_clean":"2023-09-23T17:55:57.822744+0000","last_became_active":"2023-09-23T17:55:58.870420+0000","last_became_peered":"2023-09-23T17:55:58.870420+0000","last_unstale":"2023-09-23T17:56:08.763199+0000","last_undegraded":"2023-09-23T17:55:58.862862+0000","last_fullsized":"2023-09-23T17:55:58.862757+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:31.925750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":78,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.765710+0000","last_change":"2023-09-23T17:55:58.864525+0000","last_active":"2023-09-23T17:56:08.765710+0000","last_peered":"2023-09-23T17:56:08.765710+0000","last_clean":"2023-09-23T17:53:52.724466+0000","last_became_active":"2023-09-23T17:55:58.864525+0000","last_became_peered":"2023-09-23T17:55:58.864525+0000","last_unstale":"2023-09-23T17:56:08.765710+0000","last_undegraded":"2023-09-23T17:56:08.765710+0000","last_fullsized":"2023-09-23T17:55:58.859288+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:31.785041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.a","version":"118'2781","reported_seq":2798,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:18.597280+0000","last_change":"2023-09-23T17:55:53.826278+0000","last_active":"2023-09-23T17:56:18.597280+0000","last_peered":"2023-09-23T17:56:18.597280+0000","last_clean":"2023-09-23T17:56:18.597280+0000","last_became_active":"2023-09-23T17:55:53.825962+0000","last_became_peered":"2023-09-23T17:55:53.825962+0000","last_unstale":"2023-09-23T17:56:18.597280+0000","last_undegraded":"2023-09-23T17:56:18.597280+0000","last_fullsized":"2023-09-23T17:56:18.597280+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2781,"log_dups_size":0,"ondisk_log_size":2781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:34:16.202383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182255616,"num_objects":2781,"num_object_clones":0,"num_object_copies":8343,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2781,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2781,"num_write_kb":177984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":85,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.760753+0000","last_change":"2023-09-23T17:55:58.865270+0000","last_active":"2023-09-23T17:56:08.760753+0000","last_peered":"2023-09-23T17:56:08.760753+0000","last_clean":"2023-09-23T17:55:56.845943+0000","last_became_active":"2023-09-23T17:55:58.865270+0000","last_became_peered":"2023-09-23T17:55:58.865270+0000","last_unstale":"2023-09-23T17:56:08.760753+0000","last_undegraded":"2023-09-23T17:56:08.760753+0000","last_fullsized":"2023-09-23T17:55:58.860736+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:15.861060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":61,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762760+0000","last_change":"2023-09-23T17:53:52.731322+0000","last_active":"2023-09-23T17:56:08.762760+0000","last_peered":"2023-09-23T17:56:08.762760+0000","last_clean":"2023-09-23T17:56:08.762760+0000","last_became_active":"2023-09-23T17:53:52.730995+0000","last_became_peered":"2023-09-23T17:53:52.730995+0000","last_unstale":"2023-09-23T17:56:08.762760+0000","last_undegraded":"2023-09-23T17:56:08.762760+0000","last_fullsized":"2023-09-23T17:56:08.762760+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:55.121531+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"118'2798","reported_seq":2815,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:18.590521+0000","last_change":"2023-09-23T17:55:53.830258+0000","last_active":"2023-09-23T17:56:18.590521+0000","last_peered":"2023-09-23T17:56:18.590521+0000","last_clean":"2023-09-23T17:56:18.590521+0000","last_became_active":"2023-09-23T17:55:53.829954+0000","last_became_peered":"2023-09-23T17:55:53.829954+0000","last_unstale":"2023-09-23T17:56:18.590521+0000","last_undegraded":"2023-09-23T17:56:18.590521+0000","last_fullsized":"2023-09-23T17:56:18.590521+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2798,"log_dups_size":0,"ondisk_log_size":2798,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:41.952908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183369728,"num_objects":2798,"num_object_clones":0,"num_object_copies":8394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2798,"num_write_kb":179072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":79,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.757730+0000","last_change":"2023-09-23T17:54:57.035203+0000","last_active":"2023-09-23T17:56:08.757730+0000","last_peered":"2023-09-23T17:56:08.757730+0000","last_clean":"2023-09-23T17:56:08.757730+0000","last_became_active":"2023-09-23T17:54:57.034854+0000","last_became_peered":"2023-09-23T17:54:57.034854+0000","last_unstale":"2023-09-23T17:56:08.757730+0000","last_undegraded":"2023-09-23T17:56:08.757730+0000","last_fullsized":"2023-09-23T17:56:08.757730+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:34.657128+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":85,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.759136+0000","last_change":"2023-09-23T17:55:58.864510+0000","last_active":"2023-09-23T17:56:08.759136+0000","last_peered":"2023-09-23T17:56:08.759136+0000","last_clean":"2023-09-23T17:55:57.821978+0000","last_became_active":"2023-09-23T17:55:58.864510+0000","last_became_peered":"2023-09-23T17:55:58.864510+0000","last_unstale":"2023-09-23T17:56:08.759136+0000","last_undegraded":"2023-09-23T17:56:08.759136+0000","last_fullsized":"2023-09-23T17:55:58.860623+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:02.492307+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3],"avail_no_missing":["4","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"117'2463","reported_seq":2479,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:15.742412+0000","last_change":"2023-09-23T17:55:53.828030+0000","last_active":"2023-09-23T17:56:15.742412+0000","last_peered":"2023-09-23T17:56:15.742412+0000","last_clean":"2023-09-23T17:56:15.742412+0000","last_became_active":"2023-09-23T17:55:53.827773+0000","last_became_peered":"2023-09-23T17:55:53.827773+0000","last_unstale":"2023-09-23T17:56:15.742412+0000","last_undegraded":"2023-09-23T17:56:15.742412+0000","last_fullsized":"2023-09-23T17:56:15.742412+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2463,"log_dups_size":0,"ondisk_log_size":2463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:06:26.949485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161415168,"num_objects":2463,"num_object_clones":0,"num_object_copies":7389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2463,"num_write_kb":157632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":65,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.759093+0000","last_change":"2023-09-23T17:55:58.866705+0000","last_active":"2023-09-23T17:56:08.759093+0000","last_peered":"2023-09-23T17:56:08.759093+0000","last_clean":"2023-09-23T17:53:21.027435+0000","last_became_active":"2023-09-23T17:55:58.866705+0000","last_became_peered":"2023-09-23T17:55:58.866705+0000","last_unstale":"2023-09-23T17:56:08.759093+0000","last_undegraded":"2023-09-23T17:56:08.759093+0000","last_fullsized":"2023-09-23T17:55:58.861946+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:23.295833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":88,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.759180+0000","last_change":"2023-09-23T17:54:25.603149+0000","last_active":"2023-09-23T17:56:08.759180+0000","last_peered":"2023-09-23T17:56:08.759180+0000","last_clean":"2023-09-23T17:56:08.759180+0000","last_became_active":"2023-09-23T17:54:25.602846+0000","last_became_peered":"2023-09-23T17:54:25.602846+0000","last_unstale":"2023-09-23T17:56:08.759180+0000","last_undegraded":"2023-09-23T17:56:08.759180+0000","last_fullsized":"2023-09-23T17:56:08.759180+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:04:06.955023+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"118'2654","reported_seq":2677,"reported_epoch":118,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:17.335387+0000","last_change":"2023-09-23T17:55:58.872200+0000","last_active":"2023-09-23T17:56:17.335387+0000","last_peered":"2023-09-23T17:56:17.335387+0000","last_clean":"2023-09-23T17:55:56.846049+0000","last_became_active":"2023-09-23T17:55:58.868344+0000","last_became_peered":"2023-09-23T17:55:58.868344+0000","last_unstale":"2023-09-23T17:56:17.335387+0000","last_undegraded":"2023-09-23T17:55:58.872053+0000","last_fullsized":"2023-09-23T17:55:58.864207+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2654,"log_dups_size":0,"ondisk_log_size":2654,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:00.042036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":173932544,"num_objects":2654,"num_object_clones":0,"num_object_copies":7962,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2654,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2654,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2654,"num_write_kb":169856,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":2654}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":79,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.758961+0000","last_change":"2023-09-23T17:54:25.602420+0000","last_active":"2023-09-23T17:56:08.758961+0000","last_peered":"2023-09-23T17:56:08.758961+0000","last_clean":"2023-09-23T17:56:08.758961+0000","last_became_active":"2023-09-23T17:54:25.601988+0000","last_became_peered":"2023-09-23T17:54:25.601988+0000","last_unstale":"2023-09-23T17:56:08.758961+0000","last_undegraded":"2023-09-23T17:56:08.758961+0000","last_fullsized":"2023-09-23T17:56:08.758961+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:17.703211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":85,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.759894+0000","last_change":"2023-09-23T17:55:58.866711+0000","last_active":"2023-09-23T17:56:08.759894+0000","last_peered":"2023-09-23T17:56:08.759894+0000","last_clean":"2023-09-23T17:55:57.822746+0000","last_became_active":"2023-09-23T17:55:58.866711+0000","last_became_peered":"2023-09-23T17:55:58.866711+0000","last_unstale":"2023-09-23T17:56:08.759894+0000","last_undegraded":"2023-09-23T17:56:08.759894+0000","last_fullsized":"2023-09-23T17:55:58.862846+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:38.611856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"118'2611","reported_seq":2626,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:17.333575+0000","last_change":"2023-09-23T17:55:53.829092+0000","last_active":"2023-09-23T17:56:17.333575+0000","last_peered":"2023-09-23T17:56:17.333575+0000","last_clean":"2023-09-23T17:56:17.333575+0000","last_became_active":"2023-09-23T17:55:53.828814+0000","last_became_peered":"2023-09-23T17:55:53.828814+0000","last_unstale":"2023-09-23T17:56:17.333575+0000","last_undegraded":"2023-09-23T17:56:17.333575+0000","last_fullsized":"2023-09-23T17:56:17.333575+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2611,"log_dups_size":0,"ondisk_log_size":2611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.806389+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":171114496,"num_objects":2611,"num_object_clones":0,"num_object_copies":7833,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2611,"num_write_kb":167104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":81,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.758604+0000","last_change":"2023-09-23T17:54:57.038911+0000","last_active":"2023-09-23T17:56:08.758604+0000","last_peered":"2023-09-23T17:56:08.758604+0000","last_clean":"2023-09-23T17:56:08.758604+0000","last_became_active":"2023-09-23T17:54:57.038607+0000","last_became_peered":"2023-09-23T17:54:57.038607+0000","last_unstale":"2023-09-23T17:56:08.758604+0000","last_undegraded":"2023-09-23T17:56:08.758604+0000","last_fullsized":"2023-09-23T17:56:08.758604+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:20.373213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":52,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.766237+0000","last_change":"2023-09-23T17:54:57.035100+0000","last_active":"2023-09-23T17:56:08.766237+0000","last_peered":"2023-09-23T17:56:08.766237+0000","last_clean":"2023-09-23T17:56:08.766237+0000","last_became_active":"2023-09-23T17:54:57.034733+0000","last_became_peered":"2023-09-23T17:54:57.034733+0000","last_unstale":"2023-09-23T17:56:08.766237+0000","last_undegraded":"2023-09-23T17:56:08.766237+0000","last_fullsized":"2023-09-23T17:56:08.766237+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:27.841590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":537,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T17:56:01.943317+0000","last_change":"2023-09-23T17:53:52.730761+0000","last_active":"2023-09-23T17:56:01.943317+0000","last_peered":"2023-09-23T17:56:01.943317+0000","last_clean":"2023-09-23T17:56:01.943317+0000","last_became_active":"2023-09-23T17:53:52.730121+0000","last_became_peered":"2023-09-23T17:53:52.730121+0000","last_unstale":"2023-09-23T17:56:01.943317+0000","last_undegraded":"2023-09-23T17:56:01.943317+0000","last_fullsized":"2023-09-23T17:56:01.943317+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:02.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":95,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.760928+0000","last_change":"2023-09-23T17:54:57.040788+0000","last_active":"2023-09-23T17:56:08.760928+0000","last_peered":"2023-09-23T17:56:08.760928+0000","last_clean":"2023-09-23T17:56:08.760928+0000","last_became_active":"2023-09-23T17:54:57.040271+0000","last_became_peered":"2023-09-23T17:54:57.040271+0000","last_unstale":"2023-09-23T17:56:08.760928+0000","last_undegraded":"2023-09-23T17:56:08.760928+0000","last_fullsized":"2023-09-23T17:56:08.760928+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:37.176032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":41,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T17:56:01.942592+0000","last_change":"2023-09-23T17:53:52.735673+0000","last_active":"2023-09-23T17:56:01.942592+0000","last_peered":"2023-09-23T17:56:01.942592+0000","last_clean":"2023-09-23T17:56:01.942592+0000","last_became_active":"2023-09-23T17:53:52.734890+0000","last_became_peered":"2023-09-23T17:53:52.734890+0000","last_unstale":"2023-09-23T17:56:01.942592+0000","last_undegraded":"2023-09-23T17:56:01.942592+0000","last_fullsized":"2023-09-23T17:56:01.942592+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:41:22.338546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"118'2790","reported_seq":2813,"reported_epoch":118,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:19.064256+0000","last_change":"2023-09-23T17:55:58.870796+0000","last_active":"2023-09-23T17:56:19.064256+0000","last_peered":"2023-09-23T17:56:19.064256+0000","last_clean":"2023-09-23T17:55:57.821023+0000","last_became_active":"2023-09-23T17:55:58.867778+0000","last_became_peered":"2023-09-23T17:55:58.867778+0000","last_unstale":"2023-09-23T17:56:19.064256+0000","last_undegraded":"2023-09-23T17:55:58.868868+0000","last_fullsized":"2023-09-23T17:55:58.864928+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2790,"log_dups_size":0,"ondisk_log_size":2790,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:30.503909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182845440,"num_objects":2790,"num_object_clones":0,"num_object_copies":8370,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2790,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2790,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2790,"num_write_kb":178560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":2790}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"118'2804","reported_seq":2821,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:19.053763+0000","last_change":"2023-09-23T17:55:53.826496+0000","last_active":"2023-09-23T17:56:19.053763+0000","last_peered":"2023-09-23T17:56:19.053763+0000","last_clean":"2023-09-23T17:56:19.053763+0000","last_became_active":"2023-09-23T17:55:53.826235+0000","last_became_peered":"2023-09-23T17:55:53.826235+0000","last_unstale":"2023-09-23T17:56:19.053763+0000","last_undegraded":"2023-09-23T17:56:19.053763+0000","last_fullsized":"2023-09-23T17:56:19.053763+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2804,"log_dups_size":0,"ondisk_log_size":2804,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:25.518153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183762944,"num_objects":2804,"num_object_clones":0,"num_object_copies":8412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2804,"num_write_kb":179456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1169,"reported_epoch":117,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:08.760713+0000","last_change":"2023-09-23T17:55:58.864229+0000","last_active":"2023-09-23T17:56:08.760713+0000","last_peered":"2023-09-23T17:56:08.760713+0000","last_clean":"2023-09-23T17:55:56.845670+0000","last_became_active":"2023-09-23T17:55:58.864229+0000","last_became_peered":"2023-09-23T17:55:58.864229+0000","last_unstale":"2023-09-23T17:56:08.760713+0000","last_undegraded":"2023-09-23T17:55:58.861121+0000","last_fullsized":"2023-09-23T17:55:58.861013+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:23.537845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1894,"num_read_kb":2706,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":86,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.759385+0000","last_change":"2023-09-23T17:55:58.867825+0000","last_active":"2023-09-23T17:56:08.759385+0000","last_peered":"2023-09-23T17:56:08.759385+0000","last_clean":"2023-09-23T17:55:57.822192+0000","last_became_active":"2023-09-23T17:55:58.867825+0000","last_became_peered":"2023-09-23T17:55:58.867825+0000","last_unstale":"2023-09-23T17:56:08.759385+0000","last_undegraded":"2023-09-23T17:56:08.759385+0000","last_fullsized":"2023-09-23T17:55:58.861688+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:11.479911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":71,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T17:56:01.943216+0000","last_change":"2023-09-23T17:54:57.035632+0000","last_active":"2023-09-23T17:56:01.943216+0000","last_peered":"2023-09-23T17:56:01.943216+0000","last_clean":"2023-09-23T17:56:01.943216+0000","last_became_active":"2023-09-23T17:54:57.035338+0000","last_became_peered":"2023-09-23T17:54:57.035338+0000","last_unstale":"2023-09-23T17:56:01.943216+0000","last_undegraded":"2023-09-23T17:56:01.943216+0000","last_fullsized":"2023-09-23T17:56:01.943216+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:18.773130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"118'2862","reported_seq":2879,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:18.579135+0000","last_change":"2023-09-23T17:55:53.829379+0000","last_active":"2023-09-23T17:56:18.579135+0000","last_peered":"2023-09-23T17:56:18.579135+0000","last_clean":"2023-09-23T17:56:18.579135+0000","last_became_active":"2023-09-23T17:55:53.829085+0000","last_became_peered":"2023-09-23T17:55:53.829085+0000","last_unstale":"2023-09-23T17:56:18.579135+0000","last_undegraded":"2023-09-23T17:56:18.579135+0000","last_fullsized":"2023-09-23T17:56:18.579135+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2862,"log_dups_size":0,"ondisk_log_size":2862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:22.353899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187564032,"num_objects":2862,"num_object_clones":0,"num_object_copies":8586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2862,"num_write_kb":183168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":381,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.765439+0000","last_change":"2023-09-23T17:54:57.037634+0000","last_active":"2023-09-23T17:56:08.765439+0000","last_peered":"2023-09-23T17:56:08.765439+0000","last_clean":"2023-09-23T17:56:08.765439+0000","last_became_active":"2023-09-23T17:54:57.037115+0000","last_became_peered":"2023-09-23T17:54:57.037115+0000","last_unstale":"2023-09-23T17:56:08.765439+0000","last_undegraded":"2023-09-23T17:56:08.765439+0000","last_fullsized":"2023-09-23T17:56:08.765439+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:52:58.348982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":61,"reported_epoch":116,"state":"active+undersized","last_fresh":"2023-09-23T17:56:01.942304+0000","last_change":"2023-09-23T17:55:58.865545+0000","last_active":"2023-09-23T17:56:01.942304+0000","last_peered":"2023-09-23T17:56:01.942304+0000","last_clean":"2023-09-23T17:55:57.820377+0000","last_became_active":"2023-09-23T17:55:58.865545+0000","last_became_peered":"2023-09-23T17:55:58.865545+0000","last_unstale":"2023-09-23T17:56:01.942304+0000","last_undegraded":"2023-09-23T17:56:01.942304+0000","last_fullsized":"2023-09-23T17:55:58.860734+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:55.266655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":84,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T17:56:01.943559+0000","last_change":"2023-09-23T17:53:52.728795+0000","last_active":"2023-09-23T17:56:01.943559+0000","last_peered":"2023-09-23T17:56:01.943559+0000","last_clean":"2023-09-23T17:56:01.943559+0000","last_became_active":"2023-09-23T17:53:52.728423+0000","last_became_peered":"2023-09-23T17:53:52.728423+0000","last_unstale":"2023-09-23T17:56:01.943559+0000","last_undegraded":"2023-09-23T17:56:01.943559+0000","last_fullsized":"2023-09-23T17:56:01.943559+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.759322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1032,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.760654+0000","last_change":"2023-09-23T17:54:25.601298+0000","last_active":"2023-09-23T17:56:08.760654+0000","last_peered":"2023-09-23T17:56:08.760654+0000","last_clean":"2023-09-23T17:56:08.760654+0000","last_became_active":"2023-09-23T17:54:25.600854+0000","last_became_peered":"2023-09-23T17:54:25.600854+0000","last_unstale":"2023-09-23T17:56:08.760654+0000","last_undegraded":"2023-09-23T17:56:08.760654+0000","last_fullsized":"2023-09-23T17:56:08.760654+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:56.978955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":468,"reported_epoch":117,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:08.761935+0000","last_change":"2023-09-23T17:55:58.867888+0000","last_active":"2023-09-23T17:56:08.761935+0000","last_peered":"2023-09-23T17:56:08.761935+0000","last_clean":"2023-09-23T17:55:57.822497+0000","last_became_active":"2023-09-23T17:55:58.867888+0000","last_became_peered":"2023-09-23T17:55:58.867888+0000","last_unstale":"2023-09-23T17:56:08.761935+0000","last_undegraded":"2023-09-23T17:55:58.860525+0000","last_fullsized":"2023-09-23T17:55:58.860424+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:18.913217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":37,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.757695+0000","last_change":"2023-09-23T17:55:58.866152+0000","last_active":"2023-09-23T17:56:08.757695+0000","last_peered":"2023-09-23T17:56:08.757695+0000","last_clean":"2023-09-23T17:55:56.841167+0000","last_became_active":"2023-09-23T17:55:58.866152+0000","last_became_peered":"2023-09-23T17:55:58.866152+0000","last_unstale":"2023-09-23T17:56:08.757695+0000","last_undegraded":"2023-09-23T17:56:08.757695+0000","last_fullsized":"2023-09-23T17:55:58.856680+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:13.914833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":13,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.759460+0000","last_change":"2023-09-23T17:55:58.872725+0000","last_active":"2023-09-23T17:56:08.759460+0000","last_peered":"2023-09-23T17:56:08.759460+0000","last_clean":"2023-09-23T17:45:51.086592+0000","last_became_active":"2023-09-23T17:55:58.872725+0000","last_became_peered":"2023-09-23T17:55:58.872725+0000","last_unstale":"2023-09-23T17:56:08.759460+0000","last_undegraded":"2023-09-23T17:56:08.759460+0000","last_fullsized":"2023-09-23T17:55:58.862384+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:06.504392+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.4","version":"118'2656","reported_seq":2672,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:17.324829+0000","last_change":"2023-09-23T17:55:53.829245+0000","last_active":"2023-09-23T17:56:17.324829+0000","last_peered":"2023-09-23T17:56:17.324829+0000","last_clean":"2023-09-23T17:56:17.324829+0000","last_became_active":"2023-09-23T17:55:53.829019+0000","last_became_peered":"2023-09-23T17:55:53.829019+0000","last_unstale":"2023-09-23T17:56:17.324829+0000","last_undegraded":"2023-09-23T17:56:17.324829+0000","last_fullsized":"2023-09-23T17:56:17.324829+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2656,"log_dups_size":0,"ondisk_log_size":2656,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:38.782556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174063616,"num_objects":2656,"num_object_clones":0,"num_object_copies":7968,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2656,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2656,"num_write_kb":169984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"118'2734","reported_seq":2750,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:17.333758+0000","last_change":"2023-09-23T17:55:53.828325+0000","last_active":"2023-09-23T17:56:17.333758+0000","last_peered":"2023-09-23T17:56:17.333758+0000","last_clean":"2023-09-23T17:56:17.333758+0000","last_became_active":"2023-09-23T17:55:53.828056+0000","last_became_peered":"2023-09-23T17:55:53.828056+0000","last_unstale":"2023-09-23T17:56:17.333758+0000","last_undegraded":"2023-09-23T17:56:17.333758+0000","last_fullsized":"2023-09-23T17:56:17.333758+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2734,"log_dups_size":0,"ondisk_log_size":2734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:47.523521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179175424,"num_objects":2734,"num_object_clones":0,"num_object_copies":8202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2734,"num_write_kb":174976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":316,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.761958+0000","last_change":"2023-09-23T17:53:52.734324+0000","last_active":"2023-09-23T17:56:08.761958+0000","last_peered":"2023-09-23T17:56:08.761958+0000","last_clean":"2023-09-23T17:56:08.761958+0000","last_became_active":"2023-09-23T17:53:52.733825+0000","last_became_peered":"2023-09-23T17:53:52.733825+0000","last_unstale":"2023-09-23T17:56:08.761958+0000","last_undegraded":"2023-09-23T17:56:08.761958+0000","last_fullsized":"2023-09-23T17:56:08.761958+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:34.742687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":60,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.761883+0000","last_change":"2023-09-23T17:54:57.036095+0000","last_active":"2023-09-23T17:56:08.761883+0000","last_peered":"2023-09-23T17:56:08.761883+0000","last_clean":"2023-09-23T17:56:08.761883+0000","last_became_active":"2023-09-23T17:54:57.035606+0000","last_became_peered":"2023-09-23T17:54:57.035606+0000","last_unstale":"2023-09-23T17:56:08.761883+0000","last_undegraded":"2023-09-23T17:56:08.761883+0000","last_fullsized":"2023-09-23T17:56:08.761883+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:10.936020+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":76,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762437+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T17:56:08.762437+0000","last_peered":"2023-09-23T17:56:08.762437+0000","last_clean":"2023-09-23T17:56:08.762437+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T17:56:08.762437+0000","last_undegraded":"2023-09-23T17:56:08.762437+0000","last_fullsized":"2023-09-23T17:56:08.762437+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":330,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.763099+0000","last_change":"2023-09-23T17:53:21.035484+0000","last_active":"2023-09-23T17:56:08.763099+0000","last_peered":"2023-09-23T17:56:08.763099+0000","last_clean":"2023-09-23T17:56:08.763099+0000","last_became_active":"2023-09-23T17:53:21.035035+0000","last_became_peered":"2023-09-23T17:53:21.035035+0000","last_unstale":"2023-09-23T17:56:08.763099+0000","last_undegraded":"2023-09-23T17:56:08.763099+0000","last_fullsized":"2023-09-23T17:56:08.763099+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:05.145111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":55,"reported_epoch":116,"state":"active+clean","last_fresh":"2023-09-23T17:56:01.943401+0000","last_change":"2023-09-23T17:54:25.604720+0000","last_active":"2023-09-23T17:56:01.943401+0000","last_peered":"2023-09-23T17:56:01.943401+0000","last_clean":"2023-09-23T17:56:01.943401+0000","last_became_active":"2023-09-23T17:54:25.604386+0000","last_became_peered":"2023-09-23T17:54:25.604386+0000","last_unstale":"2023-09-23T17:56:01.943401+0000","last_undegraded":"2023-09-23T17:56:01.943401+0000","last_fullsized":"2023-09-23T17:56:01.943401+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:56.869273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":46,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.763006+0000","last_change":"2023-09-23T17:53:21.039195+0000","last_active":"2023-09-23T17:56:08.763006+0000","last_peered":"2023-09-23T17:56:08.763006+0000","last_clean":"2023-09-23T17:56:08.763006+0000","last_became_active":"2023-09-23T17:53:21.038654+0000","last_became_peered":"2023-09-23T17:53:21.038654+0000","last_unstale":"2023-09-23T17:56:08.763006+0000","last_undegraded":"2023-09-23T17:56:08.763006+0000","last_fullsized":"2023-09-23T17:56:08.763006+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:34.466980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"118'2734","reported_seq":2746,"reported_epoch":118,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:19.010898+0000","last_change":"2023-09-23T17:55:58.884622+0000","last_active":"2023-09-23T17:56:19.010898+0000","last_peered":"2023-09-23T17:56:19.010898+0000","last_clean":"2023-09-23T17:55:52.789238+0000","last_became_active":"2023-09-23T17:55:58.867800+0000","last_became_peered":"2023-09-23T17:55:58.867800+0000","last_unstale":"2023-09-23T17:56:19.010898+0000","last_undegraded":"2023-09-23T17:55:58.878355+0000","last_fullsized":"2023-09-23T17:55:58.862585+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2734,"log_dups_size":0,"ondisk_log_size":2734,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:13.398530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179175424,"num_objects":2734,"num_object_clones":0,"num_object_copies":8202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2734,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2734,"num_write_kb":174976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":2734}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":79,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.758272+0000","last_change":"2023-09-23T17:54:57.041796+0000","last_active":"2023-09-23T17:56:08.758272+0000","last_peered":"2023-09-23T17:56:08.758272+0000","last_clean":"2023-09-23T17:56:08.758272+0000","last_became_active":"2023-09-23T17:54:57.041245+0000","last_became_peered":"2023-09-23T17:54:57.041245+0000","last_unstale":"2023-09-23T17:56:08.758272+0000","last_undegraded":"2023-09-23T17:56:08.758272+0000","last_fullsized":"2023-09-23T17:56:08.758272+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:26.340510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":301,"reported_epoch":116,"state":"active+undersized","last_fresh":"2023-09-23T17:56:01.943390+0000","last_change":"2023-09-23T17:55:58.866609+0000","last_active":"2023-09-23T17:56:01.943390+0000","last_peered":"2023-09-23T17:56:01.943390+0000","last_clean":"2023-09-23T17:55:57.820735+0000","last_became_active":"2023-09-23T17:55:58.866609+0000","last_became_peered":"2023-09-23T17:55:58.866609+0000","last_unstale":"2023-09-23T17:56:01.943390+0000","last_undegraded":"2023-09-23T17:56:01.943390+0000","last_fullsized":"2023-09-23T17:55:58.859636+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:37.495009+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":76,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.758524+0000","last_change":"2023-09-23T17:54:57.040450+0000","last_active":"2023-09-23T17:56:08.758524+0000","last_peered":"2023-09-23T17:56:08.758524+0000","last_clean":"2023-09-23T17:56:08.758524+0000","last_became_active":"2023-09-23T17:54:57.040134+0000","last_became_peered":"2023-09-23T17:54:57.040134+0000","last_unstale":"2023-09-23T17:56:08.758524+0000","last_undegraded":"2023-09-23T17:56:08.758524+0000","last_fullsized":"2023-09-23T17:56:08.758524+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:07.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"118'2683","reported_seq":2699,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:17.330287+0000","last_change":"2023-09-23T17:55:53.827866+0000","last_active":"2023-09-23T17:56:17.330287+0000","last_peered":"2023-09-23T17:56:17.330287+0000","last_clean":"2023-09-23T17:56:17.330287+0000","last_became_active":"2023-09-23T17:55:53.827616+0000","last_became_peered":"2023-09-23T17:55:53.827616+0000","last_unstale":"2023-09-23T17:56:17.330287+0000","last_undegraded":"2023-09-23T17:56:17.330287+0000","last_fullsized":"2023-09-23T17:56:17.330287+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":2683,"log_dups_size":0,"ondisk_log_size":2683,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:41.227976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":175833088,"num_objects":2683,"num_object_clones":0,"num_object_copies":8049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2683,"num_write_kb":171712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":301,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.762581+0000","last_change":"2023-09-23T17:55:58.866975+0000","last_active":"2023-09-23T17:56:08.762581+0000","last_peered":"2023-09-23T17:56:08.762581+0000","last_clean":"2023-09-23T17:46:59.535985+0000","last_became_active":"2023-09-23T17:55:58.866975+0000","last_became_peered":"2023-09-23T17:55:58.866975+0000","last_unstale":"2023-09-23T17:56:08.762581+0000","last_undegraded":"2023-09-23T17:56:08.762581+0000","last_fullsized":"2023-09-23T17:55:58.861027+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:48.232006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":64,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762359+0000","last_change":"2023-09-23T17:53:52.732070+0000","last_active":"2023-09-23T17:56:08.762359+0000","last_peered":"2023-09-23T17:56:08.762359+0000","last_clean":"2023-09-23T17:56:08.762359+0000","last_became_active":"2023-09-23T17:53:52.731739+0000","last_became_peered":"2023-09-23T17:53:52.731739+0000","last_unstale":"2023-09-23T17:56:08.762359+0000","last_undegraded":"2023-09-23T17:56:08.762359+0000","last_fullsized":"2023-09-23T17:56:08.762359+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:03.726754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.758992+0000","last_change":"2023-09-23T17:55:58.866331+0000","last_active":"2023-09-23T17:56:08.758992+0000","last_peered":"2023-09-23T17:56:08.758992+0000","last_clean":"2023-09-23T17:55:57.822015+0000","last_became_active":"2023-09-23T17:55:58.866331+0000","last_became_peered":"2023-09-23T17:55:58.866331+0000","last_unstale":"2023-09-23T17:56:08.758992+0000","last_undegraded":"2023-09-23T17:56:08.758992+0000","last_fullsized":"2023-09-23T17:55:58.862261+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:16.543215+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5],"avail_no_missing":["4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"118'3053","reported_seq":3070,"reported_epoch":118,"state":"active+clean","last_fresh":"2023-09-23T17:56:20.030261+0000","last_change":"2023-09-23T17:55:53.826362+0000","last_active":"2023-09-23T17:56:20.030261+0000","last_peered":"2023-09-23T17:56:20.030261+0000","last_clean":"2023-09-23T17:56:20.030261+0000","last_became_active":"2023-09-23T17:55:53.826049+0000","last_became_peered":"2023-09-23T17:55:53.826049+0000","last_unstale":"2023-09-23T17:56:20.030261+0000","last_undegraded":"2023-09-23T17:56:20.030261+0000","last_fullsized":"2023-09-23T17:56:20.030261+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":3053,"log_dups_size":0,"ondisk_log_size":3053,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:11.738622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200081408,"num_objects":3053,"num_object_clones":0,"num_object_copies":9159,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3053,"num_write_kb":195392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":49,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762933+0000","last_change":"2023-09-23T17:53:21.039717+0000","last_active":"2023-09-23T17:56:08.762933+0000","last_peered":"2023-09-23T17:56:08.762933+0000","last_clean":"2023-09-23T17:56:08.762933+0000","last_became_active":"2023-09-23T17:53:21.038793+0000","last_became_peered":"2023-09-23T17:53:21.038793+0000","last_unstale":"2023-09-23T17:56:08.762933+0000","last_undegraded":"2023-09-23T17:56:08.762933+0000","last_fullsized":"2023-09-23T17:56:08.762933+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:02.116246+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":101,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.765626+0000","last_change":"2023-09-23T17:54:57.034729+0000","last_active":"2023-09-23T17:56:08.765626+0000","last_peered":"2023-09-23T17:56:08.765626+0000","last_clean":"2023-09-23T17:56:08.765626+0000","last_became_active":"2023-09-23T17:54:57.034381+0000","last_became_peered":"2023-09-23T17:54:57.034381+0000","last_unstale":"2023-09-23T17:56:08.765626+0000","last_undegraded":"2023-09-23T17:56:08.765626+0000","last_fullsized":"2023-09-23T17:56:08.765626+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:19.369673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":80,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.760918+0000","last_change":"2023-09-23T17:53:52.732284+0000","last_active":"2023-09-23T17:56:08.760918+0000","last_peered":"2023-09-23T17:56:08.760918+0000","last_clean":"2023-09-23T17:56:08.760918+0000","last_became_active":"2023-09-23T17:53:52.731928+0000","last_became_peered":"2023-09-23T17:53:52.731928+0000","last_unstale":"2023-09-23T17:56:08.760918+0000","last_undegraded":"2023-09-23T17:56:08.760918+0000","last_fullsized":"2023-09-23T17:56:08.760918+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:35.936955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":95,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.759499+0000","last_change":"2023-09-23T17:54:57.041535+0000","last_active":"2023-09-23T17:56:08.759499+0000","last_peered":"2023-09-23T17:56:08.759499+0000","last_clean":"2023-09-23T17:56:08.759499+0000","last_became_active":"2023-09-23T17:54:57.040909+0000","last_became_peered":"2023-09-23T17:54:57.040909+0000","last_unstale":"2023-09-23T17:56:08.759499+0000","last_undegraded":"2023-09-23T17:56:08.759499+0000","last_fullsized":"2023-09-23T17:56:08.759499+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:57.504646+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":70,"reported_epoch":117,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:08.761000+0000","last_change":"2023-09-23T17:55:58.872695+0000","last_active":"2023-09-23T17:56:08.761000+0000","last_peered":"2023-09-23T17:56:08.761000+0000","last_clean":"2023-09-23T17:55:56.845914+0000","last_became_active":"2023-09-23T17:55:58.872695+0000","last_became_peered":"2023-09-23T17:55:58.872695+0000","last_unstale":"2023-09-23T17:56:08.761000+0000","last_undegraded":"2023-09-23T17:55:58.864028+0000","last_fullsized":"2023-09-23T17:55:58.863913+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:58.450245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":95,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.759250+0000","last_change":"2023-09-23T17:54:57.039895+0000","last_active":"2023-09-23T17:56:08.759250+0000","last_peered":"2023-09-23T17:56:08.759250+0000","last_clean":"2023-09-23T17:56:08.759250+0000","last_became_active":"2023-09-23T17:54:57.039581+0000","last_became_peered":"2023-09-23T17:54:57.039581+0000","last_unstale":"2023-09-23T17:56:08.759250+0000","last_undegraded":"2023-09-23T17:56:08.759250+0000","last_fullsized":"2023-09-23T17:56:08.759250+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:52.622668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":92,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.758178+0000","last_change":"2023-09-23T17:55:58.860645+0000","last_active":"2023-09-23T17:56:08.758178+0000","last_peered":"2023-09-23T17:56:08.758178+0000","last_clean":"2023-09-23T17:55:56.841110+0000","last_became_active":"2023-09-23T17:55:58.860645+0000","last_became_peered":"2023-09-23T17:55:58.860645+0000","last_unstale":"2023-09-23T17:56:08.758178+0000","last_undegraded":"2023-09-23T17:56:08.758178+0000","last_fullsized":"2023-09-23T17:55:58.855994+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:29.000041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":101,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.763226+0000","last_change":"2023-09-23T17:54:57.034722+0000","last_active":"2023-09-23T17:56:08.763226+0000","last_peered":"2023-09-23T17:56:08.763226+0000","last_clean":"2023-09-23T17:56:08.763226+0000","last_became_active":"2023-09-23T17:54:57.034289+0000","last_became_peered":"2023-09-23T17:54:57.034289+0000","last_unstale":"2023-09-23T17:56:08.763226+0000","last_undegraded":"2023-09-23T17:56:08.763226+0000","last_fullsized":"2023-09-23T17:56:08.763226+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:03.041518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":64,"reported_epoch":117,"state":"active+clean","last_fresh":"2023-09-23T17:56:08.762575+0000","last_change":"2023-09-23T17:54:25.601065+0000","last_active":"2023-09-23T17:56:08.762575+0000","last_peered":"2023-09-23T17:56:08.762575+0000","last_clean":"2023-09-23T17:56:08.762575+0000","last_became_active":"2023-09-23T17:54:25.600727+0000","last_became_peered":"2023-09-23T17:54:25.600727+0000","last_unstale":"2023-09-23T17:56:08.762575+0000","last_undegraded":"2023-09-23T17:56:08.762575+0000","last_fullsized":"2023-09-23T17:56:08.762575+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:20.919498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":85,"reported_epoch":117,"state":"active+undersized","last_fresh":"2023-09-23T17:56:08.759180+0000","last_change":"2023-09-23T17:55:58.866223+0000","last_active":"2023-09-23T17:56:08.759180+0000","last_peered":"2023-09-23T17:56:08.759180+0000","last_clean":"2023-09-23T17:55:57.821766+0000","last_became_active":"2023-09-23T17:55:58.866223+0000","last_became_peered":"2023-09-23T17:55:58.866223+0000","last_unstale":"2023-09-23T17:56:08.759180+0000","last_undegraded":"2023-09-23T17:56:08.759180+0000","last_fullsized":"2023-09-23T17:55:58.861165+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:06.895579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":2870411264,"num_objects":43799,"num_object_clones":0,"num_object_copies":131397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10797,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":43799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":43799,"num_write_kb":2803136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7809335296,"data_stored":7809335296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":43799,"ondisk_log_size":43799,"up":44,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":84,"acting":84,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":85,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":2,"acting":2,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":21,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266830,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":863592,"kb_used_data":841696,"kb_used_omap":0,"kb_used_meta":21888,"kb_avail":92881560,"statfs":{"total":95995035648,"available":95110717440,"internally_reserved":0,"allocated":861896704,"data_stored":861373121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":22413312},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:23 2023","interfaces":[{"interface":"back","average":{"1min":0.57299999999999995,"5min":0.55500000000000005,"15min":0.55200000000000005},"min":{"1min":0.34999999999999998,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":0.81200000000000006,"5min":0.81699999999999995,"15min":0.81699999999999995},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.57199999999999995,"15min":0.56899999999999995},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.752}]},{"osd":1,"last update":"Sat Sep 23 17:55:55 2023","interfaces":[{"interface":"back","average":{"1min":0.52600000000000002,"5min":0.55100000000000005,"15min":0.55500000000000005},"min":{"1min":0.38300000000000001,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.72199999999999998,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.47999999999999998,"5min":0.54000000000000004,"15min":0.55000000000000004},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.626,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.33800000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.82799999999999996,"5min":0.82799999999999996,"15min":0.82799999999999996},"last":1.5229999999999999},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.43099999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":1.4279999999999999},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":1.4159999999999999}]},{"osd":4,"last update":"Sat Sep 23 17:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.504,"15min":0.45400000000000001},"min":{"1min":0.254,"5min":0.252,"15min":0.252},"max":{"1min":0.80100000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.245},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.52900000000000003,"15min":0.48199999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.73599999999999999,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.29499999999999998}]},{"osd":5,"last update":"Sat Sep 23 17:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.52400000000000002,"15min":0.49199999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.68400000000000005,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.311},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.60999999999999999,"15min":0.53000000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29099999999999998},"max":{"1min":1.706,"5min":1.706,"15min":1.706},"last":0.35599999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":1.002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.435}]}]},{"osd":6,"up_from":45,"seq":193273528462,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12464,"kb_used_data":1704,"kb_used_omap":0,"kb_used_meta":10752,"kb_avail":93732688,"statfs":{"total":95995035648,"available":95982272512,"internally_reserved":0,"allocated":1744896,"data_stored":1230110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11010048},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.81100000000000005,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.78300000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.38}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.62},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":1.2250000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":6.9589999999999996}]},{"osd":3,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.38800000000000001,"5min":0.36799999999999999,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":0.80700000000000005,"15min":4.5599999999999996},"last":0.253},{"interface":"front","average":{"1min":0.625,"5min":0.58499999999999996,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34999999999999998,"15min":0.30399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":1.6850000000000001},"last":2.9369999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:55:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.48699999999999999,"15min":0.46000000000000002},"min":{"1min":0.377,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.82899999999999996},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.53000000000000003,"15min":0.496},"min":{"1min":0.34499999999999997,"5min":0.28899999999999998,"15min":0.27800000000000002},"max":{"1min":0.72099999999999997,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.33800000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:55:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.54300000000000004,"15min":0.503},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":0.82099999999999995,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56699999999999995,"15min":0.52700000000000002},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.33200000000000002},"max":{"1min":0.69099999999999995,"5min":0.79700000000000004,"15min":0.84599999999999997},"last":0.42199999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:55:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.58999999999999997,"15min":0.53700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":0.83299999999999996,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.57499999999999996,"15min":0.51900000000000002},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":0.81299999999999994,"5min":0.82299999999999995,"15min":1.591},"last":0.314}]}]},{"osd":1,"up_from":98,"seq":420906795039,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":836612,"kb_used_data":800828,"kb_used_omap":7,"kb_used_meta":35768,"kb_avail":92908540,"statfs":{"total":95995035648,"available":95138344960,"internally_reserved":0,"allocated":820047872,"data_stored":819492825,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":36627133},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,3,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.35399999999999998,"5min":0.38100000000000001,"15min":0.38500000000000001},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":5.3620000000000001},{"interface":"front","average":{"1min":0.38800000000000001,"5min":0.39500000000000002,"15min":0.39600000000000002},"min":{"1min":0.245,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.97099999999999997,"5min":0.97099999999999997,"15min":0.97099999999999997},"last":3.2320000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":1.976,"5min":1.976,"15min":1.976},"last":5.2859999999999996},{"interface":"front","average":{"1min":0.44,"5min":0.44,"15min":0.44},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":1.226,"5min":1.226,"15min":1.226},"last":5.4299999999999997}]},{"osd":3,"last update":"Sat Sep 23 17:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":5.2249999999999996},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.5049999999999999,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":5.3959999999999999}]},{"osd":4,"last update":"Sat Sep 23 17:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.66700000000000004,"5min":0.67900000000000005,"15min":0.68100000000000005},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.113,"5min":1.113,"15min":1.113},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.67200000000000004,"15min":0.67600000000000005},"min":{"1min":0.40600000000000003,"5min":0.40600000000000003,"15min":0.40600000000000003},"max":{"1min":1.1899999999999999,"5min":1.1899999999999999,"15min":1.1899999999999999},"last":0.38}]},{"osd":5,"last update":"Sat Sep 23 17:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.65000000000000002,"5min":0.70499999999999996,"15min":0.71399999999999997},"min":{"1min":0.46600000000000003,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":1.0289999999999999,"5min":1.0289999999999999,"15min":1.0289999999999999},"last":3.1629999999999998},{"interface":"front","average":{"1min":0.65800000000000003,"5min":0.69699999999999995,"15min":0.70299999999999996},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":1.1719999999999999,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.39600000000000002}]},{"osd":6,"last update":"Sat Sep 23 17:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.9330000000000001,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.69199999999999995,"15min":0.69899999999999995},"min":{"1min":0.47699999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.141,"5min":1.141,"15min":1.141},"last":0.77700000000000002}]},{"osd":7,"last update":"Sat Sep 23 17:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.66000000000000003,"15min":0.66600000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":1.0800000000000001,"5min":1.0800000000000001,"15min":1.0800000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.69099999999999995,"15min":0.69999999999999996},"min":{"1min":0.45100000000000001,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.94899999999999995,"5min":1.0269999999999999,"15min":1.0269999999999999},"last":0.50800000000000001}]}]},{"osd":0,"up_from":93,"seq":399431958566,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1452632,"kb_used_data":1394704,"kb_used_omap":7,"kb_used_meta":57912,"kb_avail":92292520,"statfs":{"total":95995035648,"available":94507540480,"internally_reserved":0,"allocated":1428176896,"data_stored":1427610030,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":59302628},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:55:51 2023","interfaces":[{"interface":"back","average":{"1min":0.35099999999999998,"5min":0.36599999999999999,"15min":0.36799999999999999},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.52300000000000002,"5min":0.70399999999999996,"15min":0.70399999999999996},"last":1.573},{"interface":"front","average":{"1min":0.377,"5min":0.41599999999999998,"15min":0.42199999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.48699999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.436}]},{"osd":2,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.36099999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"last":1.6599999999999999},{"interface":"front","average":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":0.53100000000000003,"5min":0.53100000000000003,"15min":0.53100000000000003},"last":0.378}]},{"osd":3,"last update":"Sat Sep 23 17:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.5910000000000002,"5min":3.5910000000000002,"15min":3.5910000000000002},"last":1.7809999999999999},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":3.879}]},{"osd":4,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.627,"15min":0.63300000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":0.82199999999999995,"5min":0.872,"15min":0.872},"last":2.3719999999999999},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.61699999999999999,"15min":0.621},"min":{"1min":0.44700000000000001,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.77700000000000002,"5min":0.85099999999999998,"15min":0.85099999999999998},"last":1.6910000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.59199999999999997,"5min":0.624,"15min":0.63},"min":{"1min":0.41799999999999998,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.73599999999999999,"5min":0.88500000000000001,"15min":0.88500000000000001},"last":1.625},{"interface":"front","average":{"1min":0.61799999999999999,"5min":0.63900000000000001,"15min":0.64300000000000002},"min":{"1min":0.46600000000000003,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.86899999999999999,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.34599999999999997}]},{"osd":6,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.42899999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65200000000000002},"min":{"1min":0.52700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.81100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":0.65900000000000003,"15min":0.66300000000000003},"min":{"1min":0.51600000000000001,"5min":0.435,"15min":0.435},"max":{"1min":0.84299999999999997,"5min":0.85599999999999998,"15min":0.85599999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.64300000000000002,"5min":0.66600000000000004,"15min":0.67000000000000004},"min":{"1min":0.437,"5min":0.437,"15min":0.437},"max":{"1min":0.86599999999999999,"5min":0.95699999999999996,"15min":0.95699999999999996},"last":2.4060000000000001}]}]},{"osd":2,"up_from":103,"seq":442381631513,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1098056,"kb_used_data":1054144,"kb_used_omap":11,"kb_used_meta":43892,"kb_avail":92647096,"statfs":{"total":95995035648,"available":94870626304,"internally_reserved":0,"allocated":1079443456,"data_stored":1078891272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":44945783},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"last":0.17399999999999999},{"interface":"front","average":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.70399999999999996}]},{"osd":1,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":0.52200000000000002,"5min":0.52200000000000002,"15min":0.52200000000000002},"last":0.33900000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":0.96499999999999997},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":0.35299999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.60299999999999998,"15min":0.60299999999999998},"min":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":3.2250000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.65900000000000003}]},{"osd":5,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.624,"5min":0.624,"15min":0.624},"min":{"1min":0.46700000000000003,"5min":0.46700000000000003,"15min":0.46700000000000003},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.375}]},{"osd":6,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":0.83499999999999996,"5min":0.83499999999999996,"15min":0.83499999999999996},"last":1.3220000000000001},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.68600000000000005,"15min":0.68600000000000005},"min":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"max":{"1min":0.91600000000000004,"5min":0.91600000000000004,"15min":0.91600000000000004},"last":0.62}]}]},{"osd":3,"up_from":108,"seq":463856467987,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":601476,"kb_used_data":561788,"kb_used_omap":3,"kb_used_meta":39676,"kb_avail":93143676,"statfs":{"total":95995035648,"available":95379124224,"internally_reserved":0,"allocated":575270912,"data_stored":574722188,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":40628968},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":1.579},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":1.9099999999999999}]},{"osd":1,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.95899999999999996},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.92600000000000005},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":1.97}]},{"osd":4,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":0.437}]},{"osd":5,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":1.083},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.98899999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":1.0229999999999999},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":1.0349999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019031,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1349412,"kb_used_data":1311324,"kb_used_omap":0,"kb_used_meta":38080,"kb_avail":92395740,"statfs":{"total":95995035648,"available":94613237760,"internally_reserved":0,"allocated":1342795776,"data_stored":1342275998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":38993920},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52400000000000002,"5min":0.51800000000000002,"15min":0.51700000000000002},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":0.75600000000000001,"5min":0.77600000000000002,"15min":0.77600000000000002},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.53800000000000003,"15min":0.52900000000000003},"min":{"1min":0.42699999999999999,"5min":0.376,"15min":0.376},"max":{"1min":1.4319999999999999,"5min":1.4319999999999999,"15min":1.4319999999999999},"last":1.2589999999999999}]},{"osd":1,"last update":"Sat Sep 23 17:55:53 2023","interfaces":[{"interface":"back","average":{"1min":0.58699999999999997,"5min":0.54200000000000004,"15min":0.53400000000000003},"min":{"1min":0.39400000000000002,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":1.5189999999999999,"5min":1.5189999999999999,"15min":1.5189999999999999},"last":0.25},{"interface":"front","average":{"1min":0.627,"5min":0.57099999999999995,"15min":0.56200000000000006},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":1.371,"5min":1.371,"15min":1.371},"last":1.7989999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":1.331,"5min":1.331,"15min":1.331},"last":0.443},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.42099999999999999,"5min":0.42099999999999999,"15min":0.42099999999999999},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.74399999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.3879999999999999,"5min":1.3879999999999999,"15min":1.3879999999999999},"last":0.432},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.65700000000000003,"15min":0.65700000000000003},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":1.5629999999999999,"5min":1.5629999999999999,"15min":1.5629999999999999},"last":1.371}]},{"osd":5,"last update":"Sat Sep 23 17:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.48599999999999999,"5min":0.48399999999999999,"15min":0.438},"min":{"1min":0.379,"5min":0.29999999999999999,"15min":0.28299999999999997},"max":{"1min":0.621,"5min":0.753,"15min":1.141},"last":0.28399999999999997},{"interface":"front","average":{"1min":0.53200000000000003,"5min":0.51700000000000002,"15min":0.44700000000000001},"min":{"1min":0.41499999999999998,"5min":0.32500000000000001,"15min":0.27500000000000002},"max":{"1min":0.749,"5min":0.76300000000000001,"15min":0.76300000000000001},"last":0.40899999999999997}]},{"osd":6,"last update":"Sat Sep 23 17:55:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54200000000000004,"15min":0.50800000000000001},"min":{"1min":0.45400000000000001,"5min":0.33200000000000002,"15min":0.28899999999999998},"max":{"1min":0.64700000000000002,"5min":0.755,"15min":1.276},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.59299999999999997,"15min":0.54700000000000004},"min":{"1min":0.52500000000000002,"5min":0.29899999999999999,"15min":0.28999999999999998},"max":{"1min":0.79400000000000004,"5min":0.80900000000000005,"15min":1.1180000000000001},"last":0.505}]},{"osd":7,"last update":"Sat Sep 23 17:56:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.56499999999999995,"15min":0.54700000000000004},"min":{"1min":0.378,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":1.1839999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.56499999999999995,"15min":0.53300000000000003},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.65200000000000002,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.45200000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757397,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1510548,"kb_used_data":1464780,"kb_used_omap":0,"kb_used_meta":45760,"kb_avail":92234604,"statfs":{"total":95995035648,"available":94448234496,"internally_reserved":0,"allocated":1499934720,"data_stored":1499426766,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46858240},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54500000000000004,"5min":0.57199999999999995,"15min":0.57599999999999996},"min":{"1min":0.39800000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":0.77100000000000002,"5min":0.98099999999999998,"15min":0.98099999999999998},"last":0.36299999999999999},{"interface":"front","average":{"1min":0.57799999999999996,"5min":0.58399999999999996,"15min":0.58499999999999996},"min":{"1min":0.40400000000000003,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.74199999999999999,"5min":0.79000000000000004,"15min":0.79000000000000004},"last":1.8320000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.68000000000000005,"15min":0.70399999999999996},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.79000000000000004,"5min":4.4279999999999999,"15min":4.4279999999999999},"last":0.41299999999999998},{"interface":"front","average":{"1min":0.52500000000000002,"5min":0.55800000000000005,"15min":0.56399999999999995},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.69299999999999995,"5min":1.45,"15min":1.45},"last":1.863}]},{"osd":2,"last update":"Sat Sep 23 17:55:24 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.75800000000000001,"5min":0.75800000000000001,"15min":0.75800000000000001},"last":2.9340000000000002},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.56699999999999995,"15min":0.56699999999999995},"min":{"1min":0.38500000000000001,"5min":0.38500000000000001,"15min":0.38500000000000001},"max":{"1min":0.69199999999999995,"5min":0.69199999999999995,"15min":0.69199999999999995},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:55:56 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.60199999999999998,"15min":0.60199999999999998},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.79900000000000004,"5min":0.79900000000000004,"15min":0.79900000000000004},"last":2.3159999999999998},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.60599999999999998,"15min":0.60599999999999998},"min":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"max":{"1min":0.80900000000000005,"5min":0.80900000000000005,"15min":0.80900000000000005},"last":1.899}]},{"osd":4,"last update":"Sat Sep 23 17:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51600000000000001,"5min":0.48899999999999999,"15min":0.44800000000000001},"min":{"1min":0.34300000000000003,"5min":0.25,"15min":0.25},"max":{"1min":0.70799999999999996,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":1.802},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.54400000000000004,"15min":0.51400000000000001},"min":{"1min":0.435,"5min":0.33100000000000002,"15min":0.28899999999999998},"max":{"1min":0.81399999999999995,"5min":0.83199999999999996,"15min":1.738},"last":2.9780000000000002}]},{"osd":6,"last update":"Sat Sep 23 17:55:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54400000000000004,"15min":0.53500000000000003},"min":{"1min":0.38800000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.66100000000000003,"5min":0.85499999999999998,"15min":1.778},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.45800000000000002,"5min":0.375,"15min":0.33200000000000002},"max":{"1min":0.72799999999999998,"5min":0.90200000000000002,"15min":1.7629999999999999},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.57699999999999996,"15min":0.53100000000000003},"min":{"1min":0.41399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.276,"5min":1.276,"15min":1.276},"last":4.1719999999999997},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.59199999999999997,"15min":0.55600000000000005},"min":{"1min":0.46300000000000002,"5min":0.313,"15min":0.313},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":1.9219999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1473839104,"data_stored":1473839104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":819265536,"data_stored":819265536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1099759616,"data_stored":1099759616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":585629696,"data_stored":585629696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1392771072,"data_stored":1392771072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1562640384,"data_stored":1562640384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":875429888,"data_stored":875429888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:56:22.022 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_0 pgp_num 16 2023-09-23T17:56:22.104 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:22.176 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:56:22.176 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:56:22.203 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:22.280 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:22.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:22.293 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:22.324 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:56:22.346 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 24 16 49971 49955 130.007 91.3125 0.00785096 0.00768544 2023-09-23T17:56:22.394 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:22.528 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:22.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:22 smithi055 ceph-mon[179668]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:56:22.817 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:22.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:22.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:22.928 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:22.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:22 smithi055 ceph-mon[184174]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:56:23.012 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:22 smithi099 ceph-mon[167076]: from='client.54586 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:56:23.290 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:23.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:23.305 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:23.346 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 25 16 51271 51255 128.057 81.25 0.016102 0.00780397 2023-09-23T17:56:23.361 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T17:56:23.406 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:23.492 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:23.610 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:23.643 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:23.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:23.748 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:23.786 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:23.836 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:23.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:23.842 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:23.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:23 smithi055 ceph-mon[184174]: pgmap v250: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s; 11687/141378 objects degraded (8.266%) 2023-09-23T17:56:23.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 11687/141378 objects degraded (8.266%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:23.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:23 smithi055 ceph-mon[179668]: pgmap v250: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s; 11687/141378 objects degraded (8.266%) 2023-09-23T17:56:23.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 11687/141378 objects degraded (8.266%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:23.943 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:24.009 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:24.035 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:24.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:23 smithi099 ceph-mon[167076]: pgmap v250: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 2.9 GiB data, 8.2 GiB used, 707 GiB / 715 GiB avail; 109 MiB/s wr, 1.74k op/s; 11687/141378 objects degraded (8.266%) 2023-09-23T17:56:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 11687/141378 objects degraded (8.266%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:24.256 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:24.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:24.261 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:24.348 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 26 16 52542 52526 126.179 79.4375 0.0157592 0.0079195 2023-09-23T17:56:24.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:24.402 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:24.446 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:24.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:24.451 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:24.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:24.593 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:24.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:24.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:24.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:24.744 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:24.823 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T17:56:24.871 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1350957330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:56:24.871 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:24 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:56:24.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1350957330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:56:24.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:24 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:56:24.876 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:25.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1350957330' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:56:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:24 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:56:25.329 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:25.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:25.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:25.350 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 27 16 53967 53951 124.796 89.0625 0.0134685 0.00800853 2023-09-23T17:56:25.435 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:25.441 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:25.507 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:25.653 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:25.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:25.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:25.767 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:25.798 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[179668]: pgmap v251: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 12518/151716 objects degraded (8.251%) 2023-09-23T17:56:25.798 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:25.798 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:56:25.798 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[179668]: osdmap e119: 8 total, 7 up, 8 in 2023-09-23T17:56:25.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[179668]: failed to encode map e119 with expected crc 2023-09-23T17:56:25.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[179668]: failed to encode map e119 with expected crc 2023-09-23T17:56:25.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[179668]: failed to encode map e119 with expected crc 2023-09-23T17:56:25.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:25.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:25.840 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:26.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[184174]: pgmap v251: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 12518/151716 objects degraded (8.251%) 2023-09-23T17:56:26.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:26.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:56:26.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[184174]: osdmap e119: 8 total, 7 up, 8 in 2023-09-23T17:56:26.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[184174]: failed to encode map e119 with expected crc 2023-09-23T17:56:26.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[184174]: failed to encode map e119 with expected crc 2023-09-23T17:56:26.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[184174]: failed to encode map e119 with expected crc 2023-09-23T17:56:26.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:26.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:25 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:26.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:25 smithi099 ceph-mon[167076]: pgmap v251: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.1 GiB data, 8.6 GiB used, 707 GiB / 715 GiB avail; 99 MiB/s wr, 1.59k op/s; 12518/151716 objects degraded (8.251%) 2023-09-23T17:56:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:25 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:25 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:56:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:25 smithi099 ceph-mon[167076]: osdmap e119: 8 total, 7 up, 8 in 2023-09-23T17:56:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:25 smithi099 ceph-mon[167076]: failed to encode map e119 with expected crc 2023-09-23T17:56:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:25 smithi099 ceph-mon[167076]: failed to encode map e119 with expected crc 2023-09-23T17:56:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:25 smithi099 ceph-mon[167076]: failed to encode map e119 with expected crc 2023-09-23T17:56:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:25 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:25 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:26.081 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:26.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:26.087 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:26.187 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:26.226 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:26.261 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:26.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:26.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:26.350 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 28 16 55262 55246 123.23 80.9375 0.0134684 0.00811056 2023-09-23T17:56:26.365 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:26.406 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:56:26.441 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:56:26.453 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:26.490 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:56:26.529 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:26.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:26.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:26.541 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:56:26.596 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:56:26.636 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:56:26.637 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:26.686 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:56:26.687 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:56:26.687 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:56:26.687 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:56:26.687 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T17:56:26.687 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:56:26.687 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:56:26.687 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:56:26.687 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T17:56:26.713 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:26.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3602335521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:26.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:26 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:56:26.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:26 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:56:26.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:26 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:26.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3602335521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:26.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:26 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:56:26.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:26 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:56:26.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:26 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:26.967 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:26.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:26.974 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:27.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3602335521' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:26 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:56:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:26 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:56:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:26 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:27.115 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:27.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:27.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:27.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:27.273 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:27.349 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:27.350 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 29 16 56616 56600 121.899 84.625 0.0171955 0.00819772 2023-09-23T17:56:27.403 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:56:27.404 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:56:27.489 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:27.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:27.495 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:27.596 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:27.641 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:27.700 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:27.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:27.705 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:27.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T17:56:27.766 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:56:27.766 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T17:56:27.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T17:56:27.766 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:56:27.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.766 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:38870 5.4 e117/110 5:2427fa21:::benchmark_data_smithi055_229117_object38869:head v 117'2437, mlcod=117'2434)", 2023-09-23T17:56:27.766 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.242596+0000", 2023-09-23T17:56:27.766 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 12.515551199000001, 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.111315684, 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242596+0000", 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.768 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242596+0000", 2023-09-23T17:56:27.768 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242595+0000", 2023-09-23T17:56:27.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:27.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242648+0000", 2023-09-23T17:56:27.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3016000000000002e-05 2023-09-23T17:56:27.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.770 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242658+0000", 2023-09-23T17:56:27.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7380000000000006e-06 2023-09-23T17:56:27.770 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.770 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.770 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242665+0000", 2023-09-23T17:56:27.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9e-06 2023-09-23T17:56:27.770 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242704+0000", 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8578999999999998e-05 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242723+0000", 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9533000000000001e-05 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244894+0000", 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021712290000000002 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353912+0000", 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.109017453 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.772 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:38873 5.6 5:675d6566:::benchmark_data_smithi055_229117_object38872:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.244482+0000", 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 12.513665751, 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11283095999999999, 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 38873 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244482+0000", 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244482+0000", 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244480+0000", 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244503+0000", 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3040999999999998e-05 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244517+0000", 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.347e-05 2023-09-23T17:56:27.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244525+0000", 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4579999999999993e-06 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244622+0000", 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7471000000000002e-05 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244781+0000", 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015900300000000001 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353163+0000", 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10838141900000001 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355058+0000", 2023-09-23T17:56:27.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001894869 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.356618+0000", 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015602529999999999 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357221+0000", 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00060303500000000003 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357272+0000", 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0523999999999999e-05 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357313+0000", 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1103999999999998e-05 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:38875 5.d 5:b97fc75e:::benchmark_data_smithi055_229117_object38874:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.244540+0000", 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 12.513606994, 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11553848899999999, 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.780 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 38875 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244540+0000", 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244540+0000", 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244540+0000", 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244562+0000", 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2147e-05 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244570+0000", 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4889999999999998e-06 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244576+0000", 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.173e-06 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.354972+0000", 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.110395935 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355110+0000", 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013866700000000001 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355252+0000", 2023-09-23T17:56:27.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014123 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.356823+0000", 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015712670000000001 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357281+0000", 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00045775600000000003 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.360000+0000", 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027189369999999998 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.360042+0000", 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2043999999999999e-05 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.360079+0000", 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7320000000000002e-05 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:38878 5.b 5:d611086b:::benchmark_data_smithi055_229117_object38877:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.244787+0000", 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 12.513360526, 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11642896699999999, 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.787 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 38878 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244787+0000", 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244787+0000", 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244786+0000", 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244814+0000", 2023-09-23T17:56:27.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8456999999999999e-05 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244828+0000", 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3489000000000001e-05 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244836+0000", 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9119999999999998e-06 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353386+0000", 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.108550427 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353559+0000", 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017319199999999999 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:27.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353725+0000", 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016569499999999999 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.360943+0000", 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.007218538 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.360994+0000", 2023-09-23T17:56:27.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0500000000000001e-05 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.361097+0000", 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000103058 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.361162+0000", 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5371999999999997e-05 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.361216+0000", 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3566000000000002e-05 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:38881 5.b 5:db8f9ecb:::benchmark_data_smithi055_229117_object38880:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.244850+0000", 2023-09-23T17:56:27.794 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 12.513297309, 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.118114012, 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 38881 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244850+0000", 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244850+0000", 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244850+0000", 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244870+0000", 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0234e-05 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355070+0000", 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.110200408 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355081+0000", 2023-09-23T17:56:27.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0318e-05 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355200+0000", 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119231 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355319+0000", 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119306 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355457+0000", 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013762799999999999 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.360983+0000", 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.005526497 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.360999+0000", 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5512e-05 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.362868+0000", 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001869154 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.362923+0000", 2023-09-23T17:56:27.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4800999999999999e-05 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.362964+0000", 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1433000000000001e-05 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.801 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:38890 5.4 e117/110 5:2fb16626:::benchmark_data_smithi055_229117_object38889:head v 117'2438, mlcod=117'2435)", 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.250951+0000", 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 12.507196782999999, 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.106575483, 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.250951+0000", 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.250951+0000", 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.250949+0000", 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.251004+0000", 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4568999999999999e-05 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.351748+0000", 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.100744036 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.351766+0000", 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7784000000000001e-05 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353945+0000", 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021791559999999998 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353964+0000", 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8994000000000001e-05 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357486+0000", 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035219499999999998 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357526+0000", 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0207999999999999e-05 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.806 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:39713 5.b 5:d4d1b561:::benchmark_data_smithi055_229117_object39712:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.775750+0000", 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.982397653, 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.071778628999999997, 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 39713 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775750+0000", 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775750+0000", 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775748+0000", 2023-09-23T17:56:27.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775776+0000", 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8136e-05 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775793+0000", 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6286999999999998e-05 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775801+0000", 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8459999999999997e-06 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775861+0000", 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9947000000000001e-05 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.776015+0000", 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015319800000000001 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.776169+0000", 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000154468 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.778225+0000", 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020556210000000001 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.785541+0000", 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0073166790000000004 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.847426+0000", 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.061884816000000002 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.847484+0000", 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8168000000000003e-05 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.847528+0000", 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4131000000000001e-05 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:39714 5.8 5:1dbd9ca0:::benchmark_data_smithi055_229117_object39713:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.775818+0000", 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.982329557, 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.076594704999999999, 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 39714 2023-09-23T17:56:27.814 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775818+0000", 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775818+0000", 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775817+0000", 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775841+0000", 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3947999999999998e-05 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775849+0000", 2023-09-23T17:56:27.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8210000000000002e-06 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775854+0000", 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2070000000000003e-06 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775916+0000", 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1563000000000007e-05 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.776037+0000", 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012085400000000001 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.776174+0000", 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137288 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.782058+0000", 2023-09-23T17:56:27.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.005883788 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.785747+0000", 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036893989999999999 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.852302+0000", 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.066554418000000004 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.852373+0000", 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1604000000000001e-05 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.852413+0000", 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9273999999999997e-05 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.820 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:39719 5.8 5:119b5100:::benchmark_data_smithi055_229117_object39718:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.775869+0000", 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.982278764, 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.076708269999999995, 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 39719 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775869+0000", 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775869+0000", 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775868+0000", 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775890+0000", 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1336999999999999e-05 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775896+0000", 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7340000000000002e-06 2023-09-23T17:56:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.775901+0000", 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0000000000000004e-06 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.778912+0000", 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030102449999999999 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.779046+0000", 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013400700000000001 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.779229+0000", 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018351500000000001 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.782255+0000", 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030259179999999998 2023-09-23T17:56:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.785753+0000", 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003497474 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.852493+0000", 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.066740280999999999 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.852542+0000", 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9178000000000002e-05 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.852577+0000", 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4941999999999999e-05 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:39733 5.b 5:db7ca648:::benchmark_data_smithi055_229117_object39732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.786044+0000", 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.972103502, 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.066404550000000007, 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 39733 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.786044+0000", 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.786044+0000", 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.786043+0000", 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:56:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.786062+0000", 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8561000000000001e-05 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.786068+0000", 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6499999999999999e-06 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.786072+0000", 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8410000000000002e-06 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.786114+0000", 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1396999999999997e-05 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.786252+0000", 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013852 2023-09-23T17:56:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.786383+0000", 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013098499999999999 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.787855+0000", 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014716519999999999 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.787888+0000", 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3189999999999999e-05 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.852157+0000", 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.064269446999999993 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.852229+0000", 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1217000000000006e-05 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.852448+0000", 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000219848 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:40113 5.d 5:bb04235d:::benchmark_data_smithi055_229117_object40112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:16.029211+0000", 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.728936086000001, 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.049744542000000003, 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 40113 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029211+0000", 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029211+0000", 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029211+0000", 2023-09-23T17:56:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029234+0000", 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3507999999999999e-05 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029243+0000", 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2069999999999995e-06 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029249+0000", 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.246e-06 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.031480+0000", 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002230779 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.031591+0000", 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111356 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.031714+0000", 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012306900000000001 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033647+0000", 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019328959999999999 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.077337+0000", 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043689617 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.078869+0000", 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001532191 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.078918+0000", 2023-09-23T17:56:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9131000000000001e-05 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.078956+0000", 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8040999999999997e-05 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:40117 5.6 5:65736939:::benchmark_data_smithi055_229117_object40116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:16.029544+0000", 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.728603776, 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047169052000000003, 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 40117 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029544+0000", 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029544+0000", 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029543+0000", 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029561+0000", 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7969999999999999e-05 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029572+0000", 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0678e-05 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029579+0000", 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.092e-06 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029619+0000", 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0200000000000001e-05 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029757+0000", 2023-09-23T17:56:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137806 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.029896+0000", 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000138992 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.032061+0000", 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021657600000000001 2023-09-23T17:56:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.035087+0000", 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030258640000000001 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076618+0000", 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.041530213000000003 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076675+0000", 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7101000000000001e-05 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076713+0000", 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8061e-05 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:40121 5.a 5:585276d2:::benchmark_data_smithi055_229117_object40120:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:16.030280+0000", 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.727867371, 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.048822412000000003, 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 40121 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.030280+0000", 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.030280+0000", 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.030280+0000", 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.030298+0000", 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8736000000000002e-05 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.030305+0000", 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9070000000000001e-06 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.030311+0000", 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6980000000000003e-06 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.031884+0000", 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015736000000000001 2023-09-23T17:56:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.032010+0000", 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012518699999999999 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.032161+0000", 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000151373 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.035279+0000", 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031184509999999999 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076473+0000", 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.041193317 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.079014+0000", 2023-09-23T17:56:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025416409999999999 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.079065+0000", 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0617e-05 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.079102+0000", 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7409e-05 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:40124 5.a 5:5ca8f945:::benchmark_data_smithi055_229117_object40123:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:16.033579+0000", 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.724568082999999, 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.045648241999999999, 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 40124 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033579+0000", 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033579+0000", 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033578+0000", 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033608+0000", 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0162999999999999e-05 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033619+0000", 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1285e-05 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033626+0000", 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7499999999999997e-06 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033680+0000", 2023-09-23T17:56:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4317999999999997e-05 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033797+0000", 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000116662 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033923+0000", 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012625399999999999 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076481+0000", 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.042557827999999999 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076516+0000", 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4365e-05 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.079140+0000", 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002624323 2023-09-23T17:56:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.079190+0000", 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0555e-05 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.079228+0000", 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7147999999999997e-05 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:40127 5.a 5:5b640b2d:::benchmark_data_smithi055_229117_object40126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:16.033639+0000", 2023-09-23T17:56:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.724508263000001, 2023-09-23T17:56:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.045731695000000003, 2023-09-23T17:56:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 40127 2023-09-23T17:56:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033639+0000", 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033639+0000", 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033639+0000", 2023-09-23T17:56:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033660+0000", 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1498999999999999e-05 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033668+0000", 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4249999999999998e-06 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.033675+0000", 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5139999999999998e-06 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.035432+0000", 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0017569409999999999 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.035523+0000", 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1378999999999994e-05 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.035620+0000", 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7040999999999995e-05 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076486+0000", 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040865831999999998 2023-09-23T17:56:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.077373+0000", 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00088648199999999996 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.079291+0000", 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001917827 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.079337+0000", 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6649e-05 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.079371+0000", 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3717000000000003e-05 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:40128 5.8 5:160fe908:::benchmark_data_smithi055_229117_object40127:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:16.034123+0000", 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.724024692, 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.044202318999999997, 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 40128 2023-09-23T17:56:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.034123+0000", 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.034123+0000", 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.034122+0000", 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.034140+0000", 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7638e-05 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.034148+0000", 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6999999999999997e-06 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.034154+0000", 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1059999999999999e-06 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.034204+0000", 2023-09-23T17:56:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0895e-05 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.034316+0000", 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011205999999999999 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.034440+0000", 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012376999999999999 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.035981+0000", 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015412690000000001 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076210+0000", 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040228886999999998 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.078250+0000", 2023-09-23T17:56:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00203981 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.078292+0000", 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1749999999999998e-05 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.078325+0000", 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3124999999999999e-05 2023-09-23T17:56:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:40132 5.b 5:d9f44754:::benchmark_data_smithi055_229117_object40131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:16.036151+0000", 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.721996844, 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.048210990000000002, 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 40132 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.036151+0000", 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.036151+0000", 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.036149+0000", 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.036170+0000", 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1109000000000001e-05 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076247+0000", 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040076410999999999 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076260+0000", 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2904e-05 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076363+0000", 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000102846 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076497+0000", 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000134929 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076654+0000", 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000156613 2023-09-23T17:56:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.079695+0000", 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003040896 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.080794+0000", 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0010992280000000001 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.084278+0000", 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034834589999999999 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.084325+0000", 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6941999999999999e-05 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.084362+0000", 2023-09-23T17:56:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6999000000000003e-05 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.44681.0:40128 5.8 e118/110 ondisk, result = 0)", 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:16.036423+0000", 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.721724053999999, 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.041908327000000002, 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.036423+0000", 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.036423+0000", 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.036423+0000", 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.036428+0000", 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9880000000000004e-06 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076850+0000", 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040421868 2023-09-23T17:56:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.076859+0000", 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2189999999999993e-06 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.078238+0000", 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013788159999999999 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.078245+0000", 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1300000000000003e-06 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.078332+0000", 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7015000000000006e-05 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:40852 5.a 5:5706de57:::benchmark_data_smithi055_229117_object40851:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:16.438425+0000", 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.319722699, 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.042082189999999998, 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 40852 2023-09-23T17:56:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.438425+0000", 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.438425+0000", 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.438424+0000", 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.438447+0000", 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3642000000000002e-05 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.438457+0000", 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0311e-05 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.438464+0000", 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0079999999999996e-06 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.439354+0000", 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00088934699999999997 2023-09-23T17:56:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.439497+0000", 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000143237 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.439637+0000", 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000139746 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.442927+0000", 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003289738 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.444905+0000", 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019787060000000002 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.445988+0000", 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0010824299999999999 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.480470+0000", 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.034482387000000003 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:16.480507+0000", 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6872999999999997e-05 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:45710 5.b 5:d38350b9:::benchmark_data_smithi055_229117_object45709:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:19.334120+0000", 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 8.4240278889999995, 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.042453266000000003, 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:56:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 45710 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.334120+0000", 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.334120+0000", 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.334118+0000", 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.334148+0000", 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0000000000000001e-05 2023-09-23T17:56:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.334164+0000", 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5891000000000001e-05 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.334172+0000", 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7390000000000005e-06 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.334241+0000", 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8385000000000003e-05 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.334409+0000", 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016795699999999999 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.334590+0000", 2023-09-23T17:56:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000181301 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.336762+0000", 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002171582 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.337710+0000", 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00094807400000000003 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.376474+0000", 2023-09-23T17:56:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038763789999999999 2023-09-23T17:56:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:56:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.376531+0000", 2023-09-23T17:56:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7720000000000003e-05 2023-09-23T17:56:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:56:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:19.376573+0000", 2023-09-23T17:56:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1520999999999998e-05 2023-09-23T17:56:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:27.905 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:56:27.906 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:27.992 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:27 smithi099 ceph-mon[167076]: pgmap v253: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 12518/154344 objects degraded (8.110%) 2023-09-23T17:56:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/88103237' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T17:56:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:27 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T17:56:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:27 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 12518/154344 objects degraded (8.110%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:28.108 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:27 smithi055 ceph-mon[179668]: pgmap v253: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 12518/154344 objects degraded (8.110%) 2023-09-23T17:56:28.109 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/88103237' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T17:56:28.109 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:27 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T17:56:28.109 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:27 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 12518/154344 objects degraded (8.110%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:28.109 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:27 smithi055 ceph-mon[184174]: pgmap v253: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 97 MiB/s wr, 1.56k op/s; 12518/154344 objects degraded (8.110%) 2023-09-23T17:56:28.109 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/88103237' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T17:56:28.109 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:27 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T17:56:28.109 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:27 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 12518/154344 objects degraded (8.110%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:28.163 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:28.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:28.171 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:28.176 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T17:56:28.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:28.316 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:28.353 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 30 16 57983 57967 120.675 85.4375 0.00894453 0.00828187 2023-09-23T17:56:28.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:28.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:28.360 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:28.461 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:28.462 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 5 pgp_num to 16 2023-09-23T17:56:28.517 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:28.532 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:28.650 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:28.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:28.659 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:28.759 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:28.796 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:28.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:28.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:28.849 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:28.950 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:29.028 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:29.086 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:29.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:29.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:29.198 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:29.264 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:29.333 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:29.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:29.338 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:29.353 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 31 16 59335 59319 119.509 84.5 0.0134215 0.00836174 2023-09-23T17:56:29.405 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:56:29.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:29.473 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:29.524 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:29.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:29.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:29.621 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[184174]: pgmap v254: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 13342/161943 objects degraded (8.239%) 2023-09-23T17:56:29.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T17:56:29.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[184174]: osdmap e120: 8 total, 7 up, 8 in 2023-09-23T17:56:29.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[184174]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[184174]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[184174]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[184174]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[179668]: pgmap v254: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 13342/161943 objects degraded (8.239%) 2023-09-23T17:56:29.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T17:56:29.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[179668]: osdmap e120: 8 total, 7 up, 8 in 2023-09-23T17:56:29.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[179668]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[179668]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[179668]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:29 smithi055 ceph-mon[179668]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.631 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:29.699 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:29.744 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:29.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:29 smithi099 ceph-mon[167076]: pgmap v254: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 13342/161943 objects degraded (8.239%) 2023-09-23T17:56:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:29 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T17:56:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:29 smithi099 ceph-mon[167076]: osdmap e120: 8 total, 7 up, 8 in 2023-09-23T17:56:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:29 smithi099 ceph-mon[167076]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:29 smithi099 ceph-mon[167076]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:29 smithi099 ceph-mon[167076]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:29 smithi099 ceph-mon[167076]: failed to encode map e120 with expected crc 2023-09-23T17:56:29.874 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:29.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:29.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:29.982 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:30.080 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:30.195 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:30.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:30.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:30.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:30.339 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:30.353 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 32 16 60638 60622 118.319 81.4375 0.0145877 0.00844751 2023-09-23T17:56:30.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:30.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:30.398 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:30.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:56:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:56:30] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-09-23T17:56:30.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:30.537 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:30.588 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:30.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:30.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:30.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:30.735 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:30.741 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:30.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:30.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:30.789 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:30.890 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:30.975 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:31.118 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:31.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:31.124 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:31.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:31.270 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:31.323 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:31.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:31.354 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 33 16 62001 61985 117.315 85.1875 0.00449342 0.00851995 2023-09-23T17:56:31.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:31.467 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:31.480 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T17:56:31.511 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:31 smithi055 ceph-mon[179668]: pgmap v256: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 14178/173100 objects degraded (8.191%) 2023-09-23T17:56:31.511 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/452430756' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:31.511 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:31 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:31.512 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:31 smithi055 ceph-mon[184174]: pgmap v256: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 14178/173100 objects degraded (8.191%) 2023-09-23T17:56:31.512 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/452430756' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:31.512 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:31 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:31.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:31.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:31.520 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:31.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:31 smithi099 ceph-mon[167076]: pgmap v256: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.5 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 83 MiB/s wr, 1.32k op/s; 14178/173100 objects degraded (8.191%) 2023-09-23T17:56:31.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/452430756' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:31.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:31 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:31.610 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:56:31.611 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:56:31.611 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.611 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:62317 5.a e121/110 5:5d06ec68:::benchmark_data_smithi055_229117_object62316:head v 121'3916, mlcod=121'3913)", 2023-09-23T17:56:31.611 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:31.610151+0000", 2023-09-23T17:56:31.611 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.000378391, 2023-09-23T17:56:31.611 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00042084700000000002, 2023-09-23T17:56:31.611 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:31.611 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T17:56:31.611 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610151+0000", 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610151+0000", 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.612 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610149+0000", 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610201+0000", 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1882999999999998e-05 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610225+0000", 2023-09-23T17:56:31.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3805e-05 2023-09-23T17:56:31.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:31.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610235+0000", 2023-09-23T17:56:31.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5859999999999994e-06 2023-09-23T17:56:31.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:56:31.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610413+0000", 2023-09-23T17:56:31.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000178541 2023-09-23T17:56:31.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:56:31.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610435+0000", 2023-09-23T17:56:31.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2133e-05 2023-09-23T17:56:31.615 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:31.615 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:31.615 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:31.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.616 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:62322 5.a e121/110)", 2023-09-23T17:56:31.616 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:31.610252+0000", 2023-09-23T17:56:31.616 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.00027709399999999997, 2023-09-23T17:56:31.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000382433, 2023-09-23T17:56:31.616 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:56:31.616 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T17:56:31.616 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:56:31.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:56:31.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610252+0000", 2023-09-23T17:56:31.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:31.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:56:31.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610252+0000", 2023-09-23T17:56:31.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:56:31.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:56:31.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610252+0000", 2023-09-23T17:56:31.618 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:31.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:56:31.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610282+0000", 2023-09-23T17:56:31.618 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0457e-05 2023-09-23T17:56:31.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:56:31.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610290+0000", 2023-09-23T17:56:31.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1980000000000001e-06 2023-09-23T17:56:31.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:56:31.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:56:31.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:56:31.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:31.610296+0000", 2023-09-23T17:56:31.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5010000000000004e-06 2023-09-23T17:56:31.620 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:31.620 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:56:31.620 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:31.620 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:56:31.620 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T17:56:31.620 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 2 2023-09-23T17:56:31.620 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:56:31.620 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:31.693 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:32.353 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 34 16 63409 63393 116.453 88 0.0103631 0.00858259 2023-09-23T17:56:32.387 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T17:56:32.394 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:32.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:32.409 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:32.509 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:32.585 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:32.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:32.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:32.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: osdmap e121: 8 total, 7 up, 8 in 2023-09-23T17:56:32.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[184174]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:32.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:32.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: osdmap e121: 8 total, 7 up, 8 in 2023-09-23T17:56:32.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:32 smithi055 ceph-mon[179668]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.683 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:32.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:32.691 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:32.761 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:56:32.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:32.795 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:56:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: osdmap e121: 8 total, 7 up, 8 in 2023-09-23T17:56:32.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:32 smithi099 ceph-mon[167076]: failed to encode map e121 with expected crc 2023-09-23T17:56:32.831 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:32.834 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:56:32.870 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:56:32.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:32.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:32.889 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:32.913 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:56:32.947 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:56:32.990 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:33.003 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:56:33.003 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:56:33.004 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:56:33.004 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:56:33.004 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T17:56:33.004 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:56:33.004 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:56:33.004 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:56:33.004 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T17:56:33.033 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:33.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:33.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:33.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:33.172 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:33.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:33.239 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:33.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:33.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:33.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:33.354 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 35 16 64702 64686 115.433 80.8125 0.0137789 0.00865868 2023-09-23T17:56:33.408 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:33.470 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:33.788 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:33.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:33.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:33.797 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:33 smithi055 ceph-mon[184174]: pgmap v258: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 15019/181830 objects degraded (8.260%) 2023-09-23T17:56:33.798 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/128900882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:33.798 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15019/181830 objects degraded (8.260%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:33.798 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:33 smithi055 ceph-mon[179668]: pgmap v258: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 15019/181830 objects degraded (8.260%) 2023-09-23T17:56:33.798 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/128900882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:33.798 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15019/181830 objects degraded (8.260%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:33.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:33 smithi099 ceph-mon[167076]: pgmap v258: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.7 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 82 MiB/s wr, 1.31k op/s; 15019/181830 objects degraded (8.260%) 2023-09-23T17:56:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:33 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/128900882' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15019/181830 objects degraded (8.260%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:33.830 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:56:33.830 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:56:33.898 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:33.961 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:34.008 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 4, 5, 6, 7] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T17:56:34.009 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:56:34.009 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 4, 5, 6, 7] 2023-09-23T17:56:34.009 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T17:56:34.187 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T17:56:34.213 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:34.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:34.219 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:34.320 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:34.357 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 36 16 65991 65975 114.455 80.5625 0.0256925 0.00873171 2023-09-23T17:56:34.397 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:34.574 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:34.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:34.579 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:34.680 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:34.798 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:34.861 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:34.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:34.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:34.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:34.980 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:35.015 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:35.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:35.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:35.060 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:35.124 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:56:35.124 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:56:35.160 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:35.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:35.187 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:35.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:35.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:35.235 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:35.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:35.357 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 37 16 67173 67157 113.359 73.875 0.00719086 0.00881739 2023-09-23T17:56:35.378 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:35.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:35.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:35.437 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:35.537 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:35.598 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:35.615 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T17:56:35.690 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:35 smithi055 ceph-mon[184174]: pgmap v259: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 15847/192171 objects degraded (8.246%) 2023-09-23T17:56:35.690 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:35 smithi055 ceph-mon[179668]: pgmap v259: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 15847/192171 objects degraded (8.246%) 2023-09-23T17:56:35.752 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:35.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:35.764 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:35.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:35 smithi099 ceph-mon[167076]: pgmap v259: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 3.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 98 MiB/s wr, 1.57k op/s; 15847/192171 objects degraded (8.246%) 2023-09-23T17:56:35.831 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:56:35.865 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:35.972 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:36.120 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:36.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:36.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:36.233 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:36.293 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:36.357 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 38 16 68283 68267 112.202 69.375 0.0140084 0.0089067 2023-09-23T17:56:36.480 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:36.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:36.484 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:36.585 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:36.641 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:36.667 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:37.133 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:37.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:37.143 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:37.244 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:37.344 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:37.357 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 39 16 69379 69363 111.082 68.5 0.0165545 0.00899608 2023-09-23T17:56:37.525 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:37.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:37.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:37.630 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:37.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:37 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T17:56:37.413+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e121 definitely_dead 0 2023-09-23T17:56:37.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:37.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:37.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:37 smithi055 ceph-mon[179668]: pgmap v260: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 4.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 15847/194523 objects degraded (8.147%) 2023-09-23T17:56:37.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:37.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:37.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:37 smithi055 ceph-mon[184174]: pgmap v260: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 4.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 15847/194523 objects degraded (8.147%) 2023-09-23T17:56:37.694 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:37.730 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:37.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:37 smithi099 ceph-mon[167076]: pgmap v260: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 4.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 85 MiB/s wr, 1.36k op/s; 15847/194523 objects degraded (8.147%) 2023-09-23T17:56:37.967 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:37.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:37.974 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:38.075 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:38.148 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:38.234 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:38.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:38.244 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:38.345 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:38.358 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T17:56:38.358908+0000 min lat: 0.00103404 max lat: 0.541726 avg lat: 0.00906866 2023-09-23T17:56:38.358 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T17:56:38.358 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 40 16 70599 70583 110.211 76.25 0.0103257 0.00906866 2023-09-23T17:56:38.448 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T17:56:38.448 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:38.457 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.4. 2023-09-23T17:56:38.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4271175877' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T17:56:38.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T17:56:38.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[179668]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T17:56:38.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15847/194523 objects degraded (8.147%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:38.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/904147859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:38.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:38.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:38.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T17:56:38.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[179668]: osdmap e122: 8 total, 7 up, 7 in 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4271175877' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[184174]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15847/194523 objects degraded (8.147%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/904147859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:56:38.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:38 smithi055 ceph-mon[184174]: osdmap e122: 8 total, 7 up, 7 in 2023-09-23T17:56:38.745 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:38.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:38.748 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4271175877' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T17:56:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T17:56:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:38 smithi099 ceph-mon[167076]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T17:56:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15847/194523 objects degraded (8.147%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/904147859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:56:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:38 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T17:56:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:56:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:38 smithi099 ceph-mon[167076]: osdmap e122: 8 total, 7 up, 7 in 2023-09-23T17:56:38.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:38.876 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:38.909 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:38.911 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:39.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:39.038 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:39.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:39.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:39.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:39.172 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:39.203 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:39.224 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:56:39.225 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:56:39.358 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 41 16 70707 70691 107.688 6.75 0.00320297 0.009077 2023-09-23T17:56:39.594 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:39.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:39.597 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:39.698 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:39.723 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:39 smithi055 ceph-mon[179668]: failed to encode map e122 with expected crc 2023-09-23T17:56:39.723 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:39 smithi055 ceph-mon[179668]: pgmap v262: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 4.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 16583/201141 objects degraded (8.244%) 2023-09-23T17:56:39.724 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:39 smithi055 ceph-mon[184174]: failed to encode map e122 with expected crc 2023-09-23T17:56:39.724 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:39 smithi055 ceph-mon[184174]: pgmap v262: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 4.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 16583/201141 objects degraded (8.244%) 2023-09-23T17:56:39.726 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:39.761 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:39.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:39.764 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:39 smithi099 ceph-mon[167076]: failed to encode map e122 with expected crc 2023-09-23T17:56:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:39 smithi099 ceph-mon[167076]: pgmap v262: 89 pgs: 10 active+undersized+degraded, 21 active+undersized, 58 active+clean; 4.1 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 73 MiB/s wr, 1.17k op/s; 16583/201141 objects degraded (8.244%) 2023-09-23T17:56:39.865 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:39.895 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:40.062 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T17:56:40.066 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:40.068 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:40.069 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:40.170 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:40.198 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:40.327 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:40.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:40.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:40.358 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 42 16 70707 70691 105.126 0 - 0.009077 2023-09-23T17:56:40.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:40.462 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:40.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:40.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:40.516 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:40.617 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:40.645 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:40.661 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:56:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: osdmap e123: 8 total, 7 up, 7 in 2023-09-23T17:56:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[184174]: osdmap e124: 8 total, 7 up, 7 in 2023-09-23T17:56:40.684 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:40.684 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: osdmap e123: 8 total, 7 up, 7 in 2023-09-23T17:56:40.684 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.686 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.686 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.686 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.686 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-mon[179668]: osdmap e124: 8 total, 7 up, 7 in 2023-09-23T17:56:40.686 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:56:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:56:40] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.43.0" 2023-09-23T17:56:40.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:40.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:40.719 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:56:40.771 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:56:40.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:40.802 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:56:40.814 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: osdmap e123: 8 total, 7 up, 7 in 2023-09-23T17:56:40.814 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.816 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: failed to encode map e123 with expected crc 2023-09-23T17:56:40.816 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:40 smithi099 ceph-mon[167076]: osdmap e124: 8 total, 7 up, 7 in 2023-09-23T17:56:40.817 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:40.845 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:56:40.884 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:56:40.908 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:40.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:40.913 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:40.928 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:56:40.978 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:56:40.978 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:56:40.978 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:56:40.978 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:56:40.978 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T17:56:40.979 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:56:40.979 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:56:40.979 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:56:40.979 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T17:56:41.014 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:41.101 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:41.332 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:41.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:41.341 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:41.359 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 43 16 71190 71174 103.382 15.0938 0.0107311 0.00966719 2023-09-23T17:56:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:56:41.430 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:56:41.442 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:41.570 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:41.634 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:41.646 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[184174]: pgmap v264: 89 pgs: 4 remapped+peering, 2 unknown, 8 peering, 10 active+undersized+degraded, 19 active+undersized, 46 active+clean; 4.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 17186/210306 objects degraded (8.172%); 0 B/s, 0 objects/s recovering 2023-09-23T17:56:41.646 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[184174]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.646 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[184174]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.646 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[184174]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[184174]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[184174]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:56:41.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1802138833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:41.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[179668]: pgmap v264: 89 pgs: 4 remapped+peering, 2 unknown, 8 peering, 10 active+undersized+degraded, 19 active+undersized, 46 active+clean; 4.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 17186/210306 objects degraded (8.172%); 0 B/s, 0 objects/s recovering 2023-09-23T17:56:41.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[179668]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[179668]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[179668]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[179668]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[179668]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:56:41.648 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:41 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1802138833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:41.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:41.648 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:41.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:41 smithi099 ceph-mon[167076]: pgmap v264: 89 pgs: 4 remapped+peering, 2 unknown, 8 peering, 10 active+undersized+degraded, 19 active+undersized, 46 active+clean; 4.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 74 MiB/s wr, 1.19k op/s; 17186/210306 objects degraded (8.172%); 0 B/s, 0 objects/s recovering 2023-09-23T17:56:41.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:41 smithi099 ceph-mon[167076]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:41 smithi099 ceph-mon[167076]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:41 smithi099 ceph-mon[167076]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:41 smithi099 ceph-mon[167076]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:41 smithi099 ceph-mon[167076]: failed to encode map e124 with expected crc 2023-09-23T17:56:41.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:41 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:56:41.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:41 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1802138833' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:41.749 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:41.831 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:41.893 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:41.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:41.905 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:41.931 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:56:41.932 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T17:56:41.932 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T17:56:41.932 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:56:42.006 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:42.047 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:42.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:42.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:42.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:42.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T17:56:42.349 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T17:56:42.359 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 44 16 72019 72003 102.21 51.8125 0.0149863 0.00977915 2023-09-23T17:56:42.403 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:42.499 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:42.576 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:42.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:42.590 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:42.691 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:42.721 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:42.764 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:42.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:42.774 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:42.874 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:42.954 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:43.014 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:56:43.039 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:43.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:43.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:43.151 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:43.225 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:43.263 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:43.362 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 45 16 72749 72733 100.948 45.625 0.017689 0.00990132 2023-09-23T17:56:43.431 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:56:43.432 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:43.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:43.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:43.539 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:43.670 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:43.752 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:43.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:43.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:43.757 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:43 smithi055 ceph-mon[184174]: pgmap v266: 89 pgs: 15 active+clean+remapped, 40 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 peering, 2 unknown, 4 remapped+peering; 4.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s; 21835/213390 objects degraded (10.232%); 13170/213390 objects misplaced (6.172%); 832 KiB/s, 16 objects/s recovering 2023-09-23T17:56:43.757 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 21835/213390 objects degraded (10.232%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:43.758 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:43 smithi055 ceph-mon[179668]: pgmap v266: 89 pgs: 15 active+clean+remapped, 40 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 peering, 2 unknown, 4 remapped+peering; 4.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s; 21835/213390 objects degraded (10.232%); 13170/213390 objects misplaced (6.172%); 832 KiB/s, 16 objects/s recovering 2023-09-23T17:56:43.758 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 21835/213390 objects degraded (10.232%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:43.783 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:43 smithi099 ceph-mon[167076]: pgmap v266: 89 pgs: 15 active+clean+remapped, 40 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 8 peering, 2 unknown, 4 remapped+peering; 4.3 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 65 MiB/s wr, 1.05k op/s; 21835/213390 objects degraded (10.232%); 13170/213390 objects misplaced (6.172%); 832 KiB/s, 16 objects/s recovering 2023-09-23T17:56:43.783 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 21835/213390 objects degraded (10.232%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:43.857 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:43.944 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:44.174 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:44.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:44.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:44.291 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:44.304 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:56:44.304 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T17:56:44.304 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T17:56:44.304 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T17:56:44.304 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:56:44.362 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 46 16 73443 73427 99.6966 43.375 0.0186051 0.0100247 2023-09-23T17:56:44.419 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:44.437 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:44.563 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:44.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:44.571 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:44.672 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:44.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T17:56:44.763 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:44.894 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7] out_osds: [4] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T17:56:44.894 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:56:44.894 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T17:56:44.895 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T17:56:44.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:44 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-09-23T17:56:44.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:44 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-09-23T17:56:45.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:44 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-09-23T17:56:45.279 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:45.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:45.287 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:45.362 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 47 16 74163 74147 98.5333 45 0.0145446 0.0101421 2023-09-23T17:56:45.384 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:56:45.388 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:45.459 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:45.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:45 smithi055 ceph-mon[179668]: pgmap v267: 89 pgs: 18 active+clean+remapped, 46 active+clean, 11 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 4 remapped+peering; 4.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s; 26595/216885 objects degraded (12.262%); 17738/216885 objects misplaced (8.179%); 841 KiB/s, 16 objects/s recovering 2023-09-23T17:56:45.704 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:45.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:45.721 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:45.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:45.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:45 smithi099 ceph-mon[167076]: pgmap v267: 89 pgs: 18 active+clean+remapped, 46 active+clean, 11 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 4 remapped+peering; 4.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s; 26595/216885 objects degraded (12.262%); 17738/216885 objects misplaced (8.179%); 841 KiB/s, 16 objects/s recovering 2023-09-23T17:56:45.884 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:45.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:45.930 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:45.931 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:45 smithi055 ceph-mon[184174]: pgmap v267: 89 pgs: 18 active+clean+remapped, 46 active+clean, 11 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded, 4 remapped+peering; 4.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 54 MiB/s wr, 871 op/s; 26595/216885 objects degraded (12.262%); 17738/216885 objects misplaced (8.179%); 841 KiB/s, 16 objects/s recovering 2023-09-23T17:56:46.031 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:46.067 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:46.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:46.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:46.165 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:46.266 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:46.309 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:46.362 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 48 16 74932 74916 97.4823 48.0625 0.017308 0.0102517 2023-09-23T17:56:46.413 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:46.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:46.418 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:46.519 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:46.595 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:46.652 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T17:56:46.687 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2714130274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:46.687 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:46 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:46.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2714130274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:46.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:46 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:46.688 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:46.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:46.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:46.757 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:56:46.757 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T17:56:46.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T17:56:46.757 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T17:56:46.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.757 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:31532 5.e 5:79717954:::benchmark_data_smithi055_229117_object31531:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:46.757 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:11.155156+0000", 2023-09-23T17:56:46.758 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 35.599382347999999, 2023-09-23T17:56:46.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018195324999999998, 2023-09-23T17:56:46.758 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.758 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.758 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.758 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.758 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.758 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 31532 2023-09-23T17:56:46.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.759 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.155156+0000", 2023-09-23T17:56:46.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.155156+0000", 2023-09-23T17:56:46.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.155156+0000", 2023-09-23T17:56:46.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:46.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.155185+0000", 2023-09-23T17:56:46.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9408e-05 2023-09-23T17:56:46.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.761 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.155190+0000", 2023-09-23T17:56:46.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.189e-06 2023-09-23T17:56:46.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.761 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.155195+0000", 2023-09-23T17:56:46.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.66e-06 2023-09-23T17:56:46.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.156527+0000", 2023-09-23T17:56:46.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001331783 2023-09-23T17:56:46.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.156645+0000", 2023-09-23T17:56:46.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011814799999999999 2023-09-23T17:56:46.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.156788+0000", 2023-09-23T17:56:46.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014264900000000001 2023-09-23T17:56:46.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.159264+0000", 2023-09-23T17:56:46.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002476065 2023-09-23T17:56:46.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.764 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.764 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.160083+0000", 2023-09-23T17:56:46.764 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00081888900000000005 2023-09-23T17:56:46.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.764 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.764 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.173248+0000", 2023-09-23T17:56:46.764 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013164825 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.173312+0000", 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4218000000000001e-05 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:11.173352+0000", 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9904000000000003e-05 2023-09-23T17:56:46.765 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.766 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.766 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.766 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:35811 5.e 5:7306e3c2:::benchmark_data_smithi055_229117_object35810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:46.766 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:13.407044+0000", 2023-09-23T17:56:46.766 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 33.347494953000002, 2023-09-23T17:56:46.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017897724, 2023-09-23T17:56:46.766 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.766 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.767 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.767 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.767 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.767 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 35811 2023-09-23T17:56:46.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.767 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.767 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.767 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407044+0000", 2023-09-23T17:56:46.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.768 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.768 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407044+0000", 2023-09-23T17:56:46.768 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.768 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.768 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407042+0000", 2023-09-23T17:56:46.768 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:46.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407073+0000", 2023-09-23T17:56:46.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0568999999999998e-05 2023-09-23T17:56:46.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407082+0000", 2023-09-23T17:56:46.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3589999999999997e-06 2023-09-23T17:56:46.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407088+0000", 2023-09-23T17:56:46.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2889999999999997e-06 2023-09-23T17:56:46.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407319+0000", 2023-09-23T17:56:46.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00023091299999999999 2023-09-23T17:56:46.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407455+0000", 2023-09-23T17:56:46.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013611999999999999 2023-09-23T17:56:46.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407619+0000", 2023-09-23T17:56:46.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000164449 2023-09-23T17:56:46.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.410388+0000", 2023-09-23T17:56:46.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002768824 2023-09-23T17:56:46.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.419574+0000", 2023-09-23T17:56:46.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0091861430000000008 2023-09-23T17:56:46.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.773 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.773 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.773 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.424861+0000", 2023-09-23T17:56:46.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052868840000000004 2023-09-23T17:56:46.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.773 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.773 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.773 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.424904+0000", 2023-09-23T17:56:46.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3282000000000003e-05 2023-09-23T17:56:46.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.774 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.774 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.774 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.424942+0000", 2023-09-23T17:56:46.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7351000000000002e-05 2023-09-23T17:56:46.774 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.774 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.774 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.775 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:35813 5.e 5:71e0e52c:::benchmark_data_smithi055_229117_object35812:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:46.775 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:13.407201+0000", 2023-09-23T17:56:46.775 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 33.347337467000003, 2023-09-23T17:56:46.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017948311000000002, 2023-09-23T17:56:46.775 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.775 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.775 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.776 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.776 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.776 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 35813 2023-09-23T17:56:46.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.776 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.776 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.776 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407201+0000", 2023-09-23T17:56:46.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407201+0000", 2023-09-23T17:56:46.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407200+0000", 2023-09-23T17:56:46.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:56:46.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407232+0000", 2023-09-23T17:56:46.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1090000000000002e-05 2023-09-23T17:56:46.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407250+0000", 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8736000000000002e-05 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.407257+0000", 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5520000000000001e-06 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.408695+0000", 2023-09-23T17:56:46.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014382959999999999 2023-09-23T17:56:46.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.408820+0000", 2023-09-23T17:56:46.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012483000000000001 2023-09-23T17:56:46.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.408966+0000", 2023-09-23T17:56:46.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014560800000000001 2023-09-23T17:56:46.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.410394+0000", 2023-09-23T17:56:46.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001428356 2023-09-23T17:56:46.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.419613+0000", 2023-09-23T17:56:46.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0092195820000000005 2023-09-23T17:56:46.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.425062+0000", 2023-09-23T17:56:46.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054480689999999998 2023-09-23T17:56:46.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.425112+0000", 2023-09-23T17:56:46.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9944000000000002e-05 2023-09-23T17:56:46.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:13.425150+0000", 2023-09-23T17:56:46.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8145999999999997e-05 2023-09-23T17:56:46.783 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.783 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.783 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:38883 5.e 5:7ce77683:::benchmark_data_smithi055_229117_object38882:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.244882+0000", 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.509656709000001, 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.111805565, 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.784 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 38883 2023-09-23T17:56:46.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.785 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244882+0000", 2023-09-23T17:56:46.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244882+0000", 2023-09-23T17:56:46.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244881+0000", 2023-09-23T17:56:46.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:46.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244920+0000", 2023-09-23T17:56:46.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9873999999999998e-05 2023-09-23T17:56:46.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244935+0000", 2023-09-23T17:56:46.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4430000000000001e-05 2023-09-23T17:56:46.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244941+0000", 2023-09-23T17:56:46.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1380000000000004e-06 2023-09-23T17:56:46.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.245154+0000", 2023-09-23T17:56:46.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021285 2023-09-23T17:56:46.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.245293+0000", 2023-09-23T17:56:46.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013899299999999999 2023-09-23T17:56:46.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.245431+0000", 2023-09-23T17:56:46.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013820099999999999 2023-09-23T17:56:46.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248486+0000", 2023-09-23T17:56:46.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003055211 2023-09-23T17:56:46.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.250192+0000", 2023-09-23T17:56:46.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017060510000000001 2023-09-23T17:56:46.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356588+0000", 2023-09-23T17:56:46.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.10639612599999999 2023-09-23T17:56:46.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356644+0000", 2023-09-23T17:56:46.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5108000000000002e-05 2023-09-23T17:56:46.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356688+0000", 2023-09-23T17:56:46.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4162000000000002e-05 2023-09-23T17:56:46.792 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.792 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.792 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.792 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:38885 5.e 5:7e62a019:::benchmark_data_smithi055_229117_object38884:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:46.793 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.244954+0000", 2023-09-23T17:56:46.793 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.509584631999999, 2023-09-23T17:56:46.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.11283467599999999, 2023-09-23T17:56:46.793 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.793 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.793 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.793 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.793 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.793 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 38885 2023-09-23T17:56:46.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.794 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244954+0000", 2023-09-23T17:56:46.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244954+0000", 2023-09-23T17:56:46.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244954+0000", 2023-09-23T17:56:46.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:46.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244981+0000", 2023-09-23T17:56:46.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6927000000000002e-05 2023-09-23T17:56:46.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244988+0000", 2023-09-23T17:56:46.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4259999999999997e-06 2023-09-23T17:56:46.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244993+0000", 2023-09-23T17:56:46.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0039999999999999e-06 2023-09-23T17:56:46.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246751+0000", 2023-09-23T17:56:46.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001757923 2023-09-23T17:56:46.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246844+0000", 2023-09-23T17:56:46.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3191999999999998e-05 2023-09-23T17:56:46.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.247004+0000", 2023-09-23T17:56:46.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016034099999999999 2023-09-23T17:56:46.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248703+0000", 2023-09-23T17:56:46.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016990029999999999 2023-09-23T17:56:46.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.250259+0000", 2023-09-23T17:56:46.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015560490000000001 2023-09-23T17:56:46.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.357695+0000", 2023-09-23T17:56:46.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.10743537 2023-09-23T17:56:46.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.357749+0000", 2023-09-23T17:56:46.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4268000000000003e-05 2023-09-23T17:56:46.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.357789+0000", 2023-09-23T17:56:46.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9761000000000001e-05 2023-09-23T17:56:46.801 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.801 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.801 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.801 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:40631 5.e 5:7cec279b:::benchmark_data_smithi055_229117_object40630:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:46.801 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:16.322037+0000", 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 30.432502007, 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018085342000000001, 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 40631 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.322037+0000", 2023-09-23T17:56:46.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.322037+0000", 2023-09-23T17:56:46.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.322036+0000", 2023-09-23T17:56:46.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:46.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.322065+0000", 2023-09-23T17:56:46.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9158000000000001e-05 2023-09-23T17:56:46.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.322080+0000", 2023-09-23T17:56:46.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5028000000000001e-05 2023-09-23T17:56:46.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.322086+0000", 2023-09-23T17:56:46.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4139999999999999e-06 2023-09-23T17:56:46.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.325199+0000", 2023-09-23T17:56:46.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031124210000000002 2023-09-23T17:56:46.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.325322+0000", 2023-09-23T17:56:46.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012333900000000001 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.325454+0000", 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132459 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.329703+0000", 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0042483249999999998 2023-09-23T17:56:46.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.329713+0000", 2023-09-23T17:56:46.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.961e-06 2023-09-23T17:56:46.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.340032+0000", 2023-09-23T17:56:46.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010319297 2023-09-23T17:56:46.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.340084+0000", 2023-09-23T17:56:46.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2547999999999999e-05 2023-09-23T17:56:46.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:16.340122+0000", 2023-09-23T17:56:46.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7670999999999999e-05 2023-09-23T17:56:46.809 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.810 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.810 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.810 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:42097 5.e 5:7a22e8c1:::benchmark_data_smithi055_229117_object42096:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:46.810 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:17.137737+0000", 2023-09-23T17:56:46.810 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 29.616801396, 2023-09-23T17:56:46.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018149297000000002, 2023-09-23T17:56:46.810 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.811 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.811 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.811 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.811 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.811 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 42097 2023-09-23T17:56:46.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.811 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.137737+0000", 2023-09-23T17:56:46.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.137737+0000", 2023-09-23T17:56:46.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.137737+0000", 2023-09-23T17:56:46.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:46.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.137765+0000", 2023-09-23T17:56:46.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8269e-05 2023-09-23T17:56:46.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.137772+0000", 2023-09-23T17:56:46.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3740000000000003e-06 2023-09-23T17:56:46.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.137776+0000", 2023-09-23T17:56:46.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0749999999999999e-06 2023-09-23T17:56:46.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.139509+0000", 2023-09-23T17:56:46.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017328580000000001 2023-09-23T17:56:46.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.139638+0000", 2023-09-23T17:56:46.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012971700000000001 2023-09-23T17:56:46.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.139772+0000", 2023-09-23T17:56:46.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013400200000000001 2023-09-23T17:56:46.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.144651+0000", 2023-09-23T17:56:46.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048783230000000004 2023-09-23T17:56:46.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.146946+0000", 2023-09-23T17:56:46.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022949350000000001 2023-09-23T17:56:46.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.155797+0000", 2023-09-23T17:56:46.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0088513940000000003 2023-09-23T17:56:46.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.155849+0000", 2023-09-23T17:56:46.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2435000000000002e-05 2023-09-23T17:56:46.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:17.155887+0000", 2023-09-23T17:56:46.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7348000000000002e-05 2023-09-23T17:56:46.818 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.818 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.818 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.819 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:44502 5.1 5:8fa7421a:::benchmark_data_smithi055_229117_object44501:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:46.819 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:18.508450+0000", 2023-09-23T17:56:46.819 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 28.246088669999999, 2023-09-23T17:56:46.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018086217000000002, 2023-09-23T17:56:46.819 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.819 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.819 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.820 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.820 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.820 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 44502 2023-09-23T17:56:46.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.820 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.508450+0000", 2023-09-23T17:56:46.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.508450+0000", 2023-09-23T17:56:46.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.508449+0000", 2023-09-23T17:56:46.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:56:46.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.508503+0000", 2023-09-23T17:56:46.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3996999999999998e-05 2023-09-23T17:56:46.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.508514+0000", 2023-09-23T17:56:46.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1404e-05 2023-09-23T17:56:46.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.823 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.508532+0000", 2023-09-23T17:56:46.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7183000000000001e-05 2023-09-23T17:56:46.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.823 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.508592+0000", 2023-09-23T17:56:46.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0232000000000002e-05 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.508705+0000", 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011343100000000001 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.508843+0000", 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013729700000000001 2023-09-23T17:56:46.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.510616+0000", 2023-09-23T17:56:46.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017738509999999999 2023-09-23T17:56:46.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.510804+0000", 2023-09-23T17:56:46.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000187505 2023-09-23T17:56:46.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.526469+0000", 2023-09-23T17:56:46.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015665281 2023-09-23T17:56:46.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.526508+0000", 2023-09-23T17:56:46.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8899999999999997e-05 2023-09-23T17:56:46.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.526536+0000", 2023-09-23T17:56:46.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8226e-05 2023-09-23T17:56:46.827 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.827 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.827 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.827 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:44868 5.e 5:7c17b6df:::benchmark_data_smithi055_229117_object44867:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:46.828 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:18.766321+0000", 2023-09-23T17:56:46.828 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 27.988217469999999, 2023-09-23T17:56:46.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019200623999999999, 2023-09-23T17:56:46.828 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.828 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.828 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.828 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.828 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.828 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 44868 2023-09-23T17:56:46.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.829 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.766321+0000", 2023-09-23T17:56:46.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.766321+0000", 2023-09-23T17:56:46.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.766320+0000", 2023-09-23T17:56:46.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:46.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.766429+0000", 2023-09-23T17:56:46.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108953 2023-09-23T17:56:46.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.831 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.766444+0000", 2023-09-23T17:56:46.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5330000000000001e-05 2023-09-23T17:56:46.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.831 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.766450+0000", 2023-09-23T17:56:46.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4790000000000001e-06 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.766499+0000", 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8226999999999998e-05 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.766640+0000", 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000141454 2023-09-23T17:56:46.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.766785+0000", 2023-09-23T17:56:46.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014521099999999999 2023-09-23T17:56:46.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.770393+0000", 2023-09-23T17:56:46.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0036075640000000002 2023-09-23T17:56:46.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.770643+0000", 2023-09-23T17:56:46.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025008400000000003 2023-09-23T17:56:46.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.785436+0000", 2023-09-23T17:56:46.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014792806 2023-09-23T17:56:46.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.785487+0000", 2023-09-23T17:56:46.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1238e-05 2023-09-23T17:56:46.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:18.785522+0000", 2023-09-23T17:56:46.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5008000000000001e-05 2023-09-23T17:56:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:45514 5.e 5:748e2010:::benchmark_data_smithi055_229117_object45513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:19.205744+0000", 2023-09-23T17:56:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 27.548794773000001, 2023-09-23T17:56:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019788193999999999, 2023-09-23T17:56:46.836 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 45514 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205744+0000", 2023-09-23T17:56:46.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205744+0000", 2023-09-23T17:56:46.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205743+0000", 2023-09-23T17:56:46.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:46.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205792+0000", 2023-09-23T17:56:46.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9276000000000001e-05 2023-09-23T17:56:46.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205806+0000", 2023-09-23T17:56:46.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4122e-05 2023-09-23T17:56:46.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205813+0000", 2023-09-23T17:56:46.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7920000000000004e-06 2023-09-23T17:56:46.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205863+0000", 2023-09-23T17:56:46.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9777000000000001e-05 2023-09-23T17:56:46.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205996+0000", 2023-09-23T17:56:46.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000133366 2023-09-23T17:56:46.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.206079+0000", 2023-09-23T17:56:46.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2461999999999998e-05 2023-09-23T17:56:46.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.208852+0000", 2023-09-23T17:56:46.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027728739999999998 2023-09-23T17:56:46.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.208895+0000", 2023-09-23T17:56:46.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3650999999999999e-05 2023-09-23T17:56:46.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.225449+0000", 2023-09-23T17:56:46.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016553924000000001 2023-09-23T17:56:46.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.225493+0000", 2023-09-23T17:56:46.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4298000000000002e-05 2023-09-23T17:56:46.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.225532+0000", 2023-09-23T17:56:46.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8847999999999998e-05 2023-09-23T17:56:46.844 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.844 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.845 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.845 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:45516 5.e 5:7c74bb6a:::benchmark_data_smithi055_229117_object45515:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:46.845 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:19.205829+0000", 2023-09-23T17:56:46.845 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 27.548709335000002, 2023-09-23T17:56:46.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019915855, 2023-09-23T17:56:46.846 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.846 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.846 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.846 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.846 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.846 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 45516 2023-09-23T17:56:46.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.846 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205829+0000", 2023-09-23T17:56:46.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205829+0000", 2023-09-23T17:56:46.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205829+0000", 2023-09-23T17:56:46.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:46.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205874+0000", 2023-09-23T17:56:46.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4854000000000001e-05 2023-09-23T17:56:46.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205884+0000", 2023-09-23T17:56:46.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9639999999999998e-06 2023-09-23T17:56:46.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.205889+0000", 2023-09-23T17:56:46.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9470000000000002e-06 2023-09-23T17:56:46.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.207509+0000", 2023-09-23T17:56:46.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016204310000000001 2023-09-23T17:56:46.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.207640+0000", 2023-09-23T17:56:46.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013061600000000001 2023-09-23T17:56:46.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.207758+0000", 2023-09-23T17:56:46.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000118084 2023-09-23T17:56:46.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.209130+0000", 2023-09-23T17:56:46.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013726000000000001 2023-09-23T17:56:46.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.210217+0000", 2023-09-23T17:56:46.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010871489999999999 2023-09-23T17:56:46.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.225581+0000", 2023-09-23T17:56:46.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015363107000000001 2023-09-23T17:56:46.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.225677+0000", 2023-09-23T17:56:46.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6127999999999998e-05 2023-09-23T17:56:46.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.225745+0000", 2023-09-23T17:56:46.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.86e-05 2023-09-23T17:56:46.853 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.853 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.853 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.854 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:45729 5.1 5:8c906439:::benchmark_data_smithi055_229117_object45728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:46.854 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:19.355571+0000", 2023-09-23T17:56:46.854 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 27.398968242999999, 2023-09-23T17:56:46.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019455494, 2023-09-23T17:56:46.854 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.854 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.854 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.854 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.855 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.855 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 45729 2023-09-23T17:56:46.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.855 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.355571+0000", 2023-09-23T17:56:46.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.355571+0000", 2023-09-23T17:56:46.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.355569+0000", 2023-09-23T17:56:46.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:46.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.355629+0000", 2023-09-23T17:56:46.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9633000000000003e-05 2023-09-23T17:56:46.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.355649+0000", 2023-09-23T17:56:46.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0177e-05 2023-09-23T17:56:46.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.355663+0000", 2023-09-23T17:56:46.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4457999999999999e-05 2023-09-23T17:56:46.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.355714+0000", 2023-09-23T17:56:46.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0170000000000002e-05 2023-09-23T17:56:46.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.355845+0000", 2023-09-23T17:56:46.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013130800000000001 2023-09-23T17:56:46.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:56:46.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.355978+0000", 2023-09-23T17:56:46.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013356300000000001 2023-09-23T17:56:46.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.357652+0000", 2023-09-23T17:56:46.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001673405 2023-09-23T17:56:46.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.357907+0000", 2023-09-23T17:56:46.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025478000000000002 2023-09-23T17:56:46.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.374916+0000", 2023-09-23T17:56:46.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017009409999999999 2023-09-23T17:56:46.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.374980+0000", 2023-09-23T17:56:46.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4189000000000005e-05 2023-09-23T17:56:46.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:19.375026+0000", 2023-09-23T17:56:46.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5896000000000002e-05 2023-09-23T17:56:46.862 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.862 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.862 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.862 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:55639 5.1 5:87365d98:::benchmark_data_smithi055_229117_object55638:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T17:56:46.862 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:26.657872+0000", 2023-09-23T17:56:46.863 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 20.096666311, 2023-09-23T17:56:46.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019979878999999999, 2023-09-23T17:56:46.863 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.863 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.863 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.863 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.863 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.863 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 55639 2023-09-23T17:56:46.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.863 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.657872+0000", 2023-09-23T17:56:46.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.657872+0000", 2023-09-23T17:56:46.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.657871+0000", 2023-09-23T17:56:46.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:46.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.657915+0000", 2023-09-23T17:56:46.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4168000000000001e-05 2023-09-23T17:56:46.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.657928+0000", 2023-09-23T17:56:46.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3468999999999999e-05 2023-09-23T17:56:46.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.657933+0000", 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1619999999999999e-06 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.657983+0000", 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9137000000000001e-05 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.658108+0000", 2023-09-23T17:56:46.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125038 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.658242+0000", 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000134766 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.660245+0000", 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020027410000000002 2023-09-23T17:56:46.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.660334+0000", 2023-09-23T17:56:46.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9295000000000005e-05 2023-09-23T17:56:46.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.677755+0000", 2023-09-23T17:56:46.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017420837000000002 2023-09-23T17:56:46.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.677811+0000", 2023-09-23T17:56:46.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5659999999999999e-05 2023-09-23T17:56:46.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.677852+0000", 2023-09-23T17:56:46.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1454e-05 2023-09-23T17:56:46.870 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.871 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.871 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.871 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:55775 5.e 5:7bd12518:::benchmark_data_smithi055_229117_object55774:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T17:56:46.871 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:26.751061+0000", 2023-09-23T17:56:46.871 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 20.0034776, 2023-09-23T17:56:46.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018414361000000001, 2023-09-23T17:56:46.871 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.871 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.872 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.872 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.872 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.872 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 55775 2023-09-23T17:56:46.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.872 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.751061+0000", 2023-09-23T17:56:46.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.751061+0000", 2023-09-23T17:56:46.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.751060+0000", 2023-09-23T17:56:46.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:46.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.751137+0000", 2023-09-23T17:56:46.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7095000000000006e-05 2023-09-23T17:56:46.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.751153+0000", 2023-09-23T17:56:46.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6113999999999999e-05 2023-09-23T17:56:46.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.751160+0000", 2023-09-23T17:56:46.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9609999999999999e-06 2023-09-23T17:56:46.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.751233+0000", 2023-09-23T17:56:46.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2976000000000001e-05 2023-09-23T17:56:46.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.751373+0000", 2023-09-23T17:56:46.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000139806 2023-09-23T17:56:46.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.751517+0000", 2023-09-23T17:56:46.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014437099999999999 2023-09-23T17:56:46.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.753244+0000", 2023-09-23T17:56:46.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017263999999999999 2023-09-23T17:56:46.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.754937+0000", 2023-09-23T17:56:46.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001693092 2023-09-23T17:56:46.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.769361+0000", 2023-09-23T17:56:46.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01442388 2023-09-23T17:56:46.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.769436+0000", 2023-09-23T17:56:46.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5438999999999994e-05 2023-09-23T17:56:46.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:26.769476+0000", 2023-09-23T17:56:46.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9504e-05 2023-09-23T17:56:46.879 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.879 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.879 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:57589 5.1 5:89f64402:::benchmark_data_smithi055_229117_object57588:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e119)", 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:28.072944+0000", 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.681595205000001, 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019957236999999999, 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.880 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 57589 2023-09-23T17:56:46.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.881 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.072944+0000", 2023-09-23T17:56:46.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.072944+0000", 2023-09-23T17:56:46.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.072943+0000", 2023-09-23T17:56:46.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:46.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.072994+0000", 2023-09-23T17:56:46.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0535999999999999e-05 2023-09-23T17:56:46.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.073005+0000", 2023-09-23T17:56:46.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1545999999999999e-05 2023-09-23T17:56:46.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.073008+0000", 2023-09-23T17:56:46.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.18e-06 2023-09-23T17:56:46.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.074753+0000", 2023-09-23T17:56:46.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017448819999999999 2023-09-23T17:56:46.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.074875+0000", 2023-09-23T17:56:46.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121206 2023-09-23T17:56:46.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:56:46.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.075017+0000", 2023-09-23T17:56:46.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014215300000000001 2023-09-23T17:56:46.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.076846+0000", 2023-09-23T17:56:46.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00182931 2023-09-23T17:56:46.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.078343+0000", 2023-09-23T17:56:46.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00149726 2023-09-23T17:56:46.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.092814+0000", 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014470635000000001 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.092859+0000", 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5293000000000002e-05 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.092901+0000", 2023-09-23T17:56:46.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1615000000000001e-05 2023-09-23T17:56:46.888 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.888 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.888 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.888 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:58116 5.e 5:7e8b1dd0:::benchmark_data_smithi055_229117_object58115:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T17:56:46.888 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:28.461378+0000", 2023-09-23T17:56:46.889 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.293161179999998, 2023-09-23T17:56:46.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021439151, 2023-09-23T17:56:46.889 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.889 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.889 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 58116 2023-09-23T17:56:46.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.889 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.461378+0000", 2023-09-23T17:56:46.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.461378+0000", 2023-09-23T17:56:46.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.461377+0000", 2023-09-23T17:56:46.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:46.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.461405+0000", 2023-09-23T17:56:46.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7441999999999999e-05 2023-09-23T17:56:46.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.461413+0000", 2023-09-23T17:56:46.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5059999999999995e-06 2023-09-23T17:56:46.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.461419+0000", 2023-09-23T17:56:46.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4689999999999996e-06 2023-09-23T17:56:46.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.461479+0000", 2023-09-23T17:56:46.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0287000000000001e-05 2023-09-23T17:56:46.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.461599+0000", 2023-09-23T17:56:46.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012048300000000001 2023-09-23T17:56:46.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.461738+0000", 2023-09-23T17:56:46.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013855700000000001 2023-09-23T17:56:46.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.463501+0000", 2023-09-23T17:56:46.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017631120000000001 2023-09-23T17:56:46.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.465280+0000", 2023-09-23T17:56:46.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017789660000000001 2023-09-23T17:56:46.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.482756+0000", 2023-09-23T17:56:46.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017476515000000001 2023-09-23T17:56:46.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.482791+0000", 2023-09-23T17:56:46.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4816e-05 2023-09-23T17:56:46.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:28.482817+0000", 2023-09-23T17:56:46.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.55e-05 2023-09-23T17:56:46.897 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.897 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.897 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.897 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:59809 5.1 5:8982bd54:::benchmark_data_smithi055_229117_object59808:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T17:56:46.897 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:29.686790+0000", 2023-09-23T17:56:46.897 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 17.067748945999998, 2023-09-23T17:56:46.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020298885999999999, 2023-09-23T17:56:46.897 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.898 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.898 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.898 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.898 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.898 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 59809 2023-09-23T17:56:46.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.898 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.686790+0000", 2023-09-23T17:56:46.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.686790+0000", 2023-09-23T17:56:46.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.686789+0000", 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.686864+0000", 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5115000000000002e-05 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.686872+0000", 2023-09-23T17:56:46.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6369999999999999e-06 2023-09-23T17:56:46.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.686877+0000", 2023-09-23T17:56:46.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6539999999999998e-06 2023-09-23T17:56:46.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.688675+0000", 2023-09-23T17:56:46.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001798355 2023-09-23T17:56:46.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.688793+0000", 2023-09-23T17:56:46.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011837600000000001 2023-09-23T17:56:46.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:56:46.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.688938+0000", 2023-09-23T17:56:46.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014486800000000001 2023-09-23T17:56:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.691528+0000", 2023-09-23T17:56:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025895940000000002 2023-09-23T17:56:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.692901+0000", 2023-09-23T17:56:46.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013725779999999999 2023-09-23T17:56:46.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.706986+0000", 2023-09-23T17:56:46.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014085284999999999 2023-09-23T17:56:46.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.707044+0000", 2023-09-23T17:56:46.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7930000000000003e-05 2023-09-23T17:56:46.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:29.707089+0000", 2023-09-23T17:56:46.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4997000000000003e-05 2023-09-23T17:56:46.905 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.905 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.905 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.906 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:61469 5.1 5:8591963e:::benchmark_data_smithi055_229117_object61468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e120)", 2023-09-23T17:56:46.906 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:30.968167+0000", 2023-09-23T17:56:46.906 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.786371645999999, 2023-09-23T17:56:46.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019857896999999999, 2023-09-23T17:56:46.906 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.906 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.906 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.906 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.906 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.906 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 61469 2023-09-23T17:56:46.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.907 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.968167+0000", 2023-09-23T17:56:46.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.968167+0000", 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.968166+0000", 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.968196+0000", 2023-09-23T17:56:46.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9714999999999999e-05 2023-09-23T17:56:46.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.968214+0000", 2023-09-23T17:56:46.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8028000000000001e-05 2023-09-23T17:56:46.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.968219+0000", 2023-09-23T17:56:46.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2449999999999998e-06 2023-09-23T17:56:46.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.968254+0000", 2023-09-23T17:56:46.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5234000000000002e-05 2023-09-23T17:56:46.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.968383+0000", 2023-09-23T17:56:46.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012859300000000001 2023-09-23T17:56:46.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:56:46.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.968496+0000", 2023-09-23T17:56:46.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112853 2023-09-23T17:56:46.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.970439+0000", 2023-09-23T17:56:46.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019435640000000001 2023-09-23T17:56:46.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.972449+0000", 2023-09-23T17:56:46.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020094969999999998 2023-09-23T17:56:46.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.987932+0000", 2023-09-23T17:56:46.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015482966000000001 2023-09-23T17:56:46.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.987980+0000", 2023-09-23T17:56:46.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8826999999999999e-05 2023-09-23T17:56:46.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:30.988025+0000", 2023-09-23T17:56:46.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.464e-05 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:62177 5.e 5:7b6edb70:::benchmark_data_smithi055_229117_object62176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:31.485499+0000", 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.269039812999999, 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017912557999999999, 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.914 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.915 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.915 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.915 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.915 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 62177 2023-09-23T17:56:46.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.915 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.485499+0000", 2023-09-23T17:56:46.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.485499+0000", 2023-09-23T17:56:46.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.485497+0000", 2023-09-23T17:56:46.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:46.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.485591+0000", 2023-09-23T17:56:46.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4303999999999997e-05 2023-09-23T17:56:46.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.485612+0000", 2023-09-23T17:56:46.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0508000000000001e-05 2023-09-23T17:56:46.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.485620+0000", 2023-09-23T17:56:46.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2810000000000008e-06 2023-09-23T17:56:46.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.485684+0000", 2023-09-23T17:56:46.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4107000000000001e-05 2023-09-23T17:56:46.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2023-09-23T17:56:46.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.485691+0000", 2023-09-23T17:56:46.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2069999999999998e-06 2023-09-23T17:56:46.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.487991+0000", 2023-09-23T17:56:46.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023000500000000001 2023-09-23T17:56:46.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.488137+0000", 2023-09-23T17:56:46.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014594 2023-09-23T17:56:46.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.489691+0000", 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015539410000000001 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.492849+0000", 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003158082 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.497283+0000", 2023-09-23T17:56:46.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044338490000000001 2023-09-23T17:56:46.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.503329+0000", 2023-09-23T17:56:46.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0060456490000000002 2023-09-23T17:56:46.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.503372+0000", 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3185000000000001e-05 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.503412+0000", 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9508999999999997e-05 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.924 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:62270 5.e 5:74744b30:::benchmark_data_smithi055_229117_object62269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e121)", 2023-09-23T17:56:46.924 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:31.558065+0000", 2023-09-23T17:56:46.924 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.196473782, 2023-09-23T17:56:46.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.023032312999999999, 2023-09-23T17:56:46.924 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:46.924 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:46.924 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:46.924 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:46.924 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:46.925 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 62270 2023-09-23T17:56:46.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.925 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:46.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:46.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.558065+0000", 2023-09-23T17:56:46.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:46.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.558065+0000", 2023-09-23T17:56:46.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:46.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:46.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.558064+0000", 2023-09-23T17:56:46.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:46.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:46.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.558095+0000", 2023-09-23T17:56:46.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0616e-05 2023-09-23T17:56:46.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:46.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.558105+0000", 2023-09-23T17:56:46.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6350000000000001e-06 2023-09-23T17:56:46.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:46.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.558109+0000", 2023-09-23T17:56:46.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8389999999999997e-06 2023-09-23T17:56:46.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:46.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.560193+0000", 2023-09-23T17:56:46.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002083087 2023-09-23T17:56:46.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:46.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.560320+0000", 2023-09-23T17:56:46.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127007 2023-09-23T17:56:46.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:56:46.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.560471+0000", 2023-09-23T17:56:46.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000151485 2023-09-23T17:56:46.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:46.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.562144+0000", 2023-09-23T17:56:46.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016725749999999999 2023-09-23T17:56:46.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.562426+0000", 2023-09-23T17:56:46.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00028249900000000002 2023-09-23T17:56:46.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.581036+0000", 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018610141 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.581069+0000", 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2824000000000003e-05 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:46.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:31.581097+0000", 2023-09-23T17:56:46.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8238999999999998e-05 2023-09-23T17:56:46.932 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.932 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.932 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.932 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:46.932 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:46.932 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:56:46.933 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:46.934 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2714130274' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:46.934 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:46 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:47.000 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:47.180 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T17:56:47.239 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:47.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:47.255 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:47.337 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:47.356 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:47.365 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 49 16 75655 75639 96.4101 45.1875 0.0252514 0.0103664 2023-09-23T17:56:47.392 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:47.437 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:47.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:47.443 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:47.543 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:47.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[179668]: pgmap v268: 89 pgs: 20 active+clean+remapped, 46 active+clean, 1 active+recovering+undersized+remapped, 11 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded; 4.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 46 MiB/s wr, 736 op/s; 31107/218820 objects degraded (14.216%); 22277/218820 objects misplaced (10.181%); 1.7 MiB/s, 29 objects/s recovering 2023-09-23T17:56:47.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:56:47.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:47.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[179668]: osdmap e125: 8 total, 7 up, 7 in 2023-09-23T17:56:47.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[179668]: failed to encode map e125 with expected crc 2023-09-23T17:56:47.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[179668]: failed to encode map e125 with expected crc 2023-09-23T17:56:47.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[179668]: failed to encode map e125 with expected crc 2023-09-23T17:56:47.630 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:47.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:47 smithi099 ceph-mon[167076]: pgmap v268: 89 pgs: 20 active+clean+remapped, 46 active+clean, 1 active+recovering+undersized+remapped, 11 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded; 4.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 46 MiB/s wr, 736 op/s; 31107/218820 objects degraded (14.216%); 22277/218820 objects misplaced (10.181%); 1.7 MiB/s, 29 objects/s recovering 2023-09-23T17:56:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:47 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:56:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:47 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:47 smithi099 ceph-mon[167076]: osdmap e125: 8 total, 7 up, 7 in 2023-09-23T17:56:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:47 smithi099 ceph-mon[167076]: failed to encode map e125 with expected crc 2023-09-23T17:56:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:47 smithi099 ceph-mon[167076]: failed to encode map e125 with expected crc 2023-09-23T17:56:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:47 smithi099 ceph-mon[167076]: failed to encode map e125 with expected crc 2023-09-23T17:56:47.858 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[184174]: pgmap v268: 89 pgs: 20 active+clean+remapped, 46 active+clean, 1 active+recovering+undersized+remapped, 11 active+undersized, 4 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 6 active+undersized+degraded; 4.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 46 MiB/s wr, 736 op/s; 31107/218820 objects degraded (14.216%); 22277/218820 objects misplaced (10.181%); 1.7 MiB/s, 29 objects/s recovering 2023-09-23T17:56:47.858 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:56:47.858 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:47.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[184174]: osdmap e125: 8 total, 7 up, 7 in 2023-09-23T17:56:47.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[184174]: failed to encode map e125 with expected crc 2023-09-23T17:56:47.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[184174]: failed to encode map e125 with expected crc 2023-09-23T17:56:47.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:47 smithi055 ceph-mon[184174]: failed to encode map e125 with expected crc 2023-09-23T17:56:47.862 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:56:47.956 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:47.961 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:47.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:48.063 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:48.150 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:56:48.182 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:56:48.200 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:48.233 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:56:48.281 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:56:48.322 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:56:48.366 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:56:48.368 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 50 16 76363 76347 95.3619 44.25 0.00917301 0.0104807 2023-09-23T17:56:48.404 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:48.407 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:56:48.410 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:56:48.410 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:56:48.410 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:56:48.410 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T17:56:48.411 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:56:48.411 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:56:48.411 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:56:48.411 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T17:56:48.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:48.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:48.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:48.546 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:48.646 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:48.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:48.656 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:48.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:48.788 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 31107/218820 objects degraded (14.216%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:48.788 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:48 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1418139078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:48.793 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:48.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 31107/218820 objects degraded (14.216%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:48.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:48 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1418139078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:48.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 31107/218820 objects degraded (14.216%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:48.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:48 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1418139078' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:48.833 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:48.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:48.836 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:48.861 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:56:48.862 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:56:48.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:48.974 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:49.072 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:49.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:49.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:49.106 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T17:56:49.168 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:56:49.168 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:56:49.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:49.210 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:49.368 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 51 16 77074 77058 94.3639 44.4375 0.00581871 0.0105913 2023-09-23T17:56:49.509 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:49.513 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:49.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T17:56:49.613 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:49.717 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:49.799 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:49.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:49.805 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:49.836 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.4. 2023-09-23T17:56:49.907 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:49.934 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:49.965 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:49.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:49.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:50.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:50.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:49 smithi099 ceph-mon[167076]: pgmap v270: 89 pgs: 20 active+clean+remapped, 46 active+clean, 1 active+recovering+undersized+remapped, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 31 MiB/s wr, 497 op/s; 31321/222249 objects degraded (14.093%); 22257/222249 objects misplaced (10.014%); 3.5 MiB/s, 59 objects/s recovering 2023-09-23T17:56:50.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4169961114' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T17:56:50.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:49 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T17:56:50.095 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:50.119 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:56:50.128 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:50.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:50.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:50.154 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:50.174 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:49 smithi055 ceph-mon[184174]: pgmap v270: 89 pgs: 20 active+clean+remapped, 46 active+clean, 1 active+recovering+undersized+remapped, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 31 MiB/s wr, 497 op/s; 31321/222249 objects degraded (14.093%); 22257/222249 objects misplaced (10.014%); 3.5 MiB/s, 59 objects/s recovering 2023-09-23T17:56:50.174 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4169961114' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T17:56:50.175 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:49 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T17:56:50.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:49 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T17:56:49.708+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e125 definitely_dead 0 2023-09-23T17:56:50.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:49 smithi055 ceph-mon[179668]: pgmap v270: 89 pgs: 20 active+clean+remapped, 46 active+clean, 1 active+recovering+undersized+remapped, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 31 MiB/s wr, 497 op/s; 31321/222249 objects degraded (14.093%); 22257/222249 objects misplaced (10.014%); 3.5 MiB/s, 59 objects/s recovering 2023-09-23T17:56:50.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4169961114' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T17:56:50.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:49 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T17:56:50.210 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T17:56:50.232 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:50.257 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:50.290 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:50.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:50.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:50.368 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 52 16 77363 77347 92.8975 18.0625 0.00432165 0.0106479 2023-09-23T17:56:50.394 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:50.423 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:50.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:56:50] "GET /metrics HTTP/1.1" 200 36519 "" "Prometheus/2.43.0" 2023-09-23T17:56:50.457 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:50.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:50.460 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:50.561 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:50.590 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:50.832 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:50.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:50.845 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:50.864 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:56:50.946 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:50.980 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:51.039 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:51.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:51.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:51.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:50 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T17:56:51.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:50 smithi099 ceph-mon[167076]: osdmap e126: 8 total, 7 up, 8 in 2023-09-23T17:56:51.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:50 smithi099 ceph-mon[167076]: failed to encode map e126 with expected crc 2023-09-23T17:56:51.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:50 smithi099 ceph-mon[167076]: failed to encode map e126 with expected crc 2023-09-23T17:56:51.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:50 smithi099 ceph-mon[167076]: failed to encode map e126 with expected crc 2023-09-23T17:56:51.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:51.172 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:51.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T17:56:51.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-mon[184174]: osdmap e126: 8 total, 7 up, 8 in 2023-09-23T17:56:51.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-mon[184174]: failed to encode map e126 with expected crc 2023-09-23T17:56:51.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-mon[184174]: failed to encode map e126 with expected crc 2023-09-23T17:56:51.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-mon[184174]: failed to encode map e126 with expected crc 2023-09-23T17:56:51.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T17:56:51.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-mon[179668]: osdmap e126: 8 total, 7 up, 8 in 2023-09-23T17:56:51.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-mon[179668]: failed to encode map e126 with expected crc 2023-09-23T17:56:51.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-mon[179668]: failed to encode map e126 with expected crc 2023-09-23T17:56:51.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:50 smithi055 ceph-mon[179668]: failed to encode map e126 with expected crc 2023-09-23T17:56:51.256 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:51.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:51.259 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:51.337 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:56:51.337 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:56:51.359 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:51.368 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 53 16 77415 77399 91.2071 3.25 0.00418455 0.0107696 2023-09-23T17:56:51.398 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:51.436 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:51.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:51.439 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:51.540 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:51.569 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:51.612 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:51.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:51.615 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:51.615 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:51.701 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T17:56:51.715 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:51.743 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:51.777 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:51.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:51.780 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:51.833 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T17:56:51.882 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:51.938 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: pgmap v272: 89 pgs: 3 remapped+peering, 2 peering, 18 active+clean+remapped, 45 active+clean, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 27268/229020 objects degraded (11.906%); 18143/229020 objects misplaced (7.922%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T17:56:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T17:56:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/969759363' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: pgmap v272: 89 pgs: 3 remapped+peering, 2 peering, 18 active+clean+remapped, 45 active+clean, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 27268/229020 objects degraded (11.906%); 18143/229020 objects misplaced (7.922%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:52.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/969759363' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: pgmap v272: 89 pgs: 3 remapped+peering, 2 peering, 18 active+clean+remapped, 45 active+clean, 11 active+undersized, 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 41 MiB/s wr, 650 op/s; 27268/229020 objects degraded (11.906%); 18143/229020 objects misplaced (7.922%); 4.2 MiB/s, 66 objects/s recovering 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: osdmap e127: 8 total, 7 up, 8 in 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: failed to encode map e127 with expected crc 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:56:52.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/969759363' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:56:52.166 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:52.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:52.173 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:52.274 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:52.347 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:56:52.371 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 54 16 77904 77888 90.0803 30.5625 0.0100813 0.0110959 2023-09-23T17:56:52.383 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:52.492 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:52.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:52.508 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:52.609 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:52.682 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:52.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:52.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: failed to encode map e128 with expected crc 2023-09-23T17:56:52.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/969759363' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:56:52.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T17:56:52.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: failed to encode map e128 with expected crc 2023-09-23T17:56:52.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: failed to encode map e128 with expected crc 2023-09-23T17:56:52.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: failed to encode map e128 with expected crc 2023-09-23T17:56:52.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: failed to encode map e128 with expected crc 2023-09-23T17:56:52.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: failed to encode map e128 with expected crc 2023-09-23T17:56:52.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: failed to encode map e128 with expected crc 2023-09-23T17:56:52.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: failed to encode map e128 with expected crc 2023-09-23T17:56:52.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: failed to encode map e128 with expected crc 2023-09-23T17:56:52.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 27268/229020 objects degraded (11.906%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:52.870 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:52.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:52.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:52.977 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:53.064 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:53.064 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.064 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/969759363' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:56:53.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T17:56:53.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:52 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 27268/229020 objects degraded (11.906%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:53.066 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:53.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:53.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/969759363' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: osdmap e128: 8 total, 7 up, 8 in 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: failed to encode map e128 with expected crc 2023-09-23T17:56:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:52 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 27268/229020 objects degraded (11.906%), 10 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:53.141 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:53.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:53.149 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:53.206 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:56:53.240 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:56:53.249 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:53.279 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:56:53.324 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:53.325 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:56:53.362 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:56:53.373 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 55 16 78904 78888 89.5759 62.5 0.0269224 0.0111578 2023-09-23T17:56:53.401 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:56:53.409 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:53.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:53.421 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:53.455 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:56:53.456 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:56:53.456 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:56:53.456 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:56:53.456 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T17:56:53.456 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:56:53.456 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:56:53.456 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:56:53.456 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T17:56:53.522 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:53.602 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:53.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:53.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:53.671 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:53.726 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:38884 5.7 5:e88075de:::benchmark_data_smithi055_229117_object38883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.244864+0000", 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 38.477417295999999, 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.111870281, 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.727 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 38884 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244864+0000", 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244864+0000", 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244864+0000", 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244895+0000", 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1819999999999997e-05 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244904+0000", 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.524e-06 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244909+0000", 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7299999999999996e-06 2023-09-23T17:56:53.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246498+0000", 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015898010000000001 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246614+0000", 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011574399999999999 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246739+0000", 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012482699999999999 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.249667+0000", 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002927972 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.249704+0000", 2023-09-23T17:56:53.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7271000000000003e-05 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356593+0000", 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.106889052 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356695+0000", 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000101992 2023-09-23T17:56:53.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356734+0000", 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9084000000000001e-05 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:38888 5.7 5:e84cc5a2:::benchmark_data_smithi055_229117_object38887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:56:53.734 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.248150+0000", 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 38.474131673000002, 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.11199930399999999, 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 38888 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248150+0000", 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248150+0000", 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248149+0000", 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:56:53.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248203+0000", 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3788000000000001e-05 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248213+0000", 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0548e-05 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248218+0000", 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7620000000000003e-06 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248262+0000", 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4316000000000001e-05 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248394+0000", 2023-09-23T17:56:53.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013165099999999999 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248541+0000", 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000147482 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.249786+0000", 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001244277 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.251395+0000", 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016097609999999999 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.360065+0000", 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.108669425 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.360114+0000", 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8878000000000002e-05 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.360149+0000", 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.54e-05 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.741 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:39725 5.7 5:ebd368fa:::benchmark_data_smithi055_229117_object39724:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.780472+0000", 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.941809695000003, 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.071684974999999998, 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 39725 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.742 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780472+0000", 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780472+0000", 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780470+0000", 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780499+0000", 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8663e-05 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780513+0000", 2023-09-23T17:56:53.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3675999999999999e-05 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780517+0000", 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.904e-06 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780574+0000", 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6681000000000002e-05 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780674+0000", 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100206 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780777+0000", 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010242300000000001 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.784071+0000", 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032940690000000002 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.785112+0000", 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010415660000000001 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.852068+0000", 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.066955485999999995 2023-09-23T17:56:53.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.852115+0000", 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7271999999999997e-05 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.852157+0000", 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1465999999999999e-05 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.748 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:39726 5.7 5:eba9343d:::benchmark_data_smithi055_229117_object39725:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.780526+0000", 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.941755389000001, 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.071728210000000001, 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.749 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 39726 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780526+0000", 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780526+0000", 2023-09-23T17:56:53.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780526+0000", 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780550+0000", 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4116000000000001e-05 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780563+0000", 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2923e-05 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780574+0000", 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1399999999999999e-05 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.781943+0000", 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013688680000000001 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.782064+0000", 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012139899999999999 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.782199+0000", 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013500299999999999 2023-09-23T17:56:53.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.784086+0000", 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018864940000000001 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.786289+0000", 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022031400000000001 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.852200+0000", 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.065911154999999999 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.852231+0000", 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0417999999999999e-05 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.852254+0000", 2023-09-23T17:56:53.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3651000000000001e-05 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:39729 5.7 5:e5c0694f:::benchmark_data_smithi055_229117_object39728:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e118)", 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.780580+0000", 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.941701004999999, 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.071767731000000001, 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.756 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 39729 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780580+0000", 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780580+0000", 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780580+0000", 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780603+0000", 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3088999999999999e-05 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780607+0000", 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.031e-06 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.780610+0000", 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.056e-06 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.782958+0000", 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002347876 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.783078+0000", 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119753 2023-09-23T17:56:53.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.783215+0000", 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000136747 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.784204+0000", 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00098930099999999994 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.786379+0000", 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021753139999999998 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.852286+0000", 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.065906919999999994 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.852324+0000", 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7632000000000002e-05 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.852348+0000", 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4408000000000001e-05 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:73290 5.e 5:7b1ff1f6:::benchmark_data_smithi055_229117_object73289:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:44.138834+0000", 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 9.5834470960000004, 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.039328847, 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 73290 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.764 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.138834+0000", 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.138834+0000", 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.138833+0000", 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.138864+0000", 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0862999999999999e-05 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.138879+0000", 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5153999999999999e-05 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.138888+0000", 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1840000000000005e-06 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.138936+0000", 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7568999999999998e-05 2023-09-23T17:56:53.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.139109+0000", 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017334199999999999 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.139329+0000", 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021975799999999999 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.143846+0000", 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045173189999999997 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.148023+0000", 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041766950000000002 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.175056+0000", 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.027033082999999999 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.175121+0000", 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5568000000000006e-05 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:44.178163+0000", 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030418099999999998 2023-09-23T17:56:53.770 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:74082 5.e 5:7a96f406:::benchmark_data_smithi055_229117_object74081:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:45.228307+0000", 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 8.4939741449999993, 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.056541363999999997, 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.771 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 74082 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.228307+0000", 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.228307+0000", 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.228306+0000", 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.228345+0000", 2023-09-23T17:56:53.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9048999999999998e-05 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.228359+0000", 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4012999999999999e-05 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.228365+0000", 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5389999999999998e-06 2023-09-23T17:56:53.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.228422+0000", 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7533e-05 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.228581+0000", 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015939400000000001 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.228696+0000", 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000114968 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.230668+0000", 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019716989999999999 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.238887+0000", 2023-09-23T17:56:53.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0082188509999999992 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.281804+0000", 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.042917314999999998 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.281882+0000", 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7488000000000001e-05 2023-09-23T17:56:53.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.284849+0000", 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029668469999999999 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:74093 5.e 5:76c0a2b1:::benchmark_data_smithi055_229117_object74092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:45.239391+0000", 2023-09-23T17:56:53.778 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 8.4828900709999999, 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.047983735, 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 74093 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.239391+0000", 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.239391+0000", 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.239390+0000", 2023-09-23T17:56:53.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.239418+0000", 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8288000000000001e-05 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.239431+0000", 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3342e-05 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.239437+0000", 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2989999999999996e-06 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.239486+0000", 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9048000000000002e-05 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.239680+0000", 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019377700000000001 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.239920+0000", 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00024075499999999999 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.242219+0000", 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022985340000000001 2023-09-23T17:56:53.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.243226+0000", 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010069829999999999 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.284910+0000", 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.041684640000000002 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.284979+0000", 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8644000000000002e-05 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.287375+0000", 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023959089999999999 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.785 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:74154 5.e 5:78fbcca3:::benchmark_data_smithi055_229117_object74153:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:45.344433+0000", 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 8.3778486599999997, 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.041973123000000001, 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 74154 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.786 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344433+0000", 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344433+0000", 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344432+0000", 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344456+0000", 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3716000000000001e-05 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344460+0000", 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1529999999999999e-06 2023-09-23T17:56:53.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344463+0000", 2023-09-23T17:56:53.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1810000000000001e-06 2023-09-23T17:56:53.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344501+0000", 2023-09-23T17:56:53.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7787000000000003e-05 2023-09-23T17:56:53.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344612+0000", 2023-09-23T17:56:53.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111261 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344740+0000", 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127167 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.348612+0000", 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0038721570000000002 2023-09-23T17:56:53.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.356124+0000", 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075117939999999996 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.384001+0000", 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.027877375999999999 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.384040+0000", 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8727999999999999e-05 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.386406+0000", 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023662319999999998 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:74158 5.e 5:71726b2b:::benchmark_data_smithi055_229117_object74157:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:45.344471+0000", 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 8.3778101330000005, 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.053686455000000001, 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 74158 2023-09-23T17:56:53.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344471+0000", 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344471+0000", 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344471+0000", 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344516+0000", 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4672999999999997e-05 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344522+0000", 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9889999999999997e-06 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.344525+0000", 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1300000000000001e-06 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.346598+0000", 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020735319999999999 2023-09-23T17:56:53.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.346696+0000", 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7602000000000005e-05 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.346794+0000", 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8172999999999997e-05 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.348752+0000", 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019581989999999999 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.356159+0000", 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0074065540000000001 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.395099+0000", 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.038939986000000003 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.395159+0000", 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0307999999999999e-05 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:45.398158+0000", 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029986399999999999 2023-09-23T17:56:53.800 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:74683 5.e 5:7e0e2e39:::benchmark_data_smithi055_229117_object74682:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:46.074803+0000", 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 7.6474780569999998, 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.039485685999999999, 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.801 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 74683 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.074803+0000", 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.074803+0000", 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.074801+0000", 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.074840+0000", 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8627999999999997e-05 2023-09-23T17:56:53.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.074853+0000", 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2413e-05 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.074859+0000", 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5899999999999996e-06 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.078997+0000", 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041383349999999999 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.079147+0000", 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014963300000000001 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T17:56:53.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.079275+0000", 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012807699999999999 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.082145+0000", 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028701529999999999 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.089135+0000", 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069898169999999997 2023-09-23T17:56:53.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.111196+0000", 2023-09-23T17:56:53.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022060913000000001 2023-09-23T17:56:53.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.111259+0000", 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2812999999999996e-05 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.114289+0000", 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003030175 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:75103 5.f 5:ff902250:::benchmark_data_smithi055_229117_object75102:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:46.586753+0000", 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 7.1355280959999998, 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.049479972999999997, 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 75103 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.811 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.586753+0000", 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.586753+0000", 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.586753+0000", 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.586786+0000", 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3169e-05 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.586796+0000", 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5759999999999995e-06 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.586800+0000", 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1749999999999998e-06 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.589568+0000", 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002768656 2023-09-23T17:56:53.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.589749+0000", 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018060999999999999 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.589882+0000", 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013319499999999999 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.592526+0000", 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026441590000000001 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.597348+0000", 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048213680000000004 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.634882+0000", 2023-09-23T17:56:53.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037533944 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.634955+0000", 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3232000000000006e-05 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.636233+0000", 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001278324 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.817 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:75106 5.f 5:f999bf27:::benchmark_data_smithi055_229117_object75105:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:46.594359+0000", 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 7.1279224790000004, 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.043384419, 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.818 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 75106 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.594359+0000", 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.594359+0000", 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.594357+0000", 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.594428+0000", 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0234999999999994e-05 2023-09-23T17:56:53.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.594441+0000", 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.376e-05 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.594448+0000", 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2419999999999997e-06 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.594493+0000", 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5469999999999997e-05 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.594669+0000", 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000176164 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.594844+0000", 2023-09-23T17:56:53.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000174358 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.597500+0000", 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026564230000000002 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.598167+0000", 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00066658600000000004 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.636292+0000", 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.038125379000000001 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.636357+0000", 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5061000000000006e-05 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.637743+0000", 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013863580000000001 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:75108 5.e 5:7fb194de:::benchmark_data_smithi055_229117_object75107:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:46.597873+0000", 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 7.1244079420000004, 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.051053765000000001, 2023-09-23T17:56:53.825 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 75108 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.597873+0000", 2023-09-23T17:56:53.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.597873+0000", 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.597872+0000", 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.597921+0000", 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8093999999999997e-05 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.597936+0000", 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5413e-05 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.597943+0000", 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7089999999999996e-06 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.598000+0000", 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7330000000000002e-05 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.598184+0000", 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000184305 2023-09-23T17:56:53.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.598391+0000", 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000206784 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.600744+0000", 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023527499999999998 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.604651+0000", 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039076579999999996 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.646620+0000", 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.041968498 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.831 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.646658+0000", 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8046000000000001e-05 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.648927+0000", 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022692070000000001 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:75114 5.e 5:7fb0320c:::benchmark_data_smithi055_229117_object75113:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.832 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:46.605406+0000", 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 7.1168753850000002, 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.045802046999999999, 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 75114 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.605406+0000", 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.605406+0000", 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.605404+0000", 2023-09-23T17:56:53.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.605466+0000", 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1928999999999996e-05 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.605482+0000", 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5423000000000001e-05 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.605488+0000", 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5060000000000001e-06 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.605539+0000", 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0685e-05 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.605731+0000", 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019177500000000001 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.605935+0000", 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000204475 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.608038+0000", 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021031029999999998 2023-09-23T17:56:53.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.612212+0000", 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041738110000000004 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.648981+0000", 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.036769090999999997 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.649020+0000", 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9026999999999998e-05 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.651208+0000", 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021877369999999999 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:75117 5.e 5:77300caf:::benchmark_data_smithi055_229117_object75116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e124)", 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:46.613126+0000", 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 7.1091553010000004, 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.039556828000000002, 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 75117 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.840 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.613126+0000", 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.613126+0000", 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.613125+0000", 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.613189+0000", 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3911000000000005e-05 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.613200+0000", 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.102e-05 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.613208+0000", 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1270000000000003e-06 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.613252+0000", 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3259e-05 2023-09-23T17:56:53.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.613352+0000", 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100011 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.613497+0000", 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014570800000000001 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.615509+0000", 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020111159999999999 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.628016+0000", 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012507781000000001 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.651251+0000", 2023-09-23T17:56:53.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.023234797000000001 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.651287+0000", 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5626000000000001e-05 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.652683+0000", 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001396179 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.846 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:75390 5.e 5:79d6b9f0:::benchmark_data_smithi055_229117_object75389:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:46.995602+0000", 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 6.7266792500000001, 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.040689143999999997, 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.847 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 75390 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.995602+0000", 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.995602+0000", 2023-09-23T17:56:53.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.995601+0000", 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.995652+0000", 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0549e-05 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.995659+0000", 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8310000000000003e-06 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:46.995668+0000", 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.516e-06 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.003252+0000", 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007583429 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.003401+0000", 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014900499999999999 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.003517+0000", 2023-09-23T17:56:53.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115968 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.007774+0000", 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0042572080000000002 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.016398+0000", 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0086238219999999997 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.033668+0000", 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017270548 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.033706+0000", 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7209999999999998e-05 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.036291+0000", 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025857020000000001 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:75579 5.f 5:fc32a912:::benchmark_data_smithi055_229117_object75578:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:47.257646+0000", 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 6.46463549, 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.044254302000000002, 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.854 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 75579 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.257646+0000", 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.257646+0000", 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.257646+0000", 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.257683+0000", 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7057000000000001e-05 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.257687+0000", 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8029999999999996e-06 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.257690+0000", 2023-09-23T17:56:53.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0489999999999999e-06 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.260819+0000", 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031289930000000001 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.261010+0000", 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019056499999999999 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.261198+0000", 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018786700000000001 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.263806+0000", 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002608572 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.265368+0000", 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015613280000000001 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.300281+0000", 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.034913548000000003 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.300345+0000", 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3351999999999999e-05 2023-09-23T17:56:53.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.301900+0000", 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015554779999999999 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:76042 5.f 5:fa89a53f:::benchmark_data_smithi055_229117_object76041:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:47.877879+0000", 2023-09-23T17:56:53.861 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 5.8444022410000001, 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.039962572000000002, 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 76042 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.877879+0000", 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.877879+0000", 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.877879+0000", 2023-09-23T17:56:53.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.877912+0000", 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3098000000000001e-05 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.877921+0000", 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3449999999999995e-06 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.877926+0000", 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.707e-06 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.884771+0000", 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0068450780000000001 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.884931+0000", 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016008099999999999 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.885067+0000", 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013562900000000001 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.887402+0000", 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002335387 2023-09-23T17:56:53.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.887942+0000", 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00054016700000000003 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.916401+0000", 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028459080000000001 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.916470+0000", 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9201000000000004e-05 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:47.917842+0000", 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013712119999999999 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:76117 5.f 5:ff7354a7:::benchmark_data_smithi055_229117_object76116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e125)", 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:48.001404+0000", 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 5.7208770629999997, 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.049401423, 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 76117 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:53.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.001404+0000", 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.001404+0000", 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.001404+0000", 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.001451+0000", 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7460999999999998e-05 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.001462+0000", 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0575e-05 2023-09-23T17:56:53.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.001474+0000", 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1993e-05 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.001539+0000", 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5209000000000005e-05 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.001678+0000", 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013928000000000001 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.001857+0000", 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000178977 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.003928+0000", 2023-09-23T17:56:53.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020706330000000001 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.004591+0000", 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00066307599999999996 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.049147+0000", 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.044556526999999999 2023-09-23T17:56:53.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.049185+0000", 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.748e-05 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:48.050806+0000", 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001620783 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.875 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.876 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.876 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:53.876 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:53.876 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:56:53.876 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:53.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:53 smithi055 ceph-mon[184174]: pgmap v275: 89 pgs: 4 remapped+peering, 4 peering, 12 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 48 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 22564/231684 objects degraded (9.739%); 6082/231684 objects misplaced (2.625%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T17:56:53.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/426507400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:53.890 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:53 smithi055 ceph-mon[179668]: pgmap v275: 89 pgs: 4 remapped+peering, 4 peering, 12 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 48 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 22564/231684 objects degraded (9.739%); 6082/231684 objects misplaced (2.625%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T17:56:53.890 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/426507400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:53.900 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:56:53.900 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:56:53.934 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:53.997 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:54.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:54.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:54.056 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:53 smithi099 ceph-mon[167076]: pgmap v275: 89 pgs: 4 remapped+peering, 4 peering, 12 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 48 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 46 MiB/s wr, 734 op/s; 22564/231684 objects degraded (9.739%); 6082/231684 objects misplaced (2.625%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T17:56:54.056 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/426507400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:54.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T17:56:54.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:54.143 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:54.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:54.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:54.191 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:54.275 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T17:56:54.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:54.321 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:54.354 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:54.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:54.357 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:54.373 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 56 16 79666 79650 88.8272 47.625 0.00270313 0.0111831 2023-09-23T17:56:54.458 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:54.487 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:54.519 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:54.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:54.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:54.622 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:54.653 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:54.685 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:54.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:54.688 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:54.714 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:56:54.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:54.815 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:54.848 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:54.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:54.851 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:54.875 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:54.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:54 smithi055 ceph-mon[179668]: osdmap e129: 8 total, 7 up, 8 in 2023-09-23T17:56:54.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:54 smithi055 ceph-mon[179668]: failed to encode map e129 with expected crc 2023-09-23T17:56:54.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:54 smithi055 ceph-mon[179668]: failed to encode map e129 with expected crc 2023-09-23T17:56:54.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:54 smithi055 ceph-mon[179668]: failed to encode map e129 with expected crc 2023-09-23T17:56:54.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:54 smithi055 ceph-mon[179668]: failed to encode map e129 with expected crc 2023-09-23T17:56:54.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:54 smithi055 ceph-mon[184174]: osdmap e129: 8 total, 7 up, 8 in 2023-09-23T17:56:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:54 smithi055 ceph-mon[184174]: failed to encode map e129 with expected crc 2023-09-23T17:56:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:54 smithi055 ceph-mon[184174]: failed to encode map e129 with expected crc 2023-09-23T17:56:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:54 smithi055 ceph-mon[184174]: failed to encode map e129 with expected crc 2023-09-23T17:56:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:54 smithi055 ceph-mon[184174]: failed to encode map e129 with expected crc 2023-09-23T17:56:54.952 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:55.021 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:55.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:54 smithi099 ceph-mon[167076]: osdmap e129: 8 total, 7 up, 8 in 2023-09-23T17:56:55.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:54 smithi099 ceph-mon[167076]: failed to encode map e129 with expected crc 2023-09-23T17:56:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:54 smithi099 ceph-mon[167076]: failed to encode map e129 with expected crc 2023-09-23T17:56:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:54 smithi099 ceph-mon[167076]: failed to encode map e129 with expected crc 2023-09-23T17:56:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:54 smithi099 ceph-mon[167076]: failed to encode map e129 with expected crc 2023-09-23T17:56:55.142 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:55.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:55.152 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:55.210 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T17:56:55.211 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:56:55.211 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T17:56:55.211 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:56:55.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:55.297 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:55.378 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 57 16 80145 80129 87.7885 29.9375 0.00811364 0.0113839 2023-09-23T17:56:55.432 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:55.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:55.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:55.539 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:55.612 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:55.746 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:55.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:55.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:55.854 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:55 smithi055 ceph-mon[179668]: pgmap v277: 89 pgs: 4 remapped+peering, 4 peering, 11 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 49 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 35 MiB/s wr, 558 op/s; 18846/235521 objects degraded (8.002%); 1318/235521 objects misplaced (0.560%); 3.7 MiB/s, 60 objects/s recovering 2023-09-23T17:56:55.883 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:55 smithi055 ceph-mon[179668]: failed to encode map e130 with expected crc 2023-09-23T17:56:55.883 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:55 smithi055 ceph-mon[179668]: failed to encode map e130 with expected crc 2023-09-23T17:56:55.883 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:55 smithi055 ceph-mon[179668]: osdmap e130: 8 total, 7 up, 8 in 2023-09-23T17:56:55.883 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:55 smithi055 ceph-mon[179668]: failed to encode map e130 with expected crc 2023-09-23T17:56:55.883 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:55.901 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:56:55.942 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:56.011 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:56:56.011 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T17:56:56.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.011 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80698 5.9 5:91ad1aec:::benchmark_data_smithi055_229117_object80697:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:56:56.011 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:56.007325+0000", 2023-09-23T17:56:56.011 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0032251329999999998, 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032699349999999999, 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80698 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:56.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007325+0000", 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007325+0000", 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007323+0000", 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T17:56:56.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007374+0000", 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0691e-05 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007391+0000", 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7238e-05 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007399+0000", 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3309999999999998e-06 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007448+0000", 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9152e-05 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007604+0000", 2023-09-23T17:56:56.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015652400000000001 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007764+0000", 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015961799999999999 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.010125+0000", 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023606949999999999 2023-09-23T17:56:56.016 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80701 5.4 5:2002f1a4:::benchmark_data_smithi055_229117_object80700:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:56.007411+0000", 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0031393160000000001, 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033091079999999998, 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:56:56.017 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80701 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007411+0000", 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007411+0000", 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007411+0000", 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007448+0000", 2023-09-23T17:56:56.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7542000000000001e-05 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007466+0000", 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.738e-05 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007472+0000", 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0480000000000004e-06 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007531+0000", 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9058000000000003e-05 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007668+0000", 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013706699999999999 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,5", 2023-09-23T17:56:56.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.007813+0000", 2023-09-23T17:56:56.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014546900000000001 2023-09-23T17:56:56.022 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:56.022 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:56:56.022 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:56.022 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:56:56.022 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T17:56:56.022 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 2 2023-09-23T17:56:56.022 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:56:56.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:56.023 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:56.024 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:56.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:55 smithi099 ceph-mon[167076]: pgmap v277: 89 pgs: 4 remapped+peering, 4 peering, 11 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 49 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 35 MiB/s wr, 558 op/s; 18846/235521 objects degraded (8.002%); 1318/235521 objects misplaced (0.560%); 3.7 MiB/s, 60 objects/s recovering 2023-09-23T17:56:56.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:55 smithi099 ceph-mon[167076]: failed to encode map e130 with expected crc 2023-09-23T17:56:56.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:55 smithi099 ceph-mon[167076]: failed to encode map e130 with expected crc 2023-09-23T17:56:56.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:55 smithi099 ceph-mon[167076]: osdmap e130: 8 total, 7 up, 8 in 2023-09-23T17:56:56.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:55 smithi099 ceph-mon[167076]: failed to encode map e130 with expected crc 2023-09-23T17:56:56.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:55 smithi055 ceph-mon[184174]: pgmap v277: 89 pgs: 4 remapped+peering, 4 peering, 11 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 49 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.8 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 35 MiB/s wr, 558 op/s; 18846/235521 objects degraded (8.002%); 1318/235521 objects misplaced (0.560%); 3.7 MiB/s, 60 objects/s recovering 2023-09-23T17:56:56.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:55 smithi055 ceph-mon[184174]: failed to encode map e130 with expected crc 2023-09-23T17:56:56.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:55 smithi055 ceph-mon[184174]: failed to encode map e130 with expected crc 2023-09-23T17:56:56.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:55 smithi055 ceph-mon[184174]: osdmap e130: 8 total, 7 up, 8 in 2023-09-23T17:56:56.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:55 smithi055 ceph-mon[184174]: failed to encode map e130 with expected crc 2023-09-23T17:56:56.120 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:56.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:56.159 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:56.328 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:56.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:56.331 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:56.377 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 58 16 80866 80850 87.0523 45.0625 0.00291124 0.0114213 2023-09-23T17:56:56.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T17:56:56.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:56.468 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:56.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:56.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:56.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:56.614 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:56.649 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:56.901 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T17:56:56.907 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:56.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:56.910 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[179668]: failed to encode map e131 with expected crc 2023-09-23T17:56:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[179668]: osdmap e131: 8 total, 7 up, 8 in 2023-09-23T17:56:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[179668]: failed to encode map e131 with expected crc 2023-09-23T17:56:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[179668]: failed to encode map e131 with expected crc 2023-09-23T17:56:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[179668]: failed to encode map e131 with expected crc 2023-09-23T17:56:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4049727284' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:56.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[184174]: failed to encode map e131 with expected crc 2023-09-23T17:56:56.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[184174]: osdmap e131: 8 total, 7 up, 8 in 2023-09-23T17:56:56.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[184174]: failed to encode map e131 with expected crc 2023-09-23T17:56:56.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[184174]: failed to encode map e131 with expected crc 2023-09-23T17:56:56.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[184174]: failed to encode map e131 with expected crc 2023-09-23T17:56:56.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4049727284' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:56.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:56 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:56.977 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:57.011 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:56:57.024 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:56:57.117 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:57.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:56 smithi099 ceph-mon[167076]: failed to encode map e131 with expected crc 2023-09-23T17:56:57.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:56 smithi099 ceph-mon[167076]: osdmap e131: 8 total, 7 up, 8 in 2023-09-23T17:56:57.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:56 smithi099 ceph-mon[167076]: failed to encode map e131 with expected crc 2023-09-23T17:56:57.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:56 smithi099 ceph-mon[167076]: failed to encode map e131 with expected crc 2023-09-23T17:56:57.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:56 smithi099 ceph-mon[167076]: failed to encode map e131 with expected crc 2023-09-23T17:56:57.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4049727284' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:57.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:56 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:56:57.377 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 59 16 81281 81265 86.0172 25.9375 0.0104371 0.0116207 2023-09-23T17:56:57.855 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:56:57.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:57.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:56:57.961 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:56:58.002 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:58.040 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:56:58.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:58.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:56:58.145 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:58.172 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:57 smithi099 ceph-mon[167076]: pgmap v280: 89 pgs: 1 remapped+peering, 12 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean, 11 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 32 MiB/s wr, 516 op/s; 19270/236655 objects degraded (8.143%); 1318/236655 objects misplaced (0.557%); 1.6 MiB/s, 27 objects/s recovering 2023-09-23T17:56:58.172 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:57 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:58.172 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:57 smithi099 ceph-mon[167076]: failed to encode map e132 with expected crc 2023-09-23T17:56:58.172 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:57 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:58.172 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:57 smithi099 ceph-mon[167076]: osdmap e132: 8 total, 7 up, 8 in 2023-09-23T17:56:58.172 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:57 smithi099 ceph-mon[167076]: failed to encode map e132 with expected crc 2023-09-23T17:56:58.173 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:57 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19270/236655 objects degraded (8.143%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:58.173 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:57 smithi099 ceph-mon[167076]: osdmap e133: 8 total, 7 up, 8 in 2023-09-23T17:56:58.173 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:57 smithi099 ceph-mon[167076]: failed to encode map e133 with expected crc 2023-09-23T17:56:58.173 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:57 smithi099 ceph-mon[167076]: failed to encode map e133 with expected crc 2023-09-23T17:56:58.175 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[179668]: pgmap v280: 89 pgs: 1 remapped+peering, 12 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean, 11 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 32 MiB/s wr, 516 op/s; 19270/236655 objects degraded (8.143%); 1318/236655 objects misplaced (0.557%); 1.6 MiB/s, 27 objects/s recovering 2023-09-23T17:56:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[179668]: failed to encode map e132 with expected crc 2023-09-23T17:56:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[179668]: osdmap e132: 8 total, 7 up, 8 in 2023-09-23T17:56:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[179668]: failed to encode map e132 with expected crc 2023-09-23T17:56:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19270/236655 objects degraded (8.143%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[179668]: osdmap e133: 8 total, 7 up, 8 in 2023-09-23T17:56:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[179668]: failed to encode map e133 with expected crc 2023-09-23T17:56:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[179668]: failed to encode map e133 with expected crc 2023-09-23T17:56:58.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[184174]: pgmap v280: 89 pgs: 1 remapped+peering, 12 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 54 active+clean, 11 active+undersized, 3 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 32 MiB/s wr, 516 op/s; 19270/236655 objects degraded (8.143%); 1318/236655 objects misplaced (0.557%); 1.6 MiB/s, 27 objects/s recovering 2023-09-23T17:56:58.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:56:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[184174]: failed to encode map e132 with expected crc 2023-09-23T17:56:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:56:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[184174]: osdmap e132: 8 total, 7 up, 8 in 2023-09-23T17:56:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[184174]: failed to encode map e132 with expected crc 2023-09-23T17:56:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19270/236655 objects degraded (8.143%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T17:56:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[184174]: osdmap e133: 8 total, 7 up, 8 in 2023-09-23T17:56:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[184174]: failed to encode map e133 with expected crc 2023-09-23T17:56:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:57 smithi055 ceph-mon[184174]: failed to encode map e133 with expected crc 2023-09-23T17:56:58.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:58.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:58.267 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:58.368 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:58.377 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T17:56:58.378696+0000 min lat: 0.00103404 max lat: 2.36625 avg lat: 0.0116386 2023-09-23T17:56:58.377 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T17:56:58.378 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 60 16 81870 81854 85.1976 36.8125 0.00310495 0.0116386 2023-09-23T17:56:58.395 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:58.420 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:56:58.421 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:56:58.434 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:58.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:58.437 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:58.461 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:56:58.461 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T17:56:58.461 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T17:56:58.461 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T17:56:58.461 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:56:58.538 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:56:58.567 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:58.654 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:56:58.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:58.657 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:56:58.758 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:58.788 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:58.826 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T17:56:58.859 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":282,"stamp":"2023-09-23T17:56:57.773577+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5259335603,"num_objects":80206,"num_object_clones":0,"num_object_copies":240618,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19947,"num_objects_misplaced":671,"num_objects_unfound":0,"num_objects_dirty":80206,"num_whiteouts":0,"num_read":6494,"num_read_kb":13112,"num_write":81324,"num_write_kb":5146533,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1605,"num_bytes_recovered":104466240,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81370,"ondisk_log_size":81370,"up":236,"acting":248,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":282,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14807292,"kb_used_data":14332796,"kb_used_omap":29,"kb_used_meta":474402,"kb_avail":735153924,"statfs":{"total":767960285184,"available":752797618176,"internally_reserved":0,"allocated":14676783104,"data_stored":14672138218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30208,"internal_metadata":485788160},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,4,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":144244736,"num_objects":2201,"num_object_clones":0,"num_object_copies":6603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1449,"num_objects_misplaced":-5411,"num_objects_unfound":0,"num_objects_dirty":2201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2201,"num_write_kb":140864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":841,"num_bytes_recovered":54984704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"3.932614"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":15,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.881843+0000","last_change":"2023-09-23T17:56:50.838338+0000","last_active":"2023-09-23T17:56:55.881843+0000","last_peered":"2023-09-23T17:56:55.881843+0000","last_clean":"2023-09-23T17:56:55.881843+0000","last_became_active":"2023-09-23T17:56:50.837963+0000","last_became_peered":"2023-09-23T17:56:50.837963+0000","last_unstale":"2023-09-23T17:56:55.881843+0000","last_undegraded":"2023-09-23T17:56:55.881843+0000","last_fullsized":"2023-09-23T17:56:55.881843+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:36.238235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":111,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.843156+0000","last_change":"2023-09-23T17:56:50.828800+0000","last_active":"2023-09-23T17:56:53.843156+0000","last_peered":"2023-09-23T17:56:53.843156+0000","last_clean":"2023-09-23T17:56:53.843156+0000","last_became_active":"2023-09-23T17:56:50.828540+0000","last_became_peered":"2023-09-23T17:56:50.828540+0000","last_unstale":"2023-09-23T17:56:53.843156+0000","last_undegraded":"2023-09-23T17:56:53.843156+0000","last_fullsized":"2023-09-23T17:56:53.843156+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:36.268257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":87,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.849510+0000","last_change":"2023-09-23T17:56:50.828650+0000","last_active":"2023-09-23T17:56:53.849510+0000","last_peered":"2023-09-23T17:56:53.849510+0000","last_clean":"2023-09-23T17:56:53.849510+0000","last_became_active":"2023-09-23T17:56:50.828313+0000","last_became_peered":"2023-09-23T17:56:50.828313+0000","last_unstale":"2023-09-23T17:56:53.849510+0000","last_undegraded":"2023-09-23T17:56:53.849510+0000","last_fullsized":"2023-09-23T17:56:53.849510+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:44:51.900293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":72,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.830238+0000","last_change":"2023-09-23T17:54:57.039438+0000","last_active":"2023-09-23T17:56:51.830238+0000","last_peered":"2023-09-23T17:56:51.830238+0000","last_clean":"2023-09-23T17:56:51.830238+0000","last_became_active":"2023-09-23T17:54:57.039163+0000","last_became_peered":"2023-09-23T17:54:57.039163+0000","last_unstale":"2023-09-23T17:56:51.830238+0000","last_undegraded":"2023-09-23T17:56:51.830238+0000","last_fullsized":"2023-09-23T17:56:51.830238+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:51.265833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":110,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.844091+0000","last_change":"2023-09-23T17:54:57.031226+0000","last_active":"2023-09-23T17:56:53.844091+0000","last_peered":"2023-09-23T17:56:53.844091+0000","last_clean":"2023-09-23T17:56:53.844091+0000","last_became_active":"2023-09-23T17:54:57.030880+0000","last_became_peered":"2023-09-23T17:54:57.030880+0000","last_unstale":"2023-09-23T17:56:53.844091+0000","last_undegraded":"2023-09-23T17:56:53.844091+0000","last_fullsized":"2023-09-23T17:56:53.844091+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:49.836129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":78,"reported_epoch":131,"state":"active+undersized","last_fresh":"2023-09-23T17:56:55.876472+0000","last_change":"2023-09-23T17:55:58.864932+0000","last_active":"2023-09-23T17:56:55.876472+0000","last_peered":"2023-09-23T17:56:55.876472+0000","last_clean":"2023-09-23T17:53:21.027423+0000","last_became_active":"2023-09-23T17:55:58.864932+0000","last_became_peered":"2023-09-23T17:55:58.864932+0000","last_unstale":"2023-09-23T17:56:55.876472+0000","last_undegraded":"2023-09-23T17:56:55.876472+0000","last_fullsized":"2023-09-23T17:55:58.862677+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:37.192868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":71,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.830401+0000","last_change":"2023-09-23T17:54:57.036930+0000","last_active":"2023-09-23T17:56:51.830401+0000","last_peered":"2023-09-23T17:56:51.830401+0000","last_clean":"2023-09-23T17:56:51.830401+0000","last_became_active":"2023-09-23T17:54:57.036631+0000","last_became_peered":"2023-09-23T17:54:57.036631+0000","last_unstale":"2023-09-23T17:56:51.830401+0000","last_undegraded":"2023-09-23T17:56:51.830401+0000","last_fullsized":"2023-09-23T17:56:51.830401+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:08.367357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":98,"reported_epoch":129,"state":"active+undersized","last_fresh":"2023-09-23T17:56:53.843981+0000","last_change":"2023-09-23T17:55:58.862654+0000","last_active":"2023-09-23T17:56:53.843981+0000","last_peered":"2023-09-23T17:56:53.843981+0000","last_clean":"2023-09-23T17:55:56.840611+0000","last_became_active":"2023-09-23T17:55:58.862654+0000","last_became_peered":"2023-09-23T17:55:58.862654+0000","last_unstale":"2023-09-23T17:56:53.843981+0000","last_undegraded":"2023-09-23T17:56:53.843981+0000","last_fullsized":"2023-09-23T17:55:58.855351+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:48:08.275429+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":92,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.877270+0000","last_change":"2023-09-23T17:53:52.732689+0000","last_active":"2023-09-23T17:56:55.877270+0000","last_peered":"2023-09-23T17:56:55.877270+0000","last_clean":"2023-09-23T17:56:55.877270+0000","last_became_active":"2023-09-23T17:53:52.732382+0000","last_became_peered":"2023-09-23T17:53:52.732382+0000","last_unstale":"2023-09-23T17:56:55.877270+0000","last_undegraded":"2023-09-23T17:56:55.877270+0000","last_fullsized":"2023-09-23T17:56:55.877270+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:34.408802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":50,"reported_epoch":129,"state":"active+undersized","last_fresh":"2023-09-23T17:56:53.845372+0000","last_change":"2023-09-23T17:55:58.861640+0000","last_active":"2023-09-23T17:56:53.845372+0000","last_peered":"2023-09-23T17:56:53.845372+0000","last_clean":"2023-09-23T17:55:56.840970+0000","last_became_active":"2023-09-23T17:55:58.861640+0000","last_became_peered":"2023-09-23T17:55:58.861640+0000","last_unstale":"2023-09-23T17:56:53.845372+0000","last_undegraded":"2023-09-23T17:56:53.845372+0000","last_fullsized":"2023-09-23T17:55:58.856676+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:53:24.404170+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":129,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.891030+0000","last_change":"2023-09-23T17:56:50.855131+0000","last_active":"2023-09-23T17:56:55.891030+0000","last_peered":"2023-09-23T17:56:55.891030+0000","last_clean":"2023-09-23T17:56:55.891030+0000","last_became_active":"2023-09-23T17:56:50.822827+0000","last_became_peered":"2023-09-23T17:56:50.822827+0000","last_unstale":"2023-09-23T17:56:55.891030+0000","last_undegraded":"2023-09-23T17:56:55.891030+0000","last_fullsized":"2023-09-23T17:56:55.891030+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:39:13.050293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":75,"reported_epoch":129,"state":"active+undersized","last_fresh":"2023-09-23T17:56:53.847611+0000","last_change":"2023-09-23T17:55:58.865485+0000","last_active":"2023-09-23T17:56:53.847611+0000","last_peered":"2023-09-23T17:56:53.847611+0000","last_clean":"2023-09-23T17:55:57.820806+0000","last_became_active":"2023-09-23T17:55:58.865485+0000","last_became_peered":"2023-09-23T17:55:58.865485+0000","last_unstale":"2023-09-23T17:56:53.847611+0000","last_undegraded":"2023-09-23T17:56:53.847611+0000","last_fullsized":"2023-09-23T17:55:58.859630+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:40.150526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":110,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.886750+0000","last_change":"2023-09-23T17:56:51.845979+0000","last_active":"2023-09-23T17:56:55.886750+0000","last_peered":"2023-09-23T17:56:55.886750+0000","last_clean":"2023-09-23T17:56:55.886750+0000","last_became_active":"2023-09-23T17:56:51.844991+0000","last_became_peered":"2023-09-23T17:56:51.844991+0000","last_unstale":"2023-09-23T17:56:55.886750+0000","last_undegraded":"2023-09-23T17:56:55.886750+0000","last_fullsized":"2023-09-23T17:56:55.886750+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:12.346538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":104,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.882164+0000","last_change":"2023-09-23T17:56:51.839104+0000","last_active":"2023-09-23T17:56:55.882164+0000","last_peered":"2023-09-23T17:56:55.882164+0000","last_clean":"2023-09-23T17:56:55.882164+0000","last_became_active":"2023-09-23T17:56:51.838692+0000","last_became_peered":"2023-09-23T17:56:51.838692+0000","last_unstale":"2023-09-23T17:56:55.882164+0000","last_undegraded":"2023-09-23T17:56:55.882164+0000","last_fullsized":"2023-09-23T17:56:55.882164+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:47.909612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,5],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":120,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.891470+0000","last_change":"2023-09-23T17:56:51.852184+0000","last_active":"2023-09-23T17:56:55.891470+0000","last_peered":"2023-09-23T17:56:55.891470+0000","last_clean":"2023-09-23T17:56:55.891470+0000","last_became_active":"2023-09-23T17:56:51.851861+0000","last_became_peered":"2023-09-23T17:56:51.851861+0000","last_unstale":"2023-09-23T17:56:55.891470+0000","last_undegraded":"2023-09-23T17:56:55.891470+0000","last_fullsized":"2023-09-23T17:56:55.891470+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:56.294669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":92,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.842969+0000","last_change":"2023-09-23T17:54:57.040422+0000","last_active":"2023-09-23T17:56:53.842969+0000","last_peered":"2023-09-23T17:56:53.842969+0000","last_clean":"2023-09-23T17:56:53.842969+0000","last_became_active":"2023-09-23T17:54:57.040085+0000","last_became_peered":"2023-09-23T17:54:57.040085+0000","last_unstale":"2023-09-23T17:56:53.842969+0000","last_undegraded":"2023-09-23T17:56:53.842969+0000","last_fullsized":"2023-09-23T17:56:53.842969+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:15:47.846454+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":71,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.830009+0000","last_change":"2023-09-23T17:54:57.038461+0000","last_active":"2023-09-23T17:56:51.830009+0000","last_peered":"2023-09-23T17:56:51.830009+0000","last_clean":"2023-09-23T17:56:51.830009+0000","last_became_active":"2023-09-23T17:54:57.038210+0000","last_became_peered":"2023-09-23T17:54:57.038210+0000","last_unstale":"2023-09-23T17:56:51.830009+0000","last_undegraded":"2023-09-23T17:56:51.830009+0000","last_fullsized":"2023-09-23T17:56:51.830009+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:01:50.077733+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":91,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.876074+0000","last_change":"2023-09-23T17:54:57.042234+0000","last_active":"2023-09-23T17:56:55.876074+0000","last_peered":"2023-09-23T17:56:55.876074+0000","last_clean":"2023-09-23T17:56:55.876074+0000","last_became_active":"2023-09-23T17:54:57.041918+0000","last_became_peered":"2023-09-23T17:54:57.041918+0000","last_unstale":"2023-09-23T17:56:55.876074+0000","last_undegraded":"2023-09-23T17:56:55.876074+0000","last_fullsized":"2023-09-23T17:56:55.876074+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:44.128871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":108,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.893307+0000","last_change":"2023-09-23T17:56:51.852722+0000","last_active":"2023-09-23T17:56:55.893307+0000","last_peered":"2023-09-23T17:56:55.893307+0000","last_clean":"2023-09-23T17:56:55.893307+0000","last_became_active":"2023-09-23T17:56:51.851886+0000","last_became_peered":"2023-09-23T17:56:51.851886+0000","last_unstale":"2023-09-23T17:56:55.893307+0000","last_undegraded":"2023-09-23T17:56:55.893307+0000","last_fullsized":"2023-09-23T17:56:55.893307+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:09.509642+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4,0],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":72,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.830149+0000","last_change":"2023-09-23T17:54:57.039517+0000","last_active":"2023-09-23T17:56:51.830149+0000","last_peered":"2023-09-23T17:56:51.830149+0000","last_clean":"2023-09-23T17:56:51.830149+0000","last_became_active":"2023-09-23T17:54:57.039236+0000","last_became_peered":"2023-09-23T17:54:57.039236+0000","last_unstale":"2023-09-23T17:56:51.830149+0000","last_undegraded":"2023-09-23T17:56:51.830149+0000","last_fullsized":"2023-09-23T17:56:51.830149+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:59.126485+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":34,"reported_epoch":128,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:51.829404+0000","last_change":"2023-09-23T17:55:58.867766+0000","last_active":"2023-09-23T17:56:51.829404+0000","last_peered":"2023-09-23T17:56:51.829404+0000","last_clean":"2023-09-23T17:45:53.616299+0000","last_became_active":"2023-09-23T17:55:58.867766+0000","last_became_peered":"2023-09-23T17:55:58.867766+0000","last_unstale":"2023-09-23T17:56:51.829404+0000","last_undegraded":"2023-09-23T17:55:58.863587+0000","last_fullsized":"2023-09-23T17:55:58.863517+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:36.202414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":92,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.843695+0000","last_change":"2023-09-23T17:54:57.041584+0000","last_active":"2023-09-23T17:56:53.843695+0000","last_peered":"2023-09-23T17:56:53.843695+0000","last_clean":"2023-09-23T17:56:53.843695+0000","last_became_active":"2023-09-23T17:54:57.040808+0000","last_became_peered":"2023-09-23T17:54:57.040808+0000","last_unstale":"2023-09-23T17:56:53.843695+0000","last_undegraded":"2023-09-23T17:56:53.843695+0000","last_fullsized":"2023-09-23T17:56:53.843695+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:01.079083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"132'4967","reported_seq":5891,"reported_epoch":132,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T17:56:57.355649+0000","last_change":"2023-09-23T17:56:57.332631+0000","last_active":"2023-09-23T17:56:57.355649+0000","last_peered":"2023-09-23T17:56:57.355649+0000","last_clean":"2023-09-23T17:55:57.822204+0000","last_became_active":"2023-09-23T17:56:51.856958+0000","last_became_peered":"2023-09-23T17:56:51.856958+0000","last_unstale":"2023-09-23T17:56:57.355649+0000","last_undegraded":"2023-09-23T17:56:51.847406+0000","last_fullsized":"2023-09-23T17:56:57.355649+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4967,"log_dups_size":0,"ondisk_log_size":4967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:26.500827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325517312,"num_objects":4967,"num_object_clones":0,"num_object_copies":14901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4967,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4967,"num_write_kb":317888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":433,"num_bytes_recovered":28377088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4,1],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"1,4,7","objects":1092},{"shards":"4,7","objects":3875}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":123,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.891298+0000","last_change":"2023-09-23T17:56:50.839197+0000","last_active":"2023-09-23T17:56:55.891298+0000","last_peered":"2023-09-23T17:56:55.891298+0000","last_clean":"2023-09-23T17:56:55.891298+0000","last_became_active":"2023-09-23T17:56:50.820626+0000","last_became_peered":"2023-09-23T17:56:50.820626+0000","last_unstale":"2023-09-23T17:56:55.891298+0000","last_undegraded":"2023-09-23T17:56:55.891298+0000","last_fullsized":"2023-09-23T17:56:55.891298+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:20.568542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":81,"reported_epoch":131,"state":"active+undersized","last_fresh":"2023-09-23T17:56:55.875945+0000","last_change":"2023-09-23T17:55:58.870416+0000","last_active":"2023-09-23T17:56:55.875945+0000","last_peered":"2023-09-23T17:56:55.875945+0000","last_clean":"2023-09-23T17:55:56.845347+0000","last_became_active":"2023-09-23T17:55:58.870416+0000","last_became_peered":"2023-09-23T17:55:58.870416+0000","last_unstale":"2023-09-23T17:56:55.875945+0000","last_undegraded":"2023-09-23T17:56:55.875945+0000","last_fullsized":"2023-09-23T17:55:58.862810+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:13.418266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"128'5000","reported_seq":5027,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.655384+0000","last_change":"2023-09-23T17:55:53.818806+0000","last_active":"2023-09-23T17:56:53.655384+0000","last_peered":"2023-09-23T17:56:53.655384+0000","last_clean":"2023-09-23T17:56:53.655384+0000","last_became_active":"2023-09-23T17:55:53.818291+0000","last_became_peered":"2023-09-23T17:55:53.818291+0000","last_unstale":"2023-09-23T17:56:53.655384+0000","last_undegraded":"2023-09-23T17:56:53.655384+0000","last_fullsized":"2023-09-23T17:56:53.655384+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5000,"log_dups_size":0,"ondisk_log_size":5000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:58.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680000,"num_objects":5000,"num_object_clones":0,"num_object_copies":15000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5000,"num_write_kb":320000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":124,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.891708+0000","last_change":"2023-09-23T17:56:50.861256+0000","last_active":"2023-09-23T17:56:55.891708+0000","last_peered":"2023-09-23T17:56:55.891708+0000","last_clean":"2023-09-23T17:56:55.891708+0000","last_became_active":"2023-09-23T17:56:50.823713+0000","last_became_peered":"2023-09-23T17:56:50.823713+0000","last_unstale":"2023-09-23T17:56:55.891708+0000","last_undegraded":"2023-09-23T17:56:55.891708+0000","last_fullsized":"2023-09-23T17:56:55.891708+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:00.463390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":114,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.887756+0000","last_change":"2023-09-23T17:56:51.835201+0000","last_active":"2023-09-23T17:56:55.887756+0000","last_peered":"2023-09-23T17:56:55.887756+0000","last_clean":"2023-09-23T17:56:55.887756+0000","last_became_active":"2023-09-23T17:56:51.834904+0000","last_became_peered":"2023-09-23T17:56:51.834904+0000","last_unstale":"2023-09-23T17:56:55.887756+0000","last_undegraded":"2023-09-23T17:56:55.887756+0000","last_fullsized":"2023-09-23T17:56:55.887756+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:52.141622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"128'4922","reported_seq":4949,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.536056+0000","last_change":"2023-09-23T17:55:53.829853+0000","last_active":"2023-09-23T17:56:53.536056+0000","last_peered":"2023-09-23T17:56:53.536056+0000","last_clean":"2023-09-23T17:56:53.536056+0000","last_became_active":"2023-09-23T17:55:53.829573+0000","last_became_peered":"2023-09-23T17:55:53.829573+0000","last_unstale":"2023-09-23T17:56:53.536056+0000","last_undegraded":"2023-09-23T17:56:53.536056+0000","last_fullsized":"2023-09-23T17:56:53.536056+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4922,"log_dups_size":0,"ondisk_log_size":4922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:39.218789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322568192,"num_objects":4922,"num_object_clones":0,"num_object_copies":14766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4922,"num_write_kb":315008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":85,"reported_epoch":131,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:55.891239+0000","last_change":"2023-09-23T17:55:58.870420+0000","last_active":"2023-09-23T17:56:55.891239+0000","last_peered":"2023-09-23T17:56:55.891239+0000","last_clean":"2023-09-23T17:55:57.822744+0000","last_became_active":"2023-09-23T17:55:58.870420+0000","last_became_peered":"2023-09-23T17:55:58.870420+0000","last_unstale":"2023-09-23T17:56:55.891239+0000","last_undegraded":"2023-09-23T17:55:58.862862+0000","last_fullsized":"2023-09-23T17:55:58.862757+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:31.925750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":88,"reported_epoch":127,"state":"active+undersized","last_fresh":"2023-09-23T17:56:50.833451+0000","last_change":"2023-09-23T17:55:58.864525+0000","last_active":"2023-09-23T17:56:50.833451+0000","last_peered":"2023-09-23T17:56:50.833451+0000","last_clean":"2023-09-23T17:53:52.724466+0000","last_became_active":"2023-09-23T17:55:58.864525+0000","last_became_peered":"2023-09-23T17:55:58.864525+0000","last_unstale":"2023-09-23T17:56:50.833451+0000","last_undegraded":"2023-09-23T17:56:50.833451+0000","last_fullsized":"2023-09-23T17:55:58.859288+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:31.785041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.a","version":"128'4959","reported_seq":5006,"reported_epoch":128,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T17:56:53.647289+0000","last_change":"2023-09-23T17:56:50.901590+0000","last_active":"2023-09-23T17:56:53.647289+0000","last_peered":"2023-09-23T17:56:53.647289+0000","last_clean":"2023-09-23T17:56:38.407983+0000","last_became_active":"2023-09-23T17:56:50.896090+0000","last_became_peered":"2023-09-23T17:56:50.896090+0000","last_unstale":"2023-09-23T17:56:53.647289+0000","last_undegraded":"2023-09-23T17:56:50.830525+0000","last_fullsized":"2023-09-23T17:56:50.807254+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4959,"log_dups_size":0,"ondisk_log_size":4959,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:22:16.409351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324993024,"num_objects":4959,"num_object_clones":0,"num_object_copies":14877,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":392,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4959,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4959,"num_write_kb":317376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,3],"avail_no_missing":["2","3"],"object_location_counts":[{"shards":"0,2,3","objects":392},{"shards":"2,3","objects":4567}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":99,"reported_epoch":131,"state":"active+undersized","last_fresh":"2023-09-23T17:56:55.876219+0000","last_change":"2023-09-23T17:55:58.865270+0000","last_active":"2023-09-23T17:56:55.876219+0000","last_peered":"2023-09-23T17:56:55.876219+0000","last_clean":"2023-09-23T17:55:56.845943+0000","last_became_active":"2023-09-23T17:55:58.865270+0000","last_became_peered":"2023-09-23T17:55:58.865270+0000","last_unstale":"2023-09-23T17:56:55.876219+0000","last_undegraded":"2023-09-23T17:56:55.876219+0000","last_fullsized":"2023-09-23T17:55:58.860736+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:15.861060+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3],"acting":[5,3],"avail_no_missing":["5","3"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":90,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.829940+0000","last_change":"2023-09-23T17:56:50.828739+0000","last_active":"2023-09-23T17:56:51.829940+0000","last_peered":"2023-09-23T17:56:51.829940+0000","last_clean":"2023-09-23T17:56:51.829940+0000","last_became_active":"2023-09-23T17:56:50.828229+0000","last_became_peered":"2023-09-23T17:56:50.828229+0000","last_unstale":"2023-09-23T17:56:51.829940+0000","last_undegraded":"2023-09-23T17:56:51.829940+0000","last_fullsized":"2023-09-23T17:56:51.829940+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:45.927953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"128'4937","reported_seq":4964,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.607102+0000","last_change":"2023-09-23T17:55:53.830258+0000","last_active":"2023-09-23T17:56:53.607102+0000","last_peered":"2023-09-23T17:56:53.607102+0000","last_clean":"2023-09-23T17:56:53.607102+0000","last_became_active":"2023-09-23T17:55:53.829954+0000","last_became_peered":"2023-09-23T17:55:53.829954+0000","last_unstale":"2023-09-23T17:56:53.607102+0000","last_undegraded":"2023-09-23T17:56:53.607102+0000","last_fullsized":"2023-09-23T17:56:53.607102+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4937,"log_dups_size":0,"ondisk_log_size":4937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:41.952908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323551232,"num_objects":4937,"num_object_clones":0,"num_object_copies":14811,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4937,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4937,"num_write_kb":315968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":117,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.844018+0000","last_change":"2023-09-23T17:56:50.867874+0000","last_active":"2023-09-23T17:56:53.844018+0000","last_peered":"2023-09-23T17:56:53.844018+0000","last_clean":"2023-09-23T17:56:53.844018+0000","last_became_active":"2023-09-23T17:56:50.828937+0000","last_became_peered":"2023-09-23T17:56:50.828937+0000","last_unstale":"2023-09-23T17:56:53.844018+0000","last_undegraded":"2023-09-23T17:56:53.844018+0000","last_fullsized":"2023-09-23T17:56:53.844018+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:59:41.039403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":104,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.886512+0000","last_change":"2023-09-23T17:56:51.837554+0000","last_active":"2023-09-23T17:56:55.886512+0000","last_peered":"2023-09-23T17:56:55.886512+0000","last_clean":"2023-09-23T17:56:55.886512+0000","last_became_active":"2023-09-23T17:56:51.837181+0000","last_became_peered":"2023-09-23T17:56:51.837181+0000","last_unstale":"2023-09-23T17:56:55.886512+0000","last_undegraded":"2023-09-23T17:56:55.886512+0000","last_fullsized":"2023-09-23T17:56:55.886512+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:27.914041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3],"acting":[4,3,1],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"130'5036","reported_seq":5083,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.750150+0000","last_change":"2023-09-23T17:56:51.840960+0000","last_active":"2023-09-23T17:56:55.750150+0000","last_peered":"2023-09-23T17:56:55.750150+0000","last_clean":"2023-09-23T17:56:55.750150+0000","last_became_active":"2023-09-23T17:56:51.835697+0000","last_became_peered":"2023-09-23T17:56:51.835697+0000","last_unstale":"2023-09-23T17:56:55.750150+0000","last_undegraded":"2023-09-23T17:56:55.750150+0000","last_fullsized":"2023-09-23T17:56:55.750150+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5036,"log_dups_size":0,"ondisk_log_size":5036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:45.804116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330039296,"num_objects":5036,"num_object_clones":0,"num_object_copies":15108,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5036,"num_write_kb":322304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":84,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.882069+0000","last_change":"2023-09-23T17:56:51.834949+0000","last_active":"2023-09-23T17:56:55.882069+0000","last_peered":"2023-09-23T17:56:55.882069+0000","last_clean":"2023-09-23T17:56:55.882069+0000","last_became_active":"2023-09-23T17:56:51.834521+0000","last_became_peered":"2023-09-23T17:56:51.834521+0000","last_unstale":"2023-09-23T17:56:55.882069+0000","last_undegraded":"2023-09-23T17:56:55.882069+0000","last_fullsized":"2023-09-23T17:56:55.882069+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:27.009536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,3],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":107,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.882385+0000","last_change":"2023-09-23T17:56:51.838748+0000","last_active":"2023-09-23T17:56:55.882385+0000","last_peered":"2023-09-23T17:56:55.882385+0000","last_clean":"2023-09-23T17:56:55.882385+0000","last_became_active":"2023-09-23T17:56:51.838511+0000","last_became_peered":"2023-09-23T17:56:51.838511+0000","last_unstale":"2023-09-23T17:56:55.882385+0000","last_undegraded":"2023-09-23T17:56:55.882385+0000","last_fullsized":"2023-09-23T17:56:55.882385+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:00.929947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"132'5091","reported_seq":5149,"reported_epoch":132,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T17:56:57.357486+0000","last_change":"2023-09-23T17:56:51.963824+0000","last_active":"2023-09-23T17:56:57.357486+0000","last_peered":"2023-09-23T17:56:57.357486+0000","last_clean":"2023-09-23T17:55:56.846049+0000","last_became_active":"2023-09-23T17:56:51.953573+0000","last_became_peered":"2023-09-23T17:56:51.953573+0000","last_unstale":"2023-09-23T17:56:57.357486+0000","last_undegraded":"2023-09-23T17:56:51.943666+0000","last_fullsized":"2023-09-23T17:56:57.357486+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5091,"log_dups_size":0,"ondisk_log_size":5091,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:24.566891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333643776,"num_objects":5091,"num_object_clones":0,"num_object_copies":15273,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4425,"num_objects_misplaced":666,"num_objects_unfound":0,"num_objects_dirty":5091,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5091,"num_write_kb":325824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,2],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"2,4,5","objects":666},{"shards":"4,5","objects":4425}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":15,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.887397+0000","last_change":"2023-09-23T17:56:50.827883+0000","last_active":"2023-09-23T17:56:55.887397+0000","last_peered":"2023-09-23T17:56:55.887397+0000","last_clean":"2023-09-23T17:56:55.887397+0000","last_became_active":"2023-09-23T17:56:50.827581+0000","last_became_peered":"2023-09-23T17:56:50.827581+0000","last_unstale":"2023-09-23T17:56:55.887397+0000","last_undegraded":"2023-09-23T17:56:55.887397+0000","last_fullsized":"2023-09-23T17:56:55.887397+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:47.279562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":104,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.888262+0000","last_change":"2023-09-23T17:56:51.838077+0000","last_active":"2023-09-23T17:56:55.888262+0000","last_peered":"2023-09-23T17:56:55.888262+0000","last_clean":"2023-09-23T17:56:55.888262+0000","last_became_active":"2023-09-23T17:56:51.837672+0000","last_became_peered":"2023-09-23T17:56:51.837672+0000","last_unstale":"2023-09-23T17:56:55.888262+0000","last_undegraded":"2023-09-23T17:56:55.888262+0000","last_fullsized":"2023-09-23T17:56:55.888262+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:22:36.748047+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,3],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"132'5067","reported_seq":5956,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-09-23T17:56:57.336800+0000","last_change":"2023-09-23T17:56:56.943738+0000","last_active":"2023-09-23T17:56:57.336800+0000","last_peered":"2023-09-23T17:56:57.336800+0000","last_clean":"2023-09-23T17:56:57.336800+0000","last_became_active":"2023-09-23T17:56:56.903740+0000","last_became_peered":"2023-09-23T17:56:56.903740+0000","last_unstale":"2023-09-23T17:56:57.336800+0000","last_undegraded":"2023-09-23T17:56:57.336800+0000","last_fullsized":"2023-09-23T17:56:57.336800+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5067,"log_dups_size":0,"ondisk_log_size":5067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:10.789289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332070912,"num_objects":5067,"num_object_clones":0,"num_object_copies":15201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5067,"num_write_kb":324288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":414,"num_bytes_recovered":27131904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":93,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.844245+0000","last_change":"2023-09-23T17:54:57.038911+0000","last_active":"2023-09-23T17:56:53.844245+0000","last_peered":"2023-09-23T17:56:53.844245+0000","last_clean":"2023-09-23T17:56:53.844245+0000","last_became_active":"2023-09-23T17:54:57.038607+0000","last_became_peered":"2023-09-23T17:54:57.038607+0000","last_unstale":"2023-09-23T17:56:53.844245+0000","last_undegraded":"2023-09-23T17:56:53.844245+0000","last_fullsized":"2023-09-23T17:56:53.844245+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:20.373213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":81,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-09-23T17:56:50.833889+0000","last_change":"2023-09-23T17:56:50.830212+0000","last_active":"2023-09-23T17:56:50.833889+0000","last_peered":"2023-09-23T17:56:50.833889+0000","last_clean":"2023-09-23T17:56:50.833889+0000","last_became_active":"2023-09-23T17:56:50.829865+0000","last_became_peered":"2023-09-23T17:56:50.829865+0000","last_unstale":"2023-09-23T17:56:50.833889+0000","last_undegraded":"2023-09-23T17:56:50.833889+0000","last_fullsized":"2023-09-23T17:56:50.833889+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:17:18.085266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":550,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.848920+0000","last_change":"2023-09-23T17:53:52.730761+0000","last_active":"2023-09-23T17:56:53.848920+0000","last_peered":"2023-09-23T17:56:53.848920+0000","last_clean":"2023-09-23T17:56:53.848920+0000","last_became_active":"2023-09-23T17:53:52.730121+0000","last_became_peered":"2023-09-23T17:53:52.730121+0000","last_unstale":"2023-09-23T17:56:53.848920+0000","last_undegraded":"2023-09-23T17:56:53.848920+0000","last_fullsized":"2023-09-23T17:56:53.848920+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:02.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":109,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.876153+0000","last_change":"2023-09-23T17:54:57.040788+0000","last_active":"2023-09-23T17:56:55.876153+0000","last_peered":"2023-09-23T17:56:55.876153+0000","last_clean":"2023-09-23T17:56:55.876153+0000","last_became_active":"2023-09-23T17:54:57.040271+0000","last_became_peered":"2023-09-23T17:54:57.040271+0000","last_unstale":"2023-09-23T17:56:55.876153+0000","last_undegraded":"2023-09-23T17:56:55.876153+0000","last_fullsized":"2023-09-23T17:56:55.876153+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:37.176032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":72,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.847521+0000","last_change":"2023-09-23T17:56:50.816700+0000","last_active":"2023-09-23T17:56:53.847521+0000","last_peered":"2023-09-23T17:56:53.847521+0000","last_clean":"2023-09-23T17:56:53.847521+0000","last_became_active":"2023-09-23T17:56:50.816422+0000","last_became_peered":"2023-09-23T17:56:50.816422+0000","last_unstale":"2023-09-23T17:56:53.847521+0000","last_undegraded":"2023-09-23T17:56:53.847521+0000","last_fullsized":"2023-09-23T17:56:53.847521+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:01.034460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"129'4922","reported_seq":4956,"reported_epoch":129,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:54.045908+0000","last_change":"2023-09-23T17:55:58.870796+0000","last_active":"2023-09-23T17:56:54.045908+0000","last_peered":"2023-09-23T17:56:54.045908+0000","last_clean":"2023-09-23T17:55:57.821023+0000","last_became_active":"2023-09-23T17:55:58.867778+0000","last_became_peered":"2023-09-23T17:55:58.867778+0000","last_unstale":"2023-09-23T17:56:54.045908+0000","last_undegraded":"2023-09-23T17:55:58.868868+0000","last_fullsized":"2023-09-23T17:55:58.864928+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4922,"log_dups_size":0,"ondisk_log_size":4922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:30.503909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322568192,"num_objects":4922,"num_object_clones":0,"num_object_copies":14766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4922,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4922,"num_write_kb":315008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4922}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"129'4918","reported_seq":4965,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:54.037580+0000","last_change":"2023-09-23T17:56:50.843235+0000","last_active":"2023-09-23T17:56:54.037580+0000","last_peered":"2023-09-23T17:56:54.037580+0000","last_clean":"2023-09-23T17:56:54.037580+0000","last_became_active":"2023-09-23T17:56:50.835153+0000","last_became_peered":"2023-09-23T17:56:50.835153+0000","last_unstale":"2023-09-23T17:56:54.037580+0000","last_undegraded":"2023-09-23T17:56:54.037580+0000","last_fullsized":"2023-09-23T17:56:54.037580+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4918,"log_dups_size":0,"ondisk_log_size":4918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:14.827228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322306048,"num_objects":4918,"num_object_clones":0,"num_object_copies":14754,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4918,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4918,"num_write_kb":314752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1183,"reported_epoch":131,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:55.876187+0000","last_change":"2023-09-23T17:55:58.864229+0000","last_active":"2023-09-23T17:56:55.876187+0000","last_peered":"2023-09-23T17:56:55.876187+0000","last_clean":"2023-09-23T17:55:56.845670+0000","last_became_active":"2023-09-23T17:55:58.864229+0000","last_became_peered":"2023-09-23T17:55:58.864229+0000","last_unstale":"2023-09-23T17:56:55.876187+0000","last_undegraded":"2023-09-23T17:55:58.861121+0000","last_fullsized":"2023-09-23T17:55:58.861013+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:23.537845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1894,"num_read_kb":2706,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":105,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.882346+0000","last_change":"2023-09-23T17:56:51.835129+0000","last_active":"2023-09-23T17:56:55.882346+0000","last_peered":"2023-09-23T17:56:55.882346+0000","last_clean":"2023-09-23T17:56:55.882346+0000","last_became_active":"2023-09-23T17:56:51.834743+0000","last_became_peered":"2023-09-23T17:56:51.834743+0000","last_unstale":"2023-09-23T17:56:55.882346+0000","last_undegraded":"2023-09-23T17:56:55.882346+0000","last_fullsized":"2023-09-23T17:56:55.882346+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:50:59.053257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":84,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.848856+0000","last_change":"2023-09-23T17:54:57.035632+0000","last_active":"2023-09-23T17:56:53.848856+0000","last_peered":"2023-09-23T17:56:53.848856+0000","last_clean":"2023-09-23T17:56:53.848856+0000","last_became_active":"2023-09-23T17:54:57.035338+0000","last_became_peered":"2023-09-23T17:54:57.035338+0000","last_unstale":"2023-09-23T17:56:53.848856+0000","last_undegraded":"2023-09-23T17:56:53.848856+0000","last_fullsized":"2023-09-23T17:56:53.848856+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:18.773130+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"128'5070","reported_seq":5097,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.634815+0000","last_change":"2023-09-23T17:55:53.829379+0000","last_active":"2023-09-23T17:56:53.634815+0000","last_peered":"2023-09-23T17:56:53.634815+0000","last_clean":"2023-09-23T17:56:53.634815+0000","last_became_active":"2023-09-23T17:55:53.829085+0000","last_became_peered":"2023-09-23T17:55:53.829085+0000","last_unstale":"2023-09-23T17:56:53.634815+0000","last_undegraded":"2023-09-23T17:56:53.634815+0000","last_fullsized":"2023-09-23T17:56:53.634815+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5070,"log_dups_size":0,"ondisk_log_size":5070,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:22.353899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":332267520,"num_objects":5070,"num_object_clones":0,"num_object_copies":15210,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5070,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5070,"num_write_kb":324480,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":392,"reported_epoch":127,"state":"active+clean","last_fresh":"2023-09-23T17:56:50.801477+0000","last_change":"2023-09-23T17:54:57.037634+0000","last_active":"2023-09-23T17:56:50.801477+0000","last_peered":"2023-09-23T17:56:50.801477+0000","last_clean":"2023-09-23T17:56:50.801477+0000","last_became_active":"2023-09-23T17:54:57.037115+0000","last_became_peered":"2023-09-23T17:54:57.037115+0000","last_unstale":"2023-09-23T17:56:50.801477+0000","last_undegraded":"2023-09-23T17:56:50.801477+0000","last_fullsized":"2023-09-23T17:56:50.801477+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:52:58.348982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":74,"reported_epoch":129,"state":"active+undersized","last_fresh":"2023-09-23T17:56:53.847237+0000","last_change":"2023-09-23T17:55:58.865545+0000","last_active":"2023-09-23T17:56:53.847237+0000","last_peered":"2023-09-23T17:56:53.847237+0000","last_clean":"2023-09-23T17:55:57.820377+0000","last_became_active":"2023-09-23T17:55:58.865545+0000","last_became_peered":"2023-09-23T17:55:58.865545+0000","last_unstale":"2023-09-23T17:56:53.847237+0000","last_undegraded":"2023-09-23T17:56:53.847237+0000","last_fullsized":"2023-09-23T17:55:58.860734+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:55.266655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":97,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.848203+0000","last_change":"2023-09-23T17:53:52.728795+0000","last_active":"2023-09-23T17:56:53.848203+0000","last_peered":"2023-09-23T17:56:53.848203+0000","last_clean":"2023-09-23T17:56:53.848203+0000","last_became_active":"2023-09-23T17:53:52.728423+0000","last_became_peered":"2023-09-23T17:53:52.728423+0000","last_unstale":"2023-09-23T17:56:53.848203+0000","last_undegraded":"2023-09-23T17:56:53.848203+0000","last_fullsized":"2023-09-23T17:56:53.848203+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.759322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1046,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.876533+0000","last_change":"2023-09-23T17:54:25.601298+0000","last_active":"2023-09-23T17:56:55.876533+0000","last_peered":"2023-09-23T17:56:55.876533+0000","last_clean":"2023-09-23T17:56:55.876533+0000","last_became_active":"2023-09-23T17:54:25.600854+0000","last_became_peered":"2023-09-23T17:54:25.600854+0000","last_unstale":"2023-09-23T17:56:55.876533+0000","last_undegraded":"2023-09-23T17:56:55.876533+0000","last_fullsized":"2023-09-23T17:56:55.876533+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:56.978955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":482,"reported_epoch":131,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:55.884460+0000","last_change":"2023-09-23T17:55:58.867888+0000","last_active":"2023-09-23T17:56:55.884460+0000","last_peered":"2023-09-23T17:56:55.884460+0000","last_clean":"2023-09-23T17:55:57.822497+0000","last_became_active":"2023-09-23T17:55:58.867888+0000","last_became_peered":"2023-09-23T17:55:58.867888+0000","last_unstale":"2023-09-23T17:56:55.884460+0000","last_undegraded":"2023-09-23T17:55:58.860525+0000","last_fullsized":"2023-09-23T17:55:58.860424+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:18.913217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":49,"reported_epoch":129,"state":"active+undersized","last_fresh":"2023-09-23T17:56:53.842676+0000","last_change":"2023-09-23T17:55:58.866152+0000","last_active":"2023-09-23T17:56:53.842676+0000","last_peered":"2023-09-23T17:56:53.842676+0000","last_clean":"2023-09-23T17:55:56.841167+0000","last_became_active":"2023-09-23T17:55:58.866152+0000","last_became_peered":"2023-09-23T17:55:58.866152+0000","last_unstale":"2023-09-23T17:56:53.842676+0000","last_undegraded":"2023-09-23T17:56:53.842676+0000","last_fullsized":"2023-09-23T17:55:58.856680+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:13.914833+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5],"acting":[3,5],"avail_no_missing":["3","5"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":32,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.882533+0000","last_change":"2023-09-23T17:56:51.845525+0000","last_active":"2023-09-23T17:56:55.882533+0000","last_peered":"2023-09-23T17:56:55.882533+0000","last_clean":"2023-09-23T17:56:55.882533+0000","last_became_active":"2023-09-23T17:56:51.845056+0000","last_became_peered":"2023-09-23T17:56:51.845056+0000","last_unstale":"2023-09-23T17:56:55.882533+0000","last_undegraded":"2023-09-23T17:56:55.882533+0000","last_fullsized":"2023-09-23T17:56:55.882533+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:09.024552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,0],"avail_no_missing":["4","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.4","version":"132'5054","reported_seq":5084,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-09-23T17:56:57.303644+0000","last_change":"2023-09-23T17:55:53.829245+0000","last_active":"2023-09-23T17:56:57.303644+0000","last_peered":"2023-09-23T17:56:57.303644+0000","last_clean":"2023-09-23T17:56:57.303644+0000","last_became_active":"2023-09-23T17:55:53.829019+0000","last_became_peered":"2023-09-23T17:55:53.829019+0000","last_unstale":"2023-09-23T17:56:57.303644+0000","last_undegraded":"2023-09-23T17:56:57.303644+0000","last_fullsized":"2023-09-23T17:56:57.303644+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5054,"log_dups_size":0,"ondisk_log_size":5054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:38.782556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":331218944,"num_objects":5054,"num_object_clones":0,"num_object_copies":15162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5054,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5054,"num_write_kb":323456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"132'5204","reported_seq":5234,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-09-23T17:56:57.325644+0000","last_change":"2023-09-23T17:55:53.828325+0000","last_active":"2023-09-23T17:56:57.325644+0000","last_peered":"2023-09-23T17:56:57.325644+0000","last_clean":"2023-09-23T17:56:57.325644+0000","last_became_active":"2023-09-23T17:55:53.828056+0000","last_became_peered":"2023-09-23T17:55:53.828056+0000","last_unstale":"2023-09-23T17:56:57.325644+0000","last_undegraded":"2023-09-23T17:56:57.325644+0000","last_fullsized":"2023-09-23T17:56:57.325644+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5204,"log_dups_size":0,"ondisk_log_size":5204,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:47.523521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":341049344,"num_objects":5204,"num_object_clones":0,"num_object_copies":15612,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5204,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5204,"num_write_kb":333056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":330,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.884606+0000","last_change":"2023-09-23T17:53:52.734324+0000","last_active":"2023-09-23T17:56:55.884606+0000","last_peered":"2023-09-23T17:56:55.884606+0000","last_clean":"2023-09-23T17:56:55.884606+0000","last_became_active":"2023-09-23T17:53:52.733825+0000","last_became_peered":"2023-09-23T17:53:52.733825+0000","last_unstale":"2023-09-23T17:56:55.884606+0000","last_undegraded":"2023-09-23T17:56:55.884606+0000","last_fullsized":"2023-09-23T17:56:55.884606+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:34.742687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":89,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.829771+0000","last_change":"2023-09-23T17:56:50.816170+0000","last_active":"2023-09-23T17:56:51.829771+0000","last_peered":"2023-09-23T17:56:51.829771+0000","last_clean":"2023-09-23T17:56:51.829771+0000","last_became_active":"2023-09-23T17:56:50.815829+0000","last_became_peered":"2023-09-23T17:56:50.815829+0000","last_unstale":"2023-09-23T17:56:51.829771+0000","last_undegraded":"2023-09-23T17:56:51.829771+0000","last_fullsized":"2023-09-23T17:56:51.829771+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:27.215024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":87,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.830333+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T17:56:51.830333+0000","last_peered":"2023-09-23T17:56:51.830333+0000","last_clean":"2023-09-23T17:56:51.830333+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T17:56:51.830333+0000","last_undegraded":"2023-09-23T17:56:51.830333+0000","last_fullsized":"2023-09-23T17:56:51.830333+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":396,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.891112+0000","last_change":"2023-09-23T17:56:50.881038+0000","last_active":"2023-09-23T17:56:55.891112+0000","last_peered":"2023-09-23T17:56:55.891112+0000","last_clean":"2023-09-23T17:56:55.891112+0000","last_became_active":"2023-09-23T17:56:50.822953+0000","last_became_peered":"2023-09-23T17:56:50.822953+0000","last_unstale":"2023-09-23T17:56:55.891112+0000","last_undegraded":"2023-09-23T17:56:55.891112+0000","last_fullsized":"2023-09-23T17:56:55.891112+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:03:47.600460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":68,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.848970+0000","last_change":"2023-09-23T17:54:25.604720+0000","last_active":"2023-09-23T17:56:53.848970+0000","last_peered":"2023-09-23T17:56:53.848970+0000","last_clean":"2023-09-23T17:56:53.848970+0000","last_became_active":"2023-09-23T17:54:25.604386+0000","last_became_peered":"2023-09-23T17:54:25.604386+0000","last_unstale":"2023-09-23T17:56:53.848970+0000","last_undegraded":"2023-09-23T17:56:53.848970+0000","last_fullsized":"2023-09-23T17:56:53.848970+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:56.869273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":57,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.830922+0000","last_change":"2023-09-23T17:53:21.039195+0000","last_active":"2023-09-23T17:56:51.830922+0000","last_peered":"2023-09-23T17:56:51.830922+0000","last_clean":"2023-09-23T17:56:51.830922+0000","last_became_active":"2023-09-23T17:53:21.038654+0000","last_became_peered":"2023-09-23T17:53:21.038654+0000","last_unstale":"2023-09-23T17:56:51.830922+0000","last_undegraded":"2023-09-23T17:56:51.830922+0000","last_fullsized":"2023-09-23T17:56:51.830922+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:34.466980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"129'4809","reported_seq":4832,"reported_epoch":129,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:56:54.092079+0000","last_change":"2023-09-23T17:55:58.884622+0000","last_active":"2023-09-23T17:56:54.092079+0000","last_peered":"2023-09-23T17:56:54.092079+0000","last_clean":"2023-09-23T17:55:52.789238+0000","last_became_active":"2023-09-23T17:55:58.867800+0000","last_became_peered":"2023-09-23T17:55:58.867800+0000","last_unstale":"2023-09-23T17:56:54.092079+0000","last_undegraded":"2023-09-23T17:55:58.878355+0000","last_fullsized":"2023-09-23T17:55:58.862585+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4809,"log_dups_size":0,"ondisk_log_size":4809,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:13.398530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315162624,"num_objects":4809,"num_object_clones":0,"num_object_copies":14427,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4809,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4809,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4809,"num_write_kb":307776,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4809}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":91,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.843781+0000","last_change":"2023-09-23T17:54:57.041796+0000","last_active":"2023-09-23T17:56:53.843781+0000","last_peered":"2023-09-23T17:56:53.843781+0000","last_clean":"2023-09-23T17:56:53.843781+0000","last_became_active":"2023-09-23T17:54:57.041245+0000","last_became_peered":"2023-09-23T17:54:57.041245+0000","last_unstale":"2023-09-23T17:56:53.843781+0000","last_undegraded":"2023-09-23T17:56:53.843781+0000","last_fullsized":"2023-09-23T17:56:53.843781+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:02:26.340510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":346,"reported_epoch":129,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:53.848045+0000","last_change":"2023-09-23T17:56:51.835757+0000","last_active":"2023-09-23T17:56:53.848045+0000","last_peered":"2023-09-23T17:56:53.848045+0000","last_clean":"2023-09-23T17:56:53.848045+0000","last_became_active":"2023-09-23T17:56:51.835165+0000","last_became_peered":"2023-09-23T17:56:51.835165+0000","last_unstale":"2023-09-23T17:56:53.848045+0000","last_undegraded":"2023-09-23T17:56:53.848045+0000","last_fullsized":"2023-09-23T17:56:53.848045+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:43.543052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":89,"reported_epoch":130,"state":"active+clean","last_fresh":"2023-09-23T17:56:54.853260+0000","last_change":"2023-09-23T17:54:57.040450+0000","last_active":"2023-09-23T17:56:54.853260+0000","last_peered":"2023-09-23T17:56:54.853260+0000","last_clean":"2023-09-23T17:56:54.853260+0000","last_became_active":"2023-09-23T17:54:57.040134+0000","last_became_peered":"2023-09-23T17:54:57.040134+0000","last_unstale":"2023-09-23T17:56:54.853260+0000","last_undegraded":"2023-09-23T17:56:54.853260+0000","last_fullsized":"2023-09-23T17:56:54.853260+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:07.182973+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"132'5117","reported_seq":6606,"reported_epoch":132,"state":"active+clean","last_fresh":"2023-09-23T17:56:57.355985+0000","last_change":"2023-09-23T17:56:54.871278+0000","last_active":"2023-09-23T17:56:57.355985+0000","last_peered":"2023-09-23T17:56:57.355985+0000","last_clean":"2023-09-23T17:56:57.355985+0000","last_became_active":"2023-09-23T17:56:54.864932+0000","last_became_peered":"2023-09-23T17:56:54.864932+0000","last_unstale":"2023-09-23T17:56:57.355985+0000","last_undegraded":"2023-09-23T17:56:57.355985+0000","last_fullsized":"2023-09-23T17:56:57.355985+0000","mapping_epoch":129,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5117,"log_dups_size":0,"ondisk_log_size":5117,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:12.979066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335347712,"num_objects":5117,"num_object_clones":0,"num_object_copies":15351,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5117,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5117,"num_write_kb":327488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":717,"num_bytes_recovered":46989312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":315,"reported_epoch":131,"state":"active+undersized","last_fresh":"2023-09-23T17:56:55.891172+0000","last_change":"2023-09-23T17:55:58.866975+0000","last_active":"2023-09-23T17:56:55.891172+0000","last_peered":"2023-09-23T17:56:55.891172+0000","last_clean":"2023-09-23T17:46:59.535985+0000","last_became_active":"2023-09-23T17:55:58.866975+0000","last_became_peered":"2023-09-23T17:55:58.866975+0000","last_unstale":"2023-09-23T17:56:55.891172+0000","last_undegraded":"2023-09-23T17:56:55.891172+0000","last_fullsized":"2023-09-23T17:55:58.861027+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:48.232006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":101,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.830291+0000","last_change":"2023-09-23T17:56:50.856836+0000","last_active":"2023-09-23T17:56:51.830291+0000","last_peered":"2023-09-23T17:56:51.830291+0000","last_clean":"2023-09-23T17:56:51.830291+0000","last_became_active":"2023-09-23T17:56:50.835155+0000","last_became_peered":"2023-09-23T17:56:50.835155+0000","last_unstale":"2023-09-23T17:56:51.830291+0000","last_undegraded":"2023-09-23T17:56:51.830291+0000","last_fullsized":"2023-09-23T17:56:51.830291+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:36.188982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":87,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.887586+0000","last_change":"2023-09-23T17:56:51.838623+0000","last_active":"2023-09-23T17:56:55.887586+0000","last_peered":"2023-09-23T17:56:55.887586+0000","last_clean":"2023-09-23T17:56:55.887586+0000","last_became_active":"2023-09-23T17:56:51.838216+0000","last_became_peered":"2023-09-23T17:56:51.838216+0000","last_unstale":"2023-09-23T17:56:55.887586+0000","last_undegraded":"2023-09-23T17:56:55.887586+0000","last_fullsized":"2023-09-23T17:56:55.887586+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:25:08.884374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,0],"avail_no_missing":["4","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"130'5105","reported_seq":5148,"reported_epoch":130,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T17:56:55.006012+0000","last_change":"2023-09-23T17:56:50.898198+0000","last_active":"2023-09-23T17:56:55.006012+0000","last_peered":"2023-09-23T17:56:55.006012+0000","last_clean":"2023-09-23T17:56:38.421727+0000","last_became_active":"2023-09-23T17:56:50.893395+0000","last_became_peered":"2023-09-23T17:56:50.893395+0000","last_unstale":"2023-09-23T17:56:55.006012+0000","last_undegraded":"2023-09-23T17:56:50.814646+0000","last_fullsized":"2023-09-23T17:56:50.798048+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5105,"log_dups_size":0,"ondisk_log_size":5105,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:20.428834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334561280,"num_objects":5105,"num_object_clones":0,"num_object_copies":15315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":424,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5105,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1],"avail_no_missing":["3","1"],"object_location_counts":[{"shards":"0,1,3","objects":424},{"shards":"1,3","objects":4681}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":78,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.830808+0000","last_change":"2023-09-23T17:56:50.828555+0000","last_active":"2023-09-23T17:56:51.830808+0000","last_peered":"2023-09-23T17:56:51.830808+0000","last_clean":"2023-09-23T17:56:51.830808+0000","last_became_active":"2023-09-23T17:56:50.827767+0000","last_became_peered":"2023-09-23T17:56:50.827767+0000","last_unstale":"2023-09-23T17:56:51.830808+0000","last_undegraded":"2023-09-23T17:56:51.830808+0000","last_fullsized":"2023-09-23T17:56:51.830808+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:59.993181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":112,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.834599+0000","last_change":"2023-09-23T17:54:57.034729+0000","last_active":"2023-09-23T17:56:51.834599+0000","last_peered":"2023-09-23T17:56:51.834599+0000","last_clean":"2023-09-23T17:56:51.834599+0000","last_became_active":"2023-09-23T17:54:57.034381+0000","last_became_peered":"2023-09-23T17:54:57.034381+0000","last_unstale":"2023-09-23T17:56:51.834599+0000","last_undegraded":"2023-09-23T17:56:51.834599+0000","last_fullsized":"2023-09-23T17:56:51.834599+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:19.369673+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":118,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.876340+0000","last_change":"2023-09-23T17:56:51.848379+0000","last_active":"2023-09-23T17:56:55.876340+0000","last_peered":"2023-09-23T17:56:55.876340+0000","last_clean":"2023-09-23T17:56:55.876340+0000","last_became_active":"2023-09-23T17:56:51.848091+0000","last_became_peered":"2023-09-23T17:56:51.848091+0000","last_unstale":"2023-09-23T17:56:55.876340+0000","last_undegraded":"2023-09-23T17:56:55.876340+0000","last_fullsized":"2023-09-23T17:56:55.876340+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:01.930989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":15,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.882091+0000","last_change":"2023-09-23T17:56:50.827982+0000","last_active":"2023-09-23T17:56:55.882091+0000","last_peered":"2023-09-23T17:56:55.882091+0000","last_clean":"2023-09-23T17:56:55.882091+0000","last_became_active":"2023-09-23T17:56:50.827720+0000","last_became_peered":"2023-09-23T17:56:50.827720+0000","last_unstale":"2023-09-23T17:56:55.882091+0000","last_undegraded":"2023-09-23T17:56:55.882091+0000","last_fullsized":"2023-09-23T17:56:55.882091+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:29.556167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":112,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.876287+0000","last_change":"2023-09-23T17:56:51.843661+0000","last_active":"2023-09-23T17:56:55.876287+0000","last_peered":"2023-09-23T17:56:55.876287+0000","last_clean":"2023-09-23T17:56:55.876287+0000","last_became_active":"2023-09-23T17:56:51.842466+0000","last_became_peered":"2023-09-23T17:56:51.842466+0000","last_unstale":"2023-09-23T17:56:55.876287+0000","last_undegraded":"2023-09-23T17:56:55.876287+0000","last_fullsized":"2023-09-23T17:56:55.876287+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:19.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,1],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":15,"reported_epoch":131,"state":"active+clean","last_fresh":"2023-09-23T17:56:55.881985+0000","last_change":"2023-09-23T17:56:50.829520+0000","last_active":"2023-09-23T17:56:55.881985+0000","last_peered":"2023-09-23T17:56:55.881985+0000","last_clean":"2023-09-23T17:56:55.881985+0000","last_became_active":"2023-09-23T17:56:50.827957+0000","last_became_peered":"2023-09-23T17:56:50.827957+0000","last_unstale":"2023-09-23T17:56:55.881985+0000","last_undegraded":"2023-09-23T17:56:55.881985+0000","last_fullsized":"2023-09-23T17:56:55.881985+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:23.181126+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":105,"reported_epoch":130,"state":"active+undersized","last_fresh":"2023-09-23T17:56:54.853541+0000","last_change":"2023-09-23T17:55:58.860645+0000","last_active":"2023-09-23T17:56:54.853541+0000","last_peered":"2023-09-23T17:56:54.853541+0000","last_clean":"2023-09-23T17:55:56.841110+0000","last_became_active":"2023-09-23T17:55:58.860645+0000","last_became_peered":"2023-09-23T17:55:58.860645+0000","last_unstale":"2023-09-23T17:56:54.853541+0000","last_undegraded":"2023-09-23T17:56:54.853541+0000","last_fullsized":"2023-09-23T17:55:58.855994+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:29.000041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2],"acting":[3,2],"avail_no_missing":["3","2"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":113,"reported_epoch":129,"state":"active+clean","last_fresh":"2023-09-23T17:56:53.843538+0000","last_change":"2023-09-23T17:54:57.034722+0000","last_active":"2023-09-23T17:56:53.843538+0000","last_peered":"2023-09-23T17:56:53.843538+0000","last_clean":"2023-09-23T17:56:53.843538+0000","last_became_active":"2023-09-23T17:54:57.034289+0000","last_became_peered":"2023-09-23T17:54:57.034289+0000","last_unstale":"2023-09-23T17:56:53.843538+0000","last_undegraded":"2023-09-23T17:56:53.843538+0000","last_fullsized":"2023-09-23T17:56:53.843538+0000","mapping_epoch":108,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:03.041518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":75,"reported_epoch":128,"state":"active+clean","last_fresh":"2023-09-23T17:56:51.830298+0000","last_change":"2023-09-23T17:54:25.601065+0000","last_active":"2023-09-23T17:56:51.830298+0000","last_peered":"2023-09-23T17:56:51.830298+0000","last_clean":"2023-09-23T17:56:51.830298+0000","last_became_active":"2023-09-23T17:54:25.600727+0000","last_became_peered":"2023-09-23T17:54:25.600727+0000","last_unstale":"2023-09-23T17:56:51.830298+0000","last_undegraded":"2023-09-23T17:56:51.830298+0000","last_fullsized":"2023-09-23T17:56:51.830298+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:20.919498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":104,"reported_epoch":131,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:56:55.881833+0000","last_change":"2023-09-23T17:56:51.835288+0000","last_active":"2023-09-23T17:56:55.881833+0000","last_peered":"2023-09-23T17:56:55.881833+0000","last_clean":"2023-09-23T17:56:55.881833+0000","last_became_active":"2023-09-23T17:56:51.834568+0000","last_became_peered":"2023-09-23T17:56:51.834568+0000","last_unstale":"2023-09-23T17:56:55.881833+0000","last_undegraded":"2023-09-23T17:56:55.881833+0000","last_fullsized":"2023-09-23T17:56:55.881833+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:49.829181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,3],"avail_no_missing":["4","1","3"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":5254545408,"num_objects":80178,"num_object_clones":0,"num_object_copies":240534,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19939,"num_objects_misplaced":666,"num_objects_unfound":0,"num_objects_dirty":80178,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":80178,"num_write_kb":5131392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1567,"num_bytes_recovered":102694912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14732165120,"data_stored":14732165120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80178,"ondisk_log_size":80178,"up":44,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":84,"acting":90,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":5,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":224,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":85,"acting":90,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":2,"acting":2,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":21,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266838,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1670400,"kb_used_data":1622200,"kb_used_omap":0,"kb_used_meta":48192,"kb_avail":92074752,"statfs":{"total":95995035648,"available":94284546048,"internally_reserved":0,"allocated":1661132800,"data_stored":1660573022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":49348608},"hb_peers":[0,1,2,3,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:25 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.61899999999999999,"15min":0.57299999999999995},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":2.6539999999999999,"5min":2.6539999999999999,"15min":2.6539999999999999},"last":0.314},{"interface":"front","average":{"1min":1.0189999999999999,"5min":0.66300000000000003,"15min":0.59899999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.7290000000000001,"5min":4.7290000000000001,"15min":4.7290000000000001},"last":0.35199999999999998}]},{"osd":1,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.59599999999999997,"15min":0.56999999999999995},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.702,"5min":2.702,"15min":2.702},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.1399999999999999,"5min":0.65700000000000003,"15min":0.58899999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":4.4240000000000004},"last":0.36199999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:56:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":0.72699999999999998,"15min":0.60799999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.98,"5min":3.98,"15min":3.98},"last":0.29899999999999999},{"interface":"front","average":{"1min":1.121,"5min":0.68999999999999995,"15min":0.61799999999999999},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":4.6180000000000003,"5min":4.6180000000000003,"15min":4.6180000000000003},"last":0.42099999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.38},{"interface":"front","average":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.625,"5min":0.625,"15min":0.625},"last":0.437}]},{"osd":4,"last update":"Sat Sep 23 17:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.52500000000000002,"5min":0.504,"15min":0.45400000000000001},"min":{"1min":0.254,"5min":0.252,"15min":0.252},"max":{"1min":0.80100000000000005,"5min":0.80300000000000005,"15min":0.80300000000000005},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.54800000000000004,"5min":0.52900000000000003,"15min":0.48199999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.73599999999999999,"5min":0.81799999999999995,"15min":0.81799999999999995},"last":0.39500000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:56:03 2023","interfaces":[{"interface":"back","average":{"1min":0.48399999999999999,"5min":0.52400000000000002,"15min":0.49199999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.68400000000000005,"5min":0.86099999999999999,"15min":0.86099999999999999},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.60999999999999999,"15min":0.53000000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29099999999999998},"max":{"1min":1.706,"5min":1.706,"15min":1.706},"last":0.45200000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":1.002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.435}]}]},{"osd":6,"up_from":45,"seq":193273528462,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12464,"kb_used_data":1704,"kb_used_omap":0,"kb_used_meta":10752,"kb_avail":93732688,"statfs":{"total":95995035648,"available":95982272512,"internally_reserved":0,"allocated":1744896,"data_stored":1230110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11010048},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.81100000000000005,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.78300000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.38}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.62},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":1.2250000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":6.9589999999999996}]},{"osd":3,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.38800000000000001,"5min":0.36799999999999999,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":0.80700000000000005,"15min":4.5599999999999996},"last":0.253},{"interface":"front","average":{"1min":0.625,"5min":0.58499999999999996,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34999999999999998,"15min":0.30399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":1.6850000000000001},"last":2.9369999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:55:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.48699999999999999,"15min":0.46000000000000002},"min":{"1min":0.377,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.82899999999999996},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.53000000000000003,"15min":0.496},"min":{"1min":0.34499999999999997,"5min":0.28899999999999998,"15min":0.27800000000000002},"max":{"1min":0.72099999999999997,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.33800000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:55:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.54300000000000004,"15min":0.503},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":0.82099999999999995,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56699999999999995,"15min":0.52700000000000002},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.33200000000000002},"max":{"1min":0.69099999999999995,"5min":0.79700000000000004,"15min":0.84599999999999997},"last":0.42199999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:55:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.58999999999999997,"15min":0.53700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":0.83299999999999996,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.57499999999999996,"15min":0.51900000000000002},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":0.81299999999999994,"5min":0.82299999999999995,"15min":1.591},"last":0.314}]}]},{"osd":1,"up_from":98,"seq":420906795047,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1744640,"kb_used_data":1676856,"kb_used_omap":7,"kb_used_meta":67768,"kb_avail":92000512,"statfs":{"total":95995035648,"available":94208524288,"internally_reserved":0,"allocated":1717100544,"data_stored":1716470990,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":69395133},"hb_peers":[0,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6800000000000002,"5min":0.83899999999999997,"15min":0.53800000000000003},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":11.384,"5min":11.384,"15min":11.384},"last":2.0089999999999999},{"interface":"front","average":{"1min":2.9550000000000001,"5min":0.90600000000000003,"15min":0.56599999999999995},"min":{"1min":0.25,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.596,"5min":11.596,"15min":11.596},"last":0.48599999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:56:31 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":0.63700000000000001,"15min":0.51300000000000001},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":5.3860000000000001,"5min":5.3860000000000001,"15min":5.3860000000000001},"last":3.2250000000000001},{"interface":"front","average":{"1min":1.675,"5min":0.68700000000000006,"15min":0.52200000000000002},"min":{"1min":0.318,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":11.305999999999999,"5min":11.305999999999999,"15min":11.305999999999999},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 23 17:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":1.589,"5min":1.589,"15min":1.589},"last":0.505},{"interface":"front","average":{"1min":0.47199999999999998,"5min":0.47199999999999998,"15min":0.47199999999999998},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":1.5049999999999999,"5min":1.5049999999999999,"15min":1.5049999999999999},"last":0.28999999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7150000000000001,"5min":0.88600000000000001,"15min":0.75},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.0090000000000003,"5min":5.0090000000000003,"15min":5.0090000000000003},"last":2.4359999999999999},{"interface":"front","average":{"1min":1.8029999999999999,"5min":0.89700000000000002,"15min":0.751},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":8.8770000000000007,"5min":8.8770000000000007,"15min":8.8770000000000007},"last":3.2429999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":0.96999999999999997,"15min":0.80200000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.4269999999999996,"5min":7.4269999999999996,"15min":7.4269999999999996},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.788,"5min":0.91300000000000003,"15min":0.77500000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":5.0430000000000001,"5min":5.0430000000000001,"15min":5.0430000000000001},"last":0.54600000000000004}]},{"osd":6,"last update":"Sat Sep 23 17:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.9330000000000001,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.69199999999999995,"15min":0.69899999999999995},"min":{"1min":0.47699999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.141,"5min":1.141,"15min":1.141},"last":0.77700000000000002}]},{"osd":7,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":0.89400000000000002,"15min":0.74399999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.9630000000000001,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":2.758},{"interface":"front","average":{"1min":1.895,"5min":0.92900000000000005,"15min":0.77900000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.2940000000000005,"5min":8.2940000000000005,"15min":8.2940000000000005},"last":0.61099999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958573,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2743344,"kb_used_data":2639464,"kb_used_omap":7,"kb_used_meta":103864,"kb_avail":91001808,"statfs":{"total":95995035648,"available":93185851392,"internally_reserved":0,"allocated":2702811136,"data_stored":2702187658,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":106357476},"hb_peers":[1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:56:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":0.66900000000000004,"15min":0.46899999999999997},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":4.7030000000000003,"5min":4.7030000000000003,"15min":4.7030000000000003},"last":0.55700000000000005},{"interface":"front","average":{"1min":1.591,"5min":0.64900000000000002,"15min":0.5},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.8450000000000006,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":0.44400000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.88,"5min":0.46400000000000002,"15min":0.39500000000000002},"min":{"1min":0.26800000000000002,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.0609999999999999,"5min":4.0609999999999999,"15min":4.0609999999999999},"last":1.6990000000000001},{"interface":"front","average":{"1min":0.78000000000000003,"5min":0.45000000000000001,"15min":0.39500000000000002},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":4.5389999999999997,"5min":4.5389999999999997,"15min":4.5389999999999997},"last":1.5009999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:55:59 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.51700000000000002,"15min":0.51700000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.5910000000000002,"5min":3.5910000000000002,"15min":3.5910000000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":1.3939999999999999}]},{"osd":4,"last update":"Sat Sep 23 17:56:27 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":0.71699999999999997,"15min":0.66300000000000003},"min":{"1min":0.42799999999999999,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":4.5810000000000004,"5min":4.5810000000000004,"15min":4.5810000000000004},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.99299999999999999,"5min":0.69099999999999995,"15min":0.64500000000000002},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":3.339,"5min":3.339,"15min":3.339},"last":2.3460000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.96499999999999997,"5min":0.69099999999999995,"15min":0.65200000000000002},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":3.3929999999999998,"5min":3.3929999999999998,"15min":3.3929999999999998},"last":2.8769999999999998},{"interface":"front","average":{"1min":0.69899999999999995,"5min":0.65000000000000002,"15min":0.64600000000000002},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":2.8809999999999998,"5min":2.8809999999999998,"15min":2.8809999999999998},"last":0.51500000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.42899999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65200000000000002},"min":{"1min":0.52700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.81100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 23 17:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.72999999999999998,"5min":0.67200000000000004,"15min":0.66800000000000004},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":2.9049999999999998,"5min":2.9049999999999998,"15min":2.9049999999999998},"last":1.7669999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":0.72699999999999998,"15min":0.68999999999999995},"min":{"1min":0.51200000000000001,"5min":0.437,"15min":0.437},"max":{"1min":2.4380000000000002,"5min":2.4380000000000002,"15min":2.4380000000000002},"last":0.67500000000000004}]}]},{"osd":2,"up_from":103,"seq":442381631520,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2024796,"kb_used_data":1952980,"kb_used_omap":11,"kb_used_meta":71796,"kb_avail":91720356,"statfs":{"total":95995035648,"available":93921644544,"internally_reserved":0,"allocated":1999851520,"data_stored":1999242572,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":73519479},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.36799999999999999,"15min":0.34899999999999998},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":2.5129999999999999,"5min":2.5129999999999999,"15min":2.5129999999999999},"last":0.22800000000000001},{"interface":"front","average":{"1min":0.86299999999999999,"5min":0.47899999999999998,"15min":0.41499999999999998},"min":{"1min":0.30099999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.0389999999999997,"5min":8.0389999999999997,"15min":8.0389999999999997},"last":0.41299999999999998}]},{"osd":1,"last update":"Sat Sep 23 17:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.625,"5min":0.44600000000000001,"15min":0.41599999999999998},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":2.7949999999999999,"5min":2.7949999999999999,"15min":2.7949999999999999},"last":0.371},{"interface":"front","average":{"1min":1.0389999999999999,"5min":0.47099999999999997,"15min":0.377},"min":{"1min":0.26100000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":11.202,"5min":11.202,"15min":11.202},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:55:57 2023","interfaces":[{"interface":"back","average":{"1min":0.371,"5min":0.371,"15min":0.371},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"last":1.2749999999999999},{"interface":"front","average":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":0.58099999999999996,"5min":0.58099999999999996,"15min":0.58099999999999996},"last":1.357}]},{"osd":4,"last update":"Sat Sep 23 17:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":0.69499999999999995,"15min":0.63300000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.7279999999999998,"5min":4.7279999999999998,"15min":4.7279999999999998},"last":0.47999999999999998},{"interface":"front","average":{"1min":0.71099999999999997,"5min":0.63,"15min":0.61599999999999999},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":2.573,"5min":2.573,"15min":2.573},"last":0.51500000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.70299999999999996,"5min":0.63900000000000001,"15min":0.629},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":2.593,"5min":2.593,"15min":2.593},"last":0.754},{"interface":"front","average":{"1min":0.97399999999999998,"5min":0.67800000000000005,"15min":0.628},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":3.8149999999999999,"5min":3.8149999999999999,"15min":3.8149999999999999},"last":0.5}]},{"osd":6,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:56:28 2023","interfaces":[{"interface":"back","average":{"1min":1.0089999999999999,"5min":0.68700000000000006,"15min":0.63300000000000001},"min":{"1min":0.441,"5min":0.441,"15min":0.441},"max":{"1min":4.6740000000000004,"5min":4.6740000000000004,"15min":4.6740000000000004},"last":2.214},{"interface":"front","average":{"1min":0.75,"5min":0.69799999999999995,"15min":0.68999999999999995},"min":{"1min":0.48899999999999999,"5min":0.48899999999999999,"15min":0.48899999999999999},"max":{"1min":2.6120000000000001,"5min":2.6120000000000001,"15min":2.6120000000000001},"last":2.6509999999999998}]}]},{"osd":3,"up_from":108,"seq":463856467994,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1002724,"kb_used_data":971228,"kb_used_omap":3,"kb_used_meta":31484,"kb_avail":92742428,"statfs":{"total":95995035648,"available":94968246272,"internally_reserved":0,"allocated":994537472,"data_stored":993925554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":32240360},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.375,"15min":0.375},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.53600000000000003,"5min":0.53600000000000003,"15min":0.53600000000000003},"last":1.355},{"interface":"front","average":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"last":3.758}]},{"osd":1,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.376,"5min":0.376,"15min":0.376},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":0.56599999999999995,"5min":0.56599999999999995,"15min":0.56599999999999995},"last":0.71399999999999997},{"interface":"front","average":{"1min":0.41599999999999998,"5min":0.41599999999999998,"15min":0.41599999999999998},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"last":0.626}]},{"osd":2,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.41099999999999998,"5min":0.41099999999999998,"15min":0.41099999999999998},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.38900000000000001}]},{"osd":4,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.92000000000000004,"5min":0.92000000000000004,"15min":0.92000000000000004},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.62,"5min":0.62,"15min":0.62},"min":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"max":{"1min":0.93500000000000005,"5min":0.93500000000000005,"15min":0.93500000000000005},"last":3.855}]},{"osd":5,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.61699999999999999},"min":{"1min":0.39600000000000002,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":3.8180000000000001},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.45100000000000001,"5min":0.45100000000000001,"15min":0.45100000000000001},"max":{"1min":0.90400000000000003,"5min":0.90400000000000003,"15min":0.90400000000000003},"last":0.45100000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:55:58 2023","interfaces":[{"interface":"back","average":{"1min":0.627,"5min":0.627,"15min":0.627},"min":{"1min":0.44,"5min":0.44,"15min":0.44},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.64800000000000002,"15min":0.64800000000000002},"min":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"max":{"1min":0.89800000000000002,"5min":0.89800000000000002,"15min":0.89800000000000002},"last":0.41699999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019039,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2629180,"kb_used_data":2561460,"kb_used_omap":0,"kb_used_meta":67712,"kb_avail":91115972,"statfs":{"total":95995035648,"available":93302755328,"internally_reserved":0,"allocated":2622935040,"data_stored":2622379067,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":69337088},"hb_peers":[0,1,2,3,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:22 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":0.53700000000000003,"15min":0.52300000000000002},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.5049999999999999,"5min":3.5049999999999999,"15min":3.5049999999999999},"last":0.245},{"interface":"front","average":{"1min":0.71499999999999997,"5min":0.57599999999999996,"15min":0.54200000000000004},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.589,"5min":3.589,"15min":3.589},"last":0.59399999999999997}]},{"osd":1,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.59099999999999997,"15min":0.55100000000000005},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.786,"5min":2.786,"15min":2.786},"last":0.75800000000000001},{"interface":"front","average":{"1min":1.234,"5min":0.70699999999999996,"15min":0.60699999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":5.6900000000000004,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":0.33200000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.86299999999999999,"5min":0.67900000000000005,"15min":0.64800000000000002},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":4.1669999999999998,"5min":4.1669999999999998,"15min":4.1669999999999998},"last":0.57499999999999996},{"interface":"front","average":{"1min":0.79400000000000004,"5min":0.626,"15min":0.59799999999999998},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":3.1829999999999998,"5min":3.1829999999999998,"15min":3.1829999999999998},"last":0.36399999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.72899999999999998,"15min":0.64900000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6130000000000004,"5min":4.6130000000000004,"15min":4.6130000000000004},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.04,"5min":0.73299999999999998,"15min":0.68200000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":3.637,"5min":3.637,"15min":3.637},"last":1.5109999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:56:28 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":0.46300000000000002,"15min":0.438},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.81899999999999995,"5min":0.81899999999999995,"15min":1.141},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.48799999999999999,"5min":0.51300000000000001,"15min":0.45400000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.27500000000000002},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 17:55:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54200000000000004,"15min":0.50800000000000001},"min":{"1min":0.45400000000000001,"5min":0.33200000000000002,"15min":0.28899999999999998},"max":{"1min":0.64700000000000002,"5min":0.755,"15min":1.276},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.59299999999999997,"15min":0.54700000000000004},"min":{"1min":0.52500000000000002,"5min":0.29899999999999999,"15min":0.28999999999999998},"max":{"1min":0.79400000000000004,"5min":0.80900000000000005,"15min":1.1180000000000001},"last":0.505}]},{"osd":7,"last update":"Sat Sep 23 17:56:08 2023","interfaces":[{"interface":"back","average":{"1min":0.55100000000000005,"5min":0.56499999999999995,"15min":0.54700000000000004},"min":{"1min":0.378,"5min":0.31,"15min":0.28399999999999997},"max":{"1min":0.82399999999999995,"5min":0.82399999999999995,"15min":1.1839999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.53600000000000003,"5min":0.56499999999999995,"15min":0.53300000000000003},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":0.65200000000000002,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.41299999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757405,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2979744,"kb_used_data":2906904,"kb_used_omap":0,"kb_used_meta":72832,"kb_avail":90765408,"statfs":{"total":95995035648,"available":92943777792,"internally_reserved":0,"allocated":2976669696,"data_stored":2976129245,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":74579968},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.60899999999999999,"15min":0.58899999999999997},"min":{"1min":0.36299999999999999,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.6040000000000001,"5min":2.6040000000000001,"15min":2.6040000000000001},"last":3.2519999999999998},{"interface":"front","average":{"1min":0.78900000000000003,"5min":0.625,"15min":0.59899999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":2.6139999999999999,"5min":2.6139999999999999,"15min":2.6139999999999999},"last":3.2240000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.70499999999999996,"15min":0.71199999999999997},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.2330000000000001,"5min":4.4279999999999999,"15min":4.4279999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":1.1879999999999999,"5min":0.68200000000000005,"15min":0.60499999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4450000000000003,"5min":5.4450000000000003,"15min":5.4450000000000003},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:56:27 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":0.59699999999999998,"15min":0.55900000000000005},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":3.7130000000000001,"5min":3.7130000000000001,"15min":3.7130000000000001},"last":38.606999999999999},{"interface":"front","average":{"1min":0.67800000000000005,"5min":0.58899999999999997,"15min":0.57399999999999995},"min":{"1min":0.36299999999999999,"5min":0.36299999999999999,"15min":0.36299999999999999},"max":{"1min":1.9059999999999999,"5min":1.9059999999999999,"15min":1.9059999999999999},"last":39.670999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:56:57 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":0.78800000000000003,"15min":0.66400000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.4969999999999999,"5min":7.4969999999999999,"15min":7.4969999999999999},"last":3.2029999999999998},{"interface":"front","average":{"1min":1.0900000000000001,"5min":0.70199999999999996,"15min":0.63800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.5510000000000002,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":0.29699999999999999}]},{"osd":4,"last update":"Sat Sep 23 17:56:31 2023","interfaces":[{"interface":"back","average":{"1min":0.71999999999999997,"5min":0.53200000000000003,"15min":0.46899999999999997},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.25},"max":{"1min":1.802,"5min":1.802,"15min":1.802},"last":0.26300000000000001},{"interface":"front","average":{"1min":0.751,"5min":0.57799999999999996,"15min":0.53100000000000003},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.9780000000000002,"5min":2.9780000000000002,"15min":2.9780000000000002},"last":0.35199999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54400000000000004,"15min":0.53500000000000003},"min":{"1min":0.38800000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.66100000000000003,"5min":0.85499999999999998,"15min":1.778},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.45800000000000002,"5min":0.375,"15min":0.33200000000000002},"max":{"1min":0.72799999999999998,"5min":0.90200000000000002,"15min":1.7629999999999999},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:56:02 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":0.57699999999999996,"15min":0.53100000000000003},"min":{"1min":0.41399999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":1.276,"5min":1.276,"15min":1.276},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.59199999999999997,"15min":0.55600000000000005},"min":{"1min":0.46300000000000002,"5min":0.313,"15min":0.313},"max":{"1min":0.84999999999999998,"5min":0.84999999999999998,"15min":0.84999999999999998},"last":0.438}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2699296768,"data_stored":2699296768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1733558272,"data_stored":1733558272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2016411648,"data_stored":2016411648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":991887360,"data_stored":991887360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2636185600,"data_stored":2636185600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2991259648,"data_stored":2991259648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1663565824,"data_stored":1663565824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:56:58.866 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 1 2023-09-23T17:56:58.948 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:58.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:58.956 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:59.057 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:56:59.058 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[179668]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.058 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[179668]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.058 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[179668]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.058 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[179668]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.058 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[179668]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.058 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[179668]: osdmap e134: 8 total, 7 up, 8 in 2023-09-23T17:56:59.058 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[184174]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.059 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[184174]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.059 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[184174]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.059 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[184174]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.059 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[184174]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.059 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:58 smithi055 ceph-mon[184174]: osdmap e134: 8 total, 7 up, 8 in 2023-09-23T17:56:59.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:58 smithi099 ceph-mon[167076]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:58 smithi099 ceph-mon[167076]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:58 smithi099 ceph-mon[167076]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:58 smithi099 ceph-mon[167076]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:58 smithi099 ceph-mon[167076]: failed to encode map e133 with expected crc 2023-09-23T17:56:59.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:58 smithi099 ceph-mon[167076]: osdmap e134: 8 total, 7 up, 8 in 2023-09-23T17:56:59.167 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:59.234 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:56:59.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:59.243 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:56:59.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:56:59.378 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 61 16 82353 82337 84.2964 30.1875 0.015764 0.0118583 2023-09-23T17:56:59.410 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:59.435 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:56:59.465 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:56:59.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:59.469 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:56:59.482 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:56:59.518 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:56:59.560 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:56:59.570 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:56:59.589 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:56:59.616 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:59.627 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:56:59.640 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:56:59.671 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:56:59.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:59.675 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:56:59.683 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:56:59.683 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:56:59.684 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:56:59.684 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:56:59.684 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T17:56:59.684 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:56:59.684 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:56:59.684 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:56:59.684 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T17:56:59.720 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:56:59.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:56:59.812 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:56:59.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:56:59.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:56:59.922 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:56:59.930 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:59 smithi099 ceph-mon[167076]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:56:59.930 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:59 smithi099 ceph-mon[167076]: pgmap v283: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 27 MiB/s wr, 424 op/s; 19947/240618 objects degraded (8.290%); 671/240618 objects misplaced (0.279%); 13 MiB/s, 209 objects/s recovering 2023-09-23T17:56:59.931 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:59 smithi099 ceph-mon[167076]: failed to encode map e134 with expected crc 2023-09-23T17:56:59.931 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:59 smithi099 ceph-mon[167076]: failed to encode map e134 with expected crc 2023-09-23T17:56:59.931 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:56:59 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2877011403' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:56:59.970 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T17:57:00.002 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:59 smithi055 ceph-mon[184174]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:57:00.002 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:59 smithi055 ceph-mon[184174]: pgmap v283: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 27 MiB/s wr, 424 op/s; 19947/240618 objects degraded (8.290%); 671/240618 objects misplaced (0.279%); 13 MiB/s, 209 objects/s recovering 2023-09-23T17:57:00.002 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:59 smithi055 ceph-mon[184174]: failed to encode map e134 with expected crc 2023-09-23T17:57:00.002 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:59 smithi055 ceph-mon[184174]: failed to encode map e134 with expected crc 2023-09-23T17:57:00.003 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:56:59 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2877011403' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:00.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:59 smithi055 ceph-mon[179668]: from='client.34872 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:57:00.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:59 smithi055 ceph-mon[179668]: pgmap v283: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean, 11 active+undersized, 2 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 27 MiB/s wr, 424 op/s; 19947/240618 objects degraded (8.290%); 671/240618 objects misplaced (0.279%); 13 MiB/s, 209 objects/s recovering 2023-09-23T17:57:00.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:59 smithi055 ceph-mon[179668]: failed to encode map e134 with expected crc 2023-09-23T17:57:00.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:59 smithi055 ceph-mon[179668]: failed to encode map e134 with expected crc 2023-09-23T17:57:00.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:56:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2877011403' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:00.023 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:57:00.125 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:00.173 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:57:00.173 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:57:00.201 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:57:00.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:00.208 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:57:00.309 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:00.368 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:00.378 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 62 16 83281 83265 83.8724 58 0.010169 0.0119178 2023-09-23T17:57:00.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:00.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:00.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:00.577 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:57:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:57:00] "GET /metrics HTTP/1.1" 200 36533 "" "Prometheus/2.43.0" 2023-09-23T17:57:00.589 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:00.660 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:00.781 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:00.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:00.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:00.889 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:57:01.001 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:01.056 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:57:01.056 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:57:01.097 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:57:01.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:01.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:57:01.108 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:01.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:01.262 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:01.380 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 63 16 84252 84236 83.5014 60.6875 0.0161686 0.0119701 2023-09-23T17:57:01.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:01.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:01.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:01.460 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T17:57:01.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:01.539 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:01.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:01.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:01.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:01.699 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:01.837 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:01.902 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:01.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:01.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:02.010 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:02.046 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:02.100 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:02.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:02.106 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:02.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:01 smithi055 ceph-mon[179668]: pgmap v285: 89 pgs: 1 peering, 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 19853/245061 objects degraded (8.101%); 671/245061 objects misplaced (0.274%); 17 MiB/s, 269 objects/s recovering 2023-09-23T17:57:02.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:01 smithi055 ceph-mon[184174]: pgmap v285: 89 pgs: 1 peering, 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 19853/245061 objects degraded (8.101%); 671/245061 objects misplaced (0.274%); 17 MiB/s, 269 objects/s recovering 2023-09-23T17:57:02.174 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:57:02.207 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:02.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:01 smithi099 ceph-mon[167076]: pgmap v285: 89 pgs: 1 peering, 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 38 MiB/s wr, 612 op/s; 19853/245061 objects degraded (8.101%); 671/245061 objects misplaced (0.274%); 17 MiB/s, 269 objects/s recovering 2023-09-23T17:57:02.345 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:02.382 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 64 16 85101 85085 83.0242 53.0625 0.00616568 0.0120391 2023-09-23T17:57:02.480 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:02.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:02.496 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:02.506 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:02.597 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:02.644 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:02.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:02.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:02.701 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:02.802 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:02.870 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:02.926 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2997176816' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T17:57:02.926 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:02 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T17:57:02.926 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:02 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19853/245061 objects degraded (8.101%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:02.971 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 1 pgp_num to 1 2023-09-23T17:57:03.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2997176816' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T17:57:03.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:02 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T17:57:03.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:02 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19853/245061 objects degraded (8.101%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2997176816' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T17:57:03.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:02 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]: dispatch 2023-09-23T17:57:03.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:02 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19853/245061 objects degraded (8.101%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:03.382 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 65 16 86062 86046 82.6711 60.0625 0.0205933 0.012091 2023-09-23T17:57:04.007 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T17:57:04.187 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:04.193 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:04.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:04.294 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:57:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:03 smithi099 ceph-mon[167076]: pgmap v286: 89 pgs: 1 peering, 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s; 20086/249939 objects degraded (8.036%); 914/249939 objects misplaced (0.366%); 16 MiB/s, 248 objects/s recovering 2023-09-23T17:57:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:03 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T17:57:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:03 smithi099 ceph-mon[167076]: osdmap e135: 8 total, 7 up, 8 in 2023-09-23T17:57:04.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:03 smithi099 ceph-mon[167076]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:03 smithi099 ceph-mon[167076]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:03 smithi099 ceph-mon[167076]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:03 smithi099 ceph-mon[167076]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3399515487' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:57:04.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:03 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:57:04.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[179668]: pgmap v286: 89 pgs: 1 peering, 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s; 20086/249939 objects degraded (8.036%); 914/249939 objects misplaced (0.366%); 16 MiB/s, 248 objects/s recovering 2023-09-23T17:57:04.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T17:57:04.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[179668]: osdmap e135: 8 total, 7 up, 8 in 2023-09-23T17:57:04.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[179668]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[179668]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.353 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[179668]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[179668]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3399515487' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:57:04.354 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:57:04.354 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[184174]: pgmap v286: 89 pgs: 1 peering, 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s; 20086/249939 objects degraded (8.036%); 914/249939 objects misplaced (0.366%); 16 MiB/s, 248 objects/s recovering 2023-09-23T17:57:04.354 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "1"}]': finished 2023-09-23T17:57:04.355 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[184174]: osdmap e135: 8 total, 7 up, 8 in 2023-09-23T17:57:04.355 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[184174]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.355 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[184174]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.355 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[184174]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.355 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[184174]: failed to encode map e135 with expected crc 2023-09-23T17:57:04.355 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3399515487' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:57:04.355 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:03 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:57:04.361 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:04.382 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 66 16 87022 87006 82.3278 60 0.0151834 0.0121424 2023-09-23T17:57:05.178 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:04 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:05.186 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:04 smithi055 ceph-mon[184174]: failed to encode map e136 with expected crc 2023-09-23T17:57:05.186 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:04 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:57:05.186 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:04 smithi055 ceph-mon[184174]: osdmap e136: 8 total, 7 up, 8 in 2023-09-23T17:57:05.186 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:04 smithi055 ceph-mon[184174]: failed to encode map e136 with expected crc 2023-09-23T17:57:05.187 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:04 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:05.187 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:04 smithi055 ceph-mon[179668]: failed to encode map e136 with expected crc 2023-09-23T17:57:05.187 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:04 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:57:05.187 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:04 smithi055 ceph-mon[179668]: osdmap e136: 8 total, 7 up, 8 in 2023-09-23T17:57:05.187 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:04 smithi055 ceph-mon[179668]: failed to encode map e136 with expected crc 2023-09-23T17:57:05.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:04 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:05.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:04 smithi099 ceph-mon[167076]: failed to encode map e136 with expected crc 2023-09-23T17:57:05.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:04 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:57:05.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:04 smithi099 ceph-mon[167076]: osdmap e136: 8 total, 7 up, 8 in 2023-09-23T17:57:05.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:04 smithi099 ceph-mon[167076]: failed to encode map e136 with expected crc 2023-09-23T17:57:05.382 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 67 16 88069 88053 82.0757 65.4375 0.0277215 0.0121791 2023-09-23T17:57:06.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:05 smithi099 ceph-mon[167076]: pgmap v289: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 20667/256785 objects degraded (8.048%); 914/256785 objects misplaced (0.356%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T17:57:06.384 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 68 16 89136 89120 81.8469 66.6875 0.0140025 0.0122134 2023-09-23T17:57:06.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:05 smithi055 ceph-mon[179668]: pgmap v289: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 20667/256785 objects degraded (8.048%); 914/256785 objects misplaced (0.356%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T17:57:06.436 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:05 smithi055 ceph-mon[184174]: pgmap v289: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 56 MiB/s wr, 897 op/s; 20667/256785 objects degraded (8.048%); 914/256785 objects misplaced (0.356%); 6.8 MiB/s, 109 objects/s recovering 2023-09-23T17:57:07.147 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:57:07.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:07.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:57:07.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:07.309 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:07.382 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:07.384 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 69 16 90150 90134 81.5793 63.375 0.0284889 0.0122533 2023-09-23T17:57:07.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:07.389 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:07.419 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:07.419 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:07.419 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:07 smithi055 ceph-mon[179668]: pgmap v290: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 20667/258558 objects degraded (7.993%); 914/258558 objects misplaced (0.353%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T17:57:07.490 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:07.563 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:07.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:07.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:07.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:07 smithi099 ceph-mon[167076]: pgmap v290: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 20667/258558 objects degraded (7.993%); 914/258558 objects misplaced (0.353%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T17:57:07.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:07.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:07.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:07 smithi055 ceph-mon[184174]: pgmap v290: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.3 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 42 MiB/s wr, 672 op/s; 20667/258558 objects degraded (7.993%); 914/258558 objects misplaced (0.353%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T17:57:08.365 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:08.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:08.384 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 70 16 91162 91146 81.3176 63.25 0.024667 0.0122923 2023-09-23T17:57:08.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:08.526 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:08.578 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:08.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:08.585 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:08.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 20667/258558 objects degraded (7.993%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:08.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 20667/258558 objects degraded (7.993%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:08.685 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:08.779 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 20667/258558 objects degraded (7.993%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:08.993 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:09.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:09.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:09.055 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:09.117 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:09.206 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:09.342 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:09.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:09.344 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:09.384 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 71 16 92108 92092 81.0051 59.125 0.0103797 0.0123401 2023-09-23T17:57:09.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:09.484 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:09.530 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:09.540 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:09.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:09 smithi055 ceph-mon[184174]: pgmap v291: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s; 20964/264150 objects degraded (7.936%); 1209/264150 objects misplaced (0.458%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T17:57:09.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:09 smithi055 ceph-mon[179668]: pgmap v291: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s; 20964/264150 objects degraded (7.936%); 1209/264150 objects misplaced (0.458%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T17:57:09.641 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:09.710 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:09.742 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T17:57:09.794 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:09.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:09.804 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:09.811 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:57:09.812 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:57:09.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:09 smithi099 ceph-mon[167076]: pgmap v291: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s; 20964/264150 objects degraded (7.936%); 1209/264150 objects misplaced (0.458%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T17:57:09.906 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:09.948 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:10.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:10.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:10.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:10.121 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:10.206 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:10.329 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:10.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:10.334 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:10.385 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 72 16 92962 92946 80.6216 53.375 0.0321669 0.0123983 2023-09-23T17:57:10.404 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T17:57:10.435 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:10.484 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:57:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:57:10] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.43.0" 2023-09-23T17:57:10.586 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:10.662 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:10.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:10.675 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:10.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:10.822 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:10.880 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:10.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:10.884 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:10.984 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:57:11.041 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:11.084 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:11.120 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:57:11.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:11.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:57:11.202 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:57:11.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:11.243 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:57:11.266 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:11.294 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:57:11.308 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:11.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:11.313 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:11.338 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:57:11.382 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:57:11.392 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 73 16 93790 93774 80.2182 51.75 0.00801914 0.0124622 2023-09-23T17:57:11.413 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:11.421 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:57:11.469 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:57:11.469 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:57:11.469 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:57:11.469 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:57:11.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T17:57:11.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:57:11.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:57:11.470 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:57:11.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T17:57:11.475 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:11.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:11 smithi055 ceph-mon[184174]: pgmap v292: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s; 21620/272700 objects degraded (7.928%); 1209/272700 objects misplaced (0.443%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T17:57:11.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:11 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3677748511' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:11.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:11 smithi055 ceph-mon[179668]: pgmap v292: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s; 21620/272700 objects degraded (7.928%); 1209/272700 objects misplaced (0.443%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T17:57:11.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:11 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3677748511' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:11.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:11 smithi099 ceph-mon[167076]: pgmap v292: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.6 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 947 op/s; 21620/272700 objects degraded (7.928%); 1209/272700 objects misplaced (0.443%); 1.9 MiB/s, 30 objects/s recovering 2023-09-23T17:57:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:11 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3677748511' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:11.831 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:12.085 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:57:12.085 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:57:12.138 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:12.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:12.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:12.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:12.299 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:12.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:12.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:12.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:12.392 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 74 16 94697 94681 79.9004 56.6875 0.0137864 0.0125116 2023-09-23T17:57:12.454 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:12.545 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:12.665 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:12.674 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:12.675 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:12.776 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:12.873 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:12.957 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:12.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:12.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:13.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:13.119 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T17:57:13.119 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:57:13.119 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T17:57:13.119 INFO:tasks.thrashosds.thrasher:Setting osd 7 primary_affinity to 0.825230 2023-09-23T17:57:13.120 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 0.8252296396859677 2023-09-23T17:57:13.152 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:13.258 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:13.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:13.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:13.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:13.392 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 75 16 95488 95472 79.4944 49.4375 0.0238287 0.0125743 2023-09-23T17:57:13.401 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:13.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:13.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:13.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:13.564 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:13.607 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:13.663 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:13.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:13.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:13.672 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:13 smithi099 ceph-mon[167076]: pgmap v293: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 21906/278709 objects degraded (7.860%); 1498/278709 objects misplaced (0.537%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T17:57:13.672 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 21906/278709 objects degraded (7.860%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:13.672 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:13 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3838471662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:13.672 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:13 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:13.773 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:13.824 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T17:57:13.828 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:13.865 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:13 smithi055 ceph-mon[184174]: pgmap v293: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 21906/278709 objects degraded (7.860%); 1498/278709 objects misplaced (0.537%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T17:57:13.865 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 21906/278709 objects degraded (7.860%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:13.865 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:13 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3838471662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:13.865 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:13 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:13.865 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:13 smithi055 ceph-mon[179668]: pgmap v293: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 21906/278709 objects degraded (7.860%); 1498/278709 objects misplaced (0.537%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T17:57:13.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 21906/278709 objects degraded (7.860%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:13.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:13 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3838471662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:13.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:13 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:13.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:13.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:13.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:13.915 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:57:13.915 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T17:57:13.915 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T17:57:13.916 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T17:57:13.916 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:57:13.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:14.031 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:14.040 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:14.086 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:57:14.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:14.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:14.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:14.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:14.230 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:14.277 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:14.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:14.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:14.344 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T17:57:14.381 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:57:14.398 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 76 16 96324 96308 79.1361 52.25 0.0139068 0.0126314 2023-09-23T17:57:14.502 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:14.571 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:57:14.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:14.576 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:57:14.677 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:14.724 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:14 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:57:14.724 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:14 smithi055 ceph-mon[179668]: failed to encode map e137 with expected crc 2023-09-23T17:57:14.724 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:14 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:57:14.724 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:14 smithi055 ceph-mon[179668]: osdmap e137: 8 total, 7 up, 8 in 2023-09-23T17:57:14.724 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:14 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:57:14.725 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:14 smithi055 ceph-mon[184174]: failed to encode map e137 with expected crc 2023-09-23T17:57:14.725 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:14 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:57:14.725 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:14 smithi055 ceph-mon[184174]: osdmap e137: 8 total, 7 up, 8 in 2023-09-23T17:57:14.817 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:14.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:14 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:57:14.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:14 smithi099 ceph-mon[167076]: failed to encode map e137 with expected crc 2023-09-23T17:57:14.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:14 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:57:14.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:14 smithi099 ceph-mon[167076]: osdmap e137: 8 total, 7 up, 8 in 2023-09-23T17:57:14.994 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:14.999 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:15.100 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:15.150 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:15.206 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:15.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:15.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:15.314 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:15.381 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:15.392 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 77 16 97132 97116 78.7643 50.5 0.0261706 0.0126911 2023-09-23T17:57:15.429 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:15.583 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:15.588 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:15.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:15.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:15.691 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:57:15.769 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:16.040 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:15 smithi055 ceph-mon[179668]: pgmap v295: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 22456/285300 objects degraded (7.871%); 1498/285300 objects misplaced (0.525%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T17:57:16.040 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:15 smithi055 ceph-mon[184174]: pgmap v295: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 22456/285300 objects degraded (7.871%); 1498/285300 objects misplaced (0.525%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T17:57:16.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:15 smithi099 ceph-mon[167076]: pgmap v295: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 949 op/s; 22456/285300 objects degraded (7.871%); 1498/285300 objects misplaced (0.525%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T17:57:16.112 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:57:16.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:16.134 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:57:16.235 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:16.278 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:16.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:16.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:16.339 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:16.395 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 78 16 97982 97966 78.4332 53.125 0.00897104 0.0127448 2023-09-23T17:57:16.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:16.477 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:16.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:16.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:16.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:16.598 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T17:57:16.633 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:16.748 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:16.853 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:16.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:16.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:16.874 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.7 primary-affinity to 0.82523 (8540822) 2023-09-23T17:57:16.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1886702067' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8252296396859677}]: dispatch 2023-09-23T17:57:16.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1886702067' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8252296396859677}]: dispatch 2023-09-23T17:57:16.961 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:17.050 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:17.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1886702067' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8252296396859677}]: dispatch 2023-09-23T17:57:17.284 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:17.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:17.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:17.394 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:17.395 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 79 16 98790 98774 78.0798 50.5 0.0170682 0.012803 2023-09-23T17:57:17.472 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:17.721 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:57:17.721 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:17.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:17.732 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:17.776 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:57:17.833 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:17.837 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:57:17.870 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:17.897 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:57:17.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:17.935 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:57:17.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:17.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:17.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:57:18.010 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:57:18.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:57:18.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:57:18.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:57:18.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T17:57:18.011 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:57:18.011 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:57:18.011 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:57:18.011 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T17:57:18.026 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: pgmap v296: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s; 22456/286881 objects degraded (7.828%); 1498/286881 objects misplaced (0.522%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T17:57:18.026 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1886702067' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8252296396859677}]': finished 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: osdmap e138: 8 total, 7 up, 8 in 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1155233775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 22456/286881 objects degraded (7.828%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[179668]: osdmap e139: 8 total, 7 up, 8 in 2023-09-23T17:57:18.027 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: pgmap v296: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s; 22456/286881 objects degraded (7.828%); 1498/286881 objects misplaced (0.522%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1886702067' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8252296396859677}]': finished 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: osdmap e138: 8 total, 7 up, 8 in 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1155233775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 22456/286881 objects degraded (7.828%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:18.028 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:17 smithi055 ceph-mon[184174]: osdmap e139: 8 total, 7 up, 8 in 2023-09-23T17:57:18.038 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:18.103 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:18.116 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:18.203 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:57:18.204 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:57:18.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: pgmap v296: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 59 MiB/s wr, 943 op/s; 22456/286881 objects degraded (7.828%); 1498/286881 objects misplaced (0.522%); 3.0 MiB/s, 47 objects/s recovering 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1886702067' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 0.8252296396859677}]': finished 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: osdmap e138: 8 total, 7 up, 8 in 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: failed to encode map e138 with expected crc 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1155233775' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 22456/286881 objects degraded (7.828%), 9 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:17 smithi099 ceph-mon[167076]: osdmap e139: 8 total, 7 up, 8 in 2023-09-23T17:57:18.395 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T17:57:18.396569+0000 min lat: 0.00103404 max lat: 2.36625 avg lat: 0.0128581 2023-09-23T17:57:18.396 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T17:57:18.396 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 80 16 99603 99587 77.7392 50.8125 0.0123391 0.0128581 2023-09-23T17:57:18.611 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:18.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:18.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:18.720 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:18.759 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:18.773 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:57:18.773 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:57:18.818 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:18.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:18.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:18.838 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T17:57:18.923 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:19.030 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:18 smithi055 ceph-mon[179668]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.031 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:18 smithi055 ceph-mon[179668]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.031 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:18 smithi055 ceph-mon[179668]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.031 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:18 smithi055 ceph-mon[179668]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.031 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:18 smithi055 ceph-mon[179668]: osdmap e140: 8 total, 7 up, 8 in 2023-09-23T17:57:19.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:18 smithi055 ceph-mon[184174]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:18 smithi055 ceph-mon[184174]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:18 smithi055 ceph-mon[184174]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:18 smithi055 ceph-mon[184174]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.031 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:18 smithi055 ceph-mon[184174]: osdmap e140: 8 total, 7 up, 8 in 2023-09-23T17:57:19.040 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:19.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:18 smithi099 ceph-mon[167076]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:18 smithi099 ceph-mon[167076]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:18 smithi099 ceph-mon[167076]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:18 smithi099 ceph-mon[167076]: failed to encode map e139 with expected crc 2023-09-23T17:57:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:18 smithi099 ceph-mon[167076]: osdmap e140: 8 total, 7 up, 8 in 2023-09-23T17:57:19.164 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:19.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:19.169 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:19.270 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:19.348 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:19.395 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 81 16 100464 100448 77.4439 53.8125 0.0272082 0.0129082 2023-09-23T17:57:19.434 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:19.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:19.442 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:19.543 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:19.577 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:19.634 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:19.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:19.643 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:19.744 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:19.815 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:19.871 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:19.891 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:19.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:19.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:20.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:20.054 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:20.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:20.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:20.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:20.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[179668]: pgmap v299: 89 pgs: 1 remapped+peering, 11 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s; 22670/291393 objects degraded (7.780%); 1742/291393 objects misplaced (0.598%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T17:57:20.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[179668]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[179668]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[179668]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[179668]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[179668]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[179668]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[184174]: pgmap v299: 89 pgs: 1 remapped+peering, 11 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s; 22670/291393 objects degraded (7.780%); 1742/291393 objects misplaced (0.598%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T17:57:20.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[184174]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[184174]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[184174]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[184174]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[184174]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:19 smithi055 ceph-mon[184174]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.215 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:57:20.314 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:20.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:19 smithi099 ceph-mon[167076]: pgmap v299: 89 pgs: 1 remapped+peering, 11 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 5.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 44 MiB/s wr, 703 op/s; 22670/291393 objects degraded (7.780%); 1742/291393 objects misplaced (0.598%); 2.5 MiB/s, 40 objects/s recovering 2023-09-23T17:57:20.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:19 smithi099 ceph-mon[167076]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:19 smithi099 ceph-mon[167076]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:19 smithi099 ceph-mon[167076]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:19 smithi099 ceph-mon[167076]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:19 smithi099 ceph-mon[167076]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:19 smithi099 ceph-mon[167076]: failed to encode map e140 with expected crc 2023-09-23T17:57:20.395 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 82 16 101286 101270 77.1261 51.375 0.00700525 0.0129599 2023-09-23T17:57:20.486 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:57:20.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:20.500 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:57:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:57:20] "GET /metrics HTTP/1.1" 200 36535 "" "Prometheus/2.43.0" 2023-09-23T17:57:20.501 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:57:20.601 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:20.665 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:20.774 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:57:20.853 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:20.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:20.859 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:20.959 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:20.966 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T17:57:21.104 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:20 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1372161311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:57:21.106 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:21.110 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:20 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1372161311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:57:21.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:20 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1372161311' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:57:21.364 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:21.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:21.369 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:21.396 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 83 16 102105 102089 76.8138 51.1875 0.0192439 0.0130137 2023-09-23T17:57:21.469 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:21.556 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:21.634 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:21.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:21.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:21.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:21.788 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:21.806 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:21.849 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:21.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:21.855 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:21.907 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[179668]: pgmap v301: 89 pgs: 1 remapped+peering, 11 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 6.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s; 23205/298338 objects degraded (7.778%); 1742/298338 objects misplaced (0.584%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T17:57:21.907 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:21.908 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1372161311' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:57:21.908 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[179668]: osdmap e141: 8 total, 7 up, 8 in 2023-09-23T17:57:21.908 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[179668]: failed to encode map e141 with expected crc 2023-09-23T17:57:21.908 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[179668]: failed to encode map e141 with expected crc 2023-09-23T17:57:21.908 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[179668]: failed to encode map e141 with expected crc 2023-09-23T17:57:21.908 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:21.937 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:57:21.938 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:57:21.956 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:22.015 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:21 smithi099 ceph-mon[167076]: pgmap v301: 89 pgs: 1 remapped+peering, 11 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 6.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s; 23205/298338 objects degraded (7.778%); 1742/298338 objects misplaced (0.584%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T17:57:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:21 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1372161311' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:57:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:21 smithi099 ceph-mon[167076]: osdmap e141: 8 total, 7 up, 8 in 2023-09-23T17:57:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:21 smithi099 ceph-mon[167076]: failed to encode map e141 with expected crc 2023-09-23T17:57:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:21 smithi099 ceph-mon[167076]: failed to encode map e141 with expected crc 2023-09-23T17:57:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:21 smithi099 ceph-mon[167076]: failed to encode map e141 with expected crc 2023-09-23T17:57:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:22.105 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:22.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:22.111 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:22.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[184174]: pgmap v301: 89 pgs: 1 remapped+peering, 11 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 6.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 45 MiB/s wr, 723 op/s; 23205/298338 objects degraded (7.778%); 1742/298338 objects misplaced (0.584%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T17:57:22.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:22.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1372161311' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:57:22.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[184174]: osdmap e141: 8 total, 7 up, 8 in 2023-09-23T17:57:22.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[184174]: failed to encode map e141 with expected crc 2023-09-23T17:57:22.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[184174]: failed to encode map e141 with expected crc 2023-09-23T17:57:22.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[184174]: failed to encode map e141 with expected crc 2023-09-23T17:57:22.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:22.212 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:22.287 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:22.357 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:22.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:22.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:22.387 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T17:57:22.396 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 84 16 102958 102942 76.5342 53.3125 0.00515946 0.0130618 2023-09-23T17:57:22.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:22.508 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:22.564 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:22.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:22.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:22.670 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T17:57:22.762 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:22.803 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6] live_osds: [0, 1, 2, 3, 4, 5, 7] 2023-09-23T17:57:22.803 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:57:22.803 INFO:tasks.thrashosds.thrasher:Killing osd 3, live_osds are [0, 1, 2, 3, 4, 5, 7] 2023-09-23T17:57:22.803 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T17:57:22.804 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T17:57:23.033 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T17:57:23.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:23.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T17:57:23.141 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:23.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:22 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 23205/298338 objects degraded (7.778%), 9 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:23.180 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:57:22 smithi055 systemd[1]: Stopping Ceph osd.3 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:57:23.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:22 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 23205/298338 objects degraded (7.778%), 9 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:23.244 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:23.312 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:23.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:23.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:23.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:22 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 23205/298338 objects degraded (7.778%), 9 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:23.396 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 85 16 103706 103690 76.1839 46.75 0.0101078 0.0131202 2023-09-23T17:57:23.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:23.474 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:23.488 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:57:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[214630]: 2023-09-23T17:57:23.450+0000 7f02e2c72700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:57:23.488 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:57:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[214630]: 2023-09-23T17:57:23.450+0000 7f02e2c72700 -1 osd.3 141 *** Got signal Terminated *** 2023-09-23T17:57:23.488 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:57:23 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[214630]: 2023-09-23T17:57:23.450+0000 7f02e2c72700 -1 osd.3 141 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T17:57:23.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:23.533 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T17:57:23.533 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:23.534 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:23.635 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:23.690 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:23.770 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:23.776 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:57:23.818 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:57:23.873 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:57:23.907 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T17:57:23.930 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:23.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:23.938 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:23.953 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:57:23.986 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:57:24.017 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:57:24.017 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:57:24.018 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:57:24.018 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:57:24.018 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T17:57:24.018 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:57:24.018 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:57:24.018 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:57:24.018 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T17:57:24.039 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:24.071 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:24.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:23 smithi099 ceph-mon[167076]: pgmap v303: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 6.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 23459/303942 objects degraded (7.718%); 1987/303942 objects misplaced (0.654%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T17:57:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:23 smithi099 ceph-mon[167076]: osd.3 marked itself down and dead 2023-09-23T17:57:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:23 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/196286699' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:24.396 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 86 16 104161 104145 75.6291 28.4375 0.00428258 0.0131539 2023-09-23T17:57:24.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:23 smithi055 ceph-mon[184174]: pgmap v303: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 6.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 23459/303942 objects degraded (7.718%); 1987/303942 objects misplaced (0.654%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T17:57:24.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:23 smithi055 ceph-mon[184174]: osd.3 marked itself down and dead 2023-09-23T17:57:24.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:23 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/196286699' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:23 smithi055 ceph-mon[179668]: pgmap v303: 89 pgs: 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 57 active+clean, 11 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 6.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 63 MiB/s wr, 1.01k op/s; 23459/303942 objects degraded (7.718%); 1987/303942 objects misplaced (0.654%); 5.4 MiB/s, 86 objects/s recovering 2023-09-23T17:57:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:23 smithi055 ceph-mon[179668]: osd.3 marked itself down and dead 2023-09-23T17:57:24.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:23 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/196286699' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:24.761 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:24.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:24.765 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:24.866 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:24.897 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:25.069 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:57:25.069 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:57:25.137 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:25.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:25.143 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:25.220 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:57:25 smithi055 bash[243300]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T17:57:25.220 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:24 smithi055 ceph-mon[184174]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T17:57:25.221 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:24 smithi055 ceph-mon[184174]: osdmap e142: 8 total, 6 up, 8 in 2023-09-23T17:57:25.221 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:24 smithi055 ceph-mon[184174]: failed to encode map e142 with expected crc 2023-09-23T17:57:25.221 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:24 smithi055 ceph-mon[179668]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T17:57:25.221 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:24 smithi055 ceph-mon[179668]: osdmap e142: 8 total, 6 up, 8 in 2023-09-23T17:57:25.221 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:24 smithi055 ceph-mon[179668]: failed to encode map e142 with expected crc 2023-09-23T17:57:25.245 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:25.269 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:24 smithi099 ceph-mon[167076]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T17:57:25.269 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:24 smithi099 ceph-mon[167076]: osdmap e142: 8 total, 6 up, 8 in 2023-09-23T17:57:25.269 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:24 smithi099 ceph-mon[167076]: failed to encode map e142 with expected crc 2023-09-23T17:57:25.271 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:25.304 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:25.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:25.307 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:25.396 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 87 16 104360 104344 74.9032 12.4375 0.00442553 0.0132829 2023-09-23T17:57:25.407 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:25.433 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:25.467 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:25.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:25.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:25.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:25.597 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:25.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:25.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:25.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:25.732 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:25.742 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:25.758 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:25.793 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:25.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:25.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:25.897 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:25.931 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:25.978 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:25.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:25.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:26.065 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:57:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T17:57:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T17:57:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T17:57:26.066 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:57:26.080 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[184174]: pgmap v305: 89 pgs: 7 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 stale+active+clean, 4 stale+active+undersized, 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 43 active+clean, 7 active+undersized, 6 active+undersized+degraded; 6.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 23961/309948 objects degraded (7.731%); 1987/309948 objects misplaced (0.641%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T17:57:26.080 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T17:57:26.081 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[184174]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.081 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[184174]: osdmap e143: 8 total, 6 up, 8 in 2023-09-23T17:57:26.081 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[184174]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.081 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[184174]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.081 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[184174]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.082 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[179668]: pgmap v305: 89 pgs: 7 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 stale+active+clean, 4 stale+active+undersized, 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 43 active+clean, 7 active+undersized, 6 active+undersized+degraded; 6.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 23961/309948 objects degraded (7.731%); 1987/309948 objects misplaced (0.641%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T17:57:26.082 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T17:57:26.082 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[179668]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.082 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[179668]: osdmap e143: 8 total, 6 up, 8 in 2023-09-23T17:57:26.082 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[179668]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.082 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[179668]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.082 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:25 smithi055 ceph-mon[179668]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.084 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:26.122 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:26.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:25 smithi099 ceph-mon[167076]: pgmap v305: 89 pgs: 7 peering, 1 stale+active+recovery_wait+undersized+degraded+remapped, 7 stale+active+clean, 4 stale+active+undersized, 12 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 43 active+clean, 7 active+undersized, 6 active+undersized+degraded; 6.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 64 MiB/s wr, 1.03k op/s; 23961/309948 objects degraded (7.731%); 1987/309948 objects misplaced (0.641%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T17:57:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:25 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T17:57:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:25 smithi099 ceph-mon[167076]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:25 smithi099 ceph-mon[167076]: osdmap e143: 8 total, 6 up, 8 in 2023-09-23T17:57:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:25 smithi099 ceph-mon[167076]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:25 smithi099 ceph-mon[167076]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:25 smithi099 ceph-mon[167076]: failed to encode map e143 with expected crc 2023-09-23T17:57:26.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 88 16 104388 104372 74.0724 1.75 0.00497557 0.0133905 2023-09-23T17:57:26.467 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:26.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:26.482 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:26.512 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T17:57:26.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:26.623 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:26.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:26.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:26.673 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:26.774 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:26.875 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:26.990 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:27.014 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:27.070 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:57:27.115 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:27.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:27 smithi055 ceph-mon[179668]: failed to encode map e144 with expected crc 2023-09-23T17:57:27.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:27 smithi055 ceph-mon[179668]: osdmap e144: 8 total, 6 up, 8 in 2023-09-23T17:57:27.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:27 smithi055 ceph-mon[179668]: failed to encode map e144 with expected crc 2023-09-23T17:57:27.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:57:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:27 smithi055 ceph-mon[184174]: failed to encode map e144 with expected crc 2023-09-23T17:57:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:27 smithi055 ceph-mon[184174]: osdmap e144: 8 total, 6 up, 8 in 2023-09-23T17:57:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:27 smithi055 ceph-mon[184174]: failed to encode map e144 with expected crc 2023-09-23T17:57:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:57:27.264 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:27.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:27 smithi099 ceph-mon[167076]: failed to encode map e144 with expected crc 2023-09-23T17:57:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:27 smithi099 ceph-mon[167076]: osdmap e144: 8 total, 6 up, 8 in 2023-09-23T17:57:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:27 smithi099 ceph-mon[167076]: failed to encode map e144 with expected crc 2023-09-23T17:57:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:57:27.376 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:27.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:27.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:27.398 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 89 16 105007 104991 73.675 38.6875 0.0182394 0.0135661 2023-09-23T17:57:27.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:27.490 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:27.521 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:27.566 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:27.570 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:27.571 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:27.672 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:27.806 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:27.911 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:27.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:27.926 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:28.027 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:28.039 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:28 smithi055 ceph-mon[179668]: pgmap v308: 89 pgs: 1 remapped+peering, 7 peering, 3 stale+active+clean, 3 stale+active+undersized, 11 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 41 active+clean, 1 undersized+peered, 14 active+undersized, 6 active+undersized+degraded; 6.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 23537/310965 objects degraded (7.569%); 1987/310965 objects misplaced (0.639%); 2.8 MiB/s, 43 objects/s recovering 2023-09-23T17:57:28.040 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 23537/310965 objects degraded (7.569%), 8 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:28.040 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:28 smithi055 ceph-mon[184174]: pgmap v308: 89 pgs: 1 remapped+peering, 7 peering, 3 stale+active+clean, 3 stale+active+undersized, 11 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 41 active+clean, 1 undersized+peered, 14 active+undersized, 6 active+undersized+degraded; 6.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 23537/310965 objects degraded (7.569%); 1987/310965 objects misplaced (0.639%); 2.8 MiB/s, 43 objects/s recovering 2023-09-23T17:57:28.040 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 23537/310965 objects degraded (7.569%), 8 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:28.072 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:28.123 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:28.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:28.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:28.226 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:28.243 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T17:57:28.269 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:28.301 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:57:28 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.3.service: Succeeded. 2023-09-23T17:57:28.301 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 17:57:28 smithi055 systemd[1]: Stopped Ceph osd.3 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:57:28.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:28.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:28 smithi099 ceph-mon[167076]: pgmap v308: 89 pgs: 1 remapped+peering, 7 peering, 3 stale+active+clean, 3 stale+active+undersized, 11 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 41 active+clean, 1 undersized+peered, 14 active+undersized, 6 active+undersized+degraded; 6.3 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 44 MiB/s wr, 705 op/s; 23537/310965 objects degraded (7.569%); 1987/310965 objects misplaced (0.639%); 2.8 MiB/s, 43 objects/s recovering 2023-09-23T17:57:28.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 23537/310965 objects degraded (7.569%), 8 pgs degraded, 15 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:28.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:28.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:28.372 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T17:57:28.372 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T17:57:28.396 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total time run: 90.0172 2023-09-23T17:57:28.396 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total writes made: 105623 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Write size: 65536 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object size: 65536 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Bandwidth (MB/sec): 73.3353 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Bandwidth: 43.6062 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max bandwidth (MB/sec): 218.625 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min bandwidth (MB/sec): 0 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average IOPS: 1173 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev IOPS: 697.706 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max IOPS: 3498 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min IOPS: 0 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average Latency(s): 0.0136319 2023-09-23T17:57:28.397 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Latency(s): 0.0464031 2023-09-23T17:57:28.398 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max latency(s): 2.41599 2023-09-23T17:57:28.398 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min latency(s): 0.00103404 2023-09-23T17:57:28.405 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T17:57:28.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:28.469 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:28.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:28.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:28.522 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:28.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:28.668 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:28.717 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:28.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:28.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:28.773 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:28.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:28.861 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:28.908 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:28.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:28.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:29.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:29.043 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2969033626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:29.047 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:29.080 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T17:57:29.089 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:29.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:29.096 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:29.197 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:29.238 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2969033626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:29.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2969033626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:29.272 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:29.338 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:29.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:29.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:29.447 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:29.534 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:29.581 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:57:29.581 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T17:57:29.581 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104107 5.0 5:08ba7c2f:::benchmark_data_smithi055_229117_object104106:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.953750+0000", 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 5.522266503, 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3969089229999998, 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.582 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104107 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953750+0000", 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.583 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953750+0000", 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953748+0000", 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953776+0000", 2023-09-23T17:57:29.584 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8515000000000001e-05 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953798+0000", 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1855999999999999e-05 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953808+0000", 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0283e-05 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.585 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953876+0000", 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7348000000000006e-05 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op must wait for map", 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953885+0000", 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0839999999999998e-06 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.586 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.962444+0000", 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0085596219999999994 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.962456+0000", 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1606999999999999e-05 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949379+0000", 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98692283300000005 2023-09-23T17:57:29.587 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949390+0000", 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1503e-05 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.325692+0000", 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.376301955 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.588 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.327734+0000", 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020413749999999998 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.327871+0000", 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137739 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336120+0000", 2023-09-23T17:57:29.589 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082488500000000003 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336181+0000", 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1007000000000001e-05 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.348130+0000", 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011948676 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.590 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.348167+0000", 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6782000000000001e-05 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350658+0000", 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024918570000000001 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.591 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104131 5.0 5:02532558:::benchmark_data_smithi055_229117_object104130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.987392+0000", 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 5.4886239779999997, 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3655978989999999, 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104131 2023-09-23T17:57:29.592 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987392+0000", 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987392+0000", 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.593 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987390+0000", 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987415+0000", 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4185e-05 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.594 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987433+0000", 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8238000000000001e-05 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987442+0000", 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9600000000000006e-06 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.988601+0000", 2023-09-23T17:57:29.595 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011587889999999999 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.988612+0000", 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1969e-05 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949474+0000", 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96086169899999996 2023-09-23T17:57:29.596 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949482+0000", 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0099999999999995e-06 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333244+0000", 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3837614760000001 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.597 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333381+0000", 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013695800000000001 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333517+0000", 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000136181 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.598 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342436+0000", 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0089189990000000004 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342466+0000", 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0561000000000001e-05 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350747+0000", 2023-09-23T17:57:29.599 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082805090000000001 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350829+0000", 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1778000000000002e-05 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.352990+0000", 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021612770000000001 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.600 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104167 5.0 5:0cc1b975:::benchmark_data_smithi055_229117_object104166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:24.986469+0000", 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.4895472630000004, 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3687401239999999, 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.601 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104167 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986469+0000", 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.602 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986469+0000", 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986467+0000", 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.603 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986500+0000", 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3046000000000002e-05 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986515+0000", 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5138e-05 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986527+0000", 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1429e-05 2023-09-23T17:57:29.604 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.026788+0000", 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040261308000000003 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.026799+0000", 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0699999999999999e-05 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.605 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336245+0000", 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3094467810000001 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336378+0000", 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013300399999999999 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336525+0000", 2023-09-23T17:57:29.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014695100000000001 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342505+0000", 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0059792530000000003 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342518+0000", 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.348e-05 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353111+0000", 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010592839 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353201+0000", 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0167999999999995e-05 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355209+0000", 2023-09-23T17:57:29.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020078090000000002 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104189 5.0 5:00d05606:::benchmark_data_smithi055_229117_object104188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.012952+0000", 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.4630644899999998, 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.354677634, 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.609 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104189 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012952+0000", 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.610 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012952+0000", 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012950+0000", 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012983+0000", 2023-09-23T17:57:29.611 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2840999999999999e-05 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012997+0000", 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.454e-05 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.013006+0000", 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2269999999999999e-06 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.612 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.030199+0000", 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017192819000000002 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.030209+0000", 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.023e-05 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339148+0000", 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3089385840000001 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339354+0000", 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020608099999999999 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339570+0000", 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021573299999999999 2023-09-23T17:57:29.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342529+0000", 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029589759999999999 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342571+0000", 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1863999999999998e-05 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365363+0000", 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022792085 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365409+0000", 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6479e-05 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367629+0000", 2023-09-23T17:57:29.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022198840000000001 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104208 5.0 5:0c574cd2:::benchmark_data_smithi055_229117_object104207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.042202+0000", 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.4338143089999997, 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3303606880000001, 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.617 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104208 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042202+0000", 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042202+0000", 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042200+0000", 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042226+0000", 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5925e-05 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042245+0000", 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9449999999999998e-05 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042255+0000", 2023-09-23T17:57:29.620 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9739999999999997e-06 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042301+0000", 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5945999999999997e-05 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042309+0000", 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7130000000000005e-06 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.621 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342600+0000", 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3002915850000001 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342725+0000", 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012474099999999999 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342853+0000", 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000127806 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353066+0000", 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010212738000000001 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355236+0000", 2023-09-23T17:57:29.623 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002170052 2023-09-23T17:57:29.624 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.624 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.624 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.624 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369924+0000", 2023-09-23T17:57:29.624 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014688012 2023-09-23T17:57:29.624 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.624 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.624 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369984+0000", 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0571e-05 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372562+0000", 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00257807 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.625 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104228 5.0 5:04306094:::benchmark_data_smithi055_229117_object104227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.063372+0000", 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.4126436370000004, 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3247219290000001, 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104228 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.626 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063372+0000", 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063372+0000", 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.627 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063371+0000", 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063394+0000", 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2929999999999999e-05 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.628 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063407+0000", 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3142000000000001e-05 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063415+0000", 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8120000000000008e-06 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063476+0000", 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0988e-05 2023-09-23T17:57:29.629 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063486+0000", 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0397999999999999e-05 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345425+0000", 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2819386879999999 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.630 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345551+0000", 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012593200000000001 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345665+0000", 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114308 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.631 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355247+0000", 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0095815529999999996 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355307+0000", 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0619999999999999e-05 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385736+0000", 2023-09-23T17:57:29.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030428261000000002 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385803+0000", 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7624000000000002e-05 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388094+0000", 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002291015 2023-09-23T17:57:29.633 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104253 5.0 5:0d680df2:::benchmark_data_smithi055_229117_object104252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.088438+0000", 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.3875780190000002, 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3025030870000001, 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.634 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104253 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088438+0000", 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.635 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088438+0000", 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088436+0000", 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.636 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088461+0000", 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4658999999999999e-05 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088478+0000", 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7805e-05 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088487+0000", 2023-09-23T17:57:29.637 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.225e-06 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.092295+0000", 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038084870000000002 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.092306+0000", 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1172000000000001e-05 2023-09-23T17:57:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355342+0000", 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.263036211 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355533+0000", 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000190767 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355663+0000", 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012937699999999999 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365284+0000", 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0096211700000000001 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367649+0000", 2023-09-23T17:57:29.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002365514 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388536+0000", 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020886503000000001 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388577+0000", 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1167000000000001e-05 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.642 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.642 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.390941+0000", 2023-09-23T17:57:29.642 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002364157 2023-09-23T17:57:29.642 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.642 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.642 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.642 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.642 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.642 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104272 5.0 5:0ec51932:::benchmark_data_smithi055_229117_object104271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.642 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.108373+0000", 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.367642569, 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2933625550000001, 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104272 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.643 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108373+0000", 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108373+0000", 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.644 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108372+0000", 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108396+0000", 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4311000000000002e-05 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108412+0000", 2023-09-23T17:57:29.645 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6049000000000002e-05 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108420+0000", 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9170000000000006e-06 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108482+0000", 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1969000000000002e-05 2023-09-23T17:57:29.646 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108491+0000", 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4779999999999997e-06 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.358777+0000", 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2502855180000001 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.647 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.358912+0000", 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013505099999999999 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.359032+0000", 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120331 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.648 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367658+0000", 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0086259400000000003 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367698+0000", 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9477999999999997e-05 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.649 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.398598+0000", 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030900179 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.398665+0000", 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7167000000000003e-05 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401736+0000", 2023-09-23T17:57:29.650 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030708630000000001 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104276 5.0 5:0ba41185:::benchmark_data_smithi055_229117_object104275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.114398+0000", 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.3616175769999996, 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2928085359999999, 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.651 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104276 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114398+0000", 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.652 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114398+0000", 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114397+0000", 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.653 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114423+0000", 2023-09-23T17:57:29.654 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6006000000000001e-05 2023-09-23T17:57:29.654 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.654 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.654 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.654 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114440+0000", 2023-09-23T17:57:29.654 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6877000000000001e-05 2023-09-23T17:57:29.654 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.654 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.654 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.654 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114448+0000", 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5120000000000008e-06 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.115268+0000", 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00081976499999999997 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.115279+0000", 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0533999999999999e-05 2023-09-23T17:57:29.655 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362251+0000", 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.24697258 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362402+0000", 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015118400000000001 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.656 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362530+0000", 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012761700000000001 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369824+0000", 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072939889999999999 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372645+0000", 2023-09-23T17:57:29.657 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028209210000000001 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.405151+0000", 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032506363000000003 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.405203+0000", 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1987000000000002e-05 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.658 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407207+0000", 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002003777 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104298 5.0 5:040384f9:::benchmark_data_smithi055_229117_object104297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.139530+0000", 2023-09-23T17:57:29.659 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.3364855479999997, 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2697862209999999, 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104298 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.660 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139530+0000", 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139530+0000", 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139529+0000", 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:29.661 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139553+0000", 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4026999999999999e-05 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139565+0000", 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2364999999999999e-05 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.662 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139573+0000", 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4200000000000001e-06 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.141901+0000", 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023278639999999998 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.141909+0000", 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5099999999999998e-06 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372679+0000", 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2307700370000001 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372803+0000", 2023-09-23T17:57:29.664 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012344299999999999 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372931+0000", 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012818500000000001 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385669+0000", 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012738027000000001 2023-09-23T17:57:29.665 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385682+0000", 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3368e-05 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407273+0000", 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021590933 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407320+0000", 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7305999999999997e-05 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409317+0000", 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019962769999999999 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.667 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104301 5.0 5:0653eda0:::benchmark_data_smithi055_229117_object104300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.142080+0000", 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.3339363779999998, 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2806010670000001, 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.668 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104301 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142080+0000", 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142080+0000", 2023-09-23T17:57:29.669 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142078+0000", 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142104+0000", 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5931e-05 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.670 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142116+0000", 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2262999999999999e-05 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142125+0000", 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5359999999999992e-06 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142238+0000", 2023-09-23T17:57:29.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000113056 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142247+0000", 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6680000000000007e-06 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.375988+0000", 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2337410959999999 2023-09-23T17:57:29.672 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.376092+0000", 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104558 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.376220+0000", 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012734399999999999 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.674 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385691+0000", 2023-09-23T17:57:29.674 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0094714069999999994 2023-09-23T17:57:29.674 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.674 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.674 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.674 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391023+0000", 2023-09-23T17:57:29.674 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053320190000000003 2023-09-23T17:57:29.674 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.674 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.674 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.419987+0000", 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028963940000000001 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.420045+0000", 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8421000000000002e-05 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.675 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422681+0000", 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026352419999999999 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104305 5.0 5:03148f53:::benchmark_data_smithi055_229117_object104304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.146575+0000", 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.3294409460000001, 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2788299140000001, 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.676 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104305 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146575+0000", 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.677 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146575+0000", 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146573+0000", 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.678 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146600+0000", 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6675000000000001e-05 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146622+0000", 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1965000000000001e-05 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.679 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146633+0000", 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.076e-05 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.147763+0000", 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011302879999999999 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.147777+0000", 2023-09-23T17:57:29.680 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4129e-05 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379012+0000", 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2312345389999999 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379258+0000", 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024643700000000001 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.681 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379508+0000", 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024977000000000002 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.390966+0000", 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011458268000000001 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.682 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391089+0000", 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000122881 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422762+0000", 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031672697999999999 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422832+0000", 2023-09-23T17:57:29.683 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9929000000000001e-05 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.425405+0000", 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002573362 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104309 5.0 5:0de80b4f:::benchmark_data_smithi055_229117_object104308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.684 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.152292+0000", 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.3237236210000001, 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2892466869999999, 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104309 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.685 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152292+0000", 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152292+0000", 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152291+0000", 2023-09-23T17:57:29.686 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152313+0000", 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2082999999999998e-05 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152326+0000", 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2493999999999999e-05 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.687 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152332+0000", 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7710000000000001e-06 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152379+0000", 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6233999999999998e-05 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.688 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152389+0000", 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0234999999999999e-05 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382503+0000", 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.230114411 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382678+0000", 2023-09-23T17:57:29.689 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000174704 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382886+0000", 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000208481 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391101+0000", 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082150169999999998 2023-09-23T17:57:29.690 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391162+0000", 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0718999999999999e-05 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.438904+0000", 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047742297000000003 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.691 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.438971+0000", 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6787999999999998e-05 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441539+0000", 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025679069999999999 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.692 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.693 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.693 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104346 5.0 5:07875566:::benchmark_data_smithi055_229117_object104345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.693 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.224009+0000", 2023-09-23T17:57:29.693 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.2520066730000003, 2023-09-23T17:57:29.693 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2202360750000001, 2023-09-23T17:57:29.693 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.693 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.693 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.694 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.694 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.694 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104346 2023-09-23T17:57:29.694 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.694 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.694 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.694 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.694 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224009+0000", 2023-09-23T17:57:29.694 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.694 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224009+0000", 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224007+0000", 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.695 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224037+0000", 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9128e-05 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224058+0000", 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1829999999999999e-05 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224068+0000", 2023-09-23T17:57:29.696 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8560000000000002e-06 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224119+0000", 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0844000000000003e-05 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224128+0000", 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4880000000000007e-06 2023-09-23T17:57:29.697 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391218+0000", 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1670905920000001 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391390+0000", 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017210399999999999 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.698 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391557+0000", 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000166694 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409338+0000", 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017781376000000002 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413075+0000", 2023-09-23T17:57:29.699 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037367199999999998 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441577+0000", 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028502169000000001 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441616+0000", 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8859999999999997e-05 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.700 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444245+0000", 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026292899999999998 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104353 5.0 5:0b824901:::benchmark_data_smithi055_229117_object104352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.238109+0000", 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.2379070509999996, 2023-09-23T17:57:29.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.208967144, 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104353 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.702 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238109+0000", 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238109+0000", 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238108+0000", 2023-09-23T17:57:29.703 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238128+0000", 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9854999999999999e-05 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238135+0000", 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1729999999999998e-06 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238140+0000", 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6199999999999998e-06 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238197+0000", 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7005000000000001e-05 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.705 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238204+0000", 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2819999999999999e-06 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.394820+0000", 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.156615231 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.394956+0000", 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013646599999999999 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.395118+0000", 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000162158 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409348+0000", 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014229883 2023-09-23T17:57:29.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413141+0000", 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00379284 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444312+0000", 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031170989 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444373+0000", 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1309000000000005e-05 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447076+0000", 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027029129999999999 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.709 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104360 5.0 5:0a7d8202:::benchmark_data_smithi055_229117_object104359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.266784+0000", 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.2092318720000002, 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1830323949999999, 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.710 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104360 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266784+0000", 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266784+0000", 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.711 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266782+0000", 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266806+0000", 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3794999999999998e-05 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266824+0000", 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7838000000000001e-05 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266833+0000", 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9469999999999992e-06 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266890+0000", 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7238999999999999e-05 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266902+0000", 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1919999999999999e-05 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401792+0000", 2023-09-23T17:57:29.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.13489014 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401976+0000", 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000184014 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.402162+0000", 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018587599999999999 2023-09-23T17:57:29.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413089+0000", 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010926544999999999 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.425460+0000", 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012371320999999999 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.716 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447163+0000", 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021703538000000001 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447231+0000", 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7675999999999994e-05 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.449817+0000", 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025853859999999998 2023-09-23T17:57:29.717 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:105649 5.0 5:0a815552:::benchmark_data_smithi055_229117_object23:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:28.424114+0000", 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.051901838, 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.125478384, 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.718 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 105649 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.424114+0000", 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.424114+0000", 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.424112+0000", 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.720 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.424118+0000", 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5879999999999997e-06 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.424132+0000", 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3868e-05 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.424141+0000", 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6859999999999995e-06 2023-09-23T17:57:29.721 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.486990+0000", 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.062849184000000002 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.487751+0000", 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00076158499999999995 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.505668+0000", 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017916522000000001 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.505758+0000", 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0613999999999997e-05 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.505930+0000", 2023-09-23T17:57:29.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017112200000000001 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.517944+0000", 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012014113 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.546519+0000", 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028575555999999998 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.724 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.546563+0000", 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3809e-05 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.546619+0000", 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5986000000000002e-05 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.549593+0000", 2023-09-23T17:57:29.725 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029735130000000001 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:105944 5.0 5:0d5acdc6:::benchmark_data_smithi055_229117_object318:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:28.838443+0000", 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.63757260400000004, 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.129844548, 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.726 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 105944 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.838443+0000", 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.727 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.838443+0000", 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.838441+0000", 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.838450+0000", 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9099999999999994e-06 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.838465+0000", 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4656e-05 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.838474+0000", 2023-09-23T17:57:29.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9579999999999996e-06 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.904984+0000", 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.066510463000000006 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.905770+0000", 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00078572600000000005 2023-09-23T17:57:29.730 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.731 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.731 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.731 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.933011+0000", 2023-09-23T17:57:29.731 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027240951999999999 2023-09-23T17:57:29.731 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.731 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.731 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.731 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.933139+0000", 2023-09-23T17:57:29.731 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012785699999999999 2023-09-23T17:57:29.731 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.933310+0000", 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017065400000000001 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.956373+0000", 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.023063324999999999 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.732 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.965173+0000", 2023-09-23T17:57:29.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0088005140000000006 2023-09-23T17:57:29.733 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.733 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.733 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.965230+0000", 2023-09-23T17:57:29.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6978000000000002e-05 2023-09-23T17:57:29.733 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.733 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.733 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.965298+0000", 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7170999999999998e-05 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.968288+0000", 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029904520000000002 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:106015 5.0 5:08046527:::benchmark_data_smithi055_229117_object389:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:28.929515+0000", 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.54650094199999999, 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.12571169500000001, 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 106015 2023-09-23T17:57:29.735 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.929515+0000", 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.929515+0000", 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.929515+0000", 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.929517+0000", 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.548e-06 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.929522+0000", 2023-09-23T17:57:29.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3440000000000003e-06 2023-09-23T17:57:29.738 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.738 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.929526+0000", 2023-09-23T17:57:29.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6659999999999999e-06 2023-09-23T17:57:29.738 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.738 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.994289+0000", 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.064762491000000005 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:28.994970+0000", 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00068077100000000005 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.739 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.027605+0000", 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032635574000000001 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.027729+0000", 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000123404 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.740 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.027891+0000", 2023-09-23T17:57:29.741 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016201099999999999 2023-09-23T17:57:29.741 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.741 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.741 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.741 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.043074+0000", 2023-09-23T17:57:29.741 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015183574 2023-09-23T17:57:29.741 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.741 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.741 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.741 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.043084+0000", 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7939999999999997e-06 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.052389+0000", 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093053349999999996 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.052453+0000", 2023-09-23T17:57:29.742 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3893000000000006e-05 2023-09-23T17:57:29.743 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.743 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.743 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.055227+0000", 2023-09-23T17:57:29.743 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027736309999999999 2023-09-23T17:57:29.743 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.743 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.743 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.743 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:106180 5.0 5:07beba41:::benchmark_data_smithi055_229117_object554:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:29.185118+0000", 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.29089841799999999, 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13283357400000001, 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 106180 2023-09-23T17:57:29.744 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.185118+0000", 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.185118+0000", 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.185116+0000", 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.185122+0000", 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1940000000000003e-06 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.746 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.185138+0000", 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6575999999999999e-05 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.185147+0000", 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4230000000000005e-06 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.229302+0000", 2023-09-23T17:57:29.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.044155624999999997 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.230114+0000", 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00081162300000000003 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.261834+0000", 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031719646999999997 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.261937+0000", 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000103342 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.262072+0000", 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013527600000000001 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.275358+0000", 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01328608 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.275402+0000", 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3986000000000002e-05 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.314674+0000", 2023-09-23T17:57:29.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.039271268999999998 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.314752+0000", 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8172999999999998e-05 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:29.317951+0000", 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031993109999999998 2023-09-23T17:57:29.751 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.752 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.752 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.752 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:29.752 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:29.752 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:57:30.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[184174]: pgmap v309: 89 pgs: 1 remapped+peering, 7 peering, 1 stale+active+clean, 1 stale+active+undersized, 7 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 34 active+clean, 4 undersized+peered, 22 active+undersized, 1 active+undersized+remapped, 9 active+undersized+degraded; 6.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 523 op/s; 26189/313380 objects degraded (8.357%); 6230/313380 objects misplaced (1.988%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T17:57:30.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:30.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2969033626' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:57:30.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[184174]: osdmap e145: 8 total, 6 up, 8 in 2023-09-23T17:57:30.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[184174]: failed to encode map e145 with expected crc 2023-09-23T17:57:30.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[184174]: failed to encode map e145 with expected crc 2023-09-23T17:57:30.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[179668]: pgmap v309: 89 pgs: 1 remapped+peering, 7 peering, 1 stale+active+clean, 1 stale+active+undersized, 7 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 34 active+clean, 4 undersized+peered, 22 active+undersized, 1 active+undersized+remapped, 9 active+undersized+degraded; 6.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 523 op/s; 26189/313380 objects degraded (8.357%); 6230/313380 objects misplaced (1.988%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T17:57:30.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:30.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2969033626' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:57:30.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[179668]: osdmap e145: 8 total, 6 up, 8 in 2023-09-23T17:57:30.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[179668]: failed to encode map e145 with expected crc 2023-09-23T17:57:30.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-mon[179668]: failed to encode map e145 with expected crc 2023-09-23T17:57:30.258 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:30.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:30.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:30 smithi099 ceph-mon[167076]: pgmap v309: 89 pgs: 1 remapped+peering, 7 peering, 1 stale+active+clean, 1 stale+active+undersized, 7 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 34 active+clean, 4 undersized+peered, 22 active+undersized, 1 active+undersized+remapped, 9 active+undersized+degraded; 6.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 33 MiB/s wr, 523 op/s; 26189/313380 objects degraded (8.357%); 6230/313380 objects misplaced (1.988%); 3.8 MiB/s, 61 objects/s recovering 2023-09-23T17:57:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:30 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2969033626' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:57:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:30 smithi099 ceph-mon[167076]: osdmap e145: 8 total, 6 up, 8 in 2023-09-23T17:57:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:30 smithi099 ceph-mon[167076]: failed to encode map e145 with expected crc 2023-09-23T17:57:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:30 smithi099 ceph-mon[167076]: failed to encode map e145 with expected crc 2023-09-23T17:57:30.364 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:30.440 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:30.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:57:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:57:30] "GET /metrics HTTP/1.1" 200 36534 "" "Prometheus/2.43.0" 2023-09-23T17:57:31.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:31 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:57:31.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:31 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:57:31.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:31 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:57:32.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:32 smithi055 ceph-mon[184174]: pgmap v311: 89 pgs: 1 remapped+peering, 7 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 170 B/s rd, 20 MiB/s wr, 349 op/s; 32869/315261 objects degraded (10.426%); 6230/315261 objects misplaced (1.976%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T17:57:32.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:32 smithi055 ceph-mon[179668]: pgmap v311: 89 pgs: 1 remapped+peering, 7 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 170 B/s rd, 20 MiB/s wr, 349 op/s; 32869/315261 objects degraded (10.426%); 6230/315261 objects misplaced (1.976%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T17:57:32.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:32 smithi099 ceph-mon[167076]: pgmap v311: 89 pgs: 1 remapped+peering, 7 active+clean+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.4 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 170 B/s rd, 20 MiB/s wr, 349 op/s; 32869/315261 objects degraded (10.426%); 6230/315261 objects misplaced (1.976%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T17:57:32.647 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T17:57:33.373 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6, 3] live_osds: [0, 1, 2, 4, 5, 7] 2023-09-23T17:57:33.373 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:57:33.373 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T17:57:33.374 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:57:33.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:33 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:33 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:33 smithi099 ceph-mon[167076]: pgmap v312: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 135 B/s rd, 19 MiB/s wr, 513 op/s; 39289/312348 objects degraded (12.579%); 6333/312348 objects misplaced (2.028%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T17:57:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 32869/315261 objects degraded (10.426%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:33.628 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:33.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:33.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:33.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:33 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:33.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:33 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:33.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:33 smithi055 ceph-mon[179668]: pgmap v312: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 135 B/s rd, 19 MiB/s wr, 513 op/s; 39289/312348 objects degraded (12.579%); 6333/312348 objects misplaced (2.028%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T17:57:33.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 32869/315261 objects degraded (10.426%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:33.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:33 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:33.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:33 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:33.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:33 smithi055 ceph-mon[184174]: pgmap v312: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.4 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 135 B/s rd, 19 MiB/s wr, 513 op/s; 39289/312348 objects degraded (12.579%); 6333/312348 objects misplaced (2.028%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T17:57:33.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 32869/315261 objects degraded (10.426%), 14 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:33.738 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:33.793 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:33.846 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:33.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:33.852 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:33.953 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:34.002 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:34.312 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T17:57:34.446 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:34.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:34.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:34.555 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:34.693 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:35.678 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:35.680 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:35.684 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:35.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:35.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:35.786 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:35.822 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:35 smithi099 ceph-mon[167076]: pgmap v313: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 119 B/s rd, 15 MiB/s wr, 636 op/s; 38580/306666 objects degraded (12.580%); 6333/306666 objects misplaced (2.065%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T17:57:35.826 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:35.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:35.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:35.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:35 smithi055 ceph-mon[184174]: pgmap v313: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 119 B/s rd, 15 MiB/s wr, 636 op/s; 38580/306666 objects degraded (12.580%); 6333/306666 objects misplaced (2.065%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T17:57:35.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:35 smithi055 ceph-mon[179668]: pgmap v313: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 119 B/s rd, 15 MiB/s wr, 636 op/s; 38580/306666 objects degraded (12.580%); 6333/306666 objects misplaced (2.065%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T17:57:35.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:36.013 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:36.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:36.107 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:36.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:36.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:36.260 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:36.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:36.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:36.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:36.411 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:36.478 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:36.953 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:36.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:36.958 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:37.059 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:37.087 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:57:37.089 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:37.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:37.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:37.130 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:37.148 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:57:37.203 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:57:37.230 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:37.231 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:57:37.273 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:57:37.310 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:57:37.310 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:57:37.311 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:57:37.311 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:57:37.311 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:57:37.311 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:57:37.311 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:57:37.311 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:57:37.311 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T17:57:37.342 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:37.420 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:37.421 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:37.421 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:37 smithi055 ceph-mon[179668]: pgmap v314: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 588 op/s; 38336/305295 objects degraded (12.557%); 6235/305295 objects misplaced (2.042%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T17:57:37.421 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1750098633' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:37.421 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:37.421 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:37.421 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:37 smithi055 ceph-mon[184174]: pgmap v314: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 588 op/s; 38336/305295 objects degraded (12.557%); 6235/305295 objects misplaced (2.042%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T17:57:37.421 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1750098633' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:37.500 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:37.505 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:37.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:37.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:37.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:37.658 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:37.700 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:37.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:37.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:37.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:37 smithi099 ceph-mon[167076]: pgmap v314: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.2 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 12 MiB/s wr, 588 op/s; 38336/305295 objects degraded (12.557%); 6235/305295 objects misplaced (2.042%); 6.0 MiB/s, 95 objects/s recovering 2023-09-23T17:57:37.702 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1750098633' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:37.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:37.703 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:37.804 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:37.881 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:38.052 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:38.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:38.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:38.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:38.192 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:38.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:38.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:38.241 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:38.341 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:38.374 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:38.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:38.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:38.427 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:38.439 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:57:38.440 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:57:38.488 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 38336/305295 objects degraded (12.557%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:38.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 38336/305295 objects degraded (12.557%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:38.529 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:38.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 38336/305295 objects degraded (12.557%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:38.621 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:39.254 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:39.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:39.268 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:39.369 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:39.442 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:39 smithi055 ceph-mon[179668]: pgmap v315: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 7.4 MiB/s wr, 644 op/s; 37830/301185 objects degraded (12.560%); 6025/301185 objects misplaced (2.000%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T17:57:39.457 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:39.648 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:39.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:39.658 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:39.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:39 smithi055 ceph-mon[184174]: pgmap v315: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 7.4 MiB/s wr, 644 op/s; 37830/301185 objects degraded (12.560%); 6025/301185 objects misplaced (2.000%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T17:57:39.753 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:57:39.759 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:39.770 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:57:39.823 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:39 smithi099 ceph-mon[167076]: pgmap v315: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.1 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 102 B/s rd, 7.4 MiB/s wr, 644 op/s; 37830/301185 objects degraded (12.560%); 6025/301185 objects misplaced (2.000%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T17:57:39.905 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:39.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:39.909 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:40.010 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:40.109 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:40.109 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:57:40.109 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:57:40.442 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:57:40.517 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":315,"stamp":"2023-09-23T17:57:38.480892+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6582376395,"num_objects":100395,"num_object_clones":0,"num_object_copies":301185,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":37830,"num_objects_misplaced":6025,"num_objects_unfound":0,"num_objects_dirty":100395,"num_whiteouts":0,"num_read":6495,"num_read_kb":13113,"num_write":112027,"num_write_kb":6775014,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4447,"num_bytes_recovered":290850624,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112073,"ondisk_log_size":112073,"up":202,"acting":213,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":279,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":18676948,"kb_used_data":18096084,"kb_used_omap":29,"kb_used_meta":580706,"kb_avail":731284268,"statfs":{"total":767960285184,"available":748835090432,"internally_reserved":0,"allocated":18530390016,"data_stored":18525207135,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":30208,"internal_metadata":594643456},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,5,4,9,4,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-266469352,"num_objects":-4065,"num_object_clones":0,"num_object_copies":-12195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11641,"num_objects_misplaced":-205,"num_objects_unfound":0,"num_objects_dirty":-4065,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6449,"num_write_kb":76225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":854,"num_bytes_recovered":55967744,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005620"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":28,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:25.961409+0000","last_change":"2023-09-23T17:56:50.838338+0000","last_active":"2023-09-23T17:57:25.961409+0000","last_peered":"2023-09-23T17:57:25.961409+0000","last_clean":"2023-09-23T17:57:25.961409+0000","last_became_active":"2023-09-23T17:56:50.837963+0000","last_became_peered":"2023-09-23T17:56:50.837963+0000","last_unstale":"2023-09-23T17:57:25.961409+0000","last_undegraded":"2023-09-23T17:57:25.961409+0000","last_fullsized":"2023-09-23T17:57:25.961409+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:36.238235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":110,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.957260+0000","last_change":"2023-09-23T17:57:24.956156+0000","last_active":"2023-09-23T17:57:25.957260+0000","last_peered":"2023-09-23T17:57:25.957260+0000","last_clean":"2023-09-23T17:56:46.613857+0000","last_became_active":"2023-09-23T17:57:24.956156+0000","last_became_peered":"2023-09-23T17:57:24.956156+0000","last_unstale":"2023-09-23T17:57:25.957260+0000","last_undegraded":"2023-09-23T17:57:25.957260+0000","last_fullsized":"2023-09-23T17:57:24.947251+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:35.949179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":108,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.954490+0000","last_change":"2023-09-23T17:57:24.956353+0000","last_active":"2023-09-23T17:57:25.954490+0000","last_peered":"2023-09-23T17:57:25.954490+0000","last_clean":"2023-09-23T17:57:20.922940+0000","last_became_active":"2023-09-23T17:57:24.956353+0000","last_became_peered":"2023-09-23T17:57:24.956353+0000","last_unstale":"2023-09-23T17:57:25.954490+0000","last_undegraded":"2023-09-23T17:57:25.954490+0000","last_fullsized":"2023-09-23T17:57:24.946863+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:41.136418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":95,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.269843+0000","last_change":"2023-09-23T17:57:24.955362+0000","last_active":"2023-09-23T17:57:26.269843+0000","last_peered":"2023-09-23T17:57:26.269843+0000","last_clean":"2023-09-23T17:57:20.931176+0000","last_became_active":"2023-09-23T17:57:24.955362+0000","last_became_peered":"2023-09-23T17:57:24.955362+0000","last_unstale":"2023-09-23T17:57:26.269843+0000","last_undegraded":"2023-09-23T17:57:26.269843+0000","last_fullsized":"2023-09-23T17:57:24.948212+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:32.961526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":91,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.313646+0000","last_change":"2023-09-23T17:57:24.965072+0000","last_active":"2023-09-23T17:57:26.313646+0000","last_peered":"2023-09-23T17:57:26.313646+0000","last_clean":"2023-09-23T17:53:21.039477+0000","last_became_active":"2023-09-23T17:57:24.965072+0000","last_became_peered":"2023-09-23T17:57:24.965072+0000","last_unstale":"2023-09-23T17:57:26.313646+0000","last_undegraded":"2023-09-23T17:57:26.313646+0000","last_fullsized":"2023-09-23T17:57:24.947630+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:41.627468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":91,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.991099+0000","last_change":"2023-09-23T17:55:58.864932+0000","last_active":"2023-09-23T17:57:25.991099+0000","last_peered":"2023-09-23T17:57:25.991099+0000","last_clean":"2023-09-23T17:53:21.027423+0000","last_became_active":"2023-09-23T17:55:58.864932+0000","last_became_peered":"2023-09-23T17:55:58.864932+0000","last_unstale":"2023-09-23T17:57:25.991099+0000","last_undegraded":"2023-09-23T17:57:25.991099+0000","last_fullsized":"2023-09-23T17:55:58.862677+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:37.192868+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":93,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.269780+0000","last_change":"2023-09-23T17:57:24.957628+0000","last_active":"2023-09-23T17:57:26.269780+0000","last_peered":"2023-09-23T17:57:26.269780+0000","last_clean":"2023-09-23T17:57:20.932733+0000","last_became_active":"2023-09-23T17:57:24.957628+0000","last_became_peered":"2023-09-23T17:57:24.957628+0000","last_unstale":"2023-09-23T17:57:26.269780+0000","last_undegraded":"2023-09-23T17:57:26.269780+0000","last_fullsized":"2023-09-23T17:57:24.946783+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:10.900318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":87,"reported_epoch":144,"state":"undersized+peered","last_fresh":"2023-09-23T17:57:25.956081+0000","last_change":"2023-09-23T17:57:24.949992+0000","last_active":"2023-09-23T17:55:57.848462+0000","last_peered":"2023-09-23T17:57:25.956081+0000","last_clean":"2023-09-23T17:55:56.840611+0000","last_became_active":"2023-09-23T17:54:57.034128+0000","last_became_peered":"2023-09-23T17:57:24.949992+0000","last_unstale":"2023-09-23T17:57:25.956081+0000","last_undegraded":"2023-09-23T17:57:25.956081+0000","last_fullsized":"2023-09-23T17:57:24.948905+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:56.976253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":105,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:25.987930+0000","last_change":"2023-09-23T17:53:52.732689+0000","last_active":"2023-09-23T17:57:25.987930+0000","last_peered":"2023-09-23T17:57:25.987930+0000","last_clean":"2023-09-23T17:57:25.987930+0000","last_became_active":"2023-09-23T17:53:52.732382+0000","last_became_peered":"2023-09-23T17:53:52.732382+0000","last_unstale":"2023-09-23T17:57:25.987930+0000","last_undegraded":"2023-09-23T17:57:25.987930+0000","last_fullsized":"2023-09-23T17:57:25.987930+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:34.408802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":40,"reported_epoch":144,"state":"undersized+peered","last_fresh":"2023-09-23T17:57:26.353649+0000","last_change":"2023-09-23T17:57:24.950382+0000","last_active":"2023-09-23T17:55:57.848062+0000","last_peered":"2023-09-23T17:57:26.353649+0000","last_clean":"2023-09-23T17:55:56.840970+0000","last_became_active":"2023-09-23T17:54:57.037551+0000","last_became_peered":"2023-09-23T17:57:24.950382+0000","last_unstale":"2023-09-23T17:57:26.353649+0000","last_undegraded":"2023-09-23T17:57:26.353649+0000","last_fullsized":"2023-09-23T17:57:24.947596+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:38.134554+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7],"acting":[7],"avail_no_missing":["7"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":149,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:26.353636+0000","last_change":"2023-09-23T17:57:24.949323+0000","last_active":"2023-09-23T17:57:26.353636+0000","last_peered":"2023-09-23T17:57:26.353636+0000","last_clean":"2023-09-23T17:57:20.919726+0000","last_became_active":"2023-09-23T17:57:24.949323+0000","last_became_peered":"2023-09-23T17:57:24.949323+0000","last_unstale":"2023-09-23T17:57:26.353636+0000","last_undegraded":"2023-09-23T17:57:24.947406+0000","last_fullsized":"2023-09-23T17:57:24.947270+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:52.316323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":90,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.954426+0000","last_change":"2023-09-23T17:55:58.865485+0000","last_active":"2023-09-23T17:57:25.954426+0000","last_peered":"2023-09-23T17:57:25.954426+0000","last_clean":"2023-09-23T17:55:57.820806+0000","last_became_active":"2023-09-23T17:55:58.865485+0000","last_became_peered":"2023-09-23T17:55:58.865485+0000","last_unstale":"2023-09-23T17:57:25.954426+0000","last_undegraded":"2023-09-23T17:57:25.954426+0000","last_fullsized":"2023-09-23T17:55:58.859630+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:06:40.150526+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":123,"reported_epoch":144,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:57:25.961922+0000","last_change":"2023-09-23T17:56:51.845979+0000","last_active":"2023-09-23T17:57:25.961922+0000","last_peered":"2023-09-23T17:57:25.961922+0000","last_clean":"2023-09-23T17:57:25.961922+0000","last_became_active":"2023-09-23T17:56:51.844991+0000","last_became_peered":"2023-09-23T17:56:51.844991+0000","last_unstale":"2023-09-23T17:57:25.961922+0000","last_undegraded":"2023-09-23T17:57:25.961922+0000","last_fullsized":"2023-09-23T17:57:25.961922+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:12.346538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":117,"reported_epoch":144,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:57:25.961727+0000","last_change":"2023-09-23T17:56:51.839104+0000","last_active":"2023-09-23T17:57:25.961727+0000","last_peered":"2023-09-23T17:57:25.961727+0000","last_clean":"2023-09-23T17:57:25.961727+0000","last_became_active":"2023-09-23T17:56:51.838692+0000","last_became_peered":"2023-09-23T17:56:51.838692+0000","last_unstale":"2023-09-23T17:57:25.961727+0000","last_undegraded":"2023-09-23T17:57:25.961727+0000","last_fullsized":"2023-09-23T17:57:25.961727+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:04:47.909612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1,5],"avail_no_missing":["4","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":133,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:26.352984+0000","last_change":"2023-09-23T17:56:51.852184+0000","last_active":"2023-09-23T17:57:26.352984+0000","last_peered":"2023-09-23T17:57:26.352984+0000","last_clean":"2023-09-23T17:57:26.352984+0000","last_became_active":"2023-09-23T17:56:51.851861+0000","last_became_peered":"2023-09-23T17:56:51.851861+0000","last_unstale":"2023-09-23T17:57:26.352984+0000","last_undegraded":"2023-09-23T17:57:26.352984+0000","last_fullsized":"2023-09-23T17:57:26.352984+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:56.294669+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":73,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.955007+0000","last_change":"2023-09-23T17:57:24.954701+0000","last_active":"2023-09-23T17:57:25.955007+0000","last_peered":"2023-09-23T17:57:25.955007+0000","last_clean":"2023-09-23T17:53:21.028332+0000","last_became_active":"2023-09-23T17:57:24.954701+0000","last_became_peered":"2023-09-23T17:57:24.954701+0000","last_unstale":"2023-09-23T17:57:25.955007+0000","last_undegraded":"2023-09-23T17:57:25.955007+0000","last_fullsized":"2023-09-23T17:57:24.946465+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:35.499119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":93,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.269392+0000","last_change":"2023-09-23T17:57:24.956897+0000","last_active":"2023-09-23T17:57:26.269392+0000","last_peered":"2023-09-23T17:57:26.269392+0000","last_clean":"2023-09-23T17:57:20.931103+0000","last_became_active":"2023-09-23T17:57:24.956897+0000","last_became_peered":"2023-09-23T17:57:24.956897+0000","last_unstale":"2023-09-23T17:57:26.269392+0000","last_undegraded":"2023-09-23T17:57:26.269392+0000","last_fullsized":"2023-09-23T17:57:24.946095+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:37.234313+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":111,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.987785+0000","last_change":"2023-09-23T17:57:24.954648+0000","last_active":"2023-09-23T17:57:25.987785+0000","last_peered":"2023-09-23T17:57:25.987785+0000","last_clean":"2023-09-23T17:57:20.925545+0000","last_became_active":"2023-09-23T17:57:24.954648+0000","last_became_peered":"2023-09-23T17:57:24.954648+0000","last_unstale":"2023-09-23T17:57:25.987785+0000","last_undegraded":"2023-09-23T17:57:25.987785+0000","last_fullsized":"2023-09-23T17:57:24.949475+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:45.012989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":113,"reported_epoch":144,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:57:25.961682+0000","last_change":"2023-09-23T17:57:18.877437+0000","last_active":"2023-09-23T17:57:25.961682+0000","last_peered":"2023-09-23T17:57:25.961682+0000","last_clean":"2023-09-23T17:57:25.961682+0000","last_became_active":"2023-09-23T17:57:18.877007+0000","last_became_peered":"2023-09-23T17:57:18.877007+0000","last_unstale":"2023-09-23T17:57:25.961682+0000","last_undegraded":"2023-09-23T17:57:25.961682+0000","last_fullsized":"2023-09-23T17:57:25.961682+0000","mapping_epoch":139,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":140,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:32.195613+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,0],"avail_no_missing":["4","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":94,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.269697+0000","last_change":"2023-09-23T17:57:24.955081+0000","last_active":"2023-09-23T17:57:26.269697+0000","last_peered":"2023-09-23T17:57:26.269697+0000","last_clean":"2023-09-23T17:57:20.931097+0000","last_became_active":"2023-09-23T17:57:24.955081+0000","last_became_peered":"2023-09-23T17:57:24.955081+0000","last_unstale":"2023-09-23T17:57:26.269697+0000","last_undegraded":"2023-09-23T17:57:26.269697+0000","last_fullsized":"2023-09-23T17:57:24.947898+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:00.439681+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7],"avail_no_missing":["0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":50,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:26.313558+0000","last_change":"2023-09-23T17:55:58.867766+0000","last_active":"2023-09-23T17:57:26.313558+0000","last_peered":"2023-09-23T17:57:26.313558+0000","last_clean":"2023-09-23T17:45:53.616299+0000","last_became_active":"2023-09-23T17:55:58.867766+0000","last_became_peered":"2023-09-23T17:55:58.867766+0000","last_unstale":"2023-09-23T17:57:26.313558+0000","last_undegraded":"2023-09-23T17:55:58.863587+0000","last_fullsized":"2023-09-23T17:55:58.863517+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:32:36.202414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":73,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.960350+0000","last_change":"2023-09-23T17:57:24.954691+0000","last_active":"2023-09-23T17:57:25.960350+0000","last_peered":"2023-09-23T17:57:25.960350+0000","last_clean":"2023-09-23T17:53:21.029434+0000","last_became_active":"2023-09-23T17:57:24.954691+0000","last_became_peered":"2023-09-23T17:57:24.954691+0000","last_unstale":"2023-09-23T17:57:25.960350+0000","last_undegraded":"2023-09-23T17:57:25.960350+0000","last_fullsized":"2023-09-23T17:57:24.947548+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:36.584518+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.9","version":"145'6781","reported_seq":10643,"reported_epoch":145,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T17:57:37.297029+0000","last_change":"2023-09-23T17:57:24.976787+0000","last_active":"2023-09-23T17:57:37.297029+0000","last_peered":"2023-09-23T17:57:37.297029+0000","last_clean":"2023-09-23T17:55:57.822204+0000","last_became_active":"2023-09-23T17:56:51.856958+0000","last_became_peered":"2023-09-23T17:56:51.856958+0000","last_unstale":"2023-09-23T17:57:37.297029+0000","last_undegraded":"2023-09-23T17:56:51.847406+0000","last_fullsized":"2023-09-23T17:57:37.297029+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6781,"log_dups_size":0,"ondisk_log_size":6781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:26.500827+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":395771904,"num_objects":6039,"num_object_clones":0,"num_object_copies":18117,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2414,"num_objects_misplaced":3625,"num_objects_unfound":0,"num_objects_dirty":6039,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6781,"num_write_kb":410240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1894,"num_bytes_recovered":124125184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4,1],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"1,4,7","objects":3625},{"shards":"4,7","objects":2414}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":136,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:26.353334+0000","last_change":"2023-09-23T17:56:50.839197+0000","last_active":"2023-09-23T17:57:26.353334+0000","last_peered":"2023-09-23T17:57:26.353334+0000","last_clean":"2023-09-23T17:57:26.353334+0000","last_became_active":"2023-09-23T17:56:50.820626+0000","last_became_peered":"2023-09-23T17:56:50.820626+0000","last_unstale":"2023-09-23T17:57:26.353334+0000","last_undegraded":"2023-09-23T17:57:26.353334+0000","last_fullsized":"2023-09-23T17:57:26.353334+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:20.568542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":94,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.987600+0000","last_change":"2023-09-23T17:55:58.870416+0000","last_active":"2023-09-23T17:57:25.987600+0000","last_peered":"2023-09-23T17:57:25.987600+0000","last_clean":"2023-09-23T17:55:56.845347+0000","last_became_active":"2023-09-23T17:55:58.870416+0000","last_became_peered":"2023-09-23T17:55:58.870416+0000","last_unstale":"2023-09-23T17:57:25.987600+0000","last_undegraded":"2023-09-23T17:57:25.987600+0000","last_fullsized":"2023-09-23T17:55:58.862810+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:06:13.418266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"145'6895","reported_seq":6939,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T17:57:33.693241+0000","last_change":"2023-09-23T17:55:53.818806+0000","last_active":"2023-09-23T17:57:33.693241+0000","last_peered":"2023-09-23T17:57:33.693241+0000","last_clean":"2023-09-23T17:57:33.693241+0000","last_became_active":"2023-09-23T17:55:53.818291+0000","last_became_peered":"2023-09-23T17:55:53.818291+0000","last_unstale":"2023-09-23T17:57:33.693241+0000","last_undegraded":"2023-09-23T17:57:33.693241+0000","last_fullsized":"2023-09-23T17:57:33.693241+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6895,"log_dups_size":0,"ondisk_log_size":6895,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:58.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419627008,"num_objects":6403,"num_object_clones":0,"num_object_copies":19209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6895,"num_write_kb":425536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":137,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:26.354580+0000","last_change":"2023-09-23T17:56:50.861256+0000","last_active":"2023-09-23T17:57:26.354580+0000","last_peered":"2023-09-23T17:57:26.354580+0000","last_clean":"2023-09-23T17:57:26.354580+0000","last_became_active":"2023-09-23T17:56:50.823713+0000","last_became_peered":"2023-09-23T17:56:50.823713+0000","last_unstale":"2023-09-23T17:57:26.354580+0000","last_undegraded":"2023-09-23T17:57:26.354580+0000","last_fullsized":"2023-09-23T17:57:26.354580+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:00.463390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":134,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.961812+0000","last_change":"2023-09-23T17:57:24.951906+0000","last_active":"2023-09-23T17:57:25.961812+0000","last_peered":"2023-09-23T17:57:25.961812+0000","last_clean":"2023-09-23T17:57:20.922147+0000","last_became_active":"2023-09-23T17:57:24.951906+0000","last_became_peered":"2023-09-23T17:57:24.951906+0000","last_unstale":"2023-09-23T17:57:25.961812+0000","last_undegraded":"2023-09-23T17:57:25.961812+0000","last_fullsized":"2023-09-23T17:57:24.945982+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:10.143229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"145'6813","reported_seq":6857,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T17:57:33.631519+0000","last_change":"2023-09-23T17:55:53.829853+0000","last_active":"2023-09-23T17:57:33.631519+0000","last_peered":"2023-09-23T17:57:33.631519+0000","last_clean":"2023-09-23T17:57:33.631519+0000","last_became_active":"2023-09-23T17:55:53.829573+0000","last_became_peered":"2023-09-23T17:55:53.829573+0000","last_unstale":"2023-09-23T17:57:33.631519+0000","last_undegraded":"2023-09-23T17:57:33.631519+0000","last_fullsized":"2023-09-23T17:57:33.631519+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6813,"log_dups_size":0,"ondisk_log_size":6813,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:39.218789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416612352,"num_objects":6357,"num_object_clones":0,"num_object_copies":19071,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6357,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6813,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":98,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:26.353838+0000","last_change":"2023-09-23T17:55:58.870420+0000","last_active":"2023-09-23T17:57:26.353838+0000","last_peered":"2023-09-23T17:57:26.353838+0000","last_clean":"2023-09-23T17:55:57.822744+0000","last_became_active":"2023-09-23T17:55:58.870420+0000","last_became_peered":"2023-09-23T17:55:58.870420+0000","last_unstale":"2023-09-23T17:57:26.353838+0000","last_undegraded":"2023-09-23T17:55:58.862862+0000","last_fullsized":"2023-09-23T17:55:58.862757+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:31.925750+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5],"acting":[7,5],"avail_no_missing":["7","5"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":105,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.001901+0000","last_change":"2023-09-23T17:55:58.864525+0000","last_active":"2023-09-23T17:57:26.001901+0000","last_peered":"2023-09-23T17:57:26.001901+0000","last_clean":"2023-09-23T17:53:52.724466+0000","last_became_active":"2023-09-23T17:55:58.864525+0000","last_became_peered":"2023-09-23T17:55:58.864525+0000","last_unstale":"2023-09-23T17:57:26.001901+0000","last_undegraded":"2023-09-23T17:57:26.001901+0000","last_fullsized":"2023-09-23T17:55:58.859288+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:15:31.785041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"5.a","version":"145'6836","reported_seq":7704,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:33.717935+0000","last_change":"2023-09-23T17:57:24.992424+0000","last_active":"2023-09-23T17:57:33.717935+0000","last_peered":"2023-09-23T17:57:33.717935+0000","last_clean":"2023-09-23T17:57:23.931663+0000","last_became_active":"2023-09-23T17:57:24.964668+0000","last_became_peered":"2023-09-23T17:57:24.964668+0000","last_unstale":"2023-09-23T17:57:33.717935+0000","last_undegraded":"2023-09-23T17:57:24.952679+0000","last_fullsized":"2023-09-23T17:57:24.947063+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6836,"log_dups_size":0,"ondisk_log_size":6836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:59.839324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413794304,"num_objects":6314,"num_object_clones":0,"num_object_copies":18942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6314,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6836,"num_write_kb":420800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":396,"num_bytes_recovered":25952256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":6314}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":118,"reported_epoch":144,"state":"undersized+peered","last_fresh":"2023-09-23T17:57:25.988064+0000","last_change":"2023-09-23T17:57:24.955397+0000","last_active":"2023-09-23T17:57:23.953569+0000","last_peered":"2023-09-23T17:57:25.988064+0000","last_clean":"2023-09-23T17:55:56.845943+0000","last_became_active":"2023-09-23T17:55:58.865270+0000","last_became_peered":"2023-09-23T17:57:24.955397+0000","last_unstale":"2023-09-23T17:57:25.988064+0000","last_undegraded":"2023-09-23T17:57:25.988064+0000","last_fullsized":"2023-09-23T17:57:24.949667+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:19:13.929332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5],"acting":[5],"avail_no_missing":["5"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":106,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:26.269371+0000","last_change":"2023-09-23T17:56:50.828739+0000","last_active":"2023-09-23T17:57:26.269371+0000","last_peered":"2023-09-23T17:57:26.269371+0000","last_clean":"2023-09-23T17:57:26.269371+0000","last_became_active":"2023-09-23T17:56:50.828229+0000","last_became_peered":"2023-09-23T17:56:50.828229+0000","last_unstale":"2023-09-23T17:57:26.269371+0000","last_undegraded":"2023-09-23T17:57:26.269371+0000","last_fullsized":"2023-09-23T17:57:26.269371+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:45.927953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"145'6834","reported_seq":6879,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T17:57:33.706297+0000","last_change":"2023-09-23T17:55:53.830258+0000","last_active":"2023-09-23T17:57:33.706297+0000","last_peered":"2023-09-23T17:57:33.706297+0000","last_clean":"2023-09-23T17:57:33.706297+0000","last_became_active":"2023-09-23T17:55:53.829954+0000","last_became_peered":"2023-09-23T17:55:53.829954+0000","last_unstale":"2023-09-23T17:57:33.706297+0000","last_undegraded":"2023-09-23T17:57:33.706297+0000","last_fullsized":"2023-09-23T17:57:33.706297+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6834,"log_dups_size":0,"ondisk_log_size":6834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:41.952908+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412811288,"num_objects":6300,"num_object_clones":0,"num_object_copies":18900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6300,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6834,"num_write_kb":420225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":30,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:26.269439+0000","last_change":"2023-09-23T17:57:24.957601+0000","last_active":"2023-09-23T17:57:26.269439+0000","last_peered":"2023-09-23T17:57:26.269439+0000","last_clean":"2023-09-23T17:45:54.465462+0000","last_became_active":"2023-09-23T17:57:24.957601+0000","last_became_peered":"2023-09-23T17:57:24.957601+0000","last_unstale":"2023-09-23T17:57:26.269439+0000","last_undegraded":"2023-09-23T17:57:24.946354+0000","last_fullsized":"2023-09-23T17:57:24.946239+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.908857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":124,"reported_epoch":144,"state":"active+undersized+remapped","last_fresh":"2023-09-23T17:57:25.961686+0000","last_change":"2023-09-23T17:57:24.951379+0000","last_active":"2023-09-23T17:57:25.961686+0000","last_peered":"2023-09-23T17:57:25.961686+0000","last_clean":"2023-09-23T17:57:20.923095+0000","last_became_active":"2023-09-23T17:57:24.951379+0000","last_became_peered":"2023-09-23T17:57:24.951379+0000","last_unstale":"2023-09-23T17:57:25.961686+0000","last_undegraded":"2023-09-23T17:57:25.961686+0000","last_fullsized":"2023-09-23T17:57:24.945727+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:12.139012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"145'6917","reported_seq":6979,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T17:57:35.748752+0000","last_change":"2023-09-23T17:56:51.840960+0000","last_active":"2023-09-23T17:57:35.748752+0000","last_peered":"2023-09-23T17:57:35.748752+0000","last_clean":"2023-09-23T17:57:35.748752+0000","last_became_active":"2023-09-23T17:56:51.835697+0000","last_became_peered":"2023-09-23T17:56:51.835697+0000","last_unstale":"2023-09-23T17:57:35.748752+0000","last_undegraded":"2023-09-23T17:57:35.748752+0000","last_fullsized":"2023-09-23T17:57:35.748752+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6917,"log_dups_size":0,"ondisk_log_size":6917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:45.804116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411631616,"num_objects":6281,"num_object_clones":0,"num_object_copies":18843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6917,"num_write_kb":422336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":104,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.961568+0000","last_change":"2023-09-23T17:57:24.959437+0000","last_active":"2023-09-23T17:57:25.961568+0000","last_peered":"2023-09-23T17:57:25.961568+0000","last_clean":"2023-09-23T17:57:20.922904+0000","last_became_active":"2023-09-23T17:57:24.959437+0000","last_became_peered":"2023-09-23T17:57:24.959437+0000","last_unstale":"2023-09-23T17:57:25.961568+0000","last_undegraded":"2023-09-23T17:57:25.961568+0000","last_fullsized":"2023-09-23T17:57:24.946245+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:21.653654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":120,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:25.961963+0000","last_change":"2023-09-23T17:56:51.838748+0000","last_active":"2023-09-23T17:57:25.961963+0000","last_peered":"2023-09-23T17:57:25.961963+0000","last_clean":"2023-09-23T17:57:25.961963+0000","last_became_active":"2023-09-23T17:56:51.838511+0000","last_became_peered":"2023-09-23T17:56:51.838511+0000","last_unstale":"2023-09-23T17:57:25.961963+0000","last_undegraded":"2023-09-23T17:57:25.961963+0000","last_fullsized":"2023-09-23T17:57:25.961963+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:00.929947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"145'7027","reported_seq":7972,"reported_epoch":145,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T17:57:37.379487+0000","last_change":"2023-09-23T17:57:26.368674+0000","last_active":"2023-09-23T17:57:37.379487+0000","last_peered":"2023-09-23T17:57:37.379487+0000","last_clean":"2023-09-23T17:55:56.846049+0000","last_became_active":"2023-09-23T17:56:51.953573+0000","last_became_peered":"2023-09-23T17:56:51.953573+0000","last_unstale":"2023-09-23T17:57:37.379487+0000","last_undegraded":"2023-09-23T17:56:51.943666+0000","last_fullsized":"2023-09-23T17:57:37.379487+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":7027,"log_dups_size":0,"ondisk_log_size":7027,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:39:24.566891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403636224,"num_objects":6159,"num_object_clones":0,"num_object_copies":18477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3990,"num_objects_misplaced":2169,"num_objects_unfound":0,"num_objects_dirty":6159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7027,"num_write_kb":421952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":436,"num_bytes_recovered":28573696,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,2],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"2,4,5","objects":2169},{"shards":"4,5","objects":3990}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":28,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:25.961305+0000","last_change":"2023-09-23T17:56:50.827883+0000","last_active":"2023-09-23T17:57:25.961305+0000","last_peered":"2023-09-23T17:57:25.961305+0000","last_clean":"2023-09-23T17:57:25.961305+0000","last_became_active":"2023-09-23T17:56:50.827581+0000","last_became_peered":"2023-09-23T17:56:50.827581+0000","last_unstale":"2023-09-23T17:57:25.961305+0000","last_undegraded":"2023-09-23T17:57:25.961305+0000","last_fullsized":"2023-09-23T17:57:25.961305+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:47.279562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":124,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.961843+0000","last_change":"2023-09-23T17:57:24.958907+0000","last_active":"2023-09-23T17:57:25.961843+0000","last_peered":"2023-09-23T17:57:25.961843+0000","last_clean":"2023-09-23T17:57:20.923043+0000","last_became_active":"2023-09-23T17:57:24.958907+0000","last_became_peered":"2023-09-23T17:57:24.958907+0000","last_unstale":"2023-09-23T17:57:25.961843+0000","last_undegraded":"2023-09-23T17:57:25.961843+0000","last_fullsized":"2023-09-23T17:57:24.946040+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:57:35.728653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"145'6991","reported_seq":7893,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T17:57:37.322836+0000","last_change":"2023-09-23T17:56:56.943738+0000","last_active":"2023-09-23T17:57:37.322836+0000","last_peered":"2023-09-23T17:57:37.322836+0000","last_clean":"2023-09-23T17:57:37.322836+0000","last_became_active":"2023-09-23T17:56:56.903740+0000","last_became_peered":"2023-09-23T17:56:56.903740+0000","last_unstale":"2023-09-23T17:57:37.322836+0000","last_undegraded":"2023-09-23T17:57:37.322836+0000","last_fullsized":"2023-09-23T17:57:37.322836+0000","mapping_epoch":131,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6991,"log_dups_size":0,"ondisk_log_size":6991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:10.789289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":406257664,"num_objects":6199,"num_object_clones":0,"num_object_copies":18597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6991,"num_write_kb":422080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":414,"num_bytes_recovered":27131904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":74,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.955749+0000","last_change":"2023-09-23T17:57:24.954589+0000","last_active":"2023-09-23T17:57:25.955749+0000","last_peered":"2023-09-23T17:57:25.955749+0000","last_clean":"2023-09-23T17:53:21.028908+0000","last_became_active":"2023-09-23T17:57:24.954589+0000","last_became_peered":"2023-09-23T17:57:24.954589+0000","last_unstale":"2023-09-23T17:57:25.955749+0000","last_undegraded":"2023-09-23T17:57:25.955749+0000","last_fullsized":"2023-09-23T17:57:24.948573+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:16.967253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":104,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.002384+0000","last_change":"2023-09-23T17:57:24.961477+0000","last_active":"2023-09-23T17:57:26.002384+0000","last_peered":"2023-09-23T17:57:26.002384+0000","last_clean":"2023-09-23T17:57:20.929549+0000","last_became_active":"2023-09-23T17:57:24.961477+0000","last_became_peered":"2023-09-23T17:57:24.961477+0000","last_unstale":"2023-09-23T17:57:26.002384+0000","last_undegraded":"2023-09-23T17:57:26.002384+0000","last_fullsized":"2023-09-23T17:57:24.954309+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:01.838528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":565,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:25.955154+0000","last_change":"2023-09-23T17:53:52.730761+0000","last_active":"2023-09-23T17:57:25.955154+0000","last_peered":"2023-09-23T17:57:25.955154+0000","last_clean":"2023-09-23T17:57:25.955154+0000","last_became_active":"2023-09-23T17:53:52.730121+0000","last_became_peered":"2023-09-23T17:53:52.730121+0000","last_unstale":"2023-09-23T17:57:25.955154+0000","last_undegraded":"2023-09-23T17:57:25.955154+0000","last_fullsized":"2023-09-23T17:57:25.955154+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:02.569422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":129,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.987805+0000","last_change":"2023-09-23T17:57:24.952712+0000","last_active":"2023-09-23T17:57:25.987805+0000","last_peered":"2023-09-23T17:57:25.987805+0000","last_clean":"2023-09-23T17:57:20.925623+0000","last_became_active":"2023-09-23T17:57:24.952712+0000","last_became_peered":"2023-09-23T17:57:24.952712+0000","last_unstale":"2023-09-23T17:57:25.987805+0000","last_undegraded":"2023-09-23T17:57:25.987805+0000","last_fullsized":"2023-09-23T17:57:24.949381+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:55.410751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":87,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:25.954347+0000","last_change":"2023-09-23T17:56:50.816700+0000","last_active":"2023-09-23T17:57:25.954347+0000","last_peered":"2023-09-23T17:57:25.954347+0000","last_clean":"2023-09-23T17:57:25.954347+0000","last_became_active":"2023-09-23T17:56:50.816422+0000","last_became_peered":"2023-09-23T17:56:50.816422+0000","last_unstale":"2023-09-23T17:57:25.954347+0000","last_undegraded":"2023-09-23T17:57:25.954347+0000","last_fullsized":"2023-09-23T17:57:25.954347+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:01.034460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"145'6814","reported_seq":6864,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:34.086093+0000","last_change":"2023-09-23T17:55:58.870796+0000","last_active":"2023-09-23T17:57:34.086093+0000","last_peered":"2023-09-23T17:57:34.086093+0000","last_clean":"2023-09-23T17:55:57.821023+0000","last_became_active":"2023-09-23T17:55:58.867778+0000","last_became_peered":"2023-09-23T17:55:58.867778+0000","last_unstale":"2023-09-23T17:57:34.086093+0000","last_undegraded":"2023-09-23T17:55:58.868868+0000","last_fullsized":"2023-09-23T17:55:58.864928+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6814,"log_dups_size":0,"ondisk_log_size":6814,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:30.503909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410517504,"num_objects":6264,"num_object_clones":0,"num_object_copies":18792,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6264,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6264,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6814,"num_write_kb":418496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":6264}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"145'6802","reported_seq":6864,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T17:57:34.104444+0000","last_change":"2023-09-23T17:56:50.843235+0000","last_active":"2023-09-23T17:57:34.104444+0000","last_peered":"2023-09-23T17:57:34.104444+0000","last_clean":"2023-09-23T17:57:34.104444+0000","last_became_active":"2023-09-23T17:56:50.835153+0000","last_became_peered":"2023-09-23T17:56:50.835153+0000","last_unstale":"2023-09-23T17:57:34.104444+0000","last_undegraded":"2023-09-23T17:57:34.104444+0000","last_fullsized":"2023-09-23T17:57:34.104444+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6802,"log_dups_size":0,"ondisk_log_size":6802,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:14.827228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":410779648,"num_objects":6268,"num_object_clones":0,"num_object_copies":18804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6802,"num_write_kb":418240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1196,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:25.987670+0000","last_change":"2023-09-23T17:55:58.864229+0000","last_active":"2023-09-23T17:57:25.987670+0000","last_peered":"2023-09-23T17:57:25.987670+0000","last_clean":"2023-09-23T17:55:56.845670+0000","last_became_active":"2023-09-23T17:55:58.864229+0000","last_became_peered":"2023-09-23T17:55:58.864229+0000","last_unstale":"2023-09-23T17:57:25.987670+0000","last_undegraded":"2023-09-23T17:55:58.861121+0000","last_fullsized":"2023-09-23T17:55:58.861013+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:23.537845+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1894,"num_read_kb":2706,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":118,"reported_epoch":144,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:57:25.961897+0000","last_change":"2023-09-23T17:56:51.835129+0000","last_active":"2023-09-23T17:57:25.961897+0000","last_peered":"2023-09-23T17:57:25.961897+0000","last_clean":"2023-09-23T17:57:25.961897+0000","last_became_active":"2023-09-23T17:56:51.834743+0000","last_became_peered":"2023-09-23T17:56:51.834743+0000","last_unstale":"2023-09-23T17:57:25.961897+0000","last_undegraded":"2023-09-23T17:57:25.961897+0000","last_fullsized":"2023-09-23T17:57:25.961897+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:50:59.053257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0,1],"avail_no_missing":["4","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":105,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.955101+0000","last_change":"2023-09-23T17:57:24.952855+0000","last_active":"2023-09-23T17:57:25.955101+0000","last_peered":"2023-09-23T17:57:25.955101+0000","last_clean":"2023-09-23T17:57:20.923267+0000","last_became_active":"2023-09-23T17:57:24.952855+0000","last_became_peered":"2023-09-23T17:57:24.952855+0000","last_unstale":"2023-09-23T17:57:25.955101+0000","last_undegraded":"2023-09-23T17:57:25.955101+0000","last_fullsized":"2023-09-23T17:57:24.946770+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:15.329593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"145'6991","reported_seq":7035,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T17:57:33.705857+0000","last_change":"2023-09-23T17:55:53.829379+0000","last_active":"2023-09-23T17:57:33.705857+0000","last_peered":"2023-09-23T17:57:33.705857+0000","last_clean":"2023-09-23T17:57:33.705857+0000","last_became_active":"2023-09-23T17:55:53.829085+0000","last_became_peered":"2023-09-23T17:55:53.829085+0000","last_unstale":"2023-09-23T17:57:33.705857+0000","last_undegraded":"2023-09-23T17:57:33.705857+0000","last_fullsized":"2023-09-23T17:57:33.705857+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6991,"log_dups_size":0,"ondisk_log_size":6991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:41:22.353899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423559168,"num_objects":6463,"num_object_clones":0,"num_object_copies":19389,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6463,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6991,"num_write_kb":430528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":415,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:26.001873+0000","last_change":"2023-09-23T17:57:24.955720+0000","last_active":"2023-09-23T17:57:26.001873+0000","last_peered":"2023-09-23T17:57:26.001873+0000","last_clean":"2023-09-23T17:57:20.931796+0000","last_became_active":"2023-09-23T17:57:24.955720+0000","last_became_peered":"2023-09-23T17:57:24.955720+0000","last_unstale":"2023-09-23T17:57:26.001873+0000","last_undegraded":"2023-09-23T17:57:24.945824+0000","last_fullsized":"2023-09-23T17:57:24.945671+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:25.614568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":89,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.955950+0000","last_change":"2023-09-23T17:55:58.865545+0000","last_active":"2023-09-23T17:57:25.955950+0000","last_peered":"2023-09-23T17:57:25.955950+0000","last_clean":"2023-09-23T17:55:57.820377+0000","last_became_active":"2023-09-23T17:55:58.865545+0000","last_became_peered":"2023-09-23T17:55:58.865545+0000","last_unstale":"2023-09-23T17:57:25.955950+0000","last_undegraded":"2023-09-23T17:57:25.955950+0000","last_fullsized":"2023-09-23T17:55:58.860734+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:55.266655+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":112,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:25.960208+0000","last_change":"2023-09-23T17:53:52.728795+0000","last_active":"2023-09-23T17:57:25.960208+0000","last_peered":"2023-09-23T17:57:25.960208+0000","last_clean":"2023-09-23T17:57:25.960208+0000","last_became_active":"2023-09-23T17:53:52.728423+0000","last_became_peered":"2023-09-23T17:53:52.728423+0000","last_unstale":"2023-09-23T17:57:25.960208+0000","last_undegraded":"2023-09-23T17:57:25.960208+0000","last_fullsized":"2023-09-23T17:57:25.960208+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.759322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1059,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:25.987605+0000","last_change":"2023-09-23T17:54:25.601298+0000","last_active":"2023-09-23T17:57:25.987605+0000","last_peered":"2023-09-23T17:57:25.987605+0000","last_clean":"2023-09-23T17:57:25.987605+0000","last_became_active":"2023-09-23T17:54:25.600854+0000","last_became_peered":"2023-09-23T17:54:25.600854+0000","last_unstale":"2023-09-23T17:57:25.987605+0000","last_undegraded":"2023-09-23T17:57:25.987605+0000","last_fullsized":"2023-09-23T17:57:25.987605+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:56.978955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":495,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:26.352658+0000","last_change":"2023-09-23T17:55:58.867888+0000","last_active":"2023-09-23T17:57:26.352658+0000","last_peered":"2023-09-23T17:57:26.352658+0000","last_clean":"2023-09-23T17:55:57.822497+0000","last_became_active":"2023-09-23T17:55:58.867888+0000","last_became_peered":"2023-09-23T17:55:58.867888+0000","last_unstale":"2023-09-23T17:57:26.352658+0000","last_undegraded":"2023-09-23T17:55:58.860525+0000","last_fullsized":"2023-09-23T17:55:58.860424+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:18.913217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":39,"reported_epoch":144,"state":"undersized+peered","last_fresh":"2023-09-23T17:57:25.987707+0000","last_change":"2023-09-23T17:57:24.952696+0000","last_active":"2023-09-23T17:55:57.847398+0000","last_peered":"2023-09-23T17:57:25.987707+0000","last_clean":"2023-09-23T17:55:56.841167+0000","last_became_active":"2023-09-23T17:54:57.035801+0000","last_became_peered":"2023-09-23T17:57:24.952696+0000","last_unstale":"2023-09-23T17:57:25.987707+0000","last_undegraded":"2023-09-23T17:57:25.987707+0000","last_fullsized":"2023-09-23T17:57:24.949891+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:08.300302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5],"acting":[5],"avail_no_missing":["5"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":45,"reported_epoch":144,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:57:25.962062+0000","last_change":"2023-09-23T17:56:51.845525+0000","last_active":"2023-09-23T17:57:25.962062+0000","last_peered":"2023-09-23T17:57:25.962062+0000","last_clean":"2023-09-23T17:57:25.962062+0000","last_became_active":"2023-09-23T17:56:51.845056+0000","last_became_peered":"2023-09-23T17:56:51.845056+0000","last_unstale":"2023-09-23T17:57:25.962062+0000","last_undegraded":"2023-09-23T17:57:25.962062+0000","last_fullsized":"2023-09-23T17:57:25.962062+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:09.024552+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7,0],"avail_no_missing":["4","0","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.4","version":"145'7073","reported_seq":7116,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T17:57:37.370667+0000","last_change":"2023-09-23T17:55:53.829245+0000","last_active":"2023-09-23T17:57:37.370667+0000","last_peered":"2023-09-23T17:57:37.370667+0000","last_clean":"2023-09-23T17:57:37.370667+0000","last_became_active":"2023-09-23T17:55:53.829019+0000","last_became_peered":"2023-09-23T17:55:53.829019+0000","last_unstale":"2023-09-23T17:57:37.370667+0000","last_undegraded":"2023-09-23T17:57:37.370667+0000","last_fullsized":"2023-09-23T17:57:37.370667+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":7073,"log_dups_size":0,"ondisk_log_size":7073,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:38.782556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":405995520,"num_objects":6195,"num_object_clones":0,"num_object_copies":18585,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6195,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7073,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"145'7168","reported_seq":7211,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T17:57:37.318342+0000","last_change":"2023-09-23T17:55:53.828325+0000","last_active":"2023-09-23T17:57:37.318342+0000","last_peered":"2023-09-23T17:57:37.318342+0000","last_clean":"2023-09-23T17:57:37.318342+0000","last_became_active":"2023-09-23T17:55:53.828056+0000","last_became_peered":"2023-09-23T17:55:53.828056+0000","last_unstale":"2023-09-23T17:57:37.318342+0000","last_undegraded":"2023-09-23T17:57:37.318342+0000","last_fullsized":"2023-09-23T17:57:37.318342+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":7168,"log_dups_size":0,"ondisk_log_size":7168,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:47.523521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412221440,"num_objects":6290,"num_object_clones":0,"num_object_copies":18870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7168,"num_write_kb":430656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":343,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:26.352690+0000","last_change":"2023-09-23T17:53:52.734324+0000","last_active":"2023-09-23T17:57:26.352690+0000","last_peered":"2023-09-23T17:57:26.352690+0000","last_clean":"2023-09-23T17:57:26.352690+0000","last_became_active":"2023-09-23T17:53:52.733825+0000","last_became_peered":"2023-09-23T17:53:52.733825+0000","last_unstale":"2023-09-23T17:57:26.352690+0000","last_undegraded":"2023-09-23T17:57:26.352690+0000","last_fullsized":"2023-09-23T17:57:26.352690+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:34.742687+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":112,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.269497+0000","last_change":"2023-09-23T17:57:24.959291+0000","last_active":"2023-09-23T17:57:26.269497+0000","last_peered":"2023-09-23T17:57:26.269497+0000","last_clean":"2023-09-23T17:57:20.930735+0000","last_became_active":"2023-09-23T17:57:24.959291+0000","last_became_peered":"2023-09-23T17:57:24.959291+0000","last_unstale":"2023-09-23T17:57:26.269497+0000","last_undegraded":"2023-09-23T17:57:26.269497+0000","last_fullsized":"2023-09-23T17:57:24.946083+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:43.225674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":103,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:26.269946+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T17:57:26.269946+0000","last_peered":"2023-09-23T17:57:26.269946+0000","last_clean":"2023-09-23T17:57:26.269946+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T17:57:26.269946+0000","last_undegraded":"2023-09-23T17:57:26.269946+0000","last_fullsized":"2023-09-23T17:57:26.269946+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":409,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:26.353753+0000","last_change":"2023-09-23T17:56:50.881038+0000","last_active":"2023-09-23T17:57:26.353753+0000","last_peered":"2023-09-23T17:57:26.353753+0000","last_clean":"2023-09-23T17:57:26.353753+0000","last_became_active":"2023-09-23T17:56:50.822953+0000","last_became_peered":"2023-09-23T17:56:50.822953+0000","last_unstale":"2023-09-23T17:57:26.353753+0000","last_undegraded":"2023-09-23T17:57:26.353753+0000","last_fullsized":"2023-09-23T17:57:26.353753+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:03:47.600460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":83,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:25.955195+0000","last_change":"2023-09-23T17:54:25.604720+0000","last_active":"2023-09-23T17:57:25.955195+0000","last_peered":"2023-09-23T17:57:25.955195+0000","last_clean":"2023-09-23T17:57:25.955195+0000","last_became_active":"2023-09-23T17:54:25.604386+0000","last_became_peered":"2023-09-23T17:54:25.604386+0000","last_unstale":"2023-09-23T17:57:25.955195+0000","last_undegraded":"2023-09-23T17:57:25.955195+0000","last_fullsized":"2023-09-23T17:57:25.955195+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:56.869273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":74,"reported_epoch":145,"state":"active+clean","last_fresh":"2023-09-23T17:57:29.079784+0000","last_change":"2023-09-23T17:53:21.039195+0000","last_active":"2023-09-23T17:57:29.079784+0000","last_peered":"2023-09-23T17:57:29.079784+0000","last_clean":"2023-09-23T17:57:29.079784+0000","last_became_active":"2023-09-23T17:53:21.038654+0000","last_became_peered":"2023-09-23T17:53:21.038654+0000","last_unstale":"2023-09-23T17:57:29.079784+0000","last_undegraded":"2023-09-23T17:57:29.079784+0000","last_fullsized":"2023-09-23T17:57:29.079784+0000","mapping_epoch":93,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:34.466980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"145'6748","reported_seq":6787,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:34.069515+0000","last_change":"2023-09-23T17:55:58.884622+0000","last_active":"2023-09-23T17:57:34.069515+0000","last_peered":"2023-09-23T17:57:34.069515+0000","last_clean":"2023-09-23T17:55:52.789238+0000","last_became_active":"2023-09-23T17:55:58.867800+0000","last_became_peered":"2023-09-23T17:55:58.867800+0000","last_unstale":"2023-09-23T17:57:34.069515+0000","last_undegraded":"2023-09-23T17:55:58.878355+0000","last_fullsized":"2023-09-23T17:55:58.862585+0000","mapping_epoch":114,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6748,"log_dups_size":0,"ondisk_log_size":6748,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:52:13.398530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":409206784,"num_objects":6244,"num_object_clones":0,"num_object_copies":18732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6244,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6748,"num_write_kb":415744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":6244}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":33,"reported_epoch":144,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:26.353090+0000","last_change":"2023-09-23T17:57:24.958874+0000","last_active":"2023-09-23T17:57:26.353090+0000","last_peered":"2023-09-23T17:57:26.353090+0000","last_clean":"2023-09-23T17:45:54.465055+0000","last_became_active":"2023-09-23T17:57:24.958874+0000","last_became_peered":"2023-09-23T17:57:24.958874+0000","last_unstale":"2023-09-23T17:57:26.353090+0000","last_undegraded":"2023-09-23T17:57:24.948018+0000","last_fullsized":"2023-09-23T17:57:24.947911+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:33.988637+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[{"shards":"0,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":367,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.960264+0000","last_change":"2023-09-23T17:57:24.951980+0000","last_active":"2023-09-23T17:57:25.960264+0000","last_peered":"2023-09-23T17:57:25.960264+0000","last_clean":"2023-09-23T17:57:20.923519+0000","last_became_active":"2023-09-23T17:57:24.951980+0000","last_became_peered":"2023-09-23T17:57:24.951980+0000","last_unstale":"2023-09-23T17:57:25.960264+0000","last_undegraded":"2023-09-23T17:57:25.960264+0000","last_fullsized":"2023-09-23T17:57:24.947444+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:21.034358+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":70,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.352541+0000","last_change":"2023-09-23T17:57:24.965626+0000","last_active":"2023-09-23T17:57:26.352541+0000","last_peered":"2023-09-23T17:57:26.352541+0000","last_clean":"2023-09-23T17:49:06.298151+0000","last_became_active":"2023-09-23T17:57:24.965626+0000","last_became_peered":"2023-09-23T17:57:24.965626+0000","last_unstale":"2023-09-23T17:57:26.352541+0000","last_undegraded":"2023-09-23T17:57:26.352541+0000","last_fullsized":"2023-09-23T17:57:24.947346+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:41.036372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0],"avail_no_missing":["7","0"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.1","version":"145'7109","reported_seq":8619,"reported_epoch":145,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:57:37.368467+0000","last_change":"2023-09-23T17:57:24.957919+0000","last_active":"2023-09-23T17:57:37.368467+0000","last_peered":"2023-09-23T17:57:37.368467+0000","last_clean":"2023-09-23T17:57:23.936571+0000","last_became_active":"2023-09-23T17:57:24.957919+0000","last_became_peered":"2023-09-23T17:57:24.957919+0000","last_unstale":"2023-09-23T17:57:37.368467+0000","last_undegraded":"2023-09-23T17:57:24.945538+0000","last_fullsized":"2023-09-23T17:57:24.945420+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":7109,"log_dups_size":0,"ondisk_log_size":7109,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:52.390092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407699456,"num_objects":6221,"num_object_clones":0,"num_object_copies":18663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6221,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7109,"num_write_kb":426560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":717,"num_bytes_recovered":46989312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[{"shards":"4,7","objects":6221}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":303,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.002431+0000","last_change":"2023-09-23T17:57:17.820817+0000","last_active":"2023-09-23T17:57:26.002431+0000","last_peered":"2023-09-23T17:57:26.002431+0000","last_clean":"2023-09-23T17:46:59.535985+0000","last_became_active":"2023-09-23T17:57:17.820817+0000","last_became_peered":"2023-09-23T17:57:17.820817+0000","last_unstale":"2023-09-23T17:57:26.002431+0000","last_undegraded":"2023-09-23T17:57:26.002431+0000","last_fullsized":"2023-09-23T17:57:17.802755+0000","mapping_epoch":138,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:12.471992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":117,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:26.269845+0000","last_change":"2023-09-23T17:56:50.856836+0000","last_active":"2023-09-23T17:57:26.269845+0000","last_peered":"2023-09-23T17:57:26.269845+0000","last_clean":"2023-09-23T17:57:26.269845+0000","last_became_active":"2023-09-23T17:56:50.835155+0000","last_became_peered":"2023-09-23T17:56:50.835155+0000","last_unstale":"2023-09-23T17:57:26.269845+0000","last_undegraded":"2023-09-23T17:57:26.269845+0000","last_fullsized":"2023-09-23T17:57:26.269845+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:36.188982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":100,"reported_epoch":144,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:57:25.961461+0000","last_change":"2023-09-23T17:56:51.838623+0000","last_active":"2023-09-23T17:57:25.961461+0000","last_peered":"2023-09-23T17:57:25.961461+0000","last_clean":"2023-09-23T17:57:25.961461+0000","last_became_active":"2023-09-23T17:56:51.838216+0000","last_became_peered":"2023-09-23T17:56:51.838216+0000","last_unstale":"2023-09-23T17:57:25.961461+0000","last_undegraded":"2023-09-23T17:57:25.961461+0000","last_fullsized":"2023-09-23T17:57:25.961461+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:25:08.884374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5],"acting":[4,5,0],"avail_no_missing":["4","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"145'7082","reported_seq":8258,"reported_epoch":145,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T17:57:35.804923+0000","last_change":"2023-09-23T17:57:26.367743+0000","last_active":"2023-09-23T17:57:35.804923+0000","last_peered":"2023-09-23T17:57:35.804923+0000","last_clean":"2023-09-23T17:56:38.421727+0000","last_became_active":"2023-09-23T17:57:26.317820+0000","last_became_peered":"2023-09-23T17:57:26.317820+0000","last_unstale":"2023-09-23T17:57:35.804923+0000","last_undegraded":"2023-09-23T17:57:26.081408+0000","last_fullsized":"2023-09-23T17:57:35.804923+0000","mapping_epoch":143,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":7082,"log_dups_size":0,"ondisk_log_size":7082,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:37.559775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417464320,"num_objects":6370,"num_object_clones":0,"num_object_copies":19110,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6370,"num_objects_misplaced":226,"num_objects_unfound":0,"num_objects_dirty":6370,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7082,"num_write_kb":430464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":554,"num_bytes_recovered":36306944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,0],"avail_no_missing":["1"],"object_location_counts":[{"shards":"0,1","objects":226},{"shards":"0,1,4","objects":284},{"shards":"1,4","objects":5860}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":94,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:26.269498+0000","last_change":"2023-09-23T17:56:50.828555+0000","last_active":"2023-09-23T17:57:26.269498+0000","last_peered":"2023-09-23T17:57:26.269498+0000","last_clean":"2023-09-23T17:57:26.269498+0000","last_became_active":"2023-09-23T17:56:50.827767+0000","last_became_peered":"2023-09-23T17:56:50.827767+0000","last_unstale":"2023-09-23T17:57:26.269498+0000","last_undegraded":"2023-09-23T17:57:26.269498+0000","last_fullsized":"2023-09-23T17:57:26.269498+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:59.993181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":135,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:26.001973+0000","last_change":"2023-09-23T17:57:24.956919+0000","last_active":"2023-09-23T17:57:26.001973+0000","last_peered":"2023-09-23T17:57:26.001973+0000","last_clean":"2023-09-23T17:57:20.928421+0000","last_became_active":"2023-09-23T17:57:24.956919+0000","last_became_peered":"2023-09-23T17:57:24.956919+0000","last_unstale":"2023-09-23T17:57:26.001973+0000","last_undegraded":"2023-09-23T17:57:26.001973+0000","last_fullsized":"2023-09-23T17:57:24.951731+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:03.928787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":131,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:25.987692+0000","last_change":"2023-09-23T17:56:51.848379+0000","last_active":"2023-09-23T17:57:25.987692+0000","last_peered":"2023-09-23T17:57:25.987692+0000","last_clean":"2023-09-23T17:57:25.987692+0000","last_became_active":"2023-09-23T17:56:51.848091+0000","last_became_peered":"2023-09-23T17:56:51.848091+0000","last_unstale":"2023-09-23T17:57:25.987692+0000","last_undegraded":"2023-09-23T17:57:25.987692+0000","last_fullsized":"2023-09-23T17:57:25.987692+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:01.930989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":35,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.961675+0000","last_change":"2023-09-23T17:57:24.951129+0000","last_active":"2023-09-23T17:57:25.961675+0000","last_peered":"2023-09-23T17:57:25.961675+0000","last_clean":"2023-09-23T17:57:20.922389+0000","last_became_active":"2023-09-23T17:57:24.951129+0000","last_became_peered":"2023-09-23T17:57:24.951129+0000","last_unstale":"2023-09-23T17:57:25.961675+0000","last_undegraded":"2023-09-23T17:57:25.961675+0000","last_fullsized":"2023-09-23T17:57:24.945440+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:43.628177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":125,"reported_epoch":144,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:57:25.987936+0000","last_change":"2023-09-23T17:56:51.843661+0000","last_active":"2023-09-23T17:57:25.987936+0000","last_peered":"2023-09-23T17:57:25.987936+0000","last_clean":"2023-09-23T17:57:25.987936+0000","last_became_active":"2023-09-23T17:56:51.842466+0000","last_became_peered":"2023-09-23T17:56:51.842466+0000","last_unstale":"2023-09-23T17:57:25.987936+0000","last_undegraded":"2023-09-23T17:57:25.987936+0000","last_fullsized":"2023-09-23T17:57:25.987936+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:19.859450+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4,1],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":35,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.961502+0000","last_change":"2023-09-23T17:57:24.957395+0000","last_active":"2023-09-23T17:57:25.961502+0000","last_peered":"2023-09-23T17:57:25.961502+0000","last_clean":"2023-09-23T17:57:20.922822+0000","last_became_active":"2023-09-23T17:57:24.957395+0000","last_became_peered":"2023-09-23T17:57:24.957395+0000","last_unstale":"2023-09-23T17:57:25.961502+0000","last_undegraded":"2023-09-23T17:57:25.961502+0000","last_fullsized":"2023-09-23T17:57:24.945910+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:25.767832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":10,"reported_epoch":144,"state":"undersized+peered","last_fresh":"2023-09-23T17:57:28.735056+0000","last_change":"2023-09-23T17:57:24.953997+0000","last_active":"2023-09-23T17:45:50.074605+0000","last_peered":"2023-09-23T17:57:28.735056+0000","last_clean":"2023-09-23T17:45:50.074605+0000","last_became_active":"0.000000","last_became_peered":"2023-09-23T17:57:24.953997+0000","last_unstale":"2023-09-23T17:57:28.735056+0000","last_undegraded":"2023-09-23T17:57:28.735056+0000","last_fullsized":"2023-09-23T17:57:24.946378+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:03:02.279182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2],"avail_no_missing":["2"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":94,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:28.734930+0000","last_change":"2023-09-23T17:57:24.955661+0000","last_active":"2023-09-23T17:57:28.734930+0000","last_peered":"2023-09-23T17:57:28.734930+0000","last_clean":"2023-09-23T17:53:52.727684+0000","last_became_active":"2023-09-23T17:57:24.955661+0000","last_became_peered":"2023-09-23T17:57:24.955661+0000","last_unstale":"2023-09-23T17:57:28.734930+0000","last_undegraded":"2023-09-23T17:57:28.734930+0000","last_fullsized":"2023-09-23T17:57:24.945672+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:04.500443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":92,"reported_epoch":144,"state":"active+clean","last_fresh":"2023-09-23T17:57:26.269755+0000","last_change":"2023-09-23T17:54:25.601065+0000","last_active":"2023-09-23T17:57:26.269755+0000","last_peered":"2023-09-23T17:57:26.269755+0000","last_clean":"2023-09-23T17:57:26.269755+0000","last_became_active":"2023-09-23T17:54:25.600727+0000","last_became_peered":"2023-09-23T17:54:25.600727+0000","last_unstale":"2023-09-23T17:57:26.269755+0000","last_undegraded":"2023-09-23T17:57:26.269755+0000","last_fullsized":"2023-09-23T17:57:26.269755+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:20.919498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":124,"reported_epoch":144,"state":"active+undersized","last_fresh":"2023-09-23T17:57:25.961297+0000","last_change":"2023-09-23T17:57:24.950899+0000","last_active":"2023-09-23T17:57:25.961297+0000","last_peered":"2023-09-23T17:57:25.961297+0000","last_clean":"2023-09-23T17:57:20.922710+0000","last_became_active":"2023-09-23T17:57:24.950899+0000","last_became_peered":"2023-09-23T17:57:24.950899+0000","last_unstale":"2023-09-23T17:57:25.961297+0000","last_undegraded":"2023-09-23T17:57:25.961297+0000","last_fullsized":"2023-09-23T17:57:24.945062+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:16.848014+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":6577586200,"num_objects":100367,"num_object_clones":0,"num_object_copies":301101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":37817,"num_objects_misplaced":6020,"num_objects_unfound":0,"num_objects_dirty":100367,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":110881,"num_write_kb":6759873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4411,"num_bytes_recovered":289079296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18456457216,"data_stored":18456445000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110881,"ondisk_log_size":110881,"up":41,"acting":44,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":67,"acting":71,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":5,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":224,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":110592,"data_stored":21246,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":25710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":72,"acting":76,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":500,"num_read_kb":924,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":161,"ondisk_log_size":161,"up":2,"acting":2,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":19,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":20,"acting":20,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266846,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2025184,"kb_used_data":1978840,"kb_used_omap":0,"kb_used_meta":46336,"kb_avail":91719968,"statfs":{"total":95995035648,"available":93921247232,"internally_reserved":0,"allocated":2026332160,"data_stored":2025719599,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":47448064},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,0,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":0.83399999999999996,"15min":0.64500000000000002},"min":{"1min":0.24099999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.5700000000000003,"5min":8.5700000000000003,"15min":8.5700000000000003},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.6499999999999999,"5min":0.879,"15min":0.67100000000000004},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.8410000000000002,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":0.32100000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.59599999999999997,"15min":0.56999999999999995},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.702,"5min":2.702,"15min":2.702},"last":0.27800000000000002},{"interface":"front","average":{"1min":1.1399999999999999,"5min":0.65700000000000003,"15min":0.58899999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":4.4240000000000004},"last":0.38300000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":0.89200000000000002,"15min":0.66300000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.2359999999999998,"5min":5.2359999999999998,"15min":5.2359999999999998},"last":0.35399999999999998},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.77900000000000003,"15min":0.64800000000000002},"min":{"1min":0.38100000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.8340000000000001,"5min":4.6180000000000003,"15min":4.6180000000000003},"last":0.39900000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:57:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.64900000000000002,"15min":0.57299999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":4.0549999999999997,"5min":4.0549999999999997,"15min":4.0549999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.665,"5min":0.77400000000000002,"15min":0.626},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.1609999999999996,"5min":5.1609999999999996,"15min":5.1609999999999996},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.53400000000000003,"15min":0.47099999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.6160000000000001,"5min":2.6160000000000001,"15min":2.6160000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.56100000000000005,"15min":0.498},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8079999999999998,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":0.34000000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.60699999999999998,"15min":0.52600000000000002},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.5840000000000001,"5min":2.5840000000000001,"15min":2.5840000000000001},"last":0.311},{"interface":"front","average":{"1min":0.749,"5min":0.64100000000000001,"15min":0.54900000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.706,"5min":2.706,"15min":2.706},"last":0.41199999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":1.002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.435}]}]},{"osd":6,"up_from":45,"seq":193273528462,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":12464,"kb_used_data":1704,"kb_used_omap":0,"kb_used_meta":10752,"kb_avail":93732688,"statfs":{"total":95995035648,"available":95982272512,"internally_reserved":0,"allocated":1744896,"data_stored":1230110,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":11010048},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.55800000000000005,"15min":0.56100000000000005},"min":{"1min":0.42599999999999999,"5min":0.378,"15min":0.378},"max":{"1min":0.81100000000000005,"5min":0.81999999999999995,"15min":0.81999999999999995},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":0.60599999999999998,"15min":0.60699999999999998},"min":{"1min":0.443,"5min":0.443,"15min":0.443},"max":{"1min":0.78300000000000003,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.38}]},{"osd":1,"last update":"Sat Sep 23 17:54:53 2023","interfaces":[{"interface":"back","average":{"1min":0.57099999999999995,"5min":0.57099999999999995,"15min":0.57099999999999995},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.82199999999999995,"5min":0.82199999999999995,"15min":0.82199999999999995},"last":0.62},{"interface":"front","average":{"1min":0.57399999999999995,"5min":0.57399999999999995,"15min":0.57399999999999995},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.76400000000000001,"5min":0.76400000000000001,"15min":0.76400000000000001},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":0.73699999999999999,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":1.2250000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.80500000000000005,"5min":0.80500000000000005,"15min":0.80500000000000005},"last":6.9589999999999996}]},{"osd":3,"last update":"Sat Sep 23 17:54:37 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":0.58399999999999996,"15min":0.58099999999999996},"min":{"1min":0.38800000000000001,"5min":0.36799999999999999,"15min":0.32700000000000001},"max":{"1min":0.73699999999999999,"5min":0.80700000000000005,"15min":4.5599999999999996},"last":0.253},{"interface":"front","average":{"1min":0.625,"5min":0.58499999999999996,"15min":0.55100000000000005},"min":{"1min":0.42399999999999999,"5min":0.34999999999999998,"15min":0.30399999999999999},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":1.6850000000000001},"last":2.9369999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:55:43 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.48699999999999999,"15min":0.46000000000000002},"min":{"1min":0.377,"5min":0.27600000000000002,"15min":0.25700000000000001},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.82899999999999996},"last":0.27200000000000002},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.53000000000000003,"15min":0.496},"min":{"1min":0.34499999999999997,"5min":0.28899999999999998,"15min":0.27800000000000002},"max":{"1min":0.72099999999999997,"5min":0.86599999999999999,"15min":0.86599999999999999},"last":0.33800000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:55:43 2023","interfaces":[{"interface":"back","average":{"1min":0.59499999999999997,"5min":0.54300000000000004,"15min":0.503},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.29999999999999999},"max":{"1min":0.82099999999999995,"5min":0.88200000000000001,"15min":0.88200000000000001},"last":0.40400000000000003},{"interface":"front","average":{"1min":0.56899999999999995,"5min":0.56699999999999995,"15min":0.52700000000000002},"min":{"1min":0.39700000000000002,"5min":0.33300000000000002,"15min":0.33200000000000002},"max":{"1min":0.69099999999999995,"5min":0.79700000000000004,"15min":0.84599999999999997},"last":0.42199999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:55:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":0.58999999999999997,"15min":0.53700000000000003},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.30599999999999999},"max":{"1min":0.83299999999999996,"5min":0.89200000000000002,"15min":0.89200000000000002},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.60599999999999998,"5min":0.57499999999999996,"15min":0.51900000000000002},"min":{"1min":0.371,"5min":0.31,"15min":0.31},"max":{"1min":0.81299999999999994,"5min":0.82299999999999995,"15min":1.591},"last":0.314}]}]},{"osd":1,"up_from":98,"seq":420906795055,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2356260,"kb_used_data":2262172,"kb_used_omap":7,"kb_used_meta":94008,"kb_avail":91388892,"statfs":{"total":95995035648,"available":93582225408,"internally_reserved":0,"allocated":2316464128,"data_stored":2315733877,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":96264893},"hb_peers":[0,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,0,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":2.6800000000000002,"5min":0.83899999999999997,"15min":0.53800000000000003},"min":{"1min":0.186,"5min":0.186,"15min":0.186},"max":{"1min":11.384,"5min":11.384,"15min":11.384},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.9550000000000001,"5min":0.90600000000000003,"15min":0.56599999999999995},"min":{"1min":0.25,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":11.596,"5min":11.596,"15min":11.596},"last":12.355}]},{"osd":2,"last update":"Sat Sep 23 17:57:33 2023","interfaces":[{"interface":"back","average":{"1min":2.8090000000000002,"5min":1.109,"15min":0.67000000000000004},"min":{"1min":0.26100000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":10.241,"5min":10.241,"15min":10.241},"last":5.3230000000000004},{"interface":"front","average":{"1min":1.869,"5min":0.97199999999999998,"15min":0.61699999999999999},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":7.9660000000000002,"5min":11.305999999999999,"15min":11.305999999999999},"last":7.9660000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:57:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8999999999999999,"5min":0.90800000000000003,"15min":0.57599999999999996},"min":{"1min":0.29899999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":14.548999999999999,"5min":14.548999999999999,"15min":14.548999999999999},"last":2.7149999999999999},{"interface":"front","average":{"1min":2.3740000000000001,"5min":0.85199999999999998,"15min":0.59799999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.250999999999999,"5min":10.250999999999999,"15min":10.250999999999999},"last":0.82899999999999996}]},{"osd":4,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.7150000000000001,"5min":0.88600000000000001,"15min":0.75},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":5.0090000000000003,"5min":5.0090000000000003,"15min":5.0090000000000003},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.8029999999999999,"5min":0.89700000000000002,"15min":0.751},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":8.8770000000000007,"5min":8.8770000000000007,"15min":8.8770000000000007},"last":0.44700000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":2.0430000000000001,"5min":0.96999999999999997,"15min":0.80200000000000005},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":7.4269999999999996,"5min":7.4269999999999996,"15min":7.4269999999999996},"last":5.383},{"interface":"front","average":{"1min":1.788,"5min":0.91300000000000003,"15min":0.77500000000000002},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":5.0430000000000001,"5min":5.0430000000000001,"15min":5.0430000000000001},"last":0.34699999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.9330000000000001,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.69199999999999995,"15min":0.69899999999999995},"min":{"1min":0.47699999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.141,"5min":1.141,"15min":1.141},"last":0.77700000000000002}]},{"osd":7,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":0.89400000000000002,"15min":0.74399999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.9630000000000001,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.895,"5min":0.92900000000000005,"15min":0.77900000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.2940000000000005,"5min":8.2940000000000005,"15min":8.2940000000000005},"last":0.50600000000000001}]}]},{"osd":0,"up_from":93,"seq":399431958581,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3435092,"kb_used_data":3299212,"kb_used_omap":7,"kb_used_meta":135864,"kb_avail":90310060,"statfs":{"total":95995035648,"available":92477501440,"internally_reserved":0,"allocated":3378393088,"data_stored":3377676518,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":139125476},"hb_peers":[1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:56:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":0.66900000000000004,"15min":0.46899999999999997},"min":{"1min":0.17899999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":4.7030000000000003,"5min":4.7030000000000003,"15min":4.7030000000000003},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.591,"5min":0.64900000000000002,"15min":0.5},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.8450000000000006,"5min":8.8450000000000006,"15min":8.8450000000000006},"last":0.28299999999999997}]},{"osd":2,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":2.5990000000000002,"5min":0.91200000000000003,"15min":0.54400000000000004},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":9.8580000000000005,"5min":9.8580000000000005,"15min":9.8580000000000005},"last":0.312},{"interface":"front","average":{"1min":3.0720000000000001,"5min":0.99099999999999999,"15min":0.57499999999999996},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":8.7349999999999994,"5min":8.7349999999999994,"15min":8.7349999999999994},"last":0.248}]},{"osd":3,"last update":"Sat Sep 23 17:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":0.77100000000000002,"15min":0.60099999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3449999999999998,"5min":4.3449999999999998,"15min":4.3449999999999998},"last":9.266},{"interface":"front","average":{"1min":2.109,"5min":0.73699999999999999,"15min":0.50900000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":8.8729999999999993},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.6279999999999999,"5min":0.91500000000000004,"15min":0.72899999999999998},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":6.4169999999999998,"5min":6.4169999999999998,"15min":6.4169999999999998},"last":3.2040000000000002},{"interface":"front","average":{"1min":1.589,"5min":0.88400000000000001,"15min":0.70999999999999996},"min":{"1min":0.309,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.1020000000000003,"5min":9.1020000000000003,"15min":9.1020000000000003},"last":0.64400000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":0.88200000000000001,"15min":0.71599999999999997},"min":{"1min":0.34699999999999998,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":9.1370000000000005,"5min":9.1370000000000005,"15min":9.1370000000000005},"last":0.61699999999999999},{"interface":"front","average":{"1min":1.762,"5min":0.873,"15min":0.72099999999999997},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.7829999999999995,"5min":8.7829999999999995,"15min":8.7829999999999995},"last":1.325}]},{"osd":6,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.42899999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":0.56599999999999995},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65200000000000002},"min":{"1min":0.52700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.81100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.58499999999999996}]},{"osd":7,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":0.90800000000000003,"15min":0.746},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.8160000000000007,"5min":8.8160000000000007,"15min":8.8160000000000007},"last":1.2470000000000001},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.01,"15min":0.78400000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":8.6470000000000002,"5min":8.6470000000000002,"15min":8.6470000000000002},"last":0.33500000000000002}]}]},{"osd":2,"up_from":103,"seq":442381631528,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2686036,"kb_used_data":2604364,"kb_used_omap":11,"kb_used_meta":81652,"kb_avail":91059116,"statfs":{"total":95995035648,"available":93244534784,"internally_reserved":0,"allocated":2666868736,"data_stored":2666155240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":83612023},"hb_peers":[0,1,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":3.145,"5min":0.92900000000000005,"15min":0.53600000000000003},"min":{"1min":0.22800000000000001,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":13.234,"5min":13.234,"15min":13.234},"last":0.17299999999999999},{"interface":"front","average":{"1min":3.202,"5min":1.0429999999999999,"15min":0.60299999999999998},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":13.364000000000001,"5min":13.364000000000001,"15min":13.364000000000001},"last":0.215}]},{"osd":1,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":3.3980000000000001,"5min":1.0449999999999999,"15min":0.61599999999999999},"min":{"1min":0.313,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":19.206,"5min":19.206,"15min":19.206},"last":0.40600000000000003},{"interface":"front","average":{"1min":3.766,"5min":1.159,"15min":0.60599999999999998},"min":{"1min":0.23899999999999999,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":15.657,"5min":15.657,"15min":15.657},"last":0.68400000000000005}]},{"osd":3,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":0.58799999999999997,"15min":0.443},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8339999999999996,"5min":4.8339999999999996,"15min":4.8339999999999996},"last":2.8849999999999998},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.59699999999999998,"15min":0.48399999999999999},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.6459999999999999,"5min":5.6459999999999999,"15min":5.6459999999999999},"last":2.5579999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.7210000000000001,"5min":0.91900000000000004,"15min":0.70799999999999996},"min":{"1min":0.29299999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":5.7080000000000002,"5min":5.7080000000000002,"15min":5.7080000000000002},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.772,"5min":0.86199999999999999,"15min":0.69399999999999995},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":13.308,"5min":13.308,"15min":13.308},"last":0.317}]},{"osd":5,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1850000000000001,"5min":0.95199999999999996,"15min":0.73299999999999998},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":13.273,"5min":13.273,"15min":13.273},"last":0.35199999999999998},{"interface":"front","average":{"1min":3.2040000000000002,"5min":1.198,"15min":0.80200000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":15.496,"5min":15.496,"15min":15.496},"last":0.71399999999999997}]},{"osd":6,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.46999999999999997}]},{"osd":7,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":0.94299999999999995,"15min":0.71899999999999997},"min":{"1min":0.505,"5min":0.441,"15min":0.441},"max":{"1min":5.7409999999999997,"5min":5.7409999999999997,"15min":5.7409999999999997},"last":0.33600000000000002},{"interface":"front","average":{"1min":2.0870000000000002,"5min":0.97899999999999998,"15min":0.78300000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":13.333,"5min":13.333,"15min":13.333},"last":0.27300000000000002}]}]},{"osd":3,"up_from":108,"seq":463856467999,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267348,"kb_used_data":1221900,"kb_used_omap":3,"kb_used_meta":45436,"kb_avail":92477804,"statfs":{"total":95995035648,"available":94697271296,"internally_reserved":0,"allocated":1251225600,"data_stored":1250531999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":46527208},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.0699999999999998,"5min":0.71399999999999997,"15min":0.48799999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":14.82,"5min":14.82,"15min":14.82},"last":9.0220000000000002},{"interface":"front","average":{"1min":2.085,"5min":0.72499999999999998,"15min":0.499},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":13.054,"5min":13.054,"15min":13.054},"last":0.46500000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":0.65700000000000003,"15min":0.46899999999999997},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.8130000000000006,"5min":8.8130000000000006,"15min":8.8130000000000006},"last":9.1489999999999991},{"interface":"front","average":{"1min":1.417,"5min":0.61599999999999999,"15min":0.48199999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4580000000000002,"5min":6.4580000000000002,"15min":6.4580000000000002},"last":9.125}]},{"osd":2,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":0.69099999999999995,"15min":0.49099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.869999999999999,"5min":10.869999999999999,"15min":10.869999999999999},"last":7.3849999999999998},{"interface":"front","average":{"1min":1.6850000000000001,"5min":0.66500000000000004,"15min":0.495},"min":{"1min":0.376,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":13.936999999999999,"5min":13.936999999999999,"15min":13.936999999999999},"last":8.9169999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":0.73699999999999999,"15min":0.65200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5380000000000003,"5min":6.5380000000000003,"15min":6.5380000000000003},"last":6.7569999999999997},{"interface":"front","average":{"1min":1.629,"5min":0.82099999999999995,"15min":0.68700000000000006},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.5940000000000003,"5min":6.5940000000000003,"15min":6.5940000000000003},"last":3.7709999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":0.94399999999999995,"15min":0.72599999999999998},"min":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":9.6489999999999991,"5min":9.6489999999999991,"15min":9.6489999999999991},"last":3.9569999999999999},{"interface":"front","average":{"1min":1.4710000000000001,"5min":0.78600000000000003,"15min":0.67200000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.3200000000000003,"5min":9.3200000000000003,"15min":9.3200000000000003},"last":6.4569999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":0.81799999999999995,"15min":0.68999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":8.8599999999999994,"5min":8.8599999999999994,"15min":8.8599999999999994},"last":6.4950000000000001},{"interface":"front","average":{"1min":1.3520000000000001,"5min":0.78800000000000003,"15min":0.69399999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.2080000000000002,"5min":9.2080000000000002,"15min":9.2080000000000002},"last":6.0709999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019047,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3236184,"kb_used_data":3157904,"kb_used_omap":0,"kb_used_meta":78272,"kb_avail":90508968,"statfs":{"total":95995035648,"available":92681183232,"internally_reserved":0,"allocated":3233693696,"data_stored":3233089012,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":80150528},"hb_peers":[0,1,2,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:57:22 2023","interfaces":[{"interface":"back","average":{"1min":0.93400000000000005,"5min":0.62,"15min":0.55100000000000005},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":4.8760000000000003,"5min":4.8760000000000003,"15min":4.8760000000000003},"last":0.86199999999999999},{"interface":"front","average":{"1min":1.6559999999999999,"5min":0.80200000000000005,"15min":0.61699999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":5.1340000000000003,"5min":5.1340000000000003,"15min":5.1340000000000003},"last":11.372}]},{"osd":1,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.77600000000000002,"5min":0.59099999999999997,"15min":0.55100000000000005},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.786,"5min":2.786,"15min":2.786},"last":0.29499999999999998},{"interface":"front","average":{"1min":1.234,"5min":0.70699999999999996,"15min":0.60699999999999998},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":5.6900000000000004,"5min":5.6900000000000004,"15min":5.6900000000000004},"last":10.83}]},{"osd":2,"last update":"Sat Sep 23 17:57:28 2023","interfaces":[{"interface":"back","average":{"1min":1.6739999999999999,"5min":0.88700000000000001,"15min":0.71699999999999997},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.7160000000000002,"5min":5.7160000000000002,"15min":5.7160000000000002},"last":1.8959999999999999},{"interface":"front","average":{"1min":1.7110000000000001,"5min":0.85199999999999998,"15min":0.67400000000000004},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":4.899,"5min":4.899,"15min":4.899},"last":3.109}]},{"osd":3,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.72899999999999998,"15min":0.64900000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6130000000000004,"5min":4.6130000000000004,"15min":4.6130000000000004},"last":1.397},{"interface":"front","average":{"1min":1.04,"5min":0.73299999999999998,"15min":0.68200000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":3.637,"5min":3.637,"15min":3.637},"last":9.4359999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:57:28 2023","interfaces":[{"interface":"back","average":{"1min":0.75,"5min":0.51900000000000002,"15min":0.46300000000000002},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.7599999999999998,"5min":2.7599999999999998,"15min":2.7599999999999998},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.2809999999999999,"5min":0.67000000000000004,"15min":0.51500000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27500000000000002},"max":{"1min":10.494,"5min":10.494,"15min":10.494},"last":2.4209999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54200000000000004,"15min":0.50800000000000001},"min":{"1min":0.45400000000000001,"5min":0.33200000000000002,"15min":0.28899999999999998},"max":{"1min":0.64700000000000002,"5min":0.755,"15min":1.276},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.59299999999999997,"15min":0.54700000000000004},"min":{"1min":0.52500000000000002,"5min":0.29899999999999999,"15min":0.28999999999999998},"max":{"1min":0.79400000000000004,"5min":0.80900000000000005,"15min":1.1180000000000001},"last":0.505}]},{"osd":7,"last update":"Sat Sep 23 17:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.57399999999999995,"15min":0.55100000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.8370000000000002,"5min":2.8370000000000002,"15min":2.8370000000000002},"last":0.88900000000000001},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.64400000000000002,"15min":0.56299999999999994},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":2.4809999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757413,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3658380,"kb_used_data":3569988,"kb_used_omap":0,"kb_used_meta":88384,"kb_avail":90086772,"statfs":{"total":95995035648,"available":92248854528,"internally_reserved":0,"allocated":3655667712,"data_stored":3655070780,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":90505216},"hb_peers":[0,1,2,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,2,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:57:27 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":0.76700000000000002,"15min":0.64100000000000001},"min":{"1min":0.223,"5min":0.223,"15min":0.223},"max":{"1min":6.5609999999999999,"5min":6.5609999999999999,"15min":6.5609999999999999},"last":3.5840000000000001},{"interface":"front","average":{"1min":2.2999999999999998,"5min":0.96699999999999997,"15min":0.71299999999999997},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":8.2680000000000007,"5min":8.2680000000000007,"15min":8.2680000000000007},"last":3.5720000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:56:57 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":0.70499999999999996,"15min":0.71199999999999997},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":3.2330000000000001,"5min":4.4279999999999999,"15min":4.4279999999999999},"last":2.8130000000000002},{"interface":"front","average":{"1min":1.1879999999999999,"5min":0.68200000000000005,"15min":0.60499999999999998},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.4450000000000003,"5min":5.4450000000000003,"15min":5.4450000000000003},"last":3.3809999999999998}]},{"osd":2,"last update":"Sat Sep 23 17:57:27 2023","interfaces":[{"interface":"back","average":{"1min":3.085,"5min":1.1060000000000001,"15min":0.72899999999999998},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":38.606999999999999,"5min":38.606999999999999,"15min":38.606999999999999},"last":7.0119999999999996},{"interface":"front","average":{"1min":3.9689999999999999,"5min":1.2689999999999999,"15min":0.80100000000000005},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":39.670999999999999,"5min":39.670999999999999,"15min":39.670999999999999},"last":1.78}]},{"osd":3,"last update":"Sat Sep 23 17:56:57 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":0.78800000000000003,"15min":0.66400000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.4969999999999999,"5min":7.4969999999999999,"15min":7.4969999999999999},"last":1.998},{"interface":"front","average":{"1min":1.0900000000000001,"5min":0.70199999999999996,"15min":0.63800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.5510000000000002,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":5.109}]},{"osd":4,"last update":"Sat Sep 23 17:57:31 2023","interfaces":[{"interface":"back","average":{"1min":1.246,"5min":0.69199999999999995,"15min":0.52400000000000002},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":6.5490000000000004,"5min":6.5490000000000004,"15min":6.5490000000000004},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.58299999999999996,"15min":0.53400000000000003},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":3.383,"5min":3.383,"15min":3.383},"last":0.25600000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:55:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54400000000000004,"15min":0.53500000000000003},"min":{"1min":0.38800000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.66100000000000003,"5min":0.85499999999999998,"15min":1.778},"last":0.55000000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.45800000000000002,"5min":0.375,"15min":0.33200000000000002},"max":{"1min":0.72799999999999998,"5min":0.90200000000000002,"15min":1.7629999999999999},"last":0.63100000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.64800000000000002,"15min":0.55800000000000005},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.1719999999999997,"5min":4.1719999999999997,"15min":4.1719999999999997},"last":0.35399999999999998},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.67400000000000004,"15min":0.58699999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.5099999999999998,"5min":5.5099999999999998,"15min":5.5099999999999998},"last":0.36099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3350528000,"data_stored":3350528000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2305556480,"data_stored":2305556480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2645889024,"data_stored":2645884952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1253441536,"data_stored":1253441536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3226337280,"data_stored":3226337280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3650818048,"data_stored":3650813976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2023886848,"data_stored":2023882776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:57:40.524 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T17:57:40.524 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 11 2023-09-23T17:57:40.545 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:40.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:40.569 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:40.643 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:57:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:57:40] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-09-23T17:57:40.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:40 smithi055 ceph-mon[179668]: from='client.34926 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:57:40.645 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:40 smithi055 ceph-mon[184174]: from='client.34926 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:57:40.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:40.700 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:40 smithi099 ceph-mon[167076]: from='client.34926 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:57:40.701 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:40.744 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:40.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:40.748 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:40.849 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:40.939 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T17:57:40.948 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:41.035 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:41.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:41.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:41.142 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:41.179 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:41.219 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:41.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:41.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:41.323 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:41.379 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:41.432 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:41.532 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T17:57:41.662 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:41.684 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:41 smithi055 ceph-mon[179668]: pgmap v316: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 3.7 MiB/s wr, 662 op/s; 37220/296001 objects degraded (12.574%); 6025/296001 objects misplaced (2.035%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T17:57:41.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:41 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/180261640' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:57:41.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:41 smithi055 ceph-mon[184174]: pgmap v316: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 3.7 MiB/s wr, 662 op/s; 37220/296001 objects degraded (12.574%); 6025/296001 objects misplaced (2.035%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T17:57:41.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:41 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/180261640' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:57:41.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:41 smithi099 ceph-mon[167076]: pgmap v316: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 6.0 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 3.7 MiB/s wr, 662 op/s; 37220/296001 objects degraded (12.574%); 6025/296001 objects misplaced (2.035%); 4.7 MiB/s, 74 objects/s recovering 2023-09-23T17:57:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:41 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/180261640' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:57:41.961 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:41.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:41.976 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:42.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:42.108 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:42.156 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:42.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:42.163 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:42.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:42.297 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:42.344 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:42.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:42.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:42.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:42.485 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:42.524 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:42.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:42.529 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:42.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:42.679 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:42.725 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:42.729 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:42 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:42.730 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:42 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/180261640' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:57:42.730 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:42 smithi099 ceph-mon[167076]: osdmap e146: 8 total, 6 up, 8 in 2023-09-23T17:57:42.730 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:42 smithi099 ceph-mon[167076]: failed to encode map e146 with expected crc 2023-09-23T17:57:42.730 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:42.730 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:42.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:42.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:42.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:42.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/180261640' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:57:42.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[184174]: osdmap e146: 8 total, 6 up, 8 in 2023-09-23T17:57:42.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[184174]: failed to encode map e146 with expected crc 2023-09-23T17:57:42.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:42.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:42.775 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:42.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/180261640' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:57:42.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[179668]: osdmap e146: 8 total, 6 up, 8 in 2023-09-23T17:57:42.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[179668]: failed to encode map e146 with expected crc 2023-09-23T17:57:42.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:42.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:42.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:42.869 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:42.909 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:42.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:42.913 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:43.014 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:43.102 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:43.140 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:43.245 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:43.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:43.258 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:43.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:43.407 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:43.448 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:43.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:43.451 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:43.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:43.581 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:43.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:43 smithi055 ceph-mon[184174]: pgmap v318: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 5.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 36607/291315 objects degraded (12.566%); 5749/291315 objects misplaced (1.973%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T17:57:43.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 37220/296001 objects degraded (12.574%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:43.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:43 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:57:43.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:43 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:57:43.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:43 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:43.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:43 smithi055 ceph-mon[179668]: pgmap v318: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 5.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 36607/291315 objects degraded (12.566%); 5749/291315 objects misplaced (1.973%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T17:57:43.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 37220/296001 objects degraded (12.574%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:43.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:43 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:57:43.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:43 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:57:43.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:43 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:43.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:43.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:43.630 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:43.731 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:43.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:43 smithi099 ceph-mon[167076]: pgmap v318: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 5.9 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 700 op/s; 36607/291315 objects degraded (12.566%); 5749/291315 objects misplaced (1.973%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T17:57:43.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 37220/296001 objects degraded (12.574%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:43.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:43 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:57:43.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:43 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:57:43.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:43 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:43.766 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:43.806 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:43.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:43.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:43.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:43.943 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:43.986 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:43.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:43.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:44.015 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:57:44.064 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:57:44.089 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:44.122 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:57:44.128 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:44.161 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:57:44.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:44.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:44.176 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:44.198 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:57:44.250 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:57:44.250 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:57:44.250 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:57:44.250 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:57:44.250 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:57:44.250 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:57:44.251 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:57:44.251 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:57:44.251 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T17:57:44.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:44.313 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:44.356 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:44.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:44.359 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:44.456 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T17:57:44.460 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:44.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1478048795' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T17:57:44.531 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/89451391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:44.531 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1478048795' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T17:57:44.531 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/89451391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:44.557 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 1 pg_num to 11 2023-09-23T17:57:44.606 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:44.737 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:44.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:44.742 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:44.795 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:57:44.796 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:57:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1478048795' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]: dispatch 2023-09-23T17:57:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/89451391' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:44.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:44.877 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:44.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:44.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:44.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:45.029 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:45.110 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:45.196 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:45.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:45.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:45.287 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:57:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T17:57:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T17:57:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T17:57:45.288 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:57:45.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:45.347 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:45.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:45.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:45.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:45.402 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:45.497 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:45.574 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:45.623 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T17:57:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:45 smithi055 ceph-mon[179668]: pgmap v319: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 36062/287007 objects degraded (12.565%); 5749/287007 objects misplaced (2.003%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T17:57:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1478048795' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T17:57:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:45 smithi055 ceph-mon[179668]: osdmap e147: 8 total, 6 up, 8 in 2023-09-23T17:57:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:45 smithi055 ceph-mon[179668]: failed to encode map e147 with expected crc 2023-09-23T17:57:45.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:45 smithi055 ceph-mon[179668]: failed to encode map e147 with expected crc 2023-09-23T17:57:45.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:45 smithi055 ceph-mon[184174]: pgmap v319: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 36062/287007 objects degraded (12.565%); 5749/287007 objects misplaced (2.003%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T17:57:45.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1478048795' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T17:57:45.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:45 smithi055 ceph-mon[184174]: osdmap e147: 8 total, 6 up, 8 in 2023-09-23T17:57:45.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:45 smithi055 ceph-mon[184174]: failed to encode map e147 with expected crc 2023-09-23T17:57:45.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:45 smithi055 ceph-mon[184174]: failed to encode map e147 with expected crc 2023-09-23T17:57:45.701 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:45.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:45.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:45.811 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:45.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:45 smithi099 ceph-mon[167076]: pgmap v319: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 36062/287007 objects degraded (12.565%); 5749/287007 objects misplaced (2.003%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T17:57:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1478048795' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "11"}]': finished 2023-09-23T17:57:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:45 smithi099 ceph-mon[167076]: osdmap e147: 8 total, 6 up, 8 in 2023-09-23T17:57:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:45 smithi099 ceph-mon[167076]: failed to encode map e147 with expected crc 2023-09-23T17:57:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:45 smithi099 ceph-mon[167076]: failed to encode map e147 with expected crc 2023-09-23T17:57:45.877 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:45.978 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:45.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:45.982 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:46.083 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:46.122 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:46.177 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:46.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:46.182 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:46.283 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:46.331 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:46.370 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:46.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:46.373 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:46.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:46.504 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:46.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:46.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:46.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:46.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:46.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:46 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T17:57:46.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:46 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T17:57:46.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:46 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]: dispatch 2023-09-23T17:57:46.687 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:46.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:46.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:46.750 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:46.797 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:57:46.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:46.896 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:46.940 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:46.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:46.943 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:46.982 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:47.044 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:47.160 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:47.298 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:47.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:47.309 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:47.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:47.457 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:47.502 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:47.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:47.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:47.572 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T17:57:47.613 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:47.725 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:47 smithi099 ceph-mon[167076]: pgmap v321: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 35845/285735 objects degraded (12.545%); 5648/285735 objects misplaced (1.977%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T17:57:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T17:57:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:47 smithi099 ceph-mon[167076]: osdmap e148: 8 total, 6 up, 8 in 2023-09-23T17:57:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:47 smithi099 ceph-mon[167076]: failed to encode map e148 with expected crc 2023-09-23T17:57:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1685363321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:47.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:47 smithi055 ceph-mon[184174]: pgmap v321: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 35845/285735 objects degraded (12.545%); 5648/285735 objects misplaced (1.977%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T17:57:47.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T17:57:47.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:47 smithi055 ceph-mon[184174]: osdmap e148: 8 total, 6 up, 8 in 2023-09-23T17:57:47.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:47 smithi055 ceph-mon[184174]: failed to encode map e148 with expected crc 2023-09-23T17:57:47.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1685363321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:47 smithi055 ceph-mon[179668]: pgmap v321: 89 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded; 5.8 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 643 op/s; 35845/285735 objects degraded (12.545%); 5648/285735 objects misplaced (1.977%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T17:57:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "11"}]': finished 2023-09-23T17:57:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:47 smithi055 ceph-mon[179668]: osdmap e148: 8 total, 6 up, 8 in 2023-09-23T17:57:47.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:47 smithi055 ceph-mon[179668]: failed to encode map e148 with expected crc 2023-09-23T17:57:47.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1685363321' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:57:48.032 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:48.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:48.039 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:48.140 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:48.183 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:48.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:48.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:48.233 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:48.279 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:48.334 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:48.445 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:48.572 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:48.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:48.578 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:48.679 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:48.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:57:48.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1685363321' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:57:48.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[179668]: osdmap e149: 8 total, 6 up, 8 in 2023-09-23T17:57:48.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[179668]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[179668]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[179668]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[179668]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 35845/285735 objects degraded (12.545%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:48.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:57:48.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1685363321' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:57:48.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[184174]: osdmap e149: 8 total, 6 up, 8 in 2023-09-23T17:57:48.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[184174]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[184174]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[184174]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[184174]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 35845/285735 objects degraded (12.545%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:48.774 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:48.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:48 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:57:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:48 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1685363321' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:57:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:48 smithi099 ceph-mon[167076]: osdmap e149: 8 total, 6 up, 8 in 2023-09-23T17:57:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:48 smithi099 ceph-mon[167076]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:48 smithi099 ceph-mon[167076]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:48 smithi099 ceph-mon[167076]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:48 smithi099 ceph-mon[167076]: failed to encode map e149 with expected crc 2023-09-23T17:57:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 35845/285735 objects degraded (12.545%), 15 pgs degraded, 11 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:48.849 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:48.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:48.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:48.956 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:48.992 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:49.036 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:49.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:49.042 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:49.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:49.164 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:57:49.164 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T17:57:49.164 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T17:57:49.164 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:57:49.164 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.164 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81746 5.a 5:548e2426:::benchmark_data_smithi055_229117_object81745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.164 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.824853+0000", 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.334652652999999, 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.981367561, 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81746 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824853+0000", 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824853+0000", 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824852+0000", 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824874+0000", 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1708e-05 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824882+0000", 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8860000000000004e-06 2023-09-23T17:57:49.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.168 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824888+0000", 2023-09-23T17:57:49.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2349999999999996e-06 2023-09-23T17:57:49.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.168 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825061+0000", 2023-09-23T17:57:49.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000172561 2023-09-23T17:57:49.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825071+0000", 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7499999999999998e-06 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.802556+0000", 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97748503399999997 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.802726+0000", 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000170237 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.802912+0000", 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018652499999999999 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806049+0000", 2023-09-23T17:57:49.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031369240000000001 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806100+0000", 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1023000000000003e-05 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806112+0000", 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1737e-05 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806175+0000", 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3121999999999996e-05 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806220+0000", 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5333000000000001e-05 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.172 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81756 5.a 5:559c0640:::benchmark_data_smithi055_229117_object81755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.824967+0000", 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.334538858999998, 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98960265000000003, 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.173 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81756 2023-09-23T17:57:49.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.174 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824967+0000", 2023-09-23T17:57:49.174 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824967+0000", 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824966+0000", 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.175 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.176 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824987+0000", 2023-09-23T17:57:49.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0706000000000001e-05 2023-09-23T17:57:49.176 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.176 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825006+0000", 2023-09-23T17:57:49.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8797e-05 2023-09-23T17:57:49.176 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825012+0000", 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8000000000000001e-06 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825091+0000", 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9028000000000003e-05 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825098+0000", 2023-09-23T17:57:49.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9299999999999997e-06 2023-09-23T17:57:49.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.178 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806272+0000", 2023-09-23T17:57:49.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98117399000000005 2023-09-23T17:57:49.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806438+0000", 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000165429 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806615+0000", 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000177312 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.180 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814377+0000", 2023-09-23T17:57:49.180 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0077615449999999999 2023-09-23T17:57:49.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.180 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.180 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814435+0000", 2023-09-23T17:57:49.180 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.817e-05 2023-09-23T17:57:49.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.181 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814447+0000", 2023-09-23T17:57:49.181 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2476999999999999e-05 2023-09-23T17:57:49.181 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.181 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.181 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814503+0000", 2023-09-23T17:57:49.181 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5831999999999997e-05 2023-09-23T17:57:49.181 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.181 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.182 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.182 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814569+0000", 2023-09-23T17:57:49.182 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6179000000000005e-05 2023-09-23T17:57:49.182 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.182 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.182 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.182 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.182 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.182 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81758 5.a 5:552239cf:::benchmark_data_smithi055_229117_object81757:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.183 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.828535+0000", 2023-09-23T17:57:49.183 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.330970731999997, 2023-09-23T17:57:49.183 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98621836399999996, 2023-09-23T17:57:49.183 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.183 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.183 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.183 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.183 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.183 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81758 2023-09-23T17:57:49.184 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.184 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.184 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.184 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.184 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828535+0000", 2023-09-23T17:57:49.184 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.185 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828535+0000", 2023-09-23T17:57:49.185 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828534+0000", 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828554+0000", 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0326000000000002e-05 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828567+0000", 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2809999999999999e-05 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828573+0000", 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8860000000000001e-06 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828630+0000", 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7315999999999998e-05 2023-09-23T17:57:49.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828642+0000", 2023-09-23T17:57:49.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1977000000000001e-05 2023-09-23T17:57:49.188 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.808815+0000", 2023-09-23T17:57:49.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98017351399999997 2023-09-23T17:57:49.188 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.808927+0000", 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111715 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.809046+0000", 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011884599999999999 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.189 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814587+0000", 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055405929999999999 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814625+0000", 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8918000000000002e-05 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814664+0000", 2023-09-23T17:57:49.190 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8680999999999998e-05 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814712+0000", 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7871000000000003e-05 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814753+0000", 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1134000000000003e-05 2023-09-23T17:57:49.191 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81760 5.a 5:5730c3c9:::benchmark_data_smithi055_229117_object81759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.837793+0000", 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.321712421999997, 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97716881300000002, 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.192 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81760 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837793+0000", 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.193 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837793+0000", 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837792+0000", 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837822+0000", 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0035e-05 2023-09-23T17:57:49.194 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837840+0000", 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.88e-05 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837850+0000", 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.007e-05 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.195 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837909+0000", 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8996000000000002e-05 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837921+0000", 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1319e-05 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.196 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.811465+0000", 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97354397400000003 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.811538+0000", 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3102000000000006e-05 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.811636+0000", 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7844999999999996e-05 2023-09-23T17:57:49.197 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814769+0000", 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003133786 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814830+0000", 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0924999999999997e-05 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.198 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814889+0000", 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8860999999999998e-05 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814930+0000", 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0921999999999999e-05 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814962+0000", 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1746999999999999e-05 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81770 5.a 5:576b5ba8:::benchmark_data_smithi055_229117_object81769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.845003+0000", 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.314503045000002, 2023-09-23T17:57:49.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98902187100000005, 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81770 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.201 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845003+0000", 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845003+0000", 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845002+0000", 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:49.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845024+0000", 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2520000000000001e-05 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845037+0000", 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2237e-05 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845044+0000", 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0110000000000003e-06 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845100+0000", 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6369999999999997e-05 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845113+0000", 2023-09-23T17:57:49.205 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.322e-05 2023-09-23T17:57:49.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.205 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.205 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.815001+0000", 2023-09-23T17:57:49.205 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96988796099999997 2023-09-23T17:57:49.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.205 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.815142+0000", 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014074700000000001 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.815314+0000", 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000171626 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.206 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.207 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827641+0000", 2023-09-23T17:57:49.207 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012327829 2023-09-23T17:57:49.207 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.207 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.207 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.207 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827724+0000", 2023-09-23T17:57:49.207 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2349999999999996e-05 2023-09-23T17:57:49.207 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.207 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.208 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.208 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.833926+0000", 2023-09-23T17:57:49.208 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006201692 2023-09-23T17:57:49.208 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.208 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.208 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.208 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.833978+0000", 2023-09-23T17:57:49.208 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2855000000000001e-05 2023-09-23T17:57:49.208 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.208 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.209 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.209 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834024+0000", 2023-09-23T17:57:49.209 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6004999999999997e-05 2023-09-23T17:57:49.209 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.209 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.209 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.209 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.209 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81774 5.a 5:516f8aea:::benchmark_data_smithi055_229117_object81773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.209 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.852382+0000", 2023-09-23T17:57:49.210 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.307123138000001, 2023-09-23T17:57:49.210 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98193056000000001, 2023-09-23T17:57:49.210 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.210 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.210 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.210 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.210 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.210 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81774 2023-09-23T17:57:49.210 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.210 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.211 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.211 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.211 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852382+0000", 2023-09-23T17:57:49.211 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.211 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.211 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.211 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.212 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852382+0000", 2023-09-23T17:57:49.212 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.212 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.212 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.212 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.212 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852381+0000", 2023-09-23T17:57:49.212 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:49.212 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.212 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.212 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.213 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852406+0000", 2023-09-23T17:57:49.213 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4822000000000001e-05 2023-09-23T17:57:49.213 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.213 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.213 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.213 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852416+0000", 2023-09-23T17:57:49.213 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0215e-05 2023-09-23T17:57:49.213 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.213 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.213 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852423+0000", 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0520000000000004e-06 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852471+0000", 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7691999999999997e-05 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852482+0000", 2023-09-23T17:57:49.214 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1433e-05 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.818150+0000", 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96566818200000004 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.818306+0000", 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015550699999999999 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.216 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.216 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.818470+0000", 2023-09-23T17:57:49.216 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016395099999999999 2023-09-23T17:57:49.216 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.216 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.216 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.216 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827682+0000", 2023-09-23T17:57:49.216 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092118649999999996 2023-09-23T17:57:49.216 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.217 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.217 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.217 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827763+0000", 2023-09-23T17:57:49.217 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0962000000000002e-05 2023-09-23T17:57:49.217 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.217 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.217 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.217 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834224+0000", 2023-09-23T17:57:49.218 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0064616430000000004 2023-09-23T17:57:49.218 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.218 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.218 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.218 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834275+0000", 2023-09-23T17:57:49.218 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0606000000000003e-05 2023-09-23T17:57:49.219 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.219 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.219 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834313+0000", 2023-09-23T17:57:49.219 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8012000000000001e-05 2023-09-23T17:57:49.219 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.219 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81777 5.a 5:5126b3af:::benchmark_data_smithi055_229117_object81776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.854699+0000", 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.304806810000002, 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97978332999999995, 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.220 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81777 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854699+0000", 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854699+0000", 2023-09-23T17:57:49.222 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.222 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854697+0000", 2023-09-23T17:57:49.222 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:49.222 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854727+0000", 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0085000000000001e-05 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854743+0000", 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5991e-05 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.223 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854753+0000", 2023-09-23T17:57:49.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3789999999999995e-06 2023-09-23T17:57:49.224 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.224 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.224 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.224 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854836+0000", 2023-09-23T17:57:49.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3842000000000001e-05 2023-09-23T17:57:49.224 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.224 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.224 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.225 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854847+0000", 2023-09-23T17:57:49.225 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0564000000000001e-05 2023-09-23T17:57:49.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.225 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.821488+0000", 2023-09-23T17:57:49.225 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96664134099999999 2023-09-23T17:57:49.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.821641+0000", 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015245499999999999 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.821806+0000", 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016498899999999999 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834142+0000", 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012336132 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834326+0000", 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018421999999999999 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834389+0000", 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2810000000000003e-05 2023-09-23T17:57:49.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834442+0000", 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3480999999999998e-05 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834482+0000", 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9580999999999999e-05 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.228 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81792 5.a 5:5acbb496:::benchmark_data_smithi055_229117_object81791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.868584+0000", 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.290921525000002, 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97274865799999999, 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.229 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81792 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868584+0000", 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868584+0000", 2023-09-23T17:57:49.230 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868583+0000", 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868604+0000", 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1667000000000001e-05 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.231 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868619+0000", 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4938e-05 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868627+0000", 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4719999999999998e-06 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868689+0000", 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2501000000000003e-05 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868702+0000", 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2771e-05 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.824681+0000", 2023-09-23T17:57:49.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95597861799999995 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.824821+0000", 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000140765 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.824973+0000", 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015180900000000001 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834213+0000", 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092394269999999997 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834334+0000", 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000121002 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.235 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841237+0000", 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0069033690000000003 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841290+0000", 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2926000000000001e-05 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841333+0000", 2023-09-23T17:57:49.236 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2759000000000002e-05 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81799 5.a 5:5098facb:::benchmark_data_smithi055_229117_object81798:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.875371+0000", 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.284134215000002, 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96608565800000001, 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.237 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.238 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.238 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.238 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81799 2023-09-23T17:57:49.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.238 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.238 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.238 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.238 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875371+0000", 2023-09-23T17:57:49.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875371+0000", 2023-09-23T17:57:49.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875370+0000", 2023-09-23T17:57:49.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:49.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875431+0000", 2023-09-23T17:57:49.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1605000000000004e-05 2023-09-23T17:57:49.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875446+0000", 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4232e-05 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875452+0000", 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1809999999999998e-06 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875509+0000", 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6781999999999999e-05 2023-09-23T17:57:49.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.242 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875518+0000", 2023-09-23T17:57:49.242 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8719999999999997e-06 2023-09-23T17:57:49.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.242 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827790+0000", 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95227146299999998 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827908+0000", 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118269 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.828034+0000", 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012610400000000001 2023-09-23T17:57:49.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834342+0000", 2023-09-23T17:57:49.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063074409999999996 2023-09-23T17:57:49.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834741+0000", 2023-09-23T17:57:49.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00039909100000000002 2023-09-23T17:57:49.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841382+0000", 2023-09-23T17:57:49.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006641152 2023-09-23T17:57:49.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841417+0000", 2023-09-23T17:57:49.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4853e-05 2023-09-23T17:57:49.246 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.246 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.246 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.246 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841457+0000", 2023-09-23T17:57:49.246 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0151999999999998e-05 2023-09-23T17:57:49.246 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.246 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.246 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.246 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.247 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.247 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81811 5.a 5:57532f39:::benchmark_data_smithi055_229117_object81810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.247 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.896344+0000", 2023-09-23T17:57:49.247 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.263161689999997, 2023-09-23T17:57:49.247 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.94522273800000001, 2023-09-23T17:57:49.247 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.247 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.247 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.247 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.247 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.248 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81811 2023-09-23T17:57:49.248 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.248 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.248 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.248 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.248 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896344+0000", 2023-09-23T17:57:49.248 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.248 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.248 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.249 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.249 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896344+0000", 2023-09-23T17:57:49.249 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.249 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.249 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.249 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.249 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896342+0000", 2023-09-23T17:57:49.249 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:49.249 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.250 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.250 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896381+0000", 2023-09-23T17:57:49.250 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8726000000000002e-05 2023-09-23T17:57:49.250 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.250 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.250 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896400+0000", 2023-09-23T17:57:49.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9658999999999999e-05 2023-09-23T17:57:49.251 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.251 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.251 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.251 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896411+0000", 2023-09-23T17:57:49.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0647e-05 2023-09-23T17:57:49.251 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.251 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896463+0000", 2023-09-23T17:57:49.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2105999999999999e-05 2023-09-23T17:57:49.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896472+0000", 2023-09-23T17:57:49.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6349999999999995e-06 2023-09-23T17:57:49.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.830796+0000", 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.93432424199999997 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.830941+0000", 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014539899999999999 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.253 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.831093+0000", 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015141699999999999 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834752+0000", 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036591800000000002 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.254 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834803+0000", 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1011999999999999e-05 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841509+0000", 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006706147 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841547+0000", 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7799000000000002e-05 2023-09-23T17:57:49.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841567+0000", 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9629000000000001e-05 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81815 5.a 5:50ec5bb9:::benchmark_data_smithi055_229117_object81814:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:57:49.256 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.900377+0000", 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 51.259128785000001, 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.94830255600000002, 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81815 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.257 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900377+0000", 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900377+0000", 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900376+0000", 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900395+0000", 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9397000000000001e-05 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900407+0000", 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2342e-05 2023-09-23T17:57:49.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900413+0000", 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4099999999999999e-06 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900470+0000", 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7207000000000003e-05 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900484+0000", 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4283e-05 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834833+0000", 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.93434895699999998 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834954+0000", 2023-09-23T17:57:49.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120833 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.835096+0000", 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000142354 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841472+0000", 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063759350000000001 2023-09-23T17:57:49.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.848551+0000", 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.007078689 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.848594+0000", 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2923000000000002e-05 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.848637+0000", 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2670999999999999e-05 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.848679+0000", 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2614999999999998e-05 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104098 5.a 5:5150641c:::benchmark_data_smithi055_229117_object104097:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.920515+0000", 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 25.238990128000001, 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.048992793, 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104098 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.265 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920538+0000", 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2770000000000001e-05 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920545+0000", 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1969999999999999e-06 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920550+0000", 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3360000000000002e-06 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927587+0000", 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0070371599999999998 2023-09-23T17:57:49.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927719+0000", 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013187600000000001 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927856+0000", 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013721800000000001 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936497+0000", 2023-09-23T17:57:49.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.008640277 2023-09-23T17:57:49.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936670+0000", 2023-09-23T17:57:49.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017294700000000001 2023-09-23T17:57:49.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959038+0000", 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022367973999999999 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959048+0000", 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0428000000000001e-05 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969404+0000", 2023-09-23T17:57:49.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0103555609999999 2023-09-23T17:57:49.272 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969508+0000", 2023-09-23T17:57:49.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010461799999999999 2023-09-23T17:57:49.272 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.273 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.273 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.273 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104100 5.a 5:58ebf1c5:::benchmark_data_smithi055_229117_object104099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:57:49.273 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.920555+0000", 2023-09-23T17:57:49.273 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 25.238950449000001, 2023-09-23T17:57:49.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0492678900000001, 2023-09-23T17:57:49.273 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.273 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:57:49.274 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.274 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.274 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.274 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104100 2023-09-23T17:57:49.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.274 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T17:57:49.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T17:57:49.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T17:57:49.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:49.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920574+0000", 2023-09-23T17:57:49.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9664e-05 2023-09-23T17:57:49.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920581+0000", 2023-09-23T17:57:49.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3260000000000001e-06 2023-09-23T17:57:49.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920586+0000", 2023-09-23T17:57:49.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7239999999999997e-06 2023-09-23T17:57:49.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.931735+0000", 2023-09-23T17:57:49.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011148345 2023-09-23T17:57:49.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.931905+0000", 2023-09-23T17:57:49.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000170466 2023-09-23T17:57:49.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.932081+0000", 2023-09-23T17:57:49.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000175623 2023-09-23T17:57:49.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936776+0000", 2023-09-23T17:57:49.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0046956209999999996 2023-09-23T17:57:49.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.280 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936996+0000", 2023-09-23T17:57:49.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021987100000000001 2023-09-23T17:57:49.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.280 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.280 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959173+0000", 2023-09-23T17:57:49.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022176392999999999 2023-09-23T17:57:49.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959189+0000", 2023-09-23T17:57:49.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.609e-05 2023-09-23T17:57:49.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969761+0000", 2023-09-23T17:57:49.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0105722159999999 2023-09-23T17:57:49.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969823+0000", 2023-09-23T17:57:49.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1938000000000002e-05 2023-09-23T17:57:49.282 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.282 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.282 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104109 5.a 5:542cd2b3:::benchmark_data_smithi055_229117_object104108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.944006+0000", 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 25.215499884, 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0419720189999999, 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.283 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104109 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944006+0000", 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944006+0000", 2023-09-23T17:57:49.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944004+0000", 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944029+0000", 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5201000000000001e-05 2023-09-23T17:57:49.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.286 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.286 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.286 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944043+0000", 2023-09-23T17:57:49.286 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3964e-05 2023-09-23T17:57:49.286 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.286 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.286 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.286 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944062+0000", 2023-09-23T17:57:49.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8190999999999999e-05 2023-09-23T17:57:49.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944258+0000", 2023-09-23T17:57:49.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019652100000000001 2023-09-23T17:57:49.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944396+0000", 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013821100000000001 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944572+0000", 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017574299999999999 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.956893+0000", 2023-09-23T17:57:49.289 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012321445 2023-09-23T17:57:49.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.289 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.289 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959227+0000", 2023-09-23T17:57:49.289 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023331559999999999 2023-09-23T17:57:49.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.289 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.289 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959235+0000", 2023-09-23T17:57:49.290 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1000000000000004e-06 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969881+0000", 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.010645899 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.970644+0000", 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000763867 2023-09-23T17:57:49.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.985881+0000", 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015236622 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.985978+0000", 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6602999999999996e-05 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.292 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104112 5.a 5:5b2e2197:::benchmark_data_smithi055_229117_object104111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.944077+0000", 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 25.215428743, 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.042230502, 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.293 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104112 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944077+0000", 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944077+0000", 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944076+0000", 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944095+0000", 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8532000000000001e-05 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944103+0000", 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8399999999999995e-06 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944108+0000", 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8510000000000004e-06 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.949704+0000", 2023-09-23T17:57:49.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055955110000000001 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.949880+0000", 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000176312 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.950043+0000", 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016271800000000001 2023-09-23T17:57:49.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959270+0000", 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092269080000000007 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959278+0000", 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0590000000000003e-06 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.970705+0000", 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.011427144 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971279+0000", 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00057403800000000002 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986229+0000", 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014949831 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986307+0000", 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8319999999999996e-05 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.300 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104117 5.a 5:5725f10b:::benchmark_data_smithi055_229117_object104116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.952810+0000", 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 25.206695642, 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.039709502, 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104117 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.301 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952810+0000", 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952810+0000", 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952808+0000", 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952833+0000", 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5307999999999999e-05 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952845+0000", 2023-09-23T17:57:49.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2145e-05 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952855+0000", 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2730000000000008e-06 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959408+0000", 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0065535530000000002 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959416+0000", 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9529999999999999e-06 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971413+0000", 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0119968539999999 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971561+0000", 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014827199999999999 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971748+0000", 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000186701 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986344+0000", 2023-09-23T17:57:49.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014596067000000001 2023-09-23T17:57:49.307 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.307 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.307 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.307 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992361+0000", 2023-09-23T17:57:49.307 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060167379999999998 2023-09-23T17:57:49.307 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.307 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.307 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.307 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992422+0000", 2023-09-23T17:57:49.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1598000000000002e-05 2023-09-23T17:57:49.308 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.308 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.308 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.308 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992519+0000", 2023-09-23T17:57:49.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6966999999999995e-05 2023-09-23T17:57:49.308 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.308 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.308 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104133 5.a 5:54e6b92a:::benchmark_data_smithi055_229117_object104132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.987398+0000", 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 25.172107691000001, 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0054708569999999, 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.309 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.310 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104133 2023-09-23T17:57:49.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.310 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987398+0000", 2023-09-23T17:57:49.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987398+0000", 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987395+0000", 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987424+0000", 2023-09-23T17:57:49.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8217000000000001e-05 2023-09-23T17:57:49.312 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.312 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987448+0000", 2023-09-23T17:57:49.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4227e-05 2023-09-23T17:57:49.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.313 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.313 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.313 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987458+0000", 2023-09-23T17:57:49.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0694e-05 2023-09-23T17:57:49.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.313 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.313 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.313 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987514+0000", 2023-09-23T17:57:49.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5435e-05 2023-09-23T17:57:49.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987524+0000", 2023-09-23T17:57:49.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0103e-05 2023-09-23T17:57:49.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.974903+0000", 2023-09-23T17:57:49.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98737907599999997 2023-09-23T17:57:49.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.975050+0000", 2023-09-23T17:57:49.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000146692 2023-09-23T17:57:49.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T17:57:49.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.975200+0000", 2023-09-23T17:57:49.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015015800000000001 2023-09-23T17:57:49.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.316 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986377+0000", 2023-09-23T17:57:49.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011176962 2023-09-23T17:57:49.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992733+0000", 2023-09-23T17:57:49.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063557029999999999 2023-09-23T17:57:49.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992783+0000", 2023-09-23T17:57:49.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0290999999999997e-05 2023-09-23T17:57:49.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992869+0000", 2023-09-23T17:57:49.318 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5835999999999996e-05 2023-09-23T17:57:49.318 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.318 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.318 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.319 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104140 5.a 5:581db13e:::benchmark_data_smithi055_229117_object104139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:57:49.319 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.998364+0000", 2023-09-23T17:57:49.319 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 25.161141097000002, 2023-09-23T17:57:49.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99464131499999997, 2023-09-23T17:57:49.319 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.319 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.319 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.319 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.319 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104140 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998364+0000", 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.320 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998364+0000", 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998363+0000", 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998390+0000", 2023-09-23T17:57:49.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7362999999999999e-05 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998404+0000", 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4058e-05 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998410+0000", 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0569999999999998e-06 2023-09-23T17:57:49.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998457+0000", 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7182999999999999e-05 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998466+0000", 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9940000000000006e-06 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.977786+0000", 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97931971699999998 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.977967+0000", 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000181263 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T17:57:49.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.978151+0000", 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018419000000000001 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986409+0000", 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082578100000000008 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992877+0000", 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0064680789999999998 2023-09-23T17:57:49.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992923+0000", 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5804999999999999e-05 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.993006+0000", 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2649000000000001e-05 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.326 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.327 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.327 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104141 5.a 5:5985f000:::benchmark_data_smithi055_229117_object104140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:57:49.327 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.998421+0000", 2023-09-23T17:57:49.327 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 25.161084429999999, 2023-09-23T17:57:49.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99472267199999997, 2023-09-23T17:57:49.327 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.327 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.327 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104141 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998421+0000", 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998421+0000", 2023-09-23T17:57:49.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998420+0000", 2023-09-23T17:57:49.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:49.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998450+0000", 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9964e-05 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998457+0000", 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7769999999999997e-06 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998461+0000", 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1649999999999999e-06 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998503+0000", 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1795999999999998e-05 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998512+0000", 2023-09-23T17:57:49.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5329999999999994e-06 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.981682+0000", 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98317060099999998 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.981856+0000", 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017383399999999999 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.982035+0000", 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017870399999999999 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986445+0000", 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044101949999999996 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.993014+0000", 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006568679 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.993061+0000", 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6820999999999998e-05 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.993144+0000", 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3287000000000004e-05 2023-09-23T17:57:49.334 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104161 5.a 5:587441b0:::benchmark_data_smithi055_229117_object104160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:24.054963+0000", 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 25.104542160000001, 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.93990519500000003, 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:49.335 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104161 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.054963+0000", 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.054963+0000", 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.054961+0000", 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.054995+0000", 2023-09-23T17:57:49.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4359000000000001e-05 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.055022+0000", 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6157e-05 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.055036+0000", 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4671999999999999e-05 2023-09-23T17:57:49.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.055101+0000", 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4274000000000002e-05 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.055111+0000", 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0294e-05 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:57:49.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986468+0000", 2023-09-23T17:57:49.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.93135727199999996 2023-09-23T17:57:49.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:57:49.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986639+0000", 2023-09-23T17:57:49.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017086799999999999 2023-09-23T17:57:49.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986812+0000", 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017333600000000001 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.993199+0000", 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063870439999999997 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:57:49.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.994759+0000", 2023-09-23T17:57:49.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015594719999999999 2023-09-23T17:57:49.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:57:49.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.994804+0000", 2023-09-23T17:57:49.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.494e-05 2023-09-23T17:57:49.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:57:49.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:57:49.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:57:49.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.994869+0000", 2023-09-23T17:57:49.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4791999999999999e-05 2023-09-23T17:57:49.343 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.343 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.343 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.343 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:57:49.343 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:57:49.343 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:57:49.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:49.345 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:49.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:49.345 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:49.446 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:49.510 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:49.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:49.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:49.564 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:49.665 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:49.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T17:57:49.783 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:49.856 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:49.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:49.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:49.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:49 smithi055 ceph-mon[184174]: pgmap v324: 99 pgs: 1 peering, 10 unknown, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 11 active+undersized+degraded; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 35449/282123 objects degraded (12.565%); 5439/282123 objects misplaced (1.928%); 5.1 MiB/s, 84 objects/s recovering 2023-09-23T17:57:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:49 smithi055 ceph-mon[179668]: pgmap v324: 99 pgs: 1 peering, 10 unknown, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 11 active+undersized+degraded; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 35449/282123 objects degraded (12.565%); 5439/282123 objects misplaced (1.928%); 5.1 MiB/s, 84 objects/s recovering 2023-09-23T17:57:49.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:50.006 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:49 smithi099 ceph-mon[167076]: pgmap v324: 99 pgs: 1 peering, 10 unknown, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 11 active+undersized+degraded; 5.7 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 35449/282123 objects degraded (12.565%); 5439/282123 objects misplaced (1.928%); 5.1 MiB/s, 84 objects/s recovering 2023-09-23T17:57:50.013 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:50.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:50.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:50.072 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:50.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T17:57:50.179 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T17:57:50.220 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:50.298 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6, 3] live_osds: [0, 1, 2, 4, 5, 7] 2023-09-23T17:57:50.298 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:57:50.299 INFO:tasks.thrashosds.thrasher:Killing osd 7, live_osds are [0, 1, 2, 4, 5, 7] 2023-09-23T17:57:50.299 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T17:57:50.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T17:57:50.313 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T17:57:50.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:50.316 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T17:57:50.359 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:57:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:57:50] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-09-23T17:57:50.364 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:57:50.417 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:50.424 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:57:50.483 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:57:50.516 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:50.527 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:57:50.577 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 17:57:50 smithi099 systemd[1]: Stopping Ceph osd.7 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:57:50.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:57:50.617 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:50.621 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:57:50.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:50.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:50.656 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T17:57:50.656 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:57:50.656 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:57:50.656 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:57:50.657 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:57:50.657 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:57:50.657 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:57:50.657 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:57:50.657 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T17:57:50.728 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:50.768 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:50.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:50 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/32998681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:50.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:50 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/32998681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:51.070 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:51.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:50 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/32998681' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:51.119 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:57:51.120 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:57:51.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:51.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:51.329 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:51.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:51.464 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:51.949 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:51.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:51.957 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:52.058 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:52.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:51 smithi099 ceph-mon[167076]: pgmap v325: 99 pgs: 1 peering, 10 unknown, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 11 active+undersized+degraded; 5.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 34803/276912 objects degraded (12.568%); 5439/276912 objects misplaced (1.964%); 5.1 MiB/s, 84 objects/s recovering 2023-09-23T17:57:52.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:52.099 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:52.099 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 17:57:52 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[127690]: 2023-09-23T17:57:52.039+0000 7f2c09d9d700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T17:57:52.100 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 17:57:52 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[127690]: 2023-09-23T17:57:52.039+0000 7f2c09d9d700 -1 osd.7 149 *** Got signal Terminated *** 2023-09-23T17:57:52.100 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 17:57:52 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[127690]: 2023-09-23T17:57:52.039+0000 7f2c09d9d700 -1 osd.7 149 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T17:57:52.103 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:52.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:51 smithi055 ceph-mon[184174]: pgmap v325: 99 pgs: 1 peering, 10 unknown, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 11 active+undersized+degraded; 5.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 34803/276912 objects degraded (12.568%); 5439/276912 objects misplaced (1.964%); 5.1 MiB/s, 84 objects/s recovering 2023-09-23T17:57:52.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:52.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:52.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:51 smithi055 ceph-mon[179668]: pgmap v325: 99 pgs: 1 peering, 10 unknown, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 11 active+undersized+degraded; 5.6 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 523 op/s; 34803/276912 objects degraded (12.568%); 5439/276912 objects misplaced (1.964%); 5.1 MiB/s, 84 objects/s recovering 2023-09-23T17:57:52.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:57:52.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:57:52.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:52.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:52.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:52.560 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:52.664 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:52.725 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:52.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:52.732 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:52.833 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:52.904 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:52 smithi055 ceph-mon[179668]: osd.7 marked itself down and dead 2023-09-23T17:57:52.904 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:52 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 34803/276912 objects degraded (12.568%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:52.904 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:52 smithi055 ceph-mon[184174]: osd.7 marked itself down and dead 2023-09-23T17:57:52.904 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:52 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 34803/276912 objects degraded (12.568%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:52.909 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:52.962 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:52.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:52.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:52.987 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:57:52.987 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:57:53.009 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:52 smithi099 ceph-mon[167076]: osd.7 marked itself down and dead 2023-09-23T17:57:53.009 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:52 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 34803/276912 objects degraded (12.568%), 14 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:53.069 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:53.098 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:53.120 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:57:53.135 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:53.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:53.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:53.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:53.270 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:53.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T17:57:53.462 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:53.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:53.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:53.566 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:53.576 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 17:57:53 smithi099 bash[184792]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7 2023-09-23T17:57:53.595 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:53.631 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:53.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:53.634 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:53.735 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:53.753 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:53.764 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:53.888 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T17:57:53.893 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:53.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:53.903 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:54.003 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:54.071 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:54.137 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:54.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[179668]: pgmap v326: 99 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 40 active+undersized, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 34133/271356 objects degraded (12.579%); 5176/271356 objects misplaced (1.907%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T17:57:54.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[179668]: Health check update: 3 osds down (OSD_DOWN) 2023-09-23T17:57:54.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[179668]: osdmap e150: 8 total, 5 up, 8 in 2023-09-23T17:57:54.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[179668]: failed to encode map e150 with expected crc 2023-09-23T17:57:54.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2543553363' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:57:54.147 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:57:54.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[184174]: pgmap v326: 99 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 40 active+undersized, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 34133/271356 objects degraded (12.579%); 5176/271356 objects misplaced (1.907%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T17:57:54.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[184174]: Health check update: 3 osds down (OSD_DOWN) 2023-09-23T17:57:54.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[184174]: osdmap e150: 8 total, 5 up, 8 in 2023-09-23T17:57:54.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[184174]: failed to encode map e150 with expected crc 2023-09-23T17:57:54.148 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2543553363' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:57:54.148 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:53 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:57:54.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:54.149 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:54.156 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:57:54.249 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:54.258 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:53 smithi099 ceph-mon[167076]: pgmap v326: 99 pgs: 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 30 active+clean, 5 undersized+peered, 40 active+undersized, 1 active+undersized+remapped, 13 active+undersized+degraded; 5.5 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 654 op/s; 34133/271356 objects degraded (12.579%); 5176/271356 objects misplaced (1.907%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T17:57:54.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:53 smithi099 ceph-mon[167076]: Health check update: 3 osds down (OSD_DOWN) 2023-09-23T17:57:54.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:53 smithi099 ceph-mon[167076]: osdmap e150: 8 total, 5 up, 8 in 2023-09-23T17:57:54.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:53 smithi099 ceph-mon[167076]: failed to encode map e150 with expected crc 2023-09-23T17:57:54.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2543553363' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:57:54.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:53 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:57:54.292 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:54.485 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:54.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:54.487 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:54.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:54.620 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:54.877 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[179668]: failed to encode map e150 with expected crc 2023-09-23T17:57:54.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:54.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[179668]: failed to encode map e151 with expected crc 2023-09-23T17:57:54.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:57:54.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[179668]: osdmap e151: 8 total, 5 up, 8 in 2023-09-23T17:57:54.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[179668]: failed to encode map e151 with expected crc 2023-09-23T17:57:54.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[179668]: failed to encode map e151 with expected crc 2023-09-23T17:57:55.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[184174]: failed to encode map e150 with expected crc 2023-09-23T17:57:55.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:55.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[184174]: failed to encode map e151 with expected crc 2023-09-23T17:57:55.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:57:55.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[184174]: osdmap e151: 8 total, 5 up, 8 in 2023-09-23T17:57:55.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[184174]: failed to encode map e151 with expected crc 2023-09-23T17:57:55.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:54 smithi055 ceph-mon[184174]: failed to encode map e151 with expected crc 2023-09-23T17:57:55.208 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:57:55.210 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:55.213 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:54 smithi099 ceph-mon[167076]: failed to encode map e150 with expected crc 2023-09-23T17:57:55.213 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:54 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:57:55.213 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:54 smithi099 ceph-mon[167076]: failed to encode map e151 with expected crc 2023-09-23T17:57:55.213 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:54 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:57:55.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:54 smithi099 ceph-mon[167076]: osdmap e151: 8 total, 5 up, 8 in 2023-09-23T17:57:55.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:54 smithi099 ceph-mon[167076]: failed to encode map e151 with expected crc 2023-09-23T17:57:55.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:54 smithi099 ceph-mon[167076]: failed to encode map e151 with expected crc 2023-09-23T17:57:55.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:55.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:55.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:57:55.249 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:57:55.274 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:57:55.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T17:57:55.294 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:57:55.294 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:57:55.295 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:57:55.295 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:57:55.295 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:57:55.295 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:57:55.295 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:57:55.295 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:57:55.295 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:57:55.316 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:57:55.325 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T17:57:55.325 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T17:57:55.350 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:55.393 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:57:55.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:55.396 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:57:55.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:55.528 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:55.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:55.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:55.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:55.626 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:57:55.626 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:57:55.715 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:57:55.715 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T17:57:55.715 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T17:57:55.715 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T17:57:55.715 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:57:55.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:55.746 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:55.782 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:55.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:55.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:55.887 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:55.917 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:55.956 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:55.959 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:55.959 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:56.060 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:56.070 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:55 smithi099 ceph-mon[167076]: pgmap v329: 99 pgs: 5 peering, 1 stale+active+recovery_wait+degraded+remapped, 4 stale+active+clean, 2 stale+active+undersized+degraded, 1 stale+undersized+peered, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 20 active+clean, 17 undersized+peered, 28 active+undersized, 3 undersized+degraded+peered, 1 active+undersized+remapped, 8 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 119 KiB/s rd, 316 KiB/s wr, 826 op/s; 33625/266898 objects degraded (12.598%); 5176/266898 objects misplaced (1.939%); 4.6 MiB/s, 68 objects/s recovering 2023-09-23T17:57:56.070 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:55 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 5 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:57:56.071 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4030473506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:56.090 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:56.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T17:57:56.128 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:56.133 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:56.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:55 smithi055 ceph-mon[184174]: pgmap v329: 99 pgs: 5 peering, 1 stale+active+recovery_wait+degraded+remapped, 4 stale+active+clean, 2 stale+active+undersized+degraded, 1 stale+undersized+peered, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 20 active+clean, 17 undersized+peered, 28 active+undersized, 3 undersized+degraded+peered, 1 active+undersized+remapped, 8 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 119 KiB/s rd, 316 KiB/s wr, 826 op/s; 33625/266898 objects degraded (12.598%); 5176/266898 objects misplaced (1.939%); 4.6 MiB/s, 68 objects/s recovering 2023-09-23T17:57:56.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:55 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 5 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:57:56.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4030473506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:56.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:55 smithi055 ceph-mon[179668]: pgmap v329: 99 pgs: 5 peering, 1 stale+active+recovery_wait+degraded+remapped, 4 stale+active+clean, 2 stale+active+undersized+degraded, 1 stale+undersized+peered, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 20 active+clean, 17 undersized+peered, 28 active+undersized, 3 undersized+degraded+peered, 1 active+undersized+remapped, 8 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 119 KiB/s rd, 316 KiB/s wr, 826 op/s; 33625/266898 objects degraded (12.598%); 5176/266898 objects misplaced (1.939%); 4.6 MiB/s, 68 objects/s recovering 2023-09-23T17:57:56.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:55 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 5 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:57:56.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4030473506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:57:56.233 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:56.262 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:56.298 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:56.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:56.302 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:56.403 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:56.433 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:56.469 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:56.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:56.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:56.543 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T17:57:56.572 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:56.598 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:56.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:56.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:56.638 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:56.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:56.743 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:57:56.767 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:56.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:56.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:56.886 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:56.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:57.016 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:57.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:57.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:57.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:57.155 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:57.163 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:57.186 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:57.295 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:57.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:57.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:57.399 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:57.432 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:57.472 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:57.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:57.475 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:57.575 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:57.606 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:57.627 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:57:57.704 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:57:57.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:57.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:57:57.807 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:57.838 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:57.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:57.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:57.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:57.949 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:57:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T17:57:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T17:57:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T17:57:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:79395 5.1 5:8a3ee8dd:::benchmark_data_smithi055_229117_object79394:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T17:57:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:53.853313+0000", 2023-09-23T17:57:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 64.093158406000001, 2023-09-23T17:57:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0221325990000001, 2023-09-23T17:57:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 79395 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853313+0000", 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853313+0000", 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853312+0000", 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853478+0000", 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016614200000000001 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853495+0000", 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7669e-05 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853502+0000", 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6939999999999998e-06 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853604+0000", 2023-09-23T17:57:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102193 2023-09-23T17:57:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2023-09-23T17:57:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853611+0000", 2023-09-23T17:57:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2110000000000001e-06 2023-09-23T17:57:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.856292+0000", 2023-09-23T17:57:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00268096 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.856301+0000", 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9290000000000004e-06 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.868742+0000", 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0124407230000001 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.868886+0000", 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143986 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.869045+0000", 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015933700000000001 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875290+0000", 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062447809999999996 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875319+0000", 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9040999999999999e-05 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875351+0000", 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1476999999999997e-05 2023-09-23T17:57:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875403+0000", 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2151000000000003e-05 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875446+0000", 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2969999999999997e-05 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:79397 5.1 5:8a26c16d:::benchmark_data_smithi055_229117_object79396:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:53.853515+0000", 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 64.092956624999999, 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0260921279999999, 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 79397 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853515+0000", 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853515+0000", 2023-09-23T17:57:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853514+0000", 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853545+0000", 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0533000000000001e-05 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853552+0000", 2023-09-23T17:57:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1890000000000001e-06 2023-09-23T17:57:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853562+0000", 2023-09-23T17:57:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3680000000000008e-06 2023-09-23T17:57:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853657+0000", 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4979e-05 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.853662+0000", 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4709999999999998e-06 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.856325+0000", 2023-09-23T17:57:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026634300000000001 2023-09-23T17:57:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.856333+0000", 2023-09-23T17:57:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8529999999999992e-06 2023-09-23T17:57:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.871522+0000", 2023-09-23T17:57:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.015188456 2023-09-23T17:57:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.871620+0000", 2023-09-23T17:57:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8437999999999995e-05 2023-09-23T17:57:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:57:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.871756+0000", 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013614700000000001 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875483+0000", 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00372676 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875580+0000", 2023-09-23T17:57:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7068e-05 2023-09-23T17:57:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.879504+0000", 2023-09-23T17:57:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039241129999999999 2023-09-23T17:57:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.879566+0000", 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1851000000000001e-05 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.879607+0000", 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1019999999999997e-05 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:79409 5.1 5:8b4f05f4:::benchmark_data_smithi055_229117_object79408:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T17:57:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:53.860636+0000", 2023-09-23T17:57:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 64.085835908000007, 2023-09-23T17:57:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0191917619999999, 2023-09-23T17:57:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 79409 2023-09-23T17:57:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.860636+0000", 2023-09-23T17:57:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.860636+0000", 2023-09-23T17:57:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.860634+0000", 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.860668+0000", 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3615999999999998e-05 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.860677+0000", 2023-09-23T17:57:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4930000000000004e-06 2023-09-23T17:57:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.860683+0000", 2023-09-23T17:57:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5389999999999998e-06 2023-09-23T17:57:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.860732+0000", 2023-09-23T17:57:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9270000000000001e-05 2023-09-23T17:57:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.860741+0000", 2023-09-23T17:57:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0759999999999992e-06 2023-09-23T17:57:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.873494+0000", 2023-09-23T17:57:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0127525749999999 2023-09-23T17:57:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.873587+0000", 2023-09-23T17:57:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3166999999999998e-05 2023-09-23T17:57:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:57:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.873672+0000", 2023-09-23T17:57:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5229999999999995e-05 2023-09-23T17:57:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875615+0000", 2023-09-23T17:57:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019432500000000001 2023-09-23T17:57:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.877291+0000", 2023-09-23T17:57:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016759190000000001 2023-09-23T17:57:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.879737+0000", 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002445418 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.879787+0000", 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0568000000000001e-05 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.879827+0000", 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0297999999999999e-05 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:79469 5.1 5:8712154f:::benchmark_data_smithi055_229117_object79468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:53.921587+0000", 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 64.024884267000004, 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96471150999999999, 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 79469 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.921587+0000", 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.921587+0000", 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.921585+0000", 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.921619+0000", 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3390999999999999e-05 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.921631+0000", 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2411e-05 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.921637+0000", 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0469999999999999e-06 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.921694+0000", 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6552000000000003e-05 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.921703+0000", 2023-09-23T17:57:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0909999999999999e-06 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875647+0000", 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95394438299999995 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875734+0000", 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7175999999999993e-05 2023-09-23T17:57:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.875812+0000", 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7948000000000007e-05 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.879882+0000", 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040701210000000003 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.879892+0000", 2023-09-23T17:57:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2310000000000002e-06 2023-09-23T17:57:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.886220+0000", 2023-09-23T17:57:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0063286460000000003 2023-09-23T17:57:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.886261+0000", 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0683999999999999e-05 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.886299+0000", 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7849999999999998e-05 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:79506 5.1 5:80f9aa0f:::benchmark_data_smithi055_229117_object79505:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:53.962078+0000", 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 63.984393658999998, 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.92436257099999997, 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 79506 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962078+0000", 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962078+0000", 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962076+0000", 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962109+0000", 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2327999999999999e-05 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962122+0000", 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3305999999999999e-05 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962126+0000", 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2429999999999999e-06 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962190+0000", 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3911000000000005e-05 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962202+0000", 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2204e-05 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.877330+0000", 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91512785100000005 2023-09-23T17:57:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.877446+0000", 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115981 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.877599+0000", 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015252199999999999 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.881949+0000", 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0043502130000000003 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.881961+0000", 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1603999999999999e-05 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.886356+0000", 2023-09-23T17:57:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0043955280000000001 2023-09-23T17:57:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.886404+0000", 2023-09-23T17:57:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7725999999999997e-05 2023-09-23T17:57:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.886441+0000", 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6659999999999998e-05 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:79509 5.1 5:839dddec:::benchmark_data_smithi055_229117_object79508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e129)", 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:53.962673+0000", 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 63.983798579000002, 2023-09-23T17:57:57.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.92851055100000002, 2023-09-23T17:57:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 79509 2023-09-23T17:57:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962673+0000", 2023-09-23T17:57:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962673+0000", 2023-09-23T17:57:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962672+0000", 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962699+0000", 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6738999999999999e-05 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962707+0000", 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6190000000000002e-06 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.962712+0000", 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9490000000000004e-06 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.964518+0000", 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018060719999999999 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:53.964527+0000", 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3209999999999994e-06 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.879929+0000", 2023-09-23T17:57:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91540160299999995 2023-09-23T17:57:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.880044+0000", 2023-09-23T17:57:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115526 2023-09-23T17:57:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.880194+0000", 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014953300000000001 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.886174+0000", 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0059806659999999999 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.886309+0000", 2023-09-23T17:57:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013471499999999999 2023-09-23T17:57:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.891116+0000", 2023-09-23T17:57:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048069599999999999 2023-09-23T17:57:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.891149+0000", 2023-09-23T17:57:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2716999999999999e-05 2023-09-23T17:57:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:54.891184+0000", 2023-09-23T17:57:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4842000000000003e-05 2023-09-23T17:57:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80604 5.e 5:70b48fb5:::benchmark_data_smithi055_229117_object80603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:57:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:55.880815+0000", 2023-09-23T17:57:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 62.065656640999997, 2023-09-23T17:57:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0631930629999999, 2023-09-23T17:57:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80604 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880815+0000", 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880815+0000", 2023-09-23T17:57:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880813+0000", 2023-09-23T17:57:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999976 2023-09-23T17:57:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880849+0000", 2023-09-23T17:57:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5797999999999998e-05 2023-09-23T17:57:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880867+0000", 2023-09-23T17:57:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8181000000000001e-05 2023-09-23T17:57:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880874+0000", 2023-09-23T17:57:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4569999999999999e-06 2023-09-23T17:57:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880938+0000", 2023-09-23T17:57:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3736000000000001e-05 2023-09-23T17:57:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2023-09-23T17:57:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880945+0000", 2023-09-23T17:57:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1189999999999999e-06 2023-09-23T17:57:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.888009+0000", 2023-09-23T17:57:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070636220000000003 2023-09-23T17:57:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.888023+0000", 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.437e-05 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.907926+0000", 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.019902603 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.908058+0000", 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013261800000000001 2023-09-23T17:57:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.908220+0000", 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016149599999999999 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.920288+0000", 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012067847 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.943890+0000", 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.023602317000000001 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.943940+0000", 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0342e-05 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.943973+0000", 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2404999999999999e-05 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944008+0000", 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5321999999999998e-05 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80612 5.e 5:70cc0f65:::benchmark_data_smithi055_229117_object80611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:55.880893+0000", 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 62.065579133999996, 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0632663899999999, 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80612 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880893+0000", 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880893+0000", 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880892+0000", 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880918+0000", 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6404999999999999e-05 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880926+0000", 2023-09-23T17:57:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1389999999999998e-06 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880930+0000", 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1520000000000002e-06 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880991+0000", 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1029999999999997e-05 2023-09-23T17:57:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880996+0000", 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7169999999999996e-06 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.888056+0000", 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070596560000000001 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.888064+0000", 2023-09-23T17:57:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7740000000000004e-06 2023-09-23T17:57:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.910586+0000", 2023-09-23T17:57:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.022522011 2023-09-23T17:57:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.910725+0000", 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013914799999999999 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.910898+0000", 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017282199999999999 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.925504+0000", 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014606192 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944059+0000", 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018555278000000001 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944095+0000", 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5148999999999998e-05 2023-09-23T17:57:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944126+0000", 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1038000000000003e-05 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944159+0000", 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3337000000000003e-05 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80630 5.e 5:791bb3fc:::benchmark_data_smithi055_229117_object80629:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:55.910375+0000", 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 62.036096665999999, 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.034055873, 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80630 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.910375+0000", 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.910375+0000", 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.910373+0000", 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.910422+0000", 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8390999999999998e-05 2023-09-23T17:57:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.910438+0000", 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6188000000000002e-05 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.910445+0000", 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7610000000000002e-06 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.910516+0000", 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1064999999999998e-05 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.910526+0000", 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9629999999999993e-06 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.913303+0000", 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0027777170000001 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.913424+0000", 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120458 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.913523+0000", 2023-09-23T17:57:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8550999999999999e-05 2023-09-23T17:57:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.925512+0000", 2023-09-23T17:57:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011989138 2023-09-23T17:57:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944259+0000", 2023-09-23T17:57:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018747083000000001 2023-09-23T17:57:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944332+0000", 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3044000000000001e-05 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944390+0000", 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7861e-05 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944431+0000", 2023-09-23T17:57:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1213e-05 2023-09-23T17:57:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80644 5.e 5:71ed47aa:::benchmark_data_smithi055_229117_object80643:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:57:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:55.934204+0000", 2023-09-23T17:57:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 62.012267366000003, 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0104634509999999, 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80644 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.934204+0000", 2023-09-23T17:57:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.934204+0000", 2023-09-23T17:57:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.934203+0000", 2023-09-23T17:57:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.934253+0000", 2023-09-23T17:57:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9703000000000002e-05 2023-09-23T17:57:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.934265+0000", 2023-09-23T17:57:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1749999999999999e-05 2023-09-23T17:57:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.934271+0000", 2023-09-23T17:57:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.596e-06 2023-09-23T17:57:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.934349+0000", 2023-09-23T17:57:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7813999999999997e-05 2023-09-23T17:57:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.934363+0000", 2023-09-23T17:57:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3995999999999999e-05 2023-09-23T17:57:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.915854+0000", 2023-09-23T17:57:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98149101699999997 2023-09-23T17:57:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.915950+0000", 2023-09-23T17:57:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6522999999999997e-05 2023-09-23T17:57:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.043 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:57:58.043 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.916051+0000", 2023-09-23T17:57:58.043 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100123 2023-09-23T17:57:58.043 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.043 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.043 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.043 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.925518+0000", 2023-09-23T17:57:58.043 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0094670329999999997 2023-09-23T17:57:58.044 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.044 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.044 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.044 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944298+0000", 2023-09-23T17:57:58.044 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018780077999999999 2023-09-23T17:57:58.044 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.044 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.044 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.044 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944560+0000", 2023-09-23T17:57:58.045 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026280500000000002 2023-09-23T17:57:58.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.045 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.045 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.045 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944630+0000", 2023-09-23T17:57:58.045 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9701000000000002e-05 2023-09-23T17:57:58.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.045 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.045 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.046 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944668+0000", 2023-09-23T17:57:58.046 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7524999999999997e-05 2023-09-23T17:57:58.046 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.046 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.046 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.046 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.046 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.046 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80655 5.e 5:76a98930:::benchmark_data_smithi055_229117_object80654:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:57:58.046 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:55.943021+0000", 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 62.003450467999997, 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0017895699999999, 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80655 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.047 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.943021+0000", 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.943021+0000", 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.048 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.943020+0000", 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.943063+0000", 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3473000000000002e-05 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.943076+0000", 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2485e-05 2023-09-23T17:57:58.049 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.943081+0000", 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.95e-06 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.943169+0000", 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8040000000000004e-05 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.050 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.943184+0000", 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.535e-05 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.918013+0000", 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97482869299999997 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.051 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.918101+0000", 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8335e-05 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.918225+0000", 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123886 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.930258+0000", 2023-09-23T17:57:58.052 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012032756 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944488+0000", 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014230474 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944724+0000", 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000235548 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.053 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944772+0000", 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8627e-05 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944811+0000", 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8284999999999997e-05 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.054 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80693 5.e 5:7aa78e64:::benchmark_data_smithi055_229117_object80692:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:55.996529+0000", 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 61.949942901999997, 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94844563100000001, 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.055 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80693 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.996529+0000", 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.996529+0000", 2023-09-23T17:57:58.056 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.996527+0000", 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.996628+0000", 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000101469 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.057 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.996647+0000", 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8376000000000002e-05 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.996655+0000", 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3350000000000007e-06 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.058 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.996717+0000", 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2281999999999998e-05 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.996730+0000", 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2811e-05 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.920334+0000", 2023-09-23T17:57:58.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.92360333500000003 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.920475+0000", 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000141182 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.920647+0000", 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000171972 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.060 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.930263+0000", 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096163529999999994 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944849+0000", 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014586328000000001 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.061 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944900+0000", 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0454999999999997e-05 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944943+0000", 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2966000000000002e-05 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944974+0000", 2023-09-23T17:57:58.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1477999999999999e-05 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80736 5.e 5:79c30e31:::benchmark_data_smithi055_229117_object80735:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:56.054370+0000", 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 61.892102031, 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.89077155799999996, 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.063 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80736 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.054370+0000", 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.064 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.054370+0000", 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.054368+0000", 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.054428+0000", 2023-09-23T17:57:58.065 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9895000000000002e-05 2023-09-23T17:57:58.066 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.066 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.066 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.066 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.054442+0000", 2023-09-23T17:57:58.066 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3781000000000001e-05 2023-09-23T17:57:58.066 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.066 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.066 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.066 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.054447+0000", 2023-09-23T17:57:58.066 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4589999999999997e-06 2023-09-23T17:57:58.067 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.067 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.054509+0000", 2023-09-23T17:57:58.067 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1130000000000006e-05 2023-09-23T17:57:58.067 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.067 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.054520+0000", 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1291000000000001e-05 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.923134+0000", 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.868614581 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.923244+0000", 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010979100000000001 2023-09-23T17:57:58.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.069 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:57:58.069 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.923345+0000", 2023-09-23T17:57:58.069 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100913 2023-09-23T17:57:58.069 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.069 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.069 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.936489+0000", 2023-09-23T17:57:58.070 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013144115 2023-09-23T17:57:58.070 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.070 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.070 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.070 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945023+0000", 2023-09-23T17:57:58.070 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085341640000000003 2023-09-23T17:57:58.070 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.070 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.070 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.070 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945066+0000", 2023-09-23T17:57:58.071 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2320000000000001e-05 2023-09-23T17:57:58.071 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.071 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.071 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.071 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945107+0000", 2023-09-23T17:57:58.071 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1255999999999999e-05 2023-09-23T17:57:58.071 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.071 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.071 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.071 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945141+0000", 2023-09-23T17:57:58.072 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4230999999999998e-05 2023-09-23T17:57:58.072 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.072 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.072 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.072 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.072 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.072 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80742 5.e 5:782f1840:::benchmark_data_smithi055_229117_object80741:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:57:58.072 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:56.062320+0000", 2023-09-23T17:57:58.072 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 61.884151627000001, 2023-09-23T17:57:58.072 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.88296560199999996, 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80742 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.062320+0000", 2023-09-23T17:57:58.073 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.062320+0000", 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.062319+0000", 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:58.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.062393+0000", 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3357999999999998e-05 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.062404+0000", 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1751e-05 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.062409+0000", 2023-09-23T17:57:58.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3730000000000003e-06 2023-09-23T17:57:58.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.076 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.062469+0000", 2023-09-23T17:57:58.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0037000000000001e-05 2023-09-23T17:57:58.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.062480+0000", 2023-09-23T17:57:58.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0695e-05 2023-09-23T17:57:58.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.925549+0000", 2023-09-23T17:57:58.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.86306945400000001 2023-09-23T17:57:58.078 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.078 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.078 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.078 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.925676+0000", 2023-09-23T17:57:58.078 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012711800000000001 2023-09-23T17:57:58.078 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.078 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.078 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:57:58.078 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.925837+0000", 2023-09-23T17:57:58.079 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000160476 2023-09-23T17:57:58.079 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.079 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.079 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.079 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.936497+0000", 2023-09-23T17:57:58.079 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010660087 2023-09-23T17:57:58.079 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.079 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.079 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.080 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945183+0000", 2023-09-23T17:57:58.080 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0086858049999999996 2023-09-23T17:57:58.080 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.080 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.080 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.080 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945221+0000", 2023-09-23T17:57:58.080 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8967000000000001e-05 2023-09-23T17:57:58.080 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.081 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.081 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.081 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945251+0000", 2023-09-23T17:57:58.081 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9754e-05 2023-09-23T17:57:58.081 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.081 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.081 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.081 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945286+0000", 2023-09-23T17:57:58.081 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4387000000000001e-05 2023-09-23T17:57:58.082 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.082 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.082 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.082 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.082 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.082 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80752 5.e 5:7510be70:::benchmark_data_smithi055_229117_object80751:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T17:57:58.082 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:56.075951+0000", 2023-09-23T17:57:58.082 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 61.870520513999999, 2023-09-23T17:57:58.082 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.86945687400000005, 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80752 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.083 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.075951+0000", 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.075951+0000", 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.075949+0000", 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T17:57:58.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.075992+0000", 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3191e-05 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.076003+0000", 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0933e-05 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.085 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.076010+0000", 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5060000000000001e-06 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.076076+0000", 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6038999999999996e-05 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.086 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.076088+0000", 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1664e-05 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.927791+0000", 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85170310800000004 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.927948+0000", 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000157687 2023-09-23T17:57:58.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.928139+0000", 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000190719 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.940002+0000", 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011862655 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.088 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945325+0000", 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0053236000000000004 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945353+0000", 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7450999999999999e-05 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945379+0000", 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5795999999999998e-05 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.945408+0000", 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9428e-05 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.090 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:104111 5.1 5:8013bcd3:::benchmark_data_smithi055_229117_object104110:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:23.953885+0000", 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 33.992586205000002, 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0219631410000001, 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 104111 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.091 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953885+0000", 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953885+0000", 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.092 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953884+0000", 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953922+0000", 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7709000000000001e-05 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.093 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953934+0000", 2023-09-23T17:57:58.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2745000000000001e-05 2023-09-23T17:57:58.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.094 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.094 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.094 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953941+0000", 2023-09-23T17:57:58.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6959999999999999e-06 2023-09-23T17:57:58.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.094 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.094 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.094 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.954104+0000", 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016321099999999999 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.954114+0000", 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5040000000000008e-06 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.963982+0000", 2023-09-23T17:57:58.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0098679930000001 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.964115+0000", 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013268000000000001 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 7", 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.964271+0000", 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000155997 2023-09-23T17:57:58.096 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.975719+0000", 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011448906 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.975751+0000", 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1894000000000003e-05 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.097 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.975787+0000", 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5431e-05 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.975849+0000", 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1834000000000005e-05 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.098 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:104113 5.1 5:881b0211:::benchmark_data_smithi055_229117_object104112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:23.953955+0000", 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 33.992516383999998, 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0253237049999999, 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 104113 2023-09-23T17:57:58.099 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953955+0000", 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953955+0000", 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.100 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953955+0000", 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953989+0000", 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4582999999999997e-05 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.101 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.953996+0000", 2023-09-23T17:57:58.102 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5599999999999999e-06 2023-09-23T17:57:58.102 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.102 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.102 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.102 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.954000+0000", 2023-09-23T17:57:58.102 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.031e-06 2023-09-23T17:57:58.102 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.102 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.102 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.102 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.954135+0000", 2023-09-23T17:57:58.103 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013543600000000001 2023-09-23T17:57:58.103 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.103 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.103 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.103 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.954139+0000", 2023-09-23T17:57:58.103 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5389999999999999e-06 2023-09-23T17:57:58.103 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.103 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.103 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.967549+0000", 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.013409875 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.967665+0000", 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116243 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 7", 2023-09-23T17:57:58.104 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.967765+0000", 2023-09-23T17:57:58.105 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9906000000000002e-05 2023-09-23T17:57:58.105 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.105 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.105 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.105 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.975924+0000", 2023-09-23T17:57:58.105 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0081588939999999999 2023-09-23T17:57:58.105 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.105 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.105 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.979135+0000", 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003211279 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.979185+0000", 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9518000000000003e-05 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.106 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.979279+0000", 2023-09-23T17:57:58.107 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4380000000000001e-05 2023-09-23T17:57:58.107 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.107 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.107 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.107 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.107 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.107 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:104127 5.1 5:8a9dd7e5:::benchmark_data_smithi055_229117_object104126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:57:58.107 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:23.976749+0000", 2023-09-23T17:57:58.107 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 33.969723006000002, 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0033371010000001, 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 104127 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.108 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.109 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.976749+0000", 2023-09-23T17:57:58.109 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.109 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.109 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.109 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.109 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.976749+0000", 2023-09-23T17:57:58.109 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.109 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.976748+0000", 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.976804+0000", 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6385999999999999e-05 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.110 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.111 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.111 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.976815+0000", 2023-09-23T17:57:58.111 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1112e-05 2023-09-23T17:57:58.111 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.111 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.111 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.111 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.976821+0000", 2023-09-23T17:57:58.111 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1189999999999996e-06 2023-09-23T17:57:58.111 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.112 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.112 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.112 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.976861+0000", 2023-09-23T17:57:58.112 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0091e-05 2023-09-23T17:57:58.112 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.112 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.112 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.112 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:23.976868+0000", 2023-09-23T17:57:58.112 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.968e-06 2023-09-23T17:57:58.113 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.113 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.113 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.113 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.970182+0000", 2023-09-23T17:57:58.113 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99331453800000002 2023-09-23T17:57:58.113 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.113 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.113 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.113 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.970316+0000", 2023-09-23T17:57:58.114 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000134065 2023-09-23T17:57:58.114 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.114 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.114 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 7", 2023-09-23T17:57:58.114 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.970487+0000", 2023-09-23T17:57:58.114 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017087800000000001 2023-09-23T17:57:58.114 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.114 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.115 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.115 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.979290+0000", 2023-09-23T17:57:58.115 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0088031800000000007 2023-09-23T17:57:58.115 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.115 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.115 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.115 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.979966+0000", 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00067619499999999999 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.980008+0000", 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1486000000000002e-05 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.980086+0000", 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7791000000000001e-05 2023-09-23T17:57:58.116 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:104151 5.1 5:8ec97295:::benchmark_data_smithi055_229117_object104150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:24.016644+0000", 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 33.929827654999997, 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96581039099999999, 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.117 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 104151 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.016644+0000", 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.118 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.016644+0000", 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.016643+0000", 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.016698+0000", 2023-09-23T17:57:58.119 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5510999999999998e-05 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.016710+0000", 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1928e-05 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.016715+0000", 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3619999999999997e-06 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.120 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.016768+0000", 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2540999999999997e-05 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.016776+0000", 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3550000000000005e-06 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.121 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.972891+0000", 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95611485699999998 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.973014+0000", 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012301900000000001 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 7", 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.973107+0000", 2023-09-23T17:57:58.122 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3201000000000004e-05 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.979296+0000", 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006188392 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.982301+0000", 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030055630000000002 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.123 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.982359+0000", 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7207999999999998e-05 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.982454+0000", 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5754000000000005e-05 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.124 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:104157 5.1 5:89885036:::benchmark_data_smithi055_229117_object104156:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:24.035437+0000", 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 33.911034708999999, 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94715633200000005, 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 104157 2023-09-23T17:57:58.125 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.035437+0000", 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.035437+0000", 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.126 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.035436+0000", 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.035493+0000", 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7816999999999999e-05 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.127 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.035507+0000", 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3207000000000001e-05 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.035513+0000", 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6390000000000003e-06 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.035550+0000", 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6704e-05 2023-09-23T17:57:58.128 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.035569+0000", 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.853e-05 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.975967+0000", 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94039883000000002 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.129 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.976095+0000", 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127484 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 7", 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.976246+0000", 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015167499999999999 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.130 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.980349+0000", 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041024399999999997 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.982498+0000", 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021487979999999999 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.982534+0000", 2023-09-23T17:57:58.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5920000000000002e-05 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:24.982593+0000", 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9608000000000003e-05 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:57:58.132 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:57:58.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:58.134 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:57 smithi099 ceph-mon[167076]: pgmap v330: 99 pgs: 5 peering, 1 stale+active+recovery_wait+degraded+remapped, 3 stale+active+clean, 2 stale+active+undersized+degraded, 1 stale+undersized+peered, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 19 active+clean, 20 undersized+peered, 26 active+undersized, 3 undersized+degraded+peered, 1 active+undersized+remapped, 9 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 KiB/s rd, 275 KiB/s wr, 746 op/s; 33501/266190 objects degraded (12.585%); 5176/266190 objects misplaced (1.944%); 6.6 MiB/s, 100 objects/s recovering 2023-09-23T17:57:58.134 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:57 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 33501/266190 objects degraded (12.585%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:58.162 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:58.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:57 smithi055 ceph-mon[179668]: pgmap v330: 99 pgs: 5 peering, 1 stale+active+recovery_wait+degraded+remapped, 3 stale+active+clean, 2 stale+active+undersized+degraded, 1 stale+undersized+peered, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 19 active+clean, 20 undersized+peered, 26 active+undersized, 3 undersized+degraded+peered, 1 active+undersized+remapped, 9 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 KiB/s rd, 275 KiB/s wr, 746 op/s; 33501/266190 objects degraded (12.585%); 5176/266190 objects misplaced (1.944%); 6.6 MiB/s, 100 objects/s recovering 2023-09-23T17:57:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:57 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 33501/266190 objects degraded (12.585%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:57 smithi055 ceph-mon[184174]: pgmap v330: 99 pgs: 5 peering, 1 stale+active+recovery_wait+degraded+remapped, 3 stale+active+clean, 2 stale+active+undersized+degraded, 1 stale+undersized+peered, 7 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 19 active+clean, 20 undersized+peered, 26 active+undersized, 3 undersized+degraded+peered, 1 active+undersized+remapped, 9 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 KiB/s rd, 275 KiB/s wr, 746 op/s; 33501/266190 objects degraded (12.585%); 5176/266190 objects misplaced (1.944%); 6.6 MiB/s, 100 objects/s recovering 2023-09-23T17:57:58.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:57 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 33501/266190 objects degraded (12.585%), 17 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T17:57:58.298 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T17:57:58.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:58.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:58.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:58.402 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:58.430 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:58.465 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:58.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:58.469 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:58.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:57:58.596 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:58.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:57:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:58.633 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:57:58.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:58.760 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:58.796 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:58.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:58.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:58.900 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:57:58.907 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:57:58.933 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:59.284 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:57:59.286 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:57:59.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:59.289 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:57:59.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:57:59.419 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:57:59.619 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:57:59.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:57:59.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:57:59.724 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:57:59.756 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:00.044 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:00.047 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:57:59 smithi055 ceph-mon[179668]: pgmap v331: 99 pgs: 5 peering, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 22 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 12 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 KiB/s rd, 275 KiB/s wr, 554 op/s; 47352/265572 objects degraded (17.830%); 2177/265572 objects misplaced (0.820%); 11 MiB/s, 163 objects/s recovering 2023-09-23T17:58:00.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:57:59 smithi055 ceph-mon[184174]: pgmap v331: 99 pgs: 5 peering, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 22 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 12 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 KiB/s rd, 275 KiB/s wr, 554 op/s; 47352/265572 objects degraded (17.830%); 2177/265572 objects misplaced (0.820%); 11 MiB/s, 163 objects/s recovering 2023-09-23T17:58:00.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:00.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:00.131 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:58:00.131 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T17:58:00.131 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T17:58:00.131 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:58:00.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:00.161 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:57:59 smithi099 ceph-mon[167076]: pgmap v331: 99 pgs: 5 peering, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 22 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 12 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 103 KiB/s rd, 275 KiB/s wr, 554 op/s; 47352/265572 objects degraded (17.830%); 2177/265572 objects misplaced (0.820%); 11 MiB/s, 163 objects/s recovering 2023-09-23T17:58:00.181 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:00.326 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6, 3, 7] live_osds: [0, 1, 2, 4, 5] 2023-09-23T17:58:00.326 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:58:00.326 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T17:58:00.326 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T17:58:00.326 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T17:58:00.327 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T17:58:00.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:00.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:00.500 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:00.553 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T17:58:00.601 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:00.602 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:58:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:58:00] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.43.0" 2023-09-23T17:58:00.634 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:00.679 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:00.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:00.683 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:00.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:00.810 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:00.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:00.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:00.847 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:00.906 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T17:58:00.948 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:00.981 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:00.991 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:01.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1424271935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:00 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:01.079 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:01.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:01.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:01.136 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:58:01.156 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:01.160 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:58:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1424271935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:00 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:01.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1424271935' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:01.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:00 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:01.183 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:01.186 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:58:01.209 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:58:01.214 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:01.231 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:58:01.232 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:58:01.232 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:58:01.232 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:58:01.232 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:58:01.232 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:58:01.232 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:58:01.233 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:58:01.233 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:58:01.368 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:01.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:01.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:01.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:01.500 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:01.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:01.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:01.587 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:01.636 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:58:01.637 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:58:01.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:01.717 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:01.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:01.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:01.914 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:02.015 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:02.048 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:01 smithi099 ceph-mon[167076]: pgmap v332: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 6.5 MiB/s, 102 objects/s recovering 2023-09-23T17:58:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:01 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:01 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:01 smithi099 ceph-mon[167076]: osdmap e152: 8 total, 5 up, 8 in 2023-09-23T17:58:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:01 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/937968809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:02.095 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:02.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:01 smithi055 ceph-mon[184174]: pgmap v332: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 6.5 MiB/s, 102 objects/s recovering 2023-09-23T17:58:02.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:01 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:02.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:01 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:02.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:01 smithi055 ceph-mon[184174]: osdmap e152: 8 total, 5 up, 8 in 2023-09-23T17:58:02.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:01 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/937968809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:02.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:01 smithi055 ceph-mon[179668]: pgmap v332: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 256 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 6.5 MiB/s, 102 objects/s recovering 2023-09-23T17:58:02.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:01 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:02.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:01 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:02.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:01 smithi055 ceph-mon[179668]: osdmap e152: 8 total, 5 up, 8 in 2023-09-23T17:58:02.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:01 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/937968809' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:02.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:02.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:02.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:02.230 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:02.314 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:02.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:02.318 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:02.368 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:58:02.368 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:58:02.419 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:02.455 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:02.592 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:02.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:02.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:02.664 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:58:02.664 INFO:teuthology.orchestra.run.smithi055.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T17:58:02.664 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:58:02.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:02.725 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:02.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:02.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:02.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:02.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T17:58:02.863 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:02.892 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:02.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:02.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:02.932 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:03.010 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:02 smithi055 ceph-mon[184174]: failed to encode map e152 with expected crc 2023-09-23T17:58:03.010 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:02 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 6 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:03.011 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:02 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 63926/265197 objects degraded (24.105%), 24 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:03.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:02 smithi055 ceph-mon[179668]: failed to encode map e152 with expected crc 2023-09-23T17:58:03.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:02 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 6 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:03.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:02 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 63926/265197 objects degraded (24.105%), 24 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:03.033 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:03.066 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:03.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:02 smithi099 ceph-mon[167076]: failed to encode map e152 with expected crc 2023-09-23T17:58:03.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:02 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 6 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:03.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:02 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 63926/265197 objects degraded (24.105%), 24 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:03.105 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:03.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:03.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:03.209 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:03.240 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:03.277 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:03.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:03.280 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:03.370 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T17:58:03.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:03.409 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:03.426 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:03.445 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:03.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:03.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:03.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:03.576 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:03.638 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:58:03.662 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:03.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:03.665 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:03.766 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:03.800 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:03.843 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:03.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:03.848 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:03.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:03.980 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:04.034 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:04.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:04.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:04.117 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:04.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:03 smithi055 ceph-mon[179668]: pgmap v334: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 15 MiB/s, 242 objects/s recovering 2023-09-23T17:58:04.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:03 smithi055 ceph-mon[184174]: pgmap v334: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 15 MiB/s, 242 objects/s recovering 2023-09-23T17:58:04.218 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:04.264 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:03 smithi099 ceph-mon[167076]: pgmap v334: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 125 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 15 MiB/s, 242 objects/s recovering 2023-09-23T17:58:04.358 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:04.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:04.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:04.464 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:04.500 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:04.530 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:04.657 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:58:04.657 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:58:04.767 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:04.771 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:04.772 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:04.873 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:04.909 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:05.050 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:05.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:05.053 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:05.078 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T17:58:05.154 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:05.184 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:05.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:05.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:05.220 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:05.321 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:05.359 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:05.531 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:05.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:05.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:05.638 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:05.684 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:05.820 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:05.822 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:05.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:05.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:05.931 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:05.937 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T17:58:05.974 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:06.021 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:06.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:06.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:06.037 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:58:06.061 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:58:06.085 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:58:06.109 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:05 smithi055 ceph-mon[184174]: pgmap v335: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 13 MiB/s, 210 objects/s recovering 2023-09-23T17:58:06.109 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2490603230' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:58:06.109 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:05 smithi055 ceph-mon[179668]: pgmap v335: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 13 MiB/s, 210 objects/s recovering 2023-09-23T17:58:06.109 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2490603230' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:58:06.110 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:58:06.125 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:06.130 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:58:06.131 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:58:06.131 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:58:06.131 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:58:06.131 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:58:06.131 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:58:06.131 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:58:06.131 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:58:06.132 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:58:06.155 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:06.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:05 smithi099 ceph-mon[167076]: pgmap v335: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 56 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 13 MiB/s, 210 objects/s recovering 2023-09-23T17:58:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2490603230' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:58:06.590 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:06.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:06.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:06.665 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:58:06.666 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:58:06.696 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:06.725 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:06.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:06.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:06.760 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:06.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:06.888 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:06.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:06.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:06.924 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:07.024 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:07.048 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:06 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:07.048 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:06 smithi099 ceph-mon[167076]: failed to encode map e153 with expected crc 2023-09-23T17:58:07.048 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:06 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2490603230' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:58:07.048 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:06 smithi099 ceph-mon[167076]: osdmap e153: 8 total, 5 up, 8 in 2023-09-23T17:58:07.049 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:06 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1619842531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:07.049 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:07.049 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:07.051 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:07.082 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:07.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:07.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:07.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:07.211 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:07.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:07.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[184174]: failed to encode map e153 with expected crc 2023-09-23T17:58:07.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2490603230' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:58:07.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[184174]: osdmap e153: 8 total, 5 up, 8 in 2023-09-23T17:58:07.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1619842531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:07.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:07.227 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:07.228 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:07.228 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[179668]: failed to encode map e153 with expected crc 2023-09-23T17:58:07.228 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2490603230' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:58:07.228 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[179668]: osdmap e153: 8 total, 5 up, 8 in 2023-09-23T17:58:07.228 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1619842531' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:07.228 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:07.228 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:07.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:07.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:07.246 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:07.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:07.373 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:07.406 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:07.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:07.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:07.440 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:58:07.440 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:58:07.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:07.440 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPush(5.0 153/143 [PushOp(5:0c4ad422:::benchmark_data_smithi055_229117_object36633:head, version: 117'2383, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(5:0c4ad422:::benchmark_data_smithi055_229117_object36633:head@117'2383, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T17:58:07.440 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:58:07.438632+0000", 2023-09-23T17:58:07.440 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0018962110000000001, 2023-09-23T17:58:07.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019332520000000001, 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:58:07.438632+0000", 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:58:07.438632+0000", 2023-09-23T17:58:07.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:58:07.438631+0000", 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:58:07.438661+0000", 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9916e-05 2023-09-23T17:58:07.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:58:07.438682+0000", 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1838999999999999e-05 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:58:07.438695+0000", 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2998e-05 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:07.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:58:07.438765+0000", 2023-09-23T17:58:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9640000000000004e-05 2023-09-23T17:58:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:58:07.438774+0000", 2023-09-23T17:58:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6519999999999995e-06 2023-09-23T17:58:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T17:58:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 1 2023-09-23T17:58:07.445 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:58:07.509 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:07.548 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:07.800 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T17:58:07.821 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:07.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:07.824 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:07.925 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:07.954 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:07.992 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:07.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:07.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:08.013 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6, 3, 7] live_osds: [0, 1, 2, 4, 5] 2023-09-23T17:58:08.013 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:58:08.013 INFO:tasks.thrashosds.thrasher:Reweighting osd 7 to 0.7492559236118755 2023-09-23T17:58:08.014 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 0.7492559236118755 2023-09-23T17:58:08.062 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:07 smithi055 ceph-mon[184174]: pgmap v337: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 15 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 13 MiB/s, 214 objects/s recovering 2023-09-23T17:58:08.062 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:07 smithi055 ceph-mon[184174]: failed to encode map e153 with expected crc 2023-09-23T17:58:08.063 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:07 smithi055 ceph-mon[179668]: pgmap v337: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 15 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 13 MiB/s, 214 objects/s recovering 2023-09-23T17:58:08.063 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:07 smithi055 ceph-mon[179668]: failed to encode map e153 with expected crc 2023-09-23T17:58:08.096 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:08.124 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:08.158 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:08.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:08.161 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:08.261 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:08.292 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:08.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:07 smithi099 ceph-mon[167076]: pgmap v337: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 15 op/s; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 13 MiB/s, 214 objects/s recovering 2023-09-23T17:58:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:07 smithi099 ceph-mon[167076]: failed to encode map e153 with expected crc 2023-09-23T17:58:08.328 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:08.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:08.332 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:08.433 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:08.460 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:08.461 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:08.495 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:08.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:08.498 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:08.598 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:08.631 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:08.667 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:58:08.698 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T17:58:08.721 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:08.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:08.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:08.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:08.854 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:08.886 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:08.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:08.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:08.891 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:08.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:09.020 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:09.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:09.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:09.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:09.157 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:09.198 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:09.585 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:09.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:09.588 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:09.688 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:09.721 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:09.862 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:09.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:09.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:09.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:09.998 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:09.999 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:10.013 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:10.035 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:10.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:10.038 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:10.049 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:58:10.050 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T17:58:10.050 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T17:58:10.050 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T17:58:10.050 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:58:10.066 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:09 smithi055 ceph-mon[184174]: pgmap v338: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 22 MiB/s, 350 objects/s recovering 2023-09-23T17:58:10.066 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:09 smithi055 ceph-mon[179668]: pgmap v338: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 22 MiB/s, 350 objects/s recovering 2023-09-23T17:58:10.139 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:10.170 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:09 smithi099 ceph-mon[167076]: pgmap v338: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 22 MiB/s, 350 objects/s recovering 2023-09-23T17:58:10.359 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:58:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:58:10] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-09-23T17:58:10.981 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.7 to 0.749256 (bfcf) 2023-09-23T17:58:11.152 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/889142342' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.7492559236118755}]: dispatch 2023-09-23T17:58:11.152 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:10 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.7492559236118755}]: dispatch 2023-09-23T17:58:11.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/889142342' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.7492559236118755}]: dispatch 2023-09-23T17:58:11.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:10 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.7492559236118755}]: dispatch 2023-09-23T17:58:11.196 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:11.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:11.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:11.301 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:11.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/889142342' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.7492559236118755}]: dispatch 2023-09-23T17:58:11.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:10 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 0.7492559236118755}]: dispatch 2023-09-23T17:58:11.335 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:11.342 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T17:58:11.822 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:11.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:11.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:11.926 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:11.957 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:11.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:11 smithi055 ceph-mon[184174]: pgmap v339: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 15 MiB/s, 246 objects/s recovering 2023-09-23T17:58:11.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:11 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.7492559236118755}]': finished 2023-09-23T17:58:11.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:11 smithi055 ceph-mon[184174]: osdmap e154: 8 total, 5 up, 8 in 2023-09-23T17:58:11.969 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:11 smithi055 ceph-mon[184174]: failed to encode map e154 with expected crc 2023-09-23T17:58:11.997 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:12.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:12.000 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:12.101 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:12.132 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:12.231 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:12.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:11 smithi055 ceph-mon[179668]: pgmap v339: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 15 MiB/s, 246 objects/s recovering 2023-09-23T17:58:12.233 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:11 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.7492559236118755}]': finished 2023-09-23T17:58:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:11 smithi055 ceph-mon[179668]: osdmap e154: 8 total, 5 up, 8 in 2023-09-23T17:58:12.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:11 smithi055 ceph-mon[179668]: failed to encode map e154 with expected crc 2023-09-23T17:58:12.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:12.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:12.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:11 smithi099 ceph-mon[167076]: pgmap v339: 99 pgs: 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 24 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 6 undersized+degraded+peered, 4 active+undersized+remapped, 15 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 63926/265197 objects degraded (24.105%); 2177/265197 objects misplaced (0.821%); 15 MiB/s, 246 objects/s recovering 2023-09-23T17:58:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:11 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 0.7492559236118755}]': finished 2023-09-23T17:58:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:11 smithi099 ceph-mon[167076]: osdmap e154: 8 total, 5 up, 8 in 2023-09-23T17:58:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:11 smithi099 ceph-mon[167076]: failed to encode map e154 with expected crc 2023-09-23T17:58:12.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:12.361 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:12.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:12.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:12.398 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:12.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:12.525 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:12.537 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:12.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:12.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:12.562 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:12.580 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:58:12.599 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:58:12.619 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:58:12.640 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:58:12.662 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:12.663 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:58:12.665 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:58:12.665 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:58:12.665 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:58:12.665 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:58:12.665 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:58:12.665 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:58:12.665 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:58:12.666 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:58:12.699 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:13.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:12 smithi055 ceph-mon[184174]: failed to encode map e155 with expected crc 2023-09-23T17:58:13.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:12 smithi055 ceph-mon[184174]: osdmap e155: 8 total, 5 up, 8 in 2023-09-23T17:58:13.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:12 smithi055 ceph-mon[184174]: failed to encode map e155 with expected crc 2023-09-23T17:58:13.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:12 smithi055 ceph-mon[184174]: failed to encode map e155 with expected crc 2023-09-23T17:58:13.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:12 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1999003363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:13.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:12 smithi055 ceph-mon[179668]: failed to encode map e155 with expected crc 2023-09-23T17:58:13.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:12 smithi055 ceph-mon[179668]: osdmap e155: 8 total, 5 up, 8 in 2023-09-23T17:58:13.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:12 smithi055 ceph-mon[179668]: failed to encode map e155 with expected crc 2023-09-23T17:58:13.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:12 smithi055 ceph-mon[179668]: failed to encode map e155 with expected crc 2023-09-23T17:58:13.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:12 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1999003363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:13.213 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:58:13.213 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:58:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:12 smithi099 ceph-mon[167076]: failed to encode map e155 with expected crc 2023-09-23T17:58:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:12 smithi099 ceph-mon[167076]: osdmap e155: 8 total, 5 up, 8 in 2023-09-23T17:58:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:12 smithi099 ceph-mon[167076]: failed to encode map e155 with expected crc 2023-09-23T17:58:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:12 smithi099 ceph-mon[167076]: failed to encode map e155 with expected crc 2023-09-23T17:58:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:12 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1999003363' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:13.378 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:13.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:13.381 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:13.482 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:13.515 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:13.569 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:13.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:13.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:13.673 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:13.704 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:13.794 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:13.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:13.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:13.865 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:58:13.865 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:58:13.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:13.925 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=sqlstore.transactions t=2023-09-23T17:58:13.669754889Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T17:58:13.925 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=cleanup t=2023-09-23T17:58:13.67203901Z level=info msg="Completed cleanup jobs" duration=12.588924ms 2023-09-23T17:58:13.928 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:13.965 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:13.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:13.968 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:14.021 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T17:58:14.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[184174]: pgmap v342: 99 pgs: 1 remapped+peering, 1 stale+undersized+peered, 4 active+clean+remapped, 2 active+recovering+degraded+remapped, 16 active+clean, 23 undersized+peered, 27 active+undersized, 4 undersized+degraded+peered, 5 active+undersized+remapped, 16 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44176/265197 objects degraded (16.658%); 10755/265197 objects misplaced (4.055%); 16 MiB/s, 251 objects/s recovering 2023-09-23T17:58:14.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 5 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:14.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 44176/265197 objects degraded (16.658%), 22 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:14.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[184174]: failed to encode map e156 with expected crc 2023-09-23T17:58:14.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[184174]: osdmap e156: 8 total, 5 up, 8 in 2023-09-23T17:58:14.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[184174]: failed to encode map e156 with expected crc 2023-09-23T17:58:14.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[184174]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 151'7754 2023-09-23T17:58:14.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[184174]: 5.1 starting backfill to osd.0 from (0'0,0'0] MAX to 149'7745 2023-09-23T17:58:14.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[184174]: failed to encode map e156 with expected crc 2023-09-23T17:58:14.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3468534917' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:14.049 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[179668]: pgmap v342: 99 pgs: 1 remapped+peering, 1 stale+undersized+peered, 4 active+clean+remapped, 2 active+recovering+degraded+remapped, 16 active+clean, 23 undersized+peered, 27 active+undersized, 4 undersized+degraded+peered, 5 active+undersized+remapped, 16 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44176/265197 objects degraded (16.658%); 10755/265197 objects misplaced (4.055%); 16 MiB/s, 251 objects/s recovering 2023-09-23T17:58:14.049 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 5 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:14.049 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 44176/265197 objects degraded (16.658%), 22 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:14.050 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[179668]: failed to encode map e156 with expected crc 2023-09-23T17:58:14.050 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[179668]: osdmap e156: 8 total, 5 up, 8 in 2023-09-23T17:58:14.050 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[179668]: failed to encode map e156 with expected crc 2023-09-23T17:58:14.050 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[179668]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 151'7754 2023-09-23T17:58:14.050 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[179668]: 5.1 starting backfill to osd.0 from (0'0,0'0] MAX to 149'7745 2023-09-23T17:58:14.050 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[179668]: failed to encode map e156 with expected crc 2023-09-23T17:58:14.050 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:13 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3468534917' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:14.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:14.095 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:14.131 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:14.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:14.136 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:14.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:14.262 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-mon[167076]: pgmap v342: 99 pgs: 1 remapped+peering, 1 stale+undersized+peered, 4 active+clean+remapped, 2 active+recovering+degraded+remapped, 16 active+clean, 23 undersized+peered, 27 active+undersized, 4 undersized+degraded+peered, 5 active+undersized+remapped, 16 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44176/265197 objects degraded (16.658%); 10755/265197 objects misplaced (4.055%); 16 MiB/s, 251 objects/s recovering 2023-09-23T17:58:14.262 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 5 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:14.262 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 44176/265197 objects degraded (16.658%), 22 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:14.262 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-mon[167076]: failed to encode map e156 with expected crc 2023-09-23T17:58:14.263 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-mon[167076]: osdmap e156: 8 total, 5 up, 8 in 2023-09-23T17:58:14.263 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-mon[167076]: failed to encode map e156 with expected crc 2023-09-23T17:58:14.263 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-mon[167076]: 5.4 starting backfill to osd.0 from (0'0,0'0] MAX to 151'7754 2023-09-23T17:58:14.263 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-mon[167076]: 5.1 starting backfill to osd.0 from (0'0,0'0] MAX to 149'7745 2023-09-23T17:58:14.263 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-mon[167076]: failed to encode map e156 with expected crc 2023-09-23T17:58:14.263 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:13 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3468534917' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:14.265 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:14.300 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:14.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:14.303 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:14.401 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T17:58:14.403 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:14.475 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:14.516 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:14.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:14.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:14.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:14.650 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:14.688 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:14.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:14.691 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:14.792 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:14.826 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:14.864 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:14.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:14.867 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:14.968 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:14.999 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:15.038 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:15.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:15.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:15.083 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:15.142 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:15.171 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:15.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:15 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:58:15.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3468534917' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:15.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:15 smithi055 ceph-mon[179668]: osdmap e157: 8 total, 5 up, 8 in 2023-09-23T17:58:15.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:15 smithi055 ceph-mon[179668]: failed to encode map e157 with expected crc 2023-09-23T17:58:15.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:15 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:58:15.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3468534917' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:15.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:15 smithi055 ceph-mon[184174]: osdmap e157: 8 total, 5 up, 8 in 2023-09-23T17:58:15.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:15 smithi055 ceph-mon[184174]: failed to encode map e157 with expected crc 2023-09-23T17:58:15.207 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:15.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:15.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:15.214 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:58:15.312 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:15.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:15 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:58:15.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3468534917' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:15.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:15 smithi099 ceph-mon[167076]: osdmap e157: 8 total, 5 up, 8 in 2023-09-23T17:58:15.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:15 smithi099 ceph-mon[167076]: failed to encode map e157 with expected crc 2023-09-23T17:58:15.345 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:15.385 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:15.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:15.389 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:15.489 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:15.524 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:15.729 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:15.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:15.734 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:15.834 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:15.865 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:16.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:16 smithi055 ceph-mon[184174]: pgmap v345: 99 pgs: 1 remapped+peering, 1 stale+undersized+peered, 4 active+clean+remapped, 2 active+recovering+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 9 undersized+peered, 41 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44174/265197 objects degraded (16.657%); 10755/265197 objects misplaced (4.055%); 8.9 MiB/s, 140 objects/s recovering 2023-09-23T17:58:16.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:16 smithi055 ceph-mon[179668]: pgmap v345: 99 pgs: 1 remapped+peering, 1 stale+undersized+peered, 4 active+clean+remapped, 2 active+recovering+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 9 undersized+peered, 41 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44174/265197 objects degraded (16.657%); 10755/265197 objects misplaced (4.055%); 8.9 MiB/s, 140 objects/s recovering 2023-09-23T17:58:16.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:16 smithi099 ceph-mon[167076]: pgmap v345: 99 pgs: 1 remapped+peering, 1 stale+undersized+peered, 4 active+clean+remapped, 2 active+recovering+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 9 undersized+peered, 41 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 44174/265197 objects degraded (16.657%); 10755/265197 objects misplaced (4.055%); 8.9 MiB/s, 140 objects/s recovering 2023-09-23T17:58:16.375 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T17:58:16.751 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:16.755 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:16.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:16.756 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:16.774 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6, 3, 7] live_osds: [0, 1, 2, 4, 5] 2023-09-23T17:58:16.774 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:58:16.774 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T17:58:16.774 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T17:58:16.774 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T17:58:16.774 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T17:58:16.858 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:16.887 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:16.990 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:16.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:16.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:17.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:17.126 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:17.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:17.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:17.165 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:17.265 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:17.293 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:17.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:17.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:17.332 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:17.433 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:17.462 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:17.600 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:17.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:17.603 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:17.704 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:17.731 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:17.766 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:17.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:17.769 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:17.869 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:17.908 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:17.974 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:58:17.974 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:58:18.013 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:18.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:18.015 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:18.116 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:18.148 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:18.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:18 smithi055 ceph-mon[184174]: pgmap v346: 99 pgs: 1 remapped+peering, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41969/265197 objects degraded (15.826%); 12960/265197 objects misplaced (4.887%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T17:58:18.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:18 smithi055 ceph-mon[179668]: pgmap v346: 99 pgs: 1 remapped+peering, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41969/265197 objects degraded (15.826%); 12960/265197 objects misplaced (4.887%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T17:58:18.302 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:18.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:18 smithi099 ceph-mon[167076]: pgmap v346: 99 pgs: 1 remapped+peering, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 41969/265197 objects degraded (15.826%); 12960/265197 objects misplaced (4.887%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T17:58:18.488 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:18.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:18.491 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:18.565 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T17:58:18.592 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:18.639 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:18.786 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:18.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:18.791 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:18.859 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:18.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:18.920 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:18.954 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:18.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:18.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:19.059 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:19.092 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:19.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:19 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 50621/265197 objects degraded (19.088%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:19 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 50621/265197 objects degraded (19.088%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:19.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:19 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 50621/265197 objects degraded (19.088%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:19.381 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:19.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:19.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:19.487 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:19.537 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:19.781 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:19.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:19.861 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:58:19.884 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:58:19.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:19.907 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:58:19.911 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:19.929 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:58:19.945 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:19.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:19.948 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:19.955 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:58:19.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:58:19.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:58:19.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:58:19.956 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:58:19.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:58:19.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:58:19.957 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:58:19.957 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:58:20.049 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:20.085 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:20.187 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:20.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:20.191 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:20.292 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:20.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:20 smithi099 ceph-mon[167076]: pgmap v347: 99 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 50621/265197 objects degraded (19.088%); 12960/265197 objects misplaced (4.887%); 32 MiB/s, 510 objects/s recovering 2023-09-23T17:58:20.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:20 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2116994236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:20.328 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:20 smithi055 ceph-mon[184174]: pgmap v347: 99 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 50621/265197 objects degraded (19.088%); 12960/265197 objects misplaced (4.887%); 32 MiB/s, 510 objects/s recovering 2023-09-23T17:58:20.328 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:20 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2116994236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:20.328 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:20 smithi055 ceph-mon[179668]: pgmap v347: 99 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 50621/265197 objects degraded (19.088%); 12960/265197 objects misplaced (4.887%); 32 MiB/s, 510 objects/s recovering 2023-09-23T17:58:20.328 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:20 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2116994236' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:20.333 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:20.471 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:20.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:20.475 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:20.538 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:58:20.538 INFO:teuthology.orchestra.run.smithi055.stdout: "success": "" 2023-09-23T17:58:20.538 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:58:20.543 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:20.576 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:20.609 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:58:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:58:20] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-09-23T17:58:20.614 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:20.756 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:20.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:20.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:20.773 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:58:20.774 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:58:20.862 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:20.895 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:21.041 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:21.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:21.044 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:21.102 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T17:58:21.142 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3682148940' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:58:21.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3682148940' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:58:21.145 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:21.176 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:21.216 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:21.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:21.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:21.321 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:21.350 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:21.388 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:21.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:21.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:21.494 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:21.524 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3682148940' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:58:21.611 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:21.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:21.614 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:21.715 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:21.746 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:21.816 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:21.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:21.821 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:21.921 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:21.954 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:21.992 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:21.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:21.996 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:22.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:22.126 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:22.127 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:58:22.127 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T17:58:22.127 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T17:58:22.127 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T17:58:22.127 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:58:22.161 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:22.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:22.164 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:22.265 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:22.281 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[184174]: pgmap v348: 99 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 50621/265197 objects degraded (19.088%); 12960/265197 objects misplaced (4.887%); 25 MiB/s, 405 objects/s recovering 2023-09-23T17:58:22.281 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:22.281 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3682148940' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:58:22.281 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[184174]: osdmap e158: 8 total, 5 up, 8 in 2023-09-23T17:58:22.281 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:22.282 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[184174]: failed to encode map e158 with expected crc 2023-09-23T17:58:22.282 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[179668]: pgmap v348: 99 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 50621/265197 objects degraded (19.088%); 12960/265197 objects misplaced (4.887%); 25 MiB/s, 405 objects/s recovering 2023-09-23T17:58:22.282 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:22.282 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3682148940' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:58:22.282 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[179668]: osdmap e158: 8 total, 5 up, 8 in 2023-09-23T17:58:22.282 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:22.283 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:22 smithi055 ceph-mon[179668]: failed to encode map e158 with expected crc 2023-09-23T17:58:22.295 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:22.512 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:22.514 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T17:58:22.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:22.516 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:22.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:22 smithi099 ceph-mon[167076]: pgmap v348: 99 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 50621/265197 objects degraded (19.088%); 12960/265197 objects misplaced (4.887%); 25 MiB/s, 405 objects/s recovering 2023-09-23T17:58:22.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:22 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3682148940' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:58:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:22 smithi099 ceph-mon[167076]: osdmap e158: 8 total, 5 up, 8 in 2023-09-23T17:58:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:22 smithi099 ceph-mon[167076]: failed to encode map e158 with expected crc 2023-09-23T17:58:22.617 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:22.648 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:22.689 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:22.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:22.692 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:22.775 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:58:22.793 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:22.821 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:22.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:22.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:22.858 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:22.959 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:22.995 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:23.044 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:23.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:23.047 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:23.126 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:23.148 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:23.182 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:23.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:23 smithi055 ceph-mon[179668]: failed to encode map e158 with expected crc 2023-09-23T17:58:23.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:23 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:23.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:23 smithi055 ceph-mon[179668]: failed to encode map e159 with expected crc 2023-09-23T17:58:23.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:23 smithi055 ceph-mon[179668]: osdmap e159: 8 total, 5 up, 8 in 2023-09-23T17:58:23.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:23 smithi055 ceph-mon[184174]: failed to encode map e158 with expected crc 2023-09-23T17:58:23.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:23 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:23.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:23 smithi055 ceph-mon[184174]: failed to encode map e159 with expected crc 2023-09-23T17:58:23.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:23 smithi055 ceph-mon[184174]: osdmap e159: 8 total, 5 up, 8 in 2023-09-23T17:58:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:23 smithi099 ceph-mon[167076]: failed to encode map e158 with expected crc 2023-09-23T17:58:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:23 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:23 smithi099 ceph-mon[167076]: failed to encode map e159 with expected crc 2023-09-23T17:58:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:23 smithi099 ceph-mon[167076]: osdmap e159: 8 total, 5 up, 8 in 2023-09-23T17:58:23.595 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T17:58:24.423 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:24.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:24 smithi055 ceph-mon[179668]: pgmap v350: 99 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 47773/265197 objects degraded (18.014%); 12960/265197 objects misplaced (4.887%); 50 MiB/s, 799 objects/s recovering 2023-09-23T17:58:24.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:24 smithi055 ceph-mon[179668]: failed to encode map e159 with expected crc 2023-09-23T17:58:24.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:24 smithi055 ceph-mon[179668]: osdmap e160: 8 total, 5 up, 8 in 2023-09-23T17:58:24.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:24 smithi055 ceph-mon[179668]: failed to encode map e160 with expected crc 2023-09-23T17:58:24.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:24 smithi055 ceph-mon[184174]: pgmap v350: 99 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 47773/265197 objects degraded (18.014%); 12960/265197 objects misplaced (4.887%); 50 MiB/s, 799 objects/s recovering 2023-09-23T17:58:24.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:24 smithi055 ceph-mon[184174]: failed to encode map e159 with expected crc 2023-09-23T17:58:24.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:24 smithi055 ceph-mon[184174]: osdmap e160: 8 total, 5 up, 8 in 2023-09-23T17:58:24.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:24 smithi055 ceph-mon[184174]: failed to encode map e160 with expected crc 2023-09-23T17:58:24.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:24.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:24.536 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:24.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:24 smithi099 ceph-mon[167076]: pgmap v350: 99 pgs: 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 47773/265197 objects degraded (18.014%); 12960/265197 objects misplaced (4.887%); 50 MiB/s, 799 objects/s recovering 2023-09-23T17:58:24.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:24 smithi099 ceph-mon[167076]: failed to encode map e159 with expected crc 2023-09-23T17:58:24.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:24 smithi099 ceph-mon[167076]: osdmap e160: 8 total, 5 up, 8 in 2023-09-23T17:58:24.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:24 smithi099 ceph-mon[167076]: failed to encode map e160 with expected crc 2023-09-23T17:58:24.624 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:24.907 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:24.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:24.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:25.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:25.053 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:25.094 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:25.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:25.097 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:25.109 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:25.198 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:25.229 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:25.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:25.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:25.278 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:25.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:25 smithi099 ceph-mon[167076]: failed to encode map e160 with expected crc 2023-09-23T17:58:25.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:25 smithi099 ceph-mon[167076]: failed to encode map e160 with expected crc 2023-09-23T17:58:25.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:25.404 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:25.410 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:25.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:25 smithi055 ceph-mon[179668]: failed to encode map e160 with expected crc 2023-09-23T17:58:25.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:25 smithi055 ceph-mon[179668]: failed to encode map e160 with expected crc 2023-09-23T17:58:25.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:25 smithi055 ceph-mon[184174]: failed to encode map e160 with expected crc 2023-09-23T17:58:25.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:25 smithi055 ceph-mon[184174]: failed to encode map e160 with expected crc 2023-09-23T17:58:25.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:25.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:25.459 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:25.560 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:25.590 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:25.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:25.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:25.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:25.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:25.766 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:25.809 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:25.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:25.811 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:25.912 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:25.946 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:25.984 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:25.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:25.990 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:26.091 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:26.143 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [6, 3, 7] live_osds: [0, 1, 2, 4, 5] 2023-09-23T17:58:26.143 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:58:26.143 INFO:tasks.thrashosds.thrasher:Reviving osd 6 2023-09-23T17:58:26.143 INFO:tasks.cephadm.osd.6:Restarting osd.6 (starting--it wasn't running)... 2023-09-23T17:58:26.144 DEBUG:teuthology.orchestra.run.smithi099:osd.6> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.6.service 2023-09-23T17:58:26.146 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl start ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T17:58:26.181 INFO:journalctl@ceph.osd.6.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T17:58:26.192 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:26.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:26 smithi055 ceph-mon[179668]: pgmap v353: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 47228/265071 objects degraded (17.817%); 12960/265071 objects misplaced (4.889%); 42 MiB/s, 675 objects/s recovering 2023-09-23T17:58:26.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:26 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 47228/265071 objects degraded (17.817%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:26.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:26 smithi055 ceph-mon[184174]: pgmap v353: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 47228/265071 objects degraded (17.817%); 12960/265071 objects misplaced (4.889%); 42 MiB/s, 675 objects/s recovering 2023-09-23T17:58:26.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:26 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 47228/265071 objects degraded (17.817%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:26.577 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:26 smithi099 systemd[1]: Starting Ceph osd.6 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T17:58:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:26 smithi099 ceph-mon[167076]: pgmap v353: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 6 op/s; 47228/265071 objects degraded (17.817%); 12960/265071 objects misplaced (4.889%); 42 MiB/s, 675 objects/s recovering 2023-09-23T17:58:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:26 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 47228/265071 objects degraded (17.817%), 23 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:26.612 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:26.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:26.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:26.724 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:26.761 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:26.897 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:58:26.897 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T17:58:26.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T17:58:26.897 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:58:26.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.897 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104107 5.0 5:08ba7c2f:::benchmark_data_smithi055_229117_object104106:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:58:26.897 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.953750+0000", 2023-09-23T17:58:26.898 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 62.937271101, 2023-09-23T17:58:26.898 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3969089229999998, 2023-09-23T17:58:26.898 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.898 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.898 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.898 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.898 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.898 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104107 2023-09-23T17:58:26.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.898 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953750+0000", 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953750+0000", 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953748+0000", 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953776+0000", 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8515000000000001e-05 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953798+0000", 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1855999999999999e-05 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953808+0000", 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0283e-05 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953876+0000", 2023-09-23T17:58:26.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7348000000000006e-05 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op must wait for map", 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953885+0000", 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0839999999999998e-06 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.962444+0000", 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0085596219999999994 2023-09-23T17:58:26.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.962456+0000", 2023-09-23T17:58:26.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1606999999999999e-05 2023-09-23T17:58:26.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949379+0000", 2023-09-23T17:58:26.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98692283300000005 2023-09-23T17:58:26.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949390+0000", 2023-09-23T17:58:26.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1503e-05 2023-09-23T17:58:26.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.325692+0000", 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.376301955 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.327734+0000", 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020413749999999998 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.327871+0000", 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137739 2023-09-23T17:58:26.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336120+0000", 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082488500000000003 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336181+0000", 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1007000000000001e-05 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.348130+0000", 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011948676 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.348167+0000", 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6782000000000001e-05 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350658+0000", 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024918570000000001 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104131 5.0 5:02532558:::benchmark_data_smithi055_229117_object104130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.987392+0000", 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 62.903628576000003, 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3655978989999999, 2023-09-23T17:58:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104131 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987392+0000", 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987392+0000", 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987390+0000", 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987415+0000", 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4185e-05 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987433+0000", 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8238000000000001e-05 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987442+0000", 2023-09-23T17:58:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9600000000000006e-06 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.988601+0000", 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011587889999999999 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.988612+0000", 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1969e-05 2023-09-23T17:58:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949474+0000", 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96086169899999996 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949482+0000", 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0099999999999995e-06 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333244+0000", 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3837614760000001 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333381+0000", 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013695800000000001 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333517+0000", 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000136181 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342436+0000", 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0089189990000000004 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342466+0000", 2023-09-23T17:58:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0561000000000001e-05 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350747+0000", 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082805090000000001 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350829+0000", 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1778000000000002e-05 2023-09-23T17:58:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.352990+0000", 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021612770000000001 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104167 5.0 5:0cc1b975:::benchmark_data_smithi055_229117_object104166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:24.986469+0000", 2023-09-23T17:58:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.904551861000002, 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3687401239999999, 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104167 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986469+0000", 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986469+0000", 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986467+0000", 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986500+0000", 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3046000000000002e-05 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986515+0000", 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5138e-05 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986527+0000", 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1429e-05 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.026788+0000", 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040261308000000003 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.026799+0000", 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0699999999999999e-05 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336245+0000", 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3094467810000001 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336378+0000", 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013300399999999999 2023-09-23T17:58:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336525+0000", 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014695100000000001 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342505+0000", 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0059792530000000003 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342518+0000", 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.348e-05 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353111+0000", 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010592839 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353201+0000", 2023-09-23T17:58:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0167999999999995e-05 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355209+0000", 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020078090000000002 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104189 5.0 5:00d05606:::benchmark_data_smithi055_229117_object104188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.012952+0000", 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.878069087999997, 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.354677634, 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104189 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.928 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012952+0000", 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012952+0000", 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012950+0000", 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012983+0000", 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2840999999999999e-05 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012997+0000", 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.454e-05 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.013006+0000", 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2269999999999999e-06 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.030199+0000", 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017192819000000002 2023-09-23T17:58:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.030209+0000", 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.023e-05 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339148+0000", 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3089385840000001 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339354+0000", 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020608099999999999 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339570+0000", 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021573299999999999 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342529+0000", 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029589759999999999 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342571+0000", 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1863999999999998e-05 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365363+0000", 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022792085 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365409+0000", 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6479e-05 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367629+0000", 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022198840000000001 2023-09-23T17:58:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104208 5.0 5:0c574cd2:::benchmark_data_smithi055_229117_object104207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.042202+0000", 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.848818907000002, 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3303606880000001, 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104208 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042202+0000", 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042202+0000", 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042200+0000", 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042226+0000", 2023-09-23T17:58:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5925e-05 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042245+0000", 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9449999999999998e-05 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042255+0000", 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9739999999999997e-06 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042301+0000", 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5945999999999997e-05 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042309+0000", 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7130000000000005e-06 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342600+0000", 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3002915850000001 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342725+0000", 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012474099999999999 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342853+0000", 2023-09-23T17:58:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000127806 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353066+0000", 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010212738000000001 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355236+0000", 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002170052 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369924+0000", 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014688012 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369984+0000", 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0571e-05 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372562+0000", 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00257807 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104228 5.0 5:04306094:::benchmark_data_smithi055_229117_object104227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.063372+0000", 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.827648234999998, 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3247219290000001, 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104228 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063372+0000", 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063372+0000", 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063371+0000", 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063394+0000", 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2929999999999999e-05 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063407+0000", 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3142000000000001e-05 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063415+0000", 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8120000000000008e-06 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063476+0000", 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0988e-05 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063486+0000", 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0397999999999999e-05 2023-09-23T17:58:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345425+0000", 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2819386879999999 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345551+0000", 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012593200000000001 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345665+0000", 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114308 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355247+0000", 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0095815529999999996 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355307+0000", 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0619999999999999e-05 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385736+0000", 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030428261000000002 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385803+0000", 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7624000000000002e-05 2023-09-23T17:58:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388094+0000", 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002291015 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104253 5.0 5:0d680df2:::benchmark_data_smithi055_229117_object104252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.088438+0000", 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.802582616999999, 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3025030870000001, 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104253 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088438+0000", 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088438+0000", 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088436+0000", 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088461+0000", 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4658999999999999e-05 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088478+0000", 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7805e-05 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088487+0000", 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.225e-06 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.092295+0000", 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038084870000000002 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.092306+0000", 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1172000000000001e-05 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355342+0000", 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.263036211 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355533+0000", 2023-09-23T17:58:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000190767 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355663+0000", 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012937699999999999 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365284+0000", 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0096211700000000001 2023-09-23T17:58:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367649+0000", 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002365514 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388536+0000", 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020886503000000001 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388577+0000", 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1167000000000001e-05 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.390941+0000", 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002364157 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104272 5.0 5:0ec51932:::benchmark_data_smithi055_229117_object104271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.108373+0000", 2023-09-23T17:58:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.782647167, 2023-09-23T17:58:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2933625550000001, 2023-09-23T17:58:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104272 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108373+0000", 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108373+0000", 2023-09-23T17:58:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108372+0000", 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108396+0000", 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4311000000000002e-05 2023-09-23T17:58:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108412+0000", 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6049000000000002e-05 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108420+0000", 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9170000000000006e-06 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108482+0000", 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1969000000000002e-05 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108491+0000", 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4779999999999997e-06 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.358777+0000", 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2502855180000001 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.358912+0000", 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013505099999999999 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.359032+0000", 2023-09-23T17:58:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120331 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367658+0000", 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0086259400000000003 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367698+0000", 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9477999999999997e-05 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.398598+0000", 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030900179 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.398665+0000", 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7167000000000003e-05 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401736+0000", 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030708630000000001 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104276 5.0 5:0ba41185:::benchmark_data_smithi055_229117_object104275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.114398+0000", 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.776622175, 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2928085359999999, 2023-09-23T17:58:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104276 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114398+0000", 2023-09-23T17:58:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114398+0000", 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114397+0000", 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114423+0000", 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6006000000000001e-05 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114440+0000", 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6877000000000001e-05 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114448+0000", 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5120000000000008e-06 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.115268+0000", 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00081976499999999997 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.115279+0000", 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0533999999999999e-05 2023-09-23T17:58:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362251+0000", 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.24697258 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362402+0000", 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015118400000000001 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362530+0000", 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012761700000000001 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369824+0000", 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072939889999999999 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372645+0000", 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028209210000000001 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.405151+0000", 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032506363000000003 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.405203+0000", 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1987000000000002e-05 2023-09-23T17:58:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407207+0000", 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002003777 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104298 5.0 5:040384f9:::benchmark_data_smithi055_229117_object104297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.139530+0000", 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.751490146000002, 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2697862209999999, 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104298 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139530+0000", 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139530+0000", 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139529+0000", 2023-09-23T17:58:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139553+0000", 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4026999999999999e-05 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139565+0000", 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2364999999999999e-05 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139573+0000", 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4200000000000001e-06 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.141901+0000", 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023278639999999998 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.141909+0000", 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5099999999999998e-06 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372679+0000", 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2307700370000001 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372803+0000", 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012344299999999999 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372931+0000", 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012818500000000001 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385669+0000", 2023-09-23T17:58:26.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012738027000000001 2023-09-23T17:58:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385682+0000", 2023-09-23T17:58:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3368e-05 2023-09-23T17:58:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407273+0000", 2023-09-23T17:58:26.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021590933 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407320+0000", 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7305999999999997e-05 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409317+0000", 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019962769999999999 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.985 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104301 5.0 5:0653eda0:::benchmark_data_smithi055_229117_object104300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.142080+0000", 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.748940976, 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2806010670000001, 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.986 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104301 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142080+0000", 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142080+0000", 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142078+0000", 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142104+0000", 2023-09-23T17:58:26.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5931e-05 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142116+0000", 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2262999999999999e-05 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142125+0000", 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5359999999999992e-06 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142238+0000", 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000113056 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142247+0000", 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6680000000000007e-06 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.375988+0000", 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2337410959999999 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.376092+0000", 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104558 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.376220+0000", 2023-09-23T17:58:26.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012734399999999999 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385691+0000", 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0094714069999999994 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391023+0000", 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053320190000000003 2023-09-23T17:58:26.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.419987+0000", 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028963940000000001 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.420045+0000", 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8421000000000002e-05 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422681+0000", 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026352419999999999 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104305 5.0 5:03148f53:::benchmark_data_smithi055_229117_object104304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.146575+0000", 2023-09-23T17:58:26.994 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.744445544000001, 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2788299140000001, 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104305 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:26.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146575+0000", 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146575+0000", 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146573+0000", 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:26.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146600+0000", 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6675000000000001e-05 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146622+0000", 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1965000000000001e-05 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146633+0000", 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.076e-05 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.147763+0000", 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011302879999999999 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:26.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.147777+0000", 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4129e-05 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379012+0000", 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2312345389999999 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379258+0000", 2023-09-23T17:58:26.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024643700000000001 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379508+0000", 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024977000000000002 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.390966+0000", 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011458268000000001 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391089+0000", 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000122881 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422762+0000", 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031672697999999999 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:27.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422832+0000", 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9929000000000001e-05 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.425405+0000", 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002573362 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104309 5.0 5:0de80b4f:::benchmark_data_smithi055_229117_object104308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.152292+0000", 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.738728219000002, 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2892466869999999, 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104309 2023-09-23T17:58:27.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152292+0000", 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152292+0000", 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152291+0000", 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152313+0000", 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2082999999999998e-05 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:27.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152326+0000", 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2493999999999999e-05 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152332+0000", 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7710000000000001e-06 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152379+0000", 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6233999999999998e-05 2023-09-23T17:58:27.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152389+0000", 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0234999999999999e-05 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382503+0000", 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.230114411 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382678+0000", 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000174704 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382886+0000", 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000208481 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391101+0000", 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082150169999999998 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391162+0000", 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0718999999999999e-05 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.438904+0000", 2023-09-23T17:58:27.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047742297000000003 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.438971+0000", 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6787999999999998e-05 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441539+0000", 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025679069999999999 2023-09-23T17:58:27.010 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104346 5.0 5:07875566:::benchmark_data_smithi055_229117_object104345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.224009+0000", 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.667011271, 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2202360750000001, 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:27.011 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104346 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224009+0000", 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224009+0000", 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224007+0000", 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:27.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224037+0000", 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9128e-05 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224058+0000", 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1829999999999999e-05 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224068+0000", 2023-09-23T17:58:27.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8560000000000002e-06 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224119+0000", 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0844000000000003e-05 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224128+0000", 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4880000000000007e-06 2023-09-23T17:58:27.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391218+0000", 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1670905920000001 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391390+0000", 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017210399999999999 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391557+0000", 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000166694 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409338+0000", 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017781376000000002 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413075+0000", 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037367199999999998 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441577+0000", 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028502169000000001 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441616+0000", 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8859999999999997e-05 2023-09-23T17:58:27.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444245+0000", 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026292899999999998 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104353 5.0 5:0b824901:::benchmark_data_smithi055_229117_object104352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:27.019 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.238109+0000", 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.652911649000004, 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.208967144, 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104353 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238109+0000", 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238109+0000", 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238108+0000", 2023-09-23T17:58:27.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238128+0000", 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9854999999999999e-05 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238135+0000", 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1729999999999998e-06 2023-09-23T17:58:27.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238140+0000", 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6199999999999998e-06 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238197+0000", 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7005000000000001e-05 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238204+0000", 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2819999999999999e-06 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.394820+0000", 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.156615231 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:27.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.394956+0000", 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013646599999999999 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.395118+0000", 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000162158 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409348+0000", 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014229883 2023-09-23T17:58:27.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413141+0000", 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00379284 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444312+0000", 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031170989 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444373+0000", 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1309000000000005e-05 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447076+0000", 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027029129999999999 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104360 5.0 5:0a7d8202:::benchmark_data_smithi055_229117_object104359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.266784+0000", 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.62423647, 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1830323949999999, 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104360 2023-09-23T17:58:27.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266784+0000", 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266784+0000", 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266782+0000", 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266806+0000", 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3794999999999998e-05 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266824+0000", 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7838000000000001e-05 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266833+0000", 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9469999999999992e-06 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266890+0000", 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7238999999999999e-05 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266902+0000", 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1919999999999999e-05 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401792+0000", 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.13489014 2023-09-23T17:58:27.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401976+0000", 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000184014 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.402162+0000", 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018587599999999999 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413089+0000", 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010926544999999999 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.425460+0000", 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012371320999999999 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447163+0000", 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021703538000000001 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447231+0000", 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7675999999999994e-05 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:27.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.449817+0000", 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025853859999999998 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:108962 5.0 5:0bab387e:::benchmark_data_smithi055_229117_object3336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.753784+0000", 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 54.137236237000003, 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.186715511, 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:27.036 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108962 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753784+0000", 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.037 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753784+0000", 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753783+0000", 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753792+0000", 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0550000000000005e-06 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753807+0000", 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4888e-05 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:27.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753815+0000", 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8659999999999995e-06 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.841808+0000", 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.087992655000000003 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.842575+0000", 2023-09-23T17:58:27.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00076655600000000005 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.871420+0000", 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028845756 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.871532+0000", 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000112043 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.041 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.871703+0000", 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017070599999999999 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.898664+0000", 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026961124 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.042 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.902238+0000", 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035741309999999999 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.938343+0000", 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.036104458999999998 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.938383+0000", 2023-09-23T17:58:27.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9885000000000002e-05 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940500+0000", 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021171979999999998 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.044 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:108965 5.0 5:0ca96563:::benchmark_data_smithi055_229117_object3339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.758821+0000", 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 54.132199833999998, 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18445447700000001, 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108965 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.045 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758821+0000", 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758821+0000", 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758820+0000", 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758825+0000", 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4580000000000001e-06 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.047 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758834+0000", 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5339999999999999e-06 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758839+0000", 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1569999999999999e-06 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.842655+0000", 2023-09-23T17:58:27.048 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.083815857999999993 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.843295+0000", 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000640651 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.887064+0000", 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043768661 2023-09-23T17:58:27.049 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.887235+0000", 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017120699999999999 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.887449+0000", 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021399400000000001 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.050 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.902251+0000", 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014802123 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940561+0000", 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038309546 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.051 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940592+0000", 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1056999999999998e-05 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940635+0000", 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3038999999999999e-05 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.943275+0000", 2023-09-23T17:58:27.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002640444 2023-09-23T17:58:27.053 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.053 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:27.053 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.053 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.053 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.053 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:108971 5.0 5:0ef24fea:::benchmark_data_smithi055_229117_object3345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T17:58:27.053 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.766589+0000", 2023-09-23T17:58:27.053 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 54.124431706999999, 2023-09-23T17:58:27.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.182688712, 2023-09-23T17:58:27.054 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:27.054 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:27.054 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:27.054 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:27.054 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:27.054 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108971 2023-09-23T17:58:27.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766589+0000", 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766589+0000", 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766588+0000", 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766593+0000", 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0699999999999997e-06 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766653+0000", 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0232000000000002e-05 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766662+0000", 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0240000000000003e-06 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.843985+0000", 2023-09-23T17:58:27.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.077322734000000004 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.844592+0000", 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00060725799999999995 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.906868+0000", 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.062275358000000003 2023-09-23T17:58:27.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.906988+0000", 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120485 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.907138+0000", 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000150216 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.921960+0000", 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014822003 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.946229+0000", 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024268112000000001 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.946287+0000", 2023-09-23T17:58:27.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7983999999999999e-05 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.946365+0000", 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8867000000000002e-05 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.949278+0000", 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029122089999999998 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.061 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPushReply(5.0 145/143 [PushReplyOp(5:023b70b2:::benchmark_data_smithi055_229117_object72678:head)])", 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.870165+0000", 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 54.020855570000002, 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18210786500000001, 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870165+0000", 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870165+0000", 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:27.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870164+0000", 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870173+0000", 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7709999999999995e-06 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870197+0000", 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4243000000000002e-05 2023-09-23T17:58:27.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870207+0000", 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8279999999999998e-06 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:33.047464+0000", 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.177257317 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:27.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:27.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:33.052273+0000", 2023-09-23T17:58:27.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048086429999999996 2023-09-23T17:58:27.066 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.066 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:27.066 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.066 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:27.066 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:27.066 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:58:27.460 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:27.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:27.464 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:27.560 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T17:58:27.565 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:27.804 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:27.868 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:58:27.909 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:27.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:27.914 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:27.919 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:58:27.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:58:27.999 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:58:28.015 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:28.052 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:58:28.059 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:58:28.060 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:58:28.060 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:58:28.060 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:58:28.060 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:58:28.060 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:58:28.060 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.6 2023-09-23T17:58:28.060 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:58:28.096 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:28.188 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:28.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:28.205 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:28.306 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:28 smithi099 ceph-mon[167076]: pgmap v354: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 47228/264414 objects degraded (17.861%); 12853/264414 objects misplaced (4.861%); 42 MiB/s, 675 objects/s recovering 2023-09-23T17:58:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:28 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2821144142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:28 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 5 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:28 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3407157492' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:28 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:28.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:28 smithi055 ceph-mon[184174]: pgmap v354: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 47228/264414 objects degraded (17.861%); 12853/264414 objects misplaced (4.861%); 42 MiB/s, 675 objects/s recovering 2023-09-23T17:58:28.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:28 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2821144142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:28.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:28 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 5 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:28.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:28 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3407157492' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:28.391 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:28 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:28.391 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:28 smithi055 ceph-mon[179668]: pgmap v354: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 17 active+undersized+degraded; 5.4 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 43 op/s; 47228/264414 objects degraded (17.861%); 12853/264414 objects misplaced (4.861%); 42 MiB/s, 675 objects/s recovering 2023-09-23T17:58:28.391 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:28 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2821144142' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:28.391 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:28 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 5 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:28.391 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:28 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3407157492' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:28.391 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:28 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:28.392 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:28.554 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:58:28.555 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:58:28.565 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:28.567 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:28.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:28.573 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:28.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:28.715 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:28.826 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:28 smithi099 bash[188066]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T17:58:28.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:28 smithi099 bash[188066]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T17:58:28.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:28 smithi099 bash[188066]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-6 --no-mon-config 2023-09-23T17:58:28.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:28 smithi099 bash[188066]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-6/block 2023-09-23T17:58:28.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:28 smithi099 bash[188066]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-6/block 2023-09-23T17:58:28.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:28 smithi099 bash[188066]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T17:58:28.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:28 smithi099 bash[188066]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-6 2023-09-23T17:58:28.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:28 smithi099 bash[188066]: --> ceph-volume lvm activate successful for osd ID: 6 2023-09-23T17:58:28.851 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T17:58:28.854 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:28.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:28.857 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:28.958 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:29.061 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:29.145 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:29 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:29.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:29 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:29.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:29 smithi055 ceph-mon[179668]: osdmap e161: 8 total, 5 up, 8 in 2023-09-23T17:58:29.303 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:29.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:29.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:29.413 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:29.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:29 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:29.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:29 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:29.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:29 smithi055 ceph-mon[184174]: osdmap e161: 8 total, 5 up, 8 in 2023-09-23T17:58:29.436 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:29 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:29.436 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:29 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:29.436 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:29 smithi099 ceph-mon[167076]: osdmap e161: 8 total, 5 up, 8 in 2023-09-23T17:58:29.436 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:29 smithi099 podman[188388]: 2023-09-23T17:58:29.436 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:29 smithi099 bash[188388]: 4eda3eaa4a8d86540d9abadbb02847cc914d2a22655c77de7beb73df65246c76 2023-09-23T17:58:29.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T17:58:29.486 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:29.615 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:29.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:29.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:29.708 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:29 smithi099 ceph-osd[188426]: -- 172.21.15.99:0/1385651478 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55bc87c40b60 con 0x55bc87c07000 2023-09-23T17:58:29.708 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:29 smithi099 systemd[1]: Started Ceph osd.6 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T17:58:29.722 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:29.807 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:29.882 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:29.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:29.896 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:29.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:30.038 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:29 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T17:58:29.707+0000 7f8a95028540 -1 Falling back to public interface 2023-09-23T17:58:30.043 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:30.099 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:30.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:30.103 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:30.166 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:30.204 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:30.288 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:30.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:30 smithi099 ceph-mon[167076]: pgmap v355: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 18 active+undersized+degraded; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 46634/261654 objects degraded (17.823%); 12503/261654 objects misplaced (4.778%); 39 MiB/s, 630 objects/s recovering 2023-09-23T17:58:30.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:30 smithi099 ceph-mon[167076]: failed to encode map e161 with expected crc 2023-09-23T17:58:30.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:30 smithi055 ceph-mon[184174]: pgmap v355: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 18 active+undersized+degraded; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 46634/261654 objects degraded (17.823%); 12503/261654 objects misplaced (4.778%); 39 MiB/s, 630 objects/s recovering 2023-09-23T17:58:30.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:30 smithi055 ceph-mon[184174]: failed to encode map e161 with expected crc 2023-09-23T17:58:30.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:30 smithi055 ceph-mon[179668]: pgmap v355: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 18 active+undersized+degraded; 5.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 159 op/s; 46634/261654 objects degraded (17.823%); 12503/261654 objects misplaced (4.778%); 39 MiB/s, 630 objects/s recovering 2023-09-23T17:58:30.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:30 smithi055 ceph-mon[179668]: failed to encode map e161 with expected crc 2023-09-23T17:58:30.432 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:58:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:58:30] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.43.0" 2023-09-23T17:58:30.556 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:58:30.644 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:58:30.645 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:58:30.690 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:30.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:30.700 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:30.801 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:30.904 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:31.084 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:31.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:31.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:31.194 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:31.269 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:31.377 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T17:58:31.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:31 smithi055 ceph-mon[179668]: pgmap v357: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 18 active+undersized+degraded; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 44703/254904 objects degraded (17.537%); 12503/254904 objects misplaced (4.905%); 13 MiB/s, 207 objects/s recovering 2023-09-23T17:58:31.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:31 smithi055 ceph-mon[184174]: pgmap v357: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 18 active+undersized+degraded; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 44703/254904 objects degraded (17.537%); 12503/254904 objects misplaced (4.905%); 13 MiB/s, 207 objects/s recovering 2023-09-23T17:58:31.478 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2023-09-23T17:58:31.478 INFO:teuthology.orchestra.run.smithi099.stderr:0 2023-09-23T17:58:31.478 INFO:teuthology.orchestra.run.smithi099.stderr:1 2023-09-23T17:58:31.478 INFO:teuthology.orchestra.run.smithi099.stderr:2 2023-09-23T17:58:31.478 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2023-09-23T17:58:31.478 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2023-09-23T17:58:31.478 INFO:teuthology.orchestra.run.smithi099.stderr:config diff 2023-09-23T17:58:31.478 INFO:teuthology.orchestra.run.smithi099.stderr:config diff get 2023-09-23T17:58:31.479 INFO:teuthology.orchestra.run.smithi099.stderr:config get 2023-09-23T17:58:31.479 INFO:teuthology.orchestra.run.smithi099.stderr:config help [] 2023-09-23T17:58:31.479 INFO:teuthology.orchestra.run.smithi099.stderr:config set ... 2023-09-23T17:58:31.479 INFO:teuthology.orchestra.run.smithi099.stderr:admin_socket: invalid command 2023-09-23T17:58:31.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:31 smithi099 ceph-mon[167076]: pgmap v357: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 18 active+undersized+degraded; 5.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 445 op/s; 44703/254904 objects degraded (17.537%); 12503/254904 objects misplaced (4.905%); 13 MiB/s, 207 objects/s recovering 2023-09-23T17:58:31.879 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T17:58:31.879 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-6, ['dump_ops_in_flight'] 2023-09-23T17:58:32.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:32 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 44703/254904 objects degraded (17.537%), 23 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:32 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 44703/254904 objects degraded (17.537%), 23 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:32.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:32 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 44703/254904 objects degraded (17.537%), 23 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:32.577 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T17:58:32.214+0000 7f8a95028540 -1 osd.6 113 log_to_monitors true 2023-09-23T17:58:33.175 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:33.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:33.180 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:33.277 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T17:58:33.280 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:33.321 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:33.359 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:33.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:33.363 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:33.464 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:33.494 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:33.577 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 17:58:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T17:58:33.267+0000 7f8a86609700 -1 osd.6 162 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T17:58:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:33 smithi099 ceph-mon[167076]: from='osd.6 [v2:172.21.15.99:6816/370560019,v1:172.21.15.99:6817/370560019]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:58:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:33 smithi099 ceph-mon[167076]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:58:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:33 smithi099 ceph-mon[167076]: pgmap v358: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 18 active+undersized+degraded; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 44185/249408 objects degraded (17.716%); 11697/249408 objects misplaced (4.690%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T17:58:33.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:33 smithi055 ceph-mon[179668]: from='osd.6 [v2:172.21.15.99:6816/370560019,v1:172.21.15.99:6817/370560019]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:58:33.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:33 smithi055 ceph-mon[179668]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:58:33.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:33 smithi055 ceph-mon[179668]: pgmap v358: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 18 active+undersized+degraded; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 44185/249408 objects degraded (17.716%); 11697/249408 objects misplaced (4.690%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T17:58:33.592 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:33 smithi055 ceph-mon[184174]: from='osd.6 [v2:172.21.15.99:6816/370560019,v1:172.21.15.99:6817/370560019]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:58:33.592 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:33 smithi055 ceph-mon[184174]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-09-23T17:58:33.592 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:33 smithi055 ceph-mon[184174]: pgmap v358: 99 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 42 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 18 active+undersized+degraded; 5.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 605 op/s; 44185/249408 objects degraded (17.716%); 11697/249408 objects misplaced (4.690%); 9.7 MiB/s, 155 objects/s recovering 2023-09-23T17:58:33.601 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:33.601 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:34 smithi099 ceph-mon[167076]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T17:58:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:34 smithi099 ceph-mon[167076]: osdmap e162: 8 total, 5 up, 8 in 2023-09-23T17:58:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:34 smithi099 ceph-mon[167076]: from='osd.6 [v2:172.21.15.99:6816/370560019,v1:172.21.15.99:6817/370560019]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:58:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:34 smithi099 ceph-mon[167076]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:58:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:34 smithi099 ceph-mon[167076]: failed to encode map e162 with expected crc 2023-09-23T17:58:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:34 smithi099 ceph-mon[167076]: failed to encode map e162 with expected crc 2023-09-23T17:58:34.596 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:34.598 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[184174]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T17:58:34.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[184174]: osdmap e162: 8 total, 5 up, 8 in 2023-09-23T17:58:34.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[184174]: from='osd.6 [v2:172.21.15.99:6816/370560019,v1:172.21.15.99:6817/370560019]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:58:34.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[184174]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:58:34.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[184174]: failed to encode map e162 with expected crc 2023-09-23T17:58:34.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[184174]: failed to encode map e162 with expected crc 2023-09-23T17:58:34.599 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[179668]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-09-23T17:58:34.599 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[179668]: osdmap e162: 8 total, 5 up, 8 in 2023-09-23T17:58:34.599 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[179668]: from='osd.6 [v2:172.21.15.99:6816/370560019,v1:172.21.15.99:6817/370560019]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:58:34.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[179668]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T17:58:34.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[179668]: failed to encode map e162 with expected crc 2023-09-23T17:58:34.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:34 smithi055 ceph-mon[179668]: failed to encode map e162 with expected crc 2023-09-23T17:58:34.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:34.601 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:34.701 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:34.733 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:34.895 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:34.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:34.904 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:35.005 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:35.039 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:35.075 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:35.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:35.078 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:35.179 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:35.208 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:35.258 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:35.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:35.261 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:35.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:35 smithi099 ceph-mon[167076]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T17:58:35.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:35 smithi099 ceph-mon[167076]: failed to encode map e163 with expected crc 2023-09-23T17:58:35.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:35 smithi099 ceph-mon[167076]: osd.6 [v2:172.21.15.99:6816/370560019,v1:172.21.15.99:6817/370560019] boot 2023-09-23T17:58:35.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:35 smithi099 ceph-mon[167076]: osdmap e163: 8 total, 6 up, 8 in 2023-09-23T17:58:35.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:35 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:58:35.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:35 smithi099 ceph-mon[167076]: pgmap v361: 99 pgs: 12 peering, 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 33 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 15 active+undersized+degraded; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 37489/243033 objects degraded (15.425%); 11697/243033 objects misplaced (4.813%); 12 MiB/s, 188 objects/s recovering 2023-09-23T17:58:35.361 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:35.400 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:35.532 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:35.535 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[184174]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T17:58:35.535 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[184174]: failed to encode map e163 with expected crc 2023-09-23T17:58:35.535 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[184174]: osd.6 [v2:172.21.15.99:6816/370560019,v1:172.21.15.99:6817/370560019] boot 2023-09-23T17:58:35.535 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[184174]: osdmap e163: 8 total, 6 up, 8 in 2023-09-23T17:58:35.535 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:58:35.535 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[184174]: pgmap v361: 99 pgs: 12 peering, 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 33 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 15 active+undersized+degraded; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 37489/243033 objects degraded (15.425%); 11697/243033 objects misplaced (4.813%); 12 MiB/s, 188 objects/s recovering 2023-09-23T17:58:35.536 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[179668]: Health check update: 2 osds down (OSD_DOWN) 2023-09-23T17:58:35.536 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[179668]: failed to encode map e163 with expected crc 2023-09-23T17:58:35.536 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[179668]: osd.6 [v2:172.21.15.99:6816/370560019,v1:172.21.15.99:6817/370560019] boot 2023-09-23T17:58:35.536 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[179668]: osdmap e163: 8 total, 6 up, 8 in 2023-09-23T17:58:35.536 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-09-23T17:58:35.536 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:35 smithi055 ceph-mon[179668]: pgmap v361: 99 pgs: 12 peering, 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 16 active+clean, 8 undersized+peered, 33 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 15 active+undersized+degraded; 4.9 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 37489/243033 objects degraded (15.425%); 11697/243033 objects misplaced (4.813%); 12 MiB/s, 188 objects/s recovering 2023-09-23T17:58:35.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:35.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:35.638 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:35.665 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:35.700 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:35.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:35.703 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:35.804 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:35.840 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:36.034 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:36.037 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:36.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:36.164 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:36.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:36.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:36.203 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:36.303 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:36.338 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:36.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:36 smithi099 ceph-mon[167076]: failed to encode map e114 with expected crc 2023-09-23T17:58:36.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:36 smithi099 ceph-mon[167076]: failed to encode map e154 with expected crc 2023-09-23T17:58:36.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:36 smithi099 ceph-mon[167076]: failed to encode map e163 with expected crc 2023-09-23T17:58:36.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:36 smithi099 ceph-mon[167076]: osdmap e164: 8 total, 6 up, 8 in 2023-09-23T17:58:36.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:36 smithi099 ceph-mon[167076]: failed to encode map e164 with expected crc 2023-09-23T17:58:36.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:36 smithi099 ceph-mon[167076]: 5.9 starting backfill to osd.6 from (0'0,0'0] MAX to 162'7981 2023-09-23T17:58:36.655 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:36.656 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:36.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:36.668 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:36.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[179668]: failed to encode map e114 with expected crc 2023-09-23T17:58:36.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[179668]: failed to encode map e154 with expected crc 2023-09-23T17:58:36.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[179668]: failed to encode map e163 with expected crc 2023-09-23T17:58:36.669 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[179668]: osdmap e164: 8 total, 6 up, 8 in 2023-09-23T17:58:36.669 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[179668]: failed to encode map e164 with expected crc 2023-09-23T17:58:36.669 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[179668]: 5.9 starting backfill to osd.6 from (0'0,0'0] MAX to 162'7981 2023-09-23T17:58:36.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[184174]: failed to encode map e114 with expected crc 2023-09-23T17:58:36.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[184174]: failed to encode map e154 with expected crc 2023-09-23T17:58:36.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[184174]: failed to encode map e163 with expected crc 2023-09-23T17:58:36.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[184174]: osdmap e164: 8 total, 6 up, 8 in 2023-09-23T17:58:36.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[184174]: failed to encode map e164 with expected crc 2023-09-23T17:58:36.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:36 smithi055 ceph-mon[184174]: 5.9 starting backfill to osd.6 from (0'0,0'0] MAX to 162'7981 2023-09-23T17:58:36.769 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:36.876 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:36.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T17:58:36.973 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:36.985 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:37.086 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:37.166 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:37.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[179668]: failed to encode map e164 with expected crc 2023-09-23T17:58:37.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[179668]: failed to encode map e164 with expected crc 2023-09-23T17:58:37.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[179668]: osdmap e165: 8 total, 6 up, 8 in 2023-09-23T17:58:37.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[179668]: failed to encode map e165 with expected crc 2023-09-23T17:58:37.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:37.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[179668]: pgmap v364: 99 pgs: 12 peering, 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 2 active+recovering+degraded+remapped, 19 active+clean, 7 undersized+peered, 31 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 15 active+undersized+degraded; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 39139/241917 objects degraded (16.179%); 9861/241917 objects misplaced (4.076%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T17:58:37.307 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:37.323 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:37.423 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:37.455 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:37.512 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:37.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:37 smithi099 ceph-mon[167076]: failed to encode map e164 with expected crc 2023-09-23T17:58:37.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:37 smithi099 ceph-mon[167076]: failed to encode map e164 with expected crc 2023-09-23T17:58:37.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:37 smithi099 ceph-mon[167076]: osdmap e165: 8 total, 6 up, 8 in 2023-09-23T17:58:37.521 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:37 smithi099 ceph-mon[167076]: failed to encode map e165 with expected crc 2023-09-23T17:58:37.521 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:37.521 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:37 smithi099 ceph-mon[167076]: pgmap v364: 99 pgs: 12 peering, 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 2 active+recovering+degraded+remapped, 19 active+clean, 7 undersized+peered, 31 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 15 active+undersized+degraded; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 39139/241917 objects degraded (16.179%); 9861/241917 objects misplaced (4.076%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T17:58:37.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:37.522 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:37.592 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[184174]: failed to encode map e164 with expected crc 2023-09-23T17:58:37.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[184174]: failed to encode map e164 with expected crc 2023-09-23T17:58:37.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[184174]: osdmap e165: 8 total, 6 up, 8 in 2023-09-23T17:58:37.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[184174]: failed to encode map e165 with expected crc 2023-09-23T17:58:37.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:37.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:37 smithi055 ceph-mon[184174]: pgmap v364: 99 pgs: 12 peering, 1 active+undersized+degraded+remapped+backfilling, 1 stale+undersized+peered, 4 active+clean+remapped, 2 active+recovering+degraded+remapped, 19 active+clean, 7 undersized+peered, 31 active+undersized, 2 undersized+degraded+peered, 5 active+undersized+remapped, 15 active+undersized+degraded; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 623 op/s; 39139/241917 objects degraded (16.179%); 9861/241917 objects misplaced (4.076%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T17:58:37.604 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:37.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:37.688 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:37.705 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:58:37.743 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:37.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:37.751 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:37.770 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:58:37.812 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:58:37.852 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:37.857 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:58:37.906 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:58:37.931 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:37.943 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:58:37.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:58:37.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:58:37.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:58:37.956 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:58:37.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:58:37.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:58:37.956 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T17:58:37.956 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:58:37.999 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:38.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:38.003 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:38.104 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:38.166 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:38.299 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T17:58:38.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[179668]: failed to encode map e165 with expected crc 2023-09-23T17:58:38.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[179668]: 5.2 starting backfill to osd.6 from (0'0,0'0] MAX to 162'8051 2023-09-23T17:58:38.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[179668]: 5.3 starting backfill to osd.6 from (0'0,0'0] MAX to 162'8086 2023-09-23T17:58:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 39139/241917 objects degraded (16.179%), 20 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1982250858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/342719060' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:58:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:58:38.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[184174]: failed to encode map e165 with expected crc 2023-09-23T17:58:38.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[184174]: 5.2 starting backfill to osd.6 from (0'0,0'0] MAX to 162'8051 2023-09-23T17:58:38.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[184174]: 5.3 starting backfill to osd.6 from (0'0,0'0] MAX to 162'8086 2023-09-23T17:58:38.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:38.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 39139/241917 objects degraded (16.179%), 20 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:38.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1982250858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:38.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/342719060' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:58:38.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:58:38.494 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:38.504 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:38.505 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:38.541 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:58:38.542 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:58:38.581 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:38 smithi099 ceph-mon[167076]: failed to encode map e165 with expected crc 2023-09-23T17:58:38.581 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:38 smithi099 ceph-mon[167076]: 5.2 starting backfill to osd.6 from (0'0,0'0] MAX to 162'8051 2023-09-23T17:58:38.581 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:38 smithi099 ceph-mon[167076]: 5.3 starting backfill to osd.6 from (0'0,0'0] MAX to 162'8086 2023-09-23T17:58:38.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:38 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 4 pgs inactive (PG_AVAILABILITY) 2023-09-23T17:58:38.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 39139/241917 objects degraded (16.179%), 20 pgs degraded, 28 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:38.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1982250858' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:38.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/342719060' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:58:38.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:58:38.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:38.651 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:38.654 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:58:38.655 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T17:58:38.655 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T17:58:38.655 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T17:58:38.655 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:58:38.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:38.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:38.765 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:38.866 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:38.932 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:39.028 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:58:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T17:58:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.44681.0:133512 5.f e166/164 5:fb1995b3:::benchmark_data_smithi055_229117_object27886:head v 166'8343, mlcod=116'1639)", 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:39.018785+0000", 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0014176010000000001, 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014480960000000001, 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:39.018785+0000", 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:39.018785+0000", 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:39.018784+0000", 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:39.018794+0000", 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0244000000000001e-05 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:39.018812+0000", 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7541e-05 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:39.018819+0000", 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7549999999999997e-06 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:39.018860+0000", 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1853000000000001e-05 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:39.018883+0000", 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2938999999999998e-05 2023-09-23T17:58:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T17:58:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 1 2023-09-23T17:58:39.033 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:58:39.054 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:39.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:39.060 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:39.108 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T17:58:39.161 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:39.266 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:39.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:39 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:39.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:39 smithi099 ceph-mon[167076]: failed to encode map e166 with expected crc 2023-09-23T17:58:39.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:39 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:58:39.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:39 smithi099 ceph-mon[167076]: osdmap e166: 8 total, 6 up, 8 in 2023-09-23T17:58:39.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:39 smithi099 ceph-mon[167076]: pgmap v366: 99 pgs: 1 down, 12 peering, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 31 active+clean, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 14 active+undersized+degraded; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 44392/239730 objects degraded (18.517%); 4781/239730 objects misplaced (1.994%); 17 MiB/s, 276 objects/s recovering 2023-09-23T17:58:39.364 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:39.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:39.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:39.377 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:39 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:39.377 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:39 smithi055 ceph-mon[179668]: failed to encode map e166 with expected crc 2023-09-23T17:58:39.377 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:39 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:58:39.377 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:39 smithi055 ceph-mon[179668]: osdmap e166: 8 total, 6 up, 8 in 2023-09-23T17:58:39.377 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:39 smithi055 ceph-mon[179668]: pgmap v366: 99 pgs: 1 down, 12 peering, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 31 active+clean, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 14 active+undersized+degraded; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 44392/239730 objects degraded (18.517%); 4781/239730 objects misplaced (1.994%); 17 MiB/s, 276 objects/s recovering 2023-09-23T17:58:39.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:39 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:39.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:39 smithi055 ceph-mon[184174]: failed to encode map e166 with expected crc 2023-09-23T17:58:39.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:39 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:58:39.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:39 smithi055 ceph-mon[184174]: osdmap e166: 8 total, 6 up, 8 in 2023-09-23T17:58:39.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:39 smithi055 ceph-mon[184174]: pgmap v366: 99 pgs: 1 down, 12 peering, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 31 active+clean, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 14 active+undersized+degraded; 4.9 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 373 op/s; 44392/239730 objects degraded (18.517%); 4781/239730 objects misplaced (1.994%); 17 MiB/s, 276 objects/s recovering 2023-09-23T17:58:39.478 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:39.567 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:39.630 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:39.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:39.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:39.740 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:39.830 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:39.890 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:39.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:39.895 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:39.996 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:40.063 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:40.113 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:40.147 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:40.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:40.155 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:40.256 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:40.321 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:40.426 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:40.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:58:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:58:40] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-09-23T17:58:40.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:40.434 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:40.535 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:40.542 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:58:40.583 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:40.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:40.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:40.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:40.753 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:40.887 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:41.025 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:41.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:41.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:41.110 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T17:58:41.144 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:41.244 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:41.367 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:41.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:41.381 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:41.482 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:41.516 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:41.562 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:41.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:41.567 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:41.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:58:41.672 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:41.714 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:41.756 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:58:41.756 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T17:58:41.756 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T17:58:41.756 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:58:41.756 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.756 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81746 5.a 5:548e2426:::benchmark_data_smithi055_229117_object81745:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.756 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.824853+0000", 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.925953788, 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.981367561, 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81746 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824853+0000", 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824853+0000", 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824852+0000", 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824874+0000", 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1708e-05 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824882+0000", 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8860000000000004e-06 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824888+0000", 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2349999999999996e-06 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825061+0000", 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000172561 2023-09-23T17:58:41.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825071+0000", 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7499999999999998e-06 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.802556+0000", 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97748503399999997 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.802726+0000", 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000170237 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.802912+0000", 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018652499999999999 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806049+0000", 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031369240000000001 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806100+0000", 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1023000000000003e-05 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806112+0000", 2023-09-23T17:58:41.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1737e-05 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806175+0000", 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3121999999999996e-05 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806220+0000", 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5333000000000001e-05 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.764 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81756 5.a 5:559c0640:::benchmark_data_smithi055_229117_object81755:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.824967+0000", 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.925839994, 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98960265000000003, 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.765 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81756 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824967+0000", 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824967+0000", 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824966+0000", 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.824987+0000", 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0706000000000001e-05 2023-09-23T17:58:41.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825006+0000", 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8797e-05 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825012+0000", 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8000000000000001e-06 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825091+0000", 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9028000000000003e-05 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.825098+0000", 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9299999999999997e-06 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806272+0000", 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98117399000000005 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806438+0000", 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000165429 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.806615+0000", 2023-09-23T17:58:41.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000177312 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814377+0000", 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0077615449999999999 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814435+0000", 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.817e-05 2023-09-23T17:58:41.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814447+0000", 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2476999999999999e-05 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814503+0000", 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5831999999999997e-05 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.773 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.773 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814569+0000", 2023-09-23T17:58:41.773 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6179000000000005e-05 2023-09-23T17:58:41.773 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.773 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.773 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.773 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.773 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81758 5.a 5:552239cf:::benchmark_data_smithi055_229117_object81757:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.773 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.828535+0000", 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.92227186700001, 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98621836399999996, 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81758 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.774 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828535+0000", 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828535+0000", 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828534+0000", 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828554+0000", 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0326000000000002e-05 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828567+0000", 2023-09-23T17:58:41.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2809999999999999e-05 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828573+0000", 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8860000000000001e-06 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828630+0000", 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7315999999999998e-05 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.828642+0000", 2023-09-23T17:58:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1977000000000001e-05 2023-09-23T17:58:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.808815+0000", 2023-09-23T17:58:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98017351399999997 2023-09-23T17:58:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.808927+0000", 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111715 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.809046+0000", 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011884599999999999 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814587+0000", 2023-09-23T17:58:41.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055405929999999999 2023-09-23T17:58:41.780 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.780 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.780 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814625+0000", 2023-09-23T17:58:41.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8918000000000002e-05 2023-09-23T17:58:41.780 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.780 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.780 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814664+0000", 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8680999999999998e-05 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814712+0000", 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7871000000000003e-05 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814753+0000", 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1134000000000003e-05 2023-09-23T17:58:41.781 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81760 5.a 5:5730c3c9:::benchmark_data_smithi055_229117_object81759:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.837793+0000", 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.913013557, 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97716881300000002, 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.782 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81760 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837793+0000", 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837793+0000", 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837792+0000", 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837822+0000", 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0035e-05 2023-09-23T17:58:41.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837840+0000", 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.88e-05 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837850+0000", 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.007e-05 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837909+0000", 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8996000000000002e-05 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.837921+0000", 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1319e-05 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.811465+0000", 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97354397400000003 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.811538+0000", 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3102000000000006e-05 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.811636+0000", 2023-09-23T17:58:41.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7844999999999996e-05 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814769+0000", 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003133786 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814830+0000", 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0924999999999997e-05 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814889+0000", 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8860999999999998e-05 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814930+0000", 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0921999999999999e-05 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.814962+0000", 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1746999999999999e-05 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81770 5.a 5:576b5ba8:::benchmark_data_smithi055_229117_object81769:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.845003+0000", 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.90580418, 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98902187100000005, 2023-09-23T17:58:41.790 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81770 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845003+0000", 2023-09-23T17:58:41.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845003+0000", 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845002+0000", 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845024+0000", 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2520000000000001e-05 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845037+0000", 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2237e-05 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845044+0000", 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0110000000000003e-06 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845100+0000", 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6369999999999997e-05 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.845113+0000", 2023-09-23T17:58:41.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.322e-05 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.815001+0000", 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96988796099999997 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.815142+0000", 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014074700000000001 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.815314+0000", 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000171626 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827641+0000", 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012327829 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827724+0000", 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2349999999999996e-05 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.833926+0000", 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006201692 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.833978+0000", 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2855000000000001e-05 2023-09-23T17:58:41.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834024+0000", 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6004999999999997e-05 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81774 5.a 5:516f8aea:::benchmark_data_smithi055_229117_object81773:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.798 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.852382+0000", 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.898424273, 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98193056000000001, 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81774 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852382+0000", 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852382+0000", 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852381+0000", 2023-09-23T17:58:41.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852406+0000", 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4822000000000001e-05 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852416+0000", 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0215e-05 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852423+0000", 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0520000000000004e-06 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852471+0000", 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7691999999999997e-05 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.852482+0000", 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1433e-05 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.818150+0000", 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96566818200000004 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.818306+0000", 2023-09-23T17:58:41.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015550699999999999 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.818470+0000", 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016395099999999999 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827682+0000", 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092118649999999996 2023-09-23T17:58:41.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827763+0000", 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0962000000000002e-05 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834224+0000", 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0064616430000000004 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834275+0000", 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0606000000000003e-05 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834313+0000", 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8012000000000001e-05 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81777 5.a 5:5126b3af:::benchmark_data_smithi055_229117_object81776:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.854699+0000", 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.896107945, 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97978332999999995, 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81777 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854699+0000", 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854699+0000", 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854697+0000", 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854727+0000", 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0085000000000001e-05 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.809 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854743+0000", 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5991e-05 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854753+0000", 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3789999999999995e-06 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854836+0000", 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3842000000000001e-05 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.854847+0000", 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0564000000000001e-05 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.821488+0000", 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96664134099999999 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.821641+0000", 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015245499999999999 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.821806+0000", 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016498899999999999 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834142+0000", 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012336132 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834326+0000", 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018421999999999999 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834389+0000", 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2810000000000003e-05 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834442+0000", 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3480999999999998e-05 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834482+0000", 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9580999999999999e-05 2023-09-23T17:58:41.814 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81792 5.a 5:5acbb496:::benchmark_data_smithi055_229117_object81791:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.868584+0000", 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.88222266, 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97274865799999999, 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.815 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81792 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868584+0000", 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868584+0000", 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868583+0000", 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868604+0000", 2023-09-23T17:58:41.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1667000000000001e-05 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868619+0000", 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4938e-05 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868627+0000", 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4719999999999998e-06 2023-09-23T17:58:41.818 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868689+0000", 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2501000000000003e-05 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.868702+0000", 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2771e-05 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.824681+0000", 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95597861799999995 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.824821+0000", 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000140765 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.824973+0000", 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015180900000000001 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834213+0000", 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092394269999999997 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834334+0000", 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000121002 2023-09-23T17:58:41.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841237+0000", 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0069033690000000003 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841290+0000", 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2926000000000001e-05 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841333+0000", 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2759000000000002e-05 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81799 5.a 5:5098facb:::benchmark_data_smithi055_229117_object81798:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.875371+0000", 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.87543535, 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96608565800000001, 2023-09-23T17:58:41.823 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81799 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875371+0000", 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875371+0000", 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875370+0000", 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875431+0000", 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1605000000000004e-05 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875446+0000", 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4232e-05 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875452+0000", 2023-09-23T17:58:41.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1809999999999998e-06 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875509+0000", 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6781999999999999e-05 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.875518+0000", 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8719999999999997e-06 2023-09-23T17:58:41.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827790+0000", 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95227146299999998 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.827908+0000", 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118269 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.828034+0000", 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012610400000000001 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834342+0000", 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063074409999999996 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834741+0000", 2023-09-23T17:58:41.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00039909100000000002 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841382+0000", 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006641152 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841417+0000", 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4853e-05 2023-09-23T17:58:41.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841457+0000", 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0151999999999998e-05 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81811 5.a 5:57532f39:::benchmark_data_smithi055_229117_object81810:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.831 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.896344+0000", 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.854462825, 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.94522273800000001, 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81811 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896344+0000", 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896344+0000", 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896342+0000", 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896381+0000", 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8726000000000002e-05 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896400+0000", 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9658999999999999e-05 2023-09-23T17:58:41.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896411+0000", 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0647e-05 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896463+0000", 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2105999999999999e-05 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.896472+0000", 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6349999999999995e-06 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.830796+0000", 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.93432424199999997 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.830941+0000", 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014539899999999999 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.831093+0000", 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015141699999999999 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834752+0000", 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036591800000000002 2023-09-23T17:58:41.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834803+0000", 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1011999999999999e-05 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841509+0000", 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006706147 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841547+0000", 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7799000000000002e-05 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841567+0000", 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9629000000000001e-05 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:81815 5.a 5:50ec5bb9:::benchmark_data_smithi055_229117_object81814:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e133)", 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:57.900377+0000", 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 103.85042992, 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.94830255600000002, 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 81815 2023-09-23T17:58:41.840 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900377+0000", 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900377+0000", 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900376+0000", 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900395+0000", 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9397000000000001e-05 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.842 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900407+0000", 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2342e-05 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900413+0000", 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4099999999999999e-06 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900470+0000", 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7207000000000003e-05 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:57.900484+0000", 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4283e-05 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834833+0000", 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.93434895699999998 2023-09-23T17:58:41.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.834954+0000", 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120833 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.835096+0000", 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000142354 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.841472+0000", 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063759350000000001 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.848551+0000", 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.007078689 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.848594+0000", 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2923000000000002e-05 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.848637+0000", 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2670999999999999e-05 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:58.848679+0000", 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2614999999999998e-05 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104098 5.a 5:5150641c:::benchmark_data_smithi055_229117_object104097:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.920515+0000", 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 77.830291263000007, 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.048992793, 2023-09-23T17:58:41.848 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104098 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.849 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920538+0000", 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2770000000000001e-05 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920545+0000", 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1969999999999999e-06 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920550+0000", 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3360000000000002e-06 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927587+0000", 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0070371599999999998 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927719+0000", 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013187600000000001 2023-09-23T17:58:41.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927856+0000", 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013721800000000001 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936497+0000", 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.008640277 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936670+0000", 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017294700000000001 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959038+0000", 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022367973999999999 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959048+0000", 2023-09-23T17:58:41.854 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0428000000000001e-05 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969404+0000", 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0103555609999999 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969508+0000", 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010461799999999999 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.855 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104100 5.a 5:58ebf1c5:::benchmark_data_smithi055_229117_object104099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.920555+0000", 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 77.830251583999996, 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0492678900000001, 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.856 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104100 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T17:58:41.857 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920574+0000", 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9664e-05 2023-09-23T17:58:41.858 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920581+0000", 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3260000000000001e-06 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920586+0000", 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7239999999999997e-06 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.861 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.861 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.931735+0000", 2023-09-23T17:58:41.861 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011148345 2023-09-23T17:58:41.862 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.862 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.931905+0000", 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000170466 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.932081+0000", 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000175623 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.863 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.864 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936776+0000", 2023-09-23T17:58:41.864 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0046956209999999996 2023-09-23T17:58:41.864 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.864 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.864 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.864 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936996+0000", 2023-09-23T17:58:41.864 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021987100000000001 2023-09-23T17:58:41.864 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.864 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.864 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959173+0000", 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022176392999999999 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959189+0000", 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.609e-05 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.865 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969761+0000", 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0105722159999999 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969823+0000", 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1938000000000002e-05 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.866 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.867 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104109 5.a 5:542cd2b3:::benchmark_data_smithi055_229117_object104108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:58:41.867 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.944006+0000", 2023-09-23T17:58:41.867 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 77.806801019000005, 2023-09-23T17:58:41.867 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0419720189999999, 2023-09-23T17:58:41.867 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.867 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:58:41.867 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.867 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.867 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.867 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104109 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944006+0000", 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944006+0000", 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.868 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944004+0000", 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944029+0000", 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5201000000000001e-05 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.869 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944043+0000", 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3964e-05 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944062+0000", 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8190999999999999e-05 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944258+0000", 2023-09-23T17:58:41.870 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019652100000000001 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944396+0000", 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013821100000000001 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944572+0000", 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017574299999999999 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.956893+0000", 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012321445 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959227+0000", 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023331559999999999 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959235+0000", 2023-09-23T17:58:41.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1000000000000004e-06 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969881+0000", 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.010645899 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.970644+0000", 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000763867 2023-09-23T17:58:41.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.985881+0000", 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015236622 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.985978+0000", 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6602999999999996e-05 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.874 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104112 5.a 5:5b2e2197:::benchmark_data_smithi055_229117_object104111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.944077+0000", 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 77.806729877999999, 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.042230502, 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104112 2023-09-23T17:58:41.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944077+0000", 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944077+0000", 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944076+0000", 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944095+0000", 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8532000000000001e-05 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944103+0000", 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8399999999999995e-06 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944108+0000", 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8510000000000004e-06 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.949704+0000", 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055955110000000001 2023-09-23T17:58:41.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.949880+0000", 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000176312 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.950043+0000", 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016271800000000001 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959270+0000", 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092269080000000007 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959278+0000", 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0590000000000003e-06 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.970705+0000", 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.011427144 2023-09-23T17:58:41.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971279+0000", 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00057403800000000002 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986229+0000", 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014949831 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.881 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986307+0000", 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8319999999999996e-05 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104117 5.a 5:5725f10b:::benchmark_data_smithi055_229117_object104116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.952810+0000", 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 77.797996776999994, 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.039709502, 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.882 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104117 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952810+0000", 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952810+0000", 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952808+0000", 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952833+0000", 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5307999999999999e-05 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952845+0000", 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2145e-05 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952855+0000", 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2730000000000008e-06 2023-09-23T17:58:41.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959408+0000", 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0065535530000000002 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959416+0000", 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9529999999999999e-06 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971413+0000", 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0119968539999999 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971561+0000", 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014827199999999999 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971748+0000", 2023-09-23T17:58:41.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000186701 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986344+0000", 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014596067000000001 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992361+0000", 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060167379999999998 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992422+0000", 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1598000000000002e-05 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992519+0000", 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6966999999999995e-05 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104133 5.a 5:54e6b92a:::benchmark_data_smithi055_229117_object104132:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.987398+0000", 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 77.763408826000003, 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0054708569999999, 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104133 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.890 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987398+0000", 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987398+0000", 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987395+0000", 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987424+0000", 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8217000000000001e-05 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987448+0000", 2023-09-23T17:58:41.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4227e-05 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987458+0000", 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0694e-05 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987514+0000", 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5435e-05 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987524+0000", 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0103e-05 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.974903+0000", 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98737907599999997 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.975050+0000", 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000146692 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.975200+0000", 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015015800000000001 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986377+0000", 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011176962 2023-09-23T17:58:41.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992733+0000", 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063557029999999999 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992783+0000", 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0290999999999997e-05 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992869+0000", 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5835999999999996e-05 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104140 5.a 5:581db13e:::benchmark_data_smithi055_229117_object104139:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.998364+0000", 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 77.752442232000007, 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99464131499999997, 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.897 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104140 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998364+0000", 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.898 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998364+0000", 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998363+0000", 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998390+0000", 2023-09-23T17:58:41.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7362999999999999e-05 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998404+0000", 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4058e-05 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998410+0000", 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0569999999999998e-06 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998457+0000", 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7182999999999999e-05 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998466+0000", 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9940000000000006e-06 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.977786+0000", 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97931971699999998 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.977967+0000", 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000181263 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.978151+0000", 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018419000000000001 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986409+0000", 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082578100000000008 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992877+0000", 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0064680789999999998 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992923+0000", 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5804999999999999e-05 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.993006+0000", 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2649000000000001e-05 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.904 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104141 5.a 5:5985f000:::benchmark_data_smithi055_229117_object104140:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.998421+0000", 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 77.752385564999997, 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99472267199999997, 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104141 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.905 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998421+0000", 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998421+0000", 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998420+0000", 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998450+0000", 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9964e-05 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998457+0000", 2023-09-23T17:58:41.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7769999999999997e-06 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998461+0000", 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1649999999999999e-06 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998503+0000", 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1795999999999998e-05 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.998512+0000", 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5329999999999994e-06 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.981682+0000", 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98317060099999998 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.981856+0000", 2023-09-23T17:58:41.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017383399999999999 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.982035+0000", 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017870399999999999 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986445+0000", 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044101949999999996 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.993014+0000", 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006568679 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.993061+0000", 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6820999999999998e-05 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.993144+0000", 2023-09-23T17:58:41.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3287000000000004e-05 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104161 5.a 5:587441b0:::benchmark_data_smithi055_229117_object104160:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:24.054963+0000", 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 77.695843295000003, 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.93990519500000003, 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:41.912 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104161 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.054963+0000", 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.054963+0000", 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.054961+0000", 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.054995+0000", 2023-09-23T17:58:41.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4359000000000001e-05 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.055022+0000", 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6157e-05 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.055036+0000", 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4671999999999999e-05 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.055101+0000", 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4274000000000002e-05 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.055111+0000", 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0294e-05 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986468+0000", 2023-09-23T17:58:41.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.93135727199999996 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986639+0000", 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017086799999999999 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986812+0000", 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017333600000000001 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.993199+0000", 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063870439999999997 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.994759+0000", 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015594719999999999 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:58:41.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.994804+0000", 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.494e-05 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.994869+0000", 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4791999999999999e-05 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:58:41.919 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:58:41.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:41.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:58:41.921 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:58:41.921 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:41 smithi099 ceph-mon[167076]: pgmap v367: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 430 op/s; 47205/235287 objects degraded (20.063%); 4781/235287 objects misplaced (2.032%); 25 MiB/s, 395 objects/s recovering 2023-09-23T17:58:41.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:41 smithi055 ceph-mon[179668]: pgmap v367: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 430 op/s; 47205/235287 objects degraded (20.063%); 4781/235287 objects misplaced (2.032%); 25 MiB/s, 395 objects/s recovering 2023-09-23T17:58:41.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:41 smithi055 ceph-mon[184174]: pgmap v367: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 430 op/s; 47205/235287 objects degraded (20.063%); 4781/235287 objects misplaced (2.032%); 25 MiB/s, 395 objects/s recovering 2023-09-23T17:58:42.022 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:42.109 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:42.180 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:42.196 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:42.196 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:42.230 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T17:58:42.244 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:42.297 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:42.334 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:42.383 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:42.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:42.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:42.494 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:42.541 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:42.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:42.604 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:42.604 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:42.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:42.785 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:42.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:42 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 45044/230436 objects degraded (19.547%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:42.883 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:42.904 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:42 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 45044/230436 objects degraded (19.547%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:42.905 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:42 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 45044/230436 objects degraded (19.547%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:42.920 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:42.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:42.924 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:43.025 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:43.106 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:43.272 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:43.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:43.291 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:43.391 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:43.498 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:43.586 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:43.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:43.591 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:43.691 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:43.819 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:43 smithi055 ceph-mon[179668]: pgmap v368: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 45044/230436 objects degraded (19.547%); 4544/230436 objects misplaced (1.972%); 35 MiB/s, 563 objects/s recovering 2023-09-23T17:58:43.819 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:43 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 1 pg inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:58:43.819 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:43 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:58:43.820 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:43 smithi055 ceph-mon[184174]: pgmap v368: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 45044/230436 objects degraded (19.547%); 4544/230436 objects misplaced (1.972%); 35 MiB/s, 563 objects/s recovering 2023-09-23T17:58:43.820 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:43 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 1 pg inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:58:43.820 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:43 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:58:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:43 smithi099 ceph-mon[167076]: pgmap v368: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 576 op/s; 45044/230436 objects degraded (19.547%); 4544/230436 objects misplaced (1.972%); 35 MiB/s, 563 objects/s recovering 2023-09-23T17:58:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:43 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 1 pg inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:58:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:43 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:58:43.838 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:43.917 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:43.917 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:58:43.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:43.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:43.969 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:58:44.000 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:58:44.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:44.061 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:58:44.081 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:44.102 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:58:44.137 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:58:44.144 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:58:44.144 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:58:44.144 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:58:44.144 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:58:44.144 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:58:44.144 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:58:44.144 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T17:58:44.145 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:58:44.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:44.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:44.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:44.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:44.331 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:58:44.331 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:58:44.358 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:44.449 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T17:58:44.449 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:58:44.449 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T17:58:44.449 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:58:44.525 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:44.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:44.529 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:44.630 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:44.690 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:58:44.691 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:58:44.721 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T17:58:44.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2463873119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:44.754 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3506580314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:44.754 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:44 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:44.754 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2463873119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:44.754 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3506580314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:44.754 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:44 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:44.768 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:44.820 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T17:58:44.859 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:44.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:44.870 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:44.971 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:45.065 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:45.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2463873119' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:45.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3506580314' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:45.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:44 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:58:45.231 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:45.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:45.242 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:45.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:45.389 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:45.430 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:45.435 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:45.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:45.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:45.543 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:45.599 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:45.628 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:45.777 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:45.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:45.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:45.884 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:45.890 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:45 smithi055 ceph-mon[184174]: pgmap v369: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 43373/224160 objects degraded (19.349%); 4544/224160 objects misplaced (2.027%); 33 MiB/s, 525 objects/s recovering 2023-09-23T17:58:45.890 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:45 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:58:45.891 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:45 smithi055 ceph-mon[184174]: failed to encode map e167 with expected crc 2023-09-23T17:58:45.891 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:45 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:45.891 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:45 smithi055 ceph-mon[184174]: osdmap e167: 8 total, 6 up, 8 in 2023-09-23T17:58:45.891 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:45 smithi055 ceph-mon[179668]: pgmap v369: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 43373/224160 objects degraded (19.349%); 4544/224160 objects misplaced (2.027%); 33 MiB/s, 525 objects/s recovering 2023-09-23T17:58:45.891 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:45 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:58:45.891 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:45 smithi055 ceph-mon[179668]: failed to encode map e167 with expected crc 2023-09-23T17:58:45.891 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:45 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:45.891 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:45 smithi055 ceph-mon[179668]: osdmap e167: 8 total, 6 up, 8 in 2023-09-23T17:58:45.968 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:46.017 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:45 smithi099 ceph-mon[167076]: pgmap v369: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.6 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 715 op/s; 43373/224160 objects degraded (19.349%); 4544/224160 objects misplaced (2.027%); 33 MiB/s, 525 objects/s recovering 2023-09-23T17:58:46.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:45 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:58:46.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:45 smithi099 ceph-mon[167076]: failed to encode map e167 with expected crc 2023-09-23T17:58:46.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:45 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:58:46.018 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:45 smithi099 ceph-mon[167076]: osdmap e167: 8 total, 6 up, 8 in 2023-09-23T17:58:46.062 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:46.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:46.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:46.227 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:46.339 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:46.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:46.346 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:46.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:46.513 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:46.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:46.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:46.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:46.692 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:58:46.736 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:46.808 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:47.008 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:47.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:47.020 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:47.120 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:47.195 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:47.198 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:58:47.198 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:58:47.237 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T17:58:47.313 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:47.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:47.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:47.422 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:47.530 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:47.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T17:58:47.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:47 smithi055 ceph-mon[179668]: pgmap v371: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 43146/222837 objects degraded (19.362%); 4544/222837 objects misplaced (2.039%); 26 MiB/s, 423 objects/s recovering 2023-09-23T17:58:47.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:47 smithi099 ceph-mon[167076]: pgmap v371: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 43146/222837 objects degraded (19.362%); 4544/222837 objects misplaced (2.039%); 26 MiB/s, 423 objects/s recovering 2023-09-23T17:58:47.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:47 smithi055 ceph-mon[184174]: pgmap v371: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 683 op/s; 43146/222837 objects degraded (19.362%); 4544/222837 objects misplaced (2.039%); 26 MiB/s, 423 objects/s recovering 2023-09-23T17:58:48.278 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:48.353 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:48.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:48.359 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:48.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:48.499 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:48.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:48.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:48.562 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:48.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:58:48.713 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:48.714 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:48.820 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:48.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 43146/222837 objects degraded (19.362%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:48.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:58:48.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:48.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:58:49.039 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:49.112 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 43146/222837 objects degraded (19.362%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:49.115 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 43146/222837 objects degraded (19.362%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:49.125 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:49.306 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:49.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:49.327 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:49.428 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:49.554 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:58:49.554 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:49.576 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:58:49.821 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:49.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:49.830 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:49.931 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:50.012 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:50.091 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:49 smithi055 ceph-mon[184174]: pgmap v372: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 41620/219468 objects degraded (18.964%); 4308/219468 objects misplaced (1.963%); 29 MiB/s, 469 objects/s recovering 2023-09-23T17:58:50.093 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:49 smithi055 ceph-mon[179668]: pgmap v372: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 41620/219468 objects degraded (18.964%); 4308/219468 objects misplaced (1.963%); 29 MiB/s, 469 objects/s recovering 2023-09-23T17:58:50.175 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:58:50.175 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:58:50.202 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:49 smithi099 ceph-mon[167076]: pgmap v372: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 675 op/s; 41620/219468 objects degraded (18.964%); 4308/219468 objects misplaced (1.963%); 29 MiB/s, 469 objects/s recovering 2023-09-23T17:58:50.274 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":372,"stamp":"2023-09-23T17:58:48.511604+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4797241291,"num_objects":73156,"num_object_clones":0,"num_object_copies":219468,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":41620,"num_objects_misplaced":4308,"num_objects_unfound":0,"num_objects_dirty":73156,"num_whiteouts":0,"num_read":6481,"num_read_kb":13101,"num_write":139266,"num_write_kb":6775014,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25079,"num_bytes_recovered":1644300618,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":121869,"ondisk_log_size":121869,"up":239,"acting":240,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":331,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15618780,"kb_used_data":14883292,"kb_used_omap":36,"kb_used_meta":735195,"kb_avail":734342436,"statfs":{"total":767960285184,"available":751966654464,"internally_reserved":0,"allocated":15240491008,"data_stored":15234572052,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37596,"internal_metadata":752839972},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,2,2,3,6,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":11,"apply_latency_ms":11,"commit_latency_ns":11000000,"apply_latency_ns":11000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-442630144,"num_objects":-6754,"num_object_clones":0,"num_object_copies":-20262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-2772,"num_objects_misplaced":-473,"num_objects_unfound":0,"num_objects_dirty":-6754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6754,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4694,"num_bytes_recovered":307625984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004424"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":50,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.267182+0000","last_change":"2023-09-23T17:56:50.838338+0000","last_active":"2023-09-23T17:58:38.267182+0000","last_peered":"2023-09-23T17:58:38.267182+0000","last_clean":"2023-09-23T17:58:38.267182+0000","last_became_active":"2023-09-23T17:56:50.837963+0000","last_became_peered":"2023-09-23T17:56:50.837963+0000","last_unstale":"2023-09-23T17:58:38.267182+0000","last_undegraded":"2023-09-23T17:58:38.267182+0000","last_fullsized":"2023-09-23T17:58:38.267182+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:36.238235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":132,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.273904+0000","last_change":"2023-09-23T17:57:24.956156+0000","last_active":"2023-09-23T17:58:38.273904+0000","last_peered":"2023-09-23T17:58:38.273904+0000","last_clean":"2023-09-23T17:56:46.613857+0000","last_became_active":"2023-09-23T17:57:24.956156+0000","last_became_peered":"2023-09-23T17:57:24.956156+0000","last_unstale":"2023-09-23T17:58:38.273904+0000","last_undegraded":"2023-09-23T17:58:38.273904+0000","last_fullsized":"2023-09-23T17:57:24.947251+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:35.949179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":131,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:40.876015+0000","last_change":"2023-09-23T17:57:24.956353+0000","last_active":"2023-09-23T17:58:40.876015+0000","last_peered":"2023-09-23T17:58:40.876015+0000","last_clean":"2023-09-23T17:57:20.922940+0000","last_became_active":"2023-09-23T17:57:24.956353+0000","last_became_peered":"2023-09-23T17:57:24.956353+0000","last_unstale":"2023-09-23T17:58:40.876015+0000","last_undegraded":"2023-09-23T17:58:40.876015+0000","last_fullsized":"2023-09-23T17:57:24.946863+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:41.136418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":127,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.244736+0000","last_change":"2023-09-23T17:58:11.996431+0000","last_active":"2023-09-23T17:58:36.244736+0000","last_peered":"2023-09-23T17:58:36.244736+0000","last_clean":"2023-09-23T17:57:20.931176+0000","last_became_active":"2023-09-23T17:58:11.996431+0000","last_became_peered":"2023-09-23T17:58:11.996431+0000","last_unstale":"2023-09-23T17:58:36.244736+0000","last_undegraded":"2023-09-23T17:58:36.244736+0000","last_fullsized":"2023-09-23T17:58:11.991094+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:21.919913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":112,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.245128+0000","last_change":"2023-09-23T17:57:24.965072+0000","last_active":"2023-09-23T17:58:36.245128+0000","last_peered":"2023-09-23T17:58:36.245128+0000","last_clean":"2023-09-23T17:53:21.039477+0000","last_became_active":"2023-09-23T17:57:24.965072+0000","last_became_peered":"2023-09-23T17:57:24.965072+0000","last_unstale":"2023-09-23T17:58:36.245128+0000","last_undegraded":"2023-09-23T17:58:36.245128+0000","last_fullsized":"2023-09-23T17:57:24.947630+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:41.627468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":77,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.266522+0000","last_change":"2023-09-23T17:58:35.243749+0000","last_active":"2023-09-23T17:58:38.266522+0000","last_peered":"2023-09-23T17:58:38.266522+0000","last_clean":"2023-09-23T17:58:38.266522+0000","last_became_active":"2023-09-23T17:58:35.243446+0000","last_became_peered":"2023-09-23T17:58:35.243446+0000","last_unstale":"2023-09-23T17:58:38.266522+0000","last_undegraded":"2023-09-23T17:58:38.266522+0000","last_fullsized":"2023-09-23T17:58:38.266522+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:28:02.984085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":115,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.244919+0000","last_change":"2023-09-23T17:57:24.957628+0000","last_active":"2023-09-23T17:58:36.244919+0000","last_peered":"2023-09-23T17:58:36.244919+0000","last_clean":"2023-09-23T17:57:20.932733+0000","last_became_active":"2023-09-23T17:57:24.957628+0000","last_became_peered":"2023-09-23T17:57:24.957628+0000","last_unstale":"2023-09-23T17:58:36.244919+0000","last_undegraded":"2023-09-23T17:58:36.244919+0000","last_fullsized":"2023-09-23T17:57:24.946783+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:51:10.900318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":115,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.279729+0000","last_change":"2023-09-23T17:58:35.245269+0000","last_active":"2023-09-23T17:58:38.279729+0000","last_peered":"2023-09-23T17:58:38.279729+0000","last_clean":"2023-09-23T17:55:56.840611+0000","last_became_active":"2023-09-23T17:58:35.245269+0000","last_became_peered":"2023-09-23T17:58:35.245269+0000","last_unstale":"2023-09-23T17:58:38.279729+0000","last_undegraded":"2023-09-23T17:58:38.279729+0000","last_fullsized":"2023-09-23T17:58:35.227362+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:14:50.366063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":134,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.272691+0000","last_change":"2023-09-23T17:57:53.878552+0000","last_active":"2023-09-23T17:58:38.272691+0000","last_peered":"2023-09-23T17:58:38.272691+0000","last_clean":"2023-09-23T17:57:47.590780+0000","last_became_active":"2023-09-23T17:57:53.878552+0000","last_became_peered":"2023-09-23T17:57:53.878552+0000","last_unstale":"2023-09-23T17:58:38.272691+0000","last_undegraded":"2023-09-23T17:58:38.272691+0000","last_fullsized":"2023-09-23T17:57:53.870081+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:44.428373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":23,"reported_epoch":166,"state":"down","last_fresh":"2023-09-23T17:58:38.264871+0000","last_change":"2023-09-23T17:58:34.212200+0000","last_active":"2023-09-23T17:45:51.086592+0000","last_peered":"2023-09-23T17:45:51.086592+0000","last_clean":"2023-09-23T17:45:51.086592+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T17:58:38.264871+0000","last_undegraded":"2023-09-23T17:58:38.264871+0000","last_fullsized":"2023-09-23T17:58:38.264871+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:32.367332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":147,"reported_epoch":166,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:38.269967+0000","last_change":"2023-09-23T17:58:12.002185+0000","last_active":"2023-09-23T17:58:38.269967+0000","last_peered":"2023-09-23T17:58:38.269967+0000","last_clean":"2023-09-23T17:56:46.621999+0000","last_became_active":"2023-09-23T17:58:11.996011+0000","last_became_peered":"2023-09-23T17:58:11.996011+0000","last_unstale":"2023-09-23T17:58:38.269967+0000","last_undegraded":"2023-09-23T17:58:11.991073+0000","last_fullsized":"2023-09-23T17:58:11.990857+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:23.701932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":122,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.272120+0000","last_change":"2023-09-23T17:58:35.248074+0000","last_active":"2023-09-23T17:58:38.272120+0000","last_peered":"2023-09-23T17:58:38.272120+0000","last_clean":"2023-09-23T17:58:38.272120+0000","last_became_active":"2023-09-23T17:58:35.247751+0000","last_became_peered":"2023-09-23T17:58:35.247751+0000","last_unstale":"2023-09-23T17:58:38.272120+0000","last_undegraded":"2023-09-23T17:58:38.272120+0000","last_fullsized":"2023-09-23T17:58:38.272120+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:08.488838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":164,"reported_epoch":166,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:58:38.267740+0000","last_change":"2023-09-23T17:58:36.257651+0000","last_active":"2023-09-23T17:58:38.267740+0000","last_peered":"2023-09-23T17:58:38.267740+0000","last_clean":"2023-09-23T17:58:38.267740+0000","last_became_active":"2023-09-23T17:58:36.257192+0000","last_became_peered":"2023-09-23T17:58:36.257192+0000","last_unstale":"2023-09-23T17:58:38.267740+0000","last_undegraded":"2023-09-23T17:58:38.267740+0000","last_fullsized":"2023-09-23T17:58:38.267740+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:26.513056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":151,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.267623+0000","last_change":"2023-09-23T17:58:36.259387+0000","last_active":"2023-09-23T17:58:38.267623+0000","last_peered":"2023-09-23T17:58:38.267623+0000","last_clean":"2023-09-23T17:58:38.267623+0000","last_became_active":"2023-09-23T17:58:36.259045+0000","last_became_peered":"2023-09-23T17:58:36.259045+0000","last_unstale":"2023-09-23T17:58:38.267623+0000","last_undegraded":"2023-09-23T17:58:38.267623+0000","last_fullsized":"2023-09-23T17:58:38.267623+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:11.094820+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":37,"reported_epoch":166,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:38.267718+0000","last_change":"2023-09-23T17:57:53.876836+0000","last_active":"2023-09-23T17:58:38.267718+0000","last_peered":"2023-09-23T17:58:38.267718+0000","last_clean":"2023-09-23T17:45:54.586105+0000","last_became_active":"2023-09-23T17:57:53.876836+0000","last_became_peered":"2023-09-23T17:57:53.876836+0000","last_unstale":"2023-09-23T17:58:38.267718+0000","last_undegraded":"2023-09-23T17:57:53.867857+0000","last_fullsized":"2023-09-23T17:57:53.867732+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:16.273804+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":100,"reported_epoch":166,"state":"undersized+peered","last_fresh":"2023-09-23T17:58:38.273036+0000","last_change":"2023-09-23T17:57:53.869041+0000","last_active":"2023-09-23T17:57:52.875213+0000","last_peered":"2023-09-23T17:58:38.273036+0000","last_clean":"2023-09-23T17:53:21.028332+0000","last_became_active":"2023-09-23T17:57:24.954701+0000","last_became_peered":"2023-09-23T17:57:53.869041+0000","last_unstale":"2023-09-23T17:58:38.273036+0000","last_undegraded":"2023-09-23T17:58:38.273036+0000","last_fullsized":"2023-09-23T17:57:53.867092+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:44.473315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":126,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.267096+0000","last_change":"2023-09-23T17:58:11.995550+0000","last_active":"2023-09-23T17:58:36.267096+0000","last_peered":"2023-09-23T17:58:36.267096+0000","last_clean":"2023-09-23T17:57:20.931103+0000","last_became_active":"2023-09-23T17:58:11.995550+0000","last_became_peered":"2023-09-23T17:58:11.995550+0000","last_unstale":"2023-09-23T17:58:36.267096+0000","last_undegraded":"2023-09-23T17:58:36.267096+0000","last_fullsized":"2023-09-23T17:58:11.986762+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:31.912263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":139,"reported_epoch":166,"state":"undersized+peered","last_fresh":"2023-09-23T17:58:38.271893+0000","last_change":"2023-09-23T17:57:53.872794+0000","last_active":"2023-09-23T17:57:52.888650+0000","last_peered":"2023-09-23T17:58:38.271893+0000","last_clean":"2023-09-23T17:57:20.925545+0000","last_became_active":"2023-09-23T17:57:24.954648+0000","last_became_peered":"2023-09-23T17:57:53.872794+0000","last_unstale":"2023-09-23T17:58:38.271893+0000","last_undegraded":"2023-09-23T17:58:38.271893+0000","last_fullsized":"2023-09-23T17:57:53.870598+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:11.091483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5],"acting":[5],"avail_no_missing":["5"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":154,"reported_epoch":166,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:58:38.270993+0000","last_change":"2023-09-23T17:58:36.251995+0000","last_active":"2023-09-23T17:58:38.270993+0000","last_peered":"2023-09-23T17:58:38.270993+0000","last_clean":"2023-09-23T17:58:38.270993+0000","last_became_active":"2023-09-23T17:58:36.251578+0000","last_became_peered":"2023-09-23T17:58:36.251578+0000","last_unstale":"2023-09-23T17:58:38.270993+0000","last_undegraded":"2023-09-23T17:58:38.270993+0000","last_fullsized":"2023-09-23T17:58:38.270993+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:45.817777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,0],"avail_no_missing":["4","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":127,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.244534+0000","last_change":"2023-09-23T17:58:11.996093+0000","last_active":"2023-09-23T17:58:36.244534+0000","last_peered":"2023-09-23T17:58:36.244534+0000","last_clean":"2023-09-23T17:57:20.931097+0000","last_became_active":"2023-09-23T17:58:11.996093+0000","last_became_peered":"2023-09-23T17:58:11.996093+0000","last_unstale":"2023-09-23T17:58:36.244534+0000","last_undegraded":"2023-09-23T17:58:36.244534+0000","last_fullsized":"2023-09-23T17:58:11.990799+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:58.128284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":81,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.264651+0000","last_change":"2023-09-23T17:58:35.251246+0000","last_active":"2023-09-23T17:58:38.264651+0000","last_peered":"2023-09-23T17:58:38.264651+0000","last_clean":"2023-09-23T17:58:38.264651+0000","last_became_active":"2023-09-23T17:58:35.250891+0000","last_became_peered":"2023-09-23T17:58:35.250891+0000","last_unstale":"2023-09-23T17:58:38.264651+0000","last_undegraded":"2023-09-23T17:58:38.264651+0000","last_fullsized":"2023-09-23T17:58:38.264651+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:41.750282+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":106,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.274170+0000","last_change":"2023-09-23T17:58:11.993687+0000","last_active":"2023-09-23T17:58:38.274170+0000","last_peered":"2023-09-23T17:58:38.274170+0000","last_clean":"2023-09-23T17:53:21.029434+0000","last_became_active":"2023-09-23T17:58:11.993687+0000","last_became_peered":"2023-09-23T17:58:11.993687+0000","last_unstale":"2023-09-23T17:58:38.274170+0000","last_undegraded":"2023-09-23T17:58:38.274170+0000","last_fullsized":"2023-09-23T17:58:11.985535+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:33.153133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.9","version":"167'8516","reported_seq":24562,"reported_epoch":167,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2023-09-23T17:58:47.411352+0000","last_change":"2023-09-23T17:58:35.333845+0000","last_active":"2023-09-23T17:58:47.411352+0000","last_peered":"2023-09-23T17:58:47.411352+0000","last_clean":"2023-09-23T17:55:57.822204+0000","last_became_active":"2023-09-23T17:58:35.311173+0000","last_became_peered":"2023-09-23T17:58:35.311173+0000","last_unstale":"2023-09-23T17:58:47.411352+0000","last_undegraded":"2023-09-23T17:58:35.251890+0000","last_fullsized":"2023-09-23T17:58:35.233548+0000","mapping_epoch":163,"log_start":"116'1285","ondisk_log_start":"116'1285","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":7231,"log_dups_size":0,"ondisk_log_size":7231,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:03.964483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282066944,"num_objects":4304,"num_object_clones":0,"num_object_copies":12912,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":890,"num_objects_misplaced":4304,"num_objects_unfound":0,"num_objects_dirty":4304,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8516,"num_write_kb":410240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7966,"num_bytes_recovered":521994240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":4304}],"blocked_by":[],"up_primary":6,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":131,"reported_epoch":166,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:38.268022+0000","last_change":"2023-09-23T17:57:53.875345+0000","last_active":"2023-09-23T17:58:38.268022+0000","last_peered":"2023-09-23T17:58:38.268022+0000","last_clean":"2023-09-23T17:56:46.621793+0000","last_became_active":"2023-09-23T17:57:53.875345+0000","last_became_peered":"2023-09-23T17:57:53.875345+0000","last_unstale":"2023-09-23T17:58:38.268022+0000","last_undegraded":"2023-09-23T17:57:53.869204+0000","last_fullsized":"2023-09-23T17:57:53.869091+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:19.033883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":139,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.271683+0000","last_change":"2023-09-23T17:58:35.237277+0000","last_active":"2023-09-23T17:58:38.271683+0000","last_peered":"2023-09-23T17:58:38.271683+0000","last_clean":"2023-09-23T17:58:38.271683+0000","last_became_active":"2023-09-23T17:58:35.236973+0000","last_became_peered":"2023-09-23T17:58:35.236973+0000","last_unstale":"2023-09-23T17:58:38.271683+0000","last_undegraded":"2023-09-23T17:58:38.271683+0000","last_fullsized":"2023-09-23T17:58:38.271683+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:05.742958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"166'8615","reported_seq":8681,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:43.741408+0000","last_change":"2023-09-23T17:55:53.818806+0000","last_active":"2023-09-23T17:58:43.741408+0000","last_peered":"2023-09-23T17:58:43.741408+0000","last_clean":"2023-09-23T17:58:43.741408+0000","last_became_active":"2023-09-23T17:55:53.818291+0000","last_became_peered":"2023-09-23T17:55:53.818291+0000","last_unstale":"2023-09-23T17:58:43.741408+0000","last_undegraded":"2023-09-23T17:58:43.741408+0000","last_fullsized":"2023-09-23T17:58:43.741408+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":8615,"log_dups_size":0,"ondisk_log_size":8615,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:58.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306905088,"num_objects":4683,"num_object_clones":0,"num_object_copies":14049,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4683,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8615,"num_write_kb":425536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":132,"reported_epoch":166,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:38.268597+0000","last_change":"2023-09-23T17:57:53.878631+0000","last_active":"2023-09-23T17:58:38.268597+0000","last_peered":"2023-09-23T17:58:38.268597+0000","last_clean":"2023-09-23T17:56:46.621682+0000","last_became_active":"2023-09-23T17:57:53.878631+0000","last_became_peered":"2023-09-23T17:57:53.878631+0000","last_unstale":"2023-09-23T17:58:38.268597+0000","last_undegraded":"2023-09-23T17:57:53.867937+0000","last_fullsized":"2023-09-23T17:57:53.867822+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:53.176515+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":156,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.270950+0000","last_change":"2023-09-23T17:57:24.951906+0000","last_active":"2023-09-23T17:58:38.270950+0000","last_peered":"2023-09-23T17:58:38.270950+0000","last_clean":"2023-09-23T17:57:20.922147+0000","last_became_active":"2023-09-23T17:57:24.951906+0000","last_became_peered":"2023-09-23T17:57:24.951906+0000","last_unstale":"2023-09-23T17:58:38.270950+0000","last_undegraded":"2023-09-23T17:58:38.270950+0000","last_fullsized":"2023-09-23T17:57:24.945982+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:10.143229+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"166'8505","reported_seq":8570,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:43.774575+0000","last_change":"2023-09-23T17:55:53.829853+0000","last_active":"2023-09-23T17:58:43.774575+0000","last_peered":"2023-09-23T17:58:43.774575+0000","last_clean":"2023-09-23T17:58:43.774575+0000","last_became_active":"2023-09-23T17:55:53.829573+0000","last_became_peered":"2023-09-23T17:55:53.829573+0000","last_unstale":"2023-09-23T17:58:43.774575+0000","last_undegraded":"2023-09-23T17:58:43.774575+0000","last_fullsized":"2023-09-23T17:58:43.774575+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":8505,"log_dups_size":0,"ondisk_log_size":8505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:39.218789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305725440,"num_objects":4665,"num_object_clones":0,"num_object_copies":13995,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4665,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8505,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":43,"reported_epoch":166,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:38.271886+0000","last_change":"2023-09-23T17:58:35.236498+0000","last_active":"2023-09-23T17:58:38.271886+0000","last_peered":"2023-09-23T17:58:38.271886+0000","last_clean":"2023-09-23T17:45:54.586241+0000","last_became_active":"2023-09-23T17:58:35.236498+0000","last_became_peered":"2023-09-23T17:58:35.236498+0000","last_unstale":"2023-09-23T17:58:38.271886+0000","last_undegraded":"2023-09-23T17:58:35.230491+0000","last_fullsized":"2023-09-23T17:58:35.230386+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:06.423143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":92,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.265810+0000","last_change":"2023-09-23T17:58:35.245786+0000","last_active":"2023-09-23T17:58:38.265810+0000","last_peered":"2023-09-23T17:58:38.265810+0000","last_clean":"2023-09-23T17:58:38.265810+0000","last_became_active":"2023-09-23T17:58:35.245496+0000","last_became_peered":"2023-09-23T17:58:35.245496+0000","last_unstale":"2023-09-23T17:58:38.265810+0000","last_undegraded":"2023-09-23T17:58:38.265810+0000","last_fullsized":"2023-09-23T17:58:38.265810+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:01.315248+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,5],"acting":[6,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"166'8543","reported_seq":9433,"reported_epoch":166,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:43.774228+0000","last_change":"2023-09-23T17:57:24.992424+0000","last_active":"2023-09-23T17:58:43.774228+0000","last_peered":"2023-09-23T17:58:43.774228+0000","last_clean":"2023-09-23T17:57:23.931663+0000","last_became_active":"2023-09-23T17:57:24.964668+0000","last_became_peered":"2023-09-23T17:57:24.964668+0000","last_unstale":"2023-09-23T17:58:43.774228+0000","last_undegraded":"2023-09-23T17:57:24.952679+0000","last_fullsized":"2023-09-23T17:57:24.947063+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":8543,"log_dups_size":0,"ondisk_log_size":8543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:59.839324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":301924352,"num_objects":4607,"num_object_clones":0,"num_object_copies":13821,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4607,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4607,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8543,"num_write_kb":420800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":396,"num_bytes_recovered":25952256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":4607}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":147,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.271848+0000","last_change":"2023-09-23T17:58:35.236466+0000","last_active":"2023-09-23T17:58:38.271848+0000","last_peered":"2023-09-23T17:58:38.271848+0000","last_clean":"2023-09-23T17:55:56.845943+0000","last_became_active":"2023-09-23T17:58:35.236466+0000","last_became_peered":"2023-09-23T17:58:35.236466+0000","last_unstale":"2023-09-23T17:58:38.271848+0000","last_undegraded":"2023-09-23T17:58:38.271848+0000","last_fullsized":"2023-09-23T17:58:35.229132+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:02.127157+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":128,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.245670+0000","last_change":"2023-09-23T17:56:50.828739+0000","last_active":"2023-09-23T17:58:36.245670+0000","last_peered":"2023-09-23T17:58:36.245670+0000","last_clean":"2023-09-23T17:58:36.245670+0000","last_became_active":"2023-09-23T17:56:50.828229+0000","last_became_peered":"2023-09-23T17:56:50.828229+0000","last_unstale":"2023-09-23T17:58:36.245670+0000","last_undegraded":"2023-09-23T17:58:36.245670+0000","last_fullsized":"2023-09-23T17:58:36.245670+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:45.927953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":452,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.243658+0000","last_change":"2023-09-23T17:58:35.252704+0000","last_active":"2023-09-23T17:58:36.243658+0000","last_peered":"2023-09-23T17:58:36.243658+0000","last_clean":"2023-09-23T17:58:36.243658+0000","last_became_active":"2023-09-23T17:58:35.252386+0000","last_became_peered":"2023-09-23T17:58:35.252386+0000","last_unstale":"2023-09-23T17:58:36.243658+0000","last_undegraded":"2023-09-23T17:58:36.243658+0000","last_fullsized":"2023-09-23T17:58:36.243658+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"166'8548","reported_seq":8619,"reported_epoch":166,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:43.798324+0000","last_change":"2023-09-23T17:57:53.880002+0000","last_active":"2023-09-23T17:58:43.798324+0000","last_peered":"2023-09-23T17:58:43.798324+0000","last_clean":"2023-09-23T17:57:52.866355+0000","last_became_active":"2023-09-23T17:57:53.880002+0000","last_became_peered":"2023-09-23T17:57:53.880002+0000","last_unstale":"2023-09-23T17:58:43.798324+0000","last_undegraded":"2023-09-23T17:57:53.873996+0000","last_fullsized":"2023-09-23T17:57:53.873843+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":8548,"log_dups_size":0,"ondisk_log_size":8548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:08.158258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300482584,"num_objects":4586,"num_object_clones":0,"num_object_copies":13758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4586,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4586,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8548,"num_write_kb":420225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":4586}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":51,"reported_epoch":165,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:36.246252+0000","last_change":"2023-09-23T17:57:24.957601+0000","last_active":"2023-09-23T17:58:36.246252+0000","last_peered":"2023-09-23T17:58:36.246252+0000","last_clean":"2023-09-23T17:45:54.465462+0000","last_became_active":"2023-09-23T17:57:24.957601+0000","last_became_peered":"2023-09-23T17:57:24.957601+0000","last_unstale":"2023-09-23T17:58:36.246252+0000","last_undegraded":"2023-09-23T17:57:24.946354+0000","last_fullsized":"2023-09-23T17:57:24.946239+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.908857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":158,"reported_epoch":166,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:58:38.267542+0000","last_change":"2023-09-23T17:58:36.259032+0000","last_active":"2023-09-23T17:58:38.267542+0000","last_peered":"2023-09-23T17:58:38.267542+0000","last_clean":"2023-09-23T17:58:38.267542+0000","last_became_active":"2023-09-23T17:58:36.258634+0000","last_became_peered":"2023-09-23T17:58:36.258634+0000","last_unstale":"2023-09-23T17:58:38.267542+0000","last_undegraded":"2023-09-23T17:58:38.267542+0000","last_fullsized":"2023-09-23T17:58:38.267542+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:59.171893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,1],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":456,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.243331+0000","last_change":"2023-09-23T17:58:35.251808+0000","last_active":"2023-09-23T17:58:36.243331+0000","last_peered":"2023-09-23T17:58:36.243331+0000","last_clean":"2023-09-23T17:58:36.243331+0000","last_became_active":"2023-09-23T17:58:35.251267+0000","last_became_peered":"2023-09-23T17:58:35.251267+0000","last_unstale":"2023-09-23T17:58:36.243331+0000","last_undegraded":"2023-09-23T17:58:36.243331+0000","last_fullsized":"2023-09-23T17:58:36.243331+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.c","version":"167'8614","reported_seq":8699,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T17:58:45.838351+0000","last_change":"2023-09-23T17:56:51.840960+0000","last_active":"2023-09-23T17:58:45.838351+0000","last_peered":"2023-09-23T17:58:45.838351+0000","last_clean":"2023-09-23T17:58:45.838351+0000","last_became_active":"2023-09-23T17:56:51.835697+0000","last_became_peered":"2023-09-23T17:56:51.835697+0000","last_unstale":"2023-09-23T17:58:45.838351+0000","last_undegraded":"2023-09-23T17:58:45.838351+0000","last_fullsized":"2023-09-23T17:58:45.838351+0000","mapping_epoch":127,"log_start":"118'3093","ondisk_log_start":"118'3093","created":110,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5521,"log_dups_size":0,"ondisk_log_size":5521,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:45.804116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300417024,"num_objects":4584,"num_object_clones":0,"num_object_copies":13752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8614,"num_write_kb":422336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":78,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.266044+0000","last_change":"2023-09-23T17:58:35.243177+0000","last_active":"2023-09-23T17:58:38.266044+0000","last_peered":"2023-09-23T17:58:38.266044+0000","last_clean":"2023-09-23T17:58:38.266044+0000","last_became_active":"2023-09-23T17:58:35.242824+0000","last_became_peered":"2023-09-23T17:58:35.242824+0000","last_unstale":"2023-09-23T17:58:38.266044+0000","last_undegraded":"2023-09-23T17:58:38.266044+0000","last_fullsized":"2023-09-23T17:58:38.266044+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:10.250046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":142,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.270077+0000","last_change":"2023-09-23T17:56:51.838748+0000","last_active":"2023-09-23T17:58:38.270077+0000","last_peered":"2023-09-23T17:58:38.270077+0000","last_clean":"2023-09-23T17:58:38.270077+0000","last_became_active":"2023-09-23T17:56:51.838511+0000","last_became_peered":"2023-09-23T17:56:51.838511+0000","last_unstale":"2023-09-23T17:58:38.270077+0000","last_undegraded":"2023-09-23T17:58:38.270077+0000","last_fullsized":"2023-09-23T17:58:38.270077+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:00.929947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"167'8704","reported_seq":15267,"reported_epoch":167,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T17:58:47.313723+0000","last_change":"2023-09-23T17:58:36.481644+0000","last_active":"2023-09-23T17:58:47.313723+0000","last_peered":"2023-09-23T17:58:47.313723+0000","last_clean":"2023-09-23T17:55:56.846049+0000","last_became_active":"2023-09-23T17:58:36.457601+0000","last_became_peered":"2023-09-23T17:58:36.457601+0000","last_unstale":"2023-09-23T17:58:47.313723+0000","last_undegraded":"2023-09-23T17:58:36.278616+0000","last_fullsized":"2023-09-23T17:58:36.241589+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":8704,"log_dups_size":0,"ondisk_log_size":8704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:14:44.421690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293732352,"num_objects":4482,"num_object_clones":0,"num_object_copies":13446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6591,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8704,"num_write_kb":421952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3229,"num_bytes_recovered":211484672,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[{"shards":"2,4,5","objects":3284},{"shards":"4,5","objects":-475}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":57,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.270713+0000","last_change":"2023-09-23T17:57:53.880278+0000","last_active":"2023-09-23T17:58:38.270713+0000","last_peered":"2023-09-23T17:58:38.270713+0000","last_clean":"2023-09-23T17:57:47.586782+0000","last_became_active":"2023-09-23T17:57:53.880278+0000","last_became_peered":"2023-09-23T17:57:53.880278+0000","last_unstale":"2023-09-23T17:58:38.270713+0000","last_undegraded":"2023-09-23T17:58:38.270713+0000","last_fullsized":"2023-09-23T17:57:53.867145+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:07.345156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":156,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.268765+0000","last_change":"2023-09-23T17:58:35.247231+0000","last_active":"2023-09-23T17:58:38.268765+0000","last_peered":"2023-09-23T17:58:38.268765+0000","last_clean":"2023-09-23T17:58:38.268765+0000","last_became_active":"2023-09-23T17:58:35.246934+0000","last_became_peered":"2023-09-23T17:58:35.246934+0000","last_unstale":"2023-09-23T17:58:38.268765+0000","last_undegraded":"2023-09-23T17:58:38.268765+0000","last_fullsized":"2023-09-23T17:58:38.268765+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:16:17.387084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":452,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.245422+0000","last_change":"2023-09-23T17:58:35.250653+0000","last_active":"2023-09-23T17:58:36.245422+0000","last_peered":"2023-09-23T17:58:36.245422+0000","last_clean":"2023-09-23T17:58:36.245422+0000","last_became_active":"2023-09-23T17:58:35.249513+0000","last_became_peered":"2023-09-23T17:58:35.249513+0000","last_unstale":"2023-09-23T17:58:36.245422+0000","last_undegraded":"2023-09-23T17:58:36.245422+0000","last_fullsized":"2023-09-23T17:58:36.245422+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"167'8706","reported_seq":9630,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T17:58:47.393272+0000","last_change":"2023-09-23T17:56:56.943738+0000","last_active":"2023-09-23T17:58:47.393272+0000","last_peered":"2023-09-23T17:58:47.393272+0000","last_clean":"2023-09-23T17:58:47.393272+0000","last_became_active":"2023-09-23T17:56:56.903740+0000","last_became_peered":"2023-09-23T17:56:56.903740+0000","last_unstale":"2023-09-23T17:58:47.393272+0000","last_undegraded":"2023-09-23T17:58:47.393272+0000","last_fullsized":"2023-09-23T17:58:47.393272+0000","mapping_epoch":131,"log_start":"116'1540","ondisk_log_start":"116'1540","created":110,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":7166,"log_dups_size":0,"ondisk_log_size":7166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:10.789289+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293863424,"num_objects":4484,"num_object_clones":0,"num_object_copies":13452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8706,"num_write_kb":422080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":414,"num_bytes_recovered":27131904,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":101,"reported_epoch":166,"state":"undersized+peered","last_fresh":"2023-09-23T17:58:38.279486+0000","last_change":"2023-09-23T17:57:53.880095+0000","last_active":"2023-09-23T17:57:52.883357+0000","last_peered":"2023-09-23T17:58:38.279486+0000","last_clean":"2023-09-23T17:53:21.028908+0000","last_became_active":"2023-09-23T17:57:24.954589+0000","last_became_peered":"2023-09-23T17:57:53.880095+0000","last_unstale":"2023-09-23T17:58:38.279486+0000","last_undegraded":"2023-09-23T17:58:38.279486+0000","last_fullsized":"2023-09-23T17:57:53.874470+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:56.618752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":125,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.254305+0000","last_change":"2023-09-23T17:57:24.961477+0000","last_active":"2023-09-23T17:58:36.254305+0000","last_peered":"2023-09-23T17:58:36.254305+0000","last_clean":"2023-09-23T17:57:20.929549+0000","last_became_active":"2023-09-23T17:57:24.961477+0000","last_became_peered":"2023-09-23T17:57:24.961477+0000","last_unstale":"2023-09-23T17:58:36.254305+0000","last_undegraded":"2023-09-23T17:58:36.254305+0000","last_fullsized":"2023-09-23T17:57:24.954309+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:01.838528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":452,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.244705+0000","last_change":"2023-09-23T17:58:35.251291+0000","last_active":"2023-09-23T17:58:36.244705+0000","last_peered":"2023-09-23T17:58:36.244705+0000","last_clean":"2023-09-23T17:58:36.244705+0000","last_became_active":"2023-09-23T17:58:35.250937+0000","last_became_peered":"2023-09-23T17:58:35.250937+0000","last_unstale":"2023-09-23T17:58:36.244705+0000","last_undegraded":"2023-09-23T17:58:36.244705+0000","last_fullsized":"2023-09-23T17:58:36.244705+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":594,"reported_epoch":166,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:38.273316+0000","last_change":"2023-09-23T17:57:53.873008+0000","last_active":"2023-09-23T17:58:38.273316+0000","last_peered":"2023-09-23T17:58:38.273316+0000","last_clean":"2023-09-23T17:57:47.562468+0000","last_became_active":"2023-09-23T17:57:53.873008+0000","last_became_peered":"2023-09-23T17:57:53.873008+0000","last_unstale":"2023-09-23T17:58:38.273316+0000","last_undegraded":"2023-09-23T17:57:53.867595+0000","last_fullsized":"2023-09-23T17:57:53.867487+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:19.562351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":151,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.272353+0000","last_change":"2023-09-23T17:57:24.952712+0000","last_active":"2023-09-23T17:58:38.272353+0000","last_peered":"2023-09-23T17:58:38.272353+0000","last_clean":"2023-09-23T17:57:20.925623+0000","last_became_active":"2023-09-23T17:57:24.952712+0000","last_became_peered":"2023-09-23T17:57:24.952712+0000","last_unstale":"2023-09-23T17:58:38.272353+0000","last_undegraded":"2023-09-23T17:58:38.272353+0000","last_fullsized":"2023-09-23T17:57:24.949381+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:55.410751+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":110,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.272199+0000","last_change":"2023-09-23T17:56:50.816700+0000","last_active":"2023-09-23T17:58:38.272199+0000","last_peered":"2023-09-23T17:58:38.272199+0000","last_clean":"2023-09-23T17:58:38.272199+0000","last_became_active":"2023-09-23T17:56:50.816422+0000","last_became_peered":"2023-09-23T17:56:50.816422+0000","last_unstale":"2023-09-23T17:58:38.272199+0000","last_undegraded":"2023-09-23T17:58:38.272199+0000","last_fullsized":"2023-09-23T17:58:38.272199+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:01.034460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"166'8434","reported_seq":8513,"reported_epoch":166,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T17:58:44.256955+0000","last_change":"2023-09-23T17:58:36.379468+0000","last_active":"2023-09-23T17:58:44.256955+0000","last_peered":"2023-09-23T17:58:44.256955+0000","last_clean":"2023-09-23T17:55:57.821023+0000","last_became_active":"2023-09-23T17:58:36.360229+0000","last_became_peered":"2023-09-23T17:58:36.360229+0000","last_unstale":"2023-09-23T17:58:44.256955+0000","last_undegraded":"2023-09-23T17:58:36.273733+0000","last_fullsized":"2023-09-23T17:58:36.243100+0000","mapping_epoch":164,"log_start":"117'2380","ondisk_log_start":"117'2380","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6054,"log_dups_size":0,"ondisk_log_size":6054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:31.842191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304349184,"num_objects":4644,"num_object_clones":0,"num_object_copies":13932,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4644,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4644,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8434,"num_write_kb":418496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4644}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":451,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.244180+0000","last_change":"2023-09-23T17:58:35.252081+0000","last_active":"2023-09-23T17:58:36.244180+0000","last_peered":"2023-09-23T17:58:36.244180+0000","last_clean":"2023-09-23T17:58:36.244180+0000","last_became_active":"2023-09-23T17:58:35.250789+0000","last_became_peered":"2023-09-23T17:58:35.250789+0000","last_unstale":"2023-09-23T17:58:36.244180+0000","last_undegraded":"2023-09-23T17:58:36.244180+0000","last_fullsized":"2023-09-23T17:58:36.244180+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"166'8490","reported_seq":8575,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:44.214517+0000","last_change":"2023-09-23T17:56:50.843235+0000","last_active":"2023-09-23T17:58:44.214517+0000","last_peered":"2023-09-23T17:58:44.214517+0000","last_clean":"2023-09-23T17:58:44.214517+0000","last_became_active":"2023-09-23T17:56:50.835153+0000","last_became_peered":"2023-09-23T17:56:50.835153+0000","last_unstale":"2023-09-23T17:58:44.214517+0000","last_undegraded":"2023-09-23T17:58:44.214517+0000","last_fullsized":"2023-09-23T17:58:44.214517+0000","mapping_epoch":126,"log_start":"118'2435","ondisk_log_start":"118'2435","created":110,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6055,"log_dups_size":0,"ondisk_log_size":6055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:14.827228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300154880,"num_objects":4580,"num_object_clones":0,"num_object_copies":13740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8490,"num_write_kb":418240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1228,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.271957+0000","last_change":"2023-09-23T17:58:35.242003+0000","last_active":"2023-09-23T17:58:38.271957+0000","last_peered":"2023-09-23T17:58:38.271957+0000","last_clean":"2023-09-23T17:58:38.271957+0000","last_became_active":"2023-09-23T17:58:35.241549+0000","last_became_peered":"2023-09-23T17:58:35.241549+0000","last_unstale":"2023-09-23T17:58:38.271957+0000","last_undegraded":"2023-09-23T17:58:38.271957+0000","last_fullsized":"2023-09-23T17:58:38.271957+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:40.693076+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1894,"num_read_kb":2706,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":152,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.267845+0000","last_change":"2023-09-23T17:58:36.275184+0000","last_active":"2023-09-23T17:58:38.267845+0000","last_peered":"2023-09-23T17:58:38.267845+0000","last_clean":"2023-09-23T17:58:38.267845+0000","last_became_active":"2023-09-23T17:58:36.274850+0000","last_became_peered":"2023-09-23T17:58:36.274850+0000","last_unstale":"2023-09-23T17:58:38.267845+0000","last_undegraded":"2023-09-23T17:58:38.267845+0000","last_fullsized":"2023-09-23T17:58:38.267845+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:15.019915+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":128,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.273158+0000","last_change":"2023-09-23T17:57:24.952855+0000","last_active":"2023-09-23T17:58:38.273158+0000","last_peered":"2023-09-23T17:58:38.273158+0000","last_clean":"2023-09-23T17:57:20.923267+0000","last_became_active":"2023-09-23T17:57:24.952855+0000","last_became_peered":"2023-09-23T17:57:24.952855+0000","last_unstale":"2023-09-23T17:58:38.273158+0000","last_undegraded":"2023-09-23T17:58:38.273158+0000","last_fullsized":"2023-09-23T17:57:24.946770+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:15.329593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":452,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.267014+0000","last_change":"2023-09-23T17:58:35.251554+0000","last_active":"2023-09-23T17:58:36.267014+0000","last_peered":"2023-09-23T17:58:36.267014+0000","last_clean":"2023-09-23T17:58:36.267014+0000","last_became_active":"2023-09-23T17:58:35.250151+0000","last_became_peered":"2023-09-23T17:58:35.250151+0000","last_unstale":"2023-09-23T17:58:36.267014+0000","last_undegraded":"2023-09-23T17:58:36.267014+0000","last_fullsized":"2023-09-23T17:58:36.267014+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":45,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"166'8775","reported_seq":8847,"reported_epoch":166,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:43.739841+0000","last_change":"2023-09-23T17:57:53.879792+0000","last_active":"2023-09-23T17:58:43.739841+0000","last_peered":"2023-09-23T17:58:43.739841+0000","last_clean":"2023-09-23T17:57:52.799975+0000","last_became_active":"2023-09-23T17:57:53.879792+0000","last_became_peered":"2023-09-23T17:57:53.879792+0000","last_unstale":"2023-09-23T17:58:43.739841+0000","last_undegraded":"2023-09-23T17:57:53.874454+0000","last_fullsized":"2023-09-23T17:57:53.874345+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":8775,"log_dups_size":0,"ondisk_log_size":8775,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:37.828270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":306642944,"num_objects":4679,"num_object_clones":0,"num_object_copies":14037,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4679,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4679,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8775,"num_write_kb":430528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":4679}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":437,"reported_epoch":165,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:36.253022+0000","last_change":"2023-09-23T17:57:24.955720+0000","last_active":"2023-09-23T17:58:36.253022+0000","last_peered":"2023-09-23T17:58:36.253022+0000","last_clean":"2023-09-23T17:57:20.931796+0000","last_became_active":"2023-09-23T17:57:24.955720+0000","last_became_peered":"2023-09-23T17:57:24.955720+0000","last_unstale":"2023-09-23T17:58:36.253022+0000","last_undegraded":"2023-09-23T17:57:24.945824+0000","last_fullsized":"2023-09-23T17:57:24.945671+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:25.614568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":121,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.279648+0000","last_change":"2023-09-23T17:58:35.246091+0000","last_active":"2023-09-23T17:58:38.279648+0000","last_peered":"2023-09-23T17:58:38.279648+0000","last_clean":"2023-09-23T17:58:38.279648+0000","last_became_active":"2023-09-23T17:58:35.245673+0000","last_became_peered":"2023-09-23T17:58:35.245673+0000","last_unstale":"2023-09-23T17:58:38.279648+0000","last_undegraded":"2023-09-23T17:58:38.279648+0000","last_fullsized":"2023-09-23T17:58:38.279648+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:53.368873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":135,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.274390+0000","last_change":"2023-09-23T17:53:52.728795+0000","last_active":"2023-09-23T17:58:38.274390+0000","last_peered":"2023-09-23T17:58:38.274390+0000","last_clean":"2023-09-23T17:58:38.274390+0000","last_became_active":"2023-09-23T17:53:52.728423+0000","last_became_peered":"2023-09-23T17:53:52.728423+0000","last_unstale":"2023-09-23T17:58:38.274390+0000","last_undegraded":"2023-09-23T17:58:38.274390+0000","last_fullsized":"2023-09-23T17:58:38.274390+0000","mapping_epoch":98,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:19.759322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1088,"reported_epoch":166,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:38.275677+0000","last_change":"2023-09-23T17:57:53.875425+0000","last_active":"2023-09-23T17:58:38.275677+0000","last_peered":"2023-09-23T17:58:38.275677+0000","last_clean":"2023-09-23T17:57:47.581803+0000","last_became_active":"2023-09-23T17:57:53.875425+0000","last_became_peered":"2023-09-23T17:57:53.875425+0000","last_unstale":"2023-09-23T17:58:38.275677+0000","last_undegraded":"2023-09-23T17:57:53.870598+0000","last_fullsized":"2023-09-23T17:57:53.870472+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:09.749828+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2],"acting":[5,2],"avail_no_missing":["5","2"],"object_location_counts":[{"shards":"2,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":452,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.244960+0000","last_change":"2023-09-23T17:58:35.251615+0000","last_active":"2023-09-23T17:58:36.244960+0000","last_peered":"2023-09-23T17:58:36.244960+0000","last_clean":"2023-09-23T17:58:36.244960+0000","last_became_active":"2023-09-23T17:58:35.250337+0000","last_became_peered":"2023-09-23T17:58:35.250337+0000","last_unstale":"2023-09-23T17:58:36.244960+0000","last_undegraded":"2023-09-23T17:58:36.244960+0000","last_fullsized":"2023-09-23T17:58:36.244960+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":82,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":68,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.272049+0000","last_change":"2023-09-23T17:58:35.236354+0000","last_active":"2023-09-23T17:58:38.272049+0000","last_peered":"2023-09-23T17:58:38.272049+0000","last_clean":"2023-09-23T17:55:56.841167+0000","last_became_active":"2023-09-23T17:58:35.236354+0000","last_became_peered":"2023-09-23T17:58:35.236354+0000","last_unstale":"2023-09-23T17:58:38.272049+0000","last_undegraded":"2023-09-23T17:58:38.272049+0000","last_fullsized":"2023-09-23T17:58:35.228453+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:20.133458+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":66,"reported_epoch":166,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:58:38.266598+0000","last_change":"2023-09-23T17:58:36.248863+0000","last_active":"2023-09-23T17:58:38.266598+0000","last_peered":"2023-09-23T17:58:38.266598+0000","last_clean":"2023-09-23T17:58:38.266598+0000","last_became_active":"2023-09-23T17:58:36.248383+0000","last_became_peered":"2023-09-23T17:58:36.248383+0000","last_unstale":"2023-09-23T17:58:38.266598+0000","last_undegraded":"2023-09-23T17:58:38.266598+0000","last_fullsized":"2023-09-23T17:58:38.266598+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:49.334942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,0],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"166'8620","reported_seq":14625,"reported_epoch":166,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2023-09-23T17:58:43.806895+0000","last_change":"2023-09-23T17:58:22.730363+0000","last_active":"2023-09-23T17:58:43.806895+0000","last_peered":"2023-09-23T17:58:43.806895+0000","last_clean":"2023-09-23T17:57:52.852296+0000","last_became_active":"2023-09-23T17:58:13.141980+0000","last_became_peered":"2023-09-23T17:58:13.141980+0000","last_unstale":"2023-09-23T17:58:43.806895+0000","last_undegraded":"2023-09-23T17:58:13.016697+0000","last_fullsized":"2023-09-23T17:58:13.003199+0000","mapping_epoch":155,"log_start":"116'971","ondisk_log_start":"116'971","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":7649,"log_dups_size":0,"ondisk_log_size":7649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:32.127769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304611328,"num_objects":4648,"num_object_clones":0,"num_object_copies":13944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1962,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8620,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2964,"num_bytes_recovered":194248704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":4648}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":452,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.245530+0000","last_change":"2023-09-23T17:58:35.251169+0000","last_active":"2023-09-23T17:58:36.245530+0000","last_peered":"2023-09-23T17:58:36.245530+0000","last_clean":"2023-09-23T17:58:36.245530+0000","last_became_active":"2023-09-23T17:58:35.250852+0000","last_became_peered":"2023-09-23T17:58:35.250852+0000","last_unstale":"2023-09-23T17:58:36.245530+0000","last_undegraded":"2023-09-23T17:58:36.245530+0000","last_fullsized":"2023-09-23T17:58:36.245530+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.7","version":"167'8924","reported_seq":8989,"reported_epoch":167,"state":"active+clean","last_fresh":"2023-09-23T17:58:47.369339+0000","last_change":"2023-09-23T17:55:53.828325+0000","last_active":"2023-09-23T17:58:47.369339+0000","last_peered":"2023-09-23T17:58:47.369339+0000","last_clean":"2023-09-23T17:58:47.369339+0000","last_became_active":"2023-09-23T17:55:53.828056+0000","last_became_peered":"2023-09-23T17:55:53.828056+0000","last_unstale":"2023-09-23T17:58:47.369339+0000","last_undegraded":"2023-09-23T17:58:47.369339+0000","last_fullsized":"2023-09-23T17:58:47.369339+0000","mapping_epoch":110,"log_start":"0'0","ondisk_log_start":"0'0","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":8924,"log_dups_size":0,"ondisk_log_size":8924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:47.523521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":297140224,"num_objects":4534,"num_object_clones":0,"num_object_copies":13602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8924,"num_write_kb":430656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":268,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.279860+0000","last_change":"2023-09-23T17:57:53.883867+0000","last_active":"2023-09-23T17:58:38.279860+0000","last_peered":"2023-09-23T17:58:38.279860+0000","last_clean":"2023-09-23T17:46:57.119141+0000","last_became_active":"2023-09-23T17:57:53.883867+0000","last_became_peered":"2023-09-23T17:57:53.883867+0000","last_unstale":"2023-09-23T17:58:38.279860+0000","last_undegraded":"2023-09-23T17:58:38.279860+0000","last_fullsized":"2023-09-23T17:57:53.875072+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:35.282181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":133,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.243869+0000","last_change":"2023-09-23T17:57:24.959291+0000","last_active":"2023-09-23T17:58:36.243869+0000","last_peered":"2023-09-23T17:58:36.243869+0000","last_clean":"2023-09-23T17:57:20.930735+0000","last_became_active":"2023-09-23T17:57:24.959291+0000","last_became_peered":"2023-09-23T17:57:24.959291+0000","last_unstale":"2023-09-23T17:58:36.243869+0000","last_undegraded":"2023-09-23T17:58:36.243869+0000","last_fullsized":"2023-09-23T17:57:24.946083+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:43.225674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":125,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.244830+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T17:58:36.244830+0000","last_peered":"2023-09-23T17:58:36.244830+0000","last_clean":"2023-09-23T17:58:36.244830+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T17:58:36.244830+0000","last_undegraded":"2023-09-23T17:58:36.244830+0000","last_fullsized":"2023-09-23T17:58:36.244830+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":452,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.244384+0000","last_change":"2023-09-23T17:58:35.250698+0000","last_active":"2023-09-23T17:58:36.244384+0000","last_peered":"2023-09-23T17:58:36.244384+0000","last_clean":"2023-09-23T17:58:36.244384+0000","last_became_active":"2023-09-23T17:58:35.250045+0000","last_became_peered":"2023-09-23T17:58:35.250045+0000","last_unstale":"2023-09-23T17:58:36.244384+0000","last_undegraded":"2023-09-23T17:58:36.244384+0000","last_fullsized":"2023-09-23T17:58:36.244384+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":300,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.244043+0000","last_change":"2023-09-23T17:57:53.875525+0000","last_active":"2023-09-23T17:58:36.244043+0000","last_peered":"2023-09-23T17:58:36.244043+0000","last_clean":"2023-09-23T17:46:59.915193+0000","last_became_active":"2023-09-23T17:57:53.875525+0000","last_became_peered":"2023-09-23T17:57:53.875525+0000","last_unstale":"2023-09-23T17:58:36.244043+0000","last_undegraded":"2023-09-23T17:58:36.244043+0000","last_fullsized":"2023-09-23T17:57:53.868097+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:57.748716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":106,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.273454+0000","last_change":"2023-09-23T17:54:25.604720+0000","last_active":"2023-09-23T17:58:38.273454+0000","last_peered":"2023-09-23T17:58:38.273454+0000","last_clean":"2023-09-23T17:58:38.273454+0000","last_became_active":"2023-09-23T17:54:25.604386+0000","last_became_peered":"2023-09-23T17:54:25.604386+0000","last_unstale":"2023-09-23T17:58:38.273454+0000","last_undegraded":"2023-09-23T17:58:38.273454+0000","last_fullsized":"2023-09-23T17:58:38.273454+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:56.869273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":101,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.246577+0000","last_change":"2023-09-23T17:57:53.873504+0000","last_active":"2023-09-23T17:58:36.246577+0000","last_peered":"2023-09-23T17:58:36.246577+0000","last_clean":"2023-09-23T17:57:47.571370+0000","last_became_active":"2023-09-23T17:57:53.873504+0000","last_became_peered":"2023-09-23T17:57:53.873504+0000","last_unstale":"2023-09-23T17:58:36.246577+0000","last_undegraded":"2023-09-23T17:58:36.246577+0000","last_fullsized":"2023-09-23T17:57:53.869657+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":94,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:09.957803+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"166'8414","reported_seq":8481,"reported_epoch":166,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T17:58:44.251689+0000","last_change":"2023-09-23T17:58:36.363577+0000","last_active":"2023-09-23T17:58:44.251689+0000","last_peered":"2023-09-23T17:58:44.251689+0000","last_clean":"2023-09-23T17:55:52.789238+0000","last_became_active":"2023-09-23T17:58:36.343368+0000","last_became_peered":"2023-09-23T17:58:36.343368+0000","last_unstale":"2023-09-23T17:58:44.251689+0000","last_undegraded":"2023-09-23T17:58:36.266675+0000","last_fullsized":"2023-09-23T17:58:36.243100+0000","mapping_epoch":164,"log_start":"117'2337","ondisk_log_start":"117'2337","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6077,"log_dups_size":0,"ondisk_log_size":6077,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:26.608406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300023808,"num_objects":4578,"num_object_clones":0,"num_object_copies":13734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4578,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8414,"num_write_kb":415744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":4578}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":452,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.243841+0000","last_change":"2023-09-23T17:58:35.251962+0000","last_active":"2023-09-23T17:58:36.243841+0000","last_peered":"2023-09-23T17:58:36.243841+0000","last_clean":"2023-09-23T17:58:36.243841+0000","last_became_active":"2023-09-23T17:58:35.251617+0000","last_became_peered":"2023-09-23T17:58:35.251617+0000","last_unstale":"2023-09-23T17:58:36.243841+0000","last_undegraded":"2023-09-23T17:58:36.243841+0000","last_fullsized":"2023-09-23T17:58:36.243841+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":161010,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":34,"reported_epoch":165,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T17:58:36.245111+0000","last_change":"2023-09-23T17:57:53.872726+0000","last_active":"2023-09-23T17:57:52.888539+0000","last_peered":"2023-09-23T17:58:36.245111+0000","last_clean":"2023-09-23T17:45:54.465055+0000","last_became_active":"2023-09-23T17:45:51.139637+0000","last_became_peered":"2023-09-23T17:57:53.872726+0000","last_unstale":"2023-09-23T17:58:36.245111+0000","last_undegraded":"2023-09-23T17:57:53.871230+0000","last_fullsized":"2023-09-23T17:57:53.871154+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:46.624357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":399,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.273432+0000","last_change":"2023-09-23T17:58:35.247928+0000","last_active":"2023-09-23T17:58:38.273432+0000","last_peered":"2023-09-23T17:58:38.273432+0000","last_clean":"2023-09-23T17:58:38.273432+0000","last_became_active":"2023-09-23T17:58:35.247482+0000","last_became_peered":"2023-09-23T17:58:35.247482+0000","last_unstale":"2023-09-23T17:58:38.273432+0000","last_undegraded":"2023-09-23T17:58:38.273432+0000","last_fullsized":"2023-09-23T17:58:38.273432+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:18.686446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":91,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.248078+0000","last_change":"2023-09-23T17:58:12.005101+0000","last_active":"2023-09-23T17:58:36.248078+0000","last_peered":"2023-09-23T17:58:36.248078+0000","last_clean":"2023-09-23T17:49:06.298151+0000","last_became_active":"2023-09-23T17:58:12.005101+0000","last_became_peered":"2023-09-23T17:58:12.005101+0000","last_unstale":"2023-09-23T17:58:36.248078+0000","last_undegraded":"2023-09-23T17:58:36.248078+0000","last_fullsized":"2023-09-23T17:58:11.987584+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:15.859050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"167'8842","reported_seq":21568,"reported_epoch":167,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:58:47.380527+0000","last_change":"2023-09-23T17:58:23.828372+0000","last_active":"2023-09-23T17:58:47.380527+0000","last_peered":"2023-09-23T17:58:47.380527+0000","last_clean":"2023-09-23T17:57:23.936571+0000","last_became_active":"2023-09-23T17:58:23.828372+0000","last_became_peered":"2023-09-23T17:58:23.828372+0000","last_unstale":"2023-09-23T17:58:47.380527+0000","last_undegraded":"2023-09-23T17:58:23.826763+0000","last_fullsized":"2023-09-23T17:58:23.826662+0000","mapping_epoch":159,"log_start":"116'1603","ondisk_log_start":"116'1603","created":110,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":7239,"log_dups_size":0,"ondisk_log_size":7239,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:59.923478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294125568,"num_objects":4488,"num_object_clones":0,"num_object_copies":13464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4488,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8842,"num_write_kb":426560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6302,"num_bytes_recovered":413007872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":4488}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":456,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.243739+0000","last_change":"2023-09-23T17:58:35.251040+0000","last_active":"2023-09-23T17:58:36.243739+0000","last_peered":"2023-09-23T17:58:36.243739+0000","last_clean":"2023-09-23T17:58:36.243739+0000","last_became_active":"2023-09-23T17:58:35.250785+0000","last_became_peered":"2023-09-23T17:58:35.250785+0000","last_unstale":"2023-09-23T17:58:36.243739+0000","last_undegraded":"2023-09-23T17:58:36.243739+0000","last_fullsized":"2023-09-23T17:58:36.243739+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":220,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":488690,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":342,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.266300+0000","last_change":"2023-09-23T17:58:35.227206+0000","last_active":"2023-09-23T17:58:38.266300+0000","last_peered":"2023-09-23T17:58:38.266300+0000","last_clean":"2023-09-23T17:46:59.535985+0000","last_became_active":"2023-09-23T17:58:35.227206+0000","last_became_peered":"2023-09-23T17:58:35.227206+0000","last_unstale":"2023-09-23T17:58:38.266300+0000","last_undegraded":"2023-09-23T17:58:38.266300+0000","last_fullsized":"2023-09-23T17:58:35.224367+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:52.549534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":139,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.244737+0000","last_change":"2023-09-23T17:56:50.856836+0000","last_active":"2023-09-23T17:58:36.244737+0000","last_peered":"2023-09-23T17:58:36.244737+0000","last_clean":"2023-09-23T17:58:36.244737+0000","last_became_active":"2023-09-23T17:56:50.835155+0000","last_became_peered":"2023-09-23T17:56:50.835155+0000","last_unstale":"2023-09-23T17:58:36.244737+0000","last_undegraded":"2023-09-23T17:58:36.244737+0000","last_fullsized":"2023-09-23T17:58:36.244737+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:36.188982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":134,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.270839+0000","last_change":"2023-09-23T17:58:36.250663+0000","last_active":"2023-09-23T17:58:38.270839+0000","last_peered":"2023-09-23T17:58:38.270839+0000","last_clean":"2023-09-23T17:58:38.270839+0000","last_became_active":"2023-09-23T17:58:36.250382+0000","last_became_peered":"2023-09-23T17:58:36.250382+0000","last_unstale":"2023-09-23T17:58:38.270839+0000","last_undegraded":"2023-09-23T17:58:38.270839+0000","last_fullsized":"2023-09-23T17:58:38.270839+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:48.471077+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"167'8870","reported_seq":16535,"reported_epoch":167,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T17:58:45.877158+0000","last_change":"2023-09-23T17:58:34.248637+0000","last_active":"2023-09-23T17:58:45.877158+0000","last_peered":"2023-09-23T17:58:45.877158+0000","last_clean":"2023-09-23T17:56:38.421727+0000","last_became_active":"2023-09-23T17:57:26.317820+0000","last_became_peered":"2023-09-23T17:57:26.317820+0000","last_unstale":"2023-09-23T17:58:45.877158+0000","last_undegraded":"2023-09-23T17:57:26.081408+0000","last_fullsized":"2023-09-23T17:58:45.877158+0000","mapping_epoch":143,"log_start":"118'3409","ondisk_log_start":"118'3409","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5461,"log_dups_size":0,"ondisk_log_size":5461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:37.559775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":300285952,"num_objects":4582,"num_object_clones":0,"num_object_copies":13746,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4582,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4582,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8870,"num_write_kb":430464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3786,"num_bytes_recovered":248119296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,0],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":1728},{"shards":"1,4","objects":2854}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":122,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.246431+0000","last_change":"2023-09-23T17:57:53.877134+0000","last_active":"2023-09-23T17:58:36.246431+0000","last_peered":"2023-09-23T17:58:36.246431+0000","last_clean":"2023-09-23T17:57:47.571295+0000","last_became_active":"2023-09-23T17:57:53.877134+0000","last_became_peered":"2023-09-23T17:57:53.877134+0000","last_unstale":"2023-09-23T17:58:36.246431+0000","last_undegraded":"2023-09-23T17:58:36.246431+0000","last_fullsized":"2023-09-23T17:57:53.869424+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:59.045696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":156,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.253144+0000","last_change":"2023-09-23T17:57:24.956919+0000","last_active":"2023-09-23T17:58:36.253144+0000","last_peered":"2023-09-23T17:58:36.253144+0000","last_clean":"2023-09-23T17:57:20.928421+0000","last_became_active":"2023-09-23T17:57:24.956919+0000","last_became_peered":"2023-09-23T17:57:24.956919+0000","last_unstale":"2023-09-23T17:58:36.253144+0000","last_undegraded":"2023-09-23T17:58:36.253144+0000","last_fullsized":"2023-09-23T17:57:24.951731+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:03.928787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":153,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.271988+0000","last_change":"2023-09-23T17:56:51.848379+0000","last_active":"2023-09-23T17:58:38.271988+0000","last_peered":"2023-09-23T17:58:38.271988+0000","last_clean":"2023-09-23T17:58:38.271988+0000","last_became_active":"2023-09-23T17:56:51.848091+0000","last_became_peered":"2023-09-23T17:56:51.848091+0000","last_unstale":"2023-09-23T17:58:38.271988+0000","last_undegraded":"2023-09-23T17:58:38.271988+0000","last_fullsized":"2023-09-23T17:58:38.271988+0000","mapping_epoch":127,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:01:01.930989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":57,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.267496+0000","last_change":"2023-09-23T17:57:24.951129+0000","last_active":"2023-09-23T17:58:38.267496+0000","last_peered":"2023-09-23T17:58:38.267496+0000","last_clean":"2023-09-23T17:57:20.922389+0000","last_became_active":"2023-09-23T17:57:24.951129+0000","last_became_peered":"2023-09-23T17:57:24.951129+0000","last_unstale":"2023-09-23T17:58:38.267496+0000","last_undegraded":"2023-09-23T17:58:38.267496+0000","last_fullsized":"2023-09-23T17:57:24.945440+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:43.628177+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":159,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.272081+0000","last_change":"2023-09-23T17:58:36.250460+0000","last_active":"2023-09-23T17:58:38.272081+0000","last_peered":"2023-09-23T17:58:38.272081+0000","last_clean":"2023-09-23T17:58:38.272081+0000","last_became_active":"2023-09-23T17:58:36.250081+0000","last_became_peered":"2023-09-23T17:58:36.250081+0000","last_unstale":"2023-09-23T17:58:38.272081+0000","last_undegraded":"2023-09-23T17:58:38.272081+0000","last_fullsized":"2023-09-23T17:58:38.272081+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:01.125167+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":57,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.269717+0000","last_change":"2023-09-23T17:57:24.957395+0000","last_active":"2023-09-23T17:58:38.269717+0000","last_peered":"2023-09-23T17:58:38.269717+0000","last_clean":"2023-09-23T17:57:20.922822+0000","last_became_active":"2023-09-23T17:57:24.957395+0000","last_became_peered":"2023-09-23T17:57:24.957395+0000","last_unstale":"2023-09-23T17:58:38.269717+0000","last_undegraded":"2023-09-23T17:58:38.269717+0000","last_fullsized":"2023-09-23T17:57:24.945910+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:25.767832+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":25,"reported_epoch":166,"state":"active+undersized","last_fresh":"2023-09-23T17:58:38.266851+0000","last_change":"2023-09-23T17:58:35.229256+0000","last_active":"2023-09-23T17:58:38.266851+0000","last_peered":"2023-09-23T17:58:38.266851+0000","last_clean":"2023-09-23T17:45:50.074605+0000","last_became_active":"2023-09-23T17:58:35.229256+0000","last_became_peered":"2023-09-23T17:58:35.229256+0000","last_unstale":"2023-09-23T17:58:38.266851+0000","last_undegraded":"2023-09-23T17:58:38.266851+0000","last_fullsized":"2023-09-23T17:58:35.224895+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:14.916976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":115,"reported_epoch":165,"state":"active+undersized","last_fresh":"2023-09-23T17:58:36.252869+0000","last_change":"2023-09-23T17:57:24.955661+0000","last_active":"2023-09-23T17:58:36.252869+0000","last_peered":"2023-09-23T17:58:36.252869+0000","last_clean":"2023-09-23T17:53:52.727684+0000","last_became_active":"2023-09-23T17:57:24.955661+0000","last_became_peered":"2023-09-23T17:57:24.955661+0000","last_unstale":"2023-09-23T17:58:36.252869+0000","last_undegraded":"2023-09-23T17:58:36.252869+0000","last_fullsized":"2023-09-23T17:57:24.945672+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:04.500443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":113,"reported_epoch":165,"state":"active+clean","last_fresh":"2023-09-23T17:58:36.245010+0000","last_change":"2023-09-23T17:54:25.601065+0000","last_active":"2023-09-23T17:58:36.245010+0000","last_peered":"2023-09-23T17:58:36.245010+0000","last_clean":"2023-09-23T17:58:36.245010+0000","last_became_active":"2023-09-23T17:54:25.600727+0000","last_became_peered":"2023-09-23T17:54:25.600727+0000","last_unstale":"2023-09-23T17:58:36.245010+0000","last_undegraded":"2023-09-23T17:58:36.245010+0000","last_fullsized":"2023-09-23T17:58:36.245010+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:20.919498+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":156,"reported_epoch":166,"state":"active+clean","last_fresh":"2023-09-23T17:58:38.267115+0000","last_change":"2023-09-23T17:58:35.243060+0000","last_active":"2023-09-23T17:58:38.267115+0000","last_peered":"2023-09-23T17:58:38.267115+0000","last_clean":"2023-09-23T17:58:38.267115+0000","last_became_active":"2023-09-23T17:58:35.242727+0000","last_became_peered":"2023-09-23T17:58:35.242727+0000","last_unstale":"2023-09-23T17:58:38.267115+0000","last_undegraded":"2023-09-23T17:58:38.267115+0000","last_fullsized":"2023-09-23T17:58:38.267115+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:23:08.850283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":4792451096,"num_objects":73128,"num_object_clones":0,"num_object_copies":219384,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":41607,"num_objects_misplaced":4304,"num_objects_unfound":0,"num_objects_dirty":73128,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138120,"num_write_kb":6759873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25057,"num_bytes_recovered":1641938944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15164977152,"data_stored":15164964936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":119067,"ondisk_log_size":119067,"up":42,"acting":39,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":74,"acting":76,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":10,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":202,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33098,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":72,"acting":74,"num_store_stats":8},{"poolid":1,"num_pg":11,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":1771,"ondisk_log_size":1771,"up":33,"acting":33,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5994,"num_read_kb":12188,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":18,"acting":18,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266849,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1840208,"kb_used_data":1778696,"kb_used_omap":0,"kb_used_meta":61504,"kb_avail":91904944,"statfs":{"total":95995035648,"available":94110662656,"internally_reserved":0,"allocated":1821384704,"data_stored":1820756859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":62980096},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":0.83399999999999996,"15min":0.64500000000000002},"min":{"1min":0.24099999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.5700000000000003,"5min":8.5700000000000003,"15min":8.5700000000000003},"last":2.2210000000000001},{"interface":"front","average":{"1min":1.6499999999999999,"5min":0.879,"15min":0.67100000000000004},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.8410000000000002,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":4.1280000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.59599999999999997,"15min":0.56999999999999995},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.702,"5min":2.702,"15min":2.702},"last":6.6890000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":0.65700000000000003,"15min":0.58899999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":4.4240000000000004},"last":0.26000000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":0.89200000000000002,"15min":0.66300000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.2359999999999998,"5min":5.2359999999999998,"15min":5.2359999999999998},"last":2.2970000000000002},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.77900000000000003,"15min":0.64800000000000002},"min":{"1min":0.38100000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.8340000000000001,"5min":4.6180000000000003,"15min":4.6180000000000003},"last":0.439}]},{"osd":3,"last update":"Sat Sep 23 17:57:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.64900000000000002,"15min":0.57299999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":4.0549999999999997,"5min":4.0549999999999997,"15min":4.0549999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.665,"5min":0.77400000000000002,"15min":0.626},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.1609999999999996,"5min":5.1609999999999996,"15min":5.1609999999999996},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.53400000000000003,"15min":0.47099999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.6160000000000001,"5min":2.6160000000000001,"15min":2.6160000000000001},"last":2.1949999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.56100000000000005,"15min":0.498},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8079999999999998,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.60699999999999998,"15min":0.52600000000000002},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.5840000000000001,"5min":2.5840000000000001,"15min":2.5840000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.749,"5min":0.64100000000000001,"15min":0.54900000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.706,"5min":2.706,"15min":2.706},"last":2.1669999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":1.002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.435}]}]},{"osd":6,"up_from":163,"seq":700079669252,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":245724,"kb_used_data":219220,"kb_used_omap":7,"kb_used_meta":26488,"kb_avail":93499428,"statfs":{"total":95995035648,"available":95743414272,"internally_reserved":0,"allocated":224481280,"data_stored":223796923,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":27124516},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.23599999999999999}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.915}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.3319999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":8.4570000000000007}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60699999999999998}]}]},{"osd":1,"up_from":98,"seq":420906795069,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1864356,"kb_used_data":1756316,"kb_used_omap":7,"kb_used_meta":107960,"kb_avail":91880796,"statfs":{"total":95995035648,"available":94085935104,"internally_reserved":0,"allocated":1798467584,"data_stored":1797603787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":110551741},"hb_peers":[0,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:57:56 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":1.115,"15min":0.63},"min":{"1min":0.215,"5min":0.186,"15min":0.186},"max":{"1min":7.1150000000000002,"5min":11.384,"15min":11.384},"last":0.91700000000000004},{"interface":"front","average":{"1min":2.2229999999999999,"5min":1.272,"15min":0.68799999999999994},"min":{"1min":0.26500000000000001,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.355,"5min":12.355,"15min":12.355},"last":1.282}]},{"osd":2,"last update":"Sat Sep 23 17:58:34 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.3069999999999999,"15min":0.73599999999999999},"min":{"1min":0.27200000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.9850000000000003,"5min":10.241,"15min":10.241},"last":0.24099999999999999},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.246,"15min":0.70799999999999996},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.6349999999999998,"5min":11.305999999999999,"15min":11.305999999999999},"last":0.98999999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:57:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8999999999999999,"5min":0.90800000000000003,"15min":0.57599999999999996},"min":{"1min":0.29899999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":14.548999999999999,"5min":14.548999999999999,"15min":14.548999999999999},"last":2.7149999999999999},{"interface":"front","average":{"1min":2.3740000000000001,"5min":0.85199999999999998,"15min":0.59799999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.250999999999999,"5min":10.250999999999999,"15min":10.250999999999999},"last":0.82899999999999996}]},{"osd":4,"last update":"Sat Sep 23 17:57:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":1.1160000000000001,"15min":0.82699999999999996},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":8.0289999999999999,"5min":8.0289999999999999,"15min":8.0289999999999999},"last":1.0960000000000001},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.1319999999999999,"15min":0.82899999999999996},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":7.2069999999999999,"5min":8.8770000000000007,"15min":8.8770000000000007},"last":0.95499999999999996}]},{"osd":5,"last update":"Sat Sep 23 17:57:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":1.1890000000000001,"15min":0.875},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":7.1189999999999998,"5min":7.4269999999999996,"15min":7.4269999999999996},"last":0.443},{"interface":"front","average":{"1min":2.2879999999999998,"5min":1.2290000000000001,"15min":0.88100000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.0790000000000006,"5min":9.0790000000000006,"15min":9.0790000000000006},"last":1.0580000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.9330000000000001,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":3.919},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.69199999999999995,"15min":0.69899999999999995},"min":{"1min":0.47699999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.141,"5min":1.141,"15min":1.141},"last":0.48099999999999998}]},{"osd":7,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":0.89400000000000002,"15min":0.74399999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.9630000000000001,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":7.8869999999999996},{"interface":"front","average":{"1min":1.895,"5min":0.92900000000000005,"15min":0.77900000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.2940000000000005,"5min":8.2940000000000005,"15min":8.2940000000000005},"last":7.234}]}]},{"osd":0,"up_from":93,"seq":399431958595,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3165184,"kb_used_data":2962808,"kb_used_omap":7,"kb_used_meta":202296,"kb_avail":90579968,"statfs":{"total":95995035648,"available":92753887232,"internally_reserved":0,"allocated":3033915392,"data_stored":3033076195,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":207151844},"hb_peers":[1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,1,2,1,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:57:56 2023","interfaces":[{"interface":"back","average":{"1min":2.173,"5min":1.0289999999999999,"15min":0.58899999999999997},"min":{"1min":0.224,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":15.644,"5min":15.644,"15min":15.644},"last":3.2730000000000001},{"interface":"front","average":{"1min":2.3199999999999998,"5min":1.028,"15min":0.626},"min":{"1min":0.28299999999999997,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":11.346,"5min":11.346,"15min":11.346},"last":2.9590000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:58:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.24,"15min":0.65400000000000003},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":14.452,"5min":14.452,"15min":14.452},"last":3.3090000000000002},{"interface":"front","average":{"1min":1.669,"5min":1.2509999999999999,"15min":0.66200000000000003},"min":{"1min":0.248,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.968,"5min":9.968,"15min":9.968},"last":3.0800000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":0.77100000000000002,"15min":0.60099999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3449999999999998,"5min":4.3449999999999998,"15min":4.3449999999999998},"last":9.266},{"interface":"front","average":{"1min":2.109,"5min":0.73699999999999999,"15min":0.50900000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":8.8729999999999993},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:58:32 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":1.0469999999999999,"15min":0.77300000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":14.081,"5min":14.081,"15min":14.081},"last":3.3540000000000001},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.0249999999999999,"15min":0.75700000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.0750000000000002,"5min":9.1020000000000003,"15min":9.1020000000000003},"last":1.3109999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:58:32 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":1.0629999999999999,"15min":0.77600000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.1829999999999998,"5min":9.1370000000000005,"15min":9.1370000000000005},"last":1.347},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.90300000000000002,"15min":0.73099999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.1219999999999999,"5min":8.7829999999999995,"15min":8.7829999999999995},"last":0.41299999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.42899999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":3.379},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65200000000000002},"min":{"1min":0.52700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.81100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":0.38400000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":0.90800000000000003,"15min":0.746},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.8160000000000007,"5min":8.8160000000000007,"15min":8.8160000000000007},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.01,"15min":0.78400000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":8.6470000000000002,"5min":8.6470000000000002,"15min":8.6470000000000002},"last":0.49099999999999999}]}]},{"osd":2,"up_from":103,"seq":442381631542,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2146756,"kb_used_data":2047036,"kb_used_omap":11,"kb_used_meta":99700,"kb_avail":91598396,"statfs":{"total":95995035648,"available":93796757504,"internally_reserved":0,"allocated":2096164864,"data_stored":2095324187,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":102093175},"hb_peers":[0,1,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.2030000000000001,"15min":0.627},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":10.853,"5min":13.234,"15min":13.234},"last":0.20699999999999999},{"interface":"front","average":{"1min":1.5269999999999999,"5min":1.272,"15min":0.68000000000000005},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":10.497,"5min":13.364000000000001,"15min":13.364000000000001},"last":1.3169999999999999}]},{"osd":1,"last update":"Sat Sep 23 17:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":1.331,"15min":0.71099999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.576000000000001,"5min":19.206,"15min":19.206},"last":1.278},{"interface":"front","average":{"1min":2.1629999999999998,"5min":1.5249999999999999,"15min":0.72799999999999998},"min":{"1min":0.22800000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.9619999999999997,"5min":15.657,"15min":15.657},"last":0.313}]},{"osd":3,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":0.58799999999999997,"15min":0.443},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8339999999999996,"5min":4.8339999999999996,"15min":4.8339999999999996},"last":2.8849999999999998},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.59699999999999998,"15min":0.48399999999999999},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.6459999999999999,"5min":5.6459999999999999,"15min":5.6459999999999999},"last":2.5579999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.135,"15min":0.78000000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.8499999999999996,"5min":6.8499999999999996,"15min":6.8499999999999996},"last":1.887},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.91700000000000004,"15min":0.71199999999999997},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.8230000000000004,"5min":13.308,"15min":13.308},"last":1.2430000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.036,"15min":0.76100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":4.7839999999999998,"5min":13.273,"15min":13.273},"last":3.8079999999999998},{"interface":"front","average":{"1min":1.885,"5min":1.454,"15min":0.88700000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.327,"5min":15.496,"15min":15.496},"last":3.863}]},{"osd":6,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":1.8440000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.35399999999999998}]},{"osd":7,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":0.94299999999999995,"15min":0.71899999999999997},"min":{"1min":0.505,"5min":0.441,"15min":0.441},"max":{"1min":5.7409999999999997,"5min":5.7409999999999997,"15min":5.7409999999999997},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.0870000000000002,"5min":0.97899999999999998,"15min":0.78300000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":13.333,"5min":13.333,"15min":13.333},"last":1.3440000000000001}]}]},{"osd":3,"up_from":108,"seq":463856467999,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267348,"kb_used_data":1221900,"kb_used_omap":3,"kb_used_meta":45436,"kb_avail":92477804,"statfs":{"total":95995035648,"available":94697271296,"internally_reserved":0,"allocated":1251225600,"data_stored":1250531999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":46527208},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.0699999999999998,"5min":0.71399999999999997,"15min":0.48799999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":14.82,"5min":14.82,"15min":14.82},"last":9.0220000000000002},{"interface":"front","average":{"1min":2.085,"5min":0.72499999999999998,"15min":0.499},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":13.054,"5min":13.054,"15min":13.054},"last":0.46500000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":0.65700000000000003,"15min":0.46899999999999997},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.8130000000000006,"5min":8.8130000000000006,"15min":8.8130000000000006},"last":9.1489999999999991},{"interface":"front","average":{"1min":1.417,"5min":0.61599999999999999,"15min":0.48199999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4580000000000002,"5min":6.4580000000000002,"15min":6.4580000000000002},"last":9.125}]},{"osd":2,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":0.69099999999999995,"15min":0.49099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.869999999999999,"5min":10.869999999999999,"15min":10.869999999999999},"last":7.3849999999999998},{"interface":"front","average":{"1min":1.6850000000000001,"5min":0.66500000000000004,"15min":0.495},"min":{"1min":0.376,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":13.936999999999999,"5min":13.936999999999999,"15min":13.936999999999999},"last":8.9169999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":0.73699999999999999,"15min":0.65200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5380000000000003,"5min":6.5380000000000003,"15min":6.5380000000000003},"last":6.7569999999999997},{"interface":"front","average":{"1min":1.629,"5min":0.82099999999999995,"15min":0.68700000000000006},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.5940000000000003,"5min":6.5940000000000003,"15min":6.5940000000000003},"last":3.7709999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":0.94399999999999995,"15min":0.72599999999999998},"min":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":9.6489999999999991,"5min":9.6489999999999991,"15min":9.6489999999999991},"last":3.9569999999999999},{"interface":"front","average":{"1min":1.4710000000000001,"5min":0.78600000000000003,"15min":0.67200000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.3200000000000003,"5min":9.3200000000000003,"15min":9.3200000000000003},"last":6.4569999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":0.81799999999999995,"15min":0.68999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":8.8599999999999994,"5min":8.8599999999999994,"15min":8.8599999999999994},"last":6.4950000000000001},{"interface":"front","average":{"1min":1.3520000000000001,"5min":0.78800000000000003,"15min":0.69399999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.2080000000000002,"5min":9.2080000000000002,"15min":9.2080000000000002},"last":6.0709999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019061,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2390944,"kb_used_data":2293720,"kb_used_omap":0,"kb_used_meta":97216,"kb_avail":91354208,"statfs":{"total":95995035648,"available":93546708992,"internally_reserved":0,"allocated":2348769280,"data_stored":2348080663,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":99549184},"hb_peers":[0,1,2,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:58:26 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":0.63900000000000001,"15min":0.55700000000000005},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":2.6880000000000002,"5min":4.8760000000000003,"15min":4.8760000000000003},"last":5.0609999999999999},{"interface":"front","average":{"1min":1.9450000000000001,"5min":1.0860000000000001,"15min":0.71199999999999997},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":11.372,"5min":11.372,"15min":11.372},"last":2.786}]},{"osd":1,"last update":"Sat Sep 23 17:57:58 2023","interfaces":[{"interface":"back","average":{"1min":2.298,"5min":0.94399999999999995,"15min":0.66800000000000004},"min":{"1min":0.26900000000000002,"5min":0.249,"15min":0.249},"max":{"1min":7.1909999999999998,"5min":7.1909999999999998,"15min":7.1909999999999998},"last":2.1629999999999998},{"interface":"front","average":{"1min":2.8879999999999999,"5min":1.173,"15min":0.76300000000000001},"min":{"1min":0.33200000000000002,"5min":0.309,"15min":0.309},"max":{"1min":10.83,"5min":10.83,"15min":10.83},"last":0.33900000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.111,"15min":0.79200000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.1790000000000003,"5min":7.1790000000000003,"15min":7.1790000000000003},"last":5.1059999999999999},{"interface":"front","average":{"1min":1.857,"5min":1.1060000000000001,"15min":0.75800000000000001},"min":{"1min":0.30099999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.5090000000000003,"5min":7.5090000000000003,"15min":7.5090000000000003},"last":0.45400000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.72899999999999998,"15min":0.64900000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6130000000000004,"5min":4.6130000000000004,"15min":4.6130000000000004},"last":1.397},{"interface":"front","average":{"1min":1.04,"5min":0.73299999999999998,"15min":0.68200000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":3.637,"5min":3.637,"15min":3.637},"last":9.4359999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.51100000000000001,"15min":0.46700000000000003},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":1.958,"5min":2.7599999999999998,"15min":2.7599999999999998},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.70499999999999996,"15min":0.53600000000000003},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.4209999999999998,"5min":10.494,"15min":10.494},"last":1.0629999999999999}]},{"osd":6,"last update":"Sat Sep 23 17:55:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54200000000000004,"15min":0.50800000000000001},"min":{"1min":0.45400000000000001,"5min":0.33200000000000002,"15min":0.28899999999999998},"max":{"1min":0.64700000000000002,"5min":0.755,"15min":1.276},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.59299999999999997,"15min":0.54700000000000004},"min":{"1min":0.52500000000000002,"5min":0.29899999999999999,"15min":0.28999999999999998},"max":{"1min":0.79400000000000004,"5min":0.80900000000000005,"15min":1.1180000000000001},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 17:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.57399999999999995,"15min":0.55100000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.8370000000000002,"5min":2.8370000000000002,"15min":2.8370000000000002},"last":0.308},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.64400000000000002,"15min":0.56299999999999994},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":0.434}]}]},{"osd":5,"up_from":38,"seq":163208757427,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2698260,"kb_used_data":2603596,"kb_used_omap":0,"kb_used_meta":94592,"kb_avail":91046892,"statfs":{"total":95995035648,"available":93232017408,"internally_reserved":0,"allocated":2666082304,"data_stored":2665401439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":96862208},"hb_peers":[0,1,2,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":0.89400000000000002,"15min":0.68400000000000005},"min":{"1min":0.22,"5min":0.22,"15min":0.22},"max":{"1min":5.0650000000000004,"5min":6.5609999999999999,"15min":6.5609999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.163,"5min":1.083,"15min":0.751},"min":{"1min":0.29299999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":3.8220000000000001,"5min":8.2680000000000007,"15min":8.2680000000000007},"last":0.30299999999999999}]},{"osd":1,"last update":"Sat Sep 23 17:57:59 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":0.80700000000000005,"15min":0.746},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.3419999999999996,"5min":4.4279999999999999,"15min":4.4279999999999999},"last":1.4099999999999999},{"interface":"front","average":{"1min":2.153,"5min":1,"15min":0.71099999999999997},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":7.1719999999999997,"5min":7.1719999999999997,"15min":7.1719999999999997},"last":1.657}]},{"osd":2,"last update":"Sat Sep 23 17:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.401,"5min":1.278,"15min":0.78600000000000003},"min":{"1min":0.29399999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.0119999999999996,"5min":38.606999999999999,"15min":38.606999999999999},"last":2.952},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.3540000000000001,"15min":0.82899999999999996},"min":{"1min":0.32600000000000001,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":5.6639999999999997,"5min":39.670999999999999,"15min":39.670999999999999},"last":0.46899999999999997}]},{"osd":3,"last update":"Sat Sep 23 17:56:57 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":0.78800000000000003,"15min":0.66400000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.4969999999999999,"5min":7.4969999999999999,"15min":7.4969999999999999},"last":1.998},{"interface":"front","average":{"1min":1.0900000000000001,"5min":0.70199999999999996,"15min":0.63800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.5510000000000002,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":5.109}]},{"osd":4,"last update":"Sat Sep 23 17:58:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.70499999999999996,"15min":0.53200000000000003},"min":{"1min":0.22800000000000001,"5min":0.224,"15min":0.224},"max":{"1min":1.8580000000000001,"5min":6.5490000000000004,"15min":6.5490000000000004},"last":0.26900000000000002},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.59099999999999997,"15min":0.53600000000000003},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.093,"5min":4.093,"15min":4.093},"last":2.2109999999999999}]},{"osd":6,"last update":"Sat Sep 23 17:55:41 2023","interfaces":[{"interface":"back","average":{"1min":0.56799999999999995,"5min":0.54400000000000004,"15min":0.53500000000000003},"min":{"1min":0.38800000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.66100000000000003,"5min":0.85499999999999998,"15min":1.778},"last":2.1760000000000002},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.60699999999999998,"15min":0.58499999999999996},"min":{"1min":0.45800000000000002,"5min":0.375,"15min":0.33200000000000002},"max":{"1min":0.72799999999999998,"5min":0.90200000000000002,"15min":1.7629999999999999},"last":0.42699999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.90800000000000003,"5min":0.64800000000000002,"15min":0.55800000000000005},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.1719999999999997,"5min":4.1719999999999997,"15min":4.1719999999999997},"last":0.92900000000000005},{"interface":"front","average":{"1min":0.99199999999999999,"5min":0.67400000000000004,"15min":0.58699999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.5099999999999998,"5min":5.5099999999999998,"15min":5.5099999999999998},"last":1.823}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3024748544,"data_stored":3024748544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1785200640,"data_stored":1785200640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2072514560,"data_stored":2072510488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1253441536,"data_stored":1253441536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2336817152,"data_stored":2336817152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2644119552,"data_stored":2644115480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":230031360,"data_stored":230031360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1818103808,"data_stored":1818099736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:58:50.282 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T17:58:50.282 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 21 2023-09-23T17:58:50.456 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:50.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:50.468 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:50.569 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:50.644 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:58:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:58:50] "GET /metrics HTTP/1.1" 200 36553 "" "Prometheus/2.43.0" 2023-09-23T17:58:50.687 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:50.834 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T17:58:50.863 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:50.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:50.873 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:50.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:50 smithi055 ceph-mon[184174]: from='client.35046 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:58:50.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:50 smithi055 ceph-mon[179668]: from='client.35046 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:58:50.973 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:51.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:50 smithi099 ceph-mon[167076]: from='client.35046 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:58:51.129 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:51.333 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:51.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:51.362 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:51.436 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:58:51.463 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:51.485 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:58:51.495 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:51.526 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:58:51.533 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:51.537 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:51.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:51.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:51.601 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:58:51.640 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:51.643 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:58:51.650 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:51.694 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:58:51.700 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:58:51.700 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:58:51.700 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:58:51.700 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:58:51.700 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:58:51.700 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:58:51.700 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T17:58:51.701 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:58:51.735 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:51.852 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:51.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:51.860 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:51.890 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T17:58:51.961 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:52.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[179668]: pgmap v373: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 39934/213336 objects degraded (18.719%); 4308/213336 objects misplaced (2.019%); 25 MiB/s, 399 objects/s recovering 2023-09-23T17:58:52.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:52.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:52.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3757768242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:52.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:52.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:52.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1740529820' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:58:52.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:52.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1740529820' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:58:52.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[179668]: osdmap e168: 8 total, 6 up, 8 in 2023-09-23T17:58:52.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[184174]: pgmap v373: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 39934/213336 objects degraded (18.719%); 4308/213336 objects misplaced (2.019%); 25 MiB/s, 399 objects/s recovering 2023-09-23T17:58:52.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:52.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:52.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3757768242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:52.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:52.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:52.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1740529820' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:58:52.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:52.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1740529820' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:58:52.054 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:51 smithi055 ceph-mon[184174]: osdmap e168: 8 total, 6 up, 8 in 2023-09-23T17:58:52.054 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:51 smithi099 ceph-mon[167076]: pgmap v373: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.3 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 731 op/s; 39934/213336 objects degraded (18.719%); 4308/213336 objects misplaced (2.019%); 25 MiB/s, 399 objects/s recovering 2023-09-23T17:58:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3757768242' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:58:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:58:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1740529820' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:58:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:51 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:58:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1740529820' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:58:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:51 smithi099 ceph-mon[167076]: osdmap e168: 8 total, 6 up, 8 in 2023-09-23T17:58:52.152 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:52.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:52.158 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:52.258 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:52.366 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:52.519 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:52.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:52.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:52.624 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:52.662 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:52.757 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:52.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:52.763 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:52.819 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:58:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T17:58:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T17:58:52.819 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:58:52.864 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:52.911 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:52.985 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:52.989 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:58:52.989 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:58:53.000 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:53.000 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:53.101 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:53.138 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:52 smithi099 ceph-mon[167076]: failed to encode map e168 with expected crc 2023-09-23T17:58:53.138 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:52 smithi099 ceph-mon[167076]: failed to encode map e168 with expected crc 2023-09-23T17:58:53.139 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:52 smithi099 ceph-mon[167076]: failed to encode map e168 with expected crc 2023-09-23T17:58:53.139 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:52 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 39934/213336 objects degraded (18.719%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:53.139 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:52 smithi099 ceph-mon[167076]: osdmap e169: 8 total, 6 up, 8 in 2023-09-23T17:58:53.146 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:53.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:52 smithi055 ceph-mon[179668]: failed to encode map e168 with expected crc 2023-09-23T17:58:53.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:52 smithi055 ceph-mon[179668]: failed to encode map e168 with expected crc 2023-09-23T17:58:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:52 smithi055 ceph-mon[179668]: failed to encode map e168 with expected crc 2023-09-23T17:58:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:52 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 39934/213336 objects degraded (18.719%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:52 smithi055 ceph-mon[179668]: osdmap e169: 8 total, 6 up, 8 in 2023-09-23T17:58:53.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:52 smithi055 ceph-mon[184174]: failed to encode map e168 with expected crc 2023-09-23T17:58:53.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:52 smithi055 ceph-mon[184174]: failed to encode map e168 with expected crc 2023-09-23T17:58:53.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:52 smithi055 ceph-mon[184174]: failed to encode map e168 with expected crc 2023-09-23T17:58:53.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:52 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 39934/213336 objects degraded (18.719%), 20 pgs degraded, 16 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:53.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:52 smithi055 ceph-mon[184174]: osdmap e169: 8 total, 6 up, 8 in 2023-09-23T17:58:53.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T17:58:53.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:53.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:53.219 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:53.319 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:53.408 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:53.732 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:53.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:53.745 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:53.846 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:53.925 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:53.933 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:53 smithi055 ceph-mon[179668]: pgmap v375: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 39491/209412 objects degraded (18.858%); 4308/209412 objects misplaced (2.057%); 16 MiB/s, 256 objects/s recovering 2023-09-23T17:58:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:53 smithi055 ceph-mon[179668]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:53 smithi055 ceph-mon[179668]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:53 smithi055 ceph-mon[179668]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:53 smithi055 ceph-mon[179668]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:53 smithi055 ceph-mon[184174]: pgmap v375: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 39491/209412 objects degraded (18.858%); 4308/209412 objects misplaced (2.057%); 16 MiB/s, 256 objects/s recovering 2023-09-23T17:58:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:53 smithi055 ceph-mon[184174]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:53 smithi055 ceph-mon[184174]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:53 smithi055 ceph-mon[184174]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:53 smithi055 ceph-mon[184174]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.191 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:53 smithi099 ceph-mon[167076]: pgmap v375: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 4 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 39491/209412 objects degraded (18.858%); 4308/209412 objects misplaced (2.057%); 16 MiB/s, 256 objects/s recovering 2023-09-23T17:58:54.191 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:53 smithi099 ceph-mon[167076]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.191 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:53 smithi099 ceph-mon[167076]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.191 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:53 smithi099 ceph-mon[167076]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.191 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:53 smithi099 ceph-mon[167076]: failed to encode map e169 with expected crc 2023-09-23T17:58:54.523 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T17:58:54.837 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:54.854 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:54.954 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:54.990 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:58:55.000 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:55.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:55.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:55.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:55.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:54 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 5 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:58:55.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:55.247 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:55.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:55.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:55.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:55.436 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:54 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 5 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:58:55.436 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:54 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 5 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:58:55.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:55.470 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:58:55.470 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T17:58:55.470 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T17:58:55.470 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T17:58:55.471 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:58:55.515 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:55.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:55.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:55.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:55.832 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:55.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T17:58:55.935 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:56.081 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:56.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:56.089 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:56.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:55 smithi055 ceph-mon[179668]: pgmap v377: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 36340/203928 objects degraded (17.820%); 4094/203928 objects misplaced (2.008%); 32 MiB/s, 508 objects/s recovering 2023-09-23T17:58:56.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:55 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:56.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:55 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:56.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:55 smithi055 ceph-mon[184174]: pgmap v377: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 36340/203928 objects degraded (17.820%); 4094/203928 objects misplaced (2.008%); 32 MiB/s, 508 objects/s recovering 2023-09-23T17:58:56.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:55 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:56.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:55 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:56.189 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:56.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:55 smithi099 ceph-mon[167076]: pgmap v377: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 787 op/s; 36340/203928 objects degraded (17.820%); 4094/203928 objects misplaced (2.008%); 32 MiB/s, 508 objects/s recovering 2023-09-23T17:58:56.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:55 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:56.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:55 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:56.217 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:56.266 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:56.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:56.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:56.371 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:56.429 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:56.556 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:56.750 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:56.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:56.760 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:56.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:58:56.896 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:56.956 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:57.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:58:57.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:58:57.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:57.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:58:57.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:58:57.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:57.018 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:58:57.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:58:57.047 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T17:58:57.047 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T17:58:57.047 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:58:57.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:57.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:58:57.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:58:57.203 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:57.335 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:58:57.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:57.344 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:58:57.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:58:57.498 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:57.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:58:57.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:57.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:58:57.655 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:58:57.753 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:57.903 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:58:57.903 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T17:58:57.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T17:58:57.903 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T17:58:57.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.903 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:38884 5.7 5:e88075de:::benchmark_data_smithi055_229117_object38883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:58:57.903 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.244864+0000", 2023-09-23T17:58:57.903 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 162.647112093, 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.111870281, 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 38884 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244864+0000", 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244864+0000", 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244864+0000", 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:57.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244895+0000", 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1819999999999997e-05 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244904+0000", 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.524e-06 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244909+0000", 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7299999999999996e-06 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246498+0000", 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015898010000000001 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246614+0000", 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011574399999999999 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246739+0000", 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012482699999999999 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.249667+0000", 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002927972 2023-09-23T17:58:57.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.249704+0000", 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7271000000000003e-05 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356593+0000", 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.106889052 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356695+0000", 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000101992 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356734+0000", 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9084000000000001e-05 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:38888 5.7 5:e84cc5a2:::benchmark_data_smithi055_229117_object38887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.248150+0000", 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 162.64382646999999, 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.11199930399999999, 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 38888 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.911 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248150+0000", 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248150+0000", 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248149+0000", 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248203+0000", 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3788000000000001e-05 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248213+0000", 2023-09-23T17:58:57.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0548e-05 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248218+0000", 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7620000000000003e-06 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248262+0000", 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4316000000000001e-05 2023-09-23T17:58:57.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248394+0000", 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013165099999999999 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248541+0000", 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000147482 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.249786+0000", 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001244277 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.251395+0000", 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016097609999999999 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.360065+0000", 2023-09-23T17:58:57.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.108669425 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.360114+0000", 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8878000000000002e-05 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.360149+0000", 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.54e-05 2023-09-23T17:58:57.917 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:104782 5.f 5:f2d70369:::benchmark_data_smithi055_229117_object104781:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e144)", 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:26.959325+0000", 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 90.932650835000004, 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.086671073000000001, 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.918 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 104782 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.959325+0000", 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.959325+0000", 2023-09-23T17:58:57.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.959325+0000", 2023-09-23T17:58:57.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:57.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.959382+0000", 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7130999999999999e-05 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.959394+0000", 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1829e-05 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.959400+0000", 2023-09-23T17:58:57.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1850000000000001e-06 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.962852+0000", 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034515850000000001 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.963015+0000", 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016342799999999999 2023-09-23T17:58:57.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.963177+0000", 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000162137 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.966354+0000", 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00317703 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:26.967112+0000", 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00075755699999999996 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:27.044189+0000", 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.077077655999999994 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:27.044231+0000", 2023-09-23T17:58:57.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2169999999999998e-05 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:27.045996+0000", 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0017649320000000001 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:110922 5.f 5:f30f8f7c:::benchmark_data_smithi055_229117_object5296:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T17:58:57.925 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:35.422405+0000", 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 82.469570988000001, 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.087947956999999993, 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 110922 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.422405+0000", 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.422405+0000", 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.422404+0000", 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.422411+0000", 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3769999999999998e-06 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.422426+0000", 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5311e-05 2023-09-23T17:58:57.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.422434+0000", 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2490000000000004e-06 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.439169+0000", 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016735314000000001 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.439889+0000", 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00071998899999999998 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.452214+0000", 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012325447 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.452319+0000", 2023-09-23T17:58:57.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104979 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.452472+0000", 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015279599999999999 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.457382+0000", 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049100070000000001 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.457887+0000", 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00050510400000000003 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.508628+0000", 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.050740873999999998 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.508665+0000", 2023-09-23T17:58:57.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6962999999999998e-05 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.510353+0000", 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001688092 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.933 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:111180 5.f 5:f3bb8a9d:::benchmark_data_smithi055_229117_object5554:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:35.804161+0000", 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 82.087814778999999, 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.080594023000000001, 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 111180 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804161+0000", 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804161+0000", 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804160+0000", 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804166+0000", 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2320000000000001e-06 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804181+0000", 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5597999999999999e-05 2023-09-23T17:58:57.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804188+0000", 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1740000000000003e-06 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.825374+0000", 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021185374999999999 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.826089+0000", 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00071500600000000002 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.847542+0000", 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021453574999999999 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.847635+0000", 2023-09-23T17:58:57.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2301999999999998e-05 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.847723+0000", 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8838000000000006e-05 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.865879+0000", 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018155907999999998 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.882225+0000", 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016346045 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.882260+0000", 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4177000000000002e-05 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.882318+0000", 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7978e-05 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.884755+0000", 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002437846 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:111181 5.f 5:fde29c90:::benchmark_data_smithi055_229117_object5555:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T17:58:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:35.804194+0000", 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 82.087781707000005, 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.082118297000000007, 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 111181 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804194+0000", 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804194+0000", 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804194+0000", 2023-09-23T17:58:57.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804197+0000", 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3069999999999998e-06 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804203+0000", 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2979999999999999e-06 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.804207+0000", 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1799999999999998e-06 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.826115+0000", 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021908249000000001 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:57.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.826736+0000", 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00062087899999999996 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.854986+0000", 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028249641999999998 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.855057+0000", 2023-09-23T17:58:57.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1237999999999998e-05 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.855148+0000", 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1351e-05 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.870750+0000", 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015602039999999999 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.884803+0000", 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014052866000000001 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.884846+0000", 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3245999999999999e-05 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.884911+0000", 2023-09-23T17:58:57.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4426999999999998e-05 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:35.886313+0000", 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014020199999999999 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:119891 5.f 5:fa4cbfd7:::benchmark_data_smithi055_229117_object14265:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T17:58:57.949 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:49.208682+0000", 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.683293746999993, 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.102729686, 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 119891 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208682+0000", 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208682+0000", 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208681+0000", 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208687+0000", 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0780000000000001e-06 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208700+0000", 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2958e-05 2023-09-23T17:58:57.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208705+0000", 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1549999999999998e-06 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.246600+0000", 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037894272 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.247270+0000", 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00066982499999999998 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.269068+0000", 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021798858000000001 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.269160+0000", 2023-09-23T17:58:57.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1817000000000005e-05 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.269293+0000", 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132445 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.279028+0000", 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097349380000000003 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.291497+0000", 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012469502 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.309305+0000", 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017807699 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.309366+0000", 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.101e-05 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.311412+0000", 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020463360000000002 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:119895 5.f 5:f7a66fc1:::benchmark_data_smithi055_229117_object14269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T17:58:57.957 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:49.208732+0000", 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.683244549999998, 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.107236875, 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 119895 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208732+0000", 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208732+0000", 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208731+0000", 2023-09-23T17:58:57.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208734+0000", 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3020000000000002e-06 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208739+0000", 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6590000000000004e-06 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208743+0000", 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5049999999999999e-06 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.247298+0000", 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.038555566999999999 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:57.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.248114+0000", 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00081573400000000003 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.275080+0000", 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.026966034 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.275177+0000", 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7153999999999998e-05 2023-09-23T17:58:57.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.275295+0000", 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117398 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.291456+0000", 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016161406999999999 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.291531+0000", 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5284000000000006e-05 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.314013+0000", 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022481944 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.314078+0000", 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5141000000000005e-05 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.315968+0000", 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018901160000000001 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.965 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:119901 5.f 5:ff2468ba:::benchmark_data_smithi055_229117_object14275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:49.217244+0000", 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.674731922999996, 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.10684498100000001, 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 119901 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217244+0000", 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217244+0000", 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217243+0000", 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217248+0000", 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9840000000000001e-06 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217262+0000", 2023-09-23T17:58:57.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3995000000000001e-05 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217269+0000", 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6540000000000002e-06 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.248139+0000", 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.030870057999999999 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.248661+0000", 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00052204900000000002 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.282008+0000", 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.033347158000000002 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.282111+0000", 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000103018 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.282254+0000", 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000142965 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.291538+0000", 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0092843200000000004 2023-09-23T17:58:57.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.291573+0000", 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4765999999999999e-05 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.322516+0000", 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.030942648 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.322567+0000", 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0631999999999999e-05 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.324089+0000", 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015227299999999999 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.973 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:119916 5.f 5:f01c1c5e:::benchmark_data_smithi055_229117_object14290:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:49.240812+0000", 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.651163957999998, 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.084711049999999996, 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 119916 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.240812+0000", 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.240812+0000", 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.240811+0000", 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.240817+0000", 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5539999999999997e-06 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.240830+0000", 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3549e-05 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.240835+0000", 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2700000000000004e-06 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.248688+0000", 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0078529680000000001 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:57.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.249285+0000", 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000596409 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.288445+0000", 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.039159897999999999 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.288517+0000", 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2664999999999996e-05 2023-09-23T17:58:57.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.288613+0000", 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5694999999999998e-05 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.292265+0000", 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0036523630000000001 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.295901+0000", 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0036357709999999999 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.324137+0000", 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028236221999999998 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.324179+0000", 2023-09-23T17:58:57.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1974999999999997e-05 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.325523+0000", 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013439400000000001 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:120284 5.f 5:f2add13b:::benchmark_data_smithi055_229117_object14658:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T17:58:57.981 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:49.708202+0000", 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.183773822000006, 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.086254085999999994, 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 120284 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.708202+0000", 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.708202+0000", 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.708201+0000", 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.708207+0000", 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5230000000000003e-06 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.708219+0000", 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2371000000000001e-05 2023-09-23T17:58:57.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.708225+0000", 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3249999999999996e-06 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.708272+0000", 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6687000000000002e-05 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.708929+0000", 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00065715800000000003 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.766008+0000", 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.057078799 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.766113+0000", 2023-09-23T17:58:57.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104627 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.766263+0000", 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015026899999999999 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.775821+0000", 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095580000000000005 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.777895+0000", 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020737749999999999 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.792549+0000", 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014654311 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.792625+0000", 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6223999999999994e-05 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.794456+0000", 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018312700000000001 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.989 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:120654 5.f 5:f2f3b65c:::benchmark_data_smithi055_229117_object15028:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:50.216588+0000", 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 67.675387862999997, 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.089080090000000001, 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 120654 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:57.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.216588+0000", 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.216588+0000", 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:57.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.216587+0000", 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.216593+0000", 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1249999999999998e-06 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.216606+0000", 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.313e-05 2023-09-23T17:58:57.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.216613+0000", 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0679999999999999e-06 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.223366+0000", 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0067529089999999996 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.224103+0000", 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00073647800000000005 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.282805+0000", 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.058702327999999998 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:57.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.282867+0000", 2023-09-23T17:58:57.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2150000000000006e-05 2023-09-23T17:58:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.282951+0000", 2023-09-23T17:58:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3341e-05 2023-09-23T17:58:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.290753+0000", 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0078026559999999998 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.290763+0000", 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8740000000000007e-06 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:57.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.303725+0000", 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012961434000000001 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.303774+0000", 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9341000000000001e-05 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:50.305668+0000", 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018944820000000001 2023-09-23T17:58:57.999 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:125249 5.f 5:f441b8a8:::benchmark_data_smithi055_229117_object19623:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e160)", 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:26.613350+0000", 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.278626151000001, 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.093854162000000005, 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:58.000 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 125249 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.613350+0000", 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.613350+0000", 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.613349+0000", 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.613354+0000", 2023-09-23T17:58:58.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5759999999999999e-06 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.613361+0000", 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3590000000000002e-06 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.613365+0000", 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9580000000000001e-06 2023-09-23T17:58:58.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.613392+0000", 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6630000000000001e-05 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.613481+0000", 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9446999999999999e-05 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.613568+0000", 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6458000000000005e-05 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.706022+0000", 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.092454819999999993 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.706030+0000", 2023-09-23T17:58:58.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0209999999999999e-06 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.706068+0000", 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7524000000000002e-05 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.706124+0000", 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6437999999999997e-05 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.707204+0000", 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010798470000000001 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.44681.0:125241 5.b e160/110 5:d8ad9291:::benchmark_data_smithi055_229117_object19615:head v 160'7771, mlcod=160'7769)", 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:26.614403+0000", 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.277573461999999, 2023-09-23T17:58:58.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.090765199000000005, 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614403+0000", 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614403+0000", 2023-09-23T17:58:58.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614402+0000", 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614411+0000", 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9570000000000008e-06 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614421+0000", 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0054999999999999e-05 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614425+0000", 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2069999999999997e-06 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:58.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614464+0000", 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8801999999999999e-05 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614481+0000", 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.721e-05 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705138+0000", 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.090656758000000004 2023-09-23T17:58:58.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705168+0000", 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9700999999999999e-05 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.44681.0:125242 5.b e160/110 5:dc7ce4b5:::benchmark_data_smithi055_229117_object19616:head v 160'7772, mlcod=160'7769)", 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:26.614438+0000", 2023-09-23T17:58:58.012 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.277538098000001, 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.090750026999999997, 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614438+0000", 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:58.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614438+0000", 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614438+0000", 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614443+0000", 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8220000000000003e-06 2023-09-23T17:58:58.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614449+0000", 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8880000000000002e-06 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614453+0000", 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2780000000000002e-06 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617322+0000", 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028693920000000001 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617341+0000", 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9313e-05 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705170+0000", 2023-09-23T17:58:58.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.087829081000000003 2023-09-23T17:58:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705188+0000", 2023-09-23T17:58:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7699000000000001e-05 2023-09-23T17:58:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "MOSDPGPushReply(5.f 160/127 [PushReplyOp(5:f08a311e:::benchmark_data_smithi055_229117_object45194:head)])", 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:26.614456+0000", 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.277520522, 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.091522088000000001, 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614456+0000", 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614456+0000", 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614455+0000", 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967296 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614459+0000", 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4769999999999999e-06 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614465+0000", 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4219999999999997e-06 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614469+0000", 2023-09-23T17:58:58.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8990000000000003e-06 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.616670+0000", 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022008269999999998 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705978+0000", 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.089307676000000003 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.021 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.44681.0:125247 5.6 e160/150 5:6cb22ff7:::benchmark_data_smithi055_229117_object19621:head v 160'7988, mlcod=160'7985)", 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:26.614595+0000", 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.277381506000001, 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.090545525000000002, 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:58.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614595+0000", 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614595+0000", 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:58.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614594+0000", 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614598+0000", 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0409999999999999e-06 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614605+0000", 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5139999999999998e-06 2023-09-23T17:58:58.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614607+0000", 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7099999999999999e-06 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614641+0000", 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3864e-05 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.614654+0000", 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2655e-05 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617728+0000", 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030742719999999999 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705140+0000", 2023-09-23T17:58:58.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.087411919000000005 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.44681.0:125249 5.f e160/127 ondisk, result = 0)", 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:26.616793+0000", 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.275182695000002, 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.089237721000000006, 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T17:58:58.027 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.616793+0000", 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.616793+0000", 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.616792+0000", 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.616799+0000", 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5230000000000001e-06 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:58.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.616809+0000", 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0265000000000001e-05 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.616815+0000", 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.677e-06 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.706006+0000", 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.089191332999999998 2023-09-23T17:58:58.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.706015+0000", 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4829999999999999e-06 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.706031+0000", 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6507e-05 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:58.031 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.44681.0:125243 5.b e160/110 5:dd4dc99e:::benchmark_data_smithi055_229117_object19617:head v 160'7773, mlcod=160'7769)", 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:26.617777+0000", 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.27419926, 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.093860759000000002, 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:58.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617777+0000", 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617777+0000", 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617776+0000", 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:58:58.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617782+0000", 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5339999999999999e-06 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705124+0000", 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.087341910999999994 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705143+0000", 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9142000000000001e-05 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705213+0000", 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9905000000000002e-05 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705229+0000", 2023-09-23T17:58:58.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6130999999999999e-05 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.711604+0000", 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0063746469999999998 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.711638+0000", 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4147999999999999e-05 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.036 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.44681.0:125254 5.2 e160/114 5:4369d7f0:::benchmark_data_smithi055_229117_object19628:head v 160'7676, mlcod=160'7674)", 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:26.627301+0000", 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.264675214, 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.081158637000000006, 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:58:58.037 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.627301+0000", 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.627301+0000", 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.627299+0000", 2023-09-23T17:58:58.038 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.627313+0000", 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3315999999999999e-05 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705125+0000", 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.077812022999999994 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705136+0000", 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1178999999999999e-05 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705176+0000", 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0404000000000001e-05 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705197+0000", 2023-09-23T17:58:58.040 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1104e-05 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.708419+0000", 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032218540000000001 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.708460+0000", 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0354e-05 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:58.041 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:58:58.042 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:58:58.042 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:58:58.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:58.043 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:58:58.043 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:58:58.064 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 1 pg_num to 21 2023-09-23T17:58:58.145 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:58.187 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:58 smithi055 ceph-mon[179668]: pgmap v378: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 36133/202710 objects degraded (17.825%); 4094/202710 objects misplaced (2.020%); 24 MiB/s, 383 objects/s recovering 2023-09-23T17:58:58.188 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:58 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/918809027' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T17:58:58.188 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:58 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T17:58:58.188 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 36133/202710 objects degraded (17.825%), 19 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:58.188 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:58 smithi055 ceph-mon[184174]: pgmap v378: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 36133/202710 objects degraded (17.825%); 4094/202710 objects misplaced (2.020%); 24 MiB/s, 383 objects/s recovering 2023-09-23T17:58:58.188 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:58 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/918809027' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T17:58:58.188 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:58 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T17:58:58.188 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 36133/202710 objects degraded (17.825%), 19 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:58.249 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:58.283 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:58 smithi099 ceph-mon[167076]: pgmap v378: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 697 op/s; 36133/202710 objects degraded (17.825%); 4094/202710 objects misplaced (2.020%); 24 MiB/s, 383 objects/s recovering 2023-09-23T17:58:58.283 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:58 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/918809027' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T17:58:58.283 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:58 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]: dispatch 2023-09-23T17:58:58.283 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 36133/202710 objects degraded (17.825%), 19 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:58:58.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T17:58:58.380 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:58.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:58.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:58:58.494 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:58:58.561 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:58.770 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:58:58.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:58.780 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:58:58.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:58.933 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:58.998 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:59.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:59.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:59.049 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:58:59.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:59.136 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:59.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:59 smithi055 ceph-mon[179668]: failed to encode map e170 with expected crc 2023-09-23T17:58:59.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:59 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T17:58:59.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:59 smithi055 ceph-mon[179668]: osdmap e170: 8 total, 6 up, 8 in 2023-09-23T17:58:59.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:58:59 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T17:58:59.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:59 smithi055 ceph-mon[184174]: failed to encode map e170 with expected crc 2023-09-23T17:58:59.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:59 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T17:58:59.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:59 smithi055 ceph-mon[184174]: osdmap e170: 8 total, 6 up, 8 in 2023-09-23T17:58:59.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:58:59 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T17:58:59.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:59.191 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:59 smithi099 ceph-mon[167076]: failed to encode map e170 with expected crc 2023-09-23T17:58:59.191 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:59 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "21"}]': finished 2023-09-23T17:58:59.191 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:59 smithi099 ceph-mon[167076]: osdmap e170: 8 total, 6 up, 8 in 2023-09-23T17:58:59.191 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:58:59 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]: dispatch 2023-09-23T17:58:59.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:59.192 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:59.201 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:58:59.256 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:58:59.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:58:59.305 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:58:59.354 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:59.360 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:58:59.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:58:59.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:59.399 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:58:59.402 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:58:59.432 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:58:59.443 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:58:59.443 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:58:59.443 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:58:59.443 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:58:59.443 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:58:59.443 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:58:59.443 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T17:58:59.444 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:58:59.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:58:59.540 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:59.633 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:58:59.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:59.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:58:59.740 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:58:59.841 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:58:59.922 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:58:59.922 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:58:59.942 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:58:59.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:58:59.947 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:00.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:00.086 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[179668]: pgmap v380: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 35953/199875 objects degraded (17.988%); 3885/199875 objects misplaced (1.944%); 22 MiB/s, 358 objects/s recovering 2023-09-23T17:59:00.086 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[179668]: failed to encode map e171 with expected crc 2023-09-23T17:59:00.086 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T17:59:00.086 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[179668]: osdmap e171: 8 total, 6 up, 8 in 2023-09-23T17:59:00.086 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[179668]: failed to encode map e171 with expected crc 2023-09-23T17:59:00.087 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2202372456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:00.087 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2255903778' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:00.087 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:00.095 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T17:59:00.103 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:00.201 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:00.204 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:00 smithi099 ceph-mon[167076]: pgmap v380: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 35953/199875 objects degraded (17.988%); 3885/199875 objects misplaced (1.944%); 22 MiB/s, 358 objects/s recovering 2023-09-23T17:59:00.204 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:00 smithi099 ceph-mon[167076]: failed to encode map e171 with expected crc 2023-09-23T17:59:00.205 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:00 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T17:59:00.205 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:00 smithi099 ceph-mon[167076]: osdmap e171: 8 total, 6 up, 8 in 2023-09-23T17:59:00.205 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:00 smithi099 ceph-mon[167076]: failed to encode map e171 with expected crc 2023-09-23T17:59:00.205 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2202372456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:00.205 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2255903778' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:00.205 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:00 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:00.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:00.206 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:00.307 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:00.361 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:59:00] "GET /metrics HTTP/1.1" 200 36557 "" "Prometheus/2.43.0" 2023-09-23T17:59:00.361 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[184174]: pgmap v380: 99 pgs: 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 42 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 4.1 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 35953/199875 objects degraded (17.988%); 3885/199875 objects misplaced (1.944%); 22 MiB/s, 358 objects/s recovering 2023-09-23T17:59:00.361 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[184174]: failed to encode map e171 with expected crc 2023-09-23T17:59:00.361 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "21"}]': finished 2023-09-23T17:59:00.361 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[184174]: osdmap e171: 8 total, 6 up, 8 in 2023-09-23T17:59:00.361 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[184174]: failed to encode map e171 with expected crc 2023-09-23T17:59:00.361 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2202372456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:00.361 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2255903778' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:00.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:00 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:00.373 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:00.468 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:59:00.469 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:59:00.508 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:00.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:00.518 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:00.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:00.673 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:00.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:00.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:00.812 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:00.862 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T17:59:00.913 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:00.984 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:01.065 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:01.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:01.070 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:01.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:01.210 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:01.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:01.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:01.270 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:01.371 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:01.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:01.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[184174]: failed to encode map e172 with expected crc 2023-09-23T17:59:01.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:01.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[184174]: osdmap e172: 8 total, 6 up, 8 in 2023-09-23T17:59:01.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[184174]: failed to encode map e172 with expected crc 2023-09-23T17:59:01.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[184174]: failed to encode map e172 with expected crc 2023-09-23T17:59:01.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:01.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[179668]: failed to encode map e172 with expected crc 2023-09-23T17:59:01.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:01.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[179668]: osdmap e172: 8 total, 6 up, 8 in 2023-09-23T17:59:01.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[179668]: failed to encode map e172 with expected crc 2023-09-23T17:59:01.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:01 smithi055 ceph-mon[179668]: failed to encode map e172 with expected crc 2023-09-23T17:59:01.434 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:01.508 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:01.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:01.518 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:01.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:01 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:01 smithi099 ceph-mon[167076]: failed to encode map e172 with expected crc 2023-09-23T17:59:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:01 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:01 smithi099 ceph-mon[167076]: osdmap e172: 8 total, 6 up, 8 in 2023-09-23T17:59:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:01 smithi099 ceph-mon[167076]: failed to encode map e172 with expected crc 2023-09-23T17:59:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:01 smithi099 ceph-mon[167076]: failed to encode map e172 with expected crc 2023-09-23T17:59:01.604 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:59:01.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:01.663 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:01.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:01.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:01.766 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:01.867 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:01.923 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:59:01.956 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:02.028 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:02.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:02.033 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:02.134 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:02.211 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:02 smithi099 ceph-mon[167076]: pgmap v383: 109 pgs: 10 unknown, 8 peering, 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 34 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 33878/193944 objects degraded (17.468%); 3885/193944 objects misplaced (2.003%); 12 MiB/s, 199 objects/s recovering 2023-09-23T17:59:02.216 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:02.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:02 smithi055 ceph-mon[184174]: pgmap v383: 109 pgs: 10 unknown, 8 peering, 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 34 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 33878/193944 objects degraded (17.468%); 3885/193944 objects misplaced (2.003%); 12 MiB/s, 199 objects/s recovering 2023-09-23T17:59:02.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:02 smithi055 ceph-mon[179668]: pgmap v383: 109 pgs: 10 unknown, 8 peering, 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 34 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 553 op/s; 33878/193944 objects degraded (17.468%); 3885/193944 objects misplaced (2.003%); 12 MiB/s, 199 objects/s recovering 2023-09-23T17:59:02.444 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:02.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:02.447 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:02.548 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:02.568 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T17:59:02.623 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:02.705 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:02.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:02.713 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:02.779 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:02.814 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:02.912 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:02.995 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T17:59:02.995 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:59:03.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 33878/193944 objects degraded (17.468%), 19 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:03.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:03 smithi055 ceph-mon[184174]: osdmap e173: 8 total, 6 up, 8 in 2023-09-23T17:59:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 33878/193944 objects degraded (17.468%), 19 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:03 smithi055 ceph-mon[179668]: osdmap e173: 8 total, 6 up, 8 in 2023-09-23T17:59:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 33878/193944 objects degraded (17.468%), 19 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:03 smithi099 ceph-mon[167076]: osdmap e173: 8 total, 6 up, 8 in 2023-09-23T17:59:03.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T17:59:03.824 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 5, 6, 7, 4] out_osds: [] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T17:59:03.825 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:59:03.825 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [0, 1, 2, 3, 5, 6, 7, 4] 2023-09-23T17:59:03.825 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T17:59:04.026 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:59:04.076 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:04.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:04.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:04.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:04.243 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:04.298 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:04.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:04.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:04.403 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:04.433 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:04 smithi099 ceph-mon[167076]: pgmap v384: 109 pgs: 10 unknown, 8 peering, 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 34 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 33416/189732 objects degraded (17.612%); 3885/189732 objects misplaced (2.048%); 12 MiB/s, 197 objects/s recovering 2023-09-23T17:59:04.433 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:04 smithi099 ceph-mon[167076]: failed to encode map e173 with expected crc 2023-09-23T17:59:04.433 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:04 smithi099 ceph-mon[167076]: osdmap e174: 8 total, 6 up, 8 in 2023-09-23T17:59:04.433 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:04 smithi099 ceph-mon[167076]: failed to encode map e174 with expected crc 2023-09-23T17:59:04.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:04 smithi055 ceph-mon[184174]: pgmap v384: 109 pgs: 10 unknown, 8 peering, 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 34 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 33416/189732 objects degraded (17.612%); 3885/189732 objects misplaced (2.048%); 12 MiB/s, 197 objects/s recovering 2023-09-23T17:59:04.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:04 smithi055 ceph-mon[184174]: failed to encode map e173 with expected crc 2023-09-23T17:59:04.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:04 smithi055 ceph-mon[184174]: osdmap e174: 8 total, 6 up, 8 in 2023-09-23T17:59:04.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:04 smithi055 ceph-mon[184174]: failed to encode map e174 with expected crc 2023-09-23T17:59:04.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:04 smithi055 ceph-mon[179668]: pgmap v384: 109 pgs: 10 unknown, 8 peering, 1 down, 2 active+undersized+degraded+remapped+backfilling, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 34 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 3.9 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 720 op/s; 33416/189732 objects degraded (17.612%); 3885/189732 objects misplaced (2.048%); 12 MiB/s, 197 objects/s recovering 2023-09-23T17:59:04.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:04 smithi055 ceph-mon[179668]: failed to encode map e173 with expected crc 2023-09-23T17:59:04.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:04 smithi055 ceph-mon[179668]: osdmap e174: 8 total, 6 up, 8 in 2023-09-23T17:59:04.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:04 smithi055 ceph-mon[179668]: failed to encode map e174 with expected crc 2023-09-23T17:59:04.436 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:04.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:04.793 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:04.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:04.895 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:04.947 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:05.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:05.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:05.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:05.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:05.200 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:05.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:05 smithi099 ceph-mon[167076]: failed to encode map e174 with expected crc 2023-09-23T17:59:05.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:05 smithi099 ceph-mon[167076]: failed to encode map e174 with expected crc 2023-09-23T17:59:05.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:05 smithi055 ceph-mon[179668]: failed to encode map e174 with expected crc 2023-09-23T17:59:05.386 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:05 smithi055 ceph-mon[179668]: failed to encode map e174 with expected crc 2023-09-23T17:59:05.386 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:05 smithi055 ceph-mon[184174]: failed to encode map e174 with expected crc 2023-09-23T17:59:05.386 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:05 smithi055 ceph-mon[184174]: failed to encode map e174 with expected crc 2023-09-23T17:59:05.674 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:05.742 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:06.309 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:06 smithi055 ceph-mon[184174]: pgmap v387: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 down, 1 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 31067/183576 objects degraded (16.923%); 3675/183576 objects misplaced (2.002%); 26 MiB/s, 410 objects/s recovering 2023-09-23T17:59:06.309 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:06 smithi055 ceph-mon[179668]: pgmap v387: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 down, 1 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 31067/183576 objects degraded (16.923%); 3675/183576 objects misplaced (2.002%); 26 MiB/s, 410 objects/s recovering 2023-09-23T17:59:06.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:06 smithi099 ceph-mon[167076]: pgmap v387: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 down, 1 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 750 op/s; 31067/183576 objects degraded (16.923%); 3675/183576 objects misplaced (2.002%); 26 MiB/s, 410 objects/s recovering 2023-09-23T17:59:06.810 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:59:06.855 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:59:06.899 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:59:06.941 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:59:06.993 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:59:07.031 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:59:07.038 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:59:07.039 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:59:07.039 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:59:07.039 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:59:07.039 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:59:07.039 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:59:07.039 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T17:59:07.039 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:59:07.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:59:07.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:07.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:07 smithi055 ceph-mon[179668]: pgmap v388: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 down, 1 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 30851/182283 objects degraded (16.925%); 3675/182283 objects misplaced (2.016%); 18 MiB/s, 286 objects/s recovering 2023-09-23T17:59:07.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:07 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3011693954' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:07.656 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:59:07.656 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:59:07.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:59:07.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:07.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:07 smithi055 ceph-mon[184174]: pgmap v388: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 down, 1 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 30851/182283 objects degraded (16.925%); 3675/182283 objects misplaced (2.016%); 18 MiB/s, 286 objects/s recovering 2023-09-23T17:59:07.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:07 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3011693954' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:07.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:59:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:07 smithi099 ceph-mon[167076]: pgmap v388: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 down, 1 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.7 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 602 op/s; 30851/182283 objects degraded (16.925%); 3675/182283 objects misplaced (2.016%); 18 MiB/s, 286 objects/s recovering 2023-09-23T17:59:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:07 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3011693954' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:07.945 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:07.952 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:07.952 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:08.053 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:08.092 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:08.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:08 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T17:59:08.307+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e174 definitely_dead 0 2023-09-23T17:59:08.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 30851/182283 objects degraded (16.925%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:08.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1952620140' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T17:59:08.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:08 smithi055 ceph-mon[179668]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T17:59:08.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 30851/182283 objects degraded (16.925%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:08.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1952620140' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T17:59:08.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:08 smithi055 ceph-mon[184174]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T17:59:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 30851/182283 objects degraded (16.925%), 18 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1952620140' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T17:59:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:08 smithi099 ceph-mon[167076]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T17:59:08.866 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.5. 2023-09-23T17:59:08.875 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T17:59:09.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:09.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:09.058 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:09.159 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:09.194 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:09.379 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:09.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:09.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:09.483 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:09.512 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:09.547 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:09.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:09.551 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:09.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:09.657 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:59:09.686 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:09.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1629147345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:59:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:09 smithi099 ceph-mon[167076]: pgmap v389: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 down, 1 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 30637/179178 objects degraded (17.099%); 3473/179178 objects misplaced (1.938%); 14 MiB/s, 230 objects/s recovering 2023-09-23T17:59:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:09 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:09 smithi099 ceph-mon[167076]: failed to encode map e175 with expected crc 2023-09-23T17:59:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1952620140' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T17:59:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1629147345' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:59:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:09 smithi099 ceph-mon[167076]: osdmap e175: 8 total, 6 up, 7 in 2023-09-23T17:59:09.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:09.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:09.881 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:09.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1629147345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:59:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[184174]: pgmap v389: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 down, 1 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 30637/179178 objects degraded (17.099%); 3473/179178 objects misplaced (1.938%); 14 MiB/s, 230 objects/s recovering 2023-09-23T17:59:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[184174]: failed to encode map e175 with expected crc 2023-09-23T17:59:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1952620140' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T17:59:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1629147345' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:59:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[184174]: osdmap e175: 8 total, 6 up, 7 in 2023-09-23T17:59:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1629147345' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:59:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[179668]: pgmap v389: 109 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 down, 1 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 3 undersized+peered, 29 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 13 active+undersized+degraded; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 614 op/s; 30637/179178 objects degraded (17.099%); 3473/179178 objects misplaced (1.938%); 14 MiB/s, 230 objects/s recovering 2023-09-23T17:59:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[179668]: failed to encode map e175 with expected crc 2023-09-23T17:59:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1952620140' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T17:59:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1629147345' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:59:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:09 smithi055 ceph-mon[179668]: osdmap e175: 8 total, 6 up, 7 in 2023-09-23T17:59:09.982 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:10.012 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:10.054 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:10.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:10.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:10.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:10.187 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:10.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:10.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:10.224 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:10.284 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T17:59:10.284 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T17:59:10.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T17:59:10.285 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T17:59:10.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.285 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131500 5.2 5.9981e372 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T17:59:10.285 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.276336+0000", 2023-09-23T17:59:10.285 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 36.005672089000001, 2023-09-23T17:59:10.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94858491499999997, 2023-09-23T17:59:10.285 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.285 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T17:59:10.285 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131500 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276336+0000", 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276336+0000", 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276335+0000", 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276340+0000", 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2310000000000004e-06 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276348+0000", 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.303e-06 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276356+0000", 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9359999999999999e-06 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276427+0000", 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0314000000000005e-05 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276438+0000", 2023-09-23T17:59:10.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1035e-05 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224903+0000", 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94846568499999995 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224921+0000", 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7461000000000001e-05 2023-09-23T17:59:10.290 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131508 5.2 5.89ad37e2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.286747+0000", 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 35.995261370000001, 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.93821040499999997, 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.291 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131508 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286747+0000", 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286747+0000", 2023-09-23T17:59:10.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286745+0000", 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286752+0000", 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0550000000000002e-06 2023-09-23T17:59:10.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286767+0000", 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4283e-05 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286775+0000", 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1780000000000003e-06 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286834+0000", 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8866999999999997e-05 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286842+0000", 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7590000000000003e-06 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224948+0000", 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.938105666 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224957+0000", 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9469999999999992e-06 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131523 5.2 5.551f4f92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.311020+0000", 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 35.970987870000002, 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91399190200000002, 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.297 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131523 2023-09-23T17:59:10.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.298 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311020+0000", 2023-09-23T17:59:10.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311020+0000", 2023-09-23T17:59:10.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311018+0000", 2023-09-23T17:59:10.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:10.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311028+0000", 2023-09-23T17:59:10.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5859999999999994e-06 2023-09-23T17:59:10.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311050+0000", 2023-09-23T17:59:10.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2079e-05 2023-09-23T17:59:10.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311059+0000", 2023-09-23T17:59:10.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9959999999999999e-06 2023-09-23T17:59:10.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311123+0000", 2023-09-23T17:59:10.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4539e-05 2023-09-23T17:59:10.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311130+0000", 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.917e-06 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.225002+0000", 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91387170399999995 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.225012+0000", 2023-09-23T17:59:10.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0132000000000001e-05 2023-09-23T17:59:10.303 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.303 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.303 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.44681.0:136262 5.f e166/164 5:f0fb8af2:::benchmark_data_smithi055_229117_object30636:head v 166'8502, mlcod=116'1639)", 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:42.664586+0000", 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 27.617421569000001, 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.041828813999999999, 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:42.664586+0000", 2023-09-23T17:59:10.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.305 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:42.664586+0000", 2023-09-23T17:59:10.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.305 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:42.664586+0000", 2023-09-23T17:59:10.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:42.664591+0000", 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6869999999999999e-06 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:42.664604+0000", 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3015e-05 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:42.664611+0000", 2023-09-23T17:59:10.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2389999999999999e-06 2023-09-23T17:59:10.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:42.677425+0000", 2023-09-23T17:59:10.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012814156 2023-09-23T17:59:10.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:42.677451+0000", 2023-09-23T17:59:10.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6489e-05 2023-09-23T17:59:10.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:42.706378+0000", 2023-09-23T17:59:10.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028926932999999998 2023-09-23T17:59:10.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:42.706415+0000", 2023-09-23T17:59:10.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7035e-05 2023-09-23T17:59:10.309 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.309 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.309 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.310 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142717 5.9 5:97ac1ed5:::benchmark_data_smithi055_229117_object37091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.310 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.884415+0000", 2023-09-23T17:59:10.310 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.397593190999999, 2023-09-23T17:59:10.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0500839660000001, 2023-09-23T17:59:10.310 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.310 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.310 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.311 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.311 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.311 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142717 2023-09-23T17:59:10.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.311 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884415+0000", 2023-09-23T17:59:10.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884415+0000", 2023-09-23T17:59:10.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884413+0000", 2023-09-23T17:59:10.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:10.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884424+0000", 2023-09-23T17:59:10.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0804e-05 2023-09-23T17:59:10.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884444+0000", 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0057000000000001e-05 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884453+0000", 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8000000000000004e-06 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884524+0000", 2023-09-23T17:59:10.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0729000000000007e-05 2023-09-23T17:59:10.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884537+0000", 2023-09-23T17:59:10.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3059e-05 2023-09-23T17:59:10.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903080+0000", 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0185432940000001 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903187+0000", 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010691700000000001 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903271+0000", 2023-09-23T17:59:10.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4103000000000004e-05 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.911979+0000", 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0087084599999999995 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934336+0000", 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022356767999999999 2023-09-23T17:59:10.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934371+0000", 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4693999999999997e-05 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934420+0000", 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8755000000000003e-05 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934499+0000", 2023-09-23T17:59:10.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9129999999999996e-05 2023-09-23T17:59:10.319 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.319 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.319 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.319 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142734 5.9 5:9b1dd87e:::benchmark_data_smithi055_229117_object37108:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.319 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.916812+0000", 2023-09-23T17:59:10.320 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.365196059999999, 2023-09-23T17:59:10.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0179610699999999, 2023-09-23T17:59:10.320 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.320 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.320 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.320 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.320 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.320 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142734 2023-09-23T17:59:10.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.320 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916812+0000", 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916812+0000", 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916810+0000", 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916818+0000", 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7209999999999994e-06 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916838+0000", 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9941000000000001e-05 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916846+0000", 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5190000000000003e-06 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917145+0000", 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00029908799999999998 2023-09-23T17:59:10.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917153+0000", 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7700000000000007e-06 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.904847+0000", 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98769368300000004 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.904960+0000", 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112802 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.905068+0000", 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010848700000000001 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915793+0000", 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010724450999999999 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934547+0000", 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018754603000000002 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934612+0000", 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4272999999999999e-05 2023-09-23T17:59:10.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934667+0000", 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4814000000000001e-05 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934773+0000", 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106517 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.327 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142738 5.9 5:9dbd7dd5:::benchmark_data_smithi055_229117_object37112:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.916861+0000", 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.365146538000001, 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018182892, 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.328 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142738 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916864+0000", 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0709999999999999e-06 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916870+0000", 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8780000000000003e-06 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916882+0000", 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2558000000000001e-05 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917176+0000", 2023-09-23T17:59:10.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00029325599999999999 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917181+0000", 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6400000000000002e-06 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906621+0000", 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98943947600000004 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.332 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906744+0000", 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012348099999999999 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906869+0000", 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012421599999999999 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915800+0000", 2023-09-23T17:59:10.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0089313259999999998 2023-09-23T17:59:10.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934813+0000", 2023-09-23T17:59:10.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019012940999999998 2023-09-23T17:59:10.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934959+0000", 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000145921 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934994+0000", 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5707999999999998e-05 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935044+0000", 2023-09-23T17:59:10.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9951000000000003e-05 2023-09-23T17:59:10.336 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.336 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.336 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.336 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.336 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.336 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142777 5.9 5:91082f54:::benchmark_data_smithi055_229117_object37151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.336 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.976270+0000", 2023-09-23T17:59:10.336 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.305737905000001, 2023-09-23T17:59:10.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.958897997, 2023-09-23T17:59:10.337 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.337 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.337 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.337 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.337 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.337 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142777 2023-09-23T17:59:10.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.337 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976270+0000", 2023-09-23T17:59:10.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976270+0000", 2023-09-23T17:59:10.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976269+0000", 2023-09-23T17:59:10.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:59:10.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976276+0000", 2023-09-23T17:59:10.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2150000000000004e-06 2023-09-23T17:59:10.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976288+0000", 2023-09-23T17:59:10.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2092999999999999e-05 2023-09-23T17:59:10.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976293+0000", 2023-09-23T17:59:10.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1610000000000002e-06 2023-09-23T17:59:10.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.341 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976338+0000", 2023-09-23T17:59:10.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4116000000000002e-05 2023-09-23T17:59:10.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.341 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.341 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976346+0000", 2023-09-23T17:59:10.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9200000000000004e-06 2023-09-23T17:59:10.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908348+0000", 2023-09-23T17:59:10.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.93200213499999995 2023-09-23T17:59:10.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908468+0000", 2023-09-23T17:59:10.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120746 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908590+0000", 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121772 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915806+0000", 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0072161990000000004 2023-09-23T17:59:10.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934909+0000", 2023-09-23T17:59:10.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019102811000000001 2023-09-23T17:59:10.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935083+0000", 2023-09-23T17:59:10.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000173499 2023-09-23T17:59:10.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.345 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935114+0000", 2023-09-23T17:59:10.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1501999999999998e-05 2023-09-23T17:59:10.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935168+0000", 2023-09-23T17:59:10.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3947999999999999e-05 2023-09-23T17:59:10.346 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.346 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.346 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.346 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142802 5.9 5:92d6f126:::benchmark_data_smithi055_229117_object37176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.030542+0000", 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.251465858, 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.90473668699999998, 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142802 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.347 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030542+0000", 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030542+0000", 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030541+0000", 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030547+0000", 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5289999999999997e-06 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030566+0000", 2023-09-23T17:59:10.349 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8630999999999998e-05 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030571+0000", 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0259999999999998e-06 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030617+0000", 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5439999999999999e-05 2023-09-23T17:59:10.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030623+0000", 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1999999999999999e-06 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910103+0000", 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.87947978999999998 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910251+0000", 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014896700000000001 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910377+0000", 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125425 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920175+0000", 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097978730000000003 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934939+0000", 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014764019 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935202+0000", 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026282399999999999 2023-09-23T17:59:10.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935229+0000", 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7793999999999999e-05 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935279+0000", 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9428999999999997e-05 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.354 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142809 5.9 5:9f3ab830:::benchmark_data_smithi055_229117_object37183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.039995+0000", 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.242012922000001, 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.902436291, 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.355 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142809 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039995+0000", 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039995+0000", 2023-09-23T17:59:10.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039994+0000", 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040001+0000", 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5259999999999996e-06 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040017+0000", 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5794000000000002e-05 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040024+0000", 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3379999999999999e-06 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040079+0000", 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4866e-05 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040091+0000", 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1362000000000001e-05 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912030+0000", 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.87193905100000002 2023-09-23T17:59:10.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912160+0000", 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013052100000000001 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912267+0000", 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010700399999999999 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.360 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920180+0000", 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079123509999999998 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935312+0000", 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01513201 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942274+0000", 2023-09-23T17:59:10.361 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069623669999999997 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942331+0000", 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7516000000000003e-05 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942431+0000", 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9986000000000001e-05 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.362 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142838 5.9 5:9cf2a92c:::benchmark_data_smithi055_229117_object37212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.111228+0000", 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.170780186000002, 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.83141186199999995, 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.363 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142838 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111228+0000", 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.364 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111228+0000", 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111227+0000", 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111233+0000", 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3160000000000002e-06 2023-09-23T17:59:10.365 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.366 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111246+0000", 2023-09-23T17:59:10.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2884000000000001e-05 2023-09-23T17:59:10.366 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.366 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111251+0000", 2023-09-23T17:59:10.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2360000000000004e-06 2023-09-23T17:59:10.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111297+0000", 2023-09-23T17:59:10.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5881000000000003e-05 2023-09-23T17:59:10.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111307+0000", 2023-09-23T17:59:10.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0046e-05 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914054+0000", 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.80274739399999995 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914133+0000", 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8448999999999994e-05 2023-09-23T17:59:10.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.369 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914206+0000", 2023-09-23T17:59:10.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3325999999999996e-05 2023-09-23T17:59:10.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.369 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925627+0000", 2023-09-23T17:59:10.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011421277000000001 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935332+0000", 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097046300000000005 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942488+0000", 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007155603 2023-09-23T17:59:10.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.371 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.371 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.371 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942537+0000", 2023-09-23T17:59:10.371 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9391000000000002e-05 2023-09-23T17:59:10.371 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.371 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.371 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942640+0000", 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102618 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142868 5.9 5:93088e3b:::benchmark_data_smithi055_229117_object37242:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.164405+0000", 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.117602631, 2023-09-23T17:59:10.372 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.78228625900000004, 2023-09-23T17:59:10.373 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.373 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.373 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.373 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.373 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.373 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142868 2023-09-23T17:59:10.373 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.373 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.373 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.374 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164405+0000", 2023-09-23T17:59:10.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.374 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.374 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.374 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164405+0000", 2023-09-23T17:59:10.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.374 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164404+0000", 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164410+0000", 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1349999999999997e-06 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.375 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164422+0000", 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1647e-05 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164437+0000", 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5234e-05 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.376 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164485+0000", 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7363e-05 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164493+0000", 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5429999999999994e-06 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915840+0000", 2023-09-23T17:59:10.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.75134721800000004 2023-09-23T17:59:10.378 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.378 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.378 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.378 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915919+0000", 2023-09-23T17:59:10.378 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8091999999999997e-05 2023-09-23T17:59:10.378 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.378 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.378 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.379 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915992+0000", 2023-09-23T17:59:10.379 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3733999999999996e-05 2023-09-23T17:59:10.379 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.379 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.379 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.379 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925638+0000", 2023-09-23T17:59:10.379 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096452810000000003 2023-09-23T17:59:10.379 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.380 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.380 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.380 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935357+0000", 2023-09-23T17:59:10.380 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097193910000000008 2023-09-23T17:59:10.380 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.380 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.380 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.380 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946534+0000", 2023-09-23T17:59:10.381 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011177302 2023-09-23T17:59:10.381 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.381 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.381 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.381 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946593+0000", 2023-09-23T17:59:10.381 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8563000000000002e-05 2023-09-23T17:59:10.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.382 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946692+0000", 2023-09-23T17:59:10.382 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8836e-05 2023-09-23T17:59:10.382 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.382 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.382 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.382 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142887 5.9 5:980e2e89:::benchmark_data_smithi055_229117_object37261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.211074+0000", 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.070933547999999, 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73576580700000005, 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142887 2023-09-23T17:59:10.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211074+0000", 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211074+0000", 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211073+0000", 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211080+0000", 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3629999999999996e-06 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211097+0000", 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6917000000000001e-05 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211103+0000", 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2499999999999997e-06 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211145+0000", 2023-09-23T17:59:10.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2466000000000003e-05 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211151+0000", 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0410000000000003e-06 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918068+0000", 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70691734399999995 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918196+0000", 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000128081 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918320+0000", 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012403300000000001 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.388 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930458+0000", 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012137652000000001 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935374+0000", 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049156570000000004 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946735+0000", 2023-09-23T17:59:10.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011361616999999999 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946767+0000", 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1952999999999997e-05 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946840+0000", 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2910000000000005e-05 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.390 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.391 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.391 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.391 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142888 5.9 5:99a82285:::benchmark_data_smithi055_229117_object37262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.391 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.211114+0000", 2023-09-23T17:59:10.391 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.070893732999998, 2023-09-23T17:59:10.391 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73588783199999996, 2023-09-23T17:59:10.391 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.391 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.391 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142888 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211117+0000", 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4680000000000001e-06 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211121+0000", 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7610000000000001e-06 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211124+0000", 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3489999999999999e-06 2023-09-23T17:59:10.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211177+0000", 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3133000000000001e-05 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211184+0000", 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6240000000000002e-06 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.396 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920219+0000", 2023-09-23T17:59:10.396 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70903441 2023-09-23T17:59:10.396 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.396 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.396 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.396 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920327+0000", 2023-09-23T17:59:10.396 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108477 2023-09-23T17:59:10.396 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.396 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.396 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920402+0000", 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5252000000000004e-05 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930467+0000", 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010064867999999999 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.937129+0000", 2023-09-23T17:59:10.397 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066614200000000004 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946878+0000", 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097489120000000002 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946919+0000", 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1031000000000001e-05 2023-09-23T17:59:10.398 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.947002+0000", 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3475999999999998e-05 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142904 5.9 5:9b2141c3:::benchmark_data_smithi055_229117_object37278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.399 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.244955+0000", 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.037052672000002, 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70391956499999997, 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142904 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.400 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244955+0000", 2023-09-23T17:59:10.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.401 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244955+0000", 2023-09-23T17:59:10.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.402 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.402 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244954+0000", 2023-09-23T17:59:10.402 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:10.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244960+0000", 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244973+0000", 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3209e-05 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.404 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244978+0000", 2023-09-23T17:59:10.404 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1930000000000001e-06 2023-09-23T17:59:10.404 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.404 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.404 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.404 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245017+0000", 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8452999999999999e-05 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245023+0000", 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.409e-06 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.921896+0000", 2023-09-23T17:59:10.405 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67687305900000005 2023-09-23T17:59:10.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.406 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.406 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.406 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.921997+0000", 2023-09-23T17:59:10.406 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100753 2023-09-23T17:59:10.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.406 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.406 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.922064+0000", 2023-09-23T17:59:10.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6978999999999997e-05 2023-09-23T17:59:10.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930475+0000", 2023-09-23T17:59:10.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0084114670000000006 2023-09-23T17:59:10.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.940046+0000", 2023-09-23T17:59:10.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095709799999999998 2023-09-23T17:59:10.408 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.408 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948781+0000", 2023-09-23T17:59:10.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0087346990000000003 2023-09-23T17:59:10.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.409 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.409 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948813+0000", 2023-09-23T17:59:10.409 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1779000000000002e-05 2023-09-23T17:59:10.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.409 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.410 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948875+0000", 2023-09-23T17:59:10.410 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2073e-05 2023-09-23T17:59:10.410 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.410 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.410 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.410 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.411 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.411 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142905 5.9 5:951a98af:::benchmark_data_smithi055_229117_object37279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.411 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.245329+0000", 2023-09-23T17:59:10.411 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.036678803000001, 2023-09-23T17:59:10.411 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71295564, 2023-09-23T17:59:10.411 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.411 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.411 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.411 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.412 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.412 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142905 2023-09-23T17:59:10.412 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.412 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.412 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.412 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T17:59:10.413 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T17:59:10.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.414 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245332+0000", 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4400000000000001e-06 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245339+0000", 2023-09-23T17:59:10.415 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4749999999999998e-06 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245343+0000", 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1370000000000004e-06 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245381+0000", 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8396999999999998e-05 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.416 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245386+0000", 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2460000000000003e-06 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923706+0000", 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67831945699999996 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.417 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923849+0000", 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143669 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923977+0000", 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012721600000000001 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933852+0000", 2023-09-23T17:59:10.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0098750190000000005 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.940069+0000", 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062175399999999997 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958137+0000", 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018067588999999998 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.419 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958189+0000", 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1981000000000003e-05 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958285+0000", 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6036e-05 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.420 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.421 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.421 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142919 5.9 5:9348e633:::benchmark_data_smithi055_229117_object37293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.421 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.270585+0000", 2023-09-23T17:59:10.421 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 18.011423171000001, 2023-09-23T17:59:10.421 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.69084338599999995, 2023-09-23T17:59:10.421 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.421 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.421 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.421 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.421 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.422 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142919 2023-09-23T17:59:10.422 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.422 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.422 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.422 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.423 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270585+0000", 2023-09-23T17:59:10.423 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.423 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.423 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.423 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.423 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270585+0000", 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270583+0000", 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270591+0000", 2023-09-23T17:59:10.424 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2910000000000002e-06 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270604+0000", 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3349999999999999e-05 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270609+0000", 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1109999999999999e-06 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.425 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270662+0000", 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3236000000000003e-05 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270673+0000", 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0560999999999999e-05 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.426 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925683+0000", 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.65501024500000005 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925803+0000", 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119807 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925920+0000", 2023-09-23T17:59:10.427 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117419 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933861+0000", 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079407209999999995 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944507+0000", 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010645846 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.428 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961259+0000", 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016751986999999999 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961315+0000", 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6477999999999997e-05 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.429 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961428+0000", 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011292400000000001 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142929 5.9 5:954e3e29:::benchmark_data_smithi055_229117_object37303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.300689+0000", 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 17.981319456000001, 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.662469262, 2023-09-23T17:59:10.430 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142929 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300689+0000", 2023-09-23T17:59:10.431 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300689+0000", 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300687+0000", 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.432 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300694+0000", 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0219999999999999e-06 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300705+0000", 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0886e-05 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.433 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300711+0000", 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9490000000000001e-06 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300749+0000", 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8077999999999997e-05 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300755+0000", 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5389999999999998e-06 2023-09-23T17:59:10.434 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.435 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.435 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.435 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928440+0000", 2023-09-23T17:59:10.435 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62768513800000003 2023-09-23T17:59:10.435 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928576+0000", 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013626900000000001 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928683+0000", 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010661400000000001 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.436 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.437 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933946+0000", 2023-09-23T17:59:10.437 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052633460000000003 2023-09-23T17:59:10.437 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.437 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.437 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.437 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944539+0000", 2023-09-23T17:59:10.437 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010592571 2023-09-23T17:59:10.438 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.438 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.438 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.438 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963005+0000", 2023-09-23T17:59:10.438 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018466598000000001 2023-09-23T17:59:10.438 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.438 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.438 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.438 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963057+0000", 2023-09-23T17:59:10.438 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1529000000000001e-05 2023-09-23T17:59:10.439 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.439 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.439 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.439 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963158+0000", 2023-09-23T17:59:10.439 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010085399999999999 2023-09-23T17:59:10.439 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.439 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.439 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.440 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.440 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.440 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142941 5.9 5:913fcfd6:::benchmark_data_smithi055_229117_object37315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.440 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.340683+0000", 2023-09-23T17:59:10.440 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 17.941325118999998, 2023-09-23T17:59:10.440 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62261945900000004, 2023-09-23T17:59:10.440 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.440 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.441 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.441 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.441 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.441 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142941 2023-09-23T17:59:10.441 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.441 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.441 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.441 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.441 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340683+0000", 2023-09-23T17:59:10.442 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.442 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.442 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.442 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340683+0000", 2023-09-23T17:59:10.442 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.442 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.442 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.442 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340682+0000", 2023-09-23T17:59:10.443 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:10.443 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.443 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.443 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.443 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340688+0000", 2023-09-23T17:59:10.443 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.319e-06 2023-09-23T17:59:10.443 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.443 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.444 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.444 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340701+0000", 2023-09-23T17:59:10.444 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3529999999999999e-05 2023-09-23T17:59:10.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.444 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.444 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.444 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340708+0000", 2023-09-23T17:59:10.444 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8449999999999997e-06 2023-09-23T17:59:10.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340755+0000", 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6428999999999999e-05 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340762+0000", 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.571e-06 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.445 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930510+0000", 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58974757200000005 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930615+0000", 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000105213 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.446 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930695+0000", 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0055000000000005e-05 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933952+0000", 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003257228 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944608+0000", 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010655243 2023-09-23T17:59:10.447 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.448 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.448 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.448 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963204+0000", 2023-09-23T17:59:10.448 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018596950000000001 2023-09-23T17:59:10.448 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.448 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.448 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.449 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963236+0000", 2023-09-23T17:59:10.449 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1894000000000003e-05 2023-09-23T17:59:10.449 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.449 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.449 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.449 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963302+0000", 2023-09-23T17:59:10.449 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6013999999999995e-05 2023-09-23T17:59:10.449 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.450 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.450 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.450 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.450 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.450 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142946 5.9 5:9099fb2d:::benchmark_data_smithi055_229117_object37320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T17:59:10.450 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.370819+0000", 2023-09-23T17:59:10.450 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 17.911189028999999, 2023-09-23T17:59:10.450 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59265797899999995, 2023-09-23T17:59:10.450 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T17:59:10.451 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:10.451 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T17:59:10.451 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T17:59:10.451 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:10.451 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142946 2023-09-23T17:59:10.451 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.451 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T17:59:10.453 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.453 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T17:59:10.453 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370819+0000", 2023-09-23T17:59:10.453 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.453 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.453 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.453 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370819+0000", 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370817+0000", 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T17:59:10.454 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370824+0000", 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8299999999999998e-06 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370837+0000", 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2716e-05 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370844+0000", 2023-09-23T17:59:10.455 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8789999999999997e-06 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370887+0000", 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2879000000000001e-05 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370893+0000", 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1120000000000001e-06 2023-09-23T17:59:10.456 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.457 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.457 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T17:59:10.457 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932242+0000", 2023-09-23T17:59:10.457 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.56134879999999998 2023-09-23T17:59:10.457 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.457 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.457 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T17:59:10.457 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932323+0000", 2023-09-23T17:59:10.457 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0871000000000005e-05 2023-09-23T17:59:10.457 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932392+0000", 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9295999999999995e-05 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934346+0000", 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019539589999999999 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.458 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944640+0000", 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010293754 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963339+0000", 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018699684000000001 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T17:59:10.459 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963384+0000", 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4499999999999997e-05 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963477+0000", 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3245000000000006e-05 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T17:59:10.460 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T17:59:10.461 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:10.462 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:10.491 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:10.650 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T17:59:10.677 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:59:10] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-09-23T17:59:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[184174]: failed to encode map e176 with expected crc 2023-09-23T17:59:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[184174]: failed to encode map e176 with expected crc 2023-09-23T17:59:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[184174]: osdmap e176: 8 total, 6 up, 7 in 2023-09-23T17:59:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[184174]: 5.3 continuing backfill to osd.6 from (124'4486,174'9534] MIN to 174'9534 2023-09-23T17:59:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[184174]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 174'9422 2023-09-23T17:59:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[184174]: 5.2 continuing backfill to osd.6 from (124'4338,174'9422] 5:4c9b24d2:::benchmark_data_smithi055_229117_object74537:head to 174'9422 2023-09-23T17:59:10.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[179668]: failed to encode map e176 with expected crc 2023-09-23T17:59:10.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[179668]: failed to encode map e176 with expected crc 2023-09-23T17:59:10.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[179668]: osdmap e176: 8 total, 6 up, 7 in 2023-09-23T17:59:10.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[179668]: 5.3 continuing backfill to osd.6 from (124'4486,174'9534] MIN to 174'9534 2023-09-23T17:59:10.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[179668]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 174'9422 2023-09-23T17:59:10.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:10 smithi055 ceph-mon[179668]: 5.2 continuing backfill to osd.6 from (124'4338,174'9422] 5:4c9b24d2:::benchmark_data_smithi055_229117_object74537:head to 174'9422 2023-09-23T17:59:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:10 smithi099 ceph-mon[167076]: failed to encode map e176 with expected crc 2023-09-23T17:59:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:10 smithi099 ceph-mon[167076]: failed to encode map e176 with expected crc 2023-09-23T17:59:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:10 smithi099 ceph-mon[167076]: osdmap e176: 8 total, 6 up, 7 in 2023-09-23T17:59:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:10 smithi099 ceph-mon[167076]: 5.3 continuing backfill to osd.6 from (124'4486,174'9534] MIN to 174'9534 2023-09-23T17:59:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:10 smithi099 ceph-mon[167076]: 5.2 starting backfill to osd.2 from (0'0,0'0] MAX to 174'9422 2023-09-23T17:59:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:10 smithi099 ceph-mon[167076]: 5.2 continuing backfill to osd.6 from (124'4338,174'9422] 5:4c9b24d2:::benchmark_data_smithi055_229117_object74537:head to 174'9422 2023-09-23T17:59:11.221 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:11.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:11.228 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:11.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:11.372 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:11.422 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:11.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:11.426 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:11.464 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T17:59:11.527 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:11.569 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:11.611 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:11.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:11.620 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:11.721 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:11.809 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:11 smithi099 ceph-mon[167076]: pgmap v392: 109 pgs: 2 remapped+peering, 1 down, 3 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 3 undersized+peered, 27 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 24611/172869 objects degraded (14.237%); 3473/172869 objects misplaced (2.009%); 19 MiB/s, 299 objects/s recovering 2023-09-23T17:59:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:11 smithi099 ceph-mon[167076]: failed to encode map e177 with expected crc 2023-09-23T17:59:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:11 smithi099 ceph-mon[167076]: failed to encode map e177 with expected crc 2023-09-23T17:59:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:11 smithi099 ceph-mon[167076]: osdmap e177: 8 total, 6 up, 7 in 2023-09-23T17:59:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:11 smithi099 ceph-mon[167076]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 174'9596 2023-09-23T17:59:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:11 smithi099 ceph-mon[167076]: 5.7 starting backfill to osd.4 from (0'0,0'0] MAX to 174'9854 2023-09-23T17:59:12.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[179668]: pgmap v392: 109 pgs: 2 remapped+peering, 1 down, 3 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 3 undersized+peered, 27 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 24611/172869 objects degraded (14.237%); 3473/172869 objects misplaced (2.009%); 19 MiB/s, 299 objects/s recovering 2023-09-23T17:59:12.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[179668]: failed to encode map e177 with expected crc 2023-09-23T17:59:12.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[179668]: failed to encode map e177 with expected crc 2023-09-23T17:59:12.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[179668]: osdmap e177: 8 total, 6 up, 7 in 2023-09-23T17:59:12.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[179668]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 174'9596 2023-09-23T17:59:12.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[179668]: 5.7 starting backfill to osd.4 from (0'0,0'0] MAX to 174'9854 2023-09-23T17:59:12.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[184174]: pgmap v392: 109 pgs: 2 remapped+peering, 1 down, 3 peering, 5 active+clean+remapped, 1 active+recovering+degraded+remapped, 52 active+clean, 3 undersized+peered, 27 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 632 op/s; 24611/172869 objects degraded (14.237%); 3473/172869 objects misplaced (2.009%); 19 MiB/s, 299 objects/s recovering 2023-09-23T17:59:12.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[184174]: failed to encode map e177 with expected crc 2023-09-23T17:59:12.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[184174]: failed to encode map e177 with expected crc 2023-09-23T17:59:12.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[184174]: osdmap e177: 8 total, 6 up, 7 in 2023-09-23T17:59:12.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[184174]: 5.e starting backfill to osd.2 from (0'0,0'0] MAX to 174'9596 2023-09-23T17:59:12.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:11 smithi055 ceph-mon[184174]: 5.7 starting backfill to osd.4 from (0'0,0'0] MAX to 174'9854 2023-09-23T17:59:12.410 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:12.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:12.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:12.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:12.553 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:12.601 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:12.604 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:12.705 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:12.749 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:12.812 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:12.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:12.818 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:12.912 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:12 smithi055 ceph-mon[184174]: 5.4 starting backfill to osd.1 from (0'0,0'0] MAX to 174'9671 2023-09-23T17:59:12.912 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:12 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 24611/172869 objects degraded (14.237%), 16 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:12.913 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:12 smithi055 ceph-mon[179668]: 5.4 starting backfill to osd.1 from (0'0,0'0] MAX to 174'9671 2023-09-23T17:59:12.913 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:12 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 24611/172869 objects degraded (14.237%), 16 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:12.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:12.959 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:13.006 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:13.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:13.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:13.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:12 smithi099 ceph-mon[167076]: 5.4 starting backfill to osd.1 from (0'0,0'0] MAX to 174'9671 2023-09-23T17:59:13.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:12 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 24611/172869 objects degraded (14.237%), 16 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:13.113 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:13.206 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:13.335 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:13.388 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:13.395 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:13.395 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:13.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:13.502 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:13.532 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:13.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:13.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:13.574 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:13.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:13.714 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:13.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:13.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:13.763 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:13.863 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:13.899 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:13.946 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:13.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:13.949 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:14.050 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:14.122 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:13 smithi055 ceph-mon[184174]: pgmap v394: 109 pgs: 1 activating+remapped, 2 remapped+peering, 1 down, 3 peering, 6 active+clean+remapped, 1 active+recovering+degraded+remapped, 48 active+clean, 3 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 18969/170709 objects degraded (11.112%); 14127/170709 objects misplaced (8.275%); 13 MiB/s, 204 objects/s recovering 2023-09-23T17:59:14.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:13 smithi055 ceph-mon[179668]: pgmap v394: 109 pgs: 1 activating+remapped, 2 remapped+peering, 1 down, 3 peering, 6 active+clean+remapped, 1 active+recovering+degraded+remapped, 48 active+clean, 3 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 18969/170709 objects degraded (11.112%); 14127/170709 objects misplaced (8.275%); 13 MiB/s, 204 objects/s recovering 2023-09-23T17:59:14.160 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:14.241 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:59:14.247 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T17:59:14.247 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:59:14.247 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T17:59:14.247 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.231169 2023-09-23T17:59:14.248 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0.2311689908880309 2023-09-23T17:59:14.298 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:59:14.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:13 smithi099 ceph-mon[167076]: pgmap v394: 109 pgs: 1 activating+remapped, 2 remapped+peering, 1 down, 3 peering, 6 active+clean+remapped, 1 active+recovering+degraded+remapped, 48 active+clean, 3 undersized+peered, 26 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+undersized+remapped, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 13 active+undersized+degraded; 3.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 18969/170709 objects degraded (11.112%); 14127/170709 objects misplaced (8.275%); 13 MiB/s, 204 objects/s recovering 2023-09-23T17:59:14.341 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:59:14.384 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:59:14.417 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:14.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:14.426 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:14.428 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:59:14.474 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:59:14.479 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:59:14.479 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:59:14.479 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:59:14.479 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:59:14.479 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:59:14.479 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:59:14.479 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T17:59:14.479 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:59:14.527 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:14.648 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:14.915 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2426487083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:15.084 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:15.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:15.092 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:15.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2426487083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:15.193 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:15.271 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:59:15.272 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:59:15.325 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:15.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2426487083' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:15.458 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:15.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:15.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:15.567 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:15.572 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:15.585 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:59:15.586 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:59:15.611 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:15.652 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:15.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:15.656 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:15.757 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:15.835 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:15.925 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:15 smithi055 ceph-mon[184174]: pgmap v395: 109 pgs: 1 down, 1 activating+remapped, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 22184/167193 objects degraded (13.268%); 41095/167193 objects misplaced (24.579%); 20 MiB/s, 313 objects/s recovering 2023-09-23T17:59:15.925 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:15 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:59:15.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:15 smithi055 ceph-mon[179668]: pgmap v395: 109 pgs: 1 down, 1 activating+remapped, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 22184/167193 objects degraded (13.268%); 41095/167193 objects misplaced (24.579%); 20 MiB/s, 313 objects/s recovering 2023-09-23T17:59:15.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:15 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:59:15.955 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:15.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:15.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:16.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:16.104 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:16.149 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:16.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:16.153 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:16.200 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T17:59:16.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:16.284 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:15 smithi099 ceph-mon[167076]: pgmap v395: 109 pgs: 1 down, 1 activating+remapped, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 665 op/s; 22184/167193 objects degraded (13.268%); 41095/167193 objects misplaced (24.579%); 20 MiB/s, 313 objects/s recovering 2023-09-23T17:59:16.284 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:15 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:59:16.288 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:16.332 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:16.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:16.335 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:16.436 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:16.466 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:16.509 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:16.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:16.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:16.613 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:16.704 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:16.971 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T17:59:17.273 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:59:17.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3484310211' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:17.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3484310211' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:17.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3484310211' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:17 smithi099 ceph-mon[167076]: pgmap v396: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 22002/165414 objects degraded (13.301%); 40868/165414 objects misplaced (24.706%); 17 MiB/s, 277 objects/s recovering 2023-09-23T17:59:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:17 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:59:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3484310211' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:17 smithi099 ceph-mon[167076]: osdmap e178: 8 total, 6 up, 7 in 2023-09-23T17:59:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:17 smithi099 ceph-mon[167076]: failed to encode map e178 with expected crc 2023-09-23T17:59:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:17 smithi099 ceph-mon[167076]: failed to encode map e178 with expected crc 2023-09-23T17:59:18.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:17 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 22002/165414 objects degraded (13.301%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[179668]: pgmap v396: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 22002/165414 objects degraded (13.301%); 40868/165414 objects misplaced (24.706%); 17 MiB/s, 277 objects/s recovering 2023-09-23T17:59:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:59:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3484310211' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:18.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[179668]: osdmap e178: 8 total, 6 up, 7 in 2023-09-23T17:59:18.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[179668]: failed to encode map e178 with expected crc 2023-09-23T17:59:18.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[179668]: failed to encode map e178 with expected crc 2023-09-23T17:59:18.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 22002/165414 objects degraded (13.301%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[184174]: pgmap v396: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.4 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 596 op/s; 22002/165414 objects degraded (13.301%); 40868/165414 objects misplaced (24.706%); 17 MiB/s, 277 objects/s recovering 2023-09-23T17:59:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:59:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3484310211' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[184174]: osdmap e178: 8 total, 6 up, 7 in 2023-09-23T17:59:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[184174]: failed to encode map e178 with expected crc 2023-09-23T17:59:18.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[184174]: failed to encode map e178 with expected crc 2023-09-23T17:59:18.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:17 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 22002/165414 objects degraded (13.301%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:19.421 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:19.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:19.428 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:19.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:19.570 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:19.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:19.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:19.621 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:19.723 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:19.803 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:19.899 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:19.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:19.907 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:20.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:20.053 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:20.078 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:20.099 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:20.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:20.103 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:20.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:19 smithi055 ceph-mon[184174]: pgmap v398: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 21752/162966 objects degraded (13.348%); 40263/162966 objects misplaced (24.706%); 13 MiB/s, 202 objects/s recovering 2023-09-23T17:59:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:19 smithi055 ceph-mon[179668]: pgmap v398: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 21752/162966 objects degraded (13.348%); 40263/162966 objects misplaced (24.706%); 13 MiB/s, 202 objects/s recovering 2023-09-23T17:59:20.183 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:20.203 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:20.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:19 smithi099 ceph-mon[167076]: pgmap v398: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.3 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 412 op/s; 21752/162966 objects degraded (13.348%); 40263/162966 objects misplaced (24.706%); 13 MiB/s, 202 objects/s recovering 2023-09-23T17:59:20.244 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:20.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:20.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:20.292 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:20.393 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:20.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:59:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:59:20] "GET /metrics HTTP/1.1" 200 36561 "" "Prometheus/2.43.0" 2023-09-23T17:59:20.435 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:20.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:20.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:20.489 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:20.590 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:20.689 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:20.788 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:20.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:20.794 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:20.895 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:20.935 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:20.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:20.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:20.983 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:21.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:21.122 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:21.169 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:21.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:21.176 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:21.276 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:21.349 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:21.558 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:21.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:21.565 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:21.666 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:21.739 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:21.862 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T17:59:21.938 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:21.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:21.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:22.007 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.4 primary-affinity to 0.231169 (8151492) 2023-09-23T17:59:22.045 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:22.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:21 smithi055 ceph-mon[179668]: pgmap v399: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 20932/157446 objects degraded (13.295%); 39419/157446 objects misplaced (25.037%); 10 MiB/s, 167 objects/s recovering 2023-09-23T17:59:22.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1319550623' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.2311689908880309}]: dispatch 2023-09-23T17:59:22.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:59:22.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:22.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:21 smithi055 ceph-mon[184174]: pgmap v399: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 20932/157446 objects degraded (13.295%); 39419/157446 objects misplaced (25.037%); 10 MiB/s, 167 objects/s recovering 2023-09-23T17:59:22.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1319550623' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.2311689908880309}]: dispatch 2023-09-23T17:59:22.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:59:22.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:22.153 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:22.207 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:59:22.207 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:59:22.244 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:22.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:22.251 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:22.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:21 smithi099 ceph-mon[167076]: pgmap v399: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.2 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 20932/157446 objects degraded (13.295%); 39419/157446 objects misplaced (25.037%); 10 MiB/s, 167 objects/s recovering 2023-09-23T17:59:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1319550623' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.2311689908880309}]: dispatch 2023-09-23T17:59:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:59:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:22.351 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:22.377 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:22.408 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:22.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:22.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:22.491 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:59:22.512 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:22.517 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:59:22.538 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:22.541 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:59:22.567 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:59:22.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:22.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:22.572 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:22.589 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:59:22.614 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:59:22.614 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:59:22.615 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:59:22.615 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:59:22.615 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:59:22.615 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:59:22.615 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:59:22.615 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T17:59:22.615 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:59:22.674 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:22.717 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:22.789 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T17:59:22.821 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:22.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:22.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:22.925 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:22.960 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:23.136 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:23.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:23.140 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:23.170 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1319550623' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.2311689908880309}]': finished 2023-09-23T17:59:23.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[184174]: osdmap e179: 8 total, 6 up, 7 in 2023-09-23T17:59:23.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[184174]: failed to encode map e179 with expected crc 2023-09-23T17:59:23.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[184174]: failed to encode map e179 with expected crc 2023-09-23T17:59:23.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/854969683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:23.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 20932/157446 objects degraded (13.295%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:23.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[184174]: osdmap e180: 8 total, 6 up, 7 in 2023-09-23T17:59:23.171 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1319550623' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.2311689908880309}]': finished 2023-09-23T17:59:23.171 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[179668]: osdmap e179: 8 total, 6 up, 7 in 2023-09-23T17:59:23.172 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[179668]: failed to encode map e179 with expected crc 2023-09-23T17:59:23.172 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[179668]: failed to encode map e179 with expected crc 2023-09-23T17:59:23.172 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/854969683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:23.172 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 20932/157446 objects degraded (13.295%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:23.172 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:22 smithi055 ceph-mon[179668]: osdmap e180: 8 total, 6 up, 7 in 2023-09-23T17:59:23.172 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:59:23.173 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:59:23.193 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:23.240 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:23.264 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1319550623' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.2311689908880309}]': finished 2023-09-23T17:59:23.265 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:22 smithi099 ceph-mon[167076]: osdmap e179: 8 total, 6 up, 7 in 2023-09-23T17:59:23.265 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:22 smithi099 ceph-mon[167076]: failed to encode map e179 with expected crc 2023-09-23T17:59:23.265 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:22 smithi099 ceph-mon[167076]: failed to encode map e179 with expected crc 2023-09-23T17:59:23.265 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/854969683' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:23.265 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:22 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 20932/157446 objects degraded (13.295%), 17 pgs degraded, 31 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:23.265 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:22 smithi099 ceph-mon[167076]: osdmap e180: 8 total, 6 up, 7 in 2023-09-23T17:59:23.267 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:23.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:23.301 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:23.302 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:23.402 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:23.443 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:23.608 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:23.636 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:23.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:23.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:23.742 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:23.777 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:23.893 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:23.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:23.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:23.998 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:24.025 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:24.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:24 smithi055 ceph-mon[184174]: pgmap v401: 109 pgs: 4 remapped+peering, 1 down, 12 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 19363/153711 objects degraded (12.597%); 32546/153711 objects misplaced (21.174%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T17:59:24.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:24 smithi055 ceph-mon[184174]: failed to encode map e180 with expected crc 2023-09-23T17:59:24.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:24 smithi055 ceph-mon[184174]: failed to encode map e180 with expected crc 2023-09-23T17:59:24.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:24 smithi055 ceph-mon[184174]: osdmap e181: 8 total, 6 up, 7 in 2023-09-23T17:59:24.053 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:24 smithi055 ceph-mon[184174]: failed to encode map e181 with expected crc 2023-09-23T17:59:24.056 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:24 smithi055 ceph-mon[179668]: pgmap v401: 109 pgs: 4 remapped+peering, 1 down, 12 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 19363/153711 objects degraded (12.597%); 32546/153711 objects misplaced (21.174%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T17:59:24.057 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:24 smithi055 ceph-mon[179668]: failed to encode map e180 with expected crc 2023-09-23T17:59:24.057 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:24 smithi055 ceph-mon[179668]: failed to encode map e180 with expected crc 2023-09-23T17:59:24.057 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:24 smithi055 ceph-mon[179668]: osdmap e181: 8 total, 6 up, 7 in 2023-09-23T17:59:24.057 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:24 smithi055 ceph-mon[179668]: failed to encode map e181 with expected crc 2023-09-23T17:59:24.063 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:24.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:24.066 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:24.167 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:24.201 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:24.309 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:24.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:24.313 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:24.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:24 smithi099 ceph-mon[167076]: pgmap v401: 109 pgs: 4 remapped+peering, 1 down, 12 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 560 op/s; 19363/153711 objects degraded (12.597%); 32546/153711 objects misplaced (21.174%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T17:59:24.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:24 smithi099 ceph-mon[167076]: failed to encode map e180 with expected crc 2023-09-23T17:59:24.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:24 smithi099 ceph-mon[167076]: failed to encode map e180 with expected crc 2023-09-23T17:59:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:24 smithi099 ceph-mon[167076]: osdmap e181: 8 total, 6 up, 7 in 2023-09-23T17:59:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:24 smithi099 ceph-mon[167076]: failed to encode map e181 with expected crc 2023-09-23T17:59:24.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:24.447 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:24.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:24.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:24.492 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:24.592 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:24.679 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:25.173 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:59:25.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:25 smithi099 ceph-mon[167076]: failed to encode map e181 with expected crc 2023-09-23T17:59:25.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:25 smithi099 ceph-mon[167076]: failed to encode map e181 with expected crc 2023-09-23T17:59:25.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:25 smithi099 ceph-mon[167076]: 5.e continuing backfill to osd.2 from (119'3330,178'10040] 5:70140cb8:::benchmark_data_smithi055_229117_object62703:head to 178'10040 2023-09-23T17:59:25.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:25 smithi055 ceph-mon[179668]: failed to encode map e181 with expected crc 2023-09-23T17:59:25.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:25 smithi055 ceph-mon[179668]: failed to encode map e181 with expected crc 2023-09-23T17:59:25.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:25 smithi055 ceph-mon[179668]: 5.e continuing backfill to osd.2 from (119'3330,178'10040] 5:70140cb8:::benchmark_data_smithi055_229117_object62703:head to 178'10040 2023-09-23T17:59:25.439 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:25 smithi055 ceph-mon[184174]: failed to encode map e181 with expected crc 2023-09-23T17:59:25.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:25 smithi055 ceph-mon[184174]: failed to encode map e181 with expected crc 2023-09-23T17:59:25.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:25 smithi055 ceph-mon[184174]: 5.e continuing backfill to osd.2 from (119'3330,178'10040] 5:70140cb8:::benchmark_data_smithi055_229117_object62703:head to 178'10040 2023-09-23T17:59:26.082 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T17:59:26.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:26 smithi099 ceph-mon[167076]: pgmap v404: 109 pgs: 1 down, 3 remapped+peering, 12 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 18822/149784 objects degraded (12.566%); 34234/149784 objects misplaced (22.856%); 13 MiB/s, 211 objects/s recovering 2023-09-23T17:59:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:26 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down, 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:59:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4002479297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:59:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:26 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:59:26.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:26 smithi055 ceph-mon[184174]: pgmap v404: 109 pgs: 1 down, 3 remapped+peering, 12 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 18822/149784 objects degraded (12.566%); 34234/149784 objects misplaced (22.856%); 13 MiB/s, 211 objects/s recovering 2023-09-23T17:59:26.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:26 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down, 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:59:26.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4002479297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:59:26.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:26 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:59:26.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:26 smithi055 ceph-mon[179668]: pgmap v404: 109 pgs: 1 down, 3 remapped+peering, 12 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.1 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 730 op/s; 18822/149784 objects degraded (12.566%); 34234/149784 objects misplaced (22.856%); 13 MiB/s, 211 objects/s recovering 2023-09-23T17:59:26.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:26 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down, 1 pg peering (PG_AVAILABILITY) 2023-09-23T17:59:26.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4002479297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:59:26.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:26 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T17:59:27.354 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:27.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:27 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:27.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:27 smithi055 ceph-mon[184174]: failed to encode map e182 with expected crc 2023-09-23T17:59:27.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:27 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:59:27.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:27 smithi055 ceph-mon[184174]: osdmap e182: 8 total, 6 up, 7 in 2023-09-23T17:59:27.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:27 smithi055 ceph-mon[184174]: failed to encode map e182 with expected crc 2023-09-23T17:59:27.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:27 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:27.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:27 smithi055 ceph-mon[179668]: failed to encode map e182 with expected crc 2023-09-23T17:59:27.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:27 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:59:27.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:27 smithi055 ceph-mon[179668]: osdmap e182: 8 total, 6 up, 7 in 2023-09-23T17:59:27.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:27 smithi055 ceph-mon[179668]: failed to encode map e182 with expected crc 2023-09-23T17:59:27.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:27.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:27.461 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:27.525 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:27.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:27 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:27.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:27 smithi099 ceph-mon[167076]: failed to encode map e182 with expected crc 2023-09-23T17:59:27.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:27 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T17:59:27.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:27 smithi099 ceph-mon[167076]: osdmap e182: 8 total, 6 up, 7 in 2023-09-23T17:59:27.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:27 smithi099 ceph-mon[167076]: failed to encode map e182 with expected crc 2023-09-23T17:59:27.751 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T17:59:27.751 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:59:27.751 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T17:59:27.751 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T17:59:28.085 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:28 smithi055 ceph-mon[179668]: pgmap v406: 109 pgs: 1 down, 3 remapped+peering, 12 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 18718/148671 objects degraded (12.590%); 34082/148671 objects misplaced (22.924%); 21 MiB/s, 339 objects/s recovering 2023-09-23T17:59:28.085 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 18718/148671 objects degraded (12.590%), 16 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:28.085 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:28 smithi055 ceph-mon[184174]: pgmap v406: 109 pgs: 1 down, 3 remapped+peering, 12 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 18718/148671 objects degraded (12.590%); 34082/148671 objects misplaced (22.924%); 21 MiB/s, 339 objects/s recovering 2023-09-23T17:59:28.086 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 18718/148671 objects degraded (12.590%), 16 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:28.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:28 smithi099 ceph-mon[167076]: pgmap v406: 109 pgs: 1 down, 3 remapped+peering, 12 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 18718/148671 objects degraded (12.590%); 34082/148671 objects misplaced (22.924%); 21 MiB/s, 339 objects/s recovering 2023-09-23T17:59:28.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 18718/148671 objects degraded (12.590%), 16 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:29.216 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:29.220 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:29.220 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:29.321 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:29.412 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:30.114 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:30.131 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:30.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:30.232 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:30.284 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:30.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:30.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:30.341 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:30.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:30 smithi055 ceph-mon[179668]: pgmap v407: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 19399/147294 objects degraded (13.170%); 36903/147294 objects misplaced (25.054%); 16 MiB/s, 253 objects/s recovering 2023-09-23T17:59:30.360 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:59:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:59:30] "GET /metrics HTTP/1.1" 200 36586 "" "Prometheus/2.43.0" 2023-09-23T17:59:30.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:30 smithi055 ceph-mon[184174]: pgmap v407: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 19399/147294 objects degraded (13.170%); 36903/147294 objects misplaced (25.054%); 16 MiB/s, 253 objects/s recovering 2023-09-23T17:59:30.398 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:30.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:30.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:30 smithi099 ceph-mon[167076]: pgmap v407: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 3.0 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 356 op/s; 19399/147294 objects degraded (13.170%); 36903/147294 objects misplaced (25.054%); 16 MiB/s, 253 objects/s recovering 2023-09-23T17:59:30.474 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:30.526 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:30.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:30.531 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:30.632 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:30.686 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:30.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:30.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:30.730 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:30.831 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:30.930 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:31.180 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:31.270 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:31.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:31.274 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:31.283 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T17:59:31.299 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:59:31.299 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T17:59:31.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T17:59:31.299 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:59:31.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.299 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:38871 5.2 5:4086753f:::benchmark_data_smithi055_229117_object38870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.242138+0000", 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.053350679, 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11274374099999999, 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 38871 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242138+0000", 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242138+0000", 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242136+0000", 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242159+0000", 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3130999999999999e-05 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242172+0000", 2023-09-23T17:59:31.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2405000000000001e-05 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242181+0000", 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1710000000000008e-06 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242234+0000", 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3562e-05 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242386+0000", 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015176400000000001 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242544+0000", 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015809600000000001 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244070+0000", 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015261820000000001 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247882+0000", 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038115940000000002 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247934+0000", 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1607000000000002e-05 2023-09-23T17:59:31.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.354881+0000", 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10694785900000001 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:38879 5.3 5:cc63ec65:::benchmark_data_smithi055_229117_object38878:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.244717+0000", 2023-09-23T17:59:31.306 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.05077175299999, 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.107349282, 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 38879 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.307 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244717+0000", 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244717+0000", 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244715+0000", 2023-09-23T17:59:31.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244741+0000", 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6139000000000002e-05 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244755+0000", 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4253000000000001e-05 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244763+0000", 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8320000000000006e-06 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244930+0000", 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000167455 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.245025+0000", 2023-09-23T17:59:31.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4429e-05 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.245114+0000", 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9466e-05 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247124+0000", 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002009294 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.351900+0000", 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.104776785 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.351964+0000", 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3223000000000001e-05 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.352066+0000", 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000102342 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.312 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:38882 5.3 5:c0aef0c4:::benchmark_data_smithi055_229117_object38881:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.244910+0000", 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.05057872200001, 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.109153715, 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.313 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 38882 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244910+0000", 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244910+0000", 2023-09-23T17:59:31.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244909+0000", 2023-09-23T17:59:31.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:59:31.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244933+0000", 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3796999999999999e-05 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244944+0000", 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1618e-05 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244952+0000", 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5989999999999996e-06 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.246293+0000", 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001341346 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.246377+0000", 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3608000000000003e-05 2023-09-23T17:59:31.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.246460+0000", 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3323000000000002e-05 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247289+0000", 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00082839199999999995 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353929+0000", 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10663998399999999 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353980+0000", 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1248000000000001e-05 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.354063+0000", 2023-09-23T17:59:31.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3555999999999998e-05 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:38883 5.e e117/110 5:7ce77683:::benchmark_data_smithi055_229117_object38882:head v 117'2409, mlcod=117'2406)", 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.246981+0000", 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.04850747699999, 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.109461428, 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.246981+0000", 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.246981+0000", 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.246980+0000", 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247046+0000", 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5328999999999997e-05 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247053+0000", 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9440000000000005e-06 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247058+0000", 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7770000000000002e-06 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247089+0000", 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0632000000000001e-05 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247103+0000", 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4151000000000001e-05 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.356401+0000", 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10929823299999999 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.356442+0000", 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1084999999999997e-05 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.324 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:38885 5.e e117/110 5:7e62a019:::benchmark_data_smithi055_229117_object38884:head v 117'2410, mlcod=117'2406)", 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.247175+0000", 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.04831365699999, 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.109947827, 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247175+0000", 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247175+0000", 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247174+0000", 2023-09-23T17:59:31.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247223+0000", 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9112e-05 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247231+0000", 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.61e-06 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247236+0000", 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8860000000000003e-06 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353831+0000", 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.106595129 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353853+0000", 2023-09-23T17:59:31.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1997e-05 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357086+0000", 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00323339 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357123+0000", 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6259e-05 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.329 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.44681.0:38871 5.2 e117/114 ondisk, result = 0)", 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.247817+0000", 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.047671772, 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.107076081, 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247817+0000", 2023-09-23T17:59:31.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247817+0000", 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247816+0000", 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247820+0000", 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5499999999999999e-06 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247825+0000", 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.604e-06 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247829+0000", 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7359999999999999e-06 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247866+0000", 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7416999999999998e-05 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247875+0000", 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5939999999999994e-06 2023-09-23T17:59:31.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.354893+0000", 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.107017691 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:38884 5.7 e117/110 5:e88075de:::benchmark_data_smithi055_229117_object38883:head v 117'2479, mlcod=117'2477)", 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.247847+0000", 2023-09-23T17:59:31.334 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.04764097399999, 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.108642849, 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247847+0000", 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247847+0000", 2023-09-23T17:59:31.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247847+0000", 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247885+0000", 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7460999999999999e-05 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247892+0000", 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2490000000000004e-06 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247897+0000", 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4550000000000003e-06 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247943+0000", 2023-09-23T17:59:31.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5413e-05 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.351677+0000", 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.103734639 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.356464+0000", 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047866510000000003 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.356490+0000", 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6251999999999999e-05 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:38888 5.7 e117/110 5:e84cc5a2:::benchmark_data_smithi055_229117_object38887:head v 117'2480, mlcod=117'2477)", 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.248707+0000", 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.046781557, 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.108458235, 2023-09-23T17:59:31.339 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.248707+0000", 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.248707+0000", 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.248706+0000", 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.248804+0000", 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7981999999999998e-05 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.351746+0000", 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.102941777 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.351764+0000", 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7669e-05 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353165+0000", 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014017579999999999 2023-09-23T17:59:31.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353233+0000", 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7902000000000001e-05 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357142+0000", 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039084819999999996 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357165+0000", 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3218e-05 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:38893 5.3 5:cae1f045:::benchmark_data_smithi055_229117_object38892:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.253941+0000", 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.04154776199999, 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10796506, 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.344 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 38893 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.253941+0000", 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.253941+0000", 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.253939+0000", 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.253968+0000", 2023-09-23T17:59:31.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9774999999999999e-05 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353317+0000", 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.099348658000000006 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353332+0000", 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4616e-05 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355479+0000", 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021470090000000001 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355623+0000", 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014422799999999999 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T17:59:31.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.355762+0000", 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013940400000000001 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357984+0000", 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022216499999999999 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.361771+0000", 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037866660000000002 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.361823+0000", 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2756000000000001e-05 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.361906+0000", 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2311999999999994e-05 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154235 5.2 5:44f142c1:::benchmark_data_smithi055_229117_object48609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.835816+0000", 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 22.459672304000001, 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.120395545, 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154235 2023-09-23T17:59:31.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835816+0000", 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835816+0000", 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835815+0000", 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835819+0000", 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7720000000000001e-06 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835827+0000", 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9359999999999999e-06 2023-09-23T17:59:31.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835835+0000", 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3760000000000002e-06 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839941+0000", 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041069510000000002 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840066+0000", 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000124762 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840214+0000", 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014806500000000001 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.845845+0000", 2023-09-23T17:59:31.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0056306910000000002 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.856676+0000", 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010831341 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.856687+0000", 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0695e-05 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956153+0000", 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0994657109999999 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956212+0000", 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8913000000000003e-05 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154236 5.2 5:46f3ddda:::benchmark_data_smithi055_229117_object48610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.835841+0000", 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 22.459647662999998, 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1205958920000001, 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154236 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835841+0000", 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835841+0000", 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835840+0000", 2023-09-23T17:59:31.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835843+0000", 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.638e-06 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835849+0000", 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9150000000000001e-06 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835854+0000", 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5650000000000001e-06 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.842866+0000", 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0070116919999999999 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.842987+0000", 2023-09-23T17:59:31.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120715 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.843124+0000", 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137155 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.846319+0000", 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031950939999999999 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.858461+0000", 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012142101000000001 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.858472+0000", 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0736e-05 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956386+0000", 2023-09-23T17:59:31.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.097913626 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956437+0000", 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1063000000000002e-05 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154274 5.5 5:aa5b5b0c:::benchmark_data_smithi055_229117_object48648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T17:59:31.364 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.105657+0000", 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 20.189830960999998, 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.20535718, 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154274 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105657+0000", 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105657+0000", 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105656+0000", 2023-09-23T17:59:31.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105663+0000", 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.742e-06 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105675+0000", 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1868000000000001e-05 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105683+0000", 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9410000000000007e-06 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298209+0000", 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.19252627899999999 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298292+0000", 2023-09-23T17:59:31.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3452e-05 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298686+0000", 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00039359900000000001 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306095+0000", 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0074089589999999997 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310633+0000", 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045383070000000001 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310919+0000", 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000285803 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310969+0000", 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0439999999999998e-05 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311015+0000", 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5250999999999998e-05 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.371 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154280 5.5 5:a78a277e:::benchmark_data_smithi055_229117_object48654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.166102+0000", 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 20.129386083, 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14501488900000001, 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154280 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166102+0000", 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166102+0000", 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166101+0000", 2023-09-23T17:59:31.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166109+0000", 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5660000000000001e-06 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166122+0000", 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3269e-05 2023-09-23T17:59:31.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166128+0000", 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5640000000000002e-06 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303566+0000", 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13743744199999999 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303721+0000", 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000155585 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303856+0000", 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000134177 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310596+0000", 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0067407539999999998 2023-09-23T17:59:31.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310889+0000", 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00029301300000000001 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311049+0000", 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016005699999999999 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311086+0000", 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6881999999999997e-05 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311117+0000", 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0936000000000003e-05 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154282 5.5 5:a1cbca1c:::benchmark_data_smithi055_229117_object48656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.171172+0000", 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 20.124316124, 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14593202499999999, 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154282 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.379 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171172+0000", 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171172+0000", 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171170+0000", 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171181+0000", 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.064e-05 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171212+0000", 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1433999999999998e-05 2023-09-23T17:59:31.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171223+0000", 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0265000000000001e-05 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306130+0000", 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13490687900000001 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306257+0000", 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012784 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306375+0000", 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011726999999999999 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310640+0000", 2023-09-23T17:59:31.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00426495 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.314542+0000", 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039025510000000002 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317023+0000", 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024812549999999999 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317068+0000", 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4632999999999998e-05 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317104+0000", 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6188e-05 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154290 5.5 5:a8c62379:::benchmark_data_smithi055_229117_object48664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.196937+0000", 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 20.098551166, 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.12027204499999999, 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154290 2023-09-23T17:59:31.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196937+0000", 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196937+0000", 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196936+0000", 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196943+0000", 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.306e-06 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196960+0000", 2023-09-23T17:59:31.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7173e-05 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196969+0000", 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7390000000000005e-06 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308177+0000", 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11120793499999999 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308311+0000", 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013440899999999999 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308441+0000", 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000129677 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.312252+0000", 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038113700000000001 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.314577+0000", 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023246730000000002 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317137+0000", 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025597609999999998 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317173+0000", 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6371000000000001e-05 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317209+0000", 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6191e-05 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161327 5.e 5:7b0ada0d:::benchmark_data_smithi055_229117_object55701:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852567+0000", 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 8.4429213270000005, 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0633735559999999, 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161327 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.393 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852567+0000", 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852567+0000", 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852565+0000", 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852572+0000", 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3729999999999998e-06 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852585+0000", 2023-09-23T17:59:31.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3426e-05 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852592+0000", 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2830000000000004e-06 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852658+0000", 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5399000000000002e-05 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op must wait for map", 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852668+0000", 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7110000000000007e-06 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872062+0000", 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019394650999999999 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872073+0000", 2023-09-23T17:59:31.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0607e-05 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.885379+0000", 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0133066719999999 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.887726+0000", 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023469929999999999 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.887957+0000", 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00023082800000000001 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915266+0000", 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027308517000000001 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915340+0000", 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4178999999999996e-05 2023-09-23T17:59:31.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915403+0000", 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2842000000000006e-05 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915811+0000", 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00040861899999999999 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915853+0000", 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1768999999999999e-05 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915941+0000", 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7410000000000005e-05 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.401 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161356 5.e 5:729afca1:::benchmark_data_smithi055_229117_object55730:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852605+0000", 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 8.442883406, 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0635528459999999, 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161356 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852607+0000", 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9570000000000001e-06 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852613+0000", 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1199999999999999e-06 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852616+0000", 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4319999999999999e-06 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852880+0000", 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026363200000000003 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T17:59:31.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852889+0000", 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8529999999999998e-06 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872097+0000", 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019208026999999999 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872104+0000", 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5739999999999998e-06 2023-09-23T17:59:31.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.905797+0000", 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0336926310000001 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.905945+0000", 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000147899 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.906095+0000", 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015013200000000001 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915327+0000", 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092321869999999993 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915351+0000", 2023-09-23T17:59:31.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.376e-05 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915452+0000", 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000101004 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915997+0000", 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00054548699999999999 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916057+0000", 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9496000000000001e-05 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916158+0000", 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000100995 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161361 5.e 5:727196ef:::benchmark_data_smithi055_229117_object55735:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852804+0000", 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 8.4426844729999999, 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0635737650000001, 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161361 2023-09-23T17:59:31.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852804+0000", 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852804+0000", 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852803+0000", 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852808+0000", 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3579999999999996e-06 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852820+0000", 2023-09-23T17:59:31.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2822e-05 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852828+0000", 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9980000000000003e-06 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111282 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852947+0000", 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3749999999999997e-06 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872127+0000", 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019180169 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:31.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872134+0000", 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.776e-06 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908641+0000", 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.036506683 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908722+0000", 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1464999999999993e-05 2023-09-23T17:59:31.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908805+0000", 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3060999999999996e-05 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915361+0000", 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00655603 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915491+0000", 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012961299999999999 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915586+0000", 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5175999999999998e-05 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916217+0000", 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00063121599999999998 2023-09-23T17:59:31.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916266+0000", 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8649000000000001e-05 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916378+0000", 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111749 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.419 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161400 5.e 5:7bd12518:::benchmark_data_smithi055_229117_object55774:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852914+0000", 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 8.4425747229999999, 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.066386922, 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.420 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161400 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852914+0000", 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852914+0000", 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852913+0000", 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852917+0000", 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1459999999999998e-06 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852927+0000", 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4700000000000008e-06 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852934+0000", 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8630000000000002e-06 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852982+0000", 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8585999999999999e-05 2023-09-23T17:59:31.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852990+0000", 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3989999999999998e-06 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872158+0000", 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019168266 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:31.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872163+0000", 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9620000000000001e-06 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.910791+0000", 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0386283650000001 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.910919+0000", 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012744899999999999 2023-09-23T17:59:31.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.911051+0000", 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013257500000000001 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915372+0000", 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004320618 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.426 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915538+0000", 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016641900000000001 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915724+0000", 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018572200000000001 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919091+0000", 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033675419999999998 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919170+0000", 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8464e-05 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919301+0000", 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013067499999999999 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161452 5.e 5:7e9dcc4b:::benchmark_data_smithi055_229117_object55826:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852940+0000", 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 8.4425485249999994, 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0665552599999999, 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161452 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.429 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:31.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852939+0000", 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852943+0000", 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1070000000000001e-06 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852949+0000", 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7900000000000002e-06 2023-09-23T17:59:31.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852955+0000", 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7760000000000003e-06 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.853019+0000", 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3619000000000002e-05 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.853023+0000", 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1899999999999997e-06 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872216+0000", 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019192753 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872224+0000", 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2779999999999993e-06 2023-09-23T17:59:31.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913001+0000", 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.040776715 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913126+0000", 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012584599999999999 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.434 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913244+0000", 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000117377 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915463+0000", 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022194480000000002 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915650+0000", 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018667299999999999 2023-09-23T17:59:31.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915759+0000", 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109165 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919339+0000", 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035801880000000002 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919386+0000", 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6295999999999999e-05 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919495+0000", 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109508 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:31.437 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:59:31.438 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:31.533 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:31.794 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:31.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:31.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:31.843 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T17:59:31.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:31.943 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:31.984 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:31.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:31.988 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:32.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:32.116 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:32.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:32.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:32.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:32.266 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:32.303 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:32 smithi099 ceph-mon[167076]: pgmap v408: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 18571/140898 objects degraded (13.180%); 35856/140898 objects misplaced (25.448%); 12 MiB/s, 197 objects/s recovering 2023-09-23T17:59:32.305 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:32.319 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:32 smithi055 ceph-mon[179668]: pgmap v408: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 18571/140898 objects degraded (13.180%); 35856/140898 objects misplaced (25.448%); 12 MiB/s, 197 objects/s recovering 2023-09-23T17:59:32.322 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:32 smithi055 ceph-mon[184174]: pgmap v408: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.9 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 554 op/s; 18571/140898 objects degraded (13.180%); 35856/140898 objects misplaced (25.448%); 12 MiB/s, 197 objects/s recovering 2023-09-23T17:59:32.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:32.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:32.352 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:32.453 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:32.524 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:32.611 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:32.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:32.618 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:32.718 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:32.769 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:32.804 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:32.940 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:32.947 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:32.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:33.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:33.082 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:33.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:33.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:33.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:33.147 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:33 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:59:33.147 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 18571/140898 objects degraded (13.180%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:33.234 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:33.252 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:33.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:33 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:59:33.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 18571/140898 objects degraded (13.180%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:33.348 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:33.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:33 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T17:59:33.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 18571/140898 objects degraded (13.180%), 17 pgs degraded, 26 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:33.433 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:59:33.496 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:59:33.534 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:59:33.591 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:59:33.630 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:59:33.677 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:59:33.685 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:33.687 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:59:33.687 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:59:33.687 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:59:33.688 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:59:33.688 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:59:33.688 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:59:33.688 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T17:59:33.688 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:59:33.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:33.689 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:33.790 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:33.935 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:34.035 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T17:59:34.053 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T17:59:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:34 smithi055 ceph-mon[179668]: pgmap v409: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 17924/137646 objects degraded (13.022%); 35412/137646 objects misplaced (25.727%); 15 MiB/s, 247 objects/s recovering 2023-09-23T17:59:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1757891861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:34.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:34 smithi055 ceph-mon[184174]: pgmap v409: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 17924/137646 objects degraded (13.022%); 35412/137646 objects misplaced (25.727%); 15 MiB/s, 247 objects/s recovering 2023-09-23T17:59:34.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1757891861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:34.447 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:59:34.447 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:59:34.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:34 smithi099 ceph-mon[167076]: pgmap v409: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.8 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 598 op/s; 17924/137646 objects degraded (13.022%); 35412/137646 objects misplaced (25.727%); 15 MiB/s, 247 objects/s recovering 2023-09-23T17:59:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1757891861' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:34.665 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":409,"stamp":"2023-09-23T17:59:32.552793+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3009812427,"num_objects":45882,"num_object_clones":0,"num_object_copies":137646,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17924,"num_objects_misplaced":35412,"num_objects_unfound":0,"num_objects_dirty":45882,"num_whiteouts":0,"num_read":6439,"num_read_kb":13059,"num_write":166540,"num_write_kb":6775014,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35935,"num_bytes_recovered":2355628826,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":112640,"ondisk_log_size":112640,"up":261,"acting":277,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":351,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10121672,"kb_used_data":9312848,"kb_used_omap":36,"kb_used_meta":808475,"kb_avail":646094392,"statfs":{"total":671965249536,"available":661600657408,"internally_reserved":0,"allocated":9536356352,"data_stored":9530748891,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37596,"internal_metadata":827878692},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,2,3,4,8,6],"upper_bound":64},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-341770240,"num_objects":-5215,"num_object_clones":0,"num_object_copies":-15645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1439,"num_objects_misplaced":3006,"num_objects_unfound":0,"num_objects_dirty":-5215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5215,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2156,"num_bytes_recovered":141492134,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.707525"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":27,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:25.943219+0000","last_change":"2023-09-23T17:59:22.862203+0000","last_active":"2023-09-23T17:59:25.943219+0000","last_peered":"2023-09-23T17:59:25.943219+0000","last_clean":"2023-09-23T17:59:25.943219+0000","last_became_active":"2023-09-23T17:59:22.861860+0000","last_became_peered":"2023-09-23T17:59:22.861860+0000","last_unstale":"2023-09-23T17:59:25.943219+0000","last_undegraded":"2023-09-23T17:59:25.943219+0000","last_fullsized":"2023-09-23T17:59:25.943219+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:35:06.644813+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":147,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.857763+0000","last_change":"2023-09-23T17:57:24.956156+0000","last_active":"2023-09-23T17:59:23.857763+0000","last_peered":"2023-09-23T17:59:23.857763+0000","last_clean":"2023-09-23T17:56:46.613857+0000","last_became_active":"2023-09-23T17:57:24.956156+0000","last_became_peered":"2023-09-23T17:57:24.956156+0000","last_unstale":"2023-09-23T17:59:23.857763+0000","last_undegraded":"2023-09-23T17:59:23.857763+0000","last_fullsized":"2023-09-23T17:57:24.947251+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:35.949179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":146,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.857083+0000","last_change":"2023-09-23T17:57:24.956353+0000","last_active":"2023-09-23T17:59:23.857083+0000","last_peered":"2023-09-23T17:59:23.857083+0000","last_clean":"2023-09-23T17:57:20.922940+0000","last_became_active":"2023-09-23T17:57:24.956353+0000","last_became_peered":"2023-09-23T17:57:24.956353+0000","last_unstale":"2023-09-23T17:59:23.857083+0000","last_undegraded":"2023-09-23T17:59:23.857083+0000","last_fullsized":"2023-09-23T17:57:24.946863+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:41.136418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":144,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.851769+0000","last_change":"2023-09-23T17:58:11.996431+0000","last_active":"2023-09-23T17:59:23.851769+0000","last_peered":"2023-09-23T17:59:23.851769+0000","last_clean":"2023-09-23T17:57:20.931176+0000","last_became_active":"2023-09-23T17:58:11.996431+0000","last_became_peered":"2023-09-23T17:58:11.996431+0000","last_unstale":"2023-09-23T17:59:23.851769+0000","last_undegraded":"2023-09-23T17:59:23.851769+0000","last_fullsized":"2023-09-23T17:58:11.991094+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:21.919913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":128,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.848698+0000","last_change":"2023-09-23T17:57:24.965072+0000","last_active":"2023-09-23T17:59:23.848698+0000","last_peered":"2023-09-23T17:59:23.848698+0000","last_clean":"2023-09-23T17:53:21.039477+0000","last_became_active":"2023-09-23T17:57:24.965072+0000","last_became_peered":"2023-09-23T17:57:24.965072+0000","last_unstale":"2023-09-23T17:59:23.848698+0000","last_undegraded":"2023-09-23T17:59:23.848698+0000","last_fullsized":"2023-09-23T17:57:24.947630+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:41.627468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":102,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.850292+0000","last_change":"2023-09-23T17:59:09.861723+0000","last_active":"2023-09-23T17:59:23.850292+0000","last_peered":"2023-09-23T17:59:23.850292+0000","last_clean":"2023-09-23T17:59:23.850292+0000","last_became_active":"2023-09-23T17:59:09.861324+0000","last_became_peered":"2023-09-23T17:59:09.861324+0000","last_unstale":"2023-09-23T17:59:23.850292+0000","last_undegraded":"2023-09-23T17:59:23.850292+0000","last_fullsized":"2023-09-23T17:59:23.850292+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:45.865590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":141,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.852302+0000","last_change":"2023-09-23T17:59:10.881778+0000","last_active":"2023-09-23T17:59:23.852302+0000","last_peered":"2023-09-23T17:59:23.852302+0000","last_clean":"2023-09-23T17:59:23.852302+0000","last_became_active":"2023-09-23T17:59:10.881369+0000","last_became_peered":"2023-09-23T17:59:10.881369+0000","last_unstale":"2023-09-23T17:59:23.852302+0000","last_undegraded":"2023-09-23T17:59:23.852302+0000","last_fullsized":"2023-09-23T17:59:23.852302+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:06.715447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":130,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.857457+0000","last_change":"2023-09-23T17:58:35.245269+0000","last_active":"2023-09-23T17:59:23.857457+0000","last_peered":"2023-09-23T17:59:23.857457+0000","last_clean":"2023-09-23T17:55:56.840611+0000","last_became_active":"2023-09-23T17:58:35.245269+0000","last_became_peered":"2023-09-23T17:58:35.245269+0000","last_unstale":"2023-09-23T17:59:23.857457+0000","last_undegraded":"2023-09-23T17:59:23.857457+0000","last_fullsized":"2023-09-23T17:58:35.227362+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:14:50.366063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":481,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.851982+0000","last_change":"2023-09-23T17:59:00.132664+0000","last_active":"2023-09-23T17:59:23.851982+0000","last_peered":"2023-09-23T17:59:23.851982+0000","last_clean":"2023-09-23T17:59:23.851982+0000","last_became_active":"2023-09-23T17:59:00.132396+0000","last_became_peered":"2023-09-23T17:59:00.132396+0000","last_unstale":"2023-09-23T17:59:23.851982+0000","last_undegraded":"2023-09-23T17:59:23.851982+0000","last_fullsized":"2023-09-23T17:59:23.851982+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":128,"reported_epoch":181,"state":"active+undersized+remapped","last_fresh":"2023-09-23T17:59:23.858116+0000","last_change":"2023-09-23T17:59:10.880170+0000","last_active":"2023-09-23T17:59:23.858116+0000","last_peered":"2023-09-23T17:59:23.858116+0000","last_clean":"2023-09-23T17:57:47.590780+0000","last_became_active":"2023-09-23T17:59:10.880170+0000","last_became_peered":"2023-09-23T17:59:10.880170+0000","last_unstale":"2023-09-23T17:59:23.858116+0000","last_undegraded":"2023-09-23T17:59:23.858116+0000","last_fullsized":"2023-09-23T17:59:10.878551+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:31.432584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":38,"reported_epoch":181,"state":"down","last_fresh":"2023-09-23T17:59:23.848941+0000","last_change":"2023-09-23T17:58:34.212200+0000","last_active":"2023-09-23T17:45:51.086592+0000","last_peered":"2023-09-23T17:45:51.086592+0000","last_clean":"2023-09-23T17:45:51.086592+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T17:59:23.848941+0000","last_undegraded":"2023-09-23T17:59:23.848941+0000","last_fullsized":"2023-09-23T17:59:23.848941+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:32.367332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":134,"reported_epoch":181,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:24.376002+0000","last_change":"2023-09-23T17:59:22.870892+0000","last_active":"2023-09-23T17:59:24.376002+0000","last_peered":"2023-09-23T17:59:24.376002+0000","last_clean":"2023-09-23T17:56:46.621999+0000","last_became_active":"2023-09-23T17:59:22.870892+0000","last_became_peered":"2023-09-23T17:59:22.870892+0000","last_unstale":"2023-09-23T17:59:24.376002+0000","last_undegraded":"2023-09-23T17:59:22.864975+0000","last_fullsized":"2023-09-23T17:59:22.864853+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:24.430932+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":90,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":2}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":137,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.857045+0000","last_change":"2023-09-23T17:58:35.248074+0000","last_active":"2023-09-23T17:59:23.857045+0000","last_peered":"2023-09-23T17:59:23.857045+0000","last_clean":"2023-09-23T17:59:23.857045+0000","last_became_active":"2023-09-23T17:58:35.247751+0000","last_became_peered":"2023-09-23T17:58:35.247751+0000","last_unstale":"2023-09-23T17:59:23.857045+0000","last_undegraded":"2023-09-23T17:59:23.857045+0000","last_fullsized":"2023-09-23T17:59:23.857045+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:08.488838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":67,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.853654+0000","last_change":"2023-09-23T17:59:23.852905+0000","last_active":"2023-09-23T17:59:23.853654+0000","last_peered":"2023-09-23T17:59:23.853654+0000","last_clean":"2023-09-23T17:59:23.853654+0000","last_became_active":"2023-09-23T17:59:23.852369+0000","last_became_peered":"2023-09-23T17:59:23.852369+0000","last_unstale":"2023-09-23T17:59:23.853654+0000","last_undegraded":"2023-09-23T17:59:23.853654+0000","last_fullsized":"2023-09-23T17:59:23.853654+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:01.253707+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,2],"avail_no_missing":["6","2","4"],"object_location_counts":[{"shards":"2,4,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":155,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.849464+0000","last_change":"2023-09-23T17:59:22.857844+0000","last_active":"2023-09-23T17:59:23.849464+0000","last_peered":"2023-09-23T17:59:23.849464+0000","last_clean":"2023-09-23T17:59:23.849464+0000","last_became_active":"2023-09-23T17:59:22.857515+0000","last_became_peered":"2023-09-23T17:59:22.857515+0000","last_unstale":"2023-09-23T17:59:23.849464+0000","last_undegraded":"2023-09-23T17:59:23.849464+0000","last_fullsized":"2023-09-23T17:59:23.849464+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:32:48.401754+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":476,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.848334+0000","last_change":"2023-09-23T17:59:00.129648+0000","last_active":"2023-09-23T17:59:23.848334+0000","last_peered":"2023-09-23T17:59:23.848334+0000","last_clean":"2023-09-23T17:59:23.848334+0000","last_became_active":"2023-09-23T17:59:00.129253+0000","last_became_peered":"2023-09-23T17:59:00.129253+0000","last_unstale":"2023-09-23T17:59:23.848334+0000","last_undegraded":"2023-09-23T17:59:23.848334+0000","last_fullsized":"2023-09-23T17:59:23.848334+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":22,"reported_epoch":181,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:23.908316+0000","last_change":"2023-09-23T17:59:22.862930+0000","last_active":"2023-09-23T17:59:23.908316+0000","last_peered":"2023-09-23T17:59:23.908316+0000","last_clean":"2023-09-23T17:45:54.586105+0000","last_became_active":"2023-09-23T17:59:22.862930+0000","last_became_peered":"2023-09-23T17:59:22.862930+0000","last_unstale":"2023-09-23T17:59:23.908316+0000","last_undegraded":"2023-09-23T17:59:22.853542+0000","last_fullsized":"2023-09-23T17:59:22.853418+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:25.383057+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":115,"reported_epoch":181,"state":"undersized+peered","last_fresh":"2023-09-23T17:59:23.858062+0000","last_change":"2023-09-23T17:57:53.869041+0000","last_active":"2023-09-23T17:57:52.875213+0000","last_peered":"2023-09-23T17:59:23.858062+0000","last_clean":"2023-09-23T17:53:21.028332+0000","last_became_active":"2023-09-23T17:57:24.954701+0000","last_became_peered":"2023-09-23T17:57:53.869041+0000","last_unstale":"2023-09-23T17:59:23.858062+0000","last_undegraded":"2023-09-23T17:59:23.858062+0000","last_fullsized":"2023-09-23T17:57:53.867092+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:44.473315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.847952+0000","last_change":"2023-09-23T17:59:00.141765+0000","last_active":"2023-09-23T17:59:23.847952+0000","last_peered":"2023-09-23T17:59:23.847952+0000","last_clean":"2023-09-23T17:59:23.847952+0000","last_became_active":"2023-09-23T17:59:00.137566+0000","last_became_peered":"2023-09-23T17:59:00.137566+0000","last_unstale":"2023-09-23T17:59:23.847952+0000","last_undegraded":"2023-09-23T17:59:23.847952+0000","last_fullsized":"2023-09-23T17:59:23.847952+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":142,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.848426+0000","last_change":"2023-09-23T17:58:11.995550+0000","last_active":"2023-09-23T17:59:23.848426+0000","last_peered":"2023-09-23T17:59:23.848426+0000","last_clean":"2023-09-23T17:57:20.931103+0000","last_became_active":"2023-09-23T17:58:11.995550+0000","last_became_peered":"2023-09-23T17:58:11.995550+0000","last_unstale":"2023-09-23T17:59:23.848426+0000","last_undegraded":"2023-09-23T17:59:23.848426+0000","last_fullsized":"2023-09-23T17:58:11.986762+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:31.912263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":15,"reported_epoch":181,"state":"active+undersized+remapped","last_fresh":"2023-09-23T17:59:23.849620+0000","last_change":"2023-09-23T17:59:10.875110+0000","last_active":"2023-09-23T17:59:23.849620+0000","last_peered":"2023-09-23T17:59:23.849620+0000","last_clean":"2023-09-23T17:57:20.925545+0000","last_became_active":"2023-09-23T17:59:10.875110+0000","last_became_peered":"2023-09-23T17:59:10.875110+0000","last_unstale":"2023-09-23T17:59:23.849620+0000","last_undegraded":"2023-09-23T17:59:23.849620+0000","last_fullsized":"2023-09-23T17:59:10.873146+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:35:28.623974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.848329+0000","last_change":"2023-09-23T17:59:00.134116+0000","last_active":"2023-09-23T17:59:23.848329+0000","last_peered":"2023-09-23T17:59:23.848329+0000","last_clean":"2023-09-23T17:59:23.848329+0000","last_became_active":"2023-09-23T17:59:00.133756+0000","last_became_peered":"2023-09-23T17:59:00.133756+0000","last_unstale":"2023-09-23T17:59:23.848329+0000","last_undegraded":"2023-09-23T17:59:23.848329+0000","last_fullsized":"2023-09-23T17:59:23.848329+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":169,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.851751+0000","last_change":"2023-09-23T17:58:36.251995+0000","last_active":"2023-09-23T17:59:23.851751+0000","last_peered":"2023-09-23T17:59:23.851751+0000","last_clean":"2023-09-23T17:59:23.851751+0000","last_became_active":"2023-09-23T17:58:36.251578+0000","last_became_peered":"2023-09-23T17:58:36.251578+0000","last_unstale":"2023-09-23T17:59:23.851751+0000","last_undegraded":"2023-09-23T17:59:23.851751+0000","last_fullsized":"2023-09-23T17:59:23.851751+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:45.817777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,0],"avail_no_missing":["4","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.851891+0000","last_change":"2023-09-23T17:58:11.996093+0000","last_active":"2023-09-23T17:59:23.851891+0000","last_peered":"2023-09-23T17:59:23.851891+0000","last_clean":"2023-09-23T17:57:20.931097+0000","last_became_active":"2023-09-23T17:58:11.996093+0000","last_became_peered":"2023-09-23T17:58:11.996093+0000","last_unstale":"2023-09-23T17:59:23.851891+0000","last_undegraded":"2023-09-23T17:59:23.851891+0000","last_fullsized":"2023-09-23T17:58:11.990799+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:58.128284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.849437+0000","last_change":"2023-09-23T17:59:00.133252+0000","last_active":"2023-09-23T17:59:23.849437+0000","last_peered":"2023-09-23T17:59:23.849437+0000","last_clean":"2023-09-23T17:59:23.849437+0000","last_became_active":"2023-09-23T17:59:00.132882+0000","last_became_peered":"2023-09-23T17:59:00.132882+0000","last_unstale":"2023-09-23T17:59:23.849437+0000","last_undegraded":"2023-09-23T17:59:23.849437+0000","last_fullsized":"2023-09-23T17:59:23.849437+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":108,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.848860+0000","last_change":"2023-09-23T17:59:10.878099+0000","last_active":"2023-09-23T17:59:23.848860+0000","last_peered":"2023-09-23T17:59:23.848860+0000","last_clean":"2023-09-23T17:59:23.848860+0000","last_became_active":"2023-09-23T17:59:10.877670+0000","last_became_peered":"2023-09-23T17:59:10.877670+0000","last_unstale":"2023-09-23T17:59:23.848860+0000","last_undegraded":"2023-09-23T17:59:23.848860+0000","last_fullsized":"2023-09-23T17:59:23.848860+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:36.244235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":121,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.858633+0000","last_change":"2023-09-23T17:58:11.993687+0000","last_active":"2023-09-23T17:59:23.858633+0000","last_peered":"2023-09-23T17:59:23.858633+0000","last_clean":"2023-09-23T17:53:21.029434+0000","last_became_active":"2023-09-23T17:58:11.993687+0000","last_became_peered":"2023-09-23T17:58:11.993687+0000","last_unstale":"2023-09-23T17:59:23.858633+0000","last_undegraded":"2023-09-23T17:59:23.858633+0000","last_fullsized":"2023-09-23T17:58:11.985535+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:33.153133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.852900+0000","last_change":"2023-09-23T17:59:00.134603+0000","last_active":"2023-09-23T17:59:23.852900+0000","last_peered":"2023-09-23T17:59:23.852900+0000","last_clean":"2023-09-23T17:59:23.852900+0000","last_became_active":"2023-09-23T17:59:00.134099+0000","last_became_peered":"2023-09-23T17:59:00.134099+0000","last_unstale":"2023-09-23T17:59:23.852900+0000","last_undegraded":"2023-09-23T17:59:23.852900+0000","last_fullsized":"2023-09-23T17:59:23.852900+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.9","version":"182'9988","reported_seq":27798,"reported_epoch":182,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:28.208709+0000","last_change":"2023-09-23T17:58:52.934142+0000","last_active":"2023-09-23T17:59:28.208709+0000","last_peered":"2023-09-23T17:59:28.208709+0000","last_clean":"2023-09-23T17:59:28.208709+0000","last_became_active":"2023-09-23T17:58:52.879112+0000","last_became_peered":"2023-09-23T17:58:52.879112+0000","last_unstale":"2023-09-23T17:59:28.208709+0000","last_undegraded":"2023-09-23T17:59:28.208709+0000","last_fullsized":"2023-09-23T17:59:28.208709+0000","mapping_epoch":168,"log_start":"125'4618","ondisk_log_start":"125'4618","created":110,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5370,"log_dups_size":0,"ondisk_log_size":5370,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:54.486328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185597952,"num_objects":2832,"num_object_clones":0,"num_object_copies":8496,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2832,"num_objects_unfound":0,"num_objects_dirty":2832,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9988,"num_write_kb":410240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8835,"num_bytes_recovered":578945024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":2832}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":146,"reported_epoch":181,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:23.853461+0000","last_change":"2023-09-23T17:57:53.875345+0000","last_active":"2023-09-23T17:59:23.853461+0000","last_peered":"2023-09-23T17:59:23.853461+0000","last_clean":"2023-09-23T17:56:46.621793+0000","last_became_active":"2023-09-23T17:57:53.875345+0000","last_became_peered":"2023-09-23T17:57:53.875345+0000","last_unstale":"2023-09-23T17:59:23.853461+0000","last_undegraded":"2023-09-23T17:57:53.869204+0000","last_fullsized":"2023-09-23T17:57:53.869091+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:19.033883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":148,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.848845+0000","last_change":"2023-09-23T17:59:10.877550+0000","last_active":"2023-09-23T17:59:23.848845+0000","last_peered":"2023-09-23T17:59:23.848845+0000","last_clean":"2023-09-23T17:59:23.848845+0000","last_became_active":"2023-09-23T17:59:10.877173+0000","last_became_peered":"2023-09-23T17:59:10.877173+0000","last_unstale":"2023-09-23T17:59:23.848845+0000","last_undegraded":"2023-09-23T17:59:23.848845+0000","last_fullsized":"2023-09-23T17:59:23.848845+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:00.740053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,5],"avail_no_missing":["6","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":482,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.851714+0000","last_change":"2023-09-23T17:59:00.136455+0000","last_active":"2023-09-23T17:59:23.851714+0000","last_peered":"2023-09-23T17:59:23.851714+0000","last_clean":"2023-09-23T17:59:23.851714+0000","last_became_active":"2023-09-23T17:59:00.136126+0000","last_became_peered":"2023-09-23T17:59:00.136126+0000","last_unstale":"2023-09-23T17:59:23.851714+0000","last_undegraded":"2023-09-23T17:59:23.851714+0000","last_fullsized":"2023-09-23T17:59:23.851714+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.8","version":"182'10360","reported_seq":10442,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T17:59:28.878766+0000","last_change":"2023-09-23T17:55:53.818806+0000","last_active":"2023-09-23T17:59:28.878766+0000","last_peered":"2023-09-23T17:59:28.878766+0000","last_clean":"2023-09-23T17:59:28.878766+0000","last_became_active":"2023-09-23T17:55:53.818291+0000","last_became_peered":"2023-09-23T17:55:53.818291+0000","last_unstale":"2023-09-23T17:59:28.878766+0000","last_undegraded":"2023-09-23T17:59:28.878766+0000","last_fullsized":"2023-09-23T17:59:28.878766+0000","mapping_epoch":110,"log_start":"115'600","ondisk_log_start":"115'600","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9760,"log_dups_size":0,"ondisk_log_size":9760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:58.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192544768,"num_objects":2938,"num_object_clones":0,"num_object_copies":8814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10360,"num_write_kb":425536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":22,"reported_epoch":181,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:25.943425+0000","last_change":"2023-09-23T17:59:22.864347+0000","last_active":"2023-09-23T17:59:25.943425+0000","last_peered":"2023-09-23T17:59:25.943425+0000","last_clean":"2023-09-23T17:45:54.586235+0000","last_became_active":"2023-09-23T17:59:22.864347+0000","last_became_peered":"2023-09-23T17:59:22.864347+0000","last_unstale":"2023-09-23T17:59:25.943425+0000","last_undegraded":"2023-09-23T17:59:22.857300+0000","last_fullsized":"2023-09-23T17:59:22.857165+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:42.778310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":137,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:25.943110+0000","last_change":"2023-09-23T17:59:22.856305+0000","last_active":"2023-09-23T17:59:25.943110+0000","last_peered":"2023-09-23T17:59:25.943110+0000","last_clean":"2023-09-23T17:57:20.922147+0000","last_became_active":"2023-09-23T17:59:22.856305+0000","last_became_peered":"2023-09-23T17:59:22.856305+0000","last_unstale":"2023-09-23T17:59:25.943110+0000","last_undegraded":"2023-09-23T17:59:25.943110+0000","last_fullsized":"2023-09-23T17:59:22.853659+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:40.156556+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.852089+0000","last_change":"2023-09-23T17:59:00.128843+0000","last_active":"2023-09-23T17:59:23.852089+0000","last_peered":"2023-09-23T17:59:23.852089+0000","last_clean":"2023-09-23T17:59:23.852089+0000","last_became_active":"2023-09-23T17:59:00.128472+0000","last_became_peered":"2023-09-23T17:59:00.128472+0000","last_unstale":"2023-09-23T17:59:23.852089+0000","last_undegraded":"2023-09-23T17:59:23.852089+0000","last_fullsized":"2023-09-23T17:59:23.852089+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.b","version":"182'10259","reported_seq":10351,"reported_epoch":182,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:28.843232+0000","last_change":"2023-09-23T17:59:10.964199+0000","last_active":"2023-09-23T17:59:28.843232+0000","last_peered":"2023-09-23T17:59:28.843232+0000","last_clean":"2023-09-23T17:59:28.843232+0000","last_became_active":"2023-09-23T17:59:10.899328+0000","last_became_peered":"2023-09-23T17:59:10.899328+0000","last_unstale":"2023-09-23T17:59:28.843232+0000","last_undegraded":"2023-09-23T17:59:28.843232+0000","last_fullsized":"2023-09-23T17:59:28.843232+0000","mapping_epoch":176,"log_start":"115'500","ondisk_log_start":"115'500","created":110,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9759,"log_dups_size":0,"ondisk_log_size":9759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:47.317955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190775296,"num_objects":2911,"num_object_clones":0,"num_object_copies":8733,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2911,"num_objects_unfound":0,"num_objects_dirty":2911,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10259,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":2911}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":29,"reported_epoch":181,"state":"active+undersized+remapped","last_fresh":"2023-09-23T17:59:23.849846+0000","last_change":"2023-09-23T17:59:10.875436+0000","last_active":"2023-09-23T17:59:23.849846+0000","last_peered":"2023-09-23T17:59:23.849846+0000","last_clean":"2023-09-23T17:45:54.586241+0000","last_became_active":"2023-09-23T17:59:10.875436+0000","last_became_peered":"2023-09-23T17:59:10.875436+0000","last_unstale":"2023-09-23T17:59:23.849846+0000","last_undegraded":"2023-09-23T17:59:23.849846+0000","last_fullsized":"2023-09-23T17:59:10.873636+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:39.737287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":117,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.849186+0000","last_change":"2023-09-23T17:59:09.884256+0000","last_active":"2023-09-23T17:59:23.849186+0000","last_peered":"2023-09-23T17:59:23.849186+0000","last_clean":"2023-09-23T17:59:23.849186+0000","last_became_active":"2023-09-23T17:59:09.883903+0000","last_became_peered":"2023-09-23T17:59:09.883903+0000","last_unstale":"2023-09-23T17:59:23.849186+0000","last_undegraded":"2023-09-23T17:59:23.849186+0000","last_fullsized":"2023-09-23T17:59:23.849186+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:08.901791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.851496+0000","last_change":"2023-09-23T17:59:00.127991+0000","last_active":"2023-09-23T17:59:23.851496+0000","last_peered":"2023-09-23T17:59:23.851496+0000","last_clean":"2023-09-23T17:59:23.851496+0000","last_became_active":"2023-09-23T17:59:00.127676+0000","last_became_peered":"2023-09-23T17:59:00.127676+0000","last_unstale":"2023-09-23T17:59:23.851496+0000","last_undegraded":"2023-09-23T17:59:23.851496+0000","last_fullsized":"2023-09-23T17:59:23.851496+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.a","version":"182'10267","reported_seq":11173,"reported_epoch":182,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:28.878892+0000","last_change":"2023-09-23T17:57:24.992424+0000","last_active":"2023-09-23T17:59:28.878892+0000","last_peered":"2023-09-23T17:59:28.878892+0000","last_clean":"2023-09-23T17:57:23.931663+0000","last_became_active":"2023-09-23T17:57:24.964668+0000","last_became_peered":"2023-09-23T17:57:24.964668+0000","last_unstale":"2023-09-23T17:59:28.878892+0000","last_undegraded":"2023-09-23T17:57:24.952679+0000","last_fullsized":"2023-09-23T17:57:24.947063+0000","mapping_epoch":142,"log_start":"115'500","ondisk_log_start":"115'500","created":110,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9767,"log_dups_size":0,"ondisk_log_size":9767,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:59.839324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188940288,"num_objects":2883,"num_object_clones":0,"num_object_copies":8649,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2883,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2883,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10267,"num_write_kb":420800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":396,"num_bytes_recovered":25952256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":2883}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":159,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.850242+0000","last_change":"2023-09-23T17:59:10.881020+0000","last_active":"2023-09-23T17:59:23.850242+0000","last_peered":"2023-09-23T17:59:23.850242+0000","last_clean":"2023-09-23T17:59:23.850242+0000","last_became_active":"2023-09-23T17:59:10.880602+0000","last_became_peered":"2023-09-23T17:59:10.880602+0000","last_unstale":"2023-09-23T17:59:23.850242+0000","last_undegraded":"2023-09-23T17:59:23.850242+0000","last_fullsized":"2023-09-23T17:59:23.850242+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:15.837348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,5],"avail_no_missing":["6","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":144,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.851276+0000","last_change":"2023-09-23T17:56:50.828739+0000","last_active":"2023-09-23T17:59:23.851276+0000","last_peered":"2023-09-23T17:59:23.851276+0000","last_clean":"2023-09-23T17:59:23.851276+0000","last_became_active":"2023-09-23T17:56:50.828229+0000","last_became_peered":"2023-09-23T17:56:50.828229+0000","last_unstale":"2023-09-23T17:59:23.851276+0000","last_undegraded":"2023-09-23T17:59:23.851276+0000","last_fullsized":"2023-09-23T17:59:23.851276+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:45.927953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":468,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.848262+0000","last_change":"2023-09-23T17:58:35.252704+0000","last_active":"2023-09-23T17:59:23.848262+0000","last_peered":"2023-09-23T17:59:23.848262+0000","last_clean":"2023-09-23T17:59:23.848262+0000","last_became_active":"2023-09-23T17:58:35.252386+0000","last_became_peered":"2023-09-23T17:58:35.252386+0000","last_unstale":"2023-09-23T17:59:23.848262+0000","last_undegraded":"2023-09-23T17:59:23.848262+0000","last_fullsized":"2023-09-23T17:59:23.848262+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"182'10264","reported_seq":10361,"reported_epoch":182,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T17:59:28.879840+0000","last_change":"2023-09-23T17:59:11.340557+0000","last_active":"2023-09-23T17:59:28.879840+0000","last_peered":"2023-09-23T17:59:28.879840+0000","last_clean":"2023-09-23T17:57:52.866355+0000","last_became_active":"2023-09-23T17:59:11.311441+0000","last_became_peered":"2023-09-23T17:59:11.311441+0000","last_unstale":"2023-09-23T17:59:28.879840+0000","last_undegraded":"2023-09-23T17:59:10.972976+0000","last_fullsized":"2023-09-23T17:59:10.890539+0000","mapping_epoch":176,"log_start":"115'500","ondisk_log_start":"115'500","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9764,"log_dups_size":0,"ondisk_log_size":9764,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:06.586919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188022808,"num_objects":2870,"num_object_clones":0,"num_object_copies":8610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2870,"num_objects_misplaced":6566,"num_objects_unfound":0,"num_objects_dirty":2870,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10264,"num_write_kb":420225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-647}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":67,"reported_epoch":181,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:23.851378+0000","last_change":"2023-09-23T17:57:24.957601+0000","last_active":"2023-09-23T17:59:23.851378+0000","last_peered":"2023-09-23T17:59:23.851378+0000","last_clean":"2023-09-23T17:45:54.465462+0000","last_became_active":"2023-09-23T17:57:24.957601+0000","last_became_peered":"2023-09-23T17:57:24.957601+0000","last_unstale":"2023-09-23T17:59:23.851378+0000","last_undegraded":"2023-09-23T17:57:24.946354+0000","last_fullsized":"2023-09-23T17:57:24.946239+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.908857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":161,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.860052+0000","last_change":"2023-09-23T17:59:23.859486+0000","last_active":"2023-09-23T17:59:23.860052+0000","last_peered":"2023-09-23T17:59:23.860052+0000","last_clean":"2023-09-23T17:59:23.860052+0000","last_became_active":"2023-09-23T17:59:23.858996+0000","last_became_peered":"2023-09-23T17:59:23.858996+0000","last_unstale":"2023-09-23T17:59:23.860052+0000","last_undegraded":"2023-09-23T17:59:23.860052+0000","last_fullsized":"2023-09-23T17:59:23.860052+0000","mapping_epoch":180,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":181,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:15:27.118811+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":472,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.847981+0000","last_change":"2023-09-23T17:58:35.251808+0000","last_active":"2023-09-23T17:59:23.847981+0000","last_peered":"2023-09-23T17:59:23.847981+0000","last_clean":"2023-09-23T17:59:23.847981+0000","last_became_active":"2023-09-23T17:58:35.251267+0000","last_became_peered":"2023-09-23T17:58:35.251267+0000","last_unstale":"2023-09-23T17:59:23.847981+0000","last_undegraded":"2023-09-23T17:59:23.847981+0000","last_fullsized":"2023-09-23T17:59:23.847981+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.c","version":"182'10405","reported_seq":10505,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T17:59:30.908085+0000","last_change":"2023-09-23T17:56:51.840960+0000","last_active":"2023-09-23T17:59:30.908085+0000","last_peered":"2023-09-23T17:59:30.908085+0000","last_clean":"2023-09-23T17:59:30.908085+0000","last_became_active":"2023-09-23T17:56:51.835697+0000","last_became_peered":"2023-09-23T17:56:51.835697+0000","last_unstale":"2023-09-23T17:59:30.908085+0000","last_undegraded":"2023-09-23T17:59:30.908085+0000","last_fullsized":"2023-09-23T17:59:30.908085+0000","mapping_epoch":127,"log_start":"137'5998","ondisk_log_start":"137'5998","created":110,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4407,"log_dups_size":0,"ondisk_log_size":4407,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:45.804116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183042048,"num_objects":2793,"num_object_clones":0,"num_object_copies":8379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10405,"num_write_kb":422336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":93,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.849373+0000","last_change":"2023-09-23T17:58:35.243177+0000","last_active":"2023-09-23T17:59:23.849373+0000","last_peered":"2023-09-23T17:59:23.849373+0000","last_clean":"2023-09-23T17:59:23.849373+0000","last_became_active":"2023-09-23T17:58:35.242824+0000","last_became_peered":"2023-09-23T17:58:35.242824+0000","last_unstale":"2023-09-23T17:59:23.849373+0000","last_undegraded":"2023-09-23T17:59:23.849373+0000","last_fullsized":"2023-09-23T17:59:23.849373+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:10.250046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":166,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.908524+0000","last_change":"2023-09-23T17:59:22.863431+0000","last_active":"2023-09-23T17:59:23.908524+0000","last_peered":"2023-09-23T17:59:23.908524+0000","last_clean":"2023-09-23T17:59:23.908524+0000","last_became_active":"2023-09-23T17:59:22.863123+0000","last_became_peered":"2023-09-23T17:59:22.863123+0000","last_unstale":"2023-09-23T17:59:23.908524+0000","last_undegraded":"2023-09-23T17:59:23.908524+0000","last_fullsized":"2023-09-23T17:59:23.908524+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:09.000763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.848385+0000","last_change":"2023-09-23T17:59:00.133828+0000","last_active":"2023-09-23T17:59:23.848385+0000","last_peered":"2023-09-23T17:59:23.848385+0000","last_clean":"2023-09-23T17:59:23.848385+0000","last_became_active":"2023-09-23T17:59:00.133283+0000","last_became_peered":"2023-09-23T17:59:00.133283+0000","last_unstale":"2023-09-23T17:59:23.848385+0000","last_undegraded":"2023-09-23T17:59:23.848385+0000","last_fullsized":"2023-09-23T17:59:23.848385+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.f","version":"182'10428","reported_seq":19276,"reported_epoch":182,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T17:59:32.405404+0000","last_change":"2023-09-23T17:59:24.085819+0000","last_active":"2023-09-23T17:59:32.405404+0000","last_peered":"2023-09-23T17:59:32.405404+0000","last_clean":"2023-09-23T17:55:56.846049+0000","last_became_active":"2023-09-23T17:59:23.979741+0000","last_became_peered":"2023-09-23T17:59:23.979741+0000","last_unstale":"2023-09-23T17:59:32.405404+0000","last_undegraded":"2023-09-23T17:59:23.972740+0000","last_fullsized":"2023-09-23T17:59:23.851330+0000","mapping_epoch":180,"log_start":"120'3747","ondisk_log_start":"120'3747","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6681,"log_dups_size":0,"ondisk_log_size":6681,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:29.332920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180748288,"num_objects":2758,"num_object_clones":0,"num_object_copies":8274,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":587,"num_objects_misplaced":2755,"num_objects_unfound":0,"num_objects_dirty":2758,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10428,"num_write_kb":421952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4049,"num_bytes_recovered":265027584,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[4,2],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"2,4","objects":3},{"shards":"2,4,5","objects":2168},{"shards":"4,5","objects":587}],"blocked_by":[],"up_primary":6,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":72,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.851471+0000","last_change":"2023-09-23T17:57:53.880278+0000","last_active":"2023-09-23T17:59:23.851471+0000","last_peered":"2023-09-23T17:59:23.851471+0000","last_clean":"2023-09-23T17:57:47.586782+0000","last_became_active":"2023-09-23T17:57:53.880278+0000","last_became_peered":"2023-09-23T17:57:53.880278+0000","last_unstale":"2023-09-23T17:59:23.851471+0000","last_undegraded":"2023-09-23T17:59:23.851471+0000","last_fullsized":"2023-09-23T17:57:53.867145+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:07.345156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":159,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.849881+0000","last_change":"2023-09-23T17:59:22.858016+0000","last_active":"2023-09-23T17:59:23.849881+0000","last_peered":"2023-09-23T17:59:23.849881+0000","last_clean":"2023-09-23T17:59:23.849881+0000","last_became_active":"2023-09-23T17:59:22.857763+0000","last_became_peered":"2023-09-23T17:59:22.857763+0000","last_unstale":"2023-09-23T17:59:23.849881+0000","last_undegraded":"2023-09-23T17:59:23.849881+0000","last_fullsized":"2023-09-23T17:59:23.849881+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:17.964492+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":468,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.849349+0000","last_change":"2023-09-23T17:58:35.250653+0000","last_active":"2023-09-23T17:59:23.849349+0000","last_peered":"2023-09-23T17:59:23.849349+0000","last_clean":"2023-09-23T17:59:23.849349+0000","last_became_active":"2023-09-23T17:58:35.249513+0000","last_became_peered":"2023-09-23T17:58:35.249513+0000","last_unstale":"2023-09-23T17:59:23.849349+0000","last_undegraded":"2023-09-23T17:59:23.849349+0000","last_fullsized":"2023-09-23T17:59:23.849349+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"182'10289","reported_seq":11288,"reported_epoch":182,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T17:59:29.354381+0000","last_change":"2023-09-23T17:59:23.905699+0000","last_active":"2023-09-23T17:59:29.354381+0000","last_peered":"2023-09-23T17:59:29.354381+0000","last_clean":"2023-09-23T17:59:08.813403+0000","last_became_active":"2023-09-23T17:59:23.859716+0000","last_became_peered":"2023-09-23T17:59:23.859716+0000","last_unstale":"2023-09-23T17:59:29.354381+0000","last_undegraded":"2023-09-23T17:59:29.354381+0000","last_fullsized":"2023-09-23T17:59:29.354381+0000","mapping_epoch":180,"log_start":"136'5763","ondisk_log_start":"136'5763","created":110,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4526,"log_dups_size":0,"ondisk_log_size":4526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:37.009579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190119936,"num_objects":2901,"num_object_clones":0,"num_object_copies":8703,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2887,"num_objects_unfound":0,"num_objects_dirty":2901,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10289,"num_write_kb":422080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":434,"num_bytes_recovered":28442624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,5],"avail_no_missing":["0","4","5"],"object_location_counts":[{"shards":"0,4,5","objects":2901}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":116,"reported_epoch":181,"state":"undersized+peered","last_fresh":"2023-09-23T17:59:23.857207+0000","last_change":"2023-09-23T17:57:53.880095+0000","last_active":"2023-09-23T17:57:52.883357+0000","last_peered":"2023-09-23T17:59:23.857207+0000","last_clean":"2023-09-23T17:53:21.028908+0000","last_became_active":"2023-09-23T17:57:24.954589+0000","last_became_peered":"2023-09-23T17:57:53.880095+0000","last_unstale":"2023-09-23T17:59:23.857207+0000","last_undegraded":"2023-09-23T17:59:23.857207+0000","last_fullsized":"2023-09-23T17:57:53.874470+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:56.618752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":142,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.852852+0000","last_change":"2023-09-23T17:57:24.961477+0000","last_active":"2023-09-23T17:59:23.852852+0000","last_peered":"2023-09-23T17:59:23.852852+0000","last_clean":"2023-09-23T17:57:20.929549+0000","last_became_active":"2023-09-23T17:57:24.961477+0000","last_became_peered":"2023-09-23T17:57:24.961477+0000","last_unstale":"2023-09-23T17:59:23.852852+0000","last_undegraded":"2023-09-23T17:59:23.852852+0000","last_fullsized":"2023-09-23T17:57:24.954309+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:01.838528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.852139+0000","last_change":"2023-09-23T17:59:00.112340+0000","last_active":"2023-09-23T17:59:23.852139+0000","last_peered":"2023-09-23T17:59:23.852139+0000","last_clean":"2023-09-23T17:59:23.852139+0000","last_became_active":"2023-09-23T17:59:00.111325+0000","last_became_peered":"2023-09-23T17:59:00.111325+0000","last_unstale":"2023-09-23T17:59:23.852139+0000","last_undegraded":"2023-09-23T17:59:23.852139+0000","last_fullsized":"2023-09-23T17:59:23.852139+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":609,"reported_epoch":181,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:23.858214+0000","last_change":"2023-09-23T17:57:53.873008+0000","last_active":"2023-09-23T17:59:23.858214+0000","last_peered":"2023-09-23T17:59:23.858214+0000","last_clean":"2023-09-23T17:57:47.562468+0000","last_became_active":"2023-09-23T17:57:53.873008+0000","last_became_peered":"2023-09-23T17:57:53.873008+0000","last_unstale":"2023-09-23T17:59:23.858214+0000","last_undegraded":"2023-09-23T17:57:53.867595+0000","last_fullsized":"2023-09-23T17:57:53.867487+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:19.562351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.852496+0000","last_change":"2023-09-23T17:59:10.975320+0000","last_active":"2023-09-23T17:59:23.852496+0000","last_peered":"2023-09-23T17:59:23.852496+0000","last_clean":"2023-09-23T17:59:23.852496+0000","last_became_active":"2023-09-23T17:59:10.974946+0000","last_became_peered":"2023-09-23T17:59:10.974946+0000","last_unstale":"2023-09-23T17:59:23.852496+0000","last_undegraded":"2023-09-23T17:59:23.852496+0000","last_fullsized":"2023-09-23T17:59:23.852496+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:05.703255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":134,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.857128+0000","last_change":"2023-09-23T17:59:09.887042+0000","last_active":"2023-09-23T17:59:23.857128+0000","last_peered":"2023-09-23T17:59:23.857128+0000","last_clean":"2023-09-23T17:59:23.857128+0000","last_became_active":"2023-09-23T17:59:09.886688+0000","last_became_peered":"2023-09-23T17:59:09.886688+0000","last_unstale":"2023-09-23T17:59:23.857128+0000","last_undegraded":"2023-09-23T17:59:23.857128+0000","last_fullsized":"2023-09-23T17:59:23.857128+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:44.443377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"182'10216","reported_seq":11555,"reported_epoch":182,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T17:59:29.350577+0000","last_change":"2023-09-23T17:59:11.194683+0000","last_active":"2023-09-23T17:59:29.350577+0000","last_peered":"2023-09-23T17:59:29.350577+0000","last_clean":"2023-09-23T17:55:57.821023+0000","last_became_active":"2023-09-23T17:59:09.854393+0000","last_became_peered":"2023-09-23T17:59:09.854393+0000","last_unstale":"2023-09-23T17:59:29.350577+0000","last_undegraded":"2023-09-23T17:59:09.846461+0000","last_fullsized":"2023-09-23T17:59:09.846277+0000","mapping_epoch":175,"log_start":"136'5657","ondisk_log_start":"136'5657","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4559,"log_dups_size":0,"ondisk_log_size":4559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:38.915731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187564032,"num_objects":2862,"num_object_clones":0,"num_object_copies":8586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2381,"num_objects_misplaced":2862,"num_objects_unfound":0,"num_objects_dirty":2862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10216,"num_write_kb":418496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":616,"num_bytes_recovered":40370176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":2862}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":476,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.848657+0000","last_change":"2023-09-23T17:59:00.117942+0000","last_active":"2023-09-23T17:59:23.848657+0000","last_peered":"2023-09-23T17:59:23.848657+0000","last_clean":"2023-09-23T17:59:23.848657+0000","last_became_active":"2023-09-23T17:59:00.117602+0000","last_became_peered":"2023-09-23T17:59:00.117602+0000","last_unstale":"2023-09-23T17:59:23.848657+0000","last_undegraded":"2023-09-23T17:59:23.848657+0000","last_fullsized":"2023-09-23T17:59:23.848657+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"182'10210","reported_seq":10311,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T17:59:29.349253+0000","last_change":"2023-09-23T17:56:50.843235+0000","last_active":"2023-09-23T17:59:29.349253+0000","last_peered":"2023-09-23T17:59:29.349253+0000","last_clean":"2023-09-23T17:59:29.349253+0000","last_became_active":"2023-09-23T17:56:50.835153+0000","last_became_peered":"2023-09-23T17:56:50.835153+0000","last_unstale":"2023-09-23T17:59:29.349253+0000","last_undegraded":"2023-09-23T17:59:29.349253+0000","last_fullsized":"2023-09-23T17:59:29.349253+0000","mapping_epoch":126,"log_start":"136'5653","ondisk_log_start":"136'5653","created":110,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4557,"log_dups_size":0,"ondisk_log_size":4557,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:14.827228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":187432960,"num_objects":2860,"num_object_clones":0,"num_object_copies":8580,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2860,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10210,"num_write_kb":418240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1085,"reported_epoch":181,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T17:59:23.857392+0000","last_change":"2023-09-23T17:59:10.894167+0000","last_active":"2023-09-23T17:59:23.857392+0000","last_peered":"2023-09-23T17:59:23.857392+0000","last_clean":"2023-09-23T17:46:59.226124+0000","last_became_active":"2023-09-23T17:59:10.893633+0000","last_became_peered":"2023-09-23T17:59:10.893633+0000","last_unstale":"2023-09-23T17:59:23.857392+0000","last_undegraded":"2023-09-23T17:59:10.880608+0000","last_fullsized":"2023-09-23T17:59:10.874138+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:13.227304+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":155,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:24.375544+0000","last_change":"2023-09-23T17:59:22.871277+0000","last_active":"2023-09-23T17:59:24.375544+0000","last_peered":"2023-09-23T17:59:24.375544+0000","last_clean":"2023-09-23T17:59:24.375544+0000","last_became_active":"2023-09-23T17:59:22.870933+0000","last_became_peered":"2023-09-23T17:59:22.870933+0000","last_unstale":"2023-09-23T17:59:24.375544+0000","last_undegraded":"2023-09-23T17:59:24.375544+0000","last_fullsized":"2023-09-23T17:59:24.375544+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:01.417962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":143,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.858127+0000","last_change":"2023-09-23T17:57:24.952855+0000","last_active":"2023-09-23T17:59:23.858127+0000","last_peered":"2023-09-23T17:59:23.858127+0000","last_clean":"2023-09-23T17:57:20.923267+0000","last_became_active":"2023-09-23T17:57:24.952855+0000","last_became_peered":"2023-09-23T17:57:24.952855+0000","last_unstale":"2023-09-23T17:59:23.858127+0000","last_undegraded":"2023-09-23T17:59:23.858127+0000","last_fullsized":"2023-09-23T17:57:24.946770+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:15.329593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.848700+0000","last_change":"2023-09-23T17:59:00.113520+0000","last_active":"2023-09-23T17:59:23.848700+0000","last_peered":"2023-09-23T17:59:23.848700+0000","last_clean":"2023-09-23T17:59:23.848700+0000","last_became_active":"2023-09-23T17:59:00.113172+0000","last_became_peered":"2023-09-23T17:59:00.113172+0000","last_unstale":"2023-09-23T17:59:23.848700+0000","last_undegraded":"2023-09-23T17:59:23.848700+0000","last_fullsized":"2023-09-23T17:59:23.848700+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"182'10492","reported_seq":10588,"reported_epoch":182,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T17:59:28.859391+0000","last_change":"2023-09-23T17:59:11.222881+0000","last_active":"2023-09-23T17:59:28.859391+0000","last_peered":"2023-09-23T17:59:28.859391+0000","last_clean":"2023-09-23T17:57:52.799975+0000","last_became_active":"2023-09-23T17:59:11.178414+0000","last_became_peered":"2023-09-23T17:59:11.178414+0000","last_unstale":"2023-09-23T17:59:28.859391+0000","last_undegraded":"2023-09-23T17:59:10.956696+0000","last_fullsized":"2023-09-23T17:59:10.884554+0000","mapping_epoch":176,"log_start":"116'773","ondisk_log_start":"116'773","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9719,"log_dups_size":0,"ondisk_log_size":9719,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:22.428899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194117632,"num_objects":2962,"num_object_clones":0,"num_object_copies":8886,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2962,"num_objects_misplaced":6727,"num_objects_unfound":0,"num_objects_dirty":2962,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10492,"num_write_kb":430528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-642}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":453,"reported_epoch":181,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:23.851221+0000","last_change":"2023-09-23T17:57:24.955720+0000","last_active":"2023-09-23T17:59:23.851221+0000","last_peered":"2023-09-23T17:59:23.851221+0000","last_clean":"2023-09-23T17:57:20.931796+0000","last_became_active":"2023-09-23T17:57:24.955720+0000","last_became_peered":"2023-09-23T17:57:24.955720+0000","last_unstale":"2023-09-23T17:59:23.851221+0000","last_undegraded":"2023-09-23T17:57:24.945824+0000","last_fullsized":"2023-09-23T17:57:24.945671+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:25.614568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":136,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.857383+0000","last_change":"2023-09-23T17:58:35.246091+0000","last_active":"2023-09-23T17:59:23.857383+0000","last_peered":"2023-09-23T17:59:23.857383+0000","last_clean":"2023-09-23T17:59:23.857383+0000","last_became_active":"2023-09-23T17:58:35.245673+0000","last_became_peered":"2023-09-23T17:58:35.245673+0000","last_unstale":"2023-09-23T17:59:23.857383+0000","last_undegraded":"2023-09-23T17:59:23.857383+0000","last_fullsized":"2023-09-23T17:59:23.857383+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:53.368873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":160,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.858901+0000","last_change":"2023-09-23T17:59:10.911985+0000","last_active":"2023-09-23T17:59:23.858901+0000","last_peered":"2023-09-23T17:59:23.858901+0000","last_clean":"2023-09-23T17:59:23.858901+0000","last_became_active":"2023-09-23T17:59:10.911584+0000","last_became_peered":"2023-09-23T17:59:10.911584+0000","last_unstale":"2023-09-23T17:59:23.858901+0000","last_undegraded":"2023-09-23T17:59:23.858901+0000","last_fullsized":"2023-09-23T17:59:23.858901+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:43.723422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,5],"avail_no_missing":["1","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":997,"reported_epoch":181,"state":"active+undersized+remapped","last_fresh":"2023-09-23T17:59:23.853205+0000","last_change":"2023-09-23T17:59:10.890970+0000","last_active":"2023-09-23T17:59:23.853205+0000","last_peered":"2023-09-23T17:59:23.853205+0000","last_clean":"2023-09-23T17:47:00.624503+0000","last_became_active":"2023-09-23T17:59:10.890970+0000","last_became_peered":"2023-09-23T17:59:10.890970+0000","last_unstale":"2023-09-23T17:59:23.853205+0000","last_undegraded":"2023-09-23T17:59:23.853205+0000","last_fullsized":"2023-09-23T17:59:10.885398+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:36.933247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.849250+0000","last_change":"2023-09-23T17:59:00.118160+0000","last_active":"2023-09-23T17:59:23.849250+0000","last_peered":"2023-09-23T17:59:23.849250+0000","last_clean":"2023-09-23T17:59:23.849250+0000","last_became_active":"2023-09-23T17:59:00.117827+0000","last_became_peered":"2023-09-23T17:59:00.117827+0000","last_unstale":"2023-09-23T17:59:23.849250+0000","last_undegraded":"2023-09-23T17:59:23.849250+0000","last_fullsized":"2023-09-23T17:59:23.849250+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":80,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.849357+0000","last_change":"2023-09-23T17:59:10.879009+0000","last_active":"2023-09-23T17:59:23.849357+0000","last_peered":"2023-09-23T17:59:23.849357+0000","last_clean":"2023-09-23T17:59:23.849357+0000","last_became_active":"2023-09-23T17:59:10.878632+0000","last_became_peered":"2023-09-23T17:59:10.878632+0000","last_unstale":"2023-09-23T17:59:23.849357+0000","last_undegraded":"2023-09-23T17:59:23.849357+0000","last_fullsized":"2023-09-23T17:59:23.849357+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:38.184201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,5],"avail_no_missing":["6","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":81,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.849776+0000","last_change":"2023-09-23T17:58:36.248863+0000","last_active":"2023-09-23T17:59:23.849776+0000","last_peered":"2023-09-23T17:59:23.849776+0000","last_clean":"2023-09-23T17:59:23.849776+0000","last_became_active":"2023-09-23T17:58:36.248383+0000","last_became_peered":"2023-09-23T17:58:36.248383+0000","last_unstale":"2023-09-23T17:59:23.849776+0000","last_undegraded":"2023-09-23T17:59:23.849776+0000","last_fullsized":"2023-09-23T17:59:23.849776+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:49.334942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,0],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"182'10322","reported_seq":20048,"reported_epoch":182,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T17:59:28.868653+0000","last_change":"2023-09-23T17:59:11.224350+0000","last_active":"2023-09-23T17:59:28.868653+0000","last_peered":"2023-09-23T17:59:28.868653+0000","last_clean":"2023-09-23T17:59:08.847410+0000","last_became_active":"2023-09-23T17:59:11.059419+0000","last_became_peered":"2023-09-23T17:59:11.059419+0000","last_unstale":"2023-09-23T17:59:28.868653+0000","last_undegraded":"2023-09-23T17:59:28.868653+0000","last_fullsized":"2023-09-23T17:59:28.868653+0000","mapping_epoch":176,"log_start":"116'971","ondisk_log_start":"116'971","created":110,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9351,"log_dups_size":0,"ondisk_log_size":9351,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:54.596656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":193069056,"num_objects":2946,"num_object_clones":0,"num_object_copies":8838,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2916,"num_objects_unfound":0,"num_objects_dirty":2946,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10322,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4804,"num_bytes_recovered":314834944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":2946}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.848603+0000","last_change":"2023-09-23T17:59:00.113658+0000","last_active":"2023-09-23T17:59:23.848603+0000","last_peered":"2023-09-23T17:59:23.848603+0000","last_clean":"2023-09-23T17:59:23.848603+0000","last_became_active":"2023-09-23T17:59:00.113034+0000","last_became_peered":"2023-09-23T17:59:00.113034+0000","last_unstale":"2023-09-23T17:59:23.848603+0000","last_undegraded":"2023-09-23T17:59:23.848603+0000","last_fullsized":"2023-09-23T17:59:23.848603+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.7","version":"182'10657","reported_seq":10744,"reported_epoch":182,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T17:59:30.908035+0000","last_change":"2023-09-23T17:59:11.276228+0000","last_active":"2023-09-23T17:59:30.908035+0000","last_peered":"2023-09-23T17:59:30.908035+0000","last_clean":"2023-09-23T17:59:08.808079+0000","last_became_active":"2023-09-23T17:59:11.192668+0000","last_became_peered":"2023-09-23T17:59:11.192668+0000","last_unstale":"2023-09-23T17:59:30.908035+0000","last_undegraded":"2023-09-23T17:59:30.908035+0000","last_fullsized":"2023-09-23T17:59:30.908035+0000","mapping_epoch":176,"log_start":"139'6307","ondisk_log_start":"139'6307","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4350,"log_dups_size":0,"ondisk_log_size":4350,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:56.972002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183566336,"num_objects":2801,"num_object_clones":0,"num_object_copies":8403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2801,"num_objects_unfound":0,"num_objects_dirty":2801,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10657,"num_write_kb":430656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,5],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1,5","objects":2801}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":283,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.857595+0000","last_change":"2023-09-23T17:57:53.883867+0000","last_active":"2023-09-23T17:59:23.857595+0000","last_peered":"2023-09-23T17:59:23.857595+0000","last_clean":"2023-09-23T17:46:57.119141+0000","last_became_active":"2023-09-23T17:57:53.883867+0000","last_became_peered":"2023-09-23T17:57:53.883867+0000","last_unstale":"2023-09-23T17:59:23.857595+0000","last_undegraded":"2023-09-23T17:59:23.857595+0000","last_fullsized":"2023-09-23T17:57:53.875072+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:35.282181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":150,"reported_epoch":182,"state":"active+undersized","last_fresh":"2023-09-23T17:59:26.065968+0000","last_change":"2023-09-23T17:57:24.959291+0000","last_active":"2023-09-23T17:59:26.065968+0000","last_peered":"2023-09-23T17:59:26.065968+0000","last_clean":"2023-09-23T17:57:20.930735+0000","last_became_active":"2023-09-23T17:57:24.959291+0000","last_became_peered":"2023-09-23T17:57:24.959291+0000","last_unstale":"2023-09-23T17:59:26.065968+0000","last_undegraded":"2023-09-23T17:59:26.065968+0000","last_fullsized":"2023-09-23T17:57:24.946083+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:43.225674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":141,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.851579+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T17:59:23.851579+0000","last_peered":"2023-09-23T17:59:23.851579+0000","last_clean":"2023-09-23T17:59:23.851579+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T17:59:23.851579+0000","last_undegraded":"2023-09-23T17:59:23.851579+0000","last_fullsized":"2023-09-23T17:59:23.851579+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.852421+0000","last_change":"2023-09-23T17:59:00.118578+0000","last_active":"2023-09-23T17:59:23.852421+0000","last_peered":"2023-09-23T17:59:23.852421+0000","last_clean":"2023-09-23T17:59:23.852421+0000","last_became_active":"2023-09-23T17:59:00.118262+0000","last_became_peered":"2023-09-23T17:59:00.118262+0000","last_unstale":"2023-09-23T17:59:23.852421+0000","last_undegraded":"2023-09-23T17:59:23.852421+0000","last_fullsized":"2023-09-23T17:59:23.852421+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":316,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.852565+0000","last_change":"2023-09-23T17:57:53.875525+0000","last_active":"2023-09-23T17:59:23.852565+0000","last_peered":"2023-09-23T17:59:23.852565+0000","last_clean":"2023-09-23T17:46:59.915193+0000","last_became_active":"2023-09-23T17:57:53.875525+0000","last_became_peered":"2023-09-23T17:57:53.875525+0000","last_unstale":"2023-09-23T17:59:23.852565+0000","last_undegraded":"2023-09-23T17:59:23.852565+0000","last_fullsized":"2023-09-23T17:57:53.868097+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:57.748716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.858350+0000","last_change":"2023-09-23T17:59:09.858216+0000","last_active":"2023-09-23T17:59:23.858350+0000","last_peered":"2023-09-23T17:59:23.858350+0000","last_clean":"2023-09-23T17:59:23.858350+0000","last_became_active":"2023-09-23T17:59:09.856443+0000","last_became_peered":"2023-09-23T17:59:09.856443+0000","last_unstale":"2023-09-23T17:59:23.858350+0000","last_undegraded":"2023-09-23T17:59:23.858350+0000","last_fullsized":"2023-09-23T17:59:23.858350+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:19.986245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":127,"reported_epoch":181,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:23.851690+0000","last_change":"2023-09-23T17:59:10.883046+0000","last_active":"2023-09-23T17:59:23.851690+0000","last_peered":"2023-09-23T17:59:23.851690+0000","last_clean":"2023-09-23T17:59:23.851690+0000","last_became_active":"2023-09-23T17:59:10.882738+0000","last_became_peered":"2023-09-23T17:59:10.882738+0000","last_unstale":"2023-09-23T17:59:23.851690+0000","last_undegraded":"2023-09-23T17:59:23.851690+0000","last_fullsized":"2023-09-23T17:59:23.851690+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:39.241547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,5],"avail_no_missing":["0","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"182'10083","reported_seq":18000,"reported_epoch":182,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T17:59:29.357703+0000","last_change":"2023-09-23T17:59:24.084532+0000","last_active":"2023-09-23T17:59:29.357703+0000","last_peered":"2023-09-23T17:59:29.357703+0000","last_clean":"2023-09-23T17:55:52.789238+0000","last_became_active":"2023-09-23T17:59:09.938603+0000","last_became_peered":"2023-09-23T17:59:09.938603+0000","last_unstale":"2023-09-23T17:59:29.357703+0000","last_undegraded":"2023-09-23T17:59:09.881270+0000","last_fullsized":"2023-09-23T17:59:09.846342+0000","mapping_epoch":175,"log_start":"136'5547","ondisk_log_start":"136'5547","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4536,"log_dups_size":0,"ondisk_log_size":4536,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:57.591912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190644224,"num_objects":2909,"num_object_clones":0,"num_object_copies":8727,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":591,"num_objects_misplaced":2143,"num_objects_unfound":0,"num_objects_dirty":2909,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10083,"num_write_kb":415744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3912,"num_bytes_recovered":256376832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":2909}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":477,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.852824+0000","last_change":"2023-09-23T17:59:00.119695+0000","last_active":"2023-09-23T17:59:23.852824+0000","last_peered":"2023-09-23T17:59:23.852824+0000","last_clean":"2023-09-23T17:59:23.852824+0000","last_became_active":"2023-09-23T17:59:00.119349+0000","last_became_peered":"2023-09-23T17:59:00.119349+0000","last_unstale":"2023-09-23T17:59:23.852824+0000","last_undegraded":"2023-09-23T17:59:23.852824+0000","last_fullsized":"2023-09-23T17:59:23.852824+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":50,"reported_epoch":181,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T17:59:23.852488+0000","last_change":"2023-09-23T17:57:53.872726+0000","last_active":"2023-09-23T17:57:52.888539+0000","last_peered":"2023-09-23T17:59:23.852488+0000","last_clean":"2023-09-23T17:45:54.465055+0000","last_became_active":"2023-09-23T17:45:51.139637+0000","last_became_peered":"2023-09-23T17:57:53.872726+0000","last_unstale":"2023-09-23T17:59:23.852488+0000","last_undegraded":"2023-09-23T17:57:53.871230+0000","last_fullsized":"2023-09-23T17:57:53.871154+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:46.624357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":414,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.858017+0000","last_change":"2023-09-23T17:58:35.247928+0000","last_active":"2023-09-23T17:59:23.858017+0000","last_peered":"2023-09-23T17:59:23.858017+0000","last_clean":"2023-09-23T17:59:23.858017+0000","last_became_active":"2023-09-23T17:58:35.247482+0000","last_became_peered":"2023-09-23T17:58:35.247482+0000","last_unstale":"2023-09-23T17:59:23.858017+0000","last_undegraded":"2023-09-23T17:59:23.858017+0000","last_fullsized":"2023-09-23T17:59:23.858017+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:18.686446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.848134+0000","last_change":"2023-09-23T17:58:12.005101+0000","last_active":"2023-09-23T17:59:23.848134+0000","last_peered":"2023-09-23T17:59:23.848134+0000","last_clean":"2023-09-23T17:49:06.298151+0000","last_became_active":"2023-09-23T17:58:12.005101+0000","last_became_peered":"2023-09-23T17:58:12.005101+0000","last_unstale":"2023-09-23T17:59:23.848134+0000","last_undegraded":"2023-09-23T17:59:23.848134+0000","last_fullsized":"2023-09-23T17:58:11.987584+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:15.859050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"182'10532","reported_seq":23273,"reported_epoch":182,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:32.354405+0000","last_change":"2023-09-23T17:58:23.828372+0000","last_active":"2023-09-23T17:59:32.354405+0000","last_peered":"2023-09-23T17:59:32.354405+0000","last_clean":"2023-09-23T17:57:23.936571+0000","last_became_active":"2023-09-23T17:58:23.828372+0000","last_became_peered":"2023-09-23T17:58:23.828372+0000","last_unstale":"2023-09-23T17:59:32.354405+0000","last_undegraded":"2023-09-23T17:58:23.826763+0000","last_fullsized":"2023-09-23T17:58:23.826662+0000","mapping_epoch":159,"log_start":"120'3777","ondisk_log_start":"120'3777","created":110,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6755,"log_dups_size":0,"ondisk_log_size":6755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:59.923478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183369728,"num_objects":2798,"num_object_clones":0,"num_object_copies":8394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2798,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10532,"num_write_kb":426560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6302,"num_bytes_recovered":413007872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":2798}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":481,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.851825+0000","last_change":"2023-09-23T17:59:00.129488+0000","last_active":"2023-09-23T17:59:23.851825+0000","last_peered":"2023-09-23T17:59:23.851825+0000","last_clean":"2023-09-23T17:59:23.851825+0000","last_became_active":"2023-09-23T17:59:00.128847+0000","last_became_peered":"2023-09-23T17:59:00.128847+0000","last_unstale":"2023-09-23T17:59:23.851825+0000","last_undegraded":"2023-09-23T17:59:23.851825+0000","last_fullsized":"2023-09-23T17:59:23.851825+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":162896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":357,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.850335+0000","last_change":"2023-09-23T17:58:35.227206+0000","last_active":"2023-09-23T17:59:23.850335+0000","last_peered":"2023-09-23T17:59:23.850335+0000","last_clean":"2023-09-23T17:46:59.535985+0000","last_became_active":"2023-09-23T17:58:35.227206+0000","last_became_peered":"2023-09-23T17:58:35.227206+0000","last_unstale":"2023-09-23T17:59:23.850335+0000","last_undegraded":"2023-09-23T17:59:23.850335+0000","last_fullsized":"2023-09-23T17:58:35.224367+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:52.549534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":155,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.851667+0000","last_change":"2023-09-23T17:56:50.856836+0000","last_active":"2023-09-23T17:59:23.851667+0000","last_peered":"2023-09-23T17:59:23.851667+0000","last_clean":"2023-09-23T17:59:23.851667+0000","last_became_active":"2023-09-23T17:56:50.835155+0000","last_became_peered":"2023-09-23T17:56:50.835155+0000","last_unstale":"2023-09-23T17:59:23.851667+0000","last_undegraded":"2023-09-23T17:59:23.851667+0000","last_fullsized":"2023-09-23T17:59:23.851667+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:36.188982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":159,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.848834+0000","last_change":"2023-09-23T17:59:22.871033+0000","last_active":"2023-09-23T17:59:23.848834+0000","last_peered":"2023-09-23T17:59:23.848834+0000","last_clean":"2023-09-23T17:59:23.848834+0000","last_became_active":"2023-09-23T17:59:22.870743+0000","last_became_peered":"2023-09-23T17:59:22.870743+0000","last_unstale":"2023-09-23T17:59:23.848834+0000","last_undegraded":"2023-09-23T17:59:23.848834+0000","last_fullsized":"2023-09-23T17:59:23.848834+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:40:42.511041+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.0","version":"182'10622","reported_seq":23861,"reported_epoch":182,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T17:59:30.940055+0000","last_change":"2023-09-23T17:58:34.248637+0000","last_active":"2023-09-23T17:59:30.940055+0000","last_peered":"2023-09-23T17:59:30.940055+0000","last_clean":"2023-09-23T17:56:38.421727+0000","last_became_active":"2023-09-23T17:57:26.317820+0000","last_became_peered":"2023-09-23T17:57:26.317820+0000","last_unstale":"2023-09-23T17:59:30.940055+0000","last_undegraded":"2023-09-23T17:57:26.081408+0000","last_fullsized":"2023-09-23T17:59:30.940055+0000","mapping_epoch":143,"log_start":"137'6255","ondisk_log_start":"137'6255","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4367,"log_dups_size":0,"ondisk_log_size":4367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:37.559775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185466880,"num_objects":2830,"num_object_clones":0,"num_object_copies":8490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2830,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10622,"num_write_kb":430464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6566,"num_bytes_recovered":430309376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,0],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"0,1,4","objects":2756},{"shards":"1,4","objects":74}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":138,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.851551+0000","last_change":"2023-09-23T17:57:53.877134+0000","last_active":"2023-09-23T17:59:23.851551+0000","last_peered":"2023-09-23T17:59:23.851551+0000","last_clean":"2023-09-23T17:57:47.571295+0000","last_became_active":"2023-09-23T17:57:53.877134+0000","last_became_peered":"2023-09-23T17:57:53.877134+0000","last_unstale":"2023-09-23T17:59:23.851551+0000","last_undegraded":"2023-09-23T17:59:23.851551+0000","last_fullsized":"2023-09-23T17:57:53.869424+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:59.045696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":173,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.908454+0000","last_change":"2023-09-23T17:57:24.956919+0000","last_active":"2023-09-23T17:59:23.908454+0000","last_peered":"2023-09-23T17:59:23.908454+0000","last_clean":"2023-09-23T17:57:20.928421+0000","last_became_active":"2023-09-23T17:57:24.956919+0000","last_became_peered":"2023-09-23T17:57:24.956919+0000","last_unstale":"2023-09-23T17:59:23.908454+0000","last_undegraded":"2023-09-23T17:59:23.908454+0000","last_fullsized":"2023-09-23T17:57:24.951731+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:03.928787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":25,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:25.943375+0000","last_change":"2023-09-23T17:59:22.863684+0000","last_active":"2023-09-23T17:59:25.943375+0000","last_peered":"2023-09-23T17:59:25.943375+0000","last_clean":"2023-09-23T17:59:25.943375+0000","last_became_active":"2023-09-23T17:59:22.856858+0000","last_became_peered":"2023-09-23T17:59:22.856858+0000","last_unstale":"2023-09-23T17:59:25.943375+0000","last_undegraded":"2023-09-23T17:59:25.943375+0000","last_fullsized":"2023-09-23T17:59:25.943375+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:27.707226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":90,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:25.943240+0000","last_change":"2023-09-23T17:59:22.857325+0000","last_active":"2023-09-23T17:59:25.943240+0000","last_peered":"2023-09-23T17:59:25.943240+0000","last_clean":"2023-09-23T17:57:20.922389+0000","last_became_active":"2023-09-23T17:59:22.857325+0000","last_became_peered":"2023-09-23T17:59:22.857325+0000","last_unstale":"2023-09-23T17:59:25.943240+0000","last_undegraded":"2023-09-23T17:59:25.943240+0000","last_fullsized":"2023-09-23T17:59:22.855531+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:54.144257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":26,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.849656+0000","last_change":"2023-09-23T17:59:22.856886+0000","last_active":"2023-09-23T17:59:23.849656+0000","last_peered":"2023-09-23T17:59:23.849656+0000","last_clean":"2023-09-23T17:59:23.849656+0000","last_became_active":"2023-09-23T17:59:22.856518+0000","last_became_peered":"2023-09-23T17:59:22.856518+0000","last_unstale":"2023-09-23T17:59:23.849656+0000","last_undegraded":"2023-09-23T17:59:23.849656+0000","last_fullsized":"2023-09-23T17:59:23.849656+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":180,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:30:55.500558+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":90,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:24.375385+0000","last_change":"2023-09-23T17:59:22.871834+0000","last_active":"2023-09-23T17:59:24.375385+0000","last_peered":"2023-09-23T17:59:24.375385+0000","last_clean":"2023-09-23T17:57:20.922822+0000","last_became_active":"2023-09-23T17:59:22.871834+0000","last_became_peered":"2023-09-23T17:59:22.871834+0000","last_unstale":"2023-09-23T17:59:24.375385+0000","last_undegraded":"2023-09-23T17:59:24.375385+0000","last_fullsized":"2023-09-23T17:59:22.864946+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:40.097366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":40,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.850106+0000","last_change":"2023-09-23T17:58:35.229256+0000","last_active":"2023-09-23T17:59:23.850106+0000","last_peered":"2023-09-23T17:59:23.850106+0000","last_clean":"2023-09-23T17:45:50.074605+0000","last_became_active":"2023-09-23T17:58:35.229256+0000","last_became_peered":"2023-09-23T17:58:35.229256+0000","last_unstale":"2023-09-23T17:59:23.850106+0000","last_undegraded":"2023-09-23T17:59:23.850106+0000","last_fullsized":"2023-09-23T17:58:35.224895+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:14.916976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":131,"reported_epoch":181,"state":"active+undersized","last_fresh":"2023-09-23T17:59:23.851035+0000","last_change":"2023-09-23T17:57:24.955661+0000","last_active":"2023-09-23T17:59:23.851035+0000","last_peered":"2023-09-23T17:59:23.851035+0000","last_clean":"2023-09-23T17:53:52.727684+0000","last_became_active":"2023-09-23T17:57:24.955661+0000","last_became_peered":"2023-09-23T17:57:24.955661+0000","last_unstale":"2023-09-23T17:59:23.851035+0000","last_undegraded":"2023-09-23T17:59:23.851035+0000","last_fullsized":"2023-09-23T17:57:24.945672+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:04.500443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":145,"reported_epoch":182,"state":"active+clean","last_fresh":"2023-09-23T17:59:26.065958+0000","last_change":"2023-09-23T17:59:09.869351+0000","last_active":"2023-09-23T17:59:26.065958+0000","last_peered":"2023-09-23T17:59:26.065958+0000","last_clean":"2023-09-23T17:59:26.065958+0000","last_became_active":"2023-09-23T17:59:09.861326+0000","last_became_peered":"2023-09-23T17:59:09.861326+0000","last_unstale":"2023-09-23T17:59:26.065958+0000","last_undegraded":"2023-09-23T17:59:26.065958+0000","last_fullsized":"2023-09-23T17:59:26.065958+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:38.142504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":171,"reported_epoch":181,"state":"active+clean","last_fresh":"2023-09-23T17:59:23.851332+0000","last_change":"2023-09-23T17:58:35.243060+0000","last_active":"2023-09-23T17:59:23.851332+0000","last_peered":"2023-09-23T17:59:23.851332+0000","last_clean":"2023-09-23T17:59:23.851332+0000","last_became_active":"2023-09-23T17:58:35.242727+0000","last_became_peered":"2023-09-23T17:58:35.242727+0000","last_unstale":"2023-09-23T17:59:23.851332+0000","last_undegraded":"2023-09-23T17:59:23.851332+0000","last_fullsized":"2023-09-23T17:59:23.851332+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:23:08.850283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":3005022232,"num_objects":45854,"num_object_clones":0,"num_object_copies":137562,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17902,"num_objects_misplaced":35400,"num_objects_unfound":0,"num_objects_dirty":45854,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":165394,"num_write_kb":6759873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35915,"num_bytes_recovered":2353266688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11174096896,"data_stored":11174084680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":108228,"ondisk_log_size":108228,"up":40,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":72,"acting":78,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":666,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33098,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":69,"acting":78,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":17,"acting":17,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266849,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1840208,"kb_used_data":1778696,"kb_used_omap":0,"kb_used_meta":61504,"kb_avail":91904944,"statfs":{"total":95995035648,"available":94110662656,"internally_reserved":0,"allocated":1821384704,"data_stored":1820756859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":62980096},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":0.83399999999999996,"15min":0.64500000000000002},"min":{"1min":0.24099999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.5700000000000003,"5min":8.5700000000000003,"15min":8.5700000000000003},"last":2.2210000000000001},{"interface":"front","average":{"1min":1.6499999999999999,"5min":0.879,"15min":0.67100000000000004},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.8410000000000002,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":4.1280000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.59599999999999997,"15min":0.56999999999999995},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.702,"5min":2.702,"15min":2.702},"last":6.6890000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":0.65700000000000003,"15min":0.58899999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":4.4240000000000004},"last":0.26000000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":0.89200000000000002,"15min":0.66300000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.2359999999999998,"5min":5.2359999999999998,"15min":5.2359999999999998},"last":2.2970000000000002},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.77900000000000003,"15min":0.64800000000000002},"min":{"1min":0.38100000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.8340000000000001,"5min":4.6180000000000003,"15min":4.6180000000000003},"last":0.439}]},{"osd":3,"last update":"Sat Sep 23 17:57:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.64900000000000002,"15min":0.57299999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":4.0549999999999997,"5min":4.0549999999999997,"15min":4.0549999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.665,"5min":0.77400000000000002,"15min":0.626},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.1609999999999996,"5min":5.1609999999999996,"15min":5.1609999999999996},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.53400000000000003,"15min":0.47099999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.6160000000000001,"5min":2.6160000000000001,"15min":2.6160000000000001},"last":2.1949999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.56100000000000005,"15min":0.498},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8079999999999998,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.60699999999999998,"15min":0.52600000000000002},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.5840000000000001,"5min":2.5840000000000001,"15min":2.5840000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.749,"5min":0.64100000000000001,"15min":0.54900000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.706,"5min":2.706,"15min":2.706},"last":2.1669999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":1.002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.435}]}]},{"osd":6,"up_from":163,"seq":700079669260,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":429716,"kb_used_data":370188,"kb_used_omap":7,"kb_used_meta":59512,"kb_avail":93315436,"statfs":{"total":95995035648,"available":95555006464,"internally_reserved":0,"allocated":379072512,"data_stored":378330212,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":60941092},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.26700000000000002}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.8780000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38100000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.60199999999999998}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.63700000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795078,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1209656,"kb_used_data":1091824,"kb_used_omap":7,"kb_used_meta":117816,"kb_avail":92535496,"statfs":{"total":95995035648,"available":94756347904,"internally_reserved":0,"allocated":1118027776,"data_stored":1117081411,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":120644285},"hb_peers":[0,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:58:56 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":1.294,"15min":0.68999999999999995},"min":{"1min":0.182,"5min":0.182,"15min":0.182},"max":{"1min":8.6039999999999992,"5min":11.384,"15min":11.384},"last":2.9180000000000001},{"interface":"front","average":{"1min":1.1830000000000001,"5min":1.429,"15min":0.74099999999999999},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.0910000000000002,"5min":12.355,"15min":12.355},"last":4.4980000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:58:34 2023","interfaces":[{"interface":"back","average":{"1min":1.444,"5min":1.3069999999999999,"15min":0.73599999999999999},"min":{"1min":0.27200000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":4.9850000000000003,"5min":10.241,"15min":10.241},"last":8.5760000000000005},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.246,"15min":0.70799999999999996},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":8.6349999999999998,"5min":11.305999999999999,"15min":11.305999999999999},"last":3.0950000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:57:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8999999999999999,"5min":0.90800000000000003,"15min":0.57599999999999996},"min":{"1min":0.29899999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":14.548999999999999,"5min":14.548999999999999,"15min":14.548999999999999},"last":2.7149999999999999},{"interface":"front","average":{"1min":2.3740000000000001,"5min":0.85199999999999998,"15min":0.59799999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.250999999999999,"5min":10.250999999999999,"15min":10.250999999999999},"last":0.82899999999999996}]},{"osd":4,"last update":"Sat Sep 23 17:58:56 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":1.1240000000000001,"15min":0.82999999999999996},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.008,"5min":8.0289999999999999,"15min":8.0289999999999999},"last":2.9340000000000002},{"interface":"front","average":{"1min":1.23,"5min":1.242,"15min":0.86599999999999999},"min":{"1min":0.28199999999999997,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":8.4870000000000001,"5min":8.8770000000000007,"15min":8.8770000000000007},"last":2.8570000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:58:56 2023","interfaces":[{"interface":"back","average":{"1min":1.276,"5min":1.3,"15min":0.91200000000000003},"min":{"1min":0.35899999999999999,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.9569999999999999,"5min":7.4269999999999996,"15min":7.4269999999999996},"last":4.4169999999999998},{"interface":"front","average":{"1min":0.84199999999999997,"5min":1.256,"15min":0.89000000000000001},"min":{"1min":0.32300000000000001,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":5.2229999999999999,"5min":9.0790000000000006,"15min":9.0790000000000006},"last":3.0760000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:55:54 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.70299999999999996,"15min":0.70299999999999996},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":1.9330000000000001,"5min":1.9330000000000001,"15min":1.9330000000000001},"last":2.9910000000000001},{"interface":"front","average":{"1min":0.64800000000000002,"5min":0.69199999999999995,"15min":0.69899999999999995},"min":{"1min":0.47699999999999998,"5min":0.44500000000000001,"15min":0.44500000000000001},"max":{"1min":1.141,"5min":1.141,"15min":1.141},"last":4.4569999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":0.89400000000000002,"15min":0.74399999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.9630000000000001,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":7.8869999999999996},{"interface":"front","average":{"1min":1.895,"5min":0.92900000000000005,"15min":0.77900000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.2940000000000005,"5min":8.2940000000000005,"15min":8.2940000000000005},"last":7.234}]}]},{"osd":0,"up_from":93,"seq":399431958604,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2306332,"kb_used_data":2030932,"kb_used_omap":7,"kb_used_meta":275384,"kb_avail":91438820,"statfs":{"total":95995035648,"available":93633351680,"internally_reserved":0,"allocated":2079674368,"data_stored":2078751647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":281993956},"hb_peers":[1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 17:58:58 2023","interfaces":[{"interface":"back","average":{"1min":2.5209999999999999,"5min":1.4590000000000001,"15min":0.73299999999999998},"min":{"1min":0.18099999999999999,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":18.154,"5min":18.154,"15min":18.154},"last":7.3419999999999996},{"interface":"front","average":{"1min":2.3530000000000002,"5min":1.413,"15min":0.755},"min":{"1min":0.25600000000000001,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":8.6890000000000001,"5min":11.346,"15min":11.346},"last":6.3460000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:58:32 2023","interfaces":[{"interface":"back","average":{"1min":2.0009999999999999,"5min":1.24,"15min":0.65400000000000003},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":14.452,"5min":14.452,"15min":14.452},"last":5.2910000000000004},{"interface":"front","average":{"1min":1.669,"5min":1.2509999999999999,"15min":0.66200000000000003},"min":{"1min":0.248,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.968,"5min":9.968,"15min":9.968},"last":3.9670000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":0.77100000000000002,"15min":0.60099999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3449999999999998,"5min":4.3449999999999998,"15min":4.3449999999999998},"last":9.266},{"interface":"front","average":{"1min":2.109,"5min":0.73699999999999999,"15min":0.50900000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":8.8729999999999993},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:58:32 2023","interfaces":[{"interface":"back","average":{"1min":1.296,"5min":1.0469999999999999,"15min":0.77300000000000002},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":14.081,"5min":14.081,"15min":14.081},"last":5.9119999999999999},{"interface":"front","average":{"1min":1.3260000000000001,"5min":1.0249999999999999,"15min":0.75700000000000001},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.0750000000000002,"5min":9.1020000000000003,"15min":9.1020000000000003},"last":7.3049999999999997}]},{"osd":5,"last update":"Sat Sep 23 17:58:32 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":1.0629999999999999,"15min":0.77600000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.1829999999999998,"5min":9.1370000000000005,"15min":9.1370000000000005},"last":5.0380000000000003},{"interface":"front","average":{"1min":0.79300000000000004,"5min":0.90300000000000002,"15min":0.73099999999999998},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.1219999999999999,"5min":8.7829999999999995,"15min":8.7829999999999995},"last":4.0049999999999999}]},{"osd":6,"last update":"Sat Sep 23 17:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.64000000000000001,"15min":0.64400000000000002},"min":{"1min":0.42899999999999999,"5min":0.371,"15min":0.371},"max":{"1min":0.84599999999999997,"5min":0.84599999999999997,"15min":0.84599999999999997},"last":6.702},{"interface":"front","average":{"1min":0.64100000000000001,"5min":0.65000000000000002,"15min":0.65200000000000002},"min":{"1min":0.52700000000000002,"5min":0.432,"15min":0.432},"max":{"1min":0.81100000000000005,"5min":0.91500000000000004,"15min":0.91500000000000004},"last":4.1139999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":0.90800000000000003,"15min":0.746},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.8160000000000007,"5min":8.8160000000000007,"15min":8.8160000000000007},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.01,"15min":0.78400000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":8.6470000000000002,"5min":8.6470000000000002,"15min":8.6470000000000002},"last":0.49099999999999999}]}]},{"osd":2,"up_from":103,"seq":442381631551,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1544232,"kb_used_data":1408160,"kb_used_omap":11,"kb_used_meta":135796,"kb_avail":92200920,"statfs":{"total":95995035648,"available":94413742080,"internally_reserved":0,"allocated":1441955840,"data_stored":1441024229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":139055479},"hb_peers":[0,1,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,0,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.2030000000000001,"15min":0.627},"min":{"1min":0.17299999999999999,"5min":0.17299999999999999,"15min":0.17299999999999999},"max":{"1min":10.853,"5min":13.234,"15min":13.234},"last":0.64600000000000002},{"interface":"front","average":{"1min":1.5269999999999999,"5min":1.272,"15min":0.68000000000000005},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":10.497,"5min":13.364000000000001,"15min":13.364000000000001},"last":0.82199999999999995}]},{"osd":1,"last update":"Sat Sep 23 17:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8320000000000001,"5min":1.331,"15min":0.71099999999999997},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.576000000000001,"5min":19.206,"15min":19.206},"last":0.68600000000000005},{"interface":"front","average":{"1min":2.1629999999999998,"5min":1.5249999999999999,"15min":0.72799999999999998},"min":{"1min":0.22800000000000001,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":8.9619999999999997,"5min":15.657,"15min":15.657},"last":0.25700000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":0.58799999999999997,"15min":0.443},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8339999999999996,"5min":4.8339999999999996,"15min":4.8339999999999996},"last":2.8849999999999998},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.59699999999999998,"15min":0.48399999999999999},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.6459999999999999,"5min":5.6459999999999999,"15min":5.6459999999999999},"last":2.5579999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":1.135,"15min":0.78000000000000003},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":6.8499999999999996,"5min":6.8499999999999996,"15min":6.8499999999999996},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.91700000000000004,"15min":0.71199999999999997},"min":{"1min":0.29199999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":4.8230000000000004,"5min":13.308,"15min":13.308},"last":0.77300000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:58:29 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.036,"15min":0.76100000000000001},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":4.7839999999999998,"5min":13.273,"15min":13.273},"last":0.79800000000000004},{"interface":"front","average":{"1min":1.885,"5min":1.454,"15min":0.88700000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":7.327,"5min":15.496,"15min":15.496},"last":2.6549999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:25 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.434,"5min":0.434,"15min":0.434},"max":{"1min":0.93200000000000005,"5min":0.93200000000000005,"15min":0.93200000000000005},"last":0.33500000000000002},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":0.88900000000000001,"5min":0.88900000000000001,"15min":0.88900000000000001},"last":0.73499999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":0.94299999999999995,"15min":0.71899999999999997},"min":{"1min":0.505,"5min":0.441,"15min":0.441},"max":{"1min":5.7409999999999997,"5min":5.7409999999999997,"15min":5.7409999999999997},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.0870000000000002,"5min":0.97899999999999998,"15min":0.78300000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":13.333,"5min":13.333,"15min":13.333},"last":1.3440000000000001}]}]},{"osd":3,"up_from":108,"seq":463856467999,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267348,"kb_used_data":1221900,"kb_used_omap":3,"kb_used_meta":45436,"kb_avail":92477804,"statfs":{"total":95995035648,"available":94697271296,"internally_reserved":0,"allocated":1251225600,"data_stored":1250531999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":46527208},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.0699999999999998,"5min":0.71399999999999997,"15min":0.48799999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":14.82,"5min":14.82,"15min":14.82},"last":9.0220000000000002},{"interface":"front","average":{"1min":2.085,"5min":0.72499999999999998,"15min":0.499},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":13.054,"5min":13.054,"15min":13.054},"last":0.46500000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":0.65700000000000003,"15min":0.46899999999999997},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.8130000000000006,"5min":8.8130000000000006,"15min":8.8130000000000006},"last":9.1489999999999991},{"interface":"front","average":{"1min":1.417,"5min":0.61599999999999999,"15min":0.48199999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4580000000000002,"5min":6.4580000000000002,"15min":6.4580000000000002},"last":9.125}]},{"osd":2,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":0.69099999999999995,"15min":0.49099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.869999999999999,"5min":10.869999999999999,"15min":10.869999999999999},"last":7.3849999999999998},{"interface":"front","average":{"1min":1.6850000000000001,"5min":0.66500000000000004,"15min":0.495},"min":{"1min":0.376,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":13.936999999999999,"5min":13.936999999999999,"15min":13.936999999999999},"last":8.9169999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":0.73699999999999999,"15min":0.65200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5380000000000003,"5min":6.5380000000000003,"15min":6.5380000000000003},"last":6.7569999999999997},{"interface":"front","average":{"1min":1.629,"5min":0.82099999999999995,"15min":0.68700000000000006},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.5940000000000003,"5min":6.5940000000000003,"15min":6.5940000000000003},"last":3.7709999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":0.94399999999999995,"15min":0.72599999999999998},"min":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":9.6489999999999991,"5min":9.6489999999999991,"15min":9.6489999999999991},"last":3.9569999999999999},{"interface":"front","average":{"1min":1.4710000000000001,"5min":0.78600000000000003,"15min":0.67200000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.3200000000000003,"5min":9.3200000000000003,"15min":9.3200000000000003},"last":6.4569999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":0.81799999999999995,"15min":0.68999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":8.8599999999999994,"5min":8.8599999999999994,"15min":8.8599999999999994},"last":6.4950000000000001},{"interface":"front","average":{"1min":1.3520000000000001,"5min":0.78800000000000003,"15min":0.69399999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.2080000000000002,"5min":9.2080000000000002,"15min":9.2080000000000002},"last":6.0709999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019070,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1524180,"kb_used_data":1411148,"kb_used_omap":0,"kb_used_meta":113024,"kb_avail":92220972,"statfs":{"total":95995035648,"available":94434275328,"internally_reserved":0,"allocated":1445015552,"data_stored":1444272534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":115736576},"hb_peers":[0,1,2,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,4,4,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":0.97399999999999998,"15min":0.66900000000000004},"min":{"1min":0.22500000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.5279999999999996,"5min":6.5279999999999996,"15min":6.5279999999999996},"last":1.5289999999999999},{"interface":"front","average":{"1min":1.744,"5min":1.3300000000000001,"15min":0.79300000000000004},"min":{"1min":0.22900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.5739999999999998,"5min":11.372,"15min":11.372},"last":4.8959999999999999}]},{"osd":1,"last update":"Sat Sep 23 17:59:01 2023","interfaces":[{"interface":"back","average":{"1min":1.4279999999999999,"5min":1.1240000000000001,"15min":0.72799999999999998},"min":{"1min":0.27100000000000002,"5min":0.249,"15min":0.249},"max":{"1min":10.429,"5min":10.429,"15min":10.429},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.6819999999999999,"5min":1.397,"15min":0.83699999999999997},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":10.519,"5min":10.83,"15min":10.83},"last":1.5649999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:58:31 2023","interfaces":[{"interface":"back","average":{"1min":1.754,"5min":1.111,"15min":0.79200000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":7.1790000000000003,"5min":7.1790000000000003,"15min":7.1790000000000003},"last":1.821},{"interface":"front","average":{"1min":1.857,"5min":1.1060000000000001,"15min":0.75800000000000001},"min":{"1min":0.30099999999999999,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.5090000000000003,"5min":7.5090000000000003,"15min":7.5090000000000003},"last":1.5980000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.72899999999999998,"15min":0.64900000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6130000000000004,"5min":4.6130000000000004,"15min":4.6130000000000004},"last":1.397},{"interface":"front","average":{"1min":1.04,"5min":0.73299999999999998,"15min":0.68200000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":3.637,"5min":3.637,"15min":3.637},"last":9.4359999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:58:31 2023","interfaces":[{"interface":"back","average":{"1min":0.45100000000000001,"5min":0.51100000000000001,"15min":0.46700000000000003},"min":{"1min":0.224,"5min":0.224,"15min":0.224},"max":{"1min":1.958,"5min":2.7599999999999998,"15min":2.7599999999999998},"last":0.505},{"interface":"front","average":{"1min":0.69199999999999995,"5min":0.70499999999999996,"15min":0.53600000000000003},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.4209999999999998,"5min":10.494,"15min":10.494},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 23 17:55:44 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.54200000000000004,"15min":0.50800000000000001},"min":{"1min":0.45400000000000001,"5min":0.33200000000000002,"15min":0.28899999999999998},"max":{"1min":0.64700000000000002,"5min":0.755,"15min":1.276},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.65500000000000003,"5min":0.59299999999999997,"15min":0.54700000000000004},"min":{"1min":0.52500000000000002,"5min":0.29899999999999999,"15min":0.28999999999999998},"max":{"1min":0.79400000000000004,"5min":0.80900000000000005,"15min":1.1180000000000001},"last":1.1240000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.57399999999999995,"15min":0.55100000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.8370000000000002,"5min":2.8370000000000002,"15min":2.8370000000000002},"last":0.308},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.64400000000000002,"15min":0.56299999999999994},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":0.434}]}]},{"osd":5,"up_from":0,"seq":163208757436,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2069168128,"data_stored":2069168128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1101529088,"data_stored":1101529088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1431441408,"data_stored":1431437336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1253441536,"data_stored":1253441536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1440415744,"data_stored":1440415744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1687621632,"data_stored":1687617560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":372375552,"data_stored":372375552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1818103808,"data_stored":1818099736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T17:59:34.673 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T17:59:34.931 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:34.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:34.936 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:35.037 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:35.070 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:35.126 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:35.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:35.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:35.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:35.278 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:35.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:35.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:35 smithi099 ceph-mon[167076]: from='client.44954 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:59:35.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:35.330 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:35.431 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:35.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:35 smithi055 ceph-mon[179668]: from='client.44954 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:59:35.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:35 smithi055 ceph-mon[184174]: from='client.44954 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T17:59:35.538 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:35.643 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:35.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:35.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:35.758 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:35.780 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:59:35.780 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:59:35.827 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:35.854 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:35.951 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:35.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:35.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:36.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:36.113 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:36.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:36.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:36.177 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:36.255 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T17:59:36.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:36.338 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:36.378 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:36.382 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:36 smithi099 ceph-mon[167076]: pgmap v410: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 17122/131094 objects degraded (13.061%); 31129/131094 objects misplaced (23.746%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T17:59:36.383 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/596265349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:36.383 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:36 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:36.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:36.384 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:36.398 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T17:59:36.448 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:59:36.469 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:36 smithi055 ceph-mon[184174]: pgmap v410: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 17122/131094 objects degraded (13.061%); 31129/131094 objects misplaced (23.746%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T17:59:36.469 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/596265349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:36.469 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:36 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:36.470 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:36 smithi055 ceph-mon[179668]: pgmap v410: 109 pgs: 1 down, 14 active+clean+remapped, 1 active+recovering+degraded+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.7 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 622 op/s; 17122/131094 objects degraded (13.061%); 31129/131094 objects misplaced (23.746%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T17:59:36.471 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/596265349' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:36.471 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:36 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:36.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:36.515 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:36.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:36.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:36.565 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:36.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:36.694 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:36.731 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:36.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:36.735 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:36.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:36.865 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:36.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:36.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:36.912 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:37.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:37.040 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:37.090 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:37.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:37.094 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:37.195 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:37.233 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:37.282 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:37.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:37.285 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:37.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:37 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:37.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:37 smithi099 ceph-mon[167076]: failed to encode map e183 with expected crc 2023-09-23T17:59:37.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:37 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:37.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:37 smithi099 ceph-mon[167076]: osdmap e183: 8 total, 6 up, 7 in 2023-09-23T17:59:37.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:59:37.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:37.363 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:37.377 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:37.377 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[184174]: failed to encode map e183 with expected crc 2023-09-23T17:59:37.377 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:37.377 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[184174]: osdmap e183: 8 total, 6 up, 7 in 2023-09-23T17:59:37.377 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:59:37.377 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:37.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:37.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[179668]: failed to encode map e183 with expected crc 2023-09-23T17:59:37.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:37.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[179668]: osdmap e183: 8 total, 6 up, 7 in 2023-09-23T17:59:37.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T17:59:37.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:37.386 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:37.516 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:37.578 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:37.711 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:37.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:37.714 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:37.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:37.859 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:37.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:37.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:37.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:38.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:38.046 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:38.096 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:38.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:38.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:38.207 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:38.292 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:38 smithi055 ceph-mon[184174]: pgmap v412: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 14292/129480 objects degraded (11.038%); 33611/129480 objects misplaced (25.958%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T17:59:38.292 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 14292/129480 objects degraded (11.038%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:38.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:38 smithi055 ceph-mon[179668]: pgmap v412: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 14292/129480 objects degraded (11.038%); 33611/129480 objects misplaced (25.958%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T17:59:38.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 14292/129480 objects degraded (11.038%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:38.299 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:38.422 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:38.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:38.435 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:38.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:38.564 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:38 smithi099 ceph-mon[167076]: pgmap v412: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 14292/129480 objects degraded (11.038%); 33611/129480 objects misplaced (25.958%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T17:59:38.564 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 14292/129480 objects degraded (11.038%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:38.566 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:38.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:38.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:38.620 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:38.721 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:38.767 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:38.820 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:38.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:38.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:38.844 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T17:59:38.930 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:39.038 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:39.124 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:39.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:39.138 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:39.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:39.240 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 1 pgp_num to 21 2023-09-23T17:59:39.267 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:39.277 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:39 smithi055 ceph-mon[179668]: pgmap v413: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 13911/127848 objects degraded (10.881%); 33271/127848 objects misplaced (26.024%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T17:59:39.277 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3883093639' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T17:59:39.277 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:39 smithi055 ceph-mon[184174]: pgmap v413: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 13911/127848 objects degraded (10.881%); 33271/127848 objects misplaced (26.024%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T17:59:39.277 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3883093639' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T17:59:39.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:39.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:39.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:39.421 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:39.485 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:59:39.485 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:59:39.518 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:39.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:39 smithi099 ceph-mon[167076]: pgmap v413: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.6 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 647 op/s; 13911/127848 objects degraded (10.881%); 33271/127848 objects misplaced (26.024%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T17:59:39.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3883093639' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T17:59:39.925 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:39.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:39.946 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:40.047 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:40.057 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:59:40.110 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:59:40.122 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:40.160 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:59:40.168 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T17:59:40.200 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:59:40.251 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:59:40.298 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:40.299 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:59:40.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:40.309 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:40.315 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:59:40.315 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:59:40.315 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:59:40.316 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:59:40.317 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:59:40.317 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:59:40.317 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T17:59:40.317 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:59:40.360 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:40.410 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:40.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[179668]: failed to encode map e184 with expected crc 2023-09-23T17:59:40.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3883093639' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T17:59:40.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[179668]: osdmap e184: 8 total, 6 up, 7 in 2023-09-23T17:59:40.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[179668]: failed to encode map e184 with expected crc 2023-09-23T17:59:40.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[179668]: failed to encode map e184 with expected crc 2023-09-23T17:59:40.439 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4242593722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:40.440 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:59:40] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-09-23T17:59:40.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[184174]: failed to encode map e184 with expected crc 2023-09-23T17:59:40.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3883093639' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T17:59:40.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[184174]: osdmap e184: 8 total, 6 up, 7 in 2023-09-23T17:59:40.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[184174]: failed to encode map e184 with expected crc 2023-09-23T17:59:40.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[184174]: failed to encode map e184 with expected crc 2023-09-23T17:59:40.440 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4242593722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:40.544 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:40 smithi099 ceph-mon[167076]: failed to encode map e184 with expected crc 2023-09-23T17:59:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3883093639' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T17:59:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:40 smithi099 ceph-mon[167076]: osdmap e184: 8 total, 6 up, 7 in 2023-09-23T17:59:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:40 smithi099 ceph-mon[167076]: failed to encode map e184 with expected crc 2023-09-23T17:59:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:40 smithi099 ceph-mon[167076]: failed to encode map e184 with expected crc 2023-09-23T17:59:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4242593722' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:40.707 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:40.718 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:40.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:40.775 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:59:40.776 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:59:40.819 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:40.896 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:40.996 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:41.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:41.004 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:41.105 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:41.180 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:41 smithi055 ceph-mon[184174]: pgmap v415: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 13185/122217 objects degraded (10.788%); 32271/122217 objects misplaced (26.405%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T17:59:41.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:41 smithi055 ceph-mon[179668]: pgmap v415: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 13185/122217 objects degraded (10.788%); 32271/122217 objects misplaced (26.405%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T17:59:41.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:41 smithi099 ceph-mon[167076]: pgmap v415: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 13185/122217 objects degraded (10.788%); 32271/122217 objects misplaced (26.405%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T17:59:42.777 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T17:59:43.000 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:43.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:43.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:43.038 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:43.109 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:43.149 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:43.194 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:43.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:43.202 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:43.303 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:43.359 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:43.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:43 smithi055 ceph-mon[179668]: pgmap v416: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.4 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 12835/119598 objects degraded (10.732%); 31734/119598 objects misplaced (26.534%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T17:59:43.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 12835/119598 objects degraded (10.732%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:43.652 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:43.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:43.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:43.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:43.798 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:43.851 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:43.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:43.858 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:43.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:43 smithi055 ceph-mon[184174]: pgmap v416: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.4 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 12835/119598 objects degraded (10.732%); 31734/119598 objects misplaced (26.534%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T17:59:43.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 12835/119598 objects degraded (10.732%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:43.959 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:43.993 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:43 smithi099 ceph-mon[167076]: pgmap v416: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.4 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 478 op/s; 12835/119598 objects degraded (10.732%); 31734/119598 objects misplaced (26.534%); 4.3 MiB/s, 69 objects/s recovering 2023-09-23T17:59:43.993 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 12835/119598 objects degraded (10.732%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:43.994 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:44.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:44.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:44.033 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:44.134 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:44.199 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:44.322 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:44.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:44.332 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:44.432 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:44.438 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:44.514 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:44.638 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:44.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:44.646 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:44.747 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:44.793 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:44.831 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:44.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:44.837 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:44.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:44.963 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T17:59:44.963 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:59:44.964 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T17:59:44.964 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.953062 2023-09-23T17:59:44.964 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0.9530624432668539 2023-09-23T17:59:44.971 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:45.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:45.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:45.021 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:45.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:45.152 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:45.190 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:45.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:45.195 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:45.295 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:45.342 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:59:45.342 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104107 5.0 5:08ba7c2f:::benchmark_data_smithi055_229117_object104106:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.953750+0000", 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 141.381255501, 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3969089229999998, 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.343 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104107 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953750+0000", 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953750+0000", 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953748+0000", 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953776+0000", 2023-09-23T17:59:45.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8515000000000001e-05 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953798+0000", 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1855999999999999e-05 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953808+0000", 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0283e-05 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953876+0000", 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7348000000000006e-05 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op must wait for map", 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953885+0000", 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0839999999999998e-06 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.962444+0000", 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0085596219999999994 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.962456+0000", 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1606999999999999e-05 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949379+0000", 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98692283300000005 2023-09-23T17:59:45.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949390+0000", 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1503e-05 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.325692+0000", 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.376301955 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.327734+0000", 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020413749999999998 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.327871+0000", 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137739 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336120+0000", 2023-09-23T17:59:45.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082488500000000003 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336181+0000", 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1007000000000001e-05 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.348130+0000", 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011948676 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.348167+0000", 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6782000000000001e-05 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350658+0000", 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024918570000000001 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104131 5.0 5:02532558:::benchmark_data_smithi055_229117_object104130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.987392+0000", 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 141.34761297599999, 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3655978989999999, 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104131 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987392+0000", 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987392+0000", 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987390+0000", 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987415+0000", 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4185e-05 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987433+0000", 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8238000000000001e-05 2023-09-23T17:59:45.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987442+0000", 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9600000000000006e-06 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.988601+0000", 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011587889999999999 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.988612+0000", 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1969e-05 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949474+0000", 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96086169899999996 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949482+0000", 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0099999999999995e-06 2023-09-23T17:59:45.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333244+0000", 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3837614760000001 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333381+0000", 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013695800000000001 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333517+0000", 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000136181 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342436+0000", 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0089189990000000004 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342466+0000", 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0561000000000001e-05 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350747+0000", 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082805090000000001 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350829+0000", 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1778000000000002e-05 2023-09-23T17:59:45.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.352990+0000", 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021612770000000001 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104167 5.0 5:0cc1b975:::benchmark_data_smithi055_229117_object104166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:24.986469+0000", 2023-09-23T17:59:45.361 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.34853626099999, 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3687401239999999, 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104167 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986469+0000", 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986469+0000", 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986467+0000", 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986500+0000", 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3046000000000002e-05 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986515+0000", 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5138e-05 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986527+0000", 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1429e-05 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.026788+0000", 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040261308000000003 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.026799+0000", 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0699999999999999e-05 2023-09-23T17:59:45.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336245+0000", 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3094467810000001 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336378+0000", 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013300399999999999 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336525+0000", 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014695100000000001 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342505+0000", 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0059792530000000003 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342518+0000", 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.348e-05 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353111+0000", 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010592839 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353201+0000", 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0167999999999995e-05 2023-09-23T17:59:45.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355209+0000", 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020078090000000002 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104189 5.0 5:00d05606:::benchmark_data_smithi055_229117_object104188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.012952+0000", 2023-09-23T17:59:45.369 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.32205348799999, 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.354677634, 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104189 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012952+0000", 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012952+0000", 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012950+0000", 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012983+0000", 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2840999999999999e-05 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012997+0000", 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.454e-05 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.013006+0000", 2023-09-23T17:59:45.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2269999999999999e-06 2023-09-23T17:59:45.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.030199+0000", 2023-09-23T17:59:45.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017192819000000002 2023-09-23T17:59:45.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.030209+0000", 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.023e-05 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339148+0000", 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3089385840000001 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339354+0000", 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020608099999999999 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339570+0000", 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021573299999999999 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342529+0000", 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029589759999999999 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342571+0000", 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1863999999999998e-05 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365363+0000", 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022792085 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365409+0000", 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6479e-05 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367629+0000", 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022198840000000001 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.377 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104208 5.0 5:0c574cd2:::benchmark_data_smithi055_229117_object104207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.042202+0000", 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.29280330700001, 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3303606880000001, 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104208 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.378 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042202+0000", 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042202+0000", 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042200+0000", 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042226+0000", 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5925e-05 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042245+0000", 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9449999999999998e-05 2023-09-23T17:59:45.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042255+0000", 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9739999999999997e-06 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042301+0000", 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5945999999999997e-05 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042309+0000", 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7130000000000005e-06 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342600+0000", 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3002915850000001 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342725+0000", 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012474099999999999 2023-09-23T17:59:45.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342853+0000", 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000127806 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353066+0000", 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010212738000000001 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355236+0000", 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002170052 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369924+0000", 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014688012 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369984+0000", 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0571e-05 2023-09-23T17:59:45.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372562+0000", 2023-09-23T17:59:45.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00257807 2023-09-23T17:59:45.385 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.385 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.385 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.385 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104228 5.0 5:04306094:::benchmark_data_smithi055_229117_object104227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.063372+0000", 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.271632635, 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3247219290000001, 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104228 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063372+0000", 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063372+0000", 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063371+0000", 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063394+0000", 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2929999999999999e-05 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063407+0000", 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3142000000000001e-05 2023-09-23T17:59:45.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063415+0000", 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8120000000000008e-06 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063476+0000", 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0988e-05 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063486+0000", 2023-09-23T17:59:45.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0397999999999999e-05 2023-09-23T17:59:45.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345425+0000", 2023-09-23T17:59:45.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2819386879999999 2023-09-23T17:59:45.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345551+0000", 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012593200000000001 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345665+0000", 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114308 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355247+0000", 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0095815529999999996 2023-09-23T17:59:45.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355307+0000", 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0619999999999999e-05 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385736+0000", 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030428261000000002 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385803+0000", 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7624000000000002e-05 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388094+0000", 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002291015 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.394 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104253 5.0 5:0d680df2:::benchmark_data_smithi055_229117_object104252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.394 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.088438+0000", 2023-09-23T17:59:45.394 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.24656701699999, 2023-09-23T17:59:45.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3025030870000001, 2023-09-23T17:59:45.394 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.394 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.394 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.394 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.394 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.394 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104253 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088438+0000", 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088438+0000", 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088436+0000", 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088461+0000", 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4658999999999999e-05 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088478+0000", 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7805e-05 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088487+0000", 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.225e-06 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.092295+0000", 2023-09-23T17:59:45.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038084870000000002 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.092306+0000", 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1172000000000001e-05 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355342+0000", 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.263036211 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355533+0000", 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000190767 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355663+0000", 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012937699999999999 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365284+0000", 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0096211700000000001 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367649+0000", 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002365514 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388536+0000", 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020886503000000001 2023-09-23T17:59:45.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388577+0000", 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1167000000000001e-05 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.390941+0000", 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002364157 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.401 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104272 5.0 5:0ec51932:::benchmark_data_smithi055_229117_object104271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.108373+0000", 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.226631567, 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2933625550000001, 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.402 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104272 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108373+0000", 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108373+0000", 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108372+0000", 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108396+0000", 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4311000000000002e-05 2023-09-23T17:59:45.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108412+0000", 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6049000000000002e-05 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108420+0000", 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9170000000000006e-06 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108482+0000", 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1969000000000002e-05 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108491+0000", 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4779999999999997e-06 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.358777+0000", 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2502855180000001 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.358912+0000", 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013505099999999999 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.359032+0000", 2023-09-23T17:59:45.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120331 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367658+0000", 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0086259400000000003 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367698+0000", 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9477999999999997e-05 2023-09-23T17:59:45.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.398598+0000", 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030900179 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.398665+0000", 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7167000000000003e-05 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401736+0000", 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030708630000000001 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104276 5.0 5:0ba41185:::benchmark_data_smithi055_229117_object104275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.114398+0000", 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.22060657500001, 2023-09-23T17:59:45.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2928085359999999, 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104276 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114398+0000", 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114398+0000", 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114397+0000", 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:45.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114423+0000", 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6006000000000001e-05 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114440+0000", 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6877000000000001e-05 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114448+0000", 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5120000000000008e-06 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.115268+0000", 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00081976499999999997 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.115279+0000", 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0533999999999999e-05 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362251+0000", 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.24697258 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362402+0000", 2023-09-23T17:59:45.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015118400000000001 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362530+0000", 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012761700000000001 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369824+0000", 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072939889999999999 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372645+0000", 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028209210000000001 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.405151+0000", 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032506363000000003 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.405203+0000", 2023-09-23T17:59:45.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1987000000000002e-05 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407207+0000", 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002003777 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104298 5.0 5:040384f9:::benchmark_data_smithi055_229117_object104297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.139530+0000", 2023-09-23T17:59:45.418 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.19547454600001, 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2697862209999999, 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104298 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139530+0000", 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139530+0000", 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139529+0000", 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139553+0000", 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4026999999999999e-05 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139565+0000", 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2364999999999999e-05 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139573+0000", 2023-09-23T17:59:45.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4200000000000001e-06 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.141901+0000", 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023278639999999998 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.141909+0000", 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5099999999999998e-06 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372679+0000", 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2307700370000001 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372803+0000", 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012344299999999999 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372931+0000", 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012818500000000001 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385669+0000", 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012738027000000001 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385682+0000", 2023-09-23T17:59:45.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3368e-05 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407273+0000", 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021590933 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407320+0000", 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7305999999999997e-05 2023-09-23T17:59:45.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409317+0000", 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019962769999999999 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104301 5.0 5:0653eda0:::benchmark_data_smithi055_229117_object104300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.426 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.142080+0000", 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.19292537600001, 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2806010670000001, 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104301 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142080+0000", 2023-09-23T17:59:45.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142080+0000", 2023-09-23T17:59:45.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142078+0000", 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142104+0000", 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5931e-05 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142116+0000", 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2262999999999999e-05 2023-09-23T17:59:45.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142125+0000", 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5359999999999992e-06 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142238+0000", 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000113056 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142247+0000", 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6680000000000007e-06 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.375988+0000", 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2337410959999999 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.376092+0000", 2023-09-23T17:59:45.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104558 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.376220+0000", 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012734399999999999 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385691+0000", 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0094714069999999994 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391023+0000", 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053320190000000003 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.419987+0000", 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028963940000000001 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.420045+0000", 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8421000000000002e-05 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422681+0000", 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026352419999999999 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104305 5.0 5:03148f53:::benchmark_data_smithi055_229117_object104304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.146575+0000", 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.18842994400001, 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2788299140000001, 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104305 2023-09-23T17:59:45.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146575+0000", 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146575+0000", 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146573+0000", 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146600+0000", 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6675000000000001e-05 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146622+0000", 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1965000000000001e-05 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146633+0000", 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.076e-05 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.147763+0000", 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011302879999999999 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.147777+0000", 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4129e-05 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379012+0000", 2023-09-23T17:59:45.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2312345389999999 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379258+0000", 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024643700000000001 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379508+0000", 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024977000000000002 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.390966+0000", 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011458268000000001 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391089+0000", 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000122881 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422762+0000", 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031672697999999999 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422832+0000", 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9929000000000001e-05 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.425405+0000", 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002573362 2023-09-23T17:59:45.442 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104309 5.0 5:0de80b4f:::benchmark_data_smithi055_229117_object104308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.152292+0000", 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.182712619, 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2892466869999999, 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.443 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104309 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152292+0000", 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152292+0000", 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152291+0000", 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152313+0000", 2023-09-23T17:59:45.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2082999999999998e-05 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152326+0000", 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2493999999999999e-05 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152332+0000", 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7710000000000001e-06 2023-09-23T17:59:45.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152379+0000", 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6233999999999998e-05 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152389+0000", 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0234999999999999e-05 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382503+0000", 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.230114411 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382678+0000", 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000174704 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382886+0000", 2023-09-23T17:59:45.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000208481 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391101+0000", 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082150169999999998 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391162+0000", 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0718999999999999e-05 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.438904+0000", 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047742297000000003 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.438971+0000", 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6787999999999998e-05 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.451 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441539+0000", 2023-09-23T17:59:45.451 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025679069999999999 2023-09-23T17:59:45.451 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.451 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.451 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.451 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.451 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.451 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104346 5.0 5:07875566:::benchmark_data_smithi055_229117_object104345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.451 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.224009+0000", 2023-09-23T17:59:45.451 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.11099567100001, 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2202360750000001, 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104346 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.452 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224009+0000", 2023-09-23T17:59:45.453 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.453 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.453 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.453 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.453 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224009+0000", 2023-09-23T17:59:45.453 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.453 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.453 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.453 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.453 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224007+0000", 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224037+0000", 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9128e-05 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224058+0000", 2023-09-23T17:59:45.454 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1829999999999999e-05 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224068+0000", 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8560000000000002e-06 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224119+0000", 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0844000000000003e-05 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224128+0000", 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4880000000000007e-06 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391218+0000", 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1670905920000001 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.456 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391390+0000", 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017210399999999999 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391557+0000", 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000166694 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409338+0000", 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017781376000000002 2023-09-23T17:59:45.457 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413075+0000", 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037367199999999998 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441577+0000", 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028502169000000001 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.458 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441616+0000", 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8859999999999997e-05 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444245+0000", 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026292899999999998 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.459 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104353 5.0 5:0b824901:::benchmark_data_smithi055_229117_object104352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.238109+0000", 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.09689604900001, 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.208967144, 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.460 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104353 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238109+0000", 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238109+0000", 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.461 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238108+0000", 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238128+0000", 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9854999999999999e-05 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.462 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238135+0000", 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1729999999999998e-06 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238140+0000", 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6199999999999998e-06 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238197+0000", 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7005000000000001e-05 2023-09-23T17:59:45.463 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238204+0000", 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2819999999999999e-06 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.394820+0000", 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.156615231 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.464 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.394956+0000", 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013646599999999999 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.395118+0000", 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000162158 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409348+0000", 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014229883 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413141+0000", 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00379284 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444312+0000", 2023-09-23T17:59:45.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031170989 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444373+0000", 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1309000000000005e-05 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447076+0000", 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027029129999999999 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.467 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104360 5.0 5:0a7d8202:::benchmark_data_smithi055_229117_object104359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.266784+0000", 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 140.06822087, 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1830323949999999, 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.468 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104360 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266784+0000", 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266784+0000", 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.469 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.470 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.470 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.470 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266782+0000", 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266806+0000", 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3794999999999998e-05 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266824+0000", 2023-09-23T17:59:45.471 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7838000000000001e-05 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266833+0000", 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9469999999999992e-06 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266890+0000", 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7238999999999999e-05 2023-09-23T17:59:45.472 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266902+0000", 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1919999999999999e-05 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401792+0000", 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.13489014 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401976+0000", 2023-09-23T17:59:45.474 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000184014 2023-09-23T17:59:45.474 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.474 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.474 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.474 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.402162+0000", 2023-09-23T17:59:45.474 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018587599999999999 2023-09-23T17:59:45.474 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.474 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.474 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.474 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413089+0000", 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010926544999999999 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.425460+0000", 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012371320999999999 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447163+0000", 2023-09-23T17:59:45.475 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021703538000000001 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447231+0000", 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7675999999999994e-05 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.449817+0000", 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025853859999999998 2023-09-23T17:59:45.476 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:108962 5.0 5:0bab387e:::benchmark_data_smithi055_229117_object3336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.753784+0000", 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 132.581220637, 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.186715511, 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.477 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108962 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753784+0000", 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.478 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753784+0000", 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753783+0000", 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753792+0000", 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0550000000000005e-06 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.479 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753807+0000", 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4888e-05 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753815+0000", 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8659999999999995e-06 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.841808+0000", 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.087992655000000003 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.842575+0000", 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00076655600000000005 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.871420+0000", 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028845756 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.871532+0000", 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000112043 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.871703+0000", 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017070599999999999 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.482 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.898664+0000", 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026961124 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.902238+0000", 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035741309999999999 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.938343+0000", 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.036104458999999998 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.483 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.938383+0000", 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9885000000000002e-05 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940500+0000", 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021171979999999998 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.484 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:108965 5.0 5:0ca96563:::benchmark_data_smithi055_229117_object3339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.758821+0000", 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 132.57618423400001, 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18445447700000001, 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108965 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.485 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758821+0000", 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758821+0000", 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.486 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758820+0000", 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758825+0000", 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4580000000000001e-06 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758834+0000", 2023-09-23T17:59:45.487 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5339999999999999e-06 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758839+0000", 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1569999999999999e-06 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.842655+0000", 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.083815857999999993 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.488 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.843295+0000", 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000640651 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.887064+0000", 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043768661 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.489 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.887235+0000", 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017120699999999999 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.887449+0000", 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021399400000000001 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.902251+0000", 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014802123 2023-09-23T17:59:45.490 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940561+0000", 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038309546 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940592+0000", 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1056999999999998e-05 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.491 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940635+0000", 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3038999999999999e-05 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.943275+0000", 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002640444 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.492 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:108971 5.0 5:0ef24fea:::benchmark_data_smithi055_229117_object3345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.766589+0000", 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 132.56841610699999, 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.182688712, 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108971 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.493 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766589+0000", 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766589+0000", 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.494 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766588+0000", 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766593+0000", 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0699999999999997e-06 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766653+0000", 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0232000000000002e-05 2023-09-23T17:59:45.495 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.496 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.496 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.496 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766662+0000", 2023-09-23T17:59:45.496 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0240000000000003e-06 2023-09-23T17:59:45.496 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.496 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.496 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.496 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.843985+0000", 2023-09-23T17:59:45.496 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.077322734000000004 2023-09-23T17:59:45.496 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.844592+0000", 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00060725799999999995 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.906868+0000", 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.062275358000000003 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.906988+0000", 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120485 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.907138+0000", 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000150216 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.921960+0000", 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014822003 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.946229+0000", 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024268112000000001 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.946287+0000", 2023-09-23T17:59:45.499 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7983999999999999e-05 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.946365+0000", 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8867000000000002e-05 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.949278+0000", 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029122089999999998 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.500 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPushReply(5.0 145/143 [PushReplyOp(5:023b70b2:::benchmark_data_smithi055_229117_object72678:head)])", 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.870165+0000", 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 132.46483997000001, 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18210786500000001, 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.501 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870165+0000", 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870165+0000", 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870164+0000", 2023-09-23T17:59:45.502 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870173+0000", 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7709999999999995e-06 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870197+0000", 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4243000000000002e-05 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.503 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870207+0000", 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8279999999999998e-06 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:33.047464+0000", 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.177257317 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:33.052273+0000", 2023-09-23T17:59:45.504 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048086429999999996 2023-09-23T17:59:45.505 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.505 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.505 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.505 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T17:59:45.505 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T17:59:45.505 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:59:45.505 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:45.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:45 smithi055 ceph-mon[179668]: pgmap v417: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 12010/113025 objects degraded (10.626%); 30465/113025 objects misplaced (26.954%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T17:59:45.637 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1213202511' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:59:45.637 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:45 smithi055 ceph-mon[184174]: pgmap v417: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 12010/113025 objects degraded (10.626%); 30465/113025 objects misplaced (26.954%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T17:59:45.637 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1213202511' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:59:45.671 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T17:59:45.830 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:45.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:45.838 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:45.901 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T17:59:45.940 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:46.067 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:46.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:45 smithi099 ceph-mon[167076]: pgmap v417: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 657 op/s; 12010/113025 objects degraded (10.626%); 30465/113025 objects misplaced (26.954%); 4.6 MiB/s, 72 objects/s recovering 2023-09-23T17:59:46.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1213202511' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T17:59:46.227 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:46.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:46.258 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:46.359 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:46.427 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:46.520 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:46.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:46.529 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:46.546 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:46.630 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:46.719 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:46.726 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:59:46.780 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:59:46.841 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:59:46.846 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:46 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:46.846 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1213202511' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:59:46.846 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:46 smithi055 ceph-mon[184174]: osdmap e185: 8 total, 6 up, 7 in 2023-09-23T17:59:46.846 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:46 smithi055 ceph-mon[184174]: failed to encode map e185 with expected crc 2023-09-23T17:59:46.846 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:46 smithi055 ceph-mon[184174]: failed to encode map e185 with expected crc 2023-09-23T17:59:46.847 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:46 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:46.847 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1213202511' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:59:46.847 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:46 smithi055 ceph-mon[179668]: osdmap e185: 8 total, 6 up, 7 in 2023-09-23T17:59:46.847 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:46 smithi055 ceph-mon[179668]: failed to encode map e185 with expected crc 2023-09-23T17:59:46.847 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:46 smithi055 ceph-mon[179668]: failed to encode map e185 with expected crc 2023-09-23T17:59:46.856 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:46.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:46.866 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:46.881 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:59:46.912 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:59:46.958 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:59:46.965 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T17:59:46.965 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T17:59:46.965 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T17:59:46.966 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:59:46.966 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T17:59:46.966 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T17:59:46.966 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T17:59:46.966 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:59:46.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:46.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:46 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T17:59:46.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1213202511' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T17:59:46.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:46 smithi099 ceph-mon[167076]: osdmap e185: 8 total, 6 up, 7 in 2023-09-23T17:59:46.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:46 smithi099 ceph-mon[167076]: failed to encode map e185 with expected crc 2023-09-23T17:59:46.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:46 smithi099 ceph-mon[167076]: failed to encode map e185 with expected crc 2023-09-23T17:59:46.997 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:47.044 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:47.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:47.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:47.107 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:47.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:47.183 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:47.238 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:47.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:47.247 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:47.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:47.396 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:47.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:47.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:47.453 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:47.554 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:47.596 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:47.780 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:47 smithi055 ceph-mon[184174]: pgmap v419: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 12010/111327 objects degraded (10.788%); 30097/111327 objects misplaced (27.035%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T17:59:47.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/44324716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:47.781 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:47 smithi055 ceph-mon[179668]: pgmap v419: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 12010/111327 objects degraded (10.788%); 30097/111327 objects misplaced (27.035%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T17:59:47.781 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/44324716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:47.848 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:47.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:47.852 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:47.938 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:59:47.938 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:59:47.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:47.984 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:47 smithi099 ceph-mon[167076]: pgmap v419: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.3 GiB data, 8.7 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 12010/111327 objects degraded (10.788%); 30097/111327 objects misplaced (27.035%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T17:59:47.984 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/44324716' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:47.990 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:48.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:48.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:48.041 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:48.142 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:48.182 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:48.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:48.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:48.233 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:48.280 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T17:59:48.333 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:48.412 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:48.565 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:48.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:48.581 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:48.681 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:48.778 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:48.858 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:48.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:48.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:48.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 12010/111327 objects degraded (10.788%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:48.867 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 12010/111327 objects degraded (10.788%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:48.965 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:48.992 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 12010/111327 objects degraded (10.788%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:48.995 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:49.039 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:49.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:49.043 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:49.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:49.187 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:49.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:49.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:49.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:49.339 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:49.385 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:49.445 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:49.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:49.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:49.460 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:49.551 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:49.632 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:49.905 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.4 primary-affinity to 0.953062 (8624592) 2023-09-23T17:59:49.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:49 smithi055 ceph-mon[184174]: pgmap v420: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 11612/109590 objects degraded (10.596%); 29705/109590 objects misplaced (27.106%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T17:59:49.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1802822673' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9530624432668539}]: dispatch 2023-09-23T17:59:49.938 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:49 smithi055 ceph-mon[179668]: pgmap v420: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 11612/109590 objects degraded (10.596%); 29705/109590 objects misplaced (27.106%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T17:59:49.938 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1802822673' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9530624432668539}]: dispatch 2023-09-23T17:59:49.939 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T17:59:50.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:49 smithi099 ceph-mon[167076]: pgmap v420: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.2 GiB data, 8.6 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 527 op/s; 11612/109590 objects degraded (10.596%); 29705/109590 objects misplaced (27.106%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T17:59:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1802822673' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9530624432668539}]: dispatch 2023-09-23T17:59:50.498 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 17:59:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:17:59:50] "GET /metrics HTTP/1.1" 200 36576 "" "Prometheus/2.43.0" 2023-09-23T17:59:51.144 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:50 smithi055 ceph-mon[184174]: failed to encode map e186 with expected crc 2023-09-23T17:59:51.144 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:50 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1802822673' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9530624432668539}]': finished 2023-09-23T17:59:51.144 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:50 smithi055 ceph-mon[184174]: osdmap e186: 8 total, 6 up, 7 in 2023-09-23T17:59:51.144 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:50 smithi055 ceph-mon[184174]: failed to encode map e186 with expected crc 2023-09-23T17:59:51.144 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:50 smithi055 ceph-mon[179668]: failed to encode map e186 with expected crc 2023-09-23T17:59:51.145 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:50 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1802822673' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9530624432668539}]': finished 2023-09-23T17:59:51.145 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:50 smithi055 ceph-mon[179668]: osdmap e186: 8 total, 6 up, 7 in 2023-09-23T17:59:51.145 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:50 smithi055 ceph-mon[179668]: failed to encode map e186 with expected crc 2023-09-23T17:59:51.259 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T17:59:51.259 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T17:59:51.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:50 smithi099 ceph-mon[167076]: failed to encode map e186 with expected crc 2023-09-23T17:59:51.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:50 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1802822673' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9530624432668539}]': finished 2023-09-23T17:59:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:50 smithi099 ceph-mon[167076]: osdmap e186: 8 total, 6 up, 7 in 2023-09-23T17:59:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:50 smithi099 ceph-mon[167076]: failed to encode map e186 with expected crc 2023-09-23T17:59:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:51 smithi055 ceph-mon[179668]: pgmap v422: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 10864/104136 objects degraded (10.433%); 28749/104136 objects misplaced (27.607%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T17:59:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:51 smithi055 ceph-mon[179668]: failed to encode map e187 with expected crc 2023-09-23T17:59:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:51 smithi055 ceph-mon[179668]: osdmap e187: 8 total, 6 up, 7 in 2023-09-23T17:59:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:52.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:51 smithi055 ceph-mon[184174]: pgmap v422: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 10864/104136 objects degraded (10.433%); 28749/104136 objects misplaced (27.607%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T17:59:52.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:51 smithi055 ceph-mon[184174]: failed to encode map e187 with expected crc 2023-09-23T17:59:52.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:51 smithi055 ceph-mon[184174]: osdmap e187: 8 total, 6 up, 7 in 2023-09-23T17:59:52.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:52.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:51 smithi099 ceph-mon[167076]: pgmap v422: 109 pgs: 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 643 op/s; 10864/104136 objects degraded (10.433%); 28749/104136 objects misplaced (27.607%); 3.7 MiB/s, 58 objects/s recovering 2023-09-23T17:59:52.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:51 smithi099 ceph-mon[167076]: failed to encode map e187 with expected crc 2023-09-23T17:59:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:51 smithi099 ceph-mon[167076]: osdmap e187: 8 total, 6 up, 7 in 2023-09-23T17:59:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T17:59:52.931 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:52.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:52.945 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:53.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:52 smithi055 ceph-mon[179668]: failed to encode map e188 with expected crc 2023-09-23T17:59:53.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:52 smithi055 ceph-mon[179668]: osdmap e188: 8 total, 6 up, 7 in 2023-09-23T17:59:53.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:52 smithi055 ceph-mon[179668]: 5.e continuing backfill to osd.2 from (144'6563,185'11062] 5:70140cb8:::benchmark_data_smithi055_229117_object62703:head to 185'11062 2023-09-23T17:59:53.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:52 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 10864/104136 objects degraded (10.433%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:53.021 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:52 smithi055 ceph-mon[184174]: failed to encode map e188 with expected crc 2023-09-23T17:59:53.021 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:52 smithi055 ceph-mon[184174]: osdmap e188: 8 total, 6 up, 7 in 2023-09-23T17:59:53.021 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:52 smithi055 ceph-mon[184174]: 5.e continuing backfill to osd.2 from (144'6563,185'11062] 5:70140cb8:::benchmark_data_smithi055_229117_object62703:head to 185'11062 2023-09-23T17:59:53.021 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:52 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 10864/104136 objects degraded (10.433%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:53.046 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:53.135 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:53.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:52 smithi099 ceph-mon[167076]: failed to encode map e188 with expected crc 2023-09-23T17:59:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:52 smithi099 ceph-mon[167076]: osdmap e188: 8 total, 6 up, 7 in 2023-09-23T17:59:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:52 smithi099 ceph-mon[167076]: 5.e continuing backfill to osd.2 from (144'6563,185'11062] 5:70140cb8:::benchmark_data_smithi055_229117_object62703:head to 185'11062 2023-09-23T17:59:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:52 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 10864/104136 objects degraded (10.433%), 16 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:53.593 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:53.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:53.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:53.640 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T17:59:53.698 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:53.794 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:53.956 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:53.963 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:53.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:53.986 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:54.087 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:54.168 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:54.319 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:53 smithi055 ceph-mon[179668]: pgmap v425: 109 pgs: 1 peering, 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 10726/101571 objects degraded (10.560%); 28221/101571 objects misplaced (27.785%); 3.8 MiB/s, 87 objects/s recovering 2023-09-23T17:59:54.320 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:53 smithi055 ceph-mon[184174]: pgmap v425: 109 pgs: 1 peering, 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 10726/101571 objects degraded (10.560%); 28221/101571 objects misplaced (27.785%); 3.8 MiB/s, 87 objects/s recovering 2023-09-23T17:59:54.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:53 smithi099 ceph-mon[167076]: pgmap v425: 109 pgs: 1 peering, 1 down, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 9 active+undersized+degraded; 2.1 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 541 op/s; 10726/101571 objects degraded (10.560%); 28221/101571 objects misplaced (27.785%); 3.8 MiB/s, 87 objects/s recovering 2023-09-23T17:59:54.334 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:54.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:54.345 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:54.445 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:54.542 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:54.620 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:54.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:54.629 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:54.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:54.778 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:54.825 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:54.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:54.834 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T17:59:54.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:54.976 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:54.984 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T17:59:55.009 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:55.021 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:55.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:55.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2966526919' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:54 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:55.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T17:59:55.171 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:55.237 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T17:59:55.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:55.243 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T17:59:55.344 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:55.369 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2966526919' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:55.369 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:54 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:55.369 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2966526919' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:55.370 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:54 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T17:59:55.418 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:55.948 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:55.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:55.954 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:56.055 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:56.111 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:56.182 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:56.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:56.188 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:56.288 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:56.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:55 smithi099 ceph-mon[167076]: pgmap v426: 109 pgs: 1 down, 1 peering, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 9695/97191 objects degraded (9.975%); 27334/97191 objects misplaced (28.124%); 8.7 MiB/s, 166 objects/s recovering 2023-09-23T17:59:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:55 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:59:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:55 smithi099 ceph-mon[167076]: failed to encode map e189 with expected crc 2023-09-23T17:59:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:55 smithi099 ceph-mon[167076]: failed to encode map e189 with expected crc 2023-09-23T17:59:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:55 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:55 smithi099 ceph-mon[167076]: osdmap e189: 8 total, 6 up, 7 in 2023-09-23T17:59:56.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[184174]: pgmap v426: 109 pgs: 1 down, 1 peering, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 9695/97191 objects degraded (9.975%); 27334/97191 objects misplaced (28.124%); 8.7 MiB/s, 166 objects/s recovering 2023-09-23T17:59:56.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:59:56.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[184174]: failed to encode map e189 with expected crc 2023-09-23T17:59:56.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[184174]: failed to encode map e189 with expected crc 2023-09-23T17:59:56.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:56.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[184174]: osdmap e189: 8 total, 6 up, 7 in 2023-09-23T17:59:56.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[179668]: pgmap v426: 109 pgs: 1 down, 1 peering, 15 active+clean+remapped, 45 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 2.0 GiB data, 8.1 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 688 op/s; 9695/97191 objects degraded (9.975%); 27334/97191 objects misplaced (28.124%); 8.7 MiB/s, 166 objects/s recovering 2023-09-23T17:59:56.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T17:59:56.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[179668]: failed to encode map e189 with expected crc 2023-09-23T17:59:56.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[179668]: failed to encode map e189 with expected crc 2023-09-23T17:59:56.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T17:59:56.343 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:55 smithi055 ceph-mon[179668]: osdmap e189: 8 total, 6 up, 7 in 2023-09-23T17:59:56.345 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:57.151 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:57.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:57.156 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:57.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T17:59:57.312 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:57.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T17:59:57.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:57.367 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T17:59:57.468 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:57.541 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:57.779 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:57.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:57.785 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:57.885 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T17:59:57.913 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T17:59:57.956 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:57.970 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T17:59:57.990 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T17:59:58.001 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T17:59:58.019 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T17:59:58.019 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T17:59:58.019 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T17:59:58.019 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T17:59:58.019 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T17:59:58.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T17:59:58.050 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:57 smithi055 ceph-mon[179668]: pgmap v428: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 9695/95640 objects degraded (10.137%); 26992/95640 objects misplaced (28.223%); 7.3 MiB/s, 144 objects/s recovering 2023-09-23T17:59:58.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:59:58.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:57 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 9695/95640 objects degraded (10.137%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:58.051 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3882827373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:58.051 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:57 smithi055 ceph-mon[184174]: pgmap v428: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 9695/95640 objects degraded (10.137%); 26992/95640 objects misplaced (28.223%); 7.3 MiB/s, 144 objects/s recovering 2023-09-23T17:59:58.051 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:59:58.051 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:57 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 9695/95640 objects degraded (10.137%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:58.051 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3882827373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:58.066 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T17:59:58.109 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T17:59:58.143 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T17:59:58.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:58.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T17:59:58.183 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T17:59:58.184 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T17:59:58.184 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T17:59:58.184 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T17:59:58.184 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T17:59:58.185 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T17:59:58.185 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T17:59:58.185 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T17:59:58.185 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T17:59:58.255 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:58.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:57 smithi099 ceph-mon[167076]: pgmap v428: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.9 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 471 op/s; 9695/95640 objects degraded (10.137%); 26992/95640 objects misplaced (28.223%); 7.3 MiB/s, 144 objects/s recovering 2023-09-23T17:59:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T17:59:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:57 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 9695/95640 objects degraded (10.137%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T17:59:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3882827373' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T17:59:58.375 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:58.697 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:58.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:58.707 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:58.774 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T17:59:58.808 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:58.841 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T17:59:58.842 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T17:59:58.906 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:59.061 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T17:59:59.074 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:59.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:59.085 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:59.186 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T17:59:59.288 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:59.428 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T17:59:59.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:59.437 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T17:59:59.483 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T17:59:59.483 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T17:59:59.483 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T17:59:59.484 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T17:59:59.484 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T17:59:59.538 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T17:59:59.641 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:59.720 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T17:59:59.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:59.726 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T17:59:59.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T17:59:59.863 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T17:59:59.962 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T17:59:59.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T17:59:59.966 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:00.067 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:00.145 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 17:59:59 smithi099 ceph-mon[167076]: pgmap v429: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 9687/93213 objects degraded (10.392%); 26390/93213 objects misplaced (28.312%); 5.7 MiB/s, 135 objects/s recovering 2023-09-23T18:00:00.155 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:00.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:00.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:00.238 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:00.249 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:00:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "" 2023-09-23T18:00:00.250 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:00:00.339 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:00.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 17:59:59 smithi055 ceph-mon[184174]: pgmap v429: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 9687/93213 objects degraded (10.392%); 26390/93213 objects misplaced (28.312%); 5.7 MiB/s, 135 objects/s recovering 2023-09-23T18:00:00.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 17:59:59 smithi055 ceph-mon[179668]: pgmap v429: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.9 GiB data, 7.9 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 473 op/s; 9687/93213 objects degraded (10.392%); 26390/93213 objects misplaced (28.312%); 5.7 MiB/s, 135 objects/s recovering 2023-09-23T18:00:00.391 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:00.685 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:00:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:00:00] "GET /metrics HTTP/1.1" 200 36569 "" "Prometheus/2.43.0" 2023-09-23T18:00:00.843 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:00:01.164 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:01.176 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: Health detail: HEALTH_WARN 2 osds down; Reduced data availability: 4 pgs inactive, 1 pg down; Degraded data redundancy: 9687/93213 objects degraded (10.392%), 15 pgs degraded, 29 pgs undersized 2023-09-23T18:00:01.176 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: [WRN] OSD_DOWN: 2 osds down 2023-09-23T18:00:01.176 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: osd.3 (root=default,host=smithi055) is down 2023-09-23T18:00:01.176 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: osd.7 (root=default,host=smithi099) is down 2023-09-23T18:00:01.177 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: [WRN] PG_AVAILABILITY: Reduced data availability: 4 pgs inactive, 1 pg down 2023-09-23T18:00:01.177 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.7 is stuck inactive for 2m, current state undersized+degraded+peered, last acting [0] 2023-09-23T18:00:01.177 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.8 is stuck inactive for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.177 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.11 is down, acting [6] 2023-09-23T18:00:01.177 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.14 is stuck inactive for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.177 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: [WRN] PG_DEGRADED: Degraded data redundancy: 9687/93213 objects degraded (10.392%), 15 pgs degraded, 29 pgs undersized 2023-09-23T18:00:01.177 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 2.0 is stuck undersized for 2m, current state active+undersized, last acting [1,0] 2023-09-23T18:00:01.177 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 2.1 is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,0] 2023-09-23T18:00:01.177 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 2.4 is stuck undersized for 2m, current state active+undersized+degraded, last acting [1,0] 2023-09-23T18:00:01.177 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 2.5 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 2.7 is stuck undersized for 83s, current state active+undersized, last acting [6,2] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.1 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.7 is stuck undersized for 2m, current state undersized+degraded+peered, last acting [0] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.8 is stuck undersized for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.9 is stuck undersized for 2m, current state active+undersized, last acting [4,2] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.b is stuck undersized for 2m, current state active+undersized+degraded, last acting [0,4] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.e is active+undersized+degraded, acting [4,1] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.f is stuck undersized for 2m, current state active+undersized+degraded, last acting [4,0] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.12 is stuck undersized for 106s, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.13 is active+undersized+degraded, acting [4,2] 2023-09-23T18:00:01.178 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.15 is active+undersized+degraded, acting [4,0] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.18 is stuck undersized for 2m, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.19 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.1b is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 3.1e is stuck undersized for 83s, current state active+undersized, last acting [6,2] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.0 is stuck undersized for 106s, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.f is stuck undersized for 2m, current state active+undersized, last acting [2,4] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.10 is stuck undersized for 83s, current state active+undersized, last acting [1,6] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.14 is stuck undersized for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.16 is stuck undersized for 106s, current state active+undersized, last acting [0,2] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.17 is stuck undersized for 106s, current state active+undersized, last acting [1,2] 2023-09-23T18:00:01.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.19 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2023-09-23T18:00:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.1c is stuck undersized for 2m, current state active+undersized, last acting [2,1] 2023-09-23T18:00:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.1d is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2023-09-23T18:00:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 4.1e is stuck undersized for 106s, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 5.1 is stuck undersized for 94s, current state active+undersized+degraded, last acting [4,0] 2023-09-23T18:00:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 5.2 is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2023-09-23T18:00:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 5.3 is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2023-09-23T18:00:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 5.6 is active+recovering+undersized+degraded+remapped, acting [2,5] 2023-09-23T18:00:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 5.a is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,4] 2023-09-23T18:00:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 5.d is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2023-09-23T18:00:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[179668]: pg 5.f is active+recovering+undersized+degraded+remapped, acting [4,2] 2023-09-23T18:00:01.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:01.181 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: Health detail: HEALTH_WARN 2 osds down; Reduced data availability: 4 pgs inactive, 1 pg down; Degraded data redundancy: 9687/93213 objects degraded (10.392%), 15 pgs degraded, 29 pgs undersized 2023-09-23T18:00:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: [WRN] OSD_DOWN: 2 osds down 2023-09-23T18:00:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: osd.3 (root=default,host=smithi055) is down 2023-09-23T18:00:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: osd.7 (root=default,host=smithi099) is down 2023-09-23T18:00:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: [WRN] PG_AVAILABILITY: Reduced data availability: 4 pgs inactive, 1 pg down 2023-09-23T18:00:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.7 is stuck inactive for 2m, current state undersized+degraded+peered, last acting [0] 2023-09-23T18:00:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.8 is stuck inactive for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.11 is down, acting [6] 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.14 is stuck inactive for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: [WRN] PG_DEGRADED: Degraded data redundancy: 9687/93213 objects degraded (10.392%), 15 pgs degraded, 29 pgs undersized 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 2.0 is stuck undersized for 2m, current state active+undersized, last acting [1,0] 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 2.1 is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,0] 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 2.4 is stuck undersized for 2m, current state active+undersized+degraded, last acting [1,0] 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 2.5 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 2.7 is stuck undersized for 83s, current state active+undersized, last acting [6,2] 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.1 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.7 is stuck undersized for 2m, current state undersized+degraded+peered, last acting [0] 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.8 is stuck undersized for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.9 is stuck undersized for 2m, current state active+undersized, last acting [4,2] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.b is stuck undersized for 2m, current state active+undersized+degraded, last acting [0,4] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.e is active+undersized+degraded, acting [4,1] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.f is stuck undersized for 2m, current state active+undersized+degraded, last acting [4,0] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.12 is stuck undersized for 106s, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.13 is active+undersized+degraded, acting [4,2] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.15 is active+undersized+degraded, acting [4,0] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.18 is stuck undersized for 2m, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.19 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.1b is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 3.1e is stuck undersized for 83s, current state active+undersized, last acting [6,2] 2023-09-23T18:00:01.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.0 is stuck undersized for 106s, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.f is stuck undersized for 2m, current state active+undersized, last acting [2,4] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.10 is stuck undersized for 83s, current state active+undersized, last acting [1,6] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.14 is stuck undersized for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.16 is stuck undersized for 106s, current state active+undersized, last acting [0,2] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.17 is stuck undersized for 106s, current state active+undersized, last acting [1,2] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.19 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.1c is stuck undersized for 2m, current state active+undersized, last acting [2,1] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.1d is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 4.1e is stuck undersized for 106s, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 5.1 is stuck undersized for 94s, current state active+undersized+degraded, last acting [4,0] 2023-09-23T18:00:01.186 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 5.2 is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2023-09-23T18:00:01.186 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 5.3 is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2023-09-23T18:00:01.186 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 5.6 is active+recovering+undersized+degraded+remapped, acting [2,5] 2023-09-23T18:00:01.186 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 5.a is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,4] 2023-09-23T18:00:01.186 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 5.d is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2023-09-23T18:00:01.186 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:01 smithi055 ceph-mon[184174]: pg 5.f is active+recovering+undersized+degraded+remapped, acting [4,2] 2023-09-23T18:00:01.221 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:00:01.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: Health detail: HEALTH_WARN 2 osds down; Reduced data availability: 4 pgs inactive, 1 pg down; Degraded data redundancy: 9687/93213 objects degraded (10.392%), 15 pgs degraded, 29 pgs undersized 2023-09-23T18:00:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: [WRN] OSD_DOWN: 2 osds down 2023-09-23T18:00:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: osd.3 (root=default,host=smithi055) is down 2023-09-23T18:00:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: osd.7 (root=default,host=smithi099) is down 2023-09-23T18:00:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: [WRN] PG_AVAILABILITY: Reduced data availability: 4 pgs inactive, 1 pg down 2023-09-23T18:00:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.7 is stuck inactive for 2m, current state undersized+degraded+peered, last acting [0] 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.8 is stuck inactive for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.11 is down, acting [6] 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.14 is stuck inactive for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: [WRN] PG_DEGRADED: Degraded data redundancy: 9687/93213 objects degraded (10.392%), 15 pgs degraded, 29 pgs undersized 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 2.0 is stuck undersized for 2m, current state active+undersized, last acting [1,0] 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 2.1 is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,0] 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 2.4 is stuck undersized for 2m, current state active+undersized+degraded, last acting [1,0] 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 2.5 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 2.7 is stuck undersized for 83s, current state active+undersized, last acting [6,2] 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.1 is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2023-09-23T18:00:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.7 is stuck undersized for 2m, current state undersized+degraded+peered, last acting [0] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.8 is stuck undersized for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.9 is stuck undersized for 2m, current state active+undersized, last acting [4,2] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.b is stuck undersized for 2m, current state active+undersized+degraded, last acting [0,4] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.e is active+undersized+degraded, acting [4,1] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.f is stuck undersized for 2m, current state active+undersized+degraded, last acting [4,0] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.12 is stuck undersized for 106s, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.13 is active+undersized+degraded, acting [4,2] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.15 is active+undersized+degraded, acting [4,0] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.18 is stuck undersized for 2m, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.19 is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2023-09-23T18:00:01.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.1b is stuck undersized for 2m, current state active+undersized, last acting [0,4] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 3.1e is stuck undersized for 83s, current state active+undersized, last acting [6,2] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.0 is stuck undersized for 106s, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.4 is stuck undersized for 2m, current state active+undersized, last acting [1,2] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.f is stuck undersized for 2m, current state active+undersized, last acting [2,4] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.10 is stuck undersized for 83s, current state active+undersized, last acting [1,6] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.14 is stuck undersized for 2m, current state undersized+peered, last acting [1] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.16 is stuck undersized for 106s, current state active+undersized, last acting [0,2] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.17 is stuck undersized for 106s, current state active+undersized, last acting [1,2] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.19 is stuck undersized for 2m, current state active+undersized, last acting [2,0] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.1c is stuck undersized for 2m, current state active+undersized, last acting [2,1] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.1d is stuck undersized for 2m, current state active+undersized, last acting [1,4] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 4.1e is stuck undersized for 106s, current state active+undersized, last acting [0,1] 2023-09-23T18:00:01.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 5.1 is stuck undersized for 94s, current state active+undersized+degraded, last acting [4,0] 2023-09-23T18:00:01.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 5.2 is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2023-09-23T18:00:01.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 5.3 is active+undersized+degraded+remapped+backfill_wait, acting [0,5] 2023-09-23T18:00:01.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 5.6 is active+recovering+undersized+degraded+remapped, acting [2,5] 2023-09-23T18:00:01.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 5.a is stuck undersized for 2m, current state active+undersized+degraded, last acting [2,4] 2023-09-23T18:00:01.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 5.d is active+recovery_wait+undersized+degraded+remapped, acting [2,5] 2023-09-23T18:00:01.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:01 smithi099 ceph-mon[167076]: pg 5.f is active+recovering+undersized+degraded+remapped, acting [4,2] 2023-09-23T18:00:01.345 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:01.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:01.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:01.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:01.503 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:01.599 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:01.672 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:01.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:01.678 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:01.779 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:01.849 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:02.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:02 smithi055 ceph-mon[179668]: pgmap v430: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 8908/87891 objects degraded (10.135%); 25536/87891 objects misplaced (29.054%); 6.0 MiB/s, 133 objects/s recovering 2023-09-23T18:00:02.187 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:02 smithi055 ceph-mon[184174]: pgmap v430: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 8908/87891 objects degraded (10.135%); 25536/87891 objects misplaced (29.054%); 6.0 MiB/s, 133 objects/s recovering 2023-09-23T18:00:02.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:02 smithi099 ceph-mon[167076]: pgmap v430: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.8 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 564 op/s; 8908/87891 objects degraded (10.135%); 25536/87891 objects misplaced (29.054%); 6.0 MiB/s, 133 objects/s recovering 2023-09-23T18:00:02.857 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:02.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:02.879 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:02.980 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:03.075 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 8908/87891 objects degraded (10.135%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:03.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 8908/87891 objects degraded (10.135%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:03.204 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:03.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:03.212 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:03.212 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:03.260 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:03.313 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:03.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 8908/87891 objects degraded (10.135%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:03.345 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:03.406 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:03.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:03.413 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:03.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:03.549 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:03.591 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:03.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:03.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:03.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:03.743 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:03.794 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:03.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:03.799 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:03.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:03.929 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:03.972 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:03.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:03.975 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:04.058 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:04 smithi055 ceph-mon[179668]: pgmap v431: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8913/84255 objects degraded (10.579%); 24713/84255 objects misplaced (29.331%); 5.1 MiB/s, 114 objects/s recovering 2023-09-23T18:00:04.058 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:04 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:04.058 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:04 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:04.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:04.110 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:04.160 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:04.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:04.165 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:04.266 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:04.313 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:04 smithi055 ceph-mon[184174]: pgmap v431: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8913/84255 objects degraded (10.579%); 24713/84255 objects misplaced (29.331%); 5.1 MiB/s, 114 objects/s recovering 2023-09-23T18:00:04.313 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:04 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:04.313 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:04 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:04.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:04 smithi099 ceph-mon[167076]: pgmap v431: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.7 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 576 op/s; 8913/84255 objects degraded (10.579%); 24713/84255 objects misplaced (29.331%); 5.1 MiB/s, 114 objects/s recovering 2023-09-23T18:00:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:04 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:04 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:04.401 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:04.486 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:04.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:04.490 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:04.591 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:04.672 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:04.815 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:04.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:04.821 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:04.921 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:04.958 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:04.998 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:05.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:05.003 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:05.103 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:00:05.104 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:05.185 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:05.266 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1520035670' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:00:05.267 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1520035670' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:00:05.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1520035670' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:00:05.383 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:00:05.429 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:00:05.462 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:00:05.492 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:00:05.531 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:00:05.567 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:00:05.583 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:00:05.583 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:00:05.583 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:00:05.583 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:00:05.583 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:00:05.583 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:00:05.583 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:00:05.584 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:00:05.693 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T18:00:05.693 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:00:05.693 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T18:00:05.693 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:00:05.781 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:00:05.781 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:00:05.795 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:05.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:05.802 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:05.903 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:06.048 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:06.118 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:06.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:06.123 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[179668]: pgmap v432: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 8138/78255 objects degraded (10.399%); 23582/78255 objects misplaced (30.135%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[179668]: failed to encode map e190 with expected crc 2023-09-23T18:00:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[179668]: failed to encode map e190 with expected crc 2023-09-23T18:00:06.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1520035670' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:00:06.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[179668]: osdmap e190: 8 total, 6 up, 7 in 2023-09-23T18:00:06.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1505414361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:06.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[184174]: pgmap v432: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 8138/78255 objects degraded (10.399%); 23582/78255 objects misplaced (30.135%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:06.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:06.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[184174]: failed to encode map e190 with expected crc 2023-09-23T18:00:06.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[184174]: failed to encode map e190 with expected crc 2023-09-23T18:00:06.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1520035670' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:00:06.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[184174]: osdmap e190: 8 total, 6 up, 7 in 2023-09-23T18:00:06.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:06 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1505414361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:06.224 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:06.238 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:00:06.239 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:00:06.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:06 smithi099 ceph-mon[167076]: pgmap v432: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 630 op/s; 8138/78255 objects degraded (10.399%); 23582/78255 objects misplaced (30.135%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:06 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:06 smithi099 ceph-mon[167076]: failed to encode map e190 with expected crc 2023-09-23T18:00:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:06 smithi099 ceph-mon[167076]: failed to encode map e190 with expected crc 2023-09-23T18:00:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:06 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1520035670' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:00:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:06 smithi099 ceph-mon[167076]: osdmap e190: 8 total, 6 up, 7 in 2023-09-23T18:00:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:06 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1505414361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:06.341 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:06.411 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:06.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:06.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:06.524 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:06.562 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:06.600 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:06.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:06.606 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:06.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:00:06.707 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:06.794 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:06.853 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:06.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:06.857 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:06.899 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:06.959 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:06.996 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:07.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:07.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:07.061 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:07.161 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:07.188 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:07.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:07.342 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:00:07.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:07.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:07.698 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:07.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:07.703 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:07.804 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:07.846 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:07.924 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:00:08.086 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:08.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:08.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:08.192 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:08.240 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:00:08.289 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:08 smithi055 ceph-mon[179668]: pgmap v434: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 8138/76449 objects degraded (10.645%); 23168/76449 objects misplaced (30.305%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:08.290 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 8138/76449 objects degraded (10.645%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:08.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:08 smithi055 ceph-mon[184174]: pgmap v434: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 8138/76449 objects degraded (10.645%); 23168/76449 objects misplaced (30.305%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:08.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 8138/76449 objects degraded (10.645%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:08.314 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:08.427 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:08 smithi099 ceph-mon[167076]: pgmap v434: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.6 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 639 op/s; 8138/76449 objects degraded (10.645%); 23168/76449 objects misplaced (30.305%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:08.428 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 8138/76449 objects degraded (10.645%), 15 pgs degraded, 29 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:08.761 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:08.763 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:00:08.763 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:00:08.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:08.767 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:08.868 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:08.912 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:09.107 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:09.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:09.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:09.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:00:09.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:09.278 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:09.328 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:09.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:09.331 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:09.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:09.479 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:09.526 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:09.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:09.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:09.632 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:09.718 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:09.737 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:09.761 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:09.805 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:09.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:09.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:09.907 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:00:09.912 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:09.981 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:10.181 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:10.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:10.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:10.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:10.330 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:10.374 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:10 smithi055 ceph-mon[179668]: pgmap v435: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.5 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 8163/74148 objects degraded (11.009%); 22585/74148 objects misplaced (30.459%); 1.7 MiB/s, 58 objects/s recovering 2023-09-23T18:00:10.375 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:00:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:00:10] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-09-23T18:00:10.375 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:10 smithi055 ceph-mon[184174]: pgmap v435: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.5 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 8163/74148 objects degraded (11.009%); 22585/74148 objects misplaced (30.459%); 1.7 MiB/s, 58 objects/s recovering 2023-09-23T18:00:10.448 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:10 smithi099 ceph-mon[167076]: pgmap v435: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.5 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 635 op/s; 8163/74148 objects degraded (11.009%); 22585/74148 objects misplaced (30.459%); 1.7 MiB/s, 58 objects/s recovering 2023-09-23T18:00:10.575 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:10.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:10.580 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:10.680 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:10.789 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:10.887 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:10.889 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:10.890 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:10.991 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:11.162 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:11.279 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:00:11.279 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:00:11.440 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:11.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:11.452 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:11.553 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:11.601 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:11.607 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:00:11.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:11.652 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:00:11.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:11.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:11.664 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:00:11.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[184174]: pgmap v436: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.4 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 7418/69009 objects degraded (10.749%); 21794/69009 objects misplaced (31.581%); 1.6 MiB/s, 57 objects/s recovering 2023-09-23T18:00:11.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:11.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:11.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:00:11.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:00:11.701 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:11.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[179668]: pgmap v436: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.4 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 7418/69009 objects degraded (10.749%); 21794/69009 objects misplaced (31.581%); 1.6 MiB/s, 57 objects/s recovering 2023-09-23T18:00:11.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:11.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:11.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:00:11.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:00:11.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:11.757 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:11.814 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:11.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:11.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:11.882 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:11.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:11 smithi099 ceph-mon[167076]: pgmap v436: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.4 GiB data, 6.8 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 628 op/s; 7418/69009 objects degraded (10.749%); 21794/69009 objects misplaced (31.581%); 1.6 MiB/s, 57 objects/s recovering 2023-09-23T18:00:11.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:11.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:11.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:00:11.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:00:11.884 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:11.983 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:12.046 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:12.335 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:00:12.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:12 smithi099 ceph-mon[167076]: from='client.35163 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:00:12.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:12 smithi055 ceph-mon[179668]: from='client.35163 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:00:12.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:12 smithi055 ceph-mon[184174]: from='client.35163 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:00:13.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:13.077 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:13.077 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:13.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:13.250 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:13.322 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:13.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:13.327 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:13.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:13.486 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:13.540 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:13.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:13.543 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:13.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:13.671 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:13 smithi055 ceph-mon[179668]: pgmap v437: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.3 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 7421/65544 objects degraded (11.322%); 21038/65544 objects misplaced (32.098%); 1.6 MiB/s, 58 objects/s recovering 2023-09-23T18:00:13.671 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7421/65544 objects degraded (11.322%), 15 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:13.713 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:13 smithi099 ceph-mon[167076]: pgmap v437: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.3 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 7421/65544 objects degraded (11.322%); 21038/65544 objects misplaced (32.098%); 1.6 MiB/s, 58 objects/s recovering 2023-09-23T18:00:13.713 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7421/65544 objects degraded (11.322%), 15 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:13.723 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:13.819 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:00:13.819 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:00:13.825 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:13.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:13.828 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:13.929 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:13.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:13 smithi055 ceph-mon[184174]: pgmap v437: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.3 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 623 op/s; 7421/65544 objects degraded (11.322%); 21038/65544 objects misplaced (32.098%); 1.6 MiB/s, 58 objects/s recovering 2023-09-23T18:00:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7421/65544 objects degraded (11.322%), 15 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:14.019 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:14.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:00:14.351 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:14.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:14.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:14.464 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:14.583 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:14.587 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:00:14.642 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:00:14.698 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:00:14.745 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:00:14.782 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:14.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:14.788 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:14.788 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:00:14.843 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:00:14.848 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2122867913' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:14.850 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":436,"stamp":"2023-09-23T18:00:10.587114+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1510414283,"num_objects":23003,"num_object_clones":0,"num_object_copies":69009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7418,"num_objects_misplaced":21794,"num_objects_unfound":0,"num_objects_dirty":23003,"num_whiteouts":0,"num_read":6439,"num_read_kb":13059,"num_write":189419,"num_write_kb":6775014,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38756,"num_bytes_recovered":2497448839,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":113819,"ondisk_log_size":113819,"up":261,"acting":278,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":350,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":7168384,"kb_used_data":6272584,"kb_used_omap":36,"kb_used_meta":895707,"kb_avail":649047680,"statfs":{"total":671965249536,"available":664624824320,"internally_reserved":0,"allocated":6423126016,"data_stored":6417362596,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37596,"internal_metadata":917204260},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-412483584,"num_objects":-6294,"num_object_clones":0,"num_object_copies":-18882,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1490,"num_objects_misplaced":-3742,"num_objects_unfound":0,"num_objects_dirty":-6294,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6294,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":577,"num_bytes_recovered":16973824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.008680"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":76,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.965545+0000","last_change":"2023-09-23T17:59:50.896420+0000","last_active":"2023-09-23T17:59:54.965545+0000","last_peered":"2023-09-23T17:59:54.965545+0000","last_clean":"2023-09-23T17:59:54.965545+0000","last_became_active":"2023-09-23T17:59:50.896017+0000","last_became_peered":"2023-09-23T17:59:50.896017+0000","last_unstale":"2023-09-23T17:59:54.965545+0000","last_undegraded":"2023-09-23T17:59:54.965545+0000","last_fullsized":"2023-09-23T17:59:54.965545+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:06.664670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":156,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:55.973674+0000","last_change":"2023-09-23T17:57:24.956156+0000","last_active":"2023-09-23T17:59:55.973674+0000","last_peered":"2023-09-23T17:59:55.973674+0000","last_clean":"2023-09-23T17:56:46.613857+0000","last_became_active":"2023-09-23T17:57:24.956156+0000","last_became_peered":"2023-09-23T17:57:24.956156+0000","last_unstale":"2023-09-23T17:59:55.973674+0000","last_undegraded":"2023-09-23T17:59:55.973674+0000","last_fullsized":"2023-09-23T17:57:24.947251+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:35.949179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":154,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.997516+0000","last_change":"2023-09-23T17:57:24.956353+0000","last_active":"2023-09-23T17:59:54.997516+0000","last_peered":"2023-09-23T17:59:54.997516+0000","last_clean":"2023-09-23T17:57:20.922940+0000","last_became_active":"2023-09-23T17:57:24.956353+0000","last_became_peered":"2023-09-23T17:57:24.956353+0000","last_unstale":"2023-09-23T17:59:54.997516+0000","last_undegraded":"2023-09-23T17:59:54.997516+0000","last_fullsized":"2023-09-23T17:57:24.946863+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:41.136418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":152,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.976817+0000","last_change":"2023-09-23T17:58:11.996431+0000","last_active":"2023-09-23T17:59:54.976817+0000","last_peered":"2023-09-23T17:59:54.976817+0000","last_clean":"2023-09-23T17:57:20.931176+0000","last_became_active":"2023-09-23T17:58:11.996431+0000","last_became_peered":"2023-09-23T17:58:11.996431+0000","last_unstale":"2023-09-23T17:59:54.976817+0000","last_undegraded":"2023-09-23T17:59:54.976817+0000","last_fullsized":"2023-09-23T17:58:11.991094+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:21.919913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":137,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.980657+0000","last_change":"2023-09-23T17:57:24.965072+0000","last_active":"2023-09-23T17:59:54.980657+0000","last_peered":"2023-09-23T17:59:54.980657+0000","last_clean":"2023-09-23T17:53:21.039477+0000","last_became_active":"2023-09-23T17:57:24.965072+0000","last_became_peered":"2023-09-23T17:57:24.965072+0000","last_unstale":"2023-09-23T17:59:54.980657+0000","last_undegraded":"2023-09-23T17:59:54.980657+0000","last_fullsized":"2023-09-23T17:57:24.947630+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:41.627468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":110,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.962680+0000","last_change":"2023-09-23T17:59:09.861723+0000","last_active":"2023-09-23T17:59:54.962680+0000","last_peered":"2023-09-23T17:59:54.962680+0000","last_clean":"2023-09-23T17:59:54.962680+0000","last_became_active":"2023-09-23T17:59:09.861324+0000","last_became_peered":"2023-09-23T17:59:09.861324+0000","last_unstale":"2023-09-23T17:59:54.962680+0000","last_undegraded":"2023-09-23T17:59:54.962680+0000","last_fullsized":"2023-09-23T17:59:54.962680+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:45.865590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":149,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.976140+0000","last_change":"2023-09-23T17:59:10.881778+0000","last_active":"2023-09-23T17:59:54.976140+0000","last_peered":"2023-09-23T17:59:54.976140+0000","last_clean":"2023-09-23T17:59:54.976140+0000","last_became_active":"2023-09-23T17:59:10.881369+0000","last_became_peered":"2023-09-23T17:59:10.881369+0000","last_unstale":"2023-09-23T17:59:54.976140+0000","last_undegraded":"2023-09-23T17:59:54.976140+0000","last_fullsized":"2023-09-23T17:59:54.976140+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:06.715447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":139,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.988423+0000","last_change":"2023-09-23T17:58:35.245269+0000","last_active":"2023-09-23T17:59:54.988423+0000","last_peered":"2023-09-23T17:59:54.988423+0000","last_clean":"2023-09-23T17:55:56.840611+0000","last_became_active":"2023-09-23T17:58:35.245269+0000","last_became_peered":"2023-09-23T17:58:35.245269+0000","last_unstale":"2023-09-23T17:59:54.988423+0000","last_undegraded":"2023-09-23T17:59:54.988423+0000","last_fullsized":"2023-09-23T17:58:35.227362+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:14:50.366063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":489,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.976295+0000","last_change":"2023-09-23T17:59:00.132664+0000","last_active":"2023-09-23T17:59:54.976295+0000","last_peered":"2023-09-23T17:59:54.976295+0000","last_clean":"2023-09-23T17:59:54.976295+0000","last_became_active":"2023-09-23T17:59:00.132396+0000","last_became_peered":"2023-09-23T17:59:00.132396+0000","last_unstale":"2023-09-23T17:59:54.976295+0000","last_undegraded":"2023-09-23T17:59:54.976295+0000","last_fullsized":"2023-09-23T17:59:54.976295+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":136,"reported_epoch":189,"state":"active+undersized+remapped","last_fresh":"2023-09-23T17:59:54.989344+0000","last_change":"2023-09-23T17:59:10.880170+0000","last_active":"2023-09-23T17:59:54.989344+0000","last_peered":"2023-09-23T17:59:54.989344+0000","last_clean":"2023-09-23T17:57:47.590780+0000","last_became_active":"2023-09-23T17:59:10.880170+0000","last_became_peered":"2023-09-23T17:59:10.880170+0000","last_unstale":"2023-09-23T17:59:54.989344+0000","last_undegraded":"2023-09-23T17:59:54.989344+0000","last_fullsized":"2023-09-23T17:59:10.878551+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:31.432584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":46,"reported_epoch":189,"state":"down","last_fresh":"2023-09-23T17:59:54.960844+0000","last_change":"2023-09-23T17:58:34.212200+0000","last_active":"2023-09-23T17:45:51.086592+0000","last_peered":"2023-09-23T17:45:51.086592+0000","last_clean":"2023-09-23T17:45:51.086592+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T17:59:54.960844+0000","last_undegraded":"2023-09-23T17:59:54.960844+0000","last_fullsized":"2023-09-23T17:59:54.960844+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:32.367332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":170,"reported_epoch":189,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:54.966507+0000","last_change":"2023-09-23T17:59:50.897099+0000","last_active":"2023-09-23T17:59:54.966507+0000","last_peered":"2023-09-23T17:59:54.966507+0000","last_clean":"2023-09-23T17:56:46.621999+0000","last_became_active":"2023-09-23T17:59:50.897099+0000","last_became_peered":"2023-09-23T17:59:50.897099+0000","last_unstale":"2023-09-23T17:59:54.966507+0000","last_undegraded":"2023-09-23T17:59:50.892775+0000","last_fullsized":"2023-09-23T17:59:50.892676+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:04.659893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":145,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.988044+0000","last_change":"2023-09-23T17:58:35.248074+0000","last_active":"2023-09-23T17:59:54.988044+0000","last_peered":"2023-09-23T17:59:54.988044+0000","last_clean":"2023-09-23T17:59:54.988044+0000","last_became_active":"2023-09-23T17:58:35.247751+0000","last_became_peered":"2023-09-23T17:58:35.247751+0000","last_unstale":"2023-09-23T17:59:54.988044+0000","last_undegraded":"2023-09-23T17:59:54.988044+0000","last_fullsized":"2023-09-23T17:59:54.988044+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:08.488838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":192,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.966754+0000","last_change":"2023-09-23T17:59:51.905994+0000","last_active":"2023-09-23T17:59:54.966754+0000","last_peered":"2023-09-23T17:59:54.966754+0000","last_clean":"2023-09-23T17:59:54.966754+0000","last_became_active":"2023-09-23T17:59:51.905695+0000","last_became_peered":"2023-09-23T17:59:51.905695+0000","last_unstale":"2023-09-23T17:59:54.966754+0000","last_undegraded":"2023-09-23T17:59:54.966754+0000","last_fullsized":"2023-09-23T17:59:54.966754+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:47.280585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":177,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.966876+0000","last_change":"2023-09-23T17:59:50.898040+0000","last_active":"2023-09-23T17:59:54.966876+0000","last_peered":"2023-09-23T17:59:54.966876+0000","last_clean":"2023-09-23T17:59:54.966876+0000","last_became_active":"2023-09-23T17:59:50.897702+0000","last_became_peered":"2023-09-23T17:59:50.897702+0000","last_unstale":"2023-09-23T17:59:54.966876+0000","last_undegraded":"2023-09-23T17:59:54.966876+0000","last_fullsized":"2023-09-23T17:59:54.966876+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:45.101467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":484,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.979343+0000","last_change":"2023-09-23T17:59:00.129648+0000","last_active":"2023-09-23T17:59:54.979343+0000","last_peered":"2023-09-23T17:59:54.979343+0000","last_clean":"2023-09-23T17:59:54.979343+0000","last_became_active":"2023-09-23T17:59:00.129253+0000","last_became_peered":"2023-09-23T17:59:00.129253+0000","last_unstale":"2023-09-23T17:59:54.979343+0000","last_undegraded":"2023-09-23T17:59:54.979343+0000","last_fullsized":"2023-09-23T17:59:54.979343+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":60,"reported_epoch":189,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:54.966471+0000","last_change":"2023-09-23T17:59:50.895581+0000","last_active":"2023-09-23T17:59:54.966471+0000","last_peered":"2023-09-23T17:59:54.966471+0000","last_clean":"2023-09-23T17:45:54.586105+0000","last_became_active":"2023-09-23T17:59:50.895581+0000","last_became_peered":"2023-09-23T17:59:50.895581+0000","last_unstale":"2023-09-23T17:59:54.966471+0000","last_undegraded":"2023-09-23T17:59:50.891295+0000","last_fullsized":"2023-09-23T17:59:50.891189+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:35.791975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":124,"reported_epoch":189,"state":"undersized+peered","last_fresh":"2023-09-23T17:59:54.989211+0000","last_change":"2023-09-23T17:57:53.869041+0000","last_active":"2023-09-23T17:57:52.875213+0000","last_peered":"2023-09-23T17:59:54.989211+0000","last_clean":"2023-09-23T17:53:21.028332+0000","last_became_active":"2023-09-23T17:57:24.954701+0000","last_became_peered":"2023-09-23T17:57:53.869041+0000","last_unstale":"2023-09-23T17:59:54.989211+0000","last_undegraded":"2023-09-23T17:59:54.989211+0000","last_fullsized":"2023-09-23T17:57:53.867092+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:44.473315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":485,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.979916+0000","last_change":"2023-09-23T17:59:00.141765+0000","last_active":"2023-09-23T17:59:54.979916+0000","last_peered":"2023-09-23T17:59:54.979916+0000","last_clean":"2023-09-23T17:59:54.979916+0000","last_became_active":"2023-09-23T17:59:00.137566+0000","last_became_peered":"2023-09-23T17:59:00.137566+0000","last_unstale":"2023-09-23T17:59:54.979916+0000","last_undegraded":"2023-09-23T17:59:54.979916+0000","last_fullsized":"2023-09-23T17:59:54.979916+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":150,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.974476+0000","last_change":"2023-09-23T17:58:11.995550+0000","last_active":"2023-09-23T17:59:54.974476+0000","last_peered":"2023-09-23T17:59:54.974476+0000","last_clean":"2023-09-23T17:57:20.931103+0000","last_became_active":"2023-09-23T17:58:11.995550+0000","last_became_peered":"2023-09-23T17:58:11.995550+0000","last_unstale":"2023-09-23T17:59:54.974476+0000","last_undegraded":"2023-09-23T17:59:54.974476+0000","last_fullsized":"2023-09-23T17:58:11.986762+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:31.912263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":23,"reported_epoch":189,"state":"active+undersized+remapped","last_fresh":"2023-09-23T17:59:54.961540+0000","last_change":"2023-09-23T17:59:10.875110+0000","last_active":"2023-09-23T17:59:54.961540+0000","last_peered":"2023-09-23T17:59:54.961540+0000","last_clean":"2023-09-23T17:57:20.925545+0000","last_became_active":"2023-09-23T17:59:10.875110+0000","last_became_peered":"2023-09-23T17:59:10.875110+0000","last_unstale":"2023-09-23T17:59:54.961540+0000","last_undegraded":"2023-09-23T17:59:54.961540+0000","last_fullsized":"2023-09-23T17:59:10.873146+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:35:28.623974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":485,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.978229+0000","last_change":"2023-09-23T17:59:00.134116+0000","last_active":"2023-09-23T17:59:54.978229+0000","last_peered":"2023-09-23T17:59:54.978229+0000","last_clean":"2023-09-23T17:59:54.978229+0000","last_became_active":"2023-09-23T17:59:00.133756+0000","last_became_peered":"2023-09-23T17:59:00.133756+0000","last_unstale":"2023-09-23T17:59:54.978229+0000","last_undegraded":"2023-09-23T17:59:54.978229+0000","last_fullsized":"2023-09-23T17:59:54.978229+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":177,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.966077+0000","last_change":"2023-09-23T17:58:36.251995+0000","last_active":"2023-09-23T17:59:54.966077+0000","last_peered":"2023-09-23T17:59:54.966077+0000","last_clean":"2023-09-23T17:59:54.966077+0000","last_became_active":"2023-09-23T17:58:36.251578+0000","last_became_peered":"2023-09-23T17:58:36.251578+0000","last_unstale":"2023-09-23T17:59:54.966077+0000","last_undegraded":"2023-09-23T17:59:54.966077+0000","last_fullsized":"2023-09-23T17:59:54.966077+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:45.817777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,0],"avail_no_missing":["4","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":151,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.976931+0000","last_change":"2023-09-23T17:58:11.996093+0000","last_active":"2023-09-23T17:59:54.976931+0000","last_peered":"2023-09-23T17:59:54.976931+0000","last_clean":"2023-09-23T17:57:20.931097+0000","last_became_active":"2023-09-23T17:58:11.996093+0000","last_became_peered":"2023-09-23T17:58:11.996093+0000","last_unstale":"2023-09-23T17:59:54.976931+0000","last_undegraded":"2023-09-23T17:59:54.976931+0000","last_fullsized":"2023-09-23T17:58:11.990799+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:58.128284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":485,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.975031+0000","last_change":"2023-09-23T17:59:00.133252+0000","last_active":"2023-09-23T17:59:54.975031+0000","last_peered":"2023-09-23T17:59:54.975031+0000","last_clean":"2023-09-23T17:59:54.975031+0000","last_became_active":"2023-09-23T17:59:00.132882+0000","last_became_peered":"2023-09-23T17:59:00.132882+0000","last_unstale":"2023-09-23T17:59:54.975031+0000","last_undegraded":"2023-09-23T17:59:54.975031+0000","last_fullsized":"2023-09-23T17:59:54.975031+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":116,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.960176+0000","last_change":"2023-09-23T17:59:10.878099+0000","last_active":"2023-09-23T17:59:54.960176+0000","last_peered":"2023-09-23T17:59:54.960176+0000","last_clean":"2023-09-23T17:59:54.960176+0000","last_became_active":"2023-09-23T17:59:10.877670+0000","last_became_peered":"2023-09-23T17:59:10.877670+0000","last_unstale":"2023-09-23T17:59:54.960176+0000","last_undegraded":"2023-09-23T17:59:54.960176+0000","last_fullsized":"2023-09-23T17:59:54.960176+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:36.244235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":130,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.990021+0000","last_change":"2023-09-23T17:58:11.993687+0000","last_active":"2023-09-23T17:59:54.990021+0000","last_peered":"2023-09-23T17:59:54.990021+0000","last_clean":"2023-09-23T17:53:21.029434+0000","last_became_active":"2023-09-23T17:58:11.993687+0000","last_became_peered":"2023-09-23T17:58:11.993687+0000","last_unstale":"2023-09-23T17:59:54.990021+0000","last_undegraded":"2023-09-23T17:59:54.990021+0000","last_fullsized":"2023-09-23T17:58:11.985535+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:33.153133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":485,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.977737+0000","last_change":"2023-09-23T17:59:00.134603+0000","last_active":"2023-09-23T17:59:54.977737+0000","last_peered":"2023-09-23T17:59:54.977737+0000","last_clean":"2023-09-23T17:59:54.977737+0000","last_became_active":"2023-09-23T17:59:00.134099+0000","last_became_peered":"2023-09-23T17:59:00.134099+0000","last_unstale":"2023-09-23T17:59:54.977737+0000","last_undegraded":"2023-09-23T17:59:54.977737+0000","last_fullsized":"2023-09-23T17:59:54.977737+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.9","version":"190'11462","reported_seq":29280,"reported_epoch":190,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:08.279815+0000","last_change":"2023-09-23T17:58:52.934142+0000","last_active":"2023-09-23T18:00:08.279815+0000","last_peered":"2023-09-23T18:00:08.279815+0000","last_clean":"2023-09-23T18:00:08.279815+0000","last_became_active":"2023-09-23T17:58:52.879112+0000","last_became_peered":"2023-09-23T17:58:52.879112+0000","last_unstale":"2023-09-23T18:00:08.279815+0000","last_undegraded":"2023-09-23T18:00:08.279815+0000","last_fullsized":"2023-09-23T18:00:08.279815+0000","mapping_epoch":168,"log_start":"138'6018","ondisk_log_start":"138'6018","created":110,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5444,"log_dups_size":0,"ondisk_log_size":5444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:54.486328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88997888,"num_objects":1358,"num_object_clones":0,"num_object_copies":4074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1358,"num_objects_unfound":0,"num_objects_dirty":1358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11462,"num_write_kb":410240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8835,"num_bytes_recovered":578945024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":1358}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":154,"reported_epoch":189,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:54.973441+0000","last_change":"2023-09-23T17:57:53.875345+0000","last_active":"2023-09-23T17:59:54.973441+0000","last_peered":"2023-09-23T17:59:54.973441+0000","last_clean":"2023-09-23T17:56:46.621793+0000","last_became_active":"2023-09-23T17:57:53.875345+0000","last_became_peered":"2023-09-23T17:57:53.875345+0000","last_unstale":"2023-09-23T17:59:54.973441+0000","last_undegraded":"2023-09-23T17:57:53.869204+0000","last_fullsized":"2023-09-23T17:57:53.869091+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:19.033883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":156,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.960208+0000","last_change":"2023-09-23T17:59:10.877550+0000","last_active":"2023-09-23T17:59:54.960208+0000","last_peered":"2023-09-23T17:59:54.960208+0000","last_clean":"2023-09-23T17:59:54.960208+0000","last_became_active":"2023-09-23T17:59:10.877173+0000","last_became_peered":"2023-09-23T17:59:10.877173+0000","last_unstale":"2023-09-23T17:59:54.960208+0000","last_undegraded":"2023-09-23T17:59:54.960208+0000","last_fullsized":"2023-09-23T17:59:54.960208+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:00.740053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,5],"avail_no_missing":["6","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":490,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.976035+0000","last_change":"2023-09-23T17:59:00.136455+0000","last_active":"2023-09-23T17:59:54.976035+0000","last_peered":"2023-09-23T17:59:54.976035+0000","last_clean":"2023-09-23T17:59:54.976035+0000","last_became_active":"2023-09-23T17:59:00.136126+0000","last_became_peered":"2023-09-23T17:59:00.136126+0000","last_unstale":"2023-09-23T17:59:54.976035+0000","last_undegraded":"2023-09-23T17:59:54.976035+0000","last_fullsized":"2023-09-23T17:59:54.976035+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.8","version":"190'11894","reported_seq":11984,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T18:00:08.939369+0000","last_change":"2023-09-23T17:55:53.818806+0000","last_active":"2023-09-23T18:00:08.939369+0000","last_peered":"2023-09-23T18:00:08.939369+0000","last_clean":"2023-09-23T18:00:08.939369+0000","last_became_active":"2023-09-23T17:55:53.818291+0000","last_became_peered":"2023-09-23T17:55:53.818291+0000","last_unstale":"2023-09-23T18:00:08.939369+0000","last_undegraded":"2023-09-23T18:00:08.939369+0000","last_fullsized":"2023-09-23T18:00:08.939369+0000","mapping_epoch":110,"log_start":"117'2200","ondisk_log_start":"117'2200","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9694,"log_dups_size":0,"ondisk_log_size":9694,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:58.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92012544,"num_objects":1404,"num_object_clones":0,"num_object_copies":4212,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1404,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11894,"num_write_kb":425536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":155,"reported_epoch":189,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:54.966766+0000","last_change":"2023-09-23T17:59:50.897959+0000","last_active":"2023-09-23T17:59:54.966766+0000","last_peered":"2023-09-23T17:59:54.966766+0000","last_clean":"2023-09-23T17:56:46.621682+0000","last_became_active":"2023-09-23T17:59:50.897959+0000","last_became_peered":"2023-09-23T17:59:50.897959+0000","last_unstale":"2023-09-23T17:59:54.966766+0000","last_undegraded":"2023-09-23T17:59:50.893310+0000","last_fullsized":"2023-09-23T17:59:50.893201+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:45.953223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":179,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.965963+0000","last_change":"2023-09-23T17:59:50.896274+0000","last_active":"2023-09-23T17:59:54.965963+0000","last_peered":"2023-09-23T17:59:54.965963+0000","last_clean":"2023-09-23T17:57:20.922147+0000","last_became_active":"2023-09-23T17:59:50.896274+0000","last_became_peered":"2023-09-23T17:59:50.896274+0000","last_unstale":"2023-09-23T17:59:54.965963+0000","last_undegraded":"2023-09-23T17:59:54.965963+0000","last_fullsized":"2023-09-23T17:59:50.890612+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:36.294543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":485,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.975855+0000","last_change":"2023-09-23T17:59:00.128843+0000","last_active":"2023-09-23T17:59:54.975855+0000","last_peered":"2023-09-23T17:59:54.975855+0000","last_clean":"2023-09-23T17:59:54.975855+0000","last_became_active":"2023-09-23T17:59:00.128472+0000","last_became_peered":"2023-09-23T17:59:00.128472+0000","last_unstale":"2023-09-23T17:59:54.975855+0000","last_undegraded":"2023-09-23T17:59:54.975855+0000","last_fullsized":"2023-09-23T17:59:54.975855+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.b","version":"190'11762","reported_seq":11862,"reported_epoch":190,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:09.122315+0000","last_change":"2023-09-23T17:59:10.964199+0000","last_active":"2023-09-23T18:00:09.122315+0000","last_peered":"2023-09-23T18:00:09.122315+0000","last_clean":"2023-09-23T18:00:09.122315+0000","last_became_active":"2023-09-23T17:59:10.899328+0000","last_became_peered":"2023-09-23T17:59:10.899328+0000","last_unstale":"2023-09-23T18:00:09.122315+0000","last_undegraded":"2023-09-23T18:00:09.122315+0000","last_fullsized":"2023-09-23T18:00:09.122315+0000","mapping_epoch":176,"log_start":"117'2000","ondisk_log_start":"117'2000","created":110,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9762,"log_dups_size":0,"ondisk_log_size":9762,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:47.317955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92274688,"num_objects":1408,"num_object_clones":0,"num_object_copies":4224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1408,"num_objects_unfound":0,"num_objects_dirty":1408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11762,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":1408}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":37,"reported_epoch":189,"state":"active+undersized+remapped","last_fresh":"2023-09-23T17:59:54.961843+0000","last_change":"2023-09-23T17:59:10.875436+0000","last_active":"2023-09-23T17:59:54.961843+0000","last_peered":"2023-09-23T17:59:54.961843+0000","last_clean":"2023-09-23T17:45:54.586241+0000","last_became_active":"2023-09-23T17:59:10.875436+0000","last_became_peered":"2023-09-23T17:59:10.875436+0000","last_unstale":"2023-09-23T17:59:54.961843+0000","last_undegraded":"2023-09-23T17:59:54.961843+0000","last_fullsized":"2023-09-23T17:59:10.873636+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:39.737287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":125,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.961214+0000","last_change":"2023-09-23T17:59:09.884256+0000","last_active":"2023-09-23T17:59:54.961214+0000","last_peered":"2023-09-23T17:59:54.961214+0000","last_clean":"2023-09-23T17:59:54.961214+0000","last_became_active":"2023-09-23T17:59:09.883903+0000","last_became_peered":"2023-09-23T17:59:09.883903+0000","last_unstale":"2023-09-23T17:59:54.961214+0000","last_undegraded":"2023-09-23T17:59:54.961214+0000","last_fullsized":"2023-09-23T17:59:54.961214+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:08.901791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":485,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.976588+0000","last_change":"2023-09-23T17:59:00.127991+0000","last_active":"2023-09-23T17:59:54.976588+0000","last_peered":"2023-09-23T17:59:54.976588+0000","last_clean":"2023-09-23T17:59:54.976588+0000","last_became_active":"2023-09-23T17:59:00.127676+0000","last_became_peered":"2023-09-23T17:59:00.127676+0000","last_unstale":"2023-09-23T17:59:54.976588+0000","last_undegraded":"2023-09-23T17:59:54.976588+0000","last_fullsized":"2023-09-23T17:59:54.976588+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.a","version":"190'11804","reported_seq":12718,"reported_epoch":190,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:09.085158+0000","last_change":"2023-09-23T17:57:24.992424+0000","last_active":"2023-09-23T18:00:09.085158+0000","last_peered":"2023-09-23T18:00:09.085158+0000","last_clean":"2023-09-23T17:57:23.931663+0000","last_became_active":"2023-09-23T17:57:24.964668+0000","last_became_peered":"2023-09-23T17:57:24.964668+0000","last_unstale":"2023-09-23T18:00:09.085158+0000","last_undegraded":"2023-09-23T17:57:24.952679+0000","last_fullsized":"2023-09-23T17:57:24.947063+0000","mapping_epoch":142,"log_start":"117'2100","ondisk_log_start":"117'2100","created":110,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9704,"log_dups_size":0,"ondisk_log_size":9704,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:59.839324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88211456,"num_objects":1346,"num_object_clones":0,"num_object_copies":4038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1346,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11804,"num_write_kb":420800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":396,"num_bytes_recovered":25952256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":1346}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":167,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.961184+0000","last_change":"2023-09-23T17:59:10.881020+0000","last_active":"2023-09-23T17:59:54.961184+0000","last_peered":"2023-09-23T17:59:54.961184+0000","last_clean":"2023-09-23T17:59:54.961184+0000","last_became_active":"2023-09-23T17:59:10.880602+0000","last_became_peered":"2023-09-23T17:59:10.880602+0000","last_unstale":"2023-09-23T17:59:54.961184+0000","last_undegraded":"2023-09-23T17:59:54.961184+0000","last_fullsized":"2023-09-23T17:59:54.961184+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:15.837348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,5],"avail_no_missing":["6","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":153,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T18:00:04.484993+0000","last_change":"2023-09-23T17:56:50.828739+0000","last_active":"2023-09-23T18:00:04.484993+0000","last_peered":"2023-09-23T18:00:04.484993+0000","last_clean":"2023-09-23T18:00:04.484993+0000","last_became_active":"2023-09-23T17:56:50.828229+0000","last_became_peered":"2023-09-23T17:56:50.828229+0000","last_unstale":"2023-09-23T18:00:04.484993+0000","last_undegraded":"2023-09-23T18:00:04.484993+0000","last_fullsized":"2023-09-23T18:00:04.484993+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:45.927953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":477,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T18:00:04.484862+0000","last_change":"2023-09-23T17:58:35.252704+0000","last_active":"2023-09-23T18:00:04.484862+0000","last_peered":"2023-09-23T18:00:04.484862+0000","last_clean":"2023-09-23T18:00:04.484862+0000","last_became_active":"2023-09-23T17:58:35.252386+0000","last_became_peered":"2023-09-23T17:58:35.252386+0000","last_unstale":"2023-09-23T18:00:04.484862+0000","last_undegraded":"2023-09-23T18:00:04.484862+0000","last_fullsized":"2023-09-23T18:00:04.484862+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"190'11747","reported_seq":11852,"reported_epoch":190,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:00:09.051846+0000","last_change":"2023-09-23T17:59:11.340557+0000","last_active":"2023-09-23T18:00:09.051846+0000","last_peered":"2023-09-23T18:00:09.051846+0000","last_clean":"2023-09-23T17:57:52.866355+0000","last_became_active":"2023-09-23T17:59:11.311441+0000","last_became_peered":"2023-09-23T17:59:11.311441+0000","last_unstale":"2023-09-23T18:00:09.051846+0000","last_undegraded":"2023-09-23T17:59:10.972976+0000","last_fullsized":"2023-09-23T17:59:10.890539+0000","mapping_epoch":176,"log_start":"117'2000","ondisk_log_start":"117'2000","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9747,"log_dups_size":0,"ondisk_log_size":9747,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:06.586919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90832920,"num_objects":1387,"num_object_clones":0,"num_object_copies":4161,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1387,"num_objects_misplaced":6566,"num_objects_unfound":0,"num_objects_dirty":1387,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11747,"num_write_kb":420225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-2130}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":76,"reported_epoch":189,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:54.978604+0000","last_change":"2023-09-23T17:57:24.957601+0000","last_active":"2023-09-23T17:59:54.978604+0000","last_peered":"2023-09-23T17:59:54.978604+0000","last_clean":"2023-09-23T17:45:54.465462+0000","last_became_active":"2023-09-23T17:57:24.957601+0000","last_became_peered":"2023-09-23T17:57:24.957601+0000","last_unstale":"2023-09-23T17:59:54.978604+0000","last_undegraded":"2023-09-23T17:57:24.946354+0000","last_fullsized":"2023-09-23T17:57:24.946239+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.908857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":186,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.966552+0000","last_change":"2023-09-23T17:59:51.905325+0000","last_active":"2023-09-23T17:59:54.966552+0000","last_peered":"2023-09-23T17:59:54.966552+0000","last_clean":"2023-09-23T17:59:54.966552+0000","last_became_active":"2023-09-23T17:59:51.904875+0000","last_became_peered":"2023-09-23T17:59:51.904875+0000","last_unstale":"2023-09-23T17:59:54.966552+0000","last_undegraded":"2023-09-23T17:59:54.966552+0000","last_fullsized":"2023-09-23T17:59:54.966552+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:49.269529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,1],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":481,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T18:00:04.484803+0000","last_change":"2023-09-23T17:58:35.251808+0000","last_active":"2023-09-23T18:00:04.484803+0000","last_peered":"2023-09-23T18:00:04.484803+0000","last_clean":"2023-09-23T18:00:04.484803+0000","last_became_active":"2023-09-23T17:58:35.251267+0000","last_became_peered":"2023-09-23T17:58:35.251267+0000","last_unstale":"2023-09-23T18:00:04.484803+0000","last_undegraded":"2023-09-23T18:00:04.484803+0000","last_fullsized":"2023-09-23T18:00:04.484803+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.c","version":"190'11676","reported_seq":11784,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T18:00:05.977332+0000","last_change":"2023-09-23T17:56:51.840960+0000","last_active":"2023-09-23T18:00:05.977332+0000","last_peered":"2023-09-23T18:00:05.977332+0000","last_clean":"2023-09-23T18:00:05.977332+0000","last_became_active":"2023-09-23T17:56:51.835697+0000","last_became_peered":"2023-09-23T17:56:51.835697+0000","last_unstale":"2023-09-23T18:00:05.977332+0000","last_undegraded":"2023-09-23T18:00:05.977332+0000","last_fullsized":"2023-09-23T18:00:05.977332+0000","mapping_epoch":127,"log_start":"146'7198","ondisk_log_start":"146'7198","created":110,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4478,"log_dups_size":0,"ondisk_log_size":4478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:45.804116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99745792,"num_objects":1522,"num_object_clones":0,"num_object_copies":4566,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1522,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11676,"num_write_kb":422336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":101,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.961497+0000","last_change":"2023-09-23T17:58:35.243177+0000","last_active":"2023-09-23T17:59:54.961497+0000","last_peered":"2023-09-23T17:59:54.961497+0000","last_clean":"2023-09-23T17:59:54.961497+0000","last_became_active":"2023-09-23T17:58:35.242824+0000","last_became_peered":"2023-09-23T17:58:35.242824+0000","last_unstale":"2023-09-23T17:59:54.961497+0000","last_undegraded":"2023-09-23T17:59:54.961497+0000","last_fullsized":"2023-09-23T17:59:54.961497+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:10.250046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":178,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.966698+0000","last_change":"2023-09-23T17:59:50.899364+0000","last_active":"2023-09-23T17:59:54.966698+0000","last_peered":"2023-09-23T17:59:54.966698+0000","last_clean":"2023-09-23T17:59:54.966698+0000","last_became_active":"2023-09-23T17:59:50.899062+0000","last_became_peered":"2023-09-23T17:59:50.899062+0000","last_unstale":"2023-09-23T17:59:54.966698+0000","last_undegraded":"2023-09-23T17:59:54.966698+0000","last_fullsized":"2023-09-23T17:59:54.966698+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:31.623206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":485,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.978310+0000","last_change":"2023-09-23T17:59:00.133828+0000","last_active":"2023-09-23T17:59:54.978310+0000","last_peered":"2023-09-23T17:59:54.978310+0000","last_clean":"2023-09-23T17:59:54.978310+0000","last_became_active":"2023-09-23T17:59:00.133283+0000","last_became_peered":"2023-09-23T17:59:00.133283+0000","last_unstale":"2023-09-23T17:59:54.978310+0000","last_undegraded":"2023-09-23T17:59:54.978310+0000","last_fullsized":"2023-09-23T17:59:54.978310+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.f","version":"190'11739","reported_seq":24291,"reported_epoch":190,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:00:07.462783+0000","last_change":"2023-09-23T17:59:51.071378+0000","last_active":"2023-09-23T18:00:07.462783+0000","last_peered":"2023-09-23T18:00:07.462783+0000","last_clean":"2023-09-23T17:55:56.846049+0000","last_became_active":"2023-09-23T17:59:50.976302+0000","last_became_peered":"2023-09-23T17:59:50.976302+0000","last_unstale":"2023-09-23T18:00:07.462783+0000","last_undegraded":"2023-09-23T17:59:50.971636+0000","last_fullsized":"2023-09-23T17:59:50.890747+0000","mapping_epoch":186,"log_start":"131'5047","ondisk_log_start":"131'5047","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6692,"log_dups_size":0,"ondisk_log_size":6692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:08.577292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94830592,"num_objects":1447,"num_object_clones":0,"num_object_copies":4341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":659,"num_objects_misplaced":1447,"num_objects_unfound":0,"num_objects_dirty":1447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11739,"num_write_kb":421952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5278,"num_bytes_recovered":303497216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,2],"avail_no_missing":["4","5"],"object_location_counts":[{"shards":"2,4","objects":-659},{"shards":"2,4,5","objects":1447}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":80,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.965498+0000","last_change":"2023-09-23T17:57:53.880278+0000","last_active":"2023-09-23T17:59:54.965498+0000","last_peered":"2023-09-23T17:59:54.965498+0000","last_clean":"2023-09-23T17:57:47.586782+0000","last_became_active":"2023-09-23T17:57:53.880278+0000","last_became_peered":"2023-09-23T17:57:53.880278+0000","last_unstale":"2023-09-23T17:59:54.965498+0000","last_undegraded":"2023-09-23T17:59:54.965498+0000","last_fullsized":"2023-09-23T17:57:53.867145+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:07.345156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":182,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.966966+0000","last_change":"2023-09-23T17:59:50.898260+0000","last_active":"2023-09-23T17:59:54.966966+0000","last_peered":"2023-09-23T17:59:54.966966+0000","last_clean":"2023-09-23T17:59:54.966966+0000","last_became_active":"2023-09-23T17:59:50.897991+0000","last_became_peered":"2023-09-23T17:59:50.897991+0000","last_unstale":"2023-09-23T17:59:54.966966+0000","last_undegraded":"2023-09-23T17:59:54.966966+0000","last_fullsized":"2023-09-23T17:59:54.966966+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:50.486896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":477,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T18:00:04.484940+0000","last_change":"2023-09-23T17:58:35.250653+0000","last_active":"2023-09-23T18:00:04.484940+0000","last_peered":"2023-09-23T18:00:04.484940+0000","last_clean":"2023-09-23T18:00:04.484940+0000","last_became_active":"2023-09-23T17:58:35.249513+0000","last_became_peered":"2023-09-23T17:58:35.249513+0000","last_unstale":"2023-09-23T18:00:04.484940+0000","last_undegraded":"2023-09-23T18:00:04.484940+0000","last_fullsized":"2023-09-23T18:00:04.484940+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"190'11714","reported_seq":12726,"reported_epoch":190,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:00:07.406393+0000","last_change":"2023-09-23T17:59:51.963319+0000","last_active":"2023-09-23T18:00:07.406393+0000","last_peered":"2023-09-23T18:00:07.406393+0000","last_clean":"2023-09-23T17:59:08.813403+0000","last_became_active":"2023-09-23T17:59:51.909158+0000","last_became_peered":"2023-09-23T17:59:51.909158+0000","last_unstale":"2023-09-23T18:00:07.406393+0000","last_undegraded":"2023-09-23T18:00:07.406393+0000","last_fullsized":"2023-09-23T18:00:07.406393+0000","mapping_epoch":187,"log_start":"144'6563","ondisk_log_start":"144'6563","created":110,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5151,"log_dups_size":0,"ondisk_log_size":5151,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:21:39.558589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96731136,"num_objects":1476,"num_object_clones":0,"num_object_copies":4428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1472,"num_objects_unfound":0,"num_objects_dirty":1476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11714,"num_write_kb":422080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":434,"num_bytes_recovered":28442624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,5],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4,5","objects":1476}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":125,"reported_epoch":189,"state":"undersized+peered","last_fresh":"2023-09-23T17:59:54.988170+0000","last_change":"2023-09-23T17:57:53.880095+0000","last_active":"2023-09-23T17:57:52.883357+0000","last_peered":"2023-09-23T17:59:54.988170+0000","last_clean":"2023-09-23T17:53:21.028908+0000","last_became_active":"2023-09-23T17:57:24.954589+0000","last_became_peered":"2023-09-23T17:57:53.880095+0000","last_unstale":"2023-09-23T17:59:54.988170+0000","last_undegraded":"2023-09-23T17:59:54.988170+0000","last_fullsized":"2023-09-23T17:57:53.874470+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:56.618752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":150,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.983537+0000","last_change":"2023-09-23T17:57:24.961477+0000","last_active":"2023-09-23T17:59:54.983537+0000","last_peered":"2023-09-23T17:59:54.983537+0000","last_clean":"2023-09-23T17:57:20.929549+0000","last_became_active":"2023-09-23T17:57:24.961477+0000","last_became_peered":"2023-09-23T17:57:24.961477+0000","last_unstale":"2023-09-23T17:59:54.983537+0000","last_undegraded":"2023-09-23T17:59:54.983537+0000","last_fullsized":"2023-09-23T17:57:24.954309+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:01.838528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":486,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T18:00:04.485279+0000","last_change":"2023-09-23T17:59:00.112340+0000","last_active":"2023-09-23T18:00:04.485279+0000","last_peered":"2023-09-23T18:00:04.485279+0000","last_clean":"2023-09-23T18:00:04.485279+0000","last_became_active":"2023-09-23T17:59:00.111325+0000","last_became_peered":"2023-09-23T17:59:00.111325+0000","last_unstale":"2023-09-23T18:00:04.485279+0000","last_undegraded":"2023-09-23T18:00:04.485279+0000","last_fullsized":"2023-09-23T18:00:04.485279+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":618,"reported_epoch":190,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:05.096100+0000","last_change":"2023-09-23T17:57:53.873008+0000","last_active":"2023-09-23T18:00:05.096100+0000","last_peered":"2023-09-23T18:00:05.096100+0000","last_clean":"2023-09-23T17:57:47.562468+0000","last_became_active":"2023-09-23T17:57:53.873008+0000","last_became_peered":"2023-09-23T17:57:53.873008+0000","last_unstale":"2023-09-23T18:00:05.096100+0000","last_undegraded":"2023-09-23T17:57:53.867595+0000","last_fullsized":"2023-09-23T17:57:53.867487+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:19.562351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":148,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.983441+0000","last_change":"2023-09-23T17:59:10.975320+0000","last_active":"2023-09-23T17:59:54.983441+0000","last_peered":"2023-09-23T17:59:54.983441+0000","last_clean":"2023-09-23T17:59:54.983441+0000","last_became_active":"2023-09-23T17:59:10.974946+0000","last_became_peered":"2023-09-23T17:59:10.974946+0000","last_unstale":"2023-09-23T17:59:54.983441+0000","last_undegraded":"2023-09-23T17:59:54.983441+0000","last_fullsized":"2023-09-23T17:59:54.983441+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:05.703255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":143,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T18:00:05.093809+0000","last_change":"2023-09-23T17:59:09.887042+0000","last_active":"2023-09-23T18:00:05.093809+0000","last_peered":"2023-09-23T18:00:05.093809+0000","last_clean":"2023-09-23T18:00:05.093809+0000","last_became_active":"2023-09-23T17:59:09.886688+0000","last_became_peered":"2023-09-23T17:59:09.886688+0000","last_unstale":"2023-09-23T18:00:05.093809+0000","last_undegraded":"2023-09-23T18:00:05.093809+0000","last_fullsized":"2023-09-23T18:00:05.093809+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:44.443377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"190'11680","reported_seq":13988,"reported_epoch":190,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:00:09.432695+0000","last_change":"2023-09-23T17:59:51.072118+0000","last_active":"2023-09-23T18:00:09.432695+0000","last_peered":"2023-09-23T18:00:09.432695+0000","last_clean":"2023-09-23T17:55:57.821023+0000","last_became_active":"2023-09-23T17:59:09.854393+0000","last_became_peered":"2023-09-23T17:59:09.854393+0000","last_unstale":"2023-09-23T18:00:09.432695+0000","last_undegraded":"2023-09-23T17:59:09.846461+0000","last_fullsized":"2023-09-23T17:59:09.846277+0000","mapping_epoch":175,"log_start":"146'7157","ondisk_log_start":"146'7157","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4523,"log_dups_size":0,"ondisk_log_size":4523,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:38.915731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91619328,"num_objects":1398,"num_object_clones":0,"num_object_copies":4194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":835,"num_objects_misplaced":1398,"num_objects_unfound":0,"num_objects_dirty":1398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11680,"num_write_kb":418496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1095,"num_bytes_recovered":71761920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":1398}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":485,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:59.466402+0000","last_change":"2023-09-23T17:59:00.117942+0000","last_active":"2023-09-23T17:59:59.466402+0000","last_peered":"2023-09-23T17:59:59.466402+0000","last_clean":"2023-09-23T17:59:59.466402+0000","last_became_active":"2023-09-23T17:59:00.117602+0000","last_became_peered":"2023-09-23T17:59:00.117602+0000","last_unstale":"2023-09-23T17:59:59.466402+0000","last_undegraded":"2023-09-23T17:59:59.466402+0000","last_fullsized":"2023-09-23T17:59:59.466402+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"190'11698","reported_seq":11807,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T18:00:09.455861+0000","last_change":"2023-09-23T17:56:50.843235+0000","last_active":"2023-09-23T18:00:09.455861+0000","last_peered":"2023-09-23T18:00:09.455861+0000","last_clean":"2023-09-23T18:00:09.455861+0000","last_became_active":"2023-09-23T17:56:50.835153+0000","last_became_peered":"2023-09-23T17:56:50.835153+0000","last_unstale":"2023-09-23T18:00:09.455861+0000","last_undegraded":"2023-09-23T18:00:09.455861+0000","last_fullsized":"2023-09-23T18:00:09.455861+0000","mapping_epoch":126,"log_start":"146'7153","ondisk_log_start":"146'7153","created":110,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4545,"log_dups_size":0,"ondisk_log_size":4545,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:14.827228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89915392,"num_objects":1372,"num_object_clones":0,"num_object_copies":4116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11698,"num_write_kb":418240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1126,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.988324+0000","last_change":"2023-09-23T17:59:51.906763+0000","last_active":"2023-09-23T17:59:54.988324+0000","last_peered":"2023-09-23T17:59:54.988324+0000","last_clean":"2023-09-23T17:59:54.988324+0000","last_became_active":"2023-09-23T17:59:51.906189+0000","last_became_peered":"2023-09-23T17:59:51.906189+0000","last_unstale":"2023-09-23T17:59:54.988324+0000","last_undegraded":"2023-09-23T17:59:54.988324+0000","last_fullsized":"2023-09-23T17:59:54.988324+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.865971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":178,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.966613+0000","last_change":"2023-09-23T17:59:50.897300+0000","last_active":"2023-09-23T17:59:54.966613+0000","last_peered":"2023-09-23T17:59:54.966613+0000","last_clean":"2023-09-23T17:59:54.966613+0000","last_became_active":"2023-09-23T17:59:50.896960+0000","last_became_peered":"2023-09-23T17:59:50.896960+0000","last_unstale":"2023-09-23T17:59:54.966613+0000","last_undegraded":"2023-09-23T17:59:54.966613+0000","last_fullsized":"2023-09-23T17:59:54.966613+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:29.201533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":152,"reported_epoch":190,"state":"active+undersized","last_fresh":"2023-09-23T18:00:05.095985+0000","last_change":"2023-09-23T17:57:24.952855+0000","last_active":"2023-09-23T18:00:05.095985+0000","last_peered":"2023-09-23T18:00:05.095985+0000","last_clean":"2023-09-23T17:57:20.923267+0000","last_became_active":"2023-09-23T17:57:24.952855+0000","last_became_peered":"2023-09-23T17:57:24.952855+0000","last_unstale":"2023-09-23T18:00:05.095985+0000","last_undegraded":"2023-09-23T18:00:05.095985+0000","last_fullsized":"2023-09-23T17:57:24.946770+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:15.329593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":486,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:59.466427+0000","last_change":"2023-09-23T17:59:00.113520+0000","last_active":"2023-09-23T17:59:59.466427+0000","last_peered":"2023-09-23T17:59:59.466427+0000","last_clean":"2023-09-23T17:59:59.466427+0000","last_became_active":"2023-09-23T17:59:00.113172+0000","last_became_peered":"2023-09-23T17:59:00.113172+0000","last_unstale":"2023-09-23T17:59:59.466427+0000","last_undegraded":"2023-09-23T17:59:59.466427+0000","last_fullsized":"2023-09-23T17:59:59.466427+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"190'12049","reported_seq":14202,"reported_epoch":190,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:00:09.114983+0000","last_change":"2023-09-23T17:59:32.085668+0000","last_active":"2023-09-23T18:00:09.114983+0000","last_peered":"2023-09-23T18:00:09.114983+0000","last_clean":"2023-09-23T17:57:52.799975+0000","last_became_active":"2023-09-23T17:59:11.178414+0000","last_became_peered":"2023-09-23T17:59:11.178414+0000","last_unstale":"2023-09-23T18:00:09.114983+0000","last_undegraded":"2023-09-23T17:59:10.956696+0000","last_fullsized":"2023-09-23T17:59:10.884554+0000","mapping_epoch":176,"log_start":"117'2273","ondisk_log_start":"117'2273","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9776,"log_dups_size":0,"ondisk_log_size":9776,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:22.428899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92078080,"num_objects":1405,"num_object_clones":0,"num_object_copies":4215,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1405,"num_objects_misplaced":2580,"num_objects_unfound":0,"num_objects_dirty":1405,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12049,"num_write_kb":430528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1024,"num_bytes_recovered":67108864,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1175}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":461,"reported_epoch":189,"state":"active+undersized+degraded","last_fresh":"2023-09-23T17:59:54.975175+0000","last_change":"2023-09-23T17:57:24.955720+0000","last_active":"2023-09-23T17:59:54.975175+0000","last_peered":"2023-09-23T17:59:54.975175+0000","last_clean":"2023-09-23T17:57:20.931796+0000","last_became_active":"2023-09-23T17:57:24.955720+0000","last_became_peered":"2023-09-23T17:57:24.955720+0000","last_unstale":"2023-09-23T17:59:54.975175+0000","last_undegraded":"2023-09-23T17:57:24.945824+0000","last_fullsized":"2023-09-23T17:57:24.945671+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:25.614568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":145,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.988340+0000","last_change":"2023-09-23T17:58:35.246091+0000","last_active":"2023-09-23T17:59:54.988340+0000","last_peered":"2023-09-23T17:59:54.988340+0000","last_clean":"2023-09-23T17:59:54.988340+0000","last_became_active":"2023-09-23T17:58:35.245673+0000","last_became_peered":"2023-09-23T17:58:35.245673+0000","last_unstale":"2023-09-23T17:59:54.988340+0000","last_undegraded":"2023-09-23T17:59:54.988340+0000","last_fullsized":"2023-09-23T17:59:54.988340+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:53.368873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":168,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.990253+0000","last_change":"2023-09-23T17:59:10.911985+0000","last_active":"2023-09-23T17:59:54.990253+0000","last_peered":"2023-09-23T17:59:54.990253+0000","last_clean":"2023-09-23T17:59:54.990253+0000","last_became_active":"2023-09-23T17:59:10.911584+0000","last_became_peered":"2023-09-23T17:59:10.911584+0000","last_unstale":"2023-09-23T17:59:54.990253+0000","last_undegraded":"2023-09-23T17:59:54.990253+0000","last_fullsized":"2023-09-23T17:59:54.990253+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:43.723422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,5],"avail_no_missing":["1","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1005,"reported_epoch":189,"state":"active+undersized+remapped","last_fresh":"2023-09-23T17:59:54.979138+0000","last_change":"2023-09-23T17:59:10.890970+0000","last_active":"2023-09-23T17:59:54.979138+0000","last_peered":"2023-09-23T17:59:54.979138+0000","last_clean":"2023-09-23T17:47:00.624503+0000","last_became_active":"2023-09-23T17:59:10.890970+0000","last_became_peered":"2023-09-23T17:59:10.890970+0000","last_unstale":"2023-09-23T17:59:54.979138+0000","last_undegraded":"2023-09-23T17:59:54.979138+0000","last_fullsized":"2023-09-23T17:59:10.885398+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:36.933247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":486,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.980429+0000","last_change":"2023-09-23T17:59:00.118160+0000","last_active":"2023-09-23T17:59:54.980429+0000","last_peered":"2023-09-23T17:59:54.980429+0000","last_clean":"2023-09-23T17:59:54.980429+0000","last_became_active":"2023-09-23T17:59:00.117827+0000","last_became_peered":"2023-09-23T17:59:00.117827+0000","last_unstale":"2023-09-23T17:59:54.980429+0000","last_undegraded":"2023-09-23T17:59:54.980429+0000","last_fullsized":"2023-09-23T17:59:54.980429+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":88,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.961385+0000","last_change":"2023-09-23T17:59:10.879009+0000","last_active":"2023-09-23T17:59:54.961385+0000","last_peered":"2023-09-23T17:59:54.961385+0000","last_clean":"2023-09-23T17:59:54.961385+0000","last_became_active":"2023-09-23T17:59:10.878632+0000","last_became_peered":"2023-09-23T17:59:10.878632+0000","last_unstale":"2023-09-23T17:59:54.961385+0000","last_undegraded":"2023-09-23T17:59:54.961385+0000","last_fullsized":"2023-09-23T17:59:54.961385+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:38.184201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,5],"avail_no_missing":["6","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":89,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.961716+0000","last_change":"2023-09-23T17:58:36.248863+0000","last_active":"2023-09-23T17:59:54.961716+0000","last_peered":"2023-09-23T17:59:54.961716+0000","last_clean":"2023-09-23T17:59:54.961716+0000","last_became_active":"2023-09-23T17:58:36.248383+0000","last_became_peered":"2023-09-23T17:58:36.248383+0000","last_unstale":"2023-09-23T17:59:54.961716+0000","last_undegraded":"2023-09-23T17:59:54.961716+0000","last_fullsized":"2023-09-23T17:59:54.961716+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:49.334942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,0],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"190'11807","reported_seq":21542,"reported_epoch":190,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:00:09.089475+0000","last_change":"2023-09-23T17:59:11.224350+0000","last_active":"2023-09-23T18:00:09.089475+0000","last_peered":"2023-09-23T18:00:09.089475+0000","last_clean":"2023-09-23T17:59:08.847410+0000","last_became_active":"2023-09-23T17:59:11.059419+0000","last_became_peered":"2023-09-23T17:59:11.059419+0000","last_unstale":"2023-09-23T18:00:09.089475+0000","last_undegraded":"2023-09-23T18:00:09.089475+0000","last_fullsized":"2023-09-23T18:00:09.089475+0000","mapping_epoch":176,"log_start":"117'2071","ondisk_log_start":"117'2071","created":110,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9736,"log_dups_size":0,"ondisk_log_size":9736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:54.596656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95748096,"num_objects":1461,"num_object_clones":0,"num_object_copies":4383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1443,"num_objects_unfound":0,"num_objects_dirty":1461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11807,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4804,"num_bytes_recovered":314834944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":1461}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":486,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:59.466496+0000","last_change":"2023-09-23T17:59:00.113658+0000","last_active":"2023-09-23T17:59:59.466496+0000","last_peered":"2023-09-23T17:59:59.466496+0000","last_clean":"2023-09-23T17:59:59.466496+0000","last_became_active":"2023-09-23T17:59:00.113034+0000","last_became_peered":"2023-09-23T17:59:00.113034+0000","last_unstale":"2023-09-23T17:59:59.466496+0000","last_undegraded":"2023-09-23T17:59:59.466496+0000","last_fullsized":"2023-09-23T17:59:59.466496+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.7","version":"190'11935","reported_seq":12031,"reported_epoch":190,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:00:05.975511+0000","last_change":"2023-09-23T17:59:11.276228+0000","last_active":"2023-09-23T18:00:05.975511+0000","last_peered":"2023-09-23T18:00:05.975511+0000","last_clean":"2023-09-23T17:59:08.808079+0000","last_became_active":"2023-09-23T17:59:11.192668+0000","last_became_peered":"2023-09-23T17:59:11.192668+0000","last_unstale":"2023-09-23T18:00:05.975511+0000","last_undegraded":"2023-09-23T18:00:05.975511+0000","last_fullsized":"2023-09-23T18:00:05.975511+0000","mapping_epoch":176,"log_start":"148'7507","ondisk_log_start":"148'7507","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4428,"log_dups_size":0,"ondisk_log_size":4428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:56.972002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99811328,"num_objects":1523,"num_object_clones":0,"num_object_copies":4569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1523,"num_objects_unfound":0,"num_objects_dirty":1523,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11935,"num_write_kb":430656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,5],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1,5","objects":1523}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":292,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:55.973530+0000","last_change":"2023-09-23T17:57:53.883867+0000","last_active":"2023-09-23T17:59:55.973530+0000","last_peered":"2023-09-23T17:59:55.973530+0000","last_clean":"2023-09-23T17:46:57.119141+0000","last_became_active":"2023-09-23T17:57:53.883867+0000","last_became_peered":"2023-09-23T17:57:53.883867+0000","last_unstale":"2023-09-23T17:59:55.973530+0000","last_undegraded":"2023-09-23T17:59:55.973530+0000","last_fullsized":"2023-09-23T17:57:53.875072+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:35.282181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":158,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.979676+0000","last_change":"2023-09-23T17:57:24.959291+0000","last_active":"2023-09-23T17:59:54.979676+0000","last_peered":"2023-09-23T17:59:54.979676+0000","last_clean":"2023-09-23T17:57:20.930735+0000","last_became_active":"2023-09-23T17:57:24.959291+0000","last_became_peered":"2023-09-23T17:57:24.959291+0000","last_unstale":"2023-09-23T17:59:54.979676+0000","last_undegraded":"2023-09-23T17:59:54.979676+0000","last_fullsized":"2023-09-23T17:57:24.946083+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:43.225674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":150,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.976658+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T17:59:54.976658+0000","last_peered":"2023-09-23T17:59:54.976658+0000","last_clean":"2023-09-23T17:59:54.976658+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T17:59:54.976658+0000","last_undegraded":"2023-09-23T17:59:54.976658+0000","last_fullsized":"2023-09-23T17:59:54.976658+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":486,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T18:00:04.485252+0000","last_change":"2023-09-23T17:59:00.118578+0000","last_active":"2023-09-23T18:00:04.485252+0000","last_peered":"2023-09-23T18:00:04.485252+0000","last_clean":"2023-09-23T18:00:04.485252+0000","last_became_active":"2023-09-23T17:59:00.118262+0000","last_became_peered":"2023-09-23T17:59:00.118262+0000","last_unstale":"2023-09-23T18:00:04.485252+0000","last_undegraded":"2023-09-23T18:00:04.485252+0000","last_fullsized":"2023-09-23T18:00:04.485252+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":325,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:59.466596+0000","last_change":"2023-09-23T17:57:53.875525+0000","last_active":"2023-09-23T17:59:59.466596+0000","last_peered":"2023-09-23T17:59:59.466596+0000","last_clean":"2023-09-23T17:46:59.915193+0000","last_became_active":"2023-09-23T17:57:53.875525+0000","last_became_peered":"2023-09-23T17:57:53.875525+0000","last_unstale":"2023-09-23T17:59:59.466596+0000","last_undegraded":"2023-09-23T17:59:59.466596+0000","last_fullsized":"2023-09-23T17:57:53.868097+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:57.748716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":139,"reported_epoch":190,"state":"active+clean","last_fresh":"2023-09-23T18:00:05.096229+0000","last_change":"2023-09-23T17:59:09.858216+0000","last_active":"2023-09-23T18:00:05.096229+0000","last_peered":"2023-09-23T18:00:05.096229+0000","last_clean":"2023-09-23T18:00:05.096229+0000","last_became_active":"2023-09-23T17:59:09.856443+0000","last_became_peered":"2023-09-23T17:59:09.856443+0000","last_unstale":"2023-09-23T18:00:05.096229+0000","last_undegraded":"2023-09-23T18:00:05.096229+0000","last_fullsized":"2023-09-23T18:00:05.096229+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:19.986245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":136,"reported_epoch":189,"state":"active+clean+remapped","last_fresh":"2023-09-23T17:59:54.978865+0000","last_change":"2023-09-23T17:59:10.883046+0000","last_active":"2023-09-23T17:59:54.978865+0000","last_peered":"2023-09-23T17:59:54.978865+0000","last_clean":"2023-09-23T17:59:54.978865+0000","last_became_active":"2023-09-23T17:59:10.882738+0000","last_became_peered":"2023-09-23T17:59:10.882738+0000","last_unstale":"2023-09-23T17:59:54.978865+0000","last_undegraded":"2023-09-23T17:59:54.978865+0000","last_fullsized":"2023-09-23T17:59:54.978865+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:39.241547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,5],"avail_no_missing":["0","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"190'11540","reported_seq":19467,"reported_epoch":190,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:00:09.480181+0000","last_change":"2023-09-23T17:59:24.084532+0000","last_active":"2023-09-23T18:00:09.480181+0000","last_peered":"2023-09-23T18:00:09.480181+0000","last_clean":"2023-09-23T17:55:52.789238+0000","last_became_active":"2023-09-23T17:59:09.938603+0000","last_became_peered":"2023-09-23T17:59:09.938603+0000","last_unstale":"2023-09-23T18:00:09.480181+0000","last_undegraded":"2023-09-23T17:59:09.881270+0000","last_fullsized":"2023-09-23T17:59:09.846342+0000","mapping_epoch":175,"log_start":"146'7047","ondisk_log_start":"146'7047","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4493,"log_dups_size":0,"ondisk_log_size":4493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:57.591912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95158272,"num_objects":1452,"num_object_clones":0,"num_object_copies":4356,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":283,"num_objects_misplaced":1063,"num_objects_unfound":0,"num_objects_dirty":1452,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11540,"num_write_kb":415744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3912,"num_bytes_recovered":256376832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":1452}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":486,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:59.466648+0000","last_change":"2023-09-23T17:59:00.119695+0000","last_active":"2023-09-23T17:59:59.466648+0000","last_peered":"2023-09-23T17:59:59.466648+0000","last_clean":"2023-09-23T17:59:59.466648+0000","last_became_active":"2023-09-23T17:59:00.119349+0000","last_became_peered":"2023-09-23T17:59:00.119349+0000","last_unstale":"2023-09-23T17:59:59.466648+0000","last_undegraded":"2023-09-23T17:59:59.466648+0000","last_fullsized":"2023-09-23T17:59:59.466648+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":59,"reported_epoch":189,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T18:00:04.485317+0000","last_change":"2023-09-23T17:57:53.872726+0000","last_active":"2023-09-23T17:57:52.888539+0000","last_peered":"2023-09-23T18:00:04.485317+0000","last_clean":"2023-09-23T17:45:54.465055+0000","last_became_active":"2023-09-23T17:45:51.139637+0000","last_became_peered":"2023-09-23T17:57:53.872726+0000","last_unstale":"2023-09-23T18:00:04.485317+0000","last_undegraded":"2023-09-23T17:57:53.871230+0000","last_fullsized":"2023-09-23T17:57:53.871154+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:46.624357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":422,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.989631+0000","last_change":"2023-09-23T17:58:35.247928+0000","last_active":"2023-09-23T17:59:54.989631+0000","last_peered":"2023-09-23T17:59:54.989631+0000","last_clean":"2023-09-23T17:59:54.989631+0000","last_became_active":"2023-09-23T17:58:35.247482+0000","last_became_peered":"2023-09-23T17:58:35.247482+0000","last_unstale":"2023-09-23T17:59:54.989631+0000","last_undegraded":"2023-09-23T17:59:54.989631+0000","last_fullsized":"2023-09-23T17:59:54.989631+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:18.686446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":116,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.980308+0000","last_change":"2023-09-23T17:58:12.005101+0000","last_active":"2023-09-23T17:59:54.980308+0000","last_peered":"2023-09-23T17:59:54.980308+0000","last_clean":"2023-09-23T17:49:06.298151+0000","last_became_active":"2023-09-23T17:58:12.005101+0000","last_became_peered":"2023-09-23T17:58:12.005101+0000","last_unstale":"2023-09-23T17:59:54.980308+0000","last_undegraded":"2023-09-23T17:59:54.980308+0000","last_fullsized":"2023-09-23T17:58:11.987584+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:15.859050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"190'11838","reported_seq":24587,"reported_epoch":190,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:07.447317+0000","last_change":"2023-09-23T17:58:23.828372+0000","last_active":"2023-09-23T18:00:07.447317+0000","last_peered":"2023-09-23T18:00:07.447317+0000","last_clean":"2023-09-23T17:57:23.936571+0000","last_became_active":"2023-09-23T17:58:23.828372+0000","last_became_peered":"2023-09-23T17:58:23.828372+0000","last_unstale":"2023-09-23T18:00:07.447317+0000","last_undegraded":"2023-09-23T17:58:23.826763+0000","last_fullsized":"2023-09-23T17:58:23.826662+0000","mapping_epoch":159,"log_start":"131'5077","ondisk_log_start":"131'5077","created":110,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6761,"log_dups_size":0,"ondisk_log_size":6761,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:59.923478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97779712,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1492,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11838,"num_write_kb":426560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6302,"num_bytes_recovered":413007872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1492}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":490,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:59.466563+0000","last_change":"2023-09-23T17:59:00.129488+0000","last_active":"2023-09-23T17:59:59.466563+0000","last_peered":"2023-09-23T17:59:59.466563+0000","last_clean":"2023-09-23T17:59:59.466563+0000","last_became_active":"2023-09-23T17:59:00.128847+0000","last_became_peered":"2023-09-23T17:59:00.128847+0000","last_unstale":"2023-09-23T17:59:59.466563+0000","last_undegraded":"2023-09-23T17:59:59.466563+0000","last_fullsized":"2023-09-23T17:59:59.466563+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":162896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":365,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.964023+0000","last_change":"2023-09-23T17:58:35.227206+0000","last_active":"2023-09-23T17:59:54.964023+0000","last_peered":"2023-09-23T17:59:54.964023+0000","last_clean":"2023-09-23T17:46:59.535985+0000","last_became_active":"2023-09-23T17:58:35.227206+0000","last_became_peered":"2023-09-23T17:58:35.227206+0000","last_unstale":"2023-09-23T17:59:54.964023+0000","last_undegraded":"2023-09-23T17:59:54.964023+0000","last_fullsized":"2023-09-23T17:58:35.224367+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:52.549534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":164,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.976734+0000","last_change":"2023-09-23T17:56:50.856836+0000","last_active":"2023-09-23T17:59:54.976734+0000","last_peered":"2023-09-23T17:59:54.976734+0000","last_clean":"2023-09-23T17:59:54.976734+0000","last_became_active":"2023-09-23T17:56:50.835155+0000","last_became_peered":"2023-09-23T17:56:50.835155+0000","last_unstale":"2023-09-23T17:59:54.976734+0000","last_undegraded":"2023-09-23T17:59:54.976734+0000","last_fullsized":"2023-09-23T17:59:54.976734+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:36.188982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":170,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.965829+0000","last_change":"2023-09-23T17:59:50.897059+0000","last_active":"2023-09-23T17:59:54.965829+0000","last_peered":"2023-09-23T17:59:54.965829+0000","last_clean":"2023-09-23T17:59:54.965829+0000","last_became_active":"2023-09-23T17:59:50.896650+0000","last_became_peered":"2023-09-23T17:59:50.896650+0000","last_unstale":"2023-09-23T17:59:54.965829+0000","last_undegraded":"2023-09-23T17:59:54.965829+0000","last_fullsized":"2023-09-23T17:59:54.965829+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:16.102787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"190'11928","reported_seq":25328,"reported_epoch":190,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:05.954080+0000","last_change":"2023-09-23T17:59:32.083467+0000","last_active":"2023-09-23T18:00:05.954080+0000","last_peered":"2023-09-23T18:00:05.954080+0000","last_clean":"2023-09-23T18:00:05.954080+0000","last_became_active":"2023-09-23T17:57:26.317820+0000","last_became_peered":"2023-09-23T17:57:26.317820+0000","last_unstale":"2023-09-23T18:00:05.954080+0000","last_undegraded":"2023-09-23T18:00:05.954080+0000","last_fullsized":"2023-09-23T18:00:05.954080+0000","mapping_epoch":143,"log_start":"146'7455","ondisk_log_start":"146'7455","created":110,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4473,"log_dups_size":0,"ondisk_log_size":4473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:37.559775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99876864,"num_objects":1524,"num_object_clones":0,"num_object_copies":4572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1524,"num_objects_unfound":0,"num_objects_dirty":1524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11928,"num_write_kb":430464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6640,"num_bytes_recovered":435159040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1524}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":146,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.978738+0000","last_change":"2023-09-23T17:57:53.877134+0000","last_active":"2023-09-23T17:59:54.978738+0000","last_peered":"2023-09-23T17:59:54.978738+0000","last_clean":"2023-09-23T17:57:47.571295+0000","last_became_active":"2023-09-23T17:57:53.877134+0000","last_became_peered":"2023-09-23T17:57:53.877134+0000","last_unstale":"2023-09-23T17:59:54.978738+0000","last_undegraded":"2023-09-23T17:59:54.978738+0000","last_fullsized":"2023-09-23T17:57:53.869424+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:59.045696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":181,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.979961+0000","last_change":"2023-09-23T17:57:24.956919+0000","last_active":"2023-09-23T17:59:54.979961+0000","last_peered":"2023-09-23T17:59:54.979961+0000","last_clean":"2023-09-23T17:57:20.928421+0000","last_became_active":"2023-09-23T17:57:24.956919+0000","last_became_peered":"2023-09-23T17:57:24.956919+0000","last_unstale":"2023-09-23T17:59:54.979961+0000","last_undegraded":"2023-09-23T17:59:54.979961+0000","last_fullsized":"2023-09-23T17:57:24.951731+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:03.928787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":45,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.966471+0000","last_change":"2023-09-23T17:59:50.901084+0000","last_active":"2023-09-23T17:59:54.966471+0000","last_peered":"2023-09-23T17:59:54.966471+0000","last_clean":"2023-09-23T17:59:54.966471+0000","last_became_active":"2023-09-23T17:59:50.900822+0000","last_became_peered":"2023-09-23T17:59:50.900822+0000","last_unstale":"2023-09-23T17:59:54.966471+0000","last_undegraded":"2023-09-23T17:59:54.966471+0000","last_fullsized":"2023-09-23T17:59:54.966471+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:08.825929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":94,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.966326+0000","last_change":"2023-09-23T17:59:50.897896+0000","last_active":"2023-09-23T17:59:54.966326+0000","last_peered":"2023-09-23T17:59:54.966326+0000","last_clean":"2023-09-23T17:57:20.922389+0000","last_became_active":"2023-09-23T17:59:50.897896+0000","last_became_peered":"2023-09-23T17:59:50.897896+0000","last_unstale":"2023-09-23T17:59:54.966326+0000","last_undegraded":"2023-09-23T17:59:54.966326+0000","last_fullsized":"2023-09-23T17:59:50.890883+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:36.213857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":41,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.966778+0000","last_change":"2023-09-23T17:59:50.899104+0000","last_active":"2023-09-23T17:59:54.966778+0000","last_peered":"2023-09-23T17:59:54.966778+0000","last_clean":"2023-09-23T17:59:54.966778+0000","last_became_active":"2023-09-23T17:59:50.898883+0000","last_became_peered":"2023-09-23T17:59:50.898883+0000","last_unstale":"2023-09-23T17:59:54.966778+0000","last_undegraded":"2023-09-23T17:59:54.966778+0000","last_fullsized":"2023-09-23T17:59:54.966778+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:31.834372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":98,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.974668+0000","last_change":"2023-09-23T17:59:22.871834+0000","last_active":"2023-09-23T17:59:54.974668+0000","last_peered":"2023-09-23T17:59:54.974668+0000","last_clean":"2023-09-23T17:57:20.922822+0000","last_became_active":"2023-09-23T17:59:22.871834+0000","last_became_peered":"2023-09-23T17:59:22.871834+0000","last_unstale":"2023-09-23T17:59:54.974668+0000","last_undegraded":"2023-09-23T17:59:54.974668+0000","last_fullsized":"2023-09-23T17:59:22.864946+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:40.097366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":48,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.962232+0000","last_change":"2023-09-23T17:58:35.229256+0000","last_active":"2023-09-23T17:59:54.962232+0000","last_peered":"2023-09-23T17:59:54.962232+0000","last_clean":"2023-09-23T17:45:50.074605+0000","last_became_active":"2023-09-23T17:58:35.229256+0000","last_became_peered":"2023-09-23T17:58:35.229256+0000","last_unstale":"2023-09-23T17:59:54.962232+0000","last_undegraded":"2023-09-23T17:59:54.962232+0000","last_fullsized":"2023-09-23T17:58:35.224895+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:14.916976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":140,"reported_epoch":189,"state":"active+undersized","last_fresh":"2023-09-23T17:59:54.975046+0000","last_change":"2023-09-23T17:57:24.955661+0000","last_active":"2023-09-23T17:59:54.975046+0000","last_peered":"2023-09-23T17:59:54.975046+0000","last_clean":"2023-09-23T17:53:52.727684+0000","last_became_active":"2023-09-23T17:57:24.955661+0000","last_became_peered":"2023-09-23T17:57:24.955661+0000","last_unstale":"2023-09-23T17:59:54.975046+0000","last_undegraded":"2023-09-23T17:59:54.975046+0000","last_fullsized":"2023-09-23T17:57:24.945672+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:04.500443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":153,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.976274+0000","last_change":"2023-09-23T17:59:09.869351+0000","last_active":"2023-09-23T17:59:54.976274+0000","last_peered":"2023-09-23T17:59:54.976274+0000","last_clean":"2023-09-23T17:59:54.976274+0000","last_became_active":"2023-09-23T17:59:09.861326+0000","last_became_peered":"2023-09-23T17:59:09.861326+0000","last_unstale":"2023-09-23T17:59:54.976274+0000","last_undegraded":"2023-09-23T17:59:54.976274+0000","last_fullsized":"2023-09-23T17:59:54.976274+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:38.142504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":179,"reported_epoch":189,"state":"active+clean","last_fresh":"2023-09-23T17:59:54.965021+0000","last_change":"2023-09-23T17:58:35.243060+0000","last_active":"2023-09-23T17:59:54.965021+0000","last_peered":"2023-09-23T17:59:54.965021+0000","last_clean":"2023-09-23T17:59:54.965021+0000","last_became_active":"2023-09-23T17:58:35.242727+0000","last_became_peered":"2023-09-23T17:58:35.242727+0000","last_unstale":"2023-09-23T17:59:54.965021+0000","last_undegraded":"2023-09-23T17:59:54.965021+0000","last_fullsized":"2023-09-23T17:59:54.965021+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:23:08.850283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1505624088,"num_objects":22975,"num_object_clones":0,"num_object_copies":68925,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7407,"num_objects_misplaced":21782,"num_objects_unfound":0,"num_objects_dirty":22975,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":188273,"num_write_kb":6759873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":38721,"num_bytes_recovered":2495086592,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7368749056,"data_stored":7368736840,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":109407,"ondisk_log_size":109407,"up":40,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":72,"acting":78,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":756,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33098,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":69,"acting":78,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":17,"acting":18,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266849,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1840208,"kb_used_data":1778696,"kb_used_omap":0,"kb_used_meta":61504,"kb_avail":91904944,"statfs":{"total":95995035648,"available":94110662656,"internally_reserved":0,"allocated":1821384704,"data_stored":1820756859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":62980096},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":0.83399999999999996,"15min":0.64500000000000002},"min":{"1min":0.24099999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.5700000000000003,"5min":8.5700000000000003,"15min":8.5700000000000003},"last":2.2210000000000001},{"interface":"front","average":{"1min":1.6499999999999999,"5min":0.879,"15min":0.67100000000000004},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.8410000000000002,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":4.1280000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.59599999999999997,"15min":0.56999999999999995},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.702,"5min":2.702,"15min":2.702},"last":6.6890000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":0.65700000000000003,"15min":0.58899999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":4.4240000000000004},"last":0.26000000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":0.89200000000000002,"15min":0.66300000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.2359999999999998,"5min":5.2359999999999998,"15min":5.2359999999999998},"last":2.2970000000000002},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.77900000000000003,"15min":0.64800000000000002},"min":{"1min":0.38100000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.8340000000000001,"5min":4.6180000000000003,"15min":4.6180000000000003},"last":0.439}]},{"osd":3,"last update":"Sat Sep 23 17:57:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.64900000000000002,"15min":0.57299999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":4.0549999999999997,"5min":4.0549999999999997,"15min":4.0549999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.665,"5min":0.77400000000000002,"15min":0.626},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.1609999999999996,"5min":5.1609999999999996,"15min":5.1609999999999996},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.53400000000000003,"15min":0.47099999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.6160000000000001,"5min":2.6160000000000001,"15min":2.6160000000000001},"last":2.1949999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.56100000000000005,"15min":0.498},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8079999999999998,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.60699999999999998,"15min":0.52600000000000002},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.5840000000000001,"5min":2.5840000000000001,"15min":2.5840000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.749,"5min":0.64100000000000001,"15min":0.54900000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.706,"5min":2.706,"15min":2.706},"last":2.1669999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":1.002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.435}]}]},{"osd":6,"up_from":163,"seq":700079669268,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":275252,"kb_used_data":205164,"kb_used_omap":7,"kb_used_meta":70072,"kb_avail":93469900,"statfs":{"total":95995035648,"available":95713177600,"internally_reserved":0,"allocated":210087936,"data_stored":209318872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":71754532},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":14.196999999999999,"5min":14.196999999999999,"15min":14.196999999999999},"last":0.26200000000000001},{"interface":"front","average":{"1min":2.7010000000000001,"5min":2.7010000000000001,"15min":2.7010000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.536,"5min":13.536,"15min":13.536},"last":3.2559999999999998}]},{"osd":1,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":2.141,"5min":2.141,"15min":2.141},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":14.233000000000001,"5min":14.233000000000001,"15min":14.233000000000001},"last":2.722},{"interface":"front","average":{"1min":2.4220000000000002,"5min":2.4220000000000002,"15min":2.4220000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":9.1099999999999994,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":0.27500000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.5169999999999999,"15min":2.5169999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.210000000000001,"5min":11.210000000000001,"15min":11.210000000000001},"last":5.8529999999999998},{"interface":"front","average":{"1min":3.0419999999999998,"5min":3.0419999999999998,"15min":3.0419999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.223000000000001,"5min":11.223000000000001,"15min":11.223000000000001},"last":5.8710000000000004}]},{"osd":4,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":8.4570000000000007,"5min":8.4570000000000007,"15min":8.4570000000000007},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.30399999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.3330000000000002,"5min":4.3330000000000002,"15min":4.3330000000000002},"last":0.28799999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.3530000000000002,"5min":2.3530000000000002,"15min":2.3530000000000002},"last":0.33900000000000002}]}]},{"osd":1,"up_from":98,"seq":420906795085,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":722332,"kb_used_data":592212,"kb_used_omap":7,"kb_used_meta":130104,"kb_avail":93022820,"statfs":{"total":95995035648,"available":95255367680,"internally_reserved":0,"allocated":606425088,"data_stored":605448535,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":133227197},"hb_peers":[0,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:59:57 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.573,"15min":0.78300000000000003},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":15.529999999999999,"5min":15.529999999999999,"15min":15.529999999999999},"last":0.19},{"interface":"front","average":{"1min":2.2799999999999998,"5min":1.8049999999999999,"15min":0.86599999999999999},"min":{"1min":0.27400000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.3239999999999998,"5min":12.355,"15min":12.355},"last":0.90000000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:59:36 2023","interfaces":[{"interface":"back","average":{"1min":3.8860000000000001,"5min":1.994,"15min":0.96499999999999997},"min":{"1min":0.24099999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":16.347999999999999,"5min":16.347999999999999,"15min":16.347999999999999},"last":1.1779999999999999},{"interface":"front","average":{"1min":1.964,"5min":1.5509999999999999,"15min":0.81000000000000005},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.8379999999999992,"5min":11.305999999999999,"15min":11.305999999999999},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:57:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8999999999999999,"5min":0.90800000000000003,"15min":0.57599999999999996},"min":{"1min":0.29899999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":14.548999999999999,"5min":14.548999999999999,"15min":14.548999999999999},"last":2.7149999999999999},{"interface":"front","average":{"1min":2.3740000000000001,"5min":0.85199999999999998,"15min":0.59799999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.250999999999999,"5min":10.250999999999999,"15min":10.250999999999999},"last":0.82899999999999996}]},{"osd":4,"last update":"Sat Sep 23 17:59:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4100000000000001,"5min":1.47,"15min":0.94499999999999995},"min":{"1min":0.33600000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.4740000000000002,"5min":9.4740000000000002,"15min":9.4740000000000002},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.718,"5min":1.45,"15min":0.93500000000000005},"min":{"1min":0.28699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.8879999999999999,"5min":9.8879999999999999,"15min":9.8879999999999999},"last":0.45700000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:59:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":1.579,"15min":1.0049999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":8.8390000000000004,"5min":8.8390000000000004,"15min":8.8390000000000004},"last":0.434},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.6359999999999999,"15min":1.016},"min":{"1min":0.34100000000000003,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":0.44500000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:59:36 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":2.2690000000000001,"15min":2.2690000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.5229999999999997,"5min":9.5229999999999997,"15min":9.5229999999999997},"last":0.40999999999999998},{"interface":"front","average":{"1min":1.746,"5min":1.746,"15min":1.746},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.5,"5min":4.5,"15min":4.5},"last":0.38100000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":0.89400000000000002,"15min":0.74399999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.9630000000000001,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":7.8869999999999996},{"interface":"front","average":{"1min":1.895,"5min":0.92900000000000005,"15min":0.77900000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.2940000000000005,"5min":8.2940000000000005,"15min":8.2940000000000005},"last":7.234}]}]},{"osd":0,"up_from":93,"seq":399431958612,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1309584,"kb_used_data":1007944,"kb_used_omap":7,"kb_used_meta":301624,"kb_avail":92435568,"statfs":{"total":95995035648,"available":94654021632,"internally_reserved":0,"allocated":1032134656,"data_stored":1031166456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":308863716},"hb_peers":[1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:00:02 2023","interfaces":[{"interface":"back","average":{"1min":3.988,"5min":2.1829999999999998,"15min":0.97399999999999998},"min":{"1min":0.186,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":13.414,"5min":18.154,"15min":18.154},"last":4.5279999999999996},{"interface":"front","average":{"1min":3.355,"5min":1.9990000000000001,"15min":0.94999999999999996},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":13.369999999999999,"5min":13.369999999999999,"15min":13.369999999999999},"last":2.4500000000000002}]},{"osd":2,"last update":"Sat Sep 23 17:59:34 2023","interfaces":[{"interface":"back","average":{"1min":4.7380000000000004,"5min":2.1150000000000002,"15min":0.94499999999999995},"min":{"1min":0.24299999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":20.882000000000001,"5min":20.882000000000001,"15min":20.882000000000001},"last":4.5039999999999996},{"interface":"front","average":{"1min":3.3620000000000001,"5min":1.8500000000000001,"15min":0.86199999999999999},"min":{"1min":0.255,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.064,"5min":15.064,"15min":15.064},"last":3.2480000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":0.77100000000000002,"15min":0.60099999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3449999999999998,"5min":4.3449999999999998,"15min":4.3449999999999998},"last":9.266},{"interface":"front","average":{"1min":2.109,"5min":0.73699999999999999,"15min":0.50900000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":8.8729999999999993},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:59:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1139999999999999,"5min":1.5429999999999999,"15min":0.93799999999999994},"min":{"1min":0.29499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":15.279999999999999,"5min":15.279999999999999,"15min":15.279999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":3.0590000000000002,"5min":1.512,"15min":0.91900000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":18.106999999999999,"5min":18.106999999999999,"15min":18.106999999999999},"last":4.4790000000000001}]},{"osd":5,"last update":"Sat Sep 23 17:59:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0419999999999998,"5min":1.5449999999999999,"15min":0.93700000000000006},"min":{"1min":0.33000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":14.720000000000001,"5min":14.720000000000001,"15min":14.720000000000001},"last":4.4390000000000001},{"interface":"front","average":{"1min":2.472,"5min":1.268,"15min":0.85199999999999998},"min":{"1min":0.35099999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.7530000000000001,"5min":8.7829999999999995,"15min":8.7829999999999995},"last":0.41799999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:59:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5179999999999998,"5min":2.5179999999999998,"15min":2.5179999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":14.956,"5min":14.956,"15min":14.956},"last":0.59399999999999997},{"interface":"front","average":{"1min":2.762,"5min":2.762,"15min":2.762},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":8.7249999999999996,"5min":8.7249999999999996,"15min":8.7249999999999996},"last":0.45400000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":0.90800000000000003,"15min":0.746},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.8160000000000007,"5min":8.8160000000000007,"15min":8.8160000000000007},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.01,"15min":0.78400000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":8.6470000000000002,"5min":8.6470000000000002,"15min":8.6470000000000002},"last":0.49099999999999999}]}]},{"osd":2,"up_from":103,"seq":442381631559,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":867592,"kb_used_data":713728,"kb_used_omap":11,"kb_used_meta":153844,"kb_avail":92877560,"statfs":{"total":95995035648,"available":95106621440,"internally_reserved":0,"allocated":730857472,"data_stored":729899097,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":157536631},"hb_peers":[0,1,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:59:33 2023","interfaces":[{"interface":"back","average":{"1min":3.2189999999999999,"5min":1.7789999999999999,"15min":0.81899999999999995},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":15.006,"5min":15.006,"15min":15.006},"last":19.727},{"interface":"front","average":{"1min":3.0960000000000001,"5min":1.8140000000000001,"15min":0.85999999999999999},"min":{"1min":0.26700000000000002,"5min":0.215,"15min":0.215},"max":{"1min":11.890000000000001,"5min":13.364000000000001,"15min":13.364000000000001},"last":21.681000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:59:33 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":1.9570000000000001,"15min":0.92000000000000004},"min":{"1min":0.318,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.339,"5min":19.206,"15min":19.206},"last":20.215},{"interface":"front","average":{"1min":1.653,"5min":1.79,"15min":0.81599999999999995},"min":{"1min":0.246,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.008,"5min":15.657,"15min":15.657},"last":15.945}]},{"osd":3,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":0.58799999999999997,"15min":0.443},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8339999999999996,"5min":4.8339999999999996,"15min":4.8339999999999996},"last":2.8849999999999998},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.59699999999999998,"15min":0.48399999999999999},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.6459999999999999,"5min":5.6459999999999999,"15min":5.6459999999999999},"last":2.5579999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:59:33 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.3560000000000001,"15min":0.85399999999999998},"min":{"1min":0.28299999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.0670000000000002,"5min":7.0670000000000002,"15min":7.0670000000000002},"last":5.6520000000000001},{"interface":"front","average":{"1min":2.3130000000000002,"5min":1.258,"15min":0.82599999999999996},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.0939999999999994,"5min":13.308,"15min":13.308},"last":20.044}]},{"osd":5,"last update":"Sat Sep 23 17:59:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7519999999999998,"5min":1.462,"15min":0.90300000000000002},"min":{"1min":0.32100000000000001,"5min":0.315,"15min":0.315},"max":{"1min":9.0030000000000001,"5min":13.273,"15min":13.273},"last":20.143000000000001},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.6359999999999999,"15min":0.94799999999999995},"min":{"1min":0.34100000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.863,"5min":15.496,"15min":15.496},"last":14.359}]},{"osd":6,"last update":"Sat Sep 23 17:59:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3220000000000001,"5min":2.3220000000000001,"15min":2.3220000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.4649999999999999,"5min":9.4649999999999999,"15min":9.4649999999999999},"last":5.6219999999999999},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.5760000000000001,"15min":2.5760000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.1910000000000007,"5min":8.1910000000000007,"15min":8.1910000000000007},"last":20.096}]},{"osd":7,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":0.94299999999999995,"15min":0.71899999999999997},"min":{"1min":0.505,"5min":0.441,"15min":0.441},"max":{"1min":5.7409999999999997,"5min":5.7409999999999997,"15min":5.7409999999999997},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.0870000000000002,"5min":0.97899999999999998,"15min":0.78300000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":13.333,"5min":13.333,"15min":13.333},"last":1.3440000000000001}]}]},{"osd":3,"up_from":108,"seq":463856467999,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267348,"kb_used_data":1221900,"kb_used_omap":3,"kb_used_meta":45436,"kb_avail":92477804,"statfs":{"total":95995035648,"available":94697271296,"internally_reserved":0,"allocated":1251225600,"data_stored":1250531999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":46527208},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.0699999999999998,"5min":0.71399999999999997,"15min":0.48799999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":14.82,"5min":14.82,"15min":14.82},"last":9.0220000000000002},{"interface":"front","average":{"1min":2.085,"5min":0.72499999999999998,"15min":0.499},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":13.054,"5min":13.054,"15min":13.054},"last":0.46500000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":0.65700000000000003,"15min":0.46899999999999997},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.8130000000000006,"5min":8.8130000000000006,"15min":8.8130000000000006},"last":9.1489999999999991},{"interface":"front","average":{"1min":1.417,"5min":0.61599999999999999,"15min":0.48199999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4580000000000002,"5min":6.4580000000000002,"15min":6.4580000000000002},"last":9.125}]},{"osd":2,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":0.69099999999999995,"15min":0.49099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.869999999999999,"5min":10.869999999999999,"15min":10.869999999999999},"last":7.3849999999999998},{"interface":"front","average":{"1min":1.6850000000000001,"5min":0.66500000000000004,"15min":0.495},"min":{"1min":0.376,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":13.936999999999999,"5min":13.936999999999999,"15min":13.936999999999999},"last":8.9169999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":0.73699999999999999,"15min":0.65200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5380000000000003,"5min":6.5380000000000003,"15min":6.5380000000000003},"last":6.7569999999999997},{"interface":"front","average":{"1min":1.629,"5min":0.82099999999999995,"15min":0.68700000000000006},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.5940000000000003,"5min":6.5940000000000003,"15min":6.5940000000000003},"last":3.7709999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":0.94399999999999995,"15min":0.72599999999999998},"min":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":9.6489999999999991,"5min":9.6489999999999991,"15min":9.6489999999999991},"last":3.9569999999999999},{"interface":"front","average":{"1min":1.4710000000000001,"5min":0.78600000000000003,"15min":0.67200000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.3200000000000003,"5min":9.3200000000000003,"15min":9.3200000000000003},"last":6.4569999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":0.81799999999999995,"15min":0.68999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":8.8599999999999994,"5min":8.8599999999999994,"15min":8.8599999999999994},"last":6.4950000000000001},{"interface":"front","average":{"1min":1.3520000000000001,"5min":0.78800000000000003,"15min":0.69399999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.2080000000000002,"5min":9.2080000000000002,"15min":9.2080000000000002},"last":6.0709999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019077,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":886068,"kb_used_data":752940,"kb_used_omap":0,"kb_used_meta":133120,"kb_avail":92859084,"statfs":{"total":95995035648,"available":95087702016,"internally_reserved":0,"allocated":771010560,"data_stored":770240778,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":136314880},"hb_peers":[0,1,2,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":0.97399999999999998,"15min":0.66900000000000004},"min":{"1min":0.22500000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.5279999999999996,"5min":6.5279999999999996,"15min":6.5279999999999996},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.744,"5min":1.3300000000000001,"15min":0.79300000000000004},"min":{"1min":0.22900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.5739999999999998,"5min":11.372,"15min":11.372},"last":2.5600000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:00:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.5289999999999999,"15min":0.86299999999999999},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":8.5259999999999998,"5min":10.429,"15min":10.429},"last":0.51600000000000001},{"interface":"front","average":{"1min":1.732,"5min":1.6319999999999999,"15min":0.91600000000000004},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.8130000000000006,"5min":10.83,"15min":10.83},"last":0.504}]},{"osd":2,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":2.5379999999999998,"5min":1.492,"15min":0.91900000000000004},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.630000000000001,"5min":11.630000000000001,"15min":11.630000000000001},"last":2.1200000000000001},{"interface":"front","average":{"1min":1.9279999999999999,"5min":1.375,"15min":0.84799999999999998},"min":{"1min":0.32000000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4909999999999997,"5min":7.5090000000000003,"15min":7.5090000000000003},"last":2.7919999999999998}]},{"osd":3,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.72899999999999998,"15min":0.64900000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6130000000000004,"5min":4.6130000000000004,"15min":4.6130000000000004},"last":1.397},{"interface":"front","average":{"1min":1.04,"5min":0.73299999999999998,"15min":0.68200000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":3.637,"5min":3.637,"15min":3.637},"last":9.4359999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.60599999999999998,"15min":0.504},"min":{"1min":0.251,"5min":0.224,"15min":0.224},"max":{"1min":5.4299999999999997,"5min":5.4299999999999997,"15min":5.4299999999999997},"last":0.55300000000000005},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.77200000000000002,"15min":0.56899999999999995},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.2610000000000001,"5min":10.494,"15min":10.494},"last":2.3500000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.3479999999999999,"5min":3.3479999999999999,"15min":3.3479999999999999},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.399,"5min":3.399,"15min":3.399},"last":0.60299999999999998}]},{"osd":7,"last update":"Sat Sep 23 17:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.57399999999999995,"15min":0.55100000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.8370000000000002,"5min":2.8370000000000002,"15min":2.8370000000000002},"last":0.308},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.64400000000000002,"15min":0.56299999999999994},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":0.434}]}]},{"osd":5,"up_from":0,"seq":163208757443,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1011613696,"data_stored":1011613696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":594280448,"data_stored":594280448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":711790592,"data_stored":711786520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1253441536,"data_stored":1253441536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":761462784,"data_stored":761462784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1012076544,"data_stored":1012072472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":205979648,"data_stored":205979648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1818103808,"data_stored":1818099736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:00:14.857 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 21 2023-09-23T18:00:14.858 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2122867913' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:14.858 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:00:14.859 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:00:14.859 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:00:14.859 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:00:14.859 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:00:14.859 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:00:14.859 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:00:14.859 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:00:14.889 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:14.954 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:00:14.987 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:15.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2122867913' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:15.259 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:15.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:15.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:15.323 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:00:15.323 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:00:15.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:15.418 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:15.469 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:15.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:15.478 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:15.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:15.622 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:15.709 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:00:15.722 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:15 smithi099 ceph-mon[167076]: pgmap v438: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.2 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 6699/59631 objects degraded (11.234%); 19960/59631 objects misplaced (33.473%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:15.722 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2142152467' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:15.722 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:15 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:15.763 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:15.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:15.770 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:15.770 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:15.871 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:15.951 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:16.074 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:16.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:16.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:16.080 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:15 smithi055 ceph-mon[184174]: pgmap v438: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.2 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 6699/59631 objects degraded (11.234%); 19960/59631 objects misplaced (33.473%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:16.081 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2142152467' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:16.081 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:15 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:16.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:15 smithi055 ceph-mon[179668]: pgmap v438: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.2 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 6699/59631 objects degraded (11.234%); 19960/59631 objects misplaced (33.473%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:16.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2142152467' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:16.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:15 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:16.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:16.242 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:16.303 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:16.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:16.312 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:16.353 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:00:16.353 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:00:16.353 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:00:16.353 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:00:16.413 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:16.503 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:16.741 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:00:16.854 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:16.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:16.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:16.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:16 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:16.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:16 smithi055 ceph-mon[179668]: failed to encode map e191 with expected crc 2023-09-23T18:00:16.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:16 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:00:16.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:16 smithi055 ceph-mon[179668]: osdmap e191: 8 total, 6 up, 7 in 2023-09-23T18:00:16.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:16 smithi055 ceph-mon[179668]: failed to encode map e191 with expected crc 2023-09-23T18:00:16.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:16 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:16.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:16 smithi055 ceph-mon[184174]: failed to encode map e191 with expected crc 2023-09-23T18:00:16.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:16 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:00:16.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:16 smithi055 ceph-mon[184174]: osdmap e191: 8 total, 6 up, 7 in 2023-09-23T18:00:16.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:16 smithi055 ceph-mon[184174]: failed to encode map e191 with expected crc 2023-09-23T18:00:16.974 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:17.064 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:17.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:16 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:17.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:16 smithi099 ceph-mon[167076]: failed to encode map e191 with expected crc 2023-09-23T18:00:17.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:16 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:00:17.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:16 smithi099 ceph-mon[167076]: osdmap e191: 8 total, 6 up, 7 in 2023-09-23T18:00:17.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:16 smithi099 ceph-mon[167076]: failed to encode map e191 with expected crc 2023-09-23T18:00:17.133 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:17.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:17.145 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:17.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:17.281 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:17.324 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:00:17.331 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:17.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:17.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:17.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:17.474 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:17.474 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:00:17.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:17.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:17.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:17.623 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:17.761 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:17.837 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:17.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:17.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:17.945 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:18.001 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:17 smithi099 ceph-mon[167076]: pgmap v440: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.2 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 6699/57882 objects degraded (11.574%); 19561/57882 objects misplaced (33.795%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:18.004 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:18.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:18.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:18.056 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:18.156 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:18.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:17 smithi055 ceph-mon[184174]: pgmap v440: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.2 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 6699/57882 objects degraded (11.574%); 19561/57882 objects misplaced (33.795%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:18.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:17 smithi055 ceph-mon[179668]: pgmap v440: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.2 GiB data, 6.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 6699/57882 objects degraded (11.574%); 19561/57882 objects misplaced (33.795%); 1.7 MiB/s, 59 objects/s recovering 2023-09-23T18:00:18.182 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:18.215 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:00:18.250 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:18.888 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:00:18.888 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:00:18.888 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:00:18.888 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:00:18.888 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:00:18.918 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 6699/57882 objects degraded (11.574%), 15 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:18.918 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:18 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/434204032' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T18:00:18.938 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 1 pgp_num to 21 2023-09-23T18:00:19.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 6699/57882 objects degraded (11.574%), 15 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:19.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:18 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/434204032' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T18:00:19.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 6699/57882 objects degraded (11.574%), 15 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:19.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:18 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/434204032' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]: dispatch 2023-09-23T18:00:19.191 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:19.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:19.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:19.300 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:19.377 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:19.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:00:20.051 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:20.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:20.057 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:20.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:20.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:19 smithi055 ceph-mon[184174]: pgmap v441: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.1 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 6682/55518 objects degraded (12.036%); 18971/55518 objects misplaced (34.171%); 1.7 MiB/s, 61 objects/s recovering 2023-09-23T18:00:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:19 smithi055 ceph-mon[184174]: failed to encode map e192 with expected crc 2023-09-23T18:00:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:19 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/434204032' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T18:00:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:19 smithi055 ceph-mon[184174]: osdmap e192: 8 total, 6 up, 7 in 2023-09-23T18:00:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:19 smithi055 ceph-mon[184174]: failed to encode map e192 with expected crc 2023-09-23T18:00:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:19 smithi055 ceph-mon[179668]: pgmap v441: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.1 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 6682/55518 objects degraded (12.036%); 18971/55518 objects misplaced (34.171%); 1.7 MiB/s, 61 objects/s recovering 2023-09-23T18:00:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:19 smithi055 ceph-mon[179668]: failed to encode map e192 with expected crc 2023-09-23T18:00:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:19 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/434204032' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T18:00:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:19 smithi055 ceph-mon[179668]: osdmap e192: 8 total, 6 up, 7 in 2023-09-23T18:00:20.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:19 smithi055 ceph-mon[179668]: failed to encode map e192 with expected crc 2023-09-23T18:00:20.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:19 smithi099 ceph-mon[167076]: pgmap v441: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.1 GiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 620 op/s; 6682/55518 objects degraded (12.036%); 18971/55518 objects misplaced (34.171%); 1.7 MiB/s, 61 objects/s recovering 2023-09-23T18:00:20.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:19 smithi099 ceph-mon[167076]: failed to encode map e192 with expected crc 2023-09-23T18:00:20.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:19 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/434204032' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "21"}]': finished 2023-09-23T18:00:20.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:19 smithi099 ceph-mon[167076]: osdmap e192: 8 total, 6 up, 7 in 2023-09-23T18:00:20.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:19 smithi099 ceph-mon[167076]: failed to encode map e192 with expected crc 2023-09-23T18:00:20.205 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:20.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:20.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:20.280 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:20.381 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:20.487 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:20.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:00:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:00:20] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-09-23T18:00:20.728 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:00:21.027 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:21.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:21.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:21.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:21.206 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:21.570 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:21.778 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:21.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:21.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:21.886 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:21.920 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:21.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:21.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:21.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:22.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:21 smithi099 ceph-mon[167076]: pgmap v443: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.0 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 5941/50046 objects degraded (11.871%); 18055/50046 objects misplaced (36.077%); 2.3 MiB/s, 59 objects/s recovering 2023-09-23T18:00:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:22.077 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:22.172 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:22.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:21 smithi055 ceph-mon[179668]: pgmap v443: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.0 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 5941/50046 objects degraded (11.871%); 18055/50046 objects misplaced (36.077%); 2.3 MiB/s, 59 objects/s recovering 2023-09-23T18:00:22.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:22.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:21 smithi055 ceph-mon[184174]: pgmap v443: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 1.0 GiB data, 6.0 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 645 op/s; 5941/50046 objects degraded (11.871%); 18055/50046 objects misplaced (36.077%); 2.3 MiB/s, 59 objects/s recovering 2023-09-23T18:00:22.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:22.350 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:22.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:22.359 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:22.393 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:00:22.441 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:00:22.460 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:22.491 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:00:22.540 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:00:22.549 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:22.585 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:00:22.627 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:00:22.628 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:00:22.628 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:00:22.628 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:00:22.628 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:00:22.628 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:00:22.628 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:00:22.629 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:00:22.629 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:00:22.678 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:22.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:22.695 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:22.795 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:22.881 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:23.078 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:23.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:23.090 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:23.123 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:00:23.123 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:00:23.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:23.211 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/648793660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:23.211 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:22 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 5941/50046 objects degraded (11.871%), 15 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:23.211 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/648793660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:23.211 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:22 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 5941/50046 objects degraded (11.871%), 15 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:23.241 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/648793660' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:23.242 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:22 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 5941/50046 objects degraded (11.871%), 15 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:23.249 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:23.303 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:23.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:23.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:23.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:23.467 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:23.530 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:23.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:23.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:23.639 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:23.686 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:23.746 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:23.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:23.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:23.828 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:00:23.828 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:00:23.850 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:23.935 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:23.994 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:00:24.027 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:24.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:24.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:23 smithi099 ceph-mon[167076]: pgmap v444: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 965 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 5939/46182 objects degraded (12.860%); 17208/46182 objects misplaced (37.261%); 1.2 MiB/s, 65 objects/s recovering 2023-09-23T18:00:24.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:23 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/255536819' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:00:24.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:23 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:00:24.132 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:24.145 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:23 smithi055 ceph-mon[184174]: pgmap v444: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 965 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 5939/46182 objects degraded (12.860%); 17208/46182 objects misplaced (37.261%); 1.2 MiB/s, 65 objects/s recovering 2023-09-23T18:00:24.145 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:23 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/255536819' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:00:24.145 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:23 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:00:24.145 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:23 smithi055 ceph-mon[179668]: pgmap v444: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 965 MiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 5939/46182 objects degraded (12.860%); 17208/46182 objects misplaced (37.261%); 1.2 MiB/s, 65 objects/s recovering 2023-09-23T18:00:24.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:23 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/255536819' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:00:24.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:23 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:00:24.184 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:24.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:00:24.408 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:24.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:24.412 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:24.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:24.577 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:24.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:24.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:24.643 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:24.743 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:24.789 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:24.834 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:24.842 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:24.842 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:24.943 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:24.991 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:25.016 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:00:25.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:25.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:25.047 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:25.124 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:00:25.148 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:25.179 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:24 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:25.179 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:24 smithi099 ceph-mon[167076]: failed to encode map e193 with expected crc 2023-09-23T18:00:25.179 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:24 smithi099 ceph-mon[167076]: failed to encode map e193 with expected crc 2023-09-23T18:00:25.179 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:24 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:00:25.179 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:24 smithi099 ceph-mon[167076]: osdmap e193: 8 total, 6 up, 7 in 2023-09-23T18:00:25.184 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:25.290 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:25.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:25.296 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:25.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:25.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:24 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:25.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:24 smithi055 ceph-mon[184174]: failed to encode map e193 with expected crc 2023-09-23T18:00:25.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:24 smithi055 ceph-mon[184174]: failed to encode map e193 with expected crc 2023-09-23T18:00:25.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:24 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:00:25.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:24 smithi055 ceph-mon[184174]: osdmap e193: 8 total, 6 up, 7 in 2023-09-23T18:00:25.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:24 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:25.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:24 smithi055 ceph-mon[179668]: failed to encode map e193 with expected crc 2023-09-23T18:00:25.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:24 smithi055 ceph-mon[179668]: failed to encode map e193 with expected crc 2023-09-23T18:00:25.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:24 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:00:25.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:24 smithi055 ceph-mon[179668]: osdmap e193: 8 total, 6 up, 7 in 2023-09-23T18:00:25.441 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:25.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:25.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:25.557 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:25.659 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:25.760 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:25.842 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:25.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:25.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:25.950 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:25.997 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:26.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:26 smithi055 ceph-mon[179668]: pgmap v446: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 837 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 5179/40059 objects degraded (12.928%); 16072/40059 objects misplaced (40.121%); 2.4 MiB/s, 84 objects/s recovering 2023-09-23T18:00:26.048 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:26.137 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T18:00:26.137 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:00:26.137 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T18:00:26.137 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:00:26.192 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:26 smithi099 ceph-mon[167076]: pgmap v446: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 837 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 5179/40059 objects degraded (12.928%); 16072/40059 objects misplaced (40.121%); 2.4 MiB/s, 84 objects/s recovering 2023-09-23T18:00:26.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:26 smithi055 ceph-mon[184174]: pgmap v446: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 837 MiB data, 5.6 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 741 op/s; 5179/40059 objects degraded (12.928%); 16072/40059 objects misplaced (40.121%); 2.4 MiB/s, 84 objects/s recovering 2023-09-23T18:00:26.459 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:00:26.460 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:00:26.502 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:00:26.847 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:00:26.902 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:26.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:26.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:27.008 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:27.080 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:27.234 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:27.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:27.250 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:27.350 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:27.436 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:27.572 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:00:27.966 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:27.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:27.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:28.076 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:28 smithi099 ceph-mon[167076]: pgmap v447: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 801 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 5179/38304 objects degraded (13.521%); 15685/38304 objects misplaced (40.949%); 2.4 MiB/s, 61 objects/s recovering 2023-09-23T18:00:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 5179/38304 objects degraded (13.521%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:28.154 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:28.251 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:28.260 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:28 smithi055 ceph-mon[179668]: pgmap v447: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 801 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 5179/38304 objects degraded (13.521%); 15685/38304 objects misplaced (40.949%); 2.4 MiB/s, 61 objects/s recovering 2023-09-23T18:00:28.260 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 5179/38304 objects degraded (13.521%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:28.261 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:28 smithi055 ceph-mon[184174]: pgmap v447: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 801 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 5179/38304 objects degraded (13.521%); 15685/38304 objects misplaced (40.949%); 2.4 MiB/s, 61 objects/s recovering 2023-09-23T18:00:28.261 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 5179/38304 objects degraded (13.521%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:28.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:28.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:28.363 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:28.426 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:28.650 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:28.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:28.659 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:28.706 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:28.760 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:28.809 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:28.878 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:00:28.878 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:00:28.879 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:29.074 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:29.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:29.081 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:29.182 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:29.245 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:29.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:00:29.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:29.317 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:29.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:29.418 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:29.459 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:29.514 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:29.521 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:29.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:29.692 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:29.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:29.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:29.740 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:29.841 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:29.914 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:00:29.922 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:29.953 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:00:29.970 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:00:30.008 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:30.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:30.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:30.030 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:00:30.082 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:00:30.120 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:00:30.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:30.164 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:30.165 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:00:30.174 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:00:30.174 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:00:30.174 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:00:30.174 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:00:30.174 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:00:30.174 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:00:30.174 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:00:30.174 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:00:30.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:30.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:30.228 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:30.269 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:30 smithi055 ceph-mon[184174]: pgmap v448: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 749 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 5179/35796 objects degraded (14.468%); 15067/35796 objects misplaced (42.091%); 2.0 MiB/s, 71 objects/s recovering 2023-09-23T18:00:30.269 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2525809986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:30.272 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:30 smithi055 ceph-mon[179668]: pgmap v448: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 749 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 5179/35796 objects degraded (14.468%); 15067/35796 objects misplaced (42.091%); 2.0 MiB/s, 71 objects/s recovering 2023-09-23T18:00:30.272 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2525809986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:30.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:30.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:30 smithi099 ceph-mon[167076]: pgmap v448: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 749 MiB data, 5.4 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 678 op/s; 5179/35796 objects degraded (14.468%); 15067/35796 objects misplaced (42.091%); 2.0 MiB/s, 71 objects/s recovering 2023-09-23T18:00:30.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2525809986' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:30.387 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:30.491 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:30.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:30.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:30.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:30.637 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:30.666 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:00:30.667 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:00:30.668 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:00:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:00:30] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2023-09-23T18:00:30.732 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:30.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:30.740 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:30.841 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:30.875 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:30.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:30.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:30.935 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:31.036 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:31.085 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:31.127 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:31.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:31.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:31.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:31.296 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:31.299 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:00:31.299 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:00:31.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:00:31.299 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:00:31.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131500 5.2 5.9981e372 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.276336+0000", 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 117.02030999599999, 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94858491499999997, 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131500 2023-09-23T18:00:31.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276336+0000", 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276336+0000", 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276335+0000", 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276340+0000", 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2310000000000004e-06 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276348+0000", 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.303e-06 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276356+0000", 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9359999999999999e-06 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276427+0000", 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0314000000000005e-05 2023-09-23T18:00:31.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276438+0000", 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1035e-05 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224903+0000", 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94846568499999995 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224921+0000", 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7461000000000001e-05 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131508 5.2 5.89ad37e2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.286747+0000", 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 117.009899277, 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.93821040499999997, 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.305 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131508 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286747+0000", 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286747+0000", 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286745+0000", 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286752+0000", 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0550000000000002e-06 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286767+0000", 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4283e-05 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286775+0000", 2023-09-23T18:00:31.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1780000000000003e-06 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286834+0000", 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8866999999999997e-05 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286842+0000", 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7590000000000003e-06 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224948+0000", 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.938105666 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224957+0000", 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9469999999999992e-06 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131523 5.2 5.551f4f92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.311020+0000", 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 116.985625777, 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91399190200000002, 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131523 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.311 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311020+0000", 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311020+0000", 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311018+0000", 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311028+0000", 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5859999999999994e-06 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311050+0000", 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2079e-05 2023-09-23T18:00:31.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311059+0000", 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9959999999999999e-06 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311123+0000", 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4539e-05 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311130+0000", 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.917e-06 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.225002+0000", 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91387170399999995 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.225012+0000", 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0132000000000001e-05 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142717 5.9 5:97ac1ed5:::benchmark_data_smithi055_229117_object37091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.884415+0000", 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.412231098000007, 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0500839660000001, 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.316 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142717 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884415+0000", 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884415+0000", 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884413+0000", 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884424+0000", 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0804e-05 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884444+0000", 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0057000000000001e-05 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884453+0000", 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8000000000000004e-06 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884524+0000", 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0729000000000007e-05 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884537+0000", 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3059e-05 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903080+0000", 2023-09-23T18:00:31.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0185432940000001 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903187+0000", 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010691700000000001 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903271+0000", 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4103000000000004e-05 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.911979+0000", 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0087084599999999995 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934336+0000", 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022356767999999999 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934371+0000", 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4693999999999997e-05 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934420+0000", 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8755000000000003e-05 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934499+0000", 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9129999999999996e-05 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.323 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142734 5.9 5:9b1dd87e:::benchmark_data_smithi055_229117_object37108:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.916812+0000", 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.379833966999996, 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0179610699999999, 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.324 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142734 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916812+0000", 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916812+0000", 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916810+0000", 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916818+0000", 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7209999999999994e-06 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916838+0000", 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9941000000000001e-05 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916846+0000", 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5190000000000003e-06 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917145+0000", 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00029908799999999998 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917153+0000", 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7700000000000007e-06 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.904847+0000", 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98769368300000004 2023-09-23T18:00:31.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.904960+0000", 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112802 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.905068+0000", 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010848700000000001 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915793+0000", 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010724450999999999 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934547+0000", 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018754603000000002 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934612+0000", 2023-09-23T18:00:31.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4272999999999999e-05 2023-09-23T18:00:31.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.331 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934667+0000", 2023-09-23T18:00:31.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4814000000000001e-05 2023-09-23T18:00:31.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.332 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934773+0000", 2023-09-23T18:00:31.332 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106517 2023-09-23T18:00:31.332 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.332 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142738 5.9 5:9dbd7dd5:::benchmark_data_smithi055_229117_object37112:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.379784444999999, 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018182892, 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.333 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142738 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916864+0000", 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0709999999999999e-06 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916870+0000", 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8780000000000003e-06 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916882+0000", 2023-09-23T18:00:31.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2558000000000001e-05 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917176+0000", 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00029325599999999999 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917181+0000", 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6400000000000002e-06 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906621+0000", 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98943947600000004 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906744+0000", 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012348099999999999 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906869+0000", 2023-09-23T18:00:31.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012421599999999999 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915800+0000", 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0089313259999999998 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934813+0000", 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019012940999999998 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934959+0000", 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000145921 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934994+0000", 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5707999999999998e-05 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935044+0000", 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9951000000000003e-05 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142777 5.9 5:91082f54:::benchmark_data_smithi055_229117_object37151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.976270+0000", 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.320375811999995, 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.958897997, 2023-09-23T18:00:31.341 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142777 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976270+0000", 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976270+0000", 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976269+0000", 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.343 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976276+0000", 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2150000000000004e-06 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976288+0000", 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2092999999999999e-05 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976293+0000", 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1610000000000002e-06 2023-09-23T18:00:31.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976338+0000", 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4116000000000002e-05 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976346+0000", 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9200000000000004e-06 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908348+0000", 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.93200213499999995 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908468+0000", 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120746 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908590+0000", 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121772 2023-09-23T18:00:31.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915806+0000", 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0072161990000000004 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934909+0000", 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019102811000000001 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935083+0000", 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000173499 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935114+0000", 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1501999999999998e-05 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935168+0000", 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3947999999999999e-05 2023-09-23T18:00:31.348 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142802 5.9 5:92d6f126:::benchmark_data_smithi055_229117_object37176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.030542+0000", 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.266103764999997, 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.90473668699999998, 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.349 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142802 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030542+0000", 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030542+0000", 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030541+0000", 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030547+0000", 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5289999999999997e-06 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030566+0000", 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8630999999999998e-05 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030571+0000", 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0259999999999998e-06 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030617+0000", 2023-09-23T18:00:31.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5439999999999999e-05 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030623+0000", 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1999999999999999e-06 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910103+0000", 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.87947978999999998 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.353 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910251+0000", 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014896700000000001 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910377+0000", 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125425 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920175+0000", 2023-09-23T18:00:31.354 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097978730000000003 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934939+0000", 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014764019 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935202+0000", 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026282399999999999 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935229+0000", 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7793999999999999e-05 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935279+0000", 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9428999999999997e-05 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142809 5.9 5:9f3ab830:::benchmark_data_smithi055_229117_object37183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.039995+0000", 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.256650828999994, 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.902436291, 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142809 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039995+0000", 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039995+0000", 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039994+0000", 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040001+0000", 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5259999999999996e-06 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040017+0000", 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5794000000000002e-05 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040024+0000", 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3379999999999999e-06 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040079+0000", 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4866e-05 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040091+0000", 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1362000000000001e-05 2023-09-23T18:00:31.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912030+0000", 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.87193905100000002 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912160+0000", 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013052100000000001 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.361 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912267+0000", 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010700399999999999 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920180+0000", 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079123509999999998 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.362 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935312+0000", 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01513201 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942274+0000", 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069623669999999997 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942331+0000", 2023-09-23T18:00:31.363 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7516000000000003e-05 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942431+0000", 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9986000000000001e-05 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.364 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142838 5.9 5:9cf2a92c:::benchmark_data_smithi055_229117_object37212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.111228+0000", 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.185418092999996, 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.83141186199999995, 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142838 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.365 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111228+0000", 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111228+0000", 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111227+0000", 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:00:31.366 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111233+0000", 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3160000000000002e-06 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111246+0000", 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2884000000000001e-05 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111251+0000", 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2360000000000004e-06 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111297+0000", 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5881000000000003e-05 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111307+0000", 2023-09-23T18:00:31.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0046e-05 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914054+0000", 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.80274739399999995 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914133+0000", 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8448999999999994e-05 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.370 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.370 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914206+0000", 2023-09-23T18:00:31.370 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3325999999999996e-05 2023-09-23T18:00:31.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.370 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.370 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925627+0000", 2023-09-23T18:00:31.370 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011421277000000001 2023-09-23T18:00:31.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935332+0000", 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097046300000000005 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942488+0000", 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007155603 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.371 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942537+0000", 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9391000000000002e-05 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942640+0000", 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102618 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.372 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142868 5.9 5:93088e3b:::benchmark_data_smithi055_229117_object37242:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.164405+0000", 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.132240538000005, 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.78228625900000004, 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142868 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.373 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164405+0000", 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164405+0000", 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164404+0000", 2023-09-23T18:00:31.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164410+0000", 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1349999999999997e-06 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164422+0000", 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1647e-05 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.375 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164437+0000", 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5234e-05 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164485+0000", 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7363e-05 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164493+0000", 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5429999999999994e-06 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915840+0000", 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.75134721800000004 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915919+0000", 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8091999999999997e-05 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915992+0000", 2023-09-23T18:00:31.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3733999999999996e-05 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925638+0000", 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096452810000000003 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935357+0000", 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097193910000000008 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.378 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946534+0000", 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011177302 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946593+0000", 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8563000000000002e-05 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946692+0000", 2023-09-23T18:00:31.379 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8836e-05 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142887 5.9 5:980e2e89:::benchmark_data_smithi055_229117_object37261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.211074+0000", 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.085571454999993, 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73576580700000005, 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.380 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142887 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211074+0000", 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.381 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211074+0000", 2023-09-23T18:00:31.382 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.382 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211073+0000", 2023-09-23T18:00:31.382 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.382 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211080+0000", 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3629999999999996e-06 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211097+0000", 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6917000000000001e-05 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211103+0000", 2023-09-23T18:00:31.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2499999999999997e-06 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211145+0000", 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2466000000000003e-05 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211151+0000", 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0410000000000003e-06 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918068+0000", 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70691734399999995 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918196+0000", 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000128081 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918320+0000", 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012403300000000001 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930458+0000", 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012137652000000001 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935374+0000", 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049156570000000004 2023-09-23T18:00:31.386 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946735+0000", 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011361616999999999 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946767+0000", 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1952999999999997e-05 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.387 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946840+0000", 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2910000000000005e-05 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142888 5.9 5:99a82285:::benchmark_data_smithi055_229117_object37262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.085531639999999, 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73588783199999996, 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.388 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142888 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211117+0000", 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4680000000000001e-06 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211121+0000", 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7610000000000001e-06 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211124+0000", 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3489999999999999e-06 2023-09-23T18:00:31.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211177+0000", 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3133000000000001e-05 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211184+0000", 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6240000000000002e-06 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920219+0000", 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70903441 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920327+0000", 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108477 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920402+0000", 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5252000000000004e-05 2023-09-23T18:00:31.393 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930467+0000", 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010064867999999999 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.937129+0000", 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066614200000000004 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.394 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946878+0000", 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097489120000000002 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946919+0000", 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1031000000000001e-05 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.947002+0000", 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3475999999999998e-05 2023-09-23T18:00:31.395 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142904 5.9 5:9b2141c3:::benchmark_data_smithi055_229117_object37278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.244955+0000", 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.051690578999995, 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70391956499999997, 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.396 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142904 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244955+0000", 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.397 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244955+0000", 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244954+0000", 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244960+0000", 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:00:31.398 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244973+0000", 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3209e-05 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244978+0000", 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1930000000000001e-06 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.399 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245017+0000", 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8452999999999999e-05 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245023+0000", 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.409e-06 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.921896+0000", 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67687305900000005 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.400 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.921997+0000", 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100753 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.922064+0000", 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6978999999999997e-05 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930475+0000", 2023-09-23T18:00:31.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0084114670000000006 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.940046+0000", 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095709799999999998 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948781+0000", 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0087346990000000003 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948813+0000", 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1779000000000002e-05 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948875+0000", 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2073e-05 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.403 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142905 5.9 5:951a98af:::benchmark_data_smithi055_229117_object37279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.051316709999995, 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71295564, 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142905 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.404 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.405 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245332+0000", 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4400000000000001e-06 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245339+0000", 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4749999999999998e-06 2023-09-23T18:00:31.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245343+0000", 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1370000000000004e-06 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245381+0000", 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8396999999999998e-05 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245386+0000", 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2460000000000003e-06 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923706+0000", 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67831945699999996 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923849+0000", 2023-09-23T18:00:31.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143669 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923977+0000", 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012721600000000001 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933852+0000", 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0098750190000000005 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.940069+0000", 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062175399999999997 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958137+0000", 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018067588999999998 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.410 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958189+0000", 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1981000000000003e-05 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958285+0000", 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6036e-05 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.411 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142919 5.9 5:9348e633:::benchmark_data_smithi055_229117_object37293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.270585+0000", 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 99.026061077999998, 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.69084338599999995, 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142919 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.412 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270585+0000", 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270585+0000", 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270583+0000", 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270591+0000", 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2910000000000002e-06 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270604+0000", 2023-09-23T18:00:31.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3349999999999999e-05 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270609+0000", 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1109999999999999e-06 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270662+0000", 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3236000000000003e-05 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.415 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270673+0000", 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0560999999999999e-05 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925683+0000", 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.65501024500000005 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925803+0000", 2023-09-23T18:00:31.416 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119807 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925920+0000", 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117419 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933861+0000", 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079407209999999995 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.417 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944507+0000", 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010645846 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961259+0000", 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016751986999999999 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961315+0000", 2023-09-23T18:00:31.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6477999999999997e-05 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961428+0000", 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011292400000000001 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142929 5.9 5:954e3e29:::benchmark_data_smithi055_229117_object37303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.419 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.300689+0000", 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 98.995957363000002, 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.662469262, 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142929 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.420 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300689+0000", 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300689+0000", 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300687+0000", 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.421 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300694+0000", 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0219999999999999e-06 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300705+0000", 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0886e-05 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300711+0000", 2023-09-23T18:00:31.422 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9490000000000001e-06 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300749+0000", 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8077999999999997e-05 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300755+0000", 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5389999999999998e-06 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.423 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928440+0000", 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62768513800000003 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928576+0000", 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013626900000000001 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928683+0000", 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010661400000000001 2023-09-23T18:00:31.424 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933946+0000", 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052633460000000003 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944539+0000", 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010592571 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.425 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963005+0000", 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018466598000000001 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963057+0000", 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1529000000000001e-05 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963158+0000", 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010085399999999999 2023-09-23T18:00:31.426 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142941 5.9 5:913fcfd6:::benchmark_data_smithi055_229117_object37315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.340683+0000", 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 98.955963026000006, 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62261945900000004, 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.427 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142941 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340683+0000", 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340683+0000", 2023-09-23T18:00:31.428 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340682+0000", 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340688+0000", 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.319e-06 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.429 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340701+0000", 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3529999999999999e-05 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340708+0000", 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8449999999999997e-06 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340755+0000", 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6428999999999999e-05 2023-09-23T18:00:31.430 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340762+0000", 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.571e-06 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930510+0000", 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58974757200000005 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930615+0000", 2023-09-23T18:00:31.431 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000105213 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930695+0000", 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0055000000000005e-05 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933952+0000", 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003257228 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.432 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944608+0000", 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010655243 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963204+0000", 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018596950000000001 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963236+0000", 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1894000000000003e-05 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.433 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963302+0000", 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6013999999999995e-05 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142946 5.9 5:9099fb2d:::benchmark_data_smithi055_229117_object37320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.370819+0000", 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 98.925826935999993, 2023-09-23T18:00:31.434 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59265797899999995, 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142946 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370819+0000", 2023-09-23T18:00:31.435 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370819+0000", 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370817+0000", 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.436 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370824+0000", 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8299999999999998e-06 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370837+0000", 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2716e-05 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370844+0000", 2023-09-23T18:00:31.437 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8789999999999997e-06 2023-09-23T18:00:31.438 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.438 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.438 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.438 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370887+0000", 2023-09-23T18:00:31.438 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2879000000000001e-05 2023-09-23T18:00:31.438 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.438 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.438 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:00:31.438 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370893+0000", 2023-09-23T18:00:31.438 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1120000000000001e-06 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932242+0000", 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.56134879999999998 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932323+0000", 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0871000000000005e-05 2023-09-23T18:00:31.439 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932392+0000", 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9295999999999995e-05 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934346+0000", 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019539589999999999 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.440 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944640+0000", 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010293754 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963339+0000", 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018699684000000001 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963384+0000", 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4499999999999997e-05 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.441 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963477+0000", 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3245000000000006e-05 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:172845 5.9 5:99e32625:::benchmark_data_smithi055_229117_object67219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:42.052464+0000", 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 49.244182207999998, 2023-09-23T18:00:31.442 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.074629107, 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 172845 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052464+0000", 2023-09-23T18:00:31.443 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052464+0000", 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052462+0000", 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.444 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052470+0000", 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.559e-06 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052486+0000", 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6611000000000001e-05 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:00:31.445 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052493+0000", 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0380000000000002e-06 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.053731+0000", 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012380329999999999 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.053877+0000", 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014536799999999999 2023-09-23T18:00:31.446 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.054023+0000", 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014621600000000001 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.055833+0000", 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001809969 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.447 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.060974+0000", 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051409100000000003 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.126977+0000", 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.066003662000000005 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.127023+0000", 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5671999999999999e-05 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:00:31.448 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:00:31.449 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.127093+0000", 2023-09-23T18:00:31.449 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9688000000000001e-05 2023-09-23T18:00:31.449 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.449 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.449 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.449 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:00:31.449 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:00:31.449 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:00:31.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:31.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:31.450 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:31.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:31.599 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:31.631 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:00:31.633 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:00:31.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:31.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:31.699 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:31.768 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:00:31.800 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:31.843 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:31.900 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:31.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:31.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:32.004 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:32.089 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:32.105 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:00:32.138 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":449,"stamp":"2023-09-23T18:00:30.601821+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":655890379,"num_objects":9964,"num_object_clones":0,"num_object_copies":29892,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4352,"num_objects_misplaced":14092,"num_objects_unfound":0,"num_objects_dirty":9964,"num_whiteouts":0,"num_read":6439,"num_read_kb":13059,"num_write":202458,"num_write_kb":6775014,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40093,"num_bytes_recovered":2536639367,"num_keys_recovered":0,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":114758,"ondisk_log_size":114758,"up":261,"acting":278,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":350,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5374716,"kb_used_data":4426116,"kb_used_omap":36,"kb_used_meta":947931,"kb_avail":650841348,"statfs":{"total":671965249536,"available":666461540352,"internally_reserved":0,"allocated":4532342784,"data_stored":4526521746,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":37596,"internal_metadata":970681636},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-440270848,"num_objects":-6718,"num_object_clones":0,"num_object_copies":-20154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1589,"num_objects_misplaced":-3963,"num_objects_unfound":0,"num_objects_dirty":-6718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6718,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":695,"num_bytes_recovered":20185088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.009639"},"pg_stats":[{"pgid":"3.1a","version":"67'1","reported_seq":79,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.921213+0000","last_change":"2023-09-23T17:59:50.896420+0000","last_active":"2023-09-23T18:00:18.921213+0000","last_peered":"2023-09-23T18:00:18.921213+0000","last_clean":"2023-09-23T18:00:18.921213+0000","last_became_active":"2023-09-23T17:59:50.896017+0000","last_became_peered":"2023-09-23T17:59:50.896017+0000","last_unstale":"2023-09-23T18:00:18.921213+0000","last_undegraded":"2023-09-23T18:00:18.921213+0000","last_fullsized":"2023-09-23T18:00:18.921213+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:06.664670+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":159,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.935665+0000","last_change":"2023-09-23T17:57:24.956156+0000","last_active":"2023-09-23T18:00:18.935665+0000","last_peered":"2023-09-23T18:00:18.935665+0000","last_clean":"2023-09-23T17:56:46.613857+0000","last_became_active":"2023-09-23T17:57:24.956156+0000","last_became_peered":"2023-09-23T17:57:24.956156+0000","last_unstale":"2023-09-23T18:00:18.935665+0000","last_undegraded":"2023-09-23T18:00:18.935665+0000","last_fullsized":"2023-09-23T17:57:24.947251+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:35.949179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":157,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.937010+0000","last_change":"2023-09-23T17:57:24.956353+0000","last_active":"2023-09-23T18:00:18.937010+0000","last_peered":"2023-09-23T18:00:18.937010+0000","last_clean":"2023-09-23T17:57:20.922940+0000","last_became_active":"2023-09-23T17:57:24.956353+0000","last_became_peered":"2023-09-23T17:57:24.956353+0000","last_unstale":"2023-09-23T18:00:18.937010+0000","last_undegraded":"2023-09-23T18:00:18.937010+0000","last_fullsized":"2023-09-23T17:57:24.946863+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:41.136418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":155,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.942977+0000","last_change":"2023-09-23T17:58:11.996431+0000","last_active":"2023-09-23T18:00:18.942977+0000","last_peered":"2023-09-23T18:00:18.942977+0000","last_clean":"2023-09-23T17:57:20.931176+0000","last_became_active":"2023-09-23T17:58:11.996431+0000","last_became_peered":"2023-09-23T17:58:11.996431+0000","last_unstale":"2023-09-23T18:00:18.942977+0000","last_undegraded":"2023-09-23T18:00:18.942977+0000","last_fullsized":"2023-09-23T17:58:11.991094+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:53:21.919913+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":140,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.934468+0000","last_change":"2023-09-23T17:57:24.965072+0000","last_active":"2023-09-23T18:00:18.934468+0000","last_peered":"2023-09-23T18:00:18.934468+0000","last_clean":"2023-09-23T17:53:21.039477+0000","last_became_active":"2023-09-23T17:57:24.965072+0000","last_became_peered":"2023-09-23T17:57:24.965072+0000","last_unstale":"2023-09-23T18:00:18.934468+0000","last_undegraded":"2023-09-23T18:00:18.934468+0000","last_fullsized":"2023-09-23T17:57:24.947630+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:40:41.627468+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":113,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.915344+0000","last_change":"2023-09-23T17:59:09.861723+0000","last_active":"2023-09-23T18:00:18.915344+0000","last_peered":"2023-09-23T18:00:18.915344+0000","last_clean":"2023-09-23T18:00:18.915344+0000","last_became_active":"2023-09-23T17:59:09.861324+0000","last_became_peered":"2023-09-23T17:59:09.861324+0000","last_unstale":"2023-09-23T18:00:18.915344+0000","last_undegraded":"2023-09-23T18:00:18.915344+0000","last_fullsized":"2023-09-23T18:00:18.915344+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:21:45.865590+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":152,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.940485+0000","last_change":"2023-09-23T17:59:10.881778+0000","last_active":"2023-09-23T18:00:18.940485+0000","last_peered":"2023-09-23T18:00:18.940485+0000","last_clean":"2023-09-23T18:00:18.940485+0000","last_became_active":"2023-09-23T17:59:10.881369+0000","last_became_peered":"2023-09-23T17:59:10.881369+0000","last_unstale":"2023-09-23T18:00:18.940485+0000","last_undegraded":"2023-09-23T18:00:18.940485+0000","last_fullsized":"2023-09-23T18:00:18.940485+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:06.715447+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,5],"avail_no_missing":["0","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":142,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.931334+0000","last_change":"2023-09-23T17:58:35.245269+0000","last_active":"2023-09-23T18:00:18.931334+0000","last_peered":"2023-09-23T18:00:18.931334+0000","last_clean":"2023-09-23T17:55:56.840611+0000","last_became_active":"2023-09-23T17:58:35.245269+0000","last_became_peered":"2023-09-23T17:58:35.245269+0000","last_unstale":"2023-09-23T18:00:18.931334+0000","last_undegraded":"2023-09-23T18:00:18.931334+0000","last_fullsized":"2023-09-23T17:58:35.227362+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:14:50.366063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":492,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.937989+0000","last_change":"2023-09-23T17:59:00.132664+0000","last_active":"2023-09-23T18:00:18.937989+0000","last_peered":"2023-09-23T18:00:18.937989+0000","last_clean":"2023-09-23T18:00:18.937989+0000","last_became_active":"2023-09-23T17:59:00.132396+0000","last_became_peered":"2023-09-23T17:59:00.132396+0000","last_unstale":"2023-09-23T18:00:18.937989+0000","last_undegraded":"2023-09-23T18:00:18.937989+0000","last_fullsized":"2023-09-23T18:00:18.937989+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":139,"reported_epoch":192,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:00:18.935026+0000","last_change":"2023-09-23T17:59:10.880170+0000","last_active":"2023-09-23T18:00:18.935026+0000","last_peered":"2023-09-23T18:00:18.935026+0000","last_clean":"2023-09-23T17:57:47.590780+0000","last_became_active":"2023-09-23T17:59:10.880170+0000","last_became_peered":"2023-09-23T17:59:10.880170+0000","last_unstale":"2023-09-23T18:00:18.935026+0000","last_undegraded":"2023-09-23T18:00:18.935026+0000","last_fullsized":"2023-09-23T17:59:10.878551+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:31.432584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1,5],"avail_no_missing":["1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":49,"reported_epoch":192,"state":"down","last_fresh":"2023-09-23T18:00:18.915075+0000","last_change":"2023-09-23T17:58:34.212200+0000","last_active":"2023-09-23T17:45:51.086592+0000","last_peered":"2023-09-23T17:45:51.086592+0000","last_clean":"2023-09-23T17:45:51.086592+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:00:18.915075+0000","last_undegraded":"2023-09-23T18:00:18.915075+0000","last_fullsized":"2023-09-23T18:00:18.915075+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:32.367332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":173,"reported_epoch":192,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:18.922795+0000","last_change":"2023-09-23T17:59:50.897099+0000","last_active":"2023-09-23T18:00:18.922795+0000","last_peered":"2023-09-23T18:00:18.922795+0000","last_clean":"2023-09-23T17:56:46.621999+0000","last_became_active":"2023-09-23T17:59:50.897099+0000","last_became_peered":"2023-09-23T17:59:50.897099+0000","last_unstale":"2023-09-23T18:00:18.922795+0000","last_undegraded":"2023-09-23T17:59:50.892775+0000","last_fullsized":"2023-09-23T17:59:50.892676+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:04.659893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":180,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":148,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.931537+0000","last_change":"2023-09-23T17:58:35.248074+0000","last_active":"2023-09-23T18:00:18.931537+0000","last_peered":"2023-09-23T18:00:18.931537+0000","last_clean":"2023-09-23T18:00:18.931537+0000","last_became_active":"2023-09-23T17:58:35.247751+0000","last_became_peered":"2023-09-23T17:58:35.247751+0000","last_unstale":"2023-09-23T18:00:18.931537+0000","last_undegraded":"2023-09-23T18:00:18.931537+0000","last_fullsized":"2023-09-23T18:00:18.931537+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:08.488838+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":195,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.922737+0000","last_change":"2023-09-23T17:59:51.905994+0000","last_active":"2023-09-23T18:00:18.922737+0000","last_peered":"2023-09-23T18:00:18.922737+0000","last_clean":"2023-09-23T18:00:18.922737+0000","last_became_active":"2023-09-23T17:59:51.905695+0000","last_became_peered":"2023-09-23T17:59:51.905695+0000","last_unstale":"2023-09-23T18:00:18.922737+0000","last_undegraded":"2023-09-23T18:00:18.922737+0000","last_fullsized":"2023-09-23T18:00:18.922737+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:47.280585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4,6","objects":4}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":180,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.921654+0000","last_change":"2023-09-23T17:59:50.898040+0000","last_active":"2023-09-23T18:00:18.921654+0000","last_peered":"2023-09-23T18:00:18.921654+0000","last_clean":"2023-09-23T18:00:18.921654+0000","last_became_active":"2023-09-23T17:59:50.897702+0000","last_became_peered":"2023-09-23T17:59:50.897702+0000","last_unstale":"2023-09-23T18:00:18.921654+0000","last_undegraded":"2023-09-23T18:00:18.921654+0000","last_fullsized":"2023-09-23T18:00:18.921654+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:45.101467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":487,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.933814+0000","last_change":"2023-09-23T17:59:00.129648+0000","last_active":"2023-09-23T18:00:18.933814+0000","last_peered":"2023-09-23T18:00:18.933814+0000","last_clean":"2023-09-23T18:00:18.933814+0000","last_became_active":"2023-09-23T17:59:00.129253+0000","last_became_peered":"2023-09-23T17:59:00.129253+0000","last_unstale":"2023-09-23T18:00:18.933814+0000","last_undegraded":"2023-09-23T18:00:18.933814+0000","last_fullsized":"2023-09-23T18:00:18.933814+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":63,"reported_epoch":192,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:18.921765+0000","last_change":"2023-09-23T17:59:50.895581+0000","last_active":"2023-09-23T18:00:18.921765+0000","last_peered":"2023-09-23T18:00:18.921765+0000","last_clean":"2023-09-23T17:45:54.586105+0000","last_became_active":"2023-09-23T17:59:50.895581+0000","last_became_peered":"2023-09-23T17:59:50.895581+0000","last_unstale":"2023-09-23T18:00:18.921765+0000","last_undegraded":"2023-09-23T17:59:50.891295+0000","last_fullsized":"2023-09-23T17:59:50.891189+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:35.791975+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[{"shards":"2,4","objects":2}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":127,"reported_epoch":192,"state":"undersized+peered","last_fresh":"2023-09-23T18:00:18.933934+0000","last_change":"2023-09-23T17:57:53.869041+0000","last_active":"2023-09-23T17:57:52.875213+0000","last_peered":"2023-09-23T18:00:18.933934+0000","last_clean":"2023-09-23T17:53:21.028332+0000","last_became_active":"2023-09-23T17:57:24.954701+0000","last_became_peered":"2023-09-23T17:57:53.869041+0000","last_unstale":"2023-09-23T18:00:18.933934+0000","last_undegraded":"2023-09-23T18:00:18.933934+0000","last_fullsized":"2023-09-23T17:57:53.867092+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:44.473315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":488,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.933956+0000","last_change":"2023-09-23T17:59:00.141765+0000","last_active":"2023-09-23T18:00:18.933956+0000","last_peered":"2023-09-23T18:00:18.933956+0000","last_clean":"2023-09-23T18:00:18.933956+0000","last_became_active":"2023-09-23T17:59:00.137566+0000","last_became_peered":"2023-09-23T17:59:00.137566+0000","last_unstale":"2023-09-23T18:00:18.933956+0000","last_undegraded":"2023-09-23T18:00:18.933956+0000","last_fullsized":"2023-09-23T18:00:18.933956+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":154,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:19.496264+0000","last_change":"2023-09-23T17:58:11.995550+0000","last_active":"2023-09-23T18:00:19.496264+0000","last_peered":"2023-09-23T18:00:19.496264+0000","last_clean":"2023-09-23T17:57:20.931103+0000","last_became_active":"2023-09-23T17:58:11.995550+0000","last_became_peered":"2023-09-23T17:58:11.995550+0000","last_unstale":"2023-09-23T18:00:19.496264+0000","last_undegraded":"2023-09-23T18:00:19.496264+0000","last_fullsized":"2023-09-23T17:58:11.986762+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:18:31.912263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":26,"reported_epoch":192,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:00:18.914138+0000","last_change":"2023-09-23T17:59:10.875110+0000","last_active":"2023-09-23T18:00:18.914138+0000","last_peered":"2023-09-23T18:00:18.914138+0000","last_clean":"2023-09-23T17:57:20.925545+0000","last_became_active":"2023-09-23T17:59:10.875110+0000","last_became_peered":"2023-09-23T17:59:10.875110+0000","last_unstale":"2023-09-23T18:00:18.914138+0000","last_undegraded":"2023-09-23T18:00:18.914138+0000","last_fullsized":"2023-09-23T17:59:10.873146+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:35:28.623974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":488,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.936761+0000","last_change":"2023-09-23T17:59:00.134116+0000","last_active":"2023-09-23T18:00:18.936761+0000","last_peered":"2023-09-23T18:00:18.936761+0000","last_clean":"2023-09-23T18:00:18.936761+0000","last_became_active":"2023-09-23T17:59:00.133756+0000","last_became_peered":"2023-09-23T17:59:00.133756+0000","last_unstale":"2023-09-23T18:00:18.936761+0000","last_undegraded":"2023-09-23T18:00:18.936761+0000","last_fullsized":"2023-09-23T18:00:18.936761+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":180,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.921771+0000","last_change":"2023-09-23T17:58:36.251995+0000","last_active":"2023-09-23T18:00:18.921771+0000","last_peered":"2023-09-23T18:00:18.921771+0000","last_clean":"2023-09-23T18:00:18.921771+0000","last_became_active":"2023-09-23T17:58:36.251578+0000","last_became_peered":"2023-09-23T17:58:36.251578+0000","last_unstale":"2023-09-23T18:00:18.921771+0000","last_undegraded":"2023-09-23T18:00:18.921771+0000","last_fullsized":"2023-09-23T18:00:18.921771+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:45.817777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,0],"avail_no_missing":["4","0","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":154,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.943060+0000","last_change":"2023-09-23T17:58:11.996093+0000","last_active":"2023-09-23T18:00:18.943060+0000","last_peered":"2023-09-23T18:00:18.943060+0000","last_clean":"2023-09-23T17:57:20.931097+0000","last_became_active":"2023-09-23T17:58:11.996093+0000","last_became_peered":"2023-09-23T17:58:11.996093+0000","last_unstale":"2023-09-23T18:00:18.943060+0000","last_undegraded":"2023-09-23T18:00:18.943060+0000","last_fullsized":"2023-09-23T17:58:11.990799+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:58.128284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":488,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.937054+0000","last_change":"2023-09-23T17:59:00.133252+0000","last_active":"2023-09-23T18:00:18.937054+0000","last_peered":"2023-09-23T18:00:18.937054+0000","last_clean":"2023-09-23T18:00:18.937054+0000","last_became_active":"2023-09-23T17:59:00.132882+0000","last_became_peered":"2023-09-23T17:59:00.132882+0000","last_unstale":"2023-09-23T18:00:18.937054+0000","last_undegraded":"2023-09-23T18:00:18.937054+0000","last_fullsized":"2023-09-23T18:00:18.937054+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":119,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.913781+0000","last_change":"2023-09-23T17:59:10.878099+0000","last_active":"2023-09-23T18:00:18.913781+0000","last_peered":"2023-09-23T18:00:18.913781+0000","last_clean":"2023-09-23T18:00:18.913781+0000","last_became_active":"2023-09-23T17:59:10.877670+0000","last_became_peered":"2023-09-23T17:59:10.877670+0000","last_unstale":"2023-09-23T18:00:18.913781+0000","last_undegraded":"2023-09-23T18:00:18.913781+0000","last_fullsized":"2023-09-23T18:00:18.913781+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:36.244235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,5],"avail_no_missing":["6","0","5"],"object_location_counts":[{"shards":"0,5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":133,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.935974+0000","last_change":"2023-09-23T17:58:11.993687+0000","last_active":"2023-09-23T18:00:18.935974+0000","last_peered":"2023-09-23T18:00:18.935974+0000","last_clean":"2023-09-23T17:53:21.029434+0000","last_became_active":"2023-09-23T17:58:11.993687+0000","last_became_peered":"2023-09-23T17:58:11.993687+0000","last_unstale":"2023-09-23T18:00:18.935974+0000","last_undegraded":"2023-09-23T18:00:18.935974+0000","last_fullsized":"2023-09-23T17:58:11.985535+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:33.153133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":488,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.945405+0000","last_change":"2023-09-23T17:59:00.134603+0000","last_active":"2023-09-23T18:00:18.945405+0000","last_peered":"2023-09-23T18:00:18.945405+0000","last_clean":"2023-09-23T18:00:18.945405+0000","last_became_active":"2023-09-23T17:59:00.134099+0000","last_became_peered":"2023-09-23T17:59:00.134099+0000","last_unstale":"2023-09-23T18:00:18.945405+0000","last_undegraded":"2023-09-23T18:00:18.945405+0000","last_fullsized":"2023-09-23T18:00:18.945405+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.9","version":"193'12245","reported_seq":30066,"reported_epoch":193,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:28.239749+0000","last_change":"2023-09-23T17:58:52.934142+0000","last_active":"2023-09-23T18:00:28.239749+0000","last_peered":"2023-09-23T18:00:28.239749+0000","last_clean":"2023-09-23T18:00:28.239749+0000","last_became_active":"2023-09-23T17:58:52.879112+0000","last_became_peered":"2023-09-23T17:58:52.879112+0000","last_unstale":"2023-09-23T18:00:28.239749+0000","last_undegraded":"2023-09-23T18:00:28.239749+0000","last_fullsized":"2023-09-23T18:00:28.239749+0000","mapping_epoch":168,"log_start":"145'6818","ondisk_log_start":"145'6818","created":110,"last_epoch_clean":169,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5427,"log_dups_size":0,"ondisk_log_size":5427,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:54.486328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37683200,"num_objects":575,"num_object_clones":0,"num_object_copies":1725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":575,"num_objects_unfound":0,"num_objects_dirty":575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12245,"num_write_kb":410240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8835,"num_bytes_recovered":578945024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,1],"avail_no_missing":["6","1","4"],"object_location_counts":[{"shards":"1,4,6","objects":575}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":157,"reported_epoch":192,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:18.921917+0000","last_change":"2023-09-23T17:57:53.875345+0000","last_active":"2023-09-23T18:00:18.921917+0000","last_peered":"2023-09-23T18:00:18.921917+0000","last_clean":"2023-09-23T17:56:46.621793+0000","last_became_active":"2023-09-23T17:57:53.875345+0000","last_became_peered":"2023-09-23T17:57:53.875345+0000","last_unstale":"2023-09-23T18:00:18.921917+0000","last_undegraded":"2023-09-23T17:57:53.869204+0000","last_fullsized":"2023-09-23T17:57:53.869091+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:19.033883+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":159,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.913780+0000","last_change":"2023-09-23T17:59:10.877550+0000","last_active":"2023-09-23T18:00:18.913780+0000","last_peered":"2023-09-23T18:00:18.913780+0000","last_clean":"2023-09-23T18:00:18.913780+0000","last_became_active":"2023-09-23T17:59:10.877173+0000","last_became_peered":"2023-09-23T17:59:10.877173+0000","last_unstale":"2023-09-23T18:00:18.913780+0000","last_undegraded":"2023-09-23T18:00:18.913780+0000","last_fullsized":"2023-09-23T18:00:18.913780+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:00.740053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,5],"avail_no_missing":["6","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":493,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.937726+0000","last_change":"2023-09-23T17:59:00.136455+0000","last_active":"2023-09-23T18:00:18.937726+0000","last_peered":"2023-09-23T18:00:18.937726+0000","last_clean":"2023-09-23T18:00:18.937726+0000","last_became_active":"2023-09-23T17:59:00.136126+0000","last_became_peered":"2023-09-23T17:59:00.136126+0000","last_unstale":"2023-09-23T18:00:18.937726+0000","last_undegraded":"2023-09-23T18:00:18.937726+0000","last_fullsized":"2023-09-23T18:00:18.937726+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.8","version":"193'12738","reported_seq":12832,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-09-23T18:00:29.177353+0000","last_change":"2023-09-23T17:55:53.818806+0000","last_active":"2023-09-23T18:00:29.177353+0000","last_peered":"2023-09-23T18:00:29.177353+0000","last_clean":"2023-09-23T18:00:29.177353+0000","last_became_active":"2023-09-23T17:55:53.818291+0000","last_became_peered":"2023-09-23T17:55:53.818291+0000","last_unstale":"2023-09-23T18:00:29.177353+0000","last_undegraded":"2023-09-23T18:00:29.177353+0000","last_fullsized":"2023-09-23T18:00:29.177353+0000","mapping_epoch":110,"log_start":"118'3000","ondisk_log_start":"118'3000","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9738,"log_dups_size":0,"ondisk_log_size":9738,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:38:58.485332+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36700160,"num_objects":560,"num_object_clones":0,"num_object_copies":1680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12738,"num_write_kb":425536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":158,"reported_epoch":192,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:18.924642+0000","last_change":"2023-09-23T17:59:50.897959+0000","last_active":"2023-09-23T18:00:18.924642+0000","last_peered":"2023-09-23T18:00:18.924642+0000","last_clean":"2023-09-23T17:56:46.621682+0000","last_became_active":"2023-09-23T17:59:50.897959+0000","last_became_peered":"2023-09-23T17:59:50.897959+0000","last_unstale":"2023-09-23T18:00:18.924642+0000","last_undegraded":"2023-09-23T17:59:50.893310+0000","last_fullsized":"2023-09-23T17:59:50.893201+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:40:45.953223+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":182,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.921644+0000","last_change":"2023-09-23T17:59:50.896274+0000","last_active":"2023-09-23T18:00:18.921644+0000","last_peered":"2023-09-23T18:00:18.921644+0000","last_clean":"2023-09-23T17:57:20.922147+0000","last_became_active":"2023-09-23T17:59:50.896274+0000","last_became_peered":"2023-09-23T17:59:50.896274+0000","last_unstale":"2023-09-23T18:00:18.921644+0000","last_undegraded":"2023-09-23T18:00:18.921644+0000","last_fullsized":"2023-09-23T17:59:50.890612+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:36.294543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":488,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.940118+0000","last_change":"2023-09-23T17:59:00.128843+0000","last_active":"2023-09-23T18:00:18.940118+0000","last_peered":"2023-09-23T18:00:18.940118+0000","last_clean":"2023-09-23T18:00:18.940118+0000","last_became_active":"2023-09-23T17:59:00.128472+0000","last_became_peered":"2023-09-23T17:59:00.128472+0000","last_unstale":"2023-09-23T18:00:18.940118+0000","last_undegraded":"2023-09-23T18:00:18.940118+0000","last_fullsized":"2023-09-23T18:00:18.940118+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.b","version":"193'12590","reported_seq":12694,"reported_epoch":193,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:29.166410+0000","last_change":"2023-09-23T17:59:10.964199+0000","last_active":"2023-09-23T18:00:29.166410+0000","last_peered":"2023-09-23T18:00:29.166410+0000","last_clean":"2023-09-23T18:00:29.166410+0000","last_became_active":"2023-09-23T17:59:10.899328+0000","last_became_peered":"2023-09-23T17:59:10.899328+0000","last_unstale":"2023-09-23T18:00:29.166410+0000","last_undegraded":"2023-09-23T18:00:29.166410+0000","last_fullsized":"2023-09-23T18:00:29.166410+0000","mapping_epoch":176,"log_start":"118'2900","ondisk_log_start":"118'2900","created":110,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9690,"log_dups_size":0,"ondisk_log_size":9690,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:47.317955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38010880,"num_objects":580,"num_object_clones":0,"num_object_copies":1740,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":580,"num_objects_unfound":0,"num_objects_dirty":580,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12590,"num_write_kb":421440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":580}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":40,"reported_epoch":192,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:00:18.914449+0000","last_change":"2023-09-23T17:59:10.875436+0000","last_active":"2023-09-23T18:00:18.914449+0000","last_peered":"2023-09-23T18:00:18.914449+0000","last_clean":"2023-09-23T17:45:54.586241+0000","last_became_active":"2023-09-23T17:59:10.875436+0000","last_became_peered":"2023-09-23T17:59:10.875436+0000","last_unstale":"2023-09-23T18:00:18.914449+0000","last_undegraded":"2023-09-23T18:00:18.914449+0000","last_fullsized":"2023-09-23T17:59:10.873636+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":64,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:39.737287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":128,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.914440+0000","last_change":"2023-09-23T17:59:09.884256+0000","last_active":"2023-09-23T18:00:18.914440+0000","last_peered":"2023-09-23T18:00:18.914440+0000","last_clean":"2023-09-23T18:00:18.914440+0000","last_became_active":"2023-09-23T17:59:09.883903+0000","last_became_peered":"2023-09-23T17:59:09.883903+0000","last_unstale":"2023-09-23T18:00:18.914440+0000","last_undegraded":"2023-09-23T18:00:18.914440+0000","last_fullsized":"2023-09-23T18:00:18.914440+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:08.901791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":488,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.942780+0000","last_change":"2023-09-23T17:59:00.127991+0000","last_active":"2023-09-23T18:00:18.942780+0000","last_peered":"2023-09-23T18:00:18.942780+0000","last_clean":"2023-09-23T18:00:18.942780+0000","last_became_active":"2023-09-23T17:59:00.127676+0000","last_became_peered":"2023-09-23T17:59:00.127676+0000","last_unstale":"2023-09-23T18:00:18.942780+0000","last_undegraded":"2023-09-23T18:00:18.942780+0000","last_fullsized":"2023-09-23T18:00:18.942780+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.a","version":"193'12577","reported_seq":13495,"reported_epoch":193,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:29.157626+0000","last_change":"2023-09-23T17:57:24.992424+0000","last_active":"2023-09-23T18:00:29.157626+0000","last_peered":"2023-09-23T18:00:29.157626+0000","last_clean":"2023-09-23T17:57:23.931663+0000","last_became_active":"2023-09-23T17:57:24.964668+0000","last_became_peered":"2023-09-23T17:57:24.964668+0000","last_unstale":"2023-09-23T18:00:29.157626+0000","last_undegraded":"2023-09-23T17:57:24.952679+0000","last_fullsized":"2023-09-23T17:57:24.947063+0000","mapping_epoch":142,"log_start":"118'2800","ondisk_log_start":"118'2800","created":110,"last_epoch_clean":134,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9777,"log_dups_size":0,"ondisk_log_size":9777,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:59.839324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37552128,"num_objects":573,"num_object_clones":0,"num_object_copies":1719,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":573,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":573,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12577,"num_write_kb":420800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":396,"num_bytes_recovered":25952256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[{"shards":"2,4","objects":573}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":170,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.914828+0000","last_change":"2023-09-23T17:59:10.881020+0000","last_active":"2023-09-23T18:00:18.914828+0000","last_peered":"2023-09-23T18:00:18.914828+0000","last_clean":"2023-09-23T18:00:18.914828+0000","last_became_active":"2023-09-23T17:59:10.880602+0000","last_became_peered":"2023-09-23T17:59:10.880602+0000","last_unstale":"2023-09-23T18:00:18.914828+0000","last_undegraded":"2023-09-23T18:00:18.914828+0000","last_fullsized":"2023-09-23T18:00:18.914828+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:15.837348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,5],"avail_no_missing":["6","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":156,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.937076+0000","last_change":"2023-09-23T17:56:50.828739+0000","last_active":"2023-09-23T18:00:18.937076+0000","last_peered":"2023-09-23T18:00:18.937076+0000","last_clean":"2023-09-23T18:00:18.937076+0000","last_became_active":"2023-09-23T17:56:50.828229+0000","last_became_peered":"2023-09-23T17:56:50.828229+0000","last_unstale":"2023-09-23T18:00:18.937076+0000","last_undegraded":"2023-09-23T18:00:18.937076+0000","last_fullsized":"2023-09-23T18:00:18.937076+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:45.927953+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":480,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.933975+0000","last_change":"2023-09-23T17:58:35.252704+0000","last_active":"2023-09-23T18:00:18.933975+0000","last_peered":"2023-09-23T18:00:18.933975+0000","last_clean":"2023-09-23T18:00:18.933975+0000","last_became_active":"2023-09-23T17:58:35.252386+0000","last_became_peered":"2023-09-23T17:58:35.252386+0000","last_unstale":"2023-09-23T18:00:18.933975+0000","last_undegraded":"2023-09-23T18:00:18.933975+0000","last_fullsized":"2023-09-23T18:00:18.933975+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"193'12572","reported_seq":12681,"reported_epoch":193,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:00:29.102510+0000","last_change":"2023-09-23T17:59:11.340557+0000","last_active":"2023-09-23T18:00:29.102510+0000","last_peered":"2023-09-23T18:00:29.102510+0000","last_clean":"2023-09-23T17:57:52.866355+0000","last_became_active":"2023-09-23T17:59:11.311441+0000","last_became_peered":"2023-09-23T17:59:11.311441+0000","last_unstale":"2023-09-23T18:00:29.102510+0000","last_undegraded":"2023-09-23T17:59:10.972976+0000","last_fullsized":"2023-09-23T17:59:10.890539+0000","mapping_epoch":176,"log_start":"118'2800","ondisk_log_start":"118'2800","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9772,"log_dups_size":0,"ondisk_log_size":9772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:06.586919+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36765720,"num_objects":562,"num_object_clones":0,"num_object_copies":1686,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":562,"num_objects_misplaced":6566,"num_objects_unfound":0,"num_objects_dirty":562,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12572,"num_write_kb":420225,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-2955}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":79,"reported_epoch":192,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:18.937169+0000","last_change":"2023-09-23T17:57:24.957601+0000","last_active":"2023-09-23T18:00:18.937169+0000","last_peered":"2023-09-23T18:00:18.937169+0000","last_clean":"2023-09-23T17:45:54.465462+0000","last_became_active":"2023-09-23T17:57:24.957601+0000","last_became_peered":"2023-09-23T17:57:24.957601+0000","last_unstale":"2023-09-23T18:00:18.937169+0000","last_undegraded":"2023-09-23T17:57:24.946354+0000","last_fullsized":"2023-09-23T17:57:24.946239+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:59.908857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,4","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":189,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.921744+0000","last_change":"2023-09-23T17:59:51.905325+0000","last_active":"2023-09-23T18:00:18.921744+0000","last_peered":"2023-09-23T18:00:18.921744+0000","last_clean":"2023-09-23T18:00:18.921744+0000","last_became_active":"2023-09-23T17:59:51.904875+0000","last_became_peered":"2023-09-23T17:59:51.904875+0000","last_unstale":"2023-09-23T18:00:18.921744+0000","last_undegraded":"2023-09-23T18:00:18.921744+0000","last_fullsized":"2023-09-23T18:00:18.921744+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:49.269529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6,1],"avail_no_missing":["4","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":484,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.934060+0000","last_change":"2023-09-23T17:58:35.251808+0000","last_active":"2023-09-23T18:00:18.934060+0000","last_peered":"2023-09-23T18:00:18.934060+0000","last_clean":"2023-09-23T18:00:18.934060+0000","last_became_active":"2023-09-23T17:58:35.251267+0000","last_became_peered":"2023-09-23T17:58:35.251267+0000","last_unstale":"2023-09-23T18:00:18.934060+0000","last_undegraded":"2023-09-23T18:00:18.934060+0000","last_fullsized":"2023-09-23T18:00:18.934060+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53670,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":22,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.c","version":"193'12495","reported_seq":12607,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-09-23T18:00:25.952386+0000","last_change":"2023-09-23T17:56:51.840960+0000","last_active":"2023-09-23T18:00:25.952386+0000","last_peered":"2023-09-23T18:00:25.952386+0000","last_clean":"2023-09-23T18:00:25.952386+0000","last_became_active":"2023-09-23T17:56:51.835697+0000","last_became_peered":"2023-09-23T17:56:51.835697+0000","last_unstale":"2023-09-23T18:00:25.952386+0000","last_undegraded":"2023-09-23T18:00:25.952386+0000","last_fullsized":"2023-09-23T18:00:25.952386+0000","mapping_epoch":127,"log_start":"161'7998","ondisk_log_start":"161'7998","created":110,"last_epoch_clean":128,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4497,"log_dups_size":0,"ondisk_log_size":4497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:45.804116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46071808,"num_objects":703,"num_object_clones":0,"num_object_copies":2109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":703,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12495,"num_write_kb":422336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":104,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.914743+0000","last_change":"2023-09-23T17:58:35.243177+0000","last_active":"2023-09-23T18:00:18.914743+0000","last_peered":"2023-09-23T18:00:18.914743+0000","last_clean":"2023-09-23T18:00:18.914743+0000","last_became_active":"2023-09-23T17:58:35.242824+0000","last_became_peered":"2023-09-23T17:58:35.242824+0000","last_unstale":"2023-09-23T18:00:18.914743+0000","last_undegraded":"2023-09-23T18:00:18.914743+0000","last_fullsized":"2023-09-23T18:00:18.914743+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:10.250046+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":181,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.922963+0000","last_change":"2023-09-23T17:59:50.899364+0000","last_active":"2023-09-23T18:00:18.922963+0000","last_peered":"2023-09-23T18:00:18.922963+0000","last_clean":"2023-09-23T18:00:18.922963+0000","last_became_active":"2023-09-23T17:59:50.899062+0000","last_became_peered":"2023-09-23T17:59:50.899062+0000","last_unstale":"2023-09-23T18:00:18.922963+0000","last_undegraded":"2023-09-23T18:00:18.922963+0000","last_fullsized":"2023-09-23T18:00:18.922963+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:36:31.623206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":488,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.936853+0000","last_change":"2023-09-23T17:59:00.133828+0000","last_active":"2023-09-23T18:00:18.936853+0000","last_peered":"2023-09-23T18:00:18.936853+0000","last_clean":"2023-09-23T18:00:18.936853+0000","last_became_active":"2023-09-23T17:59:00.133283+0000","last_became_peered":"2023-09-23T17:59:00.133283+0000","last_unstale":"2023-09-23T18:00:18.936853+0000","last_undegraded":"2023-09-23T18:00:18.936853+0000","last_fullsized":"2023-09-23T18:00:18.936853+0000","mapping_epoch":171,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":8,"num_write_kb":74,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.f","version":"193'12539","reported_seq":27312,"reported_epoch":193,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:00:27.521821+0000","last_change":"2023-09-23T17:59:51.071378+0000","last_active":"2023-09-23T18:00:27.521821+0000","last_peered":"2023-09-23T18:00:27.521821+0000","last_clean":"2023-09-23T17:55:56.846049+0000","last_became_active":"2023-09-23T17:59:50.976302+0000","last_became_peered":"2023-09-23T17:59:50.976302+0000","last_unstale":"2023-09-23T18:00:27.521821+0000","last_undegraded":"2023-09-23T17:59:50.971636+0000","last_fullsized":"2023-09-23T17:59:50.890747+0000","mapping_epoch":186,"log_start":"136'5847","ondisk_log_start":"136'5847","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6692,"log_dups_size":0,"ondisk_log_size":6692,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:08.577292+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":42401792,"num_objects":647,"num_object_clones":0,"num_object_copies":1941,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1459,"num_objects_misplaced":647,"num_objects_unfound":0,"num_objects_dirty":647,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12539,"num_write_kb":421952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6017,"num_bytes_recovered":303497216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,2],"avail_no_missing":["4","2","5"],"object_location_counts":[{"shards":"2,4","objects":-1459},{"shards":"2,4,5","objects":647}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":83,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.921133+0000","last_change":"2023-09-23T17:57:53.880278+0000","last_active":"2023-09-23T18:00:18.921133+0000","last_peered":"2023-09-23T18:00:18.921133+0000","last_clean":"2023-09-23T17:57:47.586782+0000","last_became_active":"2023-09-23T17:57:53.880278+0000","last_became_peered":"2023-09-23T17:57:53.880278+0000","last_unstale":"2023-09-23T18:00:18.921133+0000","last_undegraded":"2023-09-23T18:00:18.921133+0000","last_fullsized":"2023-09-23T17:57:53.867145+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:07.345156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":185,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.930933+0000","last_change":"2023-09-23T17:59:50.898260+0000","last_active":"2023-09-23T18:00:18.930933+0000","last_peered":"2023-09-23T18:00:18.930933+0000","last_clean":"2023-09-23T18:00:18.930933+0000","last_became_active":"2023-09-23T17:59:50.897991+0000","last_became_peered":"2023-09-23T17:59:50.897991+0000","last_unstale":"2023-09-23T18:00:18.930933+0000","last_undegraded":"2023-09-23T18:00:18.930933+0000","last_fullsized":"2023-09-23T18:00:18.930933+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:50.486896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":480,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.936960+0000","last_change":"2023-09-23T17:58:35.250653+0000","last_active":"2023-09-23T18:00:18.936960+0000","last_peered":"2023-09-23T18:00:18.936960+0000","last_clean":"2023-09-23T18:00:18.936960+0000","last_became_active":"2023-09-23T17:58:35.249513+0000","last_became_peered":"2023-09-23T17:58:35.249513+0000","last_unstale":"2023-09-23T18:00:18.936960+0000","last_undegraded":"2023-09-23T18:00:18.936960+0000","last_fullsized":"2023-09-23T18:00:18.936960+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":53669,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":161009,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.e","version":"193'12523","reported_seq":13538,"reported_epoch":193,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:00:27.504545+0000","last_change":"2023-09-23T17:59:51.963319+0000","last_active":"2023-09-23T18:00:27.504545+0000","last_peered":"2023-09-23T18:00:27.504545+0000","last_clean":"2023-09-23T17:59:08.813403+0000","last_became_active":"2023-09-23T17:59:51.909158+0000","last_became_peered":"2023-09-23T17:59:51.909158+0000","last_unstale":"2023-09-23T18:00:27.504545+0000","last_undegraded":"2023-09-23T18:00:27.504545+0000","last_fullsized":"2023-09-23T18:00:27.504545+0000","mapping_epoch":187,"log_start":"144'6563","ondisk_log_start":"144'6563","created":110,"last_epoch_clean":132,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":5960,"log_dups_size":0,"ondisk_log_size":5960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:21:39.558589+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":43712512,"num_objects":667,"num_object_clones":0,"num_object_copies":2001,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":666,"num_objects_unfound":0,"num_objects_dirty":667,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12523,"num_write_kb":422080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":434,"num_bytes_recovered":28442624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,5],"avail_no_missing":["4","0","5"],"object_location_counts":[{"shards":"0,4,5","objects":667}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":128,"reported_epoch":192,"state":"undersized+peered","last_fresh":"2023-09-23T18:00:18.931130+0000","last_change":"2023-09-23T17:57:53.880095+0000","last_active":"2023-09-23T17:57:52.883357+0000","last_peered":"2023-09-23T18:00:18.931130+0000","last_clean":"2023-09-23T17:53:21.028908+0000","last_became_active":"2023-09-23T17:57:24.954589+0000","last_became_peered":"2023-09-23T17:57:53.880095+0000","last_unstale":"2023-09-23T18:00:18.931130+0000","last_undegraded":"2023-09-23T18:00:18.931130+0000","last_fullsized":"2023-09-23T17:57:53.874470+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:56.618752+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1],"acting":[1],"avail_no_missing":["1"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":153,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.921932+0000","last_change":"2023-09-23T17:57:24.961477+0000","last_active":"2023-09-23T18:00:18.921932+0000","last_peered":"2023-09-23T18:00:18.921932+0000","last_clean":"2023-09-23T17:57:20.929549+0000","last_became_active":"2023-09-23T17:57:24.961477+0000","last_became_peered":"2023-09-23T17:57:24.961477+0000","last_unstale":"2023-09-23T18:00:18.921932+0000","last_undegraded":"2023-09-23T18:00:18.921932+0000","last_fullsized":"2023-09-23T17:57:24.954309+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:01.838528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,4],"avail_no_missing":["2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":489,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.940228+0000","last_change":"2023-09-23T17:59:00.112340+0000","last_active":"2023-09-23T18:00:18.940228+0000","last_peered":"2023-09-23T18:00:18.940228+0000","last_clean":"2023-09-23T18:00:18.940228+0000","last_became_active":"2023-09-23T17:59:00.111325+0000","last_became_peered":"2023-09-23T17:59:00.111325+0000","last_unstale":"2023-09-23T18:00:18.940228+0000","last_undegraded":"2023-09-23T18:00:18.940228+0000","last_fullsized":"2023-09-23T18:00:18.940228+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":621,"reported_epoch":192,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:18.934214+0000","last_change":"2023-09-23T17:57:53.873008+0000","last_active":"2023-09-23T18:00:18.934214+0000","last_peered":"2023-09-23T18:00:18.934214+0000","last_clean":"2023-09-23T17:57:47.562468+0000","last_became_active":"2023-09-23T17:57:53.873008+0000","last_became_peered":"2023-09-23T17:57:53.873008+0000","last_unstale":"2023-09-23T18:00:18.934214+0000","last_undegraded":"2023-09-23T17:57:53.867595+0000","last_fullsized":"2023-09-23T17:57:53.867487+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:19.562351+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":151,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.923118+0000","last_change":"2023-09-23T17:59:10.975320+0000","last_active":"2023-09-23T18:00:18.923118+0000","last_peered":"2023-09-23T18:00:18.923118+0000","last_clean":"2023-09-23T18:00:18.923118+0000","last_became_active":"2023-09-23T17:59:10.974946+0000","last_became_peered":"2023-09-23T17:59:10.974946+0000","last_unstale":"2023-09-23T18:00:18.923118+0000","last_undegraded":"2023-09-23T18:00:18.923118+0000","last_fullsized":"2023-09-23T18:00:18.923118+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:05.703255+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6],"acting":[2,6,5],"avail_no_missing":["2","5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":146,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.931609+0000","last_change":"2023-09-23T17:59:09.887042+0000","last_active":"2023-09-23T18:00:18.931609+0000","last_peered":"2023-09-23T18:00:18.931609+0000","last_clean":"2023-09-23T18:00:18.931609+0000","last_became_active":"2023-09-23T17:59:09.886688+0000","last_became_peered":"2023-09-23T17:59:09.886688+0000","last_unstale":"2023-09-23T18:00:18.931609+0000","last_undegraded":"2023-09-23T18:00:18.931609+0000","last_fullsized":"2023-09-23T18:00:18.931609+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:44.443377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"193'12500","reported_seq":14811,"reported_epoch":193,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:00:29.429073+0000","last_change":"2023-09-23T17:59:51.072118+0000","last_active":"2023-09-23T18:00:29.429073+0000","last_peered":"2023-09-23T18:00:29.429073+0000","last_clean":"2023-09-23T17:55:57.821023+0000","last_became_active":"2023-09-23T17:59:09.854393+0000","last_became_peered":"2023-09-23T17:59:09.854393+0000","last_unstale":"2023-09-23T18:00:29.429073+0000","last_undegraded":"2023-09-23T17:59:09.846461+0000","last_fullsized":"2023-09-23T17:59:09.846277+0000","mapping_epoch":175,"log_start":"161'7957","ondisk_log_start":"161'7957","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4543,"log_dups_size":0,"ondisk_log_size":4543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:38.915731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37879808,"num_objects":578,"num_object_clones":0,"num_object_copies":1734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":351,"num_objects_misplaced":578,"num_objects_unfound":0,"num_objects_dirty":578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12500,"num_write_kb":418496,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1095,"num_bytes_recovered":71761920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":578}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":488,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.934371+0000","last_change":"2023-09-23T17:59:00.117942+0000","last_active":"2023-09-23T18:00:18.934371+0000","last_peered":"2023-09-23T18:00:18.934371+0000","last_clean":"2023-09-23T18:00:18.934371+0000","last_became_active":"2023-09-23T17:59:00.117602+0000","last_became_peered":"2023-09-23T17:59:00.117602+0000","last_unstale":"2023-09-23T18:00:18.934371+0000","last_undegraded":"2023-09-23T18:00:18.934371+0000","last_fullsized":"2023-09-23T18:00:18.934371+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"193'12506","reported_seq":12619,"reported_epoch":193,"state":"active+clean","last_fresh":"2023-09-23T18:00:29.493713+0000","last_change":"2023-09-23T17:56:50.843235+0000","last_active":"2023-09-23T18:00:29.493713+0000","last_peered":"2023-09-23T18:00:29.493713+0000","last_clean":"2023-09-23T18:00:29.493713+0000","last_became_active":"2023-09-23T17:56:50.835153+0000","last_became_peered":"2023-09-23T17:56:50.835153+0000","last_unstale":"2023-09-23T18:00:29.493713+0000","last_undegraded":"2023-09-23T18:00:29.493713+0000","last_fullsized":"2023-09-23T18:00:29.493713+0000","mapping_epoch":126,"log_start":"161'7953","ondisk_log_start":"161'7953","created":110,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4553,"log_dups_size":0,"ondisk_log_size":4553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:14.827228+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36962304,"num_objects":564,"num_object_clones":0,"num_object_copies":1692,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":564,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12506,"num_write_kb":418240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1129,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.931802+0000","last_change":"2023-09-23T17:59:51.906763+0000","last_active":"2023-09-23T18:00:18.931802+0000","last_peered":"2023-09-23T18:00:18.931802+0000","last_clean":"2023-09-23T18:00:18.931802+0000","last_became_active":"2023-09-23T17:59:51.906189+0000","last_became_peered":"2023-09-23T17:59:51.906189+0000","last_unstale":"2023-09-23T18:00:18.931802+0000","last_undegraded":"2023-09-23T18:00:18.931802+0000","last_fullsized":"2023-09-23T18:00:18.931802+0000","mapping_epoch":187,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":188,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:56:57.865971+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":181,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.921907+0000","last_change":"2023-09-23T17:59:50.897300+0000","last_active":"2023-09-23T18:00:18.921907+0000","last_peered":"2023-09-23T18:00:18.921907+0000","last_clean":"2023-09-23T18:00:18.921907+0000","last_became_active":"2023-09-23T17:59:50.896960+0000","last_became_peered":"2023-09-23T17:59:50.896960+0000","last_unstale":"2023-09-23T18:00:18.921907+0000","last_undegraded":"2023-09-23T18:00:18.921907+0000","last_fullsized":"2023-09-23T18:00:18.921907+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:30:29.201533+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":155,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.934085+0000","last_change":"2023-09-23T17:57:24.952855+0000","last_active":"2023-09-23T18:00:18.934085+0000","last_peered":"2023-09-23T18:00:18.934085+0000","last_clean":"2023-09-23T17:57:20.923267+0000","last_became_active":"2023-09-23T17:57:24.952855+0000","last_became_peered":"2023-09-23T17:57:24.952855+0000","last_unstale":"2023-09-23T18:00:18.934085+0000","last_undegraded":"2023-09-23T18:00:18.934085+0000","last_fullsized":"2023-09-23T17:57:24.946770+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:15.329593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":489,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.936542+0000","last_change":"2023-09-23T17:59:00.113520+0000","last_active":"2023-09-23T18:00:18.936542+0000","last_peered":"2023-09-23T18:00:18.936542+0000","last_clean":"2023-09-23T18:00:18.936542+0000","last_became_active":"2023-09-23T17:59:00.113172+0000","last_became_peered":"2023-09-23T17:59:00.113172+0000","last_unstale":"2023-09-23T18:00:18.936542+0000","last_undegraded":"2023-09-23T18:00:18.936542+0000","last_fullsized":"2023-09-23T18:00:18.936542+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.6","version":"193'12845","reported_seq":16197,"reported_epoch":193,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:00:29.206760+0000","last_change":"2023-09-23T17:59:32.085668+0000","last_active":"2023-09-23T18:00:29.206760+0000","last_peered":"2023-09-23T18:00:29.206760+0000","last_clean":"2023-09-23T17:57:52.799975+0000","last_became_active":"2023-09-23T17:59:11.178414+0000","last_became_peered":"2023-09-23T17:59:11.178414+0000","last_unstale":"2023-09-23T18:00:29.206760+0000","last_undegraded":"2023-09-23T17:59:10.956696+0000","last_fullsized":"2023-09-23T17:59:10.884554+0000","mapping_epoch":176,"log_start":"118'3073","ondisk_log_start":"118'3073","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9772,"log_dups_size":0,"ondisk_log_size":9772,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:22.428899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39911424,"num_objects":609,"num_object_clones":0,"num_object_copies":1827,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":609,"num_objects_misplaced":1982,"num_objects_unfound":0,"num_objects_dirty":609,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12845,"num_write_kb":430528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1622,"num_bytes_recovered":106299392,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":-1373}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":464,"reported_epoch":192,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:18.920700+0000","last_change":"2023-09-23T17:57:24.955720+0000","last_active":"2023-09-23T18:00:18.920700+0000","last_peered":"2023-09-23T18:00:18.920700+0000","last_clean":"2023-09-23T17:57:20.931796+0000","last_became_active":"2023-09-23T17:57:24.955720+0000","last_became_peered":"2023-09-23T17:57:24.955720+0000","last_unstale":"2023-09-23T18:00:18.920700+0000","last_undegraded":"2023-09-23T17:57:24.945824+0000","last_fullsized":"2023-09-23T17:57:24.945671+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:25.614568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[{"shards":"0,2","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":148,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.931282+0000","last_change":"2023-09-23T17:58:35.246091+0000","last_active":"2023-09-23T18:00:18.931282+0000","last_peered":"2023-09-23T18:00:18.931282+0000","last_clean":"2023-09-23T18:00:18.931282+0000","last_became_active":"2023-09-23T17:58:35.245673+0000","last_became_peered":"2023-09-23T17:58:35.245673+0000","last_unstale":"2023-09-23T18:00:18.931282+0000","last_undegraded":"2023-09-23T18:00:18.931282+0000","last_fullsized":"2023-09-23T18:00:18.931282+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:53.368873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":172,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:20.991742+0000","last_change":"2023-09-23T17:59:10.911985+0000","last_active":"2023-09-23T18:00:20.991742+0000","last_peered":"2023-09-23T18:00:20.991742+0000","last_clean":"2023-09-23T18:00:20.991742+0000","last_became_active":"2023-09-23T17:59:10.911584+0000","last_became_peered":"2023-09-23T17:59:10.911584+0000","last_unstale":"2023-09-23T18:00:20.991742+0000","last_undegraded":"2023-09-23T18:00:20.991742+0000","last_fullsized":"2023-09-23T18:00:20.991742+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:43.723422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,5],"avail_no_missing":["1","0","5"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1008,"reported_epoch":192,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:00:18.923572+0000","last_change":"2023-09-23T17:59:10.890970+0000","last_active":"2023-09-23T18:00:18.923572+0000","last_peered":"2023-09-23T18:00:18.923572+0000","last_clean":"2023-09-23T17:47:00.624503+0000","last_became_active":"2023-09-23T17:59:10.890970+0000","last_became_peered":"2023-09-23T17:59:10.890970+0000","last_unstale":"2023-09-23T18:00:18.923572+0000","last_undegraded":"2023-09-23T18:00:18.923572+0000","last_fullsized":"2023-09-23T17:59:10.885398+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:36.933247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2],"acting":[2,5],"avail_no_missing":["2","5"],"object_location_counts":[{"shards":"2,5","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":489,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.934344+0000","last_change":"2023-09-23T17:59:00.118160+0000","last_active":"2023-09-23T18:00:18.934344+0000","last_peered":"2023-09-23T18:00:18.934344+0000","last_clean":"2023-09-23T18:00:18.934344+0000","last_became_active":"2023-09-23T17:59:00.117827+0000","last_became_peered":"2023-09-23T17:59:00.117827+0000","last_unstale":"2023-09-23T18:00:18.934344+0000","last_undegraded":"2023-09-23T18:00:18.934344+0000","last_fullsized":"2023-09-23T18:00:18.934344+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.914629+0000","last_change":"2023-09-23T17:59:10.879009+0000","last_active":"2023-09-23T18:00:18.914629+0000","last_peered":"2023-09-23T18:00:18.914629+0000","last_clean":"2023-09-23T18:00:18.914629+0000","last_became_active":"2023-09-23T17:59:10.878632+0000","last_became_peered":"2023-09-23T17:59:10.878632+0000","last_unstale":"2023-09-23T18:00:18.914629+0000","last_undegraded":"2023-09-23T18:00:18.914629+0000","last_fullsized":"2023-09-23T18:00:18.914629+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:38.184201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,5],"avail_no_missing":["6","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":92,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.914310+0000","last_change":"2023-09-23T17:58:36.248863+0000","last_active":"2023-09-23T18:00:18.914310+0000","last_peered":"2023-09-23T18:00:18.914310+0000","last_clean":"2023-09-23T18:00:18.914310+0000","last_became_active":"2023-09-23T17:58:36.248383+0000","last_became_peered":"2023-09-23T17:58:36.248383+0000","last_unstale":"2023-09-23T18:00:18.914310+0000","last_undegraded":"2023-09-23T18:00:18.914310+0000","last_fullsized":"2023-09-23T18:00:18.914310+0000","mapping_epoch":164,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":165,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:12:49.334942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4,0],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"193'12666","reported_seq":22402,"reported_epoch":193,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:00:29.145579+0000","last_change":"2023-09-23T17:59:11.224350+0000","last_active":"2023-09-23T18:00:29.145579+0000","last_peered":"2023-09-23T18:00:29.145579+0000","last_clean":"2023-09-23T17:59:08.847410+0000","last_became_active":"2023-09-23T17:59:11.059419+0000","last_became_peered":"2023-09-23T17:59:11.059419+0000","last_unstale":"2023-09-23T18:00:29.145579+0000","last_undegraded":"2023-09-23T18:00:29.145579+0000","last_fullsized":"2023-09-23T18:00:29.145579+0000","mapping_epoch":176,"log_start":"118'2971","ondisk_log_start":"118'2971","created":110,"last_epoch_clean":174,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":9695,"log_dups_size":0,"ondisk_log_size":9695,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:54.596656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":39452672,"num_objects":602,"num_object_clones":0,"num_object_copies":1806,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":594,"num_objects_unfound":0,"num_objects_dirty":602,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12666,"num_write_kb":424576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4804,"num_bytes_recovered":314834944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":602}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":489,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.936941+0000","last_change":"2023-09-23T17:59:00.113658+0000","last_active":"2023-09-23T18:00:18.936941+0000","last_peered":"2023-09-23T18:00:18.936941+0000","last_clean":"2023-09-23T18:00:18.936941+0000","last_became_active":"2023-09-23T17:59:00.113034+0000","last_became_peered":"2023-09-23T17:59:00.113034+0000","last_unstale":"2023-09-23T18:00:18.936941+0000","last_undegraded":"2023-09-23T18:00:18.936941+0000","last_fullsized":"2023-09-23T18:00:18.936941+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.7","version":"193'12711","reported_seq":12810,"reported_epoch":193,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:00:25.981475+0000","last_change":"2023-09-23T17:59:11.276228+0000","last_active":"2023-09-23T18:00:25.981475+0000","last_peered":"2023-09-23T18:00:25.981475+0000","last_clean":"2023-09-23T17:59:08.808079+0000","last_became_active":"2023-09-23T17:59:11.192668+0000","last_became_peered":"2023-09-23T17:59:11.192668+0000","last_unstale":"2023-09-23T18:00:25.981475+0000","last_undegraded":"2023-09-23T18:00:25.981475+0000","last_fullsized":"2023-09-23T18:00:25.981475+0000","mapping_epoch":176,"log_start":"161'8207","ondisk_log_start":"161'8207","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4504,"log_dups_size":0,"ondisk_log_size":4504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:56.972002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":48955392,"num_objects":747,"num_object_clones":0,"num_object_copies":2241,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":747,"num_objects_unfound":0,"num_objects_dirty":747,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12711,"num_write_kb":430656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,5],"avail_no_missing":["1","0","5"],"object_location_counts":[{"shards":"0,1,5","objects":747}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":295,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.931431+0000","last_change":"2023-09-23T17:57:53.883867+0000","last_active":"2023-09-23T18:00:18.931431+0000","last_peered":"2023-09-23T18:00:18.931431+0000","last_clean":"2023-09-23T17:46:57.119141+0000","last_became_active":"2023-09-23T17:57:53.883867+0000","last_became_peered":"2023-09-23T17:57:53.883867+0000","last_unstale":"2023-09-23T18:00:18.931431+0000","last_undegraded":"2023-09-23T18:00:18.931431+0000","last_fullsized":"2023-09-23T17:57:53.875072+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":99,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:35.282181+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":161,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.934169+0000","last_change":"2023-09-23T17:57:24.959291+0000","last_active":"2023-09-23T18:00:18.934169+0000","last_peered":"2023-09-23T18:00:18.934169+0000","last_clean":"2023-09-23T17:57:20.930735+0000","last_became_active":"2023-09-23T17:57:24.959291+0000","last_became_peered":"2023-09-23T17:57:24.959291+0000","last_unstale":"2023-09-23T18:00:18.934169+0000","last_undegraded":"2023-09-23T18:00:18.934169+0000","last_fullsized":"2023-09-23T17:57:24.946083+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:43.225674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":153,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.942831+0000","last_change":"2023-09-23T17:54:25.604509+0000","last_active":"2023-09-23T18:00:18.942831+0000","last_peered":"2023-09-23T18:00:18.942831+0000","last_clean":"2023-09-23T18:00:18.942831+0000","last_became_active":"2023-09-23T17:54:25.603754+0000","last_became_peered":"2023-09-23T17:54:25.603754+0000","last_unstale":"2023-09-23T18:00:18.942831+0000","last_undegraded":"2023-09-23T18:00:18.942831+0000","last_fullsized":"2023-09-23T18:00:18.942831+0000","mapping_epoch":103,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:25.320508+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":489,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.943498+0000","last_change":"2023-09-23T17:59:00.118578+0000","last_active":"2023-09-23T18:00:18.943498+0000","last_peered":"2023-09-23T18:00:18.943498+0000","last_clean":"2023-09-23T18:00:18.943498+0000","last_became_active":"2023-09-23T17:59:00.118262+0000","last_became_peered":"2023-09-23T17:59:00.118262+0000","last_unstale":"2023-09-23T18:00:18.943498+0000","last_undegraded":"2023-09-23T18:00:18.943498+0000","last_fullsized":"2023-09-23T18:00:18.943498+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":328,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.944967+0000","last_change":"2023-09-23T17:57:53.875525+0000","last_active":"2023-09-23T18:00:18.944967+0000","last_peered":"2023-09-23T18:00:18.944967+0000","last_clean":"2023-09-23T17:46:59.915193+0000","last_became_active":"2023-09-23T17:57:53.875525+0000","last_became_peered":"2023-09-23T17:57:53.875525+0000","last_unstale":"2023-09-23T18:00:18.944967+0000","last_undegraded":"2023-09-23T18:00:18.944967+0000","last_fullsized":"2023-09-23T17:57:53.868097+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:57.748716+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":142,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.934371+0000","last_change":"2023-09-23T17:59:09.858216+0000","last_active":"2023-09-23T18:00:18.934371+0000","last_peered":"2023-09-23T18:00:18.934371+0000","last_clean":"2023-09-23T18:00:18.934371+0000","last_became_active":"2023-09-23T17:59:09.856443+0000","last_became_peered":"2023-09-23T17:59:09.856443+0000","last_unstale":"2023-09-23T18:00:18.934371+0000","last_undegraded":"2023-09-23T18:00:18.934371+0000","last_fullsized":"2023-09-23T18:00:18.934371+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:19.986245+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":139,"reported_epoch":192,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:18.937300+0000","last_change":"2023-09-23T17:59:10.883046+0000","last_active":"2023-09-23T18:00:18.937300+0000","last_peered":"2023-09-23T18:00:18.937300+0000","last_clean":"2023-09-23T18:00:18.937300+0000","last_became_active":"2023-09-23T17:59:10.882738+0000","last_became_peered":"2023-09-23T17:59:10.882738+0000","last_unstale":"2023-09-23T18:00:18.937300+0000","last_undegraded":"2023-09-23T18:00:18.937300+0000","last_fullsized":"2023-09-23T18:00:18.937300+0000","mapping_epoch":176,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":177,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:39.241547+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1,5],"avail_no_missing":["0","1","5"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"193'12401","reported_seq":20331,"reported_epoch":193,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:00:29.461048+0000","last_change":"2023-09-23T17:59:24.084532+0000","last_active":"2023-09-23T18:00:29.461048+0000","last_peered":"2023-09-23T18:00:29.461048+0000","last_clean":"2023-09-23T17:55:52.789238+0000","last_became_active":"2023-09-23T17:59:09.938603+0000","last_became_peered":"2023-09-23T17:59:09.938603+0000","last_unstale":"2023-09-23T18:00:29.461048+0000","last_undegraded":"2023-09-23T17:59:09.881270+0000","last_fullsized":"2023-09-23T17:59:09.846342+0000","mapping_epoch":175,"log_start":"161'7847","ondisk_log_start":"161'7847","created":110,"last_epoch_clean":111,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4554,"log_dups_size":0,"ondisk_log_size":4554,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:33:57.591912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38731776,"num_objects":591,"num_object_clones":0,"num_object_copies":1773,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":111,"num_objects_misplaced":443,"num_objects_unfound":0,"num_objects_dirty":591,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12401,"num_write_kb":415744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3912,"num_bytes_recovered":256376832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,2],"acting":[0,5],"avail_no_missing":["0","5"],"object_location_counts":[{"shards":"0,5","objects":591}],"blocked_by":[],"up_primary":6,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":489,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.945312+0000","last_change":"2023-09-23T17:59:00.119695+0000","last_active":"2023-09-23T18:00:18.945312+0000","last_peered":"2023-09-23T18:00:18.945312+0000","last_clean":"2023-09-23T18:00:18.945312+0000","last_became_active":"2023-09-23T17:59:00.119349+0000","last_became_peered":"2023-09-23T17:59:00.119349+0000","last_unstale":"2023-09-23T18:00:18.945312+0000","last_undegraded":"2023-09-23T18:00:18.945312+0000","last_fullsized":"2023-09-23T18:00:18.945312+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":62,"reported_epoch":192,"state":"undersized+degraded+peered","last_fresh":"2023-09-23T18:00:18.940703+0000","last_change":"2023-09-23T17:57:53.872726+0000","last_active":"2023-09-23T17:57:52.888539+0000","last_peered":"2023-09-23T18:00:18.940703+0000","last_clean":"2023-09-23T17:45:54.465055+0000","last_became_active":"2023-09-23T17:45:51.139637+0000","last_became_peered":"2023-09-23T17:57:53.872726+0000","last_unstale":"2023-09-23T18:00:18.940703+0000","last_undegraded":"2023-09-23T17:57:53.871230+0000","last_fullsized":"2023-09-23T17:57:53.871154+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:44:46.624357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0],"acting":[0],"avail_no_missing":["0"],"object_location_counts":[{"shards":"0","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":426,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:20.991665+0000","last_change":"2023-09-23T17:58:35.247928+0000","last_active":"2023-09-23T18:00:20.991665+0000","last_peered":"2023-09-23T18:00:20.991665+0000","last_clean":"2023-09-23T18:00:20.991665+0000","last_became_active":"2023-09-23T17:58:35.247482+0000","last_became_peered":"2023-09-23T17:58:35.247482+0000","last_unstale":"2023-09-23T18:00:20.991665+0000","last_undegraded":"2023-09-23T18:00:20.991665+0000","last_fullsized":"2023-09-23T18:00:20.991665+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:18.686446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":119,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.934256+0000","last_change":"2023-09-23T17:58:12.005101+0000","last_active":"2023-09-23T18:00:18.934256+0000","last_peered":"2023-09-23T18:00:18.934256+0000","last_clean":"2023-09-23T17:49:06.298151+0000","last_became_active":"2023-09-23T17:58:12.005101+0000","last_became_peered":"2023-09-23T17:58:12.005101+0000","last_unstale":"2023-09-23T18:00:18.934256+0000","last_undegraded":"2023-09-23T18:00:18.934256+0000","last_fullsized":"2023-09-23T17:58:11.987584+0000","mapping_epoch":154,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:15.859050+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1],"acting":[0,1],"avail_no_missing":["0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.1","version":"193'12654","reported_seq":25406,"reported_epoch":193,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:00:27.465174+0000","last_change":"2023-09-23T17:58:23.828372+0000","last_active":"2023-09-23T18:00:27.465174+0000","last_peered":"2023-09-23T18:00:27.465174+0000","last_clean":"2023-09-23T17:57:23.936571+0000","last_became_active":"2023-09-23T17:58:23.828372+0000","last_became_peered":"2023-09-23T17:58:23.828372+0000","last_unstale":"2023-09-23T18:00:27.465174+0000","last_undegraded":"2023-09-23T17:58:23.826763+0000","last_fullsized":"2023-09-23T17:58:23.826662+0000","mapping_epoch":159,"log_start":"136'5977","ondisk_log_start":"136'5977","created":110,"last_epoch_clean":130,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":6677,"log_dups_size":0,"ondisk_log_size":6677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:59.923478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":44302336,"num_objects":676,"num_object_clones":0,"num_object_copies":2028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":676,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12654,"num_write_kb":426560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6302,"num_bytes_recovered":413007872,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,0],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,4","objects":676}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":493,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.937823+0000","last_change":"2023-09-23T17:59:00.129488+0000","last_active":"2023-09-23T18:00:18.937823+0000","last_peered":"2023-09-23T18:00:18.937823+0000","last_clean":"2023-09-23T18:00:18.937823+0000","last_became_active":"2023-09-23T17:59:00.128847+0000","last_became_peered":"2023-09-23T17:59:00.128847+0000","last_unstale":"2023-09-23T18:00:18.937823+0000","last_undegraded":"2023-09-23T18:00:18.937823+0000","last_fullsized":"2023-09-23T18:00:18.937823+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":172,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":162896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,1],"acting":[0,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":368,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.915493+0000","last_change":"2023-09-23T17:58:35.227206+0000","last_active":"2023-09-23T18:00:18.915493+0000","last_peered":"2023-09-23T18:00:18.915493+0000","last_clean":"2023-09-23T17:46:59.535985+0000","last_became_active":"2023-09-23T17:58:35.227206+0000","last_became_peered":"2023-09-23T17:58:35.227206+0000","last_unstale":"2023-09-23T18:00:18.915493+0000","last_undegraded":"2023-09-23T18:00:18.915493+0000","last_fullsized":"2023-09-23T17:58:35.224367+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":104,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:52.549534+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":167,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.942906+0000","last_change":"2023-09-23T17:56:50.856836+0000","last_active":"2023-09-23T18:00:18.942906+0000","last_peered":"2023-09-23T18:00:18.942906+0000","last_clean":"2023-09-23T18:00:18.942906+0000","last_became_active":"2023-09-23T17:56:50.835155+0000","last_became_peered":"2023-09-23T17:56:50.835155+0000","last_unstale":"2023-09-23T18:00:18.942906+0000","last_undegraded":"2023-09-23T18:00:18.942906+0000","last_fullsized":"2023-09-23T18:00:18.942906+0000","mapping_epoch":126,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:36.188982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":173,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.921488+0000","last_change":"2023-09-23T17:59:50.897059+0000","last_active":"2023-09-23T18:00:18.921488+0000","last_peered":"2023-09-23T18:00:18.921488+0000","last_clean":"2023-09-23T18:00:18.921488+0000","last_became_active":"2023-09-23T17:59:50.896650+0000","last_became_peered":"2023-09-23T17:59:50.896650+0000","last_unstale":"2023-09-23T18:00:18.921488+0000","last_undegraded":"2023-09-23T18:00:18.921488+0000","last_fullsized":"2023-09-23T18:00:18.921488+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:40:16.102787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"193'12750","reported_seq":26153,"reported_epoch":193,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:00:25.969333+0000","last_change":"2023-09-23T17:59:32.083467+0000","last_active":"2023-09-23T18:00:25.969333+0000","last_peered":"2023-09-23T18:00:25.969333+0000","last_clean":"2023-09-23T18:00:25.969333+0000","last_became_active":"2023-09-23T17:57:26.317820+0000","last_became_peered":"2023-09-23T17:57:26.317820+0000","last_unstale":"2023-09-23T18:00:25.969333+0000","last_undegraded":"2023-09-23T18:00:25.969333+0000","last_fullsized":"2023-09-23T18:00:25.969333+0000","mapping_epoch":143,"log_start":"161'8255","ondisk_log_start":"161'8255","created":110,"last_epoch_clean":182,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:55:52.789238+0000","last_clean_scrub_stamp":"2023-09-23T17:55:52.789238+0000","objects_scrubbed":0,"log_size":4495,"log_dups_size":0,"ondisk_log_size":4495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:37.559775+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":46006272,"num_objects":702,"num_object_clones":0,"num_object_copies":2106,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":702,"num_objects_unfound":0,"num_objects_dirty":702,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12750,"num_write_kb":430464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6640,"num_bytes_recovered":435159040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":702}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":149,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.937243+0000","last_change":"2023-09-23T17:57:53.877134+0000","last_active":"2023-09-23T18:00:18.937243+0000","last_peered":"2023-09-23T18:00:18.937243+0000","last_clean":"2023-09-23T17:57:47.571295+0000","last_became_active":"2023-09-23T17:57:53.877134+0000","last_became_peered":"2023-09-23T17:57:53.877134+0000","last_unstale":"2023-09-23T18:00:18.937243+0000","last_undegraded":"2023-09-23T18:00:18.937243+0000","last_fullsized":"2023-09-23T17:57:53.869424+0000","mapping_epoch":150,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:08:59.045696+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":184,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.921014+0000","last_change":"2023-09-23T17:57:24.956919+0000","last_active":"2023-09-23T18:00:18.921014+0000","last_peered":"2023-09-23T18:00:18.921014+0000","last_clean":"2023-09-23T17:57:20.928421+0000","last_became_active":"2023-09-23T17:57:24.956919+0000","last_became_peered":"2023-09-23T17:57:24.956919+0000","last_unstale":"2023-09-23T18:00:18.921014+0000","last_undegraded":"2023-09-23T18:00:18.921014+0000","last_fullsized":"2023-09-23T17:57:24.951731+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:03.928787+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1],"acting":[2,1],"avail_no_missing":["2","1"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":48,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.922639+0000","last_change":"2023-09-23T17:59:50.901084+0000","last_active":"2023-09-23T18:00:18.922639+0000","last_peered":"2023-09-23T18:00:18.922639+0000","last_clean":"2023-09-23T18:00:18.922639+0000","last_became_active":"2023-09-23T17:59:50.900822+0000","last_became_peered":"2023-09-23T17:59:50.900822+0000","last_unstale":"2023-09-23T18:00:18.922639+0000","last_undegraded":"2023-09-23T18:00:18.922639+0000","last_fullsized":"2023-09-23T18:00:18.922639+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:08.825929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":97,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.921654+0000","last_change":"2023-09-23T17:59:50.897896+0000","last_active":"2023-09-23T18:00:18.921654+0000","last_peered":"2023-09-23T18:00:18.921654+0000","last_clean":"2023-09-23T17:57:20.922389+0000","last_became_active":"2023-09-23T17:59:50.897896+0000","last_became_peered":"2023-09-23T17:59:50.897896+0000","last_unstale":"2023-09-23T18:00:18.921654+0000","last_undegraded":"2023-09-23T18:00:18.921654+0000","last_fullsized":"2023-09-23T17:59:50.890883+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:36.213857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":44,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.923047+0000","last_change":"2023-09-23T17:59:50.899104+0000","last_active":"2023-09-23T18:00:18.923047+0000","last_peered":"2023-09-23T18:00:18.923047+0000","last_clean":"2023-09-23T18:00:18.923047+0000","last_became_active":"2023-09-23T17:59:50.898883+0000","last_became_peered":"2023-09-23T17:59:50.898883+0000","last_unstale":"2023-09-23T18:00:18.923047+0000","last_undegraded":"2023-09-23T18:00:18.923047+0000","last_fullsized":"2023-09-23T18:00:18.923047+0000","mapping_epoch":186,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":187,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:31.834372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":101,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.936624+0000","last_change":"2023-09-23T17:59:22.871834+0000","last_active":"2023-09-23T18:00:18.936624+0000","last_peered":"2023-09-23T18:00:18.936624+0000","last_clean":"2023-09-23T17:57:20.922822+0000","last_became_active":"2023-09-23T17:59:22.871834+0000","last_became_peered":"2023-09-23T17:59:22.871834+0000","last_unstale":"2023-09-23T18:00:18.936624+0000","last_undegraded":"2023-09-23T18:00:18.936624+0000","last_fullsized":"2023-09-23T17:59:22.864946+0000","mapping_epoch":179,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":127,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:40.097366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4],"avail_no_missing":["0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":51,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.914964+0000","last_change":"2023-09-23T17:58:35.229256+0000","last_active":"2023-09-23T18:00:18.914964+0000","last_peered":"2023-09-23T18:00:18.914964+0000","last_clean":"2023-09-23T17:45:50.074605+0000","last_became_active":"2023-09-23T17:58:35.229256+0000","last_became_peered":"2023-09-23T17:58:35.229256+0000","last_unstale":"2023-09-23T18:00:18.914964+0000","last_undegraded":"2023-09-23T18:00:18.914964+0000","last_fullsized":"2023-09-23T17:58:35.224895+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:14.916976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":143,"reported_epoch":192,"state":"active+undersized","last_fresh":"2023-09-23T18:00:18.920571+0000","last_change":"2023-09-23T17:57:24.955661+0000","last_active":"2023-09-23T18:00:18.920571+0000","last_peered":"2023-09-23T18:00:18.920571+0000","last_clean":"2023-09-23T17:53:52.727684+0000","last_became_active":"2023-09-23T17:57:24.955661+0000","last_became_peered":"2023-09-23T17:57:24.955661+0000","last_unstale":"2023-09-23T18:00:18.920571+0000","last_undegraded":"2023-09-23T18:00:18.920571+0000","last_fullsized":"2023-09-23T17:57:24.945672+0000","mapping_epoch":142,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":109,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:04.500443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0],"acting":[2,0],"avail_no_missing":["2","0"],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":156,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.940612+0000","last_change":"2023-09-23T17:59:09.869351+0000","last_active":"2023-09-23T18:00:18.940612+0000","last_peered":"2023-09-23T18:00:18.940612+0000","last_clean":"2023-09-23T18:00:18.940612+0000","last_became_active":"2023-09-23T17:59:09.861326+0000","last_became_peered":"2023-09-23T17:59:09.861326+0000","last_unstale":"2023-09-23T18:00:18.940612+0000","last_undegraded":"2023-09-23T18:00:18.940612+0000","last_fullsized":"2023-09-23T18:00:18.940612+0000","mapping_epoch":175,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":176,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:56:38.142504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,6],"acting":[0,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":182,"reported_epoch":192,"state":"active+clean","last_fresh":"2023-09-23T18:00:18.921018+0000","last_change":"2023-09-23T17:58:35.243060+0000","last_active":"2023-09-23T18:00:18.921018+0000","last_peered":"2023-09-23T18:00:18.921018+0000","last_clean":"2023-09-23T18:00:18.921018+0000","last_became_active":"2023-09-23T17:58:35.242727+0000","last_became_peered":"2023-09-23T17:58:35.242727+0000","last_unstale":"2023-09-23T18:00:18.921018+0000","last_undegraded":"2023-09-23T18:00:18.921018+0000","last_fullsized":"2023-09-23T18:00:18.921018+0000","mapping_epoch":163,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":164,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:23:08.850283+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":651100184,"num_objects":9936,"num_object_clones":0,"num_object_copies":29808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4341,"num_objects_misplaced":14080,"num_objects_unfound":0,"num_objects_dirty":9936,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":201312,"num_write_kb":6759873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40058,"num_bytes_recovered":2534277120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5054738432,"data_stored":5054726216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":110346,"ondisk_log_size":110346,"up":40,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":72,"acting":78,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":10,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":756,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":114688,"data_stored":21336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":33098,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":69,"acting":78,"num_store_stats":8},{"poolid":1,"num_pg":21,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":486,"num_read_kb":912,"num_write":239,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2361472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3381,"ondisk_log_size":3381,"up":63,"acting":63,"num_store_stats":4},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":19,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":17,"acting":18,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":53,"seq":227633266849,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1840208,"kb_used_data":1778696,"kb_used_omap":0,"kb_used_meta":61504,"kb_avail":91904944,"statfs":{"total":95995035648,"available":94110662656,"internally_reserved":0,"allocated":1821384704,"data_stored":1820756859,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":62980096},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.6299999999999999,"5min":0.83399999999999996,"15min":0.64500000000000002},"min":{"1min":0.24099999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.5700000000000003,"5min":8.5700000000000003,"15min":8.5700000000000003},"last":2.2210000000000001},{"interface":"front","average":{"1min":1.6499999999999999,"5min":0.879,"15min":0.67100000000000004},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.8410000000000002,"5min":6.8410000000000002,"15min":6.8410000000000002},"last":4.1280000000000001}]},{"osd":1,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.59599999999999997,"15min":0.56999999999999995},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":2.702,"5min":2.702,"15min":2.702},"last":6.6890000000000001},{"interface":"front","average":{"1min":1.1399999999999999,"5min":0.65700000000000003,"15min":0.58899999999999997},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.4240000000000004,"5min":4.4240000000000004,"15min":4.4240000000000004},"last":0.26000000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:57:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3740000000000001,"5min":0.89200000000000002,"15min":0.66300000000000003},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":5.2359999999999998,"5min":5.2359999999999998,"15min":5.2359999999999998},"last":2.2970000000000002},{"interface":"front","average":{"1min":1.0289999999999999,"5min":0.77900000000000003,"15min":0.64800000000000002},"min":{"1min":0.38100000000000001,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":3.8340000000000001,"5min":4.6180000000000003,"15min":4.6180000000000003},"last":0.439}]},{"osd":3,"last update":"Sat Sep 23 17:57:00 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":0.64900000000000002,"15min":0.57299999999999995},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":4.0549999999999997,"5min":4.0549999999999997,"15min":4.0549999999999997},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.665,"5min":0.77400000000000002,"15min":0.626},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":5.1609999999999996,"5min":5.1609999999999996,"15min":5.1609999999999996},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.66600000000000004,"5min":0.53400000000000003,"15min":0.47099999999999997},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":2.6160000000000001,"5min":2.6160000000000001,"15min":2.6160000000000001},"last":2.1949999999999998},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.56100000000000005,"15min":0.498},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.8079999999999998,"5min":2.8079999999999998,"15min":2.8079999999999998},"last":0.68799999999999994}]},{"osd":5,"last update":"Sat Sep 23 17:57:05 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":0.60699999999999998,"15min":0.52600000000000002},"min":{"1min":0.27100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.5840000000000001,"5min":2.5840000000000001,"15min":2.5840000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.749,"5min":0.64100000000000001,"15min":0.54900000000000004},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.706,"5min":2.706,"15min":2.706},"last":2.1669999999999998}]},{"osd":6,"last update":"Sat Sep 23 17:55:01 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.59599999999999997,"15min":0.54000000000000004},"min":{"1min":0.36499999999999999,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.86399999999999999,"5min":0.871,"15min":0.91800000000000004},"last":1.002},{"interface":"front","average":{"1min":0.58099999999999996,"5min":0.59699999999999998,"15min":0.53300000000000003},"min":{"1min":0.34899999999999998,"5min":0.34000000000000002,"15min":0.315},"max":{"1min":0.71299999999999997,"5min":0.89300000000000002,"15min":0.89300000000000002},"last":0.435}]}]},{"osd":6,"up_from":163,"seq":700079669272,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":153368,"kb_used_data":90896,"kb_used_omap":7,"kb_used_meta":62456,"kb_avail":93591784,"statfs":{"total":95995035648,"available":95837986816,"internally_reserved":0,"allocated":93077504,"data_stored":92298360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":63955748},"hb_peers":[0,1,2,4,5],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":1.3500000000000001,"5min":1.3500000000000001,"15min":1.3500000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":14.196999999999999,"5min":14.196999999999999,"15min":14.196999999999999},"last":1.8049999999999999},{"interface":"front","average":{"1min":2.7010000000000001,"5min":2.7010000000000001,"15min":2.7010000000000001},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":13.536,"5min":13.536,"15min":13.536},"last":1.544}]},{"osd":1,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":2.141,"5min":2.141,"15min":2.141},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":14.233000000000001,"5min":14.233000000000001,"15min":14.233000000000001},"last":1.8819999999999999},{"interface":"front","average":{"1min":2.4220000000000002,"5min":2.4220000000000002,"15min":2.4220000000000002},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":9.1099999999999994,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":1.5129999999999999}]},{"osd":2,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":2.5169999999999999,"5min":2.5169999999999999,"15min":2.5169999999999999},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":11.210000000000001,"5min":11.210000000000001,"15min":11.210000000000001},"last":3.8650000000000002},{"interface":"front","average":{"1min":3.0419999999999998,"5min":3.0419999999999998,"15min":3.0419999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":11.223000000000001,"5min":11.223000000000001,"15min":11.223000000000001},"last":3.879}]},{"osd":4,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":0.85799999999999998,"15min":0.85799999999999998},"min":{"1min":0.248,"5min":0.248,"15min":0.248},"max":{"1min":8.4570000000000007,"5min":8.4570000000000007,"15min":8.4570000000000007},"last":0.32900000000000001},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.90300000000000002,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.27400000000000002}]},{"osd":5,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":0.79500000000000004,"5min":0.79500000000000004,"15min":0.79500000000000004},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.3330000000000002,"5min":4.3330000000000002,"15min":4.3330000000000002},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.3530000000000002,"5min":2.3530000000000002,"15min":2.3530000000000002},"last":0.314}]}]},{"osd":1,"up_from":98,"seq":420906795089,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":419584,"kb_used_data":283640,"kb_used_omap":7,"kb_used_meta":135864,"kb_avail":93325568,"statfs":{"total":95995035648,"available":95565381632,"internally_reserved":0,"allocated":290447360,"data_stored":289460727,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":139125437},"hb_peers":[0,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:59:57 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.573,"15min":0.78300000000000003},"min":{"1min":0.17699999999999999,"5min":0.17699999999999999,"15min":0.17699999999999999},"max":{"1min":15.529999999999999,"5min":15.529999999999999,"15min":15.529999999999999},"last":0.218},{"interface":"front","average":{"1min":2.2799999999999998,"5min":1.8049999999999999,"15min":0.86599999999999999},"min":{"1min":0.27400000000000002,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":8.3239999999999998,"5min":12.355,"15min":12.355},"last":0.32000000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:59:36 2023","interfaces":[{"interface":"back","average":{"1min":3.8860000000000001,"5min":1.994,"15min":0.96499999999999997},"min":{"1min":0.24099999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":16.347999999999999,"5min":16.347999999999999,"15min":16.347999999999999},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.964,"5min":1.5509999999999999,"15min":0.81000000000000005},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.21299999999999999},"max":{"1min":9.8379999999999992,"5min":11.305999999999999,"15min":11.305999999999999},"last":7.7119999999999997}]},{"osd":3,"last update":"Sat Sep 23 17:57:00 2023","interfaces":[{"interface":"back","average":{"1min":2.8999999999999999,"5min":0.90800000000000003,"15min":0.57599999999999996},"min":{"1min":0.29899999999999999,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":14.548999999999999,"5min":14.548999999999999,"15min":14.548999999999999},"last":2.7149999999999999},{"interface":"front","average":{"1min":2.3740000000000001,"5min":0.85199999999999998,"15min":0.59799999999999998},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":10.250999999999999,"5min":10.250999999999999,"15min":10.250999999999999},"last":0.82899999999999996}]},{"osd":4,"last update":"Sat Sep 23 17:59:57 2023","interfaces":[{"interface":"back","average":{"1min":2.4100000000000001,"5min":1.47,"15min":0.94499999999999995},"min":{"1min":0.33600000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":9.4740000000000002,"5min":9.4740000000000002,"15min":9.4740000000000002},"last":1.702},{"interface":"front","average":{"1min":1.718,"5min":1.45,"15min":0.93500000000000005},"min":{"1min":0.28699999999999998,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.8879999999999999,"5min":9.8879999999999999,"15min":9.8879999999999999},"last":7.6609999999999996}]},{"osd":5,"last update":"Sat Sep 23 17:59:57 2023","interfaces":[{"interface":"back","average":{"1min":2.1160000000000001,"5min":1.579,"15min":1.0049999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":8.8390000000000004,"5min":8.8390000000000004,"15min":8.8390000000000004},"last":0.55200000000000005},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.6359999999999999,"15min":1.016},"min":{"1min":0.34100000000000003,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":1.762}]},{"osd":6,"last update":"Sat Sep 23 17:59:36 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":2.2690000000000001,"15min":2.2690000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":9.5229999999999997,"5min":9.5229999999999997,"15min":9.5229999999999997},"last":1.794},{"interface":"front","average":{"1min":1.746,"5min":1.746,"15min":1.746},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":4.5,"5min":4.5,"15min":4.5},"last":2.1240000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:56:54 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":0.89400000000000002,"15min":0.74399999999999999},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":4.9630000000000001,"5min":4.9630000000000001,"15min":4.9630000000000001},"last":7.8869999999999996},{"interface":"front","average":{"1min":1.895,"5min":0.92900000000000005,"15min":0.77900000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":8.2940000000000005,"5min":8.2940000000000005,"15min":8.2940000000000005},"last":7.234}]}]},{"osd":0,"up_from":93,"seq":399431958616,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":744372,"kb_used_data":428524,"kb_used_omap":7,"kb_used_meta":315576,"kb_avail":93000780,"statfs":{"total":95995035648,"available":95232798720,"internally_reserved":0,"allocated":438808576,"data_stored":437830296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":323150564},"hb_peers":[1,2,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:00:02 2023","interfaces":[{"interface":"back","average":{"1min":3.988,"5min":2.1829999999999998,"15min":0.97399999999999998},"min":{"1min":0.186,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":13.414,"5min":18.154,"15min":18.154},"last":0.245},{"interface":"front","average":{"1min":3.355,"5min":1.9990000000000001,"15min":0.94999999999999996},"min":{"1min":0.17399999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":13.369999999999999,"5min":13.369999999999999,"15min":13.369999999999999},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Sep 23 17:59:34 2023","interfaces":[{"interface":"back","average":{"1min":4.7380000000000004,"5min":2.1150000000000002,"15min":0.94499999999999995},"min":{"1min":0.24299999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":20.882000000000001,"5min":20.882000000000001,"15min":20.882000000000001},"last":9.3900000000000006},{"interface":"front","average":{"1min":3.3620000000000001,"5min":1.8500000000000001,"15min":0.86199999999999999},"min":{"1min":0.255,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":15.064,"5min":15.064,"15min":15.064},"last":0.27600000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":0.77100000000000002,"15min":0.60099999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3449999999999998,"5min":4.3449999999999998,"15min":4.3449999999999998},"last":9.266},{"interface":"front","average":{"1min":2.109,"5min":0.73699999999999999,"15min":0.50900000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":8.8729999999999993},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 23 17:59:34 2023","interfaces":[{"interface":"back","average":{"1min":3.1139999999999999,"5min":1.5429999999999999,"15min":0.93799999999999994},"min":{"1min":0.29499999999999998,"5min":0.251,"15min":0.251},"max":{"1min":15.279999999999999,"5min":15.279999999999999,"15min":15.279999999999999},"last":0.38300000000000001},{"interface":"front","average":{"1min":3.0590000000000002,"5min":1.512,"15min":0.91900000000000004},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":18.106999999999999,"5min":18.106999999999999,"15min":18.106999999999999},"last":0.317}]},{"osd":5,"last update":"Sat Sep 23 17:59:34 2023","interfaces":[{"interface":"back","average":{"1min":3.0419999999999998,"5min":1.5449999999999999,"15min":0.93700000000000006},"min":{"1min":0.33000000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":14.720000000000001,"5min":14.720000000000001,"15min":14.720000000000001},"last":9.4649999999999999},{"interface":"front","average":{"1min":2.472,"5min":1.268,"15min":0.85199999999999998},"min":{"1min":0.35099999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":8.7530000000000001,"5min":8.7829999999999995,"15min":8.7829999999999995},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 17:59:34 2023","interfaces":[{"interface":"back","average":{"1min":2.5179999999999998,"5min":2.5179999999999998,"15min":2.5179999999999998},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":14.956,"5min":14.956,"15min":14.956},"last":0.41799999999999998},{"interface":"front","average":{"1min":2.762,"5min":2.762,"15min":2.762},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":8.7249999999999996,"5min":8.7249999999999996,"15min":8.7249999999999996},"last":0.53100000000000003}]},{"osd":7,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.8460000000000001,"5min":0.90800000000000003,"15min":0.746},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":8.8160000000000007,"5min":8.8160000000000007,"15min":8.8160000000000007},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.01,"15min":0.78400000000000003},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":8.6470000000000002,"5min":8.6470000000000002,"15min":8.6470000000000002},"last":0.49099999999999999}]}]},{"osd":2,"up_from":103,"seq":442381631563,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":469812,"kb_used_data":285484,"kb_used_omap":11,"kb_used_meta":184180,"kb_avail":93275340,"statfs":{"total":95995035648,"available":95513948160,"internally_reserved":0,"allocated":292335616,"data_stored":291359943,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":188600695},"hb_peers":[0,1,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:59:33 2023","interfaces":[{"interface":"back","average":{"1min":3.2189999999999999,"5min":1.7789999999999999,"15min":0.81899999999999995},"min":{"1min":0.17199999999999999,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":15.006,"5min":15.006,"15min":15.006},"last":12.24},{"interface":"front","average":{"1min":3.0960000000000001,"5min":1.8140000000000001,"15min":0.85999999999999999},"min":{"1min":0.26700000000000002,"5min":0.215,"15min":0.215},"max":{"1min":11.890000000000001,"5min":13.364000000000001,"15min":13.364000000000001},"last":12.141999999999999}]},{"osd":1,"last update":"Sat Sep 23 17:59:33 2023","interfaces":[{"interface":"back","average":{"1min":3.5289999999999999,"5min":1.9570000000000001,"15min":0.92000000000000004},"min":{"1min":0.318,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":10.339,"5min":19.206,"15min":19.206},"last":11.169},{"interface":"front","average":{"1min":1.653,"5min":1.79,"15min":0.81599999999999995},"min":{"1min":0.246,"5min":0.22500000000000001,"15min":0.22500000000000001},"max":{"1min":5.008,"5min":15.657,"15min":15.657},"last":0.28299999999999997}]},{"osd":3,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":0.58799999999999997,"15min":0.443},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8339999999999996,"5min":4.8339999999999996,"15min":4.8339999999999996},"last":2.8849999999999998},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.59699999999999998,"15min":0.48399999999999999},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.6459999999999999,"5min":5.6459999999999999,"15min":5.6459999999999999},"last":2.5579999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:59:33 2023","interfaces":[{"interface":"back","average":{"1min":1.71,"5min":1.3560000000000001,"15min":0.85399999999999998},"min":{"1min":0.28299999999999997,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.0670000000000002,"5min":7.0670000000000002,"15min":7.0670000000000002},"last":1.3520000000000001},{"interface":"front","average":{"1min":2.3130000000000002,"5min":1.258,"15min":0.82599999999999996},"min":{"1min":0.33200000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.0939999999999994,"5min":13.308,"15min":13.308},"last":11.112}]},{"osd":5,"last update":"Sat Sep 23 17:59:33 2023","interfaces":[{"interface":"back","average":{"1min":2.7519999999999998,"5min":1.462,"15min":0.90300000000000002},"min":{"1min":0.32100000000000001,"5min":0.315,"15min":0.315},"max":{"1min":9.0030000000000001,"5min":13.273,"15min":13.273},"last":11.066000000000001},{"interface":"front","average":{"1min":1.5169999999999999,"5min":1.6359999999999999,"15min":0.94799999999999995},"min":{"1min":0.34100000000000003,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":3.863,"5min":15.496,"15min":15.496},"last":0.33100000000000002}]},{"osd":6,"last update":"Sat Sep 23 17:59:36 2023","interfaces":[{"interface":"back","average":{"1min":2.3220000000000001,"5min":2.3220000000000001,"15min":2.3220000000000001},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":9.4649999999999999,"5min":9.4649999999999999,"15min":9.4649999999999999},"last":1.389},{"interface":"front","average":{"1min":2.5760000000000001,"5min":2.5760000000000001,"15min":2.5760000000000001},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":8.1910000000000007,"5min":8.1910000000000007,"15min":8.1910000000000007},"last":11.140000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:57:29 2023","interfaces":[{"interface":"back","average":{"1min":1.889,"5min":0.94299999999999995,"15min":0.71899999999999997},"min":{"1min":0.505,"5min":0.441,"15min":0.441},"max":{"1min":5.7409999999999997,"5min":5.7409999999999997,"15min":5.7409999999999997},"last":0.48999999999999999},{"interface":"front","average":{"1min":2.0870000000000002,"5min":0.97899999999999998,"15min":0.78300000000000003},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":13.333,"5min":13.333,"15min":13.333},"last":1.3440000000000001}]}]},{"osd":3,"up_from":108,"seq":463856467999,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267348,"kb_used_data":1221900,"kb_used_omap":3,"kb_used_meta":45436,"kb_avail":92477804,"statfs":{"total":95995035648,"available":94697271296,"internally_reserved":0,"allocated":1251225600,"data_stored":1250531999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":46527208},"hb_peers":[0,1,2,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.0699999999999998,"5min":0.71399999999999997,"15min":0.48799999999999999},"min":{"1min":0.19,"5min":0.19,"15min":0.19},"max":{"1min":14.82,"5min":14.82,"15min":14.82},"last":9.0220000000000002},{"interface":"front","average":{"1min":2.085,"5min":0.72499999999999998,"15min":0.499},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":13.054,"5min":13.054,"15min":13.054},"last":0.46500000000000002}]},{"osd":1,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.7809999999999999,"5min":0.65700000000000003,"15min":0.46899999999999997},"min":{"1min":0.254,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":8.8130000000000006,"5min":8.8130000000000006,"15min":8.8130000000000006},"last":9.1489999999999991},{"interface":"front","average":{"1min":1.417,"5min":0.61599999999999999,"15min":0.48199999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.4580000000000002,"5min":6.4580000000000002,"15min":6.4580000000000002},"last":9.125}]},{"osd":2,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.887,"5min":0.69099999999999995,"15min":0.49099999999999999},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.869999999999999,"5min":10.869999999999999,"15min":10.869999999999999},"last":7.3849999999999998},{"interface":"front","average":{"1min":1.6850000000000001,"5min":0.66500000000000004,"15min":0.495},"min":{"1min":0.376,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":13.936999999999999,"5min":13.936999999999999,"15min":13.936999999999999},"last":8.9169999999999998}]},{"osd":4,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.2450000000000001,"5min":0.73699999999999999,"15min":0.65200000000000002},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.5380000000000003,"5min":6.5380000000000003,"15min":6.5380000000000003},"last":6.7569999999999997},{"interface":"front","average":{"1min":1.629,"5min":0.82099999999999995,"15min":0.68700000000000006},"min":{"1min":0.38200000000000001,"5min":0.38200000000000001,"15min":0.38200000000000001},"max":{"1min":6.5940000000000003,"5min":6.5940000000000003,"15min":6.5940000000000003},"last":3.7709999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":2.254,"5min":0.94399999999999995,"15min":0.72599999999999998},"min":{"1min":0.41999999999999998,"5min":0.39600000000000002,"15min":0.39600000000000002},"max":{"1min":9.6489999999999991,"5min":9.6489999999999991,"15min":9.6489999999999991},"last":3.9569999999999999},{"interface":"front","average":{"1min":1.4710000000000001,"5min":0.78600000000000003,"15min":0.67200000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":9.3200000000000003,"5min":9.3200000000000003,"15min":9.3200000000000003},"last":6.4569999999999999}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":0.81799999999999995,"15min":0.68999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":8.8599999999999994,"5min":8.8599999999999994,"15min":8.8599999999999994},"last":6.4950000000000001},{"interface":"front","average":{"1min":1.3520000000000001,"5min":0.78800000000000003,"15min":0.69399999999999995},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":9.2080000000000002,"5min":9.2080000000000002,"15min":9.2080000000000002},"last":6.0709999999999997}]}]},{"osd":4,"up_from":30,"seq":128849019081,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":480024,"kb_used_data":336976,"kb_used_omap":0,"kb_used_meta":142912,"kb_avail":93265128,"statfs":{"total":95995035648,"available":95503491072,"internally_reserved":0,"allocated":345063424,"data_stored":344283562,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":146341888},"hb_peers":[0,1,2,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 17:59:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1899999999999999,"5min":0.97399999999999998,"15min":0.66900000000000004},"min":{"1min":0.22500000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.5279999999999996,"5min":6.5279999999999996,"15min":6.5279999999999996},"last":2.3700000000000001},{"interface":"front","average":{"1min":1.744,"5min":1.3300000000000001,"15min":0.79300000000000004},"min":{"1min":0.22900000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":7.5739999999999998,"5min":11.372,"15min":11.372},"last":5.6680000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:00:01 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.5289999999999999,"15min":0.86299999999999999},"min":{"1min":0.28100000000000003,"5min":0.249,"15min":0.249},"max":{"1min":8.5259999999999998,"5min":10.429,"15min":10.429},"last":3.7589999999999999},{"interface":"front","average":{"1min":1.732,"5min":1.6319999999999999,"15min":0.91600000000000004},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":8.8130000000000006,"5min":10.83,"15min":10.83},"last":3.6480000000000001}]},{"osd":2,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":2.5379999999999998,"5min":1.492,"15min":0.91900000000000004},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.630000000000001,"5min":11.630000000000001,"15min":11.630000000000001},"last":3.75},{"interface":"front","average":{"1min":1.9279999999999999,"5min":1.375,"15min":0.84799999999999998},"min":{"1min":0.32000000000000001,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4909999999999997,"5min":7.5090000000000003,"15min":7.5090000000000003},"last":5.7869999999999999}]},{"osd":3,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.72899999999999998,"15min":0.64900000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6130000000000004,"5min":4.6130000000000004,"15min":4.6130000000000004},"last":1.397},{"interface":"front","average":{"1min":1.04,"5min":0.73299999999999998,"15min":0.68200000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":3.637,"5min":3.637,"15min":3.637},"last":9.4359999999999999}]},{"osd":5,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":0.60599999999999998,"15min":0.504},"min":{"1min":0.251,"5min":0.224,"15min":0.224},"max":{"1min":5.4299999999999997,"5min":5.4299999999999997,"15min":5.4299999999999997},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.86699999999999999,"5min":0.77200000000000002,"15min":0.56899999999999995},"min":{"1min":0.27100000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.2610000000000001,"5min":10.494,"15min":10.494},"last":5.6260000000000003}]},{"osd":6,"last update":"Sat Sep 23 17:59:35 2023","interfaces":[{"interface":"back","average":{"1min":0.77000000000000002,"5min":0.77000000000000002,"15min":0.77000000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":3.3479999999999999,"5min":3.3479999999999999,"15min":3.3479999999999999},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.399,"5min":3.399,"15min":3.399},"last":0.45000000000000001}]},{"osd":7,"last update":"Sat Sep 23 17:57:10 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.57399999999999995,"15min":0.55100000000000005},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":2.8370000000000002,"5min":2.8370000000000002,"15min":2.8370000000000002},"last":0.308},{"interface":"front","average":{"1min":0.95299999999999996,"5min":0.64400000000000002,"15min":0.56299999999999994},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":4.8849999999999998,"5min":4.8849999999999998,"15min":4.8849999999999998},"last":0.434}]}]},{"osd":5,"up_from":0,"seq":163208757447,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":70,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":416284672,"data_stored":416284672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":277872640,"data_stored":277872640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":275582976,"data_stored":275578904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1253441536,"data_stored":1253441536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":337182720,"data_stored":337182720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":586420224,"data_stored":586416152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":89849856,"data_stored":89849856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1818103808,"data_stored":1818099736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:00:32.146 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:00:32.146 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 31 2023-09-23T18:00:32.167 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:32.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:32.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:32.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:32.310 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:32 smithi099 ceph-mon[167076]: pgmap v449: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 626 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 4352/29892 objects degraded (14.559%); 14092/29892 objects misplaced (47.143%); 1.9 MiB/s, 69 objects/s recovering 2023-09-23T18:00:32.310 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:32 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1712343765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:32.310 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:32 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:32.314 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:32.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:32.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:32.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:32.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:32 smithi055 ceph-mon[184174]: pgmap v449: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 626 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 4352/29892 objects degraded (14.559%); 14092/29892 objects misplaced (47.143%); 1.9 MiB/s, 69 objects/s recovering 2023-09-23T18:00:32.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:32 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1712343765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:32.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:32 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:32.436 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:32 smithi055 ceph-mon[179668]: pgmap v449: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 626 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 671 op/s; 4352/29892 objects degraded (14.559%); 14092/29892 objects misplaced (47.143%); 1.9 MiB/s, 69 objects/s recovering 2023-09-23T18:00:32.436 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:32 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1712343765' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:32.436 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:32 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:32.472 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:32.570 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:32.663 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:32.667 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:00:32.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:32.671 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:32.771 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:32.829 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:32.839 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:32.872 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:32.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:32.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:32.976 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:33.028 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:33.100 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:33.157 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:33.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:33.167 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:33.268 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:33.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[184174]: from='client.35202 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:00:33.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:00:33.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[184174]: failed to encode map e194 with expected crc 2023-09-23T18:00:33.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[184174]: failed to encode map e194 with expected crc 2023-09-23T18:00:33.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:00:33.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[184174]: osdmap e194: 8 total, 6 up, 7 in 2023-09-23T18:00:33.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 4352/29892 objects degraded (14.559%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:33.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[179668]: from='client.35202 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:00:33.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:00:33.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[179668]: failed to encode map e194 with expected crc 2023-09-23T18:00:33.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[179668]: failed to encode map e194 with expected crc 2023-09-23T18:00:33.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:00:33.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[179668]: osdmap e194: 8 total, 6 up, 7 in 2023-09-23T18:00:33.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 4352/29892 objects degraded (14.559%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:33.368 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:33 smithi099 ceph-mon[167076]: from='client.35202 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:00:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:33 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:00:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:33 smithi099 ceph-mon[167076]: failed to encode map e194 with expected crc 2023-09-23T18:00:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:33 smithi099 ceph-mon[167076]: failed to encode map e194 with expected crc 2023-09-23T18:00:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:33 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:00:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:33 smithi099 ceph-mon[167076]: osdmap e194: 8 total, 6 up, 7 in 2023-09-23T18:00:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 4352/29892 objects degraded (14.559%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:34.053 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:34.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:34.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:34.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:34.208 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:34.264 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:34.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:34.271 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:34.372 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:34.421 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:34 smithi055 ceph-mon[179668]: pgmap v451: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 551 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 4166/26310 objects degraded (15.834%); 13117/26310 objects misplaced (49.856%); 4.8 MiB/s, 99 objects/s recovering 2023-09-23T18:00:34.421 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:34 smithi055 ceph-mon[184174]: pgmap v451: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 551 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 4166/26310 objects degraded (15.834%); 13117/26310 objects misplaced (49.856%); 4.8 MiB/s, 99 objects/s recovering 2023-09-23T18:00:34.423 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:34.498 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:00:34.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:34 smithi099 ceph-mon[167076]: pgmap v451: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 551 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 742 op/s; 4166/26310 objects degraded (15.834%); 13117/26310 objects misplaced (49.856%); 4.8 MiB/s, 99 objects/s recovering 2023-09-23T18:00:36.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:36 smithi055 ceph-mon[184174]: pgmap v452: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 427 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 3387/20379 objects degraded (16.620%); 12025/20379 objects misplaced (59.007%); 4.1 MiB/s, 85 objects/s recovering 2023-09-23T18:00:36.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:36 smithi055 ceph-mon[179668]: pgmap v452: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 427 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 3387/20379 objects degraded (16.620%); 12025/20379 objects misplaced (59.007%); 4.1 MiB/s, 85 objects/s recovering 2023-09-23T18:00:36.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:36 smithi099 ceph-mon[167076]: pgmap v452: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 427 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 655 op/s; 3387/20379 objects degraded (16.620%); 12025/20379 objects misplaced (59.007%); 4.1 MiB/s, 85 objects/s recovering 2023-09-23T18:00:36.903 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:36.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:36.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:37.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:37.050 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:37.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:37.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:37.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:37.202 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:37.249 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:37.269 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:37.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:37.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:37.294 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:37.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:37.433 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:37.487 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:37.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:37.495 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:37.577 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:37.577 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:37.577 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:37 smithi055 ceph-mon[184174]: pgmap v453: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 386 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 3387/18408 objects degraded (18.400%); 11595/18408 objects misplaced (62.989%); 4.1 MiB/s, 85 objects/s recovering 2023-09-23T18:00:37.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:37.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:37.578 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:37 smithi055 ceph-mon[179668]: pgmap v453: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 386 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 3387/18408 objects degraded (18.400%); 11595/18408 objects misplaced (62.989%); 4.1 MiB/s, 85 objects/s recovering 2023-09-23T18:00:37.596 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:37.632 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:37.679 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:37.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:37.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:37.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:37 smithi099 ceph-mon[167076]: pgmap v453: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 386 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 662 op/s; 3387/18408 objects degraded (18.400%); 11595/18408 objects misplaced (62.989%); 4.1 MiB/s, 85 objects/s recovering 2023-09-23T18:00:37.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:37.684 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:37.775 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:37.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:37.818 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:37.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:37.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:37.860 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:37.871 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:00:37.871 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:00:37.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.871 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:207629 5.7 e194/176)", 2023-09-23T18:00:37.871 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:37.867521+0000", 2023-09-23T18:00:37.871 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.00063415200000000002, 2023-09-23T18:00:37.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00065858699999999997, 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867521+0000", 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867521+0000", 2023-09-23T18:00:37.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867520+0000", 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867528+0000", 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3899999999999993e-06 2023-09-23T18:00:37.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:37.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867544+0000", 2023-09-23T18:00:37.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5798e-05 2023-09-23T18:00:37.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:37.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867552+0000", 2023-09-23T18:00:37.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5290000000000003e-06 2023-09-23T18:00:37.874 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:207631 5.7 e194/176)", 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:37.867569+0000", 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.00058548200000000002, 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00068966000000000003, 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:37.875 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867569+0000", 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867569+0000", 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867569+0000", 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867574+0000", 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3020000000000002e-06 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:37.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867581+0000", 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.148e-06 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867586+0000", 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9370000000000003e-06 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:207630 5.3 5:c42fc462:::benchmark_data_smithi055_229117_object102004:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:37.858035+0000", 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.010120221, 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010269169999999999, 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 207630 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.879 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858035+0000", 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858035+0000", 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858034+0000", 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858038+0000", 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2529999999999998e-06 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:37.881 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858046+0000", 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3559999999999993e-06 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858052+0000", 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5690000000000004e-06 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.861409+0000", 2023-09-23T18:00:37.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033573119999999999 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.861540+0000", 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000130574 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.861672+0000", 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000132261 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.866680+0000", 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050078889999999997 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.867850+0000", 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011699989999999999 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:37.884 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:207628 5.1 e194/159 5:812ed989:::benchmark_data_smithi055_229117_object102002:head v 194'13095, mlcod=194'13093)", 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:37.858306+0000", 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0098492430000000006, 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010050351000000001, 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:37.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858306+0000", 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858306+0000", 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858304+0000", 2023-09-23T18:00:37.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858314+0000", 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6819999999999998e-06 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858326+0000", 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2077e-05 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858334+0000", 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6399999999999997e-06 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858395+0000", 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1317999999999998e-05 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.858420+0000", 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4538000000000001e-05 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:207638 5.c e194/127 5:3599d527:::benchmark_data_smithi055_229117_object102012:head v 194'12999, mlcod=194'12997)", 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:37.862336+0000", 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0058190960000000002, 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060564130000000001, 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:37.889 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862336+0000", 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862336+0000", 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.890 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862335+0000", 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862343+0000", 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5380000000000002e-06 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862355+0000", 2023-09-23T18:00:37.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1361e-05 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862362+0000", 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9870000000000002e-06 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862456+0000", 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4149999999999998e-05 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862478+0000", 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2016000000000001e-05 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:207636 5.0 e194/143 5:0173a417:::benchmark_data_smithi055_229117_object102010:head v 194'13243, mlcod=194'13239)", 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:37.862376+0000", 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0057786970000000002, 2023-09-23T18:00:37.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060527869999999996, 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862376+0000", 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:37.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862376+0000", 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862376+0000", 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862380+0000", 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2010000000000001e-06 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862386+0000", 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2090000000000002e-06 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862391+0000", 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0409999999999997e-06 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:37.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862535+0000", 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014335800000000001 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:37.862557+0000", 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2107999999999999e-05 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:00:37.897 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 6 2023-09-23T18:00:37.898 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:00:37.960 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:38.051 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:38.433 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:00:38.583 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:38.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:38.594 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:38.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 3387/18408 objects degraded (18.400%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:38.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 3387/18408 objects degraded (18.400%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:38.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:38.731 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 3387/18408 objects degraded (18.400%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:38.737 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:38.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:38.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:38.783 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:38.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:38.922 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:38.966 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:38.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:38.970 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:39.071 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:39.106 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:39.153 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:39.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:39.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:39.259 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:39.351 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:39.484 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 1 pg_num to 31 2023-09-23T18:00:39.523 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:39.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:39.531 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:39.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:39 smithi055 ceph-mon[184174]: pgmap v454: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 331 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 3142/15765 objects degraded (19.930%); 10885/15765 objects misplaced (69.045%); 5.9 MiB/s, 108 objects/s recovering 2023-09-23T18:00:39.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/346128023' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T18:00:39.586 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:39 smithi055 ceph-mon[179668]: pgmap v454: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 331 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 3142/15765 objects degraded (19.930%); 10885/15765 objects misplaced (69.045%); 5.9 MiB/s, 108 objects/s recovering 2023-09-23T18:00:39.586 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/346128023' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T18:00:39.631 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:39.678 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:39.733 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:39.735 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:39 smithi099 ceph-mon[167076]: pgmap v454: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 331 MiB data, 4.5 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 667 op/s; 3142/15765 objects degraded (19.930%); 10885/15765 objects misplaced (69.045%); 5.9 MiB/s, 108 objects/s recovering 2023-09-23T18:00:39.735 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/346128023' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]: dispatch 2023-09-23T18:00:39.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:39.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:39.837 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:39.874 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:39.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:39.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:39.924 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:39.951 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:00:40.001 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:00:40.009 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:40.025 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:40.066 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:00:40.139 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:40.140 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:00:40.183 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:00:40.226 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:00:40.227 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:00:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:00:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:00:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:00:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:00:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:00:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:00:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:00:40.243 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:40.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:40.254 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:40.355 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:40.391 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:00:40] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-09-23T18:00:40.442 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:40.501 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:00:40.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[179668]: failed to encode map e195 with expected crc 2023-09-23T18:00:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[179668]: failed to encode map e195 with expected crc 2023-09-23T18:00:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/346128023' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T18:00:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[179668]: osdmap e195: 8 total, 6 up, 7 in 2023-09-23T18:00:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2843435570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2016478948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:00:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[184174]: failed to encode map e195 with expected crc 2023-09-23T18:00:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[184174]: failed to encode map e195 with expected crc 2023-09-23T18:00:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/346128023' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T18:00:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[184174]: osdmap e195: 8 total, 6 up, 7 in 2023-09-23T18:00:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2843435570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2016478948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:00:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:40 smithi099 ceph-mon[167076]: failed to encode map e195 with expected crc 2023-09-23T18:00:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:40 smithi099 ceph-mon[167076]: failed to encode map e195 with expected crc 2023-09-23T18:00:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/346128023' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "31"}]': finished 2023-09-23T18:00:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:40 smithi099 ceph-mon[167076]: osdmap e195: 8 total, 6 up, 7 in 2023-09-23T18:00:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2843435570' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2016478948' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:00:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:41 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:41 smithi099 ceph-mon[167076]: failed to encode map e196 with expected crc 2023-09-23T18:00:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:41 smithi099 ceph-mon[167076]: failed to encode map e196 with expected crc 2023-09-23T18:00:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:41 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2016478948' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:00:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:41 smithi099 ceph-mon[167076]: osdmap e196: 8 total, 6 up, 7 in 2023-09-23T18:00:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:41 smithi099 ceph-mon[167076]: pgmap v457: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 216 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 2370/10236 objects degraded (23.154%); 9952/10236 objects misplaced (97.225%); 4.8 MiB/s, 93 objects/s recovering 2023-09-23T18:00:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:41 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T18:00:41.938 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:41.938 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[179668]: failed to encode map e196 with expected crc 2023-09-23T18:00:41.938 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[179668]: failed to encode map e196 with expected crc 2023-09-23T18:00:41.938 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2016478948' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:00:41.938 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[179668]: osdmap e196: 8 total, 6 up, 7 in 2023-09-23T18:00:41.938 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[179668]: pgmap v457: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 216 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 2370/10236 objects degraded (23.154%); 9952/10236 objects misplaced (97.225%); 4.8 MiB/s, 93 objects/s recovering 2023-09-23T18:00:41.938 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T18:00:41.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:41.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[184174]: failed to encode map e196 with expected crc 2023-09-23T18:00:41.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[184174]: failed to encode map e196 with expected crc 2023-09-23T18:00:41.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2016478948' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:00:41.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[184174]: osdmap e196: 8 total, 6 up, 7 in 2023-09-23T18:00:41.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[184174]: pgmap v457: 109 pgs: 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 216 MiB data, 4.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 669 op/s; 2370/10236 objects degraded (23.154%); 9952/10236 objects misplaced (97.225%); 4.8 MiB/s, 93 objects/s recovering 2023-09-23T18:00:41.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:41 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]: dispatch 2023-09-23T18:00:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:42 smithi099 ceph-mon[167076]: failed to encode map e197 with expected crc 2023-09-23T18:00:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T18:00:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:42 smithi099 ceph-mon[167076]: osdmap e197: 8 total, 6 up, 7 in 2023-09-23T18:00:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:42 smithi099 ceph-mon[167076]: failed to encode map e197 with expected crc 2023-09-23T18:00:42.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:42 smithi055 ceph-mon[179668]: failed to encode map e197 with expected crc 2023-09-23T18:00:42.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T18:00:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:42 smithi055 ceph-mon[179668]: osdmap e197: 8 total, 6 up, 7 in 2023-09-23T18:00:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:42 smithi055 ceph-mon[179668]: failed to encode map e197 with expected crc 2023-09-23T18:00:42.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:42 smithi055 ceph-mon[184174]: failed to encode map e197 with expected crc 2023-09-23T18:00:42.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num_actual", "val": "31"}]': finished 2023-09-23T18:00:42.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:42 smithi055 ceph-mon[184174]: osdmap e197: 8 total, 6 up, 7 in 2023-09-23T18:00:42.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:42 smithi055 ceph-mon[184174]: failed to encode map e197 with expected crc 2023-09-23T18:00:43.316 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Removed 105623 objects 2023-09-23T18:00:43.316 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Clean up completed and total clean up time :194.911 2023-09-23T18:00:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:43 smithi099 ceph-mon[167076]: failed to encode map e198 with expected crc 2023-09-23T18:00:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:43 smithi099 ceph-mon[167076]: osdmap e198: 8 total, 6 up, 7 in 2023-09-23T18:00:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:43 smithi099 ceph-mon[167076]: pgmap v460: 119 pgs: 10 unknown, 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 141 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 2165/6630 objects degraded (32.655%); 9347/6630 objects misplaced (140.980%); 2.7 MiB/s, 177 objects/s recovering 2023-09-23T18:00:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 2370/10236 objects degraded (23.154%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:43.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:43 smithi055 ceph-mon[184174]: failed to encode map e198 with expected crc 2023-09-23T18:00:43.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:43 smithi055 ceph-mon[184174]: osdmap e198: 8 total, 6 up, 7 in 2023-09-23T18:00:43.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:43 smithi055 ceph-mon[184174]: pgmap v460: 119 pgs: 10 unknown, 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 141 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 2165/6630 objects degraded (32.655%); 9347/6630 objects misplaced (140.980%); 2.7 MiB/s, 177 objects/s recovering 2023-09-23T18:00:43.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 2370/10236 objects degraded (23.154%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:43.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:43 smithi055 ceph-mon[179668]: failed to encode map e198 with expected crc 2023-09-23T18:00:43.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:43 smithi055 ceph-mon[179668]: osdmap e198: 8 total, 6 up, 7 in 2023-09-23T18:00:43.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:43 smithi055 ceph-mon[179668]: pgmap v460: 119 pgs: 10 unknown, 1 down, 15 active+clean+remapped, 46 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 9 active+undersized+degraded; 141 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 760 op/s; 2165/6630 objects degraded (32.655%); 9347/6630 objects misplaced (140.980%); 2.7 MiB/s, 177 objects/s recovering 2023-09-23T18:00:43.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 2370/10236 objects degraded (23.154%), 15 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:44.274 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:44.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:44.278 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:44.376 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:00:44.377 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:00:44.379 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:44.409 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:45.051 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-09-23T18:00:45.051 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-09-23T18:00:45.075 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4] out_osds: [5] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T18:00:45.075 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:00:45.075 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T18:00:45.075 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T18:00:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:45 smithi055 ceph-mon[179668]: pgmap v461: 119 pgs: 2 active+undersized+remapped+backfill_wait, 1 down, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 31 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 1510/1341 objects degraded (112.603%); 8337/1341 objects misplaced (621.700%); 1.8 MiB/s, 154 objects/s recovering 2023-09-23T18:00:45.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:45 smithi055 ceph-mon[184174]: pgmap v461: 119 pgs: 2 active+undersized+remapped+backfill_wait, 1 down, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 31 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 1510/1341 objects degraded (112.603%); 8337/1341 objects misplaced (621.700%); 1.8 MiB/s, 154 objects/s recovering 2023-09-23T18:00:46.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:45 smithi099 ceph-mon[167076]: pgmap v461: 119 pgs: 2 active+undersized+remapped+backfill_wait, 1 down, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 31 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 696 op/s; 1510/1341 objects degraded (112.603%); 8337/1341 objects misplaced (621.700%); 1.8 MiB/s, 154 objects/s recovering 2023-09-23T18:00:46.378 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:00:46.854 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:46.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:46.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:46.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:46.987 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:47.025 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:47.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:47.028 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:47.128 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:47.157 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:47.704 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:47.707 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:47.707 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:47.808 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:47.835 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:47.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:47 smithi055 ceph-mon[179668]: pgmap v462: 119 pgs: 2 active+undersized+remapped+backfill_wait, 1 down, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 13 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 1510/468 objects degraded (322.650%); 8128/468 objects misplaced (1736.752%); 1.5 MiB/s, 129 objects/s recovering 2023-09-23T18:00:47.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:47 smithi055 ceph-mon[184174]: pgmap v462: 119 pgs: 2 active+undersized+remapped+backfill_wait, 1 down, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 13 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 1510/468 objects degraded (322.650%); 8128/468 objects misplaced (1736.752%); 1.5 MiB/s, 129 objects/s recovering 2023-09-23T18:00:48.042 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:48.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:48.045 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:48.046 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:48.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:47 smithi099 ceph-mon[167076]: pgmap v462: 119 pgs: 2 active+undersized+remapped+backfill_wait, 1 down, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 13 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 530 op/s; 1510/468 objects degraded (322.650%); 8128/468 objects misplaced (1736.752%); 1.5 MiB/s, 129 objects/s recovering 2023-09-23T18:00:48.146 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:48.174 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:48.659 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:48.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:48.662 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:48.763 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:48.792 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:48.826 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:00:48.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 1510/468 objects degraded (322.650%), 11 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 1510/468 objects degraded (322.650%), 11 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:48.946 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:48.984 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:48.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:48.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:48.990 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:00:48.990 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:00:49.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 1510/468 objects degraded (322.650%), 11 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:49.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:49.123 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:49.138 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:49.170 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:49.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:49.175 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:49.276 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:49.303 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:49.385 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:00:49.395 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:49.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:49.398 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:49.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:49.525 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:49.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:49.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:49.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:49.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:49.690 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:49.725 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:49.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:49.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:49.829 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:49.861 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:49.892 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:49 smithi055 ceph-mon[179668]: pgmap v463: 119 pgs: 1 down, 2 active+undersized+remapped+backfill_wait, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 1470/84 objects degraded (1750.000%); 8084/84 objects misplaced (9623.810%); 1.2 MiB/s, 212 objects/s recovering 2023-09-23T18:00:49.951 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:49.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:49.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:50.056 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:50.096 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:50.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:50.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:50.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:50.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:49 smithi055 ceph-mon[184174]: pgmap v463: 119 pgs: 1 down, 2 active+undersized+remapped+backfill_wait, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 1470/84 objects degraded (1750.000%); 8084/84 objects misplaced (9623.810%); 1.2 MiB/s, 212 objects/s recovering 2023-09-23T18:00:50.236 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:50.261 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:49 smithi099 ceph-mon[167076]: pgmap v463: 119 pgs: 1 down, 2 active+undersized+remapped+backfill_wait, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 1470/84 objects degraded (1750.000%); 8084/84 objects misplaced (9623.810%); 1.2 MiB/s, 212 objects/s recovering 2023-09-23T18:00:50.265 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:50.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:50.302 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:50.302 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:50.403 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:50.429 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:50.462 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:50.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:50.466 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:50.566 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:50.594 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:00:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:00:50] "GET /metrics HTTP/1.1" 200 36549 "" "Prometheus/2.43.0" 2023-09-23T18:00:50.598 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:50.755 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:50.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:50.759 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:50.852 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:50.860 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:50.878 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:50.887 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:50.921 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:50.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:50.923 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:51.024 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:51.051 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:51.084 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:51.087 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:51.087 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:51.159 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:00:51.179 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:00:51.188 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:51.203 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:00:51.222 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:51.230 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:00:51.251 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:00:51.260 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:51.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:51.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:51.274 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:00:51.275 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:00:51.275 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:00:51.275 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:00:51.275 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:00:51.275 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:00:51.275 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:00:51.276 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:00:51.276 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:00:51.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:51.392 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:51.428 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:51.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:51.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:51.517 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:00:51.200+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e198 definitely_dead 0 2023-09-23T18:00:51.531 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:51.562 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:51.709 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:51.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:51.712 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:51.813 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:51.840 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:51.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:51.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:51.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:51.890 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:00:51.890 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:00:51.908 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.5. 2023-09-23T18:00:51.909 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_0' removed 2023-09-23T18:00:51.979 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:52.027 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:52.067 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:52.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:52.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:52.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:52.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[179668]: pgmap v464: 119 pgs: 1 down, 2 active+undersized+remapped+backfill_wait, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 1470/84 objects degraded (1750.000%); 8084/84 objects misplaced (9623.810%); 1.0 MiB/s, 249 objects/s recovering 2023-09-23T18:00:52.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/138125467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:52.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3930056318' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:00:52.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:52.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:52.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/769852905' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:00:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[184174]: pgmap v464: 119 pgs: 1 down, 2 active+undersized+remapped+backfill_wait, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 1470/84 objects degraded (1750.000%); 8084/84 objects misplaced (9623.810%); 1.0 MiB/s, 249 objects/s recovering 2023-09-23T18:00:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/138125467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3930056318' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:00:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/769852905' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:00:52.195 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:51 smithi099 ceph-mon[167076]: pgmap v464: 119 pgs: 1 down, 2 active+undersized+remapped+backfill_wait, 15 active+clean+remapped, 56 active+clean, 1 active+recovering+undersized+remapped, 2 undersized+peered, 24 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 undersized+degraded+peered, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+remapped, 8 active+undersized+degraded; 4.6 MiB data, 3.8 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 305 op/s; 1470/84 objects degraded (1750.000%); 8084/84 objects misplaced (9623.810%); 1.0 MiB/s, 249 objects/s recovering 2023-09-23T18:00:52.196 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/138125467' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:52.196 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3930056318' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:00:52.196 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:00:52.196 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:00:52.196 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/769852905' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:00:52.198 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:52.230 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:52.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:52.232 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:52.333 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:52.362 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:52.549 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:52.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:52.553 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:52.572 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T18:00:52.573 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T18:00:52.573 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_1 2023-09-23T18:00:52.573 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_1 16 2023-09-23T18:00:52.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:52.683 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:52.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:52.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:52.725 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:52.753 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T18:00:52.826 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:52.860 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:52.900 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:00:52.900 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:00:52.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:00:52.900 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:00:52.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.900 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:38871 5.2 5:4086753f:::benchmark_data_smithi055_229117_object38870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T18:00:52.900 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.242138+0000", 2023-09-23T18:00:52.900 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 277.65007909799999, 2023-09-23T18:00:52.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11274374099999999, 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 38871 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242138+0000", 2023-09-23T18:00:52.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242138+0000", 2023-09-23T18:00:52.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242136+0000", 2023-09-23T18:00:52.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242159+0000", 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3130999999999999e-05 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242172+0000", 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2405000000000001e-05 2023-09-23T18:00:52.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242181+0000", 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1710000000000008e-06 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242234+0000", 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3562e-05 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242386+0000", 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015176400000000001 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242544+0000", 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015809600000000001 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244070+0000", 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015261820000000001 2023-09-23T18:00:52.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247882+0000", 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038115940000000002 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247934+0000", 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1607000000000002e-05 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.354881+0000", 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10694785900000001 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:38885 5.e e117/110 5:7e62a019:::benchmark_data_smithi055_229117_object38884:head v 117'2410, mlcod=117'2406)", 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.247175+0000", 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 277.64504207599998, 2023-09-23T18:00:52.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.109947827, 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247175+0000", 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247175+0000", 2023-09-23T18:00:52.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247174+0000", 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247223+0000", 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9112e-05 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247231+0000", 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.61e-06 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247236+0000", 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8860000000000003e-06 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353831+0000", 2023-09-23T18:00:52.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.106595129 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353853+0000", 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1997e-05 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357086+0000", 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00323339 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357123+0000", 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6259e-05 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154235 5.2 5:44f142c1:::benchmark_data_smithi055_229117_object48609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.835816+0000", 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 104.056400723, 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.120395545, 2023-09-23T18:00:52.912 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154235 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835816+0000", 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835816+0000", 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835815+0000", 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835819+0000", 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7720000000000001e-06 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835827+0000", 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9359999999999999e-06 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835835+0000", 2023-09-23T18:00:52.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3760000000000002e-06 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839941+0000", 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041069510000000002 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840066+0000", 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000124762 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840214+0000", 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014806500000000001 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.845845+0000", 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0056306910000000002 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.856676+0000", 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010831341 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.856687+0000", 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0695e-05 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956153+0000", 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0994657109999999 2023-09-23T18:00:52.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956212+0000", 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8913000000000003e-05 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154236 5.2 5:46f3ddda:::benchmark_data_smithi055_229117_object48610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.835841+0000", 2023-09-23T18:00:52.919 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 104.056376082, 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1205958920000001, 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154236 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835841+0000", 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835841+0000", 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835840+0000", 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:52.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835843+0000", 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.638e-06 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835849+0000", 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9150000000000001e-06 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835854+0000", 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5650000000000001e-06 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.842866+0000", 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0070116919999999999 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.842987+0000", 2023-09-23T18:00:52.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120715 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.843124+0000", 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137155 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.846319+0000", 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031950939999999999 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.858461+0000", 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012142101000000001 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.858472+0000", 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0736e-05 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956386+0000", 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.097913626 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956437+0000", 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1063000000000002e-05 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154274 5.5 5:aa5b5b0c:::benchmark_data_smithi055_229117_object48648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.105657+0000", 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 101.78655938, 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.20535718, 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154274 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105657+0000", 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105657+0000", 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105656+0000", 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105663+0000", 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.742e-06 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105675+0000", 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1868000000000001e-05 2023-09-23T18:00:52.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105683+0000", 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9410000000000007e-06 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298209+0000", 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.19252627899999999 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298292+0000", 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3452e-05 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298686+0000", 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00039359900000000001 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306095+0000", 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0074089589999999997 2023-09-23T18:00:52.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310633+0000", 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045383070000000001 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310919+0000", 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000285803 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310969+0000", 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0439999999999998e-05 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311015+0000", 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5250999999999998e-05 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154280 5.5 5:a78a277e:::benchmark_data_smithi055_229117_object48654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.166102+0000", 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 101.726114502, 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14501488900000001, 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154280 2023-09-23T18:00:52.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166102+0000", 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166102+0000", 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166101+0000", 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166109+0000", 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5660000000000001e-06 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166122+0000", 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3269e-05 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166128+0000", 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5640000000000002e-06 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303566+0000", 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13743744199999999 2023-09-23T18:00:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303721+0000", 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000155585 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303856+0000", 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000134177 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310596+0000", 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0067407539999999998 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310889+0000", 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00029301300000000001 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311049+0000", 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016005699999999999 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311086+0000", 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6881999999999997e-05 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311117+0000", 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0936000000000003e-05 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154282 5.5 5:a1cbca1c:::benchmark_data_smithi055_229117_object48656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.171172+0000", 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 101.721044543, 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14593202499999999, 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154282 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171172+0000", 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171172+0000", 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171170+0000", 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171181+0000", 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.064e-05 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171212+0000", 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1433999999999998e-05 2023-09-23T18:00:52.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171223+0000", 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0265000000000001e-05 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306130+0000", 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13490687900000001 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306257+0000", 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012784 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306375+0000", 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011726999999999999 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310640+0000", 2023-09-23T18:00:52.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00426495 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.314542+0000", 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039025510000000002 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317023+0000", 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024812549999999999 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317068+0000", 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4632999999999998e-05 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317104+0000", 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6188e-05 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154290 5.5 5:a8c62379:::benchmark_data_smithi055_229117_object48664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.196937+0000", 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 101.69527958499999, 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.12027204499999999, 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154290 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.948 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196937+0000", 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196937+0000", 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196936+0000", 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196943+0000", 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.306e-06 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196960+0000", 2023-09-23T18:00:52.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7173e-05 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196969+0000", 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7390000000000005e-06 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308177+0000", 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11120793499999999 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308311+0000", 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013440899999999999 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308441+0000", 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000129677 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.312252+0000", 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038113700000000001 2023-09-23T18:00:52.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.314577+0000", 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023246730000000002 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317137+0000", 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025597609999999998 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317173+0000", 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6371000000000001e-05 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317209+0000", 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6191e-05 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.954 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161327 5.e 5:7b0ada0d:::benchmark_data_smithi055_229117_object55701:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852567+0000", 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 90.039649745999995, 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0633735559999999, 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.955 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161327 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852567+0000", 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852567+0000", 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852565+0000", 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852572+0000", 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3729999999999998e-06 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852585+0000", 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3426e-05 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852592+0000", 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2830000000000004e-06 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852658+0000", 2023-09-23T18:00:52.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5399000000000002e-05 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op must wait for map", 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852668+0000", 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7110000000000007e-06 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872062+0000", 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019394650999999999 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872073+0000", 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0607e-05 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.885379+0000", 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0133066719999999 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.887726+0000", 2023-09-23T18:00:52.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023469929999999999 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.887957+0000", 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00023082800000000001 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915266+0000", 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027308517000000001 2023-09-23T18:00:52.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915340+0000", 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4178999999999996e-05 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915403+0000", 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2842000000000006e-05 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915811+0000", 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00040861899999999999 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915853+0000", 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1768999999999999e-05 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915941+0000", 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7410000000000005e-05 2023-09-23T18:00:52.963 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161356 5.e 5:729afca1:::benchmark_data_smithi055_229117_object55730:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 90.039611824999994, 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0635528459999999, 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.964 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161356 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:00:52.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852607+0000", 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9570000000000001e-06 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852613+0000", 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1199999999999999e-06 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852616+0000", 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4319999999999999e-06 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852880+0000", 2023-09-23T18:00:52.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026363200000000003 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852889+0000", 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8529999999999998e-06 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872097+0000", 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019208026999999999 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872104+0000", 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5739999999999998e-06 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.905797+0000", 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0336926310000001 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.905945+0000", 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000147899 2023-09-23T18:00:52.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.906095+0000", 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015013200000000001 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915327+0000", 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092321869999999993 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915351+0000", 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.376e-05 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915452+0000", 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000101004 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915997+0000", 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00054548699999999999 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916057+0000", 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9496000000000001e-05 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916158+0000", 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000100995 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.972 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161361 5.e 5:727196ef:::benchmark_data_smithi055_229117_object55735:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852804+0000", 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 90.039412892000001, 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0635737650000001, 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.973 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161361 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852804+0000", 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852804+0000", 2023-09-23T18:00:52.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852803+0000", 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852808+0000", 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3579999999999996e-06 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852820+0000", 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2822e-05 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852828+0000", 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9980000000000003e-06 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111282 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852947+0000", 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3749999999999997e-06 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872127+0000", 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019180169 2023-09-23T18:00:52.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872134+0000", 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.776e-06 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908641+0000", 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.036506683 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908722+0000", 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1464999999999993e-05 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908805+0000", 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3060999999999996e-05 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915361+0000", 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00655603 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915491+0000", 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012961299999999999 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915586+0000", 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5175999999999998e-05 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916217+0000", 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00063121599999999998 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916266+0000", 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8649000000000001e-05 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916378+0000", 2023-09-23T18:00:52.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111749 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161400 5.e 5:7bd12518:::benchmark_data_smithi055_229117_object55774:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852914+0000", 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 90.039303141999994, 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.066386922, 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.982 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161400 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852914+0000", 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852914+0000", 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852913+0000", 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852917+0000", 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1459999999999998e-06 2023-09-23T18:00:52.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852927+0000", 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4700000000000008e-06 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852934+0000", 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8630000000000002e-06 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852982+0000", 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8585999999999999e-05 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852990+0000", 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3989999999999998e-06 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872158+0000", 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019168266 2023-09-23T18:00:52.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872163+0000", 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9620000000000001e-06 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.910791+0000", 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0386283650000001 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.910919+0000", 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012744899999999999 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.911051+0000", 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013257500000000001 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915372+0000", 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004320618 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915538+0000", 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016641900000000001 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915724+0000", 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018572200000000001 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919091+0000", 2023-09-23T18:00:52.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033675419999999998 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919170+0000", 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8464e-05 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919301+0000", 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013067499999999999 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.990 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161452 5.e 5:7e9dcc4b:::benchmark_data_smithi055_229117_object55826:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 90.039276943999994, 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0665552599999999, 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:52.991 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161452 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:52.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852939+0000", 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852943+0000", 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1070000000000001e-06 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852949+0000", 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7900000000000002e-06 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852955+0000", 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7760000000000003e-06 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.853019+0000", 2023-09-23T18:00:52.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3619000000000002e-05 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.853023+0000", 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1899999999999997e-06 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872216+0000", 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019192753 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872224+0000", 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2779999999999993e-06 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913001+0000", 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.040776715 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:52.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913126+0000", 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012584599999999999 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913244+0000", 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000117377 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915463+0000", 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022194480000000002 2023-09-23T18:00:52.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915650+0000", 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018667299999999999 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915759+0000", 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109165 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919339+0000", 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035801880000000002 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919386+0000", 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6295999999999999e-05 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919495+0000", 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109508 2023-09-23T18:00:52.999 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206353 5.5 5:a66ab0d3:::benchmark_data_smithi055_229117_object100727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.895738+0000", 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 16.996479194999999, 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.116710411, 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:53.000 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206353 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895738+0000", 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895738+0000", 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895736+0000", 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895741+0000", 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6890000000000003e-06 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895751+0000", 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0377000000000001e-05 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895759+0000", 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2420000000000003e-06 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895825+0000", 2023-09-23T18:00:53.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5957999999999994e-05 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895953+0000", 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012840799999999999 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.896065+0000", 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011191700000000001 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901012+0000", 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004946939 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901377+0000", 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00036531200000000002 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012352+0000", 2023-09-23T18:00:53.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11097481100000001 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012398+0000", 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5779000000000003e-05 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012448+0000", 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0294000000000003e-05 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.006 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206358 5.2 5:46c0880b:::benchmark_data_smithi055_229117_object100732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.895804+0000", 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 16.996412933999999, 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.119344292, 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:53.007 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206358 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895804+0000", 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895804+0000", 2023-09-23T18:00:53.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895803+0000", 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895807+0000", 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4759999999999998e-06 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895814+0000", 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.781e-06 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895819+0000", 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5269999999999998e-06 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.899325+0000", 2023-09-23T18:00:53.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003505883 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.899449+0000", 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000123726 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.899575+0000", 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000126308 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.904891+0000", 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053159310000000003 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.904948+0000", 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6798000000000001e-05 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.014957+0000", 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.110008884 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.014989+0000", 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2298999999999997e-05 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:53.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015038+0000", 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8914e-05 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015148+0000", 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000110263 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.014 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206362 5.2 5:4058eb16:::benchmark_data_smithi055_229117_object100736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 16.996385469, 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.119556432, 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206362 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.015 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895834+0000", 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.943e-06 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895840+0000", 2023-09-23T18:00:53.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4489999999999996e-06 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895846+0000", 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4999999999999999e-06 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.902010+0000", 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0061640469999999998 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.902147+0000", 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137324 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.902297+0000", 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014995699999999999 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.904939+0000", 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026419070000000002 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015191+0000", 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11025235799999999 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015223+0000", 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1665000000000003e-05 2023-09-23T18:00:53.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015245+0000", 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2456e-05 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015286+0000", 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1009e-05 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:53.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015388+0000", 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000101327 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206363 5.5 5:a478e7fe:::benchmark_data_smithi055_229117_object100737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.895850+0000", 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 16.996366446, 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11711835399999999, 2023-09-23T18:00:53.022 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206363 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895850+0000", 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895850+0000", 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895850+0000", 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895852+0000", 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4150000000000002e-06 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895858+0000", 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4709999999999998e-06 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895863+0000", 2023-09-23T18:00:53.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6829999999999999e-06 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.898438+0000", 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025751039999999999 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.898553+0000", 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000115454 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.898668+0000", 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011471 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901405+0000", 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027372210000000002 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901528+0000", 2023-09-23T18:00:53.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012334999999999999 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012883+0000", 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11135468900000001 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012931+0000", 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7821000000000002e-05 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012969+0000", 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7863e-05 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:206361 5.d e194/176 5:bb6e9a5e:::benchmark_data_smithi055_229117_object100735:head v 194'12834, mlcod=118'3050)", 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.897260+0000", 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 16.994956500000001, 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11523296600000001, 2023-09-23T18:00:53.029 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897260+0000", 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897260+0000", 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897259+0000", 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897267+0000", 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9189999999999999e-06 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897278+0000", 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0977e-05 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897286+0000", 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6830000000000008e-06 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.898147+0000", 2023-09-23T18:00:53.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00086085600000000001 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.898167+0000", 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0169999999999998e-05 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012463+0000", 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.114296261 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012493+0000", 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0060999999999999e-05 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:206365 5.e e194/187 5:75ab0aea:::benchmark_data_smithi055_229117_object100739:head v 194'12891, mlcod=194'12889)", 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.901561+0000", 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 16.990655740000001, 2023-09-23T18:00:53.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11070677700000001, 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901561+0000", 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901561+0000", 2023-09-23T18:00:53.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901560+0000", 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901567+0000", 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4710000000000004e-06 2023-09-23T18:00:53.036 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901577+0000", 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8879999999999992e-06 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901583+0000", 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3559999999999998e-06 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.037 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901622+0000", 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9186000000000001e-05 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901646+0000", 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3947e-05 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905646+0000", 2023-09-23T18:00:53.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040000670000000004 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012268+0000", 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10662147900000001 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206368 5.2 5:4617325c:::benchmark_data_smithi055_229117_object100742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:00:53.039 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.905040+0000", 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 16.987176740999999, 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.117245135, 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206368 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905040+0000", 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905040+0000", 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905038+0000", 2023-09-23T18:00:53.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905046+0000", 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2590000000000003e-06 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905057+0000", 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1406999999999999e-05 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.042 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905065+0000", 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9300000000000003e-06 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905107+0000", 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1492000000000002e-05 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905267+0000", 2023-09-23T18:00:53.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016003600000000001 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012294+0000", 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10702790199999999 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.016113+0000", 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038184 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.016149+0000", 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6520999999999998e-05 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.016543+0000", 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00039407299999999999 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.022111+0000", 2023-09-23T18:00:53.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055670880000000004 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.022163+0000", 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2528999999999998e-05 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.022285+0000", 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012215699999999999 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.046 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:53.047 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.047 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:00:53.047 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:00:53.047 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:00:53.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:53.048 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:53.048 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:52 smithi099 ceph-mon[167076]: failed to encode map e199 with expected crc 2023-09-23T18:00:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3930056318' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:00:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/769852905' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:00:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:52 smithi099 ceph-mon[167076]: osdmap e199: 8 total, 6 up, 8 in 2023-09-23T18:00:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:52 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 1470/84 objects degraded (1750.000%), 11 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:52 smithi099 ceph-mon[167076]: osdmap e200: 8 total, 6 up, 8 in 2023-09-23T18:00:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:52 smithi099 ceph-mon[167076]: failed to encode map e200 with expected crc 2023-09-23T18:00:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:52 smithi099 ceph-mon[167076]: failed to encode map e200 with expected crc 2023-09-23T18:00:53.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:52 smithi099 ceph-mon[167076]: failed to encode map e200 with expected crc 2023-09-23T18:00:53.149 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:53.174 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[179668]: failed to encode map e199 with expected crc 2023-09-23T18:00:53.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3930056318' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:00:53.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/769852905' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:00:53.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[179668]: osdmap e199: 8 total, 6 up, 8 in 2023-09-23T18:00:53.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 1470/84 objects degraded (1750.000%), 11 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:53.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[179668]: osdmap e200: 8 total, 6 up, 8 in 2023-09-23T18:00:53.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[179668]: failed to encode map e200 with expected crc 2023-09-23T18:00:53.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[179668]: failed to encode map e200 with expected crc 2023-09-23T18:00:53.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[179668]: failed to encode map e200 with expected crc 2023-09-23T18:00:53.175 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[184174]: failed to encode map e199 with expected crc 2023-09-23T18:00:53.176 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3930056318' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:00:53.176 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/769852905' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:00:53.176 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[184174]: osdmap e199: 8 total, 6 up, 8 in 2023-09-23T18:00:53.176 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 1470/84 objects degraded (1750.000%), 11 pgs degraded, 38 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:53.176 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[184174]: osdmap e200: 8 total, 6 up, 8 in 2023-09-23T18:00:53.176 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[184174]: failed to encode map e200 with expected crc 2023-09-23T18:00:53.176 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[184174]: failed to encode map e200 with expected crc 2023-09-23T18:00:53.176 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:52 smithi055 ceph-mon[184174]: failed to encode map e200 with expected crc 2023-09-23T18:00:53.177 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:53.359 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:00:53.362 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:53.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:53.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:53.466 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:53.497 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:53.535 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:53.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:53.538 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:53.639 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:53.670 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:53.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:53.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:53.709 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:53.810 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:53.813 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:53.841 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:53.876 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:53.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:53.879 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:53.891 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:00:53.981 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:54.008 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[179668]: pgmap v466: 103 pgs: 3 peering, 1 down, 12 active+clean+remapped, 50 active+clean, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 4 active+undersized+remapped, 7 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11/78 objects degraded (14.103%); 12/78 objects misplaced (15.385%) 2023-09-23T18:00:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[179668]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[179668]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[179668]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[179668]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[179668]: failed to encode map e201 with expected crc 2023-09-23T18:00:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[179668]: osdmap e201: 8 total, 6 up, 8 in 2023-09-23T18:00:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[184174]: pgmap v466: 103 pgs: 3 peering, 1 down, 12 active+clean+remapped, 50 active+clean, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 4 active+undersized+remapped, 7 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11/78 objects degraded (14.103%); 12/78 objects misplaced (15.385%) 2023-09-23T18:00:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[184174]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[184174]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[184174]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[184174]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[184174]: failed to encode map e201 with expected crc 2023-09-23T18:00:54.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:53 smithi055 ceph-mon[184174]: osdmap e201: 8 total, 6 up, 8 in 2023-09-23T18:00:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:53 smithi099 ceph-mon[167076]: pgmap v466: 103 pgs: 3 peering, 1 down, 12 active+clean+remapped, 50 active+clean, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 4 active+undersized+remapped, 7 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11/78 objects degraded (14.103%); 12/78 objects misplaced (15.385%) 2023-09-23T18:00:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:53 smithi099 ceph-mon[167076]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:53 smithi099 ceph-mon[167076]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:53 smithi099 ceph-mon[167076]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:53 smithi099 ceph-mon[167076]: failed to encode map e200 with expected crc 2023-09-23T18:00:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:53 smithi099 ceph-mon[167076]: failed to encode map e201 with expected crc 2023-09-23T18:00:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:53 smithi099 ceph-mon[167076]: osdmap e201: 8 total, 6 up, 8 in 2023-09-23T18:00:54.344 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:54.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:54.349 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:54.449 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:54.479 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:54.514 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:54.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:54.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:54.618 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:54.646 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:54.683 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:54.689 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:54.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:54.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:54.794 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:54.823 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:54.916 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:00:55.062 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:55.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:55.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:55.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:55.195 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:54 smithi099 ceph-mon[167076]: failed to encode map e201 with expected crc 2023-09-23T18:00:55.195 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:54 smithi099 ceph-mon[167076]: failed to encode map e201 with expected crc 2023-09-23T18:00:55.195 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1382793099' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:55.195 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:54 smithi099 ceph-mon[167076]: failed to encode map e201 with expected crc 2023-09-23T18:00:55.195 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:54 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:55.195 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1382793099' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:00:55.195 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:54 smithi099 ceph-mon[167076]: osdmap e202: 8 total, 6 up, 8 in 2023-09-23T18:00:55.198 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:55.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:55.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:55.238 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:55.339 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:55.356 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[184174]: failed to encode map e201 with expected crc 2023-09-23T18:00:55.356 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[184174]: failed to encode map e201 with expected crc 2023-09-23T18:00:55.356 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1382793099' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:55.356 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[184174]: failed to encode map e201 with expected crc 2023-09-23T18:00:55.356 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:55.356 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1382793099' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:00:55.356 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[184174]: osdmap e202: 8 total, 6 up, 8 in 2023-09-23T18:00:55.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[179668]: failed to encode map e201 with expected crc 2023-09-23T18:00:55.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[179668]: failed to encode map e201 with expected crc 2023-09-23T18:00:55.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1382793099' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:00:55.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[179668]: failed to encode map e201 with expected crc 2023-09-23T18:00:55.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:00:55.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1382793099' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:00:55.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:54 smithi055 ceph-mon[179668]: osdmap e202: 8 total, 6 up, 8 in 2023-09-23T18:00:55.368 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:55.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:55.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:55.408 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:55.509 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:55.539 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:55.725 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:55.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:55.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:55.829 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:00:55.862 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:55.886 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:55.902 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:00:55.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:55.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:00:56.007 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:00:56.036 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:55 smithi055 ceph-mon[179668]: pgmap v469: 103 pgs: 1 down, 3 peering, 11 active+clean+remapped, 1 active+remapped, 50 active+clean, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 4 active+undersized+remapped, 7 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11/78 objects degraded (14.103%); 8/78 objects misplaced (10.256%) 2023-09-23T18:00:56.036 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:55 smithi055 ceph-mon[179668]: failed to encode map e202 with expected crc 2023-09-23T18:00:56.036 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:55 smithi055 ceph-mon[179668]: failed to encode map e202 with expected crc 2023-09-23T18:00:56.036 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:55 smithi055 ceph-mon[179668]: failed to encode map e202 with expected crc 2023-09-23T18:00:56.036 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:55 smithi055 ceph-mon[179668]: osdmap e203: 8 total, 6 up, 8 in 2023-09-23T18:00:56.039 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:55 smithi055 ceph-mon[184174]: pgmap v469: 103 pgs: 1 down, 3 peering, 11 active+clean+remapped, 1 active+remapped, 50 active+clean, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 4 active+undersized+remapped, 7 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11/78 objects degraded (14.103%); 8/78 objects misplaced (10.256%) 2023-09-23T18:00:56.039 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:55 smithi055 ceph-mon[184174]: failed to encode map e202 with expected crc 2023-09-23T18:00:56.039 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:55 smithi055 ceph-mon[184174]: failed to encode map e202 with expected crc 2023-09-23T18:00:56.039 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:55 smithi055 ceph-mon[184174]: failed to encode map e202 with expected crc 2023-09-23T18:00:56.040 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:55 smithi055 ceph-mon[184174]: osdmap e203: 8 total, 6 up, 8 in 2023-09-23T18:00:56.040 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:56.275 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:00:56.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:56.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:00:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:55 smithi099 ceph-mon[167076]: pgmap v469: 103 pgs: 1 down, 3 peering, 11 active+clean+remapped, 1 active+remapped, 50 active+clean, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 4 active+undersized+remapped, 7 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11/78 objects degraded (14.103%); 8/78 objects misplaced (10.256%) 2023-09-23T18:00:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:55 smithi099 ceph-mon[167076]: failed to encode map e202 with expected crc 2023-09-23T18:00:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:55 smithi099 ceph-mon[167076]: failed to encode map e202 with expected crc 2023-09-23T18:00:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:55 smithi099 ceph-mon[167076]: failed to encode map e202 with expected crc 2023-09-23T18:00:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:55 smithi099 ceph-mon[167076]: osdmap e203: 8 total, 6 up, 8 in 2023-09-23T18:00:56.380 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:56.380 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:00:56.380 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:00:56.381 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:00:56.381 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:00:56.412 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:56.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:56.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:56.450 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:56.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:56.580 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:56.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:56.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:56.617 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:56.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:56.746 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:56.779 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:56.781 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:56.782 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:56.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:56.910 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:56.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:56.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:56.946 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:56.981 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_1' created 2023-09-23T18:00:57.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:57.071 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:56 smithi099 ceph-mon[167076]: failed to encode map e203 with expected crc 2023-09-23T18:00:57.071 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1271033816' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T18:00:57.072 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:56 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T18:00:57.072 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:56 smithi099 ceph-mon[167076]: failed to encode map e203 with expected crc 2023-09-23T18:00:57.074 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:57.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:57.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:57.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:57.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:56 smithi055 ceph-mon[179668]: failed to encode map e203 with expected crc 2023-09-23T18:00:57.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1271033816' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T18:00:57.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:56 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T18:00:57.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:56 smithi055 ceph-mon[179668]: failed to encode map e203 with expected crc 2023-09-23T18:00:57.155 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:56 smithi055 ceph-mon[184174]: failed to encode map e203 with expected crc 2023-09-23T18:00:57.156 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1271033816' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T18:00:57.156 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:56 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]: dispatch 2023-09-23T18:00:57.156 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:56 smithi055 ceph-mon[184174]: failed to encode map e203 with expected crc 2023-09-23T18:00:57.172 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:00:57.211 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:57.242 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:57.432 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:57.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:57.435 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:57.460 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_1 rados --yes-i-really-mean-it || true 2023-09-23T18:00:57.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:57.568 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:57.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:57.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:57.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:57.681 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:00:57.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:00:57.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:57.728 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:00:57.737 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:57.750 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:00:57.754 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 4, 5] out_osds: [] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T18:00:57.754 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:00:57.755 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [0, 1, 2, 3, 6, 7, 4, 5] 2023-09-23T18:00:57.755 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T18:00:57.769 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:00:57.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:57.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:57.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:57.789 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:00:57.789 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:00:57.789 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:00:57.789 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:00:57.789 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:00:57.790 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:00:57.790 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:00:57.790 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:00:57.790 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:00:57.844 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:57.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:57.903 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:57.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:57.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:57.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:00:58.040 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:58.043 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:00:58.065 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:58.069 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[184174]: pgmap v472: 103 pgs: 1 down, 3 peering, 10 active+clean+remapped, 1 active+remapped, 51 active+clean, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 4 active+undersized+remapped, 7 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11/78 objects degraded (14.103%); 8/78 objects misplaced (10.256%); 138 B/s, 0 objects/s recovering 2023-09-23T18:00:58.069 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[184174]: failed to encode map e204 with expected crc 2023-09-23T18:00:58.069 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T18:00:58.069 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[184174]: osdmap e204: 8 total, 6 up, 8 in 2023-09-23T18:00:58.069 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1533207022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:58.069 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 11/78 objects degraded (14.103%), 8 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:58.069 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[179668]: pgmap v472: 103 pgs: 1 down, 3 peering, 10 active+clean+remapped, 1 active+remapped, 51 active+clean, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 4 active+undersized+remapped, 7 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11/78 objects degraded (14.103%); 8/78 objects misplaced (10.256%); 138 B/s, 0 objects/s recovering 2023-09-23T18:00:58.070 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[179668]: failed to encode map e204 with expected crc 2023-09-23T18:00:58.070 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T18:00:58.070 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[179668]: osdmap e204: 8 total, 6 up, 8 in 2023-09-23T18:00:58.070 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1533207022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:58.070 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:57 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 11/78 objects degraded (14.103%), 8 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:58.077 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:00:58.078 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:00:58.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:58.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:58.100 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:58.200 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:58.234 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:58.279 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:58.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:58.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:57 smithi099 ceph-mon[167076]: pgmap v472: 103 pgs: 1 down, 3 peering, 10 active+clean+remapped, 1 active+remapped, 51 active+clean, 2 undersized+peered, 23 active+undersized, 1 undersized+degraded+peered, 4 active+undersized+remapped, 7 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 11/78 objects degraded (14.103%); 8/78 objects misplaced (10.256%); 138 B/s, 0 objects/s recovering 2023-09-23T18:00:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:57 smithi099 ceph-mon[167076]: failed to encode map e204 with expected crc 2023-09-23T18:00:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:57 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_1", "pg_num": 16}]': finished 2023-09-23T18:00:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:57 smithi099 ceph-mon[167076]: osdmap e204: 8 total, 6 up, 8 in 2023-09-23T18:00:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1533207022' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:00:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:57 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 11/78 objects degraded (14.103%), 8 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:00:58.385 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:58.411 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:58.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:58.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:58.446 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:58.484 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:58.547 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:58.573 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:58.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:58.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:58.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:58.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:58.734 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:58.765 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:58.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:58.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:58.869 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:00:58.884 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:58.896 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:00:58.905 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:59.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:59 smithi099 ceph-mon[167076]: failed to encode map e205 with expected crc 2023-09-23T18:00:59.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:59 smithi099 ceph-mon[167076]: osdmap e205: 8 total, 6 up, 8 in 2023-09-23T18:00:59.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:59 smithi099 ceph-mon[167076]: failed to encode map e205 with expected crc 2023-09-23T18:00:59.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:00:59 smithi099 ceph-mon[167076]: failed to encode map e205 with expected crc 2023-09-23T18:00:59.343 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:00:59.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:59 smithi055 ceph-mon[184174]: failed to encode map e205 with expected crc 2023-09-23T18:00:59.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:59 smithi055 ceph-mon[184174]: osdmap e205: 8 total, 6 up, 8 in 2023-09-23T18:00:59.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:59 smithi055 ceph-mon[184174]: failed to encode map e205 with expected crc 2023-09-23T18:00:59.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:00:59 smithi055 ceph-mon[184174]: failed to encode map e205 with expected crc 2023-09-23T18:00:59.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:59 smithi055 ceph-mon[179668]: failed to encode map e205 with expected crc 2023-09-23T18:00:59.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:59 smithi055 ceph-mon[179668]: osdmap e205: 8 total, 6 up, 8 in 2023-09-23T18:00:59.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:59 smithi055 ceph-mon[179668]: failed to encode map e205 with expected crc 2023-09-23T18:00:59.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:00:59 smithi055 ceph-mon[179668]: failed to encode map e205 with expected crc 2023-09-23T18:00:59.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:59.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:00:59.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:59.475 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:59.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:59.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:59.510 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:59.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:00:59.636 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:59.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:00:59.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:59.672 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:00:59.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:00:59.798 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:59.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:00:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:59.832 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:00:59.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:00:59.958 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:00:59.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:00:59.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:00:59.995 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:00 smithi099 ceph-mon[167076]: pgmap v475: 119 pgs: 1 down, 6 creating+peering, 7 unknown, 9 active+clean+remapped, 56 active+clean, 3 undersized+peered, 26 active+undersized, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 13/84 objects degraded (15.476%); 4/84 objects misplaced (4.762%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:00 smithi099 ceph-mon[167076]: failed to encode map e206 with expected crc 2023-09-23T18:01:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:00 smithi099 ceph-mon[167076]: failed to encode map e206 with expected crc 2023-09-23T18:01:00.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:00 smithi099 ceph-mon[167076]: osdmap e206: 8 total, 6 up, 8 in 2023-09-23T18:01:00.078 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:01:00.096 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:00.150 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:00.271 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:00 smithi055 ceph-mon[179668]: pgmap v475: 119 pgs: 1 down, 6 creating+peering, 7 unknown, 9 active+clean+remapped, 56 active+clean, 3 undersized+peered, 26 active+undersized, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 13/84 objects degraded (15.476%); 4/84 objects misplaced (4.762%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:00.271 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:00 smithi055 ceph-mon[179668]: failed to encode map e206 with expected crc 2023-09-23T18:01:00.271 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:00 smithi055 ceph-mon[179668]: failed to encode map e206 with expected crc 2023-09-23T18:01:00.271 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:00 smithi055 ceph-mon[179668]: osdmap e206: 8 total, 6 up, 8 in 2023-09-23T18:01:00.272 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:00 smithi055 ceph-mon[184174]: pgmap v475: 119 pgs: 1 down, 6 creating+peering, 7 unknown, 9 active+clean+remapped, 56 active+clean, 3 undersized+peered, 26 active+undersized, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 13/84 objects degraded (15.476%); 4/84 objects misplaced (4.762%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:00.272 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:00 smithi055 ceph-mon[184174]: failed to encode map e206 with expected crc 2023-09-23T18:01:00.272 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:00 smithi055 ceph-mon[184174]: failed to encode map e206 with expected crc 2023-09-23T18:01:00.272 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:00 smithi055 ceph-mon[184174]: osdmap e206: 8 total, 6 up, 8 in 2023-09-23T18:01:00.581 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:01:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:01:00] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2023-09-23T18:01:00.587 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:00.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:00.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:00.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:00.721 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:00.756 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:00.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:00.759 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:00.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:00.886 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:00.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:00.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:00.922 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:01.022 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:01.063 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:01.359 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:01.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:01.363 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:01.464 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:01.506 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:01.517 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:01:01.517 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:01:01.714 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:01.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:01.718 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:01.770 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:01.819 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:01.851 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:01.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:01 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:01:01.917+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e206 definitely_dead 0 2023-09-23T18:01:02.067 INFO:teuthology.orchestra.run.smithi055.stderr:enabled application 'rados' on pool 'unique_pool_1' 2023-09-23T18:01:02.075 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.4. 2023-09-23T18:01:02.117 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:02.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:02.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:02.221 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:02.254 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:02.286 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:02 smithi055 ceph-mon[184174]: pgmap v477: 119 pgs: 1 down, 6 creating+peering, 3 unknown, 9 active+clean+remapped, 56 active+clean, 4 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+undersized+remapped, 9 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 13/84 objects degraded (15.476%); 4/84 objects misplaced (4.762%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:02.286 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/463908141' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:01:02.286 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/813829039' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:01:02.286 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:02 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:01:02.286 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:02 smithi055 ceph-mon[184174]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T18:01:02.287 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:02 smithi055 ceph-mon[179668]: pgmap v477: 119 pgs: 1 down, 6 creating+peering, 3 unknown, 9 active+clean+remapped, 56 active+clean, 4 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+undersized+remapped, 9 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 13/84 objects degraded (15.476%); 4/84 objects misplaced (4.762%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:02.287 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/463908141' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:01:02.287 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/813829039' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:01:02.287 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:02 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:01:02.287 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:02 smithi055 ceph-mon[179668]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T18:01:02.288 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:01:02.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:02.293 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:02 smithi099 ceph-mon[167076]: pgmap v477: 119 pgs: 1 down, 6 creating+peering, 3 unknown, 9 active+clean+remapped, 56 active+clean, 4 undersized+peered, 29 active+undersized, 1 undersized+degraded+peered, 1 active+undersized+remapped, 9 active+undersized+degraded; 4.6 MiB data, 4.2 GiB used, 711 GiB / 715 GiB avail; 0 B/s wr, 1 op/s; 13/84 objects degraded (15.476%); 4/84 objects misplaced (4.762%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:02.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/463908141' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:01:02.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/813829039' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:01:02.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:02 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:01:02.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:02 smithi099 ceph-mon[167076]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T18:01:02.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:02.295 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:02.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:02.425 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:02.460 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:02.462 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:02.463 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:02.564 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:02.591 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:02.626 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:02.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:02.629 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:02.729 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:02.756 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:02.787 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:02.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:02.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:02.891 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:02.892 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:01:02.941 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:03.031 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:03.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:03.034 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:03.135 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:03.162 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:03.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[179668]: failed to encode map e207 with expected crc 2023-09-23T18:01:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[179668]: failed to encode map e207 with expected crc 2023-09-23T18:01:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/463908141' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:01:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T18:01:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[179668]: osdmap e207: 8 total, 6 up, 7 in 2023-09-23T18:01:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/882620662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 13/84 objects degraded (15.476%), 10 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/882620662' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:01:03.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[179668]: osdmap e208: 8 total, 6 up, 7 in 2023-09-23T18:01:03.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[184174]: failed to encode map e207 with expected crc 2023-09-23T18:01:03.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[184174]: failed to encode map e207 with expected crc 2023-09-23T18:01:03.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/463908141' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:01:03.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T18:01:03.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[184174]: osdmap e207: 8 total, 6 up, 7 in 2023-09-23T18:01:03.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/882620662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:03.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 13/84 objects degraded (15.476%), 10 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:03.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:03.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/882620662' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:01:03.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:03 smithi055 ceph-mon[184174]: osdmap e208: 8 total, 6 up, 7 in 2023-09-23T18:01:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:03 smithi099 ceph-mon[167076]: failed to encode map e207 with expected crc 2023-09-23T18:01:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:03 smithi099 ceph-mon[167076]: failed to encode map e207 with expected crc 2023-09-23T18:01:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/463908141' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_1", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:01:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:03 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T18:01:03.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:03 smithi099 ceph-mon[167076]: osdmap e207: 8 total, 6 up, 7 in 2023-09-23T18:01:03.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/882620662' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:03.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 13/84 objects degraded (15.476%), 10 pgs degraded, 33 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:03.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:03 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:03.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/882620662' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:01:03.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:03 smithi099 ceph-mon[167076]: osdmap e208: 8 total, 6 up, 7 in 2023-09-23T18:01:03.381 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:03.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:03.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:03.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:03.512 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:03.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:03.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:03.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:03.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:03.675 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:03.707 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:03.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:03.710 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:03.740 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:03.810 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:03.836 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:03.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:03.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:03.870 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:03.939 DEBUG:teuthology.orchestra.run.smithi055:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_1 bench 90 write' 2023-09-23T18:01:03.942 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T18:01:03.942 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T18:01:03.970 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:01:03.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:03.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:hints = 1 2023-09-23T18:01:03.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T18:01:03.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object prefix: benchmark_data_smithi055_275963 2023-09-23T18:01:03.992 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:01:03.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:01:03.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 0 0 0 0 0 0 - 0 2023-09-23T18:01:03.999 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:04.024 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:01:04.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:04.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:04.043 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:04.048 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:01:04.058 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[179668]: pgmap v479: 119 pgs: 3 remapped+peering, 5 peering, 1 down, 3 creating+peering, 6 active+clean+remapped, 55 active+clean, 4 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 13/84 objects degraded (15.476%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:04.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[179668]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[179668]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[179668]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[179668]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[179668]: failed to encode map e209 with expected crc 2023-09-23T18:01:04.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[179668]: osdmap e209: 8 total, 6 up, 7 in 2023-09-23T18:01:04.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/179597918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:04.071 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:01:04.093 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:01:04.093 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:01:04.094 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:01:04.094 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:01:04.094 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:01:04.094 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:01:04.094 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:01:04.094 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:01:04.094 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:01:04.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:04.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:04 smithi099 ceph-mon[167076]: pgmap v479: 119 pgs: 3 remapped+peering, 5 peering, 1 down, 3 creating+peering, 6 active+clean+remapped, 55 active+clean, 4 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 13/84 objects degraded (15.476%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:04.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:04 smithi099 ceph-mon[167076]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:04 smithi099 ceph-mon[167076]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:04 smithi099 ceph-mon[167076]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.167 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:04 smithi099 ceph-mon[167076]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.168 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:04 smithi099 ceph-mon[167076]: failed to encode map e209 with expected crc 2023-09-23T18:01:04.168 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:04 smithi099 ceph-mon[167076]: osdmap e209: 8 total, 6 up, 7 in 2023-09-23T18:01:04.168 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:04 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/179597918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:04.169 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:04.201 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:04.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:04.204 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:04.305 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:04.334 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[184174]: pgmap v479: 119 pgs: 3 remapped+peering, 5 peering, 1 down, 3 creating+peering, 6 active+clean+remapped, 55 active+clean, 4 undersized+peered, 31 active+undersized, 1 active+undersized+remapped, 1 undersized+degraded+peered, 9 active+undersized+degraded; 4.6 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 1 op/s; 13/84 objects degraded (15.476%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:04.334 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[184174]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.334 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[184174]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.334 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[184174]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[184174]: failed to encode map e208 with expected crc 2023-09-23T18:01:04.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[184174]: failed to encode map e209 with expected crc 2023-09-23T18:01:04.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[184174]: osdmap e209: 8 total, 6 up, 7 in 2023-09-23T18:01:04.335 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:04 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/179597918' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:04.338 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:04.426 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:04.428 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:04.429 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:04.530 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:04.556 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:04.588 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:04.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:04.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:04.642 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:01:04.642 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:01:04.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:04.719 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:04.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:04.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:04.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:04.855 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:04.880 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:04.912 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:04.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:04.914 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:04.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 1 16 121 105 6.56213 6.5625 0.0013329 0.00254723 2023-09-23T18:01:05.015 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:05.046 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:05.092 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:05.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:05.095 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:05.162 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:01:05.162 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:01:05.163 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:01:05.163 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:01:05.163 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.163 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104107 5.0 5:08ba7c2f:::benchmark_data_smithi055_229117_object104106:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T18:01:05.163 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.953750+0000", 2023-09-23T18:01:05.163 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 221.20661269999999, 2023-09-23T18:01:05.163 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3969089229999998, 2023-09-23T18:01:05.163 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.163 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.163 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.164 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.164 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.164 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104107 2023-09-23T18:01:05.164 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.164 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.164 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.164 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.164 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953750+0000", 2023-09-23T18:01:05.164 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.164 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953750+0000", 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953748+0000", 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953776+0000", 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8515000000000001e-05 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953798+0000", 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1855999999999999e-05 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953808+0000", 2023-09-23T18:01:05.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0283e-05 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953876+0000", 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7348000000000006e-05 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op must wait for map", 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953885+0000", 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0839999999999998e-06 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.962444+0000", 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0085596219999999994 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.962456+0000", 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1606999999999999e-05 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949379+0000", 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98692283300000005 2023-09-23T18:01:05.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949390+0000", 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1503e-05 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.325692+0000", 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.376301955 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.327734+0000", 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020413749999999998 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.327871+0000", 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137739 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336120+0000", 2023-09-23T18:01:05.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082488500000000003 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336181+0000", 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1007000000000001e-05 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.348130+0000", 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011948676 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.348167+0000", 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6782000000000001e-05 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350658+0000", 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024918570000000001 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104131 5.0 5:02532558:::benchmark_data_smithi055_229117_object104130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.987392+0000", 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 221.17297017499999, 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3655978989999999, 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104131 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.173 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987392+0000", 2023-09-23T18:01:05.174 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987392+0000", 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987390+0000", 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987415+0000", 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4185e-05 2023-09-23T18:01:05.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987433+0000", 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8238000000000001e-05 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987442+0000", 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9600000000000006e-06 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.988601+0000", 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011587889999999999 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.988612+0000", 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1969e-05 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949474+0000", 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96086169899999996 2023-09-23T18:01:05.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949482+0000", 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0099999999999995e-06 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333244+0000", 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3837614760000001 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.178 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333381+0000", 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013695800000000001 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333517+0000", 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000136181 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342436+0000", 2023-09-23T18:01:05.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0089189990000000004 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342466+0000", 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0561000000000001e-05 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350747+0000", 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082805090000000001 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350829+0000", 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1778000000000002e-05 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.352990+0000", 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021612770000000001 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.181 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104167 5.0 5:0cc1b975:::benchmark_data_smithi055_229117_object104166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:24.986469+0000", 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.17389345999999, 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3687401239999999, 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104167 2023-09-23T18:01:05.182 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986469+0000", 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986469+0000", 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.183 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.184 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.184 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986467+0000", 2023-09-23T18:01:05.184 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.184 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.184 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.184 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.184 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986500+0000", 2023-09-23T18:01:05.184 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3046000000000002e-05 2023-09-23T18:01:05.184 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986515+0000", 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5138e-05 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986527+0000", 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1429e-05 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.026788+0000", 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040261308000000003 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.026799+0000", 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0699999999999999e-05 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336245+0000", 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3094467810000001 2023-09-23T18:01:05.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336378+0000", 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013300399999999999 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336525+0000", 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014695100000000001 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342505+0000", 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0059792530000000003 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342518+0000", 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.348e-05 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353111+0000", 2023-09-23T18:01:05.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010592839 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353201+0000", 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0167999999999995e-05 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355209+0000", 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020078090000000002 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.189 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104189 5.0 5:00d05606:::benchmark_data_smithi055_229117_object104188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.012952+0000", 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.14741068699999, 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.354677634, 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104189 2023-09-23T18:01:05.190 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012952+0000", 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012952+0000", 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.191 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012950+0000", 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012983+0000", 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2840999999999999e-05 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012997+0000", 2023-09-23T18:01:05.192 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.454e-05 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.013006+0000", 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2269999999999999e-06 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.030199+0000", 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017192819000000002 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.193 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.030209+0000", 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.023e-05 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339148+0000", 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3089385840000001 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.194 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.195 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339354+0000", 2023-09-23T18:01:05.195 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020608099999999999 2023-09-23T18:01:05.195 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.195 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.195 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.195 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339570+0000", 2023-09-23T18:01:05.195 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021573299999999999 2023-09-23T18:01:05.195 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.195 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342529+0000", 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029589759999999999 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342571+0000", 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1863999999999998e-05 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.196 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365363+0000", 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022792085 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365409+0000", 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6479e-05 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367629+0000", 2023-09-23T18:01:05.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022198840000000001 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104208 5.0 5:0c574cd2:::benchmark_data_smithi055_229117_object104207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.042202+0000", 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.11816050600001, 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3303606880000001, 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.198 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104208 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042202+0000", 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.199 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042202+0000", 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042200+0000", 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042226+0000", 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5925e-05 2023-09-23T18:01:05.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042245+0000", 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9449999999999998e-05 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042255+0000", 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9739999999999997e-06 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.201 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042301+0000", 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5945999999999997e-05 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042309+0000", 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7130000000000005e-06 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342600+0000", 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3002915850000001 2023-09-23T18:01:05.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342725+0000", 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012474099999999999 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342853+0000", 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000127806 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353066+0000", 2023-09-23T18:01:05.204 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010212738000000001 2023-09-23T18:01:05.204 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.204 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355236+0000", 2023-09-23T18:01:05.204 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002170052 2023-09-23T18:01:05.204 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.204 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.205 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369924+0000", 2023-09-23T18:01:05.205 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014688012 2023-09-23T18:01:05.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.205 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.205 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369984+0000", 2023-09-23T18:01:05.205 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0571e-05 2023-09-23T18:01:05.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372562+0000", 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00257807 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104228 5.0 5:04306094:::benchmark_data_smithi055_229117_object104227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.063372+0000", 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.096989834, 2023-09-23T18:01:05.206 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3247219290000001, 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104228 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063372+0000", 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.207 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063372+0000", 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063371+0000", 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.208 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063394+0000", 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2929999999999999e-05 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063407+0000", 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3142000000000001e-05 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063415+0000", 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8120000000000008e-06 2023-09-23T18:01:05.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063476+0000", 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0988e-05 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063486+0000", 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0397999999999999e-05 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.210 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345425+0000", 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2819386879999999 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345551+0000", 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012593200000000001 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.211 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345665+0000", 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114308 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355247+0000", 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0095815529999999996 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355307+0000", 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0619999999999999e-05 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.212 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385736+0000", 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030428261000000002 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385803+0000", 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7624000000000002e-05 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.213 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388094+0000", 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002291015 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104253 5.0 5:0d680df2:::benchmark_data_smithi055_229117_object104252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.088438+0000", 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.07192421600001, 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3025030870000001, 2023-09-23T18:01:05.214 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104253 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088438+0000", 2023-09-23T18:01:05.215 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088438+0000", 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088436+0000", 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.216 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088461+0000", 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4658999999999999e-05 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088478+0000", 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7805e-05 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.217 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088487+0000", 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.225e-06 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.092295+0000", 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038084870000000002 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.092306+0000", 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1172000000000001e-05 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.218 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355342+0000", 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.263036211 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355533+0000", 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000190767 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355663+0000", 2023-09-23T18:01:05.219 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012937699999999999 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365284+0000", 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0096211700000000001 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367649+0000", 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002365514 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.220 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388536+0000", 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020886503000000001 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388577+0000", 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1167000000000001e-05 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.390941+0000", 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002364157 2023-09-23T18:01:05.221 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104272 5.0 5:0ec51932:::benchmark_data_smithi055_229117_object104271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.108373+0000", 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.05198876599999, 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2933625550000001, 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.222 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104272 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108373+0000", 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.223 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108373+0000", 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108372+0000", 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108396+0000", 2023-09-23T18:01:05.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4311000000000002e-05 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108412+0000", 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6049000000000002e-05 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108420+0000", 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9170000000000006e-06 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108482+0000", 2023-09-23T18:01:05.226 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1969000000000002e-05 2023-09-23T18:01:05.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.226 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.226 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108491+0000", 2023-09-23T18:01:05.226 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4779999999999997e-06 2023-09-23T18:01:05.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.358777+0000", 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2502855180000001 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.358912+0000", 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013505099999999999 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.359032+0000", 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120331 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367658+0000", 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0086259400000000003 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367698+0000", 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9477999999999997e-05 2023-09-23T18:01:05.228 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.398598+0000", 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030900179 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.398665+0000", 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7167000000000003e-05 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.229 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401736+0000", 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030708630000000001 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104276 5.0 5:0ba41185:::benchmark_data_smithi055_229117_object104275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.114398+0000", 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.045963774, 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2928085359999999, 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.230 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104276 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114398+0000", 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.231 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114398+0000", 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114397+0000", 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114423+0000", 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6006000000000001e-05 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114440+0000", 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6877000000000001e-05 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114448+0000", 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5120000000000008e-06 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.115268+0000", 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00081976499999999997 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.115279+0000", 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0533999999999999e-05 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362251+0000", 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.24697258 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362402+0000", 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015118400000000001 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362530+0000", 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012761700000000001 2023-09-23T18:01:05.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.236 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.236 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.236 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369824+0000", 2023-09-23T18:01:05.236 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072939889999999999 2023-09-23T18:01:05.236 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.236 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.236 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.236 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372645+0000", 2023-09-23T18:01:05.236 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028209210000000001 2023-09-23T18:01:05.236 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.405151+0000", 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032506363000000003 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.405203+0000", 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1987000000000002e-05 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.237 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407207+0000", 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002003777 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104298 5.0 5:040384f9:::benchmark_data_smithi055_229117_object104297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.139530+0000", 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.02083174500001, 2023-09-23T18:01:05.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2697862209999999, 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104298 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139530+0000", 2023-09-23T18:01:05.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139530+0000", 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139529+0000", 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139553+0000", 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4026999999999999e-05 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139565+0000", 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2364999999999999e-05 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139573+0000", 2023-09-23T18:01:05.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4200000000000001e-06 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.141901+0000", 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023278639999999998 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.141909+0000", 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5099999999999998e-06 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372679+0000", 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2307700370000001 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372803+0000", 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012344299999999999 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372931+0000", 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012818500000000001 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385669+0000", 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012738027000000001 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385682+0000", 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3368e-05 2023-09-23T18:01:05.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407273+0000", 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021590933 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407320+0000", 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7305999999999997e-05 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409317+0000", 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019962769999999999 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104301 5.0 5:0653eda0:::benchmark_data_smithi055_229117_object104300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.142080+0000", 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.018282575, 2023-09-23T18:01:05.246 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2806010670000001, 2023-09-23T18:01:05.247 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.247 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.247 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.247 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.247 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.247 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104301 2023-09-23T18:01:05.247 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.247 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.247 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142080+0000", 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142080+0000", 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.248 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142078+0000", 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142104+0000", 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5931e-05 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142116+0000", 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2262999999999999e-05 2023-09-23T18:01:05.249 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142125+0000", 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5359999999999992e-06 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142238+0000", 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000113056 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142247+0000", 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6680000000000007e-06 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.375988+0000", 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2337410959999999 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.376092+0000", 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104558 2023-09-23T18:01:05.251 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.376220+0000", 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012734399999999999 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385691+0000", 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0094714069999999994 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391023+0000", 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053320190000000003 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.419987+0000", 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028963940000000001 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.253 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.420045+0000", 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8421000000000002e-05 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422681+0000", 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026352419999999999 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.254 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104305 5.0 5:03148f53:::benchmark_data_smithi055_229117_object104304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.146575+0000", 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.013787143, 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2788299140000001, 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104305 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.255 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146575+0000", 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146575+0000", 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.256 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.257 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146573+0000", 2023-09-23T18:01:05.257 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.257 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.257 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146600+0000", 2023-09-23T18:01:05.257 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6675000000000001e-05 2023-09-23T18:01:05.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146622+0000", 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1965000000000001e-05 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146633+0000", 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.076e-05 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.147763+0000", 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011302879999999999 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.147777+0000", 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4129e-05 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379012+0000", 2023-09-23T18:01:05.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2312345389999999 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379258+0000", 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024643700000000001 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379508+0000", 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024977000000000002 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.390966+0000", 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011458268000000001 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391089+0000", 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000122881 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422762+0000", 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031672697999999999 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422832+0000", 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9929000000000001e-05 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.425405+0000", 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002573362 2023-09-23T18:01:05.262 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104309 5.0 5:0de80b4f:::benchmark_data_smithi055_229117_object104308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.152292+0000", 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 220.008069818, 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2892466869999999, 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.263 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104309 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152292+0000", 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152292+0000", 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152291+0000", 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152313+0000", 2023-09-23T18:01:05.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2082999999999998e-05 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152326+0000", 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2493999999999999e-05 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152332+0000", 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7710000000000001e-06 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152379+0000", 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6233999999999998e-05 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152389+0000", 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0234999999999999e-05 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382503+0000", 2023-09-23T18:01:05.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.230114411 2023-09-23T18:01:05.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382678+0000", 2023-09-23T18:01:05.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000174704 2023-09-23T18:01:05.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382886+0000", 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000208481 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391101+0000", 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082150169999999998 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391162+0000", 2023-09-23T18:01:05.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0718999999999999e-05 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.438904+0000", 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047742297000000003 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.438971+0000", 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6787999999999998e-05 2023-09-23T18:01:05.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441539+0000", 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025679069999999999 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104346 5.0 5:07875566:::benchmark_data_smithi055_229117_object104345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.224009+0000", 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 219.93635287000001, 2023-09-23T18:01:05.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2202360750000001, 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104346 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224009+0000", 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224009+0000", 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224007+0000", 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224037+0000", 2023-09-23T18:01:05.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9128e-05 2023-09-23T18:01:05.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224058+0000", 2023-09-23T18:01:05.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1829999999999999e-05 2023-09-23T18:01:05.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224068+0000", 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8560000000000002e-06 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224119+0000", 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0844000000000003e-05 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224128+0000", 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4880000000000007e-06 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391218+0000", 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1670905920000001 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391390+0000", 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017210399999999999 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391557+0000", 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000166694 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409338+0000", 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017781376000000002 2023-09-23T18:01:05.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413075+0000", 2023-09-23T18:01:05.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037367199999999998 2023-09-23T18:01:05.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441577+0000", 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028502169000000001 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441616+0000", 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8859999999999997e-05 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444245+0000", 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026292899999999998 2023-09-23T18:01:05.279 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104353 5.0 5:0b824901:::benchmark_data_smithi055_229117_object104352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.238109+0000", 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 219.922253248, 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.208967144, 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.280 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104353 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238109+0000", 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238109+0000", 2023-09-23T18:01:05.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238108+0000", 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238128+0000", 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9854999999999999e-05 2023-09-23T18:01:05.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238135+0000", 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1729999999999998e-06 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238140+0000", 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6199999999999998e-06 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238197+0000", 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7005000000000001e-05 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238204+0000", 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2819999999999999e-06 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.394820+0000", 2023-09-23T18:01:05.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.156615231 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.394956+0000", 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013646599999999999 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.395118+0000", 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000162158 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409348+0000", 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014229883 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413141+0000", 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00379284 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.286 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444312+0000", 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031170989 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444373+0000", 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1309000000000005e-05 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447076+0000", 2023-09-23T18:01:05.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027029129999999999 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104360 5.0 5:0a7d8202:::benchmark_data_smithi055_229117_object104359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.266784+0000", 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 219.893578069, 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1830323949999999, 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.288 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.289 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.289 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.289 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.289 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104360 2023-09-23T18:01:05.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.289 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.289 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.289 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266784+0000", 2023-09-23T18:01:05.289 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266784+0000", 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266782+0000", 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266806+0000", 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3794999999999998e-05 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266824+0000", 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7838000000000001e-05 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266833+0000", 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9469999999999992e-06 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266890+0000", 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7238999999999999e-05 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266902+0000", 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1919999999999999e-05 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401792+0000", 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.13489014 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401976+0000", 2023-09-23T18:01:05.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000184014 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.402162+0000", 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018587599999999999 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413089+0000", 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010926544999999999 2023-09-23T18:01:05.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.425460+0000", 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012371320999999999 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447163+0000", 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021703538000000001 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447231+0000", 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7675999999999994e-05 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.449817+0000", 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025853859999999998 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:108962 5.0 5:0bab387e:::benchmark_data_smithi055_229117_object3336:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.753784+0000", 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 212.406577836, 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.186715511, 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108962 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.297 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753784+0000", 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753784+0000", 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753783+0000", 2023-09-23T18:01:05.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:05.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.299 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753792+0000", 2023-09-23T18:01:05.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0550000000000005e-06 2023-09-23T18:01:05.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753807+0000", 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4888e-05 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.753815+0000", 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8659999999999995e-06 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.841808+0000", 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.087992655000000003 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.842575+0000", 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00076655600000000005 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.871420+0000", 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028845756 2023-09-23T18:01:05.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.871532+0000", 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000112043 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.871703+0000", 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017070599999999999 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.898664+0000", 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026961124 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.902238+0000", 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035741309999999999 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.938343+0000", 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.036104458999999998 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.938383+0000", 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9885000000000002e-05 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940500+0000", 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021171979999999998 2023-09-23T18:01:05.304 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.305 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.305 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.305 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:108965 5.0 5:0ca96563:::benchmark_data_smithi055_229117_object3339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T18:01:05.305 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.758821+0000", 2023-09-23T18:01:05.305 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 212.40154143300001, 2023-09-23T18:01:05.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18445447700000001, 2023-09-23T18:01:05.305 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108965 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758821+0000", 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758821+0000", 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758820+0000", 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.307 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758825+0000", 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4580000000000001e-06 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758834+0000", 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5339999999999999e-06 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.758839+0000", 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1569999999999999e-06 2023-09-23T18:01:05.308 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.309 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.842655+0000", 2023-09-23T18:01:05.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.083815857999999993 2023-09-23T18:01:05.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:01:05.309 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.843295+0000", 2023-09-23T18:01:05.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000640651 2023-09-23T18:01:05.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.887064+0000", 2023-09-23T18:01:05.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043768661 2023-09-23T18:01:05.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.887235+0000", 2023-09-23T18:01:05.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017120699999999999 2023-09-23T18:01:05.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.887449+0000", 2023-09-23T18:01:05.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021399400000000001 2023-09-23T18:01:05.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.902251+0000", 2023-09-23T18:01:05.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014802123 2023-09-23T18:01:05.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940561+0000", 2023-09-23T18:01:05.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038309546 2023-09-23T18:01:05.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940592+0000", 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1056999999999998e-05 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.940635+0000", 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3038999999999999e-05 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.943275+0000", 2023-09-23T18:01:05.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002640444 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:108971 5.0 5:0ef24fea:::benchmark_data_smithi055_229117_object3345:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e145)", 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.766589+0000", 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 212.39377330600001, 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.182688712, 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:05.316 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108971 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766589+0000", 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766589+0000", 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766588+0000", 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766593+0000", 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0699999999999997e-06 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766653+0000", 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0232000000000002e-05 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.766662+0000", 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0240000000000003e-06 2023-09-23T18:01:05.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.843985+0000", 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.077322734000000004 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.844592+0000", 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00060725799999999995 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.906868+0000", 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.062275358000000003 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.906988+0000", 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120485 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.907138+0000", 2023-09-23T18:01:05.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000150216 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.921960+0000", 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014822003 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.946229+0000", 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024268112000000001 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.946287+0000", 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7983999999999999e-05 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.946365+0000", 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8867000000000002e-05 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.949278+0000", 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029122089999999998 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPushReply(5.0 145/143 [PushReplyOp(5:023b70b2:::benchmark_data_smithi055_229117_object72678:head)])", 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:32.870165+0000", 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 212.29019716900001, 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18210786500000001, 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:01:05.324 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870165+0000", 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870165+0000", 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870164+0000", 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870173+0000", 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7709999999999995e-06 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:05.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870197+0000", 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4243000000000002e-05 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:32.870207+0000", 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8279999999999998e-06 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:33.047464+0000", 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.177257317 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:05.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:05.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:05.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:33.052273+0000", 2023-09-23T18:01:05.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048086429999999996 2023-09-23T18:01:05.328 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.328 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.328 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.328 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:05.328 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:05.328 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:01:05.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:05.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:05 smithi099 ceph-mon[167076]: failed to encode map e209 with expected crc 2023-09-23T18:01:05.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:05 smithi099 ceph-mon[167076]: failed to encode map e209 with expected crc 2023-09-23T18:01:05.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:05 smithi099 ceph-mon[167076]: 1.1 scrub starts 2023-09-23T18:01:05.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:05 smithi099 ceph-mon[167076]: 1.1 scrub ok 2023-09-23T18:01:05.330 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:05 smithi055 ceph-mon[179668]: failed to encode map e209 with expected crc 2023-09-23T18:01:05.330 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:05 smithi055 ceph-mon[179668]: failed to encode map e209 with expected crc 2023-09-23T18:01:05.330 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:05 smithi055 ceph-mon[179668]: 1.1 scrub starts 2023-09-23T18:01:05.330 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:05 smithi055 ceph-mon[179668]: 1.1 scrub ok 2023-09-23T18:01:05.331 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:05 smithi055 ceph-mon[184174]: failed to encode map e209 with expected crc 2023-09-23T18:01:05.331 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:05 smithi055 ceph-mon[184174]: failed to encode map e209 with expected crc 2023-09-23T18:01:05.331 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:05 smithi055 ceph-mon[184174]: 1.1 scrub starts 2023-09-23T18:01:05.331 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:05 smithi055 ceph-mon[184174]: 1.1 scrub ok 2023-09-23T18:01:05.355 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:05.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:05.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:05.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:05.486 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:01:05.491 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:05.540 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:01:05.556 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:05.595 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:05.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:05.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:05.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:05.725 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:05.756 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:05.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:05.759 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:05.860 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:05.888 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:05.925 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:05.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:05.927 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:05.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 2 16 121 105 3.28047 0 - 0.00254723 2023-09-23T18:01:06.027 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:06.053 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:06.085 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:06.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:06.088 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:06.105 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:06.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:06 smithi055 ceph-mon[179668]: pgmap v482: 119 pgs: 1 creating+remapped+peering, 7 remapped+peering, 1 down, 5 peering, 1 creating+peering, 17 active+clean+remapped, 51 active+clean, 4 undersized+peered, 22 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 7 active+undersized+degraded; 6.1 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 265 KiB/s wr, 2 op/s; 18/159 objects degraded (11.321%); 20/159 objects misplaced (12.579%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:06.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:06 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 9 pgs inactive, 1 pg down, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:01:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:06 smithi055 ceph-mon[184174]: pgmap v482: 119 pgs: 1 creating+remapped+peering, 7 remapped+peering, 1 down, 5 peering, 1 creating+peering, 17 active+clean+remapped, 51 active+clean, 4 undersized+peered, 22 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 7 active+undersized+degraded; 6.1 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 265 KiB/s wr, 2 op/s; 18/159 objects degraded (11.321%); 20/159 objects misplaced (12.579%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:06 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 9 pgs inactive, 1 pg down, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:01:06.189 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:06.214 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:06.247 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:06.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:06.250 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:06.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:06 smithi099 ceph-mon[167076]: pgmap v482: 119 pgs: 1 creating+remapped+peering, 7 remapped+peering, 1 down, 5 peering, 1 creating+peering, 17 active+clean+remapped, 51 active+clean, 4 undersized+peered, 22 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 7 active+undersized+degraded; 6.1 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 265 KiB/s wr, 2 op/s; 18/159 objects degraded (11.321%); 20/159 objects misplaced (12.579%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:06 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 9 pgs inactive, 1 pg down, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:01:06.351 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:06.379 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:06.637 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:06.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:06.641 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:06.641 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:06.643 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:01:06.741 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:06.768 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:06.800 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:06.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:06.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:06.904 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:06.929 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:06.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:06.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:06.964 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:06.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 3 16 121 105 2.18709 0 - 0.00254723 2023-09-23T18:01:07.065 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:07.099 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:07.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:07.368 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:07.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:07.371 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:07.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:07.498 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:07.501 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:07.535 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:07.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:07.538 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:07.639 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:07.669 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:07.695 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 5] out_osds: [4] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T18:01:07.695 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:01:07.695 INFO:tasks.thrashosds.thrasher:inject_pause on osd.6 2023-09-23T18:01:07.696 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:01:07.696 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:01:07.696 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 config set filestore_inject_stall 3 2023-09-23T18:01:07.813 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:07.822 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:07.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:07.825 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:07.875 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:01:07.875 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:01:07.875 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:01:07.875 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:01:07.926 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:07.951 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:07.984 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:07.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:07.987 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:07.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 4 16 121 105 1.64036 0 - 0.00254723 2023-09-23T18:01:08.088 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:08.120 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:08 smithi055 ceph-mon[184174]: pgmap v483: 119 pgs: 1 creating+remapped+peering, 7 remapped+peering, 1 down, 5 peering, 23 active+clean+remapped, 51 active+clean, 4 undersized+peered, 16 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 8 active+undersized+degraded; 8.0 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 579 KiB/s wr, 7 op/s; 31/252 objects degraded (12.302%); 21/252 objects misplaced (8.333%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 31/252 objects degraded (12.302%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:08.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:08 smithi055 ceph-mon[179668]: pgmap v483: 119 pgs: 1 creating+remapped+peering, 7 remapped+peering, 1 down, 5 peering, 23 active+clean+remapped, 51 active+clean, 4 undersized+peered, 16 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 8 active+undersized+degraded; 8.0 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 579 KiB/s wr, 7 op/s; 31/252 objects degraded (12.302%); 21/252 objects misplaced (8.333%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:08.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 31/252 objects degraded (12.302%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:08.288 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:08 smithi099 ceph-mon[167076]: pgmap v483: 119 pgs: 1 creating+remapped+peering, 7 remapped+peering, 1 down, 5 peering, 23 active+clean+remapped, 51 active+clean, 4 undersized+peered, 16 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 8 active+undersized+degraded; 8.0 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 579 KiB/s wr, 7 op/s; 31/252 objects degraded (12.302%); 21/252 objects misplaced (8.333%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 31/252 objects degraded (12.302%), 9 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:08.451 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:01:08.506 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:08.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:08.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:08.611 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:08.639 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:08.734 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:08.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:08.738 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:08.838 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:08.874 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:08.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 5 16 121 105 1.31231 0 - 0.00254723 2023-09-23T18:01:09.032 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:09.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:09.035 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:09.103 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:01:09.136 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:09.166 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:09.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:09.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:09.201 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:09.224 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:09.238 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:01:09.260 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:01:09.279 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:01:09.299 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:01:09.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:09.318 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:01:09.330 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:09.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:09 smithi099 ceph-mon[167076]: 1.3 scrub starts 2023-09-23T18:01:09.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:09 smithi099 ceph-mon[167076]: 1.3 scrub ok 2023-09-23T18:01:09.341 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1306519805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:09.342 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:01:09.342 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:01:09.342 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:01:09.343 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:01:09.343 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:01:09.343 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:01:09.343 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:01:09.343 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:01:09.343 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:01:09.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:09 smithi055 ceph-mon[184174]: 1.3 scrub starts 2023-09-23T18:01:09.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:09 smithi055 ceph-mon[184174]: 1.3 scrub ok 2023-09-23T18:01:09.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1306519805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:09.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:09 smithi055 ceph-mon[179668]: 1.3 scrub starts 2023-09-23T18:01:09.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:09 smithi055 ceph-mon[179668]: 1.3 scrub ok 2023-09-23T18:01:09.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1306519805' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:09.413 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:09.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:09.417 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:09.491 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:01:09.491 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T18:01:09.491 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:01:09.518 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:09.550 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:09.864 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:09.868 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:09.868 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:09.877 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:01:09.877 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:01:09.970 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:09.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 6 16 121 105 1.0936 0 - 0.00254723 2023-09-23T18:01:09.996 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:10.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:10.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:10.032 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:10.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:10.159 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[179668]: pgmap v484: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1006 KiB/s wr, 14 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:01:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[179668]: failed to encode map e210 with expected crc 2023-09-23T18:01:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[179668]: failed to encode map e210 with expected crc 2023-09-23T18:01:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1306519805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[179668]: osdmap e210: 8 total, 6 up, 7 in 2023-09-23T18:01:10.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2779167164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:10.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[184174]: pgmap v484: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1006 KiB/s wr, 14 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:10.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:01:10.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[184174]: failed to encode map e210 with expected crc 2023-09-23T18:01:10.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[184174]: failed to encode map e210 with expected crc 2023-09-23T18:01:10.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1306519805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:10.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[184174]: osdmap e210: 8 total, 6 up, 7 in 2023-09-23T18:01:10.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2779167164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:10.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:10.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:10.195 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:10.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:10.321 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:10.355 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:10.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:10 smithi099 ceph-mon[167076]: pgmap v484: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1006 KiB/s wr, 14 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:10.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:10 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:01:10.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:10 smithi099 ceph-mon[167076]: failed to encode map e210 with expected crc 2023-09-23T18:01:10.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:10 smithi099 ceph-mon[167076]: failed to encode map e210 with expected crc 2023-09-23T18:01:10.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1306519805' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:10.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:10 smithi099 ceph-mon[167076]: osdmap e210: 8 total, 6 up, 7 in 2023-09-23T18:01:10.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2779167164' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:10.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:10.358 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:10.459 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:10.486 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:01:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:01:10] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-09-23T18:01:10.490 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:10.574 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:10.577 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:10.578 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:10.679 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:10.707 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:10.745 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:10.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:10.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:10.785 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:01:10.785 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:01:10.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:10.877 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:10.912 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:10.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:10.915 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:10.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 7 16 121 105 0.93737 0 - 0.00254723 2023-09-23T18:01:11.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:11.041 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:11.073 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:11.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:11.076 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:11.090 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:01:11.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:11.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:11 smithi099 ceph-mon[167076]: 1.4 scrub starts 2023-09-23T18:01:11.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:11 smithi099 ceph-mon[167076]: 1.4 scrub ok 2023-09-23T18:01:11.203 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:11.237 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:11.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:11.238 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:11.339 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:11.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:11 smithi055 ceph-mon[179668]: 1.4 scrub starts 2023-09-23T18:01:11.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:11 smithi055 ceph-mon[179668]: 1.4 scrub ok 2023-09-23T18:01:11.365 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:11 smithi055 ceph-mon[184174]: 1.4 scrub starts 2023-09-23T18:01:11.365 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:11 smithi055 ceph-mon[184174]: 1.4 scrub ok 2023-09-23T18:01:11.367 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:11.401 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:11.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:11.404 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:11.505 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:11.531 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:11.564 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:11.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:11.567 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:11.668 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:11.694 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:11.727 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:11.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:11.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:11.733 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:11.831 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:11.859 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:11.878 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:01:11.895 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:01:11.898 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:11.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:11.902 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:11.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 8 16 121 105 0.820204 0 - 0.00254723 2023-09-23T18:01:12.003 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:12.029 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:12.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:12.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:12.063 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:12.164 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:12.190 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:12 smithi099 ceph-mon[167076]: pgmap v486: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 845 KiB/s wr, 11 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:12.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:12 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:01:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:12 smithi055 ceph-mon[184174]: pgmap v486: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 845 KiB/s wr, 11 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:12 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:01:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:12 smithi055 ceph-mon[179668]: pgmap v486: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 845 KiB/s wr, 11 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:01:12.624 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:12.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:12.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:12.728 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:12.765 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:12.803 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:12.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:12.807 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:12.907 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:12.938 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:12.972 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:12.974 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:12.975 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:12.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 9 16 121 105 0.729074 0 - 0.00254723 2023-09-23T18:01:13.075 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:13.103 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:13.112 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:13.139 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:13.142 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:13 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T18:01:13.143 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 47/399 objects degraded (11.779%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:13.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:13 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T18:01:13.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 47/399 objects degraded (11.779%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:13.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:13.144 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:13.245 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:13.261 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:13.277 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:13.561 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:13.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:13.564 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:13 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 4 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T18:01:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 47/399 objects degraded (11.779%), 11 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:13.665 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:13.698 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:13.975 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:13.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:13.979 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:13.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 10 16 121 105 0.65617 0 - 0.00254723 2023-09-23T18:01:14.079 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:14.110 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:14.144 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:14.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:14.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:14.248 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:14.275 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:14 smithi055 ceph-mon[179668]: pgmap v487: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 749 KiB/s wr, 9 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:14.276 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:14 smithi055 ceph-mon[184174]: pgmap v487: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 749 KiB/s wr, 9 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:14.279 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:14.354 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:01:14.354 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:01:14.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:14 smithi099 ceph-mon[167076]: pgmap v487: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 749 KiB/s wr, 9 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 1 keys/s, 1 objects/s recovering 2023-09-23T18:01:14.721 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:14.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:14.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:14.740 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:01:14.740 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:01:14.740 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:01:14.740 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:01:14.740 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:01:14.751 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:01:14.751 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:01:14.752 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:01:14.752 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:01:14.752 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:01:14.752 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:01:14.752 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:01:14.752 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:01:14.752 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:01:14.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:14.832 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 5] out_osds: [4] dead_osds: [3, 7] live_osds: [0, 1, 2, 4, 5, 6] 2023-09-23T18:01:14.833 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:01:14.833 INFO:tasks.thrashosds.thrasher:Reviving osd 7 2023-09-23T18:01:14.833 INFO:tasks.cephadm.osd.7:Restarting osd.7 (starting--it wasn't running)... 2023-09-23T18:01:14.833 DEBUG:teuthology.orchestra.run.smithi099:osd.7> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.7.service 2023-09-23T18:01:14.852 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:14.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl start ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T18:01:14.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:01:14.883 INFO:journalctl@ceph.osd.7.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T18:01:14.937 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:14.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:14.942 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:14.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 11 16 121 105 0.596519 0 - 0.00254723 2023-09-23T18:01:15.042 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:15.071 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:15.146 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:14 smithi099 systemd[1]: Starting Ceph osd.7 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:01:15.295 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:15.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:15.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:15.304 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:01:15.304 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:01:15.399 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:15.425 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:15 smithi099 ceph-mon[167076]: 1.2 scrub starts 2023-09-23T18:01:15.425 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:15 smithi099 ceph-mon[167076]: 1.2 scrub ok 2023-09-23T18:01:15.425 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4277442606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:15.428 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:15.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:15 smithi055 ceph-mon[179668]: 1.2 scrub starts 2023-09-23T18:01:15.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:15 smithi055 ceph-mon[179668]: 1.2 scrub ok 2023-09-23T18:01:15.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4277442606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:15.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:15 smithi055 ceph-mon[184174]: 1.2 scrub starts 2023-09-23T18:01:15.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:15 smithi055 ceph-mon[184174]: 1.2 scrub ok 2023-09-23T18:01:15.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4277442606' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:15.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:15.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:15.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:15.620 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:15.650 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:15.745 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:15.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:15.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:15.767 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:15.851 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:15.881 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:15.973 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:15.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:15.977 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:15.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 12 16 121 105 0.54681 0 - 0.00254723 2023-09-23T18:01:16.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:16.111 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:16.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:16 smithi055 ceph-mon[184174]: pgmap v488: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 486 KiB/s wr, 5 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:16.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:16 smithi099 ceph-mon[167076]: pgmap v488: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 486 KiB/s wr, 5 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:16.326 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:16 smithi099 bash[202543]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T18:01:16.326 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:16 smithi099 bash[202543]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-23T18:01:16.326 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:16 smithi099 bash[202543]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-7 --no-mon-config 2023-09-23T18:01:16.326 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:16 smithi099 bash[202543]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-7/block 2023-09-23T18:01:16.326 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:16 smithi099 bash[202543]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-7/block 2023-09-23T18:01:16.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:16 smithi099 bash[202543]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T18:01:16.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:16 smithi099 bash[202543]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-7 2023-09-23T18:01:16.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:16 smithi099 bash[202543]: --> ceph-volume lvm activate successful for osd ID: 7 2023-09-23T18:01:16.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:16 smithi055 ceph-mon[179668]: pgmap v488: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 486 KiB/s wr, 5 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:16.553 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:16.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:16.557 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:16.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:16.685 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:16.768 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:16.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:16.772 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:16.873 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:16.916 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:16.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 13 16 121 105 0.504749 0 - 0.00254723 2023-09-23T18:01:17.191 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:01:17.204 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:17.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:17.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:17.305 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:01:17.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:17.339 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:17.386 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:17.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:17.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:17.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1322631730' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:17.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:17 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1322631730' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:17 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:17.433 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:01:17.434 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:01:17.434 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:01:17.434 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:01:17.456 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1322631730' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:17.456 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:17 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:17.456 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:17 smithi099 podman[203220]: 2023-09-23T18:01:17.491 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:17.519 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:17.612 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:17.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:17.629 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:17.714 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:17 smithi099 ceph-osd[203420]: -- 172.21.15.99:0/3600913238 <== mon.1 v2:172.21.15.99:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55af97b86b60 con 0x55af97b4cc00 2023-09-23T18:01:17.714 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:17 smithi099 bash[203220]: 3ebd4b4b5e11201505d2af7d9795f961ce7626ae677e3fa29486852943256a20 2023-09-23T18:01:17.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:01:17.729 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:17.789 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:17.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:17.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:17.827 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:17.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:01:17.928 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:17.959 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:17.979 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:17.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 14 16 121 105 0.468697 0 - 0.00254723 2023-09-23T18:01:17.993 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:17.995 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:17 smithi099 systemd[1]: Started Ceph osd.7 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:01:17.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:17.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:18.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:18.127 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:18.160 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:18.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:18.163 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:18.264 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:18.287 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:01:18.125+0000 7f823e5ca540 -1 Falling back to public interface 2023-09-23T18:01:18.288 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:18 smithi099 ceph-mon[167076]: pgmap v489: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 288 KiB/s wr, 2 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:18.288 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:18 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:18.288 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:18 smithi099 ceph-mon[167076]: failed to encode map e211 with expected crc 2023-09-23T18:01:18.288 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:18 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:01:18.288 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:18 smithi099 ceph-mon[167076]: osdmap e211: 8 total, 6 up, 7 in 2023-09-23T18:01:18.288 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:18 smithi099 ceph-mon[167076]: failed to encode map e211 with expected crc 2023-09-23T18:01:18.288 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:18 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:18.288 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:18 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:18.290 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:18.323 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:18.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:18.326 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:18.343 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:18.426 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[179668]: pgmap v489: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 288 KiB/s wr, 2 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[179668]: failed to encode map e211 with expected crc 2023-09-23T18:01:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:01:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[179668]: osdmap e211: 8 total, 6 up, 7 in 2023-09-23T18:01:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[179668]: failed to encode map e211 with expected crc 2023-09-23T18:01:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:18.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[184174]: pgmap v489: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 288 KiB/s wr, 2 op/s; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%); 0 B/s, 0 objects/s recovering 2023-09-23T18:01:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[184174]: failed to encode map e211 with expected crc 2023-09-23T18:01:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:01:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[184174]: osdmap e211: 8 total, 6 up, 7 in 2023-09-23T18:01:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[184174]: failed to encode map e211 with expected crc 2023-09-23T18:01:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:18.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:18 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:18.452 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:18.587 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:18.587 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:18.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:18.590 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:18.691 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:18.720 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:18.806 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:18.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:18.809 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:18.909 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:18.939 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:18.994 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 15 16 121 105 0.437451 0 - 0.00254723 2023-09-23T18:01:19.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:19.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:19.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:19.341 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:01:19.377 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:01:19.401 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:01:19.420 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:01:19.429 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:19.439 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:01:19.458 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:19.459 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:01:19.460 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:01:19.460 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:01:19.460 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:01:19.460 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:01:19.460 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:01:19.460 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:01:19.460 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:01:19.461 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.7 2023-09-23T18:01:19.496 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:19.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:19.499 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:19.580 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:01:19.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:19.631 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:19.839 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:01:19.839 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:01:19.967 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2023-09-23T18:01:19.968 INFO:teuthology.orchestra.run.smithi099.stderr:0 2023-09-23T18:01:19.968 INFO:teuthology.orchestra.run.smithi099.stderr:1 2023-09-23T18:01:19.968 INFO:teuthology.orchestra.run.smithi099.stderr:2 2023-09-23T18:01:19.968 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2023-09-23T18:01:19.968 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2023-09-23T18:01:19.968 INFO:teuthology.orchestra.run.smithi099.stderr:config diff 2023-09-23T18:01:19.968 INFO:teuthology.orchestra.run.smithi099.stderr:config diff get 2023-09-23T18:01:19.968 INFO:teuthology.orchestra.run.smithi099.stderr:config get 2023-09-23T18:01:19.968 INFO:teuthology.orchestra.run.smithi099.stderr:config help [] 2023-09-23T18:01:19.969 INFO:teuthology.orchestra.run.smithi099.stderr:config set ... 2023-09-23T18:01:19.969 INFO:teuthology.orchestra.run.smithi099.stderr:admin_socket: invalid command 2023-09-23T18:01:19.988 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:19.994 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 16 16 121 105 0.410111 0 - 0.00254723 2023-09-23T18:01:19.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:19.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:20.095 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:20.126 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:20.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:20.162 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:19 smithi099 ceph-mon[167076]: pgmap v491: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%) 2023-09-23T18:01:20.162 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:19 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:20.162 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:19 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:20.162 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:19 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2858604305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:20.162 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:19 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:01:20.162 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:19 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:01:20.162 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:19 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:20.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:20.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[179668]: pgmap v491: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%) 2023-09-23T18:01:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2858604305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:01:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:01:20.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:20.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[184174]: pgmap v491: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%) 2023-09-23T18:01:20.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:20.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:20.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2858604305' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:20.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:01:20.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:01:20.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:19 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:20.187 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:20.249 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:01:20.250 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:01:20.264 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:20.291 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:20.375 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:20.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:20.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:20.478 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:20.507 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:20.551 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:01:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:01:20] "GET /metrics HTTP/1.1" 200 36508 "" "Prometheus/2.43.0" 2023-09-23T18:01:20.592 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:20.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:20.596 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:20.624 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T18:01:20.624 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-7, ['dump_ops_in_flight'] 2023-09-23T18:01:20.697 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:20.724 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:20.743 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:01:20.862 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:20.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:20.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:20.966 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:20.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 17 16 121 105 0.385987 0 - 0.00254723 2023-09-23T18:01:20.995 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:21.080 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:21.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:21.084 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:21.184 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:21.215 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:21.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:21.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:21.346 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:21.358 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:21.387 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:21.423 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:21.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:21.427 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:21.528 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:21.557 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:21.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:21.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:21.603 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:21.703 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:21.732 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:21.766 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:21.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:21.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:21.840 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:01:21.871 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:21.911 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:01:21.914 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:21.952 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:21.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:21.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:21.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 18 16 121 105 0.364544 0 - 0.00254723 2023-09-23T18:01:22.056 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:22.058 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:21 smithi055 ceph-mon[184174]: pgmap v492: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%) 2023-09-23T18:01:22.058 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:22.059 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1687066447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:22.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:21 smithi055 ceph-mon[179668]: pgmap v492: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%) 2023-09-23T18:01:22.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:22.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1687066447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:22.084 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:22.120 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:22.122 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:21 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:01:21.917+0000 7f823e5ca540 -1 osd.7 149 log_to_monitors true 2023-09-23T18:01:22.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:21 smithi099 ceph-mon[167076]: pgmap v492: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%) 2023-09-23T18:01:22.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:22.123 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1687066447' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:22.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:22.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:22.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:22.252 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:22.335 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:22.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:22.338 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:22.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:22.466 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:22.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:22.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:22.502 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:22.541 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:01:22.541 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:01:22.563 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:22.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:22.633 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:22.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:22.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:22.667 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:22.768 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:22.794 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:22.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:22.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:22.910 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:22.994 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 19 16 121 105 0.345358 0 - 0.00254723 2023-09-23T18:01:23.006 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:01:23.011 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:23.015 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[179668]: failed to encode map e212 with expected crc 2023-09-23T18:01:23.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[179668]: failed to encode map e212 with expected crc 2023-09-23T18:01:23.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1687066447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:23.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[179668]: osdmap e212: 8 total, 6 up, 7 in 2023-09-23T18:01:23.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[179668]: from='osd.7 [v2:172.21.15.99:6824/179726820,v1:172.21.15.99:6825/179726820]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:01:23.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[179668]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:01:23.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:23.016 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[184174]: failed to encode map e212 with expected crc 2023-09-23T18:01:23.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[184174]: failed to encode map e212 with expected crc 2023-09-23T18:01:23.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1687066447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:23.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[184174]: osdmap e212: 8 total, 6 up, 7 in 2023-09-23T18:01:23.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[184174]: from='osd.7 [v2:172.21.15.99:6824/179726820,v1:172.21.15.99:6825/179726820]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:01:23.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[184174]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:01:23.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:22 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:23.072 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:01:23 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:01:23.029+0000 7f822dba7700 -1 osd.7 213 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:01:23.073 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:22 smithi099 ceph-mon[167076]: failed to encode map e212 with expected crc 2023-09-23T18:01:23.073 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:22 smithi099 ceph-mon[167076]: failed to encode map e212 with expected crc 2023-09-23T18:01:23.073 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1687066447' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:23.073 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:22 smithi099 ceph-mon[167076]: osdmap e212: 8 total, 6 up, 7 in 2023-09-23T18:01:23.073 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:22 smithi099 ceph-mon[167076]: from='osd.7 [v2:172.21.15.99:6824/179726820,v1:172.21.15.99:6825/179726820]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:01:23.073 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:22 smithi099 ceph-mon[167076]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-09-23T18:01:23.073 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:22 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:23.075 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:23.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:23.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:23.112 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:23.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:23.239 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:23.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:23.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:23.274 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:23.374 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:23.400 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:23.433 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:23.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:23.436 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:23.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:23.563 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:23.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:23.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:23.597 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:23.604 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:23.699 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:23.731 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:23.905 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:23.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:23.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:23.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[179668]: pgmap v494: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%) 2023-09-23T18:01:23.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[179668]: failed to encode map e213 with expected crc 2023-09-23T18:01:23.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[179668]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T18:01:23.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[179668]: osdmap e213: 8 total, 6 up, 7 in 2023-09-23T18:01:23.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[179668]: from='osd.7 [v2:172.21.15.99:6824/179726820,v1:172.21.15.99:6825/179726820]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:01:23.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[179668]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:01:23.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[184174]: pgmap v494: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%) 2023-09-23T18:01:23.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[184174]: failed to encode map e213 with expected crc 2023-09-23T18:01:23.995 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[184174]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T18:01:23.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[184174]: osdmap e213: 8 total, 6 up, 7 in 2023-09-23T18:01:23.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[184174]: from='osd.7 [v2:172.21.15.99:6824/179726820,v1:172.21.15.99:6825/179726820]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:01:23.996 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:23 smithi055 ceph-mon[184174]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:01:23.996 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:01:23.995124+0000 min lat: 0.00103291 max lat: 0.00647296 avg lat: 0.00254723 2023-09-23T18:01:23.996 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:01:23.996 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 20 16 121 105 0.328088 0 - 0.00254723 2023-09-23T18:01:24.009 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:24.041 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:24.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:23 smithi099 ceph-mon[167076]: pgmap v494: 119 pgs: 1 down, 32 active+clean+remapped, 54 active+clean, 4 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 3 active+undersized+remapped, 10 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 47/399 objects degraded (11.779%); 58/399 objects misplaced (14.536%) 2023-09-23T18:01:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:23 smithi099 ceph-mon[167076]: failed to encode map e213 with expected crc 2023-09-23T18:01:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:23 smithi099 ceph-mon[167076]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-09-23T18:01:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:23 smithi099 ceph-mon[167076]: osdmap e213: 8 total, 6 up, 7 in 2023-09-23T18:01:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:23 smithi099 ceph-mon[167076]: from='osd.7 [v2:172.21.15.99:6824/179726820,v1:172.21.15.99:6825/179726820]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:01:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:23 smithi099 ceph-mon[167076]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:01:24.125 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:24.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:24.128 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:24.209 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:01:24.228 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:01:24.229 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:24.247 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:01:24.248 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:01:24.257 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:24.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:01:24.289 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:01:24.297 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:24.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:24.298 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:24.325 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:01:24.344 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:01:24.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:01:24.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:01:24.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:01:24.344 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:01:24.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:01:24.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:01:24.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:01:24.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:01:24.399 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:24.429 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:24.504 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:24.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:24.507 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:24.608 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:24.636 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:24.668 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:24.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:24.671 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:24.715 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:01:24.716 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:01:24.772 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:24.799 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:24.829 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:01:24.830 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:01:24.830 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:01:24.830 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:01:24.831 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:24.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:24.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:24.935 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:24.963 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:24.992 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:24.994 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 21 16 121 105 0.312465 0 - 0.00254723 2023-09-23T18:01:25.009 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:25.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:25.018 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:25.119 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:25.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:24 smithi099 ceph-mon[167076]: failed to encode map e150 with expected crc 2023-09-23T18:01:25.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:24 smithi099 ceph-mon[167076]: failed to encode map e190 with expected crc 2023-09-23T18:01:25.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:24 smithi099 ceph-mon[167076]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T18:01:25.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:24 smithi099 ceph-mon[167076]: failed to encode map e214 with expected crc 2023-09-23T18:01:25.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:24 smithi099 ceph-mon[167076]: osd.7 [v2:172.21.15.99:6824/179726820,v1:172.21.15.99:6825/179726820] boot 2023-09-23T18:01:25.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:24 smithi099 ceph-mon[167076]: osdmap e214: 8 total, 7 up, 7 in 2023-09-23T18:01:25.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:24 smithi099 ceph-mon[167076]: failed to encode map e214 with expected crc 2023-09-23T18:01:25.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:01:25.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:24 smithi099 ceph-mon[167076]: failed to encode map e214 with expected crc 2023-09-23T18:01:25.120 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/712781792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:25.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:01:25.148 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:25.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[184174]: failed to encode map e150 with expected crc 2023-09-23T18:01:25.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[184174]: failed to encode map e190 with expected crc 2023-09-23T18:01:25.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[184174]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T18:01:25.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[184174]: failed to encode map e214 with expected crc 2023-09-23T18:01:25.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[184174]: osd.7 [v2:172.21.15.99:6824/179726820,v1:172.21.15.99:6825/179726820] boot 2023-09-23T18:01:25.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[184174]: osdmap e214: 8 total, 7 up, 7 in 2023-09-23T18:01:25.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[184174]: failed to encode map e214 with expected crc 2023-09-23T18:01:25.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:01:25.215 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[184174]: failed to encode map e214 with expected crc 2023-09-23T18:01:25.215 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/712781792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:25.215 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[179668]: failed to encode map e150 with expected crc 2023-09-23T18:01:25.215 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[179668]: failed to encode map e190 with expected crc 2023-09-23T18:01:25.215 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[179668]: Health check update: 1 osds down (OSD_DOWN) 2023-09-23T18:01:25.215 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[179668]: failed to encode map e214 with expected crc 2023-09-23T18:01:25.215 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[179668]: osd.7 [v2:172.21.15.99:6824/179726820,v1:172.21.15.99:6825/179726820] boot 2023-09-23T18:01:25.215 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[179668]: osdmap e214: 8 total, 7 up, 7 in 2023-09-23T18:01:25.215 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[179668]: failed to encode map e214 with expected crc 2023-09-23T18:01:25.215 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-09-23T18:01:25.216 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[179668]: failed to encode map e214 with expected crc 2023-09-23T18:01:25.216 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/712781792' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:25.233 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:25.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:25.237 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:25.338 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:25.366 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:25.625 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:01:25.646 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:25.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:25.649 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:25.737 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:25.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:25.778 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:25.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:25.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:25.869 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:25.970 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:25.994 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 22 16 435 419 1.19021 0.934524 0.00123659 0.804436 2023-09-23T18:01:26.000 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:26.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:26.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:26.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:26.176 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:26.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[179668]: pgmap v497: 119 pgs: 1 creating+peering, 7 remapped+peering, 1 peering, 1 down, 26 active+clean+remapped, 54 active+clean, 3 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 9 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 42/399 objects degraded (10.526%); 55/399 objects misplaced (13.784%) 2023-09-23T18:01:26.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 42/399 objects degraded (10.526%), 10 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:26.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[179668]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[179668]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[179668]: osdmap e215: 8 total, 7 up, 7 in 2023-09-23T18:01:26.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[179668]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[179668]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[179668]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[184174]: pgmap v497: 119 pgs: 1 creating+peering, 7 remapped+peering, 1 peering, 1 down, 26 active+clean+remapped, 54 active+clean, 3 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 9 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 42/399 objects degraded (10.526%); 55/399 objects misplaced (13.784%) 2023-09-23T18:01:26.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 42/399 objects degraded (10.526%), 10 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:26.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[184174]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[184174]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[184174]: osdmap e215: 8 total, 7 up, 7 in 2023-09-23T18:01:26.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[184174]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[184174]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:25 smithi055 ceph-mon[184174]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.233 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:26.256 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:25 smithi099 ceph-mon[167076]: pgmap v497: 119 pgs: 1 creating+peering, 7 remapped+peering, 1 peering, 1 down, 26 active+clean+remapped, 54 active+clean, 3 undersized+peered, 14 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 9 active+undersized+degraded; 11 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 42/399 objects degraded (10.526%); 55/399 objects misplaced (13.784%) 2023-09-23T18:01:26.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:25 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 42/399 objects degraded (10.526%), 10 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:26.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:25 smithi099 ceph-mon[167076]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:25 smithi099 ceph-mon[167076]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:25 smithi099 ceph-mon[167076]: osdmap e215: 8 total, 7 up, 7 in 2023-09-23T18:01:26.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:25 smithi099 ceph-mon[167076]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:25 smithi099 ceph-mon[167076]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:25 smithi099 ceph-mon[167076]: failed to encode map e215 with expected crc 2023-09-23T18:01:26.293 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:26.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:26.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:26.400 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:26.459 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:26.508 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:26.517 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:26.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:26.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:26.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:26.669 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:26.717 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:01:26.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:26.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:26.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:26.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:26.936 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:26.995 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 23 16 3964 3948 10.7271 220.562 0.00622152 0.0932031 2023-09-23T18:01:27.033 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:01:27.046 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:27.049 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:27 smithi099 ceph-mon[167076]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.049 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:27 smithi099 ceph-mon[167076]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.049 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:27 smithi099 ceph-mon[167076]: osdmap e216: 8 total, 7 up, 7 in 2023-09-23T18:01:27.049 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:27 smithi099 ceph-mon[167076]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.050 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:27 smithi099 ceph-mon[167076]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.050 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/748817305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:27.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:27.051 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:27.151 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:27.210 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:27.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[179668]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[179668]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[179668]: osdmap e216: 8 total, 7 up, 7 in 2023-09-23T18:01:27.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[179668]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[179668]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/748817305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:27.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[184174]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[184174]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[184174]: osdmap e216: 8 total, 7 up, 7 in 2023-09-23T18:01:27.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[184174]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[184174]: failed to encode map e216 with expected crc 2023-09-23T18:01:27.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/748817305' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:27.469 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:27.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:27.474 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:27.551 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:01:27.551 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:01:27.552 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:01:27.552 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:01:27.552 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:01:27.575 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:27.647 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:27.662 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:27.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:27.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:27.714 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:27.815 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:27.868 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:27.928 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:27.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:27.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:27.994 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 24 16 7349 7333 19.0943 211.562 0.0035861 0.0523649 2023-09-23T18:01:28.020 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[179668]: pgmap v500: 119 pgs: 1 creating+peering, 8 remapped+peering, 1 peering, 1 down, 26 active+clean+remapped, 55 active+clean, 1 undersized+peered, 15 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 8 active+undersized+degraded; 16 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.3 MiB/s wr, 20 op/s; 77/642 objects degraded (11.994%); 55/642 objects misplaced (8.567%) 2023-09-23T18:01:28.020 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T18:01:28.021 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[179668]: failed to encode map e217 with expected crc 2023-09-23T18:01:28.021 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[179668]: failed to encode map e217 with expected crc 2023-09-23T18:01:28.021 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/748817305' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:01:28.021 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[179668]: osdmap e217: 8 total, 7 up, 7 in 2023-09-23T18:01:28.021 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[179668]: 1.1a scrub starts 2023-09-23T18:01:28.023 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[184174]: pgmap v500: 119 pgs: 1 creating+peering, 8 remapped+peering, 1 peering, 1 down, 26 active+clean+remapped, 55 active+clean, 1 undersized+peered, 15 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 8 active+undersized+degraded; 16 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.3 MiB/s wr, 20 op/s; 77/642 objects degraded (11.994%); 55/642 objects misplaced (8.567%) 2023-09-23T18:01:28.023 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T18:01:28.024 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[184174]: failed to encode map e217 with expected crc 2023-09-23T18:01:28.024 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[184174]: failed to encode map e217 with expected crc 2023-09-23T18:01:28.024 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/748817305' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:01:28.024 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[184174]: osdmap e217: 8 total, 7 up, 7 in 2023-09-23T18:01:28.024 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[184174]: 1.1a scrub starts 2023-09-23T18:01:28.024 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[184174]: 1.1a scrub ok 2023-09-23T18:01:28.024 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:28.034 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:28.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:01:28.080 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:28.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[179668]: 1.1a scrub ok 2023-09-23T18:01:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:28 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:28.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:28 smithi099 ceph-mon[167076]: pgmap v500: 119 pgs: 1 creating+peering, 8 remapped+peering, 1 peering, 1 down, 26 active+clean+remapped, 55 active+clean, 1 undersized+peered, 15 active+undersized, 1 undersized+degraded+peered, 2 active+undersized+remapped, 8 active+undersized+degraded; 16 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 1.3 MiB/s wr, 20 op/s; 77/642 objects degraded (11.994%); 55/642 objects misplaced (8.567%) 2023-09-23T18:01:28.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:28 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 2 pgs inactive, 1 pg down (PG_AVAILABILITY) 2023-09-23T18:01:28.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:28 smithi099 ceph-mon[167076]: failed to encode map e217 with expected crc 2023-09-23T18:01:28.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:28 smithi099 ceph-mon[167076]: failed to encode map e217 with expected crc 2023-09-23T18:01:28.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:28 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/748817305' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:01:28.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:28 smithi099 ceph-mon[167076]: osdmap e217: 8 total, 7 up, 7 in 2023-09-23T18:01:28.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:28 smithi099 ceph-mon[167076]: 1.1a scrub starts 2023-09-23T18:01:28.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:28 smithi099 ceph-mon[167076]: 1.1a scrub ok 2023-09-23T18:01:28.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:28 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:28.380 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:28.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:28.384 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:28.472 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:01:28.472 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:01:28.472 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45092.0:8845 6.9 e217/214 6:989d55f2:::benchmark_data_smithi055_275963_object8844:head v 217'560, mlcod=217'557)", 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:28.469435+0000", 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0022099540000000001, 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022402849999999998, 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.469435+0000", 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:28.473 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.469435+0000", 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.469435+0000", 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.474 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.469470+0000", 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5343999999999999e-05 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.469477+0000", 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6309999999999997e-06 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:28.475 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.469482+0000", 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9690000000000002e-06 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.469859+0000", 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00037723900000000002 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.469880+0000", 2023-09-23T18:01:28.476 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0358999999999999e-05 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:8848 6.3 6:cbdcbf11:::benchmark_data_smithi055_275963_object8847:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:28.470084+0000", 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.001561612, 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016666789999999999, 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:01:28.477 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 8848 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470084+0000", 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.478 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470084+0000", 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470082+0000", 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470144+0000", 2023-09-23T18:01:28.479 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2011e-05 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470159+0000", 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5038000000000001e-05 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470166+0000", 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.725e-06 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.480 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470220+0000", 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3879999999999999e-05 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470416+0000", 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019649500000000001 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:01:28.481 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470565+0000", 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014813599999999999 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:8851 6.5 6:ac946e46:::benchmark_data_smithi055_275963_object8850:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:28.470183+0000", 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0014622649999999999, 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016477, 2023-09-23T18:01:28.482 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 8851 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470183+0000", 2023-09-23T18:01:28.483 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470183+0000", 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470182+0000", 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.484 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470216+0000", 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3575999999999998e-05 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470223+0000", 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0439999999999998e-06 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:28.485 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470228+0000", 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7600000000000002e-06 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470264+0000", 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6489999999999998e-05 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470399+0000", 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013486499999999999 2023-09-23T18:01:28.486 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470561+0000", 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016176499999999999 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:8854 6.5 6:a1fe92da:::benchmark_data_smithi055_275963_object8853:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e217)", 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:28.470240+0000", 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.00140503, 2023-09-23T18:01:28.487 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016632369999999999, 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 8854 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470240+0000", 2023-09-23T18:01:28.488 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470240+0000", 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470240+0000", 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470295+0000", 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.486e-05 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470302+0000", 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5340000000000002e-06 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:28.490 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.470307+0000", 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.515e-06 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471472+0000", 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001165045 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471599+0000", 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012685300000000001 2023-09-23T18:01:28.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471727+0000", 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012815299999999999 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45092.0:8849 6.9 e217/214 6:922dbf09:::benchmark_data_smithi055_275963_object8848:head v 217'561, mlcod=217'557)", 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:28.471444+0000", 2023-09-23T18:01:28.492 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.000201545, 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00053736699999999997, 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471444+0000", 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.493 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471444+0000", 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471443+0000", 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471478+0000", 2023-09-23T18:01:28.494 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4898999999999999e-05 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471488+0000", 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0774000000000001e-05 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471494+0000", 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0070000000000003e-06 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.495 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471609+0000", 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000114066 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:28.471625+0000", 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6560000000000001e-05 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.496 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:28.497 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:01:28.497 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 5 2023-09-23T18:01:28.497 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:01:28.497 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:28.553 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:28.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:28.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:28.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:28.733 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:28.791 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:28.995 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 25 16 10376 10360 25.8965 189.188 0.00656037 0.038606 2023-09-23T18:01:29.025 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:29.238 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:01:29.238 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:01:29.253 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:29.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:29.260 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:29.270 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:01:29.279 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:29 smithi099 ceph-mon[167076]: 1.12 scrub starts 2023-09-23T18:01:29.279 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:29 smithi099 ceph-mon[167076]: 1.12 scrub ok 2023-09-23T18:01:29.308 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:01:29.344 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:01:29.361 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:29.381 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:01:29.409 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:29.418 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:01:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:01:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:01:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:01:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:01:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:01:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:01:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:01:29.419 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:01:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:29 smithi055 ceph-mon[179668]: 1.12 scrub starts 2023-09-23T18:01:29.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:29 smithi055 ceph-mon[179668]: 1.12 scrub ok 2023-09-23T18:01:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:29 smithi055 ceph-mon[184174]: 1.12 scrub starts 2023-09-23T18:01:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:29 smithi055 ceph-mon[184174]: 1.12 scrub ok 2023-09-23T18:01:29.499 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:01:29.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:29.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:29.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:29.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:29.669 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:29.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:29.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:29.819 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:29.920 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:29.969 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:29.995 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 26 16 13416 13400 32.2073 190 0.00417263 0.0310422 2023-09-23T18:01:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:30 smithi099 ceph-mon[167076]: pgmap v502: 119 pgs: 1 creating+peering, 5 remapped+peering, 1 peering, 24 active+clean+remapped, 68 active+clean, 13 active+undersized, 7 active+undersized+degraded; 199 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 40 MiB/s wr, 643 op/s; 1057/9444 objects degraded (11.192%); 372/9444 objects misplaced (3.939%); 300 KiB/s, 4 objects/s recovering 2023-09-23T18:01:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:30 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg down) 2023-09-23T18:01:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2428266456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:30.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:30 smithi055 ceph-mon[184174]: pgmap v502: 119 pgs: 1 creating+peering, 5 remapped+peering, 1 peering, 24 active+clean+remapped, 68 active+clean, 13 active+undersized, 7 active+undersized+degraded; 199 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 40 MiB/s wr, 643 op/s; 1057/9444 objects degraded (11.192%); 372/9444 objects misplaced (3.939%); 300 KiB/s, 4 objects/s recovering 2023-09-23T18:01:30.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:30 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg down) 2023-09-23T18:01:30.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2428266456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:30.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:30 smithi055 ceph-mon[179668]: pgmap v502: 119 pgs: 1 creating+peering, 5 remapped+peering, 1 peering, 24 active+clean+remapped, 68 active+clean, 13 active+undersized, 7 active+undersized+degraded; 199 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 40 MiB/s wr, 643 op/s; 1057/9444 objects degraded (11.192%); 372/9444 objects misplaced (3.939%); 300 KiB/s, 4 objects/s recovering 2023-09-23T18:01:30.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:30 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg down) 2023-09-23T18:01:30.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2428266456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:30.456 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:01:30.456 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:01:30.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:01:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:01:30] "GET /metrics HTTP/1.1" 200 36513 "" "Prometheus/2.43.0" 2023-09-23T18:01:30.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:01:30.995 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 27 16 16613 16597 38.414 199.812 0.00368204 0.0260274 2023-09-23T18:01:31.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:31 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 2469/19983 objects degraded (12.356%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:31.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:31 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 2469/19983 objects degraded (12.356%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:31.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:31 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 2469/19983 objects degraded (12.356%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:31.722 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:31.995 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 28 16 19380 19364 43.2177 172.938 0.00741323 0.0231337 2023-09-23T18:01:32.047 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:32.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:32.054 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:32.093 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:01:32.093 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:01:32.155 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:32.207 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:32.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:32 smithi099 ceph-mon[167076]: pgmap v503: 119 pgs: 1 remapped+peering, 29 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 419 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 2469/19983 objects degraded (12.356%); 1703/19983 objects misplaced (8.522%); 262 KiB/s, 4 objects/s recovering 2023-09-23T18:01:32.336 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:32.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:32 smithi055 ceph-mon[184174]: pgmap v503: 119 pgs: 1 remapped+peering, 29 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 419 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 2469/19983 objects degraded (12.356%); 1703/19983 objects misplaced (8.522%); 262 KiB/s, 4 objects/s recovering 2023-09-23T18:01:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:32 smithi055 ceph-mon[179668]: pgmap v503: 119 pgs: 1 remapped+peering, 29 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 419 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 68 MiB/s wr, 1.08k op/s; 2469/19983 objects degraded (12.356%); 1703/19983 objects misplaced (8.522%); 262 KiB/s, 4 objects/s recovering 2023-09-23T18:01:32.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:32.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:32.539 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:32.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:32.691 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:32.752 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:32.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:32.761 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:32.861 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:32.913 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:32.996 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 29 16 21794 21778 46.928 150.875 0.00366173 0.0213045 2023-09-23T18:01:33.247 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:01:33.247 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:01:33.395 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:33.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:33.401 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:33.502 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:33.548 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:33.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:01:33.598 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:33.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:33.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:33.703 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:33.777 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:33.835 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 5] out_osds: [4] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2023-09-23T18:01:33.835 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:01:33.835 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T18:01:33.835 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T18:01:33.857 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:33.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:33.862 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:33.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:33.996 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 30 16 24120 24104 50.209 145.375 0.00719563 0.0199123 2023-09-23T18:01:34.011 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:34.070 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:34.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:34.075 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:34.094 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:01:34.156 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:34 smithi055 ceph-mon[179668]: pgmap v504: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 849 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 4461/40602 objects degraded (10.987%); 2634/40602 objects misplaced (6.487%); 205 KiB/s, 3 objects/s recovering 2023-09-23T18:01:34.156 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:34 smithi055 ceph-mon[184174]: pgmap v504: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 849 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 4461/40602 objects degraded (10.987%); 2634/40602 objects misplaced (6.487%); 205 KiB/s, 3 objects/s recovering 2023-09-23T18:01:34.176 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:34.252 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:34.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:34 smithi099 ceph-mon[167076]: pgmap v504: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 849 MiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 109 MiB/s wr, 1.75k op/s; 4461/40602 objects degraded (10.987%); 2634/40602 objects misplaced (6.487%); 205 KiB/s, 3 objects/s recovering 2023-09-23T18:01:34.334 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:34.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:34.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:34.400 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:34.440 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:34.504 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:34.630 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:34.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:34.637 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:34.738 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:34.788 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:34.856 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:34.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:34.862 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:34.873 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:34.963 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:34.996 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 31 16 26245 26229 52.8731 132.812 0.00612909 0.0189071 2023-09-23T18:01:35.035 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:35.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:35.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:35.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:35.192 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:35.254 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:35.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:35 smithi099 ceph-mon[167076]: 1.1c scrub starts 2023-09-23T18:01:35.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:35 smithi099 ceph-mon[167076]: 1.1c scrub ok 2023-09-23T18:01:35.352 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:35.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:35 smithi055 ceph-mon[184174]: 1.1c scrub starts 2023-09-23T18:01:35.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:35 smithi055 ceph-mon[184174]: 1.1c scrub ok 2023-09-23T18:01:35.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:35 smithi055 ceph-mon[179668]: 1.1c scrub starts 2023-09-23T18:01:35.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:35 smithi055 ceph-mon[179668]: 1.1c scrub ok 2023-09-23T18:01:35.523 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:35.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:35.529 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:35.629 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:35.679 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:35.996 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 32 16 28432 28416 55.4917 136.688 0.0145764 0.0180157 2023-09-23T18:01:36.080 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:36.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:36.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:36.147 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:01:36.188 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:36.250 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:36.279 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:01:36.280 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:01:36.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:36 smithi099 ceph-mon[167076]: pgmap v505: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 154 MiB/s wr, 2.46k op/s; 7927/64095 objects degraded (12.368%); 4307/64095 objects misplaced (6.720%); 173 KiB/s, 2 objects/s recovering 2023-09-23T18:01:36.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2401749893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:36.365 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:36 smithi055 ceph-mon[184174]: pgmap v505: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 154 MiB/s wr, 2.46k op/s; 7927/64095 objects degraded (12.368%); 4307/64095 objects misplaced (6.720%); 173 KiB/s, 2 objects/s recovering 2023-09-23T18:01:36.365 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2401749893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:36.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:36 smithi055 ceph-mon[179668]: pgmap v505: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.3 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 154 MiB/s wr, 2.46k op/s; 7927/64095 objects degraded (12.368%); 4307/64095 objects misplaced (6.720%); 173 KiB/s, 2 objects/s recovering 2023-09-23T18:01:36.365 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2401749893' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:36.510 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:36.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:36.515 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:36.616 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:36.677 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:36.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:01:36.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:36.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:36.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:36.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:36.913 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:36.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:36.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:36.987 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:37.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 33 16 30573 30557 57.8585 133.812 0.00301842 0.017279 2023-09-23T18:01:37.087 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:37.158 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:37.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:01:37.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[184174]: failed to encode map e218 with expected crc 2023-09-23T18:01:37.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2401749893' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:37.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[184174]: osdmap e218: 8 total, 7 up, 7 in 2023-09-23T18:01:37.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[184174]: failed to encode map e218 with expected crc 2023-09-23T18:01:37.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:37.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:37.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:01:37.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[179668]: failed to encode map e218 with expected crc 2023-09-23T18:01:37.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2401749893' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:37.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[179668]: osdmap e218: 8 total, 7 up, 7 in 2023-09-23T18:01:37.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[179668]: failed to encode map e218 with expected crc 2023-09-23T18:01:37.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:37.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:37.226 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:37.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:37.231 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:37.281 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:01:37.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:37 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:01:37.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:37 smithi099 ceph-mon[167076]: failed to encode map e218 with expected crc 2023-09-23T18:01:37.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2401749893' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:37.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:37 smithi099 ceph-mon[167076]: osdmap e218: 8 total, 7 up, 7 in 2023-09-23T18:01:37.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:37 smithi099 ceph-mon[167076]: failed to encode map e218 with expected crc 2023-09-23T18:01:37.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:37.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:37.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:37.334 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:01:37.372 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:01:37.383 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:37.419 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:01:37.444 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:37.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:37.449 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:37.456 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:01:37.490 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:01:37.516 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:37.527 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:01:37.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:01:37.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:01:37.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:01:37.527 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:01:37.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:01:37.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:01:37.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:01:37.528 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:01:37.550 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:37.617 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:37.682 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:37.687 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:37.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:37.851 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:37.915 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:37.950 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:37 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:01:37.552+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e218 definitely_dead 0 2023-09-23T18:01:37.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:37.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:38.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 34 16 32588 32572 59.8602 125.938 0.00587116 0.0167011 2023-09-23T18:01:38.029 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:01:38.030 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:01:38.051 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:38.102 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:38.161 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:38.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:38.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:38.265 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:38.309 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:38 smithi055 ceph-mon[179668]: pgmap v507: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.5 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 157 MiB/s wr, 2.50k op/s; 9389/72864 objects degraded (12.886%); 4307/72864 objects misplaced (5.911%); 155 KiB/s, 2 objects/s recovering 2023-09-23T18:01:38.309 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2653685361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:38.309 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 9389/72864 objects degraded (12.886%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:38.309 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/162152013' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T18:01:38.309 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:38 smithi055 ceph-mon[184174]: pgmap v507: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.5 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 157 MiB/s wr, 2.50k op/s; 9389/72864 objects degraded (12.886%); 4307/72864 objects misplaced (5.911%); 155 KiB/s, 2 objects/s recovering 2023-09-23T18:01:38.309 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2653685361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:38.310 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 9389/72864 objects degraded (12.886%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:38.310 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/162152013' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T18:01:38.314 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:38.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:38 smithi099 ceph-mon[167076]: pgmap v507: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.5 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 157 MiB/s wr, 2.50k op/s; 9389/72864 objects degraded (12.886%); 4307/72864 objects misplaced (5.911%); 155 KiB/s, 2 objects/s recovering 2023-09-23T18:01:38.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2653685361' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:38.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 9389/72864 objects degraded (12.886%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:38.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/162152013' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T18:01:38.370 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:38.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:38.378 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:38.439 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.4. 2023-09-23T18:01:38.478 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:38.506 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:38.582 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:38.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:38.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:38.689 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:38.715 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:01:38.717 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:38.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:38.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:38.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:38.847 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T18:01:38.876 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:01:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:01:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:01:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:01:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131500 5.2 5.9981e372 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T18:01:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.276336+0000", 2023-09-23T18:01:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 184.59735931200001, 2023-09-23T18:01:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94858491499999997, 2023-09-23T18:01:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:01:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131500 2023-09-23T18:01:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276336+0000", 2023-09-23T18:01:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276336+0000", 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276335+0000", 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276340+0000", 2023-09-23T18:01:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2310000000000004e-06 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276348+0000", 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.303e-06 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276356+0000", 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9359999999999999e-06 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276427+0000", 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0314000000000005e-05 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276438+0000", 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1035e-05 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224903+0000", 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94846568499999995 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224921+0000", 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7461000000000001e-05 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131508 5.2 5.89ad37e2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.286747+0000", 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 184.58694859299999, 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.93821040499999997, 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131508 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286747+0000", 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286747+0000", 2023-09-23T18:01:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286745+0000", 2023-09-23T18:01:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286752+0000", 2023-09-23T18:01:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0550000000000002e-06 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286767+0000", 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4283e-05 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286775+0000", 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1780000000000003e-06 2023-09-23T18:01:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286834+0000", 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8866999999999997e-05 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286842+0000", 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7590000000000003e-06 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224948+0000", 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.938105666 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224957+0000", 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9469999999999992e-06 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131523 5.2 5.551f4f92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.311020+0000", 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 184.562675093, 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91399190200000002, 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131523 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311020+0000", 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311020+0000", 2023-09-23T18:01:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311018+0000", 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311028+0000", 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5859999999999994e-06 2023-09-23T18:01:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311050+0000", 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2079e-05 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311059+0000", 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9959999999999999e-06 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311123+0000", 2023-09-23T18:01:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4539e-05 2023-09-23T18:01:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311130+0000", 2023-09-23T18:01:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.917e-06 2023-09-23T18:01:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.225002+0000", 2023-09-23T18:01:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91387170399999995 2023-09-23T18:01:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.225012+0000", 2023-09-23T18:01:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0132000000000001e-05 2023-09-23T18:01:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142717 5.9 5:97ac1ed5:::benchmark_data_smithi055_229117_object37091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.884415+0000", 2023-09-23T18:01:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.98928041400001, 2023-09-23T18:01:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0500839660000001, 2023-09-23T18:01:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142717 2023-09-23T18:01:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884415+0000", 2023-09-23T18:01:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884415+0000", 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884413+0000", 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884424+0000", 2023-09-23T18:01:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0804e-05 2023-09-23T18:01:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884444+0000", 2023-09-23T18:01:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0057000000000001e-05 2023-09-23T18:01:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884453+0000", 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8000000000000004e-06 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884524+0000", 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0729000000000007e-05 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884537+0000", 2023-09-23T18:01:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3059e-05 2023-09-23T18:01:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903080+0000", 2023-09-23T18:01:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0185432940000001 2023-09-23T18:01:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903187+0000", 2023-09-23T18:01:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010691700000000001 2023-09-23T18:01:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903271+0000", 2023-09-23T18:01:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4103000000000004e-05 2023-09-23T18:01:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.911979+0000", 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0087084599999999995 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934336+0000", 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022356767999999999 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934371+0000", 2023-09-23T18:01:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4693999999999997e-05 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934420+0000", 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8755000000000003e-05 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934499+0000", 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9129999999999996e-05 2023-09-23T18:01:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142734 5.9 5:9b1dd87e:::benchmark_data_smithi055_229117_object37108:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.916812+0000", 2023-09-23T18:01:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.956883283, 2023-09-23T18:01:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0179610699999999, 2023-09-23T18:01:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142734 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916812+0000", 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916812+0000", 2023-09-23T18:01:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916810+0000", 2023-09-23T18:01:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916818+0000", 2023-09-23T18:01:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7209999999999994e-06 2023-09-23T18:01:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916838+0000", 2023-09-23T18:01:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9941000000000001e-05 2023-09-23T18:01:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916846+0000", 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5190000000000003e-06 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917145+0000", 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00029908799999999998 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917153+0000", 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7700000000000007e-06 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.904847+0000", 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98769368300000004 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.904960+0000", 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112802 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.905068+0000", 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010848700000000001 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915793+0000", 2023-09-23T18:01:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010724450999999999 2023-09-23T18:01:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934547+0000", 2023-09-23T18:01:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018754603000000002 2023-09-23T18:01:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934612+0000", 2023-09-23T18:01:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4272999999999999e-05 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934667+0000", 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4814000000000001e-05 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934773+0000", 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106517 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142738 5.9 5:9dbd7dd5:::benchmark_data_smithi055_229117_object37112:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:01:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.95683376100001, 2023-09-23T18:01:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018182892, 2023-09-23T18:01:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142738 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916864+0000", 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0709999999999999e-06 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916870+0000", 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8780000000000003e-06 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916882+0000", 2023-09-23T18:01:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2558000000000001e-05 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917176+0000", 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00029325599999999999 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917181+0000", 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6400000000000002e-06 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906621+0000", 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98943947600000004 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906744+0000", 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012348099999999999 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906869+0000", 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012421599999999999 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915800+0000", 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0089313259999999998 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934813+0000", 2023-09-23T18:01:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019012940999999998 2023-09-23T18:01:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934959+0000", 2023-09-23T18:01:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000145921 2023-09-23T18:01:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934994+0000", 2023-09-23T18:01:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5707999999999998e-05 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935044+0000", 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9951000000000003e-05 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142777 5.9 5:91082f54:::benchmark_data_smithi055_229117_object37151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.976270+0000", 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.89742512800001, 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.958897997, 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142777 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976270+0000", 2023-09-23T18:01:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976270+0000", 2023-09-23T18:01:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976269+0000", 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976276+0000", 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2150000000000004e-06 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976288+0000", 2023-09-23T18:01:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2092999999999999e-05 2023-09-23T18:01:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976293+0000", 2023-09-23T18:01:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1610000000000002e-06 2023-09-23T18:01:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976338+0000", 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4116000000000002e-05 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976346+0000", 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9200000000000004e-06 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908348+0000", 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.93200213499999995 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908468+0000", 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120746 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908590+0000", 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121772 2023-09-23T18:01:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915806+0000", 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0072161990000000004 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934909+0000", 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019102811000000001 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935083+0000", 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000173499 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935114+0000", 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1501999999999998e-05 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935168+0000", 2023-09-23T18:01:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3947999999999999e-05 2023-09-23T18:01:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142802 5.9 5:92d6f126:::benchmark_data_smithi055_229117_object37176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.030542+0000", 2023-09-23T18:01:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.843153081, 2023-09-23T18:01:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.90473668699999998, 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142802 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030542+0000", 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030542+0000", 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030541+0000", 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030547+0000", 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5289999999999997e-06 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030566+0000", 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8630999999999998e-05 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030571+0000", 2023-09-23T18:01:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0259999999999998e-06 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030617+0000", 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5439999999999999e-05 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030623+0000", 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1999999999999999e-06 2023-09-23T18:01:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910103+0000", 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.87947978999999998 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910251+0000", 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014896700000000001 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910377+0000", 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125425 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920175+0000", 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097978730000000003 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934939+0000", 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014764019 2023-09-23T18:01:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935202+0000", 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026282399999999999 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935229+0000", 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7793999999999999e-05 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935279+0000", 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9428999999999997e-05 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142809 5.9 5:9f3ab830:::benchmark_data_smithi055_229117_object37183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.039995+0000", 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.83370014499999, 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.902436291, 2023-09-23T18:01:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142809 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039995+0000", 2023-09-23T18:01:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039995+0000", 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039994+0000", 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040001+0000", 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5259999999999996e-06 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040017+0000", 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5794000000000002e-05 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040024+0000", 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3379999999999999e-06 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040079+0000", 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4866e-05 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040091+0000", 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1362000000000001e-05 2023-09-23T18:01:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912030+0000", 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.87193905100000002 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912160+0000", 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013052100000000001 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912267+0000", 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010700399999999999 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920180+0000", 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079123509999999998 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935312+0000", 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01513201 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942274+0000", 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069623669999999997 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942331+0000", 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7516000000000003e-05 2023-09-23T18:01:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942431+0000", 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9986000000000001e-05 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142838 5.9 5:9cf2a92c:::benchmark_data_smithi055_229117_object37212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.111228+0000", 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.76246740900001, 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.83141186199999995, 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142838 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111228+0000", 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111228+0000", 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111227+0000", 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111233+0000", 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3160000000000002e-06 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111246+0000", 2023-09-23T18:01:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2884000000000001e-05 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111251+0000", 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2360000000000004e-06 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111297+0000", 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5881000000000003e-05 2023-09-23T18:01:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111307+0000", 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0046e-05 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914054+0000", 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.80274739399999995 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914133+0000", 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8448999999999994e-05 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914206+0000", 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3325999999999996e-05 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925627+0000", 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011421277000000001 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935332+0000", 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097046300000000005 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942488+0000", 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007155603 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942537+0000", 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9391000000000002e-05 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942640+0000", 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102618 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142868 5.9 5:93088e3b:::benchmark_data_smithi055_229117_object37242:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.164405+0000", 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.70928985399999, 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.78228625900000004, 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142868 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164405+0000", 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164405+0000", 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164404+0000", 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164410+0000", 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1349999999999997e-06 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164422+0000", 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1647e-05 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164437+0000", 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5234e-05 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164485+0000", 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7363e-05 2023-09-23T18:01:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164493+0000", 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5429999999999994e-06 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915840+0000", 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.75134721800000004 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915919+0000", 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8091999999999997e-05 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915992+0000", 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3733999999999996e-05 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925638+0000", 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096452810000000003 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935357+0000", 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097193910000000008 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946534+0000", 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011177302 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946593+0000", 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8563000000000002e-05 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946692+0000", 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8836e-05 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142887 5.9 5:980e2e89:::benchmark_data_smithi055_229117_object37261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.211074+0000", 2023-09-23T18:01:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.66262077100001, 2023-09-23T18:01:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73576580700000005, 2023-09-23T18:01:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142887 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211074+0000", 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211074+0000", 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211073+0000", 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211080+0000", 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3629999999999996e-06 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211097+0000", 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6917000000000001e-05 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211103+0000", 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2499999999999997e-06 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211145+0000", 2023-09-23T18:01:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2466000000000003e-05 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211151+0000", 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0410000000000003e-06 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918068+0000", 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70691734399999995 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918196+0000", 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000128081 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918320+0000", 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012403300000000001 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930458+0000", 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012137652000000001 2023-09-23T18:01:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935374+0000", 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049156570000000004 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946735+0000", 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011361616999999999 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946767+0000", 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1952999999999997e-05 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946840+0000", 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2910000000000005e-05 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142888 5.9 5:99a82285:::benchmark_data_smithi055_229117_object37262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.662580956, 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73588783199999996, 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142888 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211117+0000", 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4680000000000001e-06 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211121+0000", 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7610000000000001e-06 2023-09-23T18:01:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211124+0000", 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3489999999999999e-06 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211177+0000", 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3133000000000001e-05 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211184+0000", 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6240000000000002e-06 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920219+0000", 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70903441 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920327+0000", 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108477 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920402+0000", 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5252000000000004e-05 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930467+0000", 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010064867999999999 2023-09-23T18:01:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.937129+0000", 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066614200000000004 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946878+0000", 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097489120000000002 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946919+0000", 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1031000000000001e-05 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.947002+0000", 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3475999999999998e-05 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142904 5.9 5:9b2141c3:::benchmark_data_smithi055_229117_object37278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.244955+0000", 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.628739895, 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70391956499999997, 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142904 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.979 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244955+0000", 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244955+0000", 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244954+0000", 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244960+0000", 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244973+0000", 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3209e-05 2023-09-23T18:01:38.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244978+0000", 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1930000000000001e-06 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245017+0000", 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8452999999999999e-05 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245023+0000", 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.409e-06 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.921896+0000", 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67687305900000005 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.921997+0000", 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100753 2023-09-23T18:01:38.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.922064+0000", 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6978999999999997e-05 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930475+0000", 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0084114670000000006 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.940046+0000", 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095709799999999998 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948781+0000", 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0087346990000000003 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948813+0000", 2023-09-23T18:01:38.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1779000000000002e-05 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948875+0000", 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2073e-05 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142905 5.9 5:951a98af:::benchmark_data_smithi055_229117_object37279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.986 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.62836602600001, 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71295564, 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142905 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:01:38.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245332+0000", 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4400000000000001e-06 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245339+0000", 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4749999999999998e-06 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245343+0000", 2023-09-23T18:01:38.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1370000000000004e-06 2023-09-23T18:01:38.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245381+0000", 2023-09-23T18:01:38.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8396999999999998e-05 2023-09-23T18:01:38.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245386+0000", 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2460000000000003e-06 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923706+0000", 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67831945699999996 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923849+0000", 2023-09-23T18:01:38.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143669 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923977+0000", 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012721600000000001 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933852+0000", 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0098750190000000005 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.940069+0000", 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062175399999999997 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958137+0000", 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018067588999999998 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:38.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958189+0000", 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1981000000000003e-05 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958285+0000", 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6036e-05 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142919 5.9 5:9348e633:::benchmark_data_smithi055_229117_object37293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.270585+0000", 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.603110394, 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.69084338599999995, 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142919 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.995 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270585+0000", 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270585+0000", 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:38.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270583+0000", 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270591+0000", 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2910000000000002e-06 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270604+0000", 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3349999999999999e-05 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270609+0000", 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1109999999999999e-06 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270662+0000", 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3236000000000003e-05 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:38.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270673+0000", 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0560999999999999e-05 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925683+0000", 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.65501024500000005 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925803+0000", 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119807 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:38.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925920+0000", 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117419 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933861+0000", 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079407209999999995 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:39.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944507+0000", 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010645846 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961259+0000", 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016751986999999999 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961315+0000", 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6477999999999997e-05 2023-09-23T18:01:39.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961428+0000", 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011292400000000001 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142929 5.9 5:954e3e29:::benchmark_data_smithi055_229117_object37303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.300689+0000", 2023-09-23T18:01:39.002 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.573006679, 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.662469262, 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142929 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300689+0000", 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300689+0000", 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300687+0000", 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:01:39.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300694+0000", 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0219999999999999e-06 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300705+0000", 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0886e-05 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300711+0000", 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9490000000000001e-06 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300749+0000", 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8077999999999997e-05 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:39.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300755+0000", 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5389999999999998e-06 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928440+0000", 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62768513800000003 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928576+0000", 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013626900000000001 2023-09-23T18:01:39.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928683+0000", 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010661400000000001 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933946+0000", 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052633460000000003 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944539+0000", 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010592571 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963005+0000", 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018466598000000001 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963057+0000", 2023-09-23T18:01:39.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1529000000000001e-05 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963158+0000", 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010085399999999999 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142941 5.9 5:913fcfd6:::benchmark_data_smithi055_229117_object37315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:39.010 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.340683+0000", 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.53301234200001, 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62261945900000004, 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142941 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340683+0000", 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340683+0000", 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340682+0000", 2023-09-23T18:01:39.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340688+0000", 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.319e-06 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340701+0000", 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3529999999999999e-05 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340708+0000", 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8449999999999997e-06 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340755+0000", 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6428999999999999e-05 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340762+0000", 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.571e-06 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930510+0000", 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58974757200000005 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930615+0000", 2023-09-23T18:01:39.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000105213 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930695+0000", 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0055000000000005e-05 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933952+0000", 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003257228 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944608+0000", 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010655243 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963204+0000", 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018596950000000001 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:39.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963236+0000", 2023-09-23T18:01:39.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1894000000000003e-05 2023-09-23T18:01:39.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:39.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963302+0000", 2023-09-23T18:01:39.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6013999999999995e-05 2023-09-23T18:01:39.018 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.018 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:39.018 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142946 5.9 5:9099fb2d:::benchmark_data_smithi055_229117_object37320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.370819+0000", 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 166.50287625199999, 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59265797899999995, 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142946 2023-09-23T18:01:39.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370819+0000", 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370819+0000", 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370817+0000", 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370824+0000", 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8299999999999998e-06 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370837+0000", 2023-09-23T18:01:39.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2716e-05 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370844+0000", 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8789999999999997e-06 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370887+0000", 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2879000000000001e-05 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:01:39.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370893+0000", 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1120000000000001e-06 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932242+0000", 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.56134879999999998 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932323+0000", 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0871000000000005e-05 2023-09-23T18:01:39.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932392+0000", 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9295999999999995e-05 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934346+0000", 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019539589999999999 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:39.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944640+0000", 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010293754 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963339+0000", 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018699684000000001 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963384+0000", 2023-09-23T18:01:39.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4499999999999997e-05 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963477+0000", 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3245000000000006e-05 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:172845 5.9 5:99e32625:::benchmark_data_smithi055_229117_object67219:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e184)", 2023-09-23T18:01:39.026 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:42.052464+0000", 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 116.821231524, 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.074629107, 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 172845 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:39.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052464+0000", 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052464+0000", 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052462+0000", 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052470+0000", 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.559e-06 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052486+0000", 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6611000000000001e-05 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:39.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.052493+0000", 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0380000000000002e-06 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.053731+0000", 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012380329999999999 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.053877+0000", 2023-09-23T18:01:39.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014536799999999999 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.054023+0000", 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014621600000000001 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.055833+0000", 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001809969 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.060974+0000", 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051409100000000003 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.126977+0000", 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.066003662000000005 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.127023+0000", 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5671999999999999e-05 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:42.127093+0000", 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9688000000000001e-05 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:39.033 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:39.034 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:01:39.034 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:39.034 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 35 16 33562 33546 59.8889 60.875 0.00223435 0.0164273 2023-09-23T18:01:39.061 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:39.278 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:39.280 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:39 smithi099 ceph-mon[167076]: 1.10 scrub starts 2023-09-23T18:01:39.280 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:39 smithi099 ceph-mon[167076]: 1.10 scrub ok 2023-09-23T18:01:39.280 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/162152013' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T18:01:39.280 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:39 smithi099 ceph-mon[167076]: osdmap e219: 8 total, 7 up, 8 in 2023-09-23T18:01:39.280 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:39.281 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:39.316 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:39.319 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:01:39.382 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:39.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:39 smithi055 ceph-mon[184174]: 1.10 scrub starts 2023-09-23T18:01:39.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:39 smithi055 ceph-mon[184174]: 1.10 scrub ok 2023-09-23T18:01:39.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/162152013' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T18:01:39.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:39 smithi055 ceph-mon[184174]: osdmap e219: 8 total, 7 up, 8 in 2023-09-23T18:01:39.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:39 smithi055 ceph-mon[179668]: 1.10 scrub starts 2023-09-23T18:01:39.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:39 smithi055 ceph-mon[179668]: 1.10 scrub ok 2023-09-23T18:01:39.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/162152013' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T18:01:39.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:39 smithi055 ceph-mon[179668]: osdmap e219: 8 total, 7 up, 8 in 2023-09-23T18:01:39.435 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:39.487 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:39.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:39.491 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:39.592 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:39.618 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:39.650 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:39.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:39.653 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:39.754 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:39.783 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:39.981 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:40.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 36 16 33562 33546 58.2256 0 - 0.0164273 2023-09-23T18:01:40.031 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:01:40.185 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:40.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:40.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:40.289 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:40.315 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[179668]: failed to encode map e219 with expected crc 2023-09-23T18:01:40.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[179668]: pgmap v509: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.8 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 174 MiB/s wr, 2.79k op/s; 10747/86925 objects degraded (12.364%); 5010/86925 objects misplaced (5.764%) 2023-09-23T18:01:40.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[179668]: 1.1b scrub starts 2023-09-23T18:01:40.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[179668]: 1.1b scrub ok 2023-09-23T18:01:40.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[179668]: osdmap e220: 8 total, 7 up, 8 in 2023-09-23T18:01:40.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[179668]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[179668]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[179668]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[179668]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[179668]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.316 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[184174]: failed to encode map e219 with expected crc 2023-09-23T18:01:40.317 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[184174]: pgmap v509: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.8 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 174 MiB/s wr, 2.79k op/s; 10747/86925 objects degraded (12.364%); 5010/86925 objects misplaced (5.764%) 2023-09-23T18:01:40.317 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[184174]: 1.1b scrub starts 2023-09-23T18:01:40.317 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[184174]: 1.1b scrub ok 2023-09-23T18:01:40.317 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[184174]: osdmap e220: 8 total, 7 up, 8 in 2023-09-23T18:01:40.317 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[184174]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.317 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[184174]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.317 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[184174]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.317 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[184174]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.317 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-mon[184174]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.319 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:40.354 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:40.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:40.356 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:40.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:40.500 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:40 smithi099 ceph-mon[167076]: failed to encode map e219 with expected crc 2023-09-23T18:01:40.500 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:40 smithi099 ceph-mon[167076]: pgmap v509: 119 pgs: 30 active+clean+remapped, 72 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.8 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 174 MiB/s wr, 2.79k op/s; 10747/86925 objects degraded (12.364%); 5010/86925 objects misplaced (5.764%) 2023-09-23T18:01:40.500 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:40 smithi099 ceph-mon[167076]: 1.1b scrub starts 2023-09-23T18:01:40.500 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:40 smithi099 ceph-mon[167076]: 1.1b scrub ok 2023-09-23T18:01:40.500 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:40 smithi099 ceph-mon[167076]: osdmap e220: 8 total, 7 up, 8 in 2023-09-23T18:01:40.500 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:40 smithi099 ceph-mon[167076]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.500 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:40 smithi099 ceph-mon[167076]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.500 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:40 smithi099 ceph-mon[167076]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:40 smithi099 ceph-mon[167076]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:40 smithi099 ceph-mon[167076]: failed to encode map e220 with expected crc 2023-09-23T18:01:40.503 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:40.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:40.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:40.550 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:40.651 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:40.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:01:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:01:40] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-09-23T18:01:40.714 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:40.771 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:40.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:40.776 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:40.837 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:40.877 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:40.926 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:41.003 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 37 16 34446 34430 58.141 27.625 0.00682037 0.0171941 2023-09-23T18:01:41.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:41 smithi099 ceph-mon[167076]: failed to encode map e220 with expected crc 2023-09-23T18:01:41.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:41 smithi099 ceph-mon[167076]: osdmap e221: 8 total, 7 up, 8 in 2023-09-23T18:01:41.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:41 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2849243558' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:41.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:41 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:41.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:41 smithi055 ceph-mon[184174]: failed to encode map e220 with expected crc 2023-09-23T18:01:41.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:41 smithi055 ceph-mon[184174]: osdmap e221: 8 total, 7 up, 8 in 2023-09-23T18:01:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:41 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2849243558' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:41 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:41 smithi055 ceph-mon[179668]: failed to encode map e220 with expected crc 2023-09-23T18:01:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:41 smithi055 ceph-mon[179668]: osdmap e221: 8 total, 7 up, 8 in 2023-09-23T18:01:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:41 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2849243558' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:41 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:01:41.455 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:01:41.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:41.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:41.489 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:41.590 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:41.657 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:41.932 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:41.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:41.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:42.003 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 38 16 36284 36268 59.6334 114.875 0.00757027 0.016763 2023-09-23T18:01:42.043 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:42.101 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:42.163 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:42.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:42.170 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:42.170 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[179668]: failed to encode map e221 with expected crc 2023-09-23T18:01:42.170 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[179668]: pgmap v512: 119 pgs: 6 remapped+peering, 23 active+clean+remapped, 73 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s; 11781/94944 objects degraded (12.408%); 1991/94944 objects misplaced (2.097%) 2023-09-23T18:01:42.170 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:42.170 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:01:42.171 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[179668]: osdmap e222: 8 total, 7 up, 8 in 2023-09-23T18:01:42.171 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[179668]: failed to encode map e222 with expected crc 2023-09-23T18:01:42.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[184174]: failed to encode map e221 with expected crc 2023-09-23T18:01:42.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[184174]: pgmap v512: 119 pgs: 6 remapped+peering, 23 active+clean+remapped, 73 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s; 11781/94944 objects degraded (12.408%); 1991/94944 objects misplaced (2.097%) 2023-09-23T18:01:42.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:42.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:01:42.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[184174]: osdmap e222: 8 total, 7 up, 8 in 2023-09-23T18:01:42.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:42 smithi055 ceph-mon[184174]: failed to encode map e222 with expected crc 2023-09-23T18:01:42.271 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:42.332 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:42.415 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:42 smithi099 ceph-mon[167076]: failed to encode map e221 with expected crc 2023-09-23T18:01:42.415 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:42 smithi099 ceph-mon[167076]: pgmap v512: 119 pgs: 6 remapped+peering, 23 active+clean+remapped, 73 active+clean, 9 active+undersized, 8 active+undersized+degraded; 1.9 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 102 MiB/s wr, 1.63k op/s; 11781/94944 objects degraded (12.408%); 1991/94944 objects misplaced (2.097%) 2023-09-23T18:01:42.416 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:42 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:42.416 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:42 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:01:42.416 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:42 smithi099 ceph-mon[167076]: osdmap e222: 8 total, 7 up, 8 in 2023-09-23T18:01:42.416 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:42 smithi099 ceph-mon[167076]: failed to encode map e222 with expected crc 2023-09-23T18:01:42.446 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:42.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:42.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:42.552 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:42.603 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:42.617 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:01:42.651 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:01:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:01:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:42.651 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45092.0:37405 6.9 e222/214 6:9d9b7e7d:::benchmark_data_smithi055_275963_object37404:head v 222'2407, mlcod=222'2405)", 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:42.650278+0000", 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0012422970000000001, 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012706530000000001, 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:42.650278+0000", 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:42.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:01:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:42.650278+0000", 2023-09-23T18:01:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:01:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:01:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:42.650278+0000", 2023-09-23T18:01:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:01:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:42.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:42.650326+0000", 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7998999999999999e-05 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:42.650336+0000", 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0127e-05 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:42.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:42.650340+0000", 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1899999999999997e-06 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:42.650395+0000", 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4879000000000001e-05 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:01:42.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:42.650421+0000", 2023-09-23T18:01:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6018000000000001e-05 2023-09-23T18:01:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:01:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:01:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:01:42.656 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 1 2023-09-23T18:01:42.656 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:01:42.658 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:01:42.665 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:42.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:42.670 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:42.695 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:01:42.733 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:01:42.767 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:01:42.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:42.805 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:01:42.829 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:42.849 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:01:42.849 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:01:42.849 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:01:42.849 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:01:42.849 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:01:42.849 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:01:42.850 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:01:42.850 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:01:42.850 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:01:43.004 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 39 16 38032 38016 60.904 109.25 0.00464658 0.0164153 2023-09-23T18:01:43.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:43.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:43.077 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:43.103 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:01:43.156 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:43 smithi055 ceph-mon[179668]: failed to encode map e222 with expected crc 2023-09-23T18:01:43.156 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/826920126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:43.156 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 11781/94944 objects degraded (12.408%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:43.178 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:43.203 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:01:43.204 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:01:43.269 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:43 smithi099 ceph-mon[167076]: failed to encode map e222 with expected crc 2023-09-23T18:01:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/826920126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 11781/94944 objects degraded (12.408%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:43.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:43.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:43.354 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:43.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:43 smithi055 ceph-mon[184174]: failed to encode map e222 with expected crc 2023-09-23T18:01:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/826920126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 11781/94944 objects degraded (12.408%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:43.455 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:43.502 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:43.563 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:43.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:43.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:43.670 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:43.733 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:43.797 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:43.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:43.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:43.847 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 5, 4] out_osds: [] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2023-09-23T18:01:43.848 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:01:43.848 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T18:01:43.848 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 1.000000 2023-09-23T18:01:43.848 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T18:01:43.902 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:43.968 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:43.974 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:44.004 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:01:44.005140+0000 min lat: 0.00103291 max lat: 21.0159 avg lat: 0.0160838 2023-09-23T18:01:44.004 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:01:44.004 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 40 16 39809 39793 62.1575 111.062 0.00429935 0.0160838 2023-09-23T18:01:44.021 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:01:44.030 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:44.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:44.038 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:44.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:44.161 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:44 smithi055 ceph-mon[179668]: pgmap v514: 119 pgs: 5 remapped+peering, 21 active+clean+remapped, 75 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 12596/104271 objects degraded (12.080%); 9058/104271 objects misplaced (8.687%); 371 KiB/s, 7 objects/s recovering 2023-09-23T18:01:44.185 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:44.237 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:44.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:44.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:44.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:44.403 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:44.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:44 smithi055 ceph-mon[184174]: pgmap v514: 119 pgs: 5 remapped+peering, 21 active+clean+remapped, 75 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 12596/104271 objects degraded (12.080%); 9058/104271 objects misplaced (8.687%); 371 KiB/s, 7 objects/s recovering 2023-09-23T18:01:44.464 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:44.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:44 smithi099 ceph-mon[167076]: pgmap v514: 119 pgs: 5 remapped+peering, 21 active+clean+remapped, 75 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.1 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 86 MiB/s wr, 1.37k op/s; 12596/104271 objects degraded (12.080%); 9058/104271 objects misplaced (8.687%); 371 KiB/s, 7 objects/s recovering 2023-09-23T18:01:44.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:44.469 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:44.570 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:44.624 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:44.659 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:44.683 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:44.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:44.690 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:44.791 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:44.839 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:45.004 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 41 16 41449 41433 63.141 102.5 0.00794003 0.0158322 2023-09-23T18:01:45.205 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:01:45.373 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:01:45.373 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:01:45.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:01:46.004 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 42 16 43122 43106 64.1266 104.562 0.00375074 0.0155885 2023-09-23T18:01:46.024 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:46.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:46.029 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:46.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:46.192 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:46.257 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:46.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:46.269 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:46.292 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:46.370 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:46.418 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:46.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:46 smithi055 ceph-mon[184174]: pgmap v515: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 14105/112287 objects degraded (12.562%); 11511/112287 objects misplaced (10.251%); 271 KiB/s, 1 keys/s, 7 objects/s recovering 2023-09-23T18:01:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:46 smithi055 ceph-mon[179668]: pgmap v515: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 14105/112287 objects degraded (12.562%); 11511/112287 objects misplaced (10.251%); 271 KiB/s, 1 keys/s, 7 objects/s recovering 2023-09-23T18:01:46.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:46.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:46.490 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:46.493 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:46 smithi099 ceph-mon[167076]: pgmap v515: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.3 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 14105/112287 objects degraded (12.562%); 11511/112287 objects misplaced (10.251%); 271 KiB/s, 1 keys/s, 7 objects/s recovering 2023-09-23T18:01:46.591 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:46.605 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:01:46.660 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:46.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:46.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:46.762 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:46.863 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:46.912 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:47.004 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 43 16 44752 44736 65.0041 101.875 0.00617034 0.0153796 2023-09-23T18:01:47.080 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:47.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:47.087 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:47.187 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:47.233 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:47.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:47.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:47.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:47.538 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:47.557 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:47.635 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:47.831 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:47.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:47.835 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:47.936 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:47.999 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:48.004 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 44 16 46334 46318 65.7735 98.875 0.00905304 0.0151979 2023-09-23T18:01:48.056 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:48.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:48.061 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:48.110 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:01:48.110 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:01:48.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:48.226 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:48 smithi055 ceph-mon[179668]: pgmap v516: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 15204/119127 objects degraded (12.763%); 11209/119127 objects misplaced (9.409%); 2.8 MiB/s, 1 keys/s, 47 objects/s recovering 2023-09-23T18:01:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15204/119127 objects degraded (12.763%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:48 smithi055 ceph-mon[184174]: pgmap v516: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 15204/119127 objects degraded (12.763%); 11209/119127 objects misplaced (9.409%); 2.8 MiB/s, 1 keys/s, 47 objects/s recovering 2023-09-23T18:01:48.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15204/119127 objects degraded (12.763%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:48.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:48.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:48.448 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:48.449 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:48 smithi099 ceph-mon[167076]: pgmap v516: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.4 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 79 MiB/s wr, 1.26k op/s; 15204/119127 objects degraded (12.763%); 11209/119127 objects misplaced (9.409%); 2.8 MiB/s, 1 keys/s, 47 objects/s recovering 2023-09-23T18:01:48.449 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15204/119127 objects degraded (12.763%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:48.452 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:01:48.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:48.601 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:48.658 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:48.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:48.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:48.763 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:48.831 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:48.948 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:48.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:49.005 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 45 16 47724 47708 66.2405 86.875 0.0100275 0.0150908 2023-09-23T18:01:49.056 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:49.099 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:49.146 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:49.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:49.152 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:49.217 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T18:01:49.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:49.290 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:49.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:49.350 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/773418927' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:01:49.351 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:49 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:01:49.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:49.352 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:49.452 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:49.467 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/773418927' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:01:49.467 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:49 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:01:49.467 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/773418927' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:01:49.467 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:49 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:01:49.505 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:49.576 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:49.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:49.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:49.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:49.749 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:49.804 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:49.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:49.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:49.825 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:01:49.859 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:49.874 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:01:49.911 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:01:49.912 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:49.957 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:01:49.983 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:49.996 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:01:50.006 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 46 16 49182 49166 66.781 91.125 0.00696266 0.0149695 2023-09-23T18:01:50.042 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:01:50.061 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:50.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:50.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:50.075 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:01:50.075 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:01:50.075 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:01:50.075 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:01:50.075 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:01:50.075 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:01:50.075 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:01:50.076 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:01:50.076 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:01:50.165 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:50.222 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:50.265 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:01:50.293 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:50.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:50.299 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:50.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[179668]: pgmap v517: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 16216/131712 objects degraded (12.312%); 12752/131712 objects misplaced (9.682%); 2.5 MiB/s, 1 keys/s, 42 objects/s recovering 2023-09-23T18:01:50.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[179668]: failed to encode map e223 with expected crc 2023-09-23T18:01:50.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:01:50.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[179668]: osdmap e223: 8 total, 7 up, 8 in 2023-09-23T18:01:50.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[179668]: failed to encode map e223 with expected crc 2023-09-23T18:01:50.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/398949464' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:50.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4165878063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:50.361 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[184174]: pgmap v517: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 16216/131712 objects degraded (12.312%); 12752/131712 objects misplaced (9.682%); 2.5 MiB/s, 1 keys/s, 42 objects/s recovering 2023-09-23T18:01:50.361 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[184174]: failed to encode map e223 with expected crc 2023-09-23T18:01:50.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:01:50.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[184174]: osdmap e223: 8 total, 7 up, 8 in 2023-09-23T18:01:50.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[184174]: failed to encode map e223 with expected crc 2023-09-23T18:01:50.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/398949464' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:50.362 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4165878063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:50.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:50.457 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:50.527 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:50.529 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:50 smithi099 ceph-mon[167076]: pgmap v517: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 93 MiB/s wr, 1.49k op/s; 16216/131712 objects degraded (12.312%); 12752/131712 objects misplaced (9.682%); 2.5 MiB/s, 1 keys/s, 42 objects/s recovering 2023-09-23T18:01:50.529 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:50 smithi099 ceph-mon[167076]: failed to encode map e223 with expected crc 2023-09-23T18:01:50.529 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:50 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:01:50.530 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:50 smithi099 ceph-mon[167076]: osdmap e223: 8 total, 7 up, 8 in 2023-09-23T18:01:50.530 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:50 smithi099 ceph-mon[167076]: failed to encode map e223 with expected crc 2023-09-23T18:01:50.530 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:50 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/398949464' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:01:50.530 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:50 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4165878063' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:50.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:50.531 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:50.632 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:50.641 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:01:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:01:50] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-09-23T18:01:50.682 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:50.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:50.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:50.743 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:50.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:50.901 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:50.951 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:50.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:50.962 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:50.973 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:01:50.974 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:01:51.006 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 47 16 50753 50737 67.4489 98.1875 0.00891711 0.0148213 2023-09-23T18:01:51.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:51.117 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:51.182 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:51.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:51.189 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:51.290 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:51.345 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:51.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:51.414 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:51.415 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:51.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:51 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:51.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:51 smithi055 ceph-mon[184174]: failed to encode map e224 with expected crc 2023-09-23T18:01:51.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/398949464' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:51.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:51 smithi055 ceph-mon[184174]: osdmap e224: 8 total, 7 up, 8 in 2023-09-23T18:01:51.500 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:51 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:51.500 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:51 smithi055 ceph-mon[179668]: failed to encode map e224 with expected crc 2023-09-23T18:01:51.500 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/398949464' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:51.500 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:51 smithi055 ceph-mon[179668]: osdmap e224: 8 total, 7 up, 8 in 2023-09-23T18:01:51.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:51.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:51 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:01:51.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:51 smithi099 ceph-mon[167076]: failed to encode map e224 with expected crc 2023-09-23T18:01:51.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/398949464' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:01:51.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:51 smithi099 ceph-mon[167076]: osdmap e224: 8 total, 7 up, 8 in 2023-09-23T18:01:51.562 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:51.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:51.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:51.625 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:51.725 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:51.793 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:51.798 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:01:51.798 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:01:51.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.798 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:51973 6.0 6:0349a741:::benchmark_data_smithi055_275963_object51972:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T18:01:51.798 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:51.794086+0000", 2023-09-23T18:01:51.798 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0042850880000000003, 2023-09-23T18:01:51.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0043228920000000001, 2023-09-23T18:01:51.799 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:51.799 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:01:51.799 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:51.799 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:01:51.799 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:01:51.799 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 51973 2023-09-23T18:01:51.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.799 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:51.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:51.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794086+0000", 2023-09-23T18:01:51.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:51.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:51.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794086+0000", 2023-09-23T18:01:51.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:51.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:51.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794084+0000", 2023-09-23T18:01:51.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:51.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:51.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794121+0000", 2023-09-23T18:01:51.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7002000000000002e-05 2023-09-23T18:01:51.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794134+0000", 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.275e-05 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794141+0000", 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4800000000000004e-06 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794217+0000", 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5537000000000005e-05 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794359+0000", 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014181299999999999 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2", 2023-09-23T18:01:51.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794500+0000", 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014157899999999999 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.796990+0000", 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002489345 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.805 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:51974 6.9 6:93f37969:::benchmark_data_smithi055_275963_object51973:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T18:01:51.805 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:51.794157+0000", 2023-09-23T18:01:51.805 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0042134989999999999, 2023-09-23T18:01:51.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0043466720000000002, 2023-09-23T18:01:51.805 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:51.805 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:01:51.805 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:51.805 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:01:51.805 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:01:51.805 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 51974 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794157+0000", 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794157+0000", 2023-09-23T18:01:51.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794157+0000", 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794180+0000", 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3725000000000001e-05 2023-09-23T18:01:51.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794211+0000", 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0295e-05 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794218+0000", 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2880000000000004e-06 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:51.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794588+0000", 2023-09-23T18:01:51.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00036981500000000001 2023-09-23T18:01:51.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.809 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:51.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794724+0000", 2023-09-23T18:01:51.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013592300000000001 2023-09-23T18:01:51.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.794880+0000", 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015600299999999999 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.797740+0000", 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002860699 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:51.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.797766+0000", 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6032000000000001e-05 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:51972 6.a e224/204 6:5301b391:::benchmark_data_smithi055_275963_object51971:head v 224'3258, mlcod=224'3256)", 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:51.796533+0000", 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.001838084, 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020356430000000002, 2023-09-23T18:01:51.811 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.796533+0000", 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.796533+0000", 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:51.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.796532+0000", 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.796586+0000", 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3575000000000001e-05 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.814 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:51.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.796598+0000", 2023-09-23T18:01:51.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2551999999999999e-05 2023-09-23T18:01:51.814 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.814 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.814 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:51.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.796606+0000", 2023-09-23T18:01:51.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2870000000000004e-06 2023-09-23T18:01:51.814 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.814 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.796656+0000", 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9960999999999998e-05 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.796678+0000", 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1645e-05 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.798534+0000", 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018555030000000001 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.798569+0000", 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4968999999999997e-05 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.817 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:51971 6.9 6:9bf8f930:::benchmark_data_smithi055_275963_object51970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e224)", 2023-09-23T18:01:51.817 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:51.791978+0000", 2023-09-23T18:01:51.817 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.006393274, 2023-09-23T18:01:51.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0066475789999999998, 2023-09-23T18:01:51.817 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:01:51.817 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:01:51.817 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:01:51.817 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:01:51.817 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:01:51.817 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 51971 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.791978+0000", 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.791978+0000", 2023-09-23T18:01:51.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.791976+0000", 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.792003+0000", 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6971e-05 2023-09-23T18:01:51.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.792025+0000", 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2429000000000002e-05 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.792036+0000", 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0370000000000001e-05 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.792133+0000", 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7522000000000006e-05 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.792311+0000", 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017771099999999999 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.792464+0000", 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015266300000000001 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.796913+0000", 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044497920000000002 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:51.797760+0000", 2023-09-23T18:01:51.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00084712399999999999 2023-09-23T18:01:51.823 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:51.823 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:01:51.823 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:51.823 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:01:51.823 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:01:51.823 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 4 2023-09-23T18:01:51.823 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:01:51.849 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:51.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:51.853 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:51.954 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:52.006 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:52.007 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 48 16 52303 52287 68.0605 96.875 0.00491639 0.014689 2023-09-23T18:01:52.126 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:52.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:52.136 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:52.146 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:01:52.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:52.281 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:52.347 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:52.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:52.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:52 smithi055 ceph-mon[179668]: pgmap v520: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 17192/137649 objects degraded (12.490%); 13236/137649 objects misplaced (9.616%); 2.4 MiB/s, 1 keys/s, 39 objects/s recovering 2023-09-23T18:01:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:52 smithi055 ceph-mon[184174]: pgmap v520: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 17192/137649 objects degraded (12.490%); 13236/137649 objects misplaced (9.616%); 2.4 MiB/s, 1 keys/s, 39 objects/s recovering 2023-09-23T18:01:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:52.463 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:52.504 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:52 smithi099 ceph-mon[167076]: pgmap v520: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 2.8 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 87 MiB/s wr, 1.39k op/s; 17192/137649 objects degraded (12.490%); 13236/137649 objects misplaced (9.616%); 2.4 MiB/s, 1 keys/s, 39 objects/s recovering 2023-09-23T18:01:52.504 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:01:52.504 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:01:52.511 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:52.558 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:52.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:52.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:52.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:01:52.713 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:52.778 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:01:52.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:52.786 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:01:52.856 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:01:52.887 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:52.942 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:52.975 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:01:52.989 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:52.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:53.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:53.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:53.007 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 49 16 53828 53812 68.6164 95.3125 0.0149188 0.0145676 2023-09-23T18:01:53.103 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:53.228 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 17192/137649 objects degraded (12.490%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:53.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 17192/137649 objects degraded (12.490%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 17192/137649 objects degraded (12.490%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:54.007 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 50 16 55300 55284 69.0838 92 0.00680973 0.0144704 2023-09-23T18:01:54.078 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:54.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:54.082 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:54.183 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:54.246 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:54.308 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:54.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:54.321 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:54.413 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:54 smithi055 ceph-mon[184174]: pgmap v521: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 18087/150363 objects degraded (12.029%); 14312/150363 objects misplaced (9.518%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T18:01:54.413 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:54 smithi055 ceph-mon[179668]: pgmap v521: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 18087/150363 objects degraded (12.029%); 14312/150363 objects misplaced (9.518%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T18:01:54.422 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:54.468 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:54.522 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:54.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:54.526 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:54 smithi099 ceph-mon[167076]: pgmap v521: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 99 MiB/s wr, 1.58k op/s; 18087/150363 objects degraded (12.029%); 14312/150363 objects misplaced (9.518%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T18:01:54.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:54.684 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:54.749 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:54.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:54.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:54.771 INFO:tasks.thrashosds.thrasher:in_osds: [0, 1, 2, 3, 6, 7, 5, 4] out_osds: [] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2023-09-23T18:01:54.772 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:01:54.772 INFO:tasks.thrashosds.thrasher:Removing osd 1, in_osds are: [0, 1, 2, 3, 6, 7, 5, 4] 2023-09-23T18:01:54.772 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 1 2023-09-23T18:01:54.859 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:54.951 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:55.007 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 51 16 56735 56719 69.4876 89.6875 0.0178141 0.0143856 2023-09-23T18:01:55.013 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:55.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:55.024 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:55.048 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:55.054 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:55.124 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:55.199 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:55.256 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:55.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:55.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:55.298 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:55 smithi055 ceph-mon[184174]: pgmap v522: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 19956/160149 objects degraded (12.461%); 14788/160149 objects misplaced (9.234%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:01:55.299 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:55 smithi055 ceph-mon[179668]: pgmap v522: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 19956/160149 objects degraded (12.461%); 14788/160149 objects misplaced (9.234%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:01:55.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:55.410 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:55.470 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:55.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:55.474 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:55.574 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:55.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:55 smithi099 ceph-mon[167076]: pgmap v522: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 107 MiB/s wr, 1.71k op/s; 19956/160149 objects degraded (12.461%); 14788/160149 objects misplaced (9.234%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:01:55.664 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:55.694 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:01:55.694 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:01:55.695 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:01:55.695 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:01:55.695 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:01:56.007 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 52 16 58062 58046 69.746 82.9375 0.00691014 0.0143331 2023-09-23T18:01:56.601 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:01:56.625 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:56.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:56.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:56.676 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:56.734 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:01:56.806 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:57.004 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:01:57.009 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 53 16 59471 59455 70.089 88.0625 0.0151403 0.0142632 2023-09-23T18:01:57.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:57.012 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:01:57.113 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:57.182 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:57.343 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:57.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:57.348 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:57.448 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:01:57.525 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:57.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:01:57.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:57.593 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:01:57.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:57.740 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:57.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:57.794 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:57 smithi099 ceph-mon[167076]: pgmap v523: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 20854/165531 objects degraded (12.598%); 14400/165531 objects misplaced (8.699%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T18:01:57.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:57.795 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:57.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:57.950 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:58.002 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:58.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:58.005 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:58.011 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 54 16 60768 60752 70.2899 81.0625 0.0195796 0.0142217 2023-09-23T18:01:58.106 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:58.139 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:57 smithi055 ceph-mon[179668]: pgmap v523: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 20854/165531 objects degraded (12.598%); 14400/165531 objects misplaced (8.699%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T18:01:58.139 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:57 smithi055 ceph-mon[184174]: pgmap v523: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.4 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 88 MiB/s wr, 1.41k op/s; 20854/165531 objects degraded (12.598%); 14400/165531 objects misplaced (8.699%); 5.7 MiB/s, 90 objects/s recovering 2023-09-23T18:01:58.140 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:01:58.166 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:01:58.184 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:01:58.184 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:58.236 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:01:58.254 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:01:58.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:58.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:01:58.282 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:01:58.322 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:01:58.355 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:01:58.372 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:01:58.392 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:01:58.392 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:01:58.392 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:01:58.392 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:01:58.392 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:01:58.393 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:01:58.393 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:01:58.393 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:01:58.393 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:01:58.421 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:58.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:01:58.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:58.494 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:01:58.594 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:01:58.655 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:58.709 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:01:58.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:58.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:01:58.814 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:01:58.866 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:01:58.950 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:01:58.981 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 20854/165531 objects degraded (12.598%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:58.981 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:58 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2768054165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:58.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:58 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/466784187' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:58.982 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:58 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:58.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 20854/165531 objects degraded (12.598%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:58.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:58 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2768054165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:58.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:58 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/466784187' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:58.982 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:58 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:59.011 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 55 16 62105 62089 70.531 83.5625 0.00871372 0.0141742 2023-09-23T18:01:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 20854/165531 objects degraded (12.598%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:01:59.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:58 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2768054165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:01:59.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:58 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/466784187' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:59.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:58 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:01:59.180 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:01:59.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:01:59.186 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:01:59.286 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:01:59.354 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:00.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 56 16 63572 63556 70.9086 91.6875 0.0117219 0.0140978 2023-09-23T18:02:00.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:59 smithi099 ceph-mon[167076]: pgmap v524: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 21731/176124 objects degraded (12.338%); 15746/176124 objects misplaced (8.940%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T18:02:00.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:59 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:00.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:59 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:02:00.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:01:59 smithi099 ceph-mon[167076]: osdmap e225: 8 total, 7 up, 8 in 2023-09-23T18:02:00.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:59 smithi055 ceph-mon[184174]: pgmap v524: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 21731/176124 objects degraded (12.338%); 15746/176124 objects misplaced (8.940%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T18:02:00.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:59 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:00.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:59 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:02:00.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:01:59 smithi055 ceph-mon[184174]: osdmap e225: 8 total, 7 up, 8 in 2023-09-23T18:02:00.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:59 smithi055 ceph-mon[179668]: pgmap v524: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 98 MiB/s wr, 1.56k op/s; 21731/176124 objects degraded (12.338%); 15746/176124 objects misplaced (8.940%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T18:02:00.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:59 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:00.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:59 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:02:00.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:59 smithi055 ceph-mon[179668]: osdmap e225: 8 total, 7 up, 8 in 2023-09-23T18:02:00.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:01:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:01:59.751+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e225 definitely_dead 0 2023-09-23T18:02:00.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:02:00] "GET /metrics HTTP/1.1" 200 36551 "" "Prometheus/2.43.0" 2023-09-23T18:02:01.011 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 57 16 64721 64705 70.9243 71.8125 0.00256613 0.0140335 2023-09-23T18:02:01.024 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.1. 2023-09-23T18:02:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:00 smithi099 ceph-mon[167076]: failed to encode map e225 with expected crc 2023-09-23T18:02:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:00 smithi099 ceph-mon[167076]: failed to encode map e225 with expected crc 2023-09-23T18:02:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3658207421' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T18:02:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:00 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T18:02:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:00 smithi099 ceph-mon[167076]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T18:02:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:00 smithi099 ceph-mon[167076]: 1.18 scrub starts 2023-09-23T18:02:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[179668]: failed to encode map e225 with expected crc 2023-09-23T18:02:01.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[179668]: failed to encode map e225 with expected crc 2023-09-23T18:02:01.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3658207421' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T18:02:01.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T18:02:01.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[179668]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T18:02:01.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[179668]: 1.18 scrub starts 2023-09-23T18:02:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[184174]: failed to encode map e225 with expected crc 2023-09-23T18:02:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[184174]: failed to encode map e225 with expected crc 2023-09-23T18:02:01.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3658207421' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T18:02:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["1"]}]: dispatch 2023-09-23T18:02:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[184174]: Client client.admin marked osd.1 out, while it was still marked up 2023-09-23T18:02:01.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:00 smithi055 ceph-mon[184174]: 1.18 scrub starts 2023-09-23T18:02:01.381 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:02:01.382 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:02:01.634 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:01.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:01.637 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:01.738 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:01.789 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:01.831 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:01.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:01.834 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:01.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:01 smithi055 ceph-mon[179668]: pgmap v526: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 22611/181401 objects degraded (12.465%); 16191/181401 objects misplaced (8.926%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:02:01.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:01 smithi055 ceph-mon[179668]: failed to encode map e226 with expected crc 2023-09-23T18:02:01.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:01 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T18:02:01.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:01 smithi055 ceph-mon[179668]: osdmap e226: 8 total, 7 up, 7 in 2023-09-23T18:02:01.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:01 smithi055 ceph-mon[179668]: failed to encode map e226 with expected crc 2023-09-23T18:02:01.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:01 smithi055 ceph-mon[184174]: pgmap v526: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 22611/181401 objects degraded (12.465%); 16191/181401 objects misplaced (8.926%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:02:01.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:01 smithi055 ceph-mon[184174]: failed to encode map e226 with expected crc 2023-09-23T18:02:01.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:01 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T18:02:01.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:01 smithi055 ceph-mon[184174]: osdmap e226: 8 total, 7 up, 7 in 2023-09-23T18:02:01.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:01 smithi055 ceph-mon[184174]: failed to encode map e226 with expected crc 2023-09-23T18:02:01.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:01.963 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:01.997 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:01.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:02.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:02.011 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 58 16 64721 64705 69.7017 0 - 0.0140335 2023-09-23T18:02:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:01 smithi099 ceph-mon[167076]: pgmap v526: 119 pgs: 25 active+clean+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 9 active+undersized, 8 active+undersized+degraded; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 91 MiB/s wr, 1.46k op/s; 22611/181401 objects degraded (12.465%); 16191/181401 objects misplaced (8.926%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:02:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:01 smithi099 ceph-mon[167076]: failed to encode map e226 with expected crc 2023-09-23T18:02:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:01 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["1"]}]': finished 2023-09-23T18:02:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:01 smithi099 ceph-mon[167076]: osdmap e226: 8 total, 7 up, 7 in 2023-09-23T18:02:02.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:01 smithi099 ceph-mon[167076]: failed to encode map e226 with expected crc 2023-09-23T18:02:02.101 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:02.132 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:02.644 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:02.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:02.648 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:02.749 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:02.785 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:02.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:02.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:02.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:02.930 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:02.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:02 smithi055 ceph-mon[184174]: failed to encode map e227 with expected crc 2023-09-23T18:02:02.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:02 smithi055 ceph-mon[184174]: failed to encode map e227 with expected crc 2023-09-23T18:02:02.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:02 smithi055 ceph-mon[184174]: osdmap e227: 8 total, 7 up, 7 in 2023-09-23T18:02:02.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:02 smithi055 ceph-mon[184174]: failed to encode map e227 with expected crc 2023-09-23T18:02:02.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:02 smithi055 ceph-mon[184174]: failed to encode map e227 with expected crc 2023-09-23T18:02:02.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:02 smithi055 ceph-mon[179668]: failed to encode map e227 with expected crc 2023-09-23T18:02:02.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:02 smithi055 ceph-mon[179668]: failed to encode map e227 with expected crc 2023-09-23T18:02:02.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:02 smithi055 ceph-mon[179668]: osdmap e227: 8 total, 7 up, 7 in 2023-09-23T18:02:02.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:02 smithi055 ceph-mon[179668]: failed to encode map e227 with expected crc 2023-09-23T18:02:02.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:02 smithi055 ceph-mon[179668]: failed to encode map e227 with expected crc 2023-09-23T18:02:02.969 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:03.007 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:03.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:03.010 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:03.011 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 59 16 64749 64733 68.5503 0.875 0.00816091 0.0143055 2023-09-23T18:02:03.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:02 smithi099 ceph-mon[167076]: failed to encode map e227 with expected crc 2023-09-23T18:02:03.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:02 smithi099 ceph-mon[167076]: failed to encode map e227 with expected crc 2023-09-23T18:02:03.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:02 smithi099 ceph-mon[167076]: osdmap e227: 8 total, 7 up, 7 in 2023-09-23T18:02:03.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:02 smithi099 ceph-mon[167076]: failed to encode map e227 with expected crc 2023-09-23T18:02:03.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:02 smithi099 ceph-mon[167076]: failed to encode map e227 with expected crc 2023-09-23T18:02:03.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:03.151 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:03.195 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:03.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:03.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:03.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:03.332 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:03.378 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:03.383 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:02:03.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:03.384 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:03.485 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:03.512 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:04.011 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:02:04.012602+0000 min lat: 0.00103291 max lat: 21.0159 avg lat: 0.014414 2023-09-23T18:02:04.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:02:04.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 60 16 64761 64745 67.4205 0.75 0.00431708 0.014414 2023-09-23T18:02:04.047 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:04.050 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: pgmap v529: 119 pgs: 13 remapped+peering, 19 active+clean+remapped, 71 active+clean, 9 active+undersized, 7 active+undersized+degraded; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 18966/188037 objects degraded (10.086%); 15908/188037 objects misplaced (8.460%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:02:04.050 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.050 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.050 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: osdmap e228: 8 total, 7 up, 7 in 2023-09-23T18:02:04.050 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.050 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.050 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: 6.4 starting backfill to osd.6 from (0'0,0'0] MAX to 225'4058 2023-09-23T18:02:04.050 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: 6.1 starting backfill to osd.7 from (0'0,0'0] MAX to 225'4104 2023-09-23T18:02:04.050 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 18966/188037 objects degraded (10.086%), 7 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:04.050 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.051 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: osdmap e229: 8 total, 7 up, 7 in 2023-09-23T18:02:04.051 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.051 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.051 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.051 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[184174]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:04.052 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:04.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: pgmap v529: 119 pgs: 13 remapped+peering, 19 active+clean+remapped, 71 active+clean, 9 active+undersized, 7 active+undersized+degraded; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 18966/188037 objects degraded (10.086%); 15908/188037 objects misplaced (8.460%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:02:04.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.052 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: osdmap e228: 8 total, 7 up, 7 in 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: 6.4 starting backfill to osd.6 from (0'0,0'0] MAX to 225'4058 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: 6.1 starting backfill to osd.7 from (0'0,0'0] MAX to 225'4104 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 18966/188037 objects degraded (10.086%), 7 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: osdmap e229: 8 total, 7 up, 7 in 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.053 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:03 smithi055 ceph-mon[179668]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: pgmap v529: 119 pgs: 13 remapped+peering, 19 active+clean+remapped, 71 active+clean, 9 active+undersized, 7 active+undersized+degraded; 3.8 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 78 MiB/s wr, 1.25k op/s; 18966/188037 objects degraded (10.086%); 15908/188037 objects misplaced (8.460%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:02:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: osdmap e228: 8 total, 7 up, 7 in 2023-09-23T18:02:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: failed to encode map e228 with expected crc 2023-09-23T18:02:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: 6.4 starting backfill to osd.6 from (0'0,0'0] MAX to 225'4058 2023-09-23T18:02:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: 6.1 starting backfill to osd.7 from (0'0,0'0] MAX to 225'4104 2023-09-23T18:02:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 18966/188037 objects degraded (10.086%), 7 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: osdmap e229: 8 total, 7 up, 7 in 2023-09-23T18:02:04.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:03 smithi099 ceph-mon[167076]: failed to encode map e229 with expected crc 2023-09-23T18:02:04.124 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:02:04.124 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:02:04.153 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:04.192 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:04.435 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:04.439 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:04.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:04.440 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:04.445 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:02:04.541 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:04.568 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:04.655 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:04.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:04.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:04.761 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:04.790 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:04.823 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:04.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:04.826 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:04.927 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:04.967 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:05.008 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:05.011 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 61 16 64761 64745 66.3155 0 - 0.014414 2023-09-23T18:02:05.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:05.016 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:05.116 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:05.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[184174]: failed to encode map e229 with expected crc 2023-09-23T18:02:05.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[184174]: failed to encode map e229 with expected crc 2023-09-23T18:02:05.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[184174]: osdmap e230: 8 total, 7 up, 7 in 2023-09-23T18:02:05.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[184174]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[184174]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[184174]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[184174]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[179668]: failed to encode map e229 with expected crc 2023-09-23T18:02:05.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[179668]: failed to encode map e229 with expected crc 2023-09-23T18:02:05.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[179668]: osdmap e230: 8 total, 7 up, 7 in 2023-09-23T18:02:05.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[179668]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[179668]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[179668]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:04 smithi055 ceph-mon[179668]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.147 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:05.183 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:05.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:05.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:05.287 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:05.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:04 smithi099 ceph-mon[167076]: failed to encode map e229 with expected crc 2023-09-23T18:02:05.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:04 smithi099 ceph-mon[167076]: failed to encode map e229 with expected crc 2023-09-23T18:02:05.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:04 smithi099 ceph-mon[167076]: osdmap e230: 8 total, 7 up, 7 in 2023-09-23T18:02:05.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:04 smithi099 ceph-mon[167076]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.319 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:04 smithi099 ceph-mon[167076]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.319 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:04 smithi099 ceph-mon[167076]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.319 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:04 smithi099 ceph-mon[167076]: failed to encode map e230 with expected crc 2023-09-23T18:02:05.321 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:05.364 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:05.367 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:05.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:05.368 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:05.469 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:05.502 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:05.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:05.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:05.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:05.649 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:05.663 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:02:05.682 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:05.687 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:02:05.714 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:02:05.724 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:05.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:05.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:05.736 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:02:05.756 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:02:05.828 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:05.857 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:06.011 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 62 16 64761 64745 65.2461 0 - 0.014414 2023-09-23T18:02:06.111 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:02:06.114 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:06.116 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:05 smithi055 ceph-mon[184174]: pgmap v533: 119 pgs: 4 activating+undersized+remapped, 12 remapped+peering, 27 active+clean+remapped, 65 active+clean, 4 active+undersized, 7 active+undersized+degraded; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s; 19456/190977 objects degraded (10.188%); 16046/190977 objects misplaced (8.402%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T18:02:06.117 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:05 smithi055 ceph-mon[184174]: osdmap e231: 8 total, 7 up, 7 in 2023-09-23T18:02:06.117 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:05 smithi055 ceph-mon[184174]: failed to encode map e231 with expected crc 2023-09-23T18:02:06.117 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:05 smithi055 ceph-mon[184174]: failed to encode map e231 with expected crc 2023-09-23T18:02:06.117 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1919560278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:06.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:05 smithi055 ceph-mon[179668]: pgmap v533: 119 pgs: 4 activating+undersized+remapped, 12 remapped+peering, 27 active+clean+remapped, 65 active+clean, 4 active+undersized, 7 active+undersized+degraded; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s; 19456/190977 objects degraded (10.188%); 16046/190977 objects misplaced (8.402%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T18:02:06.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:05 smithi055 ceph-mon[179668]: osdmap e231: 8 total, 7 up, 7 in 2023-09-23T18:02:06.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:05 smithi055 ceph-mon[179668]: failed to encode map e231 with expected crc 2023-09-23T18:02:06.118 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:05 smithi055 ceph-mon[179668]: failed to encode map e231 with expected crc 2023-09-23T18:02:06.118 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1919560278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:06.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:06.118 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:06.219 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:06.249 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:05 smithi099 ceph-mon[167076]: pgmap v533: 119 pgs: 4 activating+undersized+remapped, 12 remapped+peering, 27 active+clean+remapped, 65 active+clean, 4 active+undersized, 7 active+undersized+degraded; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 51 MiB/s wr, 814 op/s; 19456/190977 objects degraded (10.188%); 16046/190977 objects misplaced (8.402%); 6.9 MiB/s, 112 objects/s recovering 2023-09-23T18:02:06.249 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:05 smithi099 ceph-mon[167076]: osdmap e231: 8 total, 7 up, 7 in 2023-09-23T18:02:06.249 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:05 smithi099 ceph-mon[167076]: failed to encode map e231 with expected crc 2023-09-23T18:02:06.249 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:05 smithi099 ceph-mon[167076]: failed to encode map e231 with expected crc 2023-09-23T18:02:06.249 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1919560278' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:06.252 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:06.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:06.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:06.299 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:06.400 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:06.433 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:06.469 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:06.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:06.473 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:06.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:06.606 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:06.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:06.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:06.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:06.653 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:02:06.653 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:02:06.653 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:02:06.653 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:02:06.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:06.754 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:06.790 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:06.838 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:06.841 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:06.841 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:06.942 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:06.973 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:06.984 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:02:07.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 63 16 64761 64745 64.2107 0 - 0.014414 2023-09-23T18:02:07.045 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:07.047 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:07.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:07.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:07.180 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:07.223 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:07.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:07.227 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:07.227 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:07.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:07.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:07.328 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:07.360 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:07.624 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:07.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:07.628 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:07.728 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:07.769 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:07.794 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:07.818 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:07.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:07.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:07.923 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:07.955 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:07.995 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:07.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:07.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:08.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 64 16 64801 64785 63.2467 0.625 0.00288106 0.0148664 2023-09-23T18:02:08.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:07 smithi099 ceph-mon[167076]: pgmap v535: 119 pgs: 4 activating+undersized+remapped, 12 remapped+peering, 27 active+clean+remapped, 65 active+clean, 4 active+undersized, 7 active+undersized+degraded; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 15 MiB/s wr, 244 op/s; 19456/190977 objects degraded (10.188%); 16046/190977 objects misplaced (8.402%); 144 KiB/s, 4 objects/s recovering 2023-09-23T18:02:08.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:07 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19456/190977 objects degraded (10.188%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:08.100 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:08.134 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:08.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:07 smithi055 ceph-mon[184174]: pgmap v535: 119 pgs: 4 activating+undersized+remapped, 12 remapped+peering, 27 active+clean+remapped, 65 active+clean, 4 active+undersized, 7 active+undersized+degraded; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 15 MiB/s wr, 244 op/s; 19456/190977 objects degraded (10.188%); 16046/190977 objects misplaced (8.402%); 144 KiB/s, 4 objects/s recovering 2023-09-23T18:02:08.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:07 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19456/190977 objects degraded (10.188%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:08.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:07 smithi055 ceph-mon[179668]: pgmap v535: 119 pgs: 4 activating+undersized+remapped, 12 remapped+peering, 27 active+clean+remapped, 65 active+clean, 4 active+undersized, 7 active+undersized+degraded; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 15 MiB/s wr, 244 op/s; 19456/190977 objects degraded (10.188%); 16046/190977 objects misplaced (8.402%); 144 KiB/s, 4 objects/s recovering 2023-09-23T18:02:08.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:07 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19456/190977 objects degraded (10.188%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:08.175 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:08.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:08.179 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:08.279 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:08.309 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:08.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:08.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:08.353 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:08.454 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:08.484 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:08.623 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:08.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:08.626 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:08.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:08.758 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:08.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:08.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:08.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:08.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:08.935 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:08.973 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:02:08.973 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:08.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:08.977 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:09.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 65 16 64801 64785 62.2739 0 - 0.0148664 2023-09-23T18:02:09.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2001636034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:09.078 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:09.109 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:09.189 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:02:09.189 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:02:09.189 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:02:09.189 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:02:09.189 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:02:09.210 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4] out_osds: [1] dead_osds: [3] live_osds: [0, 1, 2, 4, 5, 6, 7] 2023-09-23T18:02:09.210 INFO:tasks.thrashosds.thrasher:Reviving osd 3 2023-09-23T18:02:09.210 INFO:tasks.cephadm.osd.3:Restarting osd.3 (starting--it wasn't running)... 2023-09-23T18:02:09.210 DEBUG:teuthology.orchestra.run.smithi055:osd.3> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.3.service 2023-09-23T18:02:09.212 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl start ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T18:02:09.215 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2001636034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:09.216 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2001636034' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:09.243 INFO:journalctl@ceph.osd.3.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T18:02:09.594 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:09 smithi055 systemd[1]: Starting Ceph osd.3 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:02:09.595 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:02:09.597 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:09.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:09.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:09.701 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:09.735 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:09.774 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:09.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:09.777 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:09.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:09.908 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:09.948 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:09.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:09.953 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:10.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 66 16 64801 64785 61.3305 0 - 0.0148664 2023-09-23T18:02:10.053 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:09 smithi099 ceph-mon[167076]: pgmap v536: 119 pgs: 4 activating+undersized+remapped, 35 active+clean+remapped, 66 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 11 MiB/s wr, 171 op/s; 26113/191064 objects degraded (13.667%); 23833/191064 objects misplaced (12.474%); 3.7 MiB/s, 62 objects/s recovering 2023-09-23T18:02:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:09 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:02:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2001636034' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:09 smithi099 ceph-mon[167076]: osdmap e232: 8 total, 7 up, 7 in 2023-09-23T18:02:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:09 smithi099 ceph-mon[167076]: failed to encode map e232 with expected crc 2023-09-23T18:02:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:09 smithi099 ceph-mon[167076]: failed to encode map e232 with expected crc 2023-09-23T18:02:10.083 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:10.124 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:10.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:10.128 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:10.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:10.259 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:10.301 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:10.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:10.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:10.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[179668]: pgmap v536: 119 pgs: 4 activating+undersized+remapped, 35 active+clean+remapped, 66 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 11 MiB/s wr, 171 op/s; 26113/191064 objects degraded (13.667%); 23833/191064 objects misplaced (12.474%); 3.7 MiB/s, 62 objects/s recovering 2023-09-23T18:02:10.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:02:10.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2001636034' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:10.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[179668]: osdmap e232: 8 total, 7 up, 7 in 2023-09-23T18:02:10.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[179668]: failed to encode map e232 with expected crc 2023-09-23T18:02:10.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[179668]: failed to encode map e232 with expected crc 2023-09-23T18:02:10.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[184174]: pgmap v536: 119 pgs: 4 activating+undersized+remapped, 35 active+clean+remapped, 66 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 4 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 7 active+undersized+degraded; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 11 MiB/s wr, 171 op/s; 26113/191064 objects degraded (13.667%); 23833/191064 objects misplaced (12.474%); 3.7 MiB/s, 62 objects/s recovering 2023-09-23T18:02:10.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:02:10.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2001636034' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:10.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[184174]: osdmap e232: 8 total, 7 up, 7 in 2023-09-23T18:02:10.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[184174]: failed to encode map e232 with expected crc 2023-09-23T18:02:10.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:09 smithi055 ceph-mon[184174]: failed to encode map e232 with expected crc 2023-09-23T18:02:10.376 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:10.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:10.438 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:10.478 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:02:10.484 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:10.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:10.501 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:10.502 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:02:10.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:02:10.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:02:10.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:02:10.502 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:02:10.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:02:10.503 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:02:10.503 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:02:10.503 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:02:10.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:10.650 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:10.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:02:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:02:10] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-09-23T18:02:10.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:10 smithi055 bash[286680]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T18:02:10.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:10 smithi055 bash[286680]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T18:02:10.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:10 smithi055 bash[286680]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_1 --path /var/lib/ceph/osd/ceph-3 --no-mon-config 2023-09-23T18:02:10.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:10 smithi055 bash[286680]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_1 /var/lib/ceph/osd/ceph-3/block 2023-09-23T18:02:10.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:10 smithi055 bash[286680]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-3/block 2023-09-23T18:02:10.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:10 smithi055 bash[286680]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-0 2023-09-23T18:02:10.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:10 smithi055 bash[286680]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-3 2023-09-23T18:02:10.681 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:10 smithi055 bash[286680]: --> ceph-volume lvm activate successful for osd ID: 3 2023-09-23T18:02:10.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:10.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:10.698 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:10.799 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:10.838 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:11.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 67 16 65066 65050 60.6625 5.52083 0.00695966 0.0164683 2023-09-23T18:02:11.095 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:11.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:11.099 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:11.200 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:11.234 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:02:11.235 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:02:11.246 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:11.542 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:11.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:11.547 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:11.648 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:11.681 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:11.728 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:11.735 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:11.735 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:11.748 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:02:11.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:11.875 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:11.920 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:11.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:11.924 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:11.930 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:11 smithi055 podman[287133]: 2023-09-23T18:02:11.930 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:11 smithi055 ceph-osd[287249]: -- 172.21.15.55:0/1576134422 <== mon.0 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x557d184484e0 con 0x557d18419400 2023-09-23T18:02:12.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 68 16 65484 65468 60.1547 26.125 0.0169797 0.0165978 2023-09-23T18:02:12.018 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:02:12.025 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:12.100 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:12.196 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:12.198 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:11 smithi055 bash[287133]: 4e4e70efbb731d344047aecd6f7c2aa90008717c4452407d8e0acf6980f50f08 2023-09-23T18:02:12.198 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:12 smithi055 systemd[1]: Started Ceph osd.3 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:02:12.199 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:11 smithi055 ceph-mon[179668]: pgmap v538: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 4 activating+undersized+remapped, 38 active+clean+remapped, 63 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 15 MiB/s wr, 246 op/s; 26774/194391 objects degraded (13.773%); 28249/194391 objects misplaced (14.532%); 37 MiB/s, 598 objects/s recovering 2023-09-23T18:02:12.199 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:11 smithi055 ceph-mon[179668]: failed to encode map e233 with expected crc 2023-09-23T18:02:12.199 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:11 smithi055 ceph-mon[179668]: osdmap e233: 8 total, 7 up, 7 in 2023-09-23T18:02:12.199 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:11 smithi055 ceph-mon[179668]: failed to encode map e233 with expected crc 2023-09-23T18:02:12.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:12.200 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:12.201 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:11 smithi055 ceph-mon[184174]: pgmap v538: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 4 activating+undersized+remapped, 38 active+clean+remapped, 63 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 15 MiB/s wr, 246 op/s; 26774/194391 objects degraded (13.773%); 28249/194391 objects misplaced (14.532%); 37 MiB/s, 598 objects/s recovering 2023-09-23T18:02:12.201 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:11 smithi055 ceph-mon[184174]: failed to encode map e233 with expected crc 2023-09-23T18:02:12.201 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:11 smithi055 ceph-mon[184174]: osdmap e233: 8 total, 7 up, 7 in 2023-09-23T18:02:12.201 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:11 smithi055 ceph-mon[184174]: failed to encode map e233 with expected crc 2023-09-23T18:02:12.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:12.322 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:02:12.323 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:02:12.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:02:12.323 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:02:12.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.323 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104107 5.0 5:08ba7c2f:::benchmark_data_smithi055_229117_object104106:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T18:02:12.323 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.953750+0000", 2023-09-23T18:02:12.323 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 288.36652246400001, 2023-09-23T18:02:12.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3969089229999998, 2023-09-23T18:02:12.323 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.323 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104107 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953750+0000", 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953750+0000", 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953748+0000", 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953776+0000", 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8515000000000001e-05 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953798+0000", 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1855999999999999e-05 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953808+0000", 2023-09-23T18:02:12.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0283e-05 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953876+0000", 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7348000000000006e-05 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op must wait for map", 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.953885+0000", 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0839999999999998e-06 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.962444+0000", 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0085596219999999994 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.962456+0000", 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1606999999999999e-05 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949379+0000", 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98692283300000005 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949390+0000", 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1503e-05 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.325692+0000", 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.376301955 2023-09-23T18:02:12.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.327734+0000", 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020413749999999998 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.327871+0000", 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137739 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336120+0000", 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082488500000000003 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336181+0000", 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1007000000000001e-05 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.348130+0000", 2023-09-23T18:02:12.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011948676 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.348167+0000", 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6782000000000001e-05 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350658+0000", 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024918570000000001 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.332 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104131 5.0 5:02532558:::benchmark_data_smithi055_229117_object104130:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e142)", 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.987392+0000", 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 288.33287993900001, 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3655978989999999, 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.333 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104131 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987392+0000", 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987392+0000", 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987390+0000", 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987415+0000", 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4185e-05 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987433+0000", 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8238000000000001e-05 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.987442+0000", 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9600000000000006e-06 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.988601+0000", 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011587889999999999 2023-09-23T18:02:12.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.988612+0000", 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1969e-05 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949474+0000", 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96086169899999996 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.949482+0000", 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0099999999999995e-06 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333244+0000", 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3837614760000001 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333381+0000", 2023-09-23T18:02:12.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013695800000000001 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.333517+0000", 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000136181 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342436+0000", 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0089189990000000004 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342466+0000", 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0561000000000001e-05 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350747+0000", 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082805090000000001 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.350829+0000", 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1778000000000002e-05 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.352990+0000", 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021612770000000001 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.341 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104167 5.0 5:0cc1b975:::benchmark_data_smithi055_229117_object104166:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:24.986469+0000", 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.33380322400001, 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3687401239999999, 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104167 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986469+0000", 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986469+0000", 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986467+0000", 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986500+0000", 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3046000000000002e-05 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986515+0000", 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5138e-05 2023-09-23T18:02:12.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986527+0000", 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1429e-05 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.026788+0000", 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040261308000000003 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.026799+0000", 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0699999999999999e-05 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336245+0000", 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3094467810000001 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336378+0000", 2023-09-23T18:02:12.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013300399999999999 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.336525+0000", 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014695100000000001 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342505+0000", 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0059792530000000003 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342518+0000", 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.348e-05 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353111+0000", 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010592839 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353201+0000", 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0167999999999995e-05 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355209+0000", 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020078090000000002 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.349 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104189 5.0 5:00d05606:::benchmark_data_smithi055_229117_object104188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.012952+0000", 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.30732045100001, 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.354677634, 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104189 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012952+0000", 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012952+0000", 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012950+0000", 2023-09-23T18:02:12.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012983+0000", 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2840999999999999e-05 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.012997+0000", 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.454e-05 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.013006+0000", 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2269999999999999e-06 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.030199+0000", 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017192819000000002 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.030209+0000", 2023-09-23T18:02:12.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.023e-05 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339148+0000", 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3089385840000001 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339354+0000", 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020608099999999999 2023-09-23T18:02:12.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.339570+0000", 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021573299999999999 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342529+0000", 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029589759999999999 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342571+0000", 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1863999999999998e-05 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365363+0000", 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022792085 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365409+0000", 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6479e-05 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367629+0000", 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022198840000000001 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.357 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104208 5.0 5:0c574cd2:::benchmark_data_smithi055_229117_object104207:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.042202+0000", 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.27807027, 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3303606880000001, 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104208 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.358 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042202+0000", 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042202+0000", 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042200+0000", 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042226+0000", 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5925e-05 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042245+0000", 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9449999999999998e-05 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042255+0000", 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9739999999999997e-06 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042301+0000", 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5945999999999997e-05 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.042309+0000", 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7130000000000005e-06 2023-09-23T18:02:12.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342600+0000", 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3002915850000001 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342725+0000", 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012474099999999999 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.342853+0000", 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000127806 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.353066+0000", 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010212738000000001 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355236+0000", 2023-09-23T18:02:12.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002170052 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369924+0000", 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014688012 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369984+0000", 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0571e-05 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372562+0000", 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00257807 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104228 5.0 5:04306094:::benchmark_data_smithi055_229117_object104227:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.063372+0000", 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.25689959800002, 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3247219290000001, 2023-09-23T18:02:12.365 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104228 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063372+0000", 2023-09-23T18:02:12.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063372+0000", 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063371+0000", 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063394+0000", 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2929999999999999e-05 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063407+0000", 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3142000000000001e-05 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063415+0000", 2023-09-23T18:02:12.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8120000000000008e-06 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063476+0000", 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0988e-05 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.063486+0000", 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0397999999999999e-05 2023-09-23T18:02:12.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345425+0000", 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2819386879999999 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345551+0000", 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012593200000000001 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.345665+0000", 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114308 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355247+0000", 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0095815529999999996 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355307+0000", 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0619999999999999e-05 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385736+0000", 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030428261000000002 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385803+0000", 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7624000000000002e-05 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388094+0000", 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002291015 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104253 5.0 5:0d680df2:::benchmark_data_smithi055_229117_object104252:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.088438+0000", 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.23183397999998, 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3025030870000001, 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.373 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104253 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088438+0000", 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088438+0000", 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088436+0000", 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088461+0000", 2023-09-23T18:02:12.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4658999999999999e-05 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088478+0000", 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7805e-05 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.088487+0000", 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.225e-06 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.092295+0000", 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038084870000000002 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.092306+0000", 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1172000000000001e-05 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355342+0000", 2023-09-23T18:02:12.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.263036211 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355533+0000", 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000190767 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.355663+0000", 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012937699999999999 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.365284+0000", 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0096211700000000001 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367649+0000", 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002365514 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388536+0000", 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020886503000000001 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.388577+0000", 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1167000000000001e-05 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.390941+0000", 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002364157 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.380 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104272 5.0 5:0ec51932:::benchmark_data_smithi055_229117_object104271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.108373+0000", 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.21189852999998, 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2933625550000001, 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.381 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104272 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108373+0000", 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108373+0000", 2023-09-23T18:02:12.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108372+0000", 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108396+0000", 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4311000000000002e-05 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108412+0000", 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6049000000000002e-05 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108420+0000", 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9170000000000006e-06 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108482+0000", 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1969000000000002e-05 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.108491+0000", 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4779999999999997e-06 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.358777+0000", 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2502855180000001 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.358912+0000", 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013505099999999999 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.359032+0000", 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120331 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367658+0000", 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0086259400000000003 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.367698+0000", 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9477999999999997e-05 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.398598+0000", 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030900179 2023-09-23T18:02:12.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.398665+0000", 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7167000000000003e-05 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401736+0000", 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030708630000000001 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104276 5.0 5:0ba41185:::benchmark_data_smithi055_229117_object104275:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.114398+0000", 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.20587353799999, 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2928085359999999, 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104276 2023-09-23T18:02:12.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114398+0000", 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114398+0000", 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114397+0000", 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114423+0000", 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6006000000000001e-05 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114440+0000", 2023-09-23T18:02:12.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6877000000000001e-05 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.114448+0000", 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5120000000000008e-06 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.115268+0000", 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00081976499999999997 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.115279+0000", 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0533999999999999e-05 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362251+0000", 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.24697258 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362402+0000", 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015118400000000001 2023-09-23T18:02:12.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.362530+0000", 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012761700000000001 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.369824+0000", 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072939889999999999 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372645+0000", 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028209210000000001 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.405151+0000", 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032506363000000003 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.405203+0000", 2023-09-23T18:02:12.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1987000000000002e-05 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407207+0000", 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002003777 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104298 5.0 5:040384f9:::benchmark_data_smithi055_229117_object104297:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.139530+0000", 2023-09-23T18:02:12.396 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.18074150899997, 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2697862209999999, 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104298 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139530+0000", 2023-09-23T18:02:12.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139530+0000", 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139529+0000", 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:12.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139553+0000", 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4026999999999999e-05 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139565+0000", 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2364999999999999e-05 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.139573+0000", 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4200000000000001e-06 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.141901+0000", 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023278639999999998 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.141909+0000", 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5099999999999998e-06 2023-09-23T18:02:12.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372679+0000", 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2307700370000001 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372803+0000", 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012344299999999999 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.372931+0000", 2023-09-23T18:02:12.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012818500000000001 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385669+0000", 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012738027000000001 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385682+0000", 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3368e-05 2023-09-23T18:02:12.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407273+0000", 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021590933 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.407320+0000", 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7305999999999997e-05 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409317+0000", 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019962769999999999 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104301 5.0 5:0653eda0:::benchmark_data_smithi055_229117_object104300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.142080+0000", 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.17819233900002, 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2806010670000001, 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.404 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104301 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142080+0000", 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142080+0000", 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142078+0000", 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142104+0000", 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5931e-05 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142116+0000", 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2262999999999999e-05 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142125+0000", 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5359999999999992e-06 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142238+0000", 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000113056 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.142247+0000", 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6680000000000007e-06 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.375988+0000", 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2337410959999999 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.376092+0000", 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104558 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.376220+0000", 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012734399999999999 2023-09-23T18:02:12.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.385691+0000", 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0094714069999999994 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391023+0000", 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053320190000000003 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.419987+0000", 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028963940000000001 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.420045+0000", 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8421000000000002e-05 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422681+0000", 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026352419999999999 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104305 5.0 5:03148f53:::benchmark_data_smithi055_229117_object104304:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.146575+0000", 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.17369690700002, 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2788299140000001, 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.412 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104305 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146575+0000", 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146575+0000", 2023-09-23T18:02:12.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146573+0000", 2023-09-23T18:02:12.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:12.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146600+0000", 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6675000000000001e-05 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146622+0000", 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1965000000000001e-05 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.146633+0000", 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.076e-05 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.147763+0000", 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011302879999999999 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.147777+0000", 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4129e-05 2023-09-23T18:02:12.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379012+0000", 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2312345389999999 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379258+0000", 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024643700000000001 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.379508+0000", 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024977000000000002 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.390966+0000", 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011458268000000001 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391089+0000", 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000122881 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422762+0000", 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031672697999999999 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.422832+0000", 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9929000000000001e-05 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.425405+0000", 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002573362 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104309 5.0 5:0de80b4f:::benchmark_data_smithi055_229117_object104308:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.152292+0000", 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.16797958199999, 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2892466869999999, 2023-09-23T18:02:12.420 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104309 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152292+0000", 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152292+0000", 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152291+0000", 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152313+0000", 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2082999999999998e-05 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152326+0000", 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2493999999999999e-05 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152332+0000", 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7710000000000001e-06 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152379+0000", 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6233999999999998e-05 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.152389+0000", 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0234999999999999e-05 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382503+0000", 2023-09-23T18:02:12.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.230114411 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382678+0000", 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000174704 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.382886+0000", 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000208481 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391101+0000", 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082150169999999998 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391162+0000", 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0718999999999999e-05 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.438904+0000", 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047742297000000003 2023-09-23T18:02:12.426 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.438971+0000", 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6787999999999998e-05 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441539+0000", 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025679069999999999 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104346 5.0 5:07875566:::benchmark_data_smithi055_229117_object104345:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.224009+0000", 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.09626263400003, 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2202360750000001, 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104346 2023-09-23T18:02:12.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224009+0000", 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224009+0000", 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224007+0000", 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224037+0000", 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9128e-05 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224058+0000", 2023-09-23T18:02:12.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1829999999999999e-05 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224068+0000", 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8560000000000002e-06 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224119+0000", 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0844000000000003e-05 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.224128+0000", 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4880000000000007e-06 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391218+0000", 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1670905920000001 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391390+0000", 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017210399999999999 2023-09-23T18:02:12.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.391557+0000", 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000166694 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409338+0000", 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017781376000000002 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413075+0000", 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037367199999999998 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441577+0000", 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028502169000000001 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.441616+0000", 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8859999999999997e-05 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444245+0000", 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026292899999999998 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104353 5.0 5:0b824901:::benchmark_data_smithi055_229117_object104352:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.238109+0000", 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.08216301200002, 2023-09-23T18:02:12.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.208967144, 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104353 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238109+0000", 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238109+0000", 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238108+0000", 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238128+0000", 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9854999999999999e-05 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238135+0000", 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1729999999999998e-06 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238140+0000", 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6199999999999998e-06 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238197+0000", 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7005000000000001e-05 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.238204+0000", 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2819999999999999e-06 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.394820+0000", 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.156615231 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.394956+0000", 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013646599999999999 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.395118+0000", 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000162158 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.409348+0000", 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014229883 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413141+0000", 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00379284 2023-09-23T18:02:12.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444312+0000", 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031170989 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.444373+0000", 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1309000000000005e-05 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447076+0000", 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027029129999999999 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104360 5.0 5:0a7d8202:::benchmark_data_smithi055_229117_object104359:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e143)", 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:25.266784+0000", 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 287.05348783300002, 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1830323949999999, 2023-09-23T18:02:12.443 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104360 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266784+0000", 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266784+0000", 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266782+0000", 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266806+0000", 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3794999999999998e-05 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266824+0000", 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7838000000000001e-05 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266833+0000", 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9469999999999992e-06 2023-09-23T18:02:12.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266890+0000", 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7238999999999999e-05 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:25.266902+0000", 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1919999999999999e-05 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401792+0000", 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.13489014 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.401976+0000", 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000184014 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.402162+0000", 2023-09-23T18:02:12.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018587599999999999 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.413089+0000", 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010926544999999999 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.425460+0000", 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012371320999999999 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447163+0000", 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021703538000000001 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.447231+0000", 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7675999999999994e-05 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.451 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:26.449817+0000", 2023-09-23T18:02:12.451 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025853859999999998 2023-09-23T18:02:12.451 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.451 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.451 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.451 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.451 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.451 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33540 6.e 6:757bf288:::benchmark_data_smithi055_275963_object33539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:02:12.451 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.437994+0000", 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 32.882277848000001, 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.108388157, 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33540 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.453 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.437994+0000", 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.437994+0000", 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.454 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.437991+0000", 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999967 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438029+0000", 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.824e-05 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438067+0000", 2023-09-23T18:02:12.455 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8182999999999997e-05 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438088+0000", 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0129999999999999e-05 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440561+0000", 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024736430000000002 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.456 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440572+0000", 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0878e-05 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.534316+0000", 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.093743492 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.457 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.534461+0000", 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014496100000000001 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.534632+0000", 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017151100000000001 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.539612+0000", 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049804710000000002 2023-09-23T18:02:12.458 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.543049+0000", 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003436588 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.544731+0000", 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016814320000000001 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.459 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.544791+0000", 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0006000000000001e-05 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546382+0000", 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001591765 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.460 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33544 6.e 6:7c426a47:::benchmark_data_smithi055_275963_object33543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.438104+0000", 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 32.882167723000002, 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.109720906, 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33544 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.461 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438104+0000", 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438104+0000", 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438095+0000", 2023-09-23T18:02:12.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999905 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438139+0000", 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3952999999999997e-05 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438945+0000", 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00080640499999999995 2023-09-23T18:02:12.463 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438968+0000", 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3285000000000002e-05 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440588+0000", 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001619896 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.464 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440592+0000", 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8349999999999997e-06 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.537672+0000", 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0970796060000001 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.537831+0000", 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015879199999999999 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.537997+0000", 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016669100000000001 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546403+0000", 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0084053589999999994 2023-09-23T18:02:12.466 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546451+0000", 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8115000000000003e-05 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546557+0000", 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010645300000000001 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.467 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546603+0000", 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6140000000000002e-05 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547825+0000", 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001221845 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.468 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33549 6.e 6:7eeaa6a3:::benchmark_data_smithi055_275963_object33548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.438977+0000", 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 32.881295113999997, 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.110446166, 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33549 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.469 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438977+0000", 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438977+0000", 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.470 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438976+0000", 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439029+0000", 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3094999999999999e-05 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439039+0000", 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6029999999999994e-06 2023-09-23T18:02:12.471 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439047+0000", 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5599999999999994e-06 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440614+0000", 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015667210000000001 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.472 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440620+0000", 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7189999999999998e-06 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.539732+0000", 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0991117319999999 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.539867+0000", 2023-09-23T18:02:12.473 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000135091 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.540051+0000", 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018418899999999999 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546410+0000", 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063587649999999997 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.474 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547897+0000", 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014873009999999999 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547939+0000", 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2076999999999997e-05 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547993+0000", 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4109e-05 2023-09-23T18:02:12.475 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.549423+0000", 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001429865 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33551 6.e 6:7ac7b330:::benchmark_data_smithi055_275963_object33550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.439054+0000", 2023-09-23T18:02:12.476 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 32.881218066999999, 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.113266128, 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33551 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439054+0000", 2023-09-23T18:02:12.477 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439054+0000", 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439053+0000", 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.478 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439099+0000", 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5565999999999997e-05 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439108+0000", 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.789e-06 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439114+0000", 2023-09-23T18:02:12.479 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.567e-06 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440644+0000", 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001530127 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440649+0000", 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.211e-06 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.541379+0000", 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.100730271 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.541553+0000", 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017443600000000001 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.541772+0000", 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021913 2023-09-23T18:02:12.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546460+0000", 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004687241 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.549485+0000", 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030253210000000001 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.482 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.551089+0000", 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016042529999999999 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.551163+0000", 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3696999999999996e-05 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:12.483 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.552320+0000", 2023-09-23T18:02:12.484 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00115711 2023-09-23T18:02:12.484 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.484 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.484 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.484 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:12.484 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:12.484 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:02:12.484 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:11 smithi099 ceph-mon[167076]: pgmap v538: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 4 activating+undersized+remapped, 38 active+clean+remapped, 63 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 15 MiB/s wr, 246 op/s; 26774/194391 objects degraded (13.773%); 28249/194391 objects misplaced (14.532%); 37 MiB/s, 598 objects/s recovering 2023-09-23T18:02:12.485 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:11 smithi099 ceph-mon[167076]: failed to encode map e233 with expected crc 2023-09-23T18:02:12.485 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:11 smithi099 ceph-mon[167076]: osdmap e233: 8 total, 7 up, 7 in 2023-09-23T18:02:12.485 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:11 smithi099 ceph-mon[167076]: failed to encode map e233 with expected crc 2023-09-23T18:02:12.485 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:12.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:12.486 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:12.486 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:12.486 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:12.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:12.623 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:12.633 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:12 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[287212]: 2023-09-23T18:02:12.430+0000 7f250ae8a640 -1 Falling back to public interface 2023-09-23T18:02:12.636 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:02:12.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:12.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:12.671 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:12.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:12.816 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:12.849 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:12.863 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:12.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:12.870 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:12.971 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:13.004 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:13.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 69 16 65891 65875 59.6516 25.4375 0.336221 0.0167221 2023-09-23T18:02:13.194 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:13.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:13.199 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:13.236 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:02:13.271 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:13 smithi055 ceph-mon[184174]: failed to encode map e234 with expected crc 2023-09-23T18:02:13.273 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:13 smithi055 ceph-mon[184174]: osdmap e234: 8 total, 7 up, 7 in 2023-09-23T18:02:13.273 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:13 smithi055 ceph-mon[184174]: failed to encode map e234 with expected crc 2023-09-23T18:02:13.273 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 26774/194391 objects degraded (13.773%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:13.274 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:13 smithi055 ceph-mon[179668]: failed to encode map e234 with expected crc 2023-09-23T18:02:13.274 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:13 smithi055 ceph-mon[179668]: osdmap e234: 8 total, 7 up, 7 in 2023-09-23T18:02:13.274 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:13 smithi055 ceph-mon[179668]: failed to encode map e234 with expected crc 2023-09-23T18:02:13.274 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 26774/194391 objects degraded (13.773%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:13.300 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:13.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:13 smithi099 ceph-mon[167076]: failed to encode map e234 with expected crc 2023-09-23T18:02:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:13 smithi099 ceph-mon[167076]: osdmap e234: 8 total, 7 up, 7 in 2023-09-23T18:02:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:13 smithi099 ceph-mon[167076]: failed to encode map e234 with expected crc 2023-09-23T18:02:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 26774/194391 objects degraded (13.773%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:13.332 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:13.370 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:13.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:13.375 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:13.477 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:13.515 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:13.989 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:13.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:13.996 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:14.014 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 70 16 66279 66263 59.1459 24.25 0.00757535 0.0168844 2023-09-23T18:02:14.097 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:14.136 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:14.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:14.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:14.181 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:14.193 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:14 smithi055 ceph-mon[184174]: pgmap v541: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 4 activating+undersized+remapped, 38 active+clean+remapped, 63 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 16 MiB/s wr, 261 op/s; 26825/195687 objects degraded (13.708%); 28148/195687 objects misplaced (14.384%); 45 MiB/s, 728 objects/s recovering 2023-09-23T18:02:14.193 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:14 smithi055 ceph-mon[179668]: pgmap v541: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 4 activating+undersized+remapped, 38 active+clean+remapped, 63 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 16 MiB/s wr, 261 op/s; 26825/195687 objects degraded (13.708%); 28148/195687 objects misplaced (14.384%); 45 MiB/s, 728 objects/s recovering 2023-09-23T18:02:14.212 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:14.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:14.314 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:14 smithi099 ceph-mon[167076]: pgmap v541: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 4 activating+undersized+remapped, 38 active+clean+remapped, 63 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 16 MiB/s wr, 261 op/s; 26825/195687 objects degraded (13.708%); 28148/195687 objects misplaced (14.384%); 45 MiB/s, 728 objects/s recovering 2023-09-23T18:02:14.315 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:14.317 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:14.368 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:14.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:14.376 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:14.414 INFO:teuthology.orchestra.run.smithi055.stderr:no valid command found; 10 closest matches: 2023-09-23T18:02:14.414 INFO:teuthology.orchestra.run.smithi055.stderr:0 2023-09-23T18:02:14.414 INFO:teuthology.orchestra.run.smithi055.stderr:1 2023-09-23T18:02:14.415 INFO:teuthology.orchestra.run.smithi055.stderr:2 2023-09-23T18:02:14.415 INFO:teuthology.orchestra.run.smithi055.stderr:abort 2023-09-23T18:02:14.415 INFO:teuthology.orchestra.run.smithi055.stderr:assert 2023-09-23T18:02:14.415 INFO:teuthology.orchestra.run.smithi055.stderr:config diff 2023-09-23T18:02:14.415 INFO:teuthology.orchestra.run.smithi055.stderr:config diff get 2023-09-23T18:02:14.415 INFO:teuthology.orchestra.run.smithi055.stderr:config get 2023-09-23T18:02:14.415 INFO:teuthology.orchestra.run.smithi055.stderr:config help [] 2023-09-23T18:02:14.415 INFO:teuthology.orchestra.run.smithi055.stderr:config set ... 2023-09-23T18:02:14.415 INFO:teuthology.orchestra.run.smithi055.stderr:admin_socket: invalid command 2023-09-23T18:02:14.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:14.517 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:14.568 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:14.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:14.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:14.674 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:14.738 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:14.789 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:14.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:14.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:14.895 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:14.934 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:14.982 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:14.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:14.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:15.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 71 16 66802 66786 58.7733 32.6875 0.00974766 0.0169999 2023-09-23T18:02:15.061 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:02:15.064 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T18:02:15.064 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3967009847' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:02:15.064 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:15 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:02:15.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3967009847' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:02:15.065 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:15 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:02:15.065 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-3, ['dump_ops_in_flight'] 2023-09-23T18:02:15.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:15.130 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:15.181 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:15.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:15.184 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:15.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:15.311 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3967009847' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:02:15.312 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:15 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:02:15.314 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:15.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:15.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:15.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:15.462 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:15.497 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:15.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:15.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:15.553 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:15.653 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:15.679 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:15 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[287212]: 2023-09-23T18:02:15.475+0000 7f250ae8a640 -1 osd.3 141 log_to_monitors true 2023-09-23T18:02:15.686 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:15.847 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:15.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:15.852 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:15.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:15.992 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:16.012 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 72 16 67201 67185 58.3034 24.9375 0.00348949 0.0171238 2023-09-23T18:02:16.045 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:16.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:16.049 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:16.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:16.181 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:16.225 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:16.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:16.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:16 smithi099 ceph-mon[167076]: pgmap v542: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 38 active+clean+remapped, 67 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 21 MiB/s wr, 340 op/s; 27084/197196 objects degraded (13.735%); 28231/197196 objects misplaced (14.316%); 42 MiB/s, 670 objects/s recovering 2023-09-23T18:02:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:16 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:16 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:02:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:16 smithi099 ceph-mon[167076]: osdmap e235: 8 total, 7 up, 7 in 2023-09-23T18:02:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:16 smithi099 ceph-mon[167076]: failed to encode map e235 with expected crc 2023-09-23T18:02:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:16 smithi099 ceph-mon[167076]: failed to encode map e235 with expected crc 2023-09-23T18:02:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:16 smithi099 ceph-mon[167076]: from='osd.3 [v2:172.21.15.55:6826/958494807,v1:172.21.15.55:6827/958494807]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:02:16.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:16 smithi099 ceph-mon[167076]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:02:16.330 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:16.362 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[287212]: 2023-09-23T18:02:16.148+0000 7f24faaf4700 -1 osd.3 236 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:02:16.362 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[179668]: pgmap v542: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 38 active+clean+remapped, 67 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 21 MiB/s wr, 340 op/s; 27084/197196 objects degraded (13.735%); 28231/197196 objects misplaced (14.316%); 42 MiB/s, 670 objects/s recovering 2023-09-23T18:02:16.362 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:16.362 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:02:16.362 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[179668]: osdmap e235: 8 total, 7 up, 7 in 2023-09-23T18:02:16.362 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[179668]: failed to encode map e235 with expected crc 2023-09-23T18:02:16.362 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[179668]: failed to encode map e235 with expected crc 2023-09-23T18:02:16.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[179668]: from='osd.3 [v2:172.21.15.55:6826/958494807,v1:172.21.15.55:6827/958494807]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:02:16.363 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[179668]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:02:16.363 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[184174]: pgmap v542: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+remapped+backfilling, 38 active+clean+remapped, 67 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 21 MiB/s wr, 340 op/s; 27084/197196 objects degraded (13.735%); 28231/197196 objects misplaced (14.316%); 42 MiB/s, 670 objects/s recovering 2023-09-23T18:02:16.363 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:16.363 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:02:16.363 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[184174]: osdmap e235: 8 total, 7 up, 7 in 2023-09-23T18:02:16.363 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[184174]: failed to encode map e235 with expected crc 2023-09-23T18:02:16.364 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[184174]: failed to encode map e235 with expected crc 2023-09-23T18:02:16.364 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[184174]: from='osd.3 [v2:172.21.15.55:6826/958494807,v1:172.21.15.55:6827/958494807]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:02:16.364 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:16 smithi055 ceph-mon[184174]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-09-23T18:02:16.371 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:16.379 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:02:16.407 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:02:16.434 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:02:16.456 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:02:16.469 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:16.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:16.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:16.478 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:02:16.503 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:02:16.526 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:02:16.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:02:16.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:02:16.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:02:16.526 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.3 2023-09-23T18:02:16.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:02:16.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:02:16.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:02:16.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:02:16.573 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:16.588 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:02:16.588 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:02:16.610 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:16.915 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:16.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:16.919 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:16.933 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:02:16.934 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:02:17.013 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 73 16 67493 67477 57.7548 18.25 0.00519346 0.0172804 2023-09-23T18:02:17.020 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:17.092 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:17.139 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:17.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:17.143 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[179668]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T18:02:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[179668]: osdmap e236: 8 total, 7 up, 7 in 2023-09-23T18:02:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[179668]: from='osd.3 [v2:172.21.15.55:6826/958494807,v1:172.21.15.55:6827/958494807]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:02:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[179668]: failed to encode map e236 with expected crc 2023-09-23T18:02:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[179668]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:02:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[179668]: failed to encode map e236 with expected crc 2023-09-23T18:02:17.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4293931822' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[184174]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T18:02:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[184174]: osdmap e236: 8 total, 7 up, 7 in 2023-09-23T18:02:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[184174]: from='osd.3 [v2:172.21.15.55:6826/958494807,v1:172.21.15.55:6827/958494807]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:02:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[184174]: failed to encode map e236 with expected crc 2023-09-23T18:02:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[184174]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:02:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[184174]: failed to encode map e236 with expected crc 2023-09-23T18:02:17.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4293931822' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:17.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:17.273 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:17.313 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:17.316 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:17 smithi099 ceph-mon[167076]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-09-23T18:02:17.316 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:17 smithi099 ceph-mon[167076]: osdmap e236: 8 total, 7 up, 7 in 2023-09-23T18:02:17.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:17 smithi099 ceph-mon[167076]: from='osd.3 [v2:172.21.15.55:6826/958494807,v1:172.21.15.55:6827/958494807]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:02:17.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:17 smithi099 ceph-mon[167076]: failed to encode map e236 with expected crc 2023-09-23T18:02:17.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:17 smithi099 ceph-mon[167076]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:02:17.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:17 smithi099 ceph-mon[167076]: failed to encode map e236 with expected crc 2023-09-23T18:02:17.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4293931822' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:17.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:17.318 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:17.336 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:02:17.419 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:17.450 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:17.488 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:17.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:17.491 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:17.524 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:02:17.592 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:17.619 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:17.654 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:17.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:17.657 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:17.758 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:17.785 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:17.820 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:17.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:17.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:17.924 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:17.953 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:17.979 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:18.013 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 74 16 67517 67501 56.9948 1.5 0.00368697 0.0173384 2023-09-23T18:02:18.076 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:18.079 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: pgmap v545: 119 pgs: 38 active+clean+remapped, 67 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 15 MiB/s wr, 237 op/s; 27367/198441 objects degraded (13.791%); 27857/198441 objects misplaced (14.038%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T18:02:18.079 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:02:18.079 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: failed to encode map e237 with expected crc 2023-09-23T18:02:18.079 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: failed to encode map e237 with expected crc 2023-09-23T18:02:18.079 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: osd.3 [v2:172.21.15.55:6826/958494807,v1:172.21.15.55:6827/958494807] boot 2023-09-23T18:02:18.079 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T18:02:18.080 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:02:18.080 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: failed to encode map e237 with expected crc 2023-09-23T18:02:18.080 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 27367/198441 objects degraded (13.791%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:18.080 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: failed to encode map e238 with expected crc 2023-09-23T18:02:18.080 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T18:02:18.080 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[184174]: 6.4 starting backfill to osd.3 from (0'0,0'0] MAX to 236'4219 2023-09-23T18:02:18.080 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: pgmap v545: 119 pgs: 38 active+clean+remapped, 67 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 15 MiB/s wr, 237 op/s; 27367/198441 objects degraded (13.791%); 27857/198441 objects misplaced (14.038%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T18:02:18.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:02:18.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: failed to encode map e237 with expected crc 2023-09-23T18:02:18.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: failed to encode map e237 with expected crc 2023-09-23T18:02:18.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: osd.3 [v2:172.21.15.55:6826/958494807,v1:172.21.15.55:6827/958494807] boot 2023-09-23T18:02:18.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T18:02:18.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:02:18.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: failed to encode map e237 with expected crc 2023-09-23T18:02:18.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 27367/198441 objects degraded (13.791%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:18.081 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: failed to encode map e238 with expected crc 2023-09-23T18:02:18.082 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T18:02:18.082 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:18 smithi055 ceph-mon[179668]: 6.4 starting backfill to osd.3 from (0'0,0'0] MAX to 236'4219 2023-09-23T18:02:18.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:18.083 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:18.180 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:18.184 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:18.217 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:18.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: pgmap v545: 119 pgs: 38 active+clean+remapped, 67 active+clean, 1 active+recovering+undersized+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+undersized+degraded; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 15 MiB/s wr, 237 op/s; 27367/198441 objects degraded (13.791%); 27857/198441 objects misplaced (14.038%); 8.1 MiB/s, 129 objects/s recovering 2023-09-23T18:02:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:02:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: failed to encode map e237 with expected crc 2023-09-23T18:02:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: failed to encode map e237 with expected crc 2023-09-23T18:02:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: osd.3 [v2:172.21.15.55:6826/958494807,v1:172.21.15.55:6827/958494807] boot 2023-09-23T18:02:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: osdmap e237: 8 total, 8 up, 7 in 2023-09-23T18:02:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-09-23T18:02:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: failed to encode map e237 with expected crc 2023-09-23T18:02:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 27367/198441 objects degraded (13.791%), 9 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:18.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: failed to encode map e238 with expected crc 2023-09-23T18:02:18.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: osdmap e238: 8 total, 8 up, 7 in 2023-09-23T18:02:18.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:18 smithi099 ceph-mon[167076]: 6.4 starting backfill to osd.3 from (0'0,0'0] MAX to 236'4219 2023-09-23T18:02:18.380 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:18.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:18.383 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:18.483 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:18.512 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:18.548 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:18.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:18.550 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:18.651 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:18.678 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:18.714 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:18.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:18.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:18.817 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:18.846 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:18.934 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:02:19.013 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 75 16 67522 67506 56.2392 0.3125 0.00874594 0.0173706 2023-09-23T18:02:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:19 smithi099 ceph-mon[167076]: failed to encode map e238 with expected crc 2023-09-23T18:02:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:19 smithi099 ceph-mon[167076]: 6.b starting backfill to osd.3 from (0'0,0'0] MAX to 236'4165 2023-09-23T18:02:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:19 smithi099 ceph-mon[167076]: failed to encode map e238 with expected crc 2023-09-23T18:02:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:19 smithi099 ceph-mon[167076]: 6.4 continuing backfill to osd.6 from (216'100,236'4219] 6:20225563:::benchmark_data_smithi055_275963_object20156:head to 236'4219 2023-09-23T18:02:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:19 smithi099 ceph-mon[167076]: failed to encode map e239 with expected crc 2023-09-23T18:02:19.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:19 smithi099 ceph-mon[167076]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T18:02:19.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:19 smithi099 ceph-mon[167076]: failed to encode map e239 with expected crc 2023-09-23T18:02:19.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:19 smithi099 ceph-mon[167076]: failed to encode map e239 with expected crc 2023-09-23T18:02:19.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:19 smithi099 ceph-mon[167076]: 6.c starting backfill to osd.3 from (0'0,0'0] MAX to 236'4181 2023-09-23T18:02:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[179668]: failed to encode map e238 with expected crc 2023-09-23T18:02:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[179668]: 6.b starting backfill to osd.3 from (0'0,0'0] MAX to 236'4165 2023-09-23T18:02:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[179668]: failed to encode map e238 with expected crc 2023-09-23T18:02:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[179668]: 6.4 continuing backfill to osd.6 from (216'100,236'4219] 6:20225563:::benchmark_data_smithi055_275963_object20156:head to 236'4219 2023-09-23T18:02:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[179668]: failed to encode map e239 with expected crc 2023-09-23T18:02:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[179668]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T18:02:19.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[179668]: failed to encode map e239 with expected crc 2023-09-23T18:02:19.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[179668]: failed to encode map e239 with expected crc 2023-09-23T18:02:19.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[179668]: 6.c starting backfill to osd.3 from (0'0,0'0] MAX to 236'4181 2023-09-23T18:02:19.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[184174]: failed to encode map e238 with expected crc 2023-09-23T18:02:19.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[184174]: 6.b starting backfill to osd.3 from (0'0,0'0] MAX to 236'4165 2023-09-23T18:02:19.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[184174]: failed to encode map e238 with expected crc 2023-09-23T18:02:19.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[184174]: 6.4 continuing backfill to osd.6 from (216'100,236'4219] 6:20225563:::benchmark_data_smithi055_275963_object20156:head to 236'4219 2023-09-23T18:02:19.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[184174]: failed to encode map e239 with expected crc 2023-09-23T18:02:19.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[184174]: osdmap e239: 8 total, 8 up, 7 in 2023-09-23T18:02:19.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[184174]: failed to encode map e239 with expected crc 2023-09-23T18:02:19.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[184174]: failed to encode map e239 with expected crc 2023-09-23T18:02:19.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:19 smithi055 ceph-mon[184174]: 6.c starting backfill to osd.3 from (0'0,0'0] MAX to 236'4181 2023-09-23T18:02:20.013 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 76 16 67823 67807 55.7468 18.8125 0.00500848 0.0179038 2023-09-23T18:02:20.067 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:02:20.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:20 smithi099 ceph-mon[167076]: pgmap v548: 119 pgs: 1 peering, 19 remapped+peering, 20 active+clean+remapped, 67 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+undersized+degraded; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 353 op/s; 19126/201435 objects degraded (9.495%); 11138/201435 objects misplaced (5.529%); 11 MiB/s, 181 objects/s recovering 2023-09-23T18:02:20.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:20 smithi099 ceph-mon[167076]: failed to encode map e239 with expected crc 2023-09-23T18:02:20.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:20 smithi099 ceph-mon[167076]: failed to encode map e239 with expected crc 2023-09-23T18:02:20.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:20 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:02:20.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:20 smithi055 ceph-mon[184174]: pgmap v548: 119 pgs: 1 peering, 19 remapped+peering, 20 active+clean+remapped, 67 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+undersized+degraded; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 353 op/s; 19126/201435 objects degraded (9.495%); 11138/201435 objects misplaced (5.529%); 11 MiB/s, 181 objects/s recovering 2023-09-23T18:02:20.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:20 smithi055 ceph-mon[184174]: failed to encode map e239 with expected crc 2023-09-23T18:02:20.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:20 smithi055 ceph-mon[184174]: failed to encode map e239 with expected crc 2023-09-23T18:02:20.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:20 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:02:20.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:20 smithi055 ceph-mon[179668]: pgmap v548: 119 pgs: 1 peering, 19 remapped+peering, 20 active+clean+remapped, 67 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 5 active+undersized+degraded; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 353 op/s; 19126/201435 objects degraded (9.495%); 11138/201435 objects misplaced (5.529%); 11 MiB/s, 181 objects/s recovering 2023-09-23T18:02:20.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:20 smithi055 ceph-mon[179668]: failed to encode map e239 with expected crc 2023-09-23T18:02:20.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:20 smithi055 ceph-mon[179668]: failed to encode map e239 with expected crc 2023-09-23T18:02:20.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:20 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:02:20.570 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:20.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:20.573 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:20.674 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:20.687 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:02:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:02:20] "GET /metrics HTTP/1.1" 200 36544 "" "Prometheus/2.43.0" 2023-09-23T18:02:20.766 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:20.975 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:20.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:20.984 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:21.014 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 77 16 68274 68258 55.3877 28.1875 0.0135683 0.0180375 2023-09-23T18:02:21.085 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:21.137 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:21.149 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:02:21.149 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:02:21.170 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:21.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:21.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:21.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:21.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:21.364 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.440 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:21.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:21.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:21.460 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.460 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.460 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.460 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:21.558 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:21.605 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:21.613 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:21.710 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:21.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:21.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:21.722 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:02:21.816 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:21.847 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:22.015 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 78 16 68693 68677 55.0134 26.1875 0.00335098 0.0181387 2023-09-23T18:02:22.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:22 smithi055 ceph-mon[179668]: pgmap v550: 119 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 19 remapped+peering, 8 active+clean+remapped, 79 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 19 MiB/s wr, 309 op/s; 22646/202737 objects degraded (11.170%); 6363/202737 objects misplaced (3.139%); 14 MiB/s, 224 objects/s recovering 2023-09-23T18:02:22.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:22.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:22.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2271730483' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:22.261 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:22.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:22.266 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:22.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:22.424 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:02:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:22 smithi055 ceph-mon[184174]: pgmap v550: 119 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 19 remapped+peering, 8 active+clean+remapped, 79 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 19 MiB/s wr, 309 op/s; 22646/202737 objects degraded (11.170%); 6363/202737 objects misplaced (3.139%); 14 MiB/s, 224 objects/s recovering 2023-09-23T18:02:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:22.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2271730483' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:22.435 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:22 smithi099 ceph-mon[167076]: pgmap v550: 119 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 19 remapped+peering, 8 active+clean+remapped, 79 active+clean, 1 active+recovering+undersized+remapped, 3 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+degraded; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 19 MiB/s wr, 309 op/s; 22646/202737 objects degraded (11.170%); 6363/202737 objects misplaced (3.139%); 14 MiB/s, 224 objects/s recovering 2023-09-23T18:02:22.435 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:22.436 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:22.436 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2271730483' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:22.437 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:22.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:22.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:22.525 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:22.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:22.682 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:22.734 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:22.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:22.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:22.840 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:22.875 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:23.013 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:23.015 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 79 16 69054 69038 54.6027 22.5625 0.291207 0.0182836 2023-09-23T18:02:23.015 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:23.016 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:23.097 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:02:23.097 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:02:23.097 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:23.097 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:69081 6.8 e240/238 6:1d77f467:::benchmark_data_smithi055_275963_object69080:head v 240'4364, mlcod=0'0)", 2023-09-23T18:02:23.098 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:23.095702+0000", 2023-09-23T18:02:23.098 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0018986179999999999, 2023-09-23T18:02:23.098 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001939386, 2023-09-23T18:02:23.098 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:23.098 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:02:23.098 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:23.098 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:23.098 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:23.098 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:23.095702+0000", 2023-09-23T18:02:23.098 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:23.095702+0000", 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:23.095700+0000", 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:02:23.099 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:23.095729+0000", 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.883e-05 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:23.095752+0000", 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2316000000000001e-05 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:23.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:23.095763+0000", 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1416e-05 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:23.095813+0000", 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9475999999999999e-05 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:23.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:23.095836+0000", 2023-09-23T18:02:23.102 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3456999999999999e-05 2023-09-23T18:02:23.102 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:23.102 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:23.102 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:23.102 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:23.102 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:02:23.102 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 1 2023-09-23T18:02:23.102 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:02:23.116 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:23.152 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:23.253 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:23.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:23.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:23.357 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:23.413 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:23.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:23 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:23.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:23 smithi055 ceph-mon[184174]: failed to encode map e240 with expected crc 2023-09-23T18:02:23.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:23 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2271730483' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:23.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:23 smithi055 ceph-mon[184174]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T18:02:23.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 22646/202737 objects degraded (11.170%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:23.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:23 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:23.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:23 smithi055 ceph-mon[179668]: failed to encode map e240 with expected crc 2023-09-23T18:02:23.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:23 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2271730483' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:23.427 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:23 smithi055 ceph-mon[179668]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T18:02:23.427 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 22646/202737 objects degraded (11.170%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:23.461 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:23.464 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:23 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:23.464 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:23 smithi099 ceph-mon[167076]: failed to encode map e240 with expected crc 2023-09-23T18:02:23.464 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:23 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2271730483' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:23.464 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:23 smithi099 ceph-mon[167076]: osdmap e240: 8 total, 8 up, 7 in 2023-09-23T18:02:23.464 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 22646/202737 objects degraded (11.170%), 8 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T18:02:23.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:23.465 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:23.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:23.603 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:23.642 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:23.650 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:23.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:23.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:23.680 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T18:02:23.760 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:23.803 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:24.004 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:24.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:24.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:24.015 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:02:24.016077+0000 min lat: 0.00103291 max lat: 21.0159 avg lat: 0.0184361 2023-09-23T18:02:24.015 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:02:24.015 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 80 16 69357 69341 54.1569 18.9375 0.271013 0.0184361 2023-09-23T18:02:24.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:24.148 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:24.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:24.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:24.205 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:24.306 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:24.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:24 smithi055 ceph-mon[179668]: failed to encode map e240 with expected crc 2023-09-23T18:02:24.355 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:24 smithi055 ceph-mon[179668]: pgmap v552: 119 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 1 remapped+peering, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 24 MiB/s wr, 382 op/s; 39630/204897 objects degraded (19.341%); 10284/204897 objects misplaced (5.019%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:02:24.355 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:24 smithi055 ceph-mon[184174]: failed to encode map e240 with expected crc 2023-09-23T18:02:24.355 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:24 smithi055 ceph-mon[184174]: pgmap v552: 119 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 1 remapped+peering, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 24 MiB/s wr, 382 op/s; 39630/204897 objects degraded (19.341%); 10284/204897 objects misplaced (5.019%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:02:24.359 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:24.459 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:24.463 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:02:24.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:24.467 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:24.497 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:02:24.525 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:02:24.550 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:02:24.568 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:24.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:24 smithi099 ceph-mon[167076]: failed to encode map e240 with expected crc 2023-09-23T18:02:24.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:24 smithi099 ceph-mon[167076]: pgmap v552: 119 pgs: 2 activating+undersized+degraded+remapped, 1 active+recovery_wait+degraded, 1 peering, 1 remapped+peering, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 24 MiB/s wr, 382 op/s; 39630/204897 objects degraded (19.341%); 10284/204897 objects misplaced (5.019%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:02:24.578 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:02:24.605 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:24.607 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:02:24.634 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:02:24.662 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:02:24.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:02:24.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:02:24.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:02:24.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:02:24.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:02:24.665 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:02:24.665 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:02:24.665 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:02:24.864 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:24.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:24.867 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:24.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:25.000 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:25.015 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 81 16 69645 69629 53.7106 18 0.359466 0.0185756 2023-09-23T18:02:25.042 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:25.050 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:25.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:25.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:25.108 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:02:25.155 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:25.237 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:25.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:25 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1811128528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:25.338 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:25 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1811128528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:25.339 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:25 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1811128528' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:25.462 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:25.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:25.470 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:25.571 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:25.595 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:02:25.595 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:02:25.595 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104098 5.a 5:5150641c:::benchmark_data_smithi055_229117_object104097:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.920515+0000", 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 301.67049324499999, 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.048992793, 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.596 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104098 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T18:02:25.597 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920538+0000", 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2770000000000001e-05 2023-09-23T18:02:25.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920545+0000", 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1969999999999999e-06 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920550+0000", 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3360000000000002e-06 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927587+0000", 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0070371599999999998 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927719+0000", 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013187600000000001 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927856+0000", 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013721800000000001 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936497+0000", 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.008640277 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936670+0000", 2023-09-23T18:02:25.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017294700000000001 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959038+0000", 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022367973999999999 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959048+0000", 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0428000000000001e-05 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969404+0000", 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0103555609999999 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969508+0000", 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010461799999999999 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.603 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104100 5.a 5:58ebf1c5:::benchmark_data_smithi055_229117_object104099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.920555+0000", 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 301.67045356599999, 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0492678900000001, 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.604 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.605 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104100 2023-09-23T18:02:25.605 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.605 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.605 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.605 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.605 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T18:02:25.605 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.605 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.605 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.605 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920574+0000", 2023-09-23T18:02:25.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9664e-05 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920581+0000", 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3260000000000001e-06 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920586+0000", 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7239999999999997e-06 2023-09-23T18:02:25.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.931735+0000", 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011148345 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.931905+0000", 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000170466 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.932081+0000", 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000175623 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936776+0000", 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0046956209999999996 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.609 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936996+0000", 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021987100000000001 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959173+0000", 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022176392999999999 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959189+0000", 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.609e-05 2023-09-23T18:02:25.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969761+0000", 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0105722159999999 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969823+0000", 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1938000000000002e-05 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.611 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104109 5.a 5:542cd2b3:::benchmark_data_smithi055_229117_object104108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.944006+0000", 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 301.64700300099997, 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0419720189999999, 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.612 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104109 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944006+0000", 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944006+0000", 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944004+0000", 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944029+0000", 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5201000000000001e-05 2023-09-23T18:02:25.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944043+0000", 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3964e-05 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944062+0000", 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8190999999999999e-05 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944258+0000", 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019652100000000001 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944396+0000", 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013821100000000001 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944572+0000", 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017574299999999999 2023-09-23T18:02:25.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.956893+0000", 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012321445 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959227+0000", 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023331559999999999 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959235+0000", 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1000000000000004e-06 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969881+0000", 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.010645899 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:02:25.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.970644+0000", 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000763867 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.985881+0000", 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015236622 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.985978+0000", 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6602999999999996e-05 2023-09-23T18:02:25.619 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104112 5.a 5:5b2e2197:::benchmark_data_smithi055_229117_object104111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.944077+0000", 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 301.64693186, 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.042230502, 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.620 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104112 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944077+0000", 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.621 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944077+0000", 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944076+0000", 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944095+0000", 2023-09-23T18:02:25.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8532000000000001e-05 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944103+0000", 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8399999999999995e-06 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944108+0000", 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8510000000000004e-06 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.623 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.949704+0000", 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055955110000000001 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.949880+0000", 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000176312 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:02:25.624 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.950043+0000", 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016271800000000001 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959270+0000", 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092269080000000007 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959278+0000", 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0590000000000003e-06 2023-09-23T18:02:25.625 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.970705+0000", 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.011427144 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971279+0000", 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00057403800000000002 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.626 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986229+0000", 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014949831 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986307+0000", 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8319999999999996e-05 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.627 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104117 5.a 5:5725f10b:::benchmark_data_smithi055_229117_object104116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.952810+0000", 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 301.63819875899998, 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.039709502, 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104117 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.628 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952810+0000", 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952810+0000", 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.629 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952808+0000", 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952833+0000", 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5307999999999999e-05 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952845+0000", 2023-09-23T18:02:25.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2145e-05 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952855+0000", 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2730000000000008e-06 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959408+0000", 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0065535530000000002 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.631 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959416+0000", 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9529999999999999e-06 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971413+0000", 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0119968539999999 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971561+0000", 2023-09-23T18:02:25.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014827199999999999 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971748+0000", 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000186701 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986344+0000", 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014596067000000001 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992361+0000", 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060167379999999998 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992422+0000", 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1598000000000002e-05 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.634 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992519+0000", 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6966999999999995e-05 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154239 5.d 5:b6c40944:::benchmark_data_smithi055_229117_object48613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.75121539400001, 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5202187069999997, 2023-09-23T18:02:25.635 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154239 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.636 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839792+0000", 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839799+0000", 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2969999999999998e-06 2023-09-23T18:02:25.637 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839815+0000", 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5458e-05 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839823+0000", 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.833e-06 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.638 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839864+0000", 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0865999999999998e-05 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840002+0000", 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013775199999999999 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840128+0000", 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012573200000000001 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.845223+0000", 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050947299999999996 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.854611+0000", 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093881209999999993 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.854622+0000", 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1011e-05 2023-09-23T18:02:25.640 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851119+0000", 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.996496979 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851132+0000", 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.312e-05 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.340349+0000", 2023-09-23T18:02:25.641 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4892177549999999 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.340443+0000", 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3108000000000004e-05 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.359910+0000", 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019467288999999999 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.642 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360012+0000", 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000102093 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154248 5.d 5:bb80aea8:::benchmark_data_smithi055_229117_object48622:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.855620+0000", 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.73538906900001, 2023-09-23T18:02:25.643 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5213527490000001, 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154248 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855620+0000", 2023-09-23T18:02:25.644 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855620+0000", 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855618+0000", 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.645 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855624+0000", 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3399999999999997e-06 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855643+0000", 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9772999999999999e-05 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855653+0000", 2023-09-23T18:02:25.646 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4870000000000008e-06 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855688+0000", 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5308999999999997e-05 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855697+0000", 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6910000000000003e-06 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.647 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851160+0000", 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99546285999999995 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851169+0000", 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8920000000000006e-06 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.341897+0000", 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4907282319999999 2023-09-23T18:02:25.648 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.342015+0000", 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118402 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.342112+0000", 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6391000000000006e-05 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.649 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356673+0000", 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014561579999999999 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356774+0000", 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010047699999999999 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.375006+0000", 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018232224000000002 2023-09-23T18:02:25.650 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.375055+0000", 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9345999999999998e-05 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.376972+0000", 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001916966 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.651 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154254 5.4 5:2eea31d3:::benchmark_data_smithi055_229117_object48628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.876460+0000", 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.714548334, 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2290530139999998, 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154254 2023-09-23T18:02:25.652 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876460+0000", 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876460+0000", 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.653 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876458+0000", 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876469+0000", 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0913999999999999e-05 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876489+0000", 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9649000000000001e-05 2023-09-23T18:02:25.654 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876500+0000", 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1600000000000001e-05 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876561+0000", 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0857999999999999e-05 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.655 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876573+0000", 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1836e-05 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851144+0000", 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97457099800000002 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851155+0000", 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1548e-05 2023-09-23T18:02:25.656 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.095633+0000", 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.244477501 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.095782+0000", 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014866200000000001 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T18:02:25.657 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.095879+0000", 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7288999999999996e-05 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100460+0000", 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045811979999999999 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100470+0000", 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6609999999999995e-06 2023-09-23T18:02:25.658 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105323+0000", 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048527759999999996 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105356+0000", 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3546e-05 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.659 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105407+0000", 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1372000000000002e-05 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105513+0000", 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010586699999999999 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.660 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154255 5.d 5:bf0b5457:::benchmark_data_smithi055_229117_object48629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.876519+0000", 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.714489845, 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.514717928, 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154255 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.661 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876519+0000", 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876519+0000", 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876518+0000", 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:25.662 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876522+0000", 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0609999999999997e-06 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876531+0000", 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2300000000000008e-06 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876537+0000", 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5570000000000001e-06 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876590+0000", 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2528000000000003e-05 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876600+0000", 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8800000000000003e-06 2023-09-23T18:02:25.664 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851204+0000", 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97460424499999998 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851212+0000", 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2460000000000003e-06 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.665 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.346812+0000", 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4955998619999999 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.346965+0000", 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015338600000000001 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.347122+0000", 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000156537 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356744+0000", 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0096223920000000004 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356784+0000", 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9984000000000002e-05 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.389719+0000", 2023-09-23T18:02:25.667 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032934915000000002 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.389760+0000", 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0655999999999998e-05 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391237+0000", 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014769970000000001 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.668 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154260 5.d 5:b55d3ddc:::benchmark_data_smithi055_229117_object48634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.880860+0000", 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.710148554, 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5120832480000002, 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154260 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.669 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880860+0000", 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880860+0000", 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.670 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880859+0000", 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880866+0000", 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2790000000000001e-06 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880882+0000", 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5529999999999999e-05 2023-09-23T18:02:25.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880890+0000", 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.602e-06 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880936+0000", 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5847000000000003e-05 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880946+0000", 2023-09-23T18:02:25.672 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8519999999999999e-06 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851234+0000", 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97028760300000005 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851240+0000", 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5690000000000001e-06 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.673 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.351422+0000", 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.500181193 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.351601+0000", 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017931699999999999 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:02:25.674 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.351779+0000", 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017792700000000001 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360041+0000", 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082620220000000008 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360048+0000", 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4649999999999997e-06 2023-09-23T18:02:25.675 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391364+0000", 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031316043000000002 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391424+0000", 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0053000000000003e-05 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.676 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.392943+0000", 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001518997 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154262 5.d 5:bdaa66ea:::benchmark_data_smithi055_229117_object48636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.881965+0000", 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.709043662, 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.526328737, 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.677 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154262 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881965+0000", 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.678 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881965+0000", 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881963+0000", 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881971+0000", 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8699999999999992e-06 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.679 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881990+0000", 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9173000000000001e-05 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.882000+0000", 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4210000000000001e-06 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.882051+0000", 2023-09-23T18:02:25.680 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1026999999999998e-05 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.882062+0000", 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1463e-05 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851261+0000", 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96919898400000004 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.681 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851295+0000", 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3628999999999999e-05 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360083+0000", 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5087881159999998 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360255+0000", 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017206400000000001 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.682 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360415+0000", 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015982 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.374968+0000", 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014553857999999999 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.683 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391256+0000", 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016287340000000001 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.406335+0000", 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015079231 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.406390+0000", 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4511e-05 2023-09-23T18:02:25.684 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.408294+0000", 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001904177 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154263 5.b 5:dbd24d9d:::benchmark_data_smithi055_229117_object48637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.883292+0000", 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.70771624299999, 2023-09-23T18:02:25.685 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0983152249999999, 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154263 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883292+0000", 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.686 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883292+0000", 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883291+0000", 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.687 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883300+0000", 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6999999999999997e-06 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883317+0000", 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7048999999999999e-05 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883327+0000", 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0108999999999999e-05 2023-09-23T18:02:25.688 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883392+0000", 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4751000000000004e-05 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883402+0000", 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0149000000000001e-05 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.689 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.854589+0000", 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97118778699999997 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.854601+0000", 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1292e-05 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.958490+0000", 2023-09-23T18:02:25.690 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.103889399 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.958651+0000", 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016055600000000001 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.958761+0000", 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011042799999999999 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.691 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.968153+0000", 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093921730000000002 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981288+0000", 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01313507 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981455+0000", 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000166551 2023-09-23T18:02:25.692 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981509+0000", 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4163999999999998e-05 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981608+0000", 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8573000000000006e-05 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.693 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154267 5.4 5:27d4bd8e:::benchmark_data_smithi055_229117_object48641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.891154+0000", 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.699854536, 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2238628070000002, 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154267 2023-09-23T18:02:25.694 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891154+0000", 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891154+0000", 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.695 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891152+0000", 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891163+0000", 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0482999999999999e-05 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891191+0000", 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8411e-05 2023-09-23T18:02:25.696 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891202+0000", 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0841e-05 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891258+0000", 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6165999999999997e-05 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.697 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891269+0000", 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0572e-05 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851195+0000", 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95992611100000003 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851204+0000", 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.645e-06 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.698 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100511+0000", 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.249306255 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100638+0000", 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012729000000000001 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100765+0000", 2023-09-23T18:02:25.699 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012738699999999999 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105586+0000", 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048209419999999999 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105597+0000", 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0890000000000001e-05 2023-09-23T18:02:25.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.114801+0000", 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092037030000000006 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.114844+0000", 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3130999999999998e-05 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.114898+0000", 2023-09-23T18:02:25.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4320999999999997e-05 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.115017+0000", 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118578 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154270 5.d 5:b225da75:::benchmark_data_smithi055_229117_object48644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:02:25.702 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:09.964328+0000", 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 195.626680255, 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4456077330000001, 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154270 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.703 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964328+0000", 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964328+0000", 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964327+0000", 2023-09-23T18:02:25.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964335+0000", 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9510000000000006e-06 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964352+0000", 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7091e-05 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.705 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964361+0000", 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3870000000000001e-06 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964414+0000", 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2559999999999998e-05 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964421+0000", 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0230000000000004e-06 2023-09-23T18:02:25.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.365357+0000", 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.400936304 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.365487+0000", 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013022199999999999 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.365592+0000", 2023-09-23T18:02:25.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104739 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.389646+0000", 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024053990000000001 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391266+0000", 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016202969999999999 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.708 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.408353+0000", 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017086086 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.408410+0000", 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7760999999999998e-05 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.409936+0000", 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015258559999999999 2023-09-23T18:02:25.709 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154271 5.6 5:6eca1f68:::benchmark_data_smithi055_229117_object48645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:09.964509+0000", 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 195.62649941699999, 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.263656766, 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:02:25.710 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154271 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964509+0000", 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964509+0000", 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964508+0000", 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964515+0000", 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3600000000000001e-06 2023-09-23T18:02:25.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964530+0000", 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5153e-05 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964538+0000", 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1119999999999996e-06 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964594+0000", 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5926e-05 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964603+0000", 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2220000000000008e-06 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.215843+0000", 2023-09-23T18:02:25.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2512403320000001 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.215990+0000", 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000147095 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.216730+0000", 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00073915700000000005 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225069+0000", 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0083390189999999996 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225111+0000", 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2026e-05 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225280+0000", 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000169524 2023-09-23T18:02:25.716 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225348+0000", 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8232000000000007e-05 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.228166+0000", 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028175650000000002 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33521 6.2 6:4a04cd74:::benchmark_data_smithi055_275963_object33520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.422794+0000", 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 47.168215107000002, 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.013332479, 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33521 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.718 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422794+0000", 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422794+0000", 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422792+0000", 2023-09-23T18:02:25.720 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:25.720 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.720 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.720 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.720 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422842+0000", 2023-09-23T18:02:25.720 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0219999999999997e-05 2023-09-23T18:02:25.720 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422852+0000", 2023-09-23T18:02:25.722 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4700000000000008e-06 2023-09-23T18:02:25.722 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422857+0000", 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5779999999999998e-06 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422908+0000", 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0387999999999999e-05 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423038+0000", 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013043899999999999 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423199+0000", 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016039900000000001 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.425419+0000", 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022201949999999999 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448248+0000", 2023-09-23T18:02:25.724 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022829682 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448261+0000", 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2745000000000001e-05 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.436111+0000", 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98784968799999995 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.725 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.436126+0000", 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5163e-05 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33523 6.f 6:f5fa26bf:::benchmark_data_smithi055_275963_object33522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.423670+0000", 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 47.167338845000003, 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.050894467, 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.726 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33523 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423670+0000", 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.727 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423670+0000", 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423669+0000", 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423707+0000", 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8553999999999997e-05 2023-09-23T18:02:25.728 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423723+0000", 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5512e-05 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423732+0000", 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6130000000000004e-06 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.729 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423786+0000", 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4255999999999997e-05 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423953+0000", 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016769800000000001 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.424125+0000", 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000171409 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.425758+0000", 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016331780000000001 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448278+0000", 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022520401999999998 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448286+0000", 2023-09-23T18:02:25.731 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4089999999999997e-06 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433253+0000", 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98496738100000003 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433268+0000", 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4986e-05 2023-09-23T18:02:25.732 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.463950+0000", 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0306814449999999 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465116+0000", 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011660500000000001 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474508+0000", 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093924479999999994 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474564+0000", 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6112000000000001e-05 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.734 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33545 6.f 6:f8f54476:::benchmark_data_smithi055_275963_object33544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.452774+0000", 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 47.138235010000002, 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0213147949999999, 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33545 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.735 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452774+0000", 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452774+0000", 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452773+0000", 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452804+0000", 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1013000000000003e-05 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452817+0000", 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3525e-05 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452825+0000", 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5809999999999999e-06 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452879+0000", 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4762999999999997e-05 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452893+0000", 2023-09-23T18:02:25.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3278999999999999e-05 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433299+0000", 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98040629000000001 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433308+0000", 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6859999999999995e-06 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.739 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465523+0000", 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0322154109999999 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465714+0000", 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000190941 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465951+0000", 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00023701199999999999 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.740 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.470811+0000", 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004859693 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.470872+0000", 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1622000000000001e-05 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.473905+0000", 2023-09-23T18:02:25.741 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030324229999999998 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.473979+0000", 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4177999999999994e-05 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474088+0000", 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010944500000000001 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.742 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33550 6.f 6:fa5a6c38:::benchmark_data_smithi055_275963_object33549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.456197+0000", 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 47.134811745, 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0180980869999998, 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33550 2023-09-23T18:02:25.743 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456197+0000", 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456197+0000", 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.744 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456195+0000", 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456221+0000", 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5984000000000001e-05 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456241+0000", 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0132999999999998e-05 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456251+0000", 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4180000000000003e-06 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456325+0000", 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4190000000000006e-05 2023-09-23T18:02:25.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456341+0000", 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5846e-05 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433338+0000", 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97699677799999995 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.747 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433347+0000", 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2590000000000006e-06 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.467560+0000", 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.034212833 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.467689+0000", 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012873499999999999 2023-09-23T18:02:25.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.467798+0000", 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010996400000000001 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.470822+0000", 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030237990000000002 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.471707+0000", 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00088451899999999995 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474135+0000", 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002428655 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474192+0000", 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6406999999999997e-05 2023-09-23T18:02:25.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474295+0000", 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000103137 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33562 6.f 6:f588a2ef:::benchmark_data_smithi055_275963_object33561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.476003+0000", 2023-09-23T18:02:25.751 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 47.115005306999997, 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9984609899999999, 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33562 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476003+0000", 2023-09-23T18:02:25.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476003+0000", 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476002+0000", 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:02:25.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476021+0000", 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8847000000000001e-05 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476034+0000", 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2411e-05 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476039+0000", 2023-09-23T18:02:25.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5709999999999997e-06 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476088+0000", 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8936e-05 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476097+0000", 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3600000000000002e-06 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433567+0000", 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95746954799999995 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433577+0000", 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0359e-05 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:02:25.756 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.469156+0000", 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0355791350000001 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.469366+0000", 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000209228 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.469586+0000", 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00022075999999999999 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.471026+0000", 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001439582 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.471737+0000", 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00071104700000000002 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:25.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474335+0000", 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025979190000000002 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474377+0000", 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1820000000000003e-05 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474464+0000", 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7562e-05 2023-09-23T18:02:25.759 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.760 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.760 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.760 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:02:25.760 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:02:25.760 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:02:25.760 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:25.822 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:25.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:25.826 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:25.926 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:25.972 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:26.020 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 82 16 70064 70048 53.3723 26.1875 0.0131796 0.0187173 2023-09-23T18:02:26.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:26 smithi055 ceph-mon[179668]: pgmap v553: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 19 MiB/s wr, 298 op/s; 43915/206826 objects degraded (21.233%); 10284/206826 objects misplaced (4.972%); 10 MiB/s, 159 objects/s recovering 2023-09-23T18:02:26.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:26 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:26.234 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:26 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:26.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:26 smithi055 ceph-mon[184174]: pgmap v553: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 19 MiB/s wr, 298 op/s; 43915/206826 objects degraded (21.233%); 10284/206826 objects misplaced (4.972%); 10 MiB/s, 159 objects/s recovering 2023-09-23T18:02:26.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:26 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:26.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:26 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:26 smithi099 ceph-mon[167076]: pgmap v553: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 19 MiB/s wr, 298 op/s; 43915/206826 objects degraded (21.233%); 10284/206826 objects misplaced (4.972%); 10 MiB/s, 159 objects/s recovering 2023-09-23T18:02:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:26 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:26 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:26.828 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:26.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:26.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:26.932 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:26.968 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:27.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 83 16 70448 70432 53.0154 24 0.00628412 0.0188298 2023-09-23T18:02:27.067 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:02:27.559 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:27.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:27.567 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:27.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:27.703 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:27.731 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:02:27.762 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:27.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:27.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:27.867 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:27.918 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:27.918 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:27.969 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:27.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:27.979 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:28.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 84 16 70907 70891 52.7259 28.6875 0.00636943 0.0189321 2023-09-23T18:02:28.080 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:28.118 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:28.222 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:28.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:28.226 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:28.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:28.357 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:28.378 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:02:28.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:28.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:28.411 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:28.416 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:28.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:28.547 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:28 smithi099 ceph-mon[167076]: pgmap v554: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 16 MiB/s wr, 256 op/s; 43539/207594 objects degraded (20.973%); 9908/207594 objects misplaced (4.773%); 11 MiB/s, 181 objects/s recovering 2023-09-23T18:02:28.548 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 43539/207594 objects degraded (20.973%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:28.552 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:28.651 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:28.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:28.659 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:28.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:28 smithi055 ceph-mon[184174]: pgmap v554: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 16 MiB/s wr, 256 op/s; 43539/207594 objects degraded (20.973%); 9908/207594 objects misplaced (4.773%); 11 MiB/s, 181 objects/s recovering 2023-09-23T18:02:28.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 43539/207594 objects degraded (20.973%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:28.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:28 smithi055 ceph-mon[179668]: pgmap v554: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 16 MiB/s wr, 256 op/s; 43539/207594 objects degraded (20.973%); 9908/207594 objects misplaced (4.773%); 11 MiB/s, 181 objects/s recovering 2023-09-23T18:02:28.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 43539/207594 objects degraded (20.973%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:28.760 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:28.807 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:29.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 85 16 71242 71226 52.352 20.9375 0.00663122 0.0190819 2023-09-23T18:02:29.237 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T18:02:29.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:29.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:29.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:29.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2859920939' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:02:29.344 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:29 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:02:29.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2859920939' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:02:29.345 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:29 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:02:29.378 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:29.414 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:29.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2859920939' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:02:29.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:29 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:02:29.664 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:29.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:29.673 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:29.773 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:29.846 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:29.904 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:29.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:29.910 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:29.941 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T18:02:30.011 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:30.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 86 16 71658 71642 52.0457 26 0.0127909 0.0191877 2023-09-23T18:02:30.094 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:30.194 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:30.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:30.199 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:30.239 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:02:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:02:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71640 6.6 6.eba086a6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:29.965543+0000", 2023-09-23T18:02:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.27109793900000001, 2023-09-23T18:02:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27112375700000002, 2023-09-23T18:02:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.239 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71640 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.965543+0000", 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.965543+0000", 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.965541+0000", 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.965589+0000", 2023-09-23T18:02:30.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7964999999999999e-05 2023-09-23T18:02:30.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.965607+0000", 2023-09-23T18:02:30.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.819e-05 2023-09-23T18:02:30.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.965614+0000", 2023-09-23T18:02:30.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1409999999999999e-06 2023-09-23T18:02:30.243 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.243 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.243 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.243 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71685 6.6 6.7b5e77d6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.243 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.103307+0000", 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.13333346200000001, 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.133424388, 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71685 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.103307+0000", 2023-09-23T18:02:30.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.103307+0000", 2023-09-23T18:02:30.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.103305+0000", 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.103350+0000", 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4875e-05 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.103366+0000", 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5197e-05 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.103374+0000", 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4700000000000002e-06 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71686 6.e 6.e80bb81e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.107052+0000", 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.12958887399999999, 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.129734988, 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.248 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71686 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.107052+0000", 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.107052+0000", 2023-09-23T18:02:30.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.107050+0000", 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.107103+0000", 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2469999999999997e-05 2023-09-23T18:02:30.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.107122+0000", 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9321e-05 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.107130+0000", 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6279999999999996e-06 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.251 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.45092.0:71626 6.6 e241/238)", 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.220973+0000", 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.015668259, 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015867111999999999, 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.220973+0000", 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.220973+0000", 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.220972+0000", 2023-09-23T18:02:30.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:02:30.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.220979+0000", 2023-09-23T18:02:30.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7090000000000002e-06 2023-09-23T18:02:30.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.220992+0000", 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2261e-05 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.220999+0000", 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0809999999999996e-06 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.256 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71646 6.e 6.23bcb95e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.256 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:29.981537+0000", 2023-09-23T18:02:30.256 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.25510377099999998, 2023-09-23T18:02:30.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.255350562, 2023-09-23T18:02:30.256 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.256 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.256 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.256 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.256 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71646 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.981537+0000", 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.981537+0000", 2023-09-23T18:02:30.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.981536+0000", 2023-09-23T18:02:30.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:30.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.981595+0000", 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9039000000000002e-05 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.981608+0000", 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3477e-05 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.981615+0000", 2023-09-23T18:02:30.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5760000000000002e-06 2023-09-23T18:02:30.260 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.260 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.260 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.260 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.45092.0:71626 6.6 e241/238)", 2023-09-23T18:02:30.260 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.221689+0000", 2023-09-23T18:02:30.260 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.014952151, 2023-09-23T18:02:30.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015251305999999999, 2023-09-23T18:02:30.260 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.221689+0000", 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.221689+0000", 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.221688+0000", 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.221693+0000", 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6789999999999996e-06 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.221707+0000", 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3879e-05 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.221712+0000", 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6439999999999997e-06 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71690 6.6 6.9f41c406 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.118611+0000", 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.118030256, 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.11837747699999999, 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71690 2023-09-23T18:02:30.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.118611+0000", 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.118611+0000", 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.118609+0000", 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.118653+0000", 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4212000000000003e-05 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.266 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.118671+0000", 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7425e-05 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.118678+0000", 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7739999999999999e-06 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.267 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71692 6.e 6.fea969e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.120830+0000", 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.115810919, 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.116213396, 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71692 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.120830+0000", 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.120830+0000", 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.120828+0000", 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.120918+0000", 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9951999999999995e-05 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.120934+0000", 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5688999999999998e-05 2023-09-23T18:02:30.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.120941+0000", 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9360000000000002e-06 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71654 6.6 6.84f39486 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.005768+0000", 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.23087327399999999, 2023-09-23T18:02:30.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.231327529, 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71654 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.005768+0000", 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.005768+0000", 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.005766+0000", 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.005827+0000", 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1241999999999994e-05 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.005842+0000", 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4786e-05 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.005850+0000", 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8699999999999992e-06 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71699 6.e 6.4d2eb18e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.143018+0000", 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.093622760999999999, 2023-09-23T18:02:30.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.094130280999999996, 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71699 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.143018+0000", 2023-09-23T18:02:30.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.143018+0000", 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.143016+0000", 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.143061+0000", 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5056999999999999e-05 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.143081+0000", 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9142000000000001e-05 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.143088+0000", 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1829999999999997e-06 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71624 6.e 6:778d4c93:::benchmark_data_smithi055_275963_object71623:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:29.928670+0000", 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.30797123999999998, 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.30853848499999997, 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:02:30.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71624 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.928670+0000", 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.928670+0000", 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.928668+0000", 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.928702+0000", 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3779999999999998e-05 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.928718+0000", 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5869999999999999e-05 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.928724+0000", 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9079999999999998e-06 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:30.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.217319+0000", 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28859409699999999 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.222967+0000", 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0056487270000000001 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,5", 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.233589+0000", 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010621273000000001 2023-09-23T18:02:30.283 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71701 6.6 6.6d790e16 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.149627+0000", 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.087013671000000001, 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.087649705999999994, 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.284 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71701 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.149627+0000", 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.149627+0000", 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.149626+0000", 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.149672+0000", 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6127000000000001e-05 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.149688+0000", 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5835999999999999e-05 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.149694+0000", 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3210000000000001e-06 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.287 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71626 6.6 6:6c9825f1:::benchmark_data_smithi055_275963_object71625:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:29.938881+0000", 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.29776039700000001, 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.29845409000000001, 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71626 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.938881+0000", 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.938881+0000", 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.938879+0000", 2023-09-23T18:02:30.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.938930+0000", 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1048000000000003e-05 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.938947+0000", 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6830999999999999e-05 2023-09-23T18:02:30.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.938954+0000", 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2699999999999999e-06 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.216621+0000", 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.27766722300000002 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.216780+0000", 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015862299999999999 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.216953+0000", 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017327299999999999 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71670 6.6 6.62c21f56 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.063440+0000", 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.17320052999999999, 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.173969282, 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71670 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.063440+0000", 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.063440+0000", 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.063438+0000", 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.063486+0000", 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7602000000000003e-05 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.063503+0000", 2023-09-23T18:02:30.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6739e-05 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.063511+0000", 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4880000000000007e-06 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71706 6.e 6.f7a9861e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.296 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:30.168251+0000", 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.068390184000000007, 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.069210425000000006, 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71706 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.168251+0000", 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.168251+0000", 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.168249+0000", 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.168302+0000", 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2321999999999998e-05 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.168315+0000", 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3890000000000001e-05 2023-09-23T18:02:30.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:30.168322+0000", 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.212e-06 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:71635 6.e 6.adf20fde (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e241)", 2023-09-23T18:02:30.300 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:29.956968+0000", 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.279673018, 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.280546978, 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 71635 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:30.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.956968+0000", 2023-09-23T18:02:30.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:30.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.956968+0000", 2023-09-23T18:02:30.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:30.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.956967+0000", 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.957070+0000", 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010289999999999999 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:30.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.957085+0000", 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4946999999999999e-05 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:29.957091+0000", 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3409999999999999e-06 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:02:30.304 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 16 2023-09-23T18:02:30.305 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:02:30.305 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:30.305 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:02:30.366 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:30.415 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:30.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:30.425 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:30.526 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:30.533 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:02:30.557 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:02:30.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:30 smithi099 ceph-mon[167076]: pgmap v555: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 20 MiB/s wr, 326 op/s; 43751/210981 objects degraded (20.737%); 9625/210981 objects misplaced (4.562%); 12 MiB/s, 188 objects/s recovering 2023-09-23T18:02:30.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:30 smithi099 ceph-mon[167076]: failed to encode map e241 with expected crc 2023-09-23T18:02:30.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:30 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:02:30.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:30 smithi099 ceph-mon[167076]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T18:02:30.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:30 smithi099 ceph-mon[167076]: failed to encode map e241 with expected crc 2023-09-23T18:02:30.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:30 smithi099 ceph-mon[167076]: failed to encode map e241 with expected crc 2023-09-23T18:02:30.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1438476073' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:30.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:30 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:30.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:30 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:30.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:30 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:30.585 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:02:30.587 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:30.615 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:02:30.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:02:30.639 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:02:30.661 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:30.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:30.665 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:30.670 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:02:30.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:02:30] "GET /metrics HTTP/1.1" 200 36558 "" "Prometheus/2.43.0" 2023-09-23T18:02:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[184174]: pgmap v555: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 20 MiB/s wr, 326 op/s; 43751/210981 objects degraded (20.737%); 9625/210981 objects misplaced (4.562%); 12 MiB/s, 188 objects/s recovering 2023-09-23T18:02:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[184174]: failed to encode map e241 with expected crc 2023-09-23T18:02:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:02:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[184174]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T18:02:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[184174]: failed to encode map e241 with expected crc 2023-09-23T18:02:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[184174]: failed to encode map e241 with expected crc 2023-09-23T18:02:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1438476073' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[179668]: pgmap v555: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 20 MiB/s wr, 326 op/s; 43751/210981 objects degraded (20.737%); 9625/210981 objects misplaced (4.562%); 12 MiB/s, 188 objects/s recovering 2023-09-23T18:02:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[179668]: failed to encode map e241 with expected crc 2023-09-23T18:02:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:02:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[179668]: osdmap e241: 8 total, 8 up, 7 in 2023-09-23T18:02:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[179668]: failed to encode map e241 with expected crc 2023-09-23T18:02:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[179668]: failed to encode map e241 with expected crc 2023-09-23T18:02:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1438476073' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:30.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:30.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:30.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:30 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:30.701 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:02:30.725 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:02:30.727 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:02:30.727 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:02:30.727 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:02:30.727 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:02:30.727 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:02:30.728 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:02:30.728 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:02:30.728 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:02:30.766 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:30.824 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:30.878 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:30.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:30.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:30.983 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:31.026 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 87 16 71989 71973 51.6843 20.6875 0.177224 0.0193333 2023-09-23T18:02:31.027 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:31.123 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:31.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:31.128 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:31.161 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:31.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:31.267 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:31.307 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:31.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:31.310 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:31.337 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:02:31.369 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:31.369 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[179668]: failed to encode map e242 with expected crc 2023-09-23T18:02:31.369 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:02:31.369 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[179668]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T18:02:31.370 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/734607032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:31.370 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:02:31.370 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:02:31.370 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:31.370 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:31.370 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[184174]: failed to encode map e242 with expected crc 2023-09-23T18:02:31.370 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:02:31.370 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[184174]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T18:02:31.370 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/734607032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:31.370 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:02:31.371 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:02:31.371 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:31 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:31.411 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:31.454 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:31.568 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:31 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:31.568 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:31 smithi099 ceph-mon[167076]: failed to encode map e242 with expected crc 2023-09-23T18:02:31.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:31 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:02:31.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:31 smithi099 ceph-mon[167076]: osdmap e242: 8 total, 8 up, 7 in 2023-09-23T18:02:31.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/734607032' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:31.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:31 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:02:31.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:31 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:02:31.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:31 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:31.602 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:31.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:31.607 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:31.708 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:31.739 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:31.835 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:31.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:31.843 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:31.944 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:31.986 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:32.028 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 88 16 72360 72344 51.3592 23.1875 0.111761 0.0194596 2023-09-23T18:02:32.060 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:32.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:32.064 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:32.165 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:32.211 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:32.267 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:32.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:32.272 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:32.373 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:32.411 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:32.450 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:32.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:32.455 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:32.517 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:32 smithi099 ceph-mon[167076]: pgmap v558: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 346 op/s; 43332/213210 objects degraded (20.324%); 9206/213210 objects misplaced (4.318%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T18:02:32.555 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:32.585 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:32 smithi055 ceph-mon[179668]: pgmap v558: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 346 op/s; 43332/213210 objects degraded (20.324%); 9206/213210 objects misplaced (4.318%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T18:02:32.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:32 smithi055 ceph-mon[184174]: pgmap v558: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 22 MiB/s wr, 346 op/s; 43332/213210 objects degraded (20.324%); 9206/213210 objects misplaced (4.318%); 9.2 MiB/s, 147 objects/s recovering 2023-09-23T18:02:32.588 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:32.629 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:32.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:32.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:32.734 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:02:32.735 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:32.766 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:32.801 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:02:32.802 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:02:32.817 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:32.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:32.823 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:32.923 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:32.962 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:33.007 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:33.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:33.011 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:33.028 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 89 16 72685 72669 51.0105 20.3125 0.00892827 0.019573 2023-09-23T18:02:33.111 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:33.143 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:02:33.161 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:33.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:02:33.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:33.240 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:33.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:33.255 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2457498685' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:02:33.255 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 43332/213210 objects degraded (20.324%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:33.277 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T18:02:33.341 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:33.403 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:33.457 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:33.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:33.464 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:33.515 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2457498685' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:02:33.515 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 43332/213210 objects degraded (20.324%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:33.565 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:33.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:33 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2457498685' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:02:33.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 43332/213210 objects degraded (20.324%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:33.644 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:33.689 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:33.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:33.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:33.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:33.802 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:33.861 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T18:02:33.876 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:33.905 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:02:33.979 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:33.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:33.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:34.028 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 90 8 73135 73127 50.7618 28.625 0.297232 0.0196841 2023-09-23T18:02:34.087 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:34.117 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:34.128 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:34.162 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:34.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:34.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:34.266 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:34.297 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:34.422 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:34.425 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[179668]: pgmap v559: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 21 MiB/s wr, 334 op/s; 43552/214857 objects degraded (20.270%); 8882/214857 objects misplaced (4.134%); 13 MiB/s, 200 objects/s recovering 2023-09-23T18:02:34.425 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2457498685' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:02:34.425 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[179668]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T18:02:34.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[179668]: failed to encode map e243 with expected crc 2023-09-23T18:02:34.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[179668]: failed to encode map e243 with expected crc 2023-09-23T18:02:34.426 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[179668]: failed to encode map e243 with expected crc 2023-09-23T18:02:34.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[184174]: pgmap v559: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 21 MiB/s wr, 334 op/s; 43552/214857 objects degraded (20.270%); 8882/214857 objects misplaced (4.134%); 13 MiB/s, 200 objects/s recovering 2023-09-23T18:02:34.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2457498685' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:02:34.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[184174]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T18:02:34.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[184174]: failed to encode map e243 with expected crc 2023-09-23T18:02:34.426 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[184174]: failed to encode map e243 with expected crc 2023-09-23T18:02:34.427 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:34 smithi055 ceph-mon[184174]: failed to encode map e243 with expected crc 2023-09-23T18:02:34.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:34.427 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:34.528 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:34.563 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:34.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:34 smithi099 ceph-mon[167076]: pgmap v559: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 21 MiB/s wr, 334 op/s; 43552/214857 objects degraded (20.270%); 8882/214857 objects misplaced (4.134%); 13 MiB/s, 200 objects/s recovering 2023-09-23T18:02:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2457498685' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:02:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:34 smithi099 ceph-mon[167076]: osdmap e243: 8 total, 8 up, 7 in 2023-09-23T18:02:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:34 smithi099 ceph-mon[167076]: failed to encode map e243 with expected crc 2023-09-23T18:02:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:34 smithi099 ceph-mon[167076]: failed to encode map e243 with expected crc 2023-09-23T18:02:34.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:34 smithi099 ceph-mon[167076]: failed to encode map e243 with expected crc 2023-09-23T18:02:34.659 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:34.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:34.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:34.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:34.795 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:34.801 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:35.029 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total time run: 90.1305 2023-09-23T18:02:35.029 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total writes made: 73135 2023-09-23T18:02:35.029 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Write size: 65536 2023-09-23T18:02:35.029 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object size: 65536 2023-09-23T18:02:35.029 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Bandwidth (MB/sec): 50.7146 2023-09-23T18:02:35.029 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Bandwidth: 60.0818 2023-09-23T18:02:35.029 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max bandwidth (MB/sec): 220.562 2023-09-23T18:02:35.030 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min bandwidth (MB/sec): 0 2023-09-23T18:02:35.030 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average IOPS: 811 2023-09-23T18:02:35.030 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev IOPS: 961.321 2023-09-23T18:02:35.030 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max IOPS: 3529 2023-09-23T18:02:35.030 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min IOPS: 0 2023-09-23T18:02:35.030 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average Latency(s): 0.0196987 2023-09-23T18:02:35.030 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Latency(s): 0.336302 2023-09-23T18:02:35.030 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max latency(s): 21.0159 2023-09-23T18:02:35.030 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min latency(s): 0.00103291 2023-09-23T18:02:35.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T18:02:35.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:35.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:35.297 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:35.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:35.445 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:35.888 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:02:35.913 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:02:35.944 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:02:35.968 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:02:35.992 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:02:36.019 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:02:36.044 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:02:36.067 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:02:36.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:02:36.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:02:36.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:02:36.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:02:36.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:02:36.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:02:36.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:02:36.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:02:36.291 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:02:36.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:36 smithi055 ceph-mon[184174]: pgmap v561: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 24 MiB/s wr, 384 op/s; 43552/217905 objects degraded (19.987%); 8882/217905 objects misplaced (4.076%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T18:02:36.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2842755729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:36.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:36 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:36.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/653440508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:36.318 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:36 smithi055 ceph-mon[179668]: pgmap v561: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 24 MiB/s wr, 384 op/s; 43552/217905 objects degraded (19.987%); 8882/217905 objects misplaced (4.076%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T18:02:36.318 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2842755729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:36.318 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:36 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:36.319 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/653440508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:36.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:36 smithi099 ceph-mon[167076]: pgmap v561: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 24 MiB/s wr, 384 op/s; 43552/217905 objects degraded (19.987%); 8882/217905 objects misplaced (4.076%); 8.8 MiB/s, 140 objects/s recovering 2023-09-23T18:02:36.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2842755729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:36.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:36 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:36.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/653440508' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:37.419 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T18:02:37.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:37.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:37.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:37.532 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:37 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:02:37.532 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:37 smithi099 ceph-mon[167076]: failed to encode map e244 with expected crc 2023-09-23T18:02:37.532 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:37 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:37.532 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:37 smithi099 ceph-mon[167076]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T18:02:37.532 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:37 smithi099 ceph-mon[167076]: failed to encode map e244 with expected crc 2023-09-23T18:02:37.533 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:37.533 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:37.533 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:37 smithi099 ceph-mon[167076]: pgmap v563: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 19 MiB/s wr, 312 op/s; 43055/218256 objects degraded (19.727%); 8385/218256 objects misplaced (3.842%); 13 MiB/s, 208 objects/s recovering 2023-09-23T18:02:37.533 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2879116323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:02:37.533 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:37 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:02:37.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:02:37.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[184174]: failed to encode map e244 with expected crc 2023-09-23T18:02:37.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:37.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[184174]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T18:02:37.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[184174]: failed to encode map e244 with expected crc 2023-09-23T18:02:37.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:37.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:37.549 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[184174]: pgmap v563: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 19 MiB/s wr, 312 op/s; 43055/218256 objects degraded (19.727%); 8385/218256 objects misplaced (3.842%); 13 MiB/s, 208 objects/s recovering 2023-09-23T18:02:37.550 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2879116323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:02:37.550 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:02:37.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:02:37.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[179668]: failed to encode map e244 with expected crc 2023-09-23T18:02:37.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:37.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[179668]: osdmap e244: 8 total, 8 up, 7 in 2023-09-23T18:02:37.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[179668]: failed to encode map e244 with expected crc 2023-09-23T18:02:37.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:37.551 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:37.551 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[179668]: pgmap v563: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 19 MiB/s wr, 312 op/s; 43055/218256 objects degraded (19.727%); 8385/218256 objects misplaced (3.842%); 13 MiB/s, 208 objects/s recovering 2023-09-23T18:02:37.551 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2879116323' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:02:37.551 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:37 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:02:37.632 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:37.677 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:37.722 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:37.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:37.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:37.767 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T18:02:37.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:37.873 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:38.480 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:38.549 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:02:38.619 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:38.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:38.625 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:38.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[184174]: failed to encode map e245 with expected crc 2023-09-23T18:02:38.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[184174]: failed to encode map e245 with expected crc 2023-09-23T18:02:38.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:02:38.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[184174]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T18:02:38.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[184174]: failed to encode map e245 with expected crc 2023-09-23T18:02:38.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 43055/218256 objects degraded (19.727%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:38.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[179668]: failed to encode map e245 with expected crc 2023-09-23T18:02:38.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[179668]: failed to encode map e245 with expected crc 2023-09-23T18:02:38.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:02:38.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[179668]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T18:02:38.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[179668]: failed to encode map e245 with expected crc 2023-09-23T18:02:38.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 43055/218256 objects degraded (19.727%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:38.726 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:38.755 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:38 smithi099 ceph-mon[167076]: failed to encode map e245 with expected crc 2023-09-23T18:02:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:38 smithi099 ceph-mon[167076]: failed to encode map e245 with expected crc 2023-09-23T18:02:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:02:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:38 smithi099 ceph-mon[167076]: osdmap e245: 8 total, 8 up, 7 in 2023-09-23T18:02:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:38 smithi099 ceph-mon[167076]: failed to encode map e245 with expected crc 2023-09-23T18:02:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 43055/218256 objects degraded (19.727%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:38.908 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:02:39.654 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:39.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:39.658 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:39.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:39 smithi055 ceph-mon[184174]: pgmap v565: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 170 B/s rd, 16 MiB/s wr, 377 op/s; 43006/217323 objects degraded (19.789%); 7801/217323 objects misplaced (3.590%); 11 MiB/s, 177 objects/s recovering 2023-09-23T18:02:39.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:39 smithi055 ceph-mon[179668]: pgmap v565: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 170 B/s rd, 16 MiB/s wr, 377 op/s; 43006/217323 objects degraded (19.789%); 7801/217323 objects misplaced (3.590%); 11 MiB/s, 177 objects/s recovering 2023-09-23T18:02:39.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:39 smithi099 ceph-mon[167076]: pgmap v565: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 170 B/s rd, 16 MiB/s wr, 377 op/s; 43006/217323 objects degraded (19.789%); 7801/217323 objects misplaced (3.590%); 11 MiB/s, 177 objects/s recovering 2023-09-23T18:02:39.759 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:39.786 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:39.922 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:02:39.922 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:02:39.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:02:39.922 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:02:39.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.922 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80604 5.e 5:70b48fb5:::benchmark_data_smithi055_229117_object80603:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T18:02:39.922 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:55.880815+0000", 2023-09-23T18:02:39.922 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 344.03875238000001, 2023-09-23T18:02:39.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0631930629999999, 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80604 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880815+0000", 2023-09-23T18:02:39.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880815+0000", 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880813+0000", 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999976 2023-09-23T18:02:39.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:39.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880849+0000", 2023-09-23T18:02:39.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5797999999999998e-05 2023-09-23T18:02:39.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:39.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880867+0000", 2023-09-23T18:02:39.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8181000000000001e-05 2023-09-23T18:02:39.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880874+0000", 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4569999999999999e-06 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880938+0000", 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3736000000000001e-05 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880945+0000", 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1189999999999999e-06 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.888009+0000", 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070636220000000003 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.888023+0000", 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.437e-05 2023-09-23T18:02:39.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.907926+0000", 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.019902603 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.908058+0000", 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013261800000000001 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:02:39.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.908220+0000", 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016149599999999999 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.920288+0000", 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012067847 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.943890+0000", 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.023602317000000001 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.943940+0000", 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0342e-05 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.943973+0000", 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2404999999999999e-05 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944008+0000", 2023-09-23T18:02:39.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5321999999999998e-05 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:80612 5.e 5:70cc0f65:::benchmark_data_smithi055_229117_object80611:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e131)", 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:55.880893+0000", 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 344.03867487299999, 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0632663899999999, 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:39.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 80612 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880893+0000", 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880893+0000", 2023-09-23T18:02:39.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880892+0000", 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880918+0000", 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6404999999999999e-05 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:39.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880926+0000", 2023-09-23T18:02:39.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1389999999999998e-06 2023-09-23T18:02:39.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:39.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880930+0000", 2023-09-23T18:02:39.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1520000000000002e-06 2023-09-23T18:02:39.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880991+0000", 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1029999999999997e-05 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.880996+0000", 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7169999999999996e-06 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.888056+0000", 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070596560000000001 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:55.888064+0000", 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7740000000000004e-06 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.910586+0000", 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.022522011 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.910725+0000", 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013914799999999999 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.910898+0000", 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017282199999999999 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.925504+0000", 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014606192 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944059+0000", 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018555278000000001 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944095+0000", 2023-09-23T18:02:39.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5148999999999998e-05 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944126+0000", 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1038000000000003e-05 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:56.944159+0000", 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3337000000000003e-05 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:39.939 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122746 5.1 5:8f7be469:::benchmark_data_smithi055_229117_object17120:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821428+0000", 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 257.09813933599997, 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0439162390000001, 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:39.940 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122746 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821428+0000", 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821428+0000", 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821428+0000", 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821429+0000", 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.359e-06 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821432+0000", 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0060000000000001e-06 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821435+0000", 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4499999999999998e-06 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821619+0000", 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018453499999999999 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821622+0000", 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6079999999999998e-06 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.849935+0000", 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.028313502 2023-09-23T18:02:39.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.850006+0000", 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0452999999999998e-05 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.850071+0000", 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5159000000000004e-05 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.856051+0000", 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0059799379999999997 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865192+0000", 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0091416870000000008 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865251+0000", 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8029000000000003e-05 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865344+0000", 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3762000000000001e-05 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122748 5.1 5:87ab67ee:::benchmark_data_smithi055_229117_object17122:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821437+0000", 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 257.09813068400001, 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0440240649999999, 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122748 2023-09-23T18:02:39.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821437+0000", 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821437+0000", 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821436+0000", 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967296 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821438+0000", 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.186e-06 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821441+0000", 2023-09-23T18:02:39.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9000000000000002e-06 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821444+0000", 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.163e-06 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821645+0000", 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020162199999999999 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821649+0000", 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4010000000000001e-06 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.852019+0000", 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0303700819999999 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:39.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.852085+0000", 2023-09-23T18:02:39.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6049000000000004e-05 2023-09-23T18:02:39.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:02:39.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.852150+0000", 2023-09-23T18:02:39.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4774e-05 2023-09-23T18:02:39.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:39.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:39.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.858405+0000", 2023-09-23T18:02:39.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062549090000000003 2023-09-23T18:02:39.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865386+0000", 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069814739999999997 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865414+0000", 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7761000000000001e-05 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865461+0000", 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6950000000000003e-05 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.956 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122752 5.1 5:86403481:::benchmark_data_smithi055_229117_object17126:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821461+0000", 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 257.09810682300002, 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0441718659999999, 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:39.957 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122752 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821461+0000", 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:39.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821461+0000", 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821460+0000", 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821462+0000", 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0789999999999999e-06 2023-09-23T18:02:39.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821468+0000", 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6289999999999998e-06 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821472+0000", 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6610000000000002e-06 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821667+0000", 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019562900000000001 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821670+0000", 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5119999999999998e-06 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.854001+0000", 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0323309030000001 2023-09-23T18:02:39.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.854073+0000", 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2389000000000001e-05 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.854138+0000", 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4957999999999997e-05 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.861684+0000", 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075462159999999997 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865503+0000", 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0038183340000000001 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865544+0000", 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1959000000000003e-05 2023-09-23T18:02:39.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865632+0000", 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7948000000000006e-05 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122816 5.1 5:8f041339:::benchmark_data_smithi055_229117_object17190:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:02:39.964 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821476+0000", 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 257.09809137299999, 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.044283192, 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122816 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:39.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821476+0000", 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821476+0000", 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821476+0000", 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:39.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821478+0000", 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9230000000000001e-06 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821483+0000", 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4530000000000001e-06 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:39.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821487+0000", 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.749e-06 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821705+0000", 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000217736 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821709+0000", 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2880000000000003e-06 2023-09-23T18:02:39.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.856078+0000", 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0343687239999999 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.856147+0000", 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9685999999999997e-05 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.856216+0000", 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9104999999999997e-05 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865360+0000", 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0091437619999999997 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865687+0000", 2023-09-23T18:02:39.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00032670799999999999 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865714+0000", 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6746000000000001e-05 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865759+0000", 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5586e-05 2023-09-23T18:02:39.971 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122829 5.1 5:8882a119:::benchmark_data_smithi055_229117_object17203:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821490+0000", 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 257.09807712200001, 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0444458599999999, 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:39.972 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122829 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821490+0000", 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821490+0000", 2023-09-23T18:02:39.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821490+0000", 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821492+0000", 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9659999999999999e-06 2023-09-23T18:02:39.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821497+0000", 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4500000000000003e-06 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821502+0000", 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.284e-06 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821730+0000", 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022864700000000001 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821733+0000", 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7080000000000002e-06 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.858644+0000", 2023-09-23T18:02:39.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0369113400000001 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.858773+0000", 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012875500000000001 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.858920+0000", 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014664800000000001 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865645+0000", 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0067251330000000003 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865805+0000", 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000160085 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865849+0000", 2023-09-23T18:02:39.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3593000000000001e-05 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865936+0000", 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7576000000000003e-05 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122835 5.1 5:84f7e635:::benchmark_data_smithi055_229117_object17209:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821505+0000", 2023-09-23T18:02:39.979 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 257.098062363, 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0616019000000001, 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122835 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821505+0000", 2023-09-23T18:02:39.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821505+0000", 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821505+0000", 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821506+0000", 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.598e-06 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821511+0000", 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9169999999999997e-06 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821515+0000", 2023-09-23T18:02:39.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8039999999999999e-06 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821755+0000", 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00024008300000000001 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821758+0000", 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6129999999999998e-06 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.861727+0000", 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0399696789999999 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.861855+0000", 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127136 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.861980+0000", 2023-09-23T18:02:39.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012571199999999999 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865653+0000", 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0036726630000000001 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.882952+0000", 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017299201 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883010+0000", 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7593000000000002e-05 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883107+0000", 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7178999999999999e-05 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122838 5.1 5:80ca6cb7:::benchmark_data_smithi055_229117_object17212:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821525+0000", 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 257.09804204400001, 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0617261010000001, 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122838 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.987 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821525+0000", 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821525+0000", 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:39.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821525+0000", 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821527+0000", 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9740000000000001e-06 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821532+0000", 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3750000000000002e-06 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821536+0000", 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6009999999999999e-06 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821775+0000", 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00023937399999999999 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821778+0000", 2023-09-23T18:02:39.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5059999999999998e-06 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.865966+0000", 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.044187888 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.866067+0000", 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010108800000000001 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.866142+0000", 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5085000000000004e-05 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.875604+0000", 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0094623810000000006 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883163+0000", 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075590190000000002 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883192+0000", 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8654999999999999e-05 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883251+0000", 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9472000000000003e-05 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.993 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122858 5.1 5:85c7b7f3:::benchmark_data_smithi055_229117_object17232:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821539+0000", 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 257.098027925, 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0618255270000001, 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122858 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:39.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821539+0000", 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821539+0000", 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821539+0000", 2023-09-23T18:02:39.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821541+0000", 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8899999999999999e-06 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821546+0000", 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2920000000000003e-06 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821550+0000", 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3380000000000001e-06 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821799+0000", 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00024963699999999998 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821802+0000", 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0479999999999999e-06 2023-09-23T18:02:39.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.868820+0000", 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.047017925 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.868920+0000", 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100201 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:02:39.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.868996+0000", 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5976999999999994e-05 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.877506+0000", 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085090640000000002 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883289+0000", 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057830579999999998 2023-09-23T18:02:39.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883316+0000", 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7611999999999999e-05 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883365+0000", 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8773999999999998e-05 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.000 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122887 5.1 5:862aed4f:::benchmark_data_smithi055_229117_object17261:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821565+0000", 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 257.09800244799999, 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0619787039999999, 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:40.001 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122887 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821565+0000", 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821565+0000", 2023-09-23T18:02:40.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821565+0000", 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821567+0000", 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3e-06 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821573+0000", 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6910000000000002e-06 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821576+0000", 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7440000000000001e-06 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821819+0000", 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000243157 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821822+0000", 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5100000000000001e-06 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.871475+0000", 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0496534580000001 2023-09-23T18:02:40.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.871545+0000", 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9819000000000004e-05 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.871625+0000", 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9864999999999995e-05 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.877513+0000", 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058876070000000004 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883402+0000", 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058895689999999999 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883456+0000", 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3618000000000001e-05 2023-09-23T18:02:40.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883544+0000", 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7720999999999996e-05 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122893 5.1 5:859301a2:::benchmark_data_smithi055_229117_object17267:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821588+0000", 2023-09-23T18:02:40.008 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 257.097979194, 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.072527035, 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122893 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821588+0000", 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821588+0000", 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821588+0000", 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821590+0000", 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0229999999999999e-06 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821596+0000", 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7570000000000001e-06 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:40.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821599+0000", 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3179999999999998e-06 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821833+0000", 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000234379 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821836+0000", 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.497e-06 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.873642+0000", 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0518064739999999 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.873706+0000", 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3875999999999996e-05 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.873789+0000", 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2707000000000006e-05 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.882910+0000", 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0091208369999999997 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.893965+0000", 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011055684 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:40.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.894019+0000", 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3720000000000001e-05 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.894115+0000", 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6105000000000002e-05 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.015 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154240 5.e 5:7f5ef404:::benchmark_data_smithi055_229117_object48614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.838192+0000", 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.081375116, 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2567066049999998, 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154240 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:40.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838192+0000", 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838192+0000", 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838191+0000", 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838199+0000", 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1009999999999992e-06 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:40.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838217+0000", 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8629000000000001e-05 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838225+0000", 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5510000000000002e-06 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838294+0000", 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9330000000000002e-05 2023-09-23T18:02:40.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838426+0000", 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013151499999999999 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838546+0000", 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120331 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.843064+0000", 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004517928 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.845440+0000", 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00237587 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.862351+0000", 2023-09-23T18:02:40.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016911302 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.862359+0000", 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6399999999999997e-06 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.850984+0000", 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98862541999999998 2023-09-23T18:02:40.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.850999+0000", 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4683e-05 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.094798+0000", 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.243799069 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.094899+0000", 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100995 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154257 5.e 5:7493d31c:::benchmark_data_smithi055_229117_object48631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.879851+0000", 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.03971634300001, 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2859593629999999, 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:40.024 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154257 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879851+0000", 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879851+0000", 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879850+0000", 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879855+0000", 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5829999999999997e-06 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879867+0000", 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1579e-05 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879872+0000", 2023-09-23T18:02:40.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8509999999999998e-06 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879916+0000", 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4608999999999999e-05 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879924+0000", 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8450000000000003e-06 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851029+0000", 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97110422200000002 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851036+0000", 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3780000000000003e-06 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.094924+0000", 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2438876830000001 2023-09-23T18:02:40.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.095069+0000", 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014521 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.095710+0000", 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00064142799999999999 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114512+0000", 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018801345000000001 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114543+0000", 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1266000000000002e-05 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114629+0000", 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6620999999999996e-05 2023-09-23T18:02:40.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165607+0000", 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.050977816000000002 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165673+0000", 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5704000000000007e-05 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165810+0000", 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013739199999999999 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154261 5.e 5:75cfb822:::benchmark_data_smithi055_229117_object48635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.881204+0000", 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.038363562, 2023-09-23T18:02:40.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2848199720000002, 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154261 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:40.034 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881204+0000", 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881204+0000", 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881203+0000", 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:02:40.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881208+0000", 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1699999999999996e-06 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881221+0000", 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2809000000000001e-05 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.036 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881227+0000", 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7799999999999997e-06 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881298+0000", 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1403999999999996e-05 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881305+0000", 2023-09-23T18:02:40.037 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7789999999999998e-06 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851061+0000", 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96975605099999995 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851068+0000", 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1060000000000002e-06 2023-09-23T18:02:40.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.103490+0000", 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2524223880000001 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.103624+0000", 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013401600000000001 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.039 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:02:40.040 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.103729+0000", 2023-09-23T18:02:40.040 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010425900000000001 2023-09-23T18:02:40.040 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.040 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.040 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.040 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114577+0000", 2023-09-23T18:02:40.040 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010848083999999999 2023-09-23T18:02:40.040 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.040 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.040 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114604+0000", 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6886999999999999e-05 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114655+0000", 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1703000000000003e-05 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:40.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165828+0000", 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.051172544 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165882+0000", 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4376000000000003e-05 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166024+0000", 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014146100000000001 2023-09-23T18:02:40.042 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154266 5.e 5:70dac101:::benchmark_data_smithi055_229117_object48640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.888147+0000", 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.031420537, 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.278059764, 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:40.043 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154266 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888147+0000", 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888147+0000", 2023-09-23T18:02:40.044 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888146+0000", 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888152+0000", 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4119999999999998e-06 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.045 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888162+0000", 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.679e-06 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888167+0000", 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.775e-06 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888242+0000", 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5483999999999998e-05 2023-09-23T18:02:40.046 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888250+0000", 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.926e-06 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852162+0000", 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.963912133 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.047 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852175+0000", 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2492e-05 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.107232+0000", 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2550570539999999 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.107391+0000", 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000158949 2023-09-23T18:02:40.048 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.107551+0000", 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0001605 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114820+0000", 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007268666 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.049 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.115701+0000", 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00088042600000000002 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.125929+0000", 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010228376000000001 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166033+0000", 2023-09-23T18:02:40.050 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.040103685 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166081+0000", 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8899000000000001e-05 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166207+0000", 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012513100000000001 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.051 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154238 5.f 5:fb40edf7:::benchmark_data_smithi055_229117_object48612:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:09.852458+0000", 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 210.06710959399999, 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3442088729999999, 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154238 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.052 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852458+0000", 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852458+0000", 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852456+0000", 2023-09-23T18:02:40.053 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852463+0000", 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7150000000000001e-06 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852473+0000", 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0584000000000001e-05 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.054 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852481+0000", 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0229999999999992e-06 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852600+0000", 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119177 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852610+0000", 2023-09-23T18:02:40.055 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3980000000000005e-06 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.145648+0000", 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.29303853 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.145845+0000", 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019688400000000001 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.056 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.196602+0000", 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.050756905999999997 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.196667+0000", 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4517999999999995e-05 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154243 5.f 5:f7c86625:::benchmark_data_smithi055_229117_object48617:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:02:40.057 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:09.852496+0000", 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 210.06707138799999, 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.360016345, 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154243 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:40.058 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852496+0000", 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852496+0000", 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852495+0000", 2023-09-23T18:02:40.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852499+0000", 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5109999999999999e-06 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852506+0000", 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1690000000000003e-06 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.060 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852511+0000", 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.464e-06 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852635+0000", 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123979 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.061 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852642+0000", 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0350000000000004e-06 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.197777+0000", 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.345134979 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.197894+0000", 2023-09-23T18:02:40.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116935 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.212448+0000", 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014554414999999999 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.212512+0000", 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4366e-05 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.063 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33530 6.6 6:63a1868b:::benchmark_data_smithi055_275963_object33529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:39.452220+0000", 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 60.467347271999998, 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0750561489999999, 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33530 2023-09-23T18:02:40.064 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452220+0000", 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452220+0000", 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.065 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452219+0000", 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452264+0000", 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4159999999999997e-05 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452272+0000", 2023-09-23T18:02:40.066 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4780000000000008e-06 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452277+0000", 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3499999999999996e-06 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453091+0000", 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00081378099999999997 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453100+0000", 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0420000000000008e-06 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.451792+0000", 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99869191199999996 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.453176+0000", 2023-09-23T18:02:40.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001383965 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.453351+0000", 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017501100000000001 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460176+0000", 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0068253760000000002 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.069 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460632+0000", 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00045508400000000002 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527098+0000", 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.066466833000000003 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527162+0000", 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3319999999999997e-05 2023-09-23T18:02:40.070 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527276+0000", 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000114576 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33541 6.6 6:62c48c76:::benchmark_data_smithi055_275963_object33540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:02:40.071 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 60.467277830999997, 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0751333839999999, 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33541 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:02:40.072 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452289+0000", 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:02:40.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:02:40.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452325+0000", 2023-09-23T18:02:40.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5848e-05 2023-09-23T18:02:40.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:02:40.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452332+0000", 2023-09-23T18:02:40.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5870000000000004e-06 2023-09-23T18:02:40.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452337+0000", 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8189999999999999e-06 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453126+0000", 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00078863000000000002 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453133+0000", 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5630000000000003e-06 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454584+0000", 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0014505659999999 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:02:40.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454710+0000", 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012631 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454859+0000", 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014862400000000001 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460661+0000", 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058021690000000002 2023-09-23T18:02:40.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.078 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.078 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:02:40.078 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.461337+0000", 2023-09-23T18:02:40.078 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00067573600000000002 2023-09-23T18:02:40.078 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.078 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.078 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:02:40.078 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527285+0000", 2023-09-23T18:02:40.078 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.065948602999999995 2023-09-23T18:02:40.078 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527333+0000", 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7329e-05 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527423+0000", 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0160000000000004e-05 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:02:40.079 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:02:40.080 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:02:40.080 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:40.080 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:40.180 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:40.219 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:40.363 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:02:40.539 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:02:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:02:40] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2023-09-23T18:02:40.578 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:40.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:40.581 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:40.682 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:40.717 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:40.762 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:40.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:40.769 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:40.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:40.911 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:40.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:40.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:40.974 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:41.074 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:41.111 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:41.113 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:02:41.243 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:41.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:41.249 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:41.266 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:41.320 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:41.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:41.400 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:41.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:41.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:41.452 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:41.553 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:41.591 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:41.638 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:41.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:41.641 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:41.742 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:41.775 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:41.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:41 smithi055 ceph-mon[179668]: pgmap v566: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 138 B/s rd, 8.1 MiB/s wr, 320 op/s; 42432/215277 objects degraded (19.710%); 7227/215277 objects misplaced (3.357%); 14 MiB/s, 221 objects/s recovering 2023-09-23T18:02:41.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:41 smithi055 ceph-mon[184174]: pgmap v566: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 138 B/s rd, 8.1 MiB/s wr, 320 op/s; 42432/215277 objects degraded (19.710%); 7227/215277 objects misplaced (3.357%); 14 MiB/s, 221 objects/s recovering 2023-09-23T18:02:41.912 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:41.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:41.916 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:42.017 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:42.056 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:42.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:41 smithi099 ceph-mon[167076]: pgmap v566: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.4 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 138 B/s rd, 8.1 MiB/s wr, 320 op/s; 42432/215277 objects degraded (19.710%); 7227/215277 objects misplaced (3.357%); 14 MiB/s, 221 objects/s recovering 2023-09-23T18:02:42.355 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:42.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:42.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:42.462 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:42.527 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:42.545 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:02:42.546 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:02:42.546 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:02:42.546 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:02:42.582 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:42.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:42.588 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:42.689 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:42.726 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:42.794 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T18:02:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:42 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2266177949' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:02:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:42 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:02:42.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:42 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2266177949' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:02:42.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:42 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:02:42.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:42 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2266177949' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:02:42.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:42 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:02:42.931 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:02:42.951 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:42.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:42.957 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:43.053 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:02:43.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:43.081 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:02:43.110 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:02:43.110 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:43.133 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:02:43.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:43.163 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:02:43.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:43.168 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:43.188 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:02:43.216 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:02:43.246 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:02:43.248 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:02:43.248 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:02:43.248 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:02:43.248 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:02:43.248 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:02:43.248 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:02:43.248 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:02:43.249 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:02:43.268 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:43.306 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:43.520 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:43.526 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:43.526 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:43.568 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T18:02:43.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:43.667 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:43.683 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:02:43.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:43.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:43.723 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:43.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[179668]: pgmap v567: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 127 B/s rd, 4.1 MiB/s wr, 320 op/s; 42184/213384 objects degraded (19.769%); 6417/213384 objects misplaced (3.007%); 18 MiB/s, 291 objects/s recovering 2023-09-23T18:02:43.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[179668]: failed to encode map e246 with expected crc 2023-09-23T18:02:43.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:02:43.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[179668]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T18:02:43.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[179668]: failed to encode map e246 with expected crc 2023-09-23T18:02:43.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 42184/213384 objects degraded (19.769%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:43.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3152718427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:43.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1546410257' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:02:43.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[184174]: pgmap v567: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 127 B/s rd, 4.1 MiB/s wr, 320 op/s; 42184/213384 objects degraded (19.769%); 6417/213384 objects misplaced (3.007%); 18 MiB/s, 291 objects/s recovering 2023-09-23T18:02:43.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[184174]: failed to encode map e246 with expected crc 2023-09-23T18:02:43.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:02:43.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[184174]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T18:02:43.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[184174]: failed to encode map e246 with expected crc 2023-09-23T18:02:43.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 42184/213384 objects degraded (19.769%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:43.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3152718427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:43.802 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1546410257' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:02:43.824 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:43.873 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:43.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:43.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:43.938 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:43.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:43 smithi099 ceph-mon[167076]: pgmap v567: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 127 B/s rd, 4.1 MiB/s wr, 320 op/s; 42184/213384 objects degraded (19.769%); 6417/213384 objects misplaced (3.007%); 18 MiB/s, 291 objects/s recovering 2023-09-23T18:02:43.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:43 smithi099 ceph-mon[167076]: failed to encode map e246 with expected crc 2023-09-23T18:02:43.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:43 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:02:43.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:43 smithi099 ceph-mon[167076]: osdmap e246: 8 total, 8 up, 7 in 2023-09-23T18:02:43.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:43 smithi099 ceph-mon[167076]: failed to encode map e246 with expected crc 2023-09-23T18:02:43.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 42184/213384 objects degraded (19.769%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:43.939 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3152718427' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:43.939 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1546410257' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:02:43.939 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:02:44.038 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:44.081 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:44.332 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:44.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:44.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:44.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:44.487 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:44.490 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:44.537 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:44.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:44.541 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:44.641 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:44.672 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:44.717 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:44.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:44.724 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:44.824 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:44 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1546410257' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:02:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:44 smithi099 ceph-mon[167076]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T18:02:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:44 smithi099 ceph-mon[167076]: failed to encode map e247 with expected crc 2023-09-23T18:02:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:44 smithi099 ceph-mon[167076]: failed to encode map e247 with expected crc 2023-09-23T18:02:44.856 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:44.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:44 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:44.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1546410257' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:02:44.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:44 smithi055 ceph-mon[179668]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T18:02:44.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:44 smithi055 ceph-mon[179668]: failed to encode map e247 with expected crc 2023-09-23T18:02:44.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:44 smithi055 ceph-mon[179668]: failed to encode map e247 with expected crc 2023-09-23T18:02:44.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:44 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:44.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1546410257' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:02:44.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:44 smithi055 ceph-mon[184174]: osdmap e247: 8 total, 8 up, 7 in 2023-09-23T18:02:44.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:44 smithi055 ceph-mon[184174]: failed to encode map e247 with expected crc 2023-09-23T18:02:44.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:44 smithi055 ceph-mon[184174]: failed to encode map e247 with expected crc 2023-09-23T18:02:44.997 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:45.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:45.001 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:45.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:45.106 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:02:45.106 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:02:45.147 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:45.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:45.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:45.452 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:45.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:02:45.553 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:45.589 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:45.648 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:45.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:45.652 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:45.753 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:45.783 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:45.795 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:02:45.825 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:45.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:45.832 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:45.928 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:45 smithi055 ceph-mon[179668]: pgmap v570: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 141 B/s rd, 3.2 MiB/s wr, 473 op/s; 42184/210165 objects degraded (20.072%); 6417/210165 objects misplaced (3.053%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:02:45.933 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:45.976 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:46.017 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:46.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:46.022 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:46.122 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:46.152 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:46.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:45 smithi055 ceph-mon[184174]: pgmap v570: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 141 B/s rd, 3.2 MiB/s wr, 473 op/s; 42184/210165 objects degraded (20.072%); 6417/210165 objects misplaced (3.053%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:02:46.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:46.198 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:45 smithi099 ceph-mon[167076]: pgmap v570: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 141 B/s rd, 3.2 MiB/s wr, 473 op/s; 42184/210165 objects degraded (20.072%); 6417/210165 objects misplaced (3.053%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:02:46.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:46.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:46.215 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:02:46.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:46.330 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:46.345 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:02:46.423 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:46.426 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:46.427 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:46.485 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:46.528 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:46.560 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:46.649 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:46.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:46.653 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:46.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:46.788 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:46.826 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:46.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:46.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:46.930 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:46.966 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:46.975 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T18:02:47.034 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:47.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:47.041 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:47.064 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:47.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:46 smithi099 ceph-mon[167076]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T18:02:47.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:46 smithi099 ceph-mon[167076]: failed to encode map e248 with expected crc 2023-09-23T18:02:47.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:46 smithi099 ceph-mon[167076]: failed to encode map e248 with expected crc 2023-09-23T18:02:47.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:46 smithi099 ceph-mon[167076]: failed to encode map e248 with expected crc 2023-09-23T18:02:47.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2698811565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:02:47.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:47.173 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:47.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:46 smithi055 ceph-mon[184174]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T18:02:47.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:46 smithi055 ceph-mon[184174]: failed to encode map e248 with expected crc 2023-09-23T18:02:47.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:46 smithi055 ceph-mon[184174]: failed to encode map e248 with expected crc 2023-09-23T18:02:47.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:46 smithi055 ceph-mon[184174]: failed to encode map e248 with expected crc 2023-09-23T18:02:47.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2698811565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:02:47.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:46 smithi055 ceph-mon[179668]: osdmap e248: 8 total, 8 up, 7 in 2023-09-23T18:02:47.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:46 smithi055 ceph-mon[179668]: failed to encode map e248 with expected crc 2023-09-23T18:02:47.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:46 smithi055 ceph-mon[179668]: failed to encode map e248 with expected crc 2023-09-23T18:02:47.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:46 smithi055 ceph-mon[179668]: failed to encode map e248 with expected crc 2023-09-23T18:02:47.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2698811565' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:02:47.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:47.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:47.211 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:47.312 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:47.348 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:47.409 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T18:02:47.447 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:47.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:47.450 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:47.551 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:47.581 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:47.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:47.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:47.631 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:47.670 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:02:47.670 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:02:47.732 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:47.776 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:47.963 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: failed to encode map e248 with expected crc 2023-09-23T18:02:47.964 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: pgmap v572: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 319 op/s; 41461/209526 objects degraded (19.788%); 5694/209526 objects misplaced (2.718%); 15 MiB/s, 235 objects/s recovering 2023-09-23T18:02:47.965 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: failed to encode map e249 with expected crc 2023-09-23T18:02:48.078 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:02:48.093 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:48.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:48.098 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:48.199 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:48.220 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: failed to encode map e248 with expected crc 2023-09-23T18:02:48.220 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: pgmap v572: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 319 op/s; 41461/209526 objects degraded (19.788%); 5694/209526 objects misplaced (2.718%); 15 MiB/s, 235 objects/s recovering 2023-09-23T18:02:48.220 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: failed to encode map e249 with expected crc 2023-09-23T18:02:48.221 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: failed to encode map e249 with expected crc 2023-09-23T18:02:48.221 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2698811565' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:02:48.221 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T18:02:48.221 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: failed to encode map e249 with expected crc 2023-09-23T18:02:48.221 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 41461/209526 objects degraded (19.788%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:48.221 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: failed to encode map e250 with expected crc 2023-09-23T18:02:48.221 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T18:02:48.221 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: failed to encode map e250 with expected crc 2023-09-23T18:02:48.222 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[184174]: failed to encode map e250 with expected crc 2023-09-23T18:02:48.222 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: failed to encode map e249 with expected crc 2023-09-23T18:02:48.222 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2698811565' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:02:48.222 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T18:02:48.222 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: failed to encode map e249 with expected crc 2023-09-23T18:02:48.222 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 41461/209526 objects degraded (19.788%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:48.223 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: failed to encode map e250 with expected crc 2023-09-23T18:02:48.223 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T18:02:48.223 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: failed to encode map e250 with expected crc 2023-09-23T18:02:48.223 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:47 smithi055 ceph-mon[179668]: failed to encode map e250 with expected crc 2023-09-23T18:02:48.223 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: failed to encode map e248 with expected crc 2023-09-23T18:02:48.223 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: pgmap v572: 119 pgs: 2 active+recovery_wait+degraded, 105 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.3 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 319 op/s; 41461/209526 objects degraded (19.788%); 5694/209526 objects misplaced (2.718%); 15 MiB/s, 235 objects/s recovering 2023-09-23T18:02:48.224 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: failed to encode map e249 with expected crc 2023-09-23T18:02:48.224 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: failed to encode map e249 with expected crc 2023-09-23T18:02:48.224 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2698811565' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:02:48.224 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: osdmap e249: 8 total, 8 up, 7 in 2023-09-23T18:02:48.224 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: failed to encode map e249 with expected crc 2023-09-23T18:02:48.224 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 41461/209526 objects degraded (19.788%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:48.224 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: failed to encode map e250 with expected crc 2023-09-23T18:02:48.225 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: osdmap e250: 8 total, 8 up, 7 in 2023-09-23T18:02:48.225 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: failed to encode map e250 with expected crc 2023-09-23T18:02:48.225 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:47 smithi099 ceph-mon[167076]: failed to encode map e250 with expected crc 2023-09-23T18:02:48.225 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:48.265 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:48.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:48.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:48.369 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:48.403 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:48.464 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:02:48.484 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:02:48.505 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:02:48.527 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:02:48.547 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:02:48.551 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:48.553 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:48.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:48.558 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:48.578 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:02:48.598 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:02:48.621 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:02:48.623 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:02:48.623 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:02:48.623 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:02:48.623 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:02:48.623 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:02:48.624 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:02:48.624 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:02:48.624 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:02:48.658 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:48.689 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:48.734 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:02:48.953 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:02:49.047 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:49.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:49.056 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:49.157 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:49.225 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:49.281 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:48 smithi099 ceph-mon[167076]: failed to encode map e250 with expected crc 2023-09-23T18:02:49.281 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:48 smithi099 ceph-mon[167076]: failed to encode map e250 with expected crc 2023-09-23T18:02:49.282 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:48 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2155143425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:49.282 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:48 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3600826626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:49.282 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:48 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:49.282 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:48 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:49.282 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:48 smithi099 ceph-mon[167076]: failed to encode map e251 with expected crc 2023-09-23T18:02:49.282 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:48 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:49.283 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:48 smithi099 ceph-mon[167076]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T18:02:49.302 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[184174]: failed to encode map e250 with expected crc 2023-09-23T18:02:49.302 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[184174]: failed to encode map e250 with expected crc 2023-09-23T18:02:49.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2155143425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:49.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3600826626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:49.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:49.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:49.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[184174]: failed to encode map e251 with expected crc 2023-09-23T18:02:49.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:49.304 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[184174]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T18:02:49.304 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[179668]: failed to encode map e250 with expected crc 2023-09-23T18:02:49.304 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[179668]: failed to encode map e250 with expected crc 2023-09-23T18:02:49.304 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2155143425' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:49.304 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3600826626' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:49.305 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:02:49.305 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:49.305 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[179668]: failed to encode map e251 with expected crc 2023-09-23T18:02:49.305 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:02:49.305 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:48 smithi055 ceph-mon[179668]: osdmap e251: 8 total, 8 up, 7 in 2023-09-23T18:02:49.388 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:49.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:49.393 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:49.494 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:49.543 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:49.592 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:49.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:49.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:49.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:49.744 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:49.886 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:49.891 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:49.891 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:49.992 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:50.054 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:50.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:50.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:50.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:50.125 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:02:50.125 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:02:50.126 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:02:50.126 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:02:50.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:49 smithi055 ceph-mon[184174]: pgmap v575: 119 pgs: 1 peering, 1 active+recovery_wait+degraded, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 38624/206337 objects degraded (18.719%); 7645/206337 objects misplaced (3.705%); 17 MiB/s, 277 objects/s recovering 2023-09-23T18:02:50.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:49 smithi055 ceph-mon[184174]: failed to encode map e251 with expected crc 2023-09-23T18:02:50.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:49 smithi055 ceph-mon[184174]: failed to encode map e251 with expected crc 2023-09-23T18:02:50.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:49 smithi055 ceph-mon[184174]: 6.b continuing backfill to osd.3 from (217'930,248'4810] MIN to 248'4810 2023-09-23T18:02:50.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:49 smithi055 ceph-mon[184174]: 6.b starting backfill to osd.7 from (0'0,0'0] MAX to 248'4810 2023-09-23T18:02:50.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:49 smithi055 ceph-mon[179668]: pgmap v575: 119 pgs: 1 peering, 1 active+recovery_wait+degraded, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 38624/206337 objects degraded (18.719%); 7645/206337 objects misplaced (3.705%); 17 MiB/s, 277 objects/s recovering 2023-09-23T18:02:50.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:49 smithi055 ceph-mon[179668]: failed to encode map e251 with expected crc 2023-09-23T18:02:50.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:49 smithi055 ceph-mon[179668]: failed to encode map e251 with expected crc 2023-09-23T18:02:50.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:49 smithi055 ceph-mon[179668]: 6.b continuing backfill to osd.3 from (217'930,248'4810] MIN to 248'4810 2023-09-23T18:02:50.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:49 smithi055 ceph-mon[179668]: 6.b starting backfill to osd.7 from (0'0,0'0] MAX to 248'4810 2023-09-23T18:02:50.201 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:50.245 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:50.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:49 smithi099 ceph-mon[167076]: pgmap v575: 119 pgs: 1 peering, 1 active+recovery_wait+degraded, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 7 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 367 op/s; 38624/206337 objects degraded (18.719%); 7645/206337 objects misplaced (3.705%); 17 MiB/s, 277 objects/s recovering 2023-09-23T18:02:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:49 smithi099 ceph-mon[167076]: failed to encode map e251 with expected crc 2023-09-23T18:02:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:49 smithi099 ceph-mon[167076]: failed to encode map e251 with expected crc 2023-09-23T18:02:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:49 smithi099 ceph-mon[167076]: 6.b continuing backfill to osd.3 from (217'930,248'4810] MIN to 248'4810 2023-09-23T18:02:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:49 smithi099 ceph-mon[167076]: 6.b starting backfill to osd.7 from (0'0,0'0] MAX to 248'4810 2023-09-23T18:02:50.588 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:50.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:50.591 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:50.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:02:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:02:50] "GET /metrics HTTP/1.1" 200 36575 "" "Prometheus/2.43.0" 2023-09-23T18:02:50.692 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:50.738 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:50.795 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:50.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:50.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:50.901 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:50.936 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:51.021 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[179668]: failed to encode map e252 with expected crc 2023-09-23T18:02:51.021 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[179668]: failed to encode map e252 with expected crc 2023-09-23T18:02:51.021 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[179668]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T18:02:51.022 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[179668]: failed to encode map e252 with expected crc 2023-09-23T18:02:51.022 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3924993209' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:02:51.022 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:02:51.037 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:02:51.046 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T18:02:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:51 smithi099 ceph-mon[167076]: failed to encode map e252 with expected crc 2023-09-23T18:02:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:51 smithi099 ceph-mon[167076]: failed to encode map e252 with expected crc 2023-09-23T18:02:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:51 smithi099 ceph-mon[167076]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T18:02:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:51 smithi099 ceph-mon[167076]: failed to encode map e252 with expected crc 2023-09-23T18:02:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3924993209' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:02:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:51 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:02:51.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[184174]: failed to encode map e252 with expected crc 2023-09-23T18:02:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[184174]: failed to encode map e252 with expected crc 2023-09-23T18:02:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[184174]: osdmap e252: 8 total, 8 up, 7 in 2023-09-23T18:02:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[184174]: failed to encode map e252 with expected crc 2023-09-23T18:02:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3924993209' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:02:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:51 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:02:51.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:51.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:51.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:51.472 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:02:51.502 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:02:51.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:51.576 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:51.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:51.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:51.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:51.731 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:51.776 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:51.819 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:51.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:51.822 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:51.923 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:51.960 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:51.974 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T18:02:51.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:51.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:51.998 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:52.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:52 smithi099 ceph-mon[167076]: pgmap v578: 119 pgs: 1 peering, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 29 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 38105/205023 objects degraded (18.586%); 14926/205023 objects misplaced (7.280%); 17 MiB/s, 275 objects/s recovering 2023-09-23T18:02:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:52 smithi099 ceph-mon[167076]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:52 smithi099 ceph-mon[167076]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:52 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:02:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:52 smithi099 ceph-mon[167076]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T18:02:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:52 smithi099 ceph-mon[167076]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:52 smithi099 ceph-mon[167076]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:52 smithi099 ceph-mon[167076]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:52.099 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:52.132 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:52.175 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:52.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:52.179 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[184174]: pgmap v578: 119 pgs: 1 peering, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 29 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 38105/205023 objects degraded (18.586%); 14926/205023 objects misplaced (7.280%); 17 MiB/s, 275 objects/s recovering 2023-09-23T18:02:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[184174]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[184174]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:02:52.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[184174]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T18:02:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[184174]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[184174]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[184174]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:52.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[179668]: pgmap v578: 119 pgs: 1 peering, 1 active+recovery_wait+degraded+remapped, 76 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 29 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.2 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 375 op/s; 38105/205023 objects degraded (18.586%); 14926/205023 objects misplaced (7.280%); 17 MiB/s, 275 objects/s recovering 2023-09-23T18:02:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[179668]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[179668]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:02:52.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[179668]: osdmap e253: 8 total, 8 up, 7 in 2023-09-23T18:02:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[179668]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[179668]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[179668]: failed to encode map e253 with expected crc 2023-09-23T18:02:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:02:52.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:02:52.191 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:02:52.253 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:52.281 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:52.305 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:52.332 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:52.547 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:52.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:52.555 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:52.656 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:52.708 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:52.757 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:52.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:52.761 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:52.861 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:52.896 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:53.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:53.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:53.031 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:53.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:53.166 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:53.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 38105/205023 objects degraded (18.586%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:53.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 38105/205023 objects degraded (18.586%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:53.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:53.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:53.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:53.314 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 38105/205023 objects degraded (18.586%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:53.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:53.360 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:53.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:53.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:53.451 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:53.479 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:53.532 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:02:53.532 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:02:53.532 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:02:53.532 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:02:53.532 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:02:53.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:53.583 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:53.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:02:53.901 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:53.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:53.904 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:54.005 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:54.053 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:54.113 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:54.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:54.121 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:54 smithi099 ceph-mon[167076]: pgmap v580: 119 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 29 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 37906/203526 objects degraded (18.625%); 14830/203526 objects misplaced (7.287%); 6.7 MiB/s, 110 objects/s recovering 2023-09-23T18:02:54.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:54.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:54 smithi055 ceph-mon[184174]: pgmap v580: 119 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 29 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 37906/203526 objects degraded (18.625%); 14830/203526 objects misplaced (7.287%); 6.7 MiB/s, 110 objects/s recovering 2023-09-23T18:02:54.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:54 smithi055 ceph-mon[179668]: pgmap v580: 119 pgs: 1 active+recovery_wait+degraded+remapped, 77 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 29 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 37906/203526 objects degraded (18.625%); 14830/203526 objects misplaced (7.287%); 6.7 MiB/s, 110 objects/s recovering 2023-09-23T18:02:54.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:54.256 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:54.301 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:54.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:54.306 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:54.385 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:02:54.407 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:54.408 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:02:54.433 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:02:54.435 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:54.456 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:02:54.473 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:54.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:54.478 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:54.493 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:02:54.522 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:02:54.553 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:02:54.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:54.584 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:02:54.586 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:02:54.586 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:02:54.586 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:02:54.587 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:02:54.587 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:02:54.587 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:02:54.587 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:02:54.587 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:02:54.601 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:02:54.611 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:54.655 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:54.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:54.660 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:54.760 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:54.827 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:54.884 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:54.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:54.889 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:54.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:55.024 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:55.071 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:02:55.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1074210964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1076462916' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:55 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:55.354 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:55.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:55.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:55.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1074210964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:55.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1076462916' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:55.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:55 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:55.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1074210964' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:02:55.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1076462916' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:55.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:55 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:02:55.459 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:55.515 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:55.583 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:55.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:55.587 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:55.688 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:55.745 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:55.793 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:55.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:55.798 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:55.898 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:55.904 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:02:55.904 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:02:55.944 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:55.991 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:55.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:55.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:56.073 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T18:02:56.095 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:56.127 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:56.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:56 smithi099 ceph-mon[167076]: pgmap v581: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 339 op/s; 37580/200232 objects degraded (18.768%); 14785/200232 objects misplaced (7.384%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T18:02:56.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:56 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:56.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:56 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:02:56.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:56 smithi099 ceph-mon[167076]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T18:02:56.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:56 smithi099 ceph-mon[167076]: failed to encode map e254 with expected crc 2023-09-23T18:02:56.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2464359151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:02:56.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:56 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:02:56.299 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:02:56.413 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[184174]: pgmap v581: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 339 op/s; 37580/200232 objects degraded (18.768%); 14785/200232 objects misplaced (7.384%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T18:02:56.413 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:56.414 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:02:56.414 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[184174]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T18:02:56.414 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[184174]: failed to encode map e254 with expected crc 2023-09-23T18:02:56.414 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2464359151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:02:56.414 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:02:56.414 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[179668]: pgmap v581: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 339 op/s; 37580/200232 objects degraded (18.768%); 14785/200232 objects misplaced (7.384%); 9.2 MiB/s, 146 objects/s recovering 2023-09-23T18:02:56.414 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:02:56.415 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:02:56.415 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[179668]: osdmap e254: 8 total, 8 up, 7 in 2023-09-23T18:02:56.415 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[179668]: failed to encode map e254 with expected crc 2023-09-23T18:02:56.415 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2464359151' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:02:56.415 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:56 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:02:56.426 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:56.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:56.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:56.476 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T18:02:56.476 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:02:56.533 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:56.592 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:56.661 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:56.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:56.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:56.768 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:56.800 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:56.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:56.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:56.848 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:56.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:02:56.989 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:57.005 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:02:57.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:02:57.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:57.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:02:57.042 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:02:57.140 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:57.180 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:57.232 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:02:57.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:57.237 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:02:57.252 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:02:57.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:02:57.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:57 smithi099 ceph-mon[167076]: failed to encode map e255 with expected crc 2023-09-23T18:02:57.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:57 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:02:57.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:57 smithi099 ceph-mon[167076]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T18:02:57.365 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:57.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:57 smithi055 ceph-mon[184174]: failed to encode map e255 with expected crc 2023-09-23T18:02:57.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:57 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:02:57.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:57 smithi055 ceph-mon[184174]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T18:02:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:57 smithi055 ceph-mon[179668]: failed to encode map e255 with expected crc 2023-09-23T18:02:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:57 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:02:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:57 smithi055 ceph-mon[179668]: osdmap e255: 8 total, 8 up, 7 in 2023-09-23T18:02:57.481 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:02:58.119 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:02:58.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:58.122 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:02:58.223 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:02:58.251 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:58.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:58 smithi055 ceph-mon[184174]: pgmap v584: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 37580/199425 objects degraded (18.844%); 14537/199425 objects misplaced (7.289%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T18:02:58.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:02:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 37580/199425 objects degraded (18.844%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:58.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:58 smithi055 ceph-mon[179668]: pgmap v584: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 37580/199425 objects degraded (18.844%); 14537/199425 objects misplaced (7.289%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T18:02:58.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:02:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 37580/199425 objects degraded (18.844%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:58.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:58 smithi099 ceph-mon[167076]: pgmap v584: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.1 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 310 op/s; 37580/199425 objects degraded (18.844%); 14537/199425 objects misplaced (7.289%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T18:02:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:02:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 37580/199425 objects degraded (18.844%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:02:58.612 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:02:58.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:58.617 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:02:58.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:02:58.751 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:59.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:02:59.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:59.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:02:59.249 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:02:59.290 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:59.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:02:59.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:59.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:02:59.546 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:02:59.587 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:59.639 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:02:59.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:59.643 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:02:59.744 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:02:59.779 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:02:59.823 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:02:59.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:02:59.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:02:59.931 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:02:59.992 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:00.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:00 smithi055 ceph-mon[184174]: pgmap v585: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 37282/195198 objects degraded (19.100%); 14383/195198 objects misplaced (7.368%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T18:03:00.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:00 smithi055 ceph-mon[179668]: pgmap v585: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 37282/195198 objects degraded (19.100%); 14383/195198 objects misplaced (7.368%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T18:03:00.209 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:03:00.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:00.216 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:03:00.277 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:00.317 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:00.333 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:03:00.334 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:03:00.356 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:00 smithi099 ceph-mon[167076]: pgmap v585: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 4.0 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 428 op/s; 37282/195198 objects degraded (19.100%); 14383/195198 objects misplaced (7.368%); 3.1 MiB/s, 48 objects/s recovering 2023-09-23T18:03:00.378 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:00.446 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:00.451 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:00.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:00.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:00.533 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:03:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:03:00] "GET /metrics HTTP/1.1" 200 36545 "" "Prometheus/2.43.0" 2023-09-23T18:03:00.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:00.591 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:00.684 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:00.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:00.690 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:00.734 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:03:00.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:00.835 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:00.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:00.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:00.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:00.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:01.041 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:01.084 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:01.086 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:01.087 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:01.187 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:01.205 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:03:01.207 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:03:01.222 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:01.261 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:01.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:01.266 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:01.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:01.393 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:01.428 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:01.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:01.432 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:01.448 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:03:01.533 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:01.561 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:01.585 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":586,"stamp":"2023-09-23T18:03:00.671645+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4207360518,"num_objects":64157,"num_object_clones":0,"num_object_copies":192471,"num_objects_missing_on_primary":629,"num_objects_missing":629,"num_objects_degraded":36946,"num_objects_misplaced":14097,"num_objects_unfound":0,"num_objects_dirty":64157,"num_whiteouts":0,"num_read":6284,"num_read_kb":12770,"num_write":83211,"num_write_kb":4695012,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14229,"num_bytes_recovered":928188871,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":81000,"ondisk_log_size":81000,"up":357,"acting":345,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":405,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13678272,"kb_used_data":12241160,"kb_used_omap":43,"kb_used_meta":1436948,"kb_avail":642537792,"statfs":{"total":671965249536,"available":657958699008,"internally_reserved":0,"allocated":12534947840,"data_stored":12527033224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1471435471},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-241500160,"num_objects":-3685,"num_object_clones":0,"num_object_copies":-11055,"num_objects_missing_on_primary":-46,"num_objects_missing":-46,"num_objects_degraded":-960,"num_objects_misplaced":-733,"num_objects_unfound":0,"num_objects_dirty":-3685,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3685,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":709,"num_bytes_recovered":46465024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003994"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":582,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:55.091879+0000","last_change":"2023-09-23T18:02:48.964317+0000","last_active":"2023-09-23T18:02:55.091879+0000","last_peered":"2023-09-23T18:02:55.091879+0000","last_clean":"2023-09-23T18:02:45.962259+0000","last_became_active":"2023-09-23T18:02:48.961702+0000","last_became_peered":"2023-09-23T18:02:48.961702+0000","last_unstale":"2023-09-23T18:02:55.091879+0000","last_undegraded":"2023-09-23T18:02:48.941433+0000","last_fullsized":"2023-09-23T18:02:48.940979+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":474537,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":55,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097658+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:02:55.097658+0000","last_peered":"2023-09-23T18:02:55.097658+0000","last_clean":"2023-09-23T18:02:55.097658+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:02:55.097658+0000","last_undegraded":"2023-09-23T18:02:55.097658+0000","last_fullsized":"2023-09-23T18:02:55.097658+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086508+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:02:55.086508+0000","last_peered":"2023-09-23T18:02:55.086508+0000","last_clean":"2023-09-23T18:02:55.086508+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:02:55.086508+0000","last_undegraded":"2023-09-23T18:02:55.086508+0000","last_fullsized":"2023-09-23T18:02:55.086508+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":597,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087754+0000","last_change":"2023-09-23T18:02:51.046916+0000","last_active":"2023-09-23T18:02:55.087754+0000","last_peered":"2023-09-23T18:02:55.087754+0000","last_clean":"2023-09-23T18:02:55.087754+0000","last_became_active":"2023-09-23T18:02:51.046591+0000","last_became_peered":"2023-09-23T18:02:51.046591+0000","last_unstale":"2023-09-23T18:02:55.087754+0000","last_undegraded":"2023-09-23T18:02:55.087754+0000","last_fullsized":"2023-09-23T18:02:55.087754+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:19.805099+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":104,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088134+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:02:55.088134+0000","last_peered":"2023-09-23T18:02:55.088134+0000","last_clean":"2023-09-23T18:02:55.088134+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:02:55.088134+0000","last_undegraded":"2023-09-23T18:02:55.088134+0000","last_fullsized":"2023-09-23T18:02:55.088134+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":255,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.093493+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:02:55.093493+0000","last_peered":"2023-09-23T18:02:55.093493+0000","last_clean":"2023-09-23T18:02:55.093493+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:02:55.093493+0000","last_undegraded":"2023-09-23T18:02:55.093493+0000","last_fullsized":"2023-09-23T18:02:55.093493+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":589,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087376+0000","last_change":"2023-09-23T18:02:51.051617+0000","last_active":"2023-09-23T18:02:55.087376+0000","last_peered":"2023-09-23T18:02:55.087376+0000","last_clean":"2023-09-23T18:02:55.087376+0000","last_became_active":"2023-09-23T18:02:51.048947+0000","last_became_peered":"2023-09-23T18:02:51.048947+0000","last_unstale":"2023-09-23T18:02:55.087376+0000","last_undegraded":"2023-09-23T18:02:55.087376+0000","last_fullsized":"2023-09-23T18:02:55.087376+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:49:34.962249+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":117,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.085983+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:02:55.085983+0000","last_peered":"2023-09-23T18:02:55.085983+0000","last_clean":"2023-09-23T18:02:55.085983+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:02:55.085983+0000","last_undegraded":"2023-09-23T18:02:55.085983+0000","last_fullsized":"2023-09-23T18:02:55.085983+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":195,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.071794+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:02:55.071794+0000","last_peered":"2023-09-23T18:02:55.071794+0000","last_clean":"2023-09-23T18:02:55.071794+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:02:55.071794+0000","last_undegraded":"2023-09-23T18:02:55.071794+0000","last_fullsized":"2023-09-23T18:02:55.071794+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088053+0000","last_change":"2023-09-23T18:02:51.052123+0000","last_active":"2023-09-23T18:02:55.088053+0000","last_peered":"2023-09-23T18:02:55.088053+0000","last_clean":"2023-09-23T18:02:55.088053+0000","last_became_active":"2023-09-23T18:02:51.051772+0000","last_became_peered":"2023-09-23T18:02:51.051772+0000","last_unstale":"2023-09-23T18:02:55.088053+0000","last_undegraded":"2023-09-23T18:02:55.088053+0000","last_fullsized":"2023-09-23T18:02:55.088053+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":235,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:02:56.088939+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:02:56.088939+0000","last_peered":"2023-09-23T18:02:56.088939+0000","last_clean":"2023-09-23T18:02:56.088939+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:02:56.088939+0000","last_undegraded":"2023-09-23T18:02:56.088939+0000","last_fullsized":"2023-09-23T18:02:56.088939+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":111,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.085996+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:02:55.085996+0000","last_peered":"2023-09-23T18:02:55.085996+0000","last_clean":"2023-09-23T18:02:55.085996+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:02:55.085996+0000","last_undegraded":"2023-09-23T18:02:55.085996+0000","last_fullsized":"2023-09-23T18:02:55.085996+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":581,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087470+0000","last_change":"2023-09-23T18:02:51.054208+0000","last_active":"2023-09-23T18:02:55.087470+0000","last_peered":"2023-09-23T18:02:55.087470+0000","last_clean":"2023-09-23T18:02:55.087470+0000","last_became_active":"2023-09-23T18:02:51.053676+0000","last_became_peered":"2023-09-23T18:02:51.053676+0000","last_unstale":"2023-09-23T18:02:55.087470+0000","last_undegraded":"2023-09-23T18:02:55.087470+0000","last_fullsized":"2023-09-23T18:02:55.087470+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":237,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.084272+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:02:55.084272+0000","last_peered":"2023-09-23T18:02:55.084272+0000","last_clean":"2023-09-23T18:02:55.084272+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:02:55.084272+0000","last_undegraded":"2023-09-23T18:02:55.084272+0000","last_fullsized":"2023-09-23T18:02:55.084272+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088023+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:02:55.088023+0000","last_peered":"2023-09-23T18:02:55.088023+0000","last_clean":"2023-09-23T18:02:55.088023+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:02:55.088023+0000","last_undegraded":"2023-09-23T18:02:55.088023+0000","last_fullsized":"2023-09-23T18:02:55.088023+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087511+0000","last_change":"2023-09-23T18:02:51.058624+0000","last_active":"2023-09-23T18:02:55.087511+0000","last_peered":"2023-09-23T18:02:55.087511+0000","last_clean":"2023-09-23T18:02:55.087511+0000","last_became_active":"2023-09-23T18:02:51.057352+0000","last_became_peered":"2023-09-23T18:02:51.057352+0000","last_unstale":"2023-09-23T18:02:55.087511+0000","last_undegraded":"2023-09-23T18:02:55.087511+0000","last_fullsized":"2023-09-23T18:02:55.087511+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":21,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087979+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:02:55.087979+0000","last_peered":"2023-09-23T18:02:55.087979+0000","last_clean":"2023-09-23T18:02:55.087979+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:02:55.087979+0000","last_undegraded":"2023-09-23T18:02:55.087979+0000","last_fullsized":"2023-09-23T18:02:55.087979+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":151,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.072007+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:02:55.072007+0000","last_peered":"2023-09-23T18:02:55.072007+0000","last_clean":"2023-09-23T18:02:55.072007+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:02:55.072007+0000","last_undegraded":"2023-09-23T18:02:55.072007+0000","last_fullsized":"2023-09-23T18:02:55.072007+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087993+0000","last_change":"2023-09-23T18:02:51.059983+0000","last_active":"2023-09-23T18:02:55.087993+0000","last_peered":"2023-09-23T18:02:55.087993+0000","last_clean":"2023-09-23T18:02:55.087993+0000","last_became_active":"2023-09-23T18:02:51.057768+0000","last_became_peered":"2023-09-23T18:02:51.057768+0000","last_unstale":"2023-09-23T18:02:55.087993+0000","last_undegraded":"2023-09-23T18:02:55.087993+0000","last_fullsized":"2023-09-23T18:02:55.087993+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":55,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097938+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:02:55.097938+0000","last_peered":"2023-09-23T18:02:55.097938+0000","last_clean":"2023-09-23T18:02:55.097938+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:02:55.097938+0000","last_undegraded":"2023-09-23T18:02:55.097938+0000","last_fullsized":"2023-09-23T18:02:55.097938+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":55,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097762+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:02:55.097762+0000","last_peered":"2023-09-23T18:02:55.097762+0000","last_clean":"2023-09-23T18:02:55.097762+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:02:55.097762+0000","last_undegraded":"2023-09-23T18:02:55.097762+0000","last_fullsized":"2023-09-23T18:02:55.097762+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":576,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087196+0000","last_change":"2023-09-23T18:02:51.047570+0000","last_active":"2023-09-23T18:02:55.087196+0000","last_peered":"2023-09-23T18:02:55.087196+0000","last_clean":"2023-09-23T18:02:55.087196+0000","last_became_active":"2023-09-23T18:02:51.046963+0000","last_became_peered":"2023-09-23T18:02:51.046963+0000","last_unstale":"2023-09-23T18:02:55.087196+0000","last_undegraded":"2023-09-23T18:02:55.087196+0000","last_fullsized":"2023-09-23T18:02:55.087196+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":184,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088097+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:02:55.088097+0000","last_peered":"2023-09-23T18:02:55.088097+0000","last_clean":"2023-09-23T18:02:55.088097+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:02:55.088097+0000","last_undegraded":"2023-09-23T18:02:55.088097+0000","last_fullsized":"2023-09-23T18:02:55.088097+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":99,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086468+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:02:55.086468+0000","last_peered":"2023-09-23T18:02:55.086468+0000","last_clean":"2023-09-23T18:02:55.086468+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:02:55.086468+0000","last_undegraded":"2023-09-23T18:02:55.086468+0000","last_fullsized":"2023-09-23T18:02:55.086468+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":589,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086799+0000","last_change":"2023-09-23T18:02:51.053806+0000","last_active":"2023-09-23T18:02:55.086799+0000","last_peered":"2023-09-23T18:02:55.086799+0000","last_clean":"2023-09-23T18:02:55.086799+0000","last_became_active":"2023-09-23T18:02:51.048822+0000","last_became_peered":"2023-09-23T18:02:51.048822+0000","last_unstale":"2023-09-23T18:02:55.086799+0000","last_undegraded":"2023-09-23T18:02:55.086799+0000","last_fullsized":"2023-09-23T18:02:55.086799+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:34.401104+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":254,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.093665+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:02:55.093665+0000","last_peered":"2023-09-23T18:02:55.093665+0000","last_clean":"2023-09-23T18:02:55.093665+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:02:55.093665+0000","last_undegraded":"2023-09-23T18:02:55.093665+0000","last_fullsized":"2023-09-23T18:02:55.093665+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":232,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083411+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:02:55.083411+0000","last_peered":"2023-09-23T18:02:55.083411+0000","last_clean":"2023-09-23T18:02:55.083411+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:02:55.083411+0000","last_undegraded":"2023-09-23T18:02:55.083411+0000","last_fullsized":"2023-09-23T18:02:55.083411+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":577,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087946+0000","last_change":"2023-09-23T18:02:51.046646+0000","last_active":"2023-09-23T18:02:55.087946+0000","last_peered":"2023-09-23T18:02:55.087946+0000","last_clean":"2023-09-23T18:02:55.087946+0000","last_became_active":"2023-09-23T18:02:51.046134+0000","last_became_peered":"2023-09-23T18:02:51.046134+0000","last_unstale":"2023-09-23T18:02:55.087946+0000","last_undegraded":"2023-09-23T18:02:55.087946+0000","last_fullsized":"2023-09-23T18:02:55.087946+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097488+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:02:55.097488+0000","last_peered":"2023-09-23T18:02:55.097488+0000","last_clean":"2023-09-23T18:02:55.097488+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:02:55.097488+0000","last_undegraded":"2023-09-23T18:02:55.097488+0000","last_fullsized":"2023-09-23T18:02:55.097488+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":245,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:02:56.089114+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:02:56.089114+0000","last_peered":"2023-09-23T18:02:56.089114+0000","last_clean":"2023-09-23T18:02:56.089114+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:02:56.089114+0000","last_undegraded":"2023-09-23T18:02:56.089114+0000","last_fullsized":"2023-09-23T18:02:56.089114+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":589,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087300+0000","last_change":"2023-09-23T18:02:51.051817+0000","last_active":"2023-09-23T18:02:55.087300+0000","last_peered":"2023-09-23T18:02:55.087300+0000","last_clean":"2023-09-23T18:02:55.087300+0000","last_became_active":"2023-09-23T18:02:51.048945+0000","last_became_peered":"2023-09-23T18:02:51.048945+0000","last_unstale":"2023-09-23T18:02:55.087300+0000","last_undegraded":"2023-09-23T18:02:55.087300+0000","last_fullsized":"2023-09-23T18:02:55.087300+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:47.163864+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":191,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.070864+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:02:55.070864+0000","last_peered":"2023-09-23T18:02:55.070864+0000","last_clean":"2023-09-23T18:02:55.070864+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:02:55.070864+0000","last_undegraded":"2023-09-23T18:02:55.070864+0000","last_fullsized":"2023-09-23T18:02:55.070864+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":108,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086772+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:02:55.086772+0000","last_peered":"2023-09-23T18:02:55.086772+0000","last_clean":"2023-09-23T18:02:55.086772+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:02:55.086772+0000","last_undegraded":"2023-09-23T18:02:55.086772+0000","last_fullsized":"2023-09-23T18:02:55.086772+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"255'5169","reported_seq":5227,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:02:57.551017+0000","last_change":"2023-09-23T18:00:57.996982+0000","last_active":"2023-09-23T18:02:57.551017+0000","last_peered":"2023-09-23T18:02:57.551017+0000","last_clean":"2023-09-23T18:02:57.551017+0000","last_became_active":"2023-09-23T18:00:57.996589+0000","last_became_peered":"2023-09-23T18:00:57.996589+0000","last_unstale":"2023-09-23T18:02:57.551017+0000","last_undegraded":"2023-09-23T18:02:57.551017+0000","last_fullsized":"2023-09-23T18:02:57.551017+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5169,"log_dups_size":0,"ondisk_log_size":5169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:13.306089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266928128,"num_objects":4073,"num_object_clones":0,"num_object_copies":12219,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4073,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5169,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087896+0000","last_change":"2023-09-23T18:02:51.058477+0000","last_active":"2023-09-23T18:02:55.087896+0000","last_peered":"2023-09-23T18:02:55.087896+0000","last_clean":"2023-09-23T18:02:55.087896+0000","last_became_active":"2023-09-23T18:02:51.057258+0000","last_became_peered":"2023-09-23T18:02:51.057258+0000","last_unstale":"2023-09-23T18:02:55.087896+0000","last_undegraded":"2023-09-23T18:02:55.087896+0000","last_fullsized":"2023-09-23T18:02:55.087896+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":184,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087681+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:02:55.087681+0000","last_peered":"2023-09-23T18:02:55.087681+0000","last_clean":"2023-09-23T18:02:55.087681+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:02:55.087681+0000","last_undegraded":"2023-09-23T18:02:55.087681+0000","last_fullsized":"2023-09-23T18:02:55.087681+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":243,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083165+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:02:55.083165+0000","last_peered":"2023-09-23T18:02:55.083165+0000","last_clean":"2023-09-23T18:02:55.083165+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:02:55.083165+0000","last_undegraded":"2023-09-23T18:02:55.083165+0000","last_fullsized":"2023-09-23T18:02:55.083165+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"255'5128","reported_seq":12346,"reported_epoch":255,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:03:00.455861+0000","last_change":"2023-09-23T18:02:49.039290+0000","last_active":"2023-09-23T18:03:00.455861+0000","last_peered":"2023-09-23T18:03:00.455861+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:49.001792+0000","last_became_peered":"2023-09-23T18:02:49.001792+0000","last_unstale":"2023-09-23T18:03:00.455861+0000","last_undegraded":"2023-09-23T18:02:48.968288+0000","last_fullsized":"2023-09-23T18:03:00.455861+0000","mapping_epoch":250,"log_start":"217'930","ondisk_log_start":"217'930","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4198,"log_dups_size":0,"ondisk_log_size":4198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.156979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259260416,"num_objects":3956,"num_object_clones":0,"num_object_copies":11868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":409,"num_objects_misplaced":7503,"num_objects_unfound":0,"num_objects_dirty":3956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5128,"num_write_kb":290688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3566,"num_bytes_recovered":233701376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[6,1,4],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,4,6","objects":3547},{"shards":"1,6","objects":409}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":591,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:55.093045+0000","last_change":"2023-09-23T18:02:48.968551+0000","last_active":"2023-09-23T18:02:55.093045+0000","last_peered":"2023-09-23T18:02:55.093045+0000","last_clean":"2023-09-23T18:02:45.965241+0000","last_became_active":"2023-09-23T18:02:48.968221+0000","last_became_peered":"2023-09-23T18:02:48.968221+0000","last_unstale":"2023-09-23T18:02:55.093045+0000","last_undegraded":"2023-09-23T18:02:48.944021+0000","last_fullsized":"2023-09-23T18:02:48.942326+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":409128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":199,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088601+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:02:55.088601+0000","last_peered":"2023-09-23T18:02:55.088601+0000","last_clean":"2023-09-23T18:02:55.088601+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:02:55.088601+0000","last_undegraded":"2023-09-23T18:02:55.088601+0000","last_fullsized":"2023-09-23T18:02:55.088601+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":265,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097659+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:02:55.097659+0000","last_peered":"2023-09-23T18:02:55.097659+0000","last_clean":"2023-09-23T18:02:55.097659+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:02:55.097659+0000","last_undegraded":"2023-09-23T18:02:55.097659+0000","last_fullsized":"2023-09-23T18:02:55.097659+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"255'5239","reported_seq":6838,"reported_epoch":255,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:59.259975+0000","last_change":"2023-09-23T18:02:47.987792+0000","last_active":"2023-09-23T18:02:59.259975+0000","last_peered":"2023-09-23T18:02:59.259975+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.307615+0000","last_became_peered":"2023-09-23T18:02:19.307615+0000","last_unstale":"2023-09-23T18:02:59.259975+0000","last_undegraded":"2023-09-23T18:02:18.966041+0000","last_fullsized":"2023-09-23T18:02:18.929348+0000","mapping_epoch":238,"log_start":"217'500","ondisk_log_start":"217'500","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4739,"log_dups_size":0,"ondisk_log_size":4739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:04.498873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262340608,"num_objects":4003,"num_object_clones":0,"num_object_copies":12009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3494,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5239,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":764,"num_bytes_recovered":50069504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":509}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":577,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087196+0000","last_change":"2023-09-23T18:02:51.058795+0000","last_active":"2023-09-23T18:02:55.087196+0000","last_peered":"2023-09-23T18:02:55.087196+0000","last_clean":"2023-09-23T18:02:55.087196+0000","last_became_active":"2023-09-23T18:02:51.057799+0000","last_became_peered":"2023-09-23T18:02:51.057799+0000","last_unstale":"2023-09-23T18:02:55.087196+0000","last_undegraded":"2023-09-23T18:02:55.087196+0000","last_fullsized":"2023-09-23T18:02:55.087196+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":122,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088783+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:02:55.088783+0000","last_peered":"2023-09-23T18:02:55.088783+0000","last_clean":"2023-09-23T18:02:55.088783+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:02:55.088783+0000","last_undegraded":"2023-09-23T18:02:55.088783+0000","last_fullsized":"2023-09-23T18:02:55.088783+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":220,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.071878+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:02:55.071878+0000","last_peered":"2023-09-23T18:02:55.071878+0000","last_clean":"2023-09-23T18:02:55.071878+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:02:55.071878+0000","last_undegraded":"2023-09-23T18:02:55.071878+0000","last_fullsized":"2023-09-23T18:02:55.071878+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"255'5257","reported_seq":5334,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:02:59.473944+0000","last_change":"2023-09-23T18:01:25.019784+0000","last_active":"2023-09-23T18:02:59.473944+0000","last_peered":"2023-09-23T18:02:59.473944+0000","last_clean":"2023-09-23T18:02:59.473944+0000","last_became_active":"2023-09-23T18:01:25.002073+0000","last_became_peered":"2023-09-23T18:01:25.002073+0000","last_unstale":"2023-09-23T18:02:59.473944+0000","last_undegraded":"2023-09-23T18:02:59.473944+0000","last_fullsized":"2023-09-23T18:02:59.473944+0000","mapping_epoch":214,"log_start":"217'900","ondisk_log_start":"217'900","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4357,"log_dups_size":0,"ondisk_log_size":4357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:58.981672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265093120,"num_objects":4045,"num_object_clones":0,"num_object_copies":12135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5257,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088163+0000","last_change":"2023-09-23T18:02:51.057506+0000","last_active":"2023-09-23T18:02:55.088163+0000","last_peered":"2023-09-23T18:02:55.088163+0000","last_clean":"2023-09-23T18:02:55.088163+0000","last_became_active":"2023-09-23T18:02:51.056589+0000","last_became_peered":"2023-09-23T18:02:51.056589+0000","last_unstale":"2023-09-23T18:02:55.088163+0000","last_undegraded":"2023-09-23T18:02:55.088163+0000","last_fullsized":"2023-09-23T18:02:55.088163+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":231,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.084322+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:02:55.084322+0000","last_peered":"2023-09-23T18:02:55.084322+0000","last_clean":"2023-09-23T18:02:55.084322+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:02:55.084322+0000","last_undegraded":"2023-09-23T18:02:55.084322+0000","last_fullsized":"2023-09-23T18:02:55.084322+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":257,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092395+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:02:55.092395+0000","last_peered":"2023-09-23T18:02:55.092395+0000","last_clean":"2023-09-23T18:02:55.092395+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:02:55.092395+0000","last_undegraded":"2023-09-23T18:02:55.092395+0000","last_fullsized":"2023-09-23T18:02:55.092395+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"255'5201","reported_seq":8337,"reported_epoch":255,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:02:57.813918+0000","last_change":"2023-09-23T18:02:45.688221+0000","last_active":"2023-09-23T18:02:57.813918+0000","last_peered":"2023-09-23T18:02:57.813918+0000","last_clean":"2023-09-23T18:01:38.416095+0000","last_became_active":"2023-09-23T18:02:07.547135+0000","last_became_peered":"2023-09-23T18:02:07.547135+0000","last_unstale":"2023-09-23T18:02:57.813918+0000","last_undegraded":"2023-09-23T18:02:57.813918+0000","last_fullsized":"2023-09-23T18:02:02.784733+0000","mapping_epoch":227,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5101,"log_dups_size":0,"ondisk_log_size":5101,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:27.864301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265617408,"num_objects":4053,"num_object_clones":0,"num_object_copies":12159,"num_objects_missing_on_primary":629,"num_objects_missing":629,"num_objects_degraded":0,"num_objects_misplaced":2568,"num_objects_unfound":0,"num_objects_dirty":4053,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5201,"num_write_kb":296128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1532,"num_bytes_recovered":100401152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1939},{"shards":"1,2,5","objects":629},{"shards":"2,4","objects":1485}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":578,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087118+0000","last_change":"2023-09-23T18:02:51.047609+0000","last_active":"2023-09-23T18:02:55.087118+0000","last_peered":"2023-09-23T18:02:55.087118+0000","last_clean":"2023-09-23T18:02:55.087118+0000","last_became_active":"2023-09-23T18:02:51.047209+0000","last_became_peered":"2023-09-23T18:02:51.047209+0000","last_unstale":"2023-09-23T18:02:55.087118+0000","last_undegraded":"2023-09-23T18:02:55.087118+0000","last_fullsized":"2023-09-23T18:02:55.087118+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":142,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086269+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:02:55.086269+0000","last_peered":"2023-09-23T18:02:55.086269+0000","last_clean":"2023-09-23T18:02:55.086269+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:02:55.086269+0000","last_undegraded":"2023-09-23T18:02:55.086269+0000","last_fullsized":"2023-09-23T18:02:55.086269+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":263,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097978+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:02:55.097978+0000","last_peered":"2023-09-23T18:02:55.097978+0000","last_clean":"2023-09-23T18:02:55.097978+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:02:55.097978+0000","last_undegraded":"2023-09-23T18:02:55.097978+0000","last_fullsized":"2023-09-23T18:02:55.097978+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"255'5106","reported_seq":5322,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:59.218092+0000","last_change":"2023-09-23T18:02:47.961575+0000","last_active":"2023-09-23T18:02:59.218092+0000","last_peered":"2023-09-23T18:02:59.218092+0000","last_clean":"2023-09-23T18:02:17.013863+0000","last_became_active":"2023-09-23T18:02:19.267215+0000","last_became_peered":"2023-09-23T18:02:19.267215+0000","last_unstale":"2023-09-23T18:02:59.218092+0000","last_undegraded":"2023-09-23T18:02:18.955030+0000","last_fullsized":"2023-09-23T18:02:18.924656+0000","mapping_epoch":238,"log_start":"217'400","ondisk_log_start":"217'400","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4706,"log_dups_size":0,"ondisk_log_size":4706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:46.393021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258867200,"num_objects":3950,"num_object_clones":0,"num_object_copies":11850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5106,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":4128768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-158},{"shards":"2,4,6","objects":3592}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":582,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086833+0000","last_change":"2023-09-23T18:02:51.049464+0000","last_active":"2023-09-23T18:02:55.086833+0000","last_peered":"2023-09-23T18:02:55.086833+0000","last_clean":"2023-09-23T18:02:55.086833+0000","last_became_active":"2023-09-23T18:02:51.048477+0000","last_became_peered":"2023-09-23T18:02:51.048477+0000","last_unstale":"2023-09-23T18:02:55.086833+0000","last_undegraded":"2023-09-23T18:02:55.086833+0000","last_fullsized":"2023-09-23T18:02:55.086833+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.072133+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:02:55.072133+0000","last_peered":"2023-09-23T18:02:55.072133+0000","last_clean":"2023-09-23T18:02:55.072133+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:02:55.072133+0000","last_undegraded":"2023-09-23T18:02:55.072133+0000","last_fullsized":"2023-09-23T18:02:55.072133+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":55,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.098179+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:02:55.098179+0000","last_peered":"2023-09-23T18:02:55.098179+0000","last_clean":"2023-09-23T18:02:55.098179+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:02:55.098179+0000","last_undegraded":"2023-09-23T18:02:55.098179+0000","last_fullsized":"2023-09-23T18:02:55.098179+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"255'5111","reported_seq":5174,"reported_epoch":255,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:02:57.548585+0000","last_change":"2023-09-23T18:02:19.054827+0000","last_active":"2023-09-23T18:02:57.548585+0000","last_peered":"2023-09-23T18:02:57.548585+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.048672+0000","last_became_peered":"2023-09-23T18:02:19.048672+0000","last_unstale":"2023-09-23T18:02:57.548585+0000","last_undegraded":"2023-09-23T18:02:18.951263+0000","last_fullsized":"2023-09-23T18:02:18.930851+0000","mapping_epoch":238,"log_start":"216'261","ondisk_log_start":"216'261","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4850,"log_dups_size":0,"ondisk_log_size":4850,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:23.690233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260898816,"num_objects":3981,"num_object_clones":0,"num_object_copies":11943,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3981,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3981,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5111,"num_write_kb":290944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":3981}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087643+0000","last_change":"2023-09-23T18:02:51.060858+0000","last_active":"2023-09-23T18:02:55.087643+0000","last_peered":"2023-09-23T18:02:55.087643+0000","last_clean":"2023-09-23T18:02:55.087643+0000","last_became_active":"2023-09-23T18:02:51.060554+0000","last_became_peered":"2023-09-23T18:02:51.060554+0000","last_unstale":"2023-09-23T18:02:55.087643+0000","last_undegraded":"2023-09-23T18:02:55.087643+0000","last_fullsized":"2023-09-23T18:02:55.087643+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":45,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097552+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:02:55.097552+0000","last_peered":"2023-09-23T18:02:55.097552+0000","last_clean":"2023-09-23T18:02:55.097552+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:02:55.097552+0000","last_undegraded":"2023-09-23T18:02:55.097552+0000","last_fullsized":"2023-09-23T18:02:55.097552+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":265,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.144859+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:02:55.144859+0000","last_peered":"2023-09-23T18:02:55.144859+0000","last_clean":"2023-09-23T18:02:55.144859+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:02:55.144859+0000","last_undegraded":"2023-09-23T18:02:55.144859+0000","last_fullsized":"2023-09-23T18:02:55.144859+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"255'5062","reported_seq":13174,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:02:57.520614+0000","last_change":"2023-09-23T18:02:47.017672+0000","last_active":"2023-09-23T18:02:57.520614+0000","last_peered":"2023-09-23T18:02:57.520614+0000","last_clean":"2023-09-23T18:02:57.520614+0000","last_became_active":"2023-09-23T18:02:47.009245+0000","last_became_peered":"2023-09-23T18:02:47.009245+0000","last_unstale":"2023-09-23T18:02:57.520614+0000","last_undegraded":"2023-09-23T18:02:57.520614+0000","last_fullsized":"2023-09-23T18:02:57.520614+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5062,"log_dups_size":0,"ondisk_log_size":5062,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:04.868743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263651352,"num_objects":4024,"num_object_clones":0,"num_object_copies":12072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4024,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5062,"num_write_kb":290689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4015,"num_bytes_recovered":263127040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":578,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087219+0000","last_change":"2023-09-23T18:02:51.054197+0000","last_active":"2023-09-23T18:02:55.087219+0000","last_peered":"2023-09-23T18:02:55.087219+0000","last_clean":"2023-09-23T18:02:55.087219+0000","last_became_active":"2023-09-23T18:02:51.049362+0000","last_became_peered":"2023-09-23T18:02:51.049362+0000","last_unstale":"2023-09-23T18:02:55.087219+0000","last_undegraded":"2023-09-23T18:02:55.087219+0000","last_fullsized":"2023-09-23T18:02:55.087219+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":100,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086092+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:02:55.086092+0000","last_peered":"2023-09-23T18:02:55.086092+0000","last_clean":"2023-09-23T18:02:55.086092+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:02:55.086092+0000","last_undegraded":"2023-09-23T18:02:55.086092+0000","last_fullsized":"2023-09-23T18:02:55.086092+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":100,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086970+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:02:55.086970+0000","last_peered":"2023-09-23T18:02:55.086970+0000","last_clean":"2023-09-23T18:02:55.086970+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:02:55.086970+0000","last_undegraded":"2023-09-23T18:02:55.086970+0000","last_fullsized":"2023-09-23T18:02:55.086970+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"255'5010","reported_seq":5074,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:59.546287+0000","last_change":"2023-09-23T18:02:19.315376+0000","last_active":"2023-09-23T18:02:59.546287+0000","last_peered":"2023-09-23T18:02:59.546287+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.306174+0000","last_became_peered":"2023-09-23T18:02:19.306174+0000","last_unstale":"2023-09-23T18:02:59.546287+0000","last_undegraded":"2023-09-23T18:02:18.954031+0000","last_fullsized":"2023-09-23T18:02:18.922505+0000","mapping_epoch":238,"log_start":"217'600","ondisk_log_start":"217'600","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4410,"log_dups_size":0,"ondisk_log_size":4410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:52.392034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253755392,"num_objects":3872,"num_object_clones":0,"num_object_copies":11616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5010,"num_write_kb":284224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":-223}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":587,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087103+0000","last_change":"2023-09-23T18:02:51.059934+0000","last_active":"2023-09-23T18:02:55.087103+0000","last_peered":"2023-09-23T18:02:55.087103+0000","last_clean":"2023-09-23T18:02:55.087103+0000","last_became_active":"2023-09-23T18:02:51.059638+0000","last_became_peered":"2023-09-23T18:02:51.059638+0000","last_unstale":"2023-09-23T18:02:55.087103+0000","last_undegraded":"2023-09-23T18:02:55.087103+0000","last_fullsized":"2023-09-23T18:02:55.087103+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":564,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092935+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:02:55.092935+0000","last_peered":"2023-09-23T18:02:55.092935+0000","last_clean":"2023-09-23T18:02:55.092935+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:02:55.092935+0000","last_undegraded":"2023-09-23T18:02:55.092935+0000","last_fullsized":"2023-09-23T18:02:55.092935+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":235,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083486+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:02:55.083486+0000","last_peered":"2023-09-23T18:02:55.083486+0000","last_clean":"2023-09-23T18:02:55.083486+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:02:55.083486+0000","last_undegraded":"2023-09-23T18:02:55.083486+0000","last_fullsized":"2023-09-23T18:02:55.083486+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":232,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083985+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:02:55.083985+0000","last_peered":"2023-09-23T18:02:55.083985+0000","last_clean":"2023-09-23T18:02:55.083985+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:02:55.083985+0000","last_undegraded":"2023-09-23T18:02:55.083985+0000","last_fullsized":"2023-09-23T18:02:55.083985+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"255'5031","reported_seq":5110,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:57.614212+0000","last_change":"2023-09-23T18:02:19.289610+0000","last_active":"2023-09-23T18:02:57.614212+0000","last_peered":"2023-09-23T18:02:57.614212+0000","last_clean":"2023-09-23T18:02:17.060883+0000","last_became_active":"2023-09-23T18:02:19.266879+0000","last_became_peered":"2023-09-23T18:02:19.266879+0000","last_unstale":"2023-09-23T18:02:57.614212+0000","last_undegraded":"2023-09-23T18:02:18.963390+0000","last_fullsized":"2023-09-23T18:02:18.926482+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5031,"log_dups_size":0,"ondisk_log_size":5031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.499269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257359872,"num_objects":3927,"num_object_clones":0,"num_object_copies":11781,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3927,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5031,"num_write_kb":286656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":3576},{"shards":"4,7","objects":-201}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":589,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087038+0000","last_change":"2023-09-23T18:02:51.047767+0000","last_active":"2023-09-23T18:02:55.087038+0000","last_peered":"2023-09-23T18:02:55.087038+0000","last_clean":"2023-09-23T18:02:55.087038+0000","last_became_active":"2023-09-23T18:02:51.047297+0000","last_became_peered":"2023-09-23T18:02:51.047297+0000","last_unstale":"2023-09-23T18:02:55.087038+0000","last_undegraded":"2023-09-23T18:02:55.087038+0000","last_fullsized":"2023-09-23T18:02:55.087038+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.851694+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1183,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.084031+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:02:55.084031+0000","last_peered":"2023-09-23T18:02:55.084031+0000","last_clean":"2023-09-23T18:02:55.084031+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:02:55.084031+0000","last_undegraded":"2023-09-23T18:02:55.084031+0000","last_fullsized":"2023-09-23T18:02:55.084031+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":45,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097798+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:02:55.097798+0000","last_peered":"2023-09-23T18:02:55.097798+0000","last_clean":"2023-09-23T18:02:55.097798+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:02:55.097798+0000","last_undegraded":"2023-09-23T18:02:55.097798+0000","last_fullsized":"2023-09-23T18:02:55.097798+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":148,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088461+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:02:55.088461+0000","last_peered":"2023-09-23T18:02:55.088461+0000","last_clean":"2023-09-23T18:02:55.088461+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:02:55.088461+0000","last_undegraded":"2023-09-23T18:02:55.088461+0000","last_fullsized":"2023-09-23T18:02:55.088461+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"255'5185","reported_seq":5241,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:57.847100+0000","last_change":"2023-09-23T18:02:19.260251+0000","last_active":"2023-09-23T18:02:57.847100+0000","last_peered":"2023-09-23T18:02:57.847100+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.253817+0000","last_became_peered":"2023-09-23T18:02:19.253817+0000","last_unstale":"2023-09-23T18:02:57.847100+0000","last_undegraded":"2023-09-23T18:02:18.955613+0000","last_fullsized":"2023-09-23T18:02:18.923894+0000","mapping_epoch":238,"log_start":"217'1174","ondisk_log_start":"217'1174","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4011,"log_dups_size":0,"ondisk_log_size":4011,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:18.999679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267321344,"num_objects":4079,"num_object_clones":0,"num_object_copies":12237,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4277,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4079,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5185,"num_write_kb":296448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":-198}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":590,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087142+0000","last_change":"2023-09-23T18:02:51.054622+0000","last_active":"2023-09-23T18:02:55.087142+0000","last_peered":"2023-09-23T18:02:55.087142+0000","last_clean":"2023-09-23T18:02:55.087142+0000","last_became_active":"2023-09-23T18:02:51.049345+0000","last_became_peered":"2023-09-23T18:02:51.049345+0000","last_unstale":"2023-09-23T18:02:55.087142+0000","last_undegraded":"2023-09-23T18:02:55.087142+0000","last_fullsized":"2023-09-23T18:02:55.087142+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:58.636010+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":537,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088027+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:02:55.088027+0000","last_peered":"2023-09-23T18:02:55.088027+0000","last_clean":"2023-09-23T18:02:55.088027+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:02:55.088027+0000","last_undegraded":"2023-09-23T18:02:55.088027+0000","last_fullsized":"2023-09-23T18:02:55.088027+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":149,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.095347+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:02:55.095347+0000","last_peered":"2023-09-23T18:02:55.095347+0000","last_clean":"2023-09-23T18:02:55.095347+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:02:55.095347+0000","last_undegraded":"2023-09-23T18:02:55.095347+0000","last_fullsized":"2023-09-23T18:02:55.095347+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":222,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.084446+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:02:55.084446+0000","last_peered":"2023-09-23T18:02:55.084446+0000","last_clean":"2023-09-23T18:02:55.084446+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:02:55.084446+0000","last_undegraded":"2023-09-23T18:02:55.084446+0000","last_fullsized":"2023-09-23T18:02:55.084446+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"255'5136","reported_seq":5229,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:57.521034+0000","last_change":"2023-09-23T18:02:47.986654+0000","last_active":"2023-09-23T18:02:57.521034+0000","last_peered":"2023-09-23T18:02:57.521034+0000","last_clean":"2023-09-23T18:02:16.962656+0000","last_became_active":"2023-09-23T18:02:19.272896+0000","last_became_peered":"2023-09-23T18:02:19.272896+0000","last_unstale":"2023-09-23T18:02:57.521034+0000","last_undegraded":"2023-09-23T18:02:18.972349+0000","last_fullsized":"2023-09-23T18:02:18.930868+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5136,"log_dups_size":0,"ondisk_log_size":5136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:23.745675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268959744,"num_objects":4104,"num_object_clones":0,"num_object_copies":12312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5136,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":3748},{"shards":"4,5","objects":-159}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1167,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083793+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:02:55.083793+0000","last_peered":"2023-09-23T18:02:55.083793+0000","last_clean":"2023-09-23T18:02:55.083793+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:02:55.083793+0000","last_undegraded":"2023-09-23T18:02:55.083793+0000","last_fullsized":"2023-09-23T18:02:55.083793+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":578,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086710+0000","last_change":"2023-09-23T18:02:51.054022+0000","last_active":"2023-09-23T18:02:55.086710+0000","last_peered":"2023-09-23T18:02:55.086710+0000","last_clean":"2023-09-23T18:02:55.086710+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:02:55.086710+0000","last_undegraded":"2023-09-23T18:02:55.086710+0000","last_fullsized":"2023-09-23T18:02:55.086710+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086828+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:02:55.086828+0000","last_peered":"2023-09-23T18:02:55.086828+0000","last_clean":"2023-09-23T18:02:55.086828+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:02:55.086828+0000","last_undegraded":"2023-09-23T18:02:55.086828+0000","last_fullsized":"2023-09-23T18:02:55.086828+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":188,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.071432+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:02:55.071432+0000","last_peered":"2023-09-23T18:02:55.071432+0000","last_clean":"2023-09-23T18:02:55.071432+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:02:55.071432+0000","last_undegraded":"2023-09-23T18:02:55.071432+0000","last_fullsized":"2023-09-23T18:02:55.071432+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"255'5056","reported_seq":5209,"reported_epoch":255,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:02:57.527942+0000","last_change":"2023-09-23T18:02:18.003772+0000","last_active":"2023-09-23T18:02:57.527942+0000","last_peered":"2023-09-23T18:02:57.527942+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:18.000696+0000","last_became_peered":"2023-09-23T18:02:18.000696+0000","last_unstale":"2023-09-23T18:02:57.527942+0000","last_undegraded":"2023-09-23T18:02:17.938873+0000","last_fullsized":"2023-09-23T18:02:17.919669+0000","mapping_epoch":237,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4956,"log_dups_size":0,"ondisk_log_size":4956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:48.085957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263847936,"num_objects":4026,"num_object_clones":0,"num_object_copies":12078,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3990,"num_objects_misplaced":4026,"num_objects_unfound":0,"num_objects_dirty":4026,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5056,"num_write_kb":290624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":4026}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":590,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087563+0000","last_change":"2023-09-23T18:02:51.061168+0000","last_active":"2023-09-23T18:02:55.087563+0000","last_peered":"2023-09-23T18:02:55.087563+0000","last_clean":"2023-09-23T18:02:55.087563+0000","last_became_active":"2023-09-23T18:02:51.060689+0000","last_became_peered":"2023-09-23T18:02:51.060689+0000","last_unstale":"2023-09-23T18:02:55.087563+0000","last_undegraded":"2023-09-23T18:02:55.087563+0000","last_fullsized":"2023-09-23T18:02:55.087563+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:29.406460+0000","scrub_duration":0.001568505,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":435,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086945+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:02:55.086945+0000","last_peered":"2023-09-23T18:02:55.086945+0000","last_clean":"2023-09-23T18:02:55.086945+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:02:55.086945+0000","last_undegraded":"2023-09-23T18:02:55.086945+0000","last_fullsized":"2023-09-23T18:02:55.086945+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":253,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092277+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:02:55.092277+0000","last_peered":"2023-09-23T18:02:55.092277+0000","last_clean":"2023-09-23T18:02:55.092277+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:02:55.092277+0000","last_undegraded":"2023-09-23T18:02:55.092277+0000","last_fullsized":"2023-09-23T18:02:55.092277+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":224,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.093419+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:02:55.093419+0000","last_peered":"2023-09-23T18:02:55.093419+0000","last_clean":"2023-09-23T18:02:55.093419+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:02:55.093419+0000","last_undegraded":"2023-09-23T18:02:55.093419+0000","last_fullsized":"2023-09-23T18:02:55.093419+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"255'5237","reported_seq":13573,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:03:00.486980+0000","last_change":"2023-09-23T18:02:18.942886+0000","last_active":"2023-09-23T18:03:00.486980+0000","last_peered":"2023-09-23T18:03:00.486980+0000","last_clean":"2023-09-23T18:03:00.486980+0000","last_became_active":"2023-09-23T18:02:18.936691+0000","last_became_peered":"2023-09-23T18:02:18.936691+0000","last_unstale":"2023-09-23T18:03:00.486980+0000","last_undegraded":"2023-09-23T18:03:00.486980+0000","last_fullsized":"2023-09-23T18:03:00.486980+0000","mapping_epoch":238,"log_start":"217'1058","ondisk_log_start":"217'1058","created":204,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4179,"log_dups_size":0,"ondisk_log_size":4179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:26.085859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262078464,"num_objects":3999,"num_object_clones":0,"num_object_copies":11997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5237,"num_write_kb":295552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4123,"num_bytes_recovered":270204928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":587,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088245+0000","last_change":"2023-09-23T18:02:51.057370+0000","last_active":"2023-09-23T18:02:55.088245+0000","last_peered":"2023-09-23T18:02:55.088245+0000","last_clean":"2023-09-23T18:02:55.088245+0000","last_became_active":"2023-09-23T18:02:51.056158+0000","last_became_peered":"2023-09-23T18:02:51.056158+0000","last_unstale":"2023-09-23T18:02:55.088245+0000","last_undegraded":"2023-09-23T18:02:55.088245+0000","last_fullsized":"2023-09-23T18:02:55.088245+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":469,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088698+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:02:55.088698+0000","last_peered":"2023-09-23T18:02:55.088698+0000","last_clean":"2023-09-23T18:02:55.088698+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:02:55.088698+0000","last_undegraded":"2023-09-23T18:02:55.088698+0000","last_fullsized":"2023-09-23T18:02:55.088698+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":189,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088514+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:02:55.088514+0000","last_peered":"2023-09-23T18:02:55.088514+0000","last_clean":"2023-09-23T18:02:55.088514+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:02:55.088514+0000","last_undegraded":"2023-09-23T18:02:55.088514+0000","last_fullsized":"2023-09-23T18:02:55.088514+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":221,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092617+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:02:55.092617+0000","last_peered":"2023-09-23T18:02:55.092617+0000","last_clean":"2023-09-23T18:02:55.092617+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:02:55.092617+0000","last_undegraded":"2023-09-23T18:02:55.092617+0000","last_fullsized":"2023-09-23T18:02:55.092617+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"255'5094","reported_seq":5178,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:57.652446+0000","last_change":"2023-09-23T18:02:19.279112+0000","last_active":"2023-09-23T18:02:57.652446+0000","last_peered":"2023-09-23T18:02:57.652446+0000","last_clean":"2023-09-23T18:02:17.018900+0000","last_became_active":"2023-09-23T18:02:19.267439+0000","last_became_peered":"2023-09-23T18:02:19.267439+0000","last_unstale":"2023-09-23T18:02:57.652446+0000","last_undegraded":"2023-09-23T18:02:18.959329+0000","last_fullsized":"2023-09-23T18:02:18.926801+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5094,"log_dups_size":0,"ondisk_log_size":5094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.395583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263454720,"num_objects":4020,"num_object_clones":0,"num_object_copies":12060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5094,"num_write_kb":291648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-179},{"shards":"2,4,6","objects":3662}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":587,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087817+0000","last_change":"2023-09-23T18:02:51.058680+0000","last_active":"2023-09-23T18:02:55.087817+0000","last_peered":"2023-09-23T18:02:55.087817+0000","last_clean":"2023-09-23T18:02:55.087817+0000","last_became_active":"2023-09-23T18:02:51.057706+0000","last_became_peered":"2023-09-23T18:02:51.057706+0000","last_unstale":"2023-09-23T18:02:55.087817+0000","last_undegraded":"2023-09-23T18:02:55.087817+0000","last_fullsized":"2023-09-23T18:02:55.087817+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":100,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086901+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:02:55.086901+0000","last_peered":"2023-09-23T18:02:55.086901+0000","last_clean":"2023-09-23T18:02:55.086901+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:02:55.086901+0000","last_undegraded":"2023-09-23T18:02:55.086901+0000","last_fullsized":"2023-09-23T18:02:55.086901+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":325,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.071348+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:02:55.071348+0000","last_peered":"2023-09-23T18:02:55.071348+0000","last_clean":"2023-09-23T18:02:55.071348+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:02:55.071348+0000","last_undegraded":"2023-09-23T18:02:55.071348+0000","last_fullsized":"2023-09-23T18:02:55.071348+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":104,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086059+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:02:55.086059+0000","last_peered":"2023-09-23T18:02:55.086059+0000","last_clean":"2023-09-23T18:02:55.086059+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:02:55.086059+0000","last_undegraded":"2023-09-23T18:02:55.086059+0000","last_fullsized":"2023-09-23T18:02:55.086059+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"255'5120","reported_seq":5181,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:02:57.957315+0000","last_change":"2023-09-23T18:01:25.046224+0000","last_active":"2023-09-23T18:02:57.957315+0000","last_peered":"2023-09-23T18:02:57.957315+0000","last_clean":"2023-09-23T18:02:57.957315+0000","last_became_active":"2023-09-23T18:01:25.002864+0000","last_became_peered":"2023-09-23T18:01:25.002864+0000","last_unstale":"2023-09-23T18:02:57.957315+0000","last_undegraded":"2023-09-23T18:02:57.957315+0000","last_fullsized":"2023-09-23T18:02:57.957315+0000","mapping_epoch":214,"log_start":"217'1141","ondisk_log_start":"217'1141","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":3979,"log_dups_size":0,"ondisk_log_size":3979,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.752359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":263323648,"num_objects":4018,"num_object_clones":0,"num_object_copies":12054,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4018,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5120,"num_write_kb":292416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":594,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087547+0000","last_change":"2023-09-23T18:02:51.053420+0000","last_active":"2023-09-23T18:02:55.087547+0000","last_peered":"2023-09-23T18:02:55.087547+0000","last_clean":"2023-09-23T18:02:55.087547+0000","last_became_active":"2023-09-23T18:02:51.053035+0000","last_became_peered":"2023-09-23T18:02:51.053035+0000","last_unstale":"2023-09-23T18:02:55.087547+0000","last_undegraded":"2023-09-23T18:02:55.087547+0000","last_fullsized":"2023-09-23T18:02:55.087547+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:20.456481+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":460,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.071843+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:02:55.071843+0000","last_peered":"2023-09-23T18:02:55.071843+0000","last_clean":"2023-09-23T18:02:55.071843+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:02:55.071843+0000","last_undegraded":"2023-09-23T18:02:55.071843+0000","last_fullsized":"2023-09-23T18:02:55.071843+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":281,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.093462+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:02:55.093462+0000","last_peered":"2023-09-23T18:02:55.093462+0000","last_clean":"2023-09-23T18:02:55.093462+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:02:55.093462+0000","last_undegraded":"2023-09-23T18:02:55.093462+0000","last_fullsized":"2023-09-23T18:02:55.093462+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097465+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:02:55.097465+0000","last_peered":"2023-09-23T18:02:55.097465+0000","last_clean":"2023-09-23T18:02:55.097465+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:02:55.097465+0000","last_undegraded":"2023-09-23T18:02:55.097465+0000","last_fullsized":"2023-09-23T18:02:55.097465+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":577,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087273+0000","last_change":"2023-09-23T18:02:51.046571+0000","last_active":"2023-09-23T18:02:55.087273+0000","last_peered":"2023-09-23T18:02:55.087273+0000","last_clean":"2023-09-23T18:02:55.087273+0000","last_became_active":"2023-09-23T18:02:51.046213+0000","last_became_peered":"2023-09-23T18:02:51.046213+0000","last_unstale":"2023-09-23T18:02:55.087273+0000","last_undegraded":"2023-09-23T18:02:55.087273+0000","last_fullsized":"2023-09-23T18:02:55.087273+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":240,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092542+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:02:55.092542+0000","last_peered":"2023-09-23T18:02:55.092542+0000","last_clean":"2023-09-23T18:02:55.092542+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:02:55.092542+0000","last_undegraded":"2023-09-23T18:02:55.092542+0000","last_fullsized":"2023-09-23T18:02:55.092542+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":267,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088559+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:02:55.088559+0000","last_peered":"2023-09-23T18:02:55.088559+0000","last_clean":"2023-09-23T18:02:55.088559+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:02:55.088559+0000","last_undegraded":"2023-09-23T18:02:55.088559+0000","last_fullsized":"2023-09-23T18:02:55.088559+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088083+0000","last_change":"2023-09-23T18:02:51.060198+0000","last_active":"2023-09-23T18:02:55.088083+0000","last_peered":"2023-09-23T18:02:55.088083+0000","last_clean":"2023-09-23T18:02:55.088083+0000","last_became_active":"2023-09-23T18:02:51.057046+0000","last_became_peered":"2023-09-23T18:02:51.057046+0000","last_unstale":"2023-09-23T18:02:55.088083+0000","last_undegraded":"2023-09-23T18:02:55.088083+0000","last_fullsized":"2023-09-23T18:02:55.088083+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":110,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.084305+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:02:55.084305+0000","last_peered":"2023-09-23T18:02:55.084305+0000","last_clean":"2023-09-23T18:02:55.084305+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:02:55.084305+0000","last_undegraded":"2023-09-23T18:02:55.084305+0000","last_fullsized":"2023-09-23T18:02:55.084305+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":180,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097890+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:02:55.097890+0000","last_peered":"2023-09-23T18:02:55.097890+0000","last_clean":"2023-09-23T18:02:55.097890+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:02:55.097890+0000","last_undegraded":"2023-09-23T18:02:55.097890+0000","last_fullsized":"2023-09-23T18:02:55.097890+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":96,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083286+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:02:55.083286+0000","last_peered":"2023-09-23T18:02:55.083286+0000","last_clean":"2023-09-23T18:02:55.083286+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:02:55.083286+0000","last_undegraded":"2023-09-23T18:02:55.083286+0000","last_fullsized":"2023-09-23T18:02:55.083286+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":119,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086667+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:02:55.086667+0000","last_peered":"2023-09-23T18:02:55.086667+0000","last_clean":"2023-09-23T18:02:55.086667+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:02:55.086667+0000","last_undegraded":"2023-09-23T18:02:55.086667+0000","last_fullsized":"2023-09-23T18:02:55.086667+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":602,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087712+0000","last_change":"2023-09-23T18:02:51.047476+0000","last_active":"2023-09-23T18:02:55.087712+0000","last_peered":"2023-09-23T18:02:55.087712+0000","last_clean":"2023-09-23T18:02:55.087712+0000","last_became_active":"2023-09-23T18:02:51.047152+0000","last_became_peered":"2023-09-23T18:02:51.047152+0000","last_unstale":"2023-09-23T18:02:55.087712+0000","last_undegraded":"2023-09-23T18:02:55.087712+0000","last_fullsized":"2023-09-23T18:02:55.087712+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:22.514077+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":119,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086344+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:02:55.086344+0000","last_peered":"2023-09-23T18:02:55.086344+0000","last_clean":"2023-09-23T18:02:55.086344+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:02:55.086344+0000","last_undegraded":"2023-09-23T18:02:55.086344+0000","last_fullsized":"2023-09-23T18:02:55.086344+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":122,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087868+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:02:55.087868+0000","last_peered":"2023-09-23T18:02:55.087868+0000","last_clean":"2023-09-23T18:02:55.087868+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:02:55.087868+0000","last_undegraded":"2023-09-23T18:02:55.087868+0000","last_fullsized":"2023-09-23T18:02:55.087868+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":585,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087727+0000","last_change":"2023-09-23T18:02:51.058825+0000","last_active":"2023-09-23T18:02:55.087727+0000","last_peered":"2023-09-23T18:02:55.087727+0000","last_clean":"2023-09-23T18:02:55.087727+0000","last_became_active":"2023-09-23T18:02:51.058480+0000","last_became_peered":"2023-09-23T18:02:51.058480+0000","last_unstale":"2023-09-23T18:02:55.087727+0000","last_undegraded":"2023-09-23T18:02:55.087727+0000","last_fullsized":"2023-09-23T18:02:55.087727+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":233,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092888+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:02:55.092888+0000","last_peered":"2023-09-23T18:02:55.092888+0000","last_clean":"2023-09-23T18:02:55.092888+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:02:55.092888+0000","last_undegraded":"2023-09-23T18:02:55.092888+0000","last_fullsized":"2023-09-23T18:02:55.092888+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":265,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097491+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:02:55.097491+0000","last_peered":"2023-09-23T18:02:55.097491+0000","last_clean":"2023-09-23T18:02:55.097491+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:02:55.097491+0000","last_undegraded":"2023-09-23T18:02:55.097491+0000","last_fullsized":"2023-09-23T18:02:55.097491+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4202758168,"num_objects":64130,"num_object_clones":0,"num_object_copies":192390,"num_objects_missing_on_primary":629,"num_objects_missing":629,"num_objects_degraded":36944,"num_objects_misplaced":14097,"num_objects_unfound":0,"num_objects_dirty":64130,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":82142,"num_write_kb":4680641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14129,"num_bytes_recovered":925958144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12044677120,"data_stored":12044664904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74978,"ondisk_log_size":74978,"up":48,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":402523,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":331,"num_read_kb":623,"num_write":162,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2224465,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":91,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001364,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2515700,"kb_used_data":2422316,"kb_used_omap":13,"kb_used_meta":93362,"kb_avail":91229452,"statfs":{"total":95995035648,"available":93418958848,"internally_reserved":0,"allocated":2480451584,"data_stored":2479471589,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":95602792},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.544,"5min":2.544,"15min":2.544},"last":0.316},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":0.52500000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.629,"5min":2.629,"15min":2.629},"last":0.434},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.5979999999999999,"5min":4.5979999999999999,"15min":4.5979999999999999},"last":0.371}]},{"osd":2,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.081,"5min":3.081,"15min":3.081},"last":0.68500000000000005},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.6680000000000001,"5min":2.6680000000000001,"15min":2.6680000000000001},"last":0.46999999999999997}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.78700000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":3.984,"5min":3.984,"15min":3.984},"last":0.495},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.645,"5min":1.645,"15min":1.645},"last":0.57299999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.004,"15min":1.004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.9300000000000002,"5min":3.9300000000000002,"15min":3.9300000000000002},"last":0.77300000000000002},{"interface":"front","average":{"1min":1.133,"5min":1.133,"15min":1.133},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.7080000000000002,"5min":3.7080000000000002,"15min":3.7080000000000002},"last":3.403}]},{"osd":6,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.0409999999999999,"5min":3.0409999999999999,"15min":3.0409999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.6419999999999999,"5min":3.6419999999999999,"15min":3.6419999999999999},"last":0.747}]}]},{"osd":6,"up_from":163,"seq":700079669301,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2478124,"kb_used_data":2333796,"kb_used_omap":7,"kb_used_meta":144312,"kb_avail":91267028,"statfs":{"total":95995035648,"available":93457436672,"internally_reserved":0,"allocated":2389807104,"data_stored":2388835694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":147776292},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.2270000000000001,"15min":1.3089999999999999},"min":{"1min":0.224,"5min":0.215,"15min":0.215},"max":{"1min":3.5529999999999999,"5min":14.196999999999999,"15min":14.196999999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.72,"15min":2.3740000000000001},"min":{"1min":0.32600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.8970000000000002,"5min":13.536,"15min":13.536},"last":1.827}]},{"osd":1,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.5860000000000001,"15min":1.956},"min":{"1min":0.28100000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.4910000000000001,"5min":14.233000000000001,"15min":14.233000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.01,"5min":1.6759999999999999,"15min":2.173},"min":{"1min":0.33300000000000002,"5min":0.248,"15min":0.248},"max":{"1min":4.9340000000000002,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":1.6890000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":2.0009999999999999,"15min":2.3450000000000002},"min":{"1min":0.34999999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.173,"5min":11.210000000000001,"15min":11.210000000000001},"last":5.7489999999999997},{"interface":"front","average":{"1min":1.0589999999999999,"5min":2.1899999999999999,"15min":2.758},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.9700000000000002,"5min":11.223000000000001,"15min":11.223000000000001},"last":5.7359999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.96599999999999997,"15min":0.89400000000000002},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.827,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.72499999999999998,"15min":0.503},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.0129999999999999,"5min":8.3000000000000007,"15min":8.3000000000000007},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.94199999999999995,"15min":0.84399999999999997},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.1050000000000004,"5min":11.869,"15min":11.869},"last":0.496},{"interface":"front","average":{"1min":1.21,"5min":0.875,"15min":0.63500000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.7130000000000001,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":0.48999999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.6469999999999998,"5min":3.6469999999999998,"15min":3.6469999999999998},"last":2.1150000000000002},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.5310000000000001,"5min":2.5310000000000001,"15min":2.5310000000000001},"last":1.093}]}]},{"osd":1,"up_from":0,"seq":420906795119,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958646,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1432268,"kb_used_data":1026436,"kb_used_omap":7,"kb_used_meta":405816,"kb_avail":92312884,"statfs":{"total":95995035648,"available":94528393216,"internally_reserved":0,"allocated":1051070464,"data_stored":1049781918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":415556324},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":2.238,"15min":1.0920000000000001},"min":{"1min":0.21199999999999999,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":3.786,"5min":18.154,"15min":18.154},"last":2.145},{"interface":"front","average":{"1min":0.51200000000000001,"5min":2.1739999999999999,"15min":1.083},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":1.9359999999999999,"5min":13.369999999999999,"15min":13.369999999999999},"last":0.55200000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.4460000000000002,"15min":1.24},"min":{"1min":0.25700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.5259999999999998,"5min":20.882000000000001,"15min":20.882000000000001},"last":2.1139999999999999},{"interface":"front","average":{"1min":1.234,"5min":2.0899999999999999,"15min":1.149},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":6.7439999999999998,"5min":15.064,"15min":15.064},"last":2.5219999999999998}]},{"osd":3,"last update":"Sat Sep 23 17:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":0.77100000000000002,"15min":0.60099999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3449999999999998,"5min":4.3449999999999998,"15min":4.3449999999999998},"last":3.5529999999999999},{"interface":"front","average":{"1min":2.109,"5min":0.73699999999999999,"15min":0.50900000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":8.8729999999999993},"last":3.4609999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":1.603,"15min":1.0509999999999999},"min":{"1min":0.29299999999999998,"5min":0.251,"15min":0.251},"max":{"1min":4.54,"5min":15.279999999999999,"15min":15.279999999999999},"last":1.6499999999999999},{"interface":"front","average":{"1min":1.24,"5min":1.859,"15min":1.1220000000000001},"min":{"1min":0.29899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7190000000000003,"5min":18.106999999999999,"15min":18.106999999999999},"last":2.2010000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.9710000000000001,"15min":1.1619999999999999},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.1280000000000001,"5min":14.720000000000001,"15min":14.720000000000001},"last":2.1760000000000002},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.5469999999999999,"15min":1.022},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":2.806,"5min":12.794,"15min":12.794},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.8680000000000001,"15min":2.3010000000000002},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.5069999999999997,"5min":14.956,"15min":14.956},"last":1.732},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.0419999999999998,"15min":2.5219999999999998},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.311},"max":{"1min":2.883,"5min":12.755000000000001,"15min":12.755000000000001},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:02:28 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":2.851,"5min":2.851,"15min":2.851},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.083,"5min":1.083,"15min":1.083},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.4619999999999997,"5min":4.4619999999999997,"15min":4.4619999999999997},"last":1.6919999999999999}]}]},{"osd":2,"up_from":103,"seq":442381631593,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2650368,"kb_used_data":2313080,"kb_used_omap":11,"kb_used_meta":337268,"kb_avail":91094784,"statfs":{"total":95995035648,"available":93281058816,"internally_reserved":0,"allocated":2368593920,"data_stored":2367317513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":345362807},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":2.552,"15min":1.2729999999999999},"min":{"1min":0.187,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":3.306,"5min":20.867000000000001,"15min":20.867000000000001},"last":0.247},{"interface":"front","average":{"1min":0.55300000000000005,"5min":2.3119999999999998,"15min":1.246},"min":{"1min":0.24299999999999999,"5min":0.215,"15min":0.215},"max":{"1min":1.6910000000000001,"5min":21.681000000000001,"15min":21.681000000000001},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.6349999999999998,"15min":1.361},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":4.2359999999999998,"5min":20.995000000000001,"15min":20.995000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.75,"5min":1.9530000000000001,"15min":1.147},"min":{"1min":0.24199999999999999,"5min":0.193,"15min":0.193},"max":{"1min":4.532,"5min":15.945,"15min":15.945},"last":0.32700000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":0.58799999999999997,"15min":0.443},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8339999999999996,"5min":4.8339999999999996,"15min":4.8339999999999996},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.59699999999999998,"15min":0.48399999999999999},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.6459999999999999,"5min":5.6459999999999999,"15min":5.6459999999999999},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.7649999999999999,"15min":1.095},"min":{"1min":0.29099999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.141,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":0.502},{"interface":"front","average":{"1min":0.45900000000000002,"5min":1.8899999999999999,"15min":1.1200000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78100000000000003,"5min":20.920000000000002,"15min":20.920000000000002},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.9890000000000001,"15min":1.1879999999999999},"min":{"1min":0.32900000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.972,"5min":20.738,"15min":20.738},"last":0.439},{"interface":"front","average":{"1min":1.048,"5min":1.79,"15min":1.1970000000000001},"min":{"1min":0.34100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3700000000000001,"5min":14.359,"15min":15.496},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:02:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":2.0299999999999998,"15min":2.2240000000000002},"min":{"1min":0.34699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":5.1879999999999997,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.66900000000000004,"5min":2.3290000000000002,"15min":2.4929999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.6930000000000001,"5min":20.893000000000001,"15min":20.893000000000001},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:02:25 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.411,"15min":1.411},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.3239999999999998,"5min":4.3239999999999998,"15min":4.3239999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.7130000000000001,"5min":1.7130000000000001,"15min":1.7130000000000001},"last":0.629}]}]},{"osd":3,"up_from":237,"seq":1017907249161,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1285248,"kb_used_data":1221176,"kb_used_omap":3,"kb_used_meta":64060,"kb_avail":92459904,"statfs":{"total":95995035648,"available":94678941696,"internally_reserved":0,"allocated":1250484224,"data_stored":1249029554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":65598184},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.32000000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.77200000000000002}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66600000000000004}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42699999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40899999999999997}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44400000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45500000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019111,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1682928,"kb_used_data":1482664,"kb_used_omap":0,"kb_used_meta":200256,"kb_avail":92062224,"statfs":{"total":95995035648,"available":94271717376,"internally_reserved":0,"allocated":1518247936,"data_stored":1517272266,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":205062144},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.3260000000000001,"15min":0.82099999999999995},"min":{"1min":0.24399999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.1829999999999998,"5min":14.164,"15min":14.164},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.611,"15min":0.97899999999999998},"min":{"1min":0.253,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.29,"5min":11.372,"15min":11.372},"last":0.372}]},{"osd":1,"last update":"Sat Sep 23 18:02:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.7470000000000001,"15min":0.95599999999999996},"min":{"1min":0.35699999999999998,"5min":0.26900000000000002,"15min":0.249},"max":{"1min":0.92900000000000005,"5min":10.429,"15min":10.429},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.8009999999999999,"15min":1.022},"min":{"1min":0.29099999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.0620000000000003,"5min":10.83,"15min":10.83},"last":0.45700000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.6699999999999999,"15min":1.0629999999999999},"min":{"1min":0.29899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.266,"5min":11.630000000000001,"15min":11.630000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.5289999999999999,"15min":0.98799999999999999},"min":{"1min":0.316,"5min":0.30099999999999999,"15min":0.27100000000000002},"max":{"1min":5.2450000000000001,"5min":7.5090000000000003,"15min":7.5090000000000003},"last":0.40999999999999998}]},{"osd":3,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.72899999999999998,"15min":0.64900000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6130000000000004,"5min":4.6130000000000004,"15min":4.6130000000000004},"last":0.439},{"interface":"front","average":{"1min":1.04,"5min":0.73299999999999998,"15min":0.68200000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":3.637,"5min":3.637,"15min":3.637},"last":0.51500000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.80800000000000005,"15min":0.59299999999999997},"min":{"1min":0.23799999999999999,"5min":0.224,"15min":0.224},"max":{"1min":3.2269999999999999,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0029999999999999,"15min":0.71299999999999997},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0629999999999997,"5min":7.4729999999999999,"15min":10.494},"last":0.47799999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":0.94499999999999995,"15min":0.82799999999999996},"min":{"1min":0.25,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.5579999999999998,"5min":7.1529999999999996,"15min":7.1529999999999996},"last":0.495},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0680000000000001,"15min":0.95599999999999996},"min":{"1min":0.28999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.286,"5min":10.725,"15min":10.725},"last":0.52700000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:02:25 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.8839999999999999,"5min":3.8839999999999999,"15min":3.8839999999999999},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.8090000000000002,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.58399999999999996}]}]},{"osd":5,"up_from":38,"seq":163208757477,"num_pgs":29,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1633636,"kb_used_data":1441692,"kb_used_omap":0,"kb_used_meta":191872,"kb_avail":92111516,"statfs":{"total":95995035648,"available":94322192384,"internally_reserved":0,"allocated":1476292608,"data_stored":1475324690,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":196476928},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.6779999999999999,"15min":1.008},"min":{"1min":0.28699999999999998,"5min":0.217,"15min":0.217},"max":{"1min":4.0250000000000004,"5min":21.376999999999999,"15min":21.376999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.996,"5min":1.8939999999999999,"15min":1.149},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.048,"5min":21.388999999999999,"15min":21.388999999999999},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.1990000000000001,"15min":0.873},"min":{"1min":0.315,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":2.0870000000000002,"5min":10.853999999999999,"15min":10.853999999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.518,"15min":0.92200000000000004},"min":{"1min":0.26200000000000001,"5min":0.254,"15min":0.254},"max":{"1min":2.6230000000000002,"5min":10.206,"15min":10.206},"last":0.45700000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.1320000000000001,"15min":1.2589999999999999},"min":{"1min":0.34200000000000003,"5min":0.255,"15min":0.255},"max":{"1min":4.9279999999999999,"5min":26.035,"15min":38.606999999999999},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.534,"15min":1.123},"min":{"1min":0.35999999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.885,"5min":17.085000000000001,"15min":39.670999999999999},"last":0.41699999999999998}]},{"osd":3,"last update":"Sat Sep 23 17:56:57 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":0.78800000000000003,"15min":0.66400000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.4969999999999999,"5min":7.4969999999999999,"15min":7.4969999999999999},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.0900000000000001,"5min":0.70199999999999996,"15min":0.63800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.5510000000000002,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":0.58999999999999997}]},{"osd":4,"last update":"Sat Sep 23 18:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":0.94499999999999995,"15min":0.69699999999999995},"min":{"1min":0.24199999999999999,"5min":0.22800000000000001,"15min":0.224},"max":{"1min":3.9980000000000002,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.499},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.998,"15min":0.69799999999999995},"min":{"1min":0.30499999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.1449999999999996,"5min":5.077,"15min":5.077},"last":0.40400000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.343,"15min":1.5069999999999999},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.5880000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.441},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.147,"15min":1.329},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.1710000000000003,"5min":8.266,"15min":8.266},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:02:26 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.546,"15min":1.546},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":1.222},{"interface":"front","average":{"1min":1.125,"5min":1.125,"15min":1.125},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.0380000000000003,"5min":4.0380000000000003,"15min":4.0380000000000003},"last":1.1830000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1038749696,"data_stored":1038745624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1532231680,"data_stored":1532231680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2350059520,"data_stored":2350055448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":183697408,"data_stored":183697408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1512243200,"data_stored":1512243200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1463226368,"data_stored":1463222296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2375811072,"data_stored":2375811072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1588658176,"data_stored":1588658176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:03:01.593 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:03:01.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:01.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:01.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:01.703 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:01.772 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:01.909 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:01.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:01.917 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:02.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:02.055 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:02.289 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:02 smithi055 ceph-mon[184174]: pgmap v586: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 36946/192471 objects degraded (19.196%); 14097/192471 objects misplaced (7.324%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T18:03:02.289 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:02 smithi055 ceph-mon[184174]: from='client.35538 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:02.289 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:02 smithi055 ceph-mon[179668]: pgmap v586: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 36946/192471 objects degraded (19.196%); 14097/192471 objects misplaced (7.324%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T18:03:02.289 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:02 smithi055 ceph-mon[179668]: from='client.35538 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:02.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:02.320 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:02 smithi099 ceph-mon[167076]: pgmap v586: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 460 op/s; 36946/192471 objects degraded (19.196%); 14097/192471 objects misplaced (7.324%); 5.5 MiB/s, 88 objects/s recovering 2023-09-23T18:03:02.320 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:02 smithi099 ceph-mon[167076]: from='client.35538 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:02.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:02.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:02.422 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:02.458 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:02.503 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:03:02.509 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:02.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:02.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:02.540 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:03:02.553 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:02.576 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:03:02.609 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:03:02.617 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:02.643 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:03:02.655 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:02.671 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:03:02.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:03:02.709 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:02.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:02.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:02.734 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:03:02.739 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:03:02.739 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:03:02.740 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:03:02.740 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:03:02.740 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:03:02.740 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:03:02.740 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:03:02.740 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:03:02.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:02.836 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:03:02.836 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:03:02.836 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:03:02.836 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:03:02.836 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:03:02.845 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:03.134 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:03:03.152 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:03.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:03.156 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:03.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:03:03.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:03.314 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:03.362 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:03.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:03.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:03.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2284782899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:03.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1717955790' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:03.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:03 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:03.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 36946/192471 objects degraded (19.196%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:03:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2284782899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1717955790' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:03 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 36946/192471 objects degraded (19.196%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:03:03.467 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:03.502 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2284782899' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:03.502 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1717955790' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:03.502 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:03 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:03.502 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 36946/192471 objects degraded (19.196%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:03:03.505 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:03.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:03.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:03.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:03.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:03.689 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:03.732 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:03.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:03.737 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:03.837 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:03:03.870 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:03.910 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:03:03.930 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:03:03.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:03.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:03:04.038 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:04.094 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:04.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[184174]: pgmap v587: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 36651/190203 objects degraded (19.269%); 13955/190203 objects misplaced (7.337%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:03:04.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[179668]: pgmap v587: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 36651/190203 objects degraded (19.269%); 13955/190203 objects misplaced (7.337%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:03:04.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:03:04.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[179668]: failed to encode map e256 with expected crc 2023-09-23T18:03:04.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:03:04.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[179668]: osdmap e256: 8 total, 8 up, 7 in 2023-09-23T18:03:04.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[179668]: failed to encode map e256 with expected crc 2023-09-23T18:03:04.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[179668]: failed to encode map e256 with expected crc 2023-09-23T18:03:04.151 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:04.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:04.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:04.260 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:03:04.295 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:04.344 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:03:04.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:04.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:03:04.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:03:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[184174]: failed to encode map e256 with expected crc 2023-09-23T18:03:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:03:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[184174]: osdmap e256: 8 total, 8 up, 7 in 2023-09-23T18:03:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[184174]: failed to encode map e256 with expected crc 2023-09-23T18:03:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:04 smithi055 ceph-mon[184174]: failed to encode map e256 with expected crc 2023-09-23T18:03:04.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:04.463 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:04 smithi099 ceph-mon[167076]: pgmap v587: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.9 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 417 op/s; 36651/190203 objects degraded (19.269%); 13955/190203 objects misplaced (7.337%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:03:04.463 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:04 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:03:04.463 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:04 smithi099 ceph-mon[167076]: failed to encode map e256 with expected crc 2023-09-23T18:03:04.463 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:04 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:03:04.463 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:04 smithi099 ceph-mon[167076]: osdmap e256: 8 total, 8 up, 7 in 2023-09-23T18:03:04.464 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:04 smithi099 ceph-mon[167076]: failed to encode map e256 with expected crc 2023-09-23T18:03:04.464 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:04 smithi099 ceph-mon[167076]: failed to encode map e256 with expected crc 2023-09-23T18:03:04.490 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:04.556 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:03:04.559 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:03:04.656 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:04.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:04.659 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:04.759 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:04.810 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:04.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:04.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:04.894 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:04.962 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":588,"stamp":"2023-09-23T18:03:03.123349+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4110039558,"num_objects":62672,"num_object_clones":0,"num_object_copies":188016,"num_objects_missing_on_primary":629,"num_objects_missing":629,"num_objects_degraded":36651,"num_objects_misplaced":13955,"num_objects_unfound":0,"num_objects_dirty":62672,"num_whiteouts":0,"num_read":6284,"num_read_kb":12770,"num_write":84696,"num_write_kb":4695012,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14229,"num_bytes_recovered":928188871,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82085,"ondisk_log_size":82085,"up":357,"acting":345,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":404,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":13469000,"kb_used_data":12015056,"kb_used_omap":43,"kb_used_meta":1453844,"kb_avail":642747064,"statfs":{"total":671965249536,"available":658172993536,"internally_reserved":0,"allocated":12303417344,"data_stored":12295502908,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1488736975},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-266862592,"num_objects":-4072,"num_object_clones":0,"num_object_copies":-12216,"num_objects_missing_on_primary":-1,"num_objects_missing":-1,"num_objects_degraded":-929,"num_objects_misplaced":-830,"num_objects_unfound":0,"num_objects_dirty":-4072,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4072,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":339,"num_bytes_recovered":22216704,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.062237"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":582,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:55.091879+0000","last_change":"2023-09-23T18:02:48.964317+0000","last_active":"2023-09-23T18:02:55.091879+0000","last_peered":"2023-09-23T18:02:55.091879+0000","last_clean":"2023-09-23T18:02:45.962259+0000","last_became_active":"2023-09-23T18:02:48.961702+0000","last_became_peered":"2023-09-23T18:02:48.961702+0000","last_unstale":"2023-09-23T18:02:55.091879+0000","last_undegraded":"2023-09-23T18:02:48.941433+0000","last_fullsized":"2023-09-23T18:02:48.940979+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":474537,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":55,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097658+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:02:55.097658+0000","last_peered":"2023-09-23T18:02:55.097658+0000","last_clean":"2023-09-23T18:02:55.097658+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:02:55.097658+0000","last_undegraded":"2023-09-23T18:02:55.097658+0000","last_fullsized":"2023-09-23T18:02:55.097658+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":136,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086508+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:02:55.086508+0000","last_peered":"2023-09-23T18:02:55.086508+0000","last_clean":"2023-09-23T18:02:55.086508+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:02:55.086508+0000","last_undegraded":"2023-09-23T18:02:55.086508+0000","last_fullsized":"2023-09-23T18:02:55.086508+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":597,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087754+0000","last_change":"2023-09-23T18:02:51.046916+0000","last_active":"2023-09-23T18:02:55.087754+0000","last_peered":"2023-09-23T18:02:55.087754+0000","last_clean":"2023-09-23T18:02:55.087754+0000","last_became_active":"2023-09-23T18:02:51.046591+0000","last_became_peered":"2023-09-23T18:02:51.046591+0000","last_unstale":"2023-09-23T18:02:55.087754+0000","last_undegraded":"2023-09-23T18:02:55.087754+0000","last_fullsized":"2023-09-23T18:02:55.087754+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:19.805099+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":104,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088134+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:02:55.088134+0000","last_peered":"2023-09-23T18:02:55.088134+0000","last_clean":"2023-09-23T18:02:55.088134+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:02:55.088134+0000","last_undegraded":"2023-09-23T18:02:55.088134+0000","last_fullsized":"2023-09-23T18:02:55.088134+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":255,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.093493+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:02:55.093493+0000","last_peered":"2023-09-23T18:02:55.093493+0000","last_clean":"2023-09-23T18:02:55.093493+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:02:55.093493+0000","last_undegraded":"2023-09-23T18:02:55.093493+0000","last_fullsized":"2023-09-23T18:02:55.093493+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":589,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087376+0000","last_change":"2023-09-23T18:02:51.051617+0000","last_active":"2023-09-23T18:02:55.087376+0000","last_peered":"2023-09-23T18:02:55.087376+0000","last_clean":"2023-09-23T18:02:55.087376+0000","last_became_active":"2023-09-23T18:02:51.048947+0000","last_became_peered":"2023-09-23T18:02:51.048947+0000","last_unstale":"2023-09-23T18:02:55.087376+0000","last_undegraded":"2023-09-23T18:02:55.087376+0000","last_fullsized":"2023-09-23T18:02:55.087376+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:49:34.962249+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":117,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.085983+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:02:55.085983+0000","last_peered":"2023-09-23T18:02:55.085983+0000","last_clean":"2023-09-23T18:02:55.085983+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:02:55.085983+0000","last_undegraded":"2023-09-23T18:02:55.085983+0000","last_fullsized":"2023-09-23T18:02:55.085983+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":195,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.071794+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:02:55.071794+0000","last_peered":"2023-09-23T18:02:55.071794+0000","last_clean":"2023-09-23T18:02:55.071794+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:02:55.071794+0000","last_undegraded":"2023-09-23T18:02:55.071794+0000","last_fullsized":"2023-09-23T18:02:55.071794+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088053+0000","last_change":"2023-09-23T18:02:51.052123+0000","last_active":"2023-09-23T18:02:55.088053+0000","last_peered":"2023-09-23T18:02:55.088053+0000","last_clean":"2023-09-23T18:02:55.088053+0000","last_became_active":"2023-09-23T18:02:51.051772+0000","last_became_peered":"2023-09-23T18:02:51.051772+0000","last_unstale":"2023-09-23T18:02:55.088053+0000","last_undegraded":"2023-09-23T18:02:55.088053+0000","last_fullsized":"2023-09-23T18:02:55.088053+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":235,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:02:56.088939+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:02:56.088939+0000","last_peered":"2023-09-23T18:02:56.088939+0000","last_clean":"2023-09-23T18:02:56.088939+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:02:56.088939+0000","last_undegraded":"2023-09-23T18:02:56.088939+0000","last_fullsized":"2023-09-23T18:02:56.088939+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":111,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.085996+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:02:55.085996+0000","last_peered":"2023-09-23T18:02:55.085996+0000","last_clean":"2023-09-23T18:02:55.085996+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:02:55.085996+0000","last_undegraded":"2023-09-23T18:02:55.085996+0000","last_fullsized":"2023-09-23T18:02:55.085996+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":581,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087470+0000","last_change":"2023-09-23T18:02:51.054208+0000","last_active":"2023-09-23T18:02:55.087470+0000","last_peered":"2023-09-23T18:02:55.087470+0000","last_clean":"2023-09-23T18:02:55.087470+0000","last_became_active":"2023-09-23T18:02:51.053676+0000","last_became_peered":"2023-09-23T18:02:51.053676+0000","last_unstale":"2023-09-23T18:02:55.087470+0000","last_undegraded":"2023-09-23T18:02:55.087470+0000","last_fullsized":"2023-09-23T18:02:55.087470+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":237,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.084272+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:02:55.084272+0000","last_peered":"2023-09-23T18:02:55.084272+0000","last_clean":"2023-09-23T18:02:55.084272+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:02:55.084272+0000","last_undegraded":"2023-09-23T18:02:55.084272+0000","last_fullsized":"2023-09-23T18:02:55.084272+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":65,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088023+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:02:55.088023+0000","last_peered":"2023-09-23T18:02:55.088023+0000","last_clean":"2023-09-23T18:02:55.088023+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:02:55.088023+0000","last_undegraded":"2023-09-23T18:02:55.088023+0000","last_fullsized":"2023-09-23T18:02:55.088023+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087511+0000","last_change":"2023-09-23T18:02:51.058624+0000","last_active":"2023-09-23T18:02:55.087511+0000","last_peered":"2023-09-23T18:02:55.087511+0000","last_clean":"2023-09-23T18:02:55.087511+0000","last_became_active":"2023-09-23T18:02:51.057352+0000","last_became_peered":"2023-09-23T18:02:51.057352+0000","last_unstale":"2023-09-23T18:02:55.087511+0000","last_undegraded":"2023-09-23T18:02:55.087511+0000","last_fullsized":"2023-09-23T18:02:55.087511+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":21,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087979+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:02:55.087979+0000","last_peered":"2023-09-23T18:02:55.087979+0000","last_clean":"2023-09-23T18:02:55.087979+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:02:55.087979+0000","last_undegraded":"2023-09-23T18:02:55.087979+0000","last_fullsized":"2023-09-23T18:02:55.087979+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":151,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.072007+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:02:55.072007+0000","last_peered":"2023-09-23T18:02:55.072007+0000","last_clean":"2023-09-23T18:02:55.072007+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:02:55.072007+0000","last_undegraded":"2023-09-23T18:02:55.072007+0000","last_fullsized":"2023-09-23T18:02:55.072007+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087993+0000","last_change":"2023-09-23T18:02:51.059983+0000","last_active":"2023-09-23T18:02:55.087993+0000","last_peered":"2023-09-23T18:02:55.087993+0000","last_clean":"2023-09-23T18:02:55.087993+0000","last_became_active":"2023-09-23T18:02:51.057768+0000","last_became_peered":"2023-09-23T18:02:51.057768+0000","last_unstale":"2023-09-23T18:02:55.087993+0000","last_undegraded":"2023-09-23T18:02:55.087993+0000","last_fullsized":"2023-09-23T18:02:55.087993+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":55,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097938+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:02:55.097938+0000","last_peered":"2023-09-23T18:02:55.097938+0000","last_clean":"2023-09-23T18:02:55.097938+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:02:55.097938+0000","last_undegraded":"2023-09-23T18:02:55.097938+0000","last_fullsized":"2023-09-23T18:02:55.097938+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":55,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097762+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:02:55.097762+0000","last_peered":"2023-09-23T18:02:55.097762+0000","last_clean":"2023-09-23T18:02:55.097762+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:02:55.097762+0000","last_undegraded":"2023-09-23T18:02:55.097762+0000","last_fullsized":"2023-09-23T18:02:55.097762+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":576,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087196+0000","last_change":"2023-09-23T18:02:51.047570+0000","last_active":"2023-09-23T18:02:55.087196+0000","last_peered":"2023-09-23T18:02:55.087196+0000","last_clean":"2023-09-23T18:02:55.087196+0000","last_became_active":"2023-09-23T18:02:51.046963+0000","last_became_peered":"2023-09-23T18:02:51.046963+0000","last_unstale":"2023-09-23T18:02:55.087196+0000","last_undegraded":"2023-09-23T18:02:55.087196+0000","last_fullsized":"2023-09-23T18:02:55.087196+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":184,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088097+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:02:55.088097+0000","last_peered":"2023-09-23T18:02:55.088097+0000","last_clean":"2023-09-23T18:02:55.088097+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:02:55.088097+0000","last_undegraded":"2023-09-23T18:02:55.088097+0000","last_fullsized":"2023-09-23T18:02:55.088097+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":99,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086468+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:02:55.086468+0000","last_peered":"2023-09-23T18:02:55.086468+0000","last_clean":"2023-09-23T18:02:55.086468+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:02:55.086468+0000","last_undegraded":"2023-09-23T18:02:55.086468+0000","last_fullsized":"2023-09-23T18:02:55.086468+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":589,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086799+0000","last_change":"2023-09-23T18:02:51.053806+0000","last_active":"2023-09-23T18:02:55.086799+0000","last_peered":"2023-09-23T18:02:55.086799+0000","last_clean":"2023-09-23T18:02:55.086799+0000","last_became_active":"2023-09-23T18:02:51.048822+0000","last_became_peered":"2023-09-23T18:02:51.048822+0000","last_unstale":"2023-09-23T18:02:55.086799+0000","last_undegraded":"2023-09-23T18:02:55.086799+0000","last_fullsized":"2023-09-23T18:02:55.086799+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:34.401104+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":254,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.093665+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:02:55.093665+0000","last_peered":"2023-09-23T18:02:55.093665+0000","last_clean":"2023-09-23T18:02:55.093665+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:02:55.093665+0000","last_undegraded":"2023-09-23T18:02:55.093665+0000","last_fullsized":"2023-09-23T18:02:55.093665+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":232,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083411+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:02:55.083411+0000","last_peered":"2023-09-23T18:02:55.083411+0000","last_clean":"2023-09-23T18:02:55.083411+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:02:55.083411+0000","last_undegraded":"2023-09-23T18:02:55.083411+0000","last_fullsized":"2023-09-23T18:02:55.083411+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":577,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087946+0000","last_change":"2023-09-23T18:02:51.046646+0000","last_active":"2023-09-23T18:02:55.087946+0000","last_peered":"2023-09-23T18:02:55.087946+0000","last_clean":"2023-09-23T18:02:55.087946+0000","last_became_active":"2023-09-23T18:02:51.046134+0000","last_became_peered":"2023-09-23T18:02:51.046134+0000","last_unstale":"2023-09-23T18:02:55.087946+0000","last_undegraded":"2023-09-23T18:02:55.087946+0000","last_fullsized":"2023-09-23T18:02:55.087946+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":45,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097488+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:02:55.097488+0000","last_peered":"2023-09-23T18:02:55.097488+0000","last_clean":"2023-09-23T18:02:55.097488+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:02:55.097488+0000","last_undegraded":"2023-09-23T18:02:55.097488+0000","last_fullsized":"2023-09-23T18:02:55.097488+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":245,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:02:56.089114+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:02:56.089114+0000","last_peered":"2023-09-23T18:02:56.089114+0000","last_clean":"2023-09-23T18:02:56.089114+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:02:56.089114+0000","last_undegraded":"2023-09-23T18:02:56.089114+0000","last_fullsized":"2023-09-23T18:02:56.089114+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":589,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087300+0000","last_change":"2023-09-23T18:02:51.051817+0000","last_active":"2023-09-23T18:02:55.087300+0000","last_peered":"2023-09-23T18:02:55.087300+0000","last_clean":"2023-09-23T18:02:55.087300+0000","last_became_active":"2023-09-23T18:02:51.048945+0000","last_became_peered":"2023-09-23T18:02:51.048945+0000","last_unstale":"2023-09-23T18:02:55.087300+0000","last_undegraded":"2023-09-23T18:02:55.087300+0000","last_fullsized":"2023-09-23T18:02:55.087300+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:47.163864+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":191,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.070864+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:02:55.070864+0000","last_peered":"2023-09-23T18:02:55.070864+0000","last_clean":"2023-09-23T18:02:55.070864+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:02:55.070864+0000","last_undegraded":"2023-09-23T18:02:55.070864+0000","last_fullsized":"2023-09-23T18:02:55.070864+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":108,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086772+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:02:55.086772+0000","last_peered":"2023-09-23T18:02:55.086772+0000","last_clean":"2023-09-23T18:02:55.086772+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:02:55.086772+0000","last_undegraded":"2023-09-23T18:02:55.086772+0000","last_fullsized":"2023-09-23T18:02:55.086772+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"255'5334","reported_seq":5392,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:03:02.561424+0000","last_change":"2023-09-23T18:00:57.996982+0000","last_active":"2023-09-23T18:03:02.561424+0000","last_peered":"2023-09-23T18:03:02.561424+0000","last_clean":"2023-09-23T18:03:02.561424+0000","last_became_active":"2023-09-23T18:00:57.996589+0000","last_became_peered":"2023-09-23T18:00:57.996589+0000","last_unstale":"2023-09-23T18:03:02.561424+0000","last_undegraded":"2023-09-23T18:03:02.561424+0000","last_fullsized":"2023-09-23T18:03:02.561424+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5334,"log_dups_size":0,"ondisk_log_size":5334,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:13.306089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256114688,"num_objects":3908,"num_object_clones":0,"num_object_copies":11724,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3908,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5334,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087896+0000","last_change":"2023-09-23T18:02:51.058477+0000","last_active":"2023-09-23T18:02:55.087896+0000","last_peered":"2023-09-23T18:02:55.087896+0000","last_clean":"2023-09-23T18:02:55.087896+0000","last_became_active":"2023-09-23T18:02:51.057258+0000","last_became_peered":"2023-09-23T18:02:51.057258+0000","last_unstale":"2023-09-23T18:02:55.087896+0000","last_undegraded":"2023-09-23T18:02:55.087896+0000","last_fullsized":"2023-09-23T18:02:55.087896+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":184,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087681+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:02:55.087681+0000","last_peered":"2023-09-23T18:02:55.087681+0000","last_clean":"2023-09-23T18:02:55.087681+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:02:55.087681+0000","last_undegraded":"2023-09-23T18:02:55.087681+0000","last_fullsized":"2023-09-23T18:02:55.087681+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":243,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083165+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:02:55.083165+0000","last_peered":"2023-09-23T18:02:55.083165+0000","last_clean":"2023-09-23T18:02:55.083165+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:02:55.083165+0000","last_undegraded":"2023-09-23T18:02:55.083165+0000","last_fullsized":"2023-09-23T18:02:55.083165+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"255'5128","reported_seq":12346,"reported_epoch":255,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:03:00.455861+0000","last_change":"2023-09-23T18:02:49.039290+0000","last_active":"2023-09-23T18:03:00.455861+0000","last_peered":"2023-09-23T18:03:00.455861+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:49.001792+0000","last_became_peered":"2023-09-23T18:02:49.001792+0000","last_unstale":"2023-09-23T18:03:00.455861+0000","last_undegraded":"2023-09-23T18:02:48.968288+0000","last_fullsized":"2023-09-23T18:03:00.455861+0000","mapping_epoch":250,"log_start":"217'930","ondisk_log_start":"217'930","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4198,"log_dups_size":0,"ondisk_log_size":4198,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.156979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259260416,"num_objects":3956,"num_object_clones":0,"num_object_copies":11868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":409,"num_objects_misplaced":7503,"num_objects_unfound":0,"num_objects_dirty":3956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5128,"num_write_kb":290688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3566,"num_bytes_recovered":233701376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[6,1,4],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,4,6","objects":3547},{"shards":"1,6","objects":409}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":591,"reported_epoch":254,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:55.093045+0000","last_change":"2023-09-23T18:02:48.968551+0000","last_active":"2023-09-23T18:02:55.093045+0000","last_peered":"2023-09-23T18:02:55.093045+0000","last_clean":"2023-09-23T18:02:45.965241+0000","last_became_active":"2023-09-23T18:02:48.968221+0000","last_became_peered":"2023-09-23T18:02:48.968221+0000","last_unstale":"2023-09-23T18:02:55.093045+0000","last_undegraded":"2023-09-23T18:02:48.944021+0000","last_fullsized":"2023-09-23T18:02:48.942326+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":409128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":199,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088601+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:02:55.088601+0000","last_peered":"2023-09-23T18:02:55.088601+0000","last_clean":"2023-09-23T18:02:55.088601+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:02:55.088601+0000","last_undegraded":"2023-09-23T18:02:55.088601+0000","last_fullsized":"2023-09-23T18:02:55.088601+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":265,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097659+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:02:55.097659+0000","last_peered":"2023-09-23T18:02:55.097659+0000","last_clean":"2023-09-23T18:02:55.097659+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:02:55.097659+0000","last_undegraded":"2023-09-23T18:02:55.097659+0000","last_fullsized":"2023-09-23T18:02:55.097659+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"255'5239","reported_seq":6838,"reported_epoch":255,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:59.259975+0000","last_change":"2023-09-23T18:02:47.987792+0000","last_active":"2023-09-23T18:02:59.259975+0000","last_peered":"2023-09-23T18:02:59.259975+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.307615+0000","last_became_peered":"2023-09-23T18:02:19.307615+0000","last_unstale":"2023-09-23T18:02:59.259975+0000","last_undegraded":"2023-09-23T18:02:18.966041+0000","last_fullsized":"2023-09-23T18:02:18.929348+0000","mapping_epoch":238,"log_start":"217'500","ondisk_log_start":"217'500","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4739,"log_dups_size":0,"ondisk_log_size":4739,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:04.498873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262340608,"num_objects":4003,"num_object_clones":0,"num_object_copies":12009,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3494,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4003,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5239,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":764,"num_bytes_recovered":50069504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":509}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":577,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087196+0000","last_change":"2023-09-23T18:02:51.058795+0000","last_active":"2023-09-23T18:02:55.087196+0000","last_peered":"2023-09-23T18:02:55.087196+0000","last_clean":"2023-09-23T18:02:55.087196+0000","last_became_active":"2023-09-23T18:02:51.057799+0000","last_became_peered":"2023-09-23T18:02:51.057799+0000","last_unstale":"2023-09-23T18:02:55.087196+0000","last_undegraded":"2023-09-23T18:02:55.087196+0000","last_fullsized":"2023-09-23T18:02:55.087196+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":122,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088783+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:02:55.088783+0000","last_peered":"2023-09-23T18:02:55.088783+0000","last_clean":"2023-09-23T18:02:55.088783+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:02:55.088783+0000","last_undegraded":"2023-09-23T18:02:55.088783+0000","last_fullsized":"2023-09-23T18:02:55.088783+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":220,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.071878+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:02:55.071878+0000","last_peered":"2023-09-23T18:02:55.071878+0000","last_clean":"2023-09-23T18:02:55.071878+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:02:55.071878+0000","last_undegraded":"2023-09-23T18:02:55.071878+0000","last_fullsized":"2023-09-23T18:02:55.071878+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"255'5257","reported_seq":5334,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:02:59.473944+0000","last_change":"2023-09-23T18:01:25.019784+0000","last_active":"2023-09-23T18:02:59.473944+0000","last_peered":"2023-09-23T18:02:59.473944+0000","last_clean":"2023-09-23T18:02:59.473944+0000","last_became_active":"2023-09-23T18:01:25.002073+0000","last_became_peered":"2023-09-23T18:01:25.002073+0000","last_unstale":"2023-09-23T18:02:59.473944+0000","last_undegraded":"2023-09-23T18:02:59.473944+0000","last_fullsized":"2023-09-23T18:02:59.473944+0000","mapping_epoch":214,"log_start":"217'900","ondisk_log_start":"217'900","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4357,"log_dups_size":0,"ondisk_log_size":4357,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:58.981672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265093120,"num_objects":4045,"num_object_clones":0,"num_object_copies":12135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5257,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088163+0000","last_change":"2023-09-23T18:02:51.057506+0000","last_active":"2023-09-23T18:02:55.088163+0000","last_peered":"2023-09-23T18:02:55.088163+0000","last_clean":"2023-09-23T18:02:55.088163+0000","last_became_active":"2023-09-23T18:02:51.056589+0000","last_became_peered":"2023-09-23T18:02:51.056589+0000","last_unstale":"2023-09-23T18:02:55.088163+0000","last_undegraded":"2023-09-23T18:02:55.088163+0000","last_fullsized":"2023-09-23T18:02:55.088163+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":231,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.084322+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:02:55.084322+0000","last_peered":"2023-09-23T18:02:55.084322+0000","last_clean":"2023-09-23T18:02:55.084322+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:02:55.084322+0000","last_undegraded":"2023-09-23T18:02:55.084322+0000","last_fullsized":"2023-09-23T18:02:55.084322+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":257,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092395+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:02:55.092395+0000","last_peered":"2023-09-23T18:02:55.092395+0000","last_clean":"2023-09-23T18:02:55.092395+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:02:55.092395+0000","last_undegraded":"2023-09-23T18:02:55.092395+0000","last_fullsized":"2023-09-23T18:02:55.092395+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"255'5344","reported_seq":8480,"reported_epoch":255,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:03:02.834352+0000","last_change":"2023-09-23T18:02:45.688221+0000","last_active":"2023-09-23T18:03:02.834352+0000","last_peered":"2023-09-23T18:03:02.834352+0000","last_clean":"2023-09-23T18:01:38.416095+0000","last_became_active":"2023-09-23T18:02:07.547135+0000","last_became_peered":"2023-09-23T18:02:07.547135+0000","last_unstale":"2023-09-23T18:03:02.834352+0000","last_undegraded":"2023-09-23T18:03:02.834352+0000","last_fullsized":"2023-09-23T18:02:02.784733+0000","mapping_epoch":227,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5244,"log_dups_size":0,"ondisk_log_size":5244,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:27.864301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256245760,"num_objects":3910,"num_object_clones":0,"num_object_copies":11730,"num_objects_missing_on_primary":629,"num_objects_missing":629,"num_objects_degraded":0,"num_objects_misplaced":2568,"num_objects_unfound":0,"num_objects_dirty":3910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5344,"num_write_kb":296128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1532,"num_bytes_recovered":100401152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1939},{"shards":"1,2,5","objects":629},{"shards":"2,4","objects":1342}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":578,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087118+0000","last_change":"2023-09-23T18:02:51.047609+0000","last_active":"2023-09-23T18:02:55.087118+0000","last_peered":"2023-09-23T18:02:55.087118+0000","last_clean":"2023-09-23T18:02:55.087118+0000","last_became_active":"2023-09-23T18:02:51.047209+0000","last_became_peered":"2023-09-23T18:02:51.047209+0000","last_unstale":"2023-09-23T18:02:55.087118+0000","last_undegraded":"2023-09-23T18:02:55.087118+0000","last_fullsized":"2023-09-23T18:02:55.087118+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":142,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086269+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:02:55.086269+0000","last_peered":"2023-09-23T18:02:55.086269+0000","last_clean":"2023-09-23T18:02:55.086269+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:02:55.086269+0000","last_undegraded":"2023-09-23T18:02:55.086269+0000","last_fullsized":"2023-09-23T18:02:55.086269+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":263,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097978+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:02:55.097978+0000","last_peered":"2023-09-23T18:02:55.097978+0000","last_clean":"2023-09-23T18:02:55.097978+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:02:55.097978+0000","last_undegraded":"2023-09-23T18:02:55.097978+0000","last_fullsized":"2023-09-23T18:02:55.097978+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"255'5106","reported_seq":5322,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:59.218092+0000","last_change":"2023-09-23T18:02:47.961575+0000","last_active":"2023-09-23T18:02:59.218092+0000","last_peered":"2023-09-23T18:02:59.218092+0000","last_clean":"2023-09-23T18:02:17.013863+0000","last_became_active":"2023-09-23T18:02:19.267215+0000","last_became_peered":"2023-09-23T18:02:19.267215+0000","last_unstale":"2023-09-23T18:02:59.218092+0000","last_undegraded":"2023-09-23T18:02:18.955030+0000","last_fullsized":"2023-09-23T18:02:18.924656+0000","mapping_epoch":238,"log_start":"217'400","ondisk_log_start":"217'400","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4706,"log_dups_size":0,"ondisk_log_size":4706,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:46.393021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258867200,"num_objects":3950,"num_object_clones":0,"num_object_copies":11850,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3950,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5106,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":4128768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-158},{"shards":"2,4,6","objects":3592}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":582,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086833+0000","last_change":"2023-09-23T18:02:51.049464+0000","last_active":"2023-09-23T18:02:55.086833+0000","last_peered":"2023-09-23T18:02:55.086833+0000","last_clean":"2023-09-23T18:02:55.086833+0000","last_became_active":"2023-09-23T18:02:51.048477+0000","last_became_peered":"2023-09-23T18:02:51.048477+0000","last_unstale":"2023-09-23T18:02:55.086833+0000","last_undegraded":"2023-09-23T18:02:55.086833+0000","last_fullsized":"2023-09-23T18:02:55.086833+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.072133+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:02:55.072133+0000","last_peered":"2023-09-23T18:02:55.072133+0000","last_clean":"2023-09-23T18:02:55.072133+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:02:55.072133+0000","last_undegraded":"2023-09-23T18:02:55.072133+0000","last_fullsized":"2023-09-23T18:02:55.072133+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":55,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.098179+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:02:55.098179+0000","last_peered":"2023-09-23T18:02:55.098179+0000","last_clean":"2023-09-23T18:02:55.098179+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:02:55.098179+0000","last_undegraded":"2023-09-23T18:02:55.098179+0000","last_fullsized":"2023-09-23T18:02:55.098179+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"255'5264","reported_seq":5327,"reported_epoch":255,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:03:02.501247+0000","last_change":"2023-09-23T18:02:19.054827+0000","last_active":"2023-09-23T18:03:02.501247+0000","last_peered":"2023-09-23T18:03:02.501247+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.048672+0000","last_became_peered":"2023-09-23T18:02:19.048672+0000","last_unstale":"2023-09-23T18:03:02.501247+0000","last_undegraded":"2023-09-23T18:02:18.951263+0000","last_fullsized":"2023-09-23T18:02:18.930851+0000","mapping_epoch":238,"log_start":"216'261","ondisk_log_start":"216'261","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5003,"log_dups_size":0,"ondisk_log_size":5003,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:23.690233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250871808,"num_objects":3828,"num_object_clones":0,"num_object_copies":11484,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3828,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3828,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5264,"num_write_kb":290944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":3828}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087643+0000","last_change":"2023-09-23T18:02:51.060858+0000","last_active":"2023-09-23T18:02:55.087643+0000","last_peered":"2023-09-23T18:02:55.087643+0000","last_clean":"2023-09-23T18:02:55.087643+0000","last_became_active":"2023-09-23T18:02:51.060554+0000","last_became_peered":"2023-09-23T18:02:51.060554+0000","last_unstale":"2023-09-23T18:02:55.087643+0000","last_undegraded":"2023-09-23T18:02:55.087643+0000","last_fullsized":"2023-09-23T18:02:55.087643+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":45,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097552+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:02:55.097552+0000","last_peered":"2023-09-23T18:02:55.097552+0000","last_clean":"2023-09-23T18:02:55.097552+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:02:55.097552+0000","last_undegraded":"2023-09-23T18:02:55.097552+0000","last_fullsized":"2023-09-23T18:02:55.097552+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":265,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.144859+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:02:55.144859+0000","last_peered":"2023-09-23T18:02:55.144859+0000","last_clean":"2023-09-23T18:02:55.144859+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:02:55.144859+0000","last_undegraded":"2023-09-23T18:02:55.144859+0000","last_fullsized":"2023-09-23T18:02:55.144859+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"255'5195","reported_seq":13307,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:03:02.519181+0000","last_change":"2023-09-23T18:02:47.017672+0000","last_active":"2023-09-23T18:03:02.519181+0000","last_peered":"2023-09-23T18:03:02.519181+0000","last_clean":"2023-09-23T18:03:02.519181+0000","last_became_active":"2023-09-23T18:02:47.009245+0000","last_became_peered":"2023-09-23T18:02:47.009245+0000","last_unstale":"2023-09-23T18:03:02.519181+0000","last_undegraded":"2023-09-23T18:03:02.519181+0000","last_fullsized":"2023-09-23T18:03:02.519181+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5195,"log_dups_size":0,"ondisk_log_size":5195,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:04.868743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254935064,"num_objects":3891,"num_object_clones":0,"num_object_copies":11673,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3891,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5195,"num_write_kb":290689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4015,"num_bytes_recovered":263127040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":578,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087219+0000","last_change":"2023-09-23T18:02:51.054197+0000","last_active":"2023-09-23T18:02:55.087219+0000","last_peered":"2023-09-23T18:02:55.087219+0000","last_clean":"2023-09-23T18:02:55.087219+0000","last_became_active":"2023-09-23T18:02:51.049362+0000","last_became_peered":"2023-09-23T18:02:51.049362+0000","last_unstale":"2023-09-23T18:02:55.087219+0000","last_undegraded":"2023-09-23T18:02:55.087219+0000","last_fullsized":"2023-09-23T18:02:55.087219+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":100,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086092+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:02:55.086092+0000","last_peered":"2023-09-23T18:02:55.086092+0000","last_clean":"2023-09-23T18:02:55.086092+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:02:55.086092+0000","last_undegraded":"2023-09-23T18:02:55.086092+0000","last_fullsized":"2023-09-23T18:02:55.086092+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":100,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086970+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:02:55.086970+0000","last_peered":"2023-09-23T18:02:55.086970+0000","last_clean":"2023-09-23T18:02:55.086970+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:02:55.086970+0000","last_undegraded":"2023-09-23T18:02:55.086970+0000","last_fullsized":"2023-09-23T18:02:55.086970+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"255'5010","reported_seq":5074,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:02:59.546287+0000","last_change":"2023-09-23T18:02:19.315376+0000","last_active":"2023-09-23T18:02:59.546287+0000","last_peered":"2023-09-23T18:02:59.546287+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.306174+0000","last_became_peered":"2023-09-23T18:02:19.306174+0000","last_unstale":"2023-09-23T18:02:59.546287+0000","last_undegraded":"2023-09-23T18:02:18.954031+0000","last_fullsized":"2023-09-23T18:02:18.922505+0000","mapping_epoch":238,"log_start":"217'600","ondisk_log_start":"217'600","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4410,"log_dups_size":0,"ondisk_log_size":4410,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:52.392034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253755392,"num_objects":3872,"num_object_clones":0,"num_object_copies":11616,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3872,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5010,"num_write_kb":284224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":-223}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":587,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087103+0000","last_change":"2023-09-23T18:02:51.059934+0000","last_active":"2023-09-23T18:02:55.087103+0000","last_peered":"2023-09-23T18:02:55.087103+0000","last_clean":"2023-09-23T18:02:55.087103+0000","last_became_active":"2023-09-23T18:02:51.059638+0000","last_became_peered":"2023-09-23T18:02:51.059638+0000","last_unstale":"2023-09-23T18:02:55.087103+0000","last_undegraded":"2023-09-23T18:02:55.087103+0000","last_fullsized":"2023-09-23T18:02:55.087103+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":564,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092935+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:02:55.092935+0000","last_peered":"2023-09-23T18:02:55.092935+0000","last_clean":"2023-09-23T18:02:55.092935+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:02:55.092935+0000","last_undegraded":"2023-09-23T18:02:55.092935+0000","last_fullsized":"2023-09-23T18:02:55.092935+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":235,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083486+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:02:55.083486+0000","last_peered":"2023-09-23T18:02:55.083486+0000","last_clean":"2023-09-23T18:02:55.083486+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:02:55.083486+0000","last_undegraded":"2023-09-23T18:02:55.083486+0000","last_fullsized":"2023-09-23T18:02:55.083486+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":232,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083985+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:02:55.083985+0000","last_peered":"2023-09-23T18:02:55.083985+0000","last_clean":"2023-09-23T18:02:55.083985+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:02:55.083985+0000","last_undegraded":"2023-09-23T18:02:55.083985+0000","last_fullsized":"2023-09-23T18:02:55.083985+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"255'5174","reported_seq":5253,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:02.725239+0000","last_change":"2023-09-23T18:02:19.289610+0000","last_active":"2023-09-23T18:03:02.725239+0000","last_peered":"2023-09-23T18:03:02.725239+0000","last_clean":"2023-09-23T18:02:17.060883+0000","last_became_active":"2023-09-23T18:02:19.266879+0000","last_became_peered":"2023-09-23T18:02:19.266879+0000","last_unstale":"2023-09-23T18:03:02.725239+0000","last_undegraded":"2023-09-23T18:02:18.963390+0000","last_fullsized":"2023-09-23T18:02:18.926482+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5174,"log_dups_size":0,"ondisk_log_size":5174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.499269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247988224,"num_objects":3784,"num_object_clones":0,"num_object_copies":11352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5174,"num_write_kb":286656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":3433},{"shards":"4,7","objects":-344}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":589,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087038+0000","last_change":"2023-09-23T18:02:51.047767+0000","last_active":"2023-09-23T18:02:55.087038+0000","last_peered":"2023-09-23T18:02:55.087038+0000","last_clean":"2023-09-23T18:02:55.087038+0000","last_became_active":"2023-09-23T18:02:51.047297+0000","last_became_peered":"2023-09-23T18:02:51.047297+0000","last_unstale":"2023-09-23T18:02:55.087038+0000","last_undegraded":"2023-09-23T18:02:55.087038+0000","last_fullsized":"2023-09-23T18:02:55.087038+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.851694+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1183,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.084031+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:02:55.084031+0000","last_peered":"2023-09-23T18:02:55.084031+0000","last_clean":"2023-09-23T18:02:55.084031+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:02:55.084031+0000","last_undegraded":"2023-09-23T18:02:55.084031+0000","last_fullsized":"2023-09-23T18:02:55.084031+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":45,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097798+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:02:55.097798+0000","last_peered":"2023-09-23T18:02:55.097798+0000","last_clean":"2023-09-23T18:02:55.097798+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:02:55.097798+0000","last_undegraded":"2023-09-23T18:02:55.097798+0000","last_fullsized":"2023-09-23T18:02:55.097798+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":148,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088461+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:02:55.088461+0000","last_peered":"2023-09-23T18:02:55.088461+0000","last_clean":"2023-09-23T18:02:55.088461+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:02:55.088461+0000","last_undegraded":"2023-09-23T18:02:55.088461+0000","last_fullsized":"2023-09-23T18:02:55.088461+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"255'5329","reported_seq":5385,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:02.821077+0000","last_change":"2023-09-23T18:02:19.260251+0000","last_active":"2023-09-23T18:03:02.821077+0000","last_peered":"2023-09-23T18:03:02.821077+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.253817+0000","last_became_peered":"2023-09-23T18:02:19.253817+0000","last_unstale":"2023-09-23T18:03:02.821077+0000","last_undegraded":"2023-09-23T18:02:18.955613+0000","last_fullsized":"2023-09-23T18:02:18.923894+0000","mapping_epoch":238,"log_start":"217'1374","ondisk_log_start":"217'1374","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":3955,"log_dups_size":0,"ondisk_log_size":3955,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:18.999679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257884160,"num_objects":3935,"num_object_clones":0,"num_object_copies":11805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4277,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5329,"num_write_kb":296448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":-342}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":590,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087142+0000","last_change":"2023-09-23T18:02:51.054622+0000","last_active":"2023-09-23T18:02:55.087142+0000","last_peered":"2023-09-23T18:02:55.087142+0000","last_clean":"2023-09-23T18:02:55.087142+0000","last_became_active":"2023-09-23T18:02:51.049345+0000","last_became_peered":"2023-09-23T18:02:51.049345+0000","last_unstale":"2023-09-23T18:02:55.087142+0000","last_undegraded":"2023-09-23T18:02:55.087142+0000","last_fullsized":"2023-09-23T18:02:55.087142+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:58.636010+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":537,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088027+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:02:55.088027+0000","last_peered":"2023-09-23T18:02:55.088027+0000","last_clean":"2023-09-23T18:02:55.088027+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:02:55.088027+0000","last_undegraded":"2023-09-23T18:02:55.088027+0000","last_fullsized":"2023-09-23T18:02:55.088027+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":149,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.095347+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:02:55.095347+0000","last_peered":"2023-09-23T18:02:55.095347+0000","last_clean":"2023-09-23T18:02:55.095347+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:02:55.095347+0000","last_undegraded":"2023-09-23T18:02:55.095347+0000","last_fullsized":"2023-09-23T18:02:55.095347+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":222,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.084446+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:02:55.084446+0000","last_peered":"2023-09-23T18:02:55.084446+0000","last_clean":"2023-09-23T18:02:55.084446+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:02:55.084446+0000","last_undegraded":"2023-09-23T18:02:55.084446+0000","last_fullsized":"2023-09-23T18:02:55.084446+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"255'5299","reported_seq":5392,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:02.547705+0000","last_change":"2023-09-23T18:02:47.986654+0000","last_active":"2023-09-23T18:03:02.547705+0000","last_peered":"2023-09-23T18:03:02.547705+0000","last_clean":"2023-09-23T18:02:16.962656+0000","last_became_active":"2023-09-23T18:02:19.272896+0000","last_became_peered":"2023-09-23T18:02:19.272896+0000","last_unstale":"2023-09-23T18:03:02.547705+0000","last_undegraded":"2023-09-23T18:02:18.972349+0000","last_fullsized":"2023-09-23T18:02:18.930868+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5299,"log_dups_size":0,"ondisk_log_size":5299,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:23.745675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":258277376,"num_objects":3941,"num_object_clones":0,"num_object_copies":11823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5299,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":3585},{"shards":"4,5","objects":-322}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1167,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083793+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:02:55.083793+0000","last_peered":"2023-09-23T18:02:55.083793+0000","last_clean":"2023-09-23T18:02:55.083793+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:02:55.083793+0000","last_undegraded":"2023-09-23T18:02:55.083793+0000","last_fullsized":"2023-09-23T18:02:55.083793+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":578,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086710+0000","last_change":"2023-09-23T18:02:51.054022+0000","last_active":"2023-09-23T18:02:55.086710+0000","last_peered":"2023-09-23T18:02:55.086710+0000","last_clean":"2023-09-23T18:02:55.086710+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:02:55.086710+0000","last_undegraded":"2023-09-23T18:02:55.086710+0000","last_fullsized":"2023-09-23T18:02:55.086710+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":62,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086828+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:02:55.086828+0000","last_peered":"2023-09-23T18:02:55.086828+0000","last_clean":"2023-09-23T18:02:55.086828+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:02:55.086828+0000","last_undegraded":"2023-09-23T18:02:55.086828+0000","last_fullsized":"2023-09-23T18:02:55.086828+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":188,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.071432+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:02:55.071432+0000","last_peered":"2023-09-23T18:02:55.071432+0000","last_clean":"2023-09-23T18:02:55.071432+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:02:55.071432+0000","last_undegraded":"2023-09-23T18:02:55.071432+0000","last_fullsized":"2023-09-23T18:02:55.071432+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"255'5198","reported_seq":5351,"reported_epoch":255,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:03:02.504178+0000","last_change":"2023-09-23T18:02:18.003772+0000","last_active":"2023-09-23T18:03:02.504178+0000","last_peered":"2023-09-23T18:03:02.504178+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:18.000696+0000","last_became_peered":"2023-09-23T18:02:18.000696+0000","last_unstale":"2023-09-23T18:03:02.504178+0000","last_undegraded":"2023-09-23T18:02:17.938873+0000","last_fullsized":"2023-09-23T18:02:17.919669+0000","mapping_epoch":237,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5098,"log_dups_size":0,"ondisk_log_size":5098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:48.085957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254541824,"num_objects":3884,"num_object_clones":0,"num_object_copies":11652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3848,"num_objects_misplaced":3884,"num_objects_unfound":0,"num_objects_dirty":3884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5198,"num_write_kb":290624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":3884}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":590,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087563+0000","last_change":"2023-09-23T18:02:51.061168+0000","last_active":"2023-09-23T18:02:55.087563+0000","last_peered":"2023-09-23T18:02:55.087563+0000","last_clean":"2023-09-23T18:02:55.087563+0000","last_became_active":"2023-09-23T18:02:51.060689+0000","last_became_peered":"2023-09-23T18:02:51.060689+0000","last_unstale":"2023-09-23T18:02:55.087563+0000","last_undegraded":"2023-09-23T18:02:55.087563+0000","last_fullsized":"2023-09-23T18:02:55.087563+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:29.406460+0000","scrub_duration":0.001568505,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":435,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086945+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:02:55.086945+0000","last_peered":"2023-09-23T18:02:55.086945+0000","last_clean":"2023-09-23T18:02:55.086945+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:02:55.086945+0000","last_undegraded":"2023-09-23T18:02:55.086945+0000","last_fullsized":"2023-09-23T18:02:55.086945+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":253,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092277+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:02:55.092277+0000","last_peered":"2023-09-23T18:02:55.092277+0000","last_clean":"2023-09-23T18:02:55.092277+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:02:55.092277+0000","last_undegraded":"2023-09-23T18:02:55.092277+0000","last_fullsized":"2023-09-23T18:02:55.092277+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":224,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.093419+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:02:55.093419+0000","last_peered":"2023-09-23T18:02:55.093419+0000","last_clean":"2023-09-23T18:02:55.093419+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:02:55.093419+0000","last_undegraded":"2023-09-23T18:02:55.093419+0000","last_fullsized":"2023-09-23T18:02:55.093419+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"255'5237","reported_seq":13573,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:03:00.486980+0000","last_change":"2023-09-23T18:02:18.942886+0000","last_active":"2023-09-23T18:03:00.486980+0000","last_peered":"2023-09-23T18:03:00.486980+0000","last_clean":"2023-09-23T18:03:00.486980+0000","last_became_active":"2023-09-23T18:02:18.936691+0000","last_became_peered":"2023-09-23T18:02:18.936691+0000","last_unstale":"2023-09-23T18:03:00.486980+0000","last_undegraded":"2023-09-23T18:03:00.486980+0000","last_fullsized":"2023-09-23T18:03:00.486980+0000","mapping_epoch":238,"log_start":"217'1058","ondisk_log_start":"217'1058","created":204,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4179,"log_dups_size":0,"ondisk_log_size":4179,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:26.085859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262078464,"num_objects":3999,"num_object_clones":0,"num_object_copies":11997,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3999,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5237,"num_write_kb":295552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4123,"num_bytes_recovered":270204928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":587,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088245+0000","last_change":"2023-09-23T18:02:51.057370+0000","last_active":"2023-09-23T18:02:55.088245+0000","last_peered":"2023-09-23T18:02:55.088245+0000","last_clean":"2023-09-23T18:02:55.088245+0000","last_became_active":"2023-09-23T18:02:51.056158+0000","last_became_peered":"2023-09-23T18:02:51.056158+0000","last_unstale":"2023-09-23T18:02:55.088245+0000","last_undegraded":"2023-09-23T18:02:55.088245+0000","last_fullsized":"2023-09-23T18:02:55.088245+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":469,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088698+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:02:55.088698+0000","last_peered":"2023-09-23T18:02:55.088698+0000","last_clean":"2023-09-23T18:02:55.088698+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:02:55.088698+0000","last_undegraded":"2023-09-23T18:02:55.088698+0000","last_fullsized":"2023-09-23T18:02:55.088698+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":189,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088514+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:02:55.088514+0000","last_peered":"2023-09-23T18:02:55.088514+0000","last_clean":"2023-09-23T18:02:55.088514+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:02:55.088514+0000","last_undegraded":"2023-09-23T18:02:55.088514+0000","last_fullsized":"2023-09-23T18:02:55.088514+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":221,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092617+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:02:55.092617+0000","last_peered":"2023-09-23T18:02:55.092617+0000","last_clean":"2023-09-23T18:02:55.092617+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:02:55.092617+0000","last_undegraded":"2023-09-23T18:02:55.092617+0000","last_fullsized":"2023-09-23T18:02:55.092617+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"255'5224","reported_seq":5308,"reported_epoch":255,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:02.864710+0000","last_change":"2023-09-23T18:02:19.279112+0000","last_active":"2023-09-23T18:03:02.864710+0000","last_peered":"2023-09-23T18:03:02.864710+0000","last_clean":"2023-09-23T18:02:17.018900+0000","last_became_active":"2023-09-23T18:02:19.267439+0000","last_became_peered":"2023-09-23T18:02:19.267439+0000","last_unstale":"2023-09-23T18:03:02.864710+0000","last_undegraded":"2023-09-23T18:02:18.959329+0000","last_fullsized":"2023-09-23T18:02:18.926801+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5224,"log_dups_size":0,"ondisk_log_size":5224,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.395583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254935040,"num_objects":3890,"num_object_clones":0,"num_object_copies":11670,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3890,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5224,"num_write_kb":291648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-309},{"shards":"2,4,6","objects":3532}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":587,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087817+0000","last_change":"2023-09-23T18:02:51.058680+0000","last_active":"2023-09-23T18:02:55.087817+0000","last_peered":"2023-09-23T18:02:55.087817+0000","last_clean":"2023-09-23T18:02:55.087817+0000","last_became_active":"2023-09-23T18:02:51.057706+0000","last_became_peered":"2023-09-23T18:02:51.057706+0000","last_unstale":"2023-09-23T18:02:55.087817+0000","last_undegraded":"2023-09-23T18:02:55.087817+0000","last_fullsized":"2023-09-23T18:02:55.087817+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":100,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086901+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:02:55.086901+0000","last_peered":"2023-09-23T18:02:55.086901+0000","last_clean":"2023-09-23T18:02:55.086901+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:02:55.086901+0000","last_undegraded":"2023-09-23T18:02:55.086901+0000","last_fullsized":"2023-09-23T18:02:55.086901+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":325,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.071348+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:02:55.071348+0000","last_peered":"2023-09-23T18:02:55.071348+0000","last_clean":"2023-09-23T18:02:55.071348+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:02:55.071348+0000","last_undegraded":"2023-09-23T18:02:55.071348+0000","last_fullsized":"2023-09-23T18:02:55.071348+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":104,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086059+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:02:55.086059+0000","last_peered":"2023-09-23T18:02:55.086059+0000","last_clean":"2023-09-23T18:02:55.086059+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:02:55.086059+0000","last_undegraded":"2023-09-23T18:02:55.086059+0000","last_fullsized":"2023-09-23T18:02:55.086059+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"255'5289","reported_seq":5350,"reported_epoch":255,"state":"active+clean","last_fresh":"2023-09-23T18:03:02.880857+0000","last_change":"2023-09-23T18:01:25.046224+0000","last_active":"2023-09-23T18:03:02.880857+0000","last_peered":"2023-09-23T18:03:02.880857+0000","last_clean":"2023-09-23T18:03:02.880857+0000","last_became_active":"2023-09-23T18:01:25.002864+0000","last_became_peered":"2023-09-23T18:01:25.002864+0000","last_unstale":"2023-09-23T18:03:02.880857+0000","last_undegraded":"2023-09-23T18:03:02.880857+0000","last_fullsized":"2023-09-23T18:03:02.880857+0000","mapping_epoch":214,"log_start":"217'1341","ondisk_log_start":"217'1341","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":3948,"log_dups_size":0,"ondisk_log_size":3948,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.752359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252248064,"num_objects":3849,"num_object_clones":0,"num_object_copies":11547,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3849,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5289,"num_write_kb":292416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":594,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087547+0000","last_change":"2023-09-23T18:02:51.053420+0000","last_active":"2023-09-23T18:02:55.087547+0000","last_peered":"2023-09-23T18:02:55.087547+0000","last_clean":"2023-09-23T18:02:55.087547+0000","last_became_active":"2023-09-23T18:02:51.053035+0000","last_became_peered":"2023-09-23T18:02:51.053035+0000","last_unstale":"2023-09-23T18:02:55.087547+0000","last_undegraded":"2023-09-23T18:02:55.087547+0000","last_fullsized":"2023-09-23T18:02:55.087547+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:20.456481+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":460,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.071843+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:02:55.071843+0000","last_peered":"2023-09-23T18:02:55.071843+0000","last_clean":"2023-09-23T18:02:55.071843+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:02:55.071843+0000","last_undegraded":"2023-09-23T18:02:55.071843+0000","last_fullsized":"2023-09-23T18:02:55.071843+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":281,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.093462+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:02:55.093462+0000","last_peered":"2023-09-23T18:02:55.093462+0000","last_clean":"2023-09-23T18:02:55.093462+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:02:55.093462+0000","last_undegraded":"2023-09-23T18:02:55.093462+0000","last_fullsized":"2023-09-23T18:02:55.093462+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":45,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097465+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:02:55.097465+0000","last_peered":"2023-09-23T18:02:55.097465+0000","last_clean":"2023-09-23T18:02:55.097465+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:02:55.097465+0000","last_undegraded":"2023-09-23T18:02:55.097465+0000","last_fullsized":"2023-09-23T18:02:55.097465+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":577,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087273+0000","last_change":"2023-09-23T18:02:51.046571+0000","last_active":"2023-09-23T18:02:55.087273+0000","last_peered":"2023-09-23T18:02:55.087273+0000","last_clean":"2023-09-23T18:02:55.087273+0000","last_became_active":"2023-09-23T18:02:51.046213+0000","last_became_peered":"2023-09-23T18:02:51.046213+0000","last_unstale":"2023-09-23T18:02:55.087273+0000","last_undegraded":"2023-09-23T18:02:55.087273+0000","last_fullsized":"2023-09-23T18:02:55.087273+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":240,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092542+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:02:55.092542+0000","last_peered":"2023-09-23T18:02:55.092542+0000","last_clean":"2023-09-23T18:02:55.092542+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:02:55.092542+0000","last_undegraded":"2023-09-23T18:02:55.092542+0000","last_fullsized":"2023-09-23T18:02:55.092542+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":267,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088559+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:02:55.088559+0000","last_peered":"2023-09-23T18:02:55.088559+0000","last_clean":"2023-09-23T18:02:55.088559+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:02:55.088559+0000","last_undegraded":"2023-09-23T18:02:55.088559+0000","last_fullsized":"2023-09-23T18:02:55.088559+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":586,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.088083+0000","last_change":"2023-09-23T18:02:51.060198+0000","last_active":"2023-09-23T18:02:55.088083+0000","last_peered":"2023-09-23T18:02:55.088083+0000","last_clean":"2023-09-23T18:02:55.088083+0000","last_became_active":"2023-09-23T18:02:51.057046+0000","last_became_peered":"2023-09-23T18:02:51.057046+0000","last_unstale":"2023-09-23T18:02:55.088083+0000","last_undegraded":"2023-09-23T18:02:55.088083+0000","last_fullsized":"2023-09-23T18:02:55.088083+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":110,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.084305+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:02:55.084305+0000","last_peered":"2023-09-23T18:02:55.084305+0000","last_clean":"2023-09-23T18:02:55.084305+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:02:55.084305+0000","last_undegraded":"2023-09-23T18:02:55.084305+0000","last_fullsized":"2023-09-23T18:02:55.084305+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":180,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097890+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:02:55.097890+0000","last_peered":"2023-09-23T18:02:55.097890+0000","last_clean":"2023-09-23T18:02:55.097890+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:02:55.097890+0000","last_undegraded":"2023-09-23T18:02:55.097890+0000","last_fullsized":"2023-09-23T18:02:55.097890+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":96,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.083286+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:02:55.083286+0000","last_peered":"2023-09-23T18:02:55.083286+0000","last_clean":"2023-09-23T18:02:55.083286+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:02:55.083286+0000","last_undegraded":"2023-09-23T18:02:55.083286+0000","last_fullsized":"2023-09-23T18:02:55.083286+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":119,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086667+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:02:55.086667+0000","last_peered":"2023-09-23T18:02:55.086667+0000","last_clean":"2023-09-23T18:02:55.086667+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:02:55.086667+0000","last_undegraded":"2023-09-23T18:02:55.086667+0000","last_fullsized":"2023-09-23T18:02:55.086667+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":602,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087712+0000","last_change":"2023-09-23T18:02:51.047476+0000","last_active":"2023-09-23T18:02:55.087712+0000","last_peered":"2023-09-23T18:02:55.087712+0000","last_clean":"2023-09-23T18:02:55.087712+0000","last_became_active":"2023-09-23T18:02:51.047152+0000","last_became_peered":"2023-09-23T18:02:51.047152+0000","last_unstale":"2023-09-23T18:02:55.087712+0000","last_undegraded":"2023-09-23T18:02:55.087712+0000","last_fullsized":"2023-09-23T18:02:55.087712+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:22.514077+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":119,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.086344+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:02:55.086344+0000","last_peered":"2023-09-23T18:02:55.086344+0000","last_clean":"2023-09-23T18:02:55.086344+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:02:55.086344+0000","last_undegraded":"2023-09-23T18:02:55.086344+0000","last_fullsized":"2023-09-23T18:02:55.086344+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":122,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087868+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:02:55.087868+0000","last_peered":"2023-09-23T18:02:55.087868+0000","last_clean":"2023-09-23T18:02:55.087868+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:02:55.087868+0000","last_undegraded":"2023-09-23T18:02:55.087868+0000","last_fullsized":"2023-09-23T18:02:55.087868+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":585,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.087727+0000","last_change":"2023-09-23T18:02:51.058825+0000","last_active":"2023-09-23T18:02:55.087727+0000","last_peered":"2023-09-23T18:02:55.087727+0000","last_clean":"2023-09-23T18:02:55.087727+0000","last_became_active":"2023-09-23T18:02:51.058480+0000","last_became_peered":"2023-09-23T18:02:51.058480+0000","last_unstale":"2023-09-23T18:02:55.087727+0000","last_undegraded":"2023-09-23T18:02:55.087727+0000","last_fullsized":"2023-09-23T18:02:55.087727+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":233,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.092888+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:02:55.092888+0000","last_peered":"2023-09-23T18:02:55.092888+0000","last_clean":"2023-09-23T18:02:55.092888+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:02:55.092888+0000","last_undegraded":"2023-09-23T18:02:55.092888+0000","last_fullsized":"2023-09-23T18:02:55.092888+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":265,"reported_epoch":254,"state":"active+clean","last_fresh":"2023-09-23T18:02:55.097491+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:02:55.097491+0000","last_peered":"2023-09-23T18:02:55.097491+0000","last_clean":"2023-09-23T18:02:55.097491+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:02:55.097491+0000","last_undegraded":"2023-09-23T18:02:55.097491+0000","last_fullsized":"2023-09-23T18:02:55.097491+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":4105437208,"num_objects":62645,"num_object_clones":0,"num_object_copies":187935,"num_objects_missing_on_primary":629,"num_objects_missing":629,"num_objects_degraded":36649,"num_objects_misplaced":13955,"num_objects_unfound":0,"num_objects_dirty":62645,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":83627,"num_write_kb":4680641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14129,"num_bytes_recovered":925958144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11846037504,"data_stored":11846025288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76063,"ondisk_log_size":76063,"up":48,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":402523,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":331,"num_read_kb":623,"num_write":162,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2224465,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":91,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001365,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2400944,"kb_used_data":2318824,"kb_used_omap":13,"kb_used_meta":82098,"kb_avail":91344208,"statfs":{"total":95995035648,"available":93536468992,"internally_reserved":0,"allocated":2374475776,"data_stored":2373499853,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":84068456},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.544,"5min":2.544,"15min":2.544},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":0.41499999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.629,"5min":2.629,"15min":2.629},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.5979999999999999,"5min":4.5979999999999999,"15min":4.5979999999999999},"last":0.58899999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.081,"5min":3.081,"15min":3.081},"last":0.45700000000000002},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.6680000000000001,"5min":2.6680000000000001,"15min":2.6680000000000001},"last":0.371}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.2989999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":3.984,"5min":3.984,"15min":3.984},"last":0.38900000000000001},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.645,"5min":1.645,"15min":1.645},"last":0.55800000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.004,"15min":1.004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.9300000000000002,"5min":3.9300000000000002,"15min":3.9300000000000002},"last":0.499},{"interface":"front","average":{"1min":1.133,"5min":1.133,"15min":1.133},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.7080000000000002,"5min":3.7080000000000002,"15min":3.7080000000000002},"last":0.44400000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.0409999999999999,"5min":3.0409999999999999,"15min":3.0409999999999999},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.6419999999999999,"5min":3.6419999999999999,"15min":3.6419999999999999},"last":0.54700000000000004}]}]},{"osd":6,"up_from":163,"seq":700079669301,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2478124,"kb_used_data":2333796,"kb_used_omap":7,"kb_used_meta":144312,"kb_avail":91267028,"statfs":{"total":95995035648,"available":93457436672,"internally_reserved":0,"allocated":2389807104,"data_stored":2388835694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":147776292},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.2270000000000001,"15min":1.3089999999999999},"min":{"1min":0.224,"5min":0.215,"15min":0.215},"max":{"1min":3.5529999999999999,"5min":14.196999999999999,"15min":14.196999999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.72,"15min":2.3740000000000001},"min":{"1min":0.32600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.8970000000000002,"5min":13.536,"15min":13.536},"last":1.827}]},{"osd":1,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.5860000000000001,"15min":1.956},"min":{"1min":0.28100000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.4910000000000001,"5min":14.233000000000001,"15min":14.233000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.01,"5min":1.6759999999999999,"15min":2.173},"min":{"1min":0.33300000000000002,"5min":0.248,"15min":0.248},"max":{"1min":4.9340000000000002,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":1.6890000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":2.0009999999999999,"15min":2.3450000000000002},"min":{"1min":0.34999999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.173,"5min":11.210000000000001,"15min":11.210000000000001},"last":5.7489999999999997},{"interface":"front","average":{"1min":1.0589999999999999,"5min":2.1899999999999999,"15min":2.758},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.9700000000000002,"5min":11.223000000000001,"15min":11.223000000000001},"last":5.7359999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.96599999999999997,"15min":0.89400000000000002},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.827,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.57899999999999996},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.72499999999999998,"15min":0.503},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.0129999999999999,"5min":8.3000000000000007,"15min":8.3000000000000007},"last":0.38900000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.94199999999999995,"15min":0.84399999999999997},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.1050000000000004,"5min":11.869,"15min":11.869},"last":0.496},{"interface":"front","average":{"1min":1.21,"5min":0.875,"15min":0.63500000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.7130000000000001,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":0.48999999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.6469999999999998,"5min":3.6469999999999998,"15min":3.6469999999999998},"last":2.1150000000000002},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.5310000000000001,"5min":2.5310000000000001,"15min":2.5310000000000001},"last":1.093}]}]},{"osd":1,"up_from":0,"seq":420906795120,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958646,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1432268,"kb_used_data":1026436,"kb_used_omap":7,"kb_used_meta":405816,"kb_avail":92312884,"statfs":{"total":95995035648,"available":94528393216,"internally_reserved":0,"allocated":1051070464,"data_stored":1049781918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":415556324},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:02:07 2023","interfaces":[{"interface":"back","average":{"1min":0.872,"5min":2.238,"15min":1.0920000000000001},"min":{"1min":0.21199999999999999,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":3.786,"5min":18.154,"15min":18.154},"last":2.145},{"interface":"front","average":{"1min":0.51200000000000001,"5min":2.1739999999999999,"15min":1.083},"min":{"1min":0.186,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":1.9359999999999999,"5min":13.369999999999999,"15min":13.369999999999999},"last":0.55200000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.4460000000000002,"15min":1.24},"min":{"1min":0.25700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.5259999999999998,"5min":20.882000000000001,"15min":20.882000000000001},"last":2.1139999999999999},{"interface":"front","average":{"1min":1.234,"5min":2.0899999999999999,"15min":1.149},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":6.7439999999999998,"5min":15.064,"15min":15.064},"last":2.5219999999999998}]},{"osd":3,"last update":"Sat Sep 23 17:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":0.77100000000000002,"15min":0.60099999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3449999999999998,"5min":4.3449999999999998,"15min":4.3449999999999998},"last":3.5529999999999999},{"interface":"front","average":{"1min":2.109,"5min":0.73699999999999999,"15min":0.50900000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":8.8729999999999993},"last":3.4609999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":1.603,"15min":1.0509999999999999},"min":{"1min":0.29299999999999998,"5min":0.251,"15min":0.251},"max":{"1min":4.54,"5min":15.279999999999999,"15min":15.279999999999999},"last":1.6499999999999999},{"interface":"front","average":{"1min":1.24,"5min":1.859,"15min":1.1220000000000001},"min":{"1min":0.29899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7190000000000003,"5min":18.106999999999999,"15min":18.106999999999999},"last":2.2010000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.9710000000000001,"15min":1.1619999999999999},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.1280000000000001,"5min":14.720000000000001,"15min":14.720000000000001},"last":2.1760000000000002},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.5469999999999999,"15min":1.022},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":2.806,"5min":12.794,"15min":12.794},"last":0.53100000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.8680000000000001,"15min":2.3010000000000002},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.5069999999999997,"5min":14.956,"15min":14.956},"last":1.732},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.0419999999999998,"15min":2.5219999999999998},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.311},"max":{"1min":2.883,"5min":12.755000000000001,"15min":12.755000000000001},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:02:28 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":2.851,"5min":2.851,"15min":2.851},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.083,"5min":1.083,"15min":1.083},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.4619999999999997,"5min":4.4619999999999997,"15min":4.4619999999999997},"last":1.6919999999999999}]}]},{"osd":2,"up_from":103,"seq":442381631593,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2650368,"kb_used_data":2313080,"kb_used_omap":11,"kb_used_meta":337268,"kb_avail":91094784,"statfs":{"total":95995035648,"available":93281058816,"internally_reserved":0,"allocated":2368593920,"data_stored":2367317513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":345362807},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":2.552,"15min":1.2729999999999999},"min":{"1min":0.187,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":3.306,"5min":20.867000000000001,"15min":20.867000000000001},"last":0.247},{"interface":"front","average":{"1min":0.55300000000000005,"5min":2.3119999999999998,"15min":1.246},"min":{"1min":0.24299999999999999,"5min":0.215,"15min":0.215},"max":{"1min":1.6910000000000001,"5min":21.681000000000001,"15min":21.681000000000001},"last":0.40400000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.6349999999999998,"15min":1.361},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":4.2359999999999998,"5min":20.995000000000001,"15min":20.995000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.75,"5min":1.9530000000000001,"15min":1.147},"min":{"1min":0.24199999999999999,"5min":0.193,"15min":0.193},"max":{"1min":4.532,"5min":15.945,"15min":15.945},"last":0.32700000000000001}]},{"osd":3,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":0.58799999999999997,"15min":0.443},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8339999999999996,"5min":4.8339999999999996,"15min":4.8339999999999996},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.59699999999999998,"15min":0.48399999999999999},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.6459999999999999,"5min":5.6459999999999999,"15min":5.6459999999999999},"last":0.46400000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.7649999999999999,"15min":1.095},"min":{"1min":0.29099999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.141,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":0.502},{"interface":"front","average":{"1min":0.45900000000000002,"5min":1.8899999999999999,"15min":1.1200000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78100000000000003,"5min":20.920000000000002,"15min":20.920000000000002},"last":0.47699999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.9890000000000001,"15min":1.1879999999999999},"min":{"1min":0.32900000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.972,"5min":20.738,"15min":20.738},"last":0.439},{"interface":"front","average":{"1min":1.048,"5min":1.79,"15min":1.1970000000000001},"min":{"1min":0.34100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3700000000000001,"5min":14.359,"15min":15.496},"last":0.41599999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:02:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":2.0299999999999998,"15min":2.2240000000000002},"min":{"1min":0.34699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":5.1879999999999997,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.66900000000000004,"5min":2.3290000000000002,"15min":2.4929999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.6930000000000001,"5min":20.893000000000001,"15min":20.893000000000001},"last":0.59199999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:02:25 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.411,"15min":1.411},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.3239999999999998,"5min":4.3239999999999998,"15min":4.3239999999999998},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.7130000000000001,"5min":1.7130000000000001,"15min":1.7130000000000001},"last":0.629}]}]},{"osd":3,"up_from":237,"seq":1017907249162,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1295372,"kb_used_data":1209156,"kb_used_omap":3,"kb_used_meta":86204,"kb_avail":92449780,"statfs":{"total":95995035648,"available":94668574720,"internally_reserved":0,"allocated":1238175744,"data_stored":1236717182,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":88273640},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.403}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.3149999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.35899999999999999}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.56000000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.314}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.438}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.39100000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019112,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1635184,"kb_used_data":1429800,"kb_used_omap":0,"kb_used_meta":205376,"kb_avail":92109968,"statfs":{"total":95995035648,"available":94320607232,"internally_reserved":0,"allocated":1464115200,"data_stored":1463139530,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":210305024},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.3260000000000001,"15min":0.82099999999999995},"min":{"1min":0.24399999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.1829999999999998,"5min":14.164,"15min":14.164},"last":0.27700000000000002},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.611,"15min":0.97899999999999998},"min":{"1min":0.253,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.29,"5min":11.372,"15min":11.372},"last":0.45900000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:02:03 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":1.7470000000000001,"15min":0.95599999999999996},"min":{"1min":0.35699999999999998,"5min":0.26900000000000002,"15min":0.249},"max":{"1min":0.92900000000000005,"5min":10.429,"15min":10.429},"last":1.4930000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.8009999999999999,"15min":1.022},"min":{"1min":0.29099999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":6.0620000000000003,"5min":10.83,"15min":10.83},"last":1.417}]},{"osd":2,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.6699999999999999,"15min":1.0629999999999999},"min":{"1min":0.29899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.266,"5min":11.630000000000001,"15min":11.630000000000001},"last":1.125},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.5289999999999999,"15min":0.98799999999999999},"min":{"1min":0.316,"5min":0.30099999999999999,"15min":0.27100000000000002},"max":{"1min":5.2450000000000001,"5min":7.5090000000000003,"15min":7.5090000000000003},"last":2.2799999999999998}]},{"osd":3,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.72899999999999998,"15min":0.64900000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6130000000000004,"5min":4.6130000000000004,"15min":4.6130000000000004},"last":1.712},{"interface":"front","average":{"1min":1.04,"5min":0.73299999999999998,"15min":0.68200000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":3.637,"5min":3.637,"15min":3.637},"last":2.4020000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.80800000000000005,"15min":0.59299999999999997},"min":{"1min":0.23799999999999999,"5min":0.224,"15min":0.224},"max":{"1min":3.2269999999999999,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0029999999999999,"15min":0.71299999999999997},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0629999999999997,"5min":7.4729999999999999,"15min":10.494},"last":0.39900000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":0.94499999999999995,"15min":0.82799999999999996},"min":{"1min":0.25,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.5579999999999998,"5min":7.1529999999999996,"15min":7.1529999999999996},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0680000000000001,"15min":0.95599999999999996},"min":{"1min":0.28999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.286,"5min":10.725,"15min":10.725},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:02:25 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.8839999999999999,"5min":3.8839999999999999,"15min":3.8839999999999999},"last":0.434},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.8090000000000002,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.35799999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757478,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1576740,"kb_used_data":1383964,"kb_used_omap":0,"kb_used_meta":192768,"kb_avail":92168412,"statfs":{"total":95995035648,"available":94380453888,"internally_reserved":0,"allocated":1417179136,"data_stored":1416211218,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":197394432},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.6779999999999999,"15min":1.008},"min":{"1min":0.28699999999999998,"5min":0.217,"15min":0.217},"max":{"1min":4.0250000000000004,"5min":21.376999999999999,"15min":21.376999999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":1.996,"5min":1.8939999999999999,"15min":1.149},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.048,"5min":21.388999999999999,"15min":21.388999999999999},"last":0.747}]},{"osd":1,"last update":"Sat Sep 23 18:02:04 2023","interfaces":[{"interface":"back","average":{"1min":0.751,"5min":1.1990000000000001,"15min":0.873},"min":{"1min":0.315,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":2.0870000000000002,"5min":10.853999999999999,"15min":10.853999999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.518,"15min":0.92200000000000004},"min":{"1min":0.26200000000000001,"5min":0.254,"15min":0.254},"max":{"1min":2.6230000000000002,"5min":10.206,"15min":10.206},"last":0.67700000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.1320000000000001,"15min":1.2589999999999999},"min":{"1min":0.34200000000000003,"5min":0.255,"15min":0.255},"max":{"1min":4.9279999999999999,"5min":26.035,"15min":38.606999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.534,"15min":1.123},"min":{"1min":0.35999999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.885,"5min":17.085000000000001,"15min":39.670999999999999},"last":0.434}]},{"osd":3,"last update":"Sat Sep 23 17:56:57 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":0.78800000000000003,"15min":0.66400000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.4969999999999999,"5min":7.4969999999999999,"15min":7.4969999999999999},"last":0.79200000000000004},{"interface":"front","average":{"1min":1.0900000000000001,"5min":0.70199999999999996,"15min":0.63800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.5510000000000002,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":0.34899999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":0.94499999999999995,"15min":0.69699999999999995},"min":{"1min":0.24199999999999999,"5min":0.22800000000000001,"15min":0.224},"max":{"1min":3.9980000000000002,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.998,"15min":0.69799999999999995},"min":{"1min":0.30499999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.1449999999999996,"5min":5.077,"15min":5.077},"last":0.42299999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.343,"15min":1.5069999999999999},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.5880000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.147,"15min":1.329},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.1710000000000003,"5min":8.266,"15min":8.266},"last":0.65900000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:02:26 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.546,"15min":1.546},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":1.125,"5min":1.125,"15min":1.125},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.0380000000000003,"5min":4.0380000000000003,"15min":4.0380000000000003},"last":0.77300000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1038749696,"data_stored":1038745624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1492582400,"data_stored":1492582400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2350059520,"data_stored":2350055448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":197853184,"data_stored":197853184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1456144384,"data_stored":1456144384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1404375040,"data_stored":1404370968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2375811072,"data_stored":2375811072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1530462208,"data_stored":1530462208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:03:04.969 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:03:04.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:05.026 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:05.064 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:05.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:05.069 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:05.170 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:03:05.174 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:03:05.205 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:05.209 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:03:05.209 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:03:05.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:03:05.209 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:03:05.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.209 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:5 6.5 6:aa422d9f:::benchmark_data_smithi055_275963_object4:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:03:05.209 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.962868+0000", 2023-09-23T18:03:05.209 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 101.242269024, 2023-09-23T18:03:05.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0413645090000001, 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 5 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962868+0000", 2023-09-23T18:03:05.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962868+0000", 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962867+0000", 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962943+0000", 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6361999999999998e-05 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962956+0000", 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2972999999999999e-05 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962965+0000", 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5830000000000007e-06 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985717+0000", 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022752456000000001 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985726+0000", 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8119999999999997e-06 2023-09-23T18:03:05.213 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:24.999795+0000", 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.014069508 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:24.999895+0000", 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9772999999999995e-05 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.000018+0000", 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000122621 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001650+0000", 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016320659999999999 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004113+0000", 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024632030000000002 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004166+0000", 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3303000000000001e-05 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004233+0000", 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6569000000000006e-05 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.216 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:26 6.5 6:a2aea16e:::benchmark_data_smithi055_275963_object25:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.962976+0000", 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 101.242161657, 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041444284, 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 26 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.217 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962976+0000", 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962976+0000", 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962975+0000", 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963048+0000", 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3052000000000005e-05 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963057+0000", 2023-09-23T18:03:05.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8589999999999994e-06 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963062+0000", 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1019999999999996e-06 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985743+0000", 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022680881 2023-09-23T18:03:05.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985746+0000", 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6199999999999999e-06 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.000289+0000", 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.014543376 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.000397+0000", 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010802300000000001 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.000542+0000", 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014492899999999999 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001656+0000", 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011136539999999999 2023-09-23T18:03:05.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004279+0000", 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002623776 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004330+0000", 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0961999999999998e-05 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004420+0000", 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9585000000000004e-05 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:38 6.5 6:ab4b060d:::benchmark_data_smithi055_275963_object37:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.963071+0000", 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 101.24206635900001, 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041474614, 2023-09-23T18:03:05.224 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 38 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963071+0000", 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963071+0000", 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963071+0000", 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963129+0000", 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8417e-05 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963149+0000", 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9789999999999999e-05 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963154+0000", 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3279999999999996e-06 2023-09-23T18:03:05.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985756+0000", 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022602048999999999 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985758+0000", 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2630000000000002e-06 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.228 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001146+0000", 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0153872100000001 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001256+0000", 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110223 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001395+0000", 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013967299999999999 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003007+0000", 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016115529999999999 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004466+0000", 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001459115 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004493+0000", 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6808000000000002e-05 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004546+0000", 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2677999999999999e-05 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.231 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:58 6.5 6:a3f61190:::benchmark_data_smithi055_275963_object57:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.963160+0000", 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 101.241977754, 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041575398, 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 58 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.232 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963160+0000", 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963160+0000", 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963159+0000", 2023-09-23T18:03:05.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963250+0000", 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0525999999999994e-05 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963259+0000", 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5030000000000003e-06 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963265+0000", 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8640000000000001e-06 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985768+0000", 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022502696999999999 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985770+0000", 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5100000000000001e-06 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001682+0000", 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.015911507 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001784+0000", 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010179900000000001 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001893+0000", 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010975400000000001 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003018+0000", 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011243480000000001 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004604+0000", 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015860099999999999 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004651+0000", 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6944999999999999e-05 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004735+0000", 2023-09-23T18:03:05.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4363000000000005e-05 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:76 6.5 6:a6e8b6c3:::benchmark_data_smithi055_275963_object75:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.963283+0000", 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 101.24185460299999, 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0416376890000001, 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.239 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 76 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963283+0000", 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963283+0000", 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963282+0000", 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963364+0000", 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1633999999999998e-05 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963374+0000", 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0370000000000001e-05 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963379+0000", 2023-09-23T18:03:05.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7890000000000002e-06 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985780+0000", 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022400929 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985782+0000", 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1179999999999999e-06 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002089+0000", 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0163073250000001 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002170+0000", 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0921999999999995e-05 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.244 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002280+0000", 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000109159 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003824+0000", 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001544754 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004786+0000", 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00096193199999999998 2023-09-23T18:03:05.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004835+0000", 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8804000000000002e-05 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004920+0000", 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5459999999999998e-05 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.246 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:105 6.5 6:a7b1533e:::benchmark_data_smithi055_275963_object104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.963385+0000", 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 101.24175264500001, 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041704696, 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 105 2023-09-23T18:03:05.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963385+0000", 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963385+0000", 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963384+0000", 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963433+0000", 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8285000000000003e-05 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963439+0000", 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.844e-06 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963446+0000", 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4529999999999999e-06 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985792+0000", 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022346594000000001 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985795+0000", 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1670000000000002e-06 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002533+0000", 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0167385369999999 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002654+0000", 2023-09-23T18:03:05.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120804 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002793+0000", 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000139457 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004027+0000", 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012339440000000001 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004957+0000", 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00092994099999999999 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.005009+0000", 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1399e-05 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.005089+0000", 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0758000000000002e-05 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:115 6.5 6:aa1b9814:::benchmark_data_smithi055_275963_object114:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.963452+0000", 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 101.241685531, 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.042225293, 2023-09-23T18:03:05.254 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 115 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963452+0000", 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963452+0000", 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963451+0000", 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963497+0000", 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5408999999999998e-05 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963504+0000", 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6429999999999998e-06 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963509+0000", 2023-09-23T18:03:05.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6189999999999999e-06 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985803+0000", 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022294333 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985806+0000", 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1299999999999999e-06 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003043+0000", 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0172373260000001 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003474+0000", 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00043128399999999999 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:03:05.259 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003618+0000", 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143957 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004071+0000", 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00045290599999999999 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.005592+0000", 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001521002 2023-09-23T18:03:05.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.005625+0000", 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2938999999999998e-05 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.005677+0000", 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2154000000000002e-05 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.261 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.262 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:127 6.6 6:6d064c04:::benchmark_data_smithi055_275963_object126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.262 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.008537+0000", 2023-09-23T18:03:05.262 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.19660041900001, 2023-09-23T18:03:05.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0229015530000001, 2023-09-23T18:03:05.262 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.262 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.262 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.262 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 127 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008537+0000", 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008537+0000", 2023-09-23T18:03:05.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008536+0000", 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008806+0000", 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00027006399999999998 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008818+0000", 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1547e-05 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008824+0000", 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3230000000000003e-06 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008874+0000", 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9767e-05 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008882+0000", 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8529999999999992e-06 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.020896+0000", 2023-09-23T18:03:05.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0120143370000001 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.021056+0000", 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015974199999999999 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.021277+0000", 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000221208 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.267 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024530+0000", 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032529759999999999 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031109+0000", 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065789000000000004 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031241+0000", 2023-09-23T18:03:05.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132788 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031301+0000", 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9379999999999997e-05 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031439+0000", 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000137658 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.269 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:137 6.6 6:63c48133:::benchmark_data_smithi055_275963_object136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.011129+0000", 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.194008705, 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0205278870000001, 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.270 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011129+0000", 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011129+0000", 2023-09-23T18:03:05.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011128+0000", 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011153+0000", 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4885e-05 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011161+0000", 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8529999999999992e-06 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011167+0000", 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8449999999999999e-06 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011214+0000", 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7255000000000001e-05 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011222+0000", 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5499999999999997e-06 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.021598+0000", 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0103761330000001 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.021729+0000", 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000131092 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.021866+0000", 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013756400000000001 2023-09-23T18:03:05.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024537+0000", 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026707839999999998 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031133+0000", 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065955650000000003 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031479+0000", 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00034648799999999999 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031534+0000", 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4576000000000001e-05 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031657+0000", 2023-09-23T18:03:05.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000122741 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:229 6.6 6:6dbd3e64:::benchmark_data_smithi055_275963_object228:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.036295+0000", 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.168841958, 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99556675400000005, 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.278 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 229 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036295+0000", 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.279 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036295+0000", 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036295+0000", 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036379+0000", 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4479000000000002e-05 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036387+0000", 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7559999999999999e-06 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036392+0000", 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.843e-06 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036451+0000", 2023-09-23T18:03:05.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8856e-05 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036460+0000", 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8740000000000001e-06 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.022147+0000", 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98568734999999996 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.022259+0000", 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112061 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.022422+0000", 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000163053 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024544+0000", 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002121805 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031152+0000", 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006608137 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031694+0000", 2023-09-23T18:03:05.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00054246299999999997 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031742+0000", 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7777e-05 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031862+0000", 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119978 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.285 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:234 6.6 6:6b65f21e:::benchmark_data_smithi055_275963_object233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.038246+0000", 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.166891015, 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.993822388, 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.286 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 234 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038246+0000", 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038246+0000", 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038245+0000", 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038352+0000", 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106671 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038363+0000", 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1026000000000001e-05 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038369+0000", 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8320000000000002e-06 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038416+0000", 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7846000000000002e-05 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038424+0000", 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7430000000000002e-06 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.022680+0000", 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98425543400000004 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.022807+0000", 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012723699999999999 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.023021+0000", 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021391499999999999 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025104+0000", 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002082815 2023-09-23T18:03:05.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031170+0000", 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0060661270000000002 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031901+0000", 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00073105900000000005 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031949+0000", 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8347000000000003e-05 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032069+0000", 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119639 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:265 6.6 6:6dcd9c47:::benchmark_data_smithi055_275963_object264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.046473+0000", 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.158664512, 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98580278600000004, 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.294 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 265 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046473+0000", 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046473+0000", 2023-09-23T18:03:05.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046472+0000", 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046502+0000", 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0365000000000002e-05 2023-09-23T18:03:05.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046511+0000", 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1659999999999994e-06 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046515+0000", 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.549e-06 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046582+0000", 2023-09-23T18:03:05.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6396999999999995e-05 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046590+0000", 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6680000000000007e-06 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.023319+0000", 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97672869900000003 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.023420+0000", 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010121299999999999 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.023591+0000", 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000170567 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025110+0000", 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001519187 2023-09-23T18:03:05.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031189+0000", 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0060787430000000002 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032109+0000", 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00092012799999999998 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032157+0000", 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7997000000000001e-05 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032276+0000", 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000118878 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:267 6.6 6:6e7c4e87:::benchmark_data_smithi055_275963_object266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.301 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.048471+0000", 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.15666668999999, 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98400787999999995, 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 267 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048471+0000", 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048471+0000", 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048469+0000", 2023-09-23T18:03:05.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048537+0000", 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7155000000000003e-05 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048553+0000", 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6660999999999999e-05 2023-09-23T18:03:05.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048568+0000", 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4876e-05 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048630+0000", 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1891999999999996e-05 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.305 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048639+0000", 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.482e-06 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.023970+0000", 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97533058399999994 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024119+0000", 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014856900000000001 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024293+0000", 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017458900000000001 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025610+0000", 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001316582 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031206+0000", 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055965299999999997 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032312+0000", 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011053180000000001 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032359+0000", 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7657000000000001e-05 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032479+0000", 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011932 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:270 6.6 6:6138c5d1:::benchmark_data_smithi055_275963_object269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.048591+0000", 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.15654592200001, 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98409610000000003, 2023-09-23T18:03:05.309 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 270 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048591+0000", 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048591+0000", 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048591+0000", 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048664+0000", 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3314999999999999e-05 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048675+0000", 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1086e-05 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048684+0000", 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0869999999999996e-06 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048723+0000", 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9069999999999997e-05 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048728+0000", 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8930000000000004e-06 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024585+0000", 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97585701199999997 2023-09-23T18:03:05.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024710+0000", 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012514899999999999 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024881+0000", 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017054400000000001 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.028592+0000", 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037112740000000001 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031224+0000", 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002631965 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032515+0000", 2023-09-23T18:03:05.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001290802 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032569+0000", 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3838999999999997e-05 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032688+0000", 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011871600000000001 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:272 6.6 6:6b2a1235:::benchmark_data_smithi055_275963_object271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.049079+0000", 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.156058374, 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98390465400000005, 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 272 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.317 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049079+0000", 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049079+0000", 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049078+0000", 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049166+0000", 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7336999999999994e-05 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049186+0000", 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.012e-05 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049192+0000", 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5060000000000001e-06 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049242+0000", 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9203000000000003e-05 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049251+0000", 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.38e-06 2023-09-23T18:03:05.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025142+0000", 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97589093699999996 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025265+0000", 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012347899999999999 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025399+0000", 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013394200000000001 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.029874+0000", 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044745369999999998 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032725+0000", 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028511719999999999 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032820+0000", 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5140000000000001e-05 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032868+0000", 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8102999999999997e-05 2023-09-23T18:03:05.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032984+0000", 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115351 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:301 6.6 6:6e0688bd:::benchmark_data_smithi055_275963_object300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.056725+0000", 2023-09-23T18:03:05.324 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.148412196, 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97644475600000002, 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 301 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056725+0000", 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056725+0000", 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056724+0000", 2023-09-23T18:03:05.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056770+0000", 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6489000000000002e-05 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056786+0000", 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5571000000000001e-05 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056792+0000", 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4160000000000001e-06 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056850+0000", 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7812000000000001e-05 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056862+0000", 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1591e-05 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025643+0000", 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96878142499999997 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025748+0000", 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010444 2023-09-23T18:03:05.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025882+0000", 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013407499999999999 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.029881+0000", 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039991810000000001 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032758+0000", 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002876773 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033018+0000", 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000260012 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033063+0000", 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4845000000000002e-05 2023-09-23T18:03:05.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033170+0000", 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010732500000000001 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:334 6.6 6:60a7b3fd:::benchmark_data_smithi055_275963_object333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.065215+0000", 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.139922481, 2023-09-23T18:03:05.332 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96813921400000003, 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 334 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065215+0000", 2023-09-23T18:03:05.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065215+0000", 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065214+0000", 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:03:05.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065277+0000", 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3297000000000001e-05 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065289+0000", 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1198e-05 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.336 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.336 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065293+0000", 2023-09-23T18:03:05.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6489999999999999e-06 2023-09-23T18:03:05.336 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.336 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.336 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.336 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065343+0000", 2023-09-23T18:03:05.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9872e-05 2023-09-23T18:03:05.336 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.336 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065351+0000", 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4460000000000004e-06 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.026148+0000", 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96079755200000005 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.026296+0000", 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014780800000000001 2023-09-23T18:03:05.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.026438+0000", 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014171600000000001 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.029888+0000", 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034506150000000002 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032789+0000", 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002900953 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033203+0000", 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00041394499999999998 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033246+0000", 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2812000000000003e-05 2023-09-23T18:03:05.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033354+0000", 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010816500000000001 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:359 6.6 6:65e2c22f:::benchmark_data_smithi055_275963_object358:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.072910+0000", 2023-09-23T18:03:05.340 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 100.132227239, 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96068453799999998, 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 359 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.341 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.072910+0000", 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.072910+0000", 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.072909+0000", 2023-09-23T18:03:05.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.072984+0000", 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4961000000000003e-05 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.072996+0000", 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1915e-05 2023-09-23T18:03:05.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.073001+0000", 2023-09-23T18:03:05.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8210000000000001e-06 2023-09-23T18:03:05.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.073067+0000", 2023-09-23T18:03:05.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6765000000000002e-05 2023-09-23T18:03:05.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.073077+0000", 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4739999999999994e-06 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.026727+0000", 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95364974300000005 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.026838+0000", 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011121299999999999 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.027005+0000", 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016734700000000001 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.029895+0000", 2023-09-23T18:03:05.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028893489999999998 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033387+0000", 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034925830000000001 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033444+0000", 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6764000000000001e-05 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033485+0000", 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1313999999999998e-05 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033595+0000", 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010945100000000001 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33530 6.6 6.d16185c6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.440253+0000", 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 86.764884815000002, 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.003888519, 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33530 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.349 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440253+0000", 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440253+0000", 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440252+0000", 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440284+0000", 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2190000000000002e-05 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440291+0000", 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0439999999999998e-06 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440296+0000", 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2699999999999998e-06 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441796+0000", 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015002279999999999 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441806+0000", 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.825e-06 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.444126+0000", 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0023203279999999 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.444141+0000", 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5159e-05 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.353 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33541 6.6 6.6e312346 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.450570+0000", 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 86.754567394000006, 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99379889600000004, 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:03:05.354 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33541 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450570+0000", 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:03:05.355 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450570+0000", 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450569+0000", 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450621+0000", 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2008000000000001e-05 2023-09-23T18:03:05.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450632+0000", 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1651000000000001e-05 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450638+0000", 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9839999999999998e-06 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.357 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450690+0000", 2023-09-23T18:03:05.358 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1693999999999997e-05 2023-09-23T18:03:05.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.358 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:03:05.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450699+0000", 2023-09-23T18:03:05.358 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8140000000000007e-06 2023-09-23T18:03:05.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.358 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.444300+0000", 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99360119899999999 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.444369+0000", 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9006999999999998e-05 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:03:05.359 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:03:05.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:05.360 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:03:05.361 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:03:05.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:05.493 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:05.538 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:05.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:05.543 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:05.625 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:03:05.644 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:03:05.678 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:05.706 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:03:05.724 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:05.827 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:03:05.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:05.833 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:03:05.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:05.961 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:06.004 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:06.010 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:06.010 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:06.108 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:06.111 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:03:06.168 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:06 smithi055 ceph-mon[179668]: from='client.55267 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:06.168 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:06 smithi055 ceph-mon[179668]: pgmap v589: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 36266/186486 objects degraded (19.447%); 13955/186486 objects misplaced (7.483%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T18:03:06.170 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:06 smithi055 ceph-mon[184174]: from='client.55267 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:06.170 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:06 smithi055 ceph-mon[184174]: pgmap v589: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 36266/186486 objects degraded (19.447%); 13955/186486 objects misplaced (7.483%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T18:03:06.172 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:06.220 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:03:06.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:06.225 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:03:06.326 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:03:06.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:06 smithi099 ceph-mon[167076]: from='client.55267 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:06.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:06 smithi099 ceph-mon[167076]: pgmap v589: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 500 op/s; 36266/186486 objects degraded (19.447%); 13955/186486 objects misplaced (7.483%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T18:03:06.365 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:06.996 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:03:07.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:07.002 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:03:07.054 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:07.103 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:07.133 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:07.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:07.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:07.174 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:07.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:07.279 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:07.302 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:07.349 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:07.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:07.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:07.458 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:03:07.504 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:07.633 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:03:07.657 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:03:07.677 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:03:07.679 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:03:07.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:07.680 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:03:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:03:07.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:07 smithi055 ceph-mon[184174]: pgmap v590: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 36266/185739 objects degraded (19.525%); 13743/185739 objects misplaced (7.399%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:03:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:03:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:07 smithi055 ceph-mon[179668]: pgmap v590: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 36266/185739 objects degraded (19.525%); 13743/185739 objects misplaced (7.399%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:03:07.704 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:03:07.725 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:03:07.752 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:03:07.779 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:03:07.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:07.807 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:07.807 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:03:07.807 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:07 smithi099 ceph-mon[167076]: pgmap v590: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.8 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 36266/185739 objects degraded (19.525%); 13743/185739 objects misplaced (7.399%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:03:07.810 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:07.818 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:03:07.818 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:03:07.818 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:03:07.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:03:07.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:03:07.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:03:07.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:03:07.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:03:07.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:03:07.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:07.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:07.847 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:07.948 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:03:08.017 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:08.241 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:03:08.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:08.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:03:08.348 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:03:08.397 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:08.504 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:03:08.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:08.510 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:03:08.515 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1281994408' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:08.515 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 36266/185739 objects degraded (19.525%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:03:08.516 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1281994408' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:08.516 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 36266/185739 objects degraded (19.525%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:03:08.540 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:03:08.540 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 36266/185739 objects degraded (19.525%), 12 pgs degraded, 1 pg undersized","count":13},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":678,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":256,"num_osds":8,"num_up_osds":8,"osd_up_since":1695492137,"num_in_osds":7,"osd_in_since":1695492120,"num_remapped_pgs":13},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":106},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+undersized+degraded+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+recovery_wait+degraded+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":61913,"data_bytes":4060297734,"bytes_used":13639610368,"bytes_avail":658325639168,"bytes_total":671965249536,"degraded_objects":36266,"degraded_total":185739,"degraded_ratio":0.19525247793947423,"misplaced_objects":13743,"misplaced_total":185739,"misplaced_ratio":0.073990922746434512,"recovering_objects_per_sec":72,"recovering_bytes_per_sec":4743215,"recovering_keys_per_sec":0,"num_objects_recovered":724,"num_bytes_recovered":47448064,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":456},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":22,"modified":"2023-09-23T18:02:24.658770+0000","services":{"osd":{"daemons":{"summary":"","3":{"start_epoch":0,"start_stamp":"0.000000","gid":0,"addr":"(unrecognized address family 0)/0","metadata":{},"task_status":{}}}}}},"progress_events":{"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"e54c9148-1053-4ff0-9024-eb4758d365b5":{"message":"Global Recovery Event (7m)\n [========================....] (remaining: 51s)","progress":0.89075630903244019,"add_to_ceph_s":true}}} 2023-09-23T18:03:08.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:08.652 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:08.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:08.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1281994408' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:08.696 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 36266/185739 objects degraded (19.525%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:03:08.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:08.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:08.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:08.829 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:08.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:08.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:08.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:08.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:09.010 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:09.048 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:09.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:09.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:09.153 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:09.184 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:09.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:09.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:09.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:09.328 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:03:09.328 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:03:09.331 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:09.389 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:09.417 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:03:09.417 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:03:09.529 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:09.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:09.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:09.641 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:03:09.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1259301797' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:03:09.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:09 smithi055 ceph-mon[184174]: pgmap v591: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 36034/182289 objects degraded (19.768%); 13611/182289 objects misplaced (7.467%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:03:09.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1259301797' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:03:09.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:09 smithi055 ceph-mon[179668]: pgmap v591: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 36034/182289 objects degraded (19.768%); 13611/182289 objects misplaced (7.467%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:03:09.721 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1259301797' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:03:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:09 smithi099 ceph-mon[167076]: pgmap v591: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 430 op/s; 36034/182289 objects degraded (19.768%); 13611/182289 objects misplaced (7.467%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:03:09.938 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:03:09.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:09.948 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:03:10.006 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:03:10.050 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:10.177 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:10.247 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:10.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:10.256 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:10.356 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:10.423 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:10.434 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:10.498 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:03:10.525 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:03:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:03:10] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-09-23T18:03:10.525 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4156590877' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:03:10.525 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:10 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:03:10.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4156590877' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:03:10.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:10 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:03:10.731 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:03:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4156590877' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:03:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:10 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:03:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:11 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:03:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:11 smithi099 ceph-mon[167076]: failed to encode map e257 with expected crc 2023-09-23T18:03:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:11 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:03:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:11 smithi099 ceph-mon[167076]: osdmap e257: 8 total, 8 up, 7 in 2023-09-23T18:03:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:11 smithi099 ceph-mon[167076]: failed to encode map e257 with expected crc 2023-09-23T18:03:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:11 smithi099 ceph-mon[167076]: pgmap v593: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 35634/180048 objects degraded (19.791%); 13357/180048 objects misplaced (7.419%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T18:03:11.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:03:11.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[184174]: failed to encode map e257 with expected crc 2023-09-23T18:03:11.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:03:11.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[184174]: osdmap e257: 8 total, 8 up, 7 in 2023-09-23T18:03:11.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[184174]: failed to encode map e257 with expected crc 2023-09-23T18:03:11.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[184174]: pgmap v593: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 35634/180048 objects degraded (19.791%); 13357/180048 objects misplaced (7.419%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T18:03:11.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:03:11.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[179668]: failed to encode map e257 with expected crc 2023-09-23T18:03:11.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:03:11.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[179668]: osdmap e257: 8 total, 8 up, 7 in 2023-09-23T18:03:11.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[179668]: failed to encode map e257 with expected crc 2023-09-23T18:03:11.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:11 smithi055 ceph-mon[179668]: pgmap v593: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.7 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 422 op/s; 35634/180048 objects degraded (19.791%); 13357/180048 objects misplaced (7.419%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T18:03:12.008 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:12.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:12.012 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:12.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:12.145 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:12.187 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:12.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:12.191 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:12.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:12.328 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:12.377 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:12.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:12.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:12.482 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:03:12.525 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:13.016 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:03:13.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:13.021 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:03:13.077 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 18:03:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=sqlstore.transactions t=2023-09-23T18:03:12.784630067Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-09-23T18:03:13.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:13.160 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:13.206 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:13.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:13.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:13.311 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:13.343 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:13.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:13.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:13.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:13.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:13.537 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:13.577 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:13.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:13.581 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:13.681 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:13.728 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:13 smithi099 ceph-mon[167076]: pgmap v594: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 35296/177546 objects degraded (19.880%); 13168/177546 objects misplaced (7.417%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T18:03:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 35296/177546 objects degraded (19.880%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:03:13.939 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:13 smithi055 ceph-mon[179668]: pgmap v594: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 35296/177546 objects degraded (19.880%); 13168/177546 objects misplaced (7.417%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T18:03:13.939 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 35296/177546 objects degraded (19.880%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:03:13.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:13 smithi055 ceph-mon[184174]: pgmap v594: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.6 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 365 op/s; 35296/177546 objects degraded (19.880%); 13168/177546 objects misplaced (7.417%); 5.1 MiB/s, 82 objects/s recovering 2023-09-23T18:03:13.940 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 35296/177546 objects degraded (19.880%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:03:13.980 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:13.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:13.987 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:13.987 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:13.990 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:14.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:14.143 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:14.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:14.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:14.195 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:14.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:14.335 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:14.382 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:14.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:14.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:14.487 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:14.544 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:14.757 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:14.761 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:14.762 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:14.863 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:03:14.897 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:15.012 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:03:15.260 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:03:15.274 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:03:15.380 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:03:15.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:15.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:03:15.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:15.517 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:15.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:15.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:15.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:15.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:15.706 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:15.752 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:15.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:15.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:15.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:15 smithi099 ceph-mon[167076]: pgmap v595: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 444 op/s; 34917/173148 objects degraded (20.166%); 13162/173148 objects misplaced (7.602%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:03:15.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:15 smithi099 ceph-mon[167076]: from='client.35568 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:15.856 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:03:15.888 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:15 smithi055 ceph-mon[179668]: pgmap v595: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 444 op/s; 34917/173148 objects degraded (20.166%); 13162/173148 objects misplaced (7.602%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:03:15.889 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:15 smithi055 ceph-mon[179668]: from='client.35568 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:15.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:15 smithi055 ceph-mon[184174]: pgmap v595: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 444 op/s; 34917/173148 objects degraded (20.166%); 13162/173148 objects misplaced (7.602%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:03:15.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:15 smithi055 ceph-mon[184174]: from='client.35568 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:15.893 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:15.936 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:03:15.936 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:03:15.947 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":595,"stamp":"2023-09-23T18:03:14.677919+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3785243142,"num_objects":57716,"num_object_clones":0,"num_object_copies":173148,"num_objects_missing_on_primary":623,"num_objects_missing":623,"num_objects_degraded":34917,"num_objects_misplaced":13162,"num_objects_unfound":0,"num_objects_dirty":57716,"num_whiteouts":0,"num_read":6284,"num_read_kb":12770,"num_write":89652,"num_write_kb":4695012,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15399,"num_bytes_recovered":1004865991,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84441,"ondisk_log_size":84441,"up":357,"acting":345,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":404,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":12586204,"kb_used_data":11135076,"kb_used_omap":43,"kb_used_meta":1450900,"kb_avail":643629860,"statfs":{"total":671965249536,"available":659076976640,"internally_reserved":0,"allocated":11402317824,"data_stored":11394352832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1485722319},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-291373056,"num_objects":-4446,"num_object_clones":0,"num_object_copies":-13338,"num_objects_missing_on_primary":-6,"num_objects_missing":-6,"num_objects_degraded":-1349,"num_objects_misplaced":-793,"num_objects_unfound":0,"num_objects_dirty":-4446,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4446,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":785,"num_bytes_recovered":51445760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004919"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":585,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:09.581161+0000","last_change":"2023-09-23T18:02:48.964317+0000","last_active":"2023-09-23T18:03:09.581161+0000","last_peered":"2023-09-23T18:03:09.581161+0000","last_clean":"2023-09-23T18:02:45.962259+0000","last_became_active":"2023-09-23T18:02:48.961702+0000","last_became_peered":"2023-09-23T18:02:48.961702+0000","last_unstale":"2023-09-23T18:03:09.581161+0000","last_undegraded":"2023-09-23T18:02:48.941433+0000","last_fullsized":"2023-09-23T18:02:48.940979+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":474537,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":57,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.169398+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:03:03.169398+0000","last_peered":"2023-09-23T18:03:03.169398+0000","last_clean":"2023-09-23T18:03:03.169398+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:03:03.169398+0000","last_undegraded":"2023-09-23T18:03:03.169398+0000","last_fullsized":"2023-09-23T18:03:03.169398+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":138,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.175294+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:03:03.175294+0000","last_peered":"2023-09-23T18:03:03.175294+0000","last_clean":"2023-09-23T18:03:03.175294+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:03:03.175294+0000","last_undegraded":"2023-09-23T18:03:03.175294+0000","last_fullsized":"2023-09-23T18:03:03.175294+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":599,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163531+0000","last_change":"2023-09-23T18:02:51.046916+0000","last_active":"2023-09-23T18:03:03.163531+0000","last_peered":"2023-09-23T18:03:03.163531+0000","last_clean":"2023-09-23T18:03:03.163531+0000","last_became_active":"2023-09-23T18:02:51.046591+0000","last_became_peered":"2023-09-23T18:02:51.046591+0000","last_unstale":"2023-09-23T18:03:03.163531+0000","last_undegraded":"2023-09-23T18:03:03.163531+0000","last_fullsized":"2023-09-23T18:03:03.163531+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:19.805099+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":106,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.175129+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:03:03.175129+0000","last_peered":"2023-09-23T18:03:03.175129+0000","last_clean":"2023-09-23T18:03:03.175129+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:03:03.175129+0000","last_undegraded":"2023-09-23T18:03:03.175129+0000","last_fullsized":"2023-09-23T18:03:03.175129+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":257,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.241974+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:03:03.241974+0000","last_peered":"2023-09-23T18:03:03.241974+0000","last_clean":"2023-09-23T18:03:03.241974+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:03:03.241974+0000","last_undegraded":"2023-09-23T18:03:03.241974+0000","last_fullsized":"2023-09-23T18:03:03.241974+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":591,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163380+0000","last_change":"2023-09-23T18:02:51.051617+0000","last_active":"2023-09-23T18:03:03.163380+0000","last_peered":"2023-09-23T18:03:03.163380+0000","last_clean":"2023-09-23T18:03:03.163380+0000","last_became_active":"2023-09-23T18:02:51.048947+0000","last_became_peered":"2023-09-23T18:02:51.048947+0000","last_unstale":"2023-09-23T18:03:03.163380+0000","last_undegraded":"2023-09-23T18:03:03.163380+0000","last_fullsized":"2023-09-23T18:03:03.163380+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:49:34.962249+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":119,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.175733+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:03:03.175733+0000","last_peered":"2023-09-23T18:03:03.175733+0000","last_clean":"2023-09-23T18:03:03.175733+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:03:03.175733+0000","last_undegraded":"2023-09-23T18:03:03.175733+0000","last_fullsized":"2023-09-23T18:03:03.175733+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.131737+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:03:03.131737+0000","last_peered":"2023-09-23T18:03:03.131737+0000","last_clean":"2023-09-23T18:03:03.131737+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:03:03.131737+0000","last_undegraded":"2023-09-23T18:03:03.131737+0000","last_fullsized":"2023-09-23T18:03:03.131737+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":588,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.164104+0000","last_change":"2023-09-23T18:02:51.052123+0000","last_active":"2023-09-23T18:03:03.164104+0000","last_peered":"2023-09-23T18:03:03.164104+0000","last_clean":"2023-09-23T18:03:03.164104+0000","last_became_active":"2023-09-23T18:02:51.051772+0000","last_became_peered":"2023-09-23T18:02:51.051772+0000","last_unstale":"2023-09-23T18:03:03.164104+0000","last_undegraded":"2023-09-23T18:03:03.164104+0000","last_fullsized":"2023-09-23T18:03:03.164104+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":237,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.241441+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:03:03.241441+0000","last_peered":"2023-09-23T18:03:03.241441+0000","last_clean":"2023-09-23T18:03:03.241441+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:03:03.241441+0000","last_undegraded":"2023-09-23T18:03:03.241441+0000","last_fullsized":"2023-09-23T18:03:03.241441+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":113,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.174787+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:03:03.174787+0000","last_peered":"2023-09-23T18:03:03.174787+0000","last_clean":"2023-09-23T18:03:03.174787+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:03:03.174787+0000","last_undegraded":"2023-09-23T18:03:03.174787+0000","last_fullsized":"2023-09-23T18:03:03.174787+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":583,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163524+0000","last_change":"2023-09-23T18:02:51.054208+0000","last_active":"2023-09-23T18:03:03.163524+0000","last_peered":"2023-09-23T18:03:03.163524+0000","last_clean":"2023-09-23T18:03:03.163524+0000","last_became_active":"2023-09-23T18:02:51.053676+0000","last_became_peered":"2023-09-23T18:02:51.053676+0000","last_unstale":"2023-09-23T18:03:03.163524+0000","last_undegraded":"2023-09-23T18:03:03.163524+0000","last_fullsized":"2023-09-23T18:03:03.163524+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":239,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.176984+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:03:03.176984+0000","last_peered":"2023-09-23T18:03:03.176984+0000","last_clean":"2023-09-23T18:03:03.176984+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:03:03.176984+0000","last_undegraded":"2023-09-23T18:03:03.176984+0000","last_fullsized":"2023-09-23T18:03:03.176984+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":67,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.174991+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:03:03.174991+0000","last_peered":"2023-09-23T18:03:03.174991+0000","last_clean":"2023-09-23T18:03:03.174991+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:03:03.174991+0000","last_undegraded":"2023-09-23T18:03:03.174991+0000","last_fullsized":"2023-09-23T18:03:03.174991+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":588,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.164279+0000","last_change":"2023-09-23T18:02:51.058624+0000","last_active":"2023-09-23T18:03:03.164279+0000","last_peered":"2023-09-23T18:03:03.164279+0000","last_clean":"2023-09-23T18:03:03.164279+0000","last_became_active":"2023-09-23T18:02:51.057352+0000","last_became_peered":"2023-09-23T18:02:51.057352+0000","last_unstale":"2023-09-23T18:03:03.164279+0000","last_undegraded":"2023-09-23T18:03:03.164279+0000","last_fullsized":"2023-09-23T18:03:03.164279+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":23,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.164025+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:03:03.164025+0000","last_peered":"2023-09-23T18:03:03.164025+0000","last_clean":"2023-09-23T18:03:03.164025+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:03:03.164025+0000","last_undegraded":"2023-09-23T18:03:03.164025+0000","last_fullsized":"2023-09-23T18:03:03.164025+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":153,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.131218+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:03:03.131218+0000","last_peered":"2023-09-23T18:03:03.131218+0000","last_clean":"2023-09-23T18:03:03.131218+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:03:03.131218+0000","last_undegraded":"2023-09-23T18:03:03.131218+0000","last_fullsized":"2023-09-23T18:03:03.131218+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":588,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163746+0000","last_change":"2023-09-23T18:02:51.059983+0000","last_active":"2023-09-23T18:03:03.163746+0000","last_peered":"2023-09-23T18:03:03.163746+0000","last_clean":"2023-09-23T18:03:03.163746+0000","last_became_active":"2023-09-23T18:02:51.057768+0000","last_became_peered":"2023-09-23T18:02:51.057768+0000","last_unstale":"2023-09-23T18:03:03.163746+0000","last_undegraded":"2023-09-23T18:03:03.163746+0000","last_fullsized":"2023-09-23T18:03:03.163746+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":57,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.170078+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:03:03.170078+0000","last_peered":"2023-09-23T18:03:03.170078+0000","last_clean":"2023-09-23T18:03:03.170078+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:03:03.170078+0000","last_undegraded":"2023-09-23T18:03:03.170078+0000","last_fullsized":"2023-09-23T18:03:03.170078+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":57,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.169518+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:03:03.169518+0000","last_peered":"2023-09-23T18:03:03.169518+0000","last_clean":"2023-09-23T18:03:03.169518+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:03:03.169518+0000","last_undegraded":"2023-09-23T18:03:03.169518+0000","last_fullsized":"2023-09-23T18:03:03.169518+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":578,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163057+0000","last_change":"2023-09-23T18:02:51.047570+0000","last_active":"2023-09-23T18:03:03.163057+0000","last_peered":"2023-09-23T18:03:03.163057+0000","last_clean":"2023-09-23T18:03:03.163057+0000","last_became_active":"2023-09-23T18:02:51.046963+0000","last_became_peered":"2023-09-23T18:02:51.046963+0000","last_unstale":"2023-09-23T18:03:03.163057+0000","last_undegraded":"2023-09-23T18:03:03.163057+0000","last_fullsized":"2023-09-23T18:03:03.163057+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":186,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163821+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:03:03.163821+0000","last_peered":"2023-09-23T18:03:03.163821+0000","last_clean":"2023-09-23T18:03:03.163821+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:03:03.163821+0000","last_undegraded":"2023-09-23T18:03:03.163821+0000","last_fullsized":"2023-09-23T18:03:03.163821+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":101,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.175977+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:03:03.175977+0000","last_peered":"2023-09-23T18:03:03.175977+0000","last_clean":"2023-09-23T18:03:03.175977+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:03:03.175977+0000","last_undegraded":"2023-09-23T18:03:03.175977+0000","last_fullsized":"2023-09-23T18:03:03.175977+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":591,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.162900+0000","last_change":"2023-09-23T18:02:51.053806+0000","last_active":"2023-09-23T18:03:03.162900+0000","last_peered":"2023-09-23T18:03:03.162900+0000","last_clean":"2023-09-23T18:03:03.162900+0000","last_became_active":"2023-09-23T18:02:51.048822+0000","last_became_peered":"2023-09-23T18:02:51.048822+0000","last_unstale":"2023-09-23T18:03:03.162900+0000","last_undegraded":"2023-09-23T18:03:03.162900+0000","last_fullsized":"2023-09-23T18:03:03.162900+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:34.401104+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":256,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.240597+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:03:03.240597+0000","last_peered":"2023-09-23T18:03:03.240597+0000","last_clean":"2023-09-23T18:03:03.240597+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:03:03.240597+0000","last_undegraded":"2023-09-23T18:03:03.240597+0000","last_fullsized":"2023-09-23T18:03:03.240597+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":234,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.176954+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:03:03.176954+0000","last_peered":"2023-09-23T18:03:03.176954+0000","last_clean":"2023-09-23T18:03:03.176954+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:03:03.176954+0000","last_undegraded":"2023-09-23T18:03:03.176954+0000","last_fullsized":"2023-09-23T18:03:03.176954+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":579,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163714+0000","last_change":"2023-09-23T18:02:51.046646+0000","last_active":"2023-09-23T18:03:03.163714+0000","last_peered":"2023-09-23T18:03:03.163714+0000","last_clean":"2023-09-23T18:03:03.163714+0000","last_became_active":"2023-09-23T18:02:51.046134+0000","last_became_peered":"2023-09-23T18:02:51.046134+0000","last_unstale":"2023-09-23T18:03:03.163714+0000","last_undegraded":"2023-09-23T18:03:03.163714+0000","last_fullsized":"2023-09-23T18:03:03.163714+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":47,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.169299+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:03:03.169299+0000","last_peered":"2023-09-23T18:03:03.169299+0000","last_clean":"2023-09-23T18:03:03.169299+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:03:03.169299+0000","last_undegraded":"2023-09-23T18:03:03.169299+0000","last_fullsized":"2023-09-23T18:03:03.169299+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":247,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.242014+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:03:03.242014+0000","last_peered":"2023-09-23T18:03:03.242014+0000","last_clean":"2023-09-23T18:03:03.242014+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:03:03.242014+0000","last_undegraded":"2023-09-23T18:03:03.242014+0000","last_fullsized":"2023-09-23T18:03:03.242014+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":591,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163228+0000","last_change":"2023-09-23T18:02:51.051817+0000","last_active":"2023-09-23T18:03:03.163228+0000","last_peered":"2023-09-23T18:03:03.163228+0000","last_clean":"2023-09-23T18:03:03.163228+0000","last_became_active":"2023-09-23T18:02:51.048945+0000","last_became_peered":"2023-09-23T18:02:51.048945+0000","last_unstale":"2023-09-23T18:03:03.163228+0000","last_undegraded":"2023-09-23T18:03:03.163228+0000","last_fullsized":"2023-09-23T18:03:03.163228+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:47.163864+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":193,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.130803+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:03:03.130803+0000","last_peered":"2023-09-23T18:03:03.130803+0000","last_clean":"2023-09-23T18:03:03.130803+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:03:03.130803+0000","last_undegraded":"2023-09-23T18:03:03.130803+0000","last_fullsized":"2023-09-23T18:03:03.130803+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":110,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.175258+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:03:03.175258+0000","last_peered":"2023-09-23T18:03:03.175258+0000","last_clean":"2023-09-23T18:03:03.175258+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:03:03.175258+0000","last_undegraded":"2023-09-23T18:03:03.175258+0000","last_fullsized":"2023-09-23T18:03:03.175258+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"257'5619","reported_seq":5679,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:12.494805+0000","last_change":"2023-09-23T18:00:57.996982+0000","last_active":"2023-09-23T18:03:12.494805+0000","last_peered":"2023-09-23T18:03:12.494805+0000","last_clean":"2023-09-23T18:03:12.494805+0000","last_became_active":"2023-09-23T18:00:57.996589+0000","last_became_peered":"2023-09-23T18:00:57.996589+0000","last_unstale":"2023-09-23T18:03:12.494805+0000","last_undegraded":"2023-09-23T18:03:12.494805+0000","last_fullsized":"2023-09-23T18:03:12.494805+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5619,"log_dups_size":0,"ondisk_log_size":5619,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:13.306089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237436928,"num_objects":3623,"num_object_clones":0,"num_object_copies":10869,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3623,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5619,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":588,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163936+0000","last_change":"2023-09-23T18:02:51.058477+0000","last_active":"2023-09-23T18:03:03.163936+0000","last_peered":"2023-09-23T18:03:03.163936+0000","last_clean":"2023-09-23T18:03:03.163936+0000","last_became_active":"2023-09-23T18:02:51.057258+0000","last_became_peered":"2023-09-23T18:02:51.057258+0000","last_unstale":"2023-09-23T18:03:03.163936+0000","last_undegraded":"2023-09-23T18:03:03.163936+0000","last_fullsized":"2023-09-23T18:03:03.163936+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":186,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.164350+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:03:03.164350+0000","last_peered":"2023-09-23T18:03:03.164350+0000","last_clean":"2023-09-23T18:03:03.164350+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:03:03.164350+0000","last_undegraded":"2023-09-23T18:03:03.164350+0000","last_fullsized":"2023-09-23T18:03:03.164350+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":245,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.176477+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:03:03.176477+0000","last_peered":"2023-09-23T18:03:03.176477+0000","last_clean":"2023-09-23T18:03:03.176477+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:03:03.176477+0000","last_undegraded":"2023-09-23T18:03:03.176477+0000","last_fullsized":"2023-09-23T18:03:03.176477+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"256'5361","reported_seq":12581,"reported_epoch":256,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:03:10.436385+0000","last_change":"2023-09-23T18:02:49.039290+0000","last_active":"2023-09-23T18:03:10.436385+0000","last_peered":"2023-09-23T18:03:10.436385+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:49.001792+0000","last_became_peered":"2023-09-23T18:02:49.001792+0000","last_unstale":"2023-09-23T18:03:10.436385+0000","last_undegraded":"2023-09-23T18:02:48.968288+0000","last_fullsized":"2023-09-23T18:03:10.436385+0000","mapping_epoch":250,"log_start":"217'1230","ondisk_log_start":"217'1230","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4131,"log_dups_size":0,"ondisk_log_size":4131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.156979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":243990528,"num_objects":3723,"num_object_clones":0,"num_object_copies":11169,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":409,"num_objects_misplaced":7037,"num_objects_unfound":0,"num_objects_dirty":3723,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5361,"num_write_kb":290688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3566,"num_bytes_recovered":233701376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[6,1,4],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,4,6","objects":3314},{"shards":"1,6","objects":409}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":593,"reported_epoch":256,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:03.241291+0000","last_change":"2023-09-23T18:02:48.968551+0000","last_active":"2023-09-23T18:03:03.241291+0000","last_peered":"2023-09-23T18:03:03.241291+0000","last_clean":"2023-09-23T18:02:45.965241+0000","last_became_active":"2023-09-23T18:02:48.968221+0000","last_became_peered":"2023-09-23T18:02:48.968221+0000","last_unstale":"2023-09-23T18:03:03.241291+0000","last_undegraded":"2023-09-23T18:02:48.944021+0000","last_fullsized":"2023-09-23T18:02:48.942326+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":409128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":201,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.164114+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:03:03.164114+0000","last_peered":"2023-09-23T18:03:03.164114+0000","last_clean":"2023-09-23T18:03:03.164114+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:03:03.164114+0000","last_undegraded":"2023-09-23T18:03:03.164114+0000","last_fullsized":"2023-09-23T18:03:03.164114+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":267,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.169467+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:03:03.169467+0000","last_peered":"2023-09-23T18:03:03.169467+0000","last_clean":"2023-09-23T18:03:03.169467+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:03:03.169467+0000","last_undegraded":"2023-09-23T18:03:03.169467+0000","last_fullsized":"2023-09-23T18:03:03.169467+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"257'5650","reported_seq":9580,"reported_epoch":257,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:14.286243+0000","last_change":"2023-09-23T18:02:47.987792+0000","last_active":"2023-09-23T18:03:14.286243+0000","last_peered":"2023-09-23T18:03:14.286243+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.307615+0000","last_became_peered":"2023-09-23T18:02:19.307615+0000","last_unstale":"2023-09-23T18:03:14.286243+0000","last_undegraded":"2023-09-23T18:02:18.966041+0000","last_fullsized":"2023-09-23T18:02:18.929348+0000","mapping_epoch":238,"log_start":"217'900","ondisk_log_start":"217'900","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4750,"log_dups_size":0,"ondisk_log_size":4750,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:04.498873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235405312,"num_objects":3592,"num_object_clones":0,"num_object_copies":10776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2330,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5650,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1928,"num_bytes_recovered":126353408,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":1262}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":579,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163846+0000","last_change":"2023-09-23T18:02:51.058795+0000","last_active":"2023-09-23T18:03:03.163846+0000","last_peered":"2023-09-23T18:03:03.163846+0000","last_clean":"2023-09-23T18:03:03.163846+0000","last_became_active":"2023-09-23T18:02:51.057799+0000","last_became_peered":"2023-09-23T18:02:51.057799+0000","last_unstale":"2023-09-23T18:03:03.163846+0000","last_undegraded":"2023-09-23T18:03:03.163846+0000","last_fullsized":"2023-09-23T18:03:03.163846+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":124,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.164790+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:03:03.164790+0000","last_peered":"2023-09-23T18:03:03.164790+0000","last_clean":"2023-09-23T18:03:03.164790+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:03:03.164790+0000","last_undegraded":"2023-09-23T18:03:03.164790+0000","last_fullsized":"2023-09-23T18:03:03.164790+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":222,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.131126+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:03:03.131126+0000","last_peered":"2023-09-23T18:03:03.131126+0000","last_clean":"2023-09-23T18:03:03.131126+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:03:03.131126+0000","last_undegraded":"2023-09-23T18:03:03.131126+0000","last_fullsized":"2023-09-23T18:03:03.131126+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"257'5694","reported_seq":5774,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:14.514352+0000","last_change":"2023-09-23T18:01:25.019784+0000","last_active":"2023-09-23T18:03:14.514352+0000","last_peered":"2023-09-23T18:03:14.514352+0000","last_clean":"2023-09-23T18:03:14.514352+0000","last_became_active":"2023-09-23T18:01:25.002073+0000","last_became_peered":"2023-09-23T18:01:25.002073+0000","last_unstale":"2023-09-23T18:03:14.514352+0000","last_undegraded":"2023-09-23T18:03:14.514352+0000","last_fullsized":"2023-09-23T18:03:14.514352+0000","mapping_epoch":214,"log_start":"217'1300","ondisk_log_start":"217'1300","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4394,"log_dups_size":0,"ondisk_log_size":4394,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:58.981672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236453888,"num_objects":3608,"num_object_clones":0,"num_object_copies":10824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5694,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":588,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163869+0000","last_change":"2023-09-23T18:02:51.057506+0000","last_active":"2023-09-23T18:03:03.163869+0000","last_peered":"2023-09-23T18:03:03.163869+0000","last_clean":"2023-09-23T18:03:03.163869+0000","last_became_active":"2023-09-23T18:02:51.056589+0000","last_became_peered":"2023-09-23T18:02:51.056589+0000","last_unstale":"2023-09-23T18:03:03.163869+0000","last_undegraded":"2023-09-23T18:03:03.163869+0000","last_fullsized":"2023-09-23T18:03:03.163869+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":233,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.176999+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:03:03.176999+0000","last_peered":"2023-09-23T18:03:03.176999+0000","last_clean":"2023-09-23T18:03:03.176999+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:03:03.176999+0000","last_undegraded":"2023-09-23T18:03:03.176999+0000","last_fullsized":"2023-09-23T18:03:03.176999+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":259,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.240891+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:03:03.240891+0000","last_peered":"2023-09-23T18:03:03.240891+0000","last_clean":"2023-09-23T18:03:03.240891+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:03:03.240891+0000","last_undegraded":"2023-09-23T18:03:03.240891+0000","last_fullsized":"2023-09-23T18:03:03.240891+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"257'5643","reported_seq":8793,"reported_epoch":257,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:03:12.899546+0000","last_change":"2023-09-23T18:02:45.688221+0000","last_active":"2023-09-23T18:03:12.899546+0000","last_peered":"2023-09-23T18:03:12.899546+0000","last_clean":"2023-09-23T18:01:38.416095+0000","last_became_active":"2023-09-23T18:02:07.547135+0000","last_became_peered":"2023-09-23T18:02:07.547135+0000","last_unstale":"2023-09-23T18:03:12.899546+0000","last_undegraded":"2023-09-23T18:03:12.899546+0000","last_fullsized":"2023-09-23T18:02:02.784733+0000","mapping_epoch":227,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5543,"log_dups_size":0,"ondisk_log_size":5543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:27.864301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":236650496,"num_objects":3611,"num_object_clones":0,"num_object_copies":10833,"num_objects_missing_on_primary":623,"num_objects_missing":623,"num_objects_degraded":0,"num_objects_misplaced":2562,"num_objects_unfound":0,"num_objects_dirty":3611,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5643,"num_write_kb":296128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1538,"num_bytes_recovered":100794368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1939},{"shards":"1,2,5","objects":623},{"shards":"2,4","objects":1049}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":580,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.162977+0000","last_change":"2023-09-23T18:02:51.047609+0000","last_active":"2023-09-23T18:03:03.162977+0000","last_peered":"2023-09-23T18:03:03.162977+0000","last_clean":"2023-09-23T18:03:03.162977+0000","last_became_active":"2023-09-23T18:02:51.047209+0000","last_became_peered":"2023-09-23T18:02:51.047209+0000","last_unstale":"2023-09-23T18:03:03.162977+0000","last_undegraded":"2023-09-23T18:03:03.162977+0000","last_fullsized":"2023-09-23T18:03:03.162977+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":144,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.175282+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:03:03.175282+0000","last_peered":"2023-09-23T18:03:03.175282+0000","last_clean":"2023-09-23T18:03:03.175282+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:03:03.175282+0000","last_undegraded":"2023-09-23T18:03:03.175282+0000","last_fullsized":"2023-09-23T18:03:03.175282+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":265,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.170068+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:03:03.170068+0000","last_peered":"2023-09-23T18:03:03.170068+0000","last_clean":"2023-09-23T18:03:03.170068+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:03:03.170068+0000","last_undegraded":"2023-09-23T18:03:03.170068+0000","last_fullsized":"2023-09-23T18:03:03.170068+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"257'5522","reported_seq":5739,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:14.293660+0000","last_change":"2023-09-23T18:02:47.961575+0000","last_active":"2023-09-23T18:03:14.293660+0000","last_peered":"2023-09-23T18:03:14.293660+0000","last_clean":"2023-09-23T18:02:17.013863+0000","last_became_active":"2023-09-23T18:02:19.267215+0000","last_became_peered":"2023-09-23T18:02:19.267215+0000","last_unstale":"2023-09-23T18:03:14.293660+0000","last_undegraded":"2023-09-23T18:02:18.955030+0000","last_fullsized":"2023-09-23T18:02:18.924656+0000","mapping_epoch":238,"log_start":"217'800","ondisk_log_start":"217'800","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4722,"log_dups_size":0,"ondisk_log_size":4722,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:46.393021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":231604224,"num_objects":3534,"num_object_clones":0,"num_object_copies":10602,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3534,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5522,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":4128768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-574},{"shards":"2,4,6","objects":3176}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":584,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.162912+0000","last_change":"2023-09-23T18:02:51.049464+0000","last_active":"2023-09-23T18:03:03.162912+0000","last_peered":"2023-09-23T18:03:03.162912+0000","last_clean":"2023-09-23T18:03:03.162912+0000","last_became_active":"2023-09-23T18:02:51.048477+0000","last_became_peered":"2023-09-23T18:02:51.048477+0000","last_unstale":"2023-09-23T18:03:03.162912+0000","last_undegraded":"2023-09-23T18:03:03.162912+0000","last_fullsized":"2023-09-23T18:03:03.162912+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":216,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.131318+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:03:03.131318+0000","last_peered":"2023-09-23T18:03:03.131318+0000","last_clean":"2023-09-23T18:03:03.131318+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:03:03.131318+0000","last_undegraded":"2023-09-23T18:03:03.131318+0000","last_fullsized":"2023-09-23T18:03:03.131318+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":57,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.170370+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:03:03.170370+0000","last_peered":"2023-09-23T18:03:03.170370+0000","last_clean":"2023-09-23T18:03:03.170370+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:03:03.170370+0000","last_undegraded":"2023-09-23T18:03:03.170370+0000","last_fullsized":"2023-09-23T18:03:03.170370+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"257'5516","reported_seq":5581,"reported_epoch":257,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:03:12.489186+0000","last_change":"2023-09-23T18:02:19.054827+0000","last_active":"2023-09-23T18:03:12.489186+0000","last_peered":"2023-09-23T18:03:12.489186+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.048672+0000","last_became_peered":"2023-09-23T18:02:19.048672+0000","last_unstale":"2023-09-23T18:03:12.489186+0000","last_undegraded":"2023-09-23T18:02:18.951263+0000","last_fullsized":"2023-09-23T18:02:18.930851+0000","mapping_epoch":238,"log_start":"216'261","ondisk_log_start":"216'261","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5255,"log_dups_size":0,"ondisk_log_size":5255,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:23.690233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":234356736,"num_objects":3576,"num_object_clones":0,"num_object_copies":10728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3576,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3576,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5516,"num_write_kb":290944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":3576}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":588,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163524+0000","last_change":"2023-09-23T18:02:51.060858+0000","last_active":"2023-09-23T18:03:03.163524+0000","last_peered":"2023-09-23T18:03:03.163524+0000","last_clean":"2023-09-23T18:03:03.163524+0000","last_became_active":"2023-09-23T18:02:51.060554+0000","last_became_peered":"2023-09-23T18:02:51.060554+0000","last_unstale":"2023-09-23T18:03:03.163524+0000","last_undegraded":"2023-09-23T18:03:03.163524+0000","last_fullsized":"2023-09-23T18:03:03.163524+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":47,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.169346+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:03:03.169346+0000","last_peered":"2023-09-23T18:03:03.169346+0000","last_clean":"2023-09-23T18:03:03.169346+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:03:03.169346+0000","last_undegraded":"2023-09-23T18:03:03.169346+0000","last_fullsized":"2023-09-23T18:03:03.169346+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":267,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.186047+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:03:03.186047+0000","last_peered":"2023-09-23T18:03:03.186047+0000","last_clean":"2023-09-23T18:03:03.186047+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:03:03.186047+0000","last_undegraded":"2023-09-23T18:03:03.186047+0000","last_fullsized":"2023-09-23T18:03:03.186047+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"257'5493","reported_seq":13607,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:12.545573+0000","last_change":"2023-09-23T18:02:47.017672+0000","last_active":"2023-09-23T18:03:12.545573+0000","last_peered":"2023-09-23T18:03:12.545573+0000","last_clean":"2023-09-23T18:03:12.545573+0000","last_became_active":"2023-09-23T18:02:47.009245+0000","last_became_peered":"2023-09-23T18:02:47.009245+0000","last_unstale":"2023-09-23T18:03:12.545573+0000","last_undegraded":"2023-09-23T18:03:12.545573+0000","last_fullsized":"2023-09-23T18:03:12.545573+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5493,"log_dups_size":0,"ondisk_log_size":5493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:04.868743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235405336,"num_objects":3593,"num_object_clones":0,"num_object_copies":10779,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3593,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5493,"num_write_kb":290689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4015,"num_bytes_recovered":263127040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":580,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163160+0000","last_change":"2023-09-23T18:02:51.054197+0000","last_active":"2023-09-23T18:03:03.163160+0000","last_peered":"2023-09-23T18:03:03.163160+0000","last_clean":"2023-09-23T18:03:03.163160+0000","last_became_active":"2023-09-23T18:02:51.049362+0000","last_became_peered":"2023-09-23T18:02:51.049362+0000","last_unstale":"2023-09-23T18:03:03.163160+0000","last_undegraded":"2023-09-23T18:03:03.163160+0000","last_fullsized":"2023-09-23T18:03:03.163160+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":102,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.174956+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:03:03.174956+0000","last_peered":"2023-09-23T18:03:03.174956+0000","last_clean":"2023-09-23T18:03:03.174956+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:03:03.174956+0000","last_undegraded":"2023-09-23T18:03:03.174956+0000","last_fullsized":"2023-09-23T18:03:03.174956+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":102,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.175491+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:03:03.175491+0000","last_peered":"2023-09-23T18:03:03.175491+0000","last_clean":"2023-09-23T18:03:03.175491+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:03:03.175491+0000","last_undegraded":"2023-09-23T18:03:03.175491+0000","last_fullsized":"2023-09-23T18:03:03.175491+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"257'5426","reported_seq":5493,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:14.542512+0000","last_change":"2023-09-23T18:02:19.315376+0000","last_active":"2023-09-23T18:03:14.542512+0000","last_peered":"2023-09-23T18:03:14.542512+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.306174+0000","last_became_peered":"2023-09-23T18:02:19.306174+0000","last_unstale":"2023-09-23T18:03:14.542512+0000","last_undegraded":"2023-09-23T18:02:18.954031+0000","last_fullsized":"2023-09-23T18:02:18.922505+0000","mapping_epoch":238,"log_start":"217'1000","ondisk_log_start":"217'1000","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4426,"log_dups_size":0,"ondisk_log_size":4426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:52.392034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226492416,"num_objects":3456,"num_object_clones":0,"num_object_copies":10368,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3456,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5426,"num_write_kb":284224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":-639}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":589,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163880+0000","last_change":"2023-09-23T18:02:51.059934+0000","last_active":"2023-09-23T18:03:03.163880+0000","last_peered":"2023-09-23T18:03:03.163880+0000","last_clean":"2023-09-23T18:03:03.163880+0000","last_became_active":"2023-09-23T18:02:51.059638+0000","last_became_peered":"2023-09-23T18:02:51.059638+0000","last_unstale":"2023-09-23T18:03:03.163880+0000","last_undegraded":"2023-09-23T18:03:03.163880+0000","last_fullsized":"2023-09-23T18:03:03.163880+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":566,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.241161+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:03:03.241161+0000","last_peered":"2023-09-23T18:03:03.241161+0000","last_clean":"2023-09-23T18:03:03.241161+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:03:03.241161+0000","last_undegraded":"2023-09-23T18:03:03.241161+0000","last_fullsized":"2023-09-23T18:03:03.241161+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":237,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.177041+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:03:03.177041+0000","last_peered":"2023-09-23T18:03:03.177041+0000","last_clean":"2023-09-23T18:03:03.177041+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:03:03.177041+0000","last_undegraded":"2023-09-23T18:03:03.177041+0000","last_fullsized":"2023-09-23T18:03:03.177041+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":234,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.176650+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:03:03.176650+0000","last_peered":"2023-09-23T18:03:03.176650+0000","last_clean":"2023-09-23T18:03:03.176650+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:03:03.176650+0000","last_undegraded":"2023-09-23T18:03:03.176650+0000","last_fullsized":"2023-09-23T18:03:03.176650+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"257'5453","reported_seq":5534,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:12.859147+0000","last_change":"2023-09-23T18:02:19.289610+0000","last_active":"2023-09-23T18:03:12.859147+0000","last_peered":"2023-09-23T18:03:12.859147+0000","last_clean":"2023-09-23T18:02:17.060883+0000","last_became_active":"2023-09-23T18:02:19.266879+0000","last_became_peered":"2023-09-23T18:02:19.266879+0000","last_unstale":"2023-09-23T18:03:12.859147+0000","last_undegraded":"2023-09-23T18:02:18.963390+0000","last_fullsized":"2023-09-23T18:02:18.926482+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5453,"log_dups_size":0,"ondisk_log_size":5453,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.499269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":229703680,"num_objects":3505,"num_object_clones":0,"num_object_copies":10515,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3505,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5453,"num_write_kb":286656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":3154},{"shards":"4,7","objects":-623}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":591,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.162889+0000","last_change":"2023-09-23T18:02:51.047767+0000","last_active":"2023-09-23T18:03:03.162889+0000","last_peered":"2023-09-23T18:03:03.162889+0000","last_clean":"2023-09-23T18:03:03.162889+0000","last_became_active":"2023-09-23T18:02:51.047297+0000","last_became_peered":"2023-09-23T18:02:51.047297+0000","last_unstale":"2023-09-23T18:03:03.162889+0000","last_undegraded":"2023-09-23T18:03:03.162889+0000","last_fullsized":"2023-09-23T18:03:03.162889+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.851694+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1185,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.176655+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:03:03.176655+0000","last_peered":"2023-09-23T18:03:03.176655+0000","last_clean":"2023-09-23T18:03:03.176655+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:03:03.176655+0000","last_undegraded":"2023-09-23T18:03:03.176655+0000","last_fullsized":"2023-09-23T18:03:03.176655+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":47,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.169576+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:03:03.169576+0000","last_peered":"2023-09-23T18:03:03.169576+0000","last_clean":"2023-09-23T18:03:03.169576+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:03:03.169576+0000","last_undegraded":"2023-09-23T18:03:03.169576+0000","last_fullsized":"2023-09-23T18:03:03.169576+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":150,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163909+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:03:03.163909+0000","last_peered":"2023-09-23T18:03:03.163909+0000","last_clean":"2023-09-23T18:03:03.163909+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:03:03.163909+0000","last_undegraded":"2023-09-23T18:03:03.163909+0000","last_fullsized":"2023-09-23T18:03:03.163909+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"257'5591","reported_seq":5649,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:12.872625+0000","last_change":"2023-09-23T18:02:19.260251+0000","last_active":"2023-09-23T18:03:12.872625+0000","last_peered":"2023-09-23T18:03:12.872625+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.253817+0000","last_became_peered":"2023-09-23T18:02:19.253817+0000","last_unstale":"2023-09-23T18:03:12.872625+0000","last_undegraded":"2023-09-23T18:02:18.955613+0000","last_fullsized":"2023-09-23T18:02:18.923894+0000","mapping_epoch":238,"log_start":"217'1574","ondisk_log_start":"217'1574","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4017,"log_dups_size":0,"ondisk_log_size":4017,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:18.999679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":240713728,"num_objects":3673,"num_object_clones":0,"num_object_copies":11019,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4277,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3673,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5591,"num_write_kb":296448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":-604}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":592,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163085+0000","last_change":"2023-09-23T18:02:51.054622+0000","last_active":"2023-09-23T18:03:03.163085+0000","last_peered":"2023-09-23T18:03:03.163085+0000","last_clean":"2023-09-23T18:03:03.163085+0000","last_became_active":"2023-09-23T18:02:51.049345+0000","last_became_peered":"2023-09-23T18:02:51.049345+0000","last_unstale":"2023-09-23T18:03:03.163085+0000","last_undegraded":"2023-09-23T18:03:03.163085+0000","last_fullsized":"2023-09-23T18:03:03.163085+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:58.636010+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":539,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.162883+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:03:03.162883+0000","last_peered":"2023-09-23T18:03:03.162883+0000","last_clean":"2023-09-23T18:03:03.162883+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:03:03.162883+0000","last_undegraded":"2023-09-23T18:03:03.162883+0000","last_fullsized":"2023-09-23T18:03:03.162883+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":151,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163919+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:03:03.163919+0000","last_peered":"2023-09-23T18:03:03.163919+0000","last_clean":"2023-09-23T18:03:03.163919+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:03:03.163919+0000","last_undegraded":"2023-09-23T18:03:03.163919+0000","last_fullsized":"2023-09-23T18:03:03.163919+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":224,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.177704+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:03:03.177704+0000","last_peered":"2023-09-23T18:03:03.177704+0000","last_clean":"2023-09-23T18:03:03.177704+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:03:03.177704+0000","last_undegraded":"2023-09-23T18:03:03.177704+0000","last_fullsized":"2023-09-23T18:03:03.177704+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"257'5562","reported_seq":5657,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:12.500718+0000","last_change":"2023-09-23T18:02:47.986654+0000","last_active":"2023-09-23T18:03:12.500718+0000","last_peered":"2023-09-23T18:03:12.500718+0000","last_clean":"2023-09-23T18:02:16.962656+0000","last_became_active":"2023-09-23T18:02:19.272896+0000","last_became_peered":"2023-09-23T18:02:19.272896+0000","last_unstale":"2023-09-23T18:03:12.500718+0000","last_undegraded":"2023-09-23T18:02:18.972349+0000","last_fullsized":"2023-09-23T18:02:18.930868+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5562,"log_dups_size":0,"ondisk_log_size":5562,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:23.745675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":241041408,"num_objects":3678,"num_object_clones":0,"num_object_copies":11034,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3678,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5562,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":3322},{"shards":"4,5","objects":-585}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1169,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.176733+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:03:03.176733+0000","last_peered":"2023-09-23T18:03:03.176733+0000","last_clean":"2023-09-23T18:03:03.176733+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:03:03.176733+0000","last_undegraded":"2023-09-23T18:03:03.176733+0000","last_fullsized":"2023-09-23T18:03:03.176733+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":580,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.162807+0000","last_change":"2023-09-23T18:02:51.054022+0000","last_active":"2023-09-23T18:03:03.162807+0000","last_peered":"2023-09-23T18:03:03.162807+0000","last_clean":"2023-09-23T18:03:03.162807+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:03:03.162807+0000","last_undegraded":"2023-09-23T18:03:03.162807+0000","last_fullsized":"2023-09-23T18:03:03.162807+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":64,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.178262+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:03:03.178262+0000","last_peered":"2023-09-23T18:03:03.178262+0000","last_clean":"2023-09-23T18:03:03.178262+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:03:03.178262+0000","last_undegraded":"2023-09-23T18:03:03.178262+0000","last_fullsized":"2023-09-23T18:03:03.178262+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":190,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.131486+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:03:03.131486+0000","last_peered":"2023-09-23T18:03:03.131486+0000","last_clean":"2023-09-23T18:03:03.131486+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:03:03.131486+0000","last_undegraded":"2023-09-23T18:03:03.131486+0000","last_fullsized":"2023-09-23T18:03:03.131486+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"257'5519","reported_seq":5674,"reported_epoch":257,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:03:12.516376+0000","last_change":"2023-09-23T18:02:18.003772+0000","last_active":"2023-09-23T18:03:12.516376+0000","last_peered":"2023-09-23T18:03:12.516376+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:18.000696+0000","last_became_peered":"2023-09-23T18:02:18.000696+0000","last_unstale":"2023-09-23T18:03:12.516376+0000","last_undegraded":"2023-09-23T18:02:17.938873+0000","last_fullsized":"2023-09-23T18:02:17.919669+0000","mapping_epoch":237,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5419,"log_dups_size":0,"ondisk_log_size":5419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:48.085957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":233504768,"num_objects":3563,"num_object_clones":0,"num_object_copies":10689,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3530,"num_objects_misplaced":3563,"num_objects_unfound":0,"num_objects_dirty":3563,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5519,"num_write_kb":290624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":3563}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":592,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163427+0000","last_change":"2023-09-23T18:02:51.061168+0000","last_active":"2023-09-23T18:03:03.163427+0000","last_peered":"2023-09-23T18:03:03.163427+0000","last_clean":"2023-09-23T18:03:03.163427+0000","last_became_active":"2023-09-23T18:02:51.060689+0000","last_became_peered":"2023-09-23T18:02:51.060689+0000","last_unstale":"2023-09-23T18:03:03.163427+0000","last_undegraded":"2023-09-23T18:03:03.163427+0000","last_fullsized":"2023-09-23T18:03:03.163427+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:29.406460+0000","scrub_duration":0.001568505,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":437,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163058+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:03:03.163058+0000","last_peered":"2023-09-23T18:03:03.163058+0000","last_clean":"2023-09-23T18:03:03.163058+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:03:03.163058+0000","last_undegraded":"2023-09-23T18:03:03.163058+0000","last_fullsized":"2023-09-23T18:03:03.163058+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":255,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.240733+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:03:03.240733+0000","last_peered":"2023-09-23T18:03:03.240733+0000","last_clean":"2023-09-23T18:03:03.240733+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:03:03.240733+0000","last_undegraded":"2023-09-23T18:03:03.240733+0000","last_fullsized":"2023-09-23T18:03:03.240733+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":226,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.241837+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:03:03.241837+0000","last_peered":"2023-09-23T18:03:03.241837+0000","last_clean":"2023-09-23T18:03:03.241837+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:03:03.241837+0000","last_undegraded":"2023-09-23T18:03:03.241837+0000","last_fullsized":"2023-09-23T18:03:03.241837+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"256'5501","reported_seq":13839,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.447099+0000","last_change":"2023-09-23T18:02:18.942886+0000","last_active":"2023-09-23T18:03:10.447099+0000","last_peered":"2023-09-23T18:03:10.447099+0000","last_clean":"2023-09-23T18:03:10.447099+0000","last_became_active":"2023-09-23T18:02:18.936691+0000","last_became_peered":"2023-09-23T18:02:18.936691+0000","last_unstale":"2023-09-23T18:03:10.447099+0000","last_undegraded":"2023-09-23T18:03:10.447099+0000","last_fullsized":"2023-09-23T18:03:10.447099+0000","mapping_epoch":238,"log_start":"217'1358","ondisk_log_start":"217'1358","created":204,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4143,"log_dups_size":0,"ondisk_log_size":4143,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:26.085859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244776960,"num_objects":3735,"num_object_clones":0,"num_object_copies":11205,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3735,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5501,"num_write_kb":295552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4123,"num_bytes_recovered":270204928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":589,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.164144+0000","last_change":"2023-09-23T18:02:51.057370+0000","last_active":"2023-09-23T18:03:03.164144+0000","last_peered":"2023-09-23T18:03:03.164144+0000","last_clean":"2023-09-23T18:03:03.164144+0000","last_became_active":"2023-09-23T18:02:51.056158+0000","last_became_peered":"2023-09-23T18:02:51.056158+0000","last_unstale":"2023-09-23T18:03:03.164144+0000","last_undegraded":"2023-09-23T18:03:03.164144+0000","last_fullsized":"2023-09-23T18:03:03.164144+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":471,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.164701+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:03:03.164701+0000","last_peered":"2023-09-23T18:03:03.164701+0000","last_clean":"2023-09-23T18:03:03.164701+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:03:03.164701+0000","last_undegraded":"2023-09-23T18:03:03.164701+0000","last_fullsized":"2023-09-23T18:03:03.164701+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":191,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163995+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:03:03.163995+0000","last_peered":"2023-09-23T18:03:03.163995+0000","last_clean":"2023-09-23T18:03:03.163995+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:03:03.163995+0000","last_undegraded":"2023-09-23T18:03:03.163995+0000","last_fullsized":"2023-09-23T18:03:03.163995+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":223,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.240966+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:03:03.240966+0000","last_peered":"2023-09-23T18:03:03.240966+0000","last_clean":"2023-09-23T18:03:03.240966+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:03:03.240966+0000","last_undegraded":"2023-09-23T18:03:03.240966+0000","last_fullsized":"2023-09-23T18:03:03.240966+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"257'5487","reported_seq":5573,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:12.902479+0000","last_change":"2023-09-23T18:02:19.279112+0000","last_active":"2023-09-23T18:03:12.902479+0000","last_peered":"2023-09-23T18:03:12.902479+0000","last_clean":"2023-09-23T18:02:17.018900+0000","last_became_active":"2023-09-23T18:02:19.267439+0000","last_became_peered":"2023-09-23T18:02:19.267439+0000","last_unstale":"2023-09-23T18:03:12.902479+0000","last_undegraded":"2023-09-23T18:02:18.959329+0000","last_fullsized":"2023-09-23T18:02:18.926801+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5487,"log_dups_size":0,"ondisk_log_size":5487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.395583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":237699072,"num_objects":3627,"num_object_clones":0,"num_object_copies":10881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5487,"num_write_kb":291648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-572},{"shards":"2,4,6","objects":3269}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":589,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163850+0000","last_change":"2023-09-23T18:02:51.058680+0000","last_active":"2023-09-23T18:03:03.163850+0000","last_peered":"2023-09-23T18:03:03.163850+0000","last_clean":"2023-09-23T18:03:03.163850+0000","last_became_active":"2023-09-23T18:02:51.057706+0000","last_became_peered":"2023-09-23T18:02:51.057706+0000","last_unstale":"2023-09-23T18:03:03.163850+0000","last_undegraded":"2023-09-23T18:03:03.163850+0000","last_fullsized":"2023-09-23T18:03:03.163850+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":102,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.175409+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:03:03.175409+0000","last_peered":"2023-09-23T18:03:03.175409+0000","last_clean":"2023-09-23T18:03:03.175409+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:03:03.175409+0000","last_undegraded":"2023-09-23T18:03:03.175409+0000","last_fullsized":"2023-09-23T18:03:03.175409+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":327,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.131691+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:03:03.131691+0000","last_peered":"2023-09-23T18:03:03.131691+0000","last_clean":"2023-09-23T18:03:03.131691+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:03:03.131691+0000","last_undegraded":"2023-09-23T18:03:03.131691+0000","last_fullsized":"2023-09-23T18:03:03.131691+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":106,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.174872+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:03:03.174872+0000","last_peered":"2023-09-23T18:03:03.174872+0000","last_clean":"2023-09-23T18:03:03.174872+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:03:03.174872+0000","last_undegraded":"2023-09-23T18:03:03.174872+0000","last_fullsized":"2023-09-23T18:03:03.174872+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"257'5546","reported_seq":5609,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:12.896837+0000","last_change":"2023-09-23T18:01:25.046224+0000","last_active":"2023-09-23T18:03:12.896837+0000","last_peered":"2023-09-23T18:03:12.896837+0000","last_clean":"2023-09-23T18:03:12.896837+0000","last_became_active":"2023-09-23T18:01:25.002864+0000","last_became_peered":"2023-09-23T18:01:25.002864+0000","last_unstale":"2023-09-23T18:03:12.896837+0000","last_undegraded":"2023-09-23T18:03:12.896837+0000","last_fullsized":"2023-09-23T18:03:12.896837+0000","mapping_epoch":214,"log_start":"217'1541","ondisk_log_start":"217'1541","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4005,"log_dups_size":0,"ondisk_log_size":4005,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.752359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":235405312,"num_objects":3592,"num_object_clones":0,"num_object_copies":10776,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3592,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5546,"num_write_kb":292416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":596,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163532+0000","last_change":"2023-09-23T18:02:51.053420+0000","last_active":"2023-09-23T18:03:03.163532+0000","last_peered":"2023-09-23T18:03:03.163532+0000","last_clean":"2023-09-23T18:03:03.163532+0000","last_became_active":"2023-09-23T18:02:51.053035+0000","last_became_peered":"2023-09-23T18:02:51.053035+0000","last_unstale":"2023-09-23T18:03:03.163532+0000","last_undegraded":"2023-09-23T18:03:03.163532+0000","last_fullsized":"2023-09-23T18:03:03.163532+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:20.456481+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":462,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.131876+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:03:03.131876+0000","last_peered":"2023-09-23T18:03:03.131876+0000","last_clean":"2023-09-23T18:03:03.131876+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:03:03.131876+0000","last_undegraded":"2023-09-23T18:03:03.131876+0000","last_fullsized":"2023-09-23T18:03:03.131876+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":283,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.241915+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:03:03.241915+0000","last_peered":"2023-09-23T18:03:03.241915+0000","last_clean":"2023-09-23T18:03:03.241915+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:03:03.241915+0000","last_undegraded":"2023-09-23T18:03:03.241915+0000","last_fullsized":"2023-09-23T18:03:03.241915+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":47,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.169252+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:03:03.169252+0000","last_peered":"2023-09-23T18:03:03.169252+0000","last_clean":"2023-09-23T18:03:03.169252+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:03:03.169252+0000","last_undegraded":"2023-09-23T18:03:03.169252+0000","last_fullsized":"2023-09-23T18:03:03.169252+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":579,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163131+0000","last_change":"2023-09-23T18:02:51.046571+0000","last_active":"2023-09-23T18:03:03.163131+0000","last_peered":"2023-09-23T18:03:03.163131+0000","last_clean":"2023-09-23T18:03:03.163131+0000","last_became_active":"2023-09-23T18:02:51.046213+0000","last_became_peered":"2023-09-23T18:02:51.046213+0000","last_unstale":"2023-09-23T18:03:03.163131+0000","last_undegraded":"2023-09-23T18:03:03.163131+0000","last_fullsized":"2023-09-23T18:03:03.163131+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":242,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.240990+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:03:03.240990+0000","last_peered":"2023-09-23T18:03:03.240990+0000","last_clean":"2023-09-23T18:03:03.240990+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:03:03.240990+0000","last_undegraded":"2023-09-23T18:03:03.240990+0000","last_fullsized":"2023-09-23T18:03:03.240990+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":269,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.164068+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:03:03.164068+0000","last_peered":"2023-09-23T18:03:03.164068+0000","last_clean":"2023-09-23T18:03:03.164068+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:03:03.164068+0000","last_undegraded":"2023-09-23T18:03:03.164068+0000","last_fullsized":"2023-09-23T18:03:03.164068+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":588,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163823+0000","last_change":"2023-09-23T18:02:51.060198+0000","last_active":"2023-09-23T18:03:03.163823+0000","last_peered":"2023-09-23T18:03:03.163823+0000","last_clean":"2023-09-23T18:03:03.163823+0000","last_became_active":"2023-09-23T18:02:51.057046+0000","last_became_peered":"2023-09-23T18:02:51.057046+0000","last_unstale":"2023-09-23T18:03:03.163823+0000","last_undegraded":"2023-09-23T18:03:03.163823+0000","last_fullsized":"2023-09-23T18:03:03.163823+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":112,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.176951+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:03:03.176951+0000","last_peered":"2023-09-23T18:03:03.176951+0000","last_clean":"2023-09-23T18:03:03.176951+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:03:03.176951+0000","last_undegraded":"2023-09-23T18:03:03.176951+0000","last_fullsized":"2023-09-23T18:03:03.176951+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":182,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.169667+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:03:03.169667+0000","last_peered":"2023-09-23T18:03:03.169667+0000","last_clean":"2023-09-23T18:03:03.169667+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:03:03.169667+0000","last_undegraded":"2023-09-23T18:03:03.169667+0000","last_fullsized":"2023-09-23T18:03:03.169667+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":98,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.176830+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:03:03.176830+0000","last_peered":"2023-09-23T18:03:03.176830+0000","last_clean":"2023-09-23T18:03:03.176830+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:03:03.176830+0000","last_undegraded":"2023-09-23T18:03:03.176830+0000","last_fullsized":"2023-09-23T18:03:03.176830+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":121,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.178105+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:03:03.178105+0000","last_peered":"2023-09-23T18:03:03.178105+0000","last_clean":"2023-09-23T18:03:03.178105+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:03:03.178105+0000","last_undegraded":"2023-09-23T18:03:03.178105+0000","last_fullsized":"2023-09-23T18:03:03.178105+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":604,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163650+0000","last_change":"2023-09-23T18:02:51.047476+0000","last_active":"2023-09-23T18:03:03.163650+0000","last_peered":"2023-09-23T18:03:03.163650+0000","last_clean":"2023-09-23T18:03:03.163650+0000","last_became_active":"2023-09-23T18:02:51.047152+0000","last_became_peered":"2023-09-23T18:02:51.047152+0000","last_unstale":"2023-09-23T18:03:03.163650+0000","last_undegraded":"2023-09-23T18:03:03.163650+0000","last_fullsized":"2023-09-23T18:03:03.163650+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:22.514077+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":121,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.175553+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:03:03.175553+0000","last_peered":"2023-09-23T18:03:03.175553+0000","last_clean":"2023-09-23T18:03:03.175553+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:03:03.175553+0000","last_undegraded":"2023-09-23T18:03:03.175553+0000","last_fullsized":"2023-09-23T18:03:03.175553+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":124,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.174787+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:03:03.174787+0000","last_peered":"2023-09-23T18:03:03.174787+0000","last_clean":"2023-09-23T18:03:03.174787+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:03:03.174787+0000","last_undegraded":"2023-09-23T18:03:03.174787+0000","last_fullsized":"2023-09-23T18:03:03.174787+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":587,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.163674+0000","last_change":"2023-09-23T18:02:51.058825+0000","last_active":"2023-09-23T18:03:03.163674+0000","last_peered":"2023-09-23T18:03:03.163674+0000","last_clean":"2023-09-23T18:03:03.163674+0000","last_became_active":"2023-09-23T18:02:51.058480+0000","last_became_peered":"2023-09-23T18:02:51.058480+0000","last_unstale":"2023-09-23T18:03:03.163674+0000","last_undegraded":"2023-09-23T18:03:03.163674+0000","last_fullsized":"2023-09-23T18:03:03.163674+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":235,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.241528+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:03:03.241528+0000","last_peered":"2023-09-23T18:03:03.241528+0000","last_clean":"2023-09-23T18:03:03.241528+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:03:03.241528+0000","last_undegraded":"2023-09-23T18:03:03.241528+0000","last_fullsized":"2023-09-23T18:03:03.241528+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":267,"reported_epoch":256,"state":"active+clean","last_fresh":"2023-09-23T18:03:03.169296+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:03:03.169296+0000","last_peered":"2023-09-23T18:03:03.169296+0000","last_clean":"2023-09-23T18:03:03.169296+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:03:03.169296+0000","last_undegraded":"2023-09-23T18:03:03.169296+0000","last_fullsized":"2023-09-23T18:03:03.169296+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3780640792,"num_objects":57689,"num_object_clones":0,"num_object_copies":173067,"num_objects_missing_on_primary":623,"num_objects_missing":623,"num_objects_degraded":34915,"num_objects_misplaced":13162,"num_objects_unfound":0,"num_objects_dirty":57689,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":88583,"num_write_kb":4680641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15299,"num_bytes_recovered":1002635264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11016548352,"data_stored":11016536136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78419,"ondisk_log_size":78419,"up":48,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":402523,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":331,"num_read_kb":623,"num_write":162,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2224465,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":91,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001367,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2205704,"kb_used_data":2117504,"kb_used_omap":13,"kb_used_meta":88178,"kb_avail":91539448,"statfs":{"total":95995035648,"available":93736394752,"internally_reserved":0,"allocated":2168324096,"data_stored":2167340389,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":90294376},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.544,"5min":2.544,"15min":2.544},"last":0.35599999999999998},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":0.373}]},{"osd":1,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.629,"5min":2.629,"15min":2.629},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.5979999999999999,"5min":4.5979999999999999,"15min":4.5979999999999999},"last":0.89800000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.081,"5min":3.081,"15min":3.081},"last":0.51900000000000002},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.6680000000000001,"5min":2.6680000000000001,"15min":2.6680000000000001},"last":0.504}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":3.984,"5min":3.984,"15min":3.984},"last":0.47899999999999998},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.645,"5min":1.645,"15min":1.645},"last":0.48599999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.004,"15min":1.004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.9300000000000002,"5min":3.9300000000000002,"15min":3.9300000000000002},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.133,"5min":1.133,"15min":1.133},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.7080000000000002,"5min":3.7080000000000002,"15min":3.7080000000000002},"last":0.56599999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.0409999999999999,"5min":3.0409999999999999,"15min":3.0409999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.6419999999999999,"5min":3.6419999999999999,"15min":3.6419999999999999},"last":0.58099999999999996}]}]},{"osd":6,"up_from":163,"seq":700079669303,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2375736,"kb_used_data":2237744,"kb_used_omap":7,"kb_used_meta":137976,"kb_avail":91369416,"statfs":{"total":95995035648,"available":93562281984,"internally_reserved":0,"allocated":2291449856,"data_stored":2290474554,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":141288228},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.2270000000000001,"15min":1.3089999999999999},"min":{"1min":0.224,"5min":0.215,"15min":0.215},"max":{"1min":3.5529999999999999,"5min":14.196999999999999,"15min":14.196999999999999},"last":15.407},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.72,"15min":2.3740000000000001},"min":{"1min":0.32600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.8970000000000002,"5min":13.536,"15min":13.536},"last":0.73599999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.5860000000000001,"15min":1.956},"min":{"1min":0.28100000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.4910000000000001,"5min":14.233000000000001,"15min":14.233000000000001},"last":6.7690000000000001},{"interface":"front","average":{"1min":1.01,"5min":1.6759999999999999,"15min":2.173},"min":{"1min":0.33300000000000002,"5min":0.248,"15min":0.248},"max":{"1min":4.9340000000000002,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":14.789}]},{"osd":2,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":2.0009999999999999,"15min":2.3450000000000002},"min":{"1min":0.34999999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.173,"5min":11.210000000000001,"15min":11.210000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.0589999999999999,"5min":2.1899999999999999,"15min":2.758},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.9700000000000002,"5min":11.223000000000001,"15min":11.223000000000001},"last":0.439}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51700000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.96599999999999997,"15min":0.89400000000000002},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.827,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.72499999999999998,"15min":0.503},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.0129999999999999,"5min":8.3000000000000007,"15min":8.3000000000000007},"last":0.375}]},{"osd":5,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.94199999999999995,"15min":0.84399999999999997},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.1050000000000004,"5min":11.869,"15min":11.869},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.21,"5min":0.875,"15min":0.63500000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.7130000000000001,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.6469999999999998,"5min":3.6469999999999998,"15min":3.6469999999999998},"last":0.499},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.5310000000000001,"5min":2.5310000000000001,"15min":2.5310000000000001},"last":0.41299999999999998}]}]},{"osd":1,"up_from":0,"seq":420906795122,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958649,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1310312,"kb_used_data":916768,"kb_used_omap":7,"kb_used_meta":393528,"kb_avail":92434840,"statfs":{"total":95995035648,"available":94653276160,"internally_reserved":0,"allocated":938770432,"data_stored":937470250,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":402973412},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:03:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":2.0059999999999998,"15min":1.135},"min":{"1min":0.20300000000000001,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":7.7000000000000002,"5min":18.154,"15min":18.154},"last":0.23899999999999999},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.871,"15min":1.1080000000000001},"min":{"1min":0.29599999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":6.8550000000000004,"5min":13.369999999999999,"15min":13.369999999999999},"last":0.27000000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.4460000000000002,"15min":1.24},"min":{"1min":0.25700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.5259999999999998,"5min":20.882000000000001,"15min":20.882000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.234,"5min":2.0899999999999999,"15min":1.149},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":6.7439999999999998,"5min":15.064,"15min":15.064},"last":0.34499999999999997}]},{"osd":3,"last update":"Sat Sep 23 17:57:04 2023","interfaces":[{"interface":"back","average":{"1min":1.79,"5min":0.77100000000000002,"15min":0.60099999999999998},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.3449999999999998,"5min":4.3449999999999998,"15min":4.3449999999999998},"last":1.2270000000000001},{"interface":"front","average":{"1min":2.109,"5min":0.73699999999999999,"15min":0.50900000000000001},"min":{"1min":0.28699999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.8729999999999993,"5min":8.8729999999999993,"15min":8.8729999999999993},"last":1.3220000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":1.603,"15min":1.0509999999999999},"min":{"1min":0.29299999999999998,"5min":0.251,"15min":0.251},"max":{"1min":4.54,"5min":15.279999999999999,"15min":15.279999999999999},"last":1.26},{"interface":"front","average":{"1min":1.24,"5min":1.859,"15min":1.1220000000000001},"min":{"1min":0.29899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7190000000000003,"5min":18.106999999999999,"15min":18.106999999999999},"last":0.33500000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.9710000000000001,"15min":1.1619999999999999},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.1280000000000001,"5min":14.720000000000001,"15min":14.720000000000001},"last":0.307},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.5469999999999999,"15min":1.022},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":2.806,"5min":12.794,"15min":12.794},"last":0.375}]},{"osd":6,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.8680000000000001,"15min":2.3010000000000002},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.5069999999999997,"5min":14.956,"15min":14.956},"last":1.351},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.0419999999999998,"15min":2.5219999999999998},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.311},"max":{"1min":2.883,"5min":12.755000000000001,"15min":12.755000000000001},"last":0.44400000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:02:28 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":2.851,"5min":2.851,"15min":2.851},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.083,"5min":1.083,"15min":1.083},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.4619999999999997,"5min":4.4619999999999997,"15min":4.4619999999999997},"last":1.2929999999999999}]}]},{"osd":2,"up_from":103,"seq":442381631596,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2424600,"kb_used_data":2073360,"kb_used_omap":11,"kb_used_meta":351220,"kb_avail":91320552,"statfs":{"total":95995035648,"available":93512245248,"internally_reserved":0,"allocated":2123120640,"data_stored":2121836449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":359649655},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":2.552,"15min":1.2729999999999999},"min":{"1min":0.187,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":3.306,"5min":20.867000000000001,"15min":20.867000000000001},"last":2.694},{"interface":"front","average":{"1min":0.55300000000000005,"5min":2.3119999999999998,"15min":1.246},"min":{"1min":0.24299999999999999,"5min":0.215,"15min":0.215},"max":{"1min":1.6910000000000001,"5min":21.681000000000001,"15min":21.681000000000001},"last":0.26000000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.6349999999999998,"15min":1.361},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":4.2359999999999998,"5min":20.995000000000001,"15min":20.995000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.75,"5min":1.9530000000000001,"15min":1.147},"min":{"1min":0.24199999999999999,"5min":0.193,"15min":0.193},"max":{"1min":4.532,"5min":15.945,"15min":15.945},"last":3.649}]},{"osd":3,"last update":"Sat Sep 23 17:56:59 2023","interfaces":[{"interface":"back","average":{"1min":1.4590000000000001,"5min":0.58799999999999997,"15min":0.443},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.8339999999999996,"5min":4.8339999999999996,"15min":4.8339999999999996},"last":3.6040000000000001},{"interface":"front","average":{"1min":1.2729999999999999,"5min":0.59699999999999998,"15min":0.48399999999999999},"min":{"1min":0.35299999999999998,"5min":0.34699999999999998,"15min":0.34699999999999998},"max":{"1min":5.6459999999999999,"5min":5.6459999999999999,"15min":5.6459999999999999},"last":0.51200000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.7649999999999999,"15min":1.095},"min":{"1min":0.29099999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.141,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":0.44},{"interface":"front","average":{"1min":0.45900000000000002,"5min":1.8899999999999999,"15min":1.1200000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78100000000000003,"5min":20.920000000000002,"15min":20.920000000000002},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.9890000000000001,"15min":1.1879999999999999},"min":{"1min":0.32900000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.972,"5min":20.738,"15min":20.738},"last":0.59799999999999998},{"interface":"front","average":{"1min":1.048,"5min":1.79,"15min":1.1970000000000001},"min":{"1min":0.34100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3700000000000001,"5min":14.359,"15min":15.496},"last":3.6280000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:02:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":2.0299999999999998,"15min":2.2240000000000002},"min":{"1min":0.34699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":5.1879999999999997,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":0.40200000000000002},{"interface":"front","average":{"1min":0.66900000000000004,"5min":2.3290000000000002,"15min":2.4929999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.6930000000000001,"5min":20.893000000000001,"15min":20.893000000000001},"last":0.625}]},{"osd":7,"last update":"Sat Sep 23 18:02:25 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.411,"15min":1.411},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.3239999999999998,"5min":4.3239999999999998,"15min":4.3239999999999998},"last":3.5600000000000001},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.7130000000000001,"5min":1.7130000000000001,"15min":1.7130000000000001},"last":0.57199999999999995}]}]},{"osd":3,"up_from":237,"seq":1017907249164,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1266072,"kb_used_data":1183824,"kb_used_omap":3,"kb_used_meta":82108,"kb_avail":92479080,"statfs":{"total":95995035648,"available":94698577920,"internally_reserved":0,"allocated":1212235776,"data_stored":1210773322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":84079336},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.435}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.4299999999999999}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.617}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.3759999999999999}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.38}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58599999999999997}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.44800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019114,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1540680,"kb_used_data":1329152,"kb_used_omap":0,"kb_used_meta":211520,"kb_avail":92204472,"statfs":{"total":95995035648,"available":94417379328,"internally_reserved":0,"allocated":1361051648,"data_stored":1360068194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":216596480},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.3260000000000001,"15min":0.82099999999999995},"min":{"1min":0.24399999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.1829999999999998,"5min":14.164,"15min":14.164},"last":0.26400000000000001},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.611,"15min":0.97899999999999998},"min":{"1min":0.253,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.29,"5min":11.372,"15min":11.372},"last":0.84299999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:03:04 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.532,"15min":1.002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.249},"max":{"1min":5.2190000000000003,"5min":10.429,"15min":10.429},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.46,"15min":1.0629999999999999},"min":{"1min":0.28699999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.2190000000000003,"5min":10.519,"15min":10.83},"last":0.32000000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.6699999999999999,"15min":1.0629999999999999},"min":{"1min":0.29899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.266,"5min":11.630000000000001,"15min":11.630000000000001},"last":2.294},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.5289999999999999,"15min":0.98799999999999999},"min":{"1min":0.316,"5min":0.30099999999999999,"15min":0.27100000000000002},"max":{"1min":5.2450000000000001,"5min":7.5090000000000003,"15min":7.5090000000000003},"last":0.40300000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:56:56 2023","interfaces":[{"interface":"back","average":{"1min":1.2090000000000001,"5min":0.72899999999999998,"15min":0.64900000000000002},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.6130000000000004,"5min":4.6130000000000004,"15min":4.6130000000000004},"last":0.36799999999999999},{"interface":"front","average":{"1min":1.04,"5min":0.73299999999999998,"15min":0.68200000000000005},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":3.637,"5min":3.637,"15min":3.637},"last":0.94299999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.80800000000000005,"15min":0.59299999999999997},"min":{"1min":0.23799999999999999,"5min":0.224,"15min":0.224},"max":{"1min":3.2269999999999999,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":0.30199999999999999},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0029999999999999,"15min":0.71299999999999997},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0629999999999997,"5min":7.4729999999999999,"15min":10.494},"last":0.876}]},{"osd":6,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":0.94499999999999995,"15min":0.82799999999999996},"min":{"1min":0.25,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.5579999999999998,"5min":7.1529999999999996,"15min":7.1529999999999996},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0680000000000001,"15min":0.95599999999999996},"min":{"1min":0.28999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.286,"5min":10.725,"15min":10.725},"last":0.38300000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:02:25 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.8839999999999999,"5min":3.8839999999999999,"15min":3.8839999999999999},"last":0.92100000000000004},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.8090000000000002,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.90100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757480,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1463100,"kb_used_data":1276724,"kb_used_omap":0,"kb_used_meta":186368,"kb_avail":92282052,"statfs":{"total":95995035648,"available":94496821248,"internally_reserved":0,"allocated":1307365376,"data_stored":1306389674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":190840832},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.6779999999999999,"15min":1.008},"min":{"1min":0.28699999999999998,"5min":0.217,"15min":0.217},"max":{"1min":4.0250000000000004,"5min":21.376999999999999,"15min":21.376999999999999},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.996,"5min":1.8939999999999999,"15min":1.149},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.048,"5min":21.388999999999999,"15min":21.388999999999999},"last":0.36699999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.054,"15min":0.85899999999999999},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":1.321,"5min":10.853999999999999,"15min":10.853999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":1.05,"5min":1.2969999999999999,"15min":0.95399999999999996},"min":{"1min":0.29199999999999998,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":3.6659999999999999,"5min":10.206,"15min":10.206},"last":0.30299999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.1320000000000001,"15min":1.2589999999999999},"min":{"1min":0.34200000000000003,"5min":0.255,"15min":0.255},"max":{"1min":4.9279999999999999,"5min":26.035,"15min":38.606999999999999},"last":0.91600000000000004},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.534,"15min":1.123},"min":{"1min":0.35999999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.885,"5min":17.085000000000001,"15min":39.670999999999999},"last":0.40300000000000002}]},{"osd":3,"last update":"Sat Sep 23 17:56:57 2023","interfaces":[{"interface":"back","average":{"1min":1.534,"5min":0.78800000000000003,"15min":0.66400000000000003},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":7.4969999999999999,"5min":7.4969999999999999,"15min":7.4969999999999999},"last":1.331},{"interface":"front","average":{"1min":1.0900000000000001,"5min":0.70199999999999996,"15min":0.63800000000000001},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":5.5510000000000002,"5min":5.5510000000000002,"15min":5.5510000000000002},"last":0.47799999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":0.94499999999999995,"15min":0.69699999999999995},"min":{"1min":0.24199999999999999,"5min":0.22800000000000001,"15min":0.224},"max":{"1min":3.9980000000000002,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.998,"15min":0.69799999999999995},"min":{"1min":0.30499999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.1449999999999996,"5min":5.077,"15min":5.077},"last":0.38500000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.343,"15min":1.5069999999999999},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.5880000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.42599999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.147,"15min":1.329},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.1710000000000003,"5min":8.266,"15min":8.266},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:02:26 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.546,"15min":1.546},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.125,"5min":1.125,"15min":1.125},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.0380000000000003,"5min":4.0380000000000003,"15min":4.0380000000000003},"last":0.58299999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":926224384,"data_stored":926220312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1420361728,"data_stored":1420361728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2099974144,"data_stored":2099970072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":229113856,"data_stored":229113856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1349255168,"data_stored":1349255168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1292046336,"data_stored":1292042264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2276720640,"data_stored":2276720640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1422852096,"data_stored":1422852096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:03:17.971 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2754167898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:20.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:21.390 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:03:21.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:03:21.392 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:03:21.698 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:03:21.699 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:21.700 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:03:21.700 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:03:21.700 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:03:22.006 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:03:22.006 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:03:22.006 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:03:22.007 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:03:22.007 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:03:22.008 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:22.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:03:22.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:03:22.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:03:22.011 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:03:22.011 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:03:22.011 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:03:22.011 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:03:22.011 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:03:22.012 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2754167898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:22.012 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2754167898' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:22.314 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:17 smithi099 ceph-mon[167076]: pgmap v596: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 34917/172170 objects degraded (20.281%); 12824/172170 objects misplaced (7.448%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:03:22.314 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 34917/172170 objects degraded (20.281%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:03:22.314 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:18 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/50830531' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:22.315 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:18 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:22.316 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:19 smithi099 ceph-mon[167076]: pgmap v597: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 34657/167859 objects degraded (20.646%); 12697/167859 objects misplaced (7.564%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:03:22.316 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:19 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:03:22.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:19 smithi099 ceph-mon[167076]: failed to encode map e258 with expected crc 2023-09-23T18:03:22.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:19 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:03:22.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:19 smithi099 ceph-mon[167076]: osdmap e258: 8 total, 8 up, 7 in 2023-09-23T18:03:22.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:21 smithi099 ceph-mon[167076]: pgmap v599: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 34147/165468 objects degraded (20.637%); 12435/165468 objects misplaced (7.515%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:03:22.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:22.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:03:22.319 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:17 smithi055 ceph-mon[179668]: pgmap v596: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 34917/172170 objects degraded (20.281%); 12824/172170 objects misplaced (7.448%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:03:22.319 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:17 smithi055 ceph-mon[184174]: pgmap v596: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 452 op/s; 34917/172170 objects degraded (20.281%); 12824/172170 objects misplaced (7.448%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:03:22.320 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:03:22.320 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 34917/172170 objects degraded (20.281%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:03:22.320 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:18 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/50830531' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:22.572 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:18 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:22.573 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 34917/172170 objects degraded (20.281%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:03:22.573 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:18 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/50830531' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:22.573 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:18 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:22.573 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:19 smithi055 ceph-mon[179668]: pgmap v597: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 34657/167859 objects degraded (20.646%); 12697/167859 objects misplaced (7.564%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:03:22.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:19 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:03:22.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:19 smithi055 ceph-mon[179668]: failed to encode map e258 with expected crc 2023-09-23T18:03:22.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:19 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:03:22.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:19 smithi055 ceph-mon[179668]: osdmap e258: 8 total, 8 up, 7 in 2023-09-23T18:03:22.575 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:19 smithi055 ceph-mon[184174]: pgmap v597: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 34657/167859 objects degraded (20.646%); 12697/167859 objects misplaced (7.564%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:03:22.575 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:19 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:03:22.575 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:19 smithi055 ceph-mon[184174]: failed to encode map e258 with expected crc 2023-09-23T18:03:22.575 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:19 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:03:22.576 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:19 smithi055 ceph-mon[184174]: osdmap e258: 8 total, 8 up, 7 in 2023-09-23T18:03:22.578 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:03:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:03:20] "GET /metrics HTTP/1.1" 200 36546 "" "Prometheus/2.43.0" 2023-09-23T18:03:22.579 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:21 smithi055 ceph-mon[179668]: pgmap v599: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 34147/165468 objects degraded (20.637%); 12435/165468 objects misplaced (7.515%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:03:22.579 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:22.580 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:03:22.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:22.800 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:21 smithi055 ceph-mon[184174]: pgmap v599: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.4 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 485 op/s; 34147/165468 objects degraded (20.637%); 12435/165468 objects misplaced (7.515%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:03:22.800 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:22.800 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:03:22.802 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:22.803 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:22.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:22.904 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:23.072 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:23.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:23.159 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:23.159 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:23.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:23.307 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:23.474 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:23.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:23.475 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:23.476 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:23.476 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:23.577 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:23.628 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:23.785 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:23 smithi099 ceph-mon[167076]: pgmap v600: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 33838/163209 objects degraded (20.733%); 12267/163209 objects misplaced (7.516%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:03:23.831 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 33838/163209 objects degraded (20.733%), 12 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:23.831 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:23 smithi055 ceph-mon[179668]: pgmap v600: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 33838/163209 objects degraded (20.733%); 12267/163209 objects misplaced (7.516%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:03:23.831 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 33838/163209 objects degraded (20.733%), 12 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:24.031 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:24.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:24.121 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:24.122 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:23 smithi055 ceph-mon[184174]: pgmap v600: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 477 op/s; 33838/163209 objects degraded (20.733%); 12267/163209 objects misplaced (7.516%); 5.6 MiB/s, 89 objects/s recovering 2023-09-23T18:03:24.122 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 33838/163209 objects degraded (20.733%), 12 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:24.222 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:03:24.319 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:03:24.382 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:24.578 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:03:24.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:24.881 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:03:24.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:03:24.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:25.019 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:25.064 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:25.074 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:25.074 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:03:25.133 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:03:25.175 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:03:25.221 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:26.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:25 smithi099 ceph-mon[167076]: pgmap v601: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 33195/159174 objects degraded (20.855%); 12267/159174 objects misplaced (7.707%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:03:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:25 smithi099 ceph-mon[167076]: from='client.45386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:26.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:25 smithi055 ceph-mon[184174]: pgmap v601: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 33195/159174 objects degraded (20.855%); 12267/159174 objects misplaced (7.707%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:03:26.127 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:25 smithi055 ceph-mon[184174]: from='client.45386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:26.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:25 smithi055 ceph-mon[179668]: pgmap v601: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 465 op/s; 33195/159174 objects degraded (20.855%); 12267/159174 objects misplaced (7.707%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:03:26.127 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:25 smithi055 ceph-mon[179668]: from='client.45386 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:27.275 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:03:27.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:27.279 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:03:27.380 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:27.408 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:28.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:27 smithi099 ceph-mon[167076]: pgmap v602: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 33195/158328 objects degraded (20.966%); 11999/158328 objects misplaced (7.579%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:03:28.119 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:28.121 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:27 smithi055 ceph-mon[184174]: pgmap v602: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 33195/158328 objects degraded (20.966%); 11999/158328 objects misplaced (7.579%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:03:28.122 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:27 smithi055 ceph-mon[179668]: pgmap v602: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.2 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 461 op/s; 33195/158328 objects degraded (20.966%); 11999/158328 objects misplaced (7.579%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:03:28.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:28.122 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:28.136 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":601,"stamp":"2023-09-23T18:03:24.681912+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3479976454,"num_objects":53058,"num_object_clones":0,"num_object_copies":159174,"num_objects_missing_on_primary":623,"num_objects_missing":623,"num_objects_degraded":33195,"num_objects_misplaced":12267,"num_objects_unfound":0,"num_objects_dirty":53058,"num_whiteouts":0,"num_read":6284,"num_read_kb":12770,"num_write":94310,"num_write_kb":4695012,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16552,"num_bytes_recovered":1080428999,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":86699,"ondisk_log_size":86699,"up":357,"acting":345,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":404,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":11809740,"kb_used_data":10333908,"kb_used_omap":43,"kb_used_meta":1475732,"kb_avail":644406324,"statfs":{"total":671965249536,"available":659872075776,"internally_reserved":0,"allocated":10581921792,"data_stored":10573917920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1511150287},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,0,3,1,3,1,4,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-305266688,"num_objects":-4658,"num_object_clones":0,"num_object_copies":-13974,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1722,"num_objects_misplaced":-895,"num_objects_unfound":0,"num_objects_dirty":-4658,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4658,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1153,"num_bytes_recovered":75563008,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003993"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":586,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:10.503275+0000","last_change":"2023-09-23T18:02:48.964317+0000","last_active":"2023-09-23T18:03:10.503275+0000","last_peered":"2023-09-23T18:03:10.503275+0000","last_clean":"2023-09-23T18:02:45.962259+0000","last_became_active":"2023-09-23T18:02:48.961702+0000","last_became_peered":"2023-09-23T18:02:48.961702+0000","last_unstale":"2023-09-23T18:03:10.503275+0000","last_undegraded":"2023-09-23T18:02:48.941433+0000","last_fullsized":"2023-09-23T18:02:48.940979+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":474537,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":58,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485653+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:03:10.485653+0000","last_peered":"2023-09-23T18:03:10.485653+0000","last_clean":"2023-09-23T18:03:10.485653+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:03:10.485653+0000","last_undegraded":"2023-09-23T18:03:10.485653+0000","last_fullsized":"2023-09-23T18:03:10.485653+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":139,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496794+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:03:10.496794+0000","last_peered":"2023-09-23T18:03:10.496794+0000","last_clean":"2023-09-23T18:03:10.496794+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:03:10.496794+0000","last_undegraded":"2023-09-23T18:03:10.496794+0000","last_fullsized":"2023-09-23T18:03:10.496794+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":600,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494671+0000","last_change":"2023-09-23T18:02:51.046916+0000","last_active":"2023-09-23T18:03:10.494671+0000","last_peered":"2023-09-23T18:03:10.494671+0000","last_clean":"2023-09-23T18:03:10.494671+0000","last_became_active":"2023-09-23T18:02:51.046591+0000","last_became_peered":"2023-09-23T18:02:51.046591+0000","last_unstale":"2023-09-23T18:03:10.494671+0000","last_undegraded":"2023-09-23T18:03:10.494671+0000","last_fullsized":"2023-09-23T18:03:10.494671+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:19.805099+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":107,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496612+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:03:10.496612+0000","last_peered":"2023-09-23T18:03:10.496612+0000","last_clean":"2023-09-23T18:03:10.496612+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:03:10.496612+0000","last_undegraded":"2023-09-23T18:03:10.496612+0000","last_fullsized":"2023-09-23T18:03:10.496612+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":259,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T18:03:18.977778+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:03:18.977778+0000","last_peered":"2023-09-23T18:03:18.977778+0000","last_clean":"2023-09-23T18:03:18.977778+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:03:18.977778+0000","last_undegraded":"2023-09-23T18:03:18.977778+0000","last_fullsized":"2023-09-23T18:03:18.977778+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":592,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494505+0000","last_change":"2023-09-23T18:02:51.051617+0000","last_active":"2023-09-23T18:03:10.494505+0000","last_peered":"2023-09-23T18:03:10.494505+0000","last_clean":"2023-09-23T18:03:10.494505+0000","last_became_active":"2023-09-23T18:02:51.048947+0000","last_became_peered":"2023-09-23T18:02:51.048947+0000","last_unstale":"2023-09-23T18:03:10.494505+0000","last_undegraded":"2023-09-23T18:03:10.494505+0000","last_fullsized":"2023-09-23T18:03:10.494505+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:49:34.962249+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":120,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.497491+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:03:10.497491+0000","last_peered":"2023-09-23T18:03:10.497491+0000","last_clean":"2023-09-23T18:03:10.497491+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:03:10.497491+0000","last_undegraded":"2023-09-23T18:03:10.497491+0000","last_fullsized":"2023-09-23T18:03:10.497491+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":198,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.474920+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:03:10.474920+0000","last_peered":"2023-09-23T18:03:10.474920+0000","last_clean":"2023-09-23T18:03:10.474920+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:03:10.474920+0000","last_undegraded":"2023-09-23T18:03:10.474920+0000","last_fullsized":"2023-09-23T18:03:10.474920+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":589,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496712+0000","last_change":"2023-09-23T18:02:51.052123+0000","last_active":"2023-09-23T18:03:10.496712+0000","last_peered":"2023-09-23T18:03:10.496712+0000","last_clean":"2023-09-23T18:03:10.496712+0000","last_became_active":"2023-09-23T18:02:51.051772+0000","last_became_peered":"2023-09-23T18:02:51.051772+0000","last_unstale":"2023-09-23T18:03:10.496712+0000","last_undegraded":"2023-09-23T18:03:10.496712+0000","last_fullsized":"2023-09-23T18:03:10.496712+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":238,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.501969+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:03:10.501969+0000","last_peered":"2023-09-23T18:03:10.501969+0000","last_clean":"2023-09-23T18:03:10.501969+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:03:10.501969+0000","last_undegraded":"2023-09-23T18:03:10.501969+0000","last_fullsized":"2023-09-23T18:03:10.501969+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":114,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.497567+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:03:10.497567+0000","last_peered":"2023-09-23T18:03:10.497567+0000","last_clean":"2023-09-23T18:03:10.497567+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:03:10.497567+0000","last_undegraded":"2023-09-23T18:03:10.497567+0000","last_fullsized":"2023-09-23T18:03:10.497567+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":584,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494433+0000","last_change":"2023-09-23T18:02:51.054208+0000","last_active":"2023-09-23T18:03:10.494433+0000","last_peered":"2023-09-23T18:03:10.494433+0000","last_clean":"2023-09-23T18:03:10.494433+0000","last_became_active":"2023-09-23T18:02:51.053676+0000","last_became_peered":"2023-09-23T18:02:51.053676+0000","last_unstale":"2023-09-23T18:03:10.494433+0000","last_undegraded":"2023-09-23T18:03:10.494433+0000","last_fullsized":"2023-09-23T18:03:10.494433+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":240,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.509495+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:03:10.509495+0000","last_peered":"2023-09-23T18:03:10.509495+0000","last_clean":"2023-09-23T18:03:10.509495+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:03:10.509495+0000","last_undegraded":"2023-09-23T18:03:10.509495+0000","last_fullsized":"2023-09-23T18:03:10.509495+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":68,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496464+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:03:10.496464+0000","last_peered":"2023-09-23T18:03:10.496464+0000","last_clean":"2023-09-23T18:03:10.496464+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:03:10.496464+0000","last_undegraded":"2023-09-23T18:03:10.496464+0000","last_fullsized":"2023-09-23T18:03:10.496464+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":589,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.495802+0000","last_change":"2023-09-23T18:02:51.058624+0000","last_active":"2023-09-23T18:03:10.495802+0000","last_peered":"2023-09-23T18:03:10.495802+0000","last_clean":"2023-09-23T18:03:10.495802+0000","last_became_active":"2023-09-23T18:02:51.057352+0000","last_became_peered":"2023-09-23T18:02:51.057352+0000","last_unstale":"2023-09-23T18:03:10.495802+0000","last_undegraded":"2023-09-23T18:03:10.495802+0000","last_fullsized":"2023-09-23T18:03:10.495802+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":24,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496677+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:03:10.496677+0000","last_peered":"2023-09-23T18:03:10.496677+0000","last_clean":"2023-09-23T18:03:10.496677+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:03:10.496677+0000","last_undegraded":"2023-09-23T18:03:10.496677+0000","last_fullsized":"2023-09-23T18:03:10.496677+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":154,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.476047+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:03:10.476047+0000","last_peered":"2023-09-23T18:03:10.476047+0000","last_clean":"2023-09-23T18:03:10.476047+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:03:10.476047+0000","last_undegraded":"2023-09-23T18:03:10.476047+0000","last_fullsized":"2023-09-23T18:03:10.476047+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":589,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.495919+0000","last_change":"2023-09-23T18:02:51.059983+0000","last_active":"2023-09-23T18:03:10.495919+0000","last_peered":"2023-09-23T18:03:10.495919+0000","last_clean":"2023-09-23T18:03:10.495919+0000","last_became_active":"2023-09-23T18:02:51.057768+0000","last_became_peered":"2023-09-23T18:02:51.057768+0000","last_unstale":"2023-09-23T18:03:10.495919+0000","last_undegraded":"2023-09-23T18:03:10.495919+0000","last_fullsized":"2023-09-23T18:03:10.495919+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":58,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485572+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:03:10.485572+0000","last_peered":"2023-09-23T18:03:10.485572+0000","last_clean":"2023-09-23T18:03:10.485572+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:03:10.485572+0000","last_undegraded":"2023-09-23T18:03:10.485572+0000","last_fullsized":"2023-09-23T18:03:10.485572+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":58,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485730+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:03:10.485730+0000","last_peered":"2023-09-23T18:03:10.485730+0000","last_clean":"2023-09-23T18:03:10.485730+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:03:10.485730+0000","last_undegraded":"2023-09-23T18:03:10.485730+0000","last_fullsized":"2023-09-23T18:03:10.485730+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":579,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494772+0000","last_change":"2023-09-23T18:02:51.047570+0000","last_active":"2023-09-23T18:03:10.494772+0000","last_peered":"2023-09-23T18:03:10.494772+0000","last_clean":"2023-09-23T18:03:10.494772+0000","last_became_active":"2023-09-23T18:02:51.046963+0000","last_became_peered":"2023-09-23T18:02:51.046963+0000","last_unstale":"2023-09-23T18:03:10.494772+0000","last_undegraded":"2023-09-23T18:03:10.494772+0000","last_fullsized":"2023-09-23T18:03:10.494772+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":187,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496379+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:03:10.496379+0000","last_peered":"2023-09-23T18:03:10.496379+0000","last_clean":"2023-09-23T18:03:10.496379+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:03:10.496379+0000","last_undegraded":"2023-09-23T18:03:10.496379+0000","last_fullsized":"2023-09-23T18:03:10.496379+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":102,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496136+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:03:10.496136+0000","last_peered":"2023-09-23T18:03:10.496136+0000","last_clean":"2023-09-23T18:03:10.496136+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:03:10.496136+0000","last_undegraded":"2023-09-23T18:03:10.496136+0000","last_fullsized":"2023-09-23T18:03:10.496136+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":592,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494118+0000","last_change":"2023-09-23T18:02:51.053806+0000","last_active":"2023-09-23T18:03:10.494118+0000","last_peered":"2023-09-23T18:03:10.494118+0000","last_clean":"2023-09-23T18:03:10.494118+0000","last_became_active":"2023-09-23T18:02:51.048822+0000","last_became_peered":"2023-09-23T18:02:51.048822+0000","last_unstale":"2023-09-23T18:03:10.494118+0000","last_undegraded":"2023-09-23T18:03:10.494118+0000","last_fullsized":"2023-09-23T18:03:10.494118+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:34.401104+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":257,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.502545+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:03:10.502545+0000","last_peered":"2023-09-23T18:03:10.502545+0000","last_clean":"2023-09-23T18:03:10.502545+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:03:10.502545+0000","last_undegraded":"2023-09-23T18:03:10.502545+0000","last_fullsized":"2023-09-23T18:03:10.502545+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":235,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.509019+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:03:10.509019+0000","last_peered":"2023-09-23T18:03:10.509019+0000","last_clean":"2023-09-23T18:03:10.509019+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:03:10.509019+0000","last_undegraded":"2023-09-23T18:03:10.509019+0000","last_fullsized":"2023-09-23T18:03:10.509019+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":580,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496286+0000","last_change":"2023-09-23T18:02:51.046646+0000","last_active":"2023-09-23T18:03:10.496286+0000","last_peered":"2023-09-23T18:03:10.496286+0000","last_clean":"2023-09-23T18:03:10.496286+0000","last_became_active":"2023-09-23T18:02:51.046134+0000","last_became_peered":"2023-09-23T18:02:51.046134+0000","last_unstale":"2023-09-23T18:03:10.496286+0000","last_undegraded":"2023-09-23T18:03:10.496286+0000","last_fullsized":"2023-09-23T18:03:10.496286+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":48,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485471+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:03:10.485471+0000","last_peered":"2023-09-23T18:03:10.485471+0000","last_clean":"2023-09-23T18:03:10.485471+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:03:10.485471+0000","last_undegraded":"2023-09-23T18:03:10.485471+0000","last_fullsized":"2023-09-23T18:03:10.485471+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":248,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.501513+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:03:10.501513+0000","last_peered":"2023-09-23T18:03:10.501513+0000","last_clean":"2023-09-23T18:03:10.501513+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:03:10.501513+0000","last_undegraded":"2023-09-23T18:03:10.501513+0000","last_fullsized":"2023-09-23T18:03:10.501513+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":592,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494431+0000","last_change":"2023-09-23T18:02:51.051817+0000","last_active":"2023-09-23T18:03:10.494431+0000","last_peered":"2023-09-23T18:03:10.494431+0000","last_clean":"2023-09-23T18:03:10.494431+0000","last_became_active":"2023-09-23T18:02:51.048945+0000","last_became_peered":"2023-09-23T18:02:51.048945+0000","last_unstale":"2023-09-23T18:03:10.494431+0000","last_undegraded":"2023-09-23T18:03:10.494431+0000","last_fullsized":"2023-09-23T18:03:10.494431+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:47.163864+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":194,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.474076+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:03:10.474076+0000","last_peered":"2023-09-23T18:03:10.474076+0000","last_clean":"2023-09-23T18:03:10.474076+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:03:10.474076+0000","last_undegraded":"2023-09-23T18:03:10.474076+0000","last_fullsized":"2023-09-23T18:03:10.474076+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":111,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.501367+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:03:10.501367+0000","last_peered":"2023-09-23T18:03:10.501367+0000","last_clean":"2023-09-23T18:03:10.501367+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:03:10.501367+0000","last_undegraded":"2023-09-23T18:03:10.501367+0000","last_fullsized":"2023-09-23T18:03:10.501367+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"258'5900","reported_seq":5961,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T18:03:22.493323+0000","last_change":"2023-09-23T18:00:57.996982+0000","last_active":"2023-09-23T18:03:22.493323+0000","last_peered":"2023-09-23T18:03:22.493323+0000","last_clean":"2023-09-23T18:03:22.493323+0000","last_became_active":"2023-09-23T18:00:57.996589+0000","last_became_peered":"2023-09-23T18:00:57.996589+0000","last_unstale":"2023-09-23T18:03:22.493323+0000","last_undegraded":"2023-09-23T18:03:22.493323+0000","last_fullsized":"2023-09-23T18:03:22.493323+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5900,"log_dups_size":0,"ondisk_log_size":5900,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:13.306089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219021312,"num_objects":3342,"num_object_clones":0,"num_object_copies":10026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5900,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":589,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496665+0000","last_change":"2023-09-23T18:02:51.058477+0000","last_active":"2023-09-23T18:03:10.496665+0000","last_peered":"2023-09-23T18:03:10.496665+0000","last_clean":"2023-09-23T18:03:10.496665+0000","last_became_active":"2023-09-23T18:02:51.057258+0000","last_became_peered":"2023-09-23T18:02:51.057258+0000","last_unstale":"2023-09-23T18:03:10.496665+0000","last_undegraded":"2023-09-23T18:03:10.496665+0000","last_fullsized":"2023-09-23T18:03:10.496665+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":187,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.495827+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:03:10.495827+0000","last_peered":"2023-09-23T18:03:10.495827+0000","last_clean":"2023-09-23T18:03:10.495827+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:03:10.495827+0000","last_undegraded":"2023-09-23T18:03:10.495827+0000","last_fullsized":"2023-09-23T18:03:10.495827+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":246,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.508567+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:03:10.508567+0000","last_peered":"2023-09-23T18:03:10.508567+0000","last_clean":"2023-09-23T18:03:10.508567+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:03:10.508567+0000","last_undegraded":"2023-09-23T18:03:10.508567+0000","last_fullsized":"2023-09-23T18:03:10.508567+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"258'5661","reported_seq":12882,"reported_epoch":258,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:03:20.404235+0000","last_change":"2023-09-23T18:02:49.039290+0000","last_active":"2023-09-23T18:03:20.404235+0000","last_peered":"2023-09-23T18:03:20.404235+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:49.001792+0000","last_became_peered":"2023-09-23T18:02:49.001792+0000","last_unstale":"2023-09-23T18:03:20.404235+0000","last_undegraded":"2023-09-23T18:02:48.968288+0000","last_fullsized":"2023-09-23T18:03:20.404235+0000","mapping_epoch":250,"log_start":"217'1530","ondisk_log_start":"217'1530","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4131,"log_dups_size":0,"ondisk_log_size":4131,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.156979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224329728,"num_objects":3423,"num_object_clones":0,"num_object_copies":10269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":409,"num_objects_misplaced":6437,"num_objects_unfound":0,"num_objects_dirty":3423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5661,"num_write_kb":290688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3566,"num_bytes_recovered":233701376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[6,1,4],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,4,6","objects":3014},{"shards":"1,6","objects":409}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":594,"reported_epoch":257,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:10.500831+0000","last_change":"2023-09-23T18:02:48.968551+0000","last_active":"2023-09-23T18:03:10.500831+0000","last_peered":"2023-09-23T18:03:10.500831+0000","last_clean":"2023-09-23T18:02:45.965241+0000","last_became_active":"2023-09-23T18:02:48.968221+0000","last_became_peered":"2023-09-23T18:02:48.968221+0000","last_unstale":"2023-09-23T18:03:10.500831+0000","last_undegraded":"2023-09-23T18:02:48.944021+0000","last_fullsized":"2023-09-23T18:02:48.942326+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":409128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":202,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496171+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:03:10.496171+0000","last_peered":"2023-09-23T18:03:10.496171+0000","last_clean":"2023-09-23T18:03:10.496171+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:03:10.496171+0000","last_undegraded":"2023-09-23T18:03:10.496171+0000","last_fullsized":"2023-09-23T18:03:10.496171+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":268,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485880+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:03:10.485880+0000","last_peered":"2023-09-23T18:03:10.485880+0000","last_clean":"2023-09-23T18:03:10.485880+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:03:10.485880+0000","last_undegraded":"2023-09-23T18:03:10.485880+0000","last_fullsized":"2023-09-23T18:03:10.485880+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"258'5944","reported_seq":12180,"reported_epoch":258,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:24.305629+0000","last_change":"2023-09-23T18:02:47.987792+0000","last_active":"2023-09-23T18:03:24.305629+0000","last_peered":"2023-09-23T18:03:24.305629+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.307615+0000","last_became_peered":"2023-09-23T18:02:19.307615+0000","last_unstale":"2023-09-23T18:03:24.305629+0000","last_undegraded":"2023-09-23T18:02:18.966041+0000","last_fullsized":"2023-09-23T18:02:18.929348+0000","mapping_epoch":238,"log_start":"217'1200","ondisk_log_start":"217'1200","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4744,"log_dups_size":0,"ondisk_log_size":4744,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:04.498873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216137728,"num_objects":3298,"num_object_clones":0,"num_object_copies":9894,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1177,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3298,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5944,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3081,"num_bytes_recovered":201916416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7],"avail_no_missing":["2","7"],"object_location_counts":[{"shards":"2,7","objects":2121}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":580,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.495642+0000","last_change":"2023-09-23T18:02:51.058795+0000","last_active":"2023-09-23T18:03:10.495642+0000","last_peered":"2023-09-23T18:03:10.495642+0000","last_clean":"2023-09-23T18:03:10.495642+0000","last_became_active":"2023-09-23T18:02:51.057799+0000","last_became_peered":"2023-09-23T18:02:51.057799+0000","last_unstale":"2023-09-23T18:03:10.495642+0000","last_undegraded":"2023-09-23T18:03:10.495642+0000","last_fullsized":"2023-09-23T18:03:10.495642+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":125,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496850+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:03:10.496850+0000","last_peered":"2023-09-23T18:03:10.496850+0000","last_clean":"2023-09-23T18:03:10.496850+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:03:10.496850+0000","last_undegraded":"2023-09-23T18:03:10.496850+0000","last_fullsized":"2023-09-23T18:03:10.496850+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":223,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.475977+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:03:10.475977+0000","last_peered":"2023-09-23T18:03:10.475977+0000","last_clean":"2023-09-23T18:03:10.475977+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:03:10.475977+0000","last_undegraded":"2023-09-23T18:03:10.475977+0000","last_fullsized":"2023-09-23T18:03:10.475977+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"258'5973","reported_seq":6054,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T18:03:24.525412+0000","last_change":"2023-09-23T18:01:25.019784+0000","last_active":"2023-09-23T18:03:24.525412+0000","last_peered":"2023-09-23T18:03:24.525412+0000","last_clean":"2023-09-23T18:03:24.525412+0000","last_became_active":"2023-09-23T18:01:25.002073+0000","last_became_peered":"2023-09-23T18:01:25.002073+0000","last_unstale":"2023-09-23T18:03:24.525412+0000","last_undegraded":"2023-09-23T18:03:24.525412+0000","last_fullsized":"2023-09-23T18:03:24.525412+0000","mapping_epoch":214,"log_start":"217'1600","ondisk_log_start":"217'1600","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4373,"log_dups_size":0,"ondisk_log_size":4373,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:58.981672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218169344,"num_objects":3329,"num_object_clones":0,"num_object_copies":9987,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3329,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5973,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":589,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.495965+0000","last_change":"2023-09-23T18:02:51.057506+0000","last_active":"2023-09-23T18:03:10.495965+0000","last_peered":"2023-09-23T18:03:10.495965+0000","last_clean":"2023-09-23T18:03:10.495965+0000","last_became_active":"2023-09-23T18:02:51.056589+0000","last_became_peered":"2023-09-23T18:02:51.056589+0000","last_unstale":"2023-09-23T18:03:10.495965+0000","last_undegraded":"2023-09-23T18:03:10.495965+0000","last_fullsized":"2023-09-23T18:03:10.495965+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":234,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.509291+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:03:10.509291+0000","last_peered":"2023-09-23T18:03:10.509291+0000","last_clean":"2023-09-23T18:03:10.509291+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:03:10.509291+0000","last_undegraded":"2023-09-23T18:03:10.509291+0000","last_fullsized":"2023-09-23T18:03:10.509291+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":260,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.500016+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:03:10.500016+0000","last_peered":"2023-09-23T18:03:10.500016+0000","last_clean":"2023-09-23T18:03:10.500016+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:03:10.500016+0000","last_undegraded":"2023-09-23T18:03:10.500016+0000","last_fullsized":"2023-09-23T18:03:10.500016+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"258'5948","reported_seq":9099,"reported_epoch":258,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:03:22.948509+0000","last_change":"2023-09-23T18:02:45.688221+0000","last_active":"2023-09-23T18:03:22.948509+0000","last_peered":"2023-09-23T18:03:22.948509+0000","last_clean":"2023-09-23T18:01:38.416095+0000","last_became_active":"2023-09-23T18:02:07.547135+0000","last_became_peered":"2023-09-23T18:02:07.547135+0000","last_unstale":"2023-09-23T18:03:22.948509+0000","last_undegraded":"2023-09-23T18:03:22.948509+0000","last_fullsized":"2023-09-23T18:02:02.784733+0000","mapping_epoch":227,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5848,"log_dups_size":0,"ondisk_log_size":5848,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:27.864301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216662016,"num_objects":3306,"num_object_clones":0,"num_object_copies":9918,"num_objects_missing_on_primary":623,"num_objects_missing":623,"num_objects_degraded":0,"num_objects_misplaced":2562,"num_objects_unfound":0,"num_objects_dirty":3306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5948,"num_write_kb":296128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1538,"num_bytes_recovered":100794368,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1939},{"shards":"1,2,5","objects":623},{"shards":"2,4","objects":744}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":581,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494708+0000","last_change":"2023-09-23T18:02:51.047609+0000","last_active":"2023-09-23T18:03:10.494708+0000","last_peered":"2023-09-23T18:03:10.494708+0000","last_clean":"2023-09-23T18:03:10.494708+0000","last_became_active":"2023-09-23T18:02:51.047209+0000","last_became_peered":"2023-09-23T18:02:51.047209+0000","last_unstale":"2023-09-23T18:03:10.494708+0000","last_undegraded":"2023-09-23T18:03:10.494708+0000","last_fullsized":"2023-09-23T18:03:10.494708+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":145,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496397+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:03:10.496397+0000","last_peered":"2023-09-23T18:03:10.496397+0000","last_clean":"2023-09-23T18:03:10.496397+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:03:10.496397+0000","last_undegraded":"2023-09-23T18:03:10.496397+0000","last_fullsized":"2023-09-23T18:03:10.496397+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":266,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485604+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:03:10.485604+0000","last_peered":"2023-09-23T18:03:10.485604+0000","last_clean":"2023-09-23T18:03:10.485604+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:03:10.485604+0000","last_undegraded":"2023-09-23T18:03:10.485604+0000","last_fullsized":"2023-09-23T18:03:10.485604+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"258'5821","reported_seq":6041,"reported_epoch":258,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:24.269466+0000","last_change":"2023-09-23T18:02:47.961575+0000","last_active":"2023-09-23T18:03:24.269466+0000","last_peered":"2023-09-23T18:03:24.269466+0000","last_clean":"2023-09-23T18:02:17.013863+0000","last_became_active":"2023-09-23T18:02:19.267215+0000","last_became_peered":"2023-09-23T18:02:19.267215+0000","last_unstale":"2023-09-23T18:03:24.269466+0000","last_undegraded":"2023-09-23T18:02:18.955030+0000","last_fullsized":"2023-09-23T18:02:18.924656+0000","mapping_epoch":238,"log_start":"217'1100","ondisk_log_start":"217'1100","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4721,"log_dups_size":0,"ondisk_log_size":4721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:46.393021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":212008960,"num_objects":3235,"num_object_clones":0,"num_object_copies":9705,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3235,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5821,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":4128768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-873},{"shards":"2,4,6","objects":2877}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":585,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494155+0000","last_change":"2023-09-23T18:02:51.049464+0000","last_active":"2023-09-23T18:03:10.494155+0000","last_peered":"2023-09-23T18:03:10.494155+0000","last_clean":"2023-09-23T18:03:10.494155+0000","last_became_active":"2023-09-23T18:02:51.048477+0000","last_became_peered":"2023-09-23T18:02:51.048477+0000","last_unstale":"2023-09-23T18:03:10.494155+0000","last_undegraded":"2023-09-23T18:03:10.494155+0000","last_fullsized":"2023-09-23T18:03:10.494155+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":217,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.476115+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:03:10.476115+0000","last_peered":"2023-09-23T18:03:10.476115+0000","last_clean":"2023-09-23T18:03:10.476115+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:03:10.476115+0000","last_undegraded":"2023-09-23T18:03:10.476115+0000","last_fullsized":"2023-09-23T18:03:10.476115+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":58,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485799+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:03:10.485799+0000","last_peered":"2023-09-23T18:03:10.485799+0000","last_clean":"2023-09-23T18:03:10.485799+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:03:10.485799+0000","last_undegraded":"2023-09-23T18:03:10.485799+0000","last_fullsized":"2023-09-23T18:03:10.485799+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"258'5792","reported_seq":5858,"reported_epoch":258,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:03:22.556746+0000","last_change":"2023-09-23T18:02:19.054827+0000","last_active":"2023-09-23T18:03:22.556746+0000","last_peered":"2023-09-23T18:03:22.556746+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.048672+0000","last_became_peered":"2023-09-23T18:02:19.048672+0000","last_unstale":"2023-09-23T18:03:22.556746+0000","last_undegraded":"2023-09-23T18:02:18.951263+0000","last_fullsized":"2023-09-23T18:02:18.930851+0000","mapping_epoch":238,"log_start":"216'261","ondisk_log_start":"216'261","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5531,"log_dups_size":0,"ondisk_log_size":5531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:23.690233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216268800,"num_objects":3300,"num_object_clones":0,"num_object_copies":9900,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3300,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3300,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5792,"num_write_kb":290944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":3300}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":589,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494499+0000","last_change":"2023-09-23T18:02:51.060858+0000","last_active":"2023-09-23T18:03:10.494499+0000","last_peered":"2023-09-23T18:03:10.494499+0000","last_clean":"2023-09-23T18:03:10.494499+0000","last_became_active":"2023-09-23T18:02:51.060554+0000","last_became_peered":"2023-09-23T18:02:51.060554+0000","last_unstale":"2023-09-23T18:03:10.494499+0000","last_undegraded":"2023-09-23T18:03:10.494499+0000","last_fullsized":"2023-09-23T18:03:10.494499+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":48,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485716+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:03:10.485716+0000","last_peered":"2023-09-23T18:03:10.485716+0000","last_clean":"2023-09-23T18:03:10.485716+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:03:10.485716+0000","last_undegraded":"2023-09-23T18:03:10.485716+0000","last_fullsized":"2023-09-23T18:03:10.485716+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":268,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.506258+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:03:10.506258+0000","last_peered":"2023-09-23T18:03:10.506258+0000","last_clean":"2023-09-23T18:03:10.506258+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:03:10.506258+0000","last_undegraded":"2023-09-23T18:03:10.506258+0000","last_fullsized":"2023-09-23T18:03:10.506258+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"258'5781","reported_seq":13896,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T18:03:22.537841+0000","last_change":"2023-09-23T18:02:47.017672+0000","last_active":"2023-09-23T18:03:22.537841+0000","last_peered":"2023-09-23T18:03:22.537841+0000","last_clean":"2023-09-23T18:03:22.537841+0000","last_became_active":"2023-09-23T18:02:47.009245+0000","last_became_peered":"2023-09-23T18:02:47.009245+0000","last_unstale":"2023-09-23T18:03:22.537841+0000","last_undegraded":"2023-09-23T18:03:22.537841+0000","last_fullsized":"2023-09-23T18:03:22.537841+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5781,"log_dups_size":0,"ondisk_log_size":5781,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:04.868743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":216530968,"num_objects":3305,"num_object_clones":0,"num_object_copies":9915,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3305,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":5781,"num_write_kb":290689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4015,"num_bytes_recovered":263127040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":581,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494355+0000","last_change":"2023-09-23T18:02:51.054197+0000","last_active":"2023-09-23T18:03:10.494355+0000","last_peered":"2023-09-23T18:03:10.494355+0000","last_clean":"2023-09-23T18:03:10.494355+0000","last_became_active":"2023-09-23T18:02:51.049362+0000","last_became_peered":"2023-09-23T18:02:51.049362+0000","last_unstale":"2023-09-23T18:03:10.494355+0000","last_undegraded":"2023-09-23T18:03:10.494355+0000","last_fullsized":"2023-09-23T18:03:10.494355+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":103,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.497721+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:03:10.497721+0000","last_peered":"2023-09-23T18:03:10.497721+0000","last_clean":"2023-09-23T18:03:10.497721+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:03:10.497721+0000","last_undegraded":"2023-09-23T18:03:10.497721+0000","last_fullsized":"2023-09-23T18:03:10.497721+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":103,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.501577+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:03:10.501577+0000","last_peered":"2023-09-23T18:03:10.501577+0000","last_clean":"2023-09-23T18:03:10.501577+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:03:10.501577+0000","last_undegraded":"2023-09-23T18:03:10.501577+0000","last_fullsized":"2023-09-23T18:03:10.501577+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"258'5686","reported_seq":5754,"reported_epoch":258,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:24.471234+0000","last_change":"2023-09-23T18:02:19.315376+0000","last_active":"2023-09-23T18:03:24.471234+0000","last_peered":"2023-09-23T18:03:24.471234+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.306174+0000","last_became_peered":"2023-09-23T18:02:19.306174+0000","last_unstale":"2023-09-23T18:03:24.471234+0000","last_undegraded":"2023-09-23T18:02:18.954031+0000","last_fullsized":"2023-09-23T18:02:18.922505+0000","mapping_epoch":238,"log_start":"217'1300","ondisk_log_start":"217'1300","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4386,"log_dups_size":0,"ondisk_log_size":4386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:52.392034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209453056,"num_objects":3196,"num_object_clones":0,"num_object_copies":9588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5686,"num_write_kb":284224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":-899}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":590,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.495649+0000","last_change":"2023-09-23T18:02:51.059934+0000","last_active":"2023-09-23T18:03:10.495649+0000","last_peered":"2023-09-23T18:03:10.495649+0000","last_clean":"2023-09-23T18:03:10.495649+0000","last_became_active":"2023-09-23T18:02:51.059638+0000","last_became_peered":"2023-09-23T18:02:51.059638+0000","last_unstale":"2023-09-23T18:03:10.495649+0000","last_undegraded":"2023-09-23T18:03:10.495649+0000","last_fullsized":"2023-09-23T18:03:10.495649+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":567,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.500689+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:03:10.500689+0000","last_peered":"2023-09-23T18:03:10.500689+0000","last_clean":"2023-09-23T18:03:10.500689+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:03:10.500689+0000","last_undegraded":"2023-09-23T18:03:10.500689+0000","last_fullsized":"2023-09-23T18:03:10.500689+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":238,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.508951+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:03:10.508951+0000","last_peered":"2023-09-23T18:03:10.508951+0000","last_clean":"2023-09-23T18:03:10.508951+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:03:10.508951+0000","last_undegraded":"2023-09-23T18:03:10.508951+0000","last_fullsized":"2023-09-23T18:03:10.508951+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":235,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.509202+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:03:10.509202+0000","last_peered":"2023-09-23T18:03:10.509202+0000","last_clean":"2023-09-23T18:03:10.509202+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:03:10.509202+0000","last_undegraded":"2023-09-23T18:03:10.509202+0000","last_fullsized":"2023-09-23T18:03:10.509202+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"258'5759","reported_seq":5841,"reported_epoch":258,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:22.941561+0000","last_change":"2023-09-23T18:02:19.289610+0000","last_active":"2023-09-23T18:03:22.941561+0000","last_peered":"2023-09-23T18:03:22.941561+0000","last_clean":"2023-09-23T18:02:17.060883+0000","last_became_active":"2023-09-23T18:02:19.266879+0000","last_became_peered":"2023-09-23T18:02:19.266879+0000","last_unstale":"2023-09-23T18:03:22.941561+0000","last_undegraded":"2023-09-23T18:02:18.963390+0000","last_fullsized":"2023-09-23T18:02:18.926482+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5759,"log_dups_size":0,"ondisk_log_size":5759,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.499269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":209649664,"num_objects":3199,"num_object_clones":0,"num_object_copies":9597,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3199,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5759,"num_write_kb":286656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":2848},{"shards":"4,7","objects":-929}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":592,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494704+0000","last_change":"2023-09-23T18:02:51.047767+0000","last_active":"2023-09-23T18:03:10.494704+0000","last_peered":"2023-09-23T18:03:10.494704+0000","last_clean":"2023-09-23T18:03:10.494704+0000","last_became_active":"2023-09-23T18:02:51.047297+0000","last_became_peered":"2023-09-23T18:02:51.047297+0000","last_unstale":"2023-09-23T18:03:10.494704+0000","last_undegraded":"2023-09-23T18:03:10.494704+0000","last_fullsized":"2023-09-23T18:03:10.494704+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.851694+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1186,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.509287+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:03:10.509287+0000","last_peered":"2023-09-23T18:03:10.509287+0000","last_clean":"2023-09-23T18:03:10.509287+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:03:10.509287+0000","last_undegraded":"2023-09-23T18:03:10.509287+0000","last_fullsized":"2023-09-23T18:03:10.509287+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":48,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485809+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:03:10.485809+0000","last_peered":"2023-09-23T18:03:10.485809+0000","last_clean":"2023-09-23T18:03:10.485809+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:03:10.485809+0000","last_undegraded":"2023-09-23T18:03:10.485809+0000","last_fullsized":"2023-09-23T18:03:10.485809+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":151,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494950+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:03:10.494950+0000","last_peered":"2023-09-23T18:03:10.494950+0000","last_clean":"2023-09-23T18:03:10.494950+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:03:10.494950+0000","last_undegraded":"2023-09-23T18:03:10.494950+0000","last_fullsized":"2023-09-23T18:03:10.494950+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"258'5910","reported_seq":5969,"reported_epoch":258,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:22.963784+0000","last_change":"2023-09-23T18:02:19.260251+0000","last_active":"2023-09-23T18:03:22.963784+0000","last_peered":"2023-09-23T18:03:22.963784+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.253817+0000","last_became_peered":"2023-09-23T18:02:19.253817+0000","last_unstale":"2023-09-23T18:03:22.963784+0000","last_undegraded":"2023-09-23T18:02:18.955613+0000","last_fullsized":"2023-09-23T18:02:18.923894+0000","mapping_epoch":238,"log_start":"218'1874","ondisk_log_start":"218'1874","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4036,"log_dups_size":0,"ondisk_log_size":4036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:18.999679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219807744,"num_objects":3354,"num_object_clones":0,"num_object_copies":10062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4277,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5910,"num_write_kb":296448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":-923}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":593,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494271+0000","last_change":"2023-09-23T18:02:51.054622+0000","last_active":"2023-09-23T18:03:10.494271+0000","last_peered":"2023-09-23T18:03:10.494271+0000","last_clean":"2023-09-23T18:03:10.494271+0000","last_became_active":"2023-09-23T18:02:51.049345+0000","last_became_peered":"2023-09-23T18:02:51.049345+0000","last_unstale":"2023-09-23T18:03:10.494271+0000","last_undegraded":"2023-09-23T18:03:10.494271+0000","last_fullsized":"2023-09-23T18:03:10.494271+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:58.636010+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":540,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.493785+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:03:10.493785+0000","last_peered":"2023-09-23T18:03:10.493785+0000","last_clean":"2023-09-23T18:03:10.493785+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:03:10.493785+0000","last_undegraded":"2023-09-23T18:03:10.493785+0000","last_fullsized":"2023-09-23T18:03:10.493785+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":152,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494826+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:03:10.494826+0000","last_peered":"2023-09-23T18:03:10.494826+0000","last_clean":"2023-09-23T18:03:10.494826+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:03:10.494826+0000","last_undegraded":"2023-09-23T18:03:10.494826+0000","last_fullsized":"2023-09-23T18:03:10.494826+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":225,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.510653+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:03:10.510653+0000","last_peered":"2023-09-23T18:03:10.510653+0000","last_clean":"2023-09-23T18:03:10.510653+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:03:10.510653+0000","last_undegraded":"2023-09-23T18:03:10.510653+0000","last_fullsized":"2023-09-23T18:03:10.510653+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"258'5858","reported_seq":5954,"reported_epoch":258,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:22.536490+0000","last_change":"2023-09-23T18:02:47.986654+0000","last_active":"2023-09-23T18:03:22.536490+0000","last_peered":"2023-09-23T18:03:22.536490+0000","last_clean":"2023-09-23T18:02:16.962656+0000","last_became_active":"2023-09-23T18:02:19.272896+0000","last_became_peered":"2023-09-23T18:02:19.272896+0000","last_unstale":"2023-09-23T18:03:22.536490+0000","last_undegraded":"2023-09-23T18:02:18.972349+0000","last_fullsized":"2023-09-23T18:02:18.930868+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5858,"log_dups_size":0,"ondisk_log_size":5858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:23.745675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":221642752,"num_objects":3382,"num_object_clones":0,"num_object_copies":10146,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3382,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5858,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":3026},{"shards":"4,5","objects":-881}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1170,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.508728+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:03:10.508728+0000","last_peered":"2023-09-23T18:03:10.508728+0000","last_clean":"2023-09-23T18:03:10.508728+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:03:10.508728+0000","last_undegraded":"2023-09-23T18:03:10.508728+0000","last_fullsized":"2023-09-23T18:03:10.508728+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":581,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494048+0000","last_change":"2023-09-23T18:02:51.054022+0000","last_active":"2023-09-23T18:03:10.494048+0000","last_peered":"2023-09-23T18:03:10.494048+0000","last_clean":"2023-09-23T18:03:10.494048+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:03:10.494048+0000","last_undegraded":"2023-09-23T18:03:10.494048+0000","last_fullsized":"2023-09-23T18:03:10.494048+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":65,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496770+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:03:10.496770+0000","last_peered":"2023-09-23T18:03:10.496770+0000","last_clean":"2023-09-23T18:03:10.496770+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:03:10.496770+0000","last_undegraded":"2023-09-23T18:03:10.496770+0000","last_fullsized":"2023-09-23T18:03:10.496770+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":191,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.475320+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:03:10.475320+0000","last_peered":"2023-09-23T18:03:10.475320+0000","last_clean":"2023-09-23T18:03:10.475320+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:03:10.475320+0000","last_undegraded":"2023-09-23T18:03:10.475320+0000","last_fullsized":"2023-09-23T18:03:10.475320+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"258'5814","reported_seq":5970,"reported_epoch":258,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:03:22.525686+0000","last_change":"2023-09-23T18:02:18.003772+0000","last_active":"2023-09-23T18:03:22.525686+0000","last_peered":"2023-09-23T18:03:22.525686+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:18.000696+0000","last_became_peered":"2023-09-23T18:02:18.000696+0000","last_unstale":"2023-09-23T18:03:22.525686+0000","last_undegraded":"2023-09-23T18:02:17.938873+0000","last_fullsized":"2023-09-23T18:02:17.919669+0000","mapping_epoch":237,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5714,"log_dups_size":0,"ondisk_log_size":5714,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:48.085957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214171648,"num_objects":3268,"num_object_clones":0,"num_object_copies":9804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3237,"num_objects_misplaced":3268,"num_objects_unfound":0,"num_objects_dirty":3268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5814,"num_write_kb":290624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":3268}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":593,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494424+0000","last_change":"2023-09-23T18:02:51.061168+0000","last_active":"2023-09-23T18:03:10.494424+0000","last_peered":"2023-09-23T18:03:10.494424+0000","last_clean":"2023-09-23T18:03:10.494424+0000","last_became_active":"2023-09-23T18:02:51.060689+0000","last_became_peered":"2023-09-23T18:02:51.060689+0000","last_unstale":"2023-09-23T18:03:10.494424+0000","last_undegraded":"2023-09-23T18:03:10.494424+0000","last_fullsized":"2023-09-23T18:03:10.494424+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:29.406460+0000","scrub_duration":0.001568505,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":438,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494268+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:03:10.494268+0000","last_peered":"2023-09-23T18:03:10.494268+0000","last_clean":"2023-09-23T18:03:10.494268+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:03:10.494268+0000","last_undegraded":"2023-09-23T18:03:10.494268+0000","last_fullsized":"2023-09-23T18:03:10.494268+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":256,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.499964+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:03:10.499964+0000","last_peered":"2023-09-23T18:03:10.499964+0000","last_clean":"2023-09-23T18:03:10.499964+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:03:10.499964+0000","last_undegraded":"2023-09-23T18:03:10.499964+0000","last_fullsized":"2023-09-23T18:03:10.499964+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":227,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.501306+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:03:10.501306+0000","last_peered":"2023-09-23T18:03:10.501306+0000","last_clean":"2023-09-23T18:03:10.501306+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:03:10.501306+0000","last_undegraded":"2023-09-23T18:03:10.501306+0000","last_fullsized":"2023-09-23T18:03:10.501306+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"258'5794","reported_seq":14133,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T18:03:20.398436+0000","last_change":"2023-09-23T18:02:18.942886+0000","last_active":"2023-09-23T18:03:20.398436+0000","last_peered":"2023-09-23T18:03:20.398436+0000","last_clean":"2023-09-23T18:03:20.398436+0000","last_became_active":"2023-09-23T18:02:18.936691+0000","last_became_peered":"2023-09-23T18:02:18.936691+0000","last_unstale":"2023-09-23T18:03:20.398436+0000","last_undegraded":"2023-09-23T18:03:20.398436+0000","last_fullsized":"2023-09-23T18:03:20.398436+0000","mapping_epoch":238,"log_start":"217'1658","ondisk_log_start":"217'1658","created":204,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4136,"log_dups_size":0,"ondisk_log_size":4136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:26.085859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":225574912,"num_objects":3442,"num_object_clones":0,"num_object_copies":10326,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3442,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5794,"num_write_kb":295552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4123,"num_bytes_recovered":270204928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":590,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496009+0000","last_change":"2023-09-23T18:02:51.057370+0000","last_active":"2023-09-23T18:03:10.496009+0000","last_peered":"2023-09-23T18:03:10.496009+0000","last_clean":"2023-09-23T18:03:10.496009+0000","last_became_active":"2023-09-23T18:02:51.056158+0000","last_became_peered":"2023-09-23T18:02:51.056158+0000","last_unstale":"2023-09-23T18:03:10.496009+0000","last_undegraded":"2023-09-23T18:03:10.496009+0000","last_fullsized":"2023-09-23T18:03:10.496009+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":472,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496787+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:03:10.496787+0000","last_peered":"2023-09-23T18:03:10.496787+0000","last_clean":"2023-09-23T18:03:10.496787+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:03:10.496787+0000","last_undegraded":"2023-09-23T18:03:10.496787+0000","last_fullsized":"2023-09-23T18:03:10.496787+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":192,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.495026+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:03:10.495026+0000","last_peered":"2023-09-23T18:03:10.495026+0000","last_clean":"2023-09-23T18:03:10.495026+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:03:10.495026+0000","last_undegraded":"2023-09-23T18:03:10.495026+0000","last_fullsized":"2023-09-23T18:03:10.495026+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":224,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.500269+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:03:10.500269+0000","last_peered":"2023-09-23T18:03:10.500269+0000","last_clean":"2023-09-23T18:03:10.500269+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:03:10.500269+0000","last_undegraded":"2023-09-23T18:03:10.500269+0000","last_fullsized":"2023-09-23T18:03:10.500269+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"258'5755","reported_seq":5842,"reported_epoch":258,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:22.903242+0000","last_change":"2023-09-23T18:02:19.279112+0000","last_active":"2023-09-23T18:03:22.903242+0000","last_peered":"2023-09-23T18:03:22.903242+0000","last_clean":"2023-09-23T18:02:17.018900+0000","last_became_active":"2023-09-23T18:02:19.267439+0000","last_became_peered":"2023-09-23T18:02:19.267439+0000","last_unstale":"2023-09-23T18:03:22.903242+0000","last_undegraded":"2023-09-23T18:02:18.959329+0000","last_fullsized":"2023-09-23T18:02:18.926801+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5755,"log_dups_size":0,"ondisk_log_size":5755,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.395583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":220135424,"num_objects":3359,"num_object_clones":0,"num_object_copies":10077,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3359,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5755,"num_write_kb":291648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-840},{"shards":"2,4,6","objects":3001}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":590,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496613+0000","last_change":"2023-09-23T18:02:51.058680+0000","last_active":"2023-09-23T18:03:10.496613+0000","last_peered":"2023-09-23T18:03:10.496613+0000","last_clean":"2023-09-23T18:03:10.496613+0000","last_became_active":"2023-09-23T18:02:51.057706+0000","last_became_peered":"2023-09-23T18:02:51.057706+0000","last_unstale":"2023-09-23T18:03:10.496613+0000","last_undegraded":"2023-09-23T18:03:10.496613+0000","last_fullsized":"2023-09-23T18:03:10.496613+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":103,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.501506+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:03:10.501506+0000","last_peered":"2023-09-23T18:03:10.501506+0000","last_clean":"2023-09-23T18:03:10.501506+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:03:10.501506+0000","last_undegraded":"2023-09-23T18:03:10.501506+0000","last_fullsized":"2023-09-23T18:03:10.501506+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":328,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.474480+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:03:10.474480+0000","last_peered":"2023-09-23T18:03:10.474480+0000","last_clean":"2023-09-23T18:03:10.474480+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:03:10.474480+0000","last_undegraded":"2023-09-23T18:03:10.474480+0000","last_fullsized":"2023-09-23T18:03:10.474480+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":107,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.497641+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:03:10.497641+0000","last_peered":"2023-09-23T18:03:10.497641+0000","last_clean":"2023-09-23T18:03:10.497641+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:03:10.497641+0000","last_undegraded":"2023-09-23T18:03:10.497641+0000","last_fullsized":"2023-09-23T18:03:10.497641+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"258'5845","reported_seq":5909,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T18:03:22.862810+0000","last_change":"2023-09-23T18:01:25.046224+0000","last_active":"2023-09-23T18:03:22.862810+0000","last_peered":"2023-09-23T18:03:22.862810+0000","last_clean":"2023-09-23T18:03:22.862810+0000","last_became_active":"2023-09-23T18:01:25.002864+0000","last_became_peered":"2023-09-23T18:01:25.002864+0000","last_unstale":"2023-09-23T18:03:22.862810+0000","last_undegraded":"2023-09-23T18:03:22.862810+0000","last_fullsized":"2023-09-23T18:03:22.862810+0000","mapping_epoch":214,"log_start":"218'1841","ondisk_log_start":"218'1841","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4004,"log_dups_size":0,"ondisk_log_size":4004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.752359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215810048,"num_objects":3293,"num_object_clones":0,"num_object_copies":9879,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3293,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5845,"num_write_kb":292416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":597,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494517+0000","last_change":"2023-09-23T18:02:51.053420+0000","last_active":"2023-09-23T18:03:10.494517+0000","last_peered":"2023-09-23T18:03:10.494517+0000","last_clean":"2023-09-23T18:03:10.494517+0000","last_became_active":"2023-09-23T18:02:51.053035+0000","last_became_peered":"2023-09-23T18:02:51.053035+0000","last_unstale":"2023-09-23T18:03:10.494517+0000","last_undegraded":"2023-09-23T18:03:10.494517+0000","last_fullsized":"2023-09-23T18:03:10.494517+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:20.456481+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":463,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.475041+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:03:10.475041+0000","last_peered":"2023-09-23T18:03:10.475041+0000","last_clean":"2023-09-23T18:03:10.475041+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:03:10.475041+0000","last_undegraded":"2023-09-23T18:03:10.475041+0000","last_fullsized":"2023-09-23T18:03:10.475041+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":284,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.501382+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:03:10.501382+0000","last_peered":"2023-09-23T18:03:10.501382+0000","last_clean":"2023-09-23T18:03:10.501382+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:03:10.501382+0000","last_undegraded":"2023-09-23T18:03:10.501382+0000","last_fullsized":"2023-09-23T18:03:10.501382+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":48,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485388+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:03:10.485388+0000","last_peered":"2023-09-23T18:03:10.485388+0000","last_clean":"2023-09-23T18:03:10.485388+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:03:10.485388+0000","last_undegraded":"2023-09-23T18:03:10.485388+0000","last_fullsized":"2023-09-23T18:03:10.485388+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":580,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494788+0000","last_change":"2023-09-23T18:02:51.046571+0000","last_active":"2023-09-23T18:03:10.494788+0000","last_peered":"2023-09-23T18:03:10.494788+0000","last_clean":"2023-09-23T18:03:10.494788+0000","last_became_active":"2023-09-23T18:02:51.046213+0000","last_became_peered":"2023-09-23T18:02:51.046213+0000","last_unstale":"2023-09-23T18:03:10.494788+0000","last_undegraded":"2023-09-23T18:03:10.494788+0000","last_fullsized":"2023-09-23T18:03:10.494788+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":244,"reported_epoch":258,"state":"active+clean","last_fresh":"2023-09-23T18:03:18.977896+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:03:18.977896+0000","last_peered":"2023-09-23T18:03:18.977896+0000","last_clean":"2023-09-23T18:03:18.977896+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:03:18.977896+0000","last_undegraded":"2023-09-23T18:03:18.977896+0000","last_fullsized":"2023-09-23T18:03:18.977896+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":270,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.495092+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:03:10.495092+0000","last_peered":"2023-09-23T18:03:10.495092+0000","last_clean":"2023-09-23T18:03:10.495092+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:03:10.495092+0000","last_undegraded":"2023-09-23T18:03:10.495092+0000","last_fullsized":"2023-09-23T18:03:10.495092+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":589,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.495969+0000","last_change":"2023-09-23T18:02:51.060198+0000","last_active":"2023-09-23T18:03:10.495969+0000","last_peered":"2023-09-23T18:03:10.495969+0000","last_clean":"2023-09-23T18:03:10.495969+0000","last_became_active":"2023-09-23T18:02:51.057046+0000","last_became_peered":"2023-09-23T18:02:51.057046+0000","last_unstale":"2023-09-23T18:03:10.495969+0000","last_undegraded":"2023-09-23T18:03:10.495969+0000","last_fullsized":"2023-09-23T18:03:10.495969+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":113,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.509212+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:03:10.509212+0000","last_peered":"2023-09-23T18:03:10.509212+0000","last_clean":"2023-09-23T18:03:10.509212+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:03:10.509212+0000","last_undegraded":"2023-09-23T18:03:10.509212+0000","last_fullsized":"2023-09-23T18:03:10.509212+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":183,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485970+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:03:10.485970+0000","last_peered":"2023-09-23T18:03:10.485970+0000","last_clean":"2023-09-23T18:03:10.485970+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:03:10.485970+0000","last_undegraded":"2023-09-23T18:03:10.485970+0000","last_fullsized":"2023-09-23T18:03:10.485970+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":99,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.508840+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:03:10.508840+0000","last_peered":"2023-09-23T18:03:10.508840+0000","last_clean":"2023-09-23T18:03:10.508840+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:03:10.508840+0000","last_undegraded":"2023-09-23T18:03:10.508840+0000","last_fullsized":"2023-09-23T18:03:10.508840+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":122,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496598+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:03:10.496598+0000","last_peered":"2023-09-23T18:03:10.496598+0000","last_clean":"2023-09-23T18:03:10.496598+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:03:10.496598+0000","last_undegraded":"2023-09-23T18:03:10.496598+0000","last_fullsized":"2023-09-23T18:03:10.496598+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":605,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.494772+0000","last_change":"2023-09-23T18:02:51.047476+0000","last_active":"2023-09-23T18:03:10.494772+0000","last_peered":"2023-09-23T18:03:10.494772+0000","last_clean":"2023-09-23T18:03:10.494772+0000","last_became_active":"2023-09-23T18:02:51.047152+0000","last_became_peered":"2023-09-23T18:02:51.047152+0000","last_unstale":"2023-09-23T18:03:10.494772+0000","last_undegraded":"2023-09-23T18:03:10.494772+0000","last_fullsized":"2023-09-23T18:03:10.494772+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:22.514077+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":122,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496652+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:03:10.496652+0000","last_peered":"2023-09-23T18:03:10.496652+0000","last_clean":"2023-09-23T18:03:10.496652+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:03:10.496652+0000","last_undegraded":"2023-09-23T18:03:10.496652+0000","last_fullsized":"2023-09-23T18:03:10.496652+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":125,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496269+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:03:10.496269+0000","last_peered":"2023-09-23T18:03:10.496269+0000","last_clean":"2023-09-23T18:03:10.496269+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:03:10.496269+0000","last_undegraded":"2023-09-23T18:03:10.496269+0000","last_fullsized":"2023-09-23T18:03:10.496269+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":588,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.496612+0000","last_change":"2023-09-23T18:02:51.058825+0000","last_active":"2023-09-23T18:03:10.496612+0000","last_peered":"2023-09-23T18:03:10.496612+0000","last_clean":"2023-09-23T18:03:10.496612+0000","last_became_active":"2023-09-23T18:02:51.058480+0000","last_became_peered":"2023-09-23T18:02:51.058480+0000","last_unstale":"2023-09-23T18:03:10.496612+0000","last_undegraded":"2023-09-23T18:03:10.496612+0000","last_fullsized":"2023-09-23T18:03:10.496612+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":236,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.502055+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:03:10.502055+0000","last_peered":"2023-09-23T18:03:10.502055+0000","last_clean":"2023-09-23T18:03:10.502055+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:03:10.502055+0000","last_undegraded":"2023-09-23T18:03:10.502055+0000","last_fullsized":"2023-09-23T18:03:10.502055+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":268,"reported_epoch":257,"state":"active+clean","last_fresh":"2023-09-23T18:03:10.485418+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:03:10.485418+0000","last_peered":"2023-09-23T18:03:10.485418+0000","last_clean":"2023-09-23T18:03:10.485418+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:03:10.485418+0000","last_undegraded":"2023-09-23T18:03:10.485418+0000","last_fullsized":"2023-09-23T18:03:10.485418+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3475374104,"num_objects":53031,"num_object_clones":0,"num_object_copies":159093,"num_objects_missing_on_primary":623,"num_objects_missing":623,"num_objects_degraded":33193,"num_objects_misplaced":12267,"num_objects_unfound":0,"num_objects_dirty":53031,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":93241,"num_write_kb":4680641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16452,"num_bytes_recovered":1078198272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10289754112,"data_stored":10289741896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":80677,"ondisk_log_size":80677,"up":48,"acting":38,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":402523,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":331,"num_read_kb":623,"num_write":162,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2224465,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":91,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001369,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1988308,"kb_used_data":1906124,"kb_used_omap":13,"kb_used_meta":82162,"kb_avail":91756844,"statfs":{"total":95995035648,"available":93959008256,"internally_reserved":0,"allocated":1951870976,"data_stored":1950883377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":84133992},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":2.544,"5min":2.544,"15min":2.544},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.2869999999999999,"5min":1.2869999999999999,"15min":1.2869999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":0.45600000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":2.629,"5min":2.629,"15min":2.629},"last":0.27400000000000002},{"interface":"front","average":{"1min":1.056,"5min":1.056,"15min":1.056},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":4.5979999999999999,"5min":4.5979999999999999,"15min":4.5979999999999999},"last":0.35199999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":0.93799999999999994,"15min":0.93799999999999994},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.081,"5min":3.081,"15min":3.081},"last":0.39200000000000002},{"interface":"front","average":{"1min":0.81299999999999994,"5min":0.81299999999999994,"15min":0.81299999999999994},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.6680000000000001,"5min":2.6680000000000001,"15min":2.6680000000000001},"last":0.51500000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:03:19 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.554,"15min":1.554},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":4.2670000000000003,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":0.377},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":3.0950000000000002,"5min":3.0950000000000002,"15min":3.0950000000000002},"last":0.58199999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.0720000000000001,"15min":1.0720000000000001},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":3.984,"5min":3.984,"15min":3.984},"last":0.33900000000000002},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.52600000000000002,"15min":0.52600000000000002},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":1.645,"5min":1.645,"15min":1.645},"last":0.64000000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":1.004,"5min":1.004,"15min":1.004},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":3.9300000000000002,"5min":3.9300000000000002,"15min":3.9300000000000002},"last":1.6890000000000001},{"interface":"front","average":{"1min":1.133,"5min":1.133,"15min":1.133},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.7080000000000002,"5min":3.7080000000000002,"15min":3.7080000000000002},"last":0.55000000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:02:27 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":0.70599999999999996,"15min":0.70599999999999996},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.0409999999999999,"5min":3.0409999999999999,"15min":3.0409999999999999},"last":0.49099999999999999},{"interface":"front","average":{"1min":1.1259999999999999,"5min":1.1259999999999999,"15min":1.1259999999999999},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":3.6419999999999999,"5min":3.6419999999999999,"15min":3.6419999999999999},"last":0.65000000000000002}]}]},{"osd":6,"up_from":163,"seq":700079669305,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2270352,"kb_used_data":2121928,"kb_used_omap":7,"kb_used_meta":148408,"kb_avail":91474800,"statfs":{"total":95995035648,"available":93670195200,"internally_reserved":0,"allocated":2172854272,"data_stored":2171871186,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":151970596},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.2270000000000001,"15min":1.3089999999999999},"min":{"1min":0.224,"5min":0.215,"15min":0.215},"max":{"1min":3.5529999999999999,"5min":14.196999999999999,"15min":14.196999999999999},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.72,"15min":2.3740000000000001},"min":{"1min":0.32600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.8970000000000002,"5min":13.536,"15min":13.536},"last":0.52500000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.5860000000000001,"15min":1.956},"min":{"1min":0.28100000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.4910000000000001,"5min":14.233000000000001,"15min":14.233000000000001},"last":0.316},{"interface":"front","average":{"1min":1.01,"5min":1.6759999999999999,"15min":2.173},"min":{"1min":0.33300000000000002,"5min":0.248,"15min":0.248},"max":{"1min":4.9340000000000002,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":0.58099999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":2.0009999999999999,"15min":2.3450000000000002},"min":{"1min":0.34999999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.173,"5min":11.210000000000001,"15min":11.210000000000001},"last":0.501},{"interface":"front","average":{"1min":1.0589999999999999,"5min":2.1899999999999999,"15min":2.758},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.9700000000000002,"5min":11.223000000000001,"15min":11.223000000000001},"last":0.38800000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":3.081,"5min":3.081,"15min":3.081},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":5.04,"5min":5.04,"15min":5.04},"last":0.54300000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.96599999999999997,"15min":0.89400000000000002},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.827,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.35499999999999998},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.72499999999999998,"15min":0.503},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.0129999999999999,"5min":8.3000000000000007,"15min":8.3000000000000007},"last":0.433}]},{"osd":5,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.94199999999999995,"15min":0.84399999999999997},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.1050000000000004,"5min":11.869,"15min":11.869},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.21,"5min":0.875,"15min":0.63500000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.7130000000000001,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":0.46899999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:02:24 2023","interfaces":[{"interface":"back","average":{"1min":0.96399999999999997,"5min":0.96399999999999997,"15min":0.96399999999999997},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.6469999999999998,"5min":3.6469999999999998,"15min":3.6469999999999998},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.5310000000000001,"5min":2.5310000000000001,"15min":2.5310000000000001},"last":0.68799999999999994}]}]},{"osd":1,"up_from":0,"seq":420906795124,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958651,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1249332,"kb_used_data":847596,"kb_used_omap":7,"kb_used_meta":401720,"kb_avail":92495820,"statfs":{"total":95995035648,"available":94715719680,"internally_reserved":0,"allocated":867938304,"data_stored":866634230,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":411362020},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:03:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":2.0059999999999998,"15min":1.135},"min":{"1min":0.20300000000000001,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":7.7000000000000002,"5min":18.154,"15min":18.154},"last":0.91500000000000004},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.871,"15min":1.1080000000000001},"min":{"1min":0.29599999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":6.8550000000000004,"5min":13.369999999999999,"15min":13.369999999999999},"last":0.93000000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.4460000000000002,"15min":1.24},"min":{"1min":0.25700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.5259999999999998,"5min":20.882000000000001,"15min":20.882000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.234,"5min":2.0899999999999999,"15min":1.149},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":6.7439999999999998,"5min":15.064,"15min":15.064},"last":0.35399999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.048,"5min":6.048,"15min":6.048},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.2010000000000001,"15min":1.2010000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.391,"5min":4.391,"15min":4.391},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":1.603,"15min":1.0509999999999999},"min":{"1min":0.29299999999999998,"5min":0.251,"15min":0.251},"max":{"1min":4.54,"5min":15.279999999999999,"15min":15.279999999999999},"last":2.948},{"interface":"front","average":{"1min":1.24,"5min":1.859,"15min":1.1220000000000001},"min":{"1min":0.29899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7190000000000003,"5min":18.106999999999999,"15min":18.106999999999999},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.9710000000000001,"15min":1.1619999999999999},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.1280000000000001,"5min":14.720000000000001,"15min":14.720000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.5469999999999999,"15min":1.022},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":2.806,"5min":12.794,"15min":12.794},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.8680000000000001,"15min":2.3010000000000002},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.5069999999999997,"5min":14.956,"15min":14.956},"last":0.505},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.0419999999999998,"15min":2.5219999999999998},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.311},"max":{"1min":2.883,"5min":12.755000000000001,"15min":12.755000000000001},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:02:28 2023","interfaces":[{"interface":"back","average":{"1min":0.92300000000000004,"5min":0.92300000000000004,"15min":0.92300000000000004},"min":{"1min":0.40799999999999997,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":2.851,"5min":2.851,"15min":2.851},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.083,"5min":1.083,"15min":1.083},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.4619999999999997,"5min":4.4619999999999997,"15min":4.4619999999999997},"last":2.7799999999999998}]}]},{"osd":2,"up_from":103,"seq":442381631598,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2269352,"kb_used_data":1909920,"kb_used_omap":11,"kb_used_meta":359412,"kb_avail":91475800,"statfs":{"total":95995035648,"available":93671219200,"internally_reserved":0,"allocated":1955758080,"data_stored":1954466145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":368038263},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":2.552,"15min":1.2729999999999999},"min":{"1min":0.187,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":3.306,"5min":20.867000000000001,"15min":20.867000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":2.3119999999999998,"15min":1.246},"min":{"1min":0.24299999999999999,"5min":0.215,"15min":0.215},"max":{"1min":1.6910000000000001,"5min":21.681000000000001,"15min":21.681000000000001},"last":0.39200000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.6349999999999998,"15min":1.361},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":4.2359999999999998,"5min":20.995000000000001,"15min":20.995000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.75,"5min":1.9530000000000001,"15min":1.147},"min":{"1min":0.24199999999999999,"5min":0.193,"15min":0.193},"max":{"1min":4.532,"5min":15.945,"15min":15.945},"last":0.41899999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:03:21 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.6040000000000001,"5min":3.6040000000000001,"15min":3.6040000000000001},"last":0.379},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.0790000000000002,"5min":3.0790000000000002,"15min":3.0790000000000002},"last":1.419}]},{"osd":4,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.7649999999999999,"15min":1.095},"min":{"1min":0.29099999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.141,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":1.5489999999999999},{"interface":"front","average":{"1min":0.45900000000000002,"5min":1.8899999999999999,"15min":1.1200000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78100000000000003,"5min":20.920000000000002,"15min":20.920000000000002},"last":0.60299999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.9890000000000001,"15min":1.1879999999999999},"min":{"1min":0.32900000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.972,"5min":20.738,"15min":20.738},"last":0.622},{"interface":"front","average":{"1min":1.048,"5min":1.79,"15min":1.1970000000000001},"min":{"1min":0.34100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3700000000000001,"5min":14.359,"15min":15.496},"last":0.495}]},{"osd":6,"last update":"Sat Sep 23 18:02:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":2.0299999999999998,"15min":2.2240000000000002},"min":{"1min":0.34699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":5.1879999999999997,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":1.5209999999999999},{"interface":"front","average":{"1min":0.66900000000000004,"5min":2.3290000000000002,"15min":2.4929999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.6930000000000001,"5min":20.893000000000001,"15min":20.893000000000001},"last":0.63900000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:02:25 2023","interfaces":[{"interface":"back","average":{"1min":1.411,"5min":1.411,"15min":1.411},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":4.3239999999999998,"5min":4.3239999999999998,"15min":4.3239999999999998},"last":0.51700000000000002},{"interface":"front","average":{"1min":0.63300000000000001,"5min":0.63300000000000001,"15min":0.63300000000000001},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.7130000000000001,"5min":1.7130000000000001,"15min":1.7130000000000001},"last":0.57199999999999995}]}]},{"osd":3,"up_from":237,"seq":1017907249166,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1251056,"kb_used_data":1168936,"kb_used_omap":3,"kb_used_meta":82108,"kb_avail":92494096,"statfs":{"total":95995035648,"available":94713954304,"internally_reserved":0,"allocated":1196990464,"data_stored":1195520226,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":84079336},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.6560000000000001,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":2.4239999999999999},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":4.5529999999999999,"5min":4.5529999999999999,"15min":4.5529999999999999},"last":0.40999999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.6050000000000004,"5min":4.6050000000000004,"15min":4.6050000000000004},"last":0.44},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.8810000000000002,"5min":4.8810000000000002,"15min":4.8810000000000002},"last":1.2410000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.7639999999999998,"5min":2.7639999999999998,"15min":2.7639999999999998},"last":2.7639999999999998},{"interface":"front","average":{"1min":1.246,"5min":1.246,"15min":1.246},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.157,"5min":6.157,"15min":6.157},"last":2.79}]},{"osd":4,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.633,"5min":4.633,"15min":4.633},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.7300000000000004,"5min":4.7300000000000004,"15min":4.7300000000000004},"last":0.59699999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.284,"15min":1.284},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":7.0640000000000001,"5min":7.0640000000000001,"15min":7.0640000000000001},"last":4.0389999999999997},{"interface":"front","average":{"1min":1.181,"5min":1.181,"15min":1.181},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.6150000000000002,"5min":4.6150000000000002,"15min":4.6150000000000002},"last":2.6720000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.085,"15min":1.085},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.0590000000000002,"5min":4.0590000000000002,"15min":4.0590000000000002},"last":2.7040000000000002},{"interface":"front","average":{"1min":1.48,"5min":1.48,"15min":1.48},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":7.3929999999999998,"5min":7.3929999999999998,"15min":7.3929999999999998},"last":0.46500000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.5720000000000001,"15min":1.5720000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":8.4190000000000005,"5min":8.4190000000000005,"15min":8.4190000000000005},"last":0.95099999999999996},{"interface":"front","average":{"1min":1.6200000000000001,"5min":1.6200000000000001,"15min":1.6200000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":5.1760000000000002,"5min":5.1760000000000002,"15min":5.1760000000000002},"last":2.6269999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019116,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1422356,"kb_used_data":1216844,"kb_used_omap":0,"kb_used_meta":205504,"kb_avail":92322796,"statfs":{"total":95995035648,"available":94538543104,"internally_reserved":0,"allocated":1246048256,"data_stored":1245060910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":210436096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,1,3,1,4,6],"upper_bound":512},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.3260000000000001,"15min":0.82099999999999995},"min":{"1min":0.24399999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.1829999999999998,"5min":14.164,"15min":14.164},"last":0.45700000000000002},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.611,"15min":0.97899999999999998},"min":{"1min":0.253,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.29,"5min":11.372,"15min":11.372},"last":0.28899999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:03:04 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.532,"15min":1.002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.249},"max":{"1min":5.2190000000000003,"5min":10.429,"15min":10.429},"last":0.30399999999999999},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.46,"15min":1.0629999999999999},"min":{"1min":0.28699999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.2190000000000003,"5min":10.519,"15min":10.83},"last":0.48099999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.6699999999999999,"15min":1.0629999999999999},"min":{"1min":0.29899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.266,"5min":11.630000000000001,"15min":11.630000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.5289999999999999,"15min":0.98799999999999999},"min":{"1min":0.316,"5min":0.30099999999999999,"15min":0.27100000000000002},"max":{"1min":5.2450000000000001,"5min":7.5090000000000003,"15min":7.5090000000000003},"last":0.495}]},{"osd":3,"last update":"Sat Sep 23 18:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.351,"15min":1.351},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":7.2140000000000004,"5min":7.2140000000000004,"15min":7.2140000000000004},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.373,"5min":1.373,"15min":1.373},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.1970000000000001,"5min":5.1970000000000001,"15min":5.1970000000000001},"last":0.40699999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.80800000000000005,"15min":0.59299999999999997},"min":{"1min":0.23799999999999999,"5min":0.224,"15min":0.224},"max":{"1min":3.2269999999999999,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":0.25900000000000001},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0029999999999999,"15min":0.71299999999999997},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0629999999999997,"5min":7.4729999999999999,"15min":10.494},"last":0.32700000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":0.94499999999999995,"15min":0.82799999999999996},"min":{"1min":0.25,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.5579999999999998,"5min":7.1529999999999996,"15min":7.1529999999999996},"last":0.379},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0680000000000001,"15min":0.95599999999999996},"min":{"1min":0.28999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.286,"5min":10.725,"15min":10.725},"last":0.34300000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:02:25 2023","interfaces":[{"interface":"back","average":{"1min":0.91400000000000003,"5min":0.91400000000000003,"15min":0.91400000000000003},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":3.8839999999999999,"5min":3.8839999999999999,"15min":3.8839999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.2130000000000001,"5min":1.2130000000000001,"15min":1.2130000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":5.8090000000000002,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":1.611}]}]},{"osd":5,"up_from":38,"seq":163208757482,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1358984,"kb_used_data":1162560,"kb_used_omap":0,"kb_used_meta":196416,"kb_avail":92386168,"statfs":{"total":95995035648,"available":94603436032,"internally_reserved":0,"allocated":1190461440,"data_stored":1189481846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":201129984},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.6779999999999999,"15min":1.008},"min":{"1min":0.28699999999999998,"5min":0.217,"15min":0.217},"max":{"1min":4.0250000000000004,"5min":21.376999999999999,"15min":21.376999999999999},"last":0.443},{"interface":"front","average":{"1min":1.996,"5min":1.8939999999999999,"15min":1.149},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.048,"5min":21.388999999999999,"15min":21.388999999999999},"last":0.39000000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.054,"15min":0.85899999999999999},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":1.321,"5min":10.853999999999999,"15min":10.853999999999999},"last":0.375},{"interface":"front","average":{"1min":1.05,"5min":1.2969999999999999,"15min":0.95399999999999996},"min":{"1min":0.29199999999999998,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":3.6659999999999999,"5min":10.206,"15min":10.206},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.1320000000000001,"15min":1.2589999999999999},"min":{"1min":0.34200000000000003,"5min":0.255,"15min":0.255},"max":{"1min":4.9279999999999999,"5min":26.035,"15min":38.606999999999999},"last":0.435},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.534,"15min":1.123},"min":{"1min":0.35999999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.885,"5min":17.085000000000001,"15min":39.670999999999999},"last":0.312}]},{"osd":3,"last update":"Sat Sep 23 18:03:19 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.6179999999999999,"5min":2.6179999999999999,"15min":2.6179999999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.7850000000000001,"5min":2.7850000000000001,"15min":2.7850000000000001},"last":0.48799999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":0.94499999999999995,"15min":0.69699999999999995},"min":{"1min":0.24199999999999999,"5min":0.22800000000000001,"15min":0.224},"max":{"1min":3.9980000000000002,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.26100000000000001},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.998,"15min":0.69799999999999995},"min":{"1min":0.30499999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.1449999999999996,"5min":5.077,"15min":5.077},"last":0.39900000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.343,"15min":1.5069999999999999},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.5880000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.147,"15min":1.329},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.1710000000000003,"5min":8.266,"15min":8.266},"last":0.40799999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:02:26 2023","interfaces":[{"interface":"back","average":{"1min":1.546,"5min":1.546,"15min":1.546},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":4.2050000000000001,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.125,"5min":1.125,"15min":1.125},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":4.0380000000000003,"5min":4.0380000000000003,"15min":4.0380000000000003},"last":0.47099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":856690688,"data_stored":856686616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1343881216,"data_stored":1343881216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1934233600,"data_stored":1934229528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":277282816,"data_stored":277282816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1234173952,"data_stored":1234173952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1177882624,"data_stored":1177878552,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2160394240,"data_stored":2160394240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1305214976,"data_stored":1305214976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:03:28.143 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:03:28.144 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:28.223 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:28.254 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:28.398 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:28.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:28.403 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:28.504 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:03:28.538 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:28.642 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:03:28.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:28.645 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:03:28.740 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:03:28.740 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:67505 6.c 6.b5479fac (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:17.174047+0000", 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 71.563702699000004, 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75978317699999998, 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:03:28.741 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67505 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174047+0000", 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174047+0000", 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174045+0000", 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174088+0000", 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2586000000000002e-05 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.743 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174099+0000", 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0993e-05 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174112+0000", 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2860000000000001e-05 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174191+0000", 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9517000000000005e-05 2023-09-23T18:03:28.744 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174204+0000", 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2486999999999999e-05 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.933818+0000", 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75961363900000001 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.933830+0000", 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2374999999999999e-05 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:70268 6.8 e240/238 6:10815b96:::benchmark_data_smithi055_275963_object70267:head v 240'4428, mlcod=0'0)", 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:26.478221+0000", 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 62.259528050999997, 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.039445864999999997, 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.478221+0000", 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.478221+0000", 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.478220+0000", 2023-09-23T18:03:28.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.478245+0000", 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4902999999999999e-05 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.478267+0000", 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.141e-05 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.478274+0000", 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5120000000000002e-06 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.512005+0000", 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.033730590999999997 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.512035+0000", 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9714e-05 2023-09-23T18:03:28.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.517635+0000", 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055999810000000004 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.517667+0000", 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2719999999999998e-05 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.750 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:70269 6.0 e240/238 6:0f267610:::benchmark_data_smithi055_275963_object70268:head v 240'4269, mlcod=0'0)", 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:26.478827+0000", 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 62.258922151, 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038803173000000003, 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.478827+0000", 2023-09-23T18:03:28.751 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.478827+0000", 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.478827+0000", 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.752 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.478848+0000", 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1027999999999999e-05 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.512020+0000", 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.033172183000000001 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.512028+0000", 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8779999999999998e-06 2023-09-23T18:03:28.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.512067+0000", 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9070999999999999e-05 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.512092+0000", 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5621000000000001e-05 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.754 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.517584+0000", 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0054918500000000004 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.517630+0000", 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6159999999999999e-05 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.755 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:70275 6.5 e240/238 6:ab7905ef:::benchmark_data_smithi055_275963_object70274:head v 240'4436, mlcod=0'0)", 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:26.491698+0000", 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 62.246051960000003, 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024358272, 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.491698+0000", 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.756 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.491698+0000", 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.491696+0000", 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.491804+0000", 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010739 2023-09-23T18:03:28.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.512010+0000", 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020205891 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.512021+0000", 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1218e-05 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.512068+0000", 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6955e-05 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.512085+0000", 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6895999999999999e-05 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.516001+0000", 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039162889999999999 2023-09-23T18:03:28.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:26.516056+0000", 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4753999999999998e-05 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:77864 6.f e251/238 6:f8ea5c61:::benchmark_data_smithi055_275963_object4726:head v 251'4821, mlcod=217'314)", 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:49.090304+0000", 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 39.647445806999997, 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024904899000000001, 2023-09-23T18:03:28.760 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.090304+0000", 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.090304+0000", 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.090303+0000", 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.090321+0000", 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7163000000000002e-05 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.090327+0000", 2023-09-23T18:03:28.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0270000000000001e-06 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.090331+0000", 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4519999999999999e-06 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.101720+0000", 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011388986 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.101750+0000", 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9774999999999999e-05 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.115171+0000", 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013420638 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.764 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.115209+0000", 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8040999999999997e-05 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:77888 6.8 e251/238 6:1498f737:::benchmark_data_smithi055_275963_object4750:head v 251'4924, mlcod=217'395)", 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:49.115293+0000", 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 39.622456931000002, 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024221716000000001, 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.765 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.115293+0000", 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.115293+0000", 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.115291+0000", 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.115299+0000", 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9920000000000007e-06 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.115309+0000", 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5789999999999993e-06 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.115317+0000", 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1489999999999994e-06 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.119824+0000", 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004506815 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.768 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.119848+0000", 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3855000000000001e-05 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.139483+0000", 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019635531000000001 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:49.139514+0000", 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0911999999999998e-05 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.769 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:78817 6.8 e253/238 6:1de86a04:::benchmark_data_smithi055_275963_object5679:head v 253'5002, mlcod=217'514)", 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:51.072024+0000", 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 37.665725893999998, 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024598242999999999, 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.770 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:51.072024+0000", 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:51.072024+0000", 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:51.072023+0000", 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:51.072032+0000", 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8750000000000006e-06 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:51.072048+0000", 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5981000000000002e-05 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:51.072055+0000", 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8569999999999995e-06 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:51.076462+0000", 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044063119999999999 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:51.076486+0000", 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4363999999999999e-05 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:51.096590+0000", 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020103574999999999 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:51.096622+0000", 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2159999999999997e-05 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:79890 6.6 e253/238 6:61d275d9:::benchmark_data_smithi055_275963_object6752:head v 253'4901, mlcod=0'0)", 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:53.896704+0000", 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 34.841045180999998, 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028771218000000001, 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.896704+0000", 2023-09-23T18:03:28.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.896704+0000", 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.896704+0000", 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.896710+0000", 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2389999999999999e-06 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.896718+0000", 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4770000000000003e-06 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.896726+0000", 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7570000000000004e-06 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.896771+0000", 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4391999999999998e-05 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.896796+0000", 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5528999999999999e-05 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.925448+0000", 2023-09-23T18:03:28.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028652105000000001 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.925476+0000", 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7263e-05 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:79895 6.6 e253/238 6:657d8c0d:::benchmark_data_smithi055_275963_object6757:head v 253'4902, mlcod=0'0)", 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:53.899505+0000", 2023-09-23T18:03:28.779 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 34.838244819000003, 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.025994923999999999, 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.899505+0000", 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.780 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.899505+0000", 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.899504+0000", 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.899513+0000", 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2930000000000006e-06 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.899528+0000", 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4701e-05 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.899535+0000", 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3839999999999999e-06 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.902231+0000", 2023-09-23T18:03:28.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026959610000000002 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.902259+0000", 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7495999999999999e-05 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.925478+0000", 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.023219366000000002 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:53.925500+0000", 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1588999999999999e-05 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:84702 6.6 e256/238 6:6441e8b3:::benchmark_data_smithi055_275963_object11564:head v 256'5205, mlcod=0'0)", 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:04.639029+0000", 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 24.098720318000002, 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.035795517999999998, 2023-09-23T18:03:28.784 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.639029+0000", 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.639029+0000", 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.639028+0000", 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.639041+0000", 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2897e-05 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.639057+0000", 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6745999999999999e-05 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.639067+0000", 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6509999999999996e-06 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.639122+0000", 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5191000000000001e-05 2023-09-23T18:03:28.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.639149+0000", 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7002999999999999e-05 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.674792+0000", 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.035643016 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.674825+0000", 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2388999999999998e-05 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:84710 6.6 e256/238 6:68e43675:::benchmark_data_smithi055_275963_object11572:head v 256'5206, mlcod=0'0)", 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:04.641698+0000", 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 24.096051254999999, 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.033142927000000003, 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.789 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.641698+0000", 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.641698+0000", 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.790 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.641697+0000", 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.641705+0000", 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7009999999999996e-06 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.641720+0000", 2023-09-23T18:03:28.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4977000000000001e-05 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.641732+0000", 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2019e-05 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.644168+0000", 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00243604 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.644201+0000", 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3232999999999998e-05 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.674827+0000", 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.030625957999999998 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.674841+0000", 2023-09-23T18:03:28.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3909e-05 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:84713 6.6 e256/238 6:65e6cd3a:::benchmark_data_smithi055_275963_object11575:head v 256'5207, mlcod=0'0)", 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:04.645349+0000", 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 24.092400602000001, 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.029508913000000001, 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.794 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.645349+0000", 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.645349+0000", 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.645348+0000", 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.645356+0000", 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8150000000000006e-06 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.645368+0000", 2023-09-23T18:03:28.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2119e-05 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.645376+0000", 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.354e-06 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.649069+0000", 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036925700000000001 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.649093+0000", 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4403999999999999e-05 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.674845+0000", 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.025751860000000001 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.674858+0000", 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2619999999999999e-05 2023-09-23T18:03:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:84721 6.6 e256/238 6:61e43f74:::benchmark_data_smithi055_275963_object11583:head v 256'5208, mlcod=0'0)", 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:04.647219+0000", 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 24.090530666999999, 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027650279999999999, 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.647219+0000", 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.647219+0000", 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.647218+0000", 2023-09-23T18:03:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.647226+0000", 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4379999999999998e-06 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.647239+0000", 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3312000000000001e-05 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.647247+0000", 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4610000000000008e-06 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.654046+0000", 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0067991040000000003 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.654065+0000", 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8804000000000002e-05 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.674859+0000", 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020794094999999999 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:04.674869+0000", 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7829999999999993e-06 2023-09-23T18:03:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:90383 6.6 e257/238 6:639c8de3:::benchmark_data_smithi055_275963_object17245:head v 257'5578, mlcod=0'0)", 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:17.039736+0000", 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.698013332, 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.029271958000000001, 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.039736+0000", 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.039736+0000", 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.039735+0000", 2023-09-23T18:03:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.039744+0000", 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0710000000000001e-06 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.039754+0000", 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9150000000000008e-06 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.039761+0000", 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1219999999999997e-06 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.047560+0000", 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.007798152 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.047583+0000", 2023-09-23T18:03:28.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3102999999999999e-05 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.068969+0000", 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021386398000000001 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.069008+0000", 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9075000000000001e-05 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.808 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:90408 6.6 e257/238 6:6f236814:::benchmark_data_smithi055_275963_object17270:head v 257'5579, mlcod=0'0)", 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:17.042471+0000", 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.695278158000001, 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026560509999999999, 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.042471+0000", 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.042471+0000", 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.042471+0000", 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:28.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.042477+0000", 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4880000000000004e-06 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.042485+0000", 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1580000000000005e-06 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.042491+0000", 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2939999999999996e-06 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.052964+0000", 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010473443000000001 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.052997+0000", 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2577000000000003e-05 2023-09-23T18:03:28.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.069013+0000", 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016016305000000002 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.069032+0000", 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8797999999999999e-05 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.813 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:90417 6.6 e257/238 6:6924a404:::benchmark_data_smithi055_275963_object17279:head v 257'5580, mlcod=0'0)", 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:17.045377+0000", 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.692372042000001, 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.029435161000000001, 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.045377+0000", 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.045377+0000", 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.045376+0000", 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.045385+0000", 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2770000000000005e-06 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.045402+0000", 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7584999999999999e-05 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.045411+0000", 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0329999999999997e-06 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.058284+0000", 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012872524 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.058310+0000", 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6384e-05 2023-09-23T18:03:28.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.074754+0000", 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016444271 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.074813+0000", 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8242000000000002e-05 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.818 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:90420 6.6 e257/238 6:66858a5e:::benchmark_data_smithi055_275963_object17282:head v 257'5581, mlcod=0'0)", 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:17.047653+0000", 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.690096864000001, 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027194132999999999, 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.819 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.047653+0000", 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.047653+0000", 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.047652+0000", 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.047660+0000", 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0539999999999995e-06 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.047670+0000", 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0499000000000001e-05 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.047679+0000", 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5949999999999999e-06 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.063839+0000", 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016160104000000002 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.063863+0000", 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4380999999999999e-05 2023-09-23T18:03:28.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.074818+0000", 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01095455 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:17.074847+0000", 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8790999999999999e-05 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:92962 6.6 e258/238 6:68645bfd:::benchmark_data_smithi055_275963_object19824:head v 258'5745, mlcod=0'0)", 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:22.431853+0000", 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 6.3058968780000004, 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.025797896000000001, 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:22.431853+0000", 2023-09-23T18:03:28.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:22.431853+0000", 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:22.431852+0000", 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:22.431857+0000", 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6199999999999998e-06 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:22.431862+0000", 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4029999999999998e-06 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:22.431867+0000", 2023-09-23T18:03:28.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2899999999999996e-06 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:22.436752+0000", 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048852619999999996 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:22.436773+0000", 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1197000000000001e-05 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:22.457614+0000", 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020841037 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:22.457650+0000", 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6356999999999998e-05 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.828 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:93932 6.6 e258/238 6:62246214:::benchmark_data_smithi055_275963_object20794:head v 258'5809, mlcod=0'0)", 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:24.543090+0000", 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4.1946591880000001, 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024190618000000001, 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:24.543090+0000", 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:24.543090+0000", 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:24.543090+0000", 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:24.543096+0000", 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3380000000000001e-06 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:24.543107+0000", 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0871e-05 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:24.543114+0000", 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5809999999999999e-06 2023-09-23T18:03:28.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:24.545847+0000", 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002732511 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:24.545877+0000", 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9750000000000001e-05 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:24.567234+0000", 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021357663999999998 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:24.567281+0000", 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6582999999999998e-05 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:94733 6.6 e258/238 6:673c6d8f:::benchmark_data_smithi055_275963_object21595:head v 258'5864, mlcod=0'0)", 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:03:26.221940+0000", 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 2.5158090529999999, 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026196902000000001, 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:26.221940+0000", 2023-09-23T18:03:28.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:26.221940+0000", 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:26.221939+0000", 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:26.221947+0000", 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8849999999999999e-06 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:26.221962+0000", 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4924e-05 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:26.221969+0000", 2023-09-23T18:03:28.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7510000000000002e-06 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:26.222014+0000", 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5086999999999997e-05 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:26.222042+0000", 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7872e-05 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:26.248098+0000", 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026056362999999999 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:03:26.248137+0000", 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9035000000000001e-05 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:28.838 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:28.839 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:03:28.839 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:28.839 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:28.866 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:28.907 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:28.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:28.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:29.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:29.046 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 33195/158328 objects degraded (20.966%), 12 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:29.046 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 33195/158328 objects degraded (20.966%), 12 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:29.048 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:29.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 33195/158328 objects degraded (20.966%), 12 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:29.091 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:29.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:29.099 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:29.199 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:29.203 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:03:29.229 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:29.269 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:29.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:29.273 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:29.374 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:29.449 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:29.722 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:29.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:29.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:29.831 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:03:29.883 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:30.006 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:03:30.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:30.011 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:03:30.112 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:03:30.157 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:29 smithi055 ceph-mon[184174]: pgmap v603: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 32895/153816 objects degraded (21.386%); 11849/153816 objects misplaced (7.703%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:03:30.159 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:29 smithi055 ceph-mon[179668]: pgmap v603: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 32895/153816 objects degraded (21.386%); 11849/153816 objects misplaced (7.703%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:03:30.167 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:30.199 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:30.230 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:03:30.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:30.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:03:30.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:29 smithi099 ceph-mon[167076]: pgmap v603: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 467 op/s; 32895/153816 objects degraded (21.386%); 11849/153816 objects misplaced (7.703%); 7.2 MiB/s, 115 objects/s recovering 2023-09-23T18:03:30.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:03:30.366 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:30.402 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:03:30.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:30.407 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:03:30.414 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:03:30.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:03:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:03:30] "GET /metrics HTTP/1.1" 200 36543 "" "Prometheus/2.43.0" 2023-09-23T18:03:30.430 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:03:30.455 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:03:30.484 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:03:30.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:31.434 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2487603338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:31.434 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:03:31.434 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:03:31.434 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:03:31.435 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:31.435 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:03:31.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:03:31.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:03:31.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:03:31.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:03:31.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:03:31.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:03:31.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:03:31.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:03:31.664 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2487603338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:31.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2487603338' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:31.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:31.667 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:31.667 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:31.667 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:31.768 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:31.888 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:32.097 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:03:32.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:31 smithi055 ceph-mon[179668]: pgmap v604: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 32151/150963 objects degraded (21.297%); 11525/150963 objects misplaced (7.634%); 10 MiB/s, 161 objects/s recovering 2023-09-23T18:03:32.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2721306559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:03:32.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:31 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:03:32.100 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:31 smithi055 ceph-mon[184174]: pgmap v604: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 32151/150963 objects degraded (21.297%); 11525/150963 objects misplaced (7.634%); 10 MiB/s, 161 objects/s recovering 2023-09-23T18:03:32.100 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2721306559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:03:32.100 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:31 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:03:32.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:32.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:32.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:32.201 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:32.471 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:31 smithi099 ceph-mon[167076]: pgmap v604: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.1 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 480 op/s; 32151/150963 objects degraded (21.297%); 11525/150963 objects misplaced (7.634%); 10 MiB/s, 161 objects/s recovering 2023-09-23T18:03:32.472 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2721306559' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:03:32.472 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:31 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:03:32.472 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:03:32.472 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 32151/150963 objects degraded (21.297%), 12 pgs degraded, 10 pgs undersized","count":22},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":702,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":259,"num_osds":8,"num_up_osds":8,"osd_up_since":1695492137,"num_in_osds":7,"osd_in_since":1695492120,"num_remapped_pgs":11},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":106},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":8},{"state_name":"active+undersized+degraded+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovering+undersized+remapped","count":1},{"state_name":"active+recovery_wait+degraded+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":50321,"data_bytes":3300604422,"bytes_used":11633111040,"bytes_avail":660332138496,"bytes_total":671965249536,"degraded_objects":32151,"degraded_total":150963,"degraded_ratio":0.21297271516861749,"misplaced_objects":11525,"misplaced_total":150963,"misplaced_ratio":0.076343209925610916,"recovering_objects_per_sec":161,"recovering_bytes_per_sec":10602881,"recovering_keys_per_sec":0,"num_objects_recovered":1897,"num_bytes_recovered":124321792,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":480},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":23,"modified":"2023-09-23T18:03:16.678968+0000","services":{}},"progress_events":{"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"e54c9148-1053-4ff0-9024-eb4758d365b5":{"message":"Global Recovery Event (7m)\n [========================....] (remaining: 53s)","progress":0.89075630903244019,"add_to_ceph_s":true}}} 2023-09-23T18:03:32.505 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:32.538 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:03:32.539 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:03:32.547 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:32.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:32.810 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:32.911 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:03:33.134 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:03:34.445 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:03:34.446 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:03:34.446 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[179668]: failed to encode map e259 with expected crc 2023-09-23T18:03:34.446 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2721306559' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:03:34.446 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[179668]: osdmap e259: 8 total, 8 up, 7 in 2023-09-23T18:03:34.446 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[179668]: failed to encode map e259 with expected crc 2023-09-23T18:03:34.446 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[179668]: failed to encode map e259 with expected crc 2023-09-23T18:03:34.447 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2655316389' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:03:34.447 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 32151/150963 objects degraded (21.297%), 12 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:34.447 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:32 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:03:34.447 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:32 smithi099 ceph-mon[167076]: failed to encode map e259 with expected crc 2023-09-23T18:03:34.447 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:32 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2721306559' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:03:34.447 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:32 smithi099 ceph-mon[167076]: osdmap e259: 8 total, 8 up, 7 in 2023-09-23T18:03:34.447 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:32 smithi099 ceph-mon[167076]: failed to encode map e259 with expected crc 2023-09-23T18:03:34.447 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:32 smithi099 ceph-mon[167076]: failed to encode map e259 with expected crc 2023-09-23T18:03:34.447 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:32 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2655316389' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:32 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 32151/150963 objects degraded (21.297%), 12 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:33 smithi099 ceph-mon[167076]: pgmap v606: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 31917/149202 objects degraded (21.392%); 11407/149202 objects misplaced (7.645%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:33 smithi099 ceph-mon[167076]: failed to encode map e260 with expected crc 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:33 smithi099 ceph-mon[167076]: osdmap e260: 8 total, 8 up, 7 in 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:33 smithi099 ceph-mon[167076]: failed to encode map e260 with expected crc 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:33 smithi099 ceph-mon[167076]: failed to encode map e260 with expected crc 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:33 smithi099 ceph-mon[167076]: failed to encode map e260 with expected crc 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:33 smithi099 ceph-mon[167076]: failed to encode map e260 with expected crc 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[184174]: failed to encode map e259 with expected crc 2023-09-23T18:03:34.448 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2721306559' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:03:34.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[184174]: osdmap e259: 8 total, 8 up, 7 in 2023-09-23T18:03:34.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[184174]: failed to encode map e259 with expected crc 2023-09-23T18:03:34.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[184174]: failed to encode map e259 with expected crc 2023-09-23T18:03:34.449 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2655316389' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:03:34.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:32 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 32151/150963 objects degraded (21.297%), 12 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:34.654 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:34.655 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:03:34.656 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:03:34.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:35.498 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:03:35.498 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:03:35.499 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[179668]: pgmap v606: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 31917/149202 objects degraded (21.392%); 11407/149202 objects misplaced (7.645%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T18:03:35.499 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[179668]: failed to encode map e260 with expected crc 2023-09-23T18:03:35.499 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[179668]: osdmap e260: 8 total, 8 up, 7 in 2023-09-23T18:03:35.499 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[179668]: failed to encode map e260 with expected crc 2023-09-23T18:03:35.499 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[179668]: failed to encode map e260 with expected crc 2023-09-23T18:03:35.499 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[179668]: failed to encode map e260 with expected crc 2023-09-23T18:03:35.499 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[179668]: failed to encode map e260 with expected crc 2023-09-23T18:03:35.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[184174]: pgmap v606: 119 pgs: 1 active+recovery_wait+degraded+remapped, 106 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 8 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 31917/149202 objects degraded (21.392%); 11407/149202 objects misplaced (7.645%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T18:03:35.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[184174]: failed to encode map e260 with expected crc 2023-09-23T18:03:35.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[184174]: osdmap e260: 8 total, 8 up, 7 in 2023-09-23T18:03:35.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[184174]: failed to encode map e260 with expected crc 2023-09-23T18:03:35.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[184174]: failed to encode map e260 with expected crc 2023-09-23T18:03:35.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[184174]: failed to encode map e260 with expected crc 2023-09-23T18:03:35.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:33 smithi055 ceph-mon[184174]: failed to encode map e260 with expected crc 2023-09-23T18:03:35.668 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:03:35.669 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:35.759 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:37.001 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:36 smithi099 ceph-mon[167076]: pgmap v608: 119 pgs: 1 peering, 1 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 31371/146484 objects degraded (21.416%); 11365/146484 objects misplaced (7.759%); 8.0 MiB/s, 165 objects/s recovering 2023-09-23T18:03:37.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:37.002 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:37.002 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:37.002 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:36 smithi055 ceph-mon[179668]: pgmap v608: 119 pgs: 1 peering, 1 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 31371/146484 objects degraded (21.416%); 11365/146484 objects misplaced (7.759%); 8.0 MiB/s, 165 objects/s recovering 2023-09-23T18:03:37.002 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:37.003 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:36 smithi055 ceph-mon[184174]: pgmap v608: 119 pgs: 1 peering, 1 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 493 op/s; 31371/146484 objects degraded (21.416%); 11365/146484 objects misplaced (7.759%); 8.0 MiB/s, 165 objects/s recovering 2023-09-23T18:03:37.003 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:37.003 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:37.003 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:37.103 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:37.190 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:03:37.190 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.190 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.190 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.190 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.191 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:03:37.191 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.191 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.191 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.191 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.231 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:37.389 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:37.389 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:03:37.389 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.389 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.389 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.389 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:37.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:37.391 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:37.492 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:37.652 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:37.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:37.787 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:37.787 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:37.887 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:03:37.993 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:38 smithi055 ceph-mon[184174]: pgmap v609: 119 pgs: 1 peering, 1 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 322 op/s; 31371/146067 objects degraded (21.477%); 11233/146067 objects misplaced (7.690%); 8.0 MiB/s, 165 objects/s recovering 2023-09-23T18:03:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:38 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:03:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:38 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:03:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:38 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 31371/146067 objects degraded (21.477%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:38.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:38 smithi055 ceph-mon[179668]: pgmap v609: 119 pgs: 1 peering, 1 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 322 op/s; 31371/146067 objects degraded (21.477%); 11233/146067 objects misplaced (7.690%); 8.0 MiB/s, 165 objects/s recovering 2023-09-23T18:03:38.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:38 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:03:38.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:38 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:03:38.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:38 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:38.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 31371/146067 objects degraded (21.477%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:38.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:38 smithi099 ceph-mon[167076]: pgmap v609: 119 pgs: 1 peering, 1 active+recovery_wait+degraded+remapped, 107 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 3.0 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 322 op/s; 31371/146067 objects degraded (21.477%); 11233/146067 objects misplaced (7.690%); 8.0 MiB/s, 165 objects/s recovering 2023-09-23T18:03:38.336 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:38 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:03:38.336 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:38 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:03:38.336 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:38 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:03:38.336 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 31371/146067 objects degraded (21.477%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:38.456 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:03:38.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:38.517 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:03:38.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:03:38.679 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:38.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:38.817 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:03:38.817 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:03:38.917 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:38.941 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:03:38.954 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:03:38.970 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:39.110 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:03:39.110 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:03:39.110 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:03:39.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:39.111 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:03:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:03:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:03:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:03:39.111 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:03:39.111 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:39.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:39.112 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:03:39.112 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:03:39.112 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:03:39.112 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:03:39.113 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:03:39.113 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:03:39.113 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:03:39.113 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:03:39.113 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:03:39.113 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:03:39.113 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:03:39.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:39.247 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3936596153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:39.252 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:39.290 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:39.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:39.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:39.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:39.420 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:39.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3936596153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:39.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3936596153' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:03:39.456 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:39.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:39.459 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:03:39.559 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:39.588 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:39.628 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:39.631 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:03:39.634 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:03:39.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:39.634 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:39.735 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:03:39.768 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:40.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:40 smithi099 ceph-mon[167076]: pgmap v610: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 289 op/s; 30816/144003 objects degraded (21.400%); 11089/144003 objects misplaced (7.701%); 6.1 MiB/s, 134 objects/s recovering 2023-09-23T18:03:41.089 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3846392292' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:41.090 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:40 smithi055 ceph-mon[179668]: pgmap v610: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 289 op/s; 30816/144003 objects degraded (21.400%); 11089/144003 objects misplaced (7.701%); 6.1 MiB/s, 134 objects/s recovering 2023-09-23T18:03:41.090 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3846392292' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:41.090 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:40 smithi055 ceph-mon[184174]: pgmap v610: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 289 op/s; 30816/144003 objects degraded (21.400%); 11089/144003 objects misplaced (7.701%); 6.1 MiB/s, 134 objects/s recovering 2023-09-23T18:03:41.090 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3846392292' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:03:41.091 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:03:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:03:40] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.43.0" 2023-09-23T18:03:41.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:41 smithi099 ceph-mon[167076]: from='client.35616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:41.392 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:41 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:03:41.392 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:41 smithi099 ceph-mon[167076]: failed to encode map e261 with expected crc 2023-09-23T18:03:41.392 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:41 smithi099 ceph-mon[167076]: failed to encode map e261 with expected crc 2023-09-23T18:03:41.392 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:41 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3846392292' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:03:41.392 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:41 smithi099 ceph-mon[167076]: osdmap e261: 8 total, 8 up, 7 in 2023-09-23T18:03:41.392 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:41 smithi099 ceph-mon[167076]: failed to encode map e261 with expected crc 2023-09-23T18:03:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[184174]: from='client.35616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:41.642 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:03:41.643 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[184174]: failed to encode map e261 with expected crc 2023-09-23T18:03:41.643 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[184174]: failed to encode map e261 with expected crc 2023-09-23T18:03:41.643 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3846392292' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:03:41.643 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[184174]: osdmap e261: 8 total, 8 up, 7 in 2023-09-23T18:03:41.643 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[184174]: failed to encode map e261 with expected crc 2023-09-23T18:03:41.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[179668]: from='client.35616 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:03:41.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:03:41.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[179668]: failed to encode map e261 with expected crc 2023-09-23T18:03:41.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[179668]: failed to encode map e261 with expected crc 2023-09-23T18:03:41.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3846392292' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:03:41.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[179668]: osdmap e261: 8 total, 8 up, 7 in 2023-09-23T18:03:41.644 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:41 smithi055 ceph-mon[179668]: failed to encode map e261 with expected crc 2023-09-23T18:03:41.759 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:03:42.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:42 smithi099 ceph-mon[167076]: pgmap v612: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 30816/142716 objects degraded (21.593%); 10947/142716 objects misplaced (7.670%); 6.1 MiB/s, 134 objects/s recovering 2023-09-23T18:03:43.771 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:42 smithi055 ceph-mon[179668]: pgmap v612: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 30816/142716 objects degraded (21.593%); 10947/142716 objects misplaced (7.670%); 6.1 MiB/s, 134 objects/s recovering 2023-09-23T18:03:43.772 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 30816/142716 objects degraded (21.593%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:45.266 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:42 smithi055 ceph-mon[184174]: pgmap v612: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 270 op/s; 30816/142716 objects degraded (21.593%); 10947/142716 objects misplaced (7.670%); 6.1 MiB/s, 134 objects/s recovering 2023-09-23T18:03:45.268 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:03:45.268 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 30816/142716 objects degraded (21.593%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:45.269 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 30816/142716 objects degraded (21.593%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:45.602 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:03:46.949 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:44 smithi055 ceph-mon[184174]: pgmap v613: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 235 op/s; 30673/141660 objects degraded (21.653%); 10870/141660 objects misplaced (7.673%); 4.3 MiB/s, 99 objects/s recovering 2023-09-23T18:03:46.950 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:44 smithi055 ceph-mon[179668]: pgmap v613: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 235 op/s; 30673/141660 objects degraded (21.653%); 10870/141660 objects misplaced (7.673%); 4.3 MiB/s, 99 objects/s recovering 2023-09-23T18:03:46.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:46.951 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:44 smithi099 ceph-mon[167076]: pgmap v613: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.9 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 235 op/s; 30673/141660 objects degraded (21.653%); 10870/141660 objects misplaced (7.673%); 4.3 MiB/s, 99 objects/s recovering 2023-09-23T18:03:46.951 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:46 smithi099 ceph-mon[167076]: pgmap v614: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 30172/139674 objects degraded (21.602%); 10798/139674 objects misplaced (7.731%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T18:03:46.952 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:03:47.141 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":610,"stamp":"2023-09-23T18:03:38.687809+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3148560902,"num_objects":48001,"num_object_clones":0,"num_object_copies":144003,"num_objects_missing_on_primary":504,"num_objects_missing":504,"num_objects_degraded":30816,"num_objects_misplaced":11089,"num_objects_unfound":0,"num_objects_dirty":48001,"num_whiteouts":0,"num_read":6284,"num_read_kb":12770,"num_write":99367,"num_write_kb":4695012,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18375,"num_bytes_recovered":1180503015,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":89556,"ondisk_log_size":89556,"up":357,"acting":347,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":404,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":10952528,"kb_used_data":9435992,"kb_used_omap":43,"kb_used_meta":1516436,"kb_avail":645263536,"statfs":{"total":671965249536,"available":660749860864,"internally_reserved":0,"allocated":9662455808,"data_stored":9654370194,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1552831183},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-152043520,"num_objects":-2320,"num_object_clones":0,"num_object_copies":-6960,"num_objects_missing_on_primary":-119,"num_objects_missing":-119,"num_objects_degraded":-1335,"num_objects_misplaced":-436,"num_objects_unfound":0,"num_objects_dirty":-2320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2320,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1079,"num_bytes_recovered":51315232,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.002518"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":585,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T18:03:33.004903+0000","last_change":"2023-09-23T18:03:33.004903+0000","last_active":"2023-09-23T18:03:33.004903+0000","last_peered":"2023-09-23T18:03:33.004903+0000","last_clean":"2023-09-23T18:03:33.004903+0000","last_became_active":"2023-09-23T18:03:33.004498+0000","last_became_peered":"2023-09-23T18:03:33.004498+0000","last_unstale":"2023-09-23T18:03:33.004903+0000","last_undegraded":"2023-09-23T18:03:33.004903+0000","last_fullsized":"2023-09-23T18:03:33.004903+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":737225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":60,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002435+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:03:32.002435+0000","last_peered":"2023-09-23T18:03:32.002435+0000","last_clean":"2023-09-23T18:03:32.002435+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:03:32.002435+0000","last_undegraded":"2023-09-23T18:03:32.002435+0000","last_fullsized":"2023-09-23T18:03:32.002435+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":141,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004747+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:03:32.004747+0000","last_peered":"2023-09-23T18:03:32.004747+0000","last_clean":"2023-09-23T18:03:32.004747+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:03:32.004747+0000","last_undegraded":"2023-09-23T18:03:32.004747+0000","last_fullsized":"2023-09-23T18:03:32.004747+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":602,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000280+0000","last_change":"2023-09-23T18:02:51.046916+0000","last_active":"2023-09-23T18:03:32.000280+0000","last_peered":"2023-09-23T18:03:32.000280+0000","last_clean":"2023-09-23T18:03:32.000280+0000","last_became_active":"2023-09-23T18:02:51.046591+0000","last_became_peered":"2023-09-23T18:02:51.046591+0000","last_unstale":"2023-09-23T18:03:32.000280+0000","last_undegraded":"2023-09-23T18:03:32.000280+0000","last_fullsized":"2023-09-23T18:03:32.000280+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:19.805099+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":109,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004576+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:03:32.004576+0000","last_peered":"2023-09-23T18:03:32.004576+0000","last_clean":"2023-09-23T18:03:32.004576+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:03:32.004576+0000","last_undegraded":"2023-09-23T18:03:32.004576+0000","last_fullsized":"2023-09-23T18:03:32.004576+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":261,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999261+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:03:31.999261+0000","last_peered":"2023-09-23T18:03:31.999261+0000","last_clean":"2023-09-23T18:03:31.999261+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:03:31.999261+0000","last_undegraded":"2023-09-23T18:03:31.999261+0000","last_fullsized":"2023-09-23T18:03:31.999261+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":594,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999959+0000","last_change":"2023-09-23T18:02:51.051617+0000","last_active":"2023-09-23T18:03:31.999959+0000","last_peered":"2023-09-23T18:03:31.999959+0000","last_clean":"2023-09-23T18:03:31.999959+0000","last_became_active":"2023-09-23T18:02:51.048947+0000","last_became_peered":"2023-09-23T18:02:51.048947+0000","last_unstale":"2023-09-23T18:03:31.999959+0000","last_undegraded":"2023-09-23T18:03:31.999959+0000","last_fullsized":"2023-09-23T18:03:31.999959+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:49:34.962249+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":122,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004919+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:03:32.004919+0000","last_peered":"2023-09-23T18:03:32.004919+0000","last_clean":"2023-09-23T18:03:32.004919+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:03:32.004919+0000","last_undegraded":"2023-09-23T18:03:32.004919+0000","last_fullsized":"2023-09-23T18:03:32.004919+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":200,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.998249+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:03:31.998249+0000","last_peered":"2023-09-23T18:03:31.998249+0000","last_clean":"2023-09-23T18:03:31.998249+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:03:31.998249+0000","last_undegraded":"2023-09-23T18:03:31.998249+0000","last_fullsized":"2023-09-23T18:03:31.998249+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":591,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002129+0000","last_change":"2023-09-23T18:02:51.052123+0000","last_active":"2023-09-23T18:03:32.002129+0000","last_peered":"2023-09-23T18:03:32.002129+0000","last_clean":"2023-09-23T18:03:32.002129+0000","last_became_active":"2023-09-23T18:02:51.051772+0000","last_became_peered":"2023-09-23T18:02:51.051772+0000","last_unstale":"2023-09-23T18:03:32.002129+0000","last_undegraded":"2023-09-23T18:03:32.002129+0000","last_fullsized":"2023-09-23T18:03:32.002129+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":240,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999410+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:03:31.999410+0000","last_peered":"2023-09-23T18:03:31.999410+0000","last_clean":"2023-09-23T18:03:31.999410+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:03:31.999410+0000","last_undegraded":"2023-09-23T18:03:31.999410+0000","last_fullsized":"2023-09-23T18:03:31.999410+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":116,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004993+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:03:32.004993+0000","last_peered":"2023-09-23T18:03:32.004993+0000","last_clean":"2023-09-23T18:03:32.004993+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:03:32.004993+0000","last_undegraded":"2023-09-23T18:03:32.004993+0000","last_fullsized":"2023-09-23T18:03:32.004993+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":586,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999805+0000","last_change":"2023-09-23T18:02:51.054208+0000","last_active":"2023-09-23T18:03:31.999805+0000","last_peered":"2023-09-23T18:03:31.999805+0000","last_clean":"2023-09-23T18:03:31.999805+0000","last_became_active":"2023-09-23T18:02:51.053676+0000","last_became_peered":"2023-09-23T18:02:51.053676+0000","last_unstale":"2023-09-23T18:03:31.999805+0000","last_undegraded":"2023-09-23T18:03:31.999805+0000","last_fullsized":"2023-09-23T18:03:31.999805+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":242,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.020982+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:03:32.020982+0000","last_peered":"2023-09-23T18:03:32.020982+0000","last_clean":"2023-09-23T18:03:32.020982+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:03:32.020982+0000","last_undegraded":"2023-09-23T18:03:32.020982+0000","last_fullsized":"2023-09-23T18:03:32.020982+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":70,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004477+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:03:32.004477+0000","last_peered":"2023-09-23T18:03:32.004477+0000","last_clean":"2023-09-23T18:03:32.004477+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:03:32.004477+0000","last_undegraded":"2023-09-23T18:03:32.004477+0000","last_fullsized":"2023-09-23T18:03:32.004477+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":591,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.003869+0000","last_change":"2023-09-23T18:02:51.058624+0000","last_active":"2023-09-23T18:03:32.003869+0000","last_peered":"2023-09-23T18:03:32.003869+0000","last_clean":"2023-09-23T18:03:32.003869+0000","last_became_active":"2023-09-23T18:02:51.057352+0000","last_became_peered":"2023-09-23T18:02:51.057352+0000","last_unstale":"2023-09-23T18:03:32.003869+0000","last_undegraded":"2023-09-23T18:03:32.003869+0000","last_fullsized":"2023-09-23T18:03:32.003869+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":26,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002043+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:03:32.002043+0000","last_peered":"2023-09-23T18:03:32.002043+0000","last_clean":"2023-09-23T18:03:32.002043+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:03:32.002043+0000","last_undegraded":"2023-09-23T18:03:32.002043+0000","last_fullsized":"2023-09-23T18:03:32.002043+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":156,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999052+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:03:31.999052+0000","last_peered":"2023-09-23T18:03:31.999052+0000","last_clean":"2023-09-23T18:03:31.999052+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:03:31.999052+0000","last_undegraded":"2023-09-23T18:03:31.999052+0000","last_fullsized":"2023-09-23T18:03:31.999052+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":591,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000261+0000","last_change":"2023-09-23T18:02:51.059983+0000","last_active":"2023-09-23T18:03:32.000261+0000","last_peered":"2023-09-23T18:03:32.000261+0000","last_clean":"2023-09-23T18:03:32.000261+0000","last_became_active":"2023-09-23T18:02:51.057768+0000","last_became_peered":"2023-09-23T18:02:51.057768+0000","last_unstale":"2023-09-23T18:03:32.000261+0000","last_undegraded":"2023-09-23T18:03:32.000261+0000","last_fullsized":"2023-09-23T18:03:32.000261+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":60,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002349+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:03:32.002349+0000","last_peered":"2023-09-23T18:03:32.002349+0000","last_clean":"2023-09-23T18:03:32.002349+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:03:32.002349+0000","last_undegraded":"2023-09-23T18:03:32.002349+0000","last_fullsized":"2023-09-23T18:03:32.002349+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":60,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002980+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:03:32.002980+0000","last_peered":"2023-09-23T18:03:32.002980+0000","last_clean":"2023-09-23T18:03:32.002980+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:03:32.002980+0000","last_undegraded":"2023-09-23T18:03:32.002980+0000","last_fullsized":"2023-09-23T18:03:32.002980+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":581,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999881+0000","last_change":"2023-09-23T18:02:51.047570+0000","last_active":"2023-09-23T18:03:31.999881+0000","last_peered":"2023-09-23T18:03:31.999881+0000","last_clean":"2023-09-23T18:03:31.999881+0000","last_became_active":"2023-09-23T18:02:51.046963+0000","last_became_peered":"2023-09-23T18:02:51.046963+0000","last_unstale":"2023-09-23T18:03:31.999881+0000","last_undegraded":"2023-09-23T18:03:31.999881+0000","last_fullsized":"2023-09-23T18:03:31.999881+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":189,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002102+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:03:32.002102+0000","last_peered":"2023-09-23T18:03:32.002102+0000","last_clean":"2023-09-23T18:03:32.002102+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:03:32.002102+0000","last_undegraded":"2023-09-23T18:03:32.002102+0000","last_fullsized":"2023-09-23T18:03:32.002102+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":104,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.005224+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:03:32.005224+0000","last_peered":"2023-09-23T18:03:32.005224+0000","last_clean":"2023-09-23T18:03:32.005224+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:03:32.005224+0000","last_undegraded":"2023-09-23T18:03:32.005224+0000","last_fullsized":"2023-09-23T18:03:32.005224+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":594,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.001029+0000","last_change":"2023-09-23T18:02:51.053806+0000","last_active":"2023-09-23T18:03:32.001029+0000","last_peered":"2023-09-23T18:03:32.001029+0000","last_clean":"2023-09-23T18:03:32.001029+0000","last_became_active":"2023-09-23T18:02:51.048822+0000","last_became_peered":"2023-09-23T18:02:51.048822+0000","last_unstale":"2023-09-23T18:03:32.001029+0000","last_undegraded":"2023-09-23T18:03:32.001029+0000","last_fullsized":"2023-09-23T18:03:32.001029+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:34.401104+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":259,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999466+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:03:31.999466+0000","last_peered":"2023-09-23T18:03:31.999466+0000","last_clean":"2023-09-23T18:03:31.999466+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:03:31.999466+0000","last_undegraded":"2023-09-23T18:03:31.999466+0000","last_fullsized":"2023-09-23T18:03:31.999466+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":237,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.020832+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:03:32.020832+0000","last_peered":"2023-09-23T18:03:32.020832+0000","last_clean":"2023-09-23T18:03:32.020832+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:03:32.020832+0000","last_undegraded":"2023-09-23T18:03:32.020832+0000","last_fullsized":"2023-09-23T18:03:32.020832+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":582,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000447+0000","last_change":"2023-09-23T18:02:51.046646+0000","last_active":"2023-09-23T18:03:32.000447+0000","last_peered":"2023-09-23T18:03:32.000447+0000","last_clean":"2023-09-23T18:03:32.000447+0000","last_became_active":"2023-09-23T18:02:51.046134+0000","last_became_peered":"2023-09-23T18:02:51.046134+0000","last_unstale":"2023-09-23T18:03:32.000447+0000","last_undegraded":"2023-09-23T18:03:32.000447+0000","last_fullsized":"2023-09-23T18:03:32.000447+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":50,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002380+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:03:32.002380+0000","last_peered":"2023-09-23T18:03:32.002380+0000","last_clean":"2023-09-23T18:03:32.002380+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:03:32.002380+0000","last_undegraded":"2023-09-23T18:03:32.002380+0000","last_fullsized":"2023-09-23T18:03:32.002380+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":250,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999341+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:03:31.999341+0000","last_peered":"2023-09-23T18:03:31.999341+0000","last_clean":"2023-09-23T18:03:31.999341+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:03:31.999341+0000","last_undegraded":"2023-09-23T18:03:31.999341+0000","last_fullsized":"2023-09-23T18:03:31.999341+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":594,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999896+0000","last_change":"2023-09-23T18:02:51.051817+0000","last_active":"2023-09-23T18:03:31.999896+0000","last_peered":"2023-09-23T18:03:31.999896+0000","last_clean":"2023-09-23T18:03:31.999896+0000","last_became_active":"2023-09-23T18:02:51.048945+0000","last_became_peered":"2023-09-23T18:02:51.048945+0000","last_unstale":"2023-09-23T18:03:31.999896+0000","last_undegraded":"2023-09-23T18:03:31.999896+0000","last_fullsized":"2023-09-23T18:03:31.999896+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:47.163864+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":196,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.997334+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:03:31.997334+0000","last_peered":"2023-09-23T18:03:31.997334+0000","last_clean":"2023-09-23T18:03:31.997334+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:03:31.997334+0000","last_undegraded":"2023-09-23T18:03:31.997334+0000","last_fullsized":"2023-09-23T18:03:31.997334+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":113,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004914+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:03:32.004914+0000","last_peered":"2023-09-23T18:03:32.004914+0000","last_clean":"2023-09-23T18:03:32.004914+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:03:32.004914+0000","last_undegraded":"2023-09-23T18:03:32.004914+0000","last_fullsized":"2023-09-23T18:03:32.004914+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"260'6250","reported_seq":6313,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T18:03:37.503528+0000","last_change":"2023-09-23T18:00:57.996982+0000","last_active":"2023-09-23T18:03:37.503528+0000","last_peered":"2023-09-23T18:03:37.503528+0000","last_clean":"2023-09-23T18:03:37.503528+0000","last_became_active":"2023-09-23T18:00:57.996589+0000","last_became_peered":"2023-09-23T18:00:57.996589+0000","last_unstale":"2023-09-23T18:03:37.503528+0000","last_undegraded":"2023-09-23T18:03:37.503528+0000","last_fullsized":"2023-09-23T18:03:37.503528+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6250,"log_dups_size":0,"ondisk_log_size":6250,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:13.306089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196083712,"num_objects":2992,"num_object_clones":0,"num_object_copies":8976,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2992,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6250,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":591,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.001951+0000","last_change":"2023-09-23T18:02:51.058477+0000","last_active":"2023-09-23T18:03:32.001951+0000","last_peered":"2023-09-23T18:03:32.001951+0000","last_clean":"2023-09-23T18:03:32.001951+0000","last_became_active":"2023-09-23T18:02:51.057258+0000","last_became_peered":"2023-09-23T18:02:51.057258+0000","last_unstale":"2023-09-23T18:03:32.001951+0000","last_undegraded":"2023-09-23T18:03:32.001951+0000","last_fullsized":"2023-09-23T18:03:32.001951+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":189,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.003931+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:03:32.003931+0000","last_peered":"2023-09-23T18:03:32.003931+0000","last_clean":"2023-09-23T18:03:32.003931+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:03:32.003931+0000","last_undegraded":"2023-09-23T18:03:32.003931+0000","last_fullsized":"2023-09-23T18:03:32.003931+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":248,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.019474+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:03:32.019474+0000","last_peered":"2023-09-23T18:03:32.019474+0000","last_clean":"2023-09-23T18:03:32.019474+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:03:32.019474+0000","last_undegraded":"2023-09-23T18:03:32.019474+0000","last_fullsized":"2023-09-23T18:03:32.019474+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"260'6023","reported_seq":13246,"reported_epoch":260,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:03:35.449639+0000","last_change":"2023-09-23T18:02:49.039290+0000","last_active":"2023-09-23T18:03:35.449639+0000","last_peered":"2023-09-23T18:03:35.449639+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:49.001792+0000","last_became_peered":"2023-09-23T18:02:49.001792+0000","last_unstale":"2023-09-23T18:03:35.449639+0000","last_undegraded":"2023-09-23T18:02:48.968288+0000","last_fullsized":"2023-09-23T18:03:35.449639+0000","mapping_epoch":250,"log_start":"218'1830","ondisk_log_start":"218'1830","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4193,"log_dups_size":0,"ondisk_log_size":4193,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.156979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":200605696,"num_objects":3061,"num_object_clones":0,"num_object_copies":9183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":409,"num_objects_misplaced":5713,"num_objects_unfound":0,"num_objects_dirty":3061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6023,"num_write_kb":290688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3566,"num_bytes_recovered":233701376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[6,1,4],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,4,6","objects":2652},{"shards":"1,6","objects":409}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":597,"reported_epoch":259,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:31.998592+0000","last_change":"2023-09-23T18:02:48.968551+0000","last_active":"2023-09-23T18:03:31.998592+0000","last_peered":"2023-09-23T18:03:31.998592+0000","last_clean":"2023-09-23T18:02:45.965241+0000","last_became_active":"2023-09-23T18:02:48.968221+0000","last_became_peered":"2023-09-23T18:02:48.968221+0000","last_unstale":"2023-09-23T18:03:31.998592+0000","last_undegraded":"2023-09-23T18:02:48.944021+0000","last_fullsized":"2023-09-23T18:02:48.942326+0000","mapping_epoch":250,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":409128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[0,6],"avail_no_missing":["0","2","6"],"object_location_counts":[{"shards":"0,2,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":7,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":204,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002064+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:03:32.002064+0000","last_peered":"2023-09-23T18:03:32.002064+0000","last_clean":"2023-09-23T18:03:32.002064+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:03:32.002064+0000","last_undegraded":"2023-09-23T18:03:32.002064+0000","last_fullsized":"2023-09-23T18:03:32.002064+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":270,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002694+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:03:32.002694+0000","last_peered":"2023-09-23T18:03:32.002694+0000","last_clean":"2023-09-23T18:03:32.002694+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:03:32.002694+0000","last_undegraded":"2023-09-23T18:03:32.002694+0000","last_fullsized":"2023-09-23T18:03:32.002694+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"260'6209","reported_seq":14812,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T18:03:34.122530+0000","last_change":"2023-09-23T18:03:33.017384+0000","last_active":"2023-09-23T18:03:34.122530+0000","last_peered":"2023-09-23T18:03:34.122530+0000","last_clean":"2023-09-23T18:03:34.122530+0000","last_became_active":"2023-09-23T18:03:33.004338+0000","last_became_peered":"2023-09-23T18:03:33.004338+0000","last_unstale":"2023-09-23T18:03:34.122530+0000","last_undegraded":"2023-09-23T18:03:34.122530+0000","last_fullsized":"2023-09-23T18:03:34.122530+0000","mapping_epoch":259,"log_start":"217'1500","ondisk_log_start":"217'1500","created":204,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4709,"log_dups_size":0,"ondisk_log_size":4709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:59.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198770688,"num_objects":3033,"num_object_clones":0,"num_object_copies":9099,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3033,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6209,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4258,"num_bytes_recovered":259457024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":582,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000528+0000","last_change":"2023-09-23T18:02:51.058795+0000","last_active":"2023-09-23T18:03:32.000528+0000","last_peered":"2023-09-23T18:03:32.000528+0000","last_clean":"2023-09-23T18:03:32.000528+0000","last_became_active":"2023-09-23T18:02:51.057799+0000","last_became_peered":"2023-09-23T18:02:51.057799+0000","last_unstale":"2023-09-23T18:03:32.000528+0000","last_undegraded":"2023-09-23T18:03:32.000528+0000","last_fullsized":"2023-09-23T18:03:32.000528+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":127,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004660+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:03:32.004660+0000","last_peered":"2023-09-23T18:03:32.004660+0000","last_clean":"2023-09-23T18:03:32.004660+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:03:32.004660+0000","last_undegraded":"2023-09-23T18:03:32.004660+0000","last_fullsized":"2023-09-23T18:03:32.004660+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":225,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.998920+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:03:31.998920+0000","last_peered":"2023-09-23T18:03:31.998920+0000","last_clean":"2023-09-23T18:03:31.998920+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:03:31.998920+0000","last_undegraded":"2023-09-23T18:03:31.998920+0000","last_fullsized":"2023-09-23T18:03:31.998920+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"260'6220","reported_seq":6303,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T18:03:34.342290+0000","last_change":"2023-09-23T18:01:25.019784+0000","last_active":"2023-09-23T18:03:34.342290+0000","last_peered":"2023-09-23T18:03:34.342290+0000","last_clean":"2023-09-23T18:03:34.342290+0000","last_became_active":"2023-09-23T18:01:25.002073+0000","last_became_peered":"2023-09-23T18:01:25.002073+0000","last_unstale":"2023-09-23T18:03:34.342290+0000","last_undegraded":"2023-09-23T18:03:34.342290+0000","last_fullsized":"2023-09-23T18:03:34.342290+0000","mapping_epoch":214,"log_start":"217'1800","ondisk_log_start":"217'1800","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4420,"log_dups_size":0,"ondisk_log_size":4420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:58.981672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201981952,"num_objects":3082,"num_object_clones":0,"num_object_copies":9246,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3082,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6220,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":591,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000432+0000","last_change":"2023-09-23T18:02:51.057506+0000","last_active":"2023-09-23T18:03:32.000432+0000","last_peered":"2023-09-23T18:03:32.000432+0000","last_clean":"2023-09-23T18:03:32.000432+0000","last_became_active":"2023-09-23T18:02:51.056589+0000","last_became_peered":"2023-09-23T18:02:51.056589+0000","last_unstale":"2023-09-23T18:03:32.000432+0000","last_undegraded":"2023-09-23T18:03:32.000432+0000","last_fullsized":"2023-09-23T18:03:32.000432+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":236,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.019974+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:03:32.019974+0000","last_peered":"2023-09-23T18:03:32.019974+0000","last_clean":"2023-09-23T18:03:32.019974+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:03:32.019974+0000","last_undegraded":"2023-09-23T18:03:32.019974+0000","last_fullsized":"2023-09-23T18:03:32.019974+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":262,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.998653+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:03:31.998653+0000","last_peered":"2023-09-23T18:03:31.998653+0000","last_clean":"2023-09-23T18:03:31.998653+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:03:31.998653+0000","last_undegraded":"2023-09-23T18:03:31.998653+0000","last_fullsized":"2023-09-23T18:03:31.998653+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"260'6271","reported_seq":9647,"reported_epoch":260,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:03:38.095069+0000","last_change":"2023-09-23T18:02:45.688221+0000","last_active":"2023-09-23T18:03:38.095069+0000","last_peered":"2023-09-23T18:03:38.095069+0000","last_clean":"2023-09-23T18:01:38.416095+0000","last_became_active":"2023-09-23T18:02:07.547135+0000","last_became_peered":"2023-09-23T18:02:07.547135+0000","last_unstale":"2023-09-23T18:03:38.095069+0000","last_undegraded":"2023-09-23T18:03:38.095069+0000","last_fullsized":"2023-09-23T18:02:02.784733+0000","mapping_epoch":227,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6171,"log_dups_size":0,"ondisk_log_size":6171,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:27.864301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195493888,"num_objects":2983,"num_object_clones":0,"num_object_copies":8949,"num_objects_missing_on_primary":504,"num_objects_missing":504,"num_objects_degraded":0,"num_objects_misplaced":2443,"num_objects_unfound":0,"num_objects_dirty":2983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6271,"num_write_kb":296128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1643,"num_bytes_recovered":107675648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1939},{"shards":"1,2,4,5","objects":14},{"shards":"1,2,5","objects":504},{"shards":"2,4","objects":526}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":583,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999784+0000","last_change":"2023-09-23T18:02:51.047609+0000","last_active":"2023-09-23T18:03:31.999784+0000","last_peered":"2023-09-23T18:03:31.999784+0000","last_clean":"2023-09-23T18:03:31.999784+0000","last_became_active":"2023-09-23T18:02:51.047209+0000","last_became_peered":"2023-09-23T18:02:51.047209+0000","last_unstale":"2023-09-23T18:03:31.999784+0000","last_undegraded":"2023-09-23T18:03:31.999784+0000","last_fullsized":"2023-09-23T18:03:31.999784+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":147,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004679+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:03:32.004679+0000","last_peered":"2023-09-23T18:03:32.004679+0000","last_clean":"2023-09-23T18:03:32.004679+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:03:32.004679+0000","last_undegraded":"2023-09-23T18:03:32.004679+0000","last_fullsized":"2023-09-23T18:03:32.004679+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":268,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002374+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:03:32.002374+0000","last_peered":"2023-09-23T18:03:32.002374+0000","last_clean":"2023-09-23T18:03:32.002374+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:03:32.002374+0000","last_undegraded":"2023-09-23T18:03:32.002374+0000","last_fullsized":"2023-09-23T18:03:32.002374+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"260'6073","reported_seq":6295,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:34.209076+0000","last_change":"2023-09-23T18:02:47.961575+0000","last_active":"2023-09-23T18:03:34.209076+0000","last_peered":"2023-09-23T18:03:34.209076+0000","last_clean":"2023-09-23T18:02:17.013863+0000","last_became_active":"2023-09-23T18:02:19.267215+0000","last_became_peered":"2023-09-23T18:02:19.267215+0000","last_unstale":"2023-09-23T18:03:34.209076+0000","last_undegraded":"2023-09-23T18:02:18.955030+0000","last_fullsized":"2023-09-23T18:02:18.924656+0000","mapping_epoch":238,"log_start":"217'1300","ondisk_log_start":"217'1300","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4773,"log_dups_size":0,"ondisk_log_size":4773,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:46.393021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":195493888,"num_objects":2983,"num_object_clones":0,"num_object_copies":8949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2983,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6073,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":4128768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-1125},{"shards":"2,4,6","objects":2625}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":587,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.001106+0000","last_change":"2023-09-23T18:02:51.049464+0000","last_active":"2023-09-23T18:03:32.001106+0000","last_peered":"2023-09-23T18:03:32.001106+0000","last_clean":"2023-09-23T18:03:32.001106+0000","last_became_active":"2023-09-23T18:02:51.048477+0000","last_became_peered":"2023-09-23T18:02:51.048477+0000","last_unstale":"2023-09-23T18:03:32.001106+0000","last_undegraded":"2023-09-23T18:03:32.001106+0000","last_fullsized":"2023-09-23T18:03:32.001106+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":219,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999177+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:03:31.999177+0000","last_peered":"2023-09-23T18:03:31.999177+0000","last_clean":"2023-09-23T18:03:31.999177+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:03:31.999177+0000","last_undegraded":"2023-09-23T18:03:31.999177+0000","last_fullsized":"2023-09-23T18:03:31.999177+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":60,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002589+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:03:32.002589+0000","last_peered":"2023-09-23T18:03:32.002589+0000","last_clean":"2023-09-23T18:03:32.002589+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:03:32.002589+0000","last_undegraded":"2023-09-23T18:03:32.002589+0000","last_fullsized":"2023-09-23T18:03:32.002589+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"260'6122","reported_seq":6190,"reported_epoch":260,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:03:37.497660+0000","last_change":"2023-09-23T18:02:19.054827+0000","last_active":"2023-09-23T18:03:37.497660+0000","last_peered":"2023-09-23T18:03:37.497660+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.048672+0000","last_became_peered":"2023-09-23T18:02:19.048672+0000","last_unstale":"2023-09-23T18:03:37.497660+0000","last_undegraded":"2023-09-23T18:02:18.951263+0000","last_fullsized":"2023-09-23T18:02:18.930851+0000","mapping_epoch":238,"log_start":"216'261","ondisk_log_start":"216'261","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":5861,"log_dups_size":0,"ondisk_log_size":5861,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:23.690233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194641920,"num_objects":2970,"num_object_clones":0,"num_object_copies":8910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2970,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6122,"num_write_kb":290944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":2970}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":591,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000204+0000","last_change":"2023-09-23T18:02:51.060858+0000","last_active":"2023-09-23T18:03:32.000204+0000","last_peered":"2023-09-23T18:03:32.000204+0000","last_clean":"2023-09-23T18:03:32.000204+0000","last_became_active":"2023-09-23T18:02:51.060554+0000","last_became_peered":"2023-09-23T18:02:51.060554+0000","last_unstale":"2023-09-23T18:03:32.000204+0000","last_undegraded":"2023-09-23T18:03:32.000204+0000","last_fullsized":"2023-09-23T18:03:32.000204+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":50,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002469+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:03:32.002469+0000","last_peered":"2023-09-23T18:03:32.002469+0000","last_clean":"2023-09-23T18:03:32.002469+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:03:32.002469+0000","last_undegraded":"2023-09-23T18:03:32.002469+0000","last_fullsized":"2023-09-23T18:03:32.002469+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":270,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.035753+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:03:32.035753+0000","last_peered":"2023-09-23T18:03:32.035753+0000","last_clean":"2023-09-23T18:03:32.035753+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:03:32.035753+0000","last_undegraded":"2023-09-23T18:03:32.035753+0000","last_fullsized":"2023-09-23T18:03:32.035753+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"260'6113","reported_seq":14230,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T18:03:37.479833+0000","last_change":"2023-09-23T18:02:47.017672+0000","last_active":"2023-09-23T18:03:37.479833+0000","last_peered":"2023-09-23T18:03:37.479833+0000","last_clean":"2023-09-23T18:03:37.479833+0000","last_became_active":"2023-09-23T18:02:47.009245+0000","last_became_peered":"2023-09-23T18:02:47.009245+0000","last_unstale":"2023-09-23T18:03:37.479833+0000","last_undegraded":"2023-09-23T18:03:37.479833+0000","last_fullsized":"2023-09-23T18:03:37.479833+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6113,"log_dups_size":0,"ondisk_log_size":6113,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:04.868743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194773016,"num_objects":2973,"num_object_clones":0,"num_object_copies":8919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2973,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6113,"num_write_kb":290689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4015,"num_bytes_recovered":263127040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":583,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999877+0000","last_change":"2023-09-23T18:02:51.054197+0000","last_active":"2023-09-23T18:03:31.999877+0000","last_peered":"2023-09-23T18:03:31.999877+0000","last_clean":"2023-09-23T18:03:31.999877+0000","last_became_active":"2023-09-23T18:02:51.049362+0000","last_became_peered":"2023-09-23T18:02:51.049362+0000","last_unstale":"2023-09-23T18:03:31.999877+0000","last_undegraded":"2023-09-23T18:03:31.999877+0000","last_fullsized":"2023-09-23T18:03:31.999877+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":105,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.005142+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:03:32.005142+0000","last_peered":"2023-09-23T18:03:32.005142+0000","last_clean":"2023-09-23T18:03:32.005142+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:03:32.005142+0000","last_undegraded":"2023-09-23T18:03:32.005142+0000","last_fullsized":"2023-09-23T18:03:32.005142+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":105,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.005058+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:03:32.005058+0000","last_peered":"2023-09-23T18:03:32.005058+0000","last_clean":"2023-09-23T18:03:32.005058+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:03:32.005058+0000","last_undegraded":"2023-09-23T18:03:32.005058+0000","last_fullsized":"2023-09-23T18:03:32.005058+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"260'5914","reported_seq":5984,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:34.366291+0000","last_change":"2023-09-23T18:02:19.315376+0000","last_active":"2023-09-23T18:03:34.366291+0000","last_peered":"2023-09-23T18:03:34.366291+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.306174+0000","last_became_peered":"2023-09-23T18:02:19.306174+0000","last_unstale":"2023-09-23T18:03:34.366291+0000","last_undegraded":"2023-09-23T18:02:18.954031+0000","last_fullsized":"2023-09-23T18:02:18.922505+0000","mapping_epoch":238,"log_start":"217'1500","ondisk_log_start":"217'1500","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4414,"log_dups_size":0,"ondisk_log_size":4414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:52.392034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194510848,"num_objects":2968,"num_object_clones":0,"num_object_copies":8904,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2968,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5914,"num_write_kb":284224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":-1127}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":592,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000147+0000","last_change":"2023-09-23T18:02:51.059934+0000","last_active":"2023-09-23T18:03:32.000147+0000","last_peered":"2023-09-23T18:03:32.000147+0000","last_clean":"2023-09-23T18:03:32.000147+0000","last_became_active":"2023-09-23T18:02:51.059638+0000","last_became_peered":"2023-09-23T18:02:51.059638+0000","last_unstale":"2023-09-23T18:03:32.000147+0000","last_undegraded":"2023-09-23T18:03:32.000147+0000","last_fullsized":"2023-09-23T18:03:32.000147+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":569,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.998469+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:03:31.998469+0000","last_peered":"2023-09-23T18:03:31.998469+0000","last_clean":"2023-09-23T18:03:31.998469+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:03:31.998469+0000","last_undegraded":"2023-09-23T18:03:31.998469+0000","last_fullsized":"2023-09-23T18:03:31.998469+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":240,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.020858+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:03:32.020858+0000","last_peered":"2023-09-23T18:03:32.020858+0000","last_clean":"2023-09-23T18:03:32.020858+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:03:32.020858+0000","last_undegraded":"2023-09-23T18:03:32.020858+0000","last_fullsized":"2023-09-23T18:03:32.020858+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":237,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.019586+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:03:32.019586+0000","last_peered":"2023-09-23T18:03:32.019586+0000","last_clean":"2023-09-23T18:03:32.019586+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:03:32.019586+0000","last_undegraded":"2023-09-23T18:03:32.019586+0000","last_fullsized":"2023-09-23T18:03:32.019586+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"260'6078","reported_seq":6162,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:37.880333+0000","last_change":"2023-09-23T18:02:19.289610+0000","last_active":"2023-09-23T18:03:37.880333+0000","last_peered":"2023-09-23T18:03:37.880333+0000","last_clean":"2023-09-23T18:02:17.060883+0000","last_became_active":"2023-09-23T18:02:19.266879+0000","last_became_peered":"2023-09-23T18:02:19.266879+0000","last_unstale":"2023-09-23T18:03:37.880333+0000","last_undegraded":"2023-09-23T18:02:18.963390+0000","last_fullsized":"2023-09-23T18:02:18.926482+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6078,"log_dups_size":0,"ondisk_log_size":6078,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.499269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":188743680,"num_objects":2880,"num_object_clones":0,"num_object_copies":8640,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2880,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6078,"num_write_kb":286656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":2529},{"shards":"4,7","objects":-1248}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":594,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999777+0000","last_change":"2023-09-23T18:02:51.047767+0000","last_active":"2023-09-23T18:03:31.999777+0000","last_peered":"2023-09-23T18:03:31.999777+0000","last_clean":"2023-09-23T18:03:31.999777+0000","last_became_active":"2023-09-23T18:02:51.047297+0000","last_became_peered":"2023-09-23T18:02:51.047297+0000","last_unstale":"2023-09-23T18:03:31.999777+0000","last_undegraded":"2023-09-23T18:03:31.999777+0000","last_fullsized":"2023-09-23T18:03:31.999777+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.851694+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1188,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.019639+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:03:32.019639+0000","last_peered":"2023-09-23T18:03:32.019639+0000","last_clean":"2023-09-23T18:03:32.019639+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:03:32.019639+0000","last_undegraded":"2023-09-23T18:03:32.019639+0000","last_fullsized":"2023-09-23T18:03:32.019639+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":50,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.003055+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:03:32.003055+0000","last_peered":"2023-09-23T18:03:32.003055+0000","last_clean":"2023-09-23T18:03:32.003055+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:03:32.003055+0000","last_undegraded":"2023-09-23T18:03:32.003055+0000","last_fullsized":"2023-09-23T18:03:32.003055+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":153,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.001802+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:03:32.001802+0000","last_peered":"2023-09-23T18:03:32.001802+0000","last_clean":"2023-09-23T18:03:32.001802+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:03:32.001802+0000","last_undegraded":"2023-09-23T18:03:32.001802+0000","last_fullsized":"2023-09-23T18:03:32.001802+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"260'6262","reported_seq":7404,"reported_epoch":260,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:37.985467+0000","last_change":"2023-09-23T18:03:31.606786+0000","last_active":"2023-09-23T18:03:37.985467+0000","last_peered":"2023-09-23T18:03:37.985467+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.253817+0000","last_became_peered":"2023-09-23T18:02:19.253817+0000","last_unstale":"2023-09-23T18:03:37.985467+0000","last_undegraded":"2023-09-23T18:02:18.955613+0000","last_fullsized":"2023-09-23T18:02:18.923894+0000","mapping_epoch":238,"log_start":"222'2274","ondisk_log_start":"222'2274","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":3988,"log_dups_size":0,"ondisk_log_size":3988,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:11:18.999679+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":196739072,"num_objects":3002,"num_object_clones":0,"num_object_copies":9006,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3737,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3002,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6262,"num_write_kb":296448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":540,"num_bytes_recovered":35389440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":-735}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":595,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999791+0000","last_change":"2023-09-23T18:02:51.054622+0000","last_active":"2023-09-23T18:03:31.999791+0000","last_peered":"2023-09-23T18:03:31.999791+0000","last_clean":"2023-09-23T18:03:31.999791+0000","last_became_active":"2023-09-23T18:02:51.049345+0000","last_became_peered":"2023-09-23T18:02:51.049345+0000","last_unstale":"2023-09-23T18:03:31.999791+0000","last_undegraded":"2023-09-23T18:03:31.999791+0000","last_fullsized":"2023-09-23T18:03:31.999791+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:58.636010+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":542,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.001299+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:03:32.001299+0000","last_peered":"2023-09-23T18:03:32.001299+0000","last_clean":"2023-09-23T18:03:32.001299+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:03:32.001299+0000","last_undegraded":"2023-09-23T18:03:32.001299+0000","last_fullsized":"2023-09-23T18:03:32.001299+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":154,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000757+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:03:32.000757+0000","last_peered":"2023-09-23T18:03:32.000757+0000","last_clean":"2023-09-23T18:03:32.000757+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:03:32.000757+0000","last_undegraded":"2023-09-23T18:03:32.000757+0000","last_fullsized":"2023-09-23T18:03:32.000757+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":227,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.021137+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:03:32.021137+0000","last_peered":"2023-09-23T18:03:32.021137+0000","last_clean":"2023-09-23T18:03:32.021137+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:03:32.021137+0000","last_undegraded":"2023-09-23T18:03:32.021137+0000","last_fullsized":"2023-09-23T18:03:32.021137+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"260'6196","reported_seq":6294,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:37.544850+0000","last_change":"2023-09-23T18:02:47.986654+0000","last_active":"2023-09-23T18:03:37.544850+0000","last_peered":"2023-09-23T18:03:37.544850+0000","last_clean":"2023-09-23T18:02:16.962656+0000","last_became_active":"2023-09-23T18:02:19.272896+0000","last_became_peered":"2023-09-23T18:02:19.272896+0000","last_unstale":"2023-09-23T18:03:37.544850+0000","last_undegraded":"2023-09-23T18:02:18.972349+0000","last_fullsized":"2023-09-23T18:02:18.930868+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6196,"log_dups_size":0,"ondisk_log_size":6196,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:23.745675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":199491584,"num_objects":3044,"num_object_clones":0,"num_object_copies":9132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4263,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6196,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":2688},{"shards":"4,5","objects":-1219}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1172,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.019710+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:03:32.019710+0000","last_peered":"2023-09-23T18:03:32.019710+0000","last_clean":"2023-09-23T18:03:32.019710+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:03:32.019710+0000","last_undegraded":"2023-09-23T18:03:32.019710+0000","last_fullsized":"2023-09-23T18:03:32.019710+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":583,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000903+0000","last_change":"2023-09-23T18:02:51.054022+0000","last_active":"2023-09-23T18:03:32.000903+0000","last_peered":"2023-09-23T18:03:32.000903+0000","last_clean":"2023-09-23T18:03:32.000903+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:03:32.000903+0000","last_undegraded":"2023-09-23T18:03:32.000903+0000","last_fullsized":"2023-09-23T18:03:32.000903+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":67,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.005288+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:03:32.005288+0000","last_peered":"2023-09-23T18:03:32.005288+0000","last_clean":"2023-09-23T18:03:32.005288+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:03:32.005288+0000","last_undegraded":"2023-09-23T18:03:32.005288+0000","last_fullsized":"2023-09-23T18:03:32.005288+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":193,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.997959+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:03:31.997959+0000","last_peered":"2023-09-23T18:03:31.997959+0000","last_clean":"2023-09-23T18:03:31.997959+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:03:31.997959+0000","last_undegraded":"2023-09-23T18:03:31.997959+0000","last_fullsized":"2023-09-23T18:03:31.997959+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"260'6149","reported_seq":6307,"reported_epoch":260,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:03:37.503951+0000","last_change":"2023-09-23T18:02:18.003772+0000","last_active":"2023-09-23T18:03:37.503951+0000","last_peered":"2023-09-23T18:03:37.503951+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:18.000696+0000","last_became_peered":"2023-09-23T18:02:18.000696+0000","last_unstale":"2023-09-23T18:03:37.503951+0000","last_undegraded":"2023-09-23T18:02:17.938873+0000","last_fullsized":"2023-09-23T18:02:17.919669+0000","mapping_epoch":237,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6049,"log_dups_size":0,"ondisk_log_size":6049,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:48.085957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":192217088,"num_objects":2933,"num_object_clones":0,"num_object_copies":8799,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2906,"num_objects_misplaced":2933,"num_objects_unfound":0,"num_objects_dirty":2933,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6149,"num_write_kb":290624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":2933}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":595,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000121+0000","last_change":"2023-09-23T18:02:51.061168+0000","last_active":"2023-09-23T18:03:32.000121+0000","last_peered":"2023-09-23T18:03:32.000121+0000","last_clean":"2023-09-23T18:03:32.000121+0000","last_became_active":"2023-09-23T18:02:51.060689+0000","last_became_peered":"2023-09-23T18:02:51.060689+0000","last_unstale":"2023-09-23T18:03:32.000121+0000","last_undegraded":"2023-09-23T18:03:32.000121+0000","last_fullsized":"2023-09-23T18:03:32.000121+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:29.406460+0000","scrub_duration":0.001568505,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":440,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.001646+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:03:32.001646+0000","last_peered":"2023-09-23T18:03:32.001646+0000","last_clean":"2023-09-23T18:03:32.001646+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:03:32.001646+0000","last_undegraded":"2023-09-23T18:03:32.001646+0000","last_fullsized":"2023-09-23T18:03:32.001646+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":258,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.998270+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:03:31.998270+0000","last_peered":"2023-09-23T18:03:31.998270+0000","last_clean":"2023-09-23T18:03:31.998270+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:03:31.998270+0000","last_undegraded":"2023-09-23T18:03:31.998270+0000","last_fullsized":"2023-09-23T18:03:31.998270+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":229,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999072+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:03:31.999072+0000","last_peered":"2023-09-23T18:03:31.999072+0000","last_clean":"2023-09-23T18:03:31.999072+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:03:31.999072+0000","last_undegraded":"2023-09-23T18:03:31.999072+0000","last_fullsized":"2023-09-23T18:03:31.999072+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"260'6160","reported_seq":14501,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T18:03:35.477646+0000","last_change":"2023-09-23T18:02:18.942886+0000","last_active":"2023-09-23T18:03:35.477646+0000","last_peered":"2023-09-23T18:03:35.477646+0000","last_clean":"2023-09-23T18:03:35.477646+0000","last_became_active":"2023-09-23T18:02:18.936691+0000","last_became_peered":"2023-09-23T18:02:18.936691+0000","last_unstale":"2023-09-23T18:03:35.477646+0000","last_undegraded":"2023-09-23T18:03:35.477646+0000","last_fullsized":"2023-09-23T18:03:35.477646+0000","mapping_epoch":238,"log_start":"218'1958","ondisk_log_start":"218'1958","created":204,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4202,"log_dups_size":0,"ondisk_log_size":4202,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:26.085859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":201588736,"num_objects":3076,"num_object_clones":0,"num_object_copies":9228,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3076,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6160,"num_write_kb":295552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4123,"num_bytes_recovered":270204928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":592,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000517+0000","last_change":"2023-09-23T18:02:51.057370+0000","last_active":"2023-09-23T18:03:32.000517+0000","last_peered":"2023-09-23T18:03:32.000517+0000","last_clean":"2023-09-23T18:03:32.000517+0000","last_became_active":"2023-09-23T18:02:51.056158+0000","last_became_peered":"2023-09-23T18:02:51.056158+0000","last_unstale":"2023-09-23T18:03:32.000517+0000","last_undegraded":"2023-09-23T18:03:32.000517+0000","last_fullsized":"2023-09-23T18:03:32.000517+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":474,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004534+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:03:32.004534+0000","last_peered":"2023-09-23T18:03:32.004534+0000","last_clean":"2023-09-23T18:03:32.004534+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:03:32.004534+0000","last_undegraded":"2023-09-23T18:03:32.004534+0000","last_fullsized":"2023-09-23T18:03:32.004534+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":194,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.001887+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:03:32.001887+0000","last_peered":"2023-09-23T18:03:32.001887+0000","last_clean":"2023-09-23T18:03:32.001887+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:03:32.001887+0000","last_undegraded":"2023-09-23T18:03:32.001887+0000","last_fullsized":"2023-09-23T18:03:32.001887+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":226,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.998899+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:03:31.998899+0000","last_peered":"2023-09-23T18:03:31.998899+0000","last_clean":"2023-09-23T18:03:31.998899+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:03:31.998899+0000","last_undegraded":"2023-09-23T18:03:31.998899+0000","last_fullsized":"2023-09-23T18:03:31.998899+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"260'6089","reported_seq":6178,"reported_epoch":260,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:03:37.983125+0000","last_change":"2023-09-23T18:02:19.279112+0000","last_active":"2023-09-23T18:03:37.983125+0000","last_peered":"2023-09-23T18:03:37.983125+0000","last_clean":"2023-09-23T18:02:17.018900+0000","last_became_active":"2023-09-23T18:02:19.267439+0000","last_became_peered":"2023-09-23T18:02:19.267439+0000","last_unstale":"2023-09-23T18:03:37.983125+0000","last_undegraded":"2023-09-23T18:02:18.959329+0000","last_fullsized":"2023-09-23T18:02:18.926801+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6089,"log_dups_size":0,"ondisk_log_size":6089,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.395583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":198246400,"num_objects":3025,"num_object_clones":0,"num_object_copies":9075,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3025,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6089,"num_write_kb":291648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-1174},{"shards":"2,4,6","objects":2667}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":592,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.001869+0000","last_change":"2023-09-23T18:02:51.058680+0000","last_active":"2023-09-23T18:03:32.001869+0000","last_peered":"2023-09-23T18:03:32.001869+0000","last_clean":"2023-09-23T18:03:32.001869+0000","last_became_active":"2023-09-23T18:02:51.057706+0000","last_became_peered":"2023-09-23T18:02:51.057706+0000","last_unstale":"2023-09-23T18:03:32.001869+0000","last_undegraded":"2023-09-23T18:03:32.001869+0000","last_fullsized":"2023-09-23T18:03:32.001869+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":105,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.005012+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:03:32.005012+0000","last_peered":"2023-09-23T18:03:32.005012+0000","last_clean":"2023-09-23T18:03:32.005012+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:03:32.005012+0000","last_undegraded":"2023-09-23T18:03:32.005012+0000","last_fullsized":"2023-09-23T18:03:32.005012+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":330,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.997818+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:03:31.997818+0000","last_peered":"2023-09-23T18:03:31.997818+0000","last_clean":"2023-09-23T18:03:31.997818+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:03:31.997818+0000","last_undegraded":"2023-09-23T18:03:31.997818+0000","last_fullsized":"2023-09-23T18:03:31.997818+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":109,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.005067+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:03:32.005067+0000","last_peered":"2023-09-23T18:03:32.005067+0000","last_clean":"2023-09-23T18:03:32.005067+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:03:32.005067+0000","last_undegraded":"2023-09-23T18:03:32.005067+0000","last_fullsized":"2023-09-23T18:03:32.005067+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"260'6169","reported_seq":6235,"reported_epoch":260,"state":"active+clean","last_fresh":"2023-09-23T18:03:37.852605+0000","last_change":"2023-09-23T18:01:25.046224+0000","last_active":"2023-09-23T18:03:37.852605+0000","last_peered":"2023-09-23T18:03:37.852605+0000","last_clean":"2023-09-23T18:03:37.852605+0000","last_became_active":"2023-09-23T18:01:25.002864+0000","last_became_peered":"2023-09-23T18:01:25.002864+0000","last_unstale":"2023-09-23T18:03:37.852605+0000","last_undegraded":"2023-09-23T18:03:37.852605+0000","last_fullsized":"2023-09-23T18:03:37.852605+0000","mapping_epoch":214,"log_start":"221'2141","ondisk_log_start":"221'2141","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4028,"log_dups_size":0,"ondisk_log_size":4028,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.752359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":194576384,"num_objects":2969,"num_object_clones":0,"num_object_copies":8907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6169,"num_write_kb":292416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":599,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000372+0000","last_change":"2023-09-23T18:02:51.053420+0000","last_active":"2023-09-23T18:03:32.000372+0000","last_peered":"2023-09-23T18:03:32.000372+0000","last_clean":"2023-09-23T18:03:32.000372+0000","last_became_active":"2023-09-23T18:02:51.053035+0000","last_became_peered":"2023-09-23T18:02:51.053035+0000","last_unstale":"2023-09-23T18:03:32.000372+0000","last_undegraded":"2023-09-23T18:03:32.000372+0000","last_fullsized":"2023-09-23T18:03:32.000372+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:20.456481+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":465,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.998385+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:03:31.998385+0000","last_peered":"2023-09-23T18:03:31.998385+0000","last_clean":"2023-09-23T18:03:31.998385+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:03:31.998385+0000","last_undegraded":"2023-09-23T18:03:31.998385+0000","last_fullsized":"2023-09-23T18:03:31.998385+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":286,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999165+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:03:31.999165+0000","last_peered":"2023-09-23T18:03:31.999165+0000","last_clean":"2023-09-23T18:03:31.999165+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:03:31.999165+0000","last_undegraded":"2023-09-23T18:03:31.999165+0000","last_fullsized":"2023-09-23T18:03:31.999165+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":50,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.007152+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:03:32.007152+0000","last_peered":"2023-09-23T18:03:32.007152+0000","last_clean":"2023-09-23T18:03:32.007152+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:03:32.007152+0000","last_undegraded":"2023-09-23T18:03:32.007152+0000","last_fullsized":"2023-09-23T18:03:32.007152+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":582,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999903+0000","last_change":"2023-09-23T18:02:51.046571+0000","last_active":"2023-09-23T18:03:31.999903+0000","last_peered":"2023-09-23T18:03:31.999903+0000","last_clean":"2023-09-23T18:03:31.999903+0000","last_became_active":"2023-09-23T18:02:51.046213+0000","last_became_peered":"2023-09-23T18:02:51.046213+0000","last_unstale":"2023-09-23T18:03:31.999903+0000","last_undegraded":"2023-09-23T18:03:31.999903+0000","last_fullsized":"2023-09-23T18:03:31.999903+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":246,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.998814+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:03:31.998814+0000","last_peered":"2023-09-23T18:03:31.998814+0000","last_clean":"2023-09-23T18:03:31.998814+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:03:31.998814+0000","last_undegraded":"2023-09-23T18:03:31.998814+0000","last_fullsized":"2023-09-23T18:03:31.998814+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":272,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.001962+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:03:32.001962+0000","last_peered":"2023-09-23T18:03:32.001962+0000","last_clean":"2023-09-23T18:03:32.001962+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:03:32.001962+0000","last_undegraded":"2023-09-23T18:03:32.001962+0000","last_fullsized":"2023-09-23T18:03:32.001962+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":591,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000345+0000","last_change":"2023-09-23T18:02:51.060198+0000","last_active":"2023-09-23T18:03:32.000345+0000","last_peered":"2023-09-23T18:03:32.000345+0000","last_clean":"2023-09-23T18:03:32.000345+0000","last_became_active":"2023-09-23T18:02:51.057046+0000","last_became_peered":"2023-09-23T18:02:51.057046+0000","last_unstale":"2023-09-23T18:03:32.000345+0000","last_undegraded":"2023-09-23T18:03:32.000345+0000","last_fullsized":"2023-09-23T18:03:32.000345+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":115,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.019929+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:03:32.019929+0000","last_peered":"2023-09-23T18:03:32.019929+0000","last_clean":"2023-09-23T18:03:32.019929+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:03:32.019929+0000","last_undegraded":"2023-09-23T18:03:32.019929+0000","last_fullsized":"2023-09-23T18:03:32.019929+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":185,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.003199+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:03:32.003199+0000","last_peered":"2023-09-23T18:03:32.003199+0000","last_clean":"2023-09-23T18:03:32.003199+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:03:32.003199+0000","last_undegraded":"2023-09-23T18:03:32.003199+0000","last_fullsized":"2023-09-23T18:03:32.003199+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":101,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.020689+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:03:32.020689+0000","last_peered":"2023-09-23T18:03:32.020689+0000","last_clean":"2023-09-23T18:03:32.020689+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:03:32.020689+0000","last_undegraded":"2023-09-23T18:03:32.020689+0000","last_fullsized":"2023-09-23T18:03:32.020689+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":124,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004540+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:03:32.004540+0000","last_peered":"2023-09-23T18:03:32.004540+0000","last_clean":"2023-09-23T18:03:32.004540+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:03:32.004540+0000","last_undegraded":"2023-09-23T18:03:32.004540+0000","last_fullsized":"2023-09-23T18:03:32.004540+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":607,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.000470+0000","last_change":"2023-09-23T18:02:51.047476+0000","last_active":"2023-09-23T18:03:32.000470+0000","last_peered":"2023-09-23T18:03:32.000470+0000","last_clean":"2023-09-23T18:03:32.000470+0000","last_became_active":"2023-09-23T18:02:51.047152+0000","last_became_peered":"2023-09-23T18:02:51.047152+0000","last_unstale":"2023-09-23T18:03:32.000470+0000","last_undegraded":"2023-09-23T18:03:32.000470+0000","last_fullsized":"2023-09-23T18:03:32.000470+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:22.514077+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":124,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004798+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:03:32.004798+0000","last_peered":"2023-09-23T18:03:32.004798+0000","last_clean":"2023-09-23T18:03:32.004798+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:03:32.004798+0000","last_undegraded":"2023-09-23T18:03:32.004798+0000","last_fullsized":"2023-09-23T18:03:32.004798+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":127,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.004328+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:03:32.004328+0000","last_peered":"2023-09-23T18:03:32.004328+0000","last_clean":"2023-09-23T18:03:32.004328+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:03:32.004328+0000","last_undegraded":"2023-09-23T18:03:32.004328+0000","last_fullsized":"2023-09-23T18:03:32.004328+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":590,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.001784+0000","last_change":"2023-09-23T18:02:51.058825+0000","last_active":"2023-09-23T18:03:32.001784+0000","last_peered":"2023-09-23T18:03:32.001784+0000","last_clean":"2023-09-23T18:03:32.001784+0000","last_became_active":"2023-09-23T18:02:51.058480+0000","last_became_peered":"2023-09-23T18:02:51.058480+0000","last_unstale":"2023-09-23T18:03:32.001784+0000","last_undegraded":"2023-09-23T18:03:32.001784+0000","last_fullsized":"2023-09-23T18:03:32.001784+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":238,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:31.999493+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:03:31.999493+0000","last_peered":"2023-09-23T18:03:31.999493+0000","last_clean":"2023-09-23T18:03:31.999493+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:03:31.999493+0000","last_undegraded":"2023-09-23T18:03:31.999493+0000","last_fullsized":"2023-09-23T18:03:31.999493+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":270,"reported_epoch":259,"state":"active+clean","last_fresh":"2023-09-23T18:03:32.002172+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:03:32.002172+0000","last_peered":"2023-09-23T18:03:32.002172+0000","last_clean":"2023-09-23T18:03:32.002172+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:03:32.002172+0000","last_undegraded":"2023-09-23T18:03:32.002172+0000","last_fullsized":"2023-09-23T18:03:32.002172+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":3143958552,"num_objects":47974,"num_object_clones":0,"num_object_copies":143922,"num_objects_missing_on_primary":504,"num_objects_missing":504,"num_objects_degraded":30815,"num_objects_misplaced":11089,"num_objects_unfound":0,"num_objects_dirty":47974,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":98298,"num_write_kb":4680641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18274,"num_bytes_recovered":1178009600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":9523310592,"data_stored":9523298376,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83534,"ondisk_log_size":83534,"up":48,"acting":39,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":402523,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":331,"num_read_kb":623,"num_write":162,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2487153,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001372,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1729840,"kb_used_data":1632808,"kb_used_omap":13,"kb_used_meta":97010,"kb_avail":92015312,"statfs":{"total":95995035648,"available":94223679488,"internally_reserved":0,"allocated":1671995392,"data_stored":1670996341,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":99338344},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":2.2160000000000002,"5min":2.544,"15min":2.544},"last":0.60499999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.226,"15min":1.266},"min":{"1min":0.32100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.5990000000000002,"5min":6.5990000000000002,"15min":6.5990000000000002},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.71599999999999997,"15min":0.68999999999999995},"min":{"1min":0.27400000000000002,"5min":0.252,"15min":0.252},"max":{"1min":3.2850000000000001,"5min":3.2850000000000001,"15min":3.2850000000000001},"last":0.27900000000000003},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.0649999999999999,"15min":1.0589999999999999},"min":{"1min":0.35199999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.2169999999999996,"5min":6.2169999999999996,"15min":6.2169999999999996},"last":1.6040000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":0.95699999999999996,"15min":0.94399999999999995},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.1539999999999999,"5min":4.1539999999999999,"15min":4.1539999999999999},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.161,"5min":0.88200000000000001,"15min":0.83599999999999997},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.1970000000000001,"5min":6.1970000000000001,"15min":6.1970000000000001},"last":0.31900000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:03:19 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.554,"15min":1.554},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":4.2670000000000003,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":0.59399999999999997},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":3.0950000000000002,"5min":3.0950000000000002,"15min":3.0950000000000002},"last":1.0369999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":1.014,"15min":1.052},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.4199999999999999,"5min":3.984,"15min":3.984},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.627,"5min":0.54600000000000004,"15min":0.53200000000000003},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.1040000000000001,"5min":2.1040000000000001,"15min":2.1040000000000001},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.94599999999999995,"15min":0.98399999999999999},"min":{"1min":0.33900000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.8939999999999999,"5min":3.9300000000000002,"15min":3.9300000000000002},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.87,"5min":1.0800000000000001,"15min":1.115},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.403,"5min":3.7080000000000002,"15min":3.7080000000000002},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.69799999999999995,"15min":0.70299999999999996},"min":{"1min":0.39500000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.1429999999999998,"5min":3.0409999999999999,"15min":3.0409999999999999},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.046,"15min":1.099},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.8560000000000001,"5min":3.6419999999999999,"15min":3.6419999999999999},"last":0.61499999999999999}]}]},{"osd":6,"up_from":163,"seq":700079669308,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2120936,"kb_used_data":1973792,"kb_used_omap":7,"kb_used_meta":147128,"kb_avail":91624216,"statfs":{"total":95995035648,"available":93823197184,"internally_reserved":0,"allocated":2021163008,"data_stored":2020172022,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":150659876},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.80800000000000005,"5min":1.2270000000000001,"15min":1.3089999999999999},"min":{"1min":0.224,"5min":0.215,"15min":0.215},"max":{"1min":3.5529999999999999,"5min":14.196999999999999,"15min":14.196999999999999},"last":0.505},{"interface":"front","average":{"1min":0.98999999999999999,"5min":1.72,"15min":2.3740000000000001},"min":{"1min":0.32600000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":4.8970000000000002,"5min":13.536,"15min":13.536},"last":0.59699999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.90500000000000003,"5min":1.5860000000000001,"15min":1.956},"min":{"1min":0.28100000000000003,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":2.4910000000000001,"5min":14.233000000000001,"15min":14.233000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":1.01,"5min":1.6759999999999999,"15min":2.173},"min":{"1min":0.33300000000000002,"5min":0.248,"15min":0.248},"max":{"1min":4.9340000000000002,"5min":9.1099999999999994,"15min":9.1099999999999994},"last":0.42699999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":2.0009999999999999,"15min":2.3450000000000002},"min":{"1min":0.34999999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":7.173,"5min":11.210000000000001,"15min":11.210000000000001},"last":0.68500000000000005},{"interface":"front","average":{"1min":1.0589999999999999,"5min":2.1899999999999999,"15min":2.758},"min":{"1min":0.28199999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":2.9700000000000002,"5min":11.223000000000001,"15min":11.223000000000001},"last":0.69499999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":3.081,"5min":3.081,"15min":3.081},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":5.04,"5min":5.04,"15min":5.04},"last":0.621}]},{"osd":4,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":1.157,"5min":0.96599999999999997,"15min":0.89400000000000002},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":5.827,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.372},{"interface":"front","average":{"1min":0.76700000000000002,"5min":0.72499999999999998,"15min":0.503},"min":{"1min":0.29099999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":3.0129999999999999,"5min":8.3000000000000007,"15min":8.3000000000000007},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.998,"5min":0.94199999999999995,"15min":0.84399999999999997},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":4.1050000000000004,"5min":11.869,"15min":11.869},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.21,"5min":0.875,"15min":0.63500000000000001},"min":{"1min":0.26300000000000001,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.7130000000000001,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:03:26 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.0209999999999999,"15min":0.98299999999999998},"min":{"1min":0.379,"5min":0.307,"15min":0.307},"max":{"1min":4.069,"5min":4.069,"15min":4.069},"last":0.443},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.79100000000000004,"15min":0.78500000000000003},"min":{"1min":0.39900000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.7970000000000002,"5min":2.7970000000000002,"15min":2.7970000000000002},"last":0.45000000000000001}]}]},{"osd":1,"up_from":0,"seq":420906795127,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958653,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1186132,"kb_used_data":784396,"kb_used_omap":7,"kb_used_meta":401720,"kb_avail":92559020,"statfs":{"total":95995035648,"available":94780436480,"internally_reserved":0,"allocated":803221504,"data_stored":801901900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":411362020},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:03:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":2.0059999999999998,"15min":1.135},"min":{"1min":0.20300000000000001,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":7.7000000000000002,"5min":18.154,"15min":18.154},"last":2.238},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.871,"15min":1.1080000000000001},"min":{"1min":0.29599999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":6.8550000000000004,"5min":13.369999999999999,"15min":13.369999999999999},"last":0.996}]},{"osd":2,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":2.4460000000000002,"15min":1.24},"min":{"1min":0.25700000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.5259999999999998,"5min":20.882000000000001,"15min":20.882000000000001},"last":2.2989999999999999},{"interface":"front","average":{"1min":1.234,"5min":2.0899999999999999,"15min":1.149},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":6.7439999999999998,"5min":15.064,"15min":15.064},"last":1.0760000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.048,"5min":6.048,"15min":6.048},"last":0.69599999999999995},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.2010000000000001,"15min":1.2010000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.391,"5min":4.391,"15min":4.391},"last":2.407}]},{"osd":4,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":0.72599999999999998,"5min":1.603,"15min":1.0509999999999999},"min":{"1min":0.29299999999999998,"5min":0.251,"15min":0.251},"max":{"1min":4.54,"5min":15.279999999999999,"15min":15.279999999999999},"last":0.54000000000000004},{"interface":"front","average":{"1min":1.24,"5min":1.859,"15min":1.1220000000000001},"min":{"1min":0.29899999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7190000000000003,"5min":18.106999999999999,"15min":18.106999999999999},"last":2.1989999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1080000000000001,"5min":1.9710000000000001,"15min":1.1619999999999999},"min":{"1min":0.318,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.1280000000000001,"5min":14.720000000000001,"15min":14.720000000000001},"last":2.2679999999999998},{"interface":"front","average":{"1min":0.79000000000000004,"5min":1.5469999999999999,"15min":1.022},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":2.806,"5min":12.794,"15min":12.794},"last":0.96799999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:02:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.8680000000000001,"15min":2.3010000000000002},"min":{"1min":0.36199999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.5069999999999997,"5min":14.956,"15min":14.956},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.84699999999999998,"5min":2.0419999999999998,"15min":2.5219999999999998},"min":{"1min":0.34699999999999998,"5min":0.311,"15min":0.311},"max":{"1min":2.883,"5min":12.755000000000001,"15min":12.755000000000001},"last":1.0489999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:03:28 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.93400000000000005,"15min":0.92600000000000005},"min":{"1min":0.41499999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":6.9489999999999998,"5min":6.9489999999999998,"15min":6.9489999999999998},"last":1.0209999999999999},{"interface":"front","average":{"1min":1.6439999999999999,"5min":1.1950000000000001,"15min":1.1200000000000001},"min":{"1min":0.40999999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.9880000000000004,"5min":4.9880000000000004,"15min":4.9880000000000004},"last":0.72499999999999998}]}]},{"osd":2,"up_from":103,"seq":442381631600,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2117064,"kb_used_data":1761728,"kb_used_omap":11,"kb_used_meta":355316,"kb_avail":91628088,"statfs":{"total":95995035648,"available":93827162112,"internally_reserved":0,"allocated":1804009472,"data_stored":1802702007,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":363843959},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.66100000000000003,"5min":2.552,"15min":1.2729999999999999},"min":{"1min":0.187,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":3.306,"5min":20.867000000000001,"15min":20.867000000000001},"last":1.179},{"interface":"front","average":{"1min":0.55300000000000005,"5min":2.3119999999999998,"15min":1.246},"min":{"1min":0.24299999999999999,"5min":0.215,"15min":0.215},"max":{"1min":1.6910000000000001,"5min":21.681000000000001,"15min":21.681000000000001},"last":1.365}]},{"osd":1,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.84299999999999997,"5min":2.6349999999999998,"15min":1.361},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":4.2359999999999998,"5min":20.995000000000001,"15min":20.995000000000001},"last":2.141},{"interface":"front","average":{"1min":0.75,"5min":1.9530000000000001,"15min":1.147},"min":{"1min":0.24199999999999999,"5min":0.193,"15min":0.193},"max":{"1min":4.532,"5min":15.945,"15min":15.945},"last":1.504}]},{"osd":3,"last update":"Sat Sep 23 18:03:21 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.6040000000000001,"5min":3.6040000000000001,"15min":3.6040000000000001},"last":1.3440000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.0790000000000002,"5min":3.0790000000000002,"15min":3.0790000000000002},"last":1.0629999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.87,"5min":1.7649999999999999,"15min":1.095},"min":{"1min":0.29099999999999998,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":5.141,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":1.153},{"interface":"front","average":{"1min":0.45900000000000002,"5min":1.8899999999999999,"15min":1.1200000000000001},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.78100000000000003,"5min":20.920000000000002,"15min":20.920000000000002},"last":1.2270000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.9890000000000001,"15min":1.1879999999999999},"min":{"1min":0.32900000000000001,"5min":0.315,"15min":0.315},"max":{"1min":2.972,"5min":20.738,"15min":20.738},"last":1.2649999999999999},{"interface":"front","average":{"1min":1.048,"5min":1.79,"15min":1.1970000000000001},"min":{"1min":0.34100000000000003,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.3700000000000001,"5min":14.359,"15min":15.496},"last":1.3979999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:02:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0209999999999999,"5min":2.0299999999999998,"15min":2.2240000000000002},"min":{"1min":0.34699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":5.1879999999999997,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":1.1220000000000001},{"interface":"front","average":{"1min":0.66900000000000004,"5min":2.3290000000000002,"15min":2.4929999999999999},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.6930000000000001,"5min":20.893000000000001,"15min":20.893000000000001},"last":1.3320000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:03:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.335,"15min":1.385},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":3.5600000000000001,"5min":4.3239999999999998,"15min":4.3239999999999998},"last":1.429},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.66300000000000003,"15min":0.64300000000000002},"min":{"1min":0.40699999999999997,"5min":0.309,"15min":0.309},"max":{"1min":2.7789999999999999,"5min":2.7789999999999999,"15min":2.7789999999999999},"last":1.3049999999999999}]}]},{"osd":3,"up_from":237,"seq":1017907249169,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1267280,"kb_used_data":1158920,"kb_used_omap":3,"kb_used_meta":108348,"kb_avail":92477872,"statfs":{"total":95995035648,"available":94697340928,"internally_reserved":0,"allocated":1186734080,"data_stored":1185248312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":110949096},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.6560000000000001,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.318},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":4.5529999999999999,"5min":4.5529999999999999,"15min":4.5529999999999999},"last":0.374}]},{"osd":1,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.6050000000000004,"5min":4.6050000000000004,"15min":4.6050000000000004},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.8810000000000002,"5min":4.8810000000000002,"15min":4.8810000000000002},"last":0.27600000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.7639999999999998,"5min":2.7639999999999998,"15min":2.7639999999999998},"last":0.22600000000000001},{"interface":"front","average":{"1min":1.246,"5min":1.246,"15min":1.246},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.157,"5min":6.157,"15min":6.157},"last":0.30099999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.633,"5min":4.633,"15min":4.633},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.7300000000000004,"5min":4.7300000000000004,"15min":4.7300000000000004},"last":0.33300000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.284,"15min":1.284},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":7.0640000000000001,"5min":7.0640000000000001,"15min":7.0640000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.181,"5min":1.181,"15min":1.181},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.6150000000000002,"5min":4.6150000000000002,"15min":4.6150000000000002},"last":0.47499999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.085,"15min":1.085},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.0590000000000002,"5min":4.0590000000000002,"15min":4.0590000000000002},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.48,"5min":1.48,"15min":1.48},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":7.3929999999999998,"5min":7.3929999999999998,"15min":7.3929999999999998},"last":0.42099999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.5720000000000001,"15min":1.5720000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":8.4190000000000005,"5min":8.4190000000000005,"15min":8.4190000000000005},"last":1.4319999999999999},{"interface":"front","average":{"1min":1.6200000000000001,"5min":1.6200000000000001,"15min":1.6200000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":5.1760000000000002,"5min":5.1760000000000002,"15min":5.1760000000000002},"last":0.496}]}]},{"osd":4,"up_from":30,"seq":128849019119,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1309996,"kb_used_data":1093476,"kb_used_omap":0,"kb_used_meta":216512,"kb_avail":92435156,"statfs":{"total":95995035648,"available":94653599744,"internally_reserved":0,"allocated":1119719424,"data_stored":1118724178,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":221708288},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:36 2023","interfaces":[{"interface":"back","average":{"1min":0.747,"5min":1.3260000000000001,"15min":0.82099999999999995},"min":{"1min":0.24399999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":5.1829999999999998,"5min":14.164,"15min":14.164},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.611,"15min":0.97899999999999998},"min":{"1min":0.253,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":4.29,"5min":11.372,"15min":11.372},"last":0.47499999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:03:04 2023","interfaces":[{"interface":"back","average":{"1min":1.22,"5min":1.532,"15min":1.002},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.249},"max":{"1min":5.2190000000000003,"5min":10.429,"15min":10.429},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.1819999999999999,"5min":1.46,"15min":1.0629999999999999},"min":{"1min":0.28699999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.2190000000000003,"5min":10.519,"15min":10.83},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.6699999999999999,"15min":1.0629999999999999},"min":{"1min":0.29899999999999999,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.266,"5min":11.630000000000001,"15min":11.630000000000001},"last":0.63100000000000001},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.5289999999999999,"15min":0.98799999999999999},"min":{"1min":0.316,"5min":0.30099999999999999,"15min":0.27100000000000002},"max":{"1min":5.2450000000000001,"5min":7.5090000000000003,"15min":7.5090000000000003},"last":0.49399999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.351,"15min":1.351},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":7.2140000000000004,"5min":7.2140000000000004,"15min":7.2140000000000004},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.373,"5min":1.373,"15min":1.373},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.1970000000000001,"5min":5.1970000000000001,"15min":5.1970000000000001},"last":0.433}]},{"osd":5,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":0.79200000000000004,"5min":0.80800000000000005,"15min":0.59299999999999997},"min":{"1min":0.23799999999999999,"5min":0.224,"15min":0.224},"max":{"1min":3.2269999999999999,"5min":7.0810000000000004,"15min":7.0810000000000004},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.0620000000000001,"5min":1.0029999999999999,"15min":0.71299999999999997},"min":{"1min":0.26800000000000002,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":4.0629999999999997,"5min":7.4729999999999999,"15min":10.494},"last":0.503}]},{"osd":6,"last update":"Sat Sep 23 18:02:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0840000000000001,"5min":0.94499999999999995,"15min":0.82799999999999996},"min":{"1min":0.25,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.5579999999999998,"5min":7.1529999999999996,"15min":7.1529999999999996},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.0189999999999999,"5min":1.0680000000000001,"15min":0.95599999999999996},"min":{"1min":0.28999999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.286,"5min":10.725,"15min":10.725},"last":0.51500000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:03:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":0.98099999999999998,"15min":0.93600000000000005},"min":{"1min":0.35899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.3140000000000001,"5min":5.3140000000000001,"15min":5.3140000000000001},"last":0.56000000000000005},{"interface":"front","average":{"1min":1.4550000000000001,"5min":1.2609999999999999,"15min":1.2290000000000001},"min":{"1min":0.35799999999999998,"5min":0.307,"15min":0.307},"max":{"1min":5.2770000000000001,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.53100000000000003}]}]},{"osd":5,"up_from":38,"seq":163208757485,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1221280,"kb_used_data":1030872,"kb_used_omap":0,"kb_used_meta":190400,"kb_avail":92523872,"statfs":{"total":95995035648,"available":94744444928,"internally_reserved":0,"allocated":1055612928,"data_stored":1054625434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":194969600},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":0.96999999999999997,"5min":1.6779999999999999,"15min":1.008},"min":{"1min":0.28699999999999998,"5min":0.217,"15min":0.217},"max":{"1min":4.0250000000000004,"5min":21.376999999999999,"15min":21.376999999999999},"last":0.31},{"interface":"front","average":{"1min":1.996,"5min":1.8939999999999999,"15min":1.149},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.048,"5min":21.388999999999999,"15min":21.388999999999999},"last":0.53700000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.054,"15min":0.85899999999999999},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":1.321,"5min":10.853999999999999,"15min":10.853999999999999},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.05,"5min":1.2969999999999999,"15min":0.95399999999999996},"min":{"1min":0.29199999999999998,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":3.6659999999999999,"5min":10.206,"15min":10.206},"last":0.501}]},{"osd":2,"last update":"Sat Sep 23 18:02:38 2023","interfaces":[{"interface":"back","average":{"1min":1.802,"5min":2.1320000000000001,"15min":1.2589999999999999},"min":{"1min":0.34200000000000003,"5min":0.255,"15min":0.255},"max":{"1min":4.9279999999999999,"5min":26.035,"15min":38.606999999999999},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.77400000000000002,"5min":1.534,"15min":1.123},"min":{"1min":0.35999999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.885,"5min":17.085000000000001,"15min":39.670999999999999},"last":0.55900000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:03:19 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.6179999999999999,"5min":2.6179999999999999,"15min":2.6179999999999999},"last":0.60199999999999998},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.7850000000000001,"5min":2.7850000000000001,"15min":2.7850000000000001},"last":0.42499999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6100000000000001,"5min":0.94499999999999995,"15min":0.69699999999999995},"min":{"1min":0.24199999999999999,"5min":0.22800000000000001,"15min":0.224},"max":{"1min":3.9980000000000002,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.57099999999999995},{"interface":"front","average":{"1min":1.4099999999999999,"5min":0.998,"15min":0.69799999999999995},"min":{"1min":0.30499999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":4.1449999999999996,"5min":5.077,"15min":5.077},"last":0.45600000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:02:42 2023","interfaces":[{"interface":"back","average":{"1min":1.524,"5min":1.343,"15min":1.5069999999999999},"min":{"1min":0.312,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":6.5880000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.147,"15min":1.329},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.1710000000000003,"5min":8.266,"15min":8.266},"last":0.58199999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:03:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.405,"15min":1.4990000000000001},"min":{"1min":0.39600000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.875,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.871,"5min":1.0740000000000001,"15min":1.1080000000000001},"min":{"1min":0.39100000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.7229999999999999,"5min":4.0380000000000003,"15min":4.0380000000000003},"last":0.52000000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":794169344,"data_stored":794165272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1237581824,"data_stored":1237581824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1791889408,"data_stored":1791885336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":355860480,"data_stored":355860480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1111687168,"data_stored":1111687168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1046089728,"data_stored":1046085656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2012545024,"data_stored":2012545024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1173487616,"data_stored":1173487616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:03:48.220 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:46 smithi055 ceph-mon[184174]: pgmap v614: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 30172/139674 objects degraded (21.602%); 10798/139674 objects misplaced (7.731%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T18:03:48.320 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:46 smithi055 ceph-mon[179668]: pgmap v614: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 226 op/s; 30172/139674 objects degraded (21.602%); 10798/139674 objects misplaced (7.731%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T18:03:48.321 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:48.322 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:03:48.322 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:03:48.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:48 smithi099 ceph-mon[167076]: pgmap v615: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 30172/139344 objects degraded (21.653%); 10676/139344 objects misplaced (7.662%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T18:03:48.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 30172/139344 objects degraded (21.653%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:48.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:48 smithi055 ceph-mon[184174]: pgmap v615: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 30172/139344 objects degraded (21.653%); 10676/139344 objects misplaced (7.662%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T18:03:48.986 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 30172/139344 objects degraded (21.653%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:48.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:48 smithi055 ceph-mon[179668]: pgmap v615: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 224 op/s; 30172/139344 objects degraded (21.653%); 10676/139344 objects misplaced (7.662%); 6.7 MiB/s, 107 objects/s recovering 2023-09-23T18:03:48.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 30172/139344 objects degraded (21.653%), 10 pgs degraded, 9 pgs undersized (PG_DEGRADED) 2023-09-23T18:03:48.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:48.987 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:48.988 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:03:48.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:03:49.088 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:03:49.129 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:49.419 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:03:49.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:49.538 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:03:49.639 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:03:49.739 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:50.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:50.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:03:50.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:03:50.074 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:50 smithi055 ceph-mon[184174]: pgmap v616: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 228 op/s; 29440/137157 objects degraded (21.464%); 10530/137157 objects misplaced (7.677%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T18:03:50.468 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:50 smithi055 ceph-mon[179668]: pgmap v616: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 228 op/s; 29440/137157 objects degraded (21.464%); 10530/137157 objects misplaced (7.677%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T18:03:50.469 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:03:50.469 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:03:50.470 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:03:50.470 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:50 smithi099 ceph-mon[167076]: pgmap v616: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 228 op/s; 29440/137157 objects degraded (21.464%); 10530/137157 objects misplaced (7.677%); 7.8 MiB/s, 124 objects/s recovering 2023-09-23T18:03:50.608 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:03:50.927 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:03:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:03:50] "GET /metrics HTTP/1.1" 200 36537 "" "Prometheus/2.43.0" 2023-09-23T18:03:50.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:03:50.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:03:50.929 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:03:50.929 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:38871 5.2 5:4086753f:::benchmark_data_smithi055_229117_object38870:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.242138+0000", 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 455.43678431199999, 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11274374099999999, 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:50.930 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:50.931 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:50.931 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 38871 2023-09-23T18:03:50.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:50.931 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:50.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:51.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242138+0000", 2023-09-23T18:03:51.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:51.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:51.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242138+0000", 2023-09-23T18:03:51.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:51.261 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:51.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242136+0000", 2023-09-23T18:03:51.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:51.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.543 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:51.543 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242159+0000", 2023-09-23T18:03:51.544 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3130999999999999e-05 2023-09-23T18:03:51.545 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.545 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.545 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:51.545 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242172+0000", 2023-09-23T18:03:51.545 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2405000000000001e-05 2023-09-23T18:03:51.546 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.546 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.546 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:51.546 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242181+0000", 2023-09-23T18:03:51.546 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1710000000000008e-06 2023-09-23T18:03:51.546 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.546 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242234+0000", 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3562e-05 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242386+0000", 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015176400000000001 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.547 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.242544+0000", 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015809600000000001 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.244070+0000", 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015261820000000001 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247882+0000", 2023-09-23T18:03:51.548 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038115940000000002 2023-09-23T18:03:51.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:51.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247934+0000", 2023-09-23T18:03:51.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1607000000000002e-05 2023-09-23T18:03:51.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:51.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.354881+0000", 2023-09-23T18:03:51.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10694785900000001 2023-09-23T18:03:51.875 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:51.875 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:38885 5.e e117/110 5:7e62a019:::benchmark_data_smithi055_229117_object38884:head v 117'2410, mlcod=117'2406)", 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:56:15.247175+0000", 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 455.43174728999998, 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.109947827, 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:51.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:51.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247175+0000", 2023-09-23T18:03:51.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:51.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:51.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247175+0000", 2023-09-23T18:03:51.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:51.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:51.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:51.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:51.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247174+0000", 2023-09-23T18:03:52.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:52.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.091 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:52.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247223+0000", 2023-09-23T18:03:52.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9112e-05 2023-09-23T18:03:52.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:52.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247231+0000", 2023-09-23T18:03:52.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.61e-06 2023-09-23T18:03:52.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:52.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.247236+0000", 2023-09-23T18:03:52.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8860000000000003e-06 2023-09-23T18:03:52.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:52.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353831+0000", 2023-09-23T18:03:52.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.106595129 2023-09-23T18:03:52.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:52.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.353853+0000", 2023-09-23T18:03:52.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1997e-05 2023-09-23T18:03:52.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357086+0000", 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00323339 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:56:15.357123+0000", 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6259e-05 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:52.095 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:52.096 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.096 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.096 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154235 5.2 5:44f142c1:::benchmark_data_smithi055_229117_object48609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:03:52.425 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.835816+0000", 2023-09-23T18:03:52.425 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 281.84310593700002, 2023-09-23T18:03:52.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.120395545, 2023-09-23T18:03:52.426 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:52.426 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:03:52.426 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:52.426 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:52.426 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:52.426 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154235 2023-09-23T18:03:52.426 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.426 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835816+0000", 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835816+0000", 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:52.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835815+0000", 2023-09-23T18:03:52.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:52.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:52.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835819+0000", 2023-09-23T18:03:52.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7720000000000001e-06 2023-09-23T18:03:52.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:52.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835827+0000", 2023-09-23T18:03:52.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9359999999999999e-06 2023-09-23T18:03:52.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:52.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835835+0000", 2023-09-23T18:03:52.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3760000000000002e-06 2023-09-23T18:03:52.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:52.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839941+0000", 2023-09-23T18:03:52.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041069510000000002 2023-09-23T18:03:52.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:52.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840066+0000", 2023-09-23T18:03:52.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000124762 2023-09-23T18:03:52.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840214+0000", 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014806500000000001 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.845845+0000", 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0056306910000000002 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.856676+0000", 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010831341 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.856687+0000", 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0695e-05 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956153+0000", 2023-09-23T18:03:52.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0994657109999999 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956212+0000", 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8913000000000003e-05 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.715 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154236 5.2 5:46f3ddda:::benchmark_data_smithi055_229117_object48610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.835841+0000", 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 281.84308129599998, 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1205958920000001, 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154236 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.716 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835841+0000", 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835841+0000", 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835840+0000", 2023-09-23T18:03:52.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:52.718 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.718 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.718 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:52.718 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835843+0000", 2023-09-23T18:03:52.718 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.638e-06 2023-09-23T18:03:52.718 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.718 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.718 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:52.718 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835849+0000", 2023-09-23T18:03:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9150000000000001e-06 2023-09-23T18:03:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835854+0000", 2023-09-23T18:03:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5650000000000001e-06 2023-09-23T18:03:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.842866+0000", 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0070116919999999999 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.842987+0000", 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120715 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.843124+0000", 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137155 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.846319+0000", 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031950939999999999 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.858461+0000", 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012142101000000001 2023-09-23T18:03:52.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:03:52.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.858472+0000", 2023-09-23T18:03:53.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0736e-05 2023-09-23T18:03:53.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956386+0000", 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.097913626 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956437+0000", 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1063000000000002e-05 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:53.222 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154274 5.5 5:aa5b5b0c:::benchmark_data_smithi055_229117_object48648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.105657+0000", 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 279.57326459400002, 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.20535718, 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:53.223 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154274 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105657+0000", 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105657+0000", 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:53.224 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:53.225 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105656+0000", 2023-09-23T18:03:53.225 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:53.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:53.225 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105663+0000", 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.742e-06 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105675+0000", 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1868000000000001e-05 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105683+0000", 2023-09-23T18:03:53.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9410000000000007e-06 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298209+0000", 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.19252627899999999 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298292+0000", 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3452e-05 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:03:53.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298686+0000", 2023-09-23T18:03:53.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00039359900000000001 2023-09-23T18:03:53.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306095+0000", 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0074089589999999997 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310633+0000", 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045383070000000001 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310919+0000", 2023-09-23T18:03:53.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000285803 2023-09-23T18:03:53.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:53.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310969+0000", 2023-09-23T18:03:53.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0439999999999998e-05 2023-09-23T18:03:53.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:53.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311015+0000", 2023-09-23T18:03:53.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5250999999999998e-05 2023-09-23T18:03:53.444 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:53.444 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:53.444 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:53.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.444 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154280 5.5 5:a78a277e:::benchmark_data_smithi055_229117_object48654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:03:53.444 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.166102+0000", 2023-09-23T18:03:53.444 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 279.51281971600002, 2023-09-23T18:03:53.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14501488900000001, 2023-09-23T18:03:53.733 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:53.733 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:53.733 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:53.733 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:53.733 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:53.733 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154280 2023-09-23T18:03:53.733 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.733 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166102+0000", 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166102+0000", 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.735 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:53.735 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166101+0000", 2023-09-23T18:03:53.735 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:53.735 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.735 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.735 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:53.735 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166109+0000", 2023-09-23T18:03:53.735 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5660000000000001e-06 2023-09-23T18:03:53.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:53.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166122+0000", 2023-09-23T18:03:53.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3269e-05 2023-09-23T18:03:53.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:53.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166128+0000", 2023-09-23T18:03:53.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5640000000000002e-06 2023-09-23T18:03:53.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:53.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303566+0000", 2023-09-23T18:03:53.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13743744199999999 2023-09-23T18:03:53.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:53.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303721+0000", 2023-09-23T18:03:53.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000155585 2023-09-23T18:03:53.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303856+0000", 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000134177 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310596+0000", 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0067407539999999998 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:53.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310889+0000", 2023-09-23T18:03:53.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00029301300000000001 2023-09-23T18:03:53.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:53.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311049+0000", 2023-09-23T18:03:53.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016005699999999999 2023-09-23T18:03:53.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:53.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311086+0000", 2023-09-23T18:03:53.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6881999999999997e-05 2023-09-23T18:03:53.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:53.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311117+0000", 2023-09-23T18:03:53.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0936000000000003e-05 2023-09-23T18:03:53.958 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:53.958 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154282 5.5 5:a1cbca1c:::benchmark_data_smithi055_229117_object48656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.171172+0000", 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 279.507749757, 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14593202499999999, 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:53.959 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:53.960 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:53.960 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154282 2023-09-23T18:03:53.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.960 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:53.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:53.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:53.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171172+0000", 2023-09-23T18:03:53.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:53.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:53.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:54.209 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:54.209 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171172+0000", 2023-09-23T18:03:54.209 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:54.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:54.209 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:54.209 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:54.209 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171170+0000", 2023-09-23T18:03:54.209 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:03:54.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171181+0000", 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.064e-05 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171212+0000", 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1433999999999998e-05 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:54.210 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:55.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171223+0000", 2023-09-23T18:03:55.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0265000000000001e-05 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306130+0000", 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13490687900000001 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306257+0000", 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012784 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306375+0000", 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011726999999999999 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310640+0000", 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00426495 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:55.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.314542+0000", 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039025510000000002 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317023+0000", 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024812549999999999 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317068+0000", 2023-09-23T18:03:55.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4632999999999998e-05 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317104+0000", 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6188e-05 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.359 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154290 5.5 5:a8c62379:::benchmark_data_smithi055_229117_object48664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.196937+0000", 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 279.48198479899997, 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.12027204499999999, 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154290 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:55.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196937+0000", 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196937+0000", 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:55.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196936+0000", 2023-09-23T18:03:55.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:55.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:55.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196943+0000", 2023-09-23T18:03:55.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.306e-06 2023-09-23T18:03:55.582 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.582 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.582 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:55.582 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196960+0000", 2023-09-23T18:03:55.582 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7173e-05 2023-09-23T18:03:55.582 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196969+0000", 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7390000000000005e-06 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308177+0000", 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11120793499999999 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.583 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308311+0000", 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013440899999999999 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308441+0000", 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000129677 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:55.584 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.312252+0000", 2023-09-23T18:03:55.585 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038113700000000001 2023-09-23T18:03:55.585 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.585 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.585 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:55.585 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.314577+0000", 2023-09-23T18:03:55.585 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023246730000000002 2023-09-23T18:03:55.585 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.585 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317137+0000", 2023-09-23T18:03:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025597609999999998 2023-09-23T18:03:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317173+0000", 2023-09-23T18:03:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6371000000000001e-05 2023-09-23T18:03:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317209+0000", 2023-09-23T18:03:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6191e-05 2023-09-23T18:03:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161327 5.e 5:7b0ada0d:::benchmark_data_smithi055_229117_object55701:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:03:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852567+0000", 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 267.82635496, 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0633735559999999, 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161327 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852567+0000", 2023-09-23T18:03:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852567+0000", 2023-09-23T18:03:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852565+0000", 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852572+0000", 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3729999999999998e-06 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852585+0000", 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3426e-05 2023-09-23T18:03:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852592+0000", 2023-09-23T18:03:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2830000000000004e-06 2023-09-23T18:03:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852658+0000", 2023-09-23T18:03:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5399000000000002e-05 2023-09-23T18:03:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op must wait for map", 2023-09-23T18:03:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852668+0000", 2023-09-23T18:03:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7110000000000007e-06 2023-09-23T18:03:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872062+0000", 2023-09-23T18:03:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019394650999999999 2023-09-23T18:03:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:03:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872073+0000", 2023-09-23T18:03:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0607e-05 2023-09-23T18:03:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.885379+0000", 2023-09-23T18:03:56.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0133066719999999 2023-09-23T18:03:56.176 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:56.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.887726+0000", 2023-09-23T18:03:56.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023469929999999999 2023-09-23T18:03:56.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:03:56.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.887957+0000", 2023-09-23T18:03:56.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00023082800000000001 2023-09-23T18:03:56.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915266+0000", 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027308517000000001 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915340+0000", 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4178999999999996e-05 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:56.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915403+0000", 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2842000000000006e-05 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915811+0000", 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00040861899999999999 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915853+0000", 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1768999999999999e-05 2023-09-23T18:03:56.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915941+0000", 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7410000000000005e-05 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161356 5.e 5:729afca1:::benchmark_data_smithi055_229117_object55730:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:03:56.180 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:03:56.356 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 267.826317039, 2023-09-23T18:03:56.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0635528459999999, 2023-09-23T18:03:56.356 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161356 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:03:56.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:56.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852607+0000", 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9570000000000001e-06 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852613+0000", 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1199999999999999e-06 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852616+0000", 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4319999999999999e-06 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852880+0000", 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026363200000000003 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852889+0000", 2023-09-23T18:03:56.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8529999999999998e-06 2023-09-23T18:03:56.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.512 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:56.512 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872097+0000", 2023-09-23T18:03:56.512 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019208026999999999 2023-09-23T18:03:56.512 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.512 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.512 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:03:56.513 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872104+0000", 2023-09-23T18:03:56.513 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5739999999999998e-06 2023-09-23T18:03:56.513 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.513 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.513 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:56.513 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.905797+0000", 2023-09-23T18:03:56.513 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0336926310000001 2023-09-23T18:03:56.513 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.513 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.513 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:56.514 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.905945+0000", 2023-09-23T18:03:56.514 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000147899 2023-09-23T18:03:56.514 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.514 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.514 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:03:56.514 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.906095+0000", 2023-09-23T18:03:56.514 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015013200000000001 2023-09-23T18:03:56.514 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.514 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.514 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915327+0000", 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092321869999999993 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915351+0000", 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.376e-05 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915452+0000", 2023-09-23T18:03:56.515 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000101004 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915997+0000", 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00054548699999999999 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916057+0000", 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9496000000000001e-05 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.516 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.517 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:56.517 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916158+0000", 2023-09-23T18:03:56.517 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000100995 2023-09-23T18:03:56.517 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:56.517 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:56.517 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:56.517 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.517 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.517 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161361 5.e 5:727196ef:::benchmark_data_smithi055_229117_object55735:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:03:56.630 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852804+0000", 2023-09-23T18:03:56.630 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 267.82611810600002, 2023-09-23T18:03:56.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0635737650000001, 2023-09-23T18:03:56.630 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:56.630 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:56.630 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:56.630 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161361 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852804+0000", 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:56.631 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852804+0000", 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852803+0000", 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852808+0000", 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3579999999999996e-06 2023-09-23T18:03:56.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852820+0000", 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2822e-05 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852828+0000", 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9980000000000003e-06 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111282 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852947+0000", 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3749999999999997e-06 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.634 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:56.635 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872127+0000", 2023-09-23T18:03:56.635 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019180169 2023-09-23T18:03:56.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:03:56.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872134+0000", 2023-09-23T18:03:56.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.776e-06 2023-09-23T18:03:56.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:56.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908641+0000", 2023-09-23T18:03:56.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.036506683 2023-09-23T18:03:56.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:56.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908722+0000", 2023-09-23T18:03:56.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1464999999999993e-05 2023-09-23T18:03:56.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:03:56.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908805+0000", 2023-09-23T18:03:56.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3060999999999996e-05 2023-09-23T18:03:56.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:56.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915361+0000", 2023-09-23T18:03:56.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00655603 2023-09-23T18:03:56.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:56.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915491+0000", 2023-09-23T18:03:56.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012961299999999999 2023-09-23T18:03:56.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:56.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915586+0000", 2023-09-23T18:03:56.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5175999999999998e-05 2023-09-23T18:03:56.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:56.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916217+0000", 2023-09-23T18:03:56.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00063121599999999998 2023-09-23T18:03:56.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:56.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916266+0000", 2023-09-23T18:03:56.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8649000000000001e-05 2023-09-23T18:03:56.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:56.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916378+0000", 2023-09-23T18:03:56.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111749 2023-09-23T18:03:56.905 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:56.905 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161400 5.e 5:7bd12518:::benchmark_data_smithi055_229117_object55774:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852914+0000", 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 267.82600835599999, 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.066386922, 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:56.906 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:56.907 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:56.907 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161400 2023-09-23T18:03:56.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.907 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:56.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:56.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852914+0000", 2023-09-23T18:03:56.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:56.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:56.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:56.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:56.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852914+0000", 2023-09-23T18:03:56.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:57.214 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.214 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.214 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:57.214 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852913+0000", 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852917+0000", 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1459999999999998e-06 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852927+0000", 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4700000000000008e-06 2023-09-23T18:03:57.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.216 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.216 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:57.216 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852934+0000", 2023-09-23T18:03:57.216 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8630000000000002e-06 2023-09-23T18:03:57.216 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852982+0000", 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8585999999999999e-05 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852990+0000", 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3989999999999998e-06 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.220 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:57.220 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872158+0000", 2023-09-23T18:03:57.220 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019168266 2023-09-23T18:03:57.220 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.220 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.313 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:03:57.313 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872163+0000", 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9620000000000001e-06 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.910791+0000", 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0386283650000001 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.910919+0000", 2023-09-23T18:03:57.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012744899999999999 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.911051+0000", 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013257500000000001 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915372+0000", 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004320618 2023-09-23T18:03:57.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915538+0000", 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016641900000000001 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915724+0000", 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018572200000000001 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:57.316 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919091+0000", 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033675419999999998 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919170+0000", 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8464e-05 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919301+0000", 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013067499999999999 2023-09-23T18:03:57.317 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:58.104 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:58.104 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.104 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.104 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161452 5.e 5:7e9dcc4b:::benchmark_data_smithi055_229117_object55826:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:03:58.104 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:03:58.104 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 267.82598215799999, 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0665552599999999, 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161452 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.105 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:58.106 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:03:58.106 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:58.106 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.106 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.106 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:58.106 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:03:58.106 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:58.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852939+0000", 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852943+0000", 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1070000000000001e-06 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:58.235 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852949+0000", 2023-09-23T18:03:58.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7900000000000002e-06 2023-09-23T18:03:58.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.235 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.235 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:58.235 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852955+0000", 2023-09-23T18:03:58.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7760000000000003e-06 2023-09-23T18:03:58.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.235 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.853019+0000", 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3619000000000002e-05 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.853023+0000", 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1899999999999997e-06 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:58.236 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872216+0000", 2023-09-23T18:03:58.237 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019192753 2023-09-23T18:03:58.237 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.237 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.237 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:03:58.237 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872224+0000", 2023-09-23T18:03:58.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2779999999999993e-06 2023-09-23T18:03:58.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:58.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913001+0000", 2023-09-23T18:03:58.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.040776715 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913126+0000", 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012584599999999999 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913244+0000", 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000117377 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915463+0000", 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022194480000000002 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915650+0000", 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018667299999999999 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:58.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915759+0000", 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109165 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919339+0000", 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035801880000000002 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:58.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919386+0000", 2023-09-23T18:03:58.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6295999999999999e-05 2023-09-23T18:03:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919495+0000", 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109508 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206353 5.5 5:a66ab0d3:::benchmark_data_smithi055_229117_object100727:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:03:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.895738+0000", 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 194.783184409, 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.116710411, 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206353 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895738+0000", 2023-09-23T18:03:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895738+0000", 2023-09-23T18:03:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895736+0000", 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895741+0000", 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6890000000000003e-06 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895751+0000", 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0377000000000001e-05 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895759+0000", 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2420000000000003e-06 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895825+0000", 2023-09-23T18:03:59.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5957999999999994e-05 2023-09-23T18:03:59.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.701 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.701 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:03:59.701 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895953+0000", 2023-09-23T18:03:59.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012840799999999999 2023-09-23T18:03:59.701 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.896065+0000", 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011191700000000001 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901012+0000", 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004946939 2023-09-23T18:03:59.702 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.703 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.703 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:59.703 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901377+0000", 2023-09-23T18:03:59.703 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00036531200000000002 2023-09-23T18:03:59.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.704 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:03:59.704 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012352+0000", 2023-09-23T18:03:59.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11097481100000001 2023-09-23T18:03:59.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:03:59.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012398+0000", 2023-09-23T18:03:59.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5779000000000003e-05 2023-09-23T18:03:59.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012448+0000", 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0294000000000003e-05 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206358 5.2 5:46c0880b:::benchmark_data_smithi055_229117_object100732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:03:59.917 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.895804+0000", 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 194.783118148, 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.119344292, 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206358 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.918 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895804+0000", 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895804+0000", 2023-09-23T18:03:59.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:03:59.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:03:59.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:03:59.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:03:59.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895803+0000", 2023-09-23T18:04:00.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:04:00.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:00.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895807+0000", 2023-09-23T18:04:00.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4759999999999998e-06 2023-09-23T18:04:00.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:00.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895814+0000", 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.781e-06 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895819+0000", 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5269999999999998e-06 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:04:00.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.899325+0000", 2023-09-23T18:04:00.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003505883 2023-09-23T18:04:00.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:04:00.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.899449+0000", 2023-09-23T18:04:00.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000123726 2023-09-23T18:04:00.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:04:00.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.899575+0000", 2023-09-23T18:04:00.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000126308 2023-09-23T18:04:00.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:00.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.904891+0000", 2023-09-23T18:04:00.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053159310000000003 2023-09-23T18:04:00.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:04:00.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.904948+0000", 2023-09-23T18:04:00.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6798000000000001e-05 2023-09-23T18:04:00.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.014957+0000", 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.110008884 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.014989+0000", 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2298999999999997e-05 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:04:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015038+0000", 2023-09-23T18:04:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8914e-05 2023-09-23T18:04:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:04:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015148+0000", 2023-09-23T18:04:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000110263 2023-09-23T18:04:00.400 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206362 5.2 5:4058eb16:::benchmark_data_smithi055_229117_object100736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:04:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:04:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 194.78309068300001, 2023-09-23T18:04:00.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.119556432, 2023-09-23T18:04:00.575 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:04:00.575 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:04:00.575 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:04:00.575 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206362 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.576 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:04:00.577 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:04:00.577 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:00.577 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.577 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.577 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:04:00.577 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:04:00.577 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:04:00.578 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.578 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.579 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:00.579 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895834+0000", 2023-09-23T18:04:00.579 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.943e-06 2023-09-23T18:04:00.579 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.579 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.579 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:00.794 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895840+0000", 2023-09-23T18:04:00.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4489999999999996e-06 2023-09-23T18:04:00.794 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.794 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:00.794 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895846+0000", 2023-09-23T18:04:00.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4999999999999999e-06 2023-09-23T18:04:00.794 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.794 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.902010+0000", 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0061640469999999998 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.902147+0000", 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137324 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:04:00.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.902297+0000", 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014995699999999999 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.904939+0000", 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026419070000000002 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015191+0000", 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11025235799999999 2023-09-23T18:04:00.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015223+0000", 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1665000000000003e-05 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015245+0000", 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2456e-05 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:04:00.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015286+0000", 2023-09-23T18:04:00.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1009e-05 2023-09-23T18:04:00.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:00.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:00.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:04:00.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015388+0000", 2023-09-23T18:04:00.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000101327 2023-09-23T18:04:01.026 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:01.026 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:01.026 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:01.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.026 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206363 5.5 5:a478e7fe:::benchmark_data_smithi055_229117_object100737:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:04:01.026 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.895850+0000", 2023-09-23T18:04:01.026 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 194.78307165999999, 2023-09-23T18:04:01.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11711835399999999, 2023-09-23T18:04:01.026 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:04:01.027 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:04:01.027 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:04:01.027 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:04:01.027 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:04:01.027 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206363 2023-09-23T18:04:01.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.027 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:04:01.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:04:01.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895850+0000", 2023-09-23T18:04:01.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:01.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:04:01.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895850+0000", 2023-09-23T18:04:01.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:01.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:04:01.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895850+0000", 2023-09-23T18:04:01.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:04:01.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:01.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895852+0000", 2023-09-23T18:04:01.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4150000000000002e-06 2023-09-23T18:04:01.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:01.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895858+0000", 2023-09-23T18:04:01.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4709999999999998e-06 2023-09-23T18:04:01.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.241 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:01.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895863+0000", 2023-09-23T18:04:01.242 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6829999999999999e-06 2023-09-23T18:04:01.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:04:01.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.898438+0000", 2023-09-23T18:04:01.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025751039999999999 2023-09-23T18:04:01.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:04:01.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.898553+0000", 2023-09-23T18:04:01.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000115454 2023-09-23T18:04:01.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:04:01.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.898668+0000", 2023-09-23T18:04:01.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011471 2023-09-23T18:04:01.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:01.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901405+0000", 2023-09-23T18:04:01.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027372210000000002 2023-09-23T18:04:01.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:04:01.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901528+0000", 2023-09-23T18:04:01.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012334999999999999 2023-09-23T18:04:01.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:01.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012883+0000", 2023-09-23T18:04:01.246 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11135468900000001 2023-09-23T18:04:01.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:04:01.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012931+0000", 2023-09-23T18:04:01.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7821000000000002e-05 2023-09-23T18:04:01.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.481 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:04:01.481 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012969+0000", 2023-09-23T18:04:01.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7863e-05 2023-09-23T18:04:01.481 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:01.481 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:01.481 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:01.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.481 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:206361 5.d e194/176 5:bb6e9a5e:::benchmark_data_smithi055_229117_object100735:head v 194'12834, mlcod=118'3050)", 2023-09-23T18:04:01.482 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.897260+0000", 2023-09-23T18:04:01.482 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 194.78166171399999, 2023-09-23T18:04:01.482 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11523296600000001, 2023-09-23T18:04:01.482 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:04:01.482 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:04:01.482 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:04:01.482 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.483 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:04:01.483 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897260+0000", 2023-09-23T18:04:01.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:01.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.483 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.483 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:04:01.483 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897260+0000", 2023-09-23T18:04:01.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:01.484 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.484 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.484 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:04:01.484 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897259+0000", 2023-09-23T18:04:01.484 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:04:01.484 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.484 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.484 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:01.484 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897267+0000", 2023-09-23T18:04:01.484 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9189999999999999e-06 2023-09-23T18:04:01.485 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.485 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.485 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:01.485 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897278+0000", 2023-09-23T18:04:01.485 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0977e-05 2023-09-23T18:04:01.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.705 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:01.705 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.897286+0000", 2023-09-23T18:04:01.705 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6830000000000008e-06 2023-09-23T18:04:01.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:04:01.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.898147+0000", 2023-09-23T18:04:01.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00086085600000000001 2023-09-23T18:04:01.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:04:01.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.898167+0000", 2023-09-23T18:04:01.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0169999999999998e-05 2023-09-23T18:04:01.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:04:01.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012463+0000", 2023-09-23T18:04:01.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.114296261 2023-09-23T18:04:01.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:04:01.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012493+0000", 2023-09-23T18:04:01.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0060999999999999e-05 2023-09-23T18:04:01.707 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:01.708 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:01.708 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:01.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.708 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.44681.0:206365 5.e e194/187 5:75ab0aea:::benchmark_data_smithi055_229117_object100739:head v 194'12891, mlcod=194'12889)", 2023-09-23T18:04:01.708 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.901561+0000", 2023-09-23T18:04:01.708 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 194.77736095399999, 2023-09-23T18:04:01.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11070677700000001, 2023-09-23T18:04:01.709 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:04:01.709 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:04:01.709 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:04:01.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:04:01.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901561+0000", 2023-09-23T18:04:01.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:01.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:04:01.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901561+0000", 2023-09-23T18:04:01.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:01.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:04:01.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901560+0000", 2023-09-23T18:04:01.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:04:01.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:01.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901567+0000", 2023-09-23T18:04:01.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4710000000000004e-06 2023-09-23T18:04:01.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:01.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901577+0000", 2023-09-23T18:04:01.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8879999999999992e-06 2023-09-23T18:04:01.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:01.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901583+0000", 2023-09-23T18:04:01.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3559999999999998e-06 2023-09-23T18:04:01.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:04:01.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901622+0000", 2023-09-23T18:04:01.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9186000000000001e-05 2023-09-23T18:04:01.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:01.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:01.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:04:01.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.901646+0000", 2023-09-23T18:04:01.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3947e-05 2023-09-23T18:04:01.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:04:02.290 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905646+0000", 2023-09-23T18:04:02.290 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040000670000000004 2023-09-23T18:04:02.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:04:02.290 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012268+0000", 2023-09-23T18:04:02.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10662147900000001 2023-09-23T18:04:02.291 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:02.291 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:02.291 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:02.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.292 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206368 5.2 5:4617325c:::benchmark_data_smithi055_229117_object100742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:04:02.292 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.905040+0000", 2023-09-23T18:04:02.292 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 194.77388195500001, 2023-09-23T18:04:02.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.117245135, 2023-09-23T18:04:02.292 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:04:02.292 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:04:02.292 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:04:02.292 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:04:02.293 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:04:02.293 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206368 2023-09-23T18:04:02.293 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.293 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:04:02.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:04:02.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905040+0000", 2023-09-23T18:04:02.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:02.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:04:02.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905040+0000", 2023-09-23T18:04:02.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:02.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:04:02.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905038+0000", 2023-09-23T18:04:02.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:04:02.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:02.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905046+0000", 2023-09-23T18:04:02.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2590000000000003e-06 2023-09-23T18:04:02.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:02.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905057+0000", 2023-09-23T18:04:02.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1406999999999999e-05 2023-09-23T18:04:02.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:02.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905065+0000", 2023-09-23T18:04:02.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9300000000000003e-06 2023-09-23T18:04:02.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:04:02.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905107+0000", 2023-09-23T18:04:02.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1492000000000002e-05 2023-09-23T18:04:02.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:04:02.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905267+0000", 2023-09-23T18:04:02.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016003600000000001 2023-09-23T18:04:02.606 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012294+0000", 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10702790199999999 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.016113+0000", 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038184 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:02.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.016149+0000", 2023-09-23T18:04:02.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6520999999999998e-05 2023-09-23T18:04:02.608 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.609 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.609 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:04:02.609 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.016543+0000", 2023-09-23T18:04:02.609 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00039407299999999999 2023-09-23T18:04:02.609 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.609 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.609 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:04:02.609 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.022111+0000", 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055670880000000004 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.022163+0000", 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2528999999999998e-05 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.022285+0000", 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012215699999999999 2023-09-23T18:04:02.610 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:03.881 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:03.882 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:03.882 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:03.882 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:03.882 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:04:03.882 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:04:03.882 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:04:03.885 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:04:04.085 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:04.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:52 smithi099 ceph-mon[167076]: pgmap v617: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 223 op/s; 29440/135975 objects degraded (21.651%); 10368/135975 objects misplaced (7.625%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T18:04:04.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:04.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:04.086 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2550220444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:04.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 29440/135975 objects degraded (21.651%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:04.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:54 smithi099 ceph-mon[167076]: pgmap v618: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 29260/134712 objects degraded (21.720%); 10282/134712 objects misplaced (7.633%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:04:04.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2179488450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:04:04.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:55 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:04:04.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:55 smithi099 ceph-mon[167076]: failed to encode map e262 with expected crc 2023-09-23T18:04:04.322 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2179488450' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:04:04.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:55 smithi099 ceph-mon[167076]: osdmap e262: 8 total, 8 up, 7 in 2023-09-23T18:04:04.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:55 smithi099 ceph-mon[167076]: failed to encode map e262 with expected crc 2023-09-23T18:04:04.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:56 smithi099 ceph-mon[167076]: pgmap v620: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 28575/132453 objects degraded (21.574%); 10203/132453 objects misplaced (7.703%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T18:04:04.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:58 smithi099 ceph-mon[167076]: pgmap v621: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 248 op/s; 28575/131883 objects degraded (21.667%); 10017/131883 objects misplaced (7.595%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T18:04:04.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:03:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 28575/131883 objects degraded (21.667%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:04.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:00 smithi099 ceph-mon[167076]: pgmap v622: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 27272/129237 objects degraded (21.102%); 9832/129237 objects misplaced (7.608%); 7.9 MiB/s, 194 objects/s recovering 2023-09-23T18:04:04.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:02 smithi099 ceph-mon[167076]: pgmap v623: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 271 op/s; 27272/127824 objects degraded (21.336%); 9648/127824 objects misplaced (7.548%); 7.9 MiB/s, 194 objects/s recovering 2023-09-23T18:04:04.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:02 smithi099 ceph-mon[167076]: failed to encode map e263 with expected crc 2023-09-23T18:04:04.323 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:02 smithi099 ceph-mon[167076]: osdmap e263: 8 total, 8 up, 7 in 2023-09-23T18:04:04.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:02 smithi099 ceph-mon[167076]: failed to encode map e263 with expected crc 2023-09-23T18:04:04.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:02 smithi099 ceph-mon[167076]: failed to encode map e263 with expected crc 2023-09-23T18:04:04.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:02 smithi099 ceph-mon[167076]: failed to encode map e263 with expected crc 2023-09-23T18:04:04.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:03 smithi099 ceph-mon[167076]: failed to encode map e264 with expected crc 2023-09-23T18:04:04.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:03 smithi099 ceph-mon[167076]: osdmap e264: 8 total, 8 up, 7 in 2023-09-23T18:04:04.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:03 smithi099 ceph-mon[167076]: failed to encode map e264 with expected crc 2023-09-23T18:04:04.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:03 smithi099 ceph-mon[167076]: failed to encode map e264 with expected crc 2023-09-23T18:04:04.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:03 smithi099 ceph-mon[167076]: failed to encode map e264 with expected crc 2023-09-23T18:04:04.324 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 27272/127824 objects degraded (21.336%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:04.325 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:04:04.325 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:04:04.325 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:04:04.325 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:04:04.326 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:04:04.326 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:04:04.326 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:04:04.326 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:04:04.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:04:04.507 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:04:04.507 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:04:04.507 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:04:04.507 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:04:04.507 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:04:04.507 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:04:04.507 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:04:04.507 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:04:04.508 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:52 smithi055 ceph-mon[179668]: pgmap v617: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 223 op/s; 29440/135975 objects degraded (21.651%); 10368/135975 objects misplaced (7.625%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T18:04:04.508 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:04.508 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:04.508 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2550220444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:04.508 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:52 smithi055 ceph-mon[184174]: pgmap v617: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.8 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 223 op/s; 29440/135975 objects degraded (21.651%); 10368/135975 objects misplaced (7.625%); 7.3 MiB/s, 117 objects/s recovering 2023-09-23T18:04:04.508 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:04.509 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:04.509 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2550220444' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:04.510 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 29440/135975 objects degraded (21.651%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:04.510 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 29440/135975 objects degraded (21.651%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:04.702 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:04:04.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:54 smithi055 ceph-mon[184174]: pgmap v618: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 29260/134712 objects degraded (21.720%); 10282/134712 objects misplaced (7.633%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:04:04.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2179488450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:04:04.703 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:54 smithi055 ceph-mon[179668]: pgmap v618: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.9 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 222 op/s; 29260/134712 objects degraded (21.720%); 10282/134712 objects misplaced (7.633%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:04:04.703 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2179488450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:04:04.704 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:55 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:04:04.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:55 smithi055 ceph-mon[184174]: failed to encode map e262 with expected crc 2023-09-23T18:04:04.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2179488450' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:04:04.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:55 smithi055 ceph-mon[184174]: osdmap e262: 8 total, 8 up, 7 in 2023-09-23T18:04:06.009 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:55 smithi055 ceph-mon[184174]: failed to encode map e262 with expected crc 2023-09-23T18:04:06.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:55 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:04:06.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:55 smithi055 ceph-mon[179668]: failed to encode map e262 with expected crc 2023-09-23T18:04:06.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2179488450' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:04:06.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:55 smithi055 ceph-mon[179668]: osdmap e262: 8 total, 8 up, 7 in 2023-09-23T18:04:06.010 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:55 smithi055 ceph-mon[179668]: failed to encode map e262 with expected crc 2023-09-23T18:04:06.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:56 smithi055 ceph-mon[179668]: pgmap v620: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 28575/132453 objects degraded (21.574%); 10203/132453 objects misplaced (7.703%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T18:04:06.011 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:04 smithi099 ceph-mon[167076]: pgmap v626: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 24824/126765 objects degraded (19.583%); 11692/126765 objects misplaced (9.223%); 5.3 MiB/s, 170 objects/s recovering 2023-09-23T18:04:06.012 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:56 smithi055 ceph-mon[184174]: pgmap v620: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 240 op/s; 28575/132453 objects degraded (21.574%); 10203/132453 objects misplaced (7.703%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T18:04:06.012 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:58 smithi055 ceph-mon[184174]: pgmap v621: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 248 op/s; 28575/131883 objects degraded (21.667%); 10017/131883 objects misplaced (7.595%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T18:04:06.012 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:03:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 28575/131883 objects degraded (21.667%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:06.012 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:58 smithi055 ceph-mon[179668]: pgmap v621: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.7 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 248 op/s; 28575/131883 objects degraded (21.667%); 10017/131883 objects misplaced (7.595%); 9.0 MiB/s, 143 objects/s recovering 2023-09-23T18:04:06.012 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:03:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 28575/131883 objects degraded (21.667%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:06.013 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:04:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:04:00] "GET /metrics HTTP/1.1" 200 36541 "" "Prometheus/2.43.0" 2023-09-23T18:04:06.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:06.013 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:00 smithi055 ceph-mon[179668]: pgmap v622: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 27272/129237 objects degraded (21.102%); 9832/129237 objects misplaced (7.608%); 7.9 MiB/s, 194 objects/s recovering 2023-09-23T18:04:06.014 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:07.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:04:07.806 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:00 smithi055 ceph-mon[184174]: pgmap v622: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.6 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 263 op/s; 27272/129237 objects degraded (21.102%); 9832/129237 objects misplaced (7.608%); 7.9 MiB/s, 194 objects/s recovering 2023-09-23T18:04:07.806 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:04:07.807 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:04:07.807 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:04:07.808 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[179668]: pgmap v623: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 271 op/s; 27272/127824 objects degraded (21.336%); 9648/127824 objects misplaced (7.548%); 7.9 MiB/s, 194 objects/s recovering 2023-09-23T18:04:07.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[179668]: failed to encode map e263 with expected crc 2023-09-23T18:04:07.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[179668]: osdmap e263: 8 total, 8 up, 7 in 2023-09-23T18:04:07.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[179668]: failed to encode map e263 with expected crc 2023-09-23T18:04:07.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[179668]: failed to encode map e263 with expected crc 2023-09-23T18:04:07.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[179668]: failed to encode map e263 with expected crc 2023-09-23T18:04:07.992 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:06 smithi099 ceph-mon[167076]: pgmap v627: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 23969/124701 objects degraded (19.221%); 11610/124701 objects misplaced (9.310%); 6.0 MiB/s, 287 objects/s recovering 2023-09-23T18:04:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[184174]: pgmap v623: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 6 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.5 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 271 op/s; 27272/127824 objects degraded (21.336%); 9648/127824 objects misplaced (7.548%); 7.9 MiB/s, 194 objects/s recovering 2023-09-23T18:04:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[184174]: failed to encode map e263 with expected crc 2023-09-23T18:04:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[184174]: osdmap e263: 8 total, 8 up, 7 in 2023-09-23T18:04:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[184174]: failed to encode map e263 with expected crc 2023-09-23T18:04:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[184174]: failed to encode map e263 with expected crc 2023-09-23T18:04:08.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:02 smithi055 ceph-mon[184174]: failed to encode map e263 with expected crc 2023-09-23T18:04:08.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[179668]: failed to encode map e264 with expected crc 2023-09-23T18:04:08.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[179668]: osdmap e264: 8 total, 8 up, 7 in 2023-09-23T18:04:08.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[179668]: failed to encode map e264 with expected crc 2023-09-23T18:04:08.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[179668]: failed to encode map e264 with expected crc 2023-09-23T18:04:08.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[179668]: failed to encode map e264 with expected crc 2023-09-23T18:04:08.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 27272/127824 objects degraded (21.336%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:08.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[184174]: failed to encode map e264 with expected crc 2023-09-23T18:04:08.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[184174]: osdmap e264: 8 total, 8 up, 7 in 2023-09-23T18:04:08.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[184174]: failed to encode map e264 with expected crc 2023-09-23T18:04:08.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[184174]: failed to encode map e264 with expected crc 2023-09-23T18:04:08.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[184174]: failed to encode map e264 with expected crc 2023-09-23T18:04:08.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 27272/127824 objects degraded (21.336%), 10 pgs degraded, 10 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:08.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:04 smithi055 ceph-mon[179668]: pgmap v626: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 24824/126765 objects degraded (19.583%); 11692/126765 objects misplaced (9.223%); 5.3 MiB/s, 170 objects/s recovering 2023-09-23T18:04:08.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:04 smithi055 ceph-mon[184174]: pgmap v626: 119 pgs: 1 active+recovery_wait+degraded+remapped, 108 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped; 2.6 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 236 op/s; 24824/126765 objects degraded (19.583%); 11692/126765 objects misplaced (9.223%); 5.3 MiB/s, 170 objects/s recovering 2023-09-23T18:04:08.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:06 smithi055 ceph-mon[179668]: pgmap v627: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 23969/124701 objects degraded (19.221%); 11610/124701 objects misplaced (9.310%); 6.0 MiB/s, 287 objects/s recovering 2023-09-23T18:04:08.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:06 smithi055 ceph-mon[184174]: pgmap v627: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.3 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 299 op/s; 23969/124701 objects degraded (19.221%); 11610/124701 objects misplaced (9.310%); 6.0 MiB/s, 287 objects/s recovering 2023-09-23T18:04:08.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:08.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:08.356 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:07 smithi055 ceph-mon[179668]: pgmap v628: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 23969/124188 objects degraded (19.301%); 11438/124188 objects misplaced (9.210%); 2.1 MiB/s, 140 objects/s recovering 2023-09-23T18:04:08.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:08.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:08.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:07 smithi055 ceph-mon[184174]: pgmap v628: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 23969/124188 objects degraded (19.301%); 11438/124188 objects misplaced (9.210%); 2.1 MiB/s, 140 objects/s recovering 2023-09-23T18:04:08.358 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:04:08.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:08.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:08.359 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:07 smithi099 ceph-mon[167076]: pgmap v628: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.2 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 210 op/s; 23969/124188 objects degraded (19.301%); 11438/124188 objects misplaced (9.210%); 2.1 MiB/s, 140 objects/s recovering 2023-09-23T18:04:08.475 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 23969/124188 objects degraded (19.301%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:08.531 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 23969/124188 objects degraded (19.301%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:08.531 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:08.546 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:04:08.747 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:10.031 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:04:10.032 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:10.032 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:10.033 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:10.034 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:10.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:09 smithi055 ceph-mon[179668]: pgmap v629: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 23784/121440 objects degraded (19.585%); 10870/121440 objects misplaced (8.951%); 5.8 MiB/s, 198 objects/s recovering 2023-09-23T18:04:10.194 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:09 smithi055 ceph-mon[184174]: pgmap v629: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 23784/121440 objects degraded (19.585%); 10870/121440 objects misplaced (8.951%); 5.8 MiB/s, 198 objects/s recovering 2023-09-23T18:04:10.194 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:04:10.195 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 23969/124188 objects degraded (19.301%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:10.195 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:09 smithi099 ceph-mon[167076]: pgmap v629: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.5 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 265 op/s; 23784/121440 objects degraded (19.585%); 10870/121440 objects misplaced (8.951%); 5.8 MiB/s, 198 objects/s recovering 2023-09-23T18:04:10.263 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:10.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:04:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:04:10] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-09-23T18:04:10.964 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:04:11.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:11.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:04:11.307 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:04:11.781 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:14.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:14.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:11 smithi099 ceph-mon[167076]: pgmap v630: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 286 op/s; 23784/119586 objects degraded (19.889%); 10622/119586 objects misplaced (8.882%); 4.8 MiB/s, 166 objects/s recovering 2023-09-23T18:04:14.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:12 smithi099 ceph-mon[167076]: from='client.45434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:04:14.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:13 smithi099 ceph-mon[167076]: pgmap v631: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 343 op/s; 23433/116946 objects degraded (20.037%); 10169/116946 objects misplaced (8.695%); 6.0 MiB/s, 176 objects/s recovering 2023-09-23T18:04:14.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 23433/116946 objects degraded (20.037%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:14.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:13 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/796332847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:14.521 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:04:14.521 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:04:14.521 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:11 smithi055 ceph-mon[179668]: pgmap v630: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 286 op/s; 23784/119586 objects degraded (19.889%); 10622/119586 objects misplaced (8.882%); 4.8 MiB/s, 166 objects/s recovering 2023-09-23T18:04:14.522 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:11 smithi055 ceph-mon[184174]: pgmap v630: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 9.0 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 286 op/s; 23784/119586 objects degraded (19.889%); 10622/119586 objects misplaced (8.882%); 4.8 MiB/s, 166 objects/s recovering 2023-09-23T18:04:14.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:04:15.151 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1759628881' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:04:15.153 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:04:15.155 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:04:15.477 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:04:15.477 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:04:15.478 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":630,"stamp":"2023-09-23T18:04:10.698782+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2615163398,"num_objects":39862,"num_object_clones":0,"num_object_copies":119586,"num_objects_missing_on_primary":25,"num_objects_missing":25,"num_objects_degraded":23784,"num_objects_misplaced":10622,"num_objects_unfound":0,"num_objects_dirty":39862,"num_whiteouts":0,"num_read":6284,"num_read_kb":12770,"num_write":107506,"num_write_kb":4695012,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23159,"num_bytes_recovered":1393363943,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93295,"ondisk_log_size":93295,"up":357,"acting":349,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":404,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9417332,"kb_used_data":7896956,"kb_used_omap":43,"kb_used_meta":1520276,"kb_avail":646798732,"statfs":{"total":671965249536,"available":662321901568,"internally_reserved":0,"allocated":8086482944,"data_stored":8078266782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1556763343},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-179961856,"num_objects":-2746,"num_object_clones":0,"num_object_copies":-8238,"num_objects_missing_on_primary":-170,"num_objects_missing":-170,"num_objects_degraded":-3488,"num_objects_misplaced":974,"num_objects_unfound":0,"num_objects_dirty":-2746,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2746,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1592,"num_bytes_recovered":48627712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.577125"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":589,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136832+0000","last_change":"2023-09-23T18:03:33.004903+0000","last_active":"2023-09-23T18:04:01.136832+0000","last_peered":"2023-09-23T18:04:01.136832+0000","last_clean":"2023-09-23T18:04:01.136832+0000","last_became_active":"2023-09-23T18:03:33.004498+0000","last_became_peered":"2023-09-23T18:03:33.004498+0000","last_unstale":"2023-09-23T18:04:01.136832+0000","last_undegraded":"2023-09-23T18:04:01.136832+0000","last_fullsized":"2023-09-23T18:04:01.136832+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":737225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":64,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136521+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:04:01.136521+0000","last_peered":"2023-09-23T18:04:01.136521+0000","last_clean":"2023-09-23T18:04:01.136521+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:04:01.136521+0000","last_undegraded":"2023-09-23T18:04:01.136521+0000","last_fullsized":"2023-09-23T18:04:01.136521+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":145,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137504+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:04:01.137504+0000","last_peered":"2023-09-23T18:04:01.137504+0000","last_clean":"2023-09-23T18:04:01.137504+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:04:01.137504+0000","last_undegraded":"2023-09-23T18:04:01.137504+0000","last_fullsized":"2023-09-23T18:04:01.137504+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":606,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137746+0000","last_change":"2023-09-23T18:02:51.046916+0000","last_active":"2023-09-23T18:04:01.137746+0000","last_peered":"2023-09-23T18:04:01.137746+0000","last_clean":"2023-09-23T18:04:01.137746+0000","last_became_active":"2023-09-23T18:02:51.046591+0000","last_became_peered":"2023-09-23T18:02:51.046591+0000","last_unstale":"2023-09-23T18:04:01.137746+0000","last_undegraded":"2023-09-23T18:04:01.137746+0000","last_fullsized":"2023-09-23T18:04:01.137746+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:19.805099+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":113,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137278+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:04:01.137278+0000","last_peered":"2023-09-23T18:04:01.137278+0000","last_clean":"2023-09-23T18:04:01.137278+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:04:01.137278+0000","last_undegraded":"2023-09-23T18:04:01.137278+0000","last_fullsized":"2023-09-23T18:04:01.137278+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":265,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.129538+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:04:01.129538+0000","last_peered":"2023-09-23T18:04:01.129538+0000","last_clean":"2023-09-23T18:04:01.129538+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:04:01.129538+0000","last_undegraded":"2023-09-23T18:04:01.129538+0000","last_fullsized":"2023-09-23T18:04:01.129538+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":598,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137323+0000","last_change":"2023-09-23T18:02:51.051617+0000","last_active":"2023-09-23T18:04:01.137323+0000","last_peered":"2023-09-23T18:04:01.137323+0000","last_clean":"2023-09-23T18:04:01.137323+0000","last_became_active":"2023-09-23T18:02:51.048947+0000","last_became_peered":"2023-09-23T18:02:51.048947+0000","last_unstale":"2023-09-23T18:04:01.137323+0000","last_undegraded":"2023-09-23T18:04:01.137323+0000","last_fullsized":"2023-09-23T18:04:01.137323+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:49:34.962249+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":126,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137059+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:04:01.137059+0000","last_peered":"2023-09-23T18:04:01.137059+0000","last_clean":"2023-09-23T18:04:01.137059+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:04:01.137059+0000","last_undegraded":"2023-09-23T18:04:01.137059+0000","last_fullsized":"2023-09-23T18:04:01.137059+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":204,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.126863+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:04:01.126863+0000","last_peered":"2023-09-23T18:04:01.126863+0000","last_clean":"2023-09-23T18:04:01.126863+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:04:01.126863+0000","last_undegraded":"2023-09-23T18:04:01.126863+0000","last_fullsized":"2023-09-23T18:04:01.126863+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":595,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137466+0000","last_change":"2023-09-23T18:02:51.052123+0000","last_active":"2023-09-23T18:04:01.137466+0000","last_peered":"2023-09-23T18:04:01.137466+0000","last_clean":"2023-09-23T18:04:01.137466+0000","last_became_active":"2023-09-23T18:02:51.051772+0000","last_became_peered":"2023-09-23T18:02:51.051772+0000","last_unstale":"2023-09-23T18:04:01.137466+0000","last_undegraded":"2023-09-23T18:04:01.137466+0000","last_fullsized":"2023-09-23T18:04:01.137466+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":244,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.129633+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:04:01.129633+0000","last_peered":"2023-09-23T18:04:01.129633+0000","last_clean":"2023-09-23T18:04:01.129633+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:04:01.129633+0000","last_undegraded":"2023-09-23T18:04:01.129633+0000","last_fullsized":"2023-09-23T18:04:01.129633+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":120,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137386+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:04:01.137386+0000","last_peered":"2023-09-23T18:04:01.137386+0000","last_clean":"2023-09-23T18:04:01.137386+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:04:01.137386+0000","last_undegraded":"2023-09-23T18:04:01.137386+0000","last_fullsized":"2023-09-23T18:04:01.137386+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":590,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137005+0000","last_change":"2023-09-23T18:02:51.054208+0000","last_active":"2023-09-23T18:04:01.137005+0000","last_peered":"2023-09-23T18:04:01.137005+0000","last_clean":"2023-09-23T18:04:01.137005+0000","last_became_active":"2023-09-23T18:02:51.053676+0000","last_became_peered":"2023-09-23T18:02:51.053676+0000","last_unstale":"2023-09-23T18:04:01.137005+0000","last_undegraded":"2023-09-23T18:04:01.137005+0000","last_fullsized":"2023-09-23T18:04:01.137005+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":246,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.134740+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:04:01.134740+0000","last_peered":"2023-09-23T18:04:01.134740+0000","last_clean":"2023-09-23T18:04:01.134740+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:04:01.134740+0000","last_undegraded":"2023-09-23T18:04:01.134740+0000","last_fullsized":"2023-09-23T18:04:01.134740+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":74,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137188+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:04:01.137188+0000","last_peered":"2023-09-23T18:04:01.137188+0000","last_clean":"2023-09-23T18:04:01.137188+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:04:01.137188+0000","last_undegraded":"2023-09-23T18:04:01.137188+0000","last_fullsized":"2023-09-23T18:04:01.137188+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":595,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137418+0000","last_change":"2023-09-23T18:02:51.058624+0000","last_active":"2023-09-23T18:04:01.137418+0000","last_peered":"2023-09-23T18:04:01.137418+0000","last_clean":"2023-09-23T18:04:01.137418+0000","last_became_active":"2023-09-23T18:02:51.057352+0000","last_became_peered":"2023-09-23T18:02:51.057352+0000","last_unstale":"2023-09-23T18:04:01.137418+0000","last_undegraded":"2023-09-23T18:04:01.137418+0000","last_fullsized":"2023-09-23T18:04:01.137418+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":30,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137387+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:04:01.137387+0000","last_peered":"2023-09-23T18:04:01.137387+0000","last_clean":"2023-09-23T18:04:01.137387+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:04:01.137387+0000","last_undegraded":"2023-09-23T18:04:01.137387+0000","last_fullsized":"2023-09-23T18:04:01.137387+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":160,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.127876+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:04:01.127876+0000","last_peered":"2023-09-23T18:04:01.127876+0000","last_clean":"2023-09-23T18:04:01.127876+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:04:01.127876+0000","last_undegraded":"2023-09-23T18:04:01.127876+0000","last_fullsized":"2023-09-23T18:04:01.127876+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":595,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137366+0000","last_change":"2023-09-23T18:02:51.059983+0000","last_active":"2023-09-23T18:04:01.137366+0000","last_peered":"2023-09-23T18:04:01.137366+0000","last_clean":"2023-09-23T18:04:01.137366+0000","last_became_active":"2023-09-23T18:02:51.057768+0000","last_became_peered":"2023-09-23T18:02:51.057768+0000","last_unstale":"2023-09-23T18:04:01.137366+0000","last_undegraded":"2023-09-23T18:04:01.137366+0000","last_fullsized":"2023-09-23T18:04:01.137366+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":64,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136023+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:04:01.136023+0000","last_peered":"2023-09-23T18:04:01.136023+0000","last_clean":"2023-09-23T18:04:01.136023+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:04:01.136023+0000","last_undegraded":"2023-09-23T18:04:01.136023+0000","last_fullsized":"2023-09-23T18:04:01.136023+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":64,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.135737+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:04:01.135737+0000","last_peered":"2023-09-23T18:04:01.135737+0000","last_clean":"2023-09-23T18:04:01.135737+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:04:01.135737+0000","last_undegraded":"2023-09-23T18:04:01.135737+0000","last_fullsized":"2023-09-23T18:04:01.135737+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":585,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136841+0000","last_change":"2023-09-23T18:02:51.047570+0000","last_active":"2023-09-23T18:04:01.136841+0000","last_peered":"2023-09-23T18:04:01.136841+0000","last_clean":"2023-09-23T18:04:01.136841+0000","last_became_active":"2023-09-23T18:02:51.046963+0000","last_became_peered":"2023-09-23T18:02:51.046963+0000","last_unstale":"2023-09-23T18:04:01.136841+0000","last_undegraded":"2023-09-23T18:04:01.136841+0000","last_fullsized":"2023-09-23T18:04:01.136841+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":193,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137921+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:04:01.137921+0000","last_peered":"2023-09-23T18:04:01.137921+0000","last_clean":"2023-09-23T18:04:01.137921+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:04:01.137921+0000","last_undegraded":"2023-09-23T18:04:01.137921+0000","last_fullsized":"2023-09-23T18:04:01.137921+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":108,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137656+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:04:01.137656+0000","last_peered":"2023-09-23T18:04:01.137656+0000","last_clean":"2023-09-23T18:04:01.137656+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:04:01.137656+0000","last_undegraded":"2023-09-23T18:04:01.137656+0000","last_fullsized":"2023-09-23T18:04:01.137656+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":598,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136752+0000","last_change":"2023-09-23T18:02:51.053806+0000","last_active":"2023-09-23T18:04:01.136752+0000","last_peered":"2023-09-23T18:04:01.136752+0000","last_clean":"2023-09-23T18:04:01.136752+0000","last_became_active":"2023-09-23T18:02:51.048822+0000","last_became_peered":"2023-09-23T18:02:51.048822+0000","last_unstale":"2023-09-23T18:04:01.136752+0000","last_undegraded":"2023-09-23T18:04:01.136752+0000","last_fullsized":"2023-09-23T18:04:01.136752+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:34.401104+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":263,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.129102+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:04:01.129102+0000","last_peered":"2023-09-23T18:04:01.129102+0000","last_clean":"2023-09-23T18:04:01.129102+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:04:01.129102+0000","last_undegraded":"2023-09-23T18:04:01.129102+0000","last_fullsized":"2023-09-23T18:04:01.129102+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":241,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.134497+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:04:01.134497+0000","last_peered":"2023-09-23T18:04:01.134497+0000","last_clean":"2023-09-23T18:04:01.134497+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:04:01.134497+0000","last_undegraded":"2023-09-23T18:04:01.134497+0000","last_fullsized":"2023-09-23T18:04:01.134497+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":586,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137839+0000","last_change":"2023-09-23T18:02:51.046646+0000","last_active":"2023-09-23T18:04:01.137839+0000","last_peered":"2023-09-23T18:04:01.137839+0000","last_clean":"2023-09-23T18:04:01.137839+0000","last_became_active":"2023-09-23T18:02:51.046134+0000","last_became_peered":"2023-09-23T18:02:51.046134+0000","last_unstale":"2023-09-23T18:04:01.137839+0000","last_undegraded":"2023-09-23T18:04:01.137839+0000","last_fullsized":"2023-09-23T18:04:01.137839+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":54,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.135609+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:04:01.135609+0000","last_peered":"2023-09-23T18:04:01.135609+0000","last_clean":"2023-09-23T18:04:01.135609+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:04:01.135609+0000","last_undegraded":"2023-09-23T18:04:01.135609+0000","last_fullsized":"2023-09-23T18:04:01.135609+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":254,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.129613+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:04:01.129613+0000","last_peered":"2023-09-23T18:04:01.129613+0000","last_clean":"2023-09-23T18:04:01.129613+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:04:01.129613+0000","last_undegraded":"2023-09-23T18:04:01.129613+0000","last_fullsized":"2023-09-23T18:04:01.129613+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":598,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137238+0000","last_change":"2023-09-23T18:02:51.051817+0000","last_active":"2023-09-23T18:04:01.137238+0000","last_peered":"2023-09-23T18:04:01.137238+0000","last_clean":"2023-09-23T18:04:01.137238+0000","last_became_active":"2023-09-23T18:02:51.048945+0000","last_became_peered":"2023-09-23T18:02:51.048945+0000","last_unstale":"2023-09-23T18:04:01.137238+0000","last_undegraded":"2023-09-23T18:04:01.137238+0000","last_fullsized":"2023-09-23T18:04:01.137238+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:47.163864+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":200,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.125794+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:04:01.125794+0000","last_peered":"2023-09-23T18:04:01.125794+0000","last_clean":"2023-09-23T18:04:01.125794+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:04:01.125794+0000","last_undegraded":"2023-09-23T18:04:01.125794+0000","last_fullsized":"2023-09-23T18:04:01.125794+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":117,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137734+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:04:01.137734+0000","last_peered":"2023-09-23T18:04:01.137734+0000","last_clean":"2023-09-23T18:04:01.137734+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:04:01.137734+0000","last_undegraded":"2023-09-23T18:04:01.137734+0000","last_fullsized":"2023-09-23T18:04:01.137734+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"264'6707","reported_seq":6774,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:07.514488+0000","last_change":"2023-09-23T18:00:57.996982+0000","last_active":"2023-09-23T18:04:07.514488+0000","last_peered":"2023-09-23T18:04:07.514488+0000","last_clean":"2023-09-23T18:04:07.514488+0000","last_became_active":"2023-09-23T18:00:57.996589+0000","last_became_peered":"2023-09-23T18:00:57.996589+0000","last_unstale":"2023-09-23T18:04:07.514488+0000","last_undegraded":"2023-09-23T18:04:07.514488+0000","last_fullsized":"2023-09-23T18:04:07.514488+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6707,"log_dups_size":0,"ondisk_log_size":6707,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:13.306089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166133760,"num_objects":2535,"num_object_clones":0,"num_object_copies":7605,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2535,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6707,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":595,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137295+0000","last_change":"2023-09-23T18:02:51.058477+0000","last_active":"2023-09-23T18:04:01.137295+0000","last_peered":"2023-09-23T18:04:01.137295+0000","last_clean":"2023-09-23T18:04:01.137295+0000","last_became_active":"2023-09-23T18:02:51.057258+0000","last_became_peered":"2023-09-23T18:02:51.057258+0000","last_unstale":"2023-09-23T18:04:01.137295+0000","last_undegraded":"2023-09-23T18:04:01.137295+0000","last_fullsized":"2023-09-23T18:04:01.137295+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":193,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137523+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:04:01.137523+0000","last_peered":"2023-09-23T18:04:01.137523+0000","last_clean":"2023-09-23T18:04:01.137523+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:04:01.137523+0000","last_undegraded":"2023-09-23T18:04:01.137523+0000","last_fullsized":"2023-09-23T18:04:01.137523+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":252,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.134151+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:04:01.134151+0000","last_peered":"2023-09-23T18:04:01.134151+0000","last_clean":"2023-09-23T18:04:01.134151+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:04:01.134151+0000","last_undegraded":"2023-09-23T18:04:01.134151+0000","last_fullsized":"2023-09-23T18:04:01.134151+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"264'6631","reported_seq":13858,"reported_epoch":264,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:04:10.477076+0000","last_change":"2023-09-23T18:02:49.039290+0000","last_active":"2023-09-23T18:04:10.477076+0000","last_peered":"2023-09-23T18:04:10.477076+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:49.001792+0000","last_became_peered":"2023-09-23T18:02:49.001792+0000","last_unstale":"2023-09-23T18:04:10.477076+0000","last_undegraded":"2023-09-23T18:02:48.968288+0000","last_fullsized":"2023-09-23T18:04:10.477076+0000","mapping_epoch":250,"log_start":"222'2430","ondisk_log_start":"222'2430","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4201,"log_dups_size":0,"ondisk_log_size":4201,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.156979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":160759808,"num_objects":2453,"num_object_clones":0,"num_object_copies":7359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":409,"num_objects_misplaced":4497,"num_objects_unfound":0,"num_objects_dirty":2453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6631,"num_write_kb":290688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3566,"num_bytes_recovered":233701376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[6,1,4],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,4,6","objects":2044},{"shards":"1,6","objects":409}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":594,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.152730+0000","last_change":"2023-09-23T18:04:02.152730+0000","last_active":"2023-09-23T18:04:02.152730+0000","last_peered":"2023-09-23T18:04:02.152730+0000","last_clean":"2023-09-23T18:04:02.152730+0000","last_became_active":"2023-09-23T18:04:02.152366+0000","last_became_peered":"2023-09-23T18:04:02.152366+0000","last_unstale":"2023-09-23T18:04:02.152730+0000","last_undegraded":"2023-09-23T18:04:02.152730+0000","last_fullsized":"2023-09-23T18:04:02.152730+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":736808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":208,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.138241+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:04:01.138241+0000","last_peered":"2023-09-23T18:04:01.138241+0000","last_clean":"2023-09-23T18:04:01.138241+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:04:01.138241+0000","last_undegraded":"2023-09-23T18:04:01.138241+0000","last_fullsized":"2023-09-23T18:04:01.138241+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":274,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.135980+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:04:01.135980+0000","last_peered":"2023-09-23T18:04:01.135980+0000","last_clean":"2023-09-23T18:04:01.135980+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:04:01.135980+0000","last_undegraded":"2023-09-23T18:04:01.135980+0000","last_fullsized":"2023-09-23T18:04:01.135980+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"264'6742","reported_seq":15349,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:09.038557+0000","last_change":"2023-09-23T18:03:33.017384+0000","last_active":"2023-09-23T18:04:09.038557+0000","last_peered":"2023-09-23T18:04:09.038557+0000","last_clean":"2023-09-23T18:04:09.038557+0000","last_became_active":"2023-09-23T18:03:33.004338+0000","last_became_peered":"2023-09-23T18:03:33.004338+0000","last_unstale":"2023-09-23T18:04:09.038557+0000","last_undegraded":"2023-09-23T18:04:09.038557+0000","last_fullsized":"2023-09-23T18:04:09.038557+0000","mapping_epoch":259,"log_start":"218'2000","ondisk_log_start":"218'2000","created":204,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4742,"log_dups_size":0,"ondisk_log_size":4742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:59.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840000,"num_objects":2500,"num_object_clones":0,"num_object_copies":7500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6742,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4258,"num_bytes_recovered":259457024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":586,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137236+0000","last_change":"2023-09-23T18:02:51.058795+0000","last_active":"2023-09-23T18:04:01.137236+0000","last_peered":"2023-09-23T18:04:01.137236+0000","last_clean":"2023-09-23T18:04:01.137236+0000","last_became_active":"2023-09-23T18:02:51.057799+0000","last_became_peered":"2023-09-23T18:02:51.057799+0000","last_unstale":"2023-09-23T18:04:01.137236+0000","last_undegraded":"2023-09-23T18:04:01.137236+0000","last_fullsized":"2023-09-23T18:04:01.137236+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":131,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137731+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:04:01.137731+0000","last_peered":"2023-09-23T18:04:01.137731+0000","last_clean":"2023-09-23T18:04:01.137731+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:04:01.137731+0000","last_undegraded":"2023-09-23T18:04:01.137731+0000","last_fullsized":"2023-09-23T18:04:01.137731+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":229,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.127738+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:04:01.127738+0000","last_peered":"2023-09-23T18:04:01.127738+0000","last_clean":"2023-09-23T18:04:01.127738+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:04:01.127738+0000","last_undegraded":"2023-09-23T18:04:01.127738+0000","last_fullsized":"2023-09-23T18:04:01.127738+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"264'6814","reported_seq":6901,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:09.316486+0000","last_change":"2023-09-23T18:01:25.019784+0000","last_active":"2023-09-23T18:04:09.316486+0000","last_peered":"2023-09-23T18:04:09.316486+0000","last_clean":"2023-09-23T18:04:09.316486+0000","last_became_active":"2023-09-23T18:01:25.002073+0000","last_became_peered":"2023-09-23T18:01:25.002073+0000","last_unstale":"2023-09-23T18:04:09.316486+0000","last_undegraded":"2023-09-23T18:04:09.316486+0000","last_fullsized":"2023-09-23T18:04:09.316486+0000","mapping_epoch":214,"log_start":"222'2400","ondisk_log_start":"222'2400","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4414,"log_dups_size":0,"ondisk_log_size":4414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:58.981672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163053568,"num_objects":2488,"num_object_clones":0,"num_object_copies":7464,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2488,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6814,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":595,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137981+0000","last_change":"2023-09-23T18:02:51.057506+0000","last_active":"2023-09-23T18:04:01.137981+0000","last_peered":"2023-09-23T18:04:01.137981+0000","last_clean":"2023-09-23T18:04:01.137981+0000","last_became_active":"2023-09-23T18:02:51.056589+0000","last_became_peered":"2023-09-23T18:02:51.056589+0000","last_unstale":"2023-09-23T18:04:01.137981+0000","last_undegraded":"2023-09-23T18:04:01.137981+0000","last_fullsized":"2023-09-23T18:04:01.137981+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":240,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.134759+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:04:01.134759+0000","last_peered":"2023-09-23T18:04:01.134759+0000","last_clean":"2023-09-23T18:04:01.134759+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:04:01.134759+0000","last_undegraded":"2023-09-23T18:04:01.134759+0000","last_fullsized":"2023-09-23T18:04:01.134759+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":266,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.129201+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:04:01.129201+0000","last_peered":"2023-09-23T18:04:01.129201+0000","last_clean":"2023-09-23T18:04:01.129201+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:04:01.129201+0000","last_undegraded":"2023-09-23T18:04:01.129201+0000","last_fullsized":"2023-09-23T18:04:01.129201+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"264'6752","reported_seq":11091,"reported_epoch":264,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:04:08.172934+0000","last_change":"2023-09-23T18:02:45.688221+0000","last_active":"2023-09-23T18:04:08.172934+0000","last_peered":"2023-09-23T18:04:08.172934+0000","last_clean":"2023-09-23T18:01:38.416095+0000","last_became_active":"2023-09-23T18:02:07.547135+0000","last_became_peered":"2023-09-23T18:02:07.547135+0000","last_unstale":"2023-09-23T18:04:08.172934+0000","last_undegraded":"2023-09-23T18:04:08.172934+0000","last_fullsized":"2023-09-23T18:02:02.784733+0000","mapping_epoch":227,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6652,"log_dups_size":0,"ondisk_log_size":6652,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:27.864301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163971072,"num_objects":2502,"num_object_clones":0,"num_object_copies":7506,"num_objects_missing_on_primary":25,"num_objects_missing":25,"num_objects_degraded":0,"num_objects_misplaced":1964,"num_objects_unfound":0,"num_objects_dirty":2502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6752,"num_write_kb":296128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2124,"num_bytes_recovered":139198464,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2,4","objects":1939},{"shards":"1,2,4,5","objects":12},{"shards":"1,2,5","objects":25},{"shards":"2,4","objects":526}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":587,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136817+0000","last_change":"2023-09-23T18:02:51.047609+0000","last_active":"2023-09-23T18:04:01.136817+0000","last_peered":"2023-09-23T18:04:01.136817+0000","last_clean":"2023-09-23T18:04:01.136817+0000","last_became_active":"2023-09-23T18:02:51.047209+0000","last_became_peered":"2023-09-23T18:02:51.047209+0000","last_unstale":"2023-09-23T18:04:01.136817+0000","last_undegraded":"2023-09-23T18:04:01.136817+0000","last_fullsized":"2023-09-23T18:04:01.136817+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":151,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137294+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:04:01.137294+0000","last_peered":"2023-09-23T18:04:01.137294+0000","last_clean":"2023-09-23T18:04:01.137294+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:04:01.137294+0000","last_undegraded":"2023-09-23T18:04:01.137294+0000","last_fullsized":"2023-09-23T18:04:01.137294+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":272,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136100+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:04:01.136100+0000","last_peered":"2023-09-23T18:04:01.136100+0000","last_clean":"2023-09-23T18:04:01.136100+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:04:01.136100+0000","last_undegraded":"2023-09-23T18:04:01.136100+0000","last_fullsized":"2023-09-23T18:04:01.136100+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"264'6591","reported_seq":6817,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:09.312845+0000","last_change":"2023-09-23T18:02:47.961575+0000","last_active":"2023-09-23T18:04:09.312845+0000","last_peered":"2023-09-23T18:04:09.312845+0000","last_clean":"2023-09-23T18:02:17.013863+0000","last_became_active":"2023-09-23T18:02:19.267215+0000","last_became_peered":"2023-09-23T18:02:19.267215+0000","last_unstale":"2023-09-23T18:04:09.312845+0000","last_undegraded":"2023-09-23T18:02:18.955030+0000","last_fullsized":"2023-09-23T18:02:18.924656+0000","mapping_epoch":238,"log_start":"218'1900","ondisk_log_start":"218'1900","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4691,"log_dups_size":0,"ondisk_log_size":4691,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:46.393021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":161546240,"num_objects":2465,"num_object_clones":0,"num_object_copies":7395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6591,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":4128768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-1643},{"shards":"2,4,6","objects":2107}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":591,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137688+0000","last_change":"2023-09-23T18:02:51.049464+0000","last_active":"2023-09-23T18:04:01.137688+0000","last_peered":"2023-09-23T18:04:01.137688+0000","last_clean":"2023-09-23T18:04:01.137688+0000","last_became_active":"2023-09-23T18:02:51.048477+0000","last_became_peered":"2023-09-23T18:02:51.048477+0000","last_unstale":"2023-09-23T18:04:01.137688+0000","last_undegraded":"2023-09-23T18:04:01.137688+0000","last_fullsized":"2023-09-23T18:04:01.137688+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":223,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.128006+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:04:01.128006+0000","last_peered":"2023-09-23T18:04:01.128006+0000","last_clean":"2023-09-23T18:04:01.128006+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:04:01.128006+0000","last_undegraded":"2023-09-23T18:04:01.128006+0000","last_fullsized":"2023-09-23T18:04:01.128006+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":64,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.135971+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:04:01.135971+0000","last_peered":"2023-09-23T18:04:01.135971+0000","last_clean":"2023-09-23T18:04:01.135971+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:04:01.135971+0000","last_undegraded":"2023-09-23T18:04:01.135971+0000","last_fullsized":"2023-09-23T18:04:01.135971+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"264'6605","reported_seq":6677,"reported_epoch":264,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:04:07.445105+0000","last_change":"2023-09-23T18:02:19.054827+0000","last_active":"2023-09-23T18:04:07.445105+0000","last_peered":"2023-09-23T18:04:07.445105+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.048672+0000","last_became_peered":"2023-09-23T18:02:19.048672+0000","last_unstale":"2023-09-23T18:04:07.445105+0000","last_undegraded":"2023-09-23T18:02:18.951263+0000","last_fullsized":"2023-09-23T18:02:18.930851+0000","mapping_epoch":238,"log_start":"216'261","ondisk_log_start":"216'261","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6344,"log_dups_size":0,"ondisk_log_size":6344,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:23.690233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":162988032,"num_objects":2487,"num_object_clones":0,"num_object_copies":7461,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2487,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2487,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6605,"num_write_kb":290944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":2487}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":595,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137680+0000","last_change":"2023-09-23T18:02:51.060858+0000","last_active":"2023-09-23T18:04:01.137680+0000","last_peered":"2023-09-23T18:04:01.137680+0000","last_clean":"2023-09-23T18:04:01.137680+0000","last_became_active":"2023-09-23T18:02:51.060554+0000","last_became_peered":"2023-09-23T18:02:51.060554+0000","last_unstale":"2023-09-23T18:04:01.137680+0000","last_undegraded":"2023-09-23T18:04:01.137680+0000","last_fullsized":"2023-09-23T18:04:01.137680+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":54,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.135783+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:04:01.135783+0000","last_peered":"2023-09-23T18:04:01.135783+0000","last_clean":"2023-09-23T18:04:01.135783+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:04:01.135783+0000","last_undegraded":"2023-09-23T18:04:01.135783+0000","last_fullsized":"2023-09-23T18:04:01.135783+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":274,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.184664+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:04:01.184664+0000","last_peered":"2023-09-23T18:04:01.184664+0000","last_clean":"2023-09-23T18:04:01.184664+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:04:01.184664+0000","last_undegraded":"2023-09-23T18:04:01.184664+0000","last_fullsized":"2023-09-23T18:04:01.184664+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"264'6563","reported_seq":14684,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:07.425030+0000","last_change":"2023-09-23T18:02:47.017672+0000","last_active":"2023-09-23T18:04:07.425030+0000","last_peered":"2023-09-23T18:04:07.425030+0000","last_clean":"2023-09-23T18:04:07.425030+0000","last_became_active":"2023-09-23T18:02:47.009245+0000","last_became_peered":"2023-09-23T18:02:47.009245+0000","last_unstale":"2023-09-23T18:04:07.425030+0000","last_undegraded":"2023-09-23T18:04:07.425030+0000","last_fullsized":"2023-09-23T18:04:07.425030+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6563,"log_dups_size":0,"ondisk_log_size":6563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:04.868743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165281816,"num_objects":2523,"num_object_clones":0,"num_object_copies":7569,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2523,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6563,"num_write_kb":290689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4015,"num_bytes_recovered":263127040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":587,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137152+0000","last_change":"2023-09-23T18:02:51.054197+0000","last_active":"2023-09-23T18:04:01.137152+0000","last_peered":"2023-09-23T18:04:01.137152+0000","last_clean":"2023-09-23T18:04:01.137152+0000","last_became_active":"2023-09-23T18:02:51.049362+0000","last_became_peered":"2023-09-23T18:02:51.049362+0000","last_unstale":"2023-09-23T18:04:01.137152+0000","last_undegraded":"2023-09-23T18:04:01.137152+0000","last_fullsized":"2023-09-23T18:04:01.137152+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":109,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137343+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:04:01.137343+0000","last_peered":"2023-09-23T18:04:01.137343+0000","last_clean":"2023-09-23T18:04:01.137343+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:04:01.137343+0000","last_undegraded":"2023-09-23T18:04:01.137343+0000","last_fullsized":"2023-09-23T18:04:01.137343+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":109,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137829+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:04:01.137829+0000","last_peered":"2023-09-23T18:04:01.137829+0000","last_clean":"2023-09-23T18:04:01.137829+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:04:01.137829+0000","last_undegraded":"2023-09-23T18:04:01.137829+0000","last_fullsized":"2023-09-23T18:04:01.137829+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"264'6447","reported_seq":6521,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:09.559655+0000","last_change":"2023-09-23T18:02:19.315376+0000","last_active":"2023-09-23T18:04:09.559655+0000","last_peered":"2023-09-23T18:04:09.559655+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.306174+0000","last_became_peered":"2023-09-23T18:02:19.306174+0000","last_unstale":"2023-09-23T18:04:09.559655+0000","last_undegraded":"2023-09-23T18:02:18.954031+0000","last_fullsized":"2023-09-23T18:02:18.922505+0000","mapping_epoch":238,"log_start":"218'2000","ondisk_log_start":"218'2000","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4447,"log_dups_size":0,"ondisk_log_size":4447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:52.392034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":159580160,"num_objects":2435,"num_object_clones":0,"num_object_copies":7305,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2435,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6447,"num_write_kb":284224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":-1660}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":596,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137218+0000","last_change":"2023-09-23T18:02:51.059934+0000","last_active":"2023-09-23T18:04:01.137218+0000","last_peered":"2023-09-23T18:04:01.137218+0000","last_clean":"2023-09-23T18:04:01.137218+0000","last_became_active":"2023-09-23T18:02:51.059638+0000","last_became_peered":"2023-09-23T18:02:51.059638+0000","last_unstale":"2023-09-23T18:04:01.137218+0000","last_undegraded":"2023-09-23T18:04:01.137218+0000","last_fullsized":"2023-09-23T18:04:01.137218+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":574,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151866+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:04:02.151866+0000","last_peered":"2023-09-23T18:04:02.151866+0000","last_clean":"2023-09-23T18:04:02.151866+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:04:02.151866+0000","last_undegraded":"2023-09-23T18:04:02.151866+0000","last_fullsized":"2023-09-23T18:04:02.151866+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":244,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.134583+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:04:01.134583+0000","last_peered":"2023-09-23T18:04:01.134583+0000","last_clean":"2023-09-23T18:04:01.134583+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:04:01.134583+0000","last_undegraded":"2023-09-23T18:04:01.134583+0000","last_fullsized":"2023-09-23T18:04:01.134583+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":241,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.134646+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:04:01.134646+0000","last_peered":"2023-09-23T18:04:01.134646+0000","last_clean":"2023-09-23T18:04:01.134646+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:04:01.134646+0000","last_undegraded":"2023-09-23T18:04:01.134646+0000","last_fullsized":"2023-09-23T18:04:01.134646+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"264'6559","reported_seq":6647,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:08.029237+0000","last_change":"2023-09-23T18:02:19.289610+0000","last_active":"2023-09-23T18:04:08.029237+0000","last_peered":"2023-09-23T18:04:08.029237+0000","last_clean":"2023-09-23T18:02:17.060883+0000","last_became_active":"2023-09-23T18:02:19.266879+0000","last_became_peered":"2023-09-23T18:02:19.266879+0000","last_unstale":"2023-09-23T18:04:08.029237+0000","last_undegraded":"2023-09-23T18:02:18.963390+0000","last_fullsized":"2023-09-23T18:02:18.926482+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6559,"log_dups_size":0,"ondisk_log_size":6559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.499269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":157220864,"num_objects":2399,"num_object_clones":0,"num_object_copies":7197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2399,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6559,"num_write_kb":286656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":2048},{"shards":"4,7","objects":-1729}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":598,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136749+0000","last_change":"2023-09-23T18:02:51.047767+0000","last_active":"2023-09-23T18:04:01.136749+0000","last_peered":"2023-09-23T18:04:01.136749+0000","last_clean":"2023-09-23T18:04:01.136749+0000","last_became_active":"2023-09-23T18:02:51.047297+0000","last_became_peered":"2023-09-23T18:02:51.047297+0000","last_unstale":"2023-09-23T18:04:01.136749+0000","last_undegraded":"2023-09-23T18:04:01.136749+0000","last_fullsized":"2023-09-23T18:04:01.136749+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.851694+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1192,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.134653+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:04:01.134653+0000","last_peered":"2023-09-23T18:04:01.134653+0000","last_clean":"2023-09-23T18:04:01.134653+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:04:01.134653+0000","last_undegraded":"2023-09-23T18:04:01.134653+0000","last_fullsized":"2023-09-23T18:04:01.134653+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":54,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.135801+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:04:01.135801+0000","last_peered":"2023-09-23T18:04:01.135801+0000","last_clean":"2023-09-23T18:04:01.135801+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:04:01.135801+0000","last_undegraded":"2023-09-23T18:04:01.135801+0000","last_fullsized":"2023-09-23T18:04:01.135801+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":158,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.158304+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:04:02.158304+0000","last_peered":"2023-09-23T18:04:02.158304+0000","last_clean":"2023-09-23T18:04:02.158304+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:04:02.158304+0000","last_undegraded":"2023-09-23T18:04:02.158304+0000","last_fullsized":"2023-09-23T18:04:02.158304+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"264'6727","reported_seq":15358,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:07.875953+0000","last_change":"2023-09-23T18:04:02.179479+0000","last_active":"2023-09-23T18:04:07.875953+0000","last_peered":"2023-09-23T18:04:07.875953+0000","last_clean":"2023-09-23T18:04:07.875953+0000","last_became_active":"2023-09-23T18:04:02.150813+0000","last_became_peered":"2023-09-23T18:04:02.150813+0000","last_unstale":"2023-09-23T18:04:07.875953+0000","last_undegraded":"2023-09-23T18:04:07.875953+0000","last_fullsized":"2023-09-23T18:04:07.875953+0000","mapping_epoch":263,"log_start":"222'2774","ondisk_log_start":"222'2774","created":204,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":3953,"log_dups_size":0,"ondisk_log_size":3953,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:41.194322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166264832,"num_objects":2537,"num_object_clones":0,"num_object_copies":7611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6727,"num_write_kb":296448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4277,"num_bytes_recovered":179372032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":599,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137065+0000","last_change":"2023-09-23T18:02:51.054622+0000","last_active":"2023-09-23T18:04:01.137065+0000","last_peered":"2023-09-23T18:04:01.137065+0000","last_clean":"2023-09-23T18:04:01.137065+0000","last_became_active":"2023-09-23T18:02:51.049345+0000","last_became_peered":"2023-09-23T18:02:51.049345+0000","last_unstale":"2023-09-23T18:04:01.137065+0000","last_undegraded":"2023-09-23T18:04:01.137065+0000","last_fullsized":"2023-09-23T18:04:01.137065+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:58.636010+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":546,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.125776+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:04:01.125776+0000","last_peered":"2023-09-23T18:04:01.125776+0000","last_clean":"2023-09-23T18:04:01.125776+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:04:01.125776+0000","last_undegraded":"2023-09-23T18:04:01.125776+0000","last_fullsized":"2023-09-23T18:04:01.125776+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":159,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.157910+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:04:02.157910+0000","last_peered":"2023-09-23T18:04:02.157910+0000","last_clean":"2023-09-23T18:04:02.157910+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:04:02.157910+0000","last_undegraded":"2023-09-23T18:04:02.157910+0000","last_fullsized":"2023-09-23T18:04:02.157910+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":231,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.138810+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:04:01.138810+0000","last_peered":"2023-09-23T18:04:01.138810+0000","last_clean":"2023-09-23T18:04:01.138810+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:04:01.138810+0000","last_undegraded":"2023-09-23T18:04:01.138810+0000","last_fullsized":"2023-09-23T18:04:01.138810+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"264'6697","reported_seq":7930,"reported_epoch":264,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:07.577292+0000","last_change":"2023-09-23T18:04:00.228250+0000","last_active":"2023-09-23T18:04:07.577292+0000","last_peered":"2023-09-23T18:04:07.577292+0000","last_clean":"2023-09-23T18:02:16.962656+0000","last_became_active":"2023-09-23T18:02:19.272896+0000","last_became_peered":"2023-09-23T18:02:19.272896+0000","last_unstale":"2023-09-23T18:04:07.577292+0000","last_undegraded":"2023-09-23T18:02:18.972349+0000","last_fullsized":"2023-09-23T18:02:18.930868+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6697,"log_dups_size":0,"ondisk_log_size":6697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:23.745675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166658048,"num_objects":2543,"num_object_clones":0,"num_object_copies":7629,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1959,"num_objects_misplaced":1739,"num_objects_unfound":0,"num_objects_dirty":2543,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6697,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":566,"num_bytes_recovered":37093376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":1739},{"shards":"4,5","objects":-1155}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1176,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.134390+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:04:01.134390+0000","last_peered":"2023-09-23T18:04:01.134390+0000","last_clean":"2023-09-23T18:04:01.134390+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:04:01.134390+0000","last_undegraded":"2023-09-23T18:04:01.134390+0000","last_fullsized":"2023-09-23T18:04:01.134390+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":587,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136690+0000","last_change":"2023-09-23T18:02:51.054022+0000","last_active":"2023-09-23T18:04:01.136690+0000","last_peered":"2023-09-23T18:04:01.136690+0000","last_clean":"2023-09-23T18:04:01.136690+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:04:01.136690+0000","last_undegraded":"2023-09-23T18:04:01.136690+0000","last_fullsized":"2023-09-23T18:04:01.136690+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":71,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137478+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:04:01.137478+0000","last_peered":"2023-09-23T18:04:01.137478+0000","last_clean":"2023-09-23T18:04:01.137478+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:04:01.137478+0000","last_undegraded":"2023-09-23T18:04:01.137478+0000","last_fullsized":"2023-09-23T18:04:01.137478+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":197,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.127486+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:04:01.127486+0000","last_peered":"2023-09-23T18:04:01.127486+0000","last_clean":"2023-09-23T18:04:01.127486+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:04:01.127486+0000","last_undegraded":"2023-09-23T18:04:01.127486+0000","last_fullsized":"2023-09-23T18:04:01.127486+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"264'6660","reported_seq":6822,"reported_epoch":264,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:04:07.510374+0000","last_change":"2023-09-23T18:02:18.003772+0000","last_active":"2023-09-23T18:04:07.510374+0000","last_peered":"2023-09-23T18:04:07.510374+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:18.000696+0000","last_became_peered":"2023-09-23T18:02:18.000696+0000","last_unstale":"2023-09-23T18:04:07.510374+0000","last_undegraded":"2023-09-23T18:02:17.938873+0000","last_fullsized":"2023-09-23T18:02:17.919669+0000","mapping_epoch":237,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6560,"log_dups_size":0,"ondisk_log_size":6560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:48.085957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":158728192,"num_objects":2422,"num_object_clones":0,"num_object_copies":7266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2399,"num_objects_misplaced":2422,"num_objects_unfound":0,"num_objects_dirty":2422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6660,"num_write_kb":290624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":2422}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":599,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137611+0000","last_change":"2023-09-23T18:02:51.061168+0000","last_active":"2023-09-23T18:04:01.137611+0000","last_peered":"2023-09-23T18:04:01.137611+0000","last_clean":"2023-09-23T18:04:01.137611+0000","last_became_active":"2023-09-23T18:02:51.060689+0000","last_became_peered":"2023-09-23T18:02:51.060689+0000","last_unstale":"2023-09-23T18:04:01.137611+0000","last_undegraded":"2023-09-23T18:04:01.137611+0000","last_fullsized":"2023-09-23T18:04:01.137611+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:29.406460+0000","scrub_duration":0.001568505,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":444,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136969+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:04:01.136969+0000","last_peered":"2023-09-23T18:04:01.136969+0000","last_clean":"2023-09-23T18:04:01.136969+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:04:01.136969+0000","last_undegraded":"2023-09-23T18:04:01.136969+0000","last_fullsized":"2023-09-23T18:04:01.136969+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":263,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.129011+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:04:01.129011+0000","last_peered":"2023-09-23T18:04:01.129011+0000","last_clean":"2023-09-23T18:04:01.129011+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:04:01.129011+0000","last_undegraded":"2023-09-23T18:04:01.129011+0000","last_fullsized":"2023-09-23T18:04:01.129011+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":234,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151252+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:04:02.151252+0000","last_peered":"2023-09-23T18:04:02.151252+0000","last_clean":"2023-09-23T18:04:02.151252+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:04:02.151252+0000","last_undegraded":"2023-09-23T18:04:02.151252+0000","last_fullsized":"2023-09-23T18:04:02.151252+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"264'6727","reported_seq":15072,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:10.494672+0000","last_change":"2023-09-23T18:02:18.942886+0000","last_active":"2023-09-23T18:04:10.494672+0000","last_peered":"2023-09-23T18:04:10.494672+0000","last_clean":"2023-09-23T18:04:10.494672+0000","last_became_active":"2023-09-23T18:02:18.936691+0000","last_became_peered":"2023-09-23T18:02:18.936691+0000","last_unstale":"2023-09-23T18:04:10.494672+0000","last_undegraded":"2023-09-23T18:04:10.494672+0000","last_fullsized":"2023-09-23T18:04:10.494672+0000","mapping_epoch":238,"log_start":"222'2558","ondisk_log_start":"222'2558","created":204,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4169,"log_dups_size":0,"ondisk_log_size":4169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:26.085859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":164429824,"num_objects":2509,"num_object_clones":0,"num_object_copies":7527,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2509,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6727,"num_write_kb":295552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4123,"num_bytes_recovered":270204928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":596,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.138075+0000","last_change":"2023-09-23T18:02:51.057370+0000","last_active":"2023-09-23T18:04:01.138075+0000","last_peered":"2023-09-23T18:04:01.138075+0000","last_clean":"2023-09-23T18:04:01.138075+0000","last_became_active":"2023-09-23T18:02:51.056158+0000","last_became_peered":"2023-09-23T18:02:51.056158+0000","last_unstale":"2023-09-23T18:04:01.138075+0000","last_undegraded":"2023-09-23T18:04:01.138075+0000","last_fullsized":"2023-09-23T18:04:01.138075+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":478,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137631+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:04:01.137631+0000","last_peered":"2023-09-23T18:04:01.137631+0000","last_clean":"2023-09-23T18:04:01.137631+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:04:01.137631+0000","last_undegraded":"2023-09-23T18:04:01.137631+0000","last_fullsized":"2023-09-23T18:04:01.137631+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":199,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.158331+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:04:02.158331+0000","last_peered":"2023-09-23T18:04:02.158331+0000","last_clean":"2023-09-23T18:04:02.158331+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:04:02.158331+0000","last_undegraded":"2023-09-23T18:04:02.158331+0000","last_fullsized":"2023-09-23T18:04:02.158331+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":231,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.129456+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:04:01.129456+0000","last_peered":"2023-09-23T18:04:01.129456+0000","last_clean":"2023-09-23T18:04:01.129456+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:04:01.129456+0000","last_undegraded":"2023-09-23T18:04:01.129456+0000","last_fullsized":"2023-09-23T18:04:01.129456+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"264'6577","reported_seq":6670,"reported_epoch":264,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:08.167377+0000","last_change":"2023-09-23T18:02:19.279112+0000","last_active":"2023-09-23T18:04:08.167377+0000","last_peered":"2023-09-23T18:04:08.167377+0000","last_clean":"2023-09-23T18:02:17.018900+0000","last_became_active":"2023-09-23T18:02:19.267439+0000","last_became_peered":"2023-09-23T18:02:19.267439+0000","last_unstale":"2023-09-23T18:04:08.167377+0000","last_undegraded":"2023-09-23T18:02:18.959329+0000","last_fullsized":"2023-09-23T18:02:18.926801+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6577,"log_dups_size":0,"ondisk_log_size":6577,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.395583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":166264832,"num_objects":2537,"num_object_clones":0,"num_object_copies":7611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6577,"num_write_kb":291648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-1662},{"shards":"2,4,6","objects":2179}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":596,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137208+0000","last_change":"2023-09-23T18:02:51.058680+0000","last_active":"2023-09-23T18:04:01.137208+0000","last_peered":"2023-09-23T18:04:01.137208+0000","last_clean":"2023-09-23T18:04:01.137208+0000","last_became_active":"2023-09-23T18:02:51.057706+0000","last_became_peered":"2023-09-23T18:02:51.057706+0000","last_unstale":"2023-09-23T18:04:01.137208+0000","last_undegraded":"2023-09-23T18:04:01.137208+0000","last_fullsized":"2023-09-23T18:04:01.137208+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":109,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137844+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:04:01.137844+0000","last_peered":"2023-09-23T18:04:01.137844+0000","last_clean":"2023-09-23T18:04:01.137844+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:04:01.137844+0000","last_undegraded":"2023-09-23T18:04:01.137844+0000","last_fullsized":"2023-09-23T18:04:01.137844+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":334,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.126391+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:04:01.126391+0000","last_peered":"2023-09-23T18:04:01.126391+0000","last_clean":"2023-09-23T18:04:01.126391+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:04:01.126391+0000","last_undegraded":"2023-09-23T18:04:01.126391+0000","last_fullsized":"2023-09-23T18:04:01.126391+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":113,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137147+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:04:01.137147+0000","last_peered":"2023-09-23T18:04:01.137147+0000","last_clean":"2023-09-23T18:04:01.137147+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:04:01.137147+0000","last_undegraded":"2023-09-23T18:04:01.137147+0000","last_fullsized":"2023-09-23T18:04:01.137147+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"264'6638","reported_seq":6708,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:07.947437+0000","last_change":"2023-09-23T18:01:25.046224+0000","last_active":"2023-09-23T18:04:07.947437+0000","last_peered":"2023-09-23T18:04:07.947437+0000","last_clean":"2023-09-23T18:04:07.947437+0000","last_became_active":"2023-09-23T18:01:25.002864+0000","last_became_peered":"2023-09-23T18:01:25.002864+0000","last_unstale":"2023-09-23T18:04:07.947437+0000","last_undegraded":"2023-09-23T18:04:07.947437+0000","last_fullsized":"2023-09-23T18:04:07.947437+0000","mapping_epoch":214,"log_start":"222'2641","ondisk_log_start":"222'2641","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":3997,"log_dups_size":0,"ondisk_log_size":3997,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.752359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":163840000,"num_objects":2500,"num_object_clones":0,"num_object_copies":7500,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2500,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6638,"num_write_kb":292416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":603,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137081+0000","last_change":"2023-09-23T18:02:51.053420+0000","last_active":"2023-09-23T18:04:01.137081+0000","last_peered":"2023-09-23T18:04:01.137081+0000","last_clean":"2023-09-23T18:04:01.137081+0000","last_became_active":"2023-09-23T18:02:51.053035+0000","last_became_peered":"2023-09-23T18:02:51.053035+0000","last_unstale":"2023-09-23T18:04:01.137081+0000","last_undegraded":"2023-09-23T18:04:01.137081+0000","last_fullsized":"2023-09-23T18:04:01.137081+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:20.456481+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":469,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.127000+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:04:01.127000+0000","last_peered":"2023-09-23T18:04:01.127000+0000","last_clean":"2023-09-23T18:04:01.127000+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:04:01.127000+0000","last_undegraded":"2023-09-23T18:04:01.127000+0000","last_fullsized":"2023-09-23T18:04:01.127000+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":291,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151336+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:04:02.151336+0000","last_peered":"2023-09-23T18:04:02.151336+0000","last_clean":"2023-09-23T18:04:02.151336+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:04:02.151336+0000","last_undegraded":"2023-09-23T18:04:02.151336+0000","last_fullsized":"2023-09-23T18:04:02.151336+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":54,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.135506+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:04:01.135506+0000","last_peered":"2023-09-23T18:04:01.135506+0000","last_clean":"2023-09-23T18:04:01.135506+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:04:01.135506+0000","last_undegraded":"2023-09-23T18:04:01.135506+0000","last_fullsized":"2023-09-23T18:04:01.135506+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":586,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.136894+0000","last_change":"2023-09-23T18:02:51.046571+0000","last_active":"2023-09-23T18:04:01.136894+0000","last_peered":"2023-09-23T18:04:01.136894+0000","last_clean":"2023-09-23T18:04:01.136894+0000","last_became_active":"2023-09-23T18:02:51.046213+0000","last_became_peered":"2023-09-23T18:02:51.046213+0000","last_unstale":"2023-09-23T18:04:01.136894+0000","last_undegraded":"2023-09-23T18:04:01.136894+0000","last_fullsized":"2023-09-23T18:04:01.136894+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":250,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.129374+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:04:01.129374+0000","last_peered":"2023-09-23T18:04:01.129374+0000","last_clean":"2023-09-23T18:04:01.129374+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:04:01.129374+0000","last_undegraded":"2023-09-23T18:04:01.129374+0000","last_fullsized":"2023-09-23T18:04:01.129374+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":276,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.127055+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:04:01.127055+0000","last_peered":"2023-09-23T18:04:01.127055+0000","last_clean":"2023-09-23T18:04:01.127055+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:04:01.127055+0000","last_undegraded":"2023-09-23T18:04:01.127055+0000","last_fullsized":"2023-09-23T18:04:01.127055+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":595,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137415+0000","last_change":"2023-09-23T18:02:51.060198+0000","last_active":"2023-09-23T18:04:01.137415+0000","last_peered":"2023-09-23T18:04:01.137415+0000","last_clean":"2023-09-23T18:04:01.137415+0000","last_became_active":"2023-09-23T18:02:51.057046+0000","last_became_peered":"2023-09-23T18:02:51.057046+0000","last_unstale":"2023-09-23T18:04:01.137415+0000","last_undegraded":"2023-09-23T18:04:01.137415+0000","last_fullsized":"2023-09-23T18:04:01.137415+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":119,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.134646+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:04:01.134646+0000","last_peered":"2023-09-23T18:04:01.134646+0000","last_clean":"2023-09-23T18:04:01.134646+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:04:01.134646+0000","last_undegraded":"2023-09-23T18:04:01.134646+0000","last_fullsized":"2023-09-23T18:04:01.134646+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":189,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.135898+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:04:01.135898+0000","last_peered":"2023-09-23T18:04:01.135898+0000","last_clean":"2023-09-23T18:04:01.135898+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:04:01.135898+0000","last_undegraded":"2023-09-23T18:04:01.135898+0000","last_fullsized":"2023-09-23T18:04:01.135898+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":105,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.134363+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:04:01.134363+0000","last_peered":"2023-09-23T18:04:01.134363+0000","last_clean":"2023-09-23T18:04:01.134363+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:04:01.134363+0000","last_undegraded":"2023-09-23T18:04:01.134363+0000","last_fullsized":"2023-09-23T18:04:01.134363+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":128,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137283+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:04:01.137283+0000","last_peered":"2023-09-23T18:04:01.137283+0000","last_clean":"2023-09-23T18:04:01.137283+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:04:01.137283+0000","last_undegraded":"2023-09-23T18:04:01.137283+0000","last_fullsized":"2023-09-23T18:04:01.137283+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":611,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137593+0000","last_change":"2023-09-23T18:02:51.047476+0000","last_active":"2023-09-23T18:04:01.137593+0000","last_peered":"2023-09-23T18:04:01.137593+0000","last_clean":"2023-09-23T18:04:01.137593+0000","last_became_active":"2023-09-23T18:02:51.047152+0000","last_became_peered":"2023-09-23T18:02:51.047152+0000","last_unstale":"2023-09-23T18:04:01.137593+0000","last_undegraded":"2023-09-23T18:04:01.137593+0000","last_fullsized":"2023-09-23T18:04:01.137593+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:22.514077+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":128,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137693+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:04:01.137693+0000","last_peered":"2023-09-23T18:04:01.137693+0000","last_clean":"2023-09-23T18:04:01.137693+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:04:01.137693+0000","last_undegraded":"2023-09-23T18:04:01.137693+0000","last_fullsized":"2023-09-23T18:04:01.137693+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":131,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137056+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:04:01.137056+0000","last_peered":"2023-09-23T18:04:01.137056+0000","last_clean":"2023-09-23T18:04:01.137056+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:04:01.137056+0000","last_undegraded":"2023-09-23T18:04:01.137056+0000","last_fullsized":"2023-09-23T18:04:01.137056+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":594,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.137110+0000","last_change":"2023-09-23T18:02:51.058825+0000","last_active":"2023-09-23T18:04:01.137110+0000","last_peered":"2023-09-23T18:04:01.137110+0000","last_clean":"2023-09-23T18:04:01.137110+0000","last_became_active":"2023-09-23T18:02:51.058480+0000","last_became_peered":"2023-09-23T18:02:51.058480+0000","last_unstale":"2023-09-23T18:04:01.137110+0000","last_undegraded":"2023-09-23T18:04:01.137110+0000","last_fullsized":"2023-09-23T18:04:01.137110+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":243,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.129713+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:04:01.129713+0000","last_peered":"2023-09-23T18:04:01.129713+0000","last_clean":"2023-09-23T18:04:01.129713+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:04:01.129713+0000","last_undegraded":"2023-09-23T18:04:01.129713+0000","last_fullsized":"2023-09-23T18:04:01.129713+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":274,"reported_epoch":263,"state":"active+clean","last_fresh":"2023-09-23T18:04:01.135501+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:04:01.135501+0000","last_peered":"2023-09-23T18:04:01.135501+0000","last_clean":"2023-09-23T18:04:01.135501+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:04:01.135501+0000","last_undegraded":"2023-09-23T18:04:01.135501+0000","last_fullsized":"2023-09-23T18:04:01.135501+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":2610561048,"num_objects":39835,"num_object_clones":0,"num_object_copies":119505,"num_objects_missing_on_primary":25,"num_objects_missing":25,"num_objects_degraded":23784,"num_objects_misplaced":10622,"num_objects_unfound":0,"num_objects_dirty":39835,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":106437,"num_write_kb":4680641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":23057,"num_bytes_recovered":1390542848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8232579072,"data_stored":8232566856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":87273,"ondisk_log_size":87273,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":402523,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":331,"num_read_kb":623,"num_write":162,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2814833,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001378,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1258592,"kb_used_data":1162200,"kb_used_omap":13,"kb_used_meta":96370,"kb_avail":92486560,"statfs":{"total":95995035648,"available":94706237440,"internally_reserved":0,"allocated":1190092800,"data_stored":1189077833,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":98682984},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":2.2160000000000002,"5min":2.544,"15min":2.544},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.226,"15min":1.266},"min":{"1min":0.32100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.5990000000000002,"5min":6.5990000000000002,"15min":6.5990000000000002},"last":0.317}]},{"osd":1,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.71599999999999997,"15min":0.68999999999999995},"min":{"1min":0.27400000000000002,"5min":0.252,"15min":0.252},"max":{"1min":3.2850000000000001,"5min":3.2850000000000001,"15min":3.2850000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.0649999999999999,"15min":1.0589999999999999},"min":{"1min":0.35199999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.2169999999999996,"5min":6.2169999999999996,"15min":6.2169999999999996},"last":0.35199999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":0.95699999999999996,"15min":0.94399999999999995},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.1539999999999999,"5min":4.1539999999999999,"15min":4.1539999999999999},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.161,"5min":0.88200000000000001,"15min":0.83599999999999997},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.1970000000000001,"5min":6.1970000000000001,"15min":6.1970000000000001},"last":0.40200000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:03:19 2023","interfaces":[{"interface":"back","average":{"1min":1.554,"5min":1.554,"15min":1.554},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":4.2670000000000003,"5min":4.2670000000000003,"15min":4.2670000000000003},"last":0.44900000000000001},{"interface":"front","average":{"1min":0.96999999999999997,"5min":0.96999999999999997,"15min":0.96999999999999997},"min":{"1min":0.39900000000000002,"5min":0.39900000000000002,"15min":0.39900000000000002},"max":{"1min":3.0950000000000002,"5min":3.0950000000000002,"15min":3.0950000000000002},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":1.014,"15min":1.052},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.4199999999999999,"5min":3.984,"15min":3.984},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.627,"5min":0.54600000000000004,"15min":0.53200000000000003},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.1040000000000001,"5min":2.1040000000000001,"15min":2.1040000000000001},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.94599999999999995,"15min":0.98399999999999999},"min":{"1min":0.33900000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.8939999999999999,"5min":3.9300000000000002,"15min":3.9300000000000002},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.87,"5min":1.0800000000000001,"15min":1.115},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.403,"5min":3.7080000000000002,"15min":3.7080000000000002},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.69799999999999995,"15min":0.70299999999999996},"min":{"1min":0.39500000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.1429999999999998,"5min":3.0409999999999999,"15min":3.0409999999999999},"last":0.439},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.046,"15min":1.099},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.8560000000000001,"5min":3.6419999999999999,"15min":3.6419999999999999},"last":0.36799999999999999}]}]},{"osd":6,"up_from":163,"seq":700079669315,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1906968,"kb_used_data":1759824,"kb_used_omap":7,"kb_used_meta":147128,"kb_avail":91838184,"statfs":{"total":95995035648,"available":94042300416,"internally_reserved":0,"allocated":1802059776,"data_stored":1801052874,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":150659876},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.274,"15min":1.3240000000000001},"min":{"1min":0.25700000000000001,"5min":0.215,"15min":0.215},"max":{"1min":15.407,"5min":15.407,"15min":15.407},"last":0.56699999999999995},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.347,"15min":2.2490000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.9710000000000001,"5min":13.536,"15min":13.536},"last":0.27900000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":1.458,"15min":1.913},"min":{"1min":0.29799999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7690000000000001,"5min":14.233000000000001,"15min":14.233000000000001},"last":0.29299999999999998},{"interface":"front","average":{"1min":1.905,"5min":1.5720000000000001,"15min":2.1379999999999999},"min":{"1min":0.32300000000000001,"5min":0.248,"15min":0.248},"max":{"1min":14.789,"5min":14.789,"15min":14.789},"last":2.0750000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.75,"15min":2.2610000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.7489999999999997,"5min":11.210000000000001,"15min":11.210000000000001},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.171,"5min":1.8149999999999999,"15min":2.633},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.7359999999999998,"5min":11.223000000000001,"15min":11.223000000000001},"last":1.0780000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":3.081,"5min":3.081,"15min":3.081},"last":0.32900000000000001},{"interface":"front","average":{"1min":1.1659999999999999,"5min":1.1659999999999999,"15min":1.1659999999999999},"min":{"1min":0.374,"5min":0.374,"15min":0.374},"max":{"1min":5.04,"5min":5.04,"15min":5.04},"last":0.46500000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.93700000000000006,"15min":0.88400000000000001},"min":{"1min":0.309,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.9930000000000003,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.78000000000000003,"15min":0.52100000000000002},"min":{"1min":0.35399999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.9430000000000001,"5min":8.3000000000000007,"15min":8.3000000000000007},"last":3.4159999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.89600000000000002,"15min":0.82799999999999996},"min":{"1min":0.34200000000000003,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.2440000000000002,"5min":11.869,"15min":11.869},"last":0.48599999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96499999999999997,"15min":0.66500000000000004},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.4130000000000003,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":0.53700000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:03:26 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":1.0209999999999999,"15min":0.98299999999999998},"min":{"1min":0.379,"5min":0.307,"15min":0.307},"max":{"1min":4.069,"5min":4.069,"15min":4.069},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.82899999999999996,"5min":0.79100000000000004,"15min":0.78500000000000003},"min":{"1min":0.39900000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":2.7970000000000002,"5min":2.7970000000000002,"15min":2.7970000000000002},"last":0.496}]}]},{"osd":1,"up_from":0,"seq":420906795133,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958660,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1042832,"kb_used_data":645192,"kb_used_omap":7,"kb_used_meta":397624,"kb_avail":92702320,"statfs":{"total":95995035648,"available":94927175680,"internally_reserved":0,"allocated":660676608,"data_stored":659329590,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":407167716},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:03:09 2023","interfaces":[{"interface":"back","average":{"1min":1.0149999999999999,"5min":2.0059999999999998,"15min":1.135},"min":{"1min":0.20300000000000001,"5min":0.18099999999999999,"15min":0.17899999999999999},"max":{"1min":7.7000000000000002,"5min":18.154,"15min":18.154},"last":0.20699999999999999},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.871,"15min":1.1080000000000001},"min":{"1min":0.29599999999999999,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":6.8550000000000004,"5min":13.369999999999999,"15min":13.369999999999999},"last":0.315}]},{"osd":2,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.2490000000000001,"15min":1.2829999999999999},"min":{"1min":0.21099999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.3529999999999998,"5min":20.882000000000001,"15min":20.882000000000001},"last":0.246},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.0830000000000002,"15min":1.234},"min":{"1min":0.22500000000000001,"5min":0.217,"15min":0.217},"max":{"1min":6.8869999999999996,"5min":15.064,"15min":15.064},"last":0.27800000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":1.5529999999999999,"15min":1.5529999999999999},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32800000000000001},"max":{"1min":6.048,"5min":6.048,"15min":6.048},"last":1.018},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.2010000000000001,"15min":1.2010000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":4.391,"5min":4.391,"15min":4.391},"last":1.0509999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.677,"15min":1.1200000000000001},"min":{"1min":0.436,"5min":0.29299999999999998,"15min":0.251},"max":{"1min":4.4809999999999999,"5min":15.279999999999999,"15min":15.279999999999999},"last":1.1819999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.7849999999999999,"15min":1.1439999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":5.3109999999999999,"5min":18.106999999999999,"15min":18.106999999999999},"last":0.32600000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.8939999999999999,"15min":1.196},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":5.3760000000000003,"5min":14.720000000000001,"15min":14.720000000000001},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.526,"15min":1.024},"min":{"1min":0.373,"5min":0.254,"15min":0.254},"max":{"1min":2.536,"5min":12.794,"15min":12.794},"last":0.495}]},{"osd":6,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.681,"15min":2.2389999999999999},"min":{"1min":0.35099999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.4660000000000002,"5min":14.956,"15min":14.956},"last":1.1479999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.6830000000000001,"15min":2.4020000000000001},"min":{"1min":0.38300000000000001,"5min":0.311,"15min":0.311},"max":{"1min":6.9189999999999996,"5min":12.755000000000001,"15min":12.755000000000001},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:03:28 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":0.93400000000000005,"15min":0.92600000000000005},"min":{"1min":0.41499999999999998,"5min":0.40799999999999997,"15min":0.40799999999999997},"max":{"1min":6.9489999999999998,"5min":6.9489999999999998,"15min":6.9489999999999998},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.6439999999999999,"5min":1.1950000000000001,"15min":1.1200000000000001},"min":{"1min":0.40999999999999998,"5min":0.39800000000000002,"15min":0.39800000000000002},"max":{"1min":4.9880000000000004,"5min":4.9880000000000004,"15min":4.9880000000000004},"last":1.2030000000000001}]}]},{"osd":2,"up_from":103,"seq":442381631607,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1799740,"kb_used_data":1444404,"kb_used_omap":11,"kb_used_meta":355316,"kb_avail":91945412,"statfs":{"total":95995035648,"available":94152101888,"internally_reserved":0,"allocated":1479069696,"data_stored":1477742521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":363843959},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":2.3130000000000002,"15min":1.2849999999999999},"min":{"1min":0.19800000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":2.694,"5min":20.867000000000001,"15min":20.867000000000001},"last":0.23599999999999999},{"interface":"front","average":{"1min":0.877,"5min":2.1819999999999999,"15min":1.2789999999999999},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":3.286,"5min":21.681000000000001,"15min":21.681000000000001},"last":0.5}]},{"osd":1,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":2.4540000000000002,"15min":1.395},"min":{"1min":0.26800000000000002,"5min":0.215,"15min":0.215},"max":{"1min":3.0960000000000001,"5min":20.995000000000001,"15min":20.995000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.88800000000000001,"5min":1.698,"15min":1.1839999999999999},"min":{"1min":0.26700000000000002,"5min":0.193,"15min":0.193},"max":{"1min":3.649,"5min":15.945,"15min":15.945},"last":0.29199999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:03:21 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":0.78200000000000003,"15min":0.78200000000000003},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.6040000000000001,"5min":3.6040000000000001,"15min":3.6040000000000001},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.0790000000000002,"5min":3.0790000000000002,"15min":3.0790000000000002},"last":0.53400000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.5960000000000001,"15min":1.111},"min":{"1min":0.32300000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.1440000000000001,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.622,"5min":1.837,"15min":1.121},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.6459999999999999,"5min":20.920000000000002,"15min":20.920000000000002},"last":0.35299999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.9339999999999999,"15min":1.198},"min":{"1min":0.35499999999999998,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":3.6960000000000002,"5min":20.738,"15min":20.738},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.145,"5min":1.6419999999999999,"15min":1.2330000000000001},"min":{"1min":0.32000000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.6280000000000001,"5min":14.359,"15min":15.496},"last":0.48399999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.752,"15min":2.1320000000000001},"min":{"1min":0.36599999999999999,"5min":0.308,"15min":0.308},"max":{"1min":3.1230000000000002,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":0.58899999999999997},{"interface":"front","average":{"1min":0.78400000000000003,"5min":1.97,"15min":2.3740000000000001},"min":{"1min":0.40799999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.2480000000000002,"5min":20.893000000000001,"15min":20.893000000000001},"last":0.54800000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:03:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.335,"15min":1.385},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":3.5600000000000001,"5min":4.3239999999999998,"15min":4.3239999999999998},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.66300000000000003,"15min":0.64300000000000002},"min":{"1min":0.40699999999999997,"5min":0.309,"15min":0.309},"max":{"1min":2.7789999999999999,"5min":2.7789999999999999,"15min":2.7789999999999999},"last":0.57499999999999996}]}]},{"osd":3,"up_from":237,"seq":1017907249175,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1207492,"kb_used_data":1107324,"kb_used_omap":3,"kb_used_meta":100156,"kb_avail":92537660,"statfs":{"total":95995035648,"available":94758563840,"internally_reserved":0,"allocated":1133899776,"data_stored":1132394248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":102560488},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.88400000000000001,"15min":0.88400000000000001},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":3.6560000000000001,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.378},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.0820000000000001,"15min":1.0820000000000001},"min":{"1min":0.221,"5min":0.221,"15min":0.221},"max":{"1min":4.5529999999999999,"5min":4.5529999999999999,"15min":4.5529999999999999},"last":0.36099999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0600000000000001,"5min":1.0600000000000001,"15min":1.0600000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":4.6050000000000004,"5min":4.6050000000000004,"15min":4.6050000000000004},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.94599999999999995,"5min":0.94599999999999995,"15min":0.94599999999999995},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.8810000000000002,"5min":4.8810000000000002,"15min":4.8810000000000002},"last":0.33700000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.85099999999999998,"15min":0.85099999999999998},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":2.7639999999999998,"5min":2.7639999999999998,"15min":2.7639999999999998},"last":0.307},{"interface":"front","average":{"1min":1.246,"5min":1.246,"15min":1.246},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":6.157,"5min":6.157,"15min":6.157},"last":0.46000000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.1040000000000001,"5min":1.1040000000000001,"15min":1.1040000000000001},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":4.633,"5min":4.633,"15min":4.633},"last":0.498},{"interface":"front","average":{"1min":1.0680000000000001,"5min":1.0680000000000001,"15min":1.0680000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":4.7300000000000004,"5min":4.7300000000000004,"15min":4.7300000000000004},"last":0.49299999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.284,"5min":1.284,"15min":1.284},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":7.0640000000000001,"5min":7.0640000000000001,"15min":7.0640000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.181,"5min":1.181,"15min":1.181},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":4.6150000000000002,"5min":4.6150000000000002,"15min":4.6150000000000002},"last":0.48599999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.085,"15min":1.085},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":4.0590000000000002,"5min":4.0590000000000002,"15min":4.0590000000000002},"last":0.42399999999999999},{"interface":"front","average":{"1min":1.48,"5min":1.48,"15min":1.48},"min":{"1min":0.379,"5min":0.379,"15min":0.379},"max":{"1min":7.3929999999999998,"5min":7.3929999999999998,"15min":7.3929999999999998},"last":0.57499999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:03:18 2023","interfaces":[{"interface":"back","average":{"1min":1.5720000000000001,"5min":1.5720000000000001,"15min":1.5720000000000001},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":8.4190000000000005,"5min":8.4190000000000005,"15min":8.4190000000000005},"last":0.47299999999999998},{"interface":"front","average":{"1min":1.6200000000000001,"5min":1.6200000000000001,"15min":1.6200000000000001},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":5.1760000000000002,"5min":5.1760000000000002,"15min":5.1760000000000002},"last":0.50800000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019125,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1154780,"kb_used_data":937300,"kb_used_omap":0,"kb_used_meta":217472,"kb_avail":92590372,"statfs":{"total":95995035648,"available":94812540928,"internally_reserved":0,"allocated":959795200,"data_stored":958784038,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":222691328},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.3220000000000001,"15min":0.82599999999999996},"min":{"1min":0.25,"5min":0.22,"15min":0.20100000000000001},"max":{"1min":2.238,"5min":14.164,"15min":14.164},"last":0.32400000000000001},{"interface":"front","average":{"1min":1.4450000000000001,"5min":1.5109999999999999,"15min":1.04},"min":{"1min":0.28899999999999998,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":7.1879999999999997,"5min":9.3789999999999996,"15min":11.372},"last":0.25700000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.359,"15min":1.004},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":2.2770000000000001,"5min":8.5259999999999998,"15min":10.429},"last":0.29099999999999998},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.2949999999999999,"15min":1.083},"min":{"1min":0.29699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.597,"5min":8.8130000000000006,"15min":10.83},"last":0.41099999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.5109999999999999,"15min":1.085},"min":{"1min":0.28899999999999998,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":2.7930000000000001,"5min":11.630000000000001,"15min":11.630000000000001},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.3580000000000001,"15min":1.016},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.27100000000000002},"max":{"1min":3.2250000000000001,"5min":7.4459999999999997,"15min":7.5090000000000003},"last":0.45000000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:03:17 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.351,"15min":1.351},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":7.2140000000000004,"5min":7.2140000000000004,"15min":7.2140000000000004},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.373,"5min":1.373,"15min":1.373},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.1970000000000001,"5min":5.1970000000000001,"15min":5.1970000000000001},"last":1.446}]},{"osd":5,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.90900000000000003,"15min":0.629},"min":{"1min":0.25900000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":9.9819999999999993,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.73299999999999998},{"interface":"front","average":{"1min":1.2350000000000001,"5min":1.111,"15min":0.76700000000000002},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.26100000000000001},"max":{"1min":11.170999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.495}]},{"osd":6,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.94899999999999995,"15min":0.82899999999999996},"min":{"1min":0.309,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.1849999999999996,"5min":7.1529999999999996,"15min":7.1529999999999996},"last":0.68400000000000005},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.0740000000000001,"15min":0.95799999999999996},"min":{"1min":0.34300000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.2590000000000003,"5min":10.725,"15min":10.725},"last":0.70999999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:03:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":0.98099999999999998,"15min":0.93600000000000005},"min":{"1min":0.35899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.3140000000000001,"5min":5.3140000000000001,"15min":5.3140000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.4550000000000001,"5min":1.2609999999999999,"15min":1.2290000000000001},"min":{"1min":0.35799999999999998,"5min":0.307,"15min":0.307},"max":{"1min":5.2770000000000001,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":0.52100000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757491,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1046928,"kb_used_data":840712,"kb_used_omap":0,"kb_used_meta":206208,"kb_avail":92698224,"statfs":{"total":95995035648,"available":94922981376,"internally_reserved":0,"allocated":860889088,"data_stored":859885678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":211156992},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.5349999999999999,"15min":1.002},"min":{"1min":0.27300000000000002,"5min":0.217,"15min":0.217},"max":{"1min":2.0649999999999999,"5min":21.376999999999999,"15min":21.376999999999999},"last":0.29499999999999998},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.8680000000000001,"15min":1.179},"min":{"1min":0.36699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.0310000000000006,"5min":21.388999999999999,"15min":21.388999999999999},"last":0.28399999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:03:07 2023","interfaces":[{"interface":"back","average":{"1min":0.501,"5min":1.054,"15min":0.85899999999999999},"min":{"1min":0.28399999999999997,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":1.321,"5min":10.853999999999999,"15min":10.853999999999999},"last":0.26000000000000001},{"interface":"front","average":{"1min":1.05,"5min":1.2969999999999999,"15min":0.95399999999999996},"min":{"1min":0.29199999999999998,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":3.6659999999999999,"5min":10.206,"15min":10.206},"last":0.34000000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":1.956,"15min":1.258},"min":{"1min":0.312,"5min":0.255,"15min":0.255},"max":{"1min":1.75,"5min":26.035,"15min":38.606999999999999},"last":0.38200000000000001},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.498,"15min":1.139},"min":{"1min":0.308,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9289999999999998,"5min":17.085000000000001,"15min":39.670999999999999},"last":0.40100000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:03:19 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":2.6179999999999999,"5min":2.6179999999999999,"15min":2.6179999999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.71199999999999997,"5min":0.71199999999999997,"15min":0.71199999999999997},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":2.7850000000000001,"5min":2.7850000000000001,"15min":2.7850000000000001},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.0089999999999999,"15min":0.72699999999999998},"min":{"1min":0.254,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":8.2390000000000008,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.98399999999999999,"15min":0.69999999999999996},"min":{"1min":0.35399999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":0.79900000000000004,"5min":5.077,"15min":5.077},"last":0.39700000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.127,"15min":1.4350000000000001},"min":{"1min":0.32600000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.3160000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.49099999999999999},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.97099999999999997,"15min":1.2709999999999999},"min":{"1min":0.35399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.333,"5min":8.266,"15min":8.266},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:03:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.405,"15min":1.4990000000000001},"min":{"1min":0.39600000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.875,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":0.433},{"interface":"front","average":{"1min":0.871,"5min":1.0740000000000001,"15min":1.1080000000000001},"min":{"1min":0.39100000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.7229999999999999,"5min":4.0380000000000003,"15min":4.0380000000000003},"last":0.439}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":650383360,"data_stored":650379288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1054539776,"data_stored":1054539776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1468731392,"data_stored":1468727320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":476446720,"data_stored":476446720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":953483264,"data_stored":953483264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":857477120,"data_stored":857473048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1785593856,"data_stored":1785593856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":985923584,"data_stored":985923584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:04:17.358 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:04:17.562 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:12 smithi055 ceph-mon[179668]: from='client.45434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:04:17.563 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:12 smithi055 ceph-mon[184174]: from='client.45434 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:04:17.564 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:15 smithi099 ceph-mon[167076]: pgmap v632: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 23447/110817 objects degraded (21.158%); 9925/110817 objects misplaced (8.956%); 5.6 MiB/s, 160 objects/s recovering 2023-09-23T18:04:17.564 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:15 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:04:17.564 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:15 smithi099 ceph-mon[167076]: failed to encode map e265 with expected crc 2023-09-23T18:04:17.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1759628881' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:04:17.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:15 smithi099 ceph-mon[167076]: osdmap e265: 8 total, 8 up, 7 in 2023-09-23T18:04:17.830 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:04:17.831 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:04:17.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:04:17.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:04:17.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:04:17.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:04:17.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:04:17.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:04:17.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:04:17.833 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:04:17.833 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:04:17.833 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:04:17.833 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:04:17.833 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:04:17.833 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:04:17.833 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:04:17.833 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:04:17.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:13 smithi055 ceph-mon[184174]: pgmap v631: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 343 op/s; 23433/116946 objects degraded (20.037%); 10169/116946 objects misplaced (8.695%); 6.0 MiB/s, 176 objects/s recovering 2023-09-23T18:04:17.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 23433/116946 objects degraded (20.037%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:17.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:13 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/796332847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:17.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:13 smithi055 ceph-mon[179668]: pgmap v631: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 1 active+recovering+undersized+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.4 GiB data, 8.9 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 343 op/s; 23433/116946 objects degraded (20.037%); 10169/116946 objects misplaced (8.695%); 6.0 MiB/s, 176 objects/s recovering 2023-09-23T18:04:17.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 23433/116946 objects degraded (20.037%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:17.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:13 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/796332847' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:17.836 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1759628881' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:04:19.443 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1759628881' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:04:19.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:19.445 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:15 smithi055 ceph-mon[184174]: pgmap v632: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 23447/110817 objects degraded (21.158%); 9925/110817 objects misplaced (8.956%); 5.6 MiB/s, 160 objects/s recovering 2023-09-23T18:04:19.445 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:15 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:04:19.445 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:15 smithi055 ceph-mon[184174]: failed to encode map e265 with expected crc 2023-09-23T18:04:19.445 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1759628881' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:04:19.446 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:15 smithi055 ceph-mon[184174]: osdmap e265: 8 total, 8 up, 7 in 2023-09-23T18:04:19.446 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:15 smithi055 ceph-mon[179668]: pgmap v632: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.3 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 442 op/s; 23447/110817 objects degraded (21.158%); 9925/110817 objects misplaced (8.956%); 5.6 MiB/s, 160 objects/s recovering 2023-09-23T18:04:19.446 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:15 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:04:19.446 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:15 smithi055 ceph-mon[179668]: failed to encode map e265 with expected crc 2023-09-23T18:04:19.446 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1759628881' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:04:19.449 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:15 smithi055 ceph-mon[179668]: osdmap e265: 8 total, 8 up, 7 in 2023-09-23T18:04:19.449 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:19.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:04:19.450 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:04:19.450 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:17 smithi055 ceph-mon[184174]: pgmap v634: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 23447/109056 objects degraded (21.500%); 9339/109056 objects misplaced (8.563%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T18:04:19.720 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:17 smithi099 ceph-mon[167076]: pgmap v634: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 23447/109056 objects degraded (21.500%); 9339/109056 objects misplaced (8.563%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T18:04:19.720 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 23447/109056 objects degraded (21.500%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:17 smithi055 ceph-mon[179668]: pgmap v634: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.2 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 504 op/s; 23447/109056 objects degraded (21.500%); 9339/109056 objects misplaced (8.563%); 6.1 MiB/s, 98 objects/s recovering 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 23447/109056 objects degraded (21.500%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:19.721 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 23447/109056 objects degraded (21.500%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:19.722 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:04:19.836 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:04:19.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:19.993 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:19.993 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:04:19.994 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:04:20.094 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:04:20.257 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:19 smithi055 ceph-mon[179668]: pgmap v635: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 22981/100830 objects degraded (22.792%); 8635/100830 objects misplaced (8.564%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:04:20.257 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:19 smithi055 ceph-mon[184174]: pgmap v635: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 22981/100830 objects degraded (22.792%); 8635/100830 objects misplaced (8.564%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:04:20.302 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:19 smithi099 ceph-mon[167076]: pgmap v635: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.1 GiB data, 8.0 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 686 op/s; 22981/100830 objects degraded (22.792%); 8635/100830 objects misplaced (8.564%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:04:20.535 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:20.536 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:04:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:04:20] "GET /metrics HTTP/1.1" 200 36542 "" "Prometheus/2.43.0" 2023-09-23T18:04:20.536 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:20.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:20.537 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:20.537 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:04:20.537 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:04:20.859 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:04:21.250 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:21.366 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:21.446 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:04:21.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:21.621 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:04:21.622 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:04:21.723 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:04:21.997 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:22.268 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:22 smithi055 ceph-mon[179668]: pgmap v636: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 22981/95787 objects degraded (23.992%); 8055/95787 objects misplaced (8.409%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:04:22.268 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:22.268 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:22 smithi055 ceph-mon[184174]: pgmap v636: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 22981/95787 objects degraded (23.992%); 8055/95787 objects misplaced (8.409%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:04:22.268 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:22.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:22 smithi099 ceph-mon[167076]: pgmap v636: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 2.0 GiB data, 7.7 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 792 op/s; 22981/95787 objects degraded (23.992%); 8055/95787 objects misplaced (8.409%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:04:22.623 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:22.714 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:04:22.982 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 22981/95787 objects degraded (23.992%), 9 pgs degraded, 8 pgs undersized","count":17},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":752,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":265,"num_osds":8,"num_up_osds":8,"osd_up_since":1695492137,"num_in_osds":7,"osd_in_since":1695492120,"num_remapped_pgs":9},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+recovering+undersized+degraded+remapped","count":2},{"state_name":"active+undersized+degraded+remapped+backfill_wait","count":2},{"state_name":"active+recovery_wait+degraded+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":31929,"data_bytes":2095266310,"bytes_used":8302051328,"bytes_avail":663663198208,"bytes_total":671965249536,"degraded_objects":22981,"degraded_total":95787,"degraded_ratio":0.23991773413928821,"misplaced_objects":8055,"misplaced_total":95787,"misplaced_ratio":0.08409283096871184,"recovering_objects_per_sec":94,"recovering_bytes_per_sec":6163392,"recovering_keys_per_sec":0,"num_objects_recovered":941,"num_bytes_recovered":61669376,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":792},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":23,"modified":"2023-09-23T18:03:16.678968+0000","services":{}},"progress_events":{"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"e54c9148-1053-4ff0-9024-eb4758d365b5":{"message":"Global Recovery Event (8m)\n [=========================...] (remaining: 40s)","progress":0.92436975240707397,"add_to_ceph_s":true}}} 2023-09-23T18:04:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:23 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/779559528' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:04:23.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 22981/95787 objects degraded (23.992%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:23.799 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:23 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/779559528' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:04:23.799 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 22981/95787 objects degraded (23.992%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:23.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:23.800 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:04:23.801 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:04:23.801 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:23 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/779559528' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:04:23.801 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 22981/95787 objects degraded (23.992%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:23.901 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:04:24.317 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:24 smithi055 ceph-mon[179668]: pgmap v637: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 22424/91797 objects degraded (24.428%); 7583/91797 objects misplaced (8.261%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T18:04:24.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:24 smithi099 ceph-mon[167076]: pgmap v637: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 22424/91797 objects degraded (24.428%); 7583/91797 objects misplaced (8.261%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T18:04:24.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:24 smithi055 ceph-mon[184174]: pgmap v637: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.9 GiB data, 7.6 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 22424/91797 objects degraded (24.428%); 7583/91797 objects misplaced (8.261%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T18:04:24.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:24.620 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:24.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:04:24.620 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:04:24.721 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:04:25.141 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:26.333 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:04:26.682 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:04:26.682 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:04:26.682 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:04:26.682 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:04:26.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:26 smithi099 ceph-mon[167076]: pgmap v638: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 22431/84936 objects degraded (26.409%); 7243/84936 objects misplaced (8.528%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:04:26.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:26 smithi055 ceph-mon[184174]: pgmap v638: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 22431/84936 objects degraded (26.409%); 7243/84936 objects misplaced (8.528%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:04:26.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:26 smithi055 ceph-mon[179668]: pgmap v638: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.2 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 862 op/s; 22431/84936 objects degraded (26.409%); 7243/84936 objects misplaced (8.528%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:04:26.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:26.685 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:04:27.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:04:27.060 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:04:27.060 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:04:27.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:04:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:28 smithi055 ceph-mon[179668]: pgmap v639: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 22431/83487 objects degraded (26.868%); 6793/83487 objects misplaced (8.137%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T18:04:28.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 22431/83487 objects degraded (26.868%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:28.835 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:28 smithi055 ceph-mon[184174]: pgmap v639: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 22431/83487 objects degraded (26.868%); 6793/83487 objects misplaced (8.137%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T18:04:28.835 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 22431/83487 objects degraded (26.868%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:28.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:28 smithi099 ceph-mon[167076]: pgmap v639: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.7 GiB data, 7.0 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 765 op/s; 22431/83487 objects degraded (26.868%); 6793/83487 objects misplaced (8.137%); 5.0 MiB/s, 79 objects/s recovering 2023-09-23T18:04:28.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 22431/83487 objects degraded (26.868%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:28.836 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:28.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:04:29.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:29.568 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:04:29.569 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:04:29.569 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:04:29.569 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:29.569 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:04:29.570 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:29.570 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:04:29.571 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:04:29.571 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:04:29.614 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:04:29.902 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:04:29.903 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:04:29.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:04:30.005 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:30.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:30 smithi055 ceph-mon[179668]: pgmap v640: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 937 op/s; 21928/75291 objects degraded (29.124%); 6033/75291 objects misplaced (8.013%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T18:04:30.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1193620884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:30.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:30 smithi055 ceph-mon[184174]: pgmap v640: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 937 op/s; 21928/75291 objects degraded (29.124%); 6033/75291 objects misplaced (8.013%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T18:04:30.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1193620884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:30.270 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:04:30.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:30.576 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:04:30.576 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:04:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:04:30] "GET /metrics HTTP/1.1" 200 36529 "" "Prometheus/2.43.0" 2023-09-23T18:04:30.637 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:30 smithi099 ceph-mon[167076]: pgmap v640: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.5 GiB data, 6.7 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 937 op/s; 21928/75291 objects degraded (29.124%); 6033/75291 objects misplaced (8.013%); 7.5 MiB/s, 119 objects/s recovering 2023-09-23T18:04:31.876 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1193620884' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:31.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:04:31.877 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:32.081 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:04:32.305 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:04:32.306 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:32 smithi055 ceph-mon[179668]: pgmap v641: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 21928/70665 objects degraded (31.031%); 5543/70665 objects misplaced (7.844%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T18:04:32.306 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:32 smithi055 ceph-mon[184174]: pgmap v641: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 21928/70665 objects degraded (31.031%); 5543/70665 objects misplaced (7.844%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T18:04:32.579 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:04:32.643 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:32 smithi099 ceph-mon[167076]: pgmap v641: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.4 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 837 op/s; 21928/70665 objects degraded (31.031%); 5543/70665 objects misplaced (7.844%); 5.3 MiB/s, 84 objects/s recovering 2023-09-23T18:04:34.279 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:34.279 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:33 smithi099 ceph-mon[167076]: from='client.35655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:04:34.280 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:33 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1460264654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:04:34.280 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 21928/70665 objects degraded (31.031%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:34.280 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":641,"stamp":"2023-09-23T18:04:30.708247+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1546467846,"num_objects":23555,"num_object_clones":0,"num_object_copies":70665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21928,"num_objects_misplaced":5543,"num_objects_unfound":0,"num_objects_dirty":23555,"num_whiteouts":0,"num_read":6284,"num_read_kb":12770,"num_write":123813,"num_write_kb":4695012,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25109,"num_bytes_recovered":1521159143,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":99502,"ondisk_log_size":99502,"up":357,"acting":349,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":404,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":6716368,"kb_used_data":5132696,"kb_used_omap":43,"kb_used_meta":1583444,"kb_avail":649499696,"statfs":{"total":671965249536,"available":665087688704,"internally_reserved":0,"allocated":5255880704,"data_stored":5247628782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1621447375},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,6,1,5,3,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-658964480,"num_objects":-10055,"num_object_clones":0,"num_object_copies":-30165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1053,"num_objects_misplaced":-3092,"num_objects_unfound":0,"num_objects_dirty":-10055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10055,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1009,"num_bytes_recovered":66125824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.004566"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":590,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148581+0000","last_change":"2023-09-23T18:03:33.004903+0000","last_active":"2023-09-23T18:04:02.148581+0000","last_peered":"2023-09-23T18:04:02.148581+0000","last_clean":"2023-09-23T18:04:02.148581+0000","last_became_active":"2023-09-23T18:03:33.004498+0000","last_became_peered":"2023-09-23T18:03:33.004498+0000","last_unstale":"2023-09-23T18:04:02.148581+0000","last_undegraded":"2023-09-23T18:04:02.148581+0000","last_fullsized":"2023-09-23T18:04:02.148581+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":737225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":65,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.153715+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:04:02.153715+0000","last_peered":"2023-09-23T18:04:02.153715+0000","last_clean":"2023-09-23T18:04:02.153715+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:04:02.153715+0000","last_undegraded":"2023-09-23T18:04:02.153715+0000","last_fullsized":"2023-09-23T18:04:02.153715+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":146,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.143196+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:04:02.143196+0000","last_peered":"2023-09-23T18:04:02.143196+0000","last_clean":"2023-09-23T18:04:02.143196+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:04:02.143196+0000","last_undegraded":"2023-09-23T18:04:02.143196+0000","last_fullsized":"2023-09-23T18:04:02.143196+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":607,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148830+0000","last_change":"2023-09-23T18:02:51.046916+0000","last_active":"2023-09-23T18:04:02.148830+0000","last_peered":"2023-09-23T18:04:02.148830+0000","last_clean":"2023-09-23T18:04:02.148830+0000","last_became_active":"2023-09-23T18:02:51.046591+0000","last_became_peered":"2023-09-23T18:02:51.046591+0000","last_unstale":"2023-09-23T18:04:02.148830+0000","last_undegraded":"2023-09-23T18:04:02.148830+0000","last_fullsized":"2023-09-23T18:04:02.148830+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:19.805099+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":114,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.142311+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:04:02.142311+0000","last_peered":"2023-09-23T18:04:02.142311+0000","last_clean":"2023-09-23T18:04:02.142311+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:04:02.142311+0000","last_undegraded":"2023-09-23T18:04:02.142311+0000","last_fullsized":"2023-09-23T18:04:02.142311+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":266,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151405+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:04:02.151405+0000","last_peered":"2023-09-23T18:04:02.151405+0000","last_clean":"2023-09-23T18:04:02.151405+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:04:02.151405+0000","last_undegraded":"2023-09-23T18:04:02.151405+0000","last_fullsized":"2023-09-23T18:04:02.151405+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":599,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148868+0000","last_change":"2023-09-23T18:02:51.051617+0000","last_active":"2023-09-23T18:04:02.148868+0000","last_peered":"2023-09-23T18:04:02.148868+0000","last_clean":"2023-09-23T18:04:02.148868+0000","last_became_active":"2023-09-23T18:02:51.048947+0000","last_became_peered":"2023-09-23T18:02:51.048947+0000","last_unstale":"2023-09-23T18:04:02.148868+0000","last_undegraded":"2023-09-23T18:04:02.148868+0000","last_fullsized":"2023-09-23T18:04:02.148868+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:49:34.962249+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":127,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.141965+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:04:02.141965+0000","last_peered":"2023-09-23T18:04:02.141965+0000","last_clean":"2023-09-23T18:04:02.141965+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:04:02.141965+0000","last_undegraded":"2023-09-23T18:04:02.141965+0000","last_fullsized":"2023-09-23T18:04:02.141965+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":205,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.146797+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:04:02.146797+0000","last_peered":"2023-09-23T18:04:02.146797+0000","last_clean":"2023-09-23T18:04:02.146797+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:04:02.146797+0000","last_undegraded":"2023-09-23T18:04:02.146797+0000","last_fullsized":"2023-09-23T18:04:02.146797+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":596,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149626+0000","last_change":"2023-09-23T18:02:51.052123+0000","last_active":"2023-09-23T18:04:02.149626+0000","last_peered":"2023-09-23T18:04:02.149626+0000","last_clean":"2023-09-23T18:04:02.149626+0000","last_became_active":"2023-09-23T18:02:51.051772+0000","last_became_peered":"2023-09-23T18:02:51.051772+0000","last_unstale":"2023-09-23T18:04:02.149626+0000","last_undegraded":"2023-09-23T18:04:02.149626+0000","last_fullsized":"2023-09-23T18:04:02.149626+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":245,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151614+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:04:02.151614+0000","last_peered":"2023-09-23T18:04:02.151614+0000","last_clean":"2023-09-23T18:04:02.151614+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:04:02.151614+0000","last_undegraded":"2023-09-23T18:04:02.151614+0000","last_fullsized":"2023-09-23T18:04:02.151614+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":121,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.141981+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:04:02.141981+0000","last_peered":"2023-09-23T18:04:02.141981+0000","last_clean":"2023-09-23T18:04:02.141981+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:04:02.141981+0000","last_undegraded":"2023-09-23T18:04:02.141981+0000","last_fullsized":"2023-09-23T18:04:02.141981+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":591,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148762+0000","last_change":"2023-09-23T18:02:51.054208+0000","last_active":"2023-09-23T18:04:02.148762+0000","last_peered":"2023-09-23T18:04:02.148762+0000","last_clean":"2023-09-23T18:04:02.148762+0000","last_became_active":"2023-09-23T18:02:51.053676+0000","last_became_peered":"2023-09-23T18:02:51.053676+0000","last_unstale":"2023-09-23T18:04:02.148762+0000","last_undegraded":"2023-09-23T18:04:02.148762+0000","last_fullsized":"2023-09-23T18:04:02.148762+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":247,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.158793+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:04:02.158793+0000","last_peered":"2023-09-23T18:04:02.158793+0000","last_clean":"2023-09-23T18:04:02.158793+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:04:02.158793+0000","last_undegraded":"2023-09-23T18:04:02.158793+0000","last_fullsized":"2023-09-23T18:04:02.158793+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":75,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.142167+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:04:02.142167+0000","last_peered":"2023-09-23T18:04:02.142167+0000","last_clean":"2023-09-23T18:04:02.142167+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:04:02.142167+0000","last_undegraded":"2023-09-23T18:04:02.142167+0000","last_fullsized":"2023-09-23T18:04:02.142167+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":596,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149503+0000","last_change":"2023-09-23T18:02:51.058624+0000","last_active":"2023-09-23T18:04:02.149503+0000","last_peered":"2023-09-23T18:04:02.149503+0000","last_clean":"2023-09-23T18:04:02.149503+0000","last_became_active":"2023-09-23T18:02:51.057352+0000","last_became_peered":"2023-09-23T18:02:51.057352+0000","last_unstale":"2023-09-23T18:04:02.149503+0000","last_undegraded":"2023-09-23T18:04:02.149503+0000","last_fullsized":"2023-09-23T18:04:02.149503+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":31,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149527+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:04:02.149527+0000","last_peered":"2023-09-23T18:04:02.149527+0000","last_clean":"2023-09-23T18:04:02.149527+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:04:02.149527+0000","last_undegraded":"2023-09-23T18:04:02.149527+0000","last_fullsized":"2023-09-23T18:04:02.149527+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":161,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.147080+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:04:02.147080+0000","last_peered":"2023-09-23T18:04:02.147080+0000","last_clean":"2023-09-23T18:04:02.147080+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:04:02.147080+0000","last_undegraded":"2023-09-23T18:04:02.147080+0000","last_fullsized":"2023-09-23T18:04:02.147080+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":596,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149004+0000","last_change":"2023-09-23T18:02:51.059983+0000","last_active":"2023-09-23T18:04:02.149004+0000","last_peered":"2023-09-23T18:04:02.149004+0000","last_clean":"2023-09-23T18:04:02.149004+0000","last_became_active":"2023-09-23T18:02:51.057768+0000","last_became_peered":"2023-09-23T18:02:51.057768+0000","last_unstale":"2023-09-23T18:04:02.149004+0000","last_undegraded":"2023-09-23T18:04:02.149004+0000","last_fullsized":"2023-09-23T18:04:02.149004+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":65,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.152705+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:04:02.152705+0000","last_peered":"2023-09-23T18:04:02.152705+0000","last_clean":"2023-09-23T18:04:02.152705+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:04:02.152705+0000","last_undegraded":"2023-09-23T18:04:02.152705+0000","last_fullsized":"2023-09-23T18:04:02.152705+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":65,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.152253+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:04:02.152253+0000","last_peered":"2023-09-23T18:04:02.152253+0000","last_clean":"2023-09-23T18:04:02.152253+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:04:02.152253+0000","last_undegraded":"2023-09-23T18:04:02.152253+0000","last_fullsized":"2023-09-23T18:04:02.152253+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":586,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149784+0000","last_change":"2023-09-23T18:02:51.047570+0000","last_active":"2023-09-23T18:04:02.149784+0000","last_peered":"2023-09-23T18:04:02.149784+0000","last_clean":"2023-09-23T18:04:02.149784+0000","last_became_active":"2023-09-23T18:02:51.046963+0000","last_became_peered":"2023-09-23T18:02:51.046963+0000","last_unstale":"2023-09-23T18:04:02.149784+0000","last_undegraded":"2023-09-23T18:04:02.149784+0000","last_fullsized":"2023-09-23T18:04:02.149784+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":194,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149147+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:04:02.149147+0000","last_peered":"2023-09-23T18:04:02.149147+0000","last_clean":"2023-09-23T18:04:02.149147+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:04:02.149147+0000","last_undegraded":"2023-09-23T18:04:02.149147+0000","last_fullsized":"2023-09-23T18:04:02.149147+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":109,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.143302+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:04:02.143302+0000","last_peered":"2023-09-23T18:04:02.143302+0000","last_clean":"2023-09-23T18:04:02.143302+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:04:02.143302+0000","last_undegraded":"2023-09-23T18:04:02.143302+0000","last_fullsized":"2023-09-23T18:04:02.143302+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":599,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148472+0000","last_change":"2023-09-23T18:02:51.053806+0000","last_active":"2023-09-23T18:04:02.148472+0000","last_peered":"2023-09-23T18:04:02.148472+0000","last_clean":"2023-09-23T18:04:02.148472+0000","last_became_active":"2023-09-23T18:02:51.048822+0000","last_became_peered":"2023-09-23T18:02:51.048822+0000","last_unstale":"2023-09-23T18:04:02.148472+0000","last_undegraded":"2023-09-23T18:04:02.148472+0000","last_fullsized":"2023-09-23T18:04:02.148472+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:34.401104+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":264,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151081+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:04:02.151081+0000","last_peered":"2023-09-23T18:04:02.151081+0000","last_clean":"2023-09-23T18:04:02.151081+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:04:02.151081+0000","last_undegraded":"2023-09-23T18:04:02.151081+0000","last_fullsized":"2023-09-23T18:04:02.151081+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":242,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.158519+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:04:02.158519+0000","last_peered":"2023-09-23T18:04:02.158519+0000","last_clean":"2023-09-23T18:04:02.158519+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:04:02.158519+0000","last_undegraded":"2023-09-23T18:04:02.158519+0000","last_fullsized":"2023-09-23T18:04:02.158519+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":587,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148993+0000","last_change":"2023-09-23T18:02:51.046646+0000","last_active":"2023-09-23T18:04:02.148993+0000","last_peered":"2023-09-23T18:04:02.148993+0000","last_clean":"2023-09-23T18:04:02.148993+0000","last_became_active":"2023-09-23T18:02:51.046134+0000","last_became_peered":"2023-09-23T18:02:51.046134+0000","last_unstale":"2023-09-23T18:04:02.148993+0000","last_undegraded":"2023-09-23T18:04:02.148993+0000","last_fullsized":"2023-09-23T18:04:02.148993+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":55,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151682+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:04:02.151682+0000","last_peered":"2023-09-23T18:04:02.151682+0000","last_clean":"2023-09-23T18:04:02.151682+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:04:02.151682+0000","last_undegraded":"2023-09-23T18:04:02.151682+0000","last_fullsized":"2023-09-23T18:04:02.151682+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":255,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151470+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:04:02.151470+0000","last_peered":"2023-09-23T18:04:02.151470+0000","last_clean":"2023-09-23T18:04:02.151470+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:04:02.151470+0000","last_undegraded":"2023-09-23T18:04:02.151470+0000","last_fullsized":"2023-09-23T18:04:02.151470+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":599,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148939+0000","last_change":"2023-09-23T18:02:51.051817+0000","last_active":"2023-09-23T18:04:02.148939+0000","last_peered":"2023-09-23T18:04:02.148939+0000","last_clean":"2023-09-23T18:04:02.148939+0000","last_became_active":"2023-09-23T18:02:51.048945+0000","last_became_peered":"2023-09-23T18:02:51.048945+0000","last_unstale":"2023-09-23T18:04:02.148939+0000","last_undegraded":"2023-09-23T18:04:02.148939+0000","last_fullsized":"2023-09-23T18:04:02.148939+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:47.163864+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":201,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.145549+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:04:02.145549+0000","last_peered":"2023-09-23T18:04:02.145549+0000","last_clean":"2023-09-23T18:04:02.145549+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:04:02.145549+0000","last_undegraded":"2023-09-23T18:04:02.145549+0000","last_fullsized":"2023-09-23T18:04:02.145549+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":118,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.143653+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:04:02.143653+0000","last_peered":"2023-09-23T18:04:02.143653+0000","last_clean":"2023-09-23T18:04:02.143653+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:04:02.143653+0000","last_undegraded":"2023-09-23T18:04:02.143653+0000","last_fullsized":"2023-09-23T18:04:02.143653+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"265'7667","reported_seq":7735,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:27.566737+0000","last_change":"2023-09-23T18:00:57.996982+0000","last_active":"2023-09-23T18:04:27.566737+0000","last_peered":"2023-09-23T18:04:27.566737+0000","last_clean":"2023-09-23T18:04:27.566737+0000","last_became_active":"2023-09-23T18:00:57.996589+0000","last_became_peered":"2023-09-23T18:00:57.996589+0000","last_unstale":"2023-09-23T18:04:27.566737+0000","last_undegraded":"2023-09-23T18:04:27.566737+0000","last_fullsized":"2023-09-23T18:04:27.566737+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7667,"log_dups_size":0,"ondisk_log_size":7667,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:13.306089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103219200,"num_objects":1575,"num_object_clones":0,"num_object_copies":4725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7667,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":596,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149432+0000","last_change":"2023-09-23T18:02:51.058477+0000","last_active":"2023-09-23T18:04:02.149432+0000","last_peered":"2023-09-23T18:04:02.149432+0000","last_clean":"2023-09-23T18:04:02.149432+0000","last_became_active":"2023-09-23T18:02:51.057258+0000","last_became_peered":"2023-09-23T18:02:51.057258+0000","last_unstale":"2023-09-23T18:04:02.149432+0000","last_undegraded":"2023-09-23T18:04:02.149432+0000","last_fullsized":"2023-09-23T18:04:02.149432+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":194,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149652+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:04:02.149652+0000","last_peered":"2023-09-23T18:04:02.149652+0000","last_clean":"2023-09-23T18:04:02.149652+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:04:02.149652+0000","last_undegraded":"2023-09-23T18:04:02.149652+0000","last_fullsized":"2023-09-23T18:04:02.149652+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":253,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.157424+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:04:02.157424+0000","last_peered":"2023-09-23T18:04:02.157424+0000","last_clean":"2023-09-23T18:04:02.157424+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:04:02.157424+0000","last_undegraded":"2023-09-23T18:04:02.157424+0000","last_fullsized":"2023-09-23T18:04:02.157424+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"265'7684","reported_seq":14912,"reported_epoch":265,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:04:30.484850+0000","last_change":"2023-09-23T18:02:49.039290+0000","last_active":"2023-09-23T18:04:30.484850+0000","last_peered":"2023-09-23T18:04:30.484850+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:49.001792+0000","last_became_peered":"2023-09-23T18:02:49.001792+0000","last_unstale":"2023-09-23T18:04:30.484850+0000","last_undegraded":"2023-09-23T18:02:48.968288+0000","last_fullsized":"2023-09-23T18:04:30.484850+0000","mapping_epoch":250,"log_start":"224'3530","ondisk_log_start":"224'3530","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4154,"log_dups_size":0,"ondisk_log_size":4154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.156979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91750400,"num_objects":1400,"num_object_clones":0,"num_object_copies":4200,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":409,"num_objects_misplaced":2391,"num_objects_unfound":0,"num_objects_dirty":1400,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7684,"num_write_kb":290688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3566,"num_bytes_recovered":233701376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[6,1,4],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,4,6","objects":991},{"shards":"1,6","objects":409}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":595,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.162665+0000","last_change":"2023-09-23T18:04:02.152730+0000","last_active":"2023-09-23T18:04:02.162665+0000","last_peered":"2023-09-23T18:04:02.162665+0000","last_clean":"2023-09-23T18:04:02.162665+0000","last_became_active":"2023-09-23T18:04:02.152366+0000","last_became_peered":"2023-09-23T18:04:02.152366+0000","last_unstale":"2023-09-23T18:04:02.162665+0000","last_undegraded":"2023-09-23T18:04:02.162665+0000","last_fullsized":"2023-09-23T18:04:02.162665+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":736808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":209,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149601+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:04:02.149601+0000","last_peered":"2023-09-23T18:04:02.149601+0000","last_clean":"2023-09-23T18:04:02.149601+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:04:02.149601+0000","last_undegraded":"2023-09-23T18:04:02.149601+0000","last_fullsized":"2023-09-23T18:04:02.149601+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":275,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151946+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:04:02.151946+0000","last_peered":"2023-09-23T18:04:02.151946+0000","last_clean":"2023-09-23T18:04:02.151946+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:04:02.151946+0000","last_undegraded":"2023-09-23T18:04:02.151946+0000","last_fullsized":"2023-09-23T18:04:02.151946+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"265'7789","reported_seq":16397,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:29.343373+0000","last_change":"2023-09-23T18:03:33.017384+0000","last_active":"2023-09-23T18:04:29.343373+0000","last_peered":"2023-09-23T18:04:29.343373+0000","last_clean":"2023-09-23T18:04:29.343373+0000","last_became_active":"2023-09-23T18:03:33.004338+0000","last_became_peered":"2023-09-23T18:03:33.004338+0000","last_unstale":"2023-09-23T18:04:29.343373+0000","last_undegraded":"2023-09-23T18:04:29.343373+0000","last_fullsized":"2023-09-23T18:04:29.343373+0000","mapping_epoch":259,"log_start":"223'3100","ondisk_log_start":"223'3100","created":204,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4689,"log_dups_size":0,"ondisk_log_size":4689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:59.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95223808,"num_objects":1453,"num_object_clones":0,"num_object_copies":4359,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1453,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7789,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4258,"num_bytes_recovered":259457024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":587,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149184+0000","last_change":"2023-09-23T18:02:51.058795+0000","last_active":"2023-09-23T18:04:02.149184+0000","last_peered":"2023-09-23T18:04:02.149184+0000","last_clean":"2023-09-23T18:04:02.149184+0000","last_became_active":"2023-09-23T18:02:51.057799+0000","last_became_peered":"2023-09-23T18:02:51.057799+0000","last_unstale":"2023-09-23T18:04:02.149184+0000","last_undegraded":"2023-09-23T18:04:02.149184+0000","last_fullsized":"2023-09-23T18:04:02.149184+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":132,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149866+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:04:02.149866+0000","last_peered":"2023-09-23T18:04:02.149866+0000","last_clean":"2023-09-23T18:04:02.149866+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:04:02.149866+0000","last_undegraded":"2023-09-23T18:04:02.149866+0000","last_fullsized":"2023-09-23T18:04:02.149866+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":230,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.146943+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:04:02.146943+0000","last_peered":"2023-09-23T18:04:02.146943+0000","last_clean":"2023-09-23T18:04:02.146943+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:04:02.146943+0000","last_undegraded":"2023-09-23T18:04:02.146943+0000","last_fullsized":"2023-09-23T18:04:02.146943+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"265'7888","reported_seq":7976,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:29.576413+0000","last_change":"2023-09-23T18:01:25.019784+0000","last_active":"2023-09-23T18:04:29.576413+0000","last_peered":"2023-09-23T18:04:29.576413+0000","last_clean":"2023-09-23T18:04:29.576413+0000","last_became_active":"2023-09-23T18:01:25.002073+0000","last_became_peered":"2023-09-23T18:01:25.002073+0000","last_unstale":"2023-09-23T18:04:29.576413+0000","last_undegraded":"2023-09-23T18:04:29.576413+0000","last_fullsized":"2023-09-23T18:04:29.576413+0000","mapping_epoch":214,"log_start":"224'3500","ondisk_log_start":"224'3500","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4388,"log_dups_size":0,"ondisk_log_size":4388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:58.981672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92667904,"num_objects":1414,"num_object_clones":0,"num_object_copies":4242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1414,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7888,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":596,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149150+0000","last_change":"2023-09-23T18:02:51.057506+0000","last_active":"2023-09-23T18:04:02.149150+0000","last_peered":"2023-09-23T18:04:02.149150+0000","last_clean":"2023-09-23T18:04:02.149150+0000","last_became_active":"2023-09-23T18:02:51.056589+0000","last_became_peered":"2023-09-23T18:02:51.056589+0000","last_unstale":"2023-09-23T18:04:02.149150+0000","last_undegraded":"2023-09-23T18:04:02.149150+0000","last_fullsized":"2023-09-23T18:04:02.149150+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":241,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.158704+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:04:02.158704+0000","last_peered":"2023-09-23T18:04:02.158704+0000","last_clean":"2023-09-23T18:04:02.158704+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:04:02.158704+0000","last_undegraded":"2023-09-23T18:04:02.158704+0000","last_fullsized":"2023-09-23T18:04:02.158704+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":267,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151129+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:04:02.151129+0000","last_peered":"2023-09-23T18:04:02.151129+0000","last_clean":"2023-09-23T18:04:02.151129+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:04:02.151129+0000","last_undegraded":"2023-09-23T18:04:02.151129+0000","last_fullsized":"2023-09-23T18:04:02.151129+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"265'7831","reported_seq":14393,"reported_epoch":265,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:28.221812+0000","last_change":"2023-09-23T18:04:09.659459+0000","last_active":"2023-09-23T18:04:28.221812+0000","last_peered":"2023-09-23T18:04:28.221812+0000","last_clean":"2023-09-23T18:01:38.416095+0000","last_became_active":"2023-09-23T18:02:07.547135+0000","last_became_peered":"2023-09-23T18:02:07.547135+0000","last_unstale":"2023-09-23T18:04:28.221812+0000","last_undegraded":"2023-09-23T18:04:09.654315+0000","last_fullsized":"2023-09-23T18:02:02.784733+0000","mapping_epoch":227,"log_start":"217'800","ondisk_log_start":"217'800","created":204,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7031,"log_dups_size":0,"ondisk_log_size":7031,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:27.864301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93257728,"num_objects":1423,"num_object_clones":0,"num_object_copies":4269,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":60,"num_objects_misplaced":800,"num_objects_unfound":0,"num_objects_dirty":1423,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7831,"num_write_kb":296128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3240,"num_bytes_recovered":212336640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2],"avail_no_missing":["4","1","2"],"object_location_counts":[{"shards":"1,2,4","objects":800},{"shards":"2,4","objects":563}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":588,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149733+0000","last_change":"2023-09-23T18:02:51.047609+0000","last_active":"2023-09-23T18:04:02.149733+0000","last_peered":"2023-09-23T18:04:02.149733+0000","last_clean":"2023-09-23T18:04:02.149733+0000","last_became_active":"2023-09-23T18:02:51.047209+0000","last_became_peered":"2023-09-23T18:02:51.047209+0000","last_unstale":"2023-09-23T18:04:02.149733+0000","last_undegraded":"2023-09-23T18:04:02.149733+0000","last_fullsized":"2023-09-23T18:04:02.149733+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":152,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.142287+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:04:02.142287+0000","last_peered":"2023-09-23T18:04:02.142287+0000","last_clean":"2023-09-23T18:04:02.142287+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:04:02.142287+0000","last_undegraded":"2023-09-23T18:04:02.142287+0000","last_fullsized":"2023-09-23T18:04:02.142287+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":273,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.152780+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:04:02.152780+0000","last_peered":"2023-09-23T18:04:02.152780+0000","last_clean":"2023-09-23T18:04:02.152780+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:04:02.152780+0000","last_undegraded":"2023-09-23T18:04:02.152780+0000","last_fullsized":"2023-09-23T18:04:02.152780+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"265'7599","reported_seq":7826,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:29.303997+0000","last_change":"2023-09-23T18:02:47.961575+0000","last_active":"2023-09-23T18:04:29.303997+0000","last_peered":"2023-09-23T18:04:29.303997+0000","last_clean":"2023-09-23T18:02:17.013863+0000","last_became_active":"2023-09-23T18:02:19.267215+0000","last_became_peered":"2023-09-23T18:02:19.267215+0000","last_unstale":"2023-09-23T18:04:29.303997+0000","last_undegraded":"2023-09-23T18:02:18.955030+0000","last_fullsized":"2023-09-23T18:02:18.924656+0000","mapping_epoch":238,"log_start":"222'2900","ondisk_log_start":"222'2900","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4699,"log_dups_size":0,"ondisk_log_size":4699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:46.393021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95485952,"num_objects":1457,"num_object_clones":0,"num_object_copies":4371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7599,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":4128768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-2651},{"shards":"2,4,6","objects":1099}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":592,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148475+0000","last_change":"2023-09-23T18:02:51.049464+0000","last_active":"2023-09-23T18:04:02.148475+0000","last_peered":"2023-09-23T18:04:02.148475+0000","last_clean":"2023-09-23T18:04:02.148475+0000","last_became_active":"2023-09-23T18:02:51.048477+0000","last_became_peered":"2023-09-23T18:02:51.048477+0000","last_unstale":"2023-09-23T18:04:02.148475+0000","last_undegraded":"2023-09-23T18:04:02.148475+0000","last_fullsized":"2023-09-23T18:04:02.148475+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":224,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.147159+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:04:02.147159+0000","last_peered":"2023-09-23T18:04:02.147159+0000","last_clean":"2023-09-23T18:04:02.147159+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:04:02.147159+0000","last_undegraded":"2023-09-23T18:04:02.147159+0000","last_fullsized":"2023-09-23T18:04:02.147159+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":65,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.153048+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:04:02.153048+0000","last_peered":"2023-09-23T18:04:02.153048+0000","last_clean":"2023-09-23T18:04:02.153048+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:04:02.153048+0000","last_undegraded":"2023-09-23T18:04:02.153048+0000","last_fullsized":"2023-09-23T18:04:02.153048+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"265'7555","reported_seq":7628,"reported_epoch":265,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:04:27.621574+0000","last_change":"2023-09-23T18:02:19.054827+0000","last_active":"2023-09-23T18:04:27.621574+0000","last_peered":"2023-09-23T18:04:27.621574+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.048672+0000","last_became_peered":"2023-09-23T18:02:19.048672+0000","last_unstale":"2023-09-23T18:04:27.621574+0000","last_undegraded":"2023-09-23T18:02:18.951263+0000","last_fullsized":"2023-09-23T18:02:18.930851+0000","mapping_epoch":238,"log_start":"216'261","ondisk_log_start":"216'261","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7294,"log_dups_size":0,"ondisk_log_size":7294,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:23.690233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100728832,"num_objects":1537,"num_object_clones":0,"num_object_copies":4611,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1537,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1537,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7555,"num_write_kb":290944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":1537}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":596,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148744+0000","last_change":"2023-09-23T18:02:51.060858+0000","last_active":"2023-09-23T18:04:02.148744+0000","last_peered":"2023-09-23T18:04:02.148744+0000","last_clean":"2023-09-23T18:04:02.148744+0000","last_became_active":"2023-09-23T18:02:51.060554+0000","last_became_peered":"2023-09-23T18:02:51.060554+0000","last_unstale":"2023-09-23T18:04:02.148744+0000","last_undegraded":"2023-09-23T18:04:02.148744+0000","last_fullsized":"2023-09-23T18:04:02.148744+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":55,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151762+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:04:02.151762+0000","last_peered":"2023-09-23T18:04:02.151762+0000","last_clean":"2023-09-23T18:04:02.151762+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:04:02.151762+0000","last_undegraded":"2023-09-23T18:04:02.151762+0000","last_fullsized":"2023-09-23T18:04:02.151762+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":275,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.166964+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:04:02.166964+0000","last_peered":"2023-09-23T18:04:02.166964+0000","last_clean":"2023-09-23T18:04:02.166964+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:04:02.166964+0000","last_undegraded":"2023-09-23T18:04:02.166964+0000","last_fullsized":"2023-09-23T18:04:02.166964+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"265'7594","reported_seq":15716,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:27.629121+0000","last_change":"2023-09-23T18:02:47.017672+0000","last_active":"2023-09-23T18:04:27.629121+0000","last_peered":"2023-09-23T18:04:27.629121+0000","last_clean":"2023-09-23T18:04:27.629121+0000","last_became_active":"2023-09-23T18:02:47.009245+0000","last_became_peered":"2023-09-23T18:02:47.009245+0000","last_unstale":"2023-09-23T18:04:27.629121+0000","last_undegraded":"2023-09-23T18:04:27.629121+0000","last_fullsized":"2023-09-23T18:04:27.629121+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7594,"log_dups_size":0,"ondisk_log_size":7594,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:04.868743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97714200,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7594,"num_write_kb":290689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4015,"num_bytes_recovered":263127040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":588,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148774+0000","last_change":"2023-09-23T18:02:51.054197+0000","last_active":"2023-09-23T18:04:02.148774+0000","last_peered":"2023-09-23T18:04:02.148774+0000","last_clean":"2023-09-23T18:04:02.148774+0000","last_became_active":"2023-09-23T18:02:51.049362+0000","last_became_peered":"2023-09-23T18:02:51.049362+0000","last_unstale":"2023-09-23T18:04:02.148774+0000","last_undegraded":"2023-09-23T18:04:02.148774+0000","last_fullsized":"2023-09-23T18:04:02.148774+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.811940+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:04:14.811940+0000","last_peered":"2023-09-23T18:04:14.811940+0000","last_clean":"2023-09-23T18:04:14.811940+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:04:14.811940+0000","last_undegraded":"2023-09-23T18:04:14.811940+0000","last_fullsized":"2023-09-23T18:04:14.811940+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":110,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.143750+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:04:02.143750+0000","last_peered":"2023-09-23T18:04:02.143750+0000","last_clean":"2023-09-23T18:04:02.143750+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:04:02.143750+0000","last_undegraded":"2023-09-23T18:04:02.143750+0000","last_fullsized":"2023-09-23T18:04:02.143750+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"265'7486","reported_seq":7561,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:29.547830+0000","last_change":"2023-09-23T18:02:19.315376+0000","last_active":"2023-09-23T18:04:29.547830+0000","last_peered":"2023-09-23T18:04:29.547830+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.306174+0000","last_became_peered":"2023-09-23T18:02:19.306174+0000","last_unstale":"2023-09-23T18:04:29.547830+0000","last_undegraded":"2023-09-23T18:02:18.954031+0000","last_fullsized":"2023-09-23T18:02:18.922505+0000","mapping_epoch":238,"log_start":"224'3100","ondisk_log_start":"224'3100","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4386,"log_dups_size":0,"ondisk_log_size":4386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:52.392034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91488256,"num_objects":1396,"num_object_clones":0,"num_object_copies":4188,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1396,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7486,"num_write_kb":284224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":-2699}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":597,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149105+0000","last_change":"2023-09-23T18:02:51.059934+0000","last_active":"2023-09-23T18:04:02.149105+0000","last_peered":"2023-09-23T18:04:02.149105+0000","last_clean":"2023-09-23T18:04:02.149105+0000","last_became_active":"2023-09-23T18:02:51.059638+0000","last_became_peered":"2023-09-23T18:02:51.059638+0000","last_unstale":"2023-09-23T18:04:02.149105+0000","last_undegraded":"2023-09-23T18:04:02.149105+0000","last_fullsized":"2023-09-23T18:04:02.149105+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":575,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:09.592697+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:04:09.592697+0000","last_peered":"2023-09-23T18:04:09.592697+0000","last_clean":"2023-09-23T18:04:09.592697+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:04:09.592697+0000","last_undegraded":"2023-09-23T18:04:09.592697+0000","last_fullsized":"2023-09-23T18:04:09.592697+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":245,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.158602+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:04:02.158602+0000","last_peered":"2023-09-23T18:04:02.158602+0000","last_clean":"2023-09-23T18:04:02.158602+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:04:02.158602+0000","last_undegraded":"2023-09-23T18:04:02.158602+0000","last_fullsized":"2023-09-23T18:04:02.158602+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":242,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.157959+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:04:02.157959+0000","last_peered":"2023-09-23T18:04:02.157959+0000","last_clean":"2023-09-23T18:04:02.157959+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:04:02.157959+0000","last_undegraded":"2023-09-23T18:04:02.157959+0000","last_fullsized":"2023-09-23T18:04:02.157959+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"265'7514","reported_seq":7603,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:28.195689+0000","last_change":"2023-09-23T18:02:19.289610+0000","last_active":"2023-09-23T18:04:28.195689+0000","last_peered":"2023-09-23T18:04:28.195689+0000","last_clean":"2023-09-23T18:02:17.060883+0000","last_became_active":"2023-09-23T18:02:19.266879+0000","last_became_peered":"2023-09-23T18:02:19.266879+0000","last_unstale":"2023-09-23T18:04:28.195689+0000","last_undegraded":"2023-09-23T18:02:18.963390+0000","last_fullsized":"2023-09-23T18:02:18.926482+0000","mapping_epoch":238,"log_start":"217'500","ondisk_log_start":"217'500","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.499269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94633984,"num_objects":1444,"num_object_clones":0,"num_object_copies":4332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1444,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7514,"num_write_kb":286656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":1093},{"shards":"4,7","objects":-2684}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":599,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149679+0000","last_change":"2023-09-23T18:02:51.047767+0000","last_active":"2023-09-23T18:04:02.149679+0000","last_peered":"2023-09-23T18:04:02.149679+0000","last_clean":"2023-09-23T18:04:02.149679+0000","last_became_active":"2023-09-23T18:02:51.047297+0000","last_became_peered":"2023-09-23T18:02:51.047297+0000","last_unstale":"2023-09-23T18:04:02.149679+0000","last_undegraded":"2023-09-23T18:04:02.149679+0000","last_fullsized":"2023-09-23T18:04:02.149679+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.851694+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1193,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.158038+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:04:02.158038+0000","last_peered":"2023-09-23T18:04:02.158038+0000","last_clean":"2023-09-23T18:04:02.158038+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:04:02.158038+0000","last_undegraded":"2023-09-23T18:04:02.158038+0000","last_fullsized":"2023-09-23T18:04:02.158038+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":55,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.152338+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:04:02.152338+0000","last_peered":"2023-09-23T18:04:02.152338+0000","last_clean":"2023-09-23T18:04:02.152338+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:04:02.152338+0000","last_undegraded":"2023-09-23T18:04:02.152338+0000","last_fullsized":"2023-09-23T18:04:02.152338+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":159,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:09.323985+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:04:09.323985+0000","last_peered":"2023-09-23T18:04:09.323985+0000","last_clean":"2023-09-23T18:04:09.323985+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:04:09.323985+0000","last_undegraded":"2023-09-23T18:04:09.323985+0000","last_fullsized":"2023-09-23T18:04:09.323985+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"265'7734","reported_seq":16366,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:27.965229+0000","last_change":"2023-09-23T18:04:02.179479+0000","last_active":"2023-09-23T18:04:27.965229+0000","last_peered":"2023-09-23T18:04:27.965229+0000","last_clean":"2023-09-23T18:04:27.965229+0000","last_became_active":"2023-09-23T18:04:02.150813+0000","last_became_peered":"2023-09-23T18:04:02.150813+0000","last_unstale":"2023-09-23T18:04:27.965229+0000","last_undegraded":"2023-09-23T18:04:27.965229+0000","last_fullsized":"2023-09-23T18:04:27.965229+0000","mapping_epoch":263,"log_start":"224'3674","ondisk_log_start":"224'3674","created":204,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4060,"log_dups_size":0,"ondisk_log_size":4060,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:41.194322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100270080,"num_objects":1530,"num_object_clones":0,"num_object_copies":4590,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1530,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7734,"num_write_kb":296448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4277,"num_bytes_recovered":179372032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":600,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148756+0000","last_change":"2023-09-23T18:02:51.054622+0000","last_active":"2023-09-23T18:04:02.148756+0000","last_peered":"2023-09-23T18:04:02.148756+0000","last_clean":"2023-09-23T18:04:02.148756+0000","last_became_active":"2023-09-23T18:02:51.049345+0000","last_became_peered":"2023-09-23T18:02:51.049345+0000","last_unstale":"2023-09-23T18:04:02.148756+0000","last_undegraded":"2023-09-23T18:04:02.148756+0000","last_fullsized":"2023-09-23T18:04:02.148756+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:58.636010+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":547,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.157911+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:04:02.157911+0000","last_peered":"2023-09-23T18:04:02.157911+0000","last_clean":"2023-09-23T18:04:02.157911+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:04:02.157911+0000","last_undegraded":"2023-09-23T18:04:02.157911+0000","last_fullsized":"2023-09-23T18:04:02.157911+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":160,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:04.322360+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:04:04.322360+0000","last_peered":"2023-09-23T18:04:04.322360+0000","last_clean":"2023-09-23T18:04:04.322360+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:04:04.322360+0000","last_undegraded":"2023-09-23T18:04:04.322360+0000","last_fullsized":"2023-09-23T18:04:04.322360+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":232,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.160949+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:04:02.160949+0000","last_peered":"2023-09-23T18:04:02.160949+0000","last_clean":"2023-09-23T18:04:02.160949+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:04:02.160949+0000","last_undegraded":"2023-09-23T18:04:02.160949+0000","last_fullsized":"2023-09-23T18:04:02.160949+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"265'7668","reported_seq":10570,"reported_epoch":265,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:27.605148+0000","last_change":"2023-09-23T18:04:00.228250+0000","last_active":"2023-09-23T18:04:27.605148+0000","last_peered":"2023-09-23T18:04:27.605148+0000","last_clean":"2023-09-23T18:02:16.962656+0000","last_became_active":"2023-09-23T18:02:19.272896+0000","last_became_peered":"2023-09-23T18:02:19.272896+0000","last_unstale":"2023-09-23T18:04:27.605148+0000","last_undegraded":"2023-09-23T18:02:18.972349+0000","last_fullsized":"2023-09-23T18:02:18.930868+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7668,"log_dups_size":0,"ondisk_log_size":7668,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:23.745675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":103022592,"num_objects":1572,"num_object_clones":0,"num_object_copies":4716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1959,"num_objects_misplaced":905,"num_objects_unfound":0,"num_objects_dirty":1572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7668,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1400,"num_bytes_recovered":91750400,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":905},{"shards":"4,5","objects":-1292}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1177,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.157726+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:04:02.157726+0000","last_peered":"2023-09-23T18:04:02.157726+0000","last_clean":"2023-09-23T18:04:02.157726+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:04:02.157726+0000","last_undegraded":"2023-09-23T18:04:02.157726+0000","last_fullsized":"2023-09-23T18:04:02.157726+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":588,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148357+0000","last_change":"2023-09-23T18:02:51.054022+0000","last_active":"2023-09-23T18:04:02.148357+0000","last_peered":"2023-09-23T18:04:02.148357+0000","last_clean":"2023-09-23T18:04:02.148357+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:04:02.148357+0000","last_undegraded":"2023-09-23T18:04:02.148357+0000","last_fullsized":"2023-09-23T18:04:02.148357+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":73,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.810674+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:04:14.810674+0000","last_peered":"2023-09-23T18:04:14.810674+0000","last_clean":"2023-09-23T18:04:14.810674+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:04:14.810674+0000","last_undegraded":"2023-09-23T18:04:14.810674+0000","last_fullsized":"2023-09-23T18:04:14.810674+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":198,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.146486+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:04:02.146486+0000","last_peered":"2023-09-23T18:04:02.146486+0000","last_clean":"2023-09-23T18:04:02.146486+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:04:02.146486+0000","last_undegraded":"2023-09-23T18:04:02.146486+0000","last_fullsized":"2023-09-23T18:04:02.146486+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"265'7635","reported_seq":7797,"reported_epoch":265,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:04:27.631816+0000","last_change":"2023-09-23T18:02:18.003772+0000","last_active":"2023-09-23T18:04:27.631816+0000","last_peered":"2023-09-23T18:04:27.631816+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:18.000696+0000","last_became_peered":"2023-09-23T18:02:18.000696+0000","last_unstale":"2023-09-23T18:04:27.631816+0000","last_undegraded":"2023-09-23T18:02:17.938873+0000","last_fullsized":"2023-09-23T18:02:17.919669+0000","mapping_epoch":237,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7535,"log_dups_size":0,"ondisk_log_size":7535,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:48.085957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94830592,"num_objects":1447,"num_object_clones":0,"num_object_copies":4341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1433,"num_objects_misplaced":1447,"num_objects_unfound":0,"num_objects_dirty":1447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7635,"num_write_kb":290624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":1447}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":600,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148651+0000","last_change":"2023-09-23T18:02:51.061168+0000","last_active":"2023-09-23T18:04:02.148651+0000","last_peered":"2023-09-23T18:04:02.148651+0000","last_clean":"2023-09-23T18:04:02.148651+0000","last_became_active":"2023-09-23T18:02:51.060689+0000","last_became_peered":"2023-09-23T18:02:51.060689+0000","last_unstale":"2023-09-23T18:04:02.148651+0000","last_undegraded":"2023-09-23T18:04:02.148651+0000","last_fullsized":"2023-09-23T18:04:02.148651+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:29.406460+0000","scrub_duration":0.001568505,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":445,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148893+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:04:02.148893+0000","last_peered":"2023-09-23T18:04:02.148893+0000","last_clean":"2023-09-23T18:04:02.148893+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:04:02.148893+0000","last_undegraded":"2023-09-23T18:04:02.148893+0000","last_fullsized":"2023-09-23T18:04:02.148893+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":264,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.150883+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:04:02.150883+0000","last_peered":"2023-09-23T18:04:02.150883+0000","last_clean":"2023-09-23T18:04:02.150883+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:04:02.150883+0000","last_undegraded":"2023-09-23T18:04:02.150883+0000","last_fullsized":"2023-09-23T18:04:02.150883+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":235,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:09.592712+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:04:09.592712+0000","last_peered":"2023-09-23T18:04:09.592712+0000","last_clean":"2023-09-23T18:04:09.592712+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:04:09.592712+0000","last_undegraded":"2023-09-23T18:04:09.592712+0000","last_fullsized":"2023-09-23T18:04:09.592712+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"265'7830","reported_seq":16176,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:30.470545+0000","last_change":"2023-09-23T18:02:18.942886+0000","last_active":"2023-09-23T18:04:30.470545+0000","last_peered":"2023-09-23T18:04:30.470545+0000","last_clean":"2023-09-23T18:04:30.470545+0000","last_became_active":"2023-09-23T18:02:18.936691+0000","last_became_peered":"2023-09-23T18:02:18.936691+0000","last_unstale":"2023-09-23T18:04:30.470545+0000","last_undegraded":"2023-09-23T18:04:30.470545+0000","last_fullsized":"2023-09-23T18:04:30.470545+0000","mapping_epoch":238,"log_start":"224'3658","ondisk_log_start":"224'3658","created":204,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4172,"log_dups_size":0,"ondisk_log_size":4172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:26.085859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":92143616,"num_objects":1406,"num_object_clones":0,"num_object_copies":4218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7830,"num_write_kb":295552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4123,"num_bytes_recovered":270204928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":597,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149237+0000","last_change":"2023-09-23T18:02:51.057370+0000","last_active":"2023-09-23T18:04:02.149237+0000","last_peered":"2023-09-23T18:04:02.149237+0000","last_clean":"2023-09-23T18:04:02.149237+0000","last_became_active":"2023-09-23T18:02:51.056158+0000","last_became_peered":"2023-09-23T18:02:51.056158+0000","last_unstale":"2023-09-23T18:04:02.149237+0000","last_undegraded":"2023-09-23T18:04:02.149237+0000","last_fullsized":"2023-09-23T18:04:02.149237+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":479,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149779+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:04:02.149779+0000","last_peered":"2023-09-23T18:04:02.149779+0000","last_clean":"2023-09-23T18:04:02.149779+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:04:02.149779+0000","last_undegraded":"2023-09-23T18:04:02.149779+0000","last_fullsized":"2023-09-23T18:04:02.149779+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":200,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:09.324005+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:04:09.324005+0000","last_peered":"2023-09-23T18:04:09.324005+0000","last_clean":"2023-09-23T18:04:09.324005+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:04:09.324005+0000","last_undegraded":"2023-09-23T18:04:09.324005+0000","last_fullsized":"2023-09-23T18:04:09.324005+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":232,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151307+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:04:02.151307+0000","last_peered":"2023-09-23T18:04:02.151307+0000","last_clean":"2023-09-23T18:04:02.151307+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:04:02.151307+0000","last_undegraded":"2023-09-23T18:04:02.151307+0000","last_fullsized":"2023-09-23T18:04:02.151307+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"265'7600","reported_seq":7693,"reported_epoch":265,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:28.233727+0000","last_change":"2023-09-23T18:02:19.279112+0000","last_active":"2023-09-23T18:04:28.233727+0000","last_peered":"2023-09-23T18:04:28.233727+0000","last_clean":"2023-09-23T18:02:17.018900+0000","last_became_active":"2023-09-23T18:02:19.267439+0000","last_became_peered":"2023-09-23T18:02:19.267439+0000","last_unstale":"2023-09-23T18:04:28.233727+0000","last_undegraded":"2023-09-23T18:02:18.959329+0000","last_fullsized":"2023-09-23T18:02:18.926801+0000","mapping_epoch":238,"log_start":"217'600","ondisk_log_start":"217'600","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7000,"log_dups_size":0,"ondisk_log_size":7000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.395583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99221504,"num_objects":1514,"num_object_clones":0,"num_object_copies":4542,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1514,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7600,"num_write_kb":291648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-2685},{"shards":"2,4,6","objects":1156}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":597,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149347+0000","last_change":"2023-09-23T18:02:51.058680+0000","last_active":"2023-09-23T18:04:02.149347+0000","last_peered":"2023-09-23T18:04:02.149347+0000","last_clean":"2023-09-23T18:04:02.149347+0000","last_became_active":"2023-09-23T18:02:51.057706+0000","last_became_peered":"2023-09-23T18:02:51.057706+0000","last_unstale":"2023-09-23T18:04:02.149347+0000","last_undegraded":"2023-09-23T18:04:02.149347+0000","last_fullsized":"2023-09-23T18:04:02.149347+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":111,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.823590+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:04:14.823590+0000","last_peered":"2023-09-23T18:04:14.823590+0000","last_clean":"2023-09-23T18:04:14.823590+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:04:14.823590+0000","last_undegraded":"2023-09-23T18:04:14.823590+0000","last_fullsized":"2023-09-23T18:04:14.823590+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":335,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.146774+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:04:02.146774+0000","last_peered":"2023-09-23T18:04:02.146774+0000","last_clean":"2023-09-23T18:04:02.146774+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:04:02.146774+0000","last_undegraded":"2023-09-23T18:04:02.146774+0000","last_fullsized":"2023-09-23T18:04:02.146774+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":115,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.811843+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:04:14.811843+0000","last_peered":"2023-09-23T18:04:14.811843+0000","last_clean":"2023-09-23T18:04:14.811843+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:04:14.811843+0000","last_undegraded":"2023-09-23T18:04:14.811843+0000","last_fullsized":"2023-09-23T18:04:14.811843+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"265'7670","reported_seq":7741,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:27.929706+0000","last_change":"2023-09-23T18:01:25.046224+0000","last_active":"2023-09-23T18:04:27.929706+0000","last_peered":"2023-09-23T18:04:27.929706+0000","last_clean":"2023-09-23T18:04:27.929706+0000","last_became_active":"2023-09-23T18:01:25.002864+0000","last_became_peered":"2023-09-23T18:01:25.002864+0000","last_unstale":"2023-09-23T18:04:27.929706+0000","last_undegraded":"2023-09-23T18:04:27.929706+0000","last_fullsized":"2023-09-23T18:04:27.929706+0000","mapping_epoch":214,"log_start":"224'3541","ondisk_log_start":"224'3541","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4129,"log_dups_size":0,"ondisk_log_size":4129,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.752359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96206848,"num_objects":1468,"num_object_clones":0,"num_object_copies":4404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7670,"num_write_kb":292416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":604,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148808+0000","last_change":"2023-09-23T18:02:51.053420+0000","last_active":"2023-09-23T18:04:02.148808+0000","last_peered":"2023-09-23T18:04:02.148808+0000","last_clean":"2023-09-23T18:04:02.148808+0000","last_became_active":"2023-09-23T18:02:51.053035+0000","last_became_peered":"2023-09-23T18:02:51.053035+0000","last_unstale":"2023-09-23T18:04:02.148808+0000","last_undegraded":"2023-09-23T18:04:02.148808+0000","last_fullsized":"2023-09-23T18:04:02.148808+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:20.456481+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":470,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.146926+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:04:02.146926+0000","last_peered":"2023-09-23T18:04:02.146926+0000","last_clean":"2023-09-23T18:04:02.146926+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:04:02.146926+0000","last_undegraded":"2023-09-23T18:04:02.146926+0000","last_fullsized":"2023-09-23T18:04:02.146926+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":292,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:09.592722+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:04:09.592722+0000","last_peered":"2023-09-23T18:04:09.592722+0000","last_clean":"2023-09-23T18:04:09.592722+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:04:09.592722+0000","last_undegraded":"2023-09-23T18:04:09.592722+0000","last_fullsized":"2023-09-23T18:04:09.592722+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":55,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151590+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:04:02.151590+0000","last_peered":"2023-09-23T18:04:02.151590+0000","last_clean":"2023-09-23T18:04:02.151590+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:04:02.151590+0000","last_undegraded":"2023-09-23T18:04:02.151590+0000","last_fullsized":"2023-09-23T18:04:02.151590+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":587,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149829+0000","last_change":"2023-09-23T18:02:51.046571+0000","last_active":"2023-09-23T18:04:02.149829+0000","last_peered":"2023-09-23T18:04:02.149829+0000","last_clean":"2023-09-23T18:04:02.149829+0000","last_became_active":"2023-09-23T18:02:51.046213+0000","last_became_peered":"2023-09-23T18:02:51.046213+0000","last_unstale":"2023-09-23T18:04:02.149829+0000","last_undegraded":"2023-09-23T18:04:02.149829+0000","last_fullsized":"2023-09-23T18:04:02.149829+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":251,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151260+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:04:02.151260+0000","last_peered":"2023-09-23T18:04:02.151260+0000","last_clean":"2023-09-23T18:04:02.151260+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:04:02.151260+0000","last_undegraded":"2023-09-23T18:04:02.151260+0000","last_fullsized":"2023-09-23T18:04:02.151260+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":278,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.821622+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:04:14.821622+0000","last_peered":"2023-09-23T18:04:14.821622+0000","last_clean":"2023-09-23T18:04:14.821622+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:04:14.821622+0000","last_undegraded":"2023-09-23T18:04:14.821622+0000","last_fullsized":"2023-09-23T18:04:14.821622+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":596,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149076+0000","last_change":"2023-09-23T18:02:51.060198+0000","last_active":"2023-09-23T18:04:02.149076+0000","last_peered":"2023-09-23T18:04:02.149076+0000","last_clean":"2023-09-23T18:04:02.149076+0000","last_became_active":"2023-09-23T18:02:51.057046+0000","last_became_peered":"2023-09-23T18:02:51.057046+0000","last_unstale":"2023-09-23T18:04:02.149076+0000","last_undegraded":"2023-09-23T18:04:02.149076+0000","last_fullsized":"2023-09-23T18:04:02.149076+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":120,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.158824+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:04:02.158824+0000","last_peered":"2023-09-23T18:04:02.158824+0000","last_clean":"2023-09-23T18:04:02.158824+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:04:02.158824+0000","last_undegraded":"2023-09-23T18:04:02.158824+0000","last_fullsized":"2023-09-23T18:04:02.158824+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":190,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.152536+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:04:02.152536+0000","last_peered":"2023-09-23T18:04:02.152536+0000","last_clean":"2023-09-23T18:04:02.152536+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:04:02.152536+0000","last_undegraded":"2023-09-23T18:04:02.152536+0000","last_fullsized":"2023-09-23T18:04:02.152536+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":106,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.158391+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:04:02.158391+0000","last_peered":"2023-09-23T18:04:02.158391+0000","last_clean":"2023-09-23T18:04:02.158391+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:04:02.158391+0000","last_undegraded":"2023-09-23T18:04:02.158391+0000","last_fullsized":"2023-09-23T18:04:02.158391+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":129,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.142404+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:04:02.142404+0000","last_peered":"2023-09-23T18:04:02.142404+0000","last_clean":"2023-09-23T18:04:02.142404+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:04:02.142404+0000","last_undegraded":"2023-09-23T18:04:02.142404+0000","last_fullsized":"2023-09-23T18:04:02.142404+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":612,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.148861+0000","last_change":"2023-09-23T18:02:51.047476+0000","last_active":"2023-09-23T18:04:02.148861+0000","last_peered":"2023-09-23T18:04:02.148861+0000","last_clean":"2023-09-23T18:04:02.148861+0000","last_became_active":"2023-09-23T18:02:51.047152+0000","last_became_peered":"2023-09-23T18:02:51.047152+0000","last_unstale":"2023-09-23T18:04:02.148861+0000","last_undegraded":"2023-09-23T18:04:02.148861+0000","last_fullsized":"2023-09-23T18:04:02.148861+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:22.514077+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":130,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.815003+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:04:14.815003+0000","last_peered":"2023-09-23T18:04:14.815003+0000","last_clean":"2023-09-23T18:04:14.815003+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:04:14.815003+0000","last_undegraded":"2023-09-23T18:04:14.815003+0000","last_fullsized":"2023-09-23T18:04:14.815003+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":132,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.141969+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:04:02.141969+0000","last_peered":"2023-09-23T18:04:02.141969+0000","last_clean":"2023-09-23T18:04:02.141969+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:04:02.141969+0000","last_undegraded":"2023-09-23T18:04:02.141969+0000","last_fullsized":"2023-09-23T18:04:02.141969+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":595,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.149256+0000","last_change":"2023-09-23T18:02:51.058825+0000","last_active":"2023-09-23T18:04:02.149256+0000","last_peered":"2023-09-23T18:04:02.149256+0000","last_clean":"2023-09-23T18:04:02.149256+0000","last_became_active":"2023-09-23T18:02:51.058480+0000","last_became_peered":"2023-09-23T18:02:51.058480+0000","last_unstale":"2023-09-23T18:04:02.149256+0000","last_undegraded":"2023-09-23T18:04:02.149256+0000","last_fullsized":"2023-09-23T18:04:02.149256+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":244,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151633+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:04:02.151633+0000","last_peered":"2023-09-23T18:04:02.151633+0000","last_clean":"2023-09-23T18:04:02.151633+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:04:02.151633+0000","last_undegraded":"2023-09-23T18:04:02.151633+0000","last_fullsized":"2023-09-23T18:04:02.151633+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":275,"reported_epoch":264,"state":"active+clean","last_fresh":"2023-09-23T18:04:02.151881+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:04:02.151881+0000","last_peered":"2023-09-23T18:04:02.151881+0000","last_clean":"2023-09-23T18:04:02.151881+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:04:02.151881+0000","last_undegraded":"2023-09-23T18:04:02.151881+0000","last_fullsized":"2023-09-23T18:04:02.151881+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":1541865496,"num_objects":23528,"num_object_clones":0,"num_object_copies":70584,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":21928,"num_objects_misplaced":5543,"num_objects_unfound":0,"num_objects_dirty":23528,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":122744,"num_write_kb":4680641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25007,"num_bytes_recovered":1518338048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5465452544,"data_stored":5465440328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":93480,"ondisk_log_size":93480,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":402523,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":331,"num_read_kb":623,"num_write":162,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2814833,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001382,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":694572,"kb_used_data":600292,"kb_used_omap":13,"kb_used_meta":94258,"kb_avail":93050580,"statfs":{"total":95995035648,"available":95283793920,"internally_reserved":0,"allocated":614699008,"data_stored":613680033,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":96520296},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.60799999999999998,"15min":0.61199999999999999},"min":{"1min":0.28699999999999998,"5min":0.247,"15min":0.247},"max":{"1min":2.2160000000000002,"5min":2.544,"15min":2.544},"last":0.29499999999999998},{"interface":"front","average":{"1min":0.98299999999999998,"5min":1.226,"15min":1.266},"min":{"1min":0.32100000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.5990000000000002,"5min":6.5990000000000002,"15min":6.5990000000000002},"last":1.254}]},{"osd":1,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.86899999999999999,"5min":0.71599999999999997,"15min":0.68999999999999995},"min":{"1min":0.27400000000000002,"5min":0.252,"15min":0.252},"max":{"1min":3.2850000000000001,"5min":3.2850000000000001,"15min":3.2850000000000001},"last":7.1150000000000002},{"interface":"front","average":{"1min":1.1020000000000001,"5min":1.0649999999999999,"15min":1.0589999999999999},"min":{"1min":0.35199999999999998,"5min":0.29599999999999999,"15min":0.29599999999999999},"max":{"1min":6.2169999999999996,"5min":6.2169999999999996,"15min":6.2169999999999996},"last":2.9609999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":1.036,"5min":0.95699999999999996,"15min":0.94399999999999995},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":4.1539999999999999,"5min":4.1539999999999999,"15min":4.1539999999999999},"last":1.3},{"interface":"front","average":{"1min":1.161,"5min":0.88200000000000001,"15min":0.83599999999999997},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.1970000000000001,"5min":6.1970000000000001,"15min":6.1970000000000001},"last":6.1989999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:04:20 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.4950000000000001,"15min":1.534},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.6070000000000002,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":2.3759999999999999},{"interface":"front","average":{"1min":1.5900000000000001,"5min":1.0940000000000001,"15min":1.0109999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.776999999999999,"5min":10.776999999999999,"15min":10.776999999999999},"last":0.98099999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.78200000000000003,"5min":1.014,"15min":1.052},"min":{"1min":0.27700000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.4199999999999999,"5min":3.984,"15min":3.984},"last":3.234},{"interface":"front","average":{"1min":0.627,"5min":0.54600000000000004,"15min":0.53200000000000003},"min":{"1min":0.33400000000000002,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":2.1040000000000001,"5min":2.1040000000000001,"15min":2.1040000000000001},"last":0.32800000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.71699999999999997,"5min":0.94599999999999995,"15min":0.98399999999999999},"min":{"1min":0.33900000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":1.8939999999999999,"5min":3.9300000000000002,"15min":3.9300000000000002},"last":1.3129999999999999},{"interface":"front","average":{"1min":0.87,"5min":1.0800000000000001,"15min":1.115},"min":{"1min":0.39900000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.403,"5min":3.7080000000000002,"15min":3.7080000000000002},"last":2.4119999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:03:32 2023","interfaces":[{"interface":"back","average":{"1min":0.66900000000000004,"5min":0.69799999999999995,"15min":0.70299999999999996},"min":{"1min":0.39500000000000002,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":2.1429999999999998,"5min":3.0409999999999999,"15min":3.0409999999999999},"last":1.2430000000000001},{"interface":"front","average":{"1min":0.72599999999999998,"5min":1.046,"15min":1.099},"min":{"1min":0.42099999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":1.8560000000000001,"5min":3.6419999999999999,"15min":3.6419999999999999},"last":1.2829999999999999}]}]},{"osd":6,"up_from":163,"seq":700079669319,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1519076,"kb_used_data":1353500,"kb_used_omap":7,"kb_used_meta":165560,"kb_avail":92226076,"statfs":{"total":95995035648,"available":94439501824,"internally_reserved":0,"allocated":1385984000,"data_stored":1384973090,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":169534244},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.274,"15min":1.3240000000000001},"min":{"1min":0.25700000000000001,"5min":0.215,"15min":0.215},"max":{"1min":15.407,"5min":15.407,"15min":15.407},"last":4.774},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.347,"15min":2.2490000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.9710000000000001,"5min":13.536,"15min":13.536},"last":7.5739999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":1.458,"15min":1.913},"min":{"1min":0.29799999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7690000000000001,"5min":14.233000000000001,"15min":14.233000000000001},"last":4.6890000000000001},{"interface":"front","average":{"1min":1.905,"5min":1.5720000000000001,"15min":2.1379999999999999},"min":{"1min":0.32300000000000001,"5min":0.248,"15min":0.248},"max":{"1min":14.789,"5min":14.789,"15min":14.789},"last":5.2809999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.75,"15min":2.2610000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.7489999999999997,"5min":11.210000000000001,"15min":11.210000000000001},"last":5.0060000000000002},{"interface":"front","average":{"1min":1.171,"5min":1.8149999999999999,"15min":2.633},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.7359999999999998,"5min":11.223000000000001,"15min":11.223000000000001},"last":0.40000000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:04:22 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.0449999999999999,"15min":1.0840000000000001},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.308},"max":{"1min":3.25,"5min":3.25,"15min":3.25},"last":4.7539999999999996},{"interface":"front","average":{"1min":1.472,"5min":1.2270000000000001,"15min":1.1859999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.2389999999999999,"5min":5.2389999999999999,"15min":5.2389999999999999},"last":7.484}]},{"osd":4,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.93700000000000006,"15min":0.88400000000000001},"min":{"1min":0.309,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.9930000000000003,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":4.4779999999999998},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.78000000000000003,"15min":0.52100000000000002},"min":{"1min":0.35399999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.9430000000000001,"5min":8.3000000000000007,"15min":8.3000000000000007},"last":5.7709999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.89600000000000002,"15min":0.82799999999999996},"min":{"1min":0.34200000000000003,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.2440000000000002,"5min":11.869,"15min":11.869},"last":4.4390000000000001},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96499999999999997,"15min":0.66500000000000004},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.4130000000000003,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":4.7930000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:04:28 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.024,"15min":0.98399999999999999},"min":{"1min":0.35399999999999998,"5min":0.307,"15min":0.307},"max":{"1min":4.5880000000000001,"5min":4.5880000000000001,"15min":4.5880000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.83899999999999997,"15min":0.80100000000000005},"min":{"1min":0.38100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.7190000000000003,"5min":4.7190000000000003,"15min":4.7190000000000003},"last":4.7190000000000003}]}]},{"osd":1,"up_from":0,"seq":420906795137,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958664,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":775840,"kb_used_data":382296,"kb_used_omap":7,"kb_used_meta":393528,"kb_avail":92969312,"statfs":{"total":95995035648,"available":95200575488,"internally_reserved":0,"allocated":391471104,"data_stored":390116226,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":402973412},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":1.597,"15min":1.1419999999999999},"min":{"1min":0.20699999999999999,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":2.238,"5min":13.414,"15min":18.154},"last":4.0179999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.504,"15min":1.1140000000000001},"min":{"1min":0.26800000000000002,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":1.514,"5min":13.369999999999999,"15min":13.369999999999999},"last":0.32500000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.2490000000000001,"15min":1.2829999999999999},"min":{"1min":0.21099999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.3529999999999998,"5min":20.882000000000001,"15min":20.882000000000001},"last":1.2390000000000001},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.0830000000000002,"15min":1.234},"min":{"1min":0.22500000000000001,"5min":0.217,"15min":0.217},"max":{"1min":6.8869999999999996,"5min":15.064,"15min":15.064},"last":0.249}]},{"osd":3,"last update":"Sat Sep 23 18:04:21 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.419,"15min":1.508},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.3730000000000002,"5min":6.048,"15min":6.048},"last":0.69799999999999995},{"interface":"front","average":{"1min":1.661,"5min":1.2929999999999999,"15min":1.2310000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.7300000000000004,"5min":8.7300000000000004,"15min":8.7300000000000004},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.677,"15min":1.1200000000000001},"min":{"1min":0.436,"5min":0.29299999999999998,"15min":0.251},"max":{"1min":4.4809999999999999,"5min":15.279999999999999,"15min":15.279999999999999},"last":3.766},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.7849999999999999,"15min":1.1439999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":5.3109999999999999,"5min":18.106999999999999,"15min":18.106999999999999},"last":1.532}]},{"osd":5,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.8939999999999999,"15min":1.196},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":5.3760000000000003,"5min":14.720000000000001,"15min":14.720000000000001},"last":1.3919999999999999},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.526,"15min":1.024},"min":{"1min":0.373,"5min":0.254,"15min":0.254},"max":{"1min":2.536,"5min":12.794,"15min":12.794},"last":0.36499999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.681,"15min":2.2389999999999999},"min":{"1min":0.35099999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.4660000000000002,"5min":14.956,"15min":14.956},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.6830000000000001,"15min":2.4020000000000001},"min":{"1min":0.38300000000000001,"5min":0.311,"15min":0.311},"max":{"1min":6.9189999999999996,"5min":12.755000000000001,"15min":12.755000000000001},"last":0.39200000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:04:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":0.98399999999999999,"15min":0.94299999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.4649999999999999,"5min":7.4649999999999999,"15min":7.4649999999999999},"last":7.4649999999999999},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.3080000000000001,"15min":1.1579999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.524,"5min":5.524,"15min":5.524},"last":3.8130000000000002}]}]},{"osd":2,"up_from":103,"seq":442381631611,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1210952,"kb_used_data":837440,"kb_used_omap":11,"kb_used_meta":373364,"kb_avail":92534200,"statfs":{"total":95995035648,"available":94755020800,"internally_reserved":0,"allocated":857538560,"data_stored":856207377,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":382325111},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,0,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":2.3130000000000002,"15min":1.2849999999999999},"min":{"1min":0.19800000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":2.694,"5min":20.867000000000001,"15min":20.867000000000001},"last":7.5069999999999997},{"interface":"front","average":{"1min":0.877,"5min":2.1819999999999999,"15min":1.2789999999999999},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":3.286,"5min":21.681000000000001,"15min":21.681000000000001},"last":7.6550000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":2.4540000000000002,"15min":1.395},"min":{"1min":0.26800000000000002,"5min":0.215,"15min":0.215},"max":{"1min":3.0960000000000001,"5min":20.995000000000001,"15min":20.995000000000001},"last":7.2720000000000002},{"interface":"front","average":{"1min":0.88800000000000001,"5min":1.698,"15min":1.1839999999999999},"min":{"1min":0.26700000000000002,"5min":0.193,"15min":0.193},"max":{"1min":3.649,"5min":15.945,"15min":15.945},"last":1.7829999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:04:26 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":0.98399999999999999,"15min":0.84899999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.009,"5min":12.009,"15min":12.009},"last":5.3600000000000003},{"interface":"front","average":{"1min":1.4490000000000001,"5min":0.77800000000000002,"15min":0.66600000000000004},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.0820000000000007,"5min":9.0820000000000007,"15min":9.0820000000000007},"last":0.27900000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.5960000000000001,"15min":1.111},"min":{"1min":0.32300000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.1440000000000001,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":2.448},{"interface":"front","average":{"1min":0.622,"5min":1.837,"15min":1.121},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.6459999999999999,"5min":20.920000000000002,"15min":20.920000000000002},"last":7.5609999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.9339999999999999,"15min":1.198},"min":{"1min":0.35499999999999998,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":3.6960000000000002,"5min":20.738,"15min":20.738},"last":7.3289999999999997},{"interface":"front","average":{"1min":1.145,"5min":1.6419999999999999,"15min":1.2330000000000001},"min":{"1min":0.32000000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.6280000000000001,"5min":14.359,"15min":15.496},"last":2.1549999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.752,"15min":2.1320000000000001},"min":{"1min":0.36599999999999999,"5min":0.308,"15min":0.308},"max":{"1min":3.1230000000000002,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":0.307},{"interface":"front","average":{"1min":0.78400000000000003,"5min":1.97,"15min":2.3740000000000001},"min":{"1min":0.40799999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.2480000000000002,"5min":20.893000000000001,"15min":20.893000000000001},"last":7.6870000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:03:26 2023","interfaces":[{"interface":"back","average":{"1min":1.0309999999999999,"5min":1.335,"15min":1.385},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":3.5600000000000001,"5min":4.3239999999999998,"15min":4.3239999999999998},"last":1.899},{"interface":"front","average":{"1min":0.78700000000000003,"5min":0.66300000000000003,"15min":0.64300000000000002},"min":{"1min":0.40699999999999997,"5min":0.309,"15min":0.309},"max":{"1min":2.7789999999999999,"5min":2.7789999999999999,"15min":2.7789999999999999},"last":7.5339999999999998}]}]},{"osd":3,"up_from":237,"seq":1017907249179,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":995220,"kb_used_data":877004,"kb_used_omap":3,"kb_used_meta":118204,"kb_avail":92749932,"statfs":{"total":95995035648,"available":94975930368,"internally_reserved":0,"allocated":898052096,"data_stored":896538708,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":121041640},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.79200000000000004,"15min":0.85299999999999998},"min":{"1min":0.215,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.6179999999999999,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":8.5440000000000005},{"interface":"front","average":{"1min":0.373,"5min":0.93999999999999995,"15min":1.034},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.63300000000000001,"5min":4.5529999999999999,"15min":4.5529999999999999},"last":0.33200000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.92300000000000004,"15min":1.014},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.69099999999999995,"5min":4.6050000000000004,"15min":4.6050000000000004},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.874,"15min":0.92200000000000004},"min":{"1min":0.26800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.9630000000000001,"5min":4.8810000000000002,"15min":4.8810000000000002},"last":2.2879999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.81899999999999995,"15min":0.83999999999999997},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.157,"5min":3.157,"15min":3.157},"last":1.9830000000000001},{"interface":"front","average":{"1min":0.443,"5min":1.085,"15min":1.1919999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.75600000000000001,"5min":6.157,"15min":6.157},"last":0.375}]},{"osd":4,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.016,"15min":1.0740000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.3330000000000002,"5min":4.633,"15min":4.633},"last":0.626},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.995,"15min":1.0429999999999999},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.3570000000000002,"5min":4.7300000000000004,"15min":4.7300000000000004},"last":3.129}]},{"osd":5,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.179,"15min":1.2490000000000001},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.314},"max":{"1min":2.6880000000000002,"5min":7.0640000000000001,"15min":7.0640000000000001},"last":3.0950000000000002},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.133,"15min":1.165},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.0750000000000002,"5min":4.6150000000000002,"15min":4.6150000000000002},"last":8.4900000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.0669999999999999,"15min":1.079},"min":{"1min":0.36699999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.125,"5min":4.0590000000000002,"15min":4.0590000000000002},"last":2.3439999999999999},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.3680000000000001,"15min":1.4419999999999999},"min":{"1min":0.40500000000000003,"5min":0.379,"15min":0.379},"max":{"1min":2.8679999999999999,"5min":7.3929999999999998,"15min":7.3929999999999998},"last":3.194}]},{"osd":7,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.47,"15min":1.538},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.8149999999999999,"5min":8.4190000000000005,"15min":8.4190000000000005},"last":3.274},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.512,"15min":1.5840000000000001},"min":{"1min":0.46000000000000002,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":3.8610000000000002,"5min":5.1760000000000002,"15min":5.1760000000000002},"last":2.0840000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019129,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":782120,"kb_used_data":549152,"kb_used_omap":0,"kb_used_meta":232960,"kb_avail":92963032,"statfs":{"total":95995035648,"available":95194144768,"internally_reserved":0,"allocated":562331648,"data_stored":561316478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":238551040},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.3220000000000001,"15min":0.82599999999999996},"min":{"1min":0.25,"5min":0.22,"15min":0.20100000000000001},"max":{"1min":2.238,"5min":14.164,"15min":14.164},"last":2.2330000000000001},{"interface":"front","average":{"1min":1.4450000000000001,"5min":1.5109999999999999,"15min":1.04},"min":{"1min":0.28899999999999998,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":7.1879999999999997,"5min":9.3789999999999996,"15min":11.372},"last":1.9510000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.359,"15min":1.004},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":2.2770000000000001,"5min":8.5259999999999998,"15min":10.429},"last":2.089},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.2949999999999999,"15min":1.083},"min":{"1min":0.29699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.597,"5min":8.8130000000000006,"15min":10.83},"last":2.2629999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.5109999999999999,"15min":1.085},"min":{"1min":0.28899999999999998,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":2.7930000000000001,"5min":11.630000000000001,"15min":11.630000000000001},"last":2.3050000000000002},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.3580000000000001,"15min":1.016},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.27100000000000002},"max":{"1min":3.2250000000000001,"5min":7.4459999999999997,"15min":7.5090000000000003},"last":2.2850000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":1.238,"15min":1.3129999999999999},"min":{"1min":0.378,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.6619999999999999,"5min":7.2140000000000004,"15min":7.2140000000000004},"last":2.3260000000000001},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.335,"15min":1.3600000000000001},"min":{"1min":0.34799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.165,"5min":6.165,"15min":6.165},"last":3.706}]},{"osd":5,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.90900000000000003,"15min":0.629},"min":{"1min":0.25900000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":9.9819999999999993,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":2.4430000000000001},{"interface":"front","average":{"1min":1.2350000000000001,"5min":1.111,"15min":0.76700000000000002},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.26100000000000001},"max":{"1min":11.170999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":9.1999999999999993}]},{"osd":6,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.94899999999999995,"15min":0.82899999999999996},"min":{"1min":0.309,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.1849999999999996,"5min":7.1529999999999996,"15min":7.1529999999999996},"last":9.0990000000000002},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.0740000000000001,"15min":0.95799999999999996},"min":{"1min":0.34300000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.2590000000000003,"5min":10.725,"15min":10.725},"last":9.1300000000000008}]},{"osd":7,"last update":"Sat Sep 23 18:03:28 2023","interfaces":[{"interface":"back","average":{"1min":1.2490000000000001,"5min":0.98099999999999998,"15min":0.93600000000000005},"min":{"1min":0.35899999999999999,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":5.3140000000000001,"5min":5.3140000000000001,"15min":5.3140000000000001},"last":7.6779999999999999},{"interface":"front","average":{"1min":1.4550000000000001,"5min":1.2609999999999999,"15min":1.2290000000000001},"min":{"1min":0.35799999999999998,"5min":0.307,"15min":0.307},"max":{"1min":5.2770000000000001,"5min":5.8090000000000002,"15min":5.8090000000000002},"last":7.7149999999999999}]}]},{"osd":5,"up_from":38,"seq":163208757495,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":738588,"kb_used_data":533012,"kb_used_omap":0,"kb_used_meta":205568,"kb_avail":93006564,"statfs":{"total":95995035648,"available":95238721536,"internally_reserved":0,"allocated":545804288,"data_stored":544796870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":210501632},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.5349999999999999,"15min":1.002},"min":{"1min":0.27300000000000002,"5min":0.217,"15min":0.217},"max":{"1min":2.0649999999999999,"5min":21.376999999999999,"15min":21.376999999999999},"last":2.5600000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.8680000000000001,"15min":1.179},"min":{"1min":0.36699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.0310000000000006,"5min":21.388999999999999,"15min":21.388999999999999},"last":4.5069999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:04:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.91200000000000003,"15min":0.85999999999999999},"min":{"1min":0.26000000000000001,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":3.754,"5min":4.8129999999999997,"15min":10.853999999999999},"last":1.4810000000000001},{"interface":"front","average":{"1min":1.075,"5min":1.159,"15min":0.98799999999999999},"min":{"1min":0.28000000000000003,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":8.0039999999999996,"5min":8.0039999999999996,"15min":10.206},"last":0.33800000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":1.956,"15min":1.258},"min":{"1min":0.312,"5min":0.255,"15min":0.255},"max":{"1min":1.75,"5min":26.035,"15min":38.606999999999999},"last":4.7759999999999998},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.498,"15min":1.139},"min":{"1min":0.308,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9289999999999998,"5min":17.085000000000001,"15min":39.670999999999999},"last":1.885}]},{"osd":3,"last update":"Sat Sep 23 18:04:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.85099999999999998,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.9569999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.751,"15min":0.72499999999999998},"min":{"1min":0.35699999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.1260000000000003,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":4.71}]},{"osd":4,"last update":"Sat Sep 23 18:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.0089999999999999,"15min":0.72699999999999998},"min":{"1min":0.254,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":8.2390000000000008,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.98399999999999999,"15min":0.69999999999999996},"min":{"1min":0.35399999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":0.79900000000000004,"5min":5.077,"15min":5.077},"last":4.7510000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.127,"15min":1.4350000000000001},"min":{"1min":0.32600000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.3160000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":4.673},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.97099999999999997,"15min":1.2709999999999999},"min":{"1min":0.35399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.333,"5min":8.266,"15min":8.266},"last":0.372}]},{"osd":7,"last update":"Sat Sep 23 18:03:28 2023","interfaces":[{"interface":"back","average":{"1min":0.84399999999999997,"5min":1.405,"15min":1.4990000000000001},"min":{"1min":0.39600000000000002,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":3.875,"5min":4.2050000000000001,"15min":4.2050000000000001},"last":4.798},{"interface":"front","average":{"1min":0.871,"5min":1.0740000000000001,"15min":1.1080000000000001},"min":{"1min":0.39100000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":3.7229999999999999,"5min":4.0380000000000003,"15min":4.0380000000000003},"last":8.3279999999999994}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":370675712,"data_stored":370671640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":853934080,"data_stored":853934080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":839520256,"data_stored":839516184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":350027776,"data_stored":350027776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":557252608,"data_stored":557252608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":540217344,"data_stored":540213272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1365311488,"data_stored":1365311488,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":588513280,"data_stored":588513280,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:04:36.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:36.489 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:04:37.054 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:04:37.054 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:04:37.055 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:33 smithi055 ceph-mon[179668]: from='client.35655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:04:37.055 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1460264654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:04:37.055 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 21928/70665 objects degraded (31.031%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:37.055 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:33 smithi055 ceph-mon[184174]: from='client.35655 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:04:37.055 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1460264654' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:04:37.055 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 21928/70665 objects degraded (31.031%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:37.056 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:37.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:04:37.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:04:37.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:34 smithi099 ceph-mon[167076]: pgmap v642: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 21460/66807 objects degraded (32.122%); 5135/66807 objects misplaced (7.686%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T18:04:37.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:34 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:04:37.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:34 smithi099 ceph-mon[167076]: failed to encode map e266 with expected crc 2023-09-23T18:04:37.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:34 smithi099 ceph-mon[167076]: failed to encode map e266 with expected crc 2023-09-23T18:04:37.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1460264654' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:04:37.058 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:34 smithi099 ceph-mon[167076]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T18:04:37.058 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:35 smithi099 ceph-mon[167076]: 1.8 scrub starts 2023-09-23T18:04:37.058 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:35 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2866540187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:37.058 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:36 smithi099 ceph-mon[167076]: pgmap v644: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 21460/60105 objects degraded (35.704%); 4701/60105 objects misplaced (7.821%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T18:04:37.059 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:04:37.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[184174]: pgmap v642: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 21460/66807 objects degraded (32.122%); 5135/66807 objects misplaced (7.686%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T18:04:37.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:04:37.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[184174]: failed to encode map e266 with expected crc 2023-09-23T18:04:37.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[184174]: failed to encode map e266 with expected crc 2023-09-23T18:04:37.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1460264654' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:04:37.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[184174]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T18:04:37.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[179668]: pgmap v642: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.4 GiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 804 op/s; 21460/66807 objects degraded (32.122%); 5135/66807 objects misplaced (7.686%); 6.2 MiB/s, 99 objects/s recovering 2023-09-23T18:04:37.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:04:37.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[179668]: failed to encode map e266 with expected crc 2023-09-23T18:04:38.820 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[179668]: failed to encode map e266 with expected crc 2023-09-23T18:04:38.820 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1460264654' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:04:38.820 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:34 smithi055 ceph-mon[179668]: osdmap e266: 8 total, 8 up, 7 in 2023-09-23T18:04:38.821 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:04:39.117 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:35 smithi055 ceph-mon[184174]: 1.8 scrub starts 2023-09-23T18:04:39.117 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:35 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2866540187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:39.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:35 smithi055 ceph-mon[179668]: 1.8 scrub starts 2023-09-23T18:04:39.118 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:35 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2866540187' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:39.118 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:04:39.119 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:04:39.120 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:04:39.120 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:04:39.120 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:04:39.120 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:04:39.120 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:04:39.120 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:04:39.397 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:04:39.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:39.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:37 smithi099 ceph-mon[167076]: 1.a scrub starts 2023-09-23T18:04:39.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:38 smithi099 ceph-mon[167076]: pgmap v645: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 831 op/s; 21460/58533 objects degraded (36.663%); 4199/58533 objects misplaced (7.174%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T18:04:39.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:38 smithi099 ceph-mon[167076]: 1.e scrub starts 2023-09-23T18:04:39.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:38 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:04:39.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 21460/58533 objects degraded (36.663%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:39.399 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:04:39.399 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:04:39.399 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:04:39.399 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:04:39.399 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:04:39.400 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:04:39.400 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:04:39.400 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:04:39.400 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:36 smithi055 ceph-mon[179668]: pgmap v644: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 21460/60105 objects degraded (35.704%); 4701/60105 objects misplaced (7.821%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T18:04:39.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:36 smithi055 ceph-mon[184174]: pgmap v644: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 827 op/s; 21460/60105 objects degraded (35.704%); 4701/60105 objects misplaced (7.821%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T18:04:39.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:39.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:37 smithi055 ceph-mon[179668]: 1.a scrub starts 2023-09-23T18:04:39.401 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:39.401 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:37 smithi055 ceph-mon[184174]: 1.a scrub starts 2023-09-23T18:04:39.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:38 smithi055 ceph-mon[184174]: pgmap v645: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 831 op/s; 21460/58533 objects degraded (36.663%); 4199/58533 objects misplaced (7.174%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T18:04:39.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:38 smithi055 ceph-mon[184174]: 1.e scrub starts 2023-09-23T18:04:39.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:38 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:04:39.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 21460/58533 objects degraded (36.663%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:39.402 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:38 smithi055 ceph-mon[179668]: pgmap v645: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.2 GiB data, 5.7 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 831 op/s; 21460/58533 objects degraded (36.663%); 4199/58533 objects misplaced (7.174%); 6.9 MiB/s, 110 objects/s recovering 2023-09-23T18:04:39.686 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:38 smithi055 ceph-mon[179668]: 1.e scrub starts 2023-09-23T18:04:39.686 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:38 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:04:39.686 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 21460/58533 objects degraded (36.663%), 9 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:39.687 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:04:39.724 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:39.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:40.145 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:04:40.145 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:04:40.154 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:40.418 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:04:40.498 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:40 smithi099 ceph-mon[167076]: pgmap v646: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 825 op/s; 20871/50502 objects degraded (41.327%); 3381/50502 objects misplaced (6.695%); 7.3 MiB/s, 123 objects/s recovering 2023-09-23T18:04:40.650 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:40 smithi099 ceph-mon[167076]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T18:04:40.650 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:40 smithi099 ceph-mon[167076]: failed to encode map e267 with expected crc 2023-09-23T18:04:40.650 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:40 smithi099 ceph-mon[167076]: failed to encode map e267 with expected crc 2023-09-23T18:04:40.650 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:40 smithi099 ceph-mon[167076]: failed to encode map e267 with expected crc 2023-09-23T18:04:40.650 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:40 smithi099 ceph-mon[167076]: 1.0 scrub starts 2023-09-23T18:04:40.651 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:04:40] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.43.0" 2023-09-23T18:04:40.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[184174]: pgmap v646: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 825 op/s; 20871/50502 objects degraded (41.327%); 3381/50502 objects misplaced (6.695%); 7.3 MiB/s, 123 objects/s recovering 2023-09-23T18:04:40.652 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[184174]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T18:04:40.652 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[184174]: failed to encode map e267 with expected crc 2023-09-23T18:04:40.652 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[184174]: failed to encode map e267 with expected crc 2023-09-23T18:04:40.652 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[184174]: failed to encode map e267 with expected crc 2023-09-23T18:04:40.652 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[184174]: 1.0 scrub starts 2023-09-23T18:04:40.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[179668]: pgmap v646: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 1.0 GiB data, 5.5 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 825 op/s; 20871/50502 objects degraded (41.327%); 3381/50502 objects misplaced (6.695%); 7.3 MiB/s, 123 objects/s recovering 2023-09-23T18:04:40.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[179668]: osdmap e267: 8 total, 8 up, 7 in 2023-09-23T18:04:40.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[179668]: failed to encode map e267 with expected crc 2023-09-23T18:04:40.653 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[179668]: failed to encode map e267 with expected crc 2023-09-23T18:04:40.653 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[179668]: failed to encode map e267 with expected crc 2023-09-23T18:04:40.653 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:40 smithi055 ceph-mon[179668]: 1.0 scrub starts 2023-09-23T18:04:40.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:41.014 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:41.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:04:41.017 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:04:41.017 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:41.215 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:41.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:04:41.273 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:41.561 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:04:41.562 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:41 smithi055 ceph-mon[179668]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T18:04:41.562 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:41 smithi055 ceph-mon[179668]: failed to encode map e268 with expected crc 2023-09-23T18:04:41.562 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:41 smithi055 ceph-mon[179668]: failed to encode map e268 with expected crc 2023-09-23T18:04:41.562 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:41 smithi055 ceph-mon[184174]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T18:04:41.563 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:41 smithi055 ceph-mon[184174]: failed to encode map e268 with expected crc 2023-09-23T18:04:41.563 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:41 smithi055 ceph-mon[184174]: failed to encode map e268 with expected crc 2023-09-23T18:04:41.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:41 smithi099 ceph-mon[167076]: osdmap e268: 8 total, 8 up, 7 in 2023-09-23T18:04:41.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:41 smithi099 ceph-mon[167076]: failed to encode map e268 with expected crc 2023-09-23T18:04:41.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:41 smithi099 ceph-mon[167076]: failed to encode map e268 with expected crc 2023-09-23T18:04:41.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:41.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:04:41.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:04:41.946 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:04:42.151 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:42.388 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:42 smithi055 ceph-mon[179668]: pgmap v649: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 959 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 20828/45927 objects degraded (45.350%); 2994/45927 objects misplaced (6.519%); 8.6 MiB/s, 144 objects/s recovering 2023-09-23T18:04:42.571 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:42 smithi055 ceph-mon[184174]: pgmap v649: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 959 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 20828/45927 objects degraded (45.350%); 2994/45927 objects misplaced (6.519%); 8.6 MiB/s, 144 objects/s recovering 2023-09-23T18:04:42.571 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:04:42.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:42.572 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:04:42.572 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:04:42.572 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.572 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:133411 6.f 6:fd346b1b:::benchmark_data_smithi055_275963_object60273:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T18:04:42.572 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:04:42.527550+0000", 2023-09-23T18:04:42.572 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0068868289999999997, 2023-09-23T18:04:42.572 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0069203770000000001, 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 133411 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:04:42.573 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527550+0000", 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527550+0000", 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527547+0000", 2023-09-23T18:04:42.574 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999971 2023-09-23T18:04:42.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:42.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527561+0000", 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3458999999999999e-05 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527590+0000", 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9210999999999999e-05 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527602+0000", 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2707999999999999e-05 2023-09-23T18:04:42.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:04:42.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528477+0000", 2023-09-23T18:04:42.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00087475699999999999 2023-09-23T18:04:42.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:04:42.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528684+0000", 2023-09-23T18:04:42.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020632300000000001 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528921+0000", 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00023710800000000001 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:42.838 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:133413 6.f 6.db489c7f (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e268)", 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:04:42.527619+0000", 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0068175579999999996, 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0069010340000000003, 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 133413 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:42.839 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:04:42.840 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:04:43.100 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527619+0000", 2023-09-23T18:04:43.100 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:43.100 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:04:43.100 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527619+0000", 2023-09-23T18:04:43.100 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:43.100 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:04:43.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527619+0000", 2023-09-23T18:04:43.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:04:43.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:43.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527622+0000", 2023-09-23T18:04:43.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9340000000000002e-06 2023-09-23T18:04:43.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.102 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:43.102 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527628+0000", 2023-09-23T18:04:43.102 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8769999999999998e-06 2023-09-23T18:04:43.102 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.102 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.102 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:43.102 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.527632+0000", 2023-09-23T18:04:43.102 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8160000000000001e-06 2023-09-23T18:04:43.102 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:43.102 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:43.103 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:43.103 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.103 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.103 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:133415 6.d e268/248 6:be8a0db7:::benchmark_data_smithi055_275963_object60277:head v 268'8274, mlcod=268'8272)", 2023-09-23T18:04:43.103 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:04:42.528706+0000", 2023-09-23T18:04:43.103 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0057305799999999999, 2023-09-23T18:04:43.103 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0058521520000000002, 2023-09-23T18:04:43.103 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:04:43.103 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:04:43.103 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:04:43.104 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.104 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:04:43.104 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528706+0000", 2023-09-23T18:04:43.104 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:43.104 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.104 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.104 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:04:43.104 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528706+0000", 2023-09-23T18:04:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:04:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528705+0000", 2023-09-23T18:04:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:04:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528716+0000", 2023-09-23T18:04:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.059e-05 2023-09-23T18:04:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528728+0000", 2023-09-23T18:04:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1956999999999999e-05 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528736+0000", 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3280000000000006e-06 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528798+0000", 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2345e-05 2023-09-23T18:04:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.528822+0000", 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3127000000000001e-05 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:133410 6.9 e268/214 6:990e795b:::benchmark_data_smithi055_275963_object60272:head v 268'8515, mlcod=268'8513)", 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:04:42.531834+0000", 2023-09-23T18:04:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0026025190000000002, 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027589060000000002, 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531834+0000", 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:04:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531834+0000", 2023-09-23T18:04:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:04:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531833+0000", 2023-09-23T18:04:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:04:43.612 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531842+0000", 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6640000000000004e-06 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531854+0000", 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1834999999999999e-05 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:43.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531863+0000", 2023-09-23T18:04:43.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5220000000000007e-06 2023-09-23T18:04:43.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:04:43.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.532805+0000", 2023-09-23T18:04:43.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000942639 2023-09-23T18:04:43.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.532824+0000", 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9102000000000001e-05 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45092.0:133418 6.3 e268/214 6:c6051a09:::benchmark_data_smithi055_275963_object60280:head v 268'8368, mlcod=268'8366)", 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:04:42.531885+0000", 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0025521390000000001, 2023-09-23T18:04:43.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027651609999999999, 2023-09-23T18:04:43.616 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:04:43.616 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:04:43.616 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:04:43.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:04:43.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531885+0000", 2023-09-23T18:04:43.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:43.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531885+0000", 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531884+0000", 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531891+0000", 2023-09-23T18:04:43.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2639999999999997e-06 2023-09-23T18:04:43.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:04:43.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531899+0000", 2023-09-23T18:04:43.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6600000000000001e-06 2023-09-23T18:04:43.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:04:43.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.531905+0000", 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.378e-06 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.532072+0000", 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016753100000000001 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:04:42.532096+0000", 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3482999999999999e-05 2023-09-23T18:04:43.835 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:43.836 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:04:43.836 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:43.836 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:04:43.836 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:04:43.836 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 5 2023-09-23T18:04:43.836 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:04:43.837 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:04:43.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:42 smithi099 ceph-mon[167076]: pgmap v649: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 959 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 851 op/s; 20828/45927 objects degraded (45.350%); 2994/45927 objects misplaced (6.519%); 8.6 MiB/s, 144 objects/s recovering 2023-09-23T18:04:44.052 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 20828/45927 objects degraded (45.350%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:44.054 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:04:44.056 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:04:44.057 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:04:44.057 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 20828/45927 objects degraded (45.350%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:44.058 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 20828/45927 objects degraded (45.350%), 8 pgs degraded, 8 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:44.058 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:04:44.059 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":650,"stamp":"2023-09-23T18:04:42.717739+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":941046278,"num_objects":14317,"num_object_clones":0,"num_object_copies":42951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20428,"num_objects_misplaced":2533,"num_objects_unfound":0,"num_objects_dirty":14317,"num_whiteouts":0,"num_read":6284,"num_read_kb":12770,"num_write":133051,"num_write_kb":4695012,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26641,"num_bytes_recovered":1617628135,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":103540,"ondisk_log_size":103540,"up":357,"acting":349,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":399,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":5328240,"kb_used_data":3690424,"kb_used_omap":43,"kb_used_meta":1637716,"kb_avail":650887824,"statfs":{"total":671965249536,"available":666509131776,"internally_reserved":0,"allocated":3778994176,"data_stored":3770686132,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1677021903},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,5,4,4,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-374734848,"num_objects":-5718,"num_object_clones":0,"num_object_copies":-17154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1032,"num_objects_misplaced":-2168,"num_objects_unfound":0,"num_objects_dirty":-5718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5718,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":917,"num_bytes_recovered":56164352,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.005473"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":591,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.829927+0000","last_change":"2023-09-23T18:03:33.004903+0000","last_active":"2023-09-23T18:04:14.829927+0000","last_peered":"2023-09-23T18:04:14.829927+0000","last_clean":"2023-09-23T18:04:14.829927+0000","last_became_active":"2023-09-23T18:03:33.004498+0000","last_became_peered":"2023-09-23T18:03:33.004498+0000","last_unstale":"2023-09-23T18:04:14.829927+0000","last_undegraded":"2023-09-23T18:04:14.829927+0000","last_fullsized":"2023-09-23T18:04:14.829927+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":737225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":66,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.823650+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:04:14.823650+0000","last_peered":"2023-09-23T18:04:14.823650+0000","last_clean":"2023-09-23T18:04:14.823650+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:04:14.823650+0000","last_undegraded":"2023-09-23T18:04:14.823650+0000","last_fullsized":"2023-09-23T18:04:14.823650+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":149,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.268026+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:04:39.268026+0000","last_peered":"2023-09-23T18:04:39.268026+0000","last_clean":"2023-09-23T18:04:39.268026+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:04:39.268026+0000","last_undegraded":"2023-09-23T18:04:39.268026+0000","last_fullsized":"2023-09-23T18:04:39.268026+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":608,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.827533+0000","last_change":"2023-09-23T18:02:51.046916+0000","last_active":"2023-09-23T18:04:14.827533+0000","last_peered":"2023-09-23T18:04:14.827533+0000","last_clean":"2023-09-23T18:04:14.827533+0000","last_became_active":"2023-09-23T18:02:51.046591+0000","last_became_peered":"2023-09-23T18:02:51.046591+0000","last_unstale":"2023-09-23T18:04:14.827533+0000","last_undegraded":"2023-09-23T18:04:14.827533+0000","last_fullsized":"2023-09-23T18:04:14.827533+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:19.805099+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":117,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.275912+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:04:39.275912+0000","last_peered":"2023-09-23T18:04:39.275912+0000","last_clean":"2023-09-23T18:04:39.275912+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:04:39.275912+0000","last_undegraded":"2023-09-23T18:04:39.275912+0000","last_fullsized":"2023-09-23T18:04:39.275912+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":268,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.192780+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:04:33.192780+0000","last_peered":"2023-09-23T18:04:33.192780+0000","last_clean":"2023-09-23T18:04:33.192780+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:04:33.192780+0000","last_undegraded":"2023-09-23T18:04:33.192780+0000","last_fullsized":"2023-09-23T18:04:33.192780+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":600,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828815+0000","last_change":"2023-09-23T18:02:51.051617+0000","last_active":"2023-09-23T18:04:14.828815+0000","last_peered":"2023-09-23T18:04:14.828815+0000","last_clean":"2023-09-23T18:04:14.828815+0000","last_became_active":"2023-09-23T18:02:51.048947+0000","last_became_peered":"2023-09-23T18:02:51.048947+0000","last_unstale":"2023-09-23T18:04:14.828815+0000","last_undegraded":"2023-09-23T18:04:14.828815+0000","last_fullsized":"2023-09-23T18:04:14.828815+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:49:34.962249+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":131,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.244083+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:04:40.244083+0000","last_peered":"2023-09-23T18:04:40.244083+0000","last_clean":"2023-09-23T18:04:40.244083+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:04:40.244083+0000","last_undegraded":"2023-09-23T18:04:40.244083+0000","last_fullsized":"2023-09-23T18:04:40.244083+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":208,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.271825+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:04:39.271825+0000","last_peered":"2023-09-23T18:04:39.271825+0000","last_clean":"2023-09-23T18:04:39.271825+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:04:39.271825+0000","last_undegraded":"2023-09-23T18:04:39.271825+0000","last_fullsized":"2023-09-23T18:04:39.271825+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":597,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828397+0000","last_change":"2023-09-23T18:02:51.052123+0000","last_active":"2023-09-23T18:04:14.828397+0000","last_peered":"2023-09-23T18:04:14.828397+0000","last_clean":"2023-09-23T18:04:14.828397+0000","last_became_active":"2023-09-23T18:02:51.051772+0000","last_became_peered":"2023-09-23T18:02:51.051772+0000","last_unstale":"2023-09-23T18:04:14.828397+0000","last_undegraded":"2023-09-23T18:04:14.828397+0000","last_fullsized":"2023-09-23T18:04:14.828397+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":247,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.183715+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:04:33.183715+0000","last_peered":"2023-09-23T18:04:33.183715+0000","last_clean":"2023-09-23T18:04:33.183715+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:04:33.183715+0000","last_undegraded":"2023-09-23T18:04:33.183715+0000","last_fullsized":"2023-09-23T18:04:33.183715+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":125,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.267749+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:04:39.267749+0000","last_peered":"2023-09-23T18:04:39.267749+0000","last_clean":"2023-09-23T18:04:39.267749+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:04:39.267749+0000","last_undegraded":"2023-09-23T18:04:39.267749+0000","last_fullsized":"2023-09-23T18:04:39.267749+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":592,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.826927+0000","last_change":"2023-09-23T18:02:51.054208+0000","last_active":"2023-09-23T18:04:14.826927+0000","last_peered":"2023-09-23T18:04:14.826927+0000","last_clean":"2023-09-23T18:04:14.826927+0000","last_became_active":"2023-09-23T18:02:51.053676+0000","last_became_peered":"2023-09-23T18:02:51.053676+0000","last_unstale":"2023-09-23T18:04:14.826927+0000","last_undegraded":"2023-09-23T18:04:14.826927+0000","last_fullsized":"2023-09-23T18:04:14.826927+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":250,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.256218+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:04:39.256218+0000","last_peered":"2023-09-23T18:04:39.256218+0000","last_clean":"2023-09-23T18:04:39.256218+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:04:39.256218+0000","last_undegraded":"2023-09-23T18:04:39.256218+0000","last_fullsized":"2023-09-23T18:04:39.256218+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":79,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.271879+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:04:39.271879+0000","last_peered":"2023-09-23T18:04:39.271879+0000","last_clean":"2023-09-23T18:04:39.271879+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:04:39.271879+0000","last_undegraded":"2023-09-23T18:04:39.271879+0000","last_fullsized":"2023-09-23T18:04:39.271879+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":597,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.827831+0000","last_change":"2023-09-23T18:02:51.058624+0000","last_active":"2023-09-23T18:04:14.827831+0000","last_peered":"2023-09-23T18:04:14.827831+0000","last_clean":"2023-09-23T18:04:14.827831+0000","last_became_active":"2023-09-23T18:02:51.057352+0000","last_became_peered":"2023-09-23T18:02:51.057352+0000","last_unstale":"2023-09-23T18:04:14.827831+0000","last_undegraded":"2023-09-23T18:04:14.827831+0000","last_fullsized":"2023-09-23T18:04:14.827831+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":32,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828321+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:04:14.828321+0000","last_peered":"2023-09-23T18:04:14.828321+0000","last_clean":"2023-09-23T18:04:14.828321+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:04:14.828321+0000","last_undegraded":"2023-09-23T18:04:14.828321+0000","last_fullsized":"2023-09-23T18:04:14.828321+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":164,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.271112+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:04:39.271112+0000","last_peered":"2023-09-23T18:04:39.271112+0000","last_clean":"2023-09-23T18:04:39.271112+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:04:39.271112+0000","last_undegraded":"2023-09-23T18:04:39.271112+0000","last_fullsized":"2023-09-23T18:04:39.271112+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":597,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828199+0000","last_change":"2023-09-23T18:02:51.059983+0000","last_active":"2023-09-23T18:04:14.828199+0000","last_peered":"2023-09-23T18:04:14.828199+0000","last_clean":"2023-09-23T18:04:14.828199+0000","last_became_active":"2023-09-23T18:02:51.057768+0000","last_became_peered":"2023-09-23T18:02:51.057768+0000","last_unstale":"2023-09-23T18:04:14.828199+0000","last_undegraded":"2023-09-23T18:04:14.828199+0000","last_fullsized":"2023-09-23T18:04:14.828199+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":66,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.824208+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:04:14.824208+0000","last_peered":"2023-09-23T18:04:14.824208+0000","last_clean":"2023-09-23T18:04:14.824208+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:04:14.824208+0000","last_undegraded":"2023-09-23T18:04:14.824208+0000","last_fullsized":"2023-09-23T18:04:14.824208+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":66,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.823871+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:04:14.823871+0000","last_peered":"2023-09-23T18:04:14.823871+0000","last_clean":"2023-09-23T18:04:14.823871+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:04:14.823871+0000","last_undegraded":"2023-09-23T18:04:14.823871+0000","last_fullsized":"2023-09-23T18:04:14.823871+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":587,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.826751+0000","last_change":"2023-09-23T18:02:51.047570+0000","last_active":"2023-09-23T18:04:14.826751+0000","last_peered":"2023-09-23T18:04:14.826751+0000","last_clean":"2023-09-23T18:04:14.826751+0000","last_became_active":"2023-09-23T18:02:51.046963+0000","last_became_peered":"2023-09-23T18:02:51.046963+0000","last_unstale":"2023-09-23T18:04:14.826751+0000","last_undegraded":"2023-09-23T18:04:14.826751+0000","last_fullsized":"2023-09-23T18:04:14.826751+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":195,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.829371+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:04:14.829371+0000","last_peered":"2023-09-23T18:04:14.829371+0000","last_clean":"2023-09-23T18:04:14.829371+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:04:14.829371+0000","last_undegraded":"2023-09-23T18:04:14.829371+0000","last_fullsized":"2023-09-23T18:04:14.829371+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":113,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.271223+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:04:39.271223+0000","last_peered":"2023-09-23T18:04:39.271223+0000","last_clean":"2023-09-23T18:04:39.271223+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:04:39.271223+0000","last_undegraded":"2023-09-23T18:04:39.271223+0000","last_fullsized":"2023-09-23T18:04:39.271223+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":600,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.829765+0000","last_change":"2023-09-23T18:02:51.053806+0000","last_active":"2023-09-23T18:04:14.829765+0000","last_peered":"2023-09-23T18:04:14.829765+0000","last_clean":"2023-09-23T18:04:14.829765+0000","last_became_active":"2023-09-23T18:02:51.048822+0000","last_became_peered":"2023-09-23T18:02:51.048822+0000","last_unstale":"2023-09-23T18:04:14.829765+0000","last_undegraded":"2023-09-23T18:04:14.829765+0000","last_fullsized":"2023-09-23T18:04:14.829765+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:34.401104+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":266,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.194033+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:04:33.194033+0000","last_peered":"2023-09-23T18:04:33.194033+0000","last_clean":"2023-09-23T18:04:33.194033+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:04:33.194033+0000","last_undegraded":"2023-09-23T18:04:33.194033+0000","last_fullsized":"2023-09-23T18:04:33.194033+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":245,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.255117+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:04:39.255117+0000","last_peered":"2023-09-23T18:04:39.255117+0000","last_clean":"2023-09-23T18:04:39.255117+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:04:39.255117+0000","last_undegraded":"2023-09-23T18:04:39.255117+0000","last_fullsized":"2023-09-23T18:04:39.255117+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":588,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.829231+0000","last_change":"2023-09-23T18:02:51.046646+0000","last_active":"2023-09-23T18:04:14.829231+0000","last_peered":"2023-09-23T18:04:14.829231+0000","last_clean":"2023-09-23T18:04:14.829231+0000","last_became_active":"2023-09-23T18:02:51.046134+0000","last_became_peered":"2023-09-23T18:02:51.046134+0000","last_unstale":"2023-09-23T18:04:14.829231+0000","last_undegraded":"2023-09-23T18:04:14.829231+0000","last_fullsized":"2023-09-23T18:04:14.829231+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":56,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.823821+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:04:14.823821+0000","last_peered":"2023-09-23T18:04:14.823821+0000","last_clean":"2023-09-23T18:04:14.823821+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:04:14.823821+0000","last_undegraded":"2023-09-23T18:04:14.823821+0000","last_fullsized":"2023-09-23T18:04:14.823821+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":257,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.192853+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:04:33.192853+0000","last_peered":"2023-09-23T18:04:33.192853+0000","last_clean":"2023-09-23T18:04:33.192853+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:04:33.192853+0000","last_undegraded":"2023-09-23T18:04:33.192853+0000","last_fullsized":"2023-09-23T18:04:33.192853+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":600,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828740+0000","last_change":"2023-09-23T18:02:51.051817+0000","last_active":"2023-09-23T18:04:14.828740+0000","last_peered":"2023-09-23T18:04:14.828740+0000","last_clean":"2023-09-23T18:04:14.828740+0000","last_became_active":"2023-09-23T18:02:51.048945+0000","last_became_peered":"2023-09-23T18:02:51.048945+0000","last_unstale":"2023-09-23T18:04:14.828740+0000","last_undegraded":"2023-09-23T18:04:14.828740+0000","last_fullsized":"2023-09-23T18:04:14.828740+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:47.163864+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":204,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.267268+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:04:39.267268+0000","last_peered":"2023-09-23T18:04:39.267268+0000","last_clean":"2023-09-23T18:04:39.267268+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:04:39.267268+0000","last_undegraded":"2023-09-23T18:04:39.267268+0000","last_fullsized":"2023-09-23T18:04:39.267268+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":122,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.246262+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:04:40.246262+0000","last_peered":"2023-09-23T18:04:40.246262+0000","last_clean":"2023-09-23T18:04:40.246262+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:04:40.246262+0000","last_undegraded":"2023-09-23T18:04:40.246262+0000","last_fullsized":"2023-09-23T18:04:40.246262+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"268'8443","reported_seq":8514,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:42.601607+0000","last_change":"2023-09-23T18:00:57.996982+0000","last_active":"2023-09-23T18:04:42.601607+0000","last_peered":"2023-09-23T18:04:42.601607+0000","last_clean":"2023-09-23T18:04:42.601607+0000","last_became_active":"2023-09-23T18:00:57.996589+0000","last_became_peered":"2023-09-23T18:00:57.996589+0000","last_unstale":"2023-09-23T18:04:42.601607+0000","last_undegraded":"2023-09-23T18:04:42.601607+0000","last_fullsized":"2023-09-23T18:04:42.601607+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":8443,"log_dups_size":0,"ondisk_log_size":8443,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:13.306089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52363264,"num_objects":799,"num_object_clones":0,"num_object_copies":2397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8443,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":597,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828242+0000","last_change":"2023-09-23T18:02:51.058477+0000","last_active":"2023-09-23T18:04:14.828242+0000","last_peered":"2023-09-23T18:04:14.828242+0000","last_clean":"2023-09-23T18:04:14.828242+0000","last_became_active":"2023-09-23T18:02:51.057258+0000","last_became_peered":"2023-09-23T18:02:51.057258+0000","last_unstale":"2023-09-23T18:04:14.828242+0000","last_undegraded":"2023-09-23T18:04:14.828242+0000","last_fullsized":"2023-09-23T18:04:14.828242+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":195,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.827965+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:04:14.827965+0000","last_peered":"2023-09-23T18:04:14.827965+0000","last_clean":"2023-09-23T18:04:14.827965+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:04:14.827965+0000","last_undegraded":"2023-09-23T18:04:14.827965+0000","last_fullsized":"2023-09-23T18:04:14.827965+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":256,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.254216+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:04:39.254216+0000","last_peered":"2023-09-23T18:04:39.254216+0000","last_clean":"2023-09-23T18:04:39.254216+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:04:39.254216+0000","last_undegraded":"2023-09-23T18:04:39.254216+0000","last_fullsized":"2023-09-23T18:04:39.254216+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"268'8150","reported_seq":15467,"reported_epoch":268,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:04:40.499805+0000","last_change":"2023-09-23T18:02:49.039290+0000","last_active":"2023-09-23T18:04:40.499805+0000","last_peered":"2023-09-23T18:04:40.499805+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:49.001792+0000","last_became_peered":"2023-09-23T18:02:49.001792+0000","last_unstale":"2023-09-23T18:04:40.499805+0000","last_undegraded":"2023-09-23T18:02:48.968288+0000","last_fullsized":"2023-09-23T18:04:40.499805+0000","mapping_epoch":250,"log_start":"234'4030","ondisk_log_start":"234'4030","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4120,"log_dups_size":0,"ondisk_log_size":4120,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.156979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61210624,"num_objects":934,"num_object_clones":0,"num_object_copies":2802,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":366,"num_objects_misplaced":1502,"num_objects_unfound":0,"num_objects_dirty":934,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8150,"num_write_kb":290688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3609,"num_bytes_recovered":236519424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[6,1,4],"avail_no_missing":["6","1"],"object_location_counts":[{"shards":"1,4,6","objects":568},{"shards":"1,6","objects":366}],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":596,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.827082+0000","last_change":"2023-09-23T18:04:02.152730+0000","last_active":"2023-09-23T18:04:14.827082+0000","last_peered":"2023-09-23T18:04:14.827082+0000","last_clean":"2023-09-23T18:04:14.827082+0000","last_became_active":"2023-09-23T18:04:02.152366+0000","last_became_peered":"2023-09-23T18:04:02.152366+0000","last_unstale":"2023-09-23T18:04:14.827082+0000","last_undegraded":"2023-09-23T18:04:14.827082+0000","last_fullsized":"2023-09-23T18:04:14.827082+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":736808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":210,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828807+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:04:14.828807+0000","last_peered":"2023-09-23T18:04:14.828807+0000","last_clean":"2023-09-23T18:04:14.828807+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:04:14.828807+0000","last_undegraded":"2023-09-23T18:04:14.828807+0000","last_fullsized":"2023-09-23T18:04:14.828807+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":276,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.823965+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:04:14.823965+0000","last_peered":"2023-09-23T18:04:14.823965+0000","last_clean":"2023-09-23T18:04:14.823965+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:04:14.823965+0000","last_undegraded":"2023-09-23T18:04:14.823965+0000","last_fullsized":"2023-09-23T18:04:14.823965+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"267'8329","reported_seq":16939,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.311895+0000","last_change":"2023-09-23T18:03:33.017384+0000","last_active":"2023-09-23T18:04:39.311895+0000","last_peered":"2023-09-23T18:04:39.311895+0000","last_clean":"2023-09-23T18:04:39.311895+0000","last_became_active":"2023-09-23T18:03:33.004338+0000","last_became_peered":"2023-09-23T18:03:33.004338+0000","last_unstale":"2023-09-23T18:04:39.311895+0000","last_undegraded":"2023-09-23T18:04:39.311895+0000","last_fullsized":"2023-09-23T18:04:39.311895+0000","mapping_epoch":259,"log_start":"224'3600","ondisk_log_start":"224'3600","created":204,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4729,"log_dups_size":0,"ondisk_log_size":4729,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:59.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59834368,"num_objects":913,"num_object_clones":0,"num_object_copies":2739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8329,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4258,"num_bytes_recovered":259457024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":588,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.827544+0000","last_change":"2023-09-23T18:02:51.058795+0000","last_active":"2023-09-23T18:04:14.827544+0000","last_peered":"2023-09-23T18:04:14.827544+0000","last_clean":"2023-09-23T18:04:14.827544+0000","last_became_active":"2023-09-23T18:02:51.057799+0000","last_became_peered":"2023-09-23T18:02:51.057799+0000","last_unstale":"2023-09-23T18:04:14.827544+0000","last_undegraded":"2023-09-23T18:04:14.827544+0000","last_fullsized":"2023-09-23T18:04:14.827544+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":133,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.829034+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:04:14.829034+0000","last_peered":"2023-09-23T18:04:14.829034+0000","last_clean":"2023-09-23T18:04:14.829034+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:04:14.829034+0000","last_undegraded":"2023-09-23T18:04:14.829034+0000","last_fullsized":"2023-09-23T18:04:14.829034+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":233,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.270992+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:04:39.270992+0000","last_peered":"2023-09-23T18:04:39.270992+0000","last_clean":"2023-09-23T18:04:39.270992+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:04:39.270992+0000","last_undegraded":"2023-09-23T18:04:39.270992+0000","last_fullsized":"2023-09-23T18:04:39.270992+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"267'8402","reported_seq":8492,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.526813+0000","last_change":"2023-09-23T18:01:25.019784+0000","last_active":"2023-09-23T18:04:39.526813+0000","last_peered":"2023-09-23T18:04:39.526813+0000","last_clean":"2023-09-23T18:04:39.526813+0000","last_became_active":"2023-09-23T18:01:25.002073+0000","last_became_peered":"2023-09-23T18:01:25.002073+0000","last_unstale":"2023-09-23T18:04:39.526813+0000","last_undegraded":"2023-09-23T18:04:39.526813+0000","last_fullsized":"2023-09-23T18:04:39.526813+0000","mapping_epoch":214,"log_start":"225'4000","ondisk_log_start":"225'4000","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4402,"log_dups_size":0,"ondisk_log_size":4402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:58.981672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58982400,"num_objects":900,"num_object_clones":0,"num_object_copies":2700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":900,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8402,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":600,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:37.604692+0000","last_change":"2023-09-23T18:04:37.604692+0000","last_active":"2023-09-23T18:04:37.604692+0000","last_peered":"2023-09-23T18:04:37.604692+0000","last_clean":"2023-09-23T18:04:37.604692+0000","last_became_active":"2023-09-23T18:02:51.056589+0000","last_became_peered":"2023-09-23T18:02:51.056589+0000","last_unstale":"2023-09-23T18:04:37.604692+0000","last_undegraded":"2023-09-23T18:04:37.604692+0000","last_fullsized":"2023-09-23T18:04:37.604692+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":244,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.255007+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:04:39.255007+0000","last_peered":"2023-09-23T18:04:39.255007+0000","last_clean":"2023-09-23T18:04:39.255007+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:04:39.255007+0000","last_undegraded":"2023-09-23T18:04:39.255007+0000","last_fullsized":"2023-09-23T18:04:39.255007+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":270,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:34.612865+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:04:34.612865+0000","last_peered":"2023-09-23T18:04:34.612865+0000","last_clean":"2023-09-23T18:04:34.612865+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:04:34.612865+0000","last_undegraded":"2023-09-23T18:04:34.612865+0000","last_fullsized":"2023-09-23T18:04:34.612865+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"266'8314","reported_seq":16598,"reported_epoch":266,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:04:38.227709+0000","last_change":"2023-09-23T18:04:38.227928+0000","last_active":"2023-09-23T18:04:38.227709+0000","last_peered":"2023-09-23T18:04:38.227709+0000","last_clean":"2023-09-23T18:01:38.416095+0000","last_became_active":"2023-09-23T18:02:07.547135+0000","last_became_peered":"2023-09-23T18:02:07.547135+0000","last_unstale":"2023-09-23T18:04:38.227709+0000","last_undegraded":"2023-09-23T18:04:09.654315+0000","last_fullsized":"2023-09-23T18:02:02.784733+0000","mapping_epoch":227,"log_start":"217'1300","ondisk_log_start":"217'1300","created":204,"last_epoch_clean":208,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:27.864301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61603840,"num_objects":940,"num_object_clones":0,"num_object_copies":2820,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":940,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8314,"num_write_kb":296128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4100,"num_bytes_recovered":264765440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2],"avail_no_missing":["4","1","2","5"],"object_location_counts":[{"shards":"2,4","objects":940}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":589,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.826667+0000","last_change":"2023-09-23T18:02:51.047609+0000","last_active":"2023-09-23T18:04:14.826667+0000","last_peered":"2023-09-23T18:04:14.826667+0000","last_clean":"2023-09-23T18:04:14.826667+0000","last_became_active":"2023-09-23T18:02:51.047209+0000","last_became_peered":"2023-09-23T18:02:51.047209+0000","last_unstale":"2023-09-23T18:04:14.826667+0000","last_undegraded":"2023-09-23T18:04:14.826667+0000","last_fullsized":"2023-09-23T18:04:14.826667+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":156,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.267825+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:04:39.267825+0000","last_peered":"2023-09-23T18:04:39.267825+0000","last_clean":"2023-09-23T18:04:39.267825+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:04:39.267825+0000","last_undegraded":"2023-09-23T18:04:39.267825+0000","last_fullsized":"2023-09-23T18:04:39.267825+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":274,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.824737+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:04:14.824737+0000","last_peered":"2023-09-23T18:04:14.824737+0000","last_clean":"2023-09-23T18:04:14.824737+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:04:14.824737+0000","last_undegraded":"2023-09-23T18:04:14.824737+0000","last_fullsized":"2023-09-23T18:04:14.824737+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"267'8121","reported_seq":8350,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:39.330036+0000","last_change":"2023-09-23T18:02:47.961575+0000","last_active":"2023-09-23T18:04:39.330036+0000","last_peered":"2023-09-23T18:04:39.330036+0000","last_clean":"2023-09-23T18:02:17.013863+0000","last_became_active":"2023-09-23T18:02:19.267215+0000","last_became_peered":"2023-09-23T18:02:19.267215+0000","last_unstale":"2023-09-23T18:04:39.330036+0000","last_undegraded":"2023-09-23T18:02:18.955030+0000","last_fullsized":"2023-09-23T18:02:18.924656+0000","mapping_epoch":238,"log_start":"224'3400","ondisk_log_start":"224'3400","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4721,"log_dups_size":0,"ondisk_log_size":4721,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:46.393021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":61276160,"num_objects":935,"num_object_clones":0,"num_object_copies":2805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8121,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":4128768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-3173},{"shards":"2,4,6","objects":577}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":596,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:34.596732+0000","last_change":"2023-09-23T18:04:34.596732+0000","last_active":"2023-09-23T18:04:34.596732+0000","last_peered":"2023-09-23T18:04:34.596732+0000","last_clean":"2023-09-23T18:04:34.596732+0000","last_became_active":"2023-09-23T18:02:51.048477+0000","last_became_peered":"2023-09-23T18:02:51.048477+0000","last_unstale":"2023-09-23T18:04:34.596732+0000","last_undegraded":"2023-09-23T18:04:34.596732+0000","last_fullsized":"2023-09-23T18:04:34.596732+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":227,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.271233+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:04:39.271233+0000","last_peered":"2023-09-23T18:04:39.271233+0000","last_clean":"2023-09-23T18:04:39.271233+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:04:39.271233+0000","last_undegraded":"2023-09-23T18:04:39.271233+0000","last_fullsized":"2023-09-23T18:04:39.271233+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":66,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.824593+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:04:14.824593+0000","last_peered":"2023-09-23T18:04:14.824593+0000","last_clean":"2023-09-23T18:04:14.824593+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:04:14.824593+0000","last_undegraded":"2023-09-23T18:04:14.824593+0000","last_fullsized":"2023-09-23T18:04:14.824593+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"268'8265","reported_seq":8341,"reported_epoch":268,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:04:42.642596+0000","last_change":"2023-09-23T18:02:19.054827+0000","last_active":"2023-09-23T18:04:42.642596+0000","last_peered":"2023-09-23T18:04:42.642596+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.048672+0000","last_became_peered":"2023-09-23T18:02:19.048672+0000","last_unstale":"2023-09-23T18:04:42.642596+0000","last_undegraded":"2023-09-23T18:02:18.951263+0000","last_fullsized":"2023-09-23T18:02:18.930851+0000","mapping_epoch":238,"log_start":"216'261","ondisk_log_start":"216'261","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":8004,"log_dups_size":0,"ondisk_log_size":8004,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:23.690233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54198272,"num_objects":827,"num_object_clones":0,"num_object_copies":2481,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":827,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":827,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8265,"num_write_kb":290944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[{"shards":"5,6","objects":827}],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":597,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.827456+0000","last_change":"2023-09-23T18:02:51.060858+0000","last_active":"2023-09-23T18:04:14.827456+0000","last_peered":"2023-09-23T18:04:14.827456+0000","last_clean":"2023-09-23T18:04:14.827456+0000","last_became_active":"2023-09-23T18:02:51.060554+0000","last_became_peered":"2023-09-23T18:02:51.060554+0000","last_unstale":"2023-09-23T18:04:14.827456+0000","last_undegraded":"2023-09-23T18:04:14.827456+0000","last_fullsized":"2023-09-23T18:04:14.827456+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":56,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.823864+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:04:14.823864+0000","last_peered":"2023-09-23T18:04:14.823864+0000","last_clean":"2023-09-23T18:04:14.823864+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:04:14.823864+0000","last_undegraded":"2023-09-23T18:04:14.823864+0000","last_fullsized":"2023-09-23T18:04:14.823864+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":276,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828392+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:04:14.828392+0000","last_peered":"2023-09-23T18:04:14.828392+0000","last_clean":"2023-09-23T18:04:14.828392+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:04:14.828392+0000","last_undegraded":"2023-09-23T18:04:14.828392+0000","last_fullsized":"2023-09-23T18:04:14.828392+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"268'8279","reported_seq":16404,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:42.584311+0000","last_change":"2023-09-23T18:02:47.017672+0000","last_active":"2023-09-23T18:04:42.584311+0000","last_peered":"2023-09-23T18:04:42.584311+0000","last_clean":"2023-09-23T18:04:42.584311+0000","last_became_active":"2023-09-23T18:02:47.009245+0000","last_became_peered":"2023-09-23T18:02:47.009245+0000","last_unstale":"2023-09-23T18:04:42.584311+0000","last_undegraded":"2023-09-23T18:04:42.584311+0000","last_fullsized":"2023-09-23T18:04:42.584311+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":8279,"log_dups_size":0,"ondisk_log_size":8279,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:04.868743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":52822040,"num_objects":807,"num_object_clones":0,"num_object_copies":2421,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":807,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8279,"num_write_kb":290689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4015,"num_bytes_recovered":263127040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":592,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:36.646437+0000","last_change":"2023-09-23T18:04:36.646437+0000","last_active":"2023-09-23T18:04:36.646437+0000","last_peered":"2023-09-23T18:04:36.646437+0000","last_clean":"2023-09-23T18:04:36.646437+0000","last_became_active":"2023-09-23T18:02:51.049362+0000","last_became_peered":"2023-09-23T18:02:51.049362+0000","last_unstale":"2023-09-23T18:04:36.646437+0000","last_undegraded":"2023-09-23T18:04:36.646437+0000","last_fullsized":"2023-09-23T18:04:36.646437+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":114,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.267902+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:04:39.267902+0000","last_peered":"2023-09-23T18:04:39.267902+0000","last_clean":"2023-09-23T18:04:39.267902+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:04:39.267902+0000","last_undegraded":"2023-09-23T18:04:39.267902+0000","last_fullsized":"2023-09-23T18:04:39.267902+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":114,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.273881+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:04:39.273881+0000","last_peered":"2023-09-23T18:04:39.273881+0000","last_clean":"2023-09-23T18:04:39.273881+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:04:39.273881+0000","last_undegraded":"2023-09-23T18:04:39.273881+0000","last_fullsized":"2023-09-23T18:04:39.273881+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"267'7996","reported_seq":8073,"reported_epoch":267,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:39.547827+0000","last_change":"2023-09-23T18:02:19.315376+0000","last_active":"2023-09-23T18:04:39.547827+0000","last_peered":"2023-09-23T18:04:39.547827+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.306174+0000","last_became_peered":"2023-09-23T18:02:19.306174+0000","last_unstale":"2023-09-23T18:04:39.547827+0000","last_undegraded":"2023-09-23T18:02:18.954031+0000","last_fullsized":"2023-09-23T18:02:18.922505+0000","mapping_epoch":238,"log_start":"224'3600","ondisk_log_start":"224'3600","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4396,"log_dups_size":0,"ondisk_log_size":4396,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:52.392034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":58064896,"num_objects":886,"num_object_clones":0,"num_object_copies":2658,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":886,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7996,"num_write_kb":284224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":-3209}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":598,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.827468+0000","last_change":"2023-09-23T18:02:51.059934+0000","last_active":"2023-09-23T18:04:14.827468+0000","last_peered":"2023-09-23T18:04:14.827468+0000","last_clean":"2023-09-23T18:04:14.827468+0000","last_became_active":"2023-09-23T18:02:51.059638+0000","last_became_peered":"2023-09-23T18:02:51.059638+0000","last_unstale":"2023-09-23T18:04:14.827468+0000","last_undegraded":"2023-09-23T18:04:14.827468+0000","last_fullsized":"2023-09-23T18:04:14.827468+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":577,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.183169+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:04:33.183169+0000","last_peered":"2023-09-23T18:04:33.183169+0000","last_clean":"2023-09-23T18:04:33.183169+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:04:33.183169+0000","last_undegraded":"2023-09-23T18:04:33.183169+0000","last_fullsized":"2023-09-23T18:04:33.183169+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":248,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.255200+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:04:39.255200+0000","last_peered":"2023-09-23T18:04:39.255200+0000","last_clean":"2023-09-23T18:04:39.255200+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:04:39.255200+0000","last_undegraded":"2023-09-23T18:04:39.255200+0000","last_fullsized":"2023-09-23T18:04:39.255200+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":245,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.254654+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:04:39.254654+0000","last_peered":"2023-09-23T18:04:39.254654+0000","last_clean":"2023-09-23T18:04:39.254654+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:04:39.254654+0000","last_undegraded":"2023-09-23T18:04:39.254654+0000","last_fullsized":"2023-09-23T18:04:39.254654+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"266'7986","reported_seq":8076,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:38.234201+0000","last_change":"2023-09-23T18:02:19.289610+0000","last_active":"2023-09-23T18:04:38.234201+0000","last_peered":"2023-09-23T18:04:38.234201+0000","last_clean":"2023-09-23T18:02:17.060883+0000","last_became_active":"2023-09-23T18:02:19.266879+0000","last_became_peered":"2023-09-23T18:02:19.266879+0000","last_unstale":"2023-09-23T18:04:38.234201+0000","last_undegraded":"2023-09-23T18:02:18.963390+0000","last_fullsized":"2023-09-23T18:02:18.926482+0000","mapping_epoch":238,"log_start":"217'1000","ondisk_log_start":"217'1000","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":6986,"log_dups_size":0,"ondisk_log_size":6986,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.499269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63700992,"num_objects":972,"num_object_clones":0,"num_object_copies":2916,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":972,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7986,"num_write_kb":286656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,6,7","objects":621},{"shards":"4,7","objects":-3156}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":600,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.826576+0000","last_change":"2023-09-23T18:02:51.047767+0000","last_active":"2023-09-23T18:04:14.826576+0000","last_peered":"2023-09-23T18:04:14.826576+0000","last_clean":"2023-09-23T18:04:14.826576+0000","last_became_active":"2023-09-23T18:02:51.047297+0000","last_became_peered":"2023-09-23T18:02:51.047297+0000","last_unstale":"2023-09-23T18:04:14.826576+0000","last_undegraded":"2023-09-23T18:04:14.826576+0000","last_fullsized":"2023-09-23T18:04:14.826576+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.851694+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1196,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.254740+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:04:39.254740+0000","last_peered":"2023-09-23T18:04:39.254740+0000","last_clean":"2023-09-23T18:04:39.254740+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:04:39.254740+0000","last_undegraded":"2023-09-23T18:04:39.254740+0000","last_fullsized":"2023-09-23T18:04:39.254740+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":56,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.823954+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:04:14.823954+0000","last_peered":"2023-09-23T18:04:14.823954+0000","last_clean":"2023-09-23T18:04:14.823954+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:04:14.823954+0000","last_undegraded":"2023-09-23T18:04:14.823954+0000","last_fullsized":"2023-09-23T18:04:14.823954+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":161,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.164037+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:04:33.164037+0000","last_peered":"2023-09-23T18:04:33.164037+0000","last_clean":"2023-09-23T18:04:33.164037+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:04:33.164037+0000","last_undegraded":"2023-09-23T18:04:33.164037+0000","last_fullsized":"2023-09-23T18:04:33.164037+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"266'8293","reported_seq":16926,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:37.949793+0000","last_change":"2023-09-23T18:04:02.179479+0000","last_active":"2023-09-23T18:04:37.949793+0000","last_peered":"2023-09-23T18:04:37.949793+0000","last_clean":"2023-09-23T18:04:37.949793+0000","last_became_active":"2023-09-23T18:04:02.150813+0000","last_became_peered":"2023-09-23T18:04:02.150813+0000","last_unstale":"2023-09-23T18:04:37.949793+0000","last_undegraded":"2023-09-23T18:04:37.949793+0000","last_fullsized":"2023-09-23T18:04:37.949793+0000","mapping_epoch":263,"log_start":"225'3974","ondisk_log_start":"225'3974","created":204,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4319,"log_dups_size":0,"ondisk_log_size":4319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:41.194322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":63635456,"num_objects":971,"num_object_clones":0,"num_object_copies":2913,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":971,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8293,"num_write_kb":296448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4277,"num_bytes_recovered":179372032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":601,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828579+0000","last_change":"2023-09-23T18:02:51.054622+0000","last_active":"2023-09-23T18:04:14.828579+0000","last_peered":"2023-09-23T18:04:14.828579+0000","last_clean":"2023-09-23T18:04:14.828579+0000","last_became_active":"2023-09-23T18:02:51.049345+0000","last_became_peered":"2023-09-23T18:02:51.049345+0000","last_unstale":"2023-09-23T18:04:14.828579+0000","last_undegraded":"2023-09-23T18:04:14.828579+0000","last_fullsized":"2023-09-23T18:04:14.828579+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:58.636010+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":549,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.171658+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:04:33.171658+0000","last_peered":"2023-09-23T18:04:33.171658+0000","last_clean":"2023-09-23T18:04:33.171658+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:04:33.171658+0000","last_undegraded":"2023-09-23T18:04:33.171658+0000","last_fullsized":"2023-09-23T18:04:33.171658+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":162,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.166256+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:04:33.166256+0000","last_peered":"2023-09-23T18:04:33.166256+0000","last_clean":"2023-09-23T18:04:33.166256+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:04:33.166256+0000","last_undegraded":"2023-09-23T18:04:33.166256+0000","last_fullsized":"2023-09-23T18:04:33.166256+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":235,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.256269+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:04:39.256269+0000","last_peered":"2023-09-23T18:04:39.256269+0000","last_clean":"2023-09-23T18:04:39.256269+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:04:39.256269+0000","last_undegraded":"2023-09-23T18:04:39.256269+0000","last_fullsized":"2023-09-23T18:04:39.256269+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"268'8447","reported_seq":12608,"reported_epoch":268,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:42.652775+0000","last_change":"2023-09-23T18:04:00.228250+0000","last_active":"2023-09-23T18:04:42.652775+0000","last_peered":"2023-09-23T18:04:42.652775+0000","last_clean":"2023-09-23T18:02:16.962656+0000","last_became_active":"2023-09-23T18:02:19.272896+0000","last_became_peered":"2023-09-23T18:02:19.272896+0000","last_unstale":"2023-09-23T18:04:42.652775+0000","last_undegraded":"2023-09-23T18:02:18.972349+0000","last_fullsized":"2023-09-23T18:02:18.930868+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":8447,"log_dups_size":0,"ondisk_log_size":8447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:23.745675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":51970048,"num_objects":793,"num_object_clones":0,"num_object_copies":2379,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1959,"num_objects_misplaced":276,"num_objects_unfound":0,"num_objects_dirty":793,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8447,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2029,"num_bytes_recovered":132972544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"1,4,5","objects":276},{"shards":"4,5","objects":-1442}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1180,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.254302+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:04:39.254302+0000","last_peered":"2023-09-23T18:04:39.254302+0000","last_clean":"2023-09-23T18:04:39.254302+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:04:39.254302+0000","last_undegraded":"2023-09-23T18:04:39.254302+0000","last_fullsized":"2023-09-23T18:04:39.254302+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":589,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.829630+0000","last_change":"2023-09-23T18:02:51.054022+0000","last_active":"2023-09-23T18:04:14.829630+0000","last_peered":"2023-09-23T18:04:14.829630+0000","last_clean":"2023-09-23T18:04:14.829630+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:04:14.829630+0000","last_undegraded":"2023-09-23T18:04:14.829630+0000","last_fullsized":"2023-09-23T18:04:14.829630+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":76,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.276386+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:04:39.276386+0000","last_peered":"2023-09-23T18:04:39.276386+0000","last_clean":"2023-09-23T18:04:39.276386+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:04:39.276386+0000","last_undegraded":"2023-09-23T18:04:39.276386+0000","last_fullsized":"2023-09-23T18:04:39.276386+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":201,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.268457+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:04:39.268457+0000","last_peered":"2023-09-23T18:04:39.268457+0000","last_clean":"2023-09-23T18:04:39.268457+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:04:39.268457+0000","last_undegraded":"2023-09-23T18:04:39.268457+0000","last_fullsized":"2023-09-23T18:04:39.268457+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"268'8327","reported_seq":8493,"reported_epoch":268,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:04:42.657125+0000","last_change":"2023-09-23T18:02:18.003772+0000","last_active":"2023-09-23T18:04:42.657125+0000","last_peered":"2023-09-23T18:04:42.657125+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:18.000696+0000","last_became_peered":"2023-09-23T18:02:18.000696+0000","last_unstale":"2023-09-23T18:04:42.657125+0000","last_undegraded":"2023-09-23T18:02:17.938873+0000","last_fullsized":"2023-09-23T18:02:17.919669+0000","mapping_epoch":237,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":8227,"log_dups_size":0,"ondisk_log_size":8227,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:48.085957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49479680,"num_objects":755,"num_object_clones":0,"num_object_copies":2265,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":746,"num_objects_misplaced":755,"num_objects_unfound":0,"num_objects_dirty":755,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8327,"num_write_kb":290624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,1],"avail_no_missing":["5","1"],"object_location_counts":[{"shards":"1,5","objects":755}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":601,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.827379+0000","last_change":"2023-09-23T18:02:51.061168+0000","last_active":"2023-09-23T18:04:14.827379+0000","last_peered":"2023-09-23T18:04:14.827379+0000","last_clean":"2023-09-23T18:04:14.827379+0000","last_became_active":"2023-09-23T18:02:51.060689+0000","last_became_peered":"2023-09-23T18:02:51.060689+0000","last_unstale":"2023-09-23T18:04:14.827379+0000","last_undegraded":"2023-09-23T18:04:14.827379+0000","last_fullsized":"2023-09-23T18:04:14.827379+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:29.406460+0000","scrub_duration":0.001568505,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":446,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.830296+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:04:14.830296+0000","last_peered":"2023-09-23T18:04:14.830296+0000","last_clean":"2023-09-23T18:04:14.830296+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:04:14.830296+0000","last_undegraded":"2023-09-23T18:04:14.830296+0000","last_fullsized":"2023-09-23T18:04:14.830296+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":266,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.183278+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:04:33.183278+0000","last_peered":"2023-09-23T18:04:33.183278+0000","last_clean":"2023-09-23T18:04:33.183278+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:04:33.183278+0000","last_undegraded":"2023-09-23T18:04:33.183278+0000","last_fullsized":"2023-09-23T18:04:33.183278+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":237,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.192596+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:04:33.192596+0000","last_peered":"2023-09-23T18:04:33.192596+0000","last_clean":"2023-09-23T18:04:33.192596+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:04:33.192596+0000","last_undegraded":"2023-09-23T18:04:33.192596+0000","last_fullsized":"2023-09-23T18:04:33.192596+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"268'8330","reported_seq":16679,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.431982+0000","last_change":"2023-09-23T18:02:18.942886+0000","last_active":"2023-09-23T18:04:40.431982+0000","last_peered":"2023-09-23T18:04:40.431982+0000","last_clean":"2023-09-23T18:04:40.431982+0000","last_became_active":"2023-09-23T18:02:18.936691+0000","last_became_peered":"2023-09-23T18:02:18.936691+0000","last_unstale":"2023-09-23T18:04:40.431982+0000","last_undegraded":"2023-09-23T18:04:40.431982+0000","last_fullsized":"2023-09-23T18:04:40.431982+0000","mapping_epoch":238,"log_start":"233'4158","ondisk_log_start":"233'4158","created":204,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4172,"log_dups_size":0,"ondisk_log_size":4172,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:26.085859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":59375616,"num_objects":906,"num_object_clones":0,"num_object_copies":2718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8330,"num_write_kb":295552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4123,"num_bytes_recovered":270204928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":598,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828455+0000","last_change":"2023-09-23T18:02:51.057370+0000","last_active":"2023-09-23T18:04:14.828455+0000","last_peered":"2023-09-23T18:04:14.828455+0000","last_clean":"2023-09-23T18:04:14.828455+0000","last_became_active":"2023-09-23T18:02:51.056158+0000","last_became_peered":"2023-09-23T18:02:51.056158+0000","last_unstale":"2023-09-23T18:04:14.828455+0000","last_undegraded":"2023-09-23T18:04:14.828455+0000","last_fullsized":"2023-09-23T18:04:14.828455+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":480,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828947+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:04:14.828947+0000","last_peered":"2023-09-23T18:04:14.828947+0000","last_clean":"2023-09-23T18:04:14.828947+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:04:14.828947+0000","last_undegraded":"2023-09-23T18:04:14.828947+0000","last_fullsized":"2023-09-23T18:04:14.828947+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":202,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.164120+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:04:33.164120+0000","last_peered":"2023-09-23T18:04:33.164120+0000","last_clean":"2023-09-23T18:04:33.164120+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:04:33.164120+0000","last_undegraded":"2023-09-23T18:04:33.164120+0000","last_fullsized":"2023-09-23T18:04:33.164120+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":234,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.197830+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:04:33.197830+0000","last_peered":"2023-09-23T18:04:33.197830+0000","last_clean":"2023-09-23T18:04:33.197830+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:04:33.197830+0000","last_undegraded":"2023-09-23T18:04:33.197830+0000","last_fullsized":"2023-09-23T18:04:33.197830+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"266'8110","reported_seq":8203,"reported_epoch":266,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:38.214749+0000","last_change":"2023-09-23T18:02:19.279112+0000","last_active":"2023-09-23T18:04:38.214749+0000","last_peered":"2023-09-23T18:04:38.214749+0000","last_clean":"2023-09-23T18:02:17.018900+0000","last_became_active":"2023-09-23T18:02:19.267439+0000","last_became_peered":"2023-09-23T18:02:19.267439+0000","last_unstale":"2023-09-23T18:04:38.214749+0000","last_undegraded":"2023-09-23T18:02:18.959329+0000","last_fullsized":"2023-09-23T18:02:18.926801+0000","mapping_epoch":238,"log_start":"217'1100","ondisk_log_start":"217'1100","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7010,"log_dups_size":0,"ondisk_log_size":7010,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.395583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":65798144,"num_objects":1004,"num_object_clones":0,"num_object_copies":3012,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1004,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8110,"num_write_kb":291648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-3195},{"shards":"2,4,6","objects":646}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":598,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828168+0000","last_change":"2023-09-23T18:02:51.058680+0000","last_active":"2023-09-23T18:04:14.828168+0000","last_peered":"2023-09-23T18:04:14.828168+0000","last_clean":"2023-09-23T18:04:14.828168+0000","last_became_active":"2023-09-23T18:02:51.057706+0000","last_became_peered":"2023-09-23T18:02:51.057706+0000","last_unstale":"2023-09-23T18:04:14.828168+0000","last_undegraded":"2023-09-23T18:04:14.828168+0000","last_fullsized":"2023-09-23T18:04:14.828168+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":114,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.273835+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:04:39.273835+0000","last_peered":"2023-09-23T18:04:39.273835+0000","last_clean":"2023-09-23T18:04:39.273835+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:04:39.273835+0000","last_undegraded":"2023-09-23T18:04:39.273835+0000","last_fullsized":"2023-09-23T18:04:39.273835+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":338,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.267754+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:04:39.267754+0000","last_peered":"2023-09-23T18:04:39.267754+0000","last_clean":"2023-09-23T18:04:39.267754+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:04:39.267754+0000","last_undegraded":"2023-09-23T18:04:39.267754+0000","last_fullsized":"2023-09-23T18:04:39.267754+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":118,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.267829+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:04:39.267829+0000","last_peered":"2023-09-23T18:04:39.267829+0000","last_clean":"2023-09-23T18:04:39.267829+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:04:39.267829+0000","last_undegraded":"2023-09-23T18:04:39.267829+0000","last_fullsized":"2023-09-23T18:04:39.267829+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"266'8190","reported_seq":8262,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:37.949770+0000","last_change":"2023-09-23T18:01:25.046224+0000","last_active":"2023-09-23T18:04:37.949770+0000","last_peered":"2023-09-23T18:04:37.949770+0000","last_clean":"2023-09-23T18:04:37.949770+0000","last_became_active":"2023-09-23T18:01:25.002864+0000","last_became_peered":"2023-09-23T18:01:25.002864+0000","last_unstale":"2023-09-23T18:04:37.949770+0000","last_undegraded":"2023-09-23T18:04:37.949770+0000","last_fullsized":"2023-09-23T18:04:37.949770+0000","mapping_epoch":214,"log_start":"225'3941","ondisk_log_start":"225'3941","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4249,"log_dups_size":0,"ondisk_log_size":4249,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.752359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":62128128,"num_objects":948,"num_object_clones":0,"num_object_copies":2844,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":948,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8190,"num_write_kb":292416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":605,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.827007+0000","last_change":"2023-09-23T18:02:51.053420+0000","last_active":"2023-09-23T18:04:14.827007+0000","last_peered":"2023-09-23T18:04:14.827007+0000","last_clean":"2023-09-23T18:04:14.827007+0000","last_became_active":"2023-09-23T18:02:51.053035+0000","last_became_peered":"2023-09-23T18:02:51.053035+0000","last_unstale":"2023-09-23T18:04:14.827007+0000","last_undegraded":"2023-09-23T18:04:14.827007+0000","last_fullsized":"2023-09-23T18:04:14.827007+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:20.456481+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":473,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.271946+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:04:39.271946+0000","last_peered":"2023-09-23T18:04:39.271946+0000","last_clean":"2023-09-23T18:04:39.271946+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:04:39.271946+0000","last_undegraded":"2023-09-23T18:04:39.271946+0000","last_fullsized":"2023-09-23T18:04:39.271946+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":294,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.192702+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:04:33.192702+0000","last_peered":"2023-09-23T18:04:33.192702+0000","last_clean":"2023-09-23T18:04:33.192702+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:04:33.192702+0000","last_undegraded":"2023-09-23T18:04:33.192702+0000","last_fullsized":"2023-09-23T18:04:33.192702+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":56,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.823775+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:04:14.823775+0000","last_peered":"2023-09-23T18:04:14.823775+0000","last_clean":"2023-09-23T18:04:14.823775+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:04:14.823775+0000","last_undegraded":"2023-09-23T18:04:14.823775+0000","last_fullsized":"2023-09-23T18:04:14.823775+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":588,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.826834+0000","last_change":"2023-09-23T18:02:51.046571+0000","last_active":"2023-09-23T18:04:14.826834+0000","last_peered":"2023-09-23T18:04:14.826834+0000","last_clean":"2023-09-23T18:04:14.826834+0000","last_became_active":"2023-09-23T18:02:51.046213+0000","last_became_peered":"2023-09-23T18:02:51.046213+0000","last_unstale":"2023-09-23T18:04:14.826834+0000","last_undegraded":"2023-09-23T18:04:14.826834+0000","last_fullsized":"2023-09-23T18:04:14.826834+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":253,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.197750+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:04:33.197750+0000","last_peered":"2023-09-23T18:04:33.197750+0000","last_clean":"2023-09-23T18:04:33.197750+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:04:33.197750+0000","last_undegraded":"2023-09-23T18:04:33.197750+0000","last_fullsized":"2023-09-23T18:04:33.197750+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":280,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.164194+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:04:33.164194+0000","last_peered":"2023-09-23T18:04:33.164194+0000","last_clean":"2023-09-23T18:04:33.164194+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:04:33.164194+0000","last_undegraded":"2023-09-23T18:04:33.164194+0000","last_fullsized":"2023-09-23T18:04:33.164194+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":597,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828297+0000","last_change":"2023-09-23T18:02:51.060198+0000","last_active":"2023-09-23T18:04:14.828297+0000","last_peered":"2023-09-23T18:04:14.828297+0000","last_clean":"2023-09-23T18:04:14.828297+0000","last_became_active":"2023-09-23T18:02:51.057046+0000","last_became_peered":"2023-09-23T18:02:51.057046+0000","last_unstale":"2023-09-23T18:04:14.828297+0000","last_undegraded":"2023-09-23T18:04:14.828297+0000","last_fullsized":"2023-09-23T18:04:14.828297+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":123,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.254963+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:04:39.254963+0000","last_peered":"2023-09-23T18:04:39.254963+0000","last_clean":"2023-09-23T18:04:39.254963+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:04:39.254963+0000","last_undegraded":"2023-09-23T18:04:39.254963+0000","last_fullsized":"2023-09-23T18:04:39.254963+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":191,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.824089+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:04:14.824089+0000","last_peered":"2023-09-23T18:04:14.824089+0000","last_clean":"2023-09-23T18:04:14.824089+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:04:14.824089+0000","last_undegraded":"2023-09-23T18:04:14.824089+0000","last_fullsized":"2023-09-23T18:04:14.824089+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":109,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.254969+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:04:39.254969+0000","last_peered":"2023-09-23T18:04:39.254969+0000","last_clean":"2023-09-23T18:04:39.254969+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:04:39.254969+0000","last_undegraded":"2023-09-23T18:04:39.254969+0000","last_fullsized":"2023-09-23T18:04:39.254969+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":132,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.276215+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:04:39.276215+0000","last_peered":"2023-09-23T18:04:39.276215+0000","last_clean":"2023-09-23T18:04:39.276215+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:04:39.276215+0000","last_undegraded":"2023-09-23T18:04:39.276215+0000","last_fullsized":"2023-09-23T18:04:39.276215+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":613,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.827164+0000","last_change":"2023-09-23T18:02:51.047476+0000","last_active":"2023-09-23T18:04:14.827164+0000","last_peered":"2023-09-23T18:04:14.827164+0000","last_clean":"2023-09-23T18:04:14.827164+0000","last_became_active":"2023-09-23T18:02:51.047152+0000","last_became_peered":"2023-09-23T18:02:51.047152+0000","last_unstale":"2023-09-23T18:04:14.827164+0000","last_undegraded":"2023-09-23T18:04:14.827164+0000","last_fullsized":"2023-09-23T18:04:14.827164+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:22.514077+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":133,"reported_epoch":267,"state":"active+clean","last_fresh":"2023-09-23T18:04:39.273934+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:04:39.273934+0000","last_peered":"2023-09-23T18:04:39.273934+0000","last_clean":"2023-09-23T18:04:39.273934+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:04:39.273934+0000","last_undegraded":"2023-09-23T18:04:39.273934+0000","last_fullsized":"2023-09-23T18:04:39.273934+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":136,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.244039+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:04:40.244039+0000","last_peered":"2023-09-23T18:04:40.244039+0000","last_clean":"2023-09-23T18:04:40.244039+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:04:40.244039+0000","last_undegraded":"2023-09-23T18:04:40.244039+0000","last_fullsized":"2023-09-23T18:04:40.244039+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":596,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.828086+0000","last_change":"2023-09-23T18:02:51.058825+0000","last_active":"2023-09-23T18:04:14.828086+0000","last_peered":"2023-09-23T18:04:14.828086+0000","last_clean":"2023-09-23T18:04:14.828086+0000","last_became_active":"2023-09-23T18:02:51.058480+0000","last_became_peered":"2023-09-23T18:02:51.058480+0000","last_unstale":"2023-09-23T18:04:14.828086+0000","last_undegraded":"2023-09-23T18:04:14.828086+0000","last_fullsized":"2023-09-23T18:04:14.828086+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":246,"reported_epoch":266,"state":"active+clean","last_fresh":"2023-09-23T18:04:33.183790+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:04:33.183790+0000","last_peered":"2023-09-23T18:04:33.183790+0000","last_clean":"2023-09-23T18:04:33.183790+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:04:33.183790+0000","last_undegraded":"2023-09-23T18:04:33.183790+0000","last_fullsized":"2023-09-23T18:04:33.183790+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":276,"reported_epoch":265,"state":"active+clean","last_fresh":"2023-09-23T18:04:14.823671+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:04:14.823671+0000","last_peered":"2023-09-23T18:04:14.823671+0000","last_clean":"2023-09-23T18:04:14.823671+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:04:14.823671+0000","last_undegraded":"2023-09-23T18:04:14.823671+0000","last_fullsized":"2023-09-23T18:04:14.823671+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":936443928,"num_objects":14290,"num_object_clones":0,"num_object_copies":42870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":20428,"num_objects_misplaced":2533,"num_objects_unfound":0,"num_objects_dirty":14290,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":131982,"num_write_kb":4680641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26539,"num_bytes_recovered":1614807040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3939840000,"data_stored":3939827784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":97518,"ondisk_log_size":97518,"up":48,"acting":40,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":402523,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":331,"num_read_kb":623,"num_write":162,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2814833,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001384,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":496888,"kb_used_data":386544,"kb_used_omap":13,"kb_used_meta":110322,"kb_avail":93248264,"statfs":{"total":95995035648,"available":95486222336,"internally_reserved":0,"allocated":395821056,"data_stored":394798073,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":112969832},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.64000000000000001,"15min":0.623},"min":{"1min":0.25600000000000001,"5min":0.247,"15min":0.247},"max":{"1min":6.2770000000000001,"5min":6.2770000000000001,"15min":6.2770000000000001},"last":1.923},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.198,"15min":1.2569999999999999},"min":{"1min":0.26400000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.641,"5min":6.641,"15min":6.641},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":0.90700000000000003,"15min":0.754},"min":{"1min":0.26600000000000001,"5min":0.252,"15min":0.252},"max":{"1min":7.1150000000000002,"5min":7.1150000000000002,"15min":7.1150000000000002},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.3089999999999999,"5min":1.115,"15min":1.075},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.319,"5min":6.319,"15min":6.319},"last":0.35999999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.92800000000000005,"15min":0.93400000000000005},"min":{"1min":0.29599999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.1240000000000001,"5min":4.1539999999999999,"15min":4.1539999999999999},"last":6.6630000000000003},{"interface":"front","average":{"1min":1.339,"5min":0.98699999999999999,"15min":0.871},"min":{"1min":0.30399999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.1989999999999998,"5min":6.1989999999999998,"15min":6.1989999999999998},"last":3.823}]},{"osd":3,"last update":"Sat Sep 23 18:04:20 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.4950000000000001,"15min":1.534},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.6070000000000002,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.5900000000000001,"5min":1.0940000000000001,"15min":1.0109999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.776999999999999,"5min":10.776999999999999,"15min":10.776999999999999},"last":0.51500000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.046,"15min":1.0629999999999999},"min":{"1min":0.28899999999999998,"5min":0.245,"15min":0.245},"max":{"1min":3.5409999999999999,"5min":3.984,"15min":3.984},"last":0.307},{"interface":"front","average":{"1min":0.748,"5min":0.58999999999999997,"15min":0.54700000000000004},"min":{"1min":0.32300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.3479999999999999,"5min":6.3479999999999999,"15min":6.3479999999999999},"last":0.32200000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.462,"5min":1.038,"15min":1.0149999999999999},"min":{"1min":0.34799999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.5999999999999996,"5min":9.5999999999999996,"15min":9.5999999999999996},"last":0.38},{"interface":"front","average":{"1min":1.607,"5min":1.175,"15min":1.147},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.673,"5min":5.673,"15min":5.673},"last":0.39400000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":0.78900000000000003,"15min":0.73299999999999998},"min":{"1min":0.38,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.3730000000000002,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":1.383},{"interface":"front","average":{"1min":2.016,"5min":1.224,"15min":1.1579999999999999},"min":{"1min":0.32000000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":0.54000000000000004}]}]},{"osd":6,"up_from":163,"seq":700079669321,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1317884,"kb_used_data":1153332,"kb_used_omap":7,"kb_used_meta":164536,"kb_avail":92427268,"statfs":{"total":95995035648,"available":94645522432,"internally_reserved":0,"allocated":1181011968,"data_stored":1179993013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":168485668},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":1.585,"5min":1.274,"15min":1.3240000000000001},"min":{"1min":0.25700000000000001,"5min":0.215,"15min":0.215},"max":{"1min":15.407,"5min":15.407,"15min":15.407},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.83599999999999997,"5min":1.347,"15min":2.2490000000000001},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.9710000000000001,"5min":13.536,"15min":13.536},"last":3.0619999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":1.5,"5min":1.458,"15min":1.913},"min":{"1min":0.29799999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.7690000000000001,"5min":14.233000000000001,"15min":14.233000000000001},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.905,"5min":1.5720000000000001,"15min":2.1379999999999999},"min":{"1min":0.32300000000000001,"5min":0.248,"15min":0.248},"max":{"1min":14.789,"5min":14.789,"15min":14.789},"last":0.34200000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2629999999999999,"5min":1.75,"15min":2.2610000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":5.7489999999999997,"5min":11.210000000000001,"15min":11.210000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.171,"5min":1.8149999999999999,"15min":2.633},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":5.7359999999999998,"5min":11.223000000000001,"15min":11.223000000000001},"last":0.32100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:04:22 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.0449999999999999,"15min":1.0840000000000001},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.308},"max":{"1min":3.25,"5min":3.25,"15min":3.25},"last":0.66100000000000003},{"interface":"front","average":{"1min":1.472,"5min":1.2270000000000001,"15min":1.1859999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.2389999999999999,"5min":5.2389999999999999,"15min":5.2389999999999999},"last":0.372}]},{"osd":4,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.71099999999999997,"5min":0.93700000000000006,"15min":0.88400000000000001},"min":{"1min":0.309,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.9930000000000003,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":2.2069999999999999},{"interface":"front","average":{"1min":0.67000000000000004,"5min":0.78000000000000003,"15min":0.52100000000000002},"min":{"1min":0.35399999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":2.9430000000000001,"5min":8.3000000000000007,"15min":8.3000000000000007},"last":0.41999999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.56499999999999995,"5min":0.89600000000000002,"15min":0.82799999999999996},"min":{"1min":0.34200000000000003,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":2.2440000000000002,"5min":11.869,"15min":11.869},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.96799999999999997,"5min":0.96499999999999997,"15min":0.66500000000000004},"min":{"1min":0.33300000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.4130000000000003,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":0.499}]},{"osd":7,"last update":"Sat Sep 23 18:04:28 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.024,"15min":0.98399999999999999},"min":{"1min":0.35399999999999998,"5min":0.307,"15min":0.307},"max":{"1min":4.5880000000000001,"5min":4.5880000000000001,"15min":4.5880000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.83899999999999997,"15min":0.80100000000000005},"min":{"1min":0.38100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.7190000000000003,"5min":4.7190000000000003,"15min":4.7190000000000003},"last":0.44700000000000001}]}]},{"osd":1,"up_from":0,"seq":420906795140,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958666,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":641080,"kb_used_data":239344,"kb_used_omap":7,"kb_used_meta":401720,"kb_avail":93104072,"statfs":{"total":95995035648,"available":95338569728,"internally_reserved":0,"allocated":245088256,"data_stored":243725333,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":411362020},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":1.597,"15min":1.1419999999999999},"min":{"1min":0.20699999999999999,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":2.238,"5min":13.414,"15min":18.154},"last":10.128},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.504,"15min":1.1140000000000001},"min":{"1min":0.26800000000000002,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":1.514,"5min":13.369999999999999,"15min":13.369999999999999},"last":0.32900000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":2.2490000000000001,"15min":1.2829999999999999},"min":{"1min":0.21099999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.3529999999999998,"5min":20.882000000000001,"15min":20.882000000000001},"last":10.077999999999999},{"interface":"front","average":{"1min":1.6319999999999999,"5min":2.0830000000000002,"15min":1.234},"min":{"1min":0.22500000000000001,"5min":0.217,"15min":0.217},"max":{"1min":6.8869999999999996,"5min":15.064,"15min":15.064},"last":7.9029999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:04:21 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.419,"15min":1.508},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.3730000000000002,"5min":6.048,"15min":6.048},"last":9.657},{"interface":"front","average":{"1min":1.661,"5min":1.2929999999999999,"15min":1.2310000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.7300000000000004,"5min":8.7300000000000004,"15min":8.7300000000000004},"last":0.498}]},{"osd":4,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.677,"15min":1.1200000000000001},"min":{"1min":0.436,"5min":0.29299999999999998,"15min":0.251},"max":{"1min":4.4809999999999999,"5min":15.279999999999999,"15min":15.279999999999999},"last":1.6659999999999999},{"interface":"front","average":{"1min":0.95499999999999996,"5min":1.7849999999999999,"15min":1.1439999999999999},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":5.3109999999999999,"5min":18.106999999999999,"15min":18.106999999999999},"last":10.164999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.153,"5min":1.8939999999999999,"15min":1.196},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":5.3760000000000003,"5min":14.720000000000001,"15min":14.720000000000001},"last":10.194000000000001},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.526,"15min":1.024},"min":{"1min":0.373,"5min":0.254,"15min":0.254},"max":{"1min":2.536,"5min":12.794,"15min":12.794},"last":0.372}]},{"osd":6,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.681,"15min":2.2389999999999999},"min":{"1min":0.35099999999999998,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":4.4660000000000002,"5min":14.956,"15min":14.956},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.96699999999999997,"5min":1.6830000000000001,"15min":2.4020000000000001},"min":{"1min":0.38300000000000001,"5min":0.311,"15min":0.311},"max":{"1min":6.9189999999999996,"5min":12.755000000000001,"15min":12.755000000000001},"last":0.39500000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:04:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":0.98399999999999999,"15min":0.94299999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.4649999999999999,"5min":7.4649999999999999,"15min":7.4649999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.3080000000000001,"15min":1.1579999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.524,"5min":5.524,"15min":5.524},"last":0.68500000000000005}]}]},{"osd":2,"up_from":103,"seq":442381631613,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":917716,"kb_used_data":544332,"kb_used_omap":11,"kb_used_meta":373364,"kb_avail":92827436,"statfs":{"total":95995035648,"available":95055294464,"internally_reserved":0,"allocated":557395968,"data_stored":556060777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":382325111},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,1,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.51500000000000001,"5min":2.3130000000000002,"15min":1.2849999999999999},"min":{"1min":0.19800000000000001,"5min":0.17199999999999999,"15min":0.17199999999999999},"max":{"1min":2.694,"5min":20.867000000000001,"15min":20.867000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":0.877,"5min":2.1819999999999999,"15min":1.2789999999999999},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":3.286,"5min":21.681000000000001,"15min":21.681000000000001},"last":2.8559999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":2.4540000000000002,"15min":1.395},"min":{"1min":0.26800000000000002,"5min":0.215,"15min":0.215},"max":{"1min":3.0960000000000001,"5min":20.995000000000001,"15min":20.995000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.88800000000000001,"5min":1.698,"15min":1.1839999999999999},"min":{"1min":0.26700000000000002,"5min":0.193,"15min":0.193},"max":{"1min":3.649,"5min":15.945,"15min":15.945},"last":5.2169999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:04:26 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":0.98399999999999999,"15min":0.84899999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.009,"5min":12.009,"15min":12.009},"last":2.238},{"interface":"front","average":{"1min":1.4490000000000001,"5min":0.77800000000000002,"15min":0.66600000000000004},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.0820000000000007,"5min":9.0820000000000007,"15min":9.0820000000000007},"last":6.2530000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.83999999999999997,"5min":1.5960000000000001,"15min":1.111},"min":{"1min":0.32300000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":3.1440000000000001,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":6.0899999999999999},{"interface":"front","average":{"1min":0.622,"5min":1.837,"15min":1.121},"min":{"1min":0.314,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":1.6459999999999999,"5min":20.920000000000002,"15min":20.920000000000002},"last":0.36299999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:03:39 2023","interfaces":[{"interface":"back","average":{"1min":0.77400000000000002,"5min":1.9339999999999999,"15min":1.198},"min":{"1min":0.35499999999999998,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":3.6960000000000002,"5min":20.738,"15min":20.738},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.145,"5min":1.6419999999999999,"15min":1.2330000000000001},"min":{"1min":0.32000000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":3.6280000000000001,"5min":14.359,"15min":15.496},"last":5.1769999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:03:43 2023","interfaces":[{"interface":"back","average":{"1min":0.93300000000000005,"5min":1.752,"15min":2.1320000000000001},"min":{"1min":0.36599999999999999,"5min":0.308,"15min":0.308},"max":{"1min":3.1230000000000002,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":3.7519999999999998},{"interface":"front","average":{"1min":0.78400000000000003,"5min":1.97,"15min":2.3740000000000001},"min":{"1min":0.40799999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":2.2480000000000002,"5min":20.893000000000001,"15min":20.893000000000001},"last":0.56000000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":1.3640000000000001,"15min":1.395},"min":{"1min":0.436,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":12.051,"5min":12.051,"15min":12.051},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.1859999999999999,"5min":0.77400000000000002,"15min":0.68000000000000005},"min":{"1min":0.437,"5min":0.309,"15min":0.309},"max":{"1min":7.5339999999999998,"5min":7.5339999999999998,"15min":7.5339999999999998},"last":3.5550000000000002}]}]},{"osd":3,"up_from":237,"seq":1017907249182,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":830524,"kb_used_data":681972,"kb_used_omap":3,"kb_used_meta":148540,"kb_avail":92914628,"statfs":{"total":95995035648,"available":95144579072,"internally_reserved":0,"allocated":698339328,"data_stored":696810014,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":152105704},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.79200000000000004,"15min":0.85299999999999998},"min":{"1min":0.215,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.6179999999999999,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":1.6839999999999999},{"interface":"front","average":{"1min":0.373,"5min":0.93999999999999995,"15min":1.034},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.63300000000000001,"5min":4.5529999999999999,"15min":4.5529999999999999},"last":14.885999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.92300000000000004,"15min":1.014},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.69099999999999995,"5min":4.6050000000000004,"15min":4.6050000000000004},"last":21.931999999999999},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.874,"15min":0.92200000000000004},"min":{"1min":0.26800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.9630000000000001,"5min":4.8810000000000002,"15min":4.8810000000000002},"last":12.701000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.81899999999999995,"15min":0.83999999999999997},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.157,"5min":3.157,"15min":3.157},"last":1.7270000000000001},{"interface":"front","average":{"1min":0.443,"5min":1.085,"15min":1.1919999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.75600000000000001,"5min":6.157,"15min":6.157},"last":21.988}]},{"osd":4,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.016,"15min":1.0740000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.3330000000000002,"5min":4.633,"15min":4.633},"last":14.935},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.995,"15min":1.0429999999999999},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.3570000000000002,"5min":4.7300000000000004,"15min":4.7300000000000004},"last":12.786}]},{"osd":5,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.179,"15min":1.2490000000000001},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.314},"max":{"1min":2.6880000000000002,"5min":7.0640000000000001,"15min":7.0640000000000001},"last":12.743},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.133,"15min":1.165},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.0750000000000002,"5min":4.6150000000000002,"15min":4.6150000000000002},"last":1.6259999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.0669999999999999,"15min":1.079},"min":{"1min":0.36699999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.125,"5min":4.0590000000000002,"15min":4.0590000000000002},"last":1.706},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.3680000000000001,"15min":1.4419999999999999},"min":{"1min":0.40500000000000003,"5min":0.379,"15min":0.379},"max":{"1min":2.8679999999999999,"5min":7.3929999999999998,"15min":7.3929999999999998},"last":14.964}]},{"osd":7,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.47,"15min":1.538},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.8149999999999999,"5min":8.4190000000000005,"15min":8.4190000000000005},"last":12.821},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.512,"15min":1.5840000000000001},"min":{"1min":0.46000000000000002,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":3.8610000000000002,"5min":5.1760000000000002,"15min":5.1760000000000002},"last":1.661}]}]},{"osd":4,"up_from":30,"seq":128849019131,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":601204,"kb_used_data":373164,"kb_used_omap":0,"kb_used_meta":228032,"kb_avail":93143948,"statfs":{"total":95995035648,"available":95379402752,"internally_reserved":0,"allocated":382119936,"data_stored":381100758,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":233504768},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:38 2023","interfaces":[{"interface":"back","average":{"1min":0.59699999999999998,"5min":1.3220000000000001,"15min":0.82599999999999996},"min":{"1min":0.25,"5min":0.22,"15min":0.20100000000000001},"max":{"1min":2.238,"5min":14.164,"15min":14.164},"last":6.8849999999999998},{"interface":"front","average":{"1min":1.4450000000000001,"5min":1.5109999999999999,"15min":1.04},"min":{"1min":0.28899999999999998,"5min":0.22900000000000001,"15min":0.22600000000000001},"max":{"1min":7.1879999999999997,"5min":9.3789999999999996,"15min":11.372},"last":0.46300000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.359,"15min":1.004},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":2.2770000000000001,"5min":8.5259999999999998,"15min":10.429},"last":3.101},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.2949999999999999,"15min":1.083},"min":{"1min":0.29699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.597,"5min":8.8130000000000006,"15min":10.83},"last":6.7640000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95699999999999996,"5min":1.5109999999999999,"15min":1.085},"min":{"1min":0.28899999999999998,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":2.7930000000000001,"5min":11.630000000000001,"15min":11.630000000000001},"last":6.8259999999999996},{"interface":"front","average":{"1min":1.0049999999999999,"5min":1.3580000000000001,"15min":1.016},"min":{"1min":0.34000000000000002,"5min":0.316,"15min":0.27100000000000002},"max":{"1min":3.2250000000000001,"5min":7.4459999999999997,"15min":7.5090000000000003},"last":6.9299999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":1.238,"15min":1.3129999999999999},"min":{"1min":0.378,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.6619999999999999,"5min":7.2140000000000004,"15min":7.2140000000000004},"last":6.8550000000000004},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.335,"15min":1.3600000000000001},"min":{"1min":0.34799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.165,"5min":6.165,"15min":6.165},"last":1.2290000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":0.90900000000000003,"15min":0.629},"min":{"1min":0.25900000000000001,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":9.9819999999999993,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.376},{"interface":"front","average":{"1min":1.2350000000000001,"5min":1.111,"15min":0.76700000000000002},"min":{"1min":0.28799999999999998,"5min":0.26800000000000002,"15min":0.26100000000000001},"max":{"1min":11.170999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":5.0890000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":0.94899999999999995,"15min":0.82899999999999996},"min":{"1min":0.309,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.1849999999999996,"5min":7.1529999999999996,"15min":7.1529999999999996},"last":0.34699999999999998},{"interface":"front","average":{"1min":0.93000000000000005,"5min":1.0740000000000001,"15min":0.95799999999999996},"min":{"1min":0.34300000000000003,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.2590000000000003,"5min":10.725,"15min":10.725},"last":2.1389999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:04:29 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":1.097,"15min":0.97499999999999998},"min":{"1min":0.34699999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.6779999999999999,"5min":7.6779999999999999,"15min":7.6779999999999999},"last":5.1299999999999999},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.377,"15min":1.2669999999999999},"min":{"1min":0.38500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":9.0310000000000006,"5min":9.0310000000000006,"15min":9.0310000000000006},"last":5.1580000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757498,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":522944,"kb_used_data":311736,"kb_used_omap":0,"kb_used_meta":211200,"kb_avail":93222208,"statfs":{"total":95995035648,"available":95459540992,"internally_reserved":0,"allocated":319217664,"data_stored":318198164,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":216268800},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.5,"5min":1.5349999999999999,"15min":1.002},"min":{"1min":0.27300000000000002,"5min":0.217,"15min":0.217},"max":{"1min":2.0649999999999999,"5min":21.376999999999999,"15min":21.376999999999999},"last":0.316},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.8680000000000001,"15min":1.179},"min":{"1min":0.36699999999999999,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.0310000000000006,"5min":21.388999999999999,"15min":21.388999999999999},"last":0.39100000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:04:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.91200000000000003,"15min":0.85999999999999999},"min":{"1min":0.26000000000000001,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":3.754,"5min":4.8129999999999997,"15min":10.853999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.075,"5min":1.159,"15min":0.98799999999999999},"min":{"1min":0.28000000000000003,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":8.0039999999999996,"5min":8.0039999999999996,"15min":10.206},"last":0.30099999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:03:41 2023","interfaces":[{"interface":"back","average":{"1min":0.52300000000000002,"5min":1.956,"15min":1.258},"min":{"1min":0.312,"5min":0.255,"15min":0.255},"max":{"1min":1.75,"5min":26.035,"15min":38.606999999999999},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.498,"15min":1.139},"min":{"1min":0.308,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9289999999999998,"5min":17.085000000000001,"15min":39.670999999999999},"last":0.45500000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:04:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.85099999999999998,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.9569999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.751,"15min":0.72499999999999998},"min":{"1min":0.35699999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.1260000000000003,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":0.53000000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85799999999999998,"5min":1.0089999999999999,"15min":0.72699999999999998},"min":{"1min":0.254,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":8.2390000000000008,"5min":10.092000000000001,"15min":10.092000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":0.46899999999999997,"5min":0.98399999999999999,"15min":0.69999999999999996},"min":{"1min":0.35399999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":0.79900000000000004,"5min":5.077,"15min":5.077},"last":0.34399999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:03:45 2023","interfaces":[{"interface":"back","average":{"1min":0.51200000000000001,"5min":1.127,"15min":1.4350000000000001},"min":{"1min":0.32600000000000001,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":1.3160000000000001,"5min":11.287000000000001,"15min":11.287000000000001},"last":0.442},{"interface":"front","average":{"1min":0.54000000000000004,"5min":0.97099999999999997,"15min":1.2709999999999999},"min":{"1min":0.35399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.333,"5min":8.266,"15min":8.266},"last":0.377}]},{"osd":7,"last update":"Sat Sep 23 18:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.3680000000000001,"15min":1.486},"min":{"1min":0.36399999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.8510000000000009,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.149,"15min":1.133},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":8.3279999999999994,"5min":8.3279999999999994,"15min":8.3279999999999994},"last":0.48399999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":236589056,"data_stored":236584984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":699006976,"data_stored":699006976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":537989120,"data_stored":537985048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":238551040,"data_stored":238551040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":358023168,"data_stored":358023168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":312741888,"data_stored":312737816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1173356544,"data_stored":1173356544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":383582208,"data_stored":383582208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:04:47.962 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:04:48.392 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:48.393 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:44 smithi055 ceph-mon[179668]: pgmap v650: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 897 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 20428/42951 objects degraded (47.561%); 2533/42951 objects misplaced (5.897%); 6.7 MiB/s, 114 objects/s recovering 2023-09-23T18:04:48.393 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:44 smithi055 ceph-mon[179668]: from='client.55372 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:04:48.393 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:44 smithi055 ceph-mon[184174]: pgmap v650: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 897 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 20428/42951 objects degraded (47.561%); 2533/42951 objects misplaced (5.897%); 6.7 MiB/s, 114 objects/s recovering 2023-09-23T18:04:48.394 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:44 smithi055 ceph-mon[184174]: from='client.55372 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:04:48.394 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:04:48.394 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/984412574' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:04:48.394 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:45 smithi055 ceph-mon[179668]: pgmap v651: 119 pgs: 1 active+recovery_wait+degraded+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 786 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 20428/37608 objects degraded (54.318%); 2533/37608 objects misplaced (6.735%); 6.7 MiB/s, 114 objects/s recovering 2023-09-23T18:04:48.394 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:45 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:04:48.395 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/984412574' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:04:48.395 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:45 smithi055 ceph-mon[184174]: pgmap v651: 119 pgs: 1 active+recovery_wait+degraded+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 786 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 20428/37608 objects degraded (54.318%); 2533/37608 objects misplaced (6.735%); 6.7 MiB/s, 114 objects/s recovering 2023-09-23T18:04:48.395 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:45 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:04:48.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:04:48.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[184174]: failed to encode map e269 with expected crc 2023-09-23T18:04:48.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[184174]: failed to encode map e269 with expected crc 2023-09-23T18:04:48.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:04:48.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[184174]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T18:04:48.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:48.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:48.397 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:04:48.397 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[179668]: failed to encode map e269 with expected crc 2023-09-23T18:04:48.397 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[179668]: failed to encode map e269 with expected crc 2023-09-23T18:04:48.397 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:04:48.397 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[179668]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T18:04:48.397 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:48.397 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:46 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:48.399 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:04:48.870 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:04:48.871 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:04:48.871 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:04:48.871 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:04:48.871 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:04:48.871 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:04:48.871 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:04:48.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:44 smithi099 ceph-mon[167076]: pgmap v650: 119 pgs: 1 active+recovery_wait+degraded+remapped, 110 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 897 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 714 op/s; 20428/42951 objects degraded (47.561%); 2533/42951 objects misplaced (5.897%); 6.7 MiB/s, 114 objects/s recovering 2023-09-23T18:04:48.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:44 smithi099 ceph-mon[167076]: from='client.55372 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:04:48.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/984412574' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:04:48.872 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:45 smithi099 ceph-mon[167076]: pgmap v651: 119 pgs: 1 active+recovery_wait+degraded+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 786 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 871 op/s; 20428/37608 objects degraded (54.318%); 2533/37608 objects misplaced (6.735%); 6.7 MiB/s, 114 objects/s recovering 2023-09-23T18:04:48.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:45 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:04:48.873 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:46 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:46 smithi099 ceph-mon[167076]: failed to encode map e269 with expected crc 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:46 smithi099 ceph-mon[167076]: failed to encode map e269 with expected crc 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:46 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:46 smithi099 ceph-mon[167076]: osdmap e269: 8 total, 8 up, 7 in 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:46 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:49.368 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:46 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:49.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:49.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:49.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:47 smithi099 ceph-mon[167076]: 1.16 deep-scrub starts 2023-09-23T18:04:49.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:47 smithi099 ceph-mon[167076]: pgmap v653: 119 pgs: 1 active+recovery_wait+degraded+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 757 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 20193/36195 objects degraded (55.789%); 2298/36195 objects misplaced (6.349%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:04:49.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2124218499' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:49.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:04:49.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:04:49.369 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:49.370 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:04:49.370 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:04:49.370 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:04:49.370 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:04:49.370 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:04:49.371 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:04:49.371 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:04:49.371 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:04:49.371 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:49.371 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:49.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[179668]: 1.16 deep-scrub starts 2023-09-23T18:04:49.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[179668]: pgmap v653: 119 pgs: 1 active+recovery_wait+degraded+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 757 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 20193/36195 objects degraded (55.789%); 2298/36195 objects misplaced (6.349%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:04:49.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2124218499' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:49.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:04:49.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:04:49.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:49.372 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:49.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:49.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[184174]: 1.16 deep-scrub starts 2023-09-23T18:04:49.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[184174]: pgmap v653: 119 pgs: 1 active+recovery_wait+degraded+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 757 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 637 op/s; 20193/36195 objects degraded (55.789%); 2298/36195 objects misplaced (6.349%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:04:49.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2124218499' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:04:49.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:04:49.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:04:49.373 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:53.477 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 20193/36195 objects degraded (55.789%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:56.015 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:04:56.016 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:04:56.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 20193/36195 objects degraded (55.789%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:56.017 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:56.018 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:49 smithi055 ceph-mon[179668]: pgmap v654: 119 pgs: 1 active+recovery_wait+degraded+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 609 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 19718/29127 objects degraded (67.697%); 1857/29127 objects misplaced (6.376%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T18:04:56.018 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:49 smithi055 ceph-mon[184174]: pgmap v654: 119 pgs: 1 active+recovery_wait+degraded+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 609 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 19718/29127 objects degraded (67.697%); 1857/29127 objects misplaced (6.376%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T18:04:56.018 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:04:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:04:50] "GET /metrics HTTP/1.1" 200 36536 "" "Prometheus/2.43.0" 2023-09-23T18:04:57.667 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:04:57.667 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:04:57.667 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:50 smithi055 ceph-mon[184174]: 1.6 scrub starts 2023-09-23T18:04:57.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:04:57.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:50 smithi055 ceph-mon[179668]: 1.6 scrub starts 2023-09-23T18:04:57.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:51 smithi055 ceph-mon[179668]: pgmap v655: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 516 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 19587/24639 objects degraded (79.496%); 1470/24639 objects misplaced (5.966%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T18:04:57.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:57.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:57.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:51 smithi055 ceph-mon[184174]: pgmap v655: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 516 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 19587/24639 objects degraded (79.496%); 1470/24639 objects misplaced (5.966%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T18:04:57.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:57.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:57.669 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:04:57.669 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:04:57.670 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:04:57.671 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:04:57.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:52 smithi055 ceph-mon[179668]: 1.19 scrub starts 2023-09-23T18:04:57.672 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:52 smithi055 ceph-mon[184174]: 1.19 scrub starts 2023-09-23T18:04:58.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 20193/36195 objects degraded (55.789%), 8 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:58.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:49 smithi099 ceph-mon[167076]: pgmap v654: 119 pgs: 1 active+recovery_wait+degraded+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 609 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 712 op/s; 19718/29127 objects degraded (67.697%); 1857/29127 objects misplaced (6.376%); 5.5 MiB/s, 87 objects/s recovering 2023-09-23T18:04:58.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:50 smithi099 ceph-mon[167076]: 1.6 scrub starts 2023-09-23T18:04:58.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:51 smithi099 ceph-mon[167076]: pgmap v655: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 516 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 709 op/s; 19587/24639 objects degraded (79.496%); 1470/24639 objects misplaced (5.966%); 5.2 MiB/s, 83 objects/s recovering 2023-09-23T18:04:58.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:04:58.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:04:58.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:52 smithi099 ceph-mon[167076]: 1.19 scrub starts 2023-09-23T18:04:58.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:53 smithi099 ceph-mon[167076]: pgmap v656: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 431 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 745 op/s; 18896/20565 objects degraded (91.884%); 1140/20565 objects misplaced (5.543%); 4.0 MiB/s, 79 objects/s recovering 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19587/24639 objects degraded (79.496%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:54 smithi099 ceph-mon[167076]: 1.11 scrub starts 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:55 smithi099 ceph-mon[167076]: pgmap v657: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 277 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 18896/13164 objects degraded (143.543%); 1140/13164 objects misplaced (8.660%); 4.0 MiB/s, 79 objects/s recovering 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4163548046' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:57 smithi099 ceph-mon[167076]: pgmap v658: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 240 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 18896/11394 objects degraded (165.842%); 510/11394 objects misplaced (4.476%); 3.5 MiB/s, 69 objects/s recovering 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:53 smithi055 ceph-mon[184174]: pgmap v656: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 431 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 745 op/s; 18896/20565 objects degraded (91.884%); 1140/20565 objects misplaced (5.543%); 4.0 MiB/s, 79 objects/s recovering 2023-09-23T18:04:58.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19587/24639 objects degraded (79.496%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:58.590 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:04:58.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:53 smithi055 ceph-mon[179668]: pgmap v656: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 431 MiB data, 4.0 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 745 op/s; 18896/20565 objects degraded (91.884%); 1140/20565 objects misplaced (5.543%); 4.0 MiB/s, 79 objects/s recovering 2023-09-23T18:04:58.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19587/24639 objects degraded (79.496%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:58.591 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:04:58.591 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:54 smithi055 ceph-mon[184174]: 1.11 scrub starts 2023-09-23T18:04:58.938 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:54 smithi055 ceph-mon[179668]: 1.11 scrub starts 2023-09-23T18:04:58.939 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:55 smithi055 ceph-mon[184174]: pgmap v657: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 277 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 18896/13164 objects degraded (143.543%); 1140/13164 objects misplaced (8.660%); 4.0 MiB/s, 79 objects/s recovering 2023-09-23T18:04:58.939 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:55 smithi055 ceph-mon[179668]: pgmap v657: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 277 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 814 op/s; 18896/13164 objects degraded (143.543%); 1140/13164 objects misplaced (8.660%); 4.0 MiB/s, 79 objects/s recovering 2023-09-23T18:04:58.940 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:04:58.940 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 18896/13164 objects degraded (143.543%), 7 pgs degraded, 7 pgs undersized","count":14},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":785,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":269,"num_osds":8,"num_up_osds":8,"osd_up_since":1695492137,"num_in_osds":7,"osd_in_since":1695492120,"num_remapped_pgs":8},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":111},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+undersized+degraded+remapped+backfill_wait","count":2},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":4388,"data_bytes":290339334,"bytes_used":3825438720,"bytes_avail":668139810816,"bytes_total":671965249536,"degraded_objects":18896,"degraded_total":13164,"degraded_ratio":1.4354299604983287,"misplaced_objects":1140,"misplaced_total":13164,"misplaced_ratio":0.086599817684594349,"recovering_objects_per_sec":79,"recovering_bytes_per_sec":4204986,"recovering_keys_per_sec":0,"num_objects_recovered":791,"num_bytes_recovered":42074112,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":814},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":23,"modified":"2023-09-23T18:03:16.678968+0000","services":{}},"progress_events":{"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"e54c9148-1053-4ff0-9024-eb4758d365b5":{"message":"Global Recovery Event (8m)\n [==========================..] (remaining: 37s)","progress":0.93277311325073242,"add_to_ceph_s":true}}} 2023-09-23T18:04:58.941 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:04:59.244 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:04:59.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4163548046' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:04:59.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4163548046' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:04:59.246 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 18896/11394 objects degraded (165.842%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:59.246 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Removed 73135 objects 2023-09-23T18:04:59.246 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Clean up completed and total clean up time :142.93 2023-09-23T18:04:59.247 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:57 smithi055 ceph-mon[179668]: pgmap v658: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 240 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 18896/11394 objects degraded (165.842%); 510/11394 objects misplaced (4.476%); 3.5 MiB/s, 69 objects/s recovering 2023-09-23T18:04:59.565 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:57 smithi055 ceph-mon[184174]: pgmap v658: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 240 MiB data, 3.3 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 767 op/s; 18896/11394 objects degraded (165.842%); 510/11394 objects misplaced (4.476%); 3.5 MiB/s, 69 objects/s recovering 2023-09-23T18:04:59.566 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 18896/11394 objects degraded (165.842%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:04:59.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 18896/11394 objects degraded (165.842%), 7 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:05:00.348 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:00.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:59 smithi099 ceph-mon[167076]: pgmap v659: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 74 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 18159/3402 objects degraded (533.774%); 266/3402 objects misplaced (7.819%); 2.1 MiB/s, 64 objects/s recovering 2023-09-23T18:05:00.348 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:04:59 smithi099 ceph-mon[167076]: 1.f scrub starts 2023-09-23T18:05:00.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:00.349 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:00.349 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:05:00.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:05:00.350 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:00.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:59 smithi055 ceph-mon[179668]: pgmap v659: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 74 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 18159/3402 objects degraded (533.774%); 266/3402 objects misplaced (7.819%); 2.1 MiB/s, 64 objects/s recovering 2023-09-23T18:05:00.704 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:04:59 smithi055 ceph-mon[179668]: 1.f scrub starts 2023-09-23T18:05:00.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:59 smithi055 ceph-mon[184174]: pgmap v659: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 74 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 910 op/s; 18159/3402 objects degraded (533.774%); 266/3402 objects misplaced (7.819%); 2.1 MiB/s, 64 objects/s recovering 2023-09-23T18:05:00.705 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:04:59 smithi055 ceph-mon[184174]: 1.f scrub starts 2023-09-23T18:05:00.706 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:00.706 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:05:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:05:00] "GET /metrics HTTP/1.1" 200 36528 "" "Prometheus/2.43.0" 2023-09-23T18:05:00.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:05:00.734 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:01.119 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:05:01.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:00 smithi055 ceph-mon[179668]: 1.17 scrub starts 2023-09-23T18:05:01.120 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:00 smithi055 ceph-mon[184174]: 1.17 scrub starts 2023-09-23T18:05:01.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:01.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:05:01.486 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:05:01.486 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:00 smithi099 ceph-mon[167076]: 1.17 scrub starts 2023-09-23T18:05:01.486 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:05:01.487 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:05:01.487 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:05:01.487 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:05:01.487 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:05:01.487 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:05:01.487 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:05:01.487 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:05:01.487 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:05:01.487 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:05:01.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:05:01.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:05:01.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:05:01.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:05:01.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:05:01.586 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:05:02.207 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_1 2023-09-23T18:05:02.207 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_1 unique_pool_1 --yes-i-really-really-mean-it 2023-09-23T18:05:02.208 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:01 smithi055 ceph-mon[184174]: pgmap v660: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 9.8 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 18159/342 objects degraded (5309.649%); 16/342 objects misplaced (4.678%); 1.0 MiB/s, 46 objects/s recovering 2023-09-23T18:05:02.208 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:01 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4004563310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:02.209 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:01 smithi055 ceph-mon[179668]: pgmap v660: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 9.8 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 18159/342 objects degraded (5309.649%); 16/342 objects misplaced (4.678%); 1.0 MiB/s, 46 objects/s recovering 2023-09-23T18:05:02.209 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:01 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4004563310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:02.209 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:02.283 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:02.566 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:01 smithi099 ceph-mon[167076]: pgmap v660: 119 pgs: 1 active+recovery_wait+remapped, 111 active+clean, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 9.8 MiB data, 2.8 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 799 op/s; 18159/342 objects degraded (5309.649%); 16/342 objects misplaced (4.678%); 1.0 MiB/s, 46 objects/s recovering 2023-09-23T18:05:02.566 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:01 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4004563310' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:02.775 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:05:03.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:03.235 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:05:03.237 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:05:03.237 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:05:03.237 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:05:03.237 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:05:03.237 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.237 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104098 5.a 5:5150641c:::benchmark_data_smithi055_229117_object104097:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T18:05:03.237 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.920515+0000", 2023-09-23T18:05:03.237 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 458.88945256199997, 2023-09-23T18:05:03.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.048992793, 2023-09-23T18:05:03.238 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:03.238 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:05:03.238 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:03.238 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:03.239 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:03.239 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104098 2023-09-23T18:05:03.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.239 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:03.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:03.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T18:05:03.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920515+0000", 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:03.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.241 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:03.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920538+0000", 2023-09-23T18:05:03.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2770000000000001e-05 2023-09-23T18:05:03.571 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.571 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.571 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:03.571 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920545+0000", 2023-09-23T18:05:03.571 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1969999999999999e-06 2023-09-23T18:05:03.572 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.572 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.572 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:03.574 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920550+0000", 2023-09-23T18:05:03.574 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3360000000000002e-06 2023-09-23T18:05:03.574 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.574 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927587+0000", 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0070371599999999998 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927719+0000", 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013187600000000001 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.575 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:05:03.576 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.927856+0000", 2023-09-23T18:05:03.576 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013721800000000001 2023-09-23T18:05:03.576 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.576 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.576 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:03.576 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936497+0000", 2023-09-23T18:05:03.576 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.008640277 2023-09-23T18:05:03.576 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936670+0000", 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017294700000000001 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959038+0000", 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022367973999999999 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.577 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:03.578 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959048+0000", 2023-09-23T18:05:03.578 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0428000000000001e-05 2023-09-23T18:05:03.578 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.578 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.578 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:03.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969404+0000", 2023-09-23T18:05:03.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0103555609999999 2023-09-23T18:05:03.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:03.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969508+0000", 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010461799999999999 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104100 5.a 5:58ebf1c5:::benchmark_data_smithi055_229117_object104099:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.920555+0000", 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 458.88941288299998, 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0492678900000001, 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:03.918 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:05:03.919 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:03.919 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:03.919 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:03.919 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104100 2023-09-23T18:05:06.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:06.271 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:06.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:06.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:06.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T18:05:06.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:06.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:06.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:06.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:06.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T18:05:06.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:06.272 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:06.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:06.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:06.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920555+0000", 2023-09-23T18:05:06.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:06.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:06.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:06.273 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:06.273 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920574+0000", 2023-09-23T18:05:06.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9664e-05 2023-09-23T18:05:06.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:06.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:06.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:06.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920581+0000", 2023-09-23T18:05:06.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3260000000000001e-06 2023-09-23T18:05:06.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:06.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:06.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:06.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.920586+0000", 2023-09-23T18:05:06.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7239999999999997e-06 2023-09-23T18:05:06.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:06.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:06.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:06.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.931735+0000", 2023-09-23T18:05:06.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011148345 2023-09-23T18:05:06.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:06.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:06.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:06.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.931905+0000", 2023-09-23T18:05:06.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000170466 2023-09-23T18:05:06.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:06.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:06.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:05:06.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.932081+0000", 2023-09-23T18:05:06.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000175623 2023-09-23T18:05:06.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:06.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:06.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:06.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936776+0000", 2023-09-23T18:05:06.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0046956209999999996 2023-09-23T18:05:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.936996+0000", 2023-09-23T18:05:07.444 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021987100000000001 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959173+0000", 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022176392999999999 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959189+0000", 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.609e-05 2023-09-23T18:05:07.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969761+0000", 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0105722159999999 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969823+0000", 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1938000000000002e-05 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:07.446 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:07.447 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:07.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:07.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:07.447 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104109 5.a 5:542cd2b3:::benchmark_data_smithi055_229117_object104108:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T18:05:08.113 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.944006+0000", 2023-09-23T18:05:08.113 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 458.86596231800002, 2023-09-23T18:05:08.113 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0419720189999999, 2023-09-23T18:05:08.113 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:08.113 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:05:08.113 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:08.113 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:08.113 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:08.114 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104109 2023-09-23T18:05:08.114 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.114 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:08.114 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.114 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:08.114 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944006+0000", 2023-09-23T18:05:08.114 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:08.114 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.114 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.114 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944006+0000", 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944004+0000", 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:08.115 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944029+0000", 2023-09-23T18:05:08.116 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5201000000000001e-05 2023-09-23T18:05:08.116 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.116 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.116 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:08.116 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944043+0000", 2023-09-23T18:05:08.116 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3964e-05 2023-09-23T18:05:08.116 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.116 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.116 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:08.116 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944062+0000", 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8190999999999999e-05 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944258+0000", 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019652100000000001 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944396+0000", 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013821100000000001 2023-09-23T18:05:08.117 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944572+0000", 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017574299999999999 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.956893+0000", 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012321445 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.118 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:08.119 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959227+0000", 2023-09-23T18:05:08.119 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023331559999999999 2023-09-23T18:05:08.119 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:08.119 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:08.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:08.119 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959235+0000", 2023-09-23T18:05:08.119 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1000000000000004e-06 2023-09-23T18:05:08.119 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:09.565 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:09.565 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:09.566 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.969881+0000", 2023-09-23T18:05:09.566 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.010645899 2023-09-23T18:05:09.566 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:09.566 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:12.220 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:05:12.220 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.970644+0000", 2023-09-23T18:05:12.220 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000763867 2023-09-23T18:05:12.220 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:12.220 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:12.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:12.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.985881+0000", 2023-09-23T18:05:12.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015236622 2023-09-23T18:05:12.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:12.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:12.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:12.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.985978+0000", 2023-09-23T18:05:12.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6602999999999996e-05 2023-09-23T18:05:13.572 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:13.572 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:13.572 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:13.572 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:13.572 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:13.572 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104112 5.a 5:5b2e2197:::benchmark_data_smithi055_229117_object104111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T18:05:13.572 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.944077+0000", 2023-09-23T18:05:13.572 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 458.86589117699998, 2023-09-23T18:05:13.572 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.042230502, 2023-09-23T18:05:15.987 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:15.987 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:05:15.987 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:16.454 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:16.454 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104112 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944077+0000", 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:16.455 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944077+0000", 2023-09-23T18:05:16.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:16.458 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:16.458 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:16.458 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:16.458 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944076+0000", 2023-09-23T18:05:16.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:16.459 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:16.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:16.459 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:16.459 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944095+0000", 2023-09-23T18:05:16.459 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8532000000000001e-05 2023-09-23T18:05:16.459 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:16.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944103+0000", 2023-09-23T18:05:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8399999999999995e-06 2023-09-23T18:05:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.944108+0000", 2023-09-23T18:05:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8510000000000004e-06 2023-09-23T18:05:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:19.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.949704+0000", 2023-09-23T18:05:19.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055955110000000001 2023-09-23T18:05:19.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:19.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:19.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.949880+0000", 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000176312 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.950043+0000", 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016271800000000001 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959270+0000", 2023-09-23T18:05:19.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092269080000000007 2023-09-23T18:05:19.849 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:19.849 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:20.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:20.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959278+0000", 2023-09-23T18:05:20.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0590000000000003e-06 2023-09-23T18:05:20.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:20.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:20.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:20.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.970705+0000", 2023-09-23T18:05:20.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.011427144 2023-09-23T18:05:20.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:20.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:20.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971279+0000", 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00057403800000000002 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986229+0000", 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014949831 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:20.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:20.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986307+0000", 2023-09-23T18:05:20.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8319999999999996e-05 2023-09-23T18:05:20.382 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:20.382 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:20.382 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:20.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:20.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:20.382 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:104117 5.a 5:5725f10b:::benchmark_data_smithi055_229117_object104116:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e141)", 2023-09-23T18:05:20.383 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:57:23.952810+0000", 2023-09-23T18:05:20.383 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 458.85715807600002, 2023-09-23T18:05:20.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.039709502, 2023-09-23T18:05:20.383 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:20.383 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:20.945 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:20.945 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:20.945 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:20.946 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 104117 2023-09-23T18:05:20.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:20.946 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:20.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:20.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:20.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952810+0000", 2023-09-23T18:05:20.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:20.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:20.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:20.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:20.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952810+0000", 2023-09-23T18:05:20.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:20.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:20.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:20.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:20.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952808+0000", 2023-09-23T18:05:20.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:05:21.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:21.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:21.867 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:21.867 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952833+0000", 2023-09-23T18:05:21.867 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5307999999999999e-05 2023-09-23T18:05:21.867 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:21.867 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:21.869 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:21.869 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952845+0000", 2023-09-23T18:05:21.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2145e-05 2023-09-23T18:05:21.869 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:21.869 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:21.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:21.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.952855+0000", 2023-09-23T18:05:21.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2730000000000008e-06 2023-09-23T18:05:21.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:21.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:21.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:21.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959408+0000", 2023-09-23T18:05:21.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0065535530000000002 2023-09-23T18:05:21.872 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:21.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:21.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:22.237 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:23.959416+0000", 2023-09-23T18:05:22.237 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9529999999999999e-06 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971413+0000", 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0119968539999999 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971561+0000", 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014827199999999999 2023-09-23T18:05:22.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4", 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.971748+0000", 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000186701 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.986344+0000", 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014596067000000001 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992361+0000", 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060167379999999998 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992422+0000", 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1598000000000002e-05 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:57:24.992519+0000", 2023-09-23T18:05:22.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6966999999999995e-05 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154239 5.d 5:b6c40944:::benchmark_data_smithi055_229117_object48613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 353.97017471100003, 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5202187069999997, 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:05:22.241 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:22.242 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:22.242 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:22.242 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154239 2023-09-23T18:05:22.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.242 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:22.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:22.242 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:05:22.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:22.687 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.687 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.688 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:22.688 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:05:22.688 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:22.688 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.688 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.689 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839792+0000", 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839799+0000", 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2969999999999998e-06 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839815+0000", 2023-09-23T18:05:22.690 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5458e-05 2023-09-23T18:05:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839823+0000", 2023-09-23T18:05:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.833e-06 2023-09-23T18:05:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839864+0000", 2023-09-23T18:05:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0865999999999998e-05 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840002+0000", 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013775199999999999 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840128+0000", 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012573200000000001 2023-09-23T18:05:23.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.845223+0000", 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050947299999999996 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.854611+0000", 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093881209999999993 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:23.076 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.854622+0000", 2023-09-23T18:05:23.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1011e-05 2023-09-23T18:05:23.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851119+0000", 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.996496979 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851132+0000", 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.312e-05 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.340349+0000", 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4892177549999999 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.340443+0000", 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3108000000000004e-05 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.359910+0000", 2023-09-23T18:05:23.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019467288999999999 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360012+0000", 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000102093 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.411 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154248 5.d 5:bb80aea8:::benchmark_data_smithi055_229117_object48622:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.855620+0000", 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 353.95434838599999, 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5213527490000001, 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154248 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:23.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855620+0000", 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855620+0000", 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:23.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855618+0000", 2023-09-23T18:05:23.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:05:23.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.782 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:23.782 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855624+0000", 2023-09-23T18:05:23.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3399999999999997e-06 2023-09-23T18:05:23.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:23.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855643+0000", 2023-09-23T18:05:23.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9772999999999999e-05 2023-09-23T18:05:23.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855653+0000", 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4870000000000008e-06 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855688+0000", 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5308999999999997e-05 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:23.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855697+0000", 2023-09-23T18:05:23.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6910000000000003e-06 2023-09-23T18:05:23.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:23.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:24.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851160+0000", 2023-09-23T18:05:24.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99546285999999995 2023-09-23T18:05:24.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851169+0000", 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8920000000000006e-06 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.341897+0000", 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4907282319999999 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:24.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.342015+0000", 2023-09-23T18:05:24.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118402 2023-09-23T18:05:24.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:05:24.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.342112+0000", 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6391000000000006e-05 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356673+0000", 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014561579999999999 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356774+0000", 2023-09-23T18:05:24.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010047699999999999 2023-09-23T18:05:24.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:24.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.375006+0000", 2023-09-23T18:05:24.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018232224000000002 2023-09-23T18:05:24.733 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.733 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.733 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:24.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.375055+0000", 2023-09-23T18:05:24.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9345999999999998e-05 2023-09-23T18:05:24.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.734 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:24.734 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.376972+0000", 2023-09-23T18:05:24.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001916966 2023-09-23T18:05:24.734 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:24.734 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:24.734 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:24.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154254 5.4 5:2eea31d3:::benchmark_data_smithi055_229117_object48628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.876460+0000", 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 353.93350765100001, 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2290530139999998, 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154254 2023-09-23T18:05:24.735 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876460+0000", 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876460+0000", 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:24.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876458+0000", 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876469+0000", 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0913999999999999e-05 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876489+0000", 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9649000000000001e-05 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876500+0000", 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1600000000000001e-05 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:24.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876561+0000", 2023-09-23T18:05:24.739 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0857999999999999e-05 2023-09-23T18:05:24.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:24.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876573+0000", 2023-09-23T18:05:24.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1836e-05 2023-09-23T18:05:24.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:24.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851144+0000", 2023-09-23T18:05:24.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97457099800000002 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851155+0000", 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1548e-05 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.095633+0000", 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.244477501 2023-09-23T18:05:24.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.095782+0000", 2023-09-23T18:05:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014866200000000001 2023-09-23T18:05:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T18:05:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.095879+0000", 2023-09-23T18:05:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7288999999999996e-05 2023-09-23T18:05:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100460+0000", 2023-09-23T18:05:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045811979999999999 2023-09-23T18:05:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100470+0000", 2023-09-23T18:05:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6609999999999995e-06 2023-09-23T18:05:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105323+0000", 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048527759999999996 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105356+0000", 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3546e-05 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105407+0000", 2023-09-23T18:05:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1372000000000002e-05 2023-09-23T18:05:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105513+0000", 2023-09-23T18:05:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010586699999999999 2023-09-23T18:05:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:25.237 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:25.238 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:25.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.238 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.238 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154255 5.d 5:bf0b5457:::benchmark_data_smithi055_229117_object48629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:05:25.238 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.876519+0000", 2023-09-23T18:05:25.238 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 353.93344916199999, 2023-09-23T18:05:25.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.514717928, 2023-09-23T18:05:25.238 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:25.238 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154255 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876519+0000", 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876519+0000", 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876518+0000", 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:25.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876522+0000", 2023-09-23T18:05:25.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0609999999999997e-06 2023-09-23T18:05:25.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.241 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:25.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876531+0000", 2023-09-23T18:05:25.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2300000000000008e-06 2023-09-23T18:05:25.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.241 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:25.242 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876537+0000", 2023-09-23T18:05:25.242 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5570000000000001e-06 2023-09-23T18:05:25.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:25.566 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876590+0000", 2023-09-23T18:05:25.566 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2528000000000003e-05 2023-09-23T18:05:25.566 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.566 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.566 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:25.566 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876600+0000", 2023-09-23T18:05:25.566 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8800000000000003e-06 2023-09-23T18:05:25.566 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.566 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851204+0000", 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97460424499999998 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851212+0000", 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2460000000000003e-06 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.567 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.346812+0000", 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4955998619999999 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.346965+0000", 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015338600000000001 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.347122+0000", 2023-09-23T18:05:25.568 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000156537 2023-09-23T18:05:25.569 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.569 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.569 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:25.569 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356744+0000", 2023-09-23T18:05:25.569 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0096223920000000004 2023-09-23T18:05:25.569 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.569 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.569 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:25.569 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356784+0000", 2023-09-23T18:05:25.569 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9984000000000002e-05 2023-09-23T18:05:25.570 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.570 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.570 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:25.570 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.389719+0000", 2023-09-23T18:05:25.570 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032934915000000002 2023-09-23T18:05:25.570 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.570 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.570 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:25.570 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.389760+0000", 2023-09-23T18:05:25.570 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0655999999999998e-05 2023-09-23T18:05:25.571 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.571 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.571 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:25.571 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391237+0000", 2023-09-23T18:05:25.571 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014769970000000001 2023-09-23T18:05:25.571 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:25.571 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:25.571 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:25.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.784 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154260 5.d 5:b55d3ddc:::benchmark_data_smithi055_229117_object48634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:05:25.784 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.880860+0000", 2023-09-23T18:05:25.784 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 353.92910787099999, 2023-09-23T18:05:25.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5120832480000002, 2023-09-23T18:05:25.784 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:25.784 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:25.784 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:25.785 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:25.785 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:25.785 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154260 2023-09-23T18:05:25.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.785 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:25.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:25.785 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880860+0000", 2023-09-23T18:05:25.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:25.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880860+0000", 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880859+0000", 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880866+0000", 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2790000000000001e-06 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880882+0000", 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5529999999999999e-05 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880890+0000", 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.602e-06 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880936+0000", 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5847000000000003e-05 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:25.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880946+0000", 2023-09-23T18:05:25.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8519999999999999e-06 2023-09-23T18:05:25.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:25.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:25.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:25.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851234+0000", 2023-09-23T18:05:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97028760300000005 2023-09-23T18:05:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851240+0000", 2023-09-23T18:05:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5690000000000001e-06 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.351422+0000", 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.500181193 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.351601+0000", 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017931699999999999 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.351779+0000", 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017792700000000001 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360041+0000", 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082620220000000008 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360048+0000", 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4649999999999997e-06 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391364+0000", 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031316043000000002 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391424+0000", 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0053000000000003e-05 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.392943+0000", 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001518997 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154262 5.d 5:bdaa66ea:::benchmark_data_smithi055_229117_object48636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:05:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.881965+0000", 2023-09-23T18:05:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 353.92800297899998, 2023-09-23T18:05:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.526328737, 2023-09-23T18:05:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154262 2023-09-23T18:05:26.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.242 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:26.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:26.242 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881965+0000", 2023-09-23T18:05:26.242 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:26.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:26.242 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881965+0000", 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881963+0000", 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881971+0000", 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8699999999999992e-06 2023-09-23T18:05:26.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881990+0000", 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9173000000000001e-05 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.882000+0000", 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4210000000000001e-06 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.882051+0000", 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1026999999999998e-05 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.882062+0000", 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1463e-05 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:26.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851261+0000", 2023-09-23T18:05:26.246 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96919898400000004 2023-09-23T18:05:26.246 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.246 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.246 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:26.246 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851295+0000", 2023-09-23T18:05:26.246 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3628999999999999e-05 2023-09-23T18:05:26.496 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.496 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.496 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:26.496 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360083+0000", 2023-09-23T18:05:26.496 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5087881159999998 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360255+0000", 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017206400000000001 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360415+0000", 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015982 2023-09-23T18:05:26.497 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.374968+0000", 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014553857999999999 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391256+0000", 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016287340000000001 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.498 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.406335+0000", 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015079231 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.406390+0000", 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4511e-05 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.499 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.408294+0000", 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001904177 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154263 5.b 5:dbd24d9d:::benchmark_data_smithi055_229117_object48637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.883292+0000", 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 353.92667555999998, 2023-09-23T18:05:26.500 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0983152249999999, 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154263 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883292+0000", 2023-09-23T18:05:26.501 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:26.502 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.502 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:26.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883292+0000", 2023-09-23T18:05:26.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:26.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.752 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.752 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:26.752 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883291+0000", 2023-09-23T18:05:26.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:05:26.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.753 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:26.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883300+0000", 2023-09-23T18:05:26.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6999999999999997e-06 2023-09-23T18:05:26.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.753 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:26.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883317+0000", 2023-09-23T18:05:26.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7048999999999999e-05 2023-09-23T18:05:26.754 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.754 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:26.754 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883327+0000", 2023-09-23T18:05:26.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0108999999999999e-05 2023-09-23T18:05:26.754 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:26.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883392+0000", 2023-09-23T18:05:26.755 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4751000000000004e-05 2023-09-23T18:05:26.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:26.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883402+0000", 2023-09-23T18:05:26.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0149000000000001e-05 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.854589+0000", 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97118778699999997 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.854601+0000", 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1292e-05 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:26.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:26.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:27.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.958490+0000", 2023-09-23T18:05:27.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.103889399 2023-09-23T18:05:27.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:27.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.958651+0000", 2023-09-23T18:05:27.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016055600000000001 2023-09-23T18:05:27.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.958761+0000", 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011042799999999999 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.968153+0000", 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093921730000000002 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:27.091 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981288+0000", 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01313507 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981455+0000", 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000166551 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981509+0000", 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4163999999999998e-05 2023-09-23T18:05:27.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981608+0000", 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8573000000000006e-05 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154267 5.4 5:27d4bd8e:::benchmark_data_smithi055_229117_object48641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.891154+0000", 2023-09-23T18:05:27.093 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 353.91881385300002, 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2238628070000002, 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154267 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:27.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891154+0000", 2023-09-23T18:05:27.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:27.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:27.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891154+0000", 2023-09-23T18:05:27.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:27.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:27.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:27.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891152+0000", 2023-09-23T18:05:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:05:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:28.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:28.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891163+0000", 2023-09-23T18:05:28.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0482999999999999e-05 2023-09-23T18:05:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891191+0000", 2023-09-23T18:05:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8411e-05 2023-09-23T18:05:28.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:28.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:28.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891202+0000", 2023-09-23T18:05:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0841e-05 2023-09-23T18:05:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:28.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:29.126 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891258+0000", 2023-09-23T18:05:29.127 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6165999999999997e-05 2023-09-23T18:05:29.127 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.127 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.127 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:29.127 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891269+0000", 2023-09-23T18:05:29.127 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0572e-05 2023-09-23T18:05:29.127 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.127 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.128 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:29.128 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851195+0000", 2023-09-23T18:05:29.128 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95992611100000003 2023-09-23T18:05:29.128 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.128 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851204+0000", 2023-09-23T18:05:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.645e-06 2023-09-23T18:05:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100511+0000", 2023-09-23T18:05:29.638 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.249306255 2023-09-23T18:05:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100638+0000", 2023-09-23T18:05:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012729000000000001 2023-09-23T18:05:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T18:05:29.639 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100765+0000", 2023-09-23T18:05:29.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012738699999999999 2023-09-23T18:05:29.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:29.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105586+0000", 2023-09-23T18:05:29.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048209419999999999 2023-09-23T18:05:29.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105597+0000", 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0890000000000001e-05 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.114801+0000", 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092037030000000006 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.114844+0000", 2023-09-23T18:05:29.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3130999999999998e-05 2023-09-23T18:05:29.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:29.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.114898+0000", 2023-09-23T18:05:29.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4320999999999997e-05 2023-09-23T18:05:29.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.115017+0000", 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118578 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154270 5.d 5:b225da75:::benchmark_data_smithi055_229117_object48644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:09.964328+0000", 2023-09-23T18:05:29.975 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 352.84563957199998, 2023-09-23T18:05:29.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4456077330000001, 2023-09-23T18:05:29.976 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:29.976 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:29.976 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:29.976 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:29.976 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:29.976 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154270 2023-09-23T18:05:29.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.199 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:31.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:31.199 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964328+0000", 2023-09-23T18:05:31.199 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:31.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964328+0000", 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964327+0000", 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:31.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964335+0000", 2023-09-23T18:05:31.201 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9510000000000006e-06 2023-09-23T18:05:31.201 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.201 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.201 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:31.201 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964352+0000", 2023-09-23T18:05:31.201 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7091e-05 2023-09-23T18:05:31.201 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.204 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964361+0000", 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3870000000000001e-06 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964414+0000", 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2559999999999998e-05 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:31.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964421+0000", 2023-09-23T18:05:31.206 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0230000000000004e-06 2023-09-23T18:05:31.206 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.206 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.206 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:31.206 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.365357+0000", 2023-09-23T18:05:31.206 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.400936304 2023-09-23T18:05:31.206 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.206 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.206 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:31.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.365487+0000", 2023-09-23T18:05:31.699 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013022199999999999 2023-09-23T18:05:31.699 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.699 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.699 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:05:31.699 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.365592+0000", 2023-09-23T18:05:31.699 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104739 2023-09-23T18:05:31.699 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.699 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.699 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:31.700 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.389646+0000", 2023-09-23T18:05:31.700 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024053990000000001 2023-09-23T18:05:31.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.700 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.700 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:31.700 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391266+0000", 2023-09-23T18:05:31.700 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016202969999999999 2023-09-23T18:05:31.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.704 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:31.704 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.408353+0000", 2023-09-23T18:05:31.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017086086 2023-09-23T18:05:31.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.704 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:31.704 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.408410+0000", 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7760999999999998e-05 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.409936+0000", 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015258559999999999 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:31.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.033 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154271 5.6 5:6eca1f68:::benchmark_data_smithi055_229117_object48645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:05:32.033 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:09.964509+0000", 2023-09-23T18:05:32.033 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 352.84545873399998, 2023-09-23T18:05:32.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.263656766, 2023-09-23T18:05:32.033 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154271 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964509+0000", 2023-09-23T18:05:32.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:32.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:32.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964509+0000", 2023-09-23T18:05:32.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:32.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.036 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:32.036 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964508+0000", 2023-09-23T18:05:32.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:05:32.036 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.036 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.036 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:32.036 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964515+0000", 2023-09-23T18:05:32.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3600000000000001e-06 2023-09-23T18:05:32.036 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.037 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:32.037 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964530+0000", 2023-09-23T18:05:32.037 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5153e-05 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964538+0000", 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1119999999999996e-06 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964594+0000", 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5926e-05 2023-09-23T18:05:32.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:32.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964603+0000", 2023-09-23T18:05:32.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2220000000000008e-06 2023-09-23T18:05:32.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:32.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.215843+0000", 2023-09-23T18:05:32.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2512403320000001 2023-09-23T18:05:32.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:32.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.215990+0000", 2023-09-23T18:05:32.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000147095 2023-09-23T18:05:32.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:05:32.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.216730+0000", 2023-09-23T18:05:32.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00073915700000000005 2023-09-23T18:05:32.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:32.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225069+0000", 2023-09-23T18:05:32.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0083390189999999996 2023-09-23T18:05:32.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:32.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225111+0000", 2023-09-23T18:05:32.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2026e-05 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225280+0000", 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000169524 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225348+0000", 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8232000000000007e-05 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:32.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.228166+0000", 2023-09-23T18:05:32.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028175650000000002 2023-09-23T18:05:32.377 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:32.377 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:32.377 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:32.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.377 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33521 6.2 6:4a04cd74:::benchmark_data_smithi055_275963_object33520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:05:32.378 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.422794+0000", 2023-09-23T18:05:32.378 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 204.38717442399999, 2023-09-23T18:05:32.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.013332479, 2023-09-23T18:05:32.378 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:32.378 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:05:32.699 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:32.699 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:05:32.699 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33521 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422794+0000", 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422794+0000", 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422792+0000", 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422842+0000", 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0219999999999997e-05 2023-09-23T18:05:32.701 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422852+0000", 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4700000000000008e-06 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422857+0000", 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5779999999999998e-06 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.702 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422908+0000", 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0387999999999999e-05 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423038+0000", 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013043899999999999 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:05:32.703 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423199+0000", 2023-09-23T18:05:32.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016039900000000001 2023-09-23T18:05:32.704 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:32.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:32.704 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:33.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.425419+0000", 2023-09-23T18:05:33.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022201949999999999 2023-09-23T18:05:33.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:33.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448248+0000", 2023-09-23T18:05:33.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022829682 2023-09-23T18:05:33.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448261+0000", 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2745000000000001e-05 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.436111+0000", 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98784968799999995 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:33.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.436126+0000", 2023-09-23T18:05:33.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5163e-05 2023-09-23T18:05:33.041 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:33.041 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:33.041 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:33.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.041 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.041 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33523 6.f 6:f5fa26bf:::benchmark_data_smithi055_275963_object33522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:05:33.285 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.423670+0000", 2023-09-23T18:05:33.614 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 204.386298162, 2023-09-23T18:05:33.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.050894467, 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33523 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423670+0000", 2023-09-23T18:05:33.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:33.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:33.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423670+0000", 2023-09-23T18:05:33.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:33.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:33.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423669+0000", 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423707+0000", 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8553999999999997e-05 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423723+0000", 2023-09-23T18:05:33.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5512e-05 2023-09-23T18:05:33.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:33.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423732+0000", 2023-09-23T18:05:33.618 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6130000000000004e-06 2023-09-23T18:05:33.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:33.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423786+0000", 2023-09-23T18:05:33.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4255999999999997e-05 2023-09-23T18:05:33.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:33.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423953+0000", 2023-09-23T18:05:33.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016769800000000001 2023-09-23T18:05:33.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:05:33.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.424125+0000", 2023-09-23T18:05:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000171409 2023-09-23T18:05:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.425758+0000", 2023-09-23T18:05:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016331780000000001 2023-09-23T18:05:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:33.620 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448278+0000", 2023-09-23T18:05:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022520401999999998 2023-09-23T18:05:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448286+0000", 2023-09-23T18:05:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4089999999999997e-06 2023-09-23T18:05:33.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433253+0000", 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98496738100000003 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433268+0000", 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4986e-05 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:33.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:33.623 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:33.623 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.463950+0000", 2023-09-23T18:05:33.623 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0306814449999999 2023-09-23T18:05:33.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:05:35.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465116+0000", 2023-09-23T18:05:35.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011660500000000001 2023-09-23T18:05:35.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:35.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474508+0000", 2023-09-23T18:05:35.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093924479999999994 2023-09-23T18:05:35.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:35.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474564+0000", 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6112000000000001e-05 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33545 6.f 6:f8f54476:::benchmark_data_smithi055_275963_object33544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.452774+0000", 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 204.357194327, 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0213147949999999, 2023-09-23T18:05:35.707 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33545 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452774+0000", 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:35.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452774+0000", 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452773+0000", 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452804+0000", 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1013000000000003e-05 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452817+0000", 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3525e-05 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.710 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452825+0000", 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5809999999999999e-06 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452879+0000", 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4762999999999997e-05 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452893+0000", 2023-09-23T18:05:35.711 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3278999999999999e-05 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433299+0000", 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98040629000000001 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433308+0000", 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6859999999999995e-06 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:35.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465523+0000", 2023-09-23T18:05:35.713 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0322154109999999 2023-09-23T18:05:35.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:35.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465714+0000", 2023-09-23T18:05:35.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000190941 2023-09-23T18:05:35.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:05:35.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465951+0000", 2023-09-23T18:05:35.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00023701199999999999 2023-09-23T18:05:35.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:35.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.470811+0000", 2023-09-23T18:05:35.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004859693 2023-09-23T18:05:35.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:35.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:35.904 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.470872+0000", 2023-09-23T18:05:35.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1622000000000001e-05 2023-09-23T18:05:35.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:35.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:36.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.473905+0000", 2023-09-23T18:05:36.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030324229999999998 2023-09-23T18:05:36.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.257 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:36.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.473979+0000", 2023-09-23T18:05:36.451 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4177999999999994e-05 2023-09-23T18:05:36.451 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.451 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.451 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:36.451 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474088+0000", 2023-09-23T18:05:36.451 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010944500000000001 2023-09-23T18:05:36.451 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:36.452 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:36.454 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:36.454 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.454 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.454 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33550 6.f 6:fa5a6c38:::benchmark_data_smithi055_275963_object33549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:05:36.454 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.456197+0000", 2023-09-23T18:05:36.454 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 204.35377106199999, 2023-09-23T18:05:36.455 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0180980869999998, 2023-09-23T18:05:36.455 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:36.455 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:36.455 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:36.455 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:05:36.455 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:36.455 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33550 2023-09-23T18:05:36.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.455 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:36.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.456 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:36.456 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456197+0000", 2023-09-23T18:05:36.456 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:36.456 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.456 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.456 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:36.456 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456197+0000", 2023-09-23T18:05:36.456 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:36.456 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.456 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:36.457 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456195+0000", 2023-09-23T18:05:36.457 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:05:36.655 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.655 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.655 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:36.655 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456221+0000", 2023-09-23T18:05:36.655 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5984000000000001e-05 2023-09-23T18:05:36.655 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.656 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.656 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:36.656 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456241+0000", 2023-09-23T18:05:36.656 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0132999999999998e-05 2023-09-23T18:05:36.656 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.656 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.656 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:36.656 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456251+0000", 2023-09-23T18:05:36.656 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4180000000000003e-06 2023-09-23T18:05:36.657 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.657 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.657 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:36.657 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456325+0000", 2023-09-23T18:05:36.657 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4190000000000006e-05 2023-09-23T18:05:36.657 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.657 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.657 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:36.657 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456341+0000", 2023-09-23T18:05:36.658 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5846e-05 2023-09-23T18:05:36.658 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.658 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.658 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:36.658 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433338+0000", 2023-09-23T18:05:36.658 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97699677799999995 2023-09-23T18:05:36.658 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.658 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.659 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:36.659 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433347+0000", 2023-09-23T18:05:36.659 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2590000000000006e-06 2023-09-23T18:05:36.659 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.659 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.659 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:36.659 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.467560+0000", 2023-09-23T18:05:36.659 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.034212833 2023-09-23T18:05:36.660 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.660 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.660 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:36.660 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.467689+0000", 2023-09-23T18:05:36.660 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012873499999999999 2023-09-23T18:05:36.660 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.660 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.660 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:05:36.660 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.467798+0000", 2023-09-23T18:05:36.660 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010996400000000001 2023-09-23T18:05:36.661 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.661 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.661 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:36.661 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.470822+0000", 2023-09-23T18:05:36.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030237990000000002 2023-09-23T18:05:36.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:36.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.471707+0000", 2023-09-23T18:05:36.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00088451899999999995 2023-09-23T18:05:36.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:36.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474135+0000", 2023-09-23T18:05:36.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002428655 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474192+0000", 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6406999999999997e-05 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474295+0000", 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000103137 2023-09-23T18:05:36.979 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:36.980 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:36.980 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:36.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.980 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33562 6.f 6:f588a2ef:::benchmark_data_smithi055_275963_object33561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:05:36.980 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.476003+0000", 2023-09-23T18:05:36.981 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 204.333964624, 2023-09-23T18:05:36.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9984609899999999, 2023-09-23T18:05:36.981 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:05:36.981 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:36.981 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:05:36.981 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:05:36.981 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:36.981 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33562 2023-09-23T18:05:36.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:36.981 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:05:36.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:36.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:05:37.197 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476003+0000", 2023-09-23T18:05:37.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:37.198 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.198 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.198 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:05:37.199 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476003+0000", 2023-09-23T18:05:37.199 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:05:37.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:05:37.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476002+0000", 2023-09-23T18:05:37.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:05:37.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.200 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.200 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:05:37.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476021+0000", 2023-09-23T18:05:37.201 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8847000000000001e-05 2023-09-23T18:05:37.201 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.201 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.201 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:05:37.202 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476034+0000", 2023-09-23T18:05:37.202 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2411e-05 2023-09-23T18:05:37.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.202 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.202 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:05:37.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476039+0000", 2023-09-23T18:05:37.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5709999999999997e-06 2023-09-23T18:05:37.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:37.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476088+0000", 2023-09-23T18:05:37.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8936e-05 2023-09-23T18:05:37.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:37.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476097+0000", 2023-09-23T18:05:37.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3600000000000002e-06 2023-09-23T18:05:37.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433567+0000", 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95746954799999995 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433577+0000", 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0359e-05 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.469156+0000", 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0355791350000001 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.469366+0000", 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000209228 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.469586+0000", 2023-09-23T18:05:37.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00022075999999999999 2023-09-23T18:05:37.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.556 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.556 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:05:37.556 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.471026+0000", 2023-09-23T18:05:37.556 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001439582 2023-09-23T18:05:37.556 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.556 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.556 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:37.557 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.471737+0000", 2023-09-23T18:05:37.557 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00071104700000000002 2023-09-23T18:05:37.557 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.557 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.557 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:37.557 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474335+0000", 2023-09-23T18:05:37.558 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025979190000000002 2023-09-23T18:05:37.558 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.558 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.558 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:05:37.558 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474377+0000", 2023-09-23T18:05:37.558 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1820000000000003e-05 2023-09-23T18:05:37.559 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:05:37.559 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:05:37.559 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:05:37.559 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474464+0000", 2023-09-23T18:05:37.559 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7562e-05 2023-09-23T18:05:37.559 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:37.559 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:37.560 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:37.560 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:05:37.560 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:05:37.560 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:05:37.560 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:05:37.561 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4082852707' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:05:37.561 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:02 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:05:37.562 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4082852707' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:05:37.562 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:02 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:05:37.776 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[184174]: pgmap v661: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+remapped, 111 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 681 op/s; 17069/81 objects degraded (21072.840%); 357 KiB/s, 123 objects/s recovering 2023-09-23T18:05:37.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:05:37.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[184174]: failed to encode map e270 with expected crc 2023-09-23T18:05:37.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:05:37.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[184174]: osdmap e270: 8 total, 8 up, 7 in 2023-09-23T18:05:37.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[184174]: failed to encode map e270 with expected crc 2023-09-23T18:05:37.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 17069/81 objects degraded (21072.840%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:05:37.778 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:05:37.779 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:05:37.780 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[179668]: pgmap v661: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+remapped, 111 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 681 op/s; 17069/81 objects degraded (21072.840%); 357 KiB/s, 123 objects/s recovering 2023-09-23T18:05:37.780 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:05:37.780 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[179668]: failed to encode map e270 with expected crc 2023-09-23T18:05:37.780 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:05:37.780 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[179668]: osdmap e270: 8 total, 8 up, 7 in 2023-09-23T18:05:37.780 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[179668]: failed to encode map e270 with expected crc 2023-09-23T18:05:37.780 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 17069/81 objects degraded (21072.840%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:05:37.780 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:05 smithi055 ceph-mon[179668]: pgmap v663: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+remapped, 111 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 17069/81 objects degraded (21072.840%); 0 B/s, 127 objects/s recovering 2023-09-23T18:05:37.780 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:05 smithi055 ceph-mon[179668]: failed to encode map e271 with expected crc 2023-09-23T18:05:37.780 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:05 smithi055 ceph-mon[179668]: failed to encode map e271 with expected crc 2023-09-23T18:05:37.781 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:05 smithi055 ceph-mon[179668]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T18:05:37.781 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:05 smithi055 ceph-mon[179668]: failed to encode map e271 with expected crc 2023-09-23T18:05:37.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:05 smithi055 ceph-mon[184174]: pgmap v663: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+remapped, 111 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 17069/81 objects degraded (21072.840%); 0 B/s, 127 objects/s recovering 2023-09-23T18:05:37.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:05 smithi055 ceph-mon[184174]: failed to encode map e271 with expected crc 2023-09-23T18:05:37.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:05 smithi055 ceph-mon[184174]: failed to encode map e271 with expected crc 2023-09-23T18:05:37.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:05 smithi055 ceph-mon[184174]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T18:05:37.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:05 smithi055 ceph-mon[184174]: failed to encode map e271 with expected crc 2023-09-23T18:05:37.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[179668]: failed to encode map e272 with expected crc 2023-09-23T18:05:37.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[179668]: failed to encode map e272 with expected crc 2023-09-23T18:05:37.988 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[179668]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T18:05:37.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[179668]: failed to encode map e272 with expected crc 2023-09-23T18:05:37.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:37.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:37.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[184174]: failed to encode map e272 with expected crc 2023-09-23T18:05:37.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[184174]: failed to encode map e272 with expected crc 2023-09-23T18:05:37.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[184174]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T18:05:37.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[184174]: failed to encode map e272 with expected crc 2023-09-23T18:05:37.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:37.991 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:37.993 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:05:38.712 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:38.719 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4082852707' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:05:38.839 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:02 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:05:38.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:03 smithi099 ceph-mon[167076]: pgmap v661: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+remapped, 111 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 681 op/s; 17069/81 objects degraded (21072.840%); 357 KiB/s, 123 objects/s recovering 2023-09-23T18:05:38.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:03 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:05:38.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:03 smithi099 ceph-mon[167076]: failed to encode map e270 with expected crc 2023-09-23T18:05:38.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:03 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:05:38.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:03 smithi099 ceph-mon[167076]: osdmap e270: 8 total, 8 up, 7 in 2023-09-23T18:05:38.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:03 smithi099 ceph-mon[167076]: failed to encode map e270 with expected crc 2023-09-23T18:05:38.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 17069/81 objects degraded (21072.840%), 6 pgs degraded, 7 pgs undersized (PG_DEGRADED) 2023-09-23T18:05:38.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:05 smithi099 ceph-mon[167076]: pgmap v663: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+recovery_wait+remapped, 111 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 435 op/s; 17069/81 objects degraded (21072.840%); 0 B/s, 127 objects/s recovering 2023-09-23T18:05:38.840 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:05 smithi099 ceph-mon[167076]: failed to encode map e271 with expected crc 2023-09-23T18:05:38.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:05 smithi099 ceph-mon[167076]: failed to encode map e271 with expected crc 2023-09-23T18:05:38.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:05 smithi099 ceph-mon[167076]: osdmap e271: 8 total, 8 up, 7 in 2023-09-23T18:05:38.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:05 smithi099 ceph-mon[167076]: failed to encode map e271 with expected crc 2023-09-23T18:05:38.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:06 smithi099 ceph-mon[167076]: failed to encode map e272 with expected crc 2023-09-23T18:05:38.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:06 smithi099 ceph-mon[167076]: failed to encode map e272 with expected crc 2023-09-23T18:05:38.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:06 smithi099 ceph-mon[167076]: osdmap e272: 8 total, 8 up, 7 in 2023-09-23T18:05:38.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:06 smithi099 ceph-mon[167076]: failed to encode map e272 with expected crc 2023-09-23T18:05:38.841 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:38.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:38.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:07 smithi099 ceph-mon[167076]: pgmap v666: 119 pgs: 1 active+undersized+remapped+backfill_wait, 111 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 14 op/s; 17069/81 objects degraded (21072.840%); 0 B/s, 175 objects/s recovering 2023-09-23T18:05:38.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:08 smithi099 ceph-mon[167076]: from='client.45500 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:05:38.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:09 smithi099 ceph-mon[167076]: pgmap v667: 119 pgs: 2 active+undersized+remapped+backfill_wait, 112 active+clean, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 16530/81 objects degraded (20407.407%); 0 B/s, 89 objects/s recovering 2023-09-23T18:05:38.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:09 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16530/81 objects degraded (20407.407%), 4 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:05:38.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3694469409' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:05:38.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:11 smithi099 ceph-mon[167076]: failed to encode map e273 with expected crc 2023-09-23T18:05:38.842 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:11 smithi099 ceph-mon[167076]: failed to encode map e273 with expected crc 2023-09-23T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:11 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3694469409' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:11 smithi099 ceph-mon[167076]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:12 smithi099 ceph-mon[167076]: pgmap v669: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:12 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16530/81 objects degraded (20407.407%), 4 pgs degraded, 6 pgs undersized) 2023-09-23T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:12 smithi099 ceph-mon[167076]: failed to encode map e274 with expected crc 2023-09-23T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:12 smithi099 ceph-mon[167076]: failed to encode map e274 with expected crc 2023-09-23T18:05:38.843 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:12 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:12 smithi099 ceph-mon[167076]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:13 smithi099 ceph-mon[167076]: failed to encode map e275 with expected crc 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:13 smithi099 ceph-mon[167076]: failed to encode map e275 with expected crc 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:13 smithi099 ceph-mon[167076]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:13 smithi099 ceph-mon[167076]: failed to encode map e275 with expected crc 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:13 smithi099 ceph-mon[167076]: failed to encode map e275 with expected crc 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:13 smithi099 ceph-mon[167076]: failed to encode map e275 with expected crc 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:13 smithi099 ceph-mon[167076]: failed to encode map e275 with expected crc 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:14 smithi099 ceph-mon[167076]: pgmap v672: 103 pgs: 6 unknown, 97 active+clean; 4.3 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:14 smithi099 ceph-mon[167076]: 1.18 continuing backfill to osd.2 from (0'0,89'161] MIN to 89'161 2023-09-23T18:05:38.844 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:14 smithi099 ceph-mon[167076]: failed to encode map e276 with expected crc 2023-09-23T18:05:38.983 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:14 smithi099 ceph-mon[167076]: failed to encode map e276 with expected crc 2023-09-23T18:05:38.983 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:14 smithi099 ceph-mon[167076]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T18:05:38.983 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:14 smithi099 ceph-mon[167076]: failed to encode map e276 with expected crc 2023-09-23T18:05:38.984 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:14 smithi099 ceph-mon[167076]: failed to encode map e276 with expected crc 2023-09-23T18:05:38.984 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:14 smithi099 ceph-mon[167076]: failed to encode map e276 with expected crc 2023-09-23T18:05:38.984 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:14 smithi099 ceph-mon[167076]: failed to encode map e276 with expected crc 2023-09-23T18:05:38.984 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:14 smithi099 ceph-mon[167076]: failed to encode map e276 with expected crc 2023-09-23T18:05:38.984 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:15 smithi099 ceph-mon[167076]: failed to encode map e277 with expected crc 2023-09-23T18:05:38.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:15 smithi099 ceph-mon[167076]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T18:05:38.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:15 smithi099 ceph-mon[167076]: failed to encode map e277 with expected crc 2023-09-23T18:05:38.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: pgmap v675: 103 pgs: 5 peering, 4 remapped+peering, 6 unknown, 88 active+clean; 4.3 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:38.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: failed to encode map e278 with expected crc 2023-09-23T18:05:38.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: failed to encode map e278 with expected crc 2023-09-23T18:05:38.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: failed to encode map e278 with expected crc 2023-09-23T18:05:38.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: failed to encode map e278 with expected crc 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: failed to encode map e278 with expected crc 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: failed to encode map e278 with expected crc 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: failed to encode map e278 with expected crc 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: failed to encode map e278 with expected crc 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:16 smithi099 ceph-mon[167076]: failed to encode map e278 with expected crc 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:17 smithi099 ceph-mon[167076]: pgmap v677: 103 pgs: 5 peering, 4 remapped+peering, 94 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 33 KiB/s rd, 86 KiB/s wr, 25 op/s; 277 KiB/s, 1 objects/s recovering 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:20 smithi099 ceph-mon[167076]: pgmap v678: 103 pgs: 5 peering, 98 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 25 KiB/s rd, 67 KiB/s wr, 20 op/s; 302 KiB/s, 1 objects/s recovering 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:22 smithi099 ceph-mon[167076]: pgmap v679: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 15 op/s; 236 KiB/s, 0 objects/s recovering 2023-09-23T18:05:38.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:24 smithi099 ceph-mon[167076]: pgmap v680: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 18 KiB/s rd, 46 KiB/s wr, 13 op/s; 209 KiB/s, 0 objects/s recovering 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:26 smithi099 ceph-mon[167076]: pgmap v681: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 12 op/s; 181 KiB/s, 0 objects/s recovering 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:27 smithi099 ceph-mon[167076]: pgmap v682: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 13 KiB/s rd, 35 KiB/s wr, 10 op/s; 156 KiB/s, 0 objects/s recovering 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:29 smithi099 ceph-mon[167076]: pgmap v683: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:32 smithi099 ceph-mon[167076]: pgmap v684: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:34 smithi099 ceph-mon[167076]: pgmap v685: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:36 smithi099 ceph-mon[167076]: pgmap v686: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:38.987 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:38 smithi099 ceph-mon[167076]: pgmap v687: 103 pgs: 103 active+clean; 4.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:38.990 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:05:39.112 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:05:39.112 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":666,"stamp":"2023-09-23T18:05:06.729833+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4602350,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17069,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":6284,"num_read_kb":12770,"num_write":147341,"num_write_kb":4695012,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28703,"num_bytes_recovered":1659702247,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":107730,"ondisk_log_size":107730,"up":357,"acting":350,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":399,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2762208,"kb_used_data":1122088,"kb_used_omap":43,"kb_used_meta":1640020,"kb_avail":653453856,"statfs":{"total":671965249536,"available":669136748544,"internally_reserved":0,"allocated":1149018112,"data_stored":1140637449,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1679381199},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-5636120,"num_objects":-87,"num_object_clones":0,"num_object_copies":-261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1090,"num_objects_misplaced":-16,"num_objects_unfound":0,"num_objects_dirty":-87,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":87,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1056,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.003316"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":594,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337491+0000","last_change":"2023-09-23T18:03:33.004903+0000","last_active":"2023-09-23T18:04:40.337491+0000","last_peered":"2023-09-23T18:04:40.337491+0000","last_clean":"2023-09-23T18:04:40.337491+0000","last_became_active":"2023-09-23T18:03:33.004498+0000","last_became_peered":"2023-09-23T18:03:33.004498+0000","last_unstale":"2023-09-23T18:04:40.337491+0000","last_undegraded":"2023-09-23T18:04:40.337491+0000","last_fullsized":"2023-09-23T18:04:40.337491+0000","mapping_epoch":259,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":737225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":70,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.375685+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:04:45.375685+0000","last_peered":"2023-09-23T18:04:45.375685+0000","last_clean":"2023-09-23T18:04:45.375685+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:04:45.375685+0000","last_undegraded":"2023-09-23T18:04:45.375685+0000","last_fullsized":"2023-09-23T18:04:45.375685+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":154,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.859463+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:05:04.859463+0000","last_peered":"2023-09-23T18:05:04.859463+0000","last_clean":"2023-09-23T18:05:04.859463+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:05:04.859463+0000","last_undegraded":"2023-09-23T18:05:04.859463+0000","last_fullsized":"2023-09-23T18:05:04.859463+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":611,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337197+0000","last_change":"2023-09-23T18:02:51.046916+0000","last_active":"2023-09-23T18:04:40.337197+0000","last_peered":"2023-09-23T18:04:40.337197+0000","last_clean":"2023-09-23T18:04:40.337197+0000","last_became_active":"2023-09-23T18:02:51.046591+0000","last_became_peered":"2023-09-23T18:02:51.046591+0000","last_unstale":"2023-09-23T18:04:40.337197+0000","last_undegraded":"2023-09-23T18:04:40.337197+0000","last_fullsized":"2023-09-23T18:04:40.337197+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:46:19.805099+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":122,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.859301+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:05:04.859301+0000","last_peered":"2023-09-23T18:05:04.859301+0000","last_clean":"2023-09-23T18:05:04.859301+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:05:04.859301+0000","last_undegraded":"2023-09-23T18:05:04.859301+0000","last_fullsized":"2023-09-23T18:05:04.859301+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":271,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.385624+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:04:45.385624+0000","last_peered":"2023-09-23T18:04:45.385624+0000","last_clean":"2023-09-23T18:04:45.385624+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:04:45.385624+0000","last_undegraded":"2023-09-23T18:04:45.385624+0000","last_fullsized":"2023-09-23T18:04:45.385624+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":603,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337821+0000","last_change":"2023-09-23T18:02:51.051617+0000","last_active":"2023-09-23T18:04:40.337821+0000","last_peered":"2023-09-23T18:04:40.337821+0000","last_clean":"2023-09-23T18:04:40.337821+0000","last_became_active":"2023-09-23T18:02:51.048947+0000","last_became_peered":"2023-09-23T18:02:51.048947+0000","last_unstale":"2023-09-23T18:04:40.337821+0000","last_undegraded":"2023-09-23T18:04:40.337821+0000","last_fullsized":"2023-09-23T18:04:40.337821+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:49:34.962249+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":135,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.858899+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:05:04.858899+0000","last_peered":"2023-09-23T18:05:04.858899+0000","last_clean":"2023-09-23T18:05:04.858899+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:05:04.858899+0000","last_undegraded":"2023-09-23T18:05:04.858899+0000","last_fullsized":"2023-09-23T18:05:04.858899+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":211,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-09-23T18:05:02.862322+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:05:02.862322+0000","last_peered":"2023-09-23T18:05:02.862322+0000","last_clean":"2023-09-23T18:05:02.862322+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:05:02.862322+0000","last_undegraded":"2023-09-23T18:05:02.862322+0000","last_fullsized":"2023-09-23T18:05:02.862322+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":600,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336848+0000","last_change":"2023-09-23T18:02:51.052123+0000","last_active":"2023-09-23T18:04:40.336848+0000","last_peered":"2023-09-23T18:04:40.336848+0000","last_clean":"2023-09-23T18:04:40.336848+0000","last_became_active":"2023-09-23T18:02:51.051772+0000","last_became_peered":"2023-09-23T18:02:51.051772+0000","last_unstale":"2023-09-23T18:04:40.336848+0000","last_undegraded":"2023-09-23T18:04:40.336848+0000","last_fullsized":"2023-09-23T18:04:40.336848+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":250,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.379557+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:04:45.379557+0000","last_peered":"2023-09-23T18:04:45.379557+0000","last_clean":"2023-09-23T18:04:45.379557+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:04:45.379557+0000","last_undegraded":"2023-09-23T18:04:45.379557+0000","last_fullsized":"2023-09-23T18:04:45.379557+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":129,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.859397+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:05:04.859397+0000","last_peered":"2023-09-23T18:05:04.859397+0000","last_clean":"2023-09-23T18:05:04.859397+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:05:04.859397+0000","last_undegraded":"2023-09-23T18:05:04.859397+0000","last_fullsized":"2023-09-23T18:05:04.859397+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":595,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336393+0000","last_change":"2023-09-23T18:02:51.054208+0000","last_active":"2023-09-23T18:04:40.336393+0000","last_peered":"2023-09-23T18:04:40.336393+0000","last_clean":"2023-09-23T18:04:40.336393+0000","last_became_active":"2023-09-23T18:02:51.053676+0000","last_became_peered":"2023-09-23T18:02:51.053676+0000","last_unstale":"2023-09-23T18:04:40.336393+0000","last_undegraded":"2023-09-23T18:04:40.336393+0000","last_fullsized":"2023-09-23T18:04:40.336393+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":251,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.256391+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:04:40.256391+0000","last_peered":"2023-09-23T18:04:40.256391+0000","last_clean":"2023-09-23T18:04:40.256391+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:04:40.256391+0000","last_undegraded":"2023-09-23T18:04:40.256391+0000","last_fullsized":"2023-09-23T18:04:40.256391+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":83,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.859150+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:05:04.859150+0000","last_peered":"2023-09-23T18:05:04.859150+0000","last_clean":"2023-09-23T18:05:04.859150+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:05:04.859150+0000","last_undegraded":"2023-09-23T18:05:04.859150+0000","last_fullsized":"2023-09-23T18:05:04.859150+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":603,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:59.750841+0000","last_change":"2023-09-23T18:04:59.750841+0000","last_active":"2023-09-23T18:04:59.750841+0000","last_peered":"2023-09-23T18:04:59.750841+0000","last_clean":"2023-09-23T18:04:59.750841+0000","last_became_active":"2023-09-23T18:02:51.057352+0000","last_became_peered":"2023-09-23T18:02:51.057352+0000","last_unstale":"2023-09-23T18:04:59.750841+0000","last_undegraded":"2023-09-23T18:04:59.750841+0000","last_fullsized":"2023-09-23T18:04:59.750841+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":35,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336804+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:04:40.336804+0000","last_peered":"2023-09-23T18:04:40.336804+0000","last_clean":"2023-09-23T18:04:40.336804+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:04:40.336804+0000","last_undegraded":"2023-09-23T18:04:40.336804+0000","last_fullsized":"2023-09-23T18:04:40.336804+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":167,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-09-23T18:05:02.861884+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:05:02.861884+0000","last_peered":"2023-09-23T18:05:02.861884+0000","last_clean":"2023-09-23T18:05:02.861884+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:05:02.861884+0000","last_undegraded":"2023-09-23T18:05:02.861884+0000","last_fullsized":"2023-09-23T18:05:02.861884+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":603,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:46.691964+0000","last_change":"2023-09-23T18:04:46.691964+0000","last_active":"2023-09-23T18:04:46.691964+0000","last_peered":"2023-09-23T18:04:46.691964+0000","last_clean":"2023-09-23T18:04:46.691964+0000","last_became_active":"2023-09-23T18:02:51.057768+0000","last_became_peered":"2023-09-23T18:02:51.057768+0000","last_unstale":"2023-09-23T18:04:46.691964+0000","last_undegraded":"2023-09-23T18:04:46.691964+0000","last_fullsized":"2023-09-23T18:04:46.691964+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":70,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.374659+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:04:45.374659+0000","last_peered":"2023-09-23T18:04:45.374659+0000","last_clean":"2023-09-23T18:04:45.374659+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:04:45.374659+0000","last_undegraded":"2023-09-23T18:04:45.374659+0000","last_fullsized":"2023-09-23T18:04:45.374659+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":70,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.374908+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:04:45.374908+0000","last_peered":"2023-09-23T18:04:45.374908+0000","last_clean":"2023-09-23T18:04:45.374908+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:04:45.374908+0000","last_undegraded":"2023-09-23T18:04:45.374908+0000","last_fullsized":"2023-09-23T18:04:45.374908+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":593,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:53.806479+0000","last_change":"2023-09-23T18:04:53.806479+0000","last_active":"2023-09-23T18:04:53.806479+0000","last_peered":"2023-09-23T18:04:53.806479+0000","last_clean":"2023-09-23T18:04:53.806479+0000","last_became_active":"2023-09-23T18:02:51.046963+0000","last_became_peered":"2023-09-23T18:02:51.046963+0000","last_unstale":"2023-09-23T18:04:53.806479+0000","last_undegraded":"2023-09-23T18:04:53.806479+0000","last_fullsized":"2023-09-23T18:04:53.806479+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":198,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337545+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:04:40.337545+0000","last_peered":"2023-09-23T18:04:40.337545+0000","last_clean":"2023-09-23T18:04:40.337545+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:04:40.337545+0000","last_undegraded":"2023-09-23T18:04:40.337545+0000","last_fullsized":"2023-09-23T18:04:40.337545+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":117,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.859476+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:05:04.859476+0000","last_peered":"2023-09-23T18:05:04.859476+0000","last_clean":"2023-09-23T18:05:04.859476+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:05:04.859476+0000","last_undegraded":"2023-09-23T18:05:04.859476+0000","last_fullsized":"2023-09-23T18:05:04.859476+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":603,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.338820+0000","last_change":"2023-09-23T18:02:51.053806+0000","last_active":"2023-09-23T18:04:40.338820+0000","last_peered":"2023-09-23T18:04:40.338820+0000","last_clean":"2023-09-23T18:04:40.338820+0000","last_became_active":"2023-09-23T18:02:51.048822+0000","last_became_peered":"2023-09-23T18:02:51.048822+0000","last_unstale":"2023-09-23T18:04:40.338820+0000","last_undegraded":"2023-09-23T18:04:40.338820+0000","last_fullsized":"2023-09-23T18:04:40.338820+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:34.401104+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":270,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:59.629267+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:04:59.629267+0000","last_peered":"2023-09-23T18:04:59.629267+0000","last_clean":"2023-09-23T18:04:59.629267+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:04:59.629267+0000","last_undegraded":"2023-09-23T18:04:59.629267+0000","last_fullsized":"2023-09-23T18:04:59.629267+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":246,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.256412+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:04:40.256412+0000","last_peered":"2023-09-23T18:04:40.256412+0000","last_clean":"2023-09-23T18:04:40.256412+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:04:40.256412+0000","last_undegraded":"2023-09-23T18:04:40.256412+0000","last_fullsized":"2023-09-23T18:04:40.256412+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":591,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337380+0000","last_change":"2023-09-23T18:02:51.046646+0000","last_active":"2023-09-23T18:04:40.337380+0000","last_peered":"2023-09-23T18:04:40.337380+0000","last_clean":"2023-09-23T18:04:40.337380+0000","last_became_active":"2023-09-23T18:02:51.046134+0000","last_became_peered":"2023-09-23T18:02:51.046134+0000","last_unstale":"2023-09-23T18:04:40.337380+0000","last_undegraded":"2023-09-23T18:04:40.337380+0000","last_fullsized":"2023-09-23T18:04:40.337380+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":60,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.374633+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:04:45.374633+0000","last_peered":"2023-09-23T18:04:45.374633+0000","last_clean":"2023-09-23T18:04:45.374633+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:04:45.374633+0000","last_undegraded":"2023-09-23T18:04:45.374633+0000","last_fullsized":"2023-09-23T18:04:45.374633+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":260,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.385688+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:04:45.385688+0000","last_peered":"2023-09-23T18:04:45.385688+0000","last_clean":"2023-09-23T18:04:45.385688+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:04:45.385688+0000","last_undegraded":"2023-09-23T18:04:45.385688+0000","last_fullsized":"2023-09-23T18:04:45.385688+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":603,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337740+0000","last_change":"2023-09-23T18:02:51.051817+0000","last_active":"2023-09-23T18:04:40.337740+0000","last_peered":"2023-09-23T18:04:40.337740+0000","last_clean":"2023-09-23T18:04:40.337740+0000","last_became_active":"2023-09-23T18:02:51.048945+0000","last_became_peered":"2023-09-23T18:02:51.048945+0000","last_unstale":"2023-09-23T18:04:40.337740+0000","last_undegraded":"2023-09-23T18:04:40.337740+0000","last_fullsized":"2023-09-23T18:04:40.337740+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:02:47.163864+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":207,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-09-23T18:05:02.861426+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:05:02.861426+0000","last_peered":"2023-09-23T18:05:02.861426+0000","last_clean":"2023-09-23T18:05:02.861426+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:05:02.861426+0000","last_undegraded":"2023-09-23T18:05:02.861426+0000","last_fullsized":"2023-09-23T18:05:02.861426+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":126,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.860326+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:05:04.860326+0000","last_peered":"2023-09-23T18:05:04.860326+0000","last_clean":"2023-09-23T18:05:04.860326+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:05:04.860326+0000","last_undegraded":"2023-09-23T18:05:04.860326+0000","last_fullsized":"2023-09-23T18:05:04.860326+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.a","version":"269'9242","reported_seq":9314,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:57.964723+0000","last_change":"2023-09-23T18:00:57.996982+0000","last_active":"2023-09-23T18:04:57.964723+0000","last_peered":"2023-09-23T18:04:57.964723+0000","last_clean":"2023-09-23T18:04:57.964723+0000","last_became_active":"2023-09-23T18:00:57.996589+0000","last_became_peered":"2023-09-23T18:00:57.996589+0000","last_unstale":"2023-09-23T18:04:57.964723+0000","last_undegraded":"2023-09-23T18:04:57.964723+0000","last_fullsized":"2023-09-23T18:04:57.964723+0000","mapping_epoch":204,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":205,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":9242,"log_dups_size":0,"ondisk_log_size":9242,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:13.306089+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9242,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":600,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336755+0000","last_change":"2023-09-23T18:02:51.058477+0000","last_active":"2023-09-23T18:04:40.336755+0000","last_peered":"2023-09-23T18:04:40.336755+0000","last_clean":"2023-09-23T18:04:40.336755+0000","last_became_active":"2023-09-23T18:02:51.057258+0000","last_became_peered":"2023-09-23T18:02:51.057258+0000","last_unstale":"2023-09-23T18:04:40.336755+0000","last_undegraded":"2023-09-23T18:04:40.336755+0000","last_fullsized":"2023-09-23T18:04:40.336755+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":198,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337591+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:04:40.337591+0000","last_peered":"2023-09-23T18:04:40.337591+0000","last_clean":"2023-09-23T18:04:40.337591+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:04:40.337591+0000","last_undegraded":"2023-09-23T18:04:40.337591+0000","last_fullsized":"2023-09-23T18:04:40.337591+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":257,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.255879+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:04:40.255879+0000","last_peered":"2023-09-23T18:04:40.255879+0000","last_clean":"2023-09-23T18:04:40.255879+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:04:40.255879+0000","last_undegraded":"2023-09-23T18:04:40.255879+0000","last_fullsized":"2023-09-23T18:04:40.255879+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.b","version":"269'9084","reported_seq":17140,"reported_epoch":270,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:05:04.386177+0000","last_change":"2023-09-23T18:05:04.386060+0000","last_active":"2023-09-23T18:05:04.386177+0000","last_peered":"2023-09-23T18:05:04.386177+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:49.001792+0000","last_became_peered":"2023-09-23T18:02:49.001792+0000","last_unstale":"2023-09-23T18:05:04.386177+0000","last_undegraded":"2023-09-23T18:05:04.386177+0000","last_fullsized":"2023-09-23T18:05:04.386177+0000","mapping_epoch":250,"log_start":"253'4930","ondisk_log_start":"253'4930","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4154,"log_dups_size":0,"ondisk_log_size":4154,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:35.156979+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9084,"num_write_kb":290688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3975,"num_bytes_recovered":260505600,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[6,1,4],"avail_no_missing":["6","1","3","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":599,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336477+0000","last_change":"2023-09-23T18:04:02.152730+0000","last_active":"2023-09-23T18:04:40.336477+0000","last_peered":"2023-09-23T18:04:40.336477+0000","last_clean":"2023-09-23T18:04:40.336477+0000","last_became_active":"2023-09-23T18:04:02.152366+0000","last_became_peered":"2023-09-23T18:04:02.152366+0000","last_unstale":"2023-09-23T18:04:40.336477+0000","last_undegraded":"2023-09-23T18:04:40.336477+0000","last_fullsized":"2023-09-23T18:04:40.336477+0000","mapping_epoch":263,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":736808,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":213,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.338271+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:04:40.338271+0000","last_peered":"2023-09-23T18:04:40.338271+0000","last_clean":"2023-09-23T18:04:40.338271+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:04:40.338271+0000","last_undegraded":"2023-09-23T18:04:40.338271+0000","last_fullsized":"2023-09-23T18:04:40.338271+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":280,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.374880+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:04:45.374880+0000","last_peered":"2023-09-23T18:04:45.374880+0000","last_clean":"2023-09-23T18:04:45.374880+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:04:45.374880+0000","last_undegraded":"2023-09-23T18:04:45.374880+0000","last_fullsized":"2023-09-23T18:04:45.374880+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.8","version":"269'9242","reported_seq":17855,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:57.935817+0000","last_change":"2023-09-23T18:03:33.017384+0000","last_active":"2023-09-23T18:04:57.935817+0000","last_peered":"2023-09-23T18:04:57.935817+0000","last_clean":"2023-09-23T18:04:57.935817+0000","last_became_active":"2023-09-23T18:03:33.004338+0000","last_became_peered":"2023-09-23T18:03:33.004338+0000","last_unstale":"2023-09-23T18:04:57.935817+0000","last_undegraded":"2023-09-23T18:04:57.935817+0000","last_fullsized":"2023-09-23T18:04:57.935817+0000","mapping_epoch":259,"log_start":"240'4500","ondisk_log_start":"240'4500","created":204,"last_epoch_clean":260,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4742,"log_dups_size":0,"ondisk_log_size":4742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:35:59.468205+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9242,"num_write_kb":295744,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4258,"num_bytes_recovered":259457024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,3],"acting":[2,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":594,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:58.792847+0000","last_change":"2023-09-23T18:04:58.792847+0000","last_active":"2023-09-23T18:04:58.792847+0000","last_peered":"2023-09-23T18:04:58.792847+0000","last_clean":"2023-09-23T18:04:58.792847+0000","last_became_active":"2023-09-23T18:02:51.057799+0000","last_became_peered":"2023-09-23T18:02:51.057799+0000","last_unstale":"2023-09-23T18:04:58.792847+0000","last_undegraded":"2023-09-23T18:04:58.792847+0000","last_fullsized":"2023-09-23T18:04:58.792847+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":136,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337196+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:04:40.337196+0000","last_peered":"2023-09-23T18:04:40.337196+0000","last_clean":"2023-09-23T18:04:40.337196+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:04:40.337196+0000","last_undegraded":"2023-09-23T18:04:40.337196+0000","last_fullsized":"2023-09-23T18:04:40.337196+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":236,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-09-23T18:05:02.861810+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:05:02.861810+0000","last_peered":"2023-09-23T18:05:02.861810+0000","last_clean":"2023-09-23T18:05:02.861810+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:05:02.861810+0000","last_undegraded":"2023-09-23T18:05:02.861810+0000","last_fullsized":"2023-09-23T18:05:02.861810+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.9","version":"269'9302","reported_seq":9395,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:57.911876+0000","last_change":"2023-09-23T18:01:25.019784+0000","last_active":"2023-09-23T18:04:57.911876+0000","last_peered":"2023-09-23T18:04:57.911876+0000","last_clean":"2023-09-23T18:04:57.911876+0000","last_became_active":"2023-09-23T18:01:25.002073+0000","last_became_peered":"2023-09-23T18:01:25.002073+0000","last_unstale":"2023-09-23T18:04:57.911876+0000","last_undegraded":"2023-09-23T18:04:57.911876+0000","last_fullsized":"2023-09-23T18:04:57.911876+0000","mapping_epoch":214,"log_start":"247'4900","ondisk_log_start":"247'4900","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4402,"log_dups_size":0,"ondisk_log_size":4402,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:46:58.981672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9302,"num_write_kb":297664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":327680,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":603,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337840+0000","last_change":"2023-09-23T18:04:37.604692+0000","last_active":"2023-09-23T18:04:40.337840+0000","last_peered":"2023-09-23T18:04:40.337840+0000","last_clean":"2023-09-23T18:04:40.337840+0000","last_became_active":"2023-09-23T18:02:51.056589+0000","last_became_peered":"2023-09-23T18:02:51.056589+0000","last_unstale":"2023-09-23T18:04:40.337840+0000","last_undegraded":"2023-09-23T18:04:40.337840+0000","last_fullsized":"2023-09-23T18:04:40.337840+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":245,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.256880+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:04:40.256880+0000","last_peered":"2023-09-23T18:04:40.256880+0000","last_clean":"2023-09-23T18:04:40.256880+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:04:40.256880+0000","last_undegraded":"2023-09-23T18:04:40.256880+0000","last_fullsized":"2023-09-23T18:04:40.256880+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":273,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.385206+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:04:45.385206+0000","last_peered":"2023-09-23T18:04:45.385206+0000","last_clean":"2023-09-23T18:04:45.385206+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:04:45.385206+0000","last_undegraded":"2023-09-23T18:04:45.385206+0000","last_fullsized":"2023-09-23T18:04:45.385206+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.e","version":"269'9254","reported_seq":17552,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:57.967176+0000","last_change":"2023-09-23T18:04:40.332205+0000","last_active":"2023-09-23T18:04:57.967176+0000","last_peered":"2023-09-23T18:04:57.967176+0000","last_clean":"2023-09-23T18:04:57.967176+0000","last_became_active":"2023-09-23T18:04:40.257475+0000","last_became_peered":"2023-09-23T18:04:40.257475+0000","last_unstale":"2023-09-23T18:04:57.967176+0000","last_undegraded":"2023-09-23T18:04:57.967176+0000","last_fullsized":"2023-09-23T18:04:57.967176+0000","mapping_epoch":267,"log_start":"221'2200","ondisk_log_start":"221'2200","created":204,"last_epoch_clean":268,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7054,"log_dups_size":0,"ondisk_log_size":7054,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:48:17.161093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9254,"num_write_kb":296128,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4100,"num_bytes_recovered":264765440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,5],"acting":[4,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":592,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336294+0000","last_change":"2023-09-23T18:02:51.047609+0000","last_active":"2023-09-23T18:04:40.336294+0000","last_peered":"2023-09-23T18:04:40.336294+0000","last_clean":"2023-09-23T18:04:40.336294+0000","last_became_active":"2023-09-23T18:02:51.047209+0000","last_became_peered":"2023-09-23T18:02:51.047209+0000","last_unstale":"2023-09-23T18:04:40.336294+0000","last_undegraded":"2023-09-23T18:04:40.336294+0000","last_fullsized":"2023-09-23T18:04:40.336294+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":160,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.859131+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:05:04.859131+0000","last_peered":"2023-09-23T18:05:04.859131+0000","last_clean":"2023-09-23T18:05:04.859131+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:05:04.859131+0000","last_undegraded":"2023-09-23T18:05:04.859131+0000","last_fullsized":"2023-09-23T18:05:04.859131+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":278,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.374740+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:04:45.374740+0000","last_peered":"2023-09-23T18:04:45.374740+0000","last_clean":"2023-09-23T18:04:45.374740+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:04:45.374740+0000","last_undegraded":"2023-09-23T18:04:45.374740+0000","last_fullsized":"2023-09-23T18:04:45.374740+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.f","version":"269'9056","reported_seq":9288,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:57.969045+0000","last_change":"2023-09-23T18:02:47.961575+0000","last_active":"2023-09-23T18:04:57.969045+0000","last_peered":"2023-09-23T18:04:57.969045+0000","last_clean":"2023-09-23T18:02:17.013863+0000","last_became_active":"2023-09-23T18:02:19.267215+0000","last_became_peered":"2023-09-23T18:02:19.267215+0000","last_unstale":"2023-09-23T18:04:57.969045+0000","last_undegraded":"2023-09-23T18:02:18.955030+0000","last_fullsized":"2023-09-23T18:02:18.924656+0000","mapping_epoch":238,"log_start":"240'4300","ondisk_log_start":"240'4300","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4756,"log_dups_size":0,"ondisk_log_size":4756,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:53:46.393021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4108,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9056,"num_write_kb":289792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":63,"num_bytes_recovered":4128768,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,4],"avail_no_missing":["2","4","6"],"object_location_counts":[{"shards":"2,4","objects":-4108}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":599,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337410+0000","last_change":"2023-09-23T18:04:34.596732+0000","last_active":"2023-09-23T18:04:40.337410+0000","last_peered":"2023-09-23T18:04:40.337410+0000","last_clean":"2023-09-23T18:04:40.337410+0000","last_became_active":"2023-09-23T18:02:51.048477+0000","last_became_peered":"2023-09-23T18:02:51.048477+0000","last_unstale":"2023-09-23T18:04:40.337410+0000","last_undegraded":"2023-09-23T18:04:40.337410+0000","last_fullsized":"2023-09-23T18:04:40.337410+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":230,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-09-23T18:05:02.861962+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:05:02.861962+0000","last_peered":"2023-09-23T18:05:02.861962+0000","last_clean":"2023-09-23T18:05:02.861962+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:05:02.861962+0000","last_undegraded":"2023-09-23T18:05:02.861962+0000","last_fullsized":"2023-09-23T18:05:02.861962+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":70,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.374858+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:04:45.374858+0000","last_peered":"2023-09-23T18:04:45.374858+0000","last_clean":"2023-09-23T18:04:45.374858+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:04:45.374858+0000","last_undegraded":"2023-09-23T18:04:45.374858+0000","last_fullsized":"2023-09-23T18:04:45.374858+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.c","version":"269'9092","reported_seq":9169,"reported_epoch":269,"state":"active+undersized+remapped+backfill_wait","last_fresh":"2023-09-23T18:04:57.972062+0000","last_change":"2023-09-23T18:04:57.972894+0000","last_active":"2023-09-23T18:04:57.972062+0000","last_peered":"2023-09-23T18:04:57.972062+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.048672+0000","last_became_peered":"2023-09-23T18:02:19.048672+0000","last_unstale":"2023-09-23T18:04:57.972062+0000","last_undegraded":"2023-09-23T18:02:18.951263+0000","last_fullsized":"2023-09-23T18:02:18.930851+0000","mapping_epoch":238,"log_start":"216'261","ondisk_log_start":"216'261","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":8831,"log_dups_size":0,"ondisk_log_size":8831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:23.690233+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9092,"num_write_kb":290944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[5,6],"avail_no_missing":["5","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":600,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337113+0000","last_change":"2023-09-23T18:02:51.060858+0000","last_active":"2023-09-23T18:04:40.337113+0000","last_peered":"2023-09-23T18:04:40.337113+0000","last_clean":"2023-09-23T18:04:40.337113+0000","last_became_active":"2023-09-23T18:02:51.060554+0000","last_became_peered":"2023-09-23T18:02:51.060554+0000","last_unstale":"2023-09-23T18:04:40.337113+0000","last_undegraded":"2023-09-23T18:04:40.337113+0000","last_fullsized":"2023-09-23T18:04:40.337113+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":60,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.374705+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:04:45.374705+0000","last_peered":"2023-09-23T18:04:45.374705+0000","last_clean":"2023-09-23T18:04:45.374705+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:04:45.374705+0000","last_undegraded":"2023-09-23T18:04:45.374705+0000","last_fullsized":"2023-09-23T18:04:45.374705+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":280,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.382957+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:04:45.382957+0000","last_peered":"2023-09-23T18:04:45.382957+0000","last_clean":"2023-09-23T18:04:45.382957+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:04:45.382957+0000","last_undegraded":"2023-09-23T18:04:45.382957+0000","last_fullsized":"2023-09-23T18:04:45.382957+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"6.d","version":"269'9086","reported_seq":17212,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:57.959631+0000","last_change":"2023-09-23T18:02:47.017672+0000","last_active":"2023-09-23T18:04:57.959631+0000","last_peered":"2023-09-23T18:04:57.959631+0000","last_clean":"2023-09-23T18:04:57.959631+0000","last_became_active":"2023-09-23T18:02:47.009245+0000","last_became_peered":"2023-09-23T18:02:47.009245+0000","last_unstale":"2023-09-23T18:04:57.959631+0000","last_undegraded":"2023-09-23T18:04:57.959631+0000","last_fullsized":"2023-09-23T18:04:57.959631+0000","mapping_epoch":248,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":249,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":9086,"log_dups_size":0,"ondisk_log_size":9086,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:10:04.868743+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9086,"num_write_kb":290689,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4015,"num_bytes_recovered":263127040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":595,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336834+0000","last_change":"2023-09-23T18:04:36.646437+0000","last_active":"2023-09-23T18:04:40.336834+0000","last_peered":"2023-09-23T18:04:40.336834+0000","last_clean":"2023-09-23T18:04:40.336834+0000","last_became_active":"2023-09-23T18:02:51.049362+0000","last_became_peered":"2023-09-23T18:02:51.049362+0000","last_unstale":"2023-09-23T18:04:40.336834+0000","last_undegraded":"2023-09-23T18:04:40.336834+0000","last_fullsized":"2023-09-23T18:04:40.336834+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":118,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.859080+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:05:04.859080+0000","last_peered":"2023-09-23T18:05:04.859080+0000","last_clean":"2023-09-23T18:05:04.859080+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:05:04.859080+0000","last_undegraded":"2023-09-23T18:05:04.859080+0000","last_fullsized":"2023-09-23T18:05:04.859080+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":118,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.860445+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:05:04.860445+0000","last_peered":"2023-09-23T18:05:04.860445+0000","last_clean":"2023-09-23T18:05:04.860445+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:05:04.860445+0000","last_undegraded":"2023-09-23T18:05:04.860445+0000","last_fullsized":"2023-09-23T18:05:04.860445+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.0","version":"269'8882","reported_seq":8963,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:59.627901+0000","last_change":"2023-09-23T18:02:19.315376+0000","last_active":"2023-09-23T18:04:59.627901+0000","last_peered":"2023-09-23T18:04:59.627901+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:19.306174+0000","last_became_peered":"2023-09-23T18:02:19.306174+0000","last_unstale":"2023-09-23T18:04:59.627901+0000","last_undegraded":"2023-09-23T18:02:18.954031+0000","last_fullsized":"2023-09-23T18:02:18.922505+0000","mapping_epoch":238,"log_start":"244'4500","ondisk_log_start":"244'4500","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4382,"log_dups_size":0,"ondisk_log_size":4382,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:52.392034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4095,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8882,"num_write_kb":284224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,2],"acting":[0,2],"avail_no_missing":["0","2"],"object_location_counts":[{"shards":"0,2","objects":-4095}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":601,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337077+0000","last_change":"2023-09-23T18:02:51.059934+0000","last_active":"2023-09-23T18:04:40.337077+0000","last_peered":"2023-09-23T18:04:40.337077+0000","last_clean":"2023-09-23T18:04:40.337077+0000","last_became_active":"2023-09-23T18:02:51.059638+0000","last_became_peered":"2023-09-23T18:02:51.059638+0000","last_unstale":"2023-09-23T18:04:40.337077+0000","last_undegraded":"2023-09-23T18:04:40.337077+0000","last_fullsized":"2023-09-23T18:04:40.337077+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":580,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.379577+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:04:45.379577+0000","last_peered":"2023-09-23T18:04:45.379577+0000","last_clean":"2023-09-23T18:04:45.379577+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:04:45.379577+0000","last_undegraded":"2023-09-23T18:04:45.379577+0000","last_fullsized":"2023-09-23T18:04:45.379577+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":249,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.256472+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:04:40.256472+0000","last_peered":"2023-09-23T18:04:40.256472+0000","last_clean":"2023-09-23T18:04:40.256472+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:04:40.256472+0000","last_undegraded":"2023-09-23T18:04:40.256472+0000","last_fullsized":"2023-09-23T18:04:40.256472+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":246,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.256089+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:04:40.256089+0000","last_peered":"2023-09-23T18:04:40.256089+0000","last_clean":"2023-09-23T18:04:40.256089+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:04:40.256089+0000","last_undegraded":"2023-09-23T18:04:40.256089+0000","last_fullsized":"2023-09-23T18:04:40.256089+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.6","version":"269'8958","reported_seq":9052,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:57.969339+0000","last_change":"2023-09-23T18:02:19.289610+0000","last_active":"2023-09-23T18:04:57.969339+0000","last_peered":"2023-09-23T18:04:57.969339+0000","last_clean":"2023-09-23T18:02:17.060883+0000","last_became_active":"2023-09-23T18:02:19.266879+0000","last_became_peered":"2023-09-23T18:02:19.266879+0000","last_unstale":"2023-09-23T18:04:57.969339+0000","last_undegraded":"2023-09-23T18:02:18.963390+0000","last_fullsized":"2023-09-23T18:02:18.926482+0000","mapping_epoch":238,"log_start":"218'1900","ondisk_log_start":"218'1900","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7058,"log_dups_size":0,"ondisk_log_size":7058,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:13.499269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4128,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8958,"num_write_kb":286656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":851968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[4,7],"avail_no_missing":["4","6","7"],"object_location_counts":[{"shards":"4,7","objects":-4128}],"blocked_by":[],"up_primary":3,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":603,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336198+0000","last_change":"2023-09-23T18:02:51.047767+0000","last_active":"2023-09-23T18:04:40.336198+0000","last_peered":"2023-09-23T18:04:40.336198+0000","last_clean":"2023-09-23T18:04:40.336198+0000","last_became_active":"2023-09-23T18:02:51.047297+0000","last_became_peered":"2023-09-23T18:02:51.047297+0000","last_unstale":"2023-09-23T18:04:40.336198+0000","last_undegraded":"2023-09-23T18:04:40.336198+0000","last_fullsized":"2023-09-23T18:04:40.336198+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:07.851694+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1197,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.256192+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:04:40.256192+0000","last_peered":"2023-09-23T18:04:40.256192+0000","last_clean":"2023-09-23T18:04:40.256192+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:04:40.256192+0000","last_undegraded":"2023-09-23T18:04:40.256192+0000","last_fullsized":"2023-09-23T18:04:40.256192+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":60,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.374989+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:04:45.374989+0000","last_peered":"2023-09-23T18:04:45.374989+0000","last_clean":"2023-09-23T18:04:45.374989+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:04:45.374989+0000","last_undegraded":"2023-09-23T18:04:45.374989+0000","last_fullsized":"2023-09-23T18:04:45.374989+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":164,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.388053+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:04:45.388053+0000","last_peered":"2023-09-23T18:04:45.388053+0000","last_clean":"2023-09-23T18:04:45.388053+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:04:45.388053+0000","last_undegraded":"2023-09-23T18:04:45.388053+0000","last_fullsized":"2023-09-23T18:04:45.388053+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"6.5","version":"269'9264","reported_seq":17900,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:57.949467+0000","last_change":"2023-09-23T18:04:02.179479+0000","last_active":"2023-09-23T18:04:57.949467+0000","last_peered":"2023-09-23T18:04:57.949467+0000","last_clean":"2023-09-23T18:04:57.949467+0000","last_became_active":"2023-09-23T18:04:02.150813+0000","last_became_peered":"2023-09-23T18:04:02.150813+0000","last_unstale":"2023-09-23T18:04:57.949467+0000","last_undegraded":"2023-09-23T18:04:57.949467+0000","last_fullsized":"2023-09-23T18:04:57.949467+0000","mapping_epoch":263,"log_start":"253'4974","ondisk_log_start":"253'4974","created":204,"last_epoch_clean":264,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4290,"log_dups_size":0,"ondisk_log_size":4290,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:41.194322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9264,"num_write_kb":296448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4277,"num_bytes_recovered":179372032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":604,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336759+0000","last_change":"2023-09-23T18:02:51.054622+0000","last_active":"2023-09-23T18:04:40.336759+0000","last_peered":"2023-09-23T18:04:40.336759+0000","last_clean":"2023-09-23T18:04:40.336759+0000","last_became_active":"2023-09-23T18:02:51.049345+0000","last_became_peered":"2023-09-23T18:02:51.049345+0000","last_unstale":"2023-09-23T18:04:40.336759+0000","last_undegraded":"2023-09-23T18:04:40.336759+0000","last_fullsized":"2023-09-23T18:04:40.336759+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:57:58.636010+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":553,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:49.358130+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:04:49.358130+0000","last_peered":"2023-09-23T18:04:49.358130+0000","last_clean":"2023-09-23T18:04:49.358130+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:04:49.358130+0000","last_undegraded":"2023-09-23T18:04:49.358130+0000","last_fullsized":"2023-09-23T18:04:49.358130+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":165,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.387368+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:04:45.387368+0000","last_peered":"2023-09-23T18:04:45.387368+0000","last_clean":"2023-09-23T18:04:45.387368+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:04:45.387368+0000","last_undegraded":"2023-09-23T18:04:45.387368+0000","last_fullsized":"2023-09-23T18:04:45.387368+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":236,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.256193+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:04:40.256193+0000","last_peered":"2023-09-23T18:04:40.256193+0000","last_clean":"2023-09-23T18:04:40.256193+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:04:40.256193+0000","last_undegraded":"2023-09-23T18:04:40.256193+0000","last_fullsized":"2023-09-23T18:04:40.256193+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"6.7","version":"269'9240","reported_seq":16796,"reported_epoch":269,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:05:02.690605+0000","last_change":"2023-09-23T18:04:00.228250+0000","last_active":"2023-09-23T18:05:02.690605+0000","last_peered":"2023-09-23T18:05:02.690605+0000","last_clean":"2023-09-23T18:02:16.962656+0000","last_became_active":"2023-09-23T18:02:19.272896+0000","last_became_peered":"2023-09-23T18:02:19.272896+0000","last_unstale":"2023-09-23T18:05:02.690605+0000","last_undegraded":"2023-09-23T18:02:18.972349+0000","last_fullsized":"2023-09-23T18:02:18.930868+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":9240,"log_dups_size":0,"ondisk_log_size":9240,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:31:23.745675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":539,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9240,"num_write_kb":295680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3725,"num_bytes_recovered":151060480,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","1","4"],"object_location_counts":[{"shards":"4,5","objects":-539}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1181,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.256499+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:04:40.256499+0000","last_peered":"2023-09-23T18:04:40.256499+0000","last_clean":"2023-09-23T18:04:40.256499+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:04:40.256499+0000","last_undegraded":"2023-09-23T18:04:40.256499+0000","last_fullsized":"2023-09-23T18:04:40.256499+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":595,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336198+0000","last_change":"2023-09-23T18:04:39.585751+0000","last_active":"2023-09-23T18:04:40.336198+0000","last_peered":"2023-09-23T18:04:40.336198+0000","last_clean":"2023-09-23T18:04:40.336198+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:04:40.336198+0000","last_undegraded":"2023-09-23T18:04:40.336198+0000","last_fullsized":"2023-09-23T18:04:40.336198+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":80,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.859752+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:05:04.859752+0000","last_peered":"2023-09-23T18:05:04.859752+0000","last_clean":"2023-09-23T18:05:04.859752+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:05:04.859752+0000","last_undegraded":"2023-09-23T18:05:04.859752+0000","last_fullsized":"2023-09-23T18:05:04.859752+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":204,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-09-23T18:05:02.862044+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:05:02.862044+0000","last_peered":"2023-09-23T18:05:02.862044+0000","last_clean":"2023-09-23T18:05:02.862044+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:05:02.862044+0000","last_undegraded":"2023-09-23T18:05:02.862044+0000","last_fullsized":"2023-09-23T18:05:02.862044+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"6.4","version":"269'9082","reported_seq":9249,"reported_epoch":269,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:04:57.942384+0000","last_change":"2023-09-23T18:02:18.003772+0000","last_active":"2023-09-23T18:04:57.942384+0000","last_peered":"2023-09-23T18:04:57.942384+0000","last_clean":"2023-09-23T18:00:56.942864+0000","last_became_active":"2023-09-23T18:02:18.000696+0000","last_became_peered":"2023-09-23T18:02:18.000696+0000","last_unstale":"2023-09-23T18:04:57.942384+0000","last_undegraded":"2023-09-23T18:02:17.938873+0000","last_fullsized":"2023-09-23T18:02:17.919669+0000","mapping_epoch":237,"log_start":"216'100","ondisk_log_start":"216'100","created":204,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":8982,"log_dups_size":0,"ondisk_log_size":8982,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:48.085957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9082,"num_write_kb":290624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":37,"num_bytes_recovered":2424832,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,1],"avail_no_missing":["5","1","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":604,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337026+0000","last_change":"2023-09-23T18:02:51.061168+0000","last_active":"2023-09-23T18:04:40.337026+0000","last_peered":"2023-09-23T18:04:40.337026+0000","last_clean":"2023-09-23T18:04:40.337026+0000","last_became_active":"2023-09-23T18:02:51.060689+0000","last_became_peered":"2023-09-23T18:02:51.060689+0000","last_unstale":"2023-09-23T18:04:40.337026+0000","last_undegraded":"2023-09-23T18:04:40.337026+0000","last_fullsized":"2023-09-23T18:04:40.337026+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:54:29.406460+0000","scrub_duration":0.001568505,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":449,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337638+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:04:40.337638+0000","last_peered":"2023-09-23T18:04:40.337638+0000","last_clean":"2023-09-23T18:04:40.337638+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:04:40.337638+0000","last_undegraded":"2023-09-23T18:04:40.337638+0000","last_fullsized":"2023-09-23T18:04:40.337638+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":269,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.385044+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:04:45.385044+0000","last_peered":"2023-09-23T18:04:45.385044+0000","last_clean":"2023-09-23T18:04:45.385044+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:04:45.385044+0000","last_undegraded":"2023-09-23T18:04:45.385044+0000","last_fullsized":"2023-09-23T18:04:45.385044+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":240,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.385469+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:04:45.385469+0000","last_peered":"2023-09-23T18:04:45.385469+0000","last_clean":"2023-09-23T18:04:45.385469+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:04:45.385469+0000","last_undegraded":"2023-09-23T18:04:45.385469+0000","last_fullsized":"2023-09-23T18:04:45.385469+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.1","version":"269'9236","reported_seq":17588,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-09-23T18:05:02.861457+0000","last_change":"2023-09-23T18:02:18.942886+0000","last_active":"2023-09-23T18:05:02.861457+0000","last_peered":"2023-09-23T18:05:02.861457+0000","last_clean":"2023-09-23T18:05:02.861457+0000","last_became_active":"2023-09-23T18:02:18.936691+0000","last_became_peered":"2023-09-23T18:02:18.936691+0000","last_unstale":"2023-09-23T18:05:02.861457+0000","last_undegraded":"2023-09-23T18:05:02.861457+0000","last_fullsized":"2023-09-23T18:05:02.861457+0000","mapping_epoch":238,"log_start":"253'5058","ondisk_log_start":"253'5058","created":204,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4178,"log_dups_size":0,"ondisk_log_size":4178,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:26.085859+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9236,"num_write_kb":295552,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4123,"num_bytes_recovered":270204928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":604,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:49.648491+0000","last_change":"2023-09-23T18:04:49.648491+0000","last_active":"2023-09-23T18:04:49.648491+0000","last_peered":"2023-09-23T18:04:49.648491+0000","last_clean":"2023-09-23T18:04:49.648491+0000","last_became_active":"2023-09-23T18:02:51.056158+0000","last_became_peered":"2023-09-23T18:02:51.056158+0000","last_unstale":"2023-09-23T18:04:49.648491+0000","last_undegraded":"2023-09-23T18:04:49.648491+0000","last_fullsized":"2023-09-23T18:04:49.648491+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":483,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337114+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:04:40.337114+0000","last_peered":"2023-09-23T18:04:40.337114+0000","last_clean":"2023-09-23T18:04:40.337114+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:04:40.337114+0000","last_undegraded":"2023-09-23T18:04:40.337114+0000","last_fullsized":"2023-09-23T18:04:40.337114+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":205,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.388144+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:04:45.388144+0000","last_peered":"2023-09-23T18:04:45.388144+0000","last_clean":"2023-09-23T18:04:45.388144+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:04:45.388144+0000","last_undegraded":"2023-09-23T18:04:45.388144+0000","last_fullsized":"2023-09-23T18:04:45.388144+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":237,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.385375+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:04:45.385375+0000","last_peered":"2023-09-23T18:04:45.385375+0000","last_clean":"2023-09-23T18:04:45.385375+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:04:45.385375+0000","last_undegraded":"2023-09-23T18:04:45.385375+0000","last_fullsized":"2023-09-23T18:04:45.385375+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"6.2","version":"269'9114","reported_seq":9213,"reported_epoch":269,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:04:57.971018+0000","last_change":"2023-09-23T18:02:19.279112+0000","last_active":"2023-09-23T18:04:57.971018+0000","last_peered":"2023-09-23T18:04:57.971018+0000","last_clean":"2023-09-23T18:02:17.018900+0000","last_became_active":"2023-09-23T18:02:19.267439+0000","last_became_peered":"2023-09-23T18:02:19.267439+0000","last_unstale":"2023-09-23T18:04:57.971018+0000","last_undegraded":"2023-09-23T18:02:18.959329+0000","last_fullsized":"2023-09-23T18:02:18.926801+0000","mapping_epoch":238,"log_start":"221'2100","ondisk_log_start":"221'2100","created":204,"last_epoch_clean":221,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":7014,"log_dups_size":0,"ondisk_log_size":7014,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:02:07.395583+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4199,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9114,"num_write_kb":291648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,2],"avail_no_missing":["4","2","6"],"object_location_counts":[{"shards":"2,4","objects":-4199}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":601,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336710+0000","last_change":"2023-09-23T18:02:51.058680+0000","last_active":"2023-09-23T18:04:40.336710+0000","last_peered":"2023-09-23T18:04:40.336710+0000","last_clean":"2023-09-23T18:04:40.336710+0000","last_became_active":"2023-09-23T18:02:51.057706+0000","last_became_peered":"2023-09-23T18:02:51.057706+0000","last_unstale":"2023-09-23T18:04:40.336710+0000","last_undegraded":"2023-09-23T18:04:40.336710+0000","last_fullsized":"2023-09-23T18:04:40.336710+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":118,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.860413+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:05:04.860413+0000","last_peered":"2023-09-23T18:05:04.860413+0000","last_clean":"2023-09-23T18:05:04.860413+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:05:04.860413+0000","last_undegraded":"2023-09-23T18:05:04.860413+0000","last_fullsized":"2023-09-23T18:05:04.860413+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":341,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-09-23T18:05:02.862192+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:05:02.862192+0000","last_peered":"2023-09-23T18:05:02.862192+0000","last_clean":"2023-09-23T18:05:02.862192+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:05:02.862192+0000","last_undegraded":"2023-09-23T18:05:02.862192+0000","last_fullsized":"2023-09-23T18:05:02.862192+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":122,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.858986+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:05:04.858986+0000","last_peered":"2023-09-23T18:05:04.858986+0000","last_clean":"2023-09-23T18:05:04.858986+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:05:04.858986+0000","last_undegraded":"2023-09-23T18:05:04.858986+0000","last_fullsized":"2023-09-23T18:05:04.858986+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"6.3","version":"269'9138","reported_seq":9213,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:57.938457+0000","last_change":"2023-09-23T18:01:25.046224+0000","last_active":"2023-09-23T18:04:57.938457+0000","last_peered":"2023-09-23T18:04:57.938457+0000","last_clean":"2023-09-23T18:04:57.938457+0000","last_became_active":"2023-09-23T18:01:25.002864+0000","last_became_peered":"2023-09-23T18:01:25.002864+0000","last_unstale":"2023-09-23T18:04:57.938457+0000","last_undegraded":"2023-09-23T18:04:57.938457+0000","last_fullsized":"2023-09-23T18:04:57.938457+0000","mapping_epoch":214,"log_start":"247'4841","ondisk_log_start":"247'4841","created":204,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:00:56.942864+0000","last_clean_scrub_stamp":"2023-09-23T18:00:56.942864+0000","objects_scrubbed":0,"log_size":4297,"log_dups_size":0,"ondisk_log_size":4297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:51:13.752359+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9138,"num_write_kb":292416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":589824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,2],"acting":[7,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":608,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336444+0000","last_change":"2023-09-23T18:02:51.053420+0000","last_active":"2023-09-23T18:04:40.336444+0000","last_peered":"2023-09-23T18:04:40.336444+0000","last_clean":"2023-09-23T18:04:40.336444+0000","last_became_active":"2023-09-23T18:02:51.053035+0000","last_became_peered":"2023-09-23T18:02:51.053035+0000","last_unstale":"2023-09-23T18:04:40.336444+0000","last_undegraded":"2023-09-23T18:04:40.336444+0000","last_fullsized":"2023-09-23T18:04:40.336444+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:50:20.456481+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":476,"reported_epoch":270,"state":"active+clean","last_fresh":"2023-09-23T18:05:02.862370+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:05:02.862370+0000","last_peered":"2023-09-23T18:05:02.862370+0000","last_clean":"2023-09-23T18:05:02.862370+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:05:02.862370+0000","last_undegraded":"2023-09-23T18:05:02.862370+0000","last_fullsized":"2023-09-23T18:05:02.862370+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":297,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.385558+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:04:45.385558+0000","last_peered":"2023-09-23T18:04:45.385558+0000","last_clean":"2023-09-23T18:04:45.385558+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:04:45.385558+0000","last_undegraded":"2023-09-23T18:04:45.385558+0000","last_fullsized":"2023-09-23T18:04:45.385558+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":60,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.374533+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:04:45.374533+0000","last_peered":"2023-09-23T18:04:45.374533+0000","last_clean":"2023-09-23T18:04:45.374533+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:04:45.374533+0000","last_undegraded":"2023-09-23T18:04:45.374533+0000","last_fullsized":"2023-09-23T18:04:45.374533+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":594,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:51.732776+0000","last_change":"2023-09-23T18:04:51.732776+0000","last_active":"2023-09-23T18:04:51.732776+0000","last_peered":"2023-09-23T18:04:51.732776+0000","last_clean":"2023-09-23T18:04:51.732776+0000","last_became_active":"2023-09-23T18:02:51.046213+0000","last_became_peered":"2023-09-23T18:02:51.046213+0000","last_unstale":"2023-09-23T18:04:51.732776+0000","last_undegraded":"2023-09-23T18:04:51.732776+0000","last_fullsized":"2023-09-23T18:04:51.732776+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":256,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.385313+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:04:45.385313+0000","last_peered":"2023-09-23T18:04:45.385313+0000","last_clean":"2023-09-23T18:04:45.385313+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:04:45.385313+0000","last_undegraded":"2023-09-23T18:04:45.385313+0000","last_fullsized":"2023-09-23T18:04:45.385313+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":283,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.388251+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:04:45.388251+0000","last_peered":"2023-09-23T18:04:45.388251+0000","last_clean":"2023-09-23T18:04:45.388251+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:04:45.388251+0000","last_undegraded":"2023-09-23T18:04:45.388251+0000","last_fullsized":"2023-09-23T18:04:45.388251+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":600,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.337764+0000","last_change":"2023-09-23T18:02:51.060198+0000","last_active":"2023-09-23T18:04:40.337764+0000","last_peered":"2023-09-23T18:04:40.337764+0000","last_clean":"2023-09-23T18:04:40.337764+0000","last_became_active":"2023-09-23T18:02:51.057046+0000","last_became_peered":"2023-09-23T18:02:51.057046+0000","last_unstale":"2023-09-23T18:04:40.337764+0000","last_undegraded":"2023-09-23T18:04:40.337764+0000","last_fullsized":"2023-09-23T18:04:40.337764+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":124,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.256816+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:04:40.256816+0000","last_peered":"2023-09-23T18:04:40.256816+0000","last_clean":"2023-09-23T18:04:40.256816+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:04:40.256816+0000","last_undegraded":"2023-09-23T18:04:40.256816+0000","last_fullsized":"2023-09-23T18:04:40.256816+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":195,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.375174+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:04:45.375174+0000","last_peered":"2023-09-23T18:04:45.375174+0000","last_clean":"2023-09-23T18:04:45.375174+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:04:45.375174+0000","last_undegraded":"2023-09-23T18:04:45.375174+0000","last_fullsized":"2023-09-23T18:04:45.375174+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":110,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.256170+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:04:40.256170+0000","last_peered":"2023-09-23T18:04:40.256170+0000","last_clean":"2023-09-23T18:04:40.256170+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:04:40.256170+0000","last_undegraded":"2023-09-23T18:04:40.256170+0000","last_fullsized":"2023-09-23T18:04:40.256170+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":137,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.859643+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:05:04.859643+0000","last_peered":"2023-09-23T18:05:04.859643+0000","last_clean":"2023-09-23T18:05:04.859643+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:05:04.859643+0000","last_undegraded":"2023-09-23T18:05:04.859643+0000","last_fullsized":"2023-09-23T18:05:04.859643+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":616,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336537+0000","last_change":"2023-09-23T18:02:51.047476+0000","last_active":"2023-09-23T18:04:40.336537+0000","last_peered":"2023-09-23T18:04:40.336537+0000","last_clean":"2023-09-23T18:04:40.336537+0000","last_became_active":"2023-09-23T18:02:51.047152+0000","last_became_peered":"2023-09-23T18:02:51.047152+0000","last_unstale":"2023-09-23T18:04:40.336537+0000","last_undegraded":"2023-09-23T18:04:40.336537+0000","last_fullsized":"2023-09-23T18:04:40.336537+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:22.514077+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":137,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.859695+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:05:04.859695+0000","last_peered":"2023-09-23T18:05:04.859695+0000","last_clean":"2023-09-23T18:05:04.859695+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:05:04.859695+0000","last_undegraded":"2023-09-23T18:05:04.859695+0000","last_fullsized":"2023-09-23T18:05:04.859695+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":140,"reported_epoch":271,"state":"active+clean","last_fresh":"2023-09-23T18:05:04.858929+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:05:04.858929+0000","last_peered":"2023-09-23T18:05:04.858929+0000","last_clean":"2023-09-23T18:05:04.858929+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:05:04.858929+0000","last_undegraded":"2023-09-23T18:05:04.858929+0000","last_fullsized":"2023-09-23T18:05:04.858929+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":599,"reported_epoch":268,"state":"active+clean","last_fresh":"2023-09-23T18:04:40.336640+0000","last_change":"2023-09-23T18:02:51.058825+0000","last_active":"2023-09-23T18:04:40.336640+0000","last_peered":"2023-09-23T18:04:40.336640+0000","last_clean":"2023-09-23T18:04:40.336640+0000","last_became_active":"2023-09-23T18:02:51.058480+0000","last_became_peered":"2023-09-23T18:02:51.058480+0000","last_unstale":"2023-09-23T18:04:40.336640+0000","last_undegraded":"2023-09-23T18:04:40.336640+0000","last_fullsized":"2023-09-23T18:04:40.336640+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":249,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.379644+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:04:45.379644+0000","last_peered":"2023-09-23T18:04:45.379644+0000","last_clean":"2023-09-23T18:04:45.379644+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:04:45.379644+0000","last_undegraded":"2023-09-23T18:04:45.379644+0000","last_fullsized":"2023-09-23T18:04:45.379644+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":280,"reported_epoch":269,"state":"active+clean","last_fresh":"2023-09-23T18:04:45.376770+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:04:45.376770+0000","last_peered":"2023-09-23T18:04:45.376770+0000","last_clean":"2023-09-23T18:04:45.376770+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:04:45.376770+0000","last_undegraded":"2023-09-23T18:04:45.376770+0000","last_fullsized":"2023-09-23T18:04:45.376770+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":6,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":17069,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":146272,"num_write_kb":4680641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28601,"num_bytes_recovered":1656881152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1316225024,"data_stored":1316225024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":101708,"ondisk_log_size":101708,"up":48,"acting":41,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":402523,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":331,"num_read_kb":623,"num_write":162,"num_write_kb":1646,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9,"num_bytes_recovered":2814833,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":5},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001389,"num_pgs":71,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":111260,"kb_used_data":3540,"kb_used_omap":13,"kb_used_meta":107698,"kb_avail":93633892,"statfs":{"total":95995035648,"available":95881105408,"internally_reserved":0,"allocated":3624960,"data_stored":2589866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":110282856},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.77200000000000002,"5min":0.64000000000000001,"15min":0.623},"min":{"1min":0.25600000000000001,"5min":0.247,"15min":0.247},"max":{"1min":6.2770000000000001,"5min":6.2770000000000001,"15min":6.2770000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.1499999999999999,"5min":1.198,"15min":1.2569999999999999},"min":{"1min":0.26400000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":6.641,"5min":6.641,"15min":6.641},"last":0.309}]},{"osd":1,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6339999999999999,"5min":0.90700000000000003,"15min":0.754},"min":{"1min":0.26600000000000001,"5min":0.252,"15min":0.252},"max":{"1min":7.1150000000000002,"5min":7.1150000000000002,"15min":7.1150000000000002},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.3089999999999999,"5min":1.115,"15min":1.075},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.319,"5min":6.319,"15min":6.319},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":0.92800000000000005,"15min":0.93400000000000005},"min":{"1min":0.29599999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":3.1240000000000001,"5min":4.1539999999999999,"15min":4.1539999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.339,"5min":0.98699999999999999,"15min":0.871},"min":{"1min":0.30399999999999999,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.1989999999999998,"5min":6.1989999999999998,"15min":6.1989999999999998},"last":0.46700000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:04:20 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.4950000000000001,"15min":1.534},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":7.6070000000000002,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.5900000000000001,"5min":1.0940000000000001,"15min":1.0109999999999999},"min":{"1min":0.35299999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":10.776999999999999,"5min":10.776999999999999,"15min":10.776999999999999},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.236,"5min":1.046,"15min":1.0629999999999999},"min":{"1min":0.28899999999999998,"5min":0.245,"15min":0.245},"max":{"1min":3.5409999999999999,"5min":3.984,"15min":3.984},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.748,"5min":0.58999999999999997,"15min":0.54700000000000004},"min":{"1min":0.32300000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":6.3479999999999999,"5min":6.3479999999999999,"15min":6.3479999999999999},"last":0.40300000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.462,"5min":1.038,"15min":1.0149999999999999},"min":{"1min":0.34799999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":9.5999999999999996,"5min":9.5999999999999996,"15min":9.5999999999999996},"last":0.41499999999999998},{"interface":"front","average":{"1min":1.607,"5min":1.175,"15min":1.147},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.673,"5min":5.673,"15min":5.673},"last":0.434}]},{"osd":6,"last update":"Sat Sep 23 18:04:33 2023","interfaces":[{"interface":"back","average":{"1min":1.1619999999999999,"5min":0.78900000000000003,"15min":0.73299999999999998},"min":{"1min":0.38,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":6.3730000000000002,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":2.016,"5min":1.224,"15min":1.1579999999999999},"min":{"1min":0.32000000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":9.5519999999999996,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":0.45300000000000001}]}]},{"osd":6,"up_from":163,"seq":700079669326,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":961056,"kb_used_data":803416,"kb_used_omap":7,"kb_used_meta":157624,"kb_avail":92784096,"statfs":{"total":95995035648,"available":95010914304,"internally_reserved":0,"allocated":822697984,"data_stored":821666918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":161407780},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:40 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.1930000000000001,"15min":1.2969999999999999},"min":{"1min":0.26200000000000001,"5min":0.215,"15min":0.215},"max":{"1min":4.774,"5min":15.407,"15min":15.407},"last":0.48299999999999998},{"interface":"front","average":{"1min":1.595,"5min":1.1259999999999999,"15min":2.1760000000000002},"min":{"1min":0.27100000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":7.5739999999999998,"5min":7.633,"15min":13.536},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:04:40 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.278,"15min":1.853},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":4.6890000000000001,"5min":6.7690000000000001,"15min":14.233000000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.405,"5min":1.369,"15min":2.0710000000000002},"min":{"1min":0.317,"5min":0.248,"15min":0.248},"max":{"1min":5.2809999999999997,"5min":14.789,"15min":14.789},"last":0.44}]},{"osd":2,"last update":"Sat Sep 23 18:04:40 2023","interfaces":[{"interface":"back","average":{"1min":1.756,"5min":1.5980000000000001,"15min":2.21},"min":{"1min":0.27400000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":5.7140000000000004,"5min":10.837,"15min":11.210000000000001},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.669,"5min":1.5409999999999999,"15min":2.5409999999999999},"min":{"1min":0.32100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":9.3620000000000001,"5min":10.861000000000001,"15min":11.223000000000001},"last":0.53900000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:04:22 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":1.0449999999999999,"15min":1.0840000000000001},"min":{"1min":0.32900000000000001,"5min":0.308,"15min":0.308},"max":{"1min":3.25,"5min":3.25,"15min":3.25},"last":0.52100000000000002},{"interface":"front","average":{"1min":1.472,"5min":1.2270000000000001,"15min":1.1859999999999999},"min":{"1min":0.32300000000000001,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.2389999999999999,"5min":5.2389999999999999,"15min":5.2389999999999999},"last":0.55700000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:04:40 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.0029999999999999,"15min":0.90600000000000003},"min":{"1min":0.27500000000000002,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.4779999999999998,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.028,"5min":0.90700000000000003,"15min":0.56299999999999994},"min":{"1min":0.33400000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":5.7709999999999999,"5min":8.3000000000000007,"15min":8.3000000000000007},"last":0.34799999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:04:40 2023","interfaces":[{"interface":"back","average":{"1min":1.196,"5min":0.97599999999999998,"15min":0.85499999999999998},"min":{"1min":0.307,"5min":0.28799999999999998,"15min":0.28599999999999998},"max":{"1min":6.6159999999999997,"5min":11.869,"15min":11.869},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.2,"5min":1.1020000000000001,"15min":0.71099999999999997},"min":{"1min":0.39400000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":4.7930000000000001,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":0.377}]},{"osd":7,"last update":"Sat Sep 23 18:04:28 2023","interfaces":[{"interface":"back","average":{"1min":0.97899999999999998,"5min":1.024,"15min":0.98399999999999999},"min":{"1min":0.35399999999999998,"5min":0.307,"15min":0.307},"max":{"1min":4.5880000000000001,"5min":4.5880000000000001,"15min":4.5880000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.0209999999999999,"5min":0.83899999999999997,"15min":0.80100000000000005},"min":{"1min":0.38100000000000001,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":4.7190000000000003,"5min":4.7190000000000003,"15min":4.7190000000000003},"last":0.40100000000000002}]}]},{"osd":1,"up_from":0,"seq":420906795145,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958671,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":409752,"kb_used_data":8016,"kb_used_omap":7,"kb_used_meta":401720,"kb_avail":93335400,"statfs":{"total":95995035648,"available":95575449600,"internally_reserved":0,"allocated":8208384,"data_stored":6837422,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":411362020},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:04:11 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":1.597,"15min":1.1419999999999999},"min":{"1min":0.20699999999999999,"5min":0.186,"15min":0.17899999999999999},"max":{"1min":2.238,"5min":13.414,"15min":18.154},"last":0.29299999999999998},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.504,"15min":1.1140000000000001},"min":{"1min":0.26800000000000002,"5min":0.17399999999999999,"15min":0.17399999999999999},"max":{"1min":1.514,"5min":13.369999999999999,"15min":13.369999999999999},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.6279999999999999,"15min":1.3680000000000001},"min":{"1min":0.23400000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":10.077999999999999,"5min":11.757999999999999,"15min":20.882000000000001},"last":0.34000000000000002},{"interface":"front","average":{"1min":1.3999999999999999,"5min":1.6899999999999999,"15min":1.302},"min":{"1min":0.249,"5min":0.217,"15min":0.217},"max":{"1min":7.9029999999999996,"5min":12.843,"15min":15.064},"last":0.32300000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:04:21 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":1.419,"15min":1.508},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.3730000000000002,"5min":6.048,"15min":6.048},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.661,"5min":1.2929999999999999,"15min":1.2310000000000001},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":8.7300000000000004,"5min":8.7300000000000004,"15min":8.7300000000000004},"last":0.49299999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":1.4379999999999999,"15min":1.206},"min":{"1min":0.32800000000000001,"5min":0.29299999999999998,"15min":0.251},"max":{"1min":5.4720000000000004,"5min":8.4700000000000006,"15min":15.279999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.827,"5min":1.538,"15min":1.224},"min":{"1min":0.315,"5min":0.247,"15min":0.247},"max":{"1min":10.164999999999999,"5min":11.521000000000001,"15min":18.106999999999999},"last":0.379}]},{"osd":5,"last update":"Sat Sep 23 18:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.6830000000000001,"15min":1.2869999999999999},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":10.194000000000001,"5min":11.48,"15min":14.720000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.4019999999999999,"15min":1.105},"min":{"1min":0.36499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":7.742,"5min":12.794,"15min":12.794},"last":0.43099999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.4279999999999999,"15min":2.1539999999999999},"min":{"1min":0.36299999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.5599999999999996,"5min":8.4410000000000007,"15min":14.956},"last":0.64100000000000001},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.3899999999999999,"15min":2.3039999999999998},"min":{"1min":0.35799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":4.2759999999999998,"5min":12.755000000000001,"15min":12.755000000000001},"last":0.56699999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:04:29 2023","interfaces":[{"interface":"back","average":{"1min":1.1719999999999999,"5min":0.98399999999999999,"15min":0.94299999999999995},"min":{"1min":0.36799999999999999,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":7.4649999999999999,"5min":7.4649999999999999,"15min":7.4649999999999999},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.6499999999999999,"5min":1.3080000000000001,"15min":1.1579999999999999},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.524,"5min":5.524,"15min":5.524},"last":0.66300000000000003}]}]},{"osd":2,"up_from":103,"seq":442381631618,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":377280,"kb_used_data":7992,"kb_used_omap":11,"kb_used_meta":369268,"kb_avail":93367872,"statfs":{"total":95995035648,"available":95608700928,"internally_reserved":0,"allocated":8183808,"data_stored":6836541,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":378130807},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":2.1230000000000002,"15min":1.4139999999999999},"min":{"1min":0.214,"5min":0.187,"15min":0.17199999999999999},"max":{"1min":12.395,"5min":20.867000000000001,"15min":20.867000000000001},"last":0.443},{"interface":"front","average":{"1min":1.5820000000000001,"5min":1.879,"15min":1.359},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":7.6550000000000002,"5min":21.681000000000001,"15min":21.681000000000001},"last":0.52900000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":2.0049999999999999,"15min":1.454},"min":{"1min":0.317,"5min":0.215,"15min":0.215},"max":{"1min":7.2720000000000002,"5min":20.995000000000001,"15min":20.995000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.6120000000000001,"15min":1.244},"min":{"1min":0.25900000000000001,"5min":0.193,"15min":0.193},"max":{"1min":7.5720000000000001,"5min":15.945,"15min":15.945},"last":0.41799999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:04:26 2023","interfaces":[{"interface":"back","average":{"1min":1.794,"5min":0.98399999999999999,"15min":0.84899999999999998},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":12.009,"5min":12.009,"15min":12.009},"last":0.47199999999999998},{"interface":"front","average":{"1min":1.4490000000000001,"5min":0.77800000000000002,"15min":0.66600000000000004},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":9.0820000000000007,"5min":9.0820000000000007,"15min":9.0820000000000007},"last":0.45900000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.472,"15min":1.143},"min":{"1min":0.34599999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.0899999999999999,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.4039999999999999,"5min":1.655,"15min":1.1739999999999999},"min":{"1min":0.35299999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.5609999999999999,"5min":20.920000000000002,"15min":20.920000000000002},"last":0.60999999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.661,"15min":1.2490000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":7.3289999999999997,"5min":20.738,"15min":20.738},"last":0.55700000000000005},{"interface":"front","average":{"1min":2.0190000000000001,"5min":1.742,"15min":1.327},"min":{"1min":0.36299999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.103999999999999,"5min":14.359,"15min":15.496},"last":0.85799999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.554,"15min":2.0659999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":7.931,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":0.622},{"interface":"front","average":{"1min":1.907,"5min":1.8360000000000001,"15min":2.3290000000000002},"min":{"1min":0.40500000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.4649999999999999,"5min":20.893000000000001,"15min":20.893000000000001},"last":0.877}]},{"osd":7,"last update":"Sat Sep 23 18:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.5600000000000001,"5min":1.3640000000000001,"15min":1.395},"min":{"1min":0.436,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":12.051,"5min":12.051,"15min":12.051},"last":0.90100000000000002},{"interface":"front","average":{"1min":1.1859999999999999,"5min":0.77400000000000002,"15min":0.68000000000000005},"min":{"1min":0.437,"5min":0.309,"15min":0.309},"max":{"1min":7.5339999999999998,"5min":7.5339999999999998,"15min":7.5339999999999998},"last":0.91500000000000004}]}]},{"osd":3,"up_from":237,"seq":1017907249187,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":437676,"kb_used_data":293220,"kb_used_omap":3,"kb_used_meta":144444,"kb_avail":93307476,"statfs":{"total":95995035648,"available":95546855424,"internally_reserved":0,"allocated":300257280,"data_stored":298711760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":147911400},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":0.79200000000000004,"15min":0.85299999999999998},"min":{"1min":0.215,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":2.6179999999999999,"5min":3.6560000000000001,"15min":3.6560000000000001},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.373,"5min":0.93999999999999995,"15min":1.034},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.63300000000000001,"5min":4.5529999999999999,"15min":4.5529999999999999},"last":0.27000000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":0.92300000000000004,"15min":1.014},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.69099999999999995,"5min":4.6050000000000004,"15min":4.6050000000000004},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":0.874,"15min":0.92200000000000004},"min":{"1min":0.26800000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.9630000000000001,"5min":4.8810000000000002,"15min":4.8810000000000002},"last":0.35599999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.81899999999999995,"15min":0.83999999999999997},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.157,"5min":3.157,"15min":3.157},"last":0.39600000000000002},{"interface":"front","average":{"1min":0.443,"5min":1.085,"15min":1.1919999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":0.75600000000000001,"5min":6.157,"15min":6.157},"last":0.38300000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.66400000000000003,"5min":1.016,"15min":1.0740000000000001},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.3330000000000002,"5min":4.633,"15min":4.633},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.70299999999999996,"5min":0.995,"15min":1.0429999999999999},"min":{"1min":0.33200000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.3570000000000002,"5min":4.7300000000000004,"15min":4.7300000000000004},"last":0.54800000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.76200000000000001,"5min":1.179,"15min":1.2490000000000001},"min":{"1min":0.36099999999999999,"5min":0.314,"15min":0.314},"max":{"1min":2.6880000000000002,"5min":7.0640000000000001,"15min":7.0640000000000001},"last":0.504},{"interface":"front","average":{"1min":0.94499999999999995,"5min":1.133,"15min":1.165},"min":{"1min":0.373,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":3.0750000000000002,"5min":4.6150000000000002,"15min":4.6150000000000002},"last":0.60199999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":0.997,"5min":1.0669999999999999,"15min":1.079},"min":{"1min":0.36699999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":3.125,"5min":4.0590000000000002,"15min":4.0590000000000002},"last":0.66200000000000003},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.3680000000000001,"15min":1.4419999999999999},"min":{"1min":0.40500000000000003,"5min":0.379,"15min":0.379},"max":{"1min":2.8679999999999999,"5min":7.3929999999999998,"15min":7.3929999999999998},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:04:18 2023","interfaces":[{"interface":"back","average":{"1min":1.0620000000000001,"5min":1.47,"15min":1.538},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":2.8149999999999999,"5min":8.4190000000000005,"15min":8.4190000000000005},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.0840000000000001,"5min":1.512,"15min":1.5840000000000001},"min":{"1min":0.46000000000000002,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":3.8610000000000002,"5min":5.1760000000000002,"15min":5.1760000000000002},"last":0.64100000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019136,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":246808,"kb_used_data":2960,"kb_used_omap":0,"kb_used_meta":243840,"kb_avail":93498344,"statfs":{"total":95995035648,"available":95742304256,"internally_reserved":0,"allocated":3031040,"data_stored":1999751,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":249692160},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:39 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":1.1220000000000001,"15min":0.871},"min":{"1min":0.215,"5min":0.215,"15min":0.20100000000000001},"max":{"1min":6.8849999999999998,"5min":14.164,"15min":14.164},"last":0.28100000000000003},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.3740000000000001,"15min":1.0760000000000001},"min":{"1min":0.25700000000000001,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":6.1989999999999998,"5min":9.3789999999999996,"15min":11.372},"last":0.46000000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:04:06 2023","interfaces":[{"interface":"back","average":{"1min":0.56699999999999995,"5min":1.359,"15min":1.004},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":2.2770000000000001,"5min":8.5259999999999998,"15min":10.429},"last":0.29599999999999999},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.2949999999999999,"15min":1.083},"min":{"1min":0.29699999999999999,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.597,"5min":8.8130000000000006,"15min":10.83},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":1.319,"15min":1.1479999999999999},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":6.8259999999999996,"5min":9.6739999999999995,"15min":11.630000000000001},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.3580000000000001,"5min":1.244,"15min":1.0680000000000001},"min":{"1min":0.32100000000000001,"5min":0.316,"15min":0.27100000000000002},"max":{"1min":6.9299999999999997,"5min":7.4459999999999997,"15min":7.5090000000000003},"last":0.42999999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:04:17 2023","interfaces":[{"interface":"back","average":{"1min":0.78900000000000003,"5min":1.238,"15min":1.3129999999999999},"min":{"1min":0.378,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":3.6619999999999999,"5min":7.2140000000000004,"15min":7.2140000000000004},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.1839999999999999,"5min":1.335,"15min":1.3600000000000001},"min":{"1min":0.34799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":6.165,"5min":6.165,"15min":6.165},"last":0.628}]},{"osd":5,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":0.99099999999999999,"15min":0.69399999999999995},"min":{"1min":0.27800000000000002,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":3.786,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.194,"5min":1.377,"15min":0.88500000000000001},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.26100000000000001},"max":{"1min":9.1999999999999993,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.59999999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":1.2030000000000001,"15min":0.91400000000000003},"min":{"1min":0.34699999999999998,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":9.0990000000000002,"5min":9.0990000000000002,"15min":9.0990000000000002},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.667,"5min":1.2270000000000001,"15min":1.0089999999999999},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":9.1300000000000008,"5min":10.725,"15min":10.725},"last":0.32100000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:04:29 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":1.097,"15min":0.97499999999999998},"min":{"1min":0.34699999999999998,"5min":0.32700000000000001,"15min":0.32700000000000001},"max":{"1min":7.6779999999999999,"5min":7.6779999999999999,"15min":7.6779999999999999},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.7929999999999999,"5min":1.377,"15min":1.2669999999999999},"min":{"1min":0.38500000000000001,"5min":0.307,"15min":0.307},"max":{"1min":9.0310000000000006,"5min":9.0310000000000006,"15min":9.0310000000000006},"last":0.68500000000000005}]}]},{"osd":5,"up_from":38,"seq":163208757502,"num_pgs":28,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":218376,"kb_used_data":2944,"kb_used_omap":0,"kb_used_meta":215424,"kb_avail":93526776,"statfs":{"total":95995035648,"available":95771418624,"internally_reserved":0,"allocated":3014656,"data_stored":1995191,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":220594176},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.3360000000000001,"15min":1.0269999999999999},"min":{"1min":0.29499999999999998,"5min":0.217,"15min":0.217},"max":{"1min":3.2360000000000002,"5min":21.376999999999999,"15min":21.376999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.4530000000000001,"5min":1.7410000000000001,"15min":1.2370000000000001},"min":{"1min":0.28399999999999997,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.5099999999999998,"5min":21.388999999999999,"15min":21.388999999999999},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:04:12 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":0.91200000000000003,"15min":0.85999999999999999},"min":{"1min":0.26000000000000001,"5min":0.24099999999999999,"15min":0.23899999999999999},"max":{"1min":3.754,"5min":4.8129999999999997,"15min":10.853999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.075,"5min":1.159,"15min":0.98799999999999999},"min":{"1min":0.28000000000000003,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":8.0039999999999996,"5min":8.0039999999999996,"15min":10.206},"last":0.44700000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":1.851,"15min":1.319},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":5.5309999999999997,"5min":26.035,"15min":38.606999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":1.736,"5min":1.407,"15min":1.2170000000000001},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":10.065,"5min":17.085000000000001,"15min":39.670999999999999},"last":0.46300000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:04:21 2023","interfaces":[{"interface":"back","average":{"1min":1.1870000000000001,"5min":0.85099999999999998,"15min":0.79500000000000004},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.9569999999999999,"5min":7.9569999999999999,"15min":7.9569999999999999},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.90800000000000003,"5min":0.751,"15min":0.72499999999999998},"min":{"1min":0.35699999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":7.1260000000000003,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":0.47599999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.0629999999999999,"15min":0.78800000000000003},"min":{"1min":0.26900000000000002,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":8.6229999999999993,"5min":8.6229999999999993,"15min":10.092000000000001},"last":0.376},{"interface":"front","average":{"1min":1.5229999999999999,"5min":1.0549999999999999,"15min":0.77200000000000002},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.6379999999999999,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.085,"15min":1.421},"min":{"1min":0.35299999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.673,"5min":6.5880000000000001,"15min":11.287000000000001},"last":0.438},{"interface":"front","average":{"1min":1.2629999999999999,"5min":0.93999999999999995,"15min":1.26},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.7939999999999996,"5min":5.7939999999999996,"15min":8.266},"last":0.48399999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:04:30 2023","interfaces":[{"interface":"back","average":{"1min":1.361,"5min":1.3680000000000001,"15min":1.486},"min":{"1min":0.36399999999999999,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.8510000000000009,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.496},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.149,"15min":1.133},"min":{"1min":0.40400000000000003,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":8.3279999999999994,"5min":8.3279999999999994,"15min":8.3279999999999994},"last":0.503}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":497156096,"data_stored":497156096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":819068928,"data_stored":819068928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":6,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:05:39.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:07 smithi055 ceph-mon[184174]: pgmap v666: 119 pgs: 1 active+undersized+remapped+backfill_wait, 111 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 14 op/s; 17069/81 objects degraded (21072.840%); 0 B/s, 175 objects/s recovering 2023-09-23T18:05:40.089 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:07 smithi055 ceph-mon[179668]: pgmap v666: 119 pgs: 1 active+undersized+remapped+backfill_wait, 111 active+clean, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 0 B/s wr, 14 op/s; 17069/81 objects degraded (21072.840%); 0 B/s, 175 objects/s recovering 2023-09-23T18:05:40.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:40.090 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:08 smithi055 ceph-mon[184174]: from='client.45500 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:05:40.091 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:08 smithi055 ceph-mon[179668]: from='client.45500 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:05:40.091 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_1' removed 2023-09-23T18:05:40.091 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:09 smithi055 ceph-mon[179668]: pgmap v667: 119 pgs: 2 active+undersized+remapped+backfill_wait, 112 active+clean, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 16530/81 objects degraded (20407.407%); 0 B/s, 89 objects/s recovering 2023-09-23T18:05:40.092 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:09 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16530/81 objects degraded (20407.407%), 4 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:05:40.092 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3694469409' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:05:40.092 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:09 smithi055 ceph-mon[184174]: pgmap v667: 119 pgs: 2 active+undersized+remapped+backfill_wait, 112 active+clean, 1 active+remapped+backfill_wait, 4 active+recovery_wait+undersized+degraded+remapped; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 16530/81 objects degraded (20407.407%); 0 B/s, 89 objects/s recovering 2023-09-23T18:05:40.092 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:09 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16530/81 objects degraded (20407.407%), 4 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:05:40.092 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3694469409' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:05:40.093 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:40.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:05:40.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:05:40.094 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:05:40.094 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T18:05:40.095 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T18:05:40.095 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_2 2023-09-23T18:05:40.095 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_2 16 2023-09-23T18:05:40.096 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:05:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:05:10] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-09-23T18:05:40.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:11 smithi055 ceph-mon[179668]: failed to encode map e273 with expected crc 2023-09-23T18:05:40.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:11 smithi055 ceph-mon[179668]: failed to encode map e273 with expected crc 2023-09-23T18:05:40.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:11 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3694469409' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:05:40.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:11 smithi055 ceph-mon[179668]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T18:05:40.096 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T18:05:40.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:11 smithi055 ceph-mon[184174]: failed to encode map e273 with expected crc 2023-09-23T18:05:40.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:11 smithi055 ceph-mon[184174]: failed to encode map e273 with expected crc 2023-09-23T18:05:40.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:11 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3694469409' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_1", "pool2": "unique_pool_1", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:05:40.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:11 smithi055 ceph-mon[184174]: osdmap e273: 8 total, 8 up, 7 in 2023-09-23T18:05:40.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]: dispatch 2023-09-23T18:05:40.097 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[179668]: pgmap v669: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.097 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16530/81 objects degraded (20407.407%), 4 pgs degraded, 6 pgs undersized) 2023-09-23T18:05:40.097 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[179668]: failed to encode map e274 with expected crc 2023-09-23T18:05:40.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[179668]: failed to encode map e274 with expected crc 2023-09-23T18:05:40.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T18:05:40.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[179668]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T18:05:40.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[184174]: pgmap v669: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 16530/81 objects degraded (20407.407%), 4 pgs degraded, 6 pgs undersized) 2023-09-23T18:05:40.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[184174]: failed to encode map e274 with expected crc 2023-09-23T18:05:40.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[184174]: failed to encode map e274 with expected crc 2023-09-23T18:05:40.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num_actual", "val": "31"}]': finished 2023-09-23T18:05:40.099 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:12 smithi055 ceph-mon[184174]: osdmap e274: 8 total, 8 up, 7 in 2023-09-23T18:05:40.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[179668]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[179668]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[179668]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T18:05:40.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[179668]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[179668]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[179668]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[179668]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.100 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[184174]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.100 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[184174]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.100 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[184174]: osdmap e275: 8 total, 8 up, 7 in 2023-09-23T18:05:40.100 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[184174]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.100 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[184174]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.100 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[184174]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.100 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:13 smithi055 ceph-mon[184174]: failed to encode map e275 with expected crc 2023-09-23T18:05:40.100 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[179668]: pgmap v672: 103 pgs: 6 unknown, 97 active+clean; 4.3 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.100 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[179668]: 1.18 continuing backfill to osd.2 from (0'0,89'161] MIN to 89'161 2023-09-23T18:05:40.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[179668]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[179668]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[179668]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T18:05:40.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[179668]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[179668]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[179668]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[179668]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.101 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[179668]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.101 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[184174]: pgmap v672: 103 pgs: 6 unknown, 97 active+clean; 4.3 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.101 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[184174]: 1.18 continuing backfill to osd.2 from (0'0,89'161] MIN to 89'161 2023-09-23T18:05:40.102 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[184174]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.102 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[184174]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.102 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[184174]: osdmap e276: 8 total, 8 up, 7 in 2023-09-23T18:05:40.102 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[184174]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.102 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[184174]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.102 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[184174]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.102 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[184174]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.102 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:14 smithi055 ceph-mon[184174]: failed to encode map e276 with expected crc 2023-09-23T18:05:40.102 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:15 smithi055 ceph-mon[179668]: failed to encode map e277 with expected crc 2023-09-23T18:05:40.103 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:15 smithi055 ceph-mon[179668]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T18:05:40.103 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:15 smithi055 ceph-mon[179668]: failed to encode map e277 with expected crc 2023-09-23T18:05:40.103 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:15 smithi055 ceph-mon[184174]: failed to encode map e277 with expected crc 2023-09-23T18:05:40.103 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:15 smithi055 ceph-mon[184174]: osdmap e277: 8 total, 8 up, 7 in 2023-09-23T18:05:40.103 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:15 smithi055 ceph-mon[184174]: failed to encode map e277 with expected crc 2023-09-23T18:05:40.103 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: pgmap v675: 103 pgs: 5 peering, 4 remapped+peering, 6 unknown, 88 active+clean; 4.3 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T18:05:40.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.104 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[179668]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: pgmap v675: 103 pgs: 5 peering, 4 remapped+peering, 6 unknown, 88 active+clean; 4.3 MiB data, 2.7 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: osdmap e278: 8 total, 8 up, 7 in 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.105 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:16 smithi055 ceph-mon[184174]: failed to encode map e278 with expected crc 2023-09-23T18:05:40.106 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:40.106 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:17 smithi055 ceph-mon[179668]: pgmap v677: 103 pgs: 5 peering, 4 remapped+peering, 94 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 33 KiB/s rd, 86 KiB/s wr, 25 op/s; 277 KiB/s, 1 objects/s recovering 2023-09-23T18:05:40.106 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:40.106 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:17 smithi055 ceph-mon[184174]: pgmap v677: 103 pgs: 5 peering, 4 remapped+peering, 94 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 33 KiB/s rd, 86 KiB/s wr, 25 op/s; 277 KiB/s, 1 objects/s recovering 2023-09-23T18:05:40.106 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:20 smithi055 ceph-mon[179668]: pgmap v678: 103 pgs: 5 peering, 98 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 25 KiB/s rd, 67 KiB/s wr, 20 op/s; 302 KiB/s, 1 objects/s recovering 2023-09-23T18:05:40.107 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:20 smithi055 ceph-mon[184174]: pgmap v678: 103 pgs: 5 peering, 98 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 25 KiB/s rd, 67 KiB/s wr, 20 op/s; 302 KiB/s, 1 objects/s recovering 2023-09-23T18:05:40.107 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:05:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:05:20] "GET /metrics HTTP/1.1" 200 36514 "" "Prometheus/2.43.0" 2023-09-23T18:05:40.107 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:22 smithi055 ceph-mon[179668]: pgmap v679: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 15 op/s; 236 KiB/s, 0 objects/s recovering 2023-09-23T18:05:40.107 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:40.107 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:40.108 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:22 smithi055 ceph-mon[184174]: pgmap v679: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 20 KiB/s rd, 52 KiB/s wr, 15 op/s; 236 KiB/s, 0 objects/s recovering 2023-09-23T18:05:40.108 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:40.108 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:40.108 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:24 smithi055 ceph-mon[179668]: pgmap v680: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 18 KiB/s rd, 46 KiB/s wr, 13 op/s; 209 KiB/s, 0 objects/s recovering 2023-09-23T18:05:40.108 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:24 smithi055 ceph-mon[184174]: pgmap v680: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 18 KiB/s rd, 46 KiB/s wr, 13 op/s; 209 KiB/s, 0 objects/s recovering 2023-09-23T18:05:40.109 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:26 smithi055 ceph-mon[179668]: pgmap v681: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 12 op/s; 181 KiB/s, 0 objects/s recovering 2023-09-23T18:05:40.109 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:26 smithi055 ceph-mon[184174]: pgmap v681: 103 pgs: 103 active+clean; 4.4 MiB data, 2.6 GiB used, 623 GiB / 626 GiB avail; 15 KiB/s rd, 40 KiB/s wr, 12 op/s; 181 KiB/s, 0 objects/s recovering 2023-09-23T18:05:40.109 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:40.109 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:27 smithi055 ceph-mon[179668]: pgmap v682: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 13 KiB/s rd, 35 KiB/s wr, 10 op/s; 156 KiB/s, 0 objects/s recovering 2023-09-23T18:05:40.110 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:40.110 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:27 smithi055 ceph-mon[184174]: pgmap v682: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 13 KiB/s rd, 35 KiB/s wr, 10 op/s; 156 KiB/s, 0 objects/s recovering 2023-09-23T18:05:40.110 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:29 smithi055 ceph-mon[179668]: pgmap v683: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-09-23T18:05:40.110 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:29 smithi055 ceph-mon[184174]: pgmap v683: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail; 43 KiB/s, 0 objects/s recovering 2023-09-23T18:05:40.111 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:05:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:05:30] "GET /metrics HTTP/1.1" 200 34165 "" "Prometheus/2.43.0" 2023-09-23T18:05:40.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:32 smithi055 ceph-mon[179668]: pgmap v684: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.111 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:32 smithi055 ceph-mon[184174]: pgmap v684: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:34 smithi055 ceph-mon[179668]: pgmap v685: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.112 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:34 smithi055 ceph-mon[184174]: pgmap v685: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:36 smithi055 ceph-mon[179668]: pgmap v686: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.112 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:36 smithi055 ceph-mon[184174]: pgmap v686: 103 pgs: 103 active+clean; 4.4 MiB data, 2.5 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:40.113 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:40.113 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:38 smithi055 ceph-mon[179668]: pgmap v687: 103 pgs: 103 active+clean; 4.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.114 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:38 smithi055 ceph-mon[184174]: pgmap v687: 103 pgs: 103 active+clean; 4.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.121 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:40 smithi099 ceph-mon[167076]: pgmap v688: 103 pgs: 103 active+clean; 4.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:05:40.324 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:40.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:40 smithi055 ceph-mon[179668]: pgmap v688: 103 pgs: 103 active+clean; 4.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:40 smithi055 ceph-mon[184174]: pgmap v688: 103 pgs: 103 active+clean; 4.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:40.410 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:05:40.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:40.467 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:05:40.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:05:40.595 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:40.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:40.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:05:40.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:05:40.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:05:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:05:40] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.43.0" 2023-09-23T18:05:40.735 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:05:40.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:40.744 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:40.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:40.836 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:40.836 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:40.836 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:40.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:05:41.263 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:05:41.263 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:05:41.263 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:05:41.263 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:05:41.292 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:41.326 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:05:41.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:41.430 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:05:41.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:05:41.530 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:05:41.586 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:41.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:41.662 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:41.663 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:05:41.663 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:05:41.764 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:05:41.795 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:42.060 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:05:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:42 smithi099 ceph-mon[167076]: pgmap v689: 103 pgs: 103 active+clean; 4.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:42.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:42 smithi055 ceph-mon[179668]: pgmap v689: 103 pgs: 103 active+clean; 4.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:42.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:42 smithi055 ceph-mon[184174]: pgmap v689: 103 pgs: 103 active+clean; 4.4 MiB data, 2.4 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:42.720 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:05:42.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:42.724 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:05:42.778 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:05:42.825 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:05:42.854 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:42.940 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:05:42.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:42.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:05:43.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:05:43.077 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:43.113 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:05:43.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:43.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:05:43.218 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:05:43.246 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:43.274 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:05:43.274 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:05:43.274 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:05:43.275 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:05:43.275 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:05:43.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:05:43.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:43.282 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:05:43.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:05:43.411 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:43.446 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:05:43.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:43.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:05:43.550 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:05:43.579 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:43.612 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:43.613 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:43.635 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:05:43.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:43.641 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:05:43.701 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:05:43.742 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:05:43.768 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:43.801 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:05:43.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:43.805 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:05:43.905 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:05:43.951 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:44.040 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:05:44.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:44.043 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:05:44.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:05:44.171 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:44.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:44 smithi055 ceph-mon[179668]: pgmap v690: 103 pgs: 103 active+clean; 4.4 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:44.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:44 smithi055 ceph-mon[184174]: pgmap v690: 103 pgs: 103 active+clean; 4.4 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:44.189 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:05:44.212 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:05:44.239 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:05:44.254 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:05:44.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:44.258 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:05:44.265 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:05:44.284 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:05:44.301 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:05:44.304 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:05:44.323 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:05:44.342 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:05:44.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:05:44.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:05:44.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:05:44.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:05:44.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:05:44.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:05:44.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:05:44.344 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:05:44.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:05:44.381 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:44 smithi099 ceph-mon[167076]: pgmap v690: 103 pgs: 103 active+clean; 4.4 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:44.384 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:44.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:05:44.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:44.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:05:44.535 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:44.563 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:44.596 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:44.599 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:44.600 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:44.700 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:05:44.730 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:45.118 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_2' created 2023-09-23T18:05:45.226 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:05:45.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:45.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:05:45.261 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/106395413' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:45.261 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3808504863' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T18:05:45.261 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/106395413' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:45.261 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3808504863' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T18:05:45.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:05:45.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/106395413' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:45.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3808504863' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]: dispatch 2023-09-23T18:05:45.360 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:45.444 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:05:45.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:45.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:05:45.548 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:05:45.578 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:45.633 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:05:45.633 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:05:45.864 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_2 rados --yes-i-really-mean-it || true 2023-09-23T18:05:45.913 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:05:45.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:45.917 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:05:46.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:46.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:05:46.048 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:46.081 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:46.083 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:46.083 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:46.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[179668]: pgmap v691: 103 pgs: 103 active+clean; 4.4 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:46.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[179668]: failed to encode map e279 with expected crc 2023-09-23T18:05:46.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3808504863' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T18:05:46.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[179668]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T18:05:46.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1436282153' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:05:46.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:05:46.132 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:05:46.184 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:46.212 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:46.245 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:05:46.247 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:05:46.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:46.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:46.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:46.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:46 smithi099 ceph-mon[167076]: pgmap v691: 103 pgs: 103 active+clean; 4.4 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:46 smithi099 ceph-mon[167076]: failed to encode map e279 with expected crc 2023-09-23T18:05:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3808504863' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T18:05:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:46 smithi099 ceph-mon[167076]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T18:05:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1436282153' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:05:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:46 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:05:46.348 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:05:46.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[184174]: pgmap v691: 103 pgs: 103 active+clean; 4.4 MiB data, 2.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:05:46.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[184174]: failed to encode map e279 with expected crc 2023-09-23T18:05:46.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3808504863' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_2", "pg_num": 16}]': finished 2023-09-23T18:05:46.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[184174]: osdmap e279: 8 total, 8 up, 7 in 2023-09-23T18:05:46.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1436282153' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:05:46.377 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:46 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:05:46.379 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:46.664 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:05:46.666 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:46.667 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:05:46.682 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:05:46.768 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:05:46.796 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:46.799 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":693,"stamp":"2023-09-23T18:05:46.105060+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4620240,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":6297,"num_read_kb":12796,"num_write":1076,"num_write_kb":14444,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":3727821,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":318,"acting":318,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2422364,"kb_used_data":719076,"kb_used_omap":43,"kb_used_meta":1703188,"kb_avail":653793700,"statfs":{"total":671965249536,"available":669484748800,"internally_reserved":0,"allocated":736333824,"data_stored":727767121,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1744065231},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.365371"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":605,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.087182+0000","last_change":"2023-09-23T18:05:15.087182+0000","last_active":"2023-09-23T18:05:15.087182+0000","last_peered":"2023-09-23T18:05:15.087182+0000","last_clean":"2023-09-23T18:05:15.087182+0000","last_became_active":"2023-09-23T18:05:15.086678+0000","last_became_peered":"2023-09-23T18:05:15.086678+0000","last_unstale":"2023-09-23T18:05:15.087182+0000","last_undegraded":"2023-09-23T18:05:15.087182+0000","last_fullsized":"2023-09-23T18:05:15.087182+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1262601,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":78,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.073522+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:05:14.073522+0000","last_peered":"2023-09-23T18:05:14.073522+0000","last_clean":"2023-09-23T18:05:14.073522+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:05:14.073522+0000","last_undegraded":"2023-09-23T18:05:14.073522+0000","last_fullsized":"2023-09-23T18:05:14.073522+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":160,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.065491+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:05:14.065491+0000","last_peered":"2023-09-23T18:05:14.065491+0000","last_clean":"2023-09-23T18:05:14.065491+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:05:14.065491+0000","last_undegraded":"2023-09-23T18:05:14.065491+0000","last_fullsized":"2023-09-23T18:05:14.065491+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":603,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.096602+0000","last_change":"2023-09-23T18:05:15.096602+0000","last_active":"2023-09-23T18:05:15.096602+0000","last_peered":"2023-09-23T18:05:15.096602+0000","last_clean":"2023-09-23T18:05:15.096602+0000","last_became_active":"2023-09-23T18:05:15.096205+0000","last_became_peered":"2023-09-23T18:05:15.096205+0000","last_unstale":"2023-09-23T18:05:15.096602+0000","last_undegraded":"2023-09-23T18:05:15.096602+0000","last_fullsized":"2023-09-23T18:05:15.096602+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:31:18.329681+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":128,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.069561+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:05:14.069561+0000","last_peered":"2023-09-23T18:05:14.069561+0000","last_clean":"2023-09-23T18:05:14.069561+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:05:14.069561+0000","last_undegraded":"2023-09-23T18:05:14.069561+0000","last_fullsized":"2023-09-23T18:05:14.069561+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":279,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.074617+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:05:15.074617+0000","last_peered":"2023-09-23T18:05:15.074617+0000","last_clean":"2023-09-23T18:05:15.074617+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:05:15.074617+0000","last_undegraded":"2023-09-23T18:05:15.074617+0000","last_fullsized":"2023-09-23T18:05:15.074617+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":595,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.096807+0000","last_change":"2023-09-23T18:05:15.096807+0000","last_active":"2023-09-23T18:05:15.096807+0000","last_peered":"2023-09-23T18:05:15.096807+0000","last_clean":"2023-09-23T18:05:15.096807+0000","last_became_active":"2023-09-23T18:05:15.096022+0000","last_became_peered":"2023-09-23T18:05:15.096022+0000","last_unstale":"2023-09-23T18:05:15.096807+0000","last_undegraded":"2023-09-23T18:05:15.096807+0000","last_fullsized":"2023-09-23T18:05:15.096807+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:32.387626+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":141,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.059446+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:05:14.059446+0000","last_peered":"2023-09-23T18:05:14.059446+0000","last_clean":"2023-09-23T18:05:14.059446+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:05:14.059446+0000","last_undegraded":"2023-09-23T18:05:14.059446+0000","last_fullsized":"2023-09-23T18:05:14.059446+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":219,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.078252+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:05:15.078252+0000","last_peered":"2023-09-23T18:05:15.078252+0000","last_clean":"2023-09-23T18:05:15.078252+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:05:15.078252+0000","last_undegraded":"2023-09-23T18:05:15.078252+0000","last_fullsized":"2023-09-23T18:05:15.078252+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":628,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.086647+0000","last_change":"2023-09-23T18:05:15.086647+0000","last_active":"2023-09-23T18:05:15.086647+0000","last_peered":"2023-09-23T18:05:15.086647+0000","last_clean":"2023-09-23T18:05:15.086647+0000","last_became_active":"2023-09-23T18:05:15.086214+0000","last_became_peered":"2023-09-23T18:05:15.086214+0000","last_unstale":"2023-09-23T18:05:15.086647+0000","last_undegraded":"2023-09-23T18:05:15.086647+0000","last_fullsized":"2023-09-23T18:05:15.086647+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":258,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.074997+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:05:15.074997+0000","last_peered":"2023-09-23T18:05:15.074997+0000","last_clean":"2023-09-23T18:05:15.074997+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:05:15.074997+0000","last_undegraded":"2023-09-23T18:05:15.074997+0000","last_fullsized":"2023-09-23T18:05:15.074997+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":135,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.059546+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:05:14.059546+0000","last_peered":"2023-09-23T18:05:14.059546+0000","last_clean":"2023-09-23T18:05:14.059546+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:05:14.059546+0000","last_undegraded":"2023-09-23T18:05:14.059546+0000","last_fullsized":"2023-09-23T18:05:14.059546+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":586,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.088596+0000","last_change":"2023-09-23T18:05:15.088596+0000","last_active":"2023-09-23T18:05:15.088596+0000","last_peered":"2023-09-23T18:05:15.088596+0000","last_clean":"2023-09-23T18:05:15.088596+0000","last_became_active":"2023-09-23T18:05:15.088205+0000","last_became_peered":"2023-09-23T18:05:15.088205+0000","last_unstale":"2023-09-23T18:05:15.088596+0000","last_undegraded":"2023-09-23T18:05:15.088596+0000","last_fullsized":"2023-09-23T18:05:15.088596+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":260,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.059226+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:05:14.059226+0000","last_peered":"2023-09-23T18:05:14.059226+0000","last_clean":"2023-09-23T18:05:14.059226+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:05:14.059226+0000","last_undegraded":"2023-09-23T18:05:14.059226+0000","last_fullsized":"2023-09-23T18:05:14.059226+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":89,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.069475+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:05:14.069475+0000","last_peered":"2023-09-23T18:05:14.069475+0000","last_clean":"2023-09-23T18:05:14.069475+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:05:14.069475+0000","last_undegraded":"2023-09-23T18:05:14.069475+0000","last_fullsized":"2023-09-23T18:05:14.069475+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":592,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.100844+0000","last_change":"2023-09-23T18:05:15.100844+0000","last_active":"2023-09-23T18:05:15.100844+0000","last_peered":"2023-09-23T18:05:15.100844+0000","last_clean":"2023-09-23T18:05:15.100844+0000","last_became_active":"2023-09-23T18:05:15.100365+0000","last_became_peered":"2023-09-23T18:05:15.100365+0000","last_unstale":"2023-09-23T18:05:15.100844+0000","last_undegraded":"2023-09-23T18:05:15.100844+0000","last_fullsized":"2023-09-23T18:05:15.100844+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":43,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.065773+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:05:14.065773+0000","last_peered":"2023-09-23T18:05:14.065773+0000","last_clean":"2023-09-23T18:05:14.065773+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:05:14.065773+0000","last_undegraded":"2023-09-23T18:05:14.065773+0000","last_fullsized":"2023-09-23T18:05:14.065773+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":175,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.081805+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:05:15.081805+0000","last_peered":"2023-09-23T18:05:15.081805+0000","last_clean":"2023-09-23T18:05:15.081805+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:05:15.081805+0000","last_undegraded":"2023-09-23T18:05:15.081805+0000","last_fullsized":"2023-09-23T18:05:15.081805+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":599,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.096980+0000","last_change":"2023-09-23T18:05:15.096980+0000","last_active":"2023-09-23T18:05:15.096980+0000","last_peered":"2023-09-23T18:05:15.096980+0000","last_clean":"2023-09-23T18:05:15.096980+0000","last_became_active":"2023-09-23T18:05:15.096456+0000","last_became_peered":"2023-09-23T18:05:15.096456+0000","last_unstale":"2023-09-23T18:05:15.096980+0000","last_undegraded":"2023-09-23T18:05:15.096980+0000","last_fullsized":"2023-09-23T18:05:15.096980+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":78,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.065799+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:05:14.065799+0000","last_peered":"2023-09-23T18:05:14.065799+0000","last_clean":"2023-09-23T18:05:14.065799+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:05:14.065799+0000","last_undegraded":"2023-09-23T18:05:14.065799+0000","last_fullsized":"2023-09-23T18:05:14.065799+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":78,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.069964+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:05:14.069964+0000","last_peered":"2023-09-23T18:05:14.069964+0000","last_clean":"2023-09-23T18:05:14.069964+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:05:14.069964+0000","last_undegraded":"2023-09-23T18:05:14.069964+0000","last_fullsized":"2023-09-23T18:05:14.069964+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":589,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.095829+0000","last_change":"2023-09-23T18:05:15.095829+0000","last_active":"2023-09-23T18:05:15.095829+0000","last_peered":"2023-09-23T18:05:15.095829+0000","last_clean":"2023-09-23T18:05:15.095829+0000","last_became_active":"2023-09-23T18:05:15.095414+0000","last_became_peered":"2023-09-23T18:05:15.095414+0000","last_unstale":"2023-09-23T18:05:15.095829+0000","last_undegraded":"2023-09-23T18:05:15.095829+0000","last_fullsized":"2023-09-23T18:05:15.095829+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":206,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.077484+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:05:14.077484+0000","last_peered":"2023-09-23T18:05:14.077484+0000","last_clean":"2023-09-23T18:05:14.077484+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:05:14.077484+0000","last_undegraded":"2023-09-23T18:05:14.077484+0000","last_fullsized":"2023-09-23T18:05:14.077484+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":123,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.066235+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:05:14.066235+0000","last_peered":"2023-09-23T18:05:14.066235+0000","last_clean":"2023-09-23T18:05:14.066235+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:05:14.066235+0000","last_undegraded":"2023-09-23T18:05:14.066235+0000","last_fullsized":"2023-09-23T18:05:14.066235+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":595,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.091218+0000","last_change":"2023-09-23T18:05:15.091218+0000","last_active":"2023-09-23T18:05:15.091218+0000","last_peered":"2023-09-23T18:05:15.091218+0000","last_clean":"2023-09-23T18:05:15.091218+0000","last_became_active":"2023-09-23T18:05:15.090420+0000","last_became_peered":"2023-09-23T18:05:15.090420+0000","last_unstale":"2023-09-23T18:05:15.091218+0000","last_undegraded":"2023-09-23T18:05:15.091218+0000","last_fullsized":"2023-09-23T18:05:15.091218+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:57.754408+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":277,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.056306+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:05:14.056306+0000","last_peered":"2023-09-23T18:05:14.056306+0000","last_clean":"2023-09-23T18:05:14.056306+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:05:14.056306+0000","last_undegraded":"2023-09-23T18:05:14.056306+0000","last_fullsized":"2023-09-23T18:05:14.056306+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":255,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.059580+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:05:14.059580+0000","last_peered":"2023-09-23T18:05:14.059580+0000","last_clean":"2023-09-23T18:05:14.059580+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:05:14.059580+0000","last_undegraded":"2023-09-23T18:05:14.059580+0000","last_fullsized":"2023-09-23T18:05:14.059580+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":580,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.097196+0000","last_change":"2023-09-23T18:05:15.097196+0000","last_active":"2023-09-23T18:05:15.097196+0000","last_peered":"2023-09-23T18:05:15.097196+0000","last_clean":"2023-09-23T18:05:15.097196+0000","last_became_active":"2023-09-23T18:05:15.096800+0000","last_became_peered":"2023-09-23T18:05:15.096800+0000","last_unstale":"2023-09-23T18:05:15.097196+0000","last_undegraded":"2023-09-23T18:05:15.097196+0000","last_fullsized":"2023-09-23T18:05:15.097196+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":68,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.066067+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:05:14.066067+0000","last_peered":"2023-09-23T18:05:14.066067+0000","last_clean":"2023-09-23T18:05:14.066067+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:05:14.066067+0000","last_undegraded":"2023-09-23T18:05:14.066067+0000","last_fullsized":"2023-09-23T18:05:14.066067+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":268,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.074704+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:05:15.074704+0000","last_peered":"2023-09-23T18:05:15.074704+0000","last_clean":"2023-09-23T18:05:15.074704+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:05:15.074704+0000","last_undegraded":"2023-09-23T18:05:15.074704+0000","last_fullsized":"2023-09-23T18:05:15.074704+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":595,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.090180+0000","last_change":"2023-09-23T18:05:15.090180+0000","last_active":"2023-09-23T18:05:15.090180+0000","last_peered":"2023-09-23T18:05:15.090180+0000","last_clean":"2023-09-23T18:05:15.090180+0000","last_became_active":"2023-09-23T18:05:15.089754+0000","last_became_peered":"2023-09-23T18:05:15.089754+0000","last_unstale":"2023-09-23T18:05:15.090180+0000","last_undegraded":"2023-09-23T18:05:15.090180+0000","last_fullsized":"2023-09-23T18:05:15.090180+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:06.678700+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":215,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.073087+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:05:15.073087+0000","last_peered":"2023-09-23T18:05:15.073087+0000","last_clean":"2023-09-23T18:05:15.073087+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:05:15.073087+0000","last_undegraded":"2023-09-23T18:05:15.073087+0000","last_fullsized":"2023-09-23T18:05:15.073087+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":132,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.066424+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:05:14.066424+0000","last_peered":"2023-09-23T18:05:14.066424+0000","last_clean":"2023-09-23T18:05:14.066424+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:05:14.066424+0000","last_undegraded":"2023-09-23T18:05:14.066424+0000","last_fullsized":"2023-09-23T18:05:14.066424+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":628,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.096714+0000","last_change":"2023-09-23T18:05:15.096714+0000","last_active":"2023-09-23T18:05:15.096714+0000","last_peered":"2023-09-23T18:05:15.096714+0000","last_clean":"2023-09-23T18:05:15.096714+0000","last_became_active":"2023-09-23T18:05:15.096260+0000","last_became_peered":"2023-09-23T18:05:15.096260+0000","last_unstale":"2023-09-23T18:05:15.096714+0000","last_undegraded":"2023-09-23T18:05:15.096714+0000","last_fullsized":"2023-09-23T18:05:15.096714+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":206,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.072679+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:05:14.072679+0000","last_peered":"2023-09-23T18:05:14.072679+0000","last_clean":"2023-09-23T18:05:14.072679+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:05:14.072679+0000","last_undegraded":"2023-09-23T18:05:14.072679+0000","last_fullsized":"2023-09-23T18:05:14.072679+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":266,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.072252+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:05:14.072252+0000","last_peered":"2023-09-23T18:05:14.072252+0000","last_clean":"2023-09-23T18:05:14.072252+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:05:14.072252+0000","last_undegraded":"2023-09-23T18:05:14.072252+0000","last_fullsized":"2023-09-23T18:05:14.072252+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":616,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.083980+0000","last_change":"2023-09-23T18:05:15.083980+0000","last_active":"2023-09-23T18:05:15.083980+0000","last_peered":"2023-09-23T18:05:15.083980+0000","last_clean":"2023-09-23T18:05:15.083980+0000","last_became_active":"2023-09-23T18:05:15.083516+0000","last_became_peered":"2023-09-23T18:05:15.083516+0000","last_unstale":"2023-09-23T18:05:15.083980+0000","last_undegraded":"2023-09-23T18:05:15.083980+0000","last_fullsized":"2023-09-23T18:05:15.083980+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1064488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":221,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.076590+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:05:14.076590+0000","last_peered":"2023-09-23T18:05:14.076590+0000","last_clean":"2023-09-23T18:05:14.076590+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:05:14.076590+0000","last_undegraded":"2023-09-23T18:05:14.076590+0000","last_fullsized":"2023-09-23T18:05:14.076590+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":288,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.066350+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:05:14.066350+0000","last_peered":"2023-09-23T18:05:14.066350+0000","last_clean":"2023-09-23T18:05:14.066350+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:05:14.066350+0000","last_undegraded":"2023-09-23T18:05:14.066350+0000","last_fullsized":"2023-09-23T18:05:14.066350+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":440,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.100703+0000","last_change":"2023-09-23T18:05:15.099144+0000","last_active":"2023-09-23T18:05:15.100703+0000","last_peered":"2023-09-23T18:05:15.100703+0000","last_clean":"2023-09-23T18:05:15.100703+0000","last_became_active":"2023-09-23T18:05:15.098234+0000","last_became_peered":"2023-09-23T18:05:15.098234+0000","last_unstale":"2023-09-23T18:05:15.100703+0000","last_undegraded":"2023-09-23T18:05:15.100703+0000","last_fullsized":"2023-09-23T18:05:15.100703+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":144,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.072620+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:05:14.072620+0000","last_peered":"2023-09-23T18:05:14.072620+0000","last_clean":"2023-09-23T18:05:14.072620+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:05:14.072620+0000","last_undegraded":"2023-09-23T18:05:14.072620+0000","last_fullsized":"2023-09-23T18:05:14.072620+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":244,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.081201+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:05:15.081201+0000","last_peered":"2023-09-23T18:05:15.081201+0000","last_clean":"2023-09-23T18:05:15.081201+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:05:15.081201+0000","last_undegraded":"2023-09-23T18:05:15.081201+0000","last_fullsized":"2023-09-23T18:05:15.081201+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":592,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.102453+0000","last_change":"2023-09-23T18:05:15.102453+0000","last_active":"2023-09-23T18:05:15.102453+0000","last_peered":"2023-09-23T18:05:15.102453+0000","last_clean":"2023-09-23T18:05:15.102453+0000","last_became_active":"2023-09-23T18:05:15.102072+0000","last_became_peered":"2023-09-23T18:05:15.102072+0000","last_unstale":"2023-09-23T18:05:15.102453+0000","last_undegraded":"2023-09-23T18:05:15.102453+0000","last_fullsized":"2023-09-23T18:05:15.102453+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":254,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.059874+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:05:14.059874+0000","last_peered":"2023-09-23T18:05:14.059874+0000","last_clean":"2023-09-23T18:05:14.059874+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:05:14.059874+0000","last_undegraded":"2023-09-23T18:05:14.059874+0000","last_fullsized":"2023-09-23T18:05:14.059874+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":280,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.058844+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:05:14.058844+0000","last_peered":"2023-09-23T18:05:14.058844+0000","last_clean":"2023-09-23T18:05:14.058844+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:05:14.058844+0000","last_undegraded":"2023-09-23T18:05:14.058844+0000","last_fullsized":"2023-09-23T18:05:14.058844+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":620,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.086621+0000","last_change":"2023-09-23T18:05:15.086621+0000","last_active":"2023-09-23T18:05:15.086621+0000","last_peered":"2023-09-23T18:05:15.086621+0000","last_clean":"2023-09-23T18:05:15.086621+0000","last_became_active":"2023-09-23T18:05:15.086216+0000","last_became_peered":"2023-09-23T18:05:15.086216+0000","last_unstale":"2023-09-23T18:05:15.086621+0000","last_undegraded":"2023-09-23T18:05:15.086621+0000","last_fullsized":"2023-09-23T18:05:15.086621+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":166,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.065946+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:05:14.065946+0000","last_peered":"2023-09-23T18:05:14.065946+0000","last_clean":"2023-09-23T18:05:14.065946+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:05:14.065946+0000","last_undegraded":"2023-09-23T18:05:14.065946+0000","last_fullsized":"2023-09-23T18:05:14.065946+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":286,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.065884+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:05:14.065884+0000","last_peered":"2023-09-23T18:05:14.065884+0000","last_clean":"2023-09-23T18:05:14.065884+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:05:14.065884+0000","last_undegraded":"2023-09-23T18:05:14.065884+0000","last_fullsized":"2023-09-23T18:05:14.065884+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":588,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.083605+0000","last_change":"2023-09-23T18:05:15.083605+0000","last_active":"2023-09-23T18:05:15.083605+0000","last_peered":"2023-09-23T18:05:15.083605+0000","last_clean":"2023-09-23T18:05:15.083605+0000","last_became_active":"2023-09-23T18:05:15.083184+0000","last_became_peered":"2023-09-23T18:05:15.083184+0000","last_unstale":"2023-09-23T18:05:15.083605+0000","last_undegraded":"2023-09-23T18:05:15.083605+0000","last_fullsized":"2023-09-23T18:05:15.083605+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":238,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.081905+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:05:15.081905+0000","last_peered":"2023-09-23T18:05:15.081905+0000","last_clean":"2023-09-23T18:05:15.081905+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:05:15.081905+0000","last_undegraded":"2023-09-23T18:05:15.081905+0000","last_fullsized":"2023-09-23T18:05:15.081905+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":78,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.072466+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:05:14.072466+0000","last_peered":"2023-09-23T18:05:14.072466+0000","last_clean":"2023-09-23T18:05:14.072466+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:05:14.072466+0000","last_undegraded":"2023-09-23T18:05:14.072466+0000","last_fullsized":"2023-09-23T18:05:14.072466+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.110840+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:03:05.839506+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":599,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.082278+0000","last_change":"2023-09-23T18:05:15.082278+0000","last_active":"2023-09-23T18:05:15.082278+0000","last_peered":"2023-09-23T18:05:15.082278+0000","last_clean":"2023-09-23T18:05:15.082278+0000","last_became_active":"2023-09-23T18:05:15.081901+0000","last_became_peered":"2023-09-23T18:05:15.081901+0000","last_unstale":"2023-09-23T18:05:15.082278+0000","last_undegraded":"2023-09-23T18:05:15.082278+0000","last_fullsized":"2023-09-23T18:05:15.082278+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":68,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.066150+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:05:14.066150+0000","last_peered":"2023-09-23T18:05:14.066150+0000","last_clean":"2023-09-23T18:05:14.066150+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:05:14.066150+0000","last_undegraded":"2023-09-23T18:05:14.066150+0000","last_fullsized":"2023-09-23T18:05:14.066150+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":288,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.068254+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:05:14.068254+0000","last_peered":"2023-09-23T18:05:14.068254+0000","last_clean":"2023-09-23T18:05:14.068254+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:05:14.068254+0000","last_undegraded":"2023-09-23T18:05:14.068254+0000","last_fullsized":"2023-09-23T18:05:14.068254+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.110847+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:25.713210+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3,7],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":584,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.089881+0000","last_change":"2023-09-23T18:05:15.089881+0000","last_active":"2023-09-23T18:05:15.089881+0000","last_peered":"2023-09-23T18:05:15.089881+0000","last_clean":"2023-09-23T18:05:15.089881+0000","last_became_active":"2023-09-23T18:05:15.089224+0000","last_became_peered":"2023-09-23T18:05:15.089224+0000","last_unstale":"2023-09-23T18:05:15.089881+0000","last_undegraded":"2023-09-23T18:05:15.089881+0000","last_fullsized":"2023-09-23T18:05:15.089881+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":124,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.059754+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:05:14.059754+0000","last_peered":"2023-09-23T18:05:14.059754+0000","last_clean":"2023-09-23T18:05:14.059754+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:05:14.059754+0000","last_undegraded":"2023-09-23T18:05:14.059754+0000","last_fullsized":"2023-09-23T18:05:14.059754+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":124,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.066691+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:05:14.066691+0000","last_peered":"2023-09-23T18:05:14.066691+0000","last_clean":"2023-09-23T18:05:14.066691+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:05:14.066691+0000","last_undegraded":"2023-09-23T18:05:14.066691+0000","last_fullsized":"2023-09-23T18:05:14.066691+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":440,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.100757+0000","last_change":"2023-09-23T18:05:15.099360+0000","last_active":"2023-09-23T18:05:15.100757+0000","last_peered":"2023-09-23T18:05:15.100757+0000","last_clean":"2023-09-23T18:05:15.100757+0000","last_became_active":"2023-09-23T18:05:15.098750+0000","last_became_peered":"2023-09-23T18:05:15.098750+0000","last_unstale":"2023-09-23T18:05:15.100757+0000","last_undegraded":"2023-09-23T18:05:15.100757+0000","last_fullsized":"2023-09-23T18:05:15.100757+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":587,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.058235+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:05:14.058235+0000","last_peered":"2023-09-23T18:05:14.058235+0000","last_clean":"2023-09-23T18:05:14.058235+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:05:14.058235+0000","last_undegraded":"2023-09-23T18:05:14.058235+0000","last_fullsized":"2023-09-23T18:05:14.058235+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":258,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.060204+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:05:14.060204+0000","last_peered":"2023-09-23T18:05:14.060204+0000","last_clean":"2023-09-23T18:05:14.060204+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:05:14.060204+0000","last_undegraded":"2023-09-23T18:05:14.060204+0000","last_fullsized":"2023-09-23T18:05:14.060204+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":255,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.073547+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:05:14.073547+0000","last_peered":"2023-09-23T18:05:14.073547+0000","last_clean":"2023-09-23T18:05:14.073547+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:05:14.073547+0000","last_undegraded":"2023-09-23T18:05:14.073547+0000","last_fullsized":"2023-09-23T18:05:14.073547+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"creating+peering","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.110853+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:07.605432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,3],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":595,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.083813+0000","last_change":"2023-09-23T18:05:15.083813+0000","last_active":"2023-09-23T18:05:15.083813+0000","last_peered":"2023-09-23T18:05:15.083813+0000","last_clean":"2023-09-23T18:05:15.083813+0000","last_became_active":"2023-09-23T18:05:15.083395+0000","last_became_peered":"2023-09-23T18:05:15.083395+0000","last_unstale":"2023-09-23T18:05:15.083813+0000","last_undegraded":"2023-09-23T18:05:15.083813+0000","last_fullsized":"2023-09-23T18:05:15.083813+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:24.402252+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1206,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.073454+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:05:14.073454+0000","last_peered":"2023-09-23T18:05:14.073454+0000","last_clean":"2023-09-23T18:05:14.073454+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:05:14.073454+0000","last_undegraded":"2023-09-23T18:05:14.073454+0000","last_fullsized":"2023-09-23T18:05:14.073454+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":68,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.070057+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:05:14.070057+0000","last_peered":"2023-09-23T18:05:14.070057+0000","last_clean":"2023-09-23T18:05:14.070057+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:05:14.070057+0000","last_undegraded":"2023-09-23T18:05:14.070057+0000","last_fullsized":"2023-09-23T18:05:14.070057+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":172,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.064686+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:05:14.064686+0000","last_peered":"2023-09-23T18:05:14.064686+0000","last_clean":"2023-09-23T18:05:14.064686+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:05:14.064686+0000","last_undegraded":"2023-09-23T18:05:14.064686+0000","last_fullsized":"2023-09-23T18:05:14.064686+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":603,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.087882+0000","last_change":"2023-09-23T18:05:15.087882+0000","last_active":"2023-09-23T18:05:15.087882+0000","last_peered":"2023-09-23T18:05:15.087882+0000","last_clean":"2023-09-23T18:05:15.087882+0000","last_became_active":"2023-09-23T18:05:15.087524+0000","last_became_peered":"2023-09-23T18:05:15.087524+0000","last_unstale":"2023-09-23T18:05:15.087882+0000","last_undegraded":"2023-09-23T18:05:15.087882+0000","last_fullsized":"2023-09-23T18:05:15.087882+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:55.015941+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":561,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.065065+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:05:14.065065+0000","last_peered":"2023-09-23T18:05:14.065065+0000","last_clean":"2023-09-23T18:05:14.065065+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:05:14.065065+0000","last_undegraded":"2023-09-23T18:05:14.065065+0000","last_fullsized":"2023-09-23T18:05:14.065065+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":173,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.064082+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:05:14.064082+0000","last_peered":"2023-09-23T18:05:14.064082+0000","last_clean":"2023-09-23T18:05:14.064082+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:05:14.064082+0000","last_undegraded":"2023-09-23T18:05:14.064082+0000","last_fullsized":"2023-09-23T18:05:14.064082+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":245,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.062511+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:05:14.062511+0000","last_peered":"2023-09-23T18:05:14.062511+0000","last_clean":"2023-09-23T18:05:14.062511+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:05:14.062511+0000","last_undegraded":"2023-09-23T18:05:14.062511+0000","last_fullsized":"2023-09-23T18:05:14.062511+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1190,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.063324+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:05:14.063324+0000","last_peered":"2023-09-23T18:05:14.063324+0000","last_clean":"2023-09-23T18:05:14.063324+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:05:14.063324+0000","last_undegraded":"2023-09-23T18:05:14.063324+0000","last_fullsized":"2023-09-23T18:05:14.063324+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":603,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.069232+0000","last_change":"2023-09-23T18:04:39.585751+0000","last_active":"2023-09-23T18:05:14.069232+0000","last_peered":"2023-09-23T18:05:14.069232+0000","last_clean":"2023-09-23T18:05:14.069232+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:05:14.069232+0000","last_undegraded":"2023-09-23T18:05:14.069232+0000","last_fullsized":"2023-09-23T18:05:14.069232+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":86,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.065607+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:05:14.065607+0000","last_peered":"2023-09-23T18:05:14.065607+0000","last_clean":"2023-09-23T18:05:14.065607+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:05:14.065607+0000","last_undegraded":"2023-09-23T18:05:14.065607+0000","last_fullsized":"2023-09-23T18:05:14.065607+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":212,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.081375+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:05:15.081375+0000","last_peered":"2023-09-23T18:05:15.081375+0000","last_clean":"2023-09-23T18:05:15.081375+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:05:15.081375+0000","last_undegraded":"2023-09-23T18:05:15.081375+0000","last_fullsized":"2023-09-23T18:05:15.081375+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":440,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.099310+0000","last_change":"2023-09-23T18:05:15.096283+0000","last_active":"2023-09-23T18:05:15.099310+0000","last_peered":"2023-09-23T18:05:15.099310+0000","last_clean":"2023-09-23T18:05:15.099310+0000","last_became_active":"2023-09-23T18:05:15.095885+0000","last_became_peered":"2023-09-23T18:05:15.095885+0000","last_unstale":"2023-09-23T18:05:15.099310+0000","last_undegraded":"2023-09-23T18:05:15.099310+0000","last_fullsized":"2023-09-23T18:05:15.099310+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":457,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.069706+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:05:14.069706+0000","last_peered":"2023-09-23T18:05:14.069706+0000","last_clean":"2023-09-23T18:05:14.069706+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:05:14.069706+0000","last_undegraded":"2023-09-23T18:05:14.069706+0000","last_fullsized":"2023-09-23T18:05:14.069706+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":277,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.075650+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:05:15.075650+0000","last_peered":"2023-09-23T18:05:15.075650+0000","last_clean":"2023-09-23T18:05:15.075650+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:05:15.075650+0000","last_undegraded":"2023-09-23T18:05:15.075650+0000","last_fullsized":"2023-09-23T18:05:15.075650+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":247,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.062939+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:05:14.062939+0000","last_peered":"2023-09-23T18:05:14.062939+0000","last_clean":"2023-09-23T18:05:14.062939+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:05:14.062939+0000","last_undegraded":"2023-09-23T18:05:14.062939+0000","last_fullsized":"2023-09-23T18:05:14.062939+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":440,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.091137+0000","last_change":"2023-09-23T18:05:15.085455+0000","last_active":"2023-09-23T18:05:15.091137+0000","last_peered":"2023-09-23T18:05:15.091137+0000","last_clean":"2023-09-23T18:05:15.091137+0000","last_became_active":"2023-09-23T18:05:15.085106+0000","last_became_peered":"2023-09-23T18:05:15.085106+0000","last_unstale":"2023-09-23T18:05:15.091137+0000","last_undegraded":"2023-09-23T18:05:15.091137+0000","last_fullsized":"2023-09-23T18:05:15.091137+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":491,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.072505+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:05:14.072505+0000","last_peered":"2023-09-23T18:05:14.072505+0000","last_clean":"2023-09-23T18:05:14.072505+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:05:14.072505+0000","last_undegraded":"2023-09-23T18:05:14.072505+0000","last_fullsized":"2023-09-23T18:05:14.072505+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":213,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.064717+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:05:14.064717+0000","last_peered":"2023-09-23T18:05:14.064717+0000","last_clean":"2023-09-23T18:05:14.064717+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:05:14.064717+0000","last_undegraded":"2023-09-23T18:05:14.064717+0000","last_fullsized":"2023-09-23T18:05:14.064717+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":244,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.059219+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:05:14.059219+0000","last_peered":"2023-09-23T18:05:14.059219+0000","last_clean":"2023-09-23T18:05:14.059219+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:05:14.059219+0000","last_undegraded":"2023-09-23T18:05:14.059219+0000","last_fullsized":"2023-09-23T18:05:14.059219+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":593,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.096447+0000","last_change":"2023-09-23T18:05:15.096447+0000","last_active":"2023-09-23T18:05:15.096447+0000","last_peered":"2023-09-23T18:05:15.096447+0000","last_clean":"2023-09-23T18:05:15.096447+0000","last_became_active":"2023-09-23T18:05:15.096059+0000","last_became_peered":"2023-09-23T18:05:15.096059+0000","last_unstale":"2023-09-23T18:05:15.096447+0000","last_undegraded":"2023-09-23T18:05:15.096447+0000","last_fullsized":"2023-09-23T18:05:15.096447+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":124,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.066604+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:05:14.066604+0000","last_peered":"2023-09-23T18:05:14.066604+0000","last_clean":"2023-09-23T18:05:14.066604+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:05:14.066604+0000","last_undegraded":"2023-09-23T18:05:14.066604+0000","last_fullsized":"2023-09-23T18:05:14.066604+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":349,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.073387+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:05:15.073387+0000","last_peered":"2023-09-23T18:05:15.073387+0000","last_clean":"2023-09-23T18:05:15.073387+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:05:15.073387+0000","last_undegraded":"2023-09-23T18:05:15.073387+0000","last_fullsized":"2023-09-23T18:05:15.073387+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":128,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.059641+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:05:14.059641+0000","last_peered":"2023-09-23T18:05:14.059641+0000","last_clean":"2023-09-23T18:05:14.059641+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:05:14.059641+0000","last_undegraded":"2023-09-23T18:05:14.059641+0000","last_fullsized":"2023-09-23T18:05:14.059641+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"0'0","reported_seq":0,"reported_epoch":0,"state":"unknown","last_fresh":"2023-09-23T18:05:45.094891+0000","last_change":"2023-09-23T18:05:45.094891+0000","last_active":"2023-09-23T18:05:45.094891+0000","last_peered":"2023-09-23T18:05:45.094891+0000","last_clean":"2023-09-23T18:05:45.094891+0000","last_became_active":"0.000000","last_became_peered":"0.000000","last_unstale":"2023-09-23T18:05:45.094891+0000","last_undegraded":"2023-09-23T18:05:45.094891+0000","last_fullsized":"2023-09-23T18:05:45.094891+0000","mapping_epoch":0,"log_start":"0'0","ondisk_log_start":"0'0","created":0,"last_epoch_clean":0,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"--","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[],"acting":[],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":-1,"acting_primary":-1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":598,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.088351+0000","last_change":"2023-09-23T18:05:15.088351+0000","last_active":"2023-09-23T18:05:15.088351+0000","last_peered":"2023-09-23T18:05:15.088351+0000","last_clean":"2023-09-23T18:05:15.088351+0000","last_became_active":"2023-09-23T18:05:15.087902+0000","last_became_peered":"2023-09-23T18:05:15.087902+0000","last_unstale":"2023-09-23T18:05:15.088351+0000","last_undegraded":"2023-09-23T18:05:15.088351+0000","last_fullsized":"2023-09-23T18:05:15.088351+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:02:50.479420+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":484,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.078774+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:05:15.078774+0000","last_peered":"2023-09-23T18:05:15.078774+0000","last_clean":"2023-09-23T18:05:15.078774+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:05:15.078774+0000","last_undegraded":"2023-09-23T18:05:15.078774+0000","last_fullsized":"2023-09-23T18:05:15.078774+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":304,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.063041+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:05:14.063041+0000","last_peered":"2023-09-23T18:05:14.063041+0000","last_clean":"2023-09-23T18:05:14.063041+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:05:14.063041+0000","last_undegraded":"2023-09-23T18:05:14.063041+0000","last_fullsized":"2023-09-23T18:05:14.063041+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":68,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.065980+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:05:14.065980+0000","last_peered":"2023-09-23T18:05:14.065980+0000","last_clean":"2023-09-23T18:05:14.065980+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:05:14.065980+0000","last_undegraded":"2023-09-23T18:05:14.065980+0000","last_fullsized":"2023-09-23T18:05:14.065980+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":581,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.083744+0000","last_change":"2023-09-23T18:05:15.083744+0000","last_active":"2023-09-23T18:05:15.083744+0000","last_peered":"2023-09-23T18:05:15.083744+0000","last_clean":"2023-09-23T18:05:15.083744+0000","last_became_active":"2023-09-23T18:05:15.083337+0000","last_became_peered":"2023-09-23T18:05:15.083337+0000","last_unstale":"2023-09-23T18:05:15.083744+0000","last_undegraded":"2023-09-23T18:05:15.083744+0000","last_fullsized":"2023-09-23T18:05:15.083744+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":263,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.059099+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:05:14.059099+0000","last_peered":"2023-09-23T18:05:14.059099+0000","last_clean":"2023-09-23T18:05:14.059099+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:05:14.059099+0000","last_undegraded":"2023-09-23T18:05:14.059099+0000","last_fullsized":"2023-09-23T18:05:14.059099+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":291,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.064765+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:05:14.064765+0000","last_peered":"2023-09-23T18:05:14.064765+0000","last_clean":"2023-09-23T18:05:14.064765+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:05:14.064765+0000","last_undegraded":"2023-09-23T18:05:14.064765+0000","last_fullsized":"2023-09-23T18:05:14.064765+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":590,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.092746+0000","last_change":"2023-09-23T18:05:15.092746+0000","last_active":"2023-09-23T18:05:15.092746+0000","last_peered":"2023-09-23T18:05:15.092746+0000","last_clean":"2023-09-23T18:05:15.092746+0000","last_became_active":"2023-09-23T18:05:15.092486+0000","last_became_peered":"2023-09-23T18:05:15.092486+0000","last_unstale":"2023-09-23T18:05:15.092746+0000","last_undegraded":"2023-09-23T18:05:15.092746+0000","last_fullsized":"2023-09-23T18:05:15.092746+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":133,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.060370+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:05:14.060370+0000","last_peered":"2023-09-23T18:05:14.060370+0000","last_clean":"2023-09-23T18:05:14.060370+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:05:14.060370+0000","last_undegraded":"2023-09-23T18:05:14.060370+0000","last_fullsized":"2023-09-23T18:05:14.060370+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":203,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.071671+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:05:14.071671+0000","last_peered":"2023-09-23T18:05:14.071671+0000","last_clean":"2023-09-23T18:05:14.071671+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:05:14.071671+0000","last_undegraded":"2023-09-23T18:05:14.071671+0000","last_fullsized":"2023-09-23T18:05:14.071671+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":119,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.059262+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:05:14.059262+0000","last_peered":"2023-09-23T18:05:14.059262+0000","last_clean":"2023-09-23T18:05:14.059262+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:05:14.059262+0000","last_undegraded":"2023-09-23T18:05:14.059262+0000","last_fullsized":"2023-09-23T18:05:14.059262+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":143,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.065206+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:05:14.065206+0000","last_peered":"2023-09-23T18:05:14.065206+0000","last_clean":"2023-09-23T18:05:14.065206+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:05:14.065206+0000","last_undegraded":"2023-09-23T18:05:14.065206+0000","last_fullsized":"2023-09-23T18:05:14.065206+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":644,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.087831+0000","last_change":"2023-09-23T18:05:15.087831+0000","last_active":"2023-09-23T18:05:15.087831+0000","last_peered":"2023-09-23T18:05:15.087831+0000","last_clean":"2023-09-23T18:05:15.087831+0000","last_became_active":"2023-09-23T18:05:15.087299+0000","last_became_peered":"2023-09-23T18:05:15.087299+0000","last_unstale":"2023-09-23T18:05:15.087831+0000","last_undegraded":"2023-09-23T18:05:15.087831+0000","last_fullsized":"2023-09-23T18:05:15.087831+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:31.678814+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":143,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.067052+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:05:14.067052+0000","last_peered":"2023-09-23T18:05:14.067052+0000","last_clean":"2023-09-23T18:05:14.067052+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:05:14.067052+0000","last_undegraded":"2023-09-23T18:05:14.067052+0000","last_fullsized":"2023-09-23T18:05:14.067052+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":146,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.069276+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:05:14.069276+0000","last_peered":"2023-09-23T18:05:14.069276+0000","last_clean":"2023-09-23T18:05:14.069276+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:05:14.069276+0000","last_undegraded":"2023-09-23T18:05:14.069276+0000","last_fullsized":"2023-09-23T18:05:14.069276+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":589,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.098282+0000","last_change":"2023-09-23T18:05:15.098282+0000","last_active":"2023-09-23T18:05:15.098282+0000","last_peered":"2023-09-23T18:05:15.098282+0000","last_clean":"2023-09-23T18:05:15.098282+0000","last_became_active":"2023-09-23T18:05:15.097779+0000","last_became_peered":"2023-09-23T18:05:15.097779+0000","last_unstale":"2023-09-23T18:05:15.098282+0000","last_undegraded":"2023-09-23T18:05:15.098282+0000","last_fullsized":"2023-09-23T18:05:15.098282+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":257,"reported_epoch":278,"state":"active+clean","last_fresh":"2023-09-23T18:05:15.075014+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:05:15.075014+0000","last_peered":"2023-09-23T18:05:15.075014+0000","last_clean":"2023-09-23T18:05:15.075014+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:05:15.075014+0000","last_undegraded":"2023-09-23T18:05:15.075014+0000","last_fullsized":"2023-09-23T18:05:15.075014+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":288,"reported_epoch":277,"state":"active+clean","last_fresh":"2023-09-23T18:05:14.064934+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:05:14.064934+0000","last_peered":"2023-09-23T18:05:14.064934+0000","last_clean":"2023-09-23T18:05:14.064934+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:05:14.064934+0000","last_undegraded":"2023-09-23T18:05:14.064934+0000","last_fullsized":"2023-09-23T18:05:14.064934+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":9,"acting":9,"num_store_stats":1},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":420413,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":345,"num_read_kb":650,"num_write":169,"num_write_kb":1719,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3721559,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001397,"num_pgs":48,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":116860,"kb_used_data":3316,"kb_used_omap":13,"kb_used_meta":113522,"kb_avail":93628292,"statfs":{"total":95995035648,"available":95875371008,"internally_reserved":0,"allocated":3395584,"data_stored":2334910,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":116246632},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":0.76100000000000001,"15min":0.66300000000000003},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.9889999999999999,"5min":6.2770000000000001,"15min":6.2770000000000001},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.266,"5min":1.194,"15min":1.256},"min":{"1min":0.309,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.8209999999999997,"5min":6.641,"15min":6.641},"last":0.56299999999999994}]},{"osd":1,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.97699999999999998,"15min":0.77700000000000002},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":7.0309999999999997,"5min":7.1150000000000002,"15min":7.1150000000000002},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.466,"5min":1.1970000000000001,"15min":1.103},"min":{"1min":0.27600000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.9589999999999996,"5min":6.9589999999999996,"15min":6.9589999999999996},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.071,"15min":0.98199999999999998},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.6630000000000003,"5min":6.6630000000000003,"15min":6.6630000000000003},"last":0.61499999999999999},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.1100000000000001,"15min":0.91200000000000003},"min":{"1min":0.316,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.9160000000000004,"5min":7.9160000000000004,"15min":7.9160000000000004},"last":0.64200000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:05:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.472,"15min":1.526},"min":{"1min":0.34599999999999997,"5min":0.316,"15min":0.316},"max":{"1min":7.0579999999999998,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":0.753},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.391,"15min":1.1100000000000001},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":15.349,"5min":15.349,"15min":15.349},"last":0.59799999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.073,"15min":1.0720000000000001},"min":{"1min":0.307,"5min":0.245,"15min":0.245},"max":{"1min":6.9729999999999999,"5min":6.9729999999999999,"15min":6.9729999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":1.1459999999999999,"5min":0.71399999999999997,"15min":0.58799999999999997},"min":{"1min":0.32200000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0289999999999999,"5min":6.3479999999999999,"15min":6.3479999999999999},"last":0.73699999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.1759999999999999,"15min":1.0609999999999999},"min":{"1min":0.38,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.4749999999999996,"5min":9.5999999999999996,"15min":9.5999999999999996},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.2709999999999999,"15min":1.179},"min":{"1min":0.39400000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.3739999999999997,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.77800000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":0.94599999999999995,"15min":0.78600000000000003},"min":{"1min":0.438,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.2469999999999999,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.628},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.232,"15min":1.161},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.9539999999999997,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":0.70099999999999996}]}]},{"osd":6,"up_from":163,"seq":700079669334,"num_pgs":49,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":761904,"kb_used_data":601768,"kb_used_omap":7,"kb_used_meta":160120,"kb_avail":92983248,"statfs":{"total":95995035648,"available":95214845952,"internally_reserved":0,"allocated":616210432,"data_stored":615161375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":163963684},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.89900000000000002,"15min":1.2529999999999999},"min":{"1min":0.23100000000000001,"5min":0.224,"15min":0.215},"max":{"1min":1.7,"5min":15.407,"15min":15.407},"last":0.441},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.96899999999999997,"15min":2.0470000000000002},"min":{"1min":0.29199999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.5600000000000001,"5min":7.5739999999999998,"15min":13.536},"last":0.53300000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.044,"15min":1.7649999999999999},"min":{"1min":0.27300000000000002,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.161,"5min":6.7690000000000001,"15min":14.233000000000001},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.1739999999999999,"15min":1.9650000000000001},"min":{"1min":0.44,"5min":0.317,"15min":0.248},"max":{"1min":3.5830000000000002,"5min":14.789,"15min":14.789},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.1919999999999999,"15min":2.1030000000000002},"min":{"1min":0.41199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.8410000000000002,"5min":7.173,"15min":11.210000000000001},"last":0.55400000000000005},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.1439999999999999,"15min":2.4209999999999998},"min":{"1min":0.313,"5min":0.245,"15min":0.245},"max":{"1min":5.4829999999999997,"5min":9.3620000000000001,"15min":11.223000000000001},"last":0.54600000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:05:26 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.097,"15min":1.101},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":4.7539999999999996,"5min":4.7539999999999996,"15min":4.7539999999999996},"last":0.47599999999999998},{"interface":"front","average":{"1min":1.6619999999999999,"5min":1.3260000000000001,"15min":1.2190000000000001},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.484,"5min":7.484,"15min":7.484},"last":0.46300000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0109999999999999,"15min":0.91800000000000004},"min":{"1min":0.34200000000000003,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":5.5069999999999997,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.52300000000000002},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.81299999999999994,"15min":0.59999999999999998},"min":{"1min":0.32300000000000001,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":6.3920000000000003,"5min":6.3920000000000003,"15min":8.3000000000000007},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":0.91000000000000003,"15min":0.89400000000000002},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.28599999999999998},"max":{"1min":5.4249999999999998,"5min":6.6159999999999997,"15min":11.869},"last":0.44800000000000001},{"interface":"front","average":{"1min":0.625,"5min":0.95799999999999996,"15min":0.71799999999999997},"min":{"1min":0.377,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.73,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":0.41999999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:05:31 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.075,"15min":1.0009999999999999},"min":{"1min":0.38400000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.5380000000000003,"5min":5.5380000000000003,"15min":5.5380000000000003},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.647,"5min":1.012,"15min":0.85799999999999998},"min":{"1min":0.40100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.411,"5min":10.411,"15min":10.411},"last":0.51200000000000001}]}]},{"osd":1,"up_from":0,"seq":420906795153,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958679,"num_pgs":46,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":409592,"kb_used_data":7856,"kb_used_omap":7,"kb_used_meta":401720,"kb_avail":93335560,"statfs":{"total":95995035648,"available":95575613440,"internally_reserved":0,"allocated":8044544,"data_stored":6641855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":411362020},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:05:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":1.2909999999999999,"15min":1.2809999999999999},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.17899999999999999},"max":{"1min":10.128,"5min":10.128,"15min":18.154},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.101,"15min":1.175},"min":{"1min":0.26900000000000002,"5min":0.186,"15min":0.17399999999999999},"max":{"1min":5.5490000000000004,"5min":12.641999999999999,"15min":13.369999999999999},"last":0.57199999999999995}]},{"osd":2,"last update":"Sat Sep 23 18:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.631,"5min":1.6279999999999999,"15min":1.3680000000000001},"min":{"1min":0.23400000000000001,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":10.077999999999999,"5min":11.757999999999999,"15min":20.882000000000001},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.3999999999999999,"5min":1.6899999999999999,"15min":1.302},"min":{"1min":0.249,"5min":0.217,"15min":0.217},"max":{"1min":7.9029999999999996,"5min":12.843,"15min":15.064},"last":0.48099999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:05:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.5649999999999999,"15min":1.5569999999999999},"min":{"1min":0.245,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":11.948,"5min":11.948,"15min":11.948},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.347,"15min":1.2490000000000001},"min":{"1min":0.30599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.9699999999999998,"5min":8.7300000000000004,"15min":8.7300000000000004},"last":0.54600000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.921,"5min":1.4379999999999999,"15min":1.206},"min":{"1min":0.32800000000000001,"5min":0.29299999999999998,"15min":0.251},"max":{"1min":5.4720000000000004,"5min":8.4700000000000006,"15min":15.279999999999999},"last":0.65300000000000002},{"interface":"front","average":{"1min":1.827,"5min":1.538,"15min":1.224},"min":{"1min":0.315,"5min":0.247,"15min":0.247},"max":{"1min":10.164999999999999,"5min":11.521000000000001,"15min":18.106999999999999},"last":0.67000000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9890000000000001,"5min":1.6830000000000001,"15min":1.2869999999999999},"min":{"1min":0.34000000000000002,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":10.194000000000001,"5min":11.48,"15min":14.720000000000001},"last":0.60799999999999998},{"interface":"front","average":{"1min":1.8500000000000001,"5min":1.4019999999999999,"15min":1.105},"min":{"1min":0.36499999999999999,"5min":0.254,"15min":0.254},"max":{"1min":7.742,"5min":12.794,"15min":12.794},"last":0.63}]},{"osd":6,"last update":"Sat Sep 23 18:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.254,"5min":1.4279999999999999,"15min":2.1539999999999999},"min":{"1min":0.36299999999999999,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.5599999999999996,"5min":8.4410000000000007,"15min":14.956},"last":0.59299999999999997},{"interface":"front","average":{"1min":1.2949999999999999,"5min":1.3899999999999999,"15min":2.3039999999999998},"min":{"1min":0.35799999999999998,"5min":0.311,"15min":0.311},"max":{"1min":4.2759999999999998,"5min":12.755000000000001,"15min":12.755000000000001},"last":0.68899999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:05:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.036,"15min":0.95999999999999996},"min":{"1min":0.41999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.4180000000000001,"5min":7.4649999999999999,"15min":7.4649999999999999},"last":0.72199999999999998},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.385,"15min":1.1830000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.468,"5min":5.524,"15min":5.524},"last":0.70799999999999996}]}]},{"osd":2,"up_from":103,"seq":442381631626,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":395104,"kb_used_data":7768,"kb_used_omap":11,"kb_used_meta":387316,"kb_avail":93350048,"statfs":{"total":95995035648,"available":95590449152,"internally_reserved":0,"allocated":7954432,"data_stored":6575084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":396611959},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":2.2690000000000001,"5min":2.1230000000000002,"15min":1.4139999999999999},"min":{"1min":0.214,"5min":0.187,"15min":0.17199999999999999},"max":{"1min":12.395,"5min":20.867000000000001,"15min":20.867000000000001},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.5820000000000001,"5min":1.879,"15min":1.359},"min":{"1min":0.29199999999999998,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":7.6550000000000002,"5min":21.681000000000001,"15min":21.681000000000001},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.286,"5min":2.0049999999999999,"15min":1.454},"min":{"1min":0.317,"5min":0.215,"15min":0.215},"max":{"1min":7.2720000000000002,"5min":20.995000000000001,"15min":20.995000000000001},"last":0.77000000000000002},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.6120000000000001,"15min":1.244},"min":{"1min":0.25900000000000001,"5min":0.193,"15min":0.193},"max":{"1min":7.5720000000000001,"5min":15.945,"15min":15.945},"last":0.50800000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.98899999999999999,"15min":0.85099999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.238,"5min":12.009,"15min":12.009},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.702,"5min":0.996,"15min":0.73899999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.162000000000001,"5min":11.162000000000001,"15min":11.162000000000001},"last":0.60199999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0880000000000001,"5min":1.472,"15min":1.143},"min":{"1min":0.34599999999999997,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":6.0899999999999999,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":0.69399999999999995},{"interface":"front","average":{"1min":1.4039999999999999,"5min":1.655,"15min":1.1739999999999999},"min":{"1min":0.35299999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":7.5609999999999999,"5min":20.920000000000002,"15min":20.920000000000002},"last":0.48699999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":1.661,"15min":1.2490000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":7.3289999999999997,"5min":20.738,"15min":20.738},"last":0.57699999999999996},{"interface":"front","average":{"1min":2.0190000000000001,"5min":1.742,"15min":1.327},"min":{"1min":0.36299999999999999,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":12.103999999999999,"5min":14.359,"15min":15.496},"last":0.67800000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:04:44 2023","interfaces":[{"interface":"back","average":{"1min":1.329,"5min":1.554,"15min":2.0659999999999998},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":7.931,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.907,"5min":1.8360000000000001,"15min":2.3290000000000002},"min":{"1min":0.40500000000000003,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.4649999999999999,"5min":20.893000000000001,"15min":20.893000000000001},"last":0.621}]},{"osd":7,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.3320000000000001,"15min":1.3839999999999999},"min":{"1min":0.42099999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":7.4989999999999997,"5min":12.051,"15min":12.051},"last":0.71199999999999997},{"interface":"front","average":{"1min":1.766,"5min":1.0009999999999999,"15min":0.755},"min":{"1min":0.45300000000000001,"5min":0.309,"15min":0.309},"max":{"1min":8.5039999999999996,"5min":8.5039999999999996,"15min":8.5039999999999996},"last":0.72799999999999998}]}]},{"osd":3,"up_from":237,"seq":1017907249194,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":268980,"kb_used_data":92012,"kb_used_omap":3,"kb_used_meta":176956,"kb_avail":93476172,"statfs":{"total":95995035648,"available":95719600128,"internally_reserved":0,"allocated":94220288,"data_stored":92650819,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":181203688},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":0.91000000000000003,"15min":0.89200000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.5440000000000005,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.371},{"interface":"front","average":{"1min":2.0179999999999998,"5min":1.127,"15min":1.097},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.885999999999999,"5min":14.885999999999999,"15min":14.885999999999999},"last":0.442}]},{"osd":1,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":1.2130000000000001,"15min":1.111},"min":{"1min":0.29799999999999999,"5min":0.245,"15min":0.245},"max":{"1min":21.931999999999999,"5min":21.931999999999999,"15min":21.931999999999999},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.625,"5min":1.01,"15min":0.96699999999999997},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":12.701000000000001,"5min":12.701000000000001,"15min":12.701000000000001},"last":0.33900000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":0.94899999999999995,"15min":0.88300000000000001},"min":{"1min":0.32500000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2200000000000006,"5min":8.2200000000000006,"15min":8.2200000000000006},"last":0.42099999999999999},{"interface":"front","average":{"1min":2.46,"5min":1.3280000000000001,"15min":1.2729999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":21.988,"5min":21.988,"15min":21.988},"last":0.47399999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.2509999999999999,"15min":1.153},"min":{"1min":0.375,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":14.935,"5min":14.935,"15min":14.935},"last":0.77800000000000002},{"interface":"front","average":{"1min":2.6600000000000001,"5min":1.3129999999999999,"15min":1.149},"min":{"1min":0.36899999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.786,"5min":12.786,"15min":12.786},"last":0.63300000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":1.3879999999999999,"15min":1.3180000000000001},"min":{"1min":0.48399999999999999,"5min":0.314,"15min":0.314},"max":{"1min":12.743,"5min":12.743,"15min":12.743},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.827,"5min":1.2629999999999999,"15min":1.208},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.6479999999999997,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.71199999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.125,"15min":1.0980000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.5789999999999997,"5min":7.5789999999999997,"15min":7.5789999999999997},"last":0.57399999999999995},{"interface":"front","average":{"1min":2.5,"5min":1.5720000000000001,"15min":1.51},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":0.73099999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":1.625,"15min":1.589},"min":{"1min":0.42599999999999999,"5min":0.377,"15min":0.377},"max":{"1min":12.821,"5min":12.821,"15min":12.821},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.5780000000000001,"15min":1.6060000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":8.6039999999999992,"5min":8.6039999999999992,"15min":8.6039999999999992},"last":0.75700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019144,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":248760,"kb_used_data":3056,"kb_used_omap":0,"kb_used_meta":245696,"kb_avail":93496392,"statfs":{"total":95995035648,"available":95740305408,"internally_reserved":0,"allocated":3129344,"data_stored":2072475,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":251592704},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.75800000000000001,"15min":0.88700000000000001},"min":{"1min":0.26000000000000001,"5min":0.215,"15min":0.20100000000000001},"max":{"1min":2.2850000000000001,"5min":6.8849999999999998,"15min":14.164},"last":0.65100000000000002},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.0640000000000001,"15min":1.1220000000000001},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":5.7750000000000004,"5min":7.1879999999999997,"15min":11.372},"last":0.443}]},{"osd":1,"last update":"Sat Sep 23 18:05:09 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.204,"15min":1.0880000000000001},"min":{"1min":0.29099999999999998,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":5.9740000000000002,"5min":7.2640000000000002,"15min":10.429},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.2430000000000001,"15min":1.1439999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":6.7640000000000002,"5min":6.7640000000000002,"15min":10.83},"last":0.73799999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.579,"5min":1.319,"15min":1.1479999999999999},"min":{"1min":0.29899999999999999,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":6.8259999999999996,"5min":9.6739999999999995,"15min":11.630000000000001},"last":0.628},{"interface":"front","average":{"1min":1.3580000000000001,"5min":1.244,"15min":1.0680000000000001},"min":{"1min":0.32100000000000001,"5min":0.316,"15min":0.27100000000000002},"max":{"1min":6.9299999999999997,"5min":7.4459999999999997,"15min":7.5090000000000003},"last":0.70799999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:05:19 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.3620000000000001,"15min":1.3540000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.6750000000000007,"5min":8.6750000000000007,"15min":8.6750000000000007},"last":0.68000000000000005},{"interface":"front","average":{"1min":1.8140000000000001,"5min":1.423,"15min":1.389},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.8170000000000002,"5min":6.165,"15min":6.165},"last":0.53200000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.359,"5min":0.99099999999999999,"15min":0.69399999999999995},"min":{"1min":0.27800000000000002,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":3.786,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.54200000000000004},{"interface":"front","average":{"1min":2.194,"5min":1.377,"15min":0.88500000000000001},"min":{"1min":0.29799999999999999,"5min":0.26800000000000002,"15min":0.26100000000000001},"max":{"1min":9.1999999999999993,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.378}]},{"osd":6,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":2.0409999999999999,"5min":1.2030000000000001,"15min":0.91400000000000003},"min":{"1min":0.34699999999999998,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":9.0990000000000002,"5min":9.0990000000000002,"15min":9.0990000000000002},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.667,"5min":1.2270000000000001,"15min":1.0089999999999999},"min":{"1min":0.34799999999999998,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":9.1300000000000008,"5min":10.725,"15min":10.725},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:05:30 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":1.284,"15min":1.0369999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":13.298999999999999,"5min":13.298999999999999,"15min":13.298999999999999},"last":0.60699999999999998},{"interface":"front","average":{"1min":1.7470000000000001,"5min":1.484,"15min":1.3029999999999999},"min":{"1min":0.379,"5min":0.307,"15min":0.307},"max":{"1min":10.192,"5min":10.192,"15min":10.192},"last":0.502}]}]},{"osd":5,"up_from":38,"seq":163208757510,"num_pgs":38,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":221164,"kb_used_data":3300,"kb_used_omap":0,"kb_used_meta":217856,"kb_avail":93523988,"statfs":{"total":95995035648,"available":95768563712,"internally_reserved":0,"allocated":3379200,"data_stored":2330603,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":223084544},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":0.95299999999999996,"5min":1.3360000000000001,"15min":1.0269999999999999},"min":{"1min":0.29499999999999998,"5min":0.217,"15min":0.217},"max":{"1min":3.2360000000000002,"5min":21.376999999999999,"15min":21.376999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":1.4530000000000001,"5min":1.7410000000000001,"15min":1.2370000000000001},"min":{"1min":0.28399999999999997,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":8.5099999999999998,"5min":21.388999999999999,"15min":21.388999999999999},"last":0.372}]},{"osd":1,"last update":"Sat Sep 23 18:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.80800000000000005,"15min":0.875},"min":{"1min":0.34999999999999998,"5min":0.26000000000000001,"15min":0.23899999999999999},"max":{"1min":4.6150000000000002,"5min":4.8129999999999997,"15min":10.853999999999999},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.1519999999999999,"15min":1.0569999999999999},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":8.5619999999999994,"5min":8.5619999999999994,"15min":10.206},"last":0.32600000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:04:43 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":1.851,"15min":1.319},"min":{"1min":0.28999999999999998,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":5.5309999999999997,"5min":26.035,"15min":38.606999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.736,"5min":1.407,"15min":1.2170000000000001},"min":{"1min":0.35899999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":10.065,"5min":17.085000000000001,"15min":39.670999999999999},"last":0.36099999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:05:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.0289999999999999,"15min":0.85399999999999998},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.593,"5min":8.593,"15min":8.593},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.224,"5min":0.85299999999999998,"15min":0.75900000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.0439999999999996,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":0.47499999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.325,"5min":1.0629999999999999,"15min":0.78800000000000003},"min":{"1min":0.26900000000000002,"5min":0.23300000000000001,"15min":0.224},"max":{"1min":8.6229999999999993,"5min":8.6229999999999993,"15min":10.092000000000001},"last":0.28699999999999998},{"interface":"front","average":{"1min":1.5229999999999999,"5min":1.0549999999999999,"15min":0.77200000000000002},"min":{"1min":0.29499999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.6379999999999999,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.40500000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:04:47 2023","interfaces":[{"interface":"back","average":{"1min":1.381,"5min":1.085,"15min":1.421},"min":{"1min":0.35299999999999998,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":4.673,"5min":6.5880000000000001,"15min":11.287000000000001},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.2629999999999999,"5min":0.93999999999999995,"15min":1.26},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":5.7939999999999996,"5min":5.7939999999999996,"15min":8.266},"last":0.46200000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:05:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.298,"15min":1.4630000000000001},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.0999999999999996,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.439},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.218,"15min":1.1559999999999999},"min":{"1min":0.42999999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":17.684000000000001,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.44400000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:05:46.806 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:05:46.829 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:05:46.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:46.832 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:05:46.850 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:46.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:46.961 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:46.995 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:46.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:46.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:47.099 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:05:47.125 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:47.157 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:05:47.159 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:47 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:05:47.159 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:47 smithi099 ceph-mon[167076]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.159 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:47 smithi099 ceph-mon[167076]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.160 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:47 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:05:47.160 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:47 smithi099 ceph-mon[167076]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T18:05:47.160 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:47 smithi099 ceph-mon[167076]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.160 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:47 smithi099 ceph-mon[167076]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.160 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:47 smithi099 ceph-mon[167076]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.160 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:47 smithi099 ceph-mon[167076]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.160 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:47 smithi099 ceph-mon[167076]: from='client.45530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:05:47.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:47.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:05:47.241 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:05:47.262 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:05:47.289 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:47.399 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:05:47.399 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[184174]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.399 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[184174]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:05:47.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[184174]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T18:05:47.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[184174]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[184174]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[184174]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[184174]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.400 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[184174]: from='client.45530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:05:47.400 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:05:47.400 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[179668]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.400 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[179668]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:05:47.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[179668]: osdmap e280: 8 total, 8 up, 7 in 2023-09-23T18:05:47.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[179668]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[179668]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[179668]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[179668]: failed to encode map e280 with expected crc 2023-09-23T18:05:47.401 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:47 smithi055 ceph-mon[179668]: from='client.45530 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:05:47.440 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:05:47.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:47.444 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:05:47.544 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:47.572 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:47.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:47.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:47.607 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:47.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:47.735 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:47.807 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:47.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:47.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:47.874 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:47.946 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:05:47.946 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:05:47.975 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:05:48.015 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:48.203 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:05:48.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:48.207 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:05:48.214 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:48.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[179668]: pgmap v694: 119 pgs: 3 creating+peering, 10 unknown, 106 active+clean; 4.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:05:48.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[179668]: failed to encode map e281 with expected crc 2023-09-23T18:05:48.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[179668]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T18:05:48.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[179668]: failed to encode map e281 with expected crc 2023-09-23T18:05:48.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[179668]: failed to encode map e281 with expected crc 2023-09-23T18:05:48.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:05:48.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[184174]: pgmap v694: 119 pgs: 3 creating+peering, 10 unknown, 106 active+clean; 4.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:05:48.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[184174]: failed to encode map e281 with expected crc 2023-09-23T18:05:48.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[184174]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T18:05:48.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[184174]: failed to encode map e281 with expected crc 2023-09-23T18:05:48.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[184174]: failed to encode map e281 with expected crc 2023-09-23T18:05:48.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:48 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:05:48.307 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:05:48.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:48 smithi099 ceph-mon[167076]: pgmap v694: 119 pgs: 3 creating+peering, 10 unknown, 106 active+clean; 4.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:05:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:48 smithi099 ceph-mon[167076]: failed to encode map e281 with expected crc 2023-09-23T18:05:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:48 smithi099 ceph-mon[167076]: osdmap e281: 8 total, 8 up, 7 in 2023-09-23T18:05:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:48 smithi099 ceph-mon[167076]: failed to encode map e281 with expected crc 2023-09-23T18:05:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:48 smithi099 ceph-mon[167076]: failed to encode map e281 with expected crc 2023-09-23T18:05:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:48 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:05:48.339 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:48.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:05:48.375 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:05:48.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:48.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:05:48.479 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:48.505 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:48.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:48.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:48.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:48.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:48.680 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:48.703 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:05:48.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:48.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:48.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:48.820 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:05:48.850 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:49.014 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:05:49.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:49.017 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:05:49.033 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:05:49.118 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:05:49.152 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:49.165 INFO:teuthology.orchestra.run.smithi055.stderr:enabled application 'rados' on pool 'unique_pool_2' 2023-09-23T18:05:49.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3976951388' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:05:49.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:49 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:05:49.292 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3976951388' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:05:49.292 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:49 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:05:49.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3976951388' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:05:49.378 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:49 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:05:49.560 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:05:49.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:49.563 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:05:49.664 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:05:49.695 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:49.789 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:05:49.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:49.792 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:05:49.829 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:05:49.829 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":839,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":282,"num_osds":8,"num_up_osds":8,"osd_up_since":1695492137,"num_in_osds":7,"osd_in_since":1695492120,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"unknown","count":6},{"state_name":"creating+peering","count":3}],"num_pgs":119,"num_pools":5,"num_objects":27,"data_bytes":4620240,"bytes_used":2451271680,"bytes_avail":669513977856,"bytes_total":671965249536,"unknown_pgs_ratio":0.050420168787240982,"inactive_pgs_ratio":0.025210084393620491},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":24,"modified":"2023-09-23T18:05:08.730605+0000","services":{}},"progress_events":{"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true},"9cc9c1ad-c8db-41a9-831f-1f2d0b6579b5":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true}}} 2023-09-23T18:05:49.841 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:49.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:05:50.434 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:50 smithi099 ceph-mon[167076]: pgmap v696: 119 pgs: 3 creating+peering, 6 unknown, 110 active+clean; 4.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:05:50.434 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:50 smithi099 ceph-mon[167076]: failed to encode map e282 with expected crc 2023-09-23T18:05:50.434 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:50 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:05:50.434 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:50 smithi099 ceph-mon[167076]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T18:05:50.434 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:50 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/452178682' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:05:50.434 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:50 smithi099 ceph-mon[167076]: failed to encode map e282 with expected crc 2023-09-23T18:05:50.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[179668]: pgmap v696: 119 pgs: 3 creating+peering, 6 unknown, 110 active+clean; 4.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:05:50.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[179668]: failed to encode map e282 with expected crc 2023-09-23T18:05:50.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:05:50.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[179668]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T18:05:50.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/452178682' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:05:50.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[179668]: failed to encode map e282 with expected crc 2023-09-23T18:05:50.436 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:05:50.436 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:05:50.436 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:05:50.436 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:05:50.436 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:05:50.437 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:05:50.437 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:05:50.437 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:05:50.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:05:50.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:05:50.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:05:50.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:05:50.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:05:50.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:05:50.438 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:05:50.438 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:05:50.438 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:05:50] "GET /metrics HTTP/1.1" 200 34156 "" "Prometheus/2.43.0" 2023-09-23T18:05:50.438 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[184174]: pgmap v696: 119 pgs: 3 creating+peering, 6 unknown, 110 active+clean; 4.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:05:50.438 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[184174]: failed to encode map e282 with expected crc 2023-09-23T18:05:50.438 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_2", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:05:50.438 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[184174]: osdmap e282: 8 total, 8 up, 7 in 2023-09-23T18:05:50.439 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/452178682' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:05:50.439 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:50 smithi055 ceph-mon[184174]: failed to encode map e282 with expected crc 2023-09-23T18:05:50.470 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:50.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:05:50.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:50.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:05:50.658 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:05:50.658 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:05:50.658 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:05:50.658 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:05:50.658 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:05:50.711 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:05:50.745 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:50.771 DEBUG:teuthology.orchestra.run.smithi055:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_2 bench 90 write' 2023-09-23T18:05:50.773 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T18:05:50.773 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T18:05:50.801 INFO:tasks.cephadm.ceph_manager.ceph:no progress seen, keeping timeout for now 2023-09-23T18:05:50.801 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:05:50.825 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:hints = 1 2023-09-23T18:05:50.825 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T18:05:50.825 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object prefix: benchmark_data_smithi055_311528 2023-09-23T18:05:50.826 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:05:50.826 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 0 0 0 0 0 0 - 0 2023-09-23T18:05:51.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:05:51.153 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:05:51.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:51.157 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:05:51.258 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:05:51.307 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:51 smithi099 ceph-mon[167076]: failed to encode map e282 with expected crc 2023-09-23T18:05:51.308 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/183232105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:51.314 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:51.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:05:51.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:51.380 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:05:51.413 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:51 smithi055 ceph-mon[179668]: failed to encode map e282 with expected crc 2023-09-23T18:05:51.413 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/183232105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:51.413 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:51 smithi055 ceph-mon[184174]: failed to encode map e282 with expected crc 2023-09-23T18:05:51.413 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/183232105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:51.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:05:51.533 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:51.593 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:05:51.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:51.598 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:05:51.699 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:05:51.774 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:51.777 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:51.826 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 1 16 2599 2583 161.351 161.438 0.0145972 0.0061512 2023-09-23T18:05:51.861 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:05:51.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:51.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:05:51.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:05:52.011 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:52.206 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:05:52.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:52 smithi099 ceph-mon[167076]: pgmap v698: 119 pgs: 119 active+clean; 4.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:05:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3432227487' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:05:52.374 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:52 smithi055 ceph-mon[184174]: pgmap v698: 119 pgs: 119 active+clean; 4.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:05:52.374 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:52.374 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3432227487' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:05:52.374 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:52 smithi055 ceph-mon[179668]: pgmap v698: 119 pgs: 119 active+clean; 4.4 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail 2023-09-23T18:05:52.375 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:05:52.375 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3432227487' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:05:52.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:05:52.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:52.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:05:52.562 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:05:52.570 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:52.640 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:52.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:52.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:52.804 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:52.826 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 2 16 5120 5104 159.45 157.562 0.0026324 0.00624749 2023-09-23T18:05:52.844 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:05:52.904 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:05:52.993 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:53.388 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:05:53.399 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:53.399 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:05:53.500 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:05:53.558 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:53 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:05:53.558 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:53 smithi055 ceph-mon[184174]: failed to encode map e283 with expected crc 2023-09-23T18:05:53.558 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3432227487' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:05:53.558 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:53 smithi055 ceph-mon[184174]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T18:05:53.559 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:53 smithi055 ceph-mon[184174]: failed to encode map e283 with expected crc 2023-09-23T18:05:53.559 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:53 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:05:53.559 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:53 smithi055 ceph-mon[179668]: failed to encode map e283 with expected crc 2023-09-23T18:05:53.559 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3432227487' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:05:53.559 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:53 smithi055 ceph-mon[179668]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T18:05:53.559 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:53 smithi055 ceph-mon[179668]: failed to encode map e283 with expected crc 2023-09-23T18:05:53.566 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:53 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:05:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:53 smithi099 ceph-mon[167076]: failed to encode map e283 with expected crc 2023-09-23T18:05:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3432227487' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:05:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:53 smithi099 ceph-mon[167076]: osdmap e283: 8 total, 8 up, 7 in 2023-09-23T18:05:53.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:53 smithi099 ceph-mon[167076]: failed to encode map e283 with expected crc 2023-09-23T18:05:53.629 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:05:53.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:53.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:05:53.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:53.826 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 3 16 7813 7797 162.399 168.312 0.00701557 0.00614354 2023-09-23T18:05:53.834 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:54.132 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:05:54.134 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:05:54.142 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:54.386 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:54.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:54.390 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:54 smithi055 ceph-mon[179668]: pgmap v700: 119 pgs: 119 active+clean; 55 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 8.5 MiB/s wr, 135 op/s 2023-09-23T18:05:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:54 smithi055 ceph-mon[179668]: 1.0 scrub starts 2023-09-23T18:05:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:54 smithi055 ceph-mon[179668]: 1.0 scrub ok 2023-09-23T18:05:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:54 smithi055 ceph-mon[179668]: 1.a scrub starts 2023-09-23T18:05:54.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:54 smithi055 ceph-mon[179668]: 1.a scrub ok 2023-09-23T18:05:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:54 smithi055 ceph-mon[184174]: pgmap v700: 119 pgs: 119 active+clean; 55 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 8.5 MiB/s wr, 135 op/s 2023-09-23T18:05:54.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:54 smithi055 ceph-mon[184174]: 1.0 scrub starts 2023-09-23T18:05:54.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:54 smithi055 ceph-mon[184174]: 1.0 scrub ok 2023-09-23T18:05:54.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:54 smithi055 ceph-mon[184174]: 1.a scrub starts 2023-09-23T18:05:54.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:54 smithi055 ceph-mon[184174]: 1.a scrub ok 2023-09-23T18:05:54.485 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":700,"stamp":"2023-09-23T18:05:52.744410+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":57901008,"num_objects":840,"num_object_clones":0,"num_object_copies":2520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":840,"num_whiteouts":0,"num_read":6297,"num_read_kb":12796,"num_write":1889,"num_write_kb":66476,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":3727821,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6835,"ondisk_log_size":6835,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":358,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":2414064,"kb_used_data":705400,"kb_used_omap":43,"kb_used_meta":1708564,"kb_avail":653802000,"statfs":{"total":671965249536,"available":669493248000,"internally_reserved":0,"allocated":722329600,"data_stored":713740104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1749570255},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":53280768,"num_objects":813,"num_object_clones":0,"num_object_copies":2439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":813,"num_write_kb":52032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001793"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":610,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885794+0000","last_change":"2023-09-23T18:05:15.087182+0000","last_active":"2023-09-23T18:05:49.885794+0000","last_peered":"2023-09-23T18:05:49.885794+0000","last_clean":"2023-09-23T18:05:49.885794+0000","last_became_active":"2023-09-23T18:05:15.086678+0000","last_became_peered":"2023-09-23T18:05:15.086678+0000","last_unstale":"2023-09-23T18:05:49.885794+0000","last_undegraded":"2023-09-23T18:05:49.885794+0000","last_fullsized":"2023-09-23T18:05:49.885794+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1262601,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":81,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.122531+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:05:46.122531+0000","last_peered":"2023-09-23T18:05:46.122531+0000","last_clean":"2023-09-23T18:05:46.122531+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:05:46.122531+0000","last_undegraded":"2023-09-23T18:05:46.122531+0000","last_fullsized":"2023-09-23T18:05:46.122531+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":164,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775553+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:05:47.775553+0000","last_peered":"2023-09-23T18:05:47.775553+0000","last_clean":"2023-09-23T18:05:47.775553+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:05:47.775553+0000","last_undegraded":"2023-09-23T18:05:47.775553+0000","last_fullsized":"2023-09-23T18:05:47.775553+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":608,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886187+0000","last_change":"2023-09-23T18:05:15.096602+0000","last_active":"2023-09-23T18:05:49.886187+0000","last_peered":"2023-09-23T18:05:49.886187+0000","last_clean":"2023-09-23T18:05:49.886187+0000","last_became_active":"2023-09-23T18:05:15.096205+0000","last_became_peered":"2023-09-23T18:05:15.096205+0000","last_unstale":"2023-09-23T18:05:49.886187+0000","last_undegraded":"2023-09-23T18:05:49.886187+0000","last_fullsized":"2023-09-23T18:05:49.886187+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:31:18.329681+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":132,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775955+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:05:47.775955+0000","last_peered":"2023-09-23T18:05:47.775955+0000","last_clean":"2023-09-23T18:05:47.775955+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:05:47.775955+0000","last_undegraded":"2023-09-23T18:05:47.775955+0000","last_fullsized":"2023-09-23T18:05:47.775955+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":282,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118529+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:05:46.118529+0000","last_peered":"2023-09-23T18:05:46.118529+0000","last_clean":"2023-09-23T18:05:46.118529+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:05:46.118529+0000","last_undegraded":"2023-09-23T18:05:46.118529+0000","last_fullsized":"2023-09-23T18:05:46.118529+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":598,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.122647+0000","last_change":"2023-09-23T18:05:15.096807+0000","last_active":"2023-09-23T18:05:46.122647+0000","last_peered":"2023-09-23T18:05:46.122647+0000","last_clean":"2023-09-23T18:05:46.122647+0000","last_became_active":"2023-09-23T18:05:15.096022+0000","last_became_peered":"2023-09-23T18:05:15.096022+0000","last_unstale":"2023-09-23T18:05:46.122647+0000","last_undegraded":"2023-09-23T18:05:46.122647+0000","last_fullsized":"2023-09-23T18:05:46.122647+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:32.387626+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":145,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775750+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:05:47.775750+0000","last_peered":"2023-09-23T18:05:47.775750+0000","last_clean":"2023-09-23T18:05:47.775750+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:05:47.775750+0000","last_undegraded":"2023-09-23T18:05:47.775750+0000","last_fullsized":"2023-09-23T18:05:47.775750+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":222,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.150029+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:05:47.150029+0000","last_peered":"2023-09-23T18:05:47.150029+0000","last_clean":"2023-09-23T18:05:47.150029+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:05:47.150029+0000","last_undegraded":"2023-09-23T18:05:47.150029+0000","last_fullsized":"2023-09-23T18:05:47.150029+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":631,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.119757+0000","last_change":"2023-09-23T18:05:15.086647+0000","last_active":"2023-09-23T18:05:46.119757+0000","last_peered":"2023-09-23T18:05:46.119757+0000","last_clean":"2023-09-23T18:05:46.119757+0000","last_became_active":"2023-09-23T18:05:15.086214+0000","last_became_peered":"2023-09-23T18:05:15.086214+0000","last_unstale":"2023-09-23T18:05:46.119757+0000","last_undegraded":"2023-09-23T18:05:46.119757+0000","last_fullsized":"2023-09-23T18:05:46.119757+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":261,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118733+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:05:46.118733+0000","last_peered":"2023-09-23T18:05:46.118733+0000","last_clean":"2023-09-23T18:05:46.118733+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:05:46.118733+0000","last_undegraded":"2023-09-23T18:05:46.118733+0000","last_fullsized":"2023-09-23T18:05:46.118733+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":139,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775835+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:05:47.775835+0000","last_peered":"2023-09-23T18:05:47.775835+0000","last_clean":"2023-09-23T18:05:47.775835+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:05:47.775835+0000","last_undegraded":"2023-09-23T18:05:47.775835+0000","last_fullsized":"2023-09-23T18:05:47.775835+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":590,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.776232+0000","last_change":"2023-09-23T18:05:15.088596+0000","last_active":"2023-09-23T18:05:47.776232+0000","last_peered":"2023-09-23T18:05:47.776232+0000","last_clean":"2023-09-23T18:05:47.776232+0000","last_became_active":"2023-09-23T18:05:15.088205+0000","last_became_peered":"2023-09-23T18:05:15.088205+0000","last_unstale":"2023-09-23T18:05:47.776232+0000","last_undegraded":"2023-09-23T18:05:47.776232+0000","last_fullsized":"2023-09-23T18:05:47.776232+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":265,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886263+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:05:49.886263+0000","last_peered":"2023-09-23T18:05:49.886263+0000","last_clean":"2023-09-23T18:05:49.886263+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:05:49.886263+0000","last_undegraded":"2023-09-23T18:05:49.886263+0000","last_fullsized":"2023-09-23T18:05:49.886263+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":93,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775880+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:05:47.775880+0000","last_peered":"2023-09-23T18:05:47.775880+0000","last_clean":"2023-09-23T18:05:47.775880+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:05:47.775880+0000","last_undegraded":"2023-09-23T18:05:47.775880+0000","last_fullsized":"2023-09-23T18:05:47.775880+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":597,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886503+0000","last_change":"2023-09-23T18:05:15.100844+0000","last_active":"2023-09-23T18:05:49.886503+0000","last_peered":"2023-09-23T18:05:49.886503+0000","last_clean":"2023-09-23T18:05:49.886503+0000","last_became_active":"2023-09-23T18:05:15.100365+0000","last_became_peered":"2023-09-23T18:05:15.100365+0000","last_unstale":"2023-09-23T18:05:49.886503+0000","last_undegraded":"2023-09-23T18:05:49.886503+0000","last_fullsized":"2023-09-23T18:05:49.886503+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":46,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.119745+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:05:46.119745+0000","last_peered":"2023-09-23T18:05:46.119745+0000","last_clean":"2023-09-23T18:05:46.119745+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:05:46.119745+0000","last_undegraded":"2023-09-23T18:05:46.119745+0000","last_fullsized":"2023-09-23T18:05:46.119745+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":178,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.149938+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:05:47.149938+0000","last_peered":"2023-09-23T18:05:47.149938+0000","last_clean":"2023-09-23T18:05:47.149938+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:05:47.149938+0000","last_undegraded":"2023-09-23T18:05:47.149938+0000","last_fullsized":"2023-09-23T18:05:47.149938+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":602,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118391+0000","last_change":"2023-09-23T18:05:15.096980+0000","last_active":"2023-09-23T18:05:46.118391+0000","last_peered":"2023-09-23T18:05:46.118391+0000","last_clean":"2023-09-23T18:05:46.118391+0000","last_became_active":"2023-09-23T18:05:15.096456+0000","last_became_peered":"2023-09-23T18:05:15.096456+0000","last_unstale":"2023-09-23T18:05:46.118391+0000","last_undegraded":"2023-09-23T18:05:46.118391+0000","last_fullsized":"2023-09-23T18:05:46.118391+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":81,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121520+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:05:46.121520+0000","last_peered":"2023-09-23T18:05:46.121520+0000","last_clean":"2023-09-23T18:05:46.121520+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:05:46.121520+0000","last_undegraded":"2023-09-23T18:05:46.121520+0000","last_fullsized":"2023-09-23T18:05:46.121520+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":81,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.123111+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:05:46.123111+0000","last_peered":"2023-09-23T18:05:46.123111+0000","last_clean":"2023-09-23T18:05:46.123111+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:05:46.123111+0000","last_undegraded":"2023-09-23T18:05:46.123111+0000","last_fullsized":"2023-09-23T18:05:46.123111+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":592,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.119429+0000","last_change":"2023-09-23T18:05:15.095829+0000","last_active":"2023-09-23T18:05:46.119429+0000","last_peered":"2023-09-23T18:05:46.119429+0000","last_clean":"2023-09-23T18:05:46.119429+0000","last_became_active":"2023-09-23T18:05:15.095414+0000","last_became_peered":"2023-09-23T18:05:15.095414+0000","last_unstale":"2023-09-23T18:05:46.119429+0000","last_undegraded":"2023-09-23T18:05:46.119429+0000","last_fullsized":"2023-09-23T18:05:46.119429+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":209,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.119859+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:05:46.119859+0000","last_peered":"2023-09-23T18:05:46.119859+0000","last_clean":"2023-09-23T18:05:46.119859+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:05:46.119859+0000","last_undegraded":"2023-09-23T18:05:46.119859+0000","last_fullsized":"2023-09-23T18:05:46.119859+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":127,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.777381+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:05:47.777381+0000","last_peered":"2023-09-23T18:05:47.777381+0000","last_clean":"2023-09-23T18:05:47.777381+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:05:47.777381+0000","last_undegraded":"2023-09-23T18:05:47.777381+0000","last_fullsized":"2023-09-23T18:05:47.777381+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":600,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885883+0000","last_change":"2023-09-23T18:05:15.091218+0000","last_active":"2023-09-23T18:05:49.885883+0000","last_peered":"2023-09-23T18:05:49.885883+0000","last_clean":"2023-09-23T18:05:49.885883+0000","last_became_active":"2023-09-23T18:05:15.090420+0000","last_became_peered":"2023-09-23T18:05:15.090420+0000","last_unstale":"2023-09-23T18:05:49.885883+0000","last_undegraded":"2023-09-23T18:05:49.885883+0000","last_fullsized":"2023-09-23T18:05:49.885883+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:57.754408+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":280,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.117449+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:05:46.117449+0000","last_peered":"2023-09-23T18:05:46.117449+0000","last_clean":"2023-09-23T18:05:46.117449+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:05:46.117449+0000","last_undegraded":"2023-09-23T18:05:46.117449+0000","last_fullsized":"2023-09-23T18:05:46.117449+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":260,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886364+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:05:49.886364+0000","last_peered":"2023-09-23T18:05:49.886364+0000","last_clean":"2023-09-23T18:05:49.886364+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:05:49.886364+0000","last_undegraded":"2023-09-23T18:05:49.886364+0000","last_fullsized":"2023-09-23T18:05:49.886364+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":584,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775548+0000","last_change":"2023-09-23T18:05:15.097196+0000","last_active":"2023-09-23T18:05:47.775548+0000","last_peered":"2023-09-23T18:05:47.775548+0000","last_clean":"2023-09-23T18:05:47.775548+0000","last_became_active":"2023-09-23T18:05:15.096800+0000","last_became_peered":"2023-09-23T18:05:15.096800+0000","last_unstale":"2023-09-23T18:05:47.775548+0000","last_undegraded":"2023-09-23T18:05:47.775548+0000","last_fullsized":"2023-09-23T18:05:47.775548+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":71,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121392+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:05:46.121392+0000","last_peered":"2023-09-23T18:05:46.121392+0000","last_clean":"2023-09-23T18:05:46.121392+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:05:46.121392+0000","last_undegraded":"2023-09-23T18:05:46.121392+0000","last_fullsized":"2023-09-23T18:05:46.121392+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":271,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118613+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:05:46.118613+0000","last_peered":"2023-09-23T18:05:46.118613+0000","last_clean":"2023-09-23T18:05:46.118613+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:05:46.118613+0000","last_undegraded":"2023-09-23T18:05:46.118613+0000","last_fullsized":"2023-09-23T18:05:46.118613+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":600,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885987+0000","last_change":"2023-09-23T18:05:15.090180+0000","last_active":"2023-09-23T18:05:49.885987+0000","last_peered":"2023-09-23T18:05:49.885987+0000","last_clean":"2023-09-23T18:05:49.885987+0000","last_became_active":"2023-09-23T18:05:15.089754+0000","last_became_peered":"2023-09-23T18:05:15.089754+0000","last_unstale":"2023-09-23T18:05:49.885987+0000","last_undegraded":"2023-09-23T18:05:49.885987+0000","last_fullsized":"2023-09-23T18:05:49.885987+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:06.678700+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":218,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.124277+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:05:47.124277+0000","last_peered":"2023-09-23T18:05:47.124277+0000","last_clean":"2023-09-23T18:05:47.124277+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:05:47.124277+0000","last_undegraded":"2023-09-23T18:05:47.124277+0000","last_fullsized":"2023-09-23T18:05:47.124277+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":136,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775662+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:05:47.775662+0000","last_peered":"2023-09-23T18:05:47.775662+0000","last_clean":"2023-09-23T18:05:47.775662+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:05:47.775662+0000","last_undegraded":"2023-09-23T18:05:47.775662+0000","last_fullsized":"2023-09-23T18:05:47.775662+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"283'315","reported_seq":327,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:52.669915+0000","last_change":"2023-09-23T18:05:46.127510+0000","last_active":"2023-09-23T18:05:52.669915+0000","last_peered":"2023-09-23T18:05:52.669915+0000","last_clean":"2023-09-23T18:05:52.669915+0000","last_became_active":"2023-09-23T18:05:46.127237+0000","last_became_peered":"2023-09-23T18:05:46.127237+0000","last_unstale":"2023-09-23T18:05:52.669915+0000","last_undegraded":"2023-09-23T18:05:52.669915+0000","last_fullsized":"2023-09-23T18:05:52.669915+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":315,"log_dups_size":0,"ondisk_log_size":315,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:46.020682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20643840,"num_objects":315,"num_object_clones":0,"num_object_copies":945,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":315,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":315,"num_write_kb":20160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":631,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.119683+0000","last_change":"2023-09-23T18:05:15.096714+0000","last_active":"2023-09-23T18:05:46.119683+0000","last_peered":"2023-09-23T18:05:46.119683+0000","last_clean":"2023-09-23T18:05:46.119683+0000","last_became_active":"2023-09-23T18:05:15.096260+0000","last_became_peered":"2023-09-23T18:05:15.096260+0000","last_unstale":"2023-09-23T18:05:46.119683+0000","last_undegraded":"2023-09-23T18:05:46.119683+0000","last_fullsized":"2023-09-23T18:05:46.119683+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":209,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121442+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:05:46.121442+0000","last_peered":"2023-09-23T18:05:46.121442+0000","last_clean":"2023-09-23T18:05:46.121442+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:05:46.121442+0000","last_undegraded":"2023-09-23T18:05:46.121442+0000","last_fullsized":"2023-09-23T18:05:46.121442+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":271,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885990+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:05:49.885990+0000","last_peered":"2023-09-23T18:05:49.885990+0000","last_clean":"2023-09-23T18:05:49.885990+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:05:49.885990+0000","last_undegraded":"2023-09-23T18:05:49.885990+0000","last_fullsized":"2023-09-23T18:05:49.885990+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"282'62","reported_seq":74,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:51.201424+0000","last_change":"2023-09-23T18:05:46.126616+0000","last_active":"2023-09-23T18:05:51.201424+0000","last_peered":"2023-09-23T18:05:51.201424+0000","last_clean":"2023-09-23T18:05:51.201424+0000","last_became_active":"2023-09-23T18:05:46.126346+0000","last_became_peered":"2023-09-23T18:05:46.126346+0000","last_unstale":"2023-09-23T18:05:51.201424+0000","last_undegraded":"2023-09-23T18:05:51.201424+0000","last_fullsized":"2023-09-23T18:05:51.201424+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":62,"log_dups_size":0,"ondisk_log_size":62,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:57.668631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4063232,"num_objects":62,"num_object_clones":0,"num_object_copies":186,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":62,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62,"num_write_kb":3968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":620,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.777243+0000","last_change":"2023-09-23T18:05:15.083980+0000","last_active":"2023-09-23T18:05:47.777243+0000","last_peered":"2023-09-23T18:05:47.777243+0000","last_clean":"2023-09-23T18:05:47.777243+0000","last_became_active":"2023-09-23T18:05:15.083516+0000","last_became_peered":"2023-09-23T18:05:15.083516+0000","last_unstale":"2023-09-23T18:05:47.777243+0000","last_undegraded":"2023-09-23T18:05:47.777243+0000","last_fullsized":"2023-09-23T18:05:47.777243+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1064488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":224,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121510+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:05:46.121510+0000","last_peered":"2023-09-23T18:05:46.121510+0000","last_clean":"2023-09-23T18:05:46.121510+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:05:46.121510+0000","last_undegraded":"2023-09-23T18:05:46.121510+0000","last_fullsized":"2023-09-23T18:05:46.121510+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":291,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121635+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:05:46.121635+0000","last_peered":"2023-09-23T18:05:46.121635+0000","last_clean":"2023-09-23T18:05:46.121635+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:05:46.121635+0000","last_undegraded":"2023-09-23T18:05:46.121635+0000","last_fullsized":"2023-09-23T18:05:46.121635+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"0'0","reported_seq":11,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.558676+0000","last_change":"2023-09-23T18:05:46.130561+0000","last_active":"2023-09-23T18:05:47.558676+0000","last_peered":"2023-09-23T18:05:47.558676+0000","last_clean":"2023-09-23T18:05:47.558676+0000","last_became_active":"2023-09-23T18:05:46.130243+0000","last_became_peered":"2023-09-23T18:05:46.130243+0000","last_unstale":"2023-09-23T18:05:47.558676+0000","last_undegraded":"2023-09-23T18:05:47.558676+0000","last_fullsized":"2023-09-23T18:05:47.558676+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:29.290251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":443,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.150226+0000","last_change":"2023-09-23T18:05:15.099144+0000","last_active":"2023-09-23T18:05:47.150226+0000","last_peered":"2023-09-23T18:05:47.150226+0000","last_clean":"2023-09-23T18:05:47.150226+0000","last_became_active":"2023-09-23T18:05:15.098234+0000","last_became_peered":"2023-09-23T18:05:15.098234+0000","last_unstale":"2023-09-23T18:05:47.150226+0000","last_undegraded":"2023-09-23T18:05:47.150226+0000","last_fullsized":"2023-09-23T18:05:47.150226+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":147,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.119844+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:05:46.119844+0000","last_peered":"2023-09-23T18:05:46.119844+0000","last_clean":"2023-09-23T18:05:46.119844+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:05:46.119844+0000","last_undegraded":"2023-09-23T18:05:46.119844+0000","last_fullsized":"2023-09-23T18:05:46.119844+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":247,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.149578+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:05:47.149578+0000","last_peered":"2023-09-23T18:05:47.149578+0000","last_clean":"2023-09-23T18:05:47.149578+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:05:47.149578+0000","last_undegraded":"2023-09-23T18:05:47.149578+0000","last_fullsized":"2023-09-23T18:05:47.149578+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"0'0","reported_seq":12,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.212904+0000","last_change":"2023-09-23T18:05:46.123859+0000","last_active":"2023-09-23T18:05:49.212904+0000","last_peered":"2023-09-23T18:05:49.212904+0000","last_clean":"2023-09-23T18:05:49.212904+0000","last_became_active":"2023-09-23T18:05:46.123482+0000","last_became_peered":"2023-09-23T18:05:46.123482+0000","last_unstale":"2023-09-23T18:05:49.212904+0000","last_undegraded":"2023-09-23T18:05:49.212904+0000","last_fullsized":"2023-09-23T18:05:49.212904+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:50.537709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":595,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121851+0000","last_change":"2023-09-23T18:05:15.102453+0000","last_active":"2023-09-23T18:05:46.121851+0000","last_peered":"2023-09-23T18:05:46.121851+0000","last_clean":"2023-09-23T18:05:46.121851+0000","last_became_active":"2023-09-23T18:05:15.102072+0000","last_became_peered":"2023-09-23T18:05:15.102072+0000","last_unstale":"2023-09-23T18:05:46.121851+0000","last_undegraded":"2023-09-23T18:05:46.121851+0000","last_fullsized":"2023-09-23T18:05:46.121851+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":259,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886106+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:05:49.886106+0000","last_peered":"2023-09-23T18:05:49.886106+0000","last_clean":"2023-09-23T18:05:49.886106+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:05:49.886106+0000","last_undegraded":"2023-09-23T18:05:49.886106+0000","last_fullsized":"2023-09-23T18:05:49.886106+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":283,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118001+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:05:46.118001+0000","last_peered":"2023-09-23T18:05:46.118001+0000","last_clean":"2023-09-23T18:05:46.118001+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:05:46.118001+0000","last_undegraded":"2023-09-23T18:05:46.118001+0000","last_fullsized":"2023-09-23T18:05:46.118001+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"0'0","reported_seq":11,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.559371+0000","last_change":"2023-09-23T18:05:46.128878+0000","last_active":"2023-09-23T18:05:47.559371+0000","last_peered":"2023-09-23T18:05:47.559371+0000","last_clean":"2023-09-23T18:05:47.559371+0000","last_became_active":"2023-09-23T18:05:46.128615+0000","last_became_peered":"2023-09-23T18:05:46.128615+0000","last_unstale":"2023-09-23T18:05:47.559371+0000","last_undegraded":"2023-09-23T18:05:47.559371+0000","last_fullsized":"2023-09-23T18:05:47.559371+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:44.175736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":623,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118478+0000","last_change":"2023-09-23T18:05:15.086621+0000","last_active":"2023-09-23T18:05:46.118478+0000","last_peered":"2023-09-23T18:05:46.118478+0000","last_clean":"2023-09-23T18:05:46.118478+0000","last_became_active":"2023-09-23T18:05:15.086216+0000","last_became_peered":"2023-09-23T18:05:15.086216+0000","last_unstale":"2023-09-23T18:05:46.118478+0000","last_undegraded":"2023-09-23T18:05:46.118478+0000","last_fullsized":"2023-09-23T18:05:46.118478+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":170,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775500+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:05:47.775500+0000","last_peered":"2023-09-23T18:05:47.775500+0000","last_clean":"2023-09-23T18:05:47.775500+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:05:47.775500+0000","last_undegraded":"2023-09-23T18:05:47.775500+0000","last_fullsized":"2023-09-23T18:05:47.775500+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":289,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121613+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:05:46.121613+0000","last_peered":"2023-09-23T18:05:46.121613+0000","last_clean":"2023-09-23T18:05:46.121613+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:05:46.121613+0000","last_undegraded":"2023-09-23T18:05:46.121613+0000","last_fullsized":"2023-09-23T18:05:46.121613+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"0'0","reported_seq":12,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.213364+0000","last_change":"2023-09-23T18:05:46.128982+0000","last_active":"2023-09-23T18:05:49.213364+0000","last_peered":"2023-09-23T18:05:49.213364+0000","last_clean":"2023-09-23T18:05:49.213364+0000","last_became_active":"2023-09-23T18:05:46.128696+0000","last_became_peered":"2023-09-23T18:05:46.128696+0000","last_unstale":"2023-09-23T18:05:49.213364+0000","last_undegraded":"2023-09-23T18:05:49.213364+0000","last_fullsized":"2023-09-23T18:05:49.213364+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:44.112131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":593,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885875+0000","last_change":"2023-09-23T18:05:15.083605+0000","last_active":"2023-09-23T18:05:49.885875+0000","last_peered":"2023-09-23T18:05:49.885875+0000","last_clean":"2023-09-23T18:05:49.885875+0000","last_became_active":"2023-09-23T18:05:15.083184+0000","last_became_peered":"2023-09-23T18:05:15.083184+0000","last_unstale":"2023-09-23T18:05:49.885875+0000","last_undegraded":"2023-09-23T18:05:49.885875+0000","last_fullsized":"2023-09-23T18:05:49.885875+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":241,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.150053+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:05:47.150053+0000","last_peered":"2023-09-23T18:05:47.150053+0000","last_clean":"2023-09-23T18:05:47.150053+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:05:47.150053+0000","last_undegraded":"2023-09-23T18:05:47.150053+0000","last_fullsized":"2023-09-23T18:05:47.150053+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":81,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.122086+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:05:46.122086+0000","last_peered":"2023-09-23T18:05:46.122086+0000","last_clean":"2023-09-23T18:05:46.122086+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:05:46.122086+0000","last_undegraded":"2023-09-23T18:05:46.122086+0000","last_fullsized":"2023-09-23T18:05:46.122086+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"0'0","reported_seq":11,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.149763+0000","last_change":"2023-09-23T18:05:46.129778+0000","last_active":"2023-09-23T18:05:47.149763+0000","last_peered":"2023-09-23T18:05:47.149763+0000","last_clean":"2023-09-23T18:05:47.149763+0000","last_became_active":"2023-09-23T18:05:46.129527+0000","last_became_peered":"2023-09-23T18:05:46.129527+0000","last_unstale":"2023-09-23T18:05:47.149763+0000","last_undegraded":"2023-09-23T18:05:47.149763+0000","last_fullsized":"2023-09-23T18:05:47.149763+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:17.756683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":602,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.117743+0000","last_change":"2023-09-23T18:05:15.082278+0000","last_active":"2023-09-23T18:05:46.117743+0000","last_peered":"2023-09-23T18:05:46.117743+0000","last_clean":"2023-09-23T18:05:46.117743+0000","last_became_active":"2023-09-23T18:05:15.081901+0000","last_became_peered":"2023-09-23T18:05:15.081901+0000","last_unstale":"2023-09-23T18:05:46.117743+0000","last_undegraded":"2023-09-23T18:05:46.117743+0000","last_fullsized":"2023-09-23T18:05:46.117743+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":71,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121466+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:05:46.121466+0000","last_peered":"2023-09-23T18:05:46.121466+0000","last_clean":"2023-09-23T18:05:46.121466+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:05:46.121466+0000","last_undegraded":"2023-09-23T18:05:46.121466+0000","last_fullsized":"2023-09-23T18:05:46.121466+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":291,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121981+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:05:46.121981+0000","last_peered":"2023-09-23T18:05:46.121981+0000","last_clean":"2023-09-23T18:05:46.121981+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:05:46.121981+0000","last_undegraded":"2023-09-23T18:05:46.121981+0000","last_fullsized":"2023-09-23T18:05:46.121981+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"0'0","reported_seq":11,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.150080+0000","last_change":"2023-09-23T18:05:46.123519+0000","last_active":"2023-09-23T18:05:47.150080+0000","last_peered":"2023-09-23T18:05:47.150080+0000","last_clean":"2023-09-23T18:05:47.150080+0000","last_became_active":"2023-09-23T18:05:46.123243+0000","last_became_peered":"2023-09-23T18:05:46.123243+0000","last_unstale":"2023-09-23T18:05:47.150080+0000","last_undegraded":"2023-09-23T18:05:47.150080+0000","last_fullsized":"2023-09-23T18:05:47.150080+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.325296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":589,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886176+0000","last_change":"2023-09-23T18:05:15.089881+0000","last_active":"2023-09-23T18:05:49.886176+0000","last_peered":"2023-09-23T18:05:49.886176+0000","last_clean":"2023-09-23T18:05:49.886176+0000","last_became_active":"2023-09-23T18:05:15.089224+0000","last_became_peered":"2023-09-23T18:05:15.089224+0000","last_unstale":"2023-09-23T18:05:49.886176+0000","last_undegraded":"2023-09-23T18:05:49.886176+0000","last_fullsized":"2023-09-23T18:05:49.886176+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":128,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775997+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:05:47.775997+0000","last_peered":"2023-09-23T18:05:47.775997+0000","last_clean":"2023-09-23T18:05:47.775997+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:05:47.775997+0000","last_undegraded":"2023-09-23T18:05:47.775997+0000","last_fullsized":"2023-09-23T18:05:47.775997+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":128,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775779+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:05:47.775779+0000","last_peered":"2023-09-23T18:05:47.775779+0000","last_clean":"2023-09-23T18:05:47.775779+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:05:47.775779+0000","last_undegraded":"2023-09-23T18:05:47.775779+0000","last_fullsized":"2023-09-23T18:05:47.775779+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"0'0","reported_seq":10,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.125339+0000","last_change":"2023-09-23T18:05:46.124349+0000","last_active":"2023-09-23T18:05:46.125339+0000","last_peered":"2023-09-23T18:05:46.125339+0000","last_clean":"2023-09-23T18:05:46.125339+0000","last_became_active":"2023-09-23T18:05:46.124081+0000","last_became_peered":"2023-09-23T18:05:46.124081+0000","last_unstale":"2023-09-23T18:05:46.125339+0000","last_undegraded":"2023-09-23T18:05:46.125339+0000","last_fullsized":"2023-09-23T18:05:46.125339+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:46.818802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":443,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.149949+0000","last_change":"2023-09-23T18:05:15.099360+0000","last_active":"2023-09-23T18:05:47.149949+0000","last_peered":"2023-09-23T18:05:47.149949+0000","last_clean":"2023-09-23T18:05:47.149949+0000","last_became_active":"2023-09-23T18:05:15.098750+0000","last_became_peered":"2023-09-23T18:05:15.098750+0000","last_unstale":"2023-09-23T18:05:47.149949+0000","last_undegraded":"2023-09-23T18:05:47.149949+0000","last_fullsized":"2023-09-23T18:05:47.149949+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":590,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.119142+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:05:46.119142+0000","last_peered":"2023-09-23T18:05:46.119142+0000","last_clean":"2023-09-23T18:05:46.119142+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:05:46.119142+0000","last_undegraded":"2023-09-23T18:05:46.119142+0000","last_fullsized":"2023-09-23T18:05:46.119142+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":263,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886448+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:05:49.886448+0000","last_peered":"2023-09-23T18:05:49.886448+0000","last_clean":"2023-09-23T18:05:49.886448+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:05:49.886448+0000","last_undegraded":"2023-09-23T18:05:49.886448+0000","last_fullsized":"2023-09-23T18:05:49.886448+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":260,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886133+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:05:49.886133+0000","last_peered":"2023-09-23T18:05:49.886133+0000","last_clean":"2023-09-23T18:05:49.886133+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:05:49.886133+0000","last_undegraded":"2023-09-23T18:05:49.886133+0000","last_fullsized":"2023-09-23T18:05:49.886133+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"0'0","reported_seq":11,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.149804+0000","last_change":"2023-09-23T18:05:46.129518+0000","last_active":"2023-09-23T18:05:47.149804+0000","last_peered":"2023-09-23T18:05:47.149804+0000","last_clean":"2023-09-23T18:05:47.149804+0000","last_became_active":"2023-09-23T18:05:46.129180+0000","last_became_peered":"2023-09-23T18:05:46.129180+0000","last_unstale":"2023-09-23T18:05:47.149804+0000","last_undegraded":"2023-09-23T18:05:47.149804+0000","last_fullsized":"2023-09-23T18:05:47.149804+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:27.845759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":600,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885834+0000","last_change":"2023-09-23T18:05:15.083813+0000","last_active":"2023-09-23T18:05:49.885834+0000","last_peered":"2023-09-23T18:05:49.885834+0000","last_clean":"2023-09-23T18:05:49.885834+0000","last_became_active":"2023-09-23T18:05:15.083395+0000","last_became_peered":"2023-09-23T18:05:15.083395+0000","last_unstale":"2023-09-23T18:05:49.885834+0000","last_undegraded":"2023-09-23T18:05:49.885834+0000","last_fullsized":"2023-09-23T18:05:49.885834+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:24.402252+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1211,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886092+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:05:49.886092+0000","last_peered":"2023-09-23T18:05:49.886092+0000","last_clean":"2023-09-23T18:05:49.886092+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:05:49.886092+0000","last_undegraded":"2023-09-23T18:05:49.886092+0000","last_fullsized":"2023-09-23T18:05:49.886092+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":71,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.123200+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:05:46.123200+0000","last_peered":"2023-09-23T18:05:46.123200+0000","last_clean":"2023-09-23T18:05:46.123200+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:05:46.123200+0000","last_undegraded":"2023-09-23T18:05:46.123200+0000","last_fullsized":"2023-09-23T18:05:46.123200+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":176,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.776428+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:05:47.776428+0000","last_peered":"2023-09-23T18:05:47.776428+0000","last_clean":"2023-09-23T18:05:47.776428+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:05:47.776428+0000","last_undegraded":"2023-09-23T18:05:47.776428+0000","last_fullsized":"2023-09-23T18:05:47.776428+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"0'0","reported_seq":11,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.559094+0000","last_change":"2023-09-23T18:05:46.128848+0000","last_active":"2023-09-23T18:05:47.559094+0000","last_peered":"2023-09-23T18:05:47.559094+0000","last_clean":"2023-09-23T18:05:47.559094+0000","last_became_active":"2023-09-23T18:05:46.128536+0000","last_became_peered":"2023-09-23T18:05:46.128536+0000","last_unstale":"2023-09-23T18:05:47.559094+0000","last_undegraded":"2023-09-23T18:05:47.559094+0000","last_fullsized":"2023-09-23T18:05:47.559094+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:01.485297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":606,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.117538+0000","last_change":"2023-09-23T18:05:15.087882+0000","last_active":"2023-09-23T18:05:46.117538+0000","last_peered":"2023-09-23T18:05:46.117538+0000","last_clean":"2023-09-23T18:05:46.117538+0000","last_became_active":"2023-09-23T18:05:15.087524+0000","last_became_peered":"2023-09-23T18:05:15.087524+0000","last_unstale":"2023-09-23T18:05:46.117538+0000","last_undegraded":"2023-09-23T18:05:46.117538+0000","last_fullsized":"2023-09-23T18:05:46.117538+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:55.015941+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":565,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775211+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:05:47.775211+0000","last_peered":"2023-09-23T18:05:47.775211+0000","last_clean":"2023-09-23T18:05:47.775211+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:05:47.775211+0000","last_undegraded":"2023-09-23T18:05:47.775211+0000","last_fullsized":"2023-09-23T18:05:47.775211+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":177,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.776494+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:05:47.776494+0000","last_peered":"2023-09-23T18:05:47.776494+0000","last_clean":"2023-09-23T18:05:47.776494+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:05:47.776494+0000","last_undegraded":"2023-09-23T18:05:47.776494+0000","last_fullsized":"2023-09-23T18:05:47.776494+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":250,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886440+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:05:49.886440+0000","last_peered":"2023-09-23T18:05:49.886440+0000","last_clean":"2023-09-23T18:05:49.886440+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:05:49.886440+0000","last_undegraded":"2023-09-23T18:05:49.886440+0000","last_fullsized":"2023-09-23T18:05:49.886440+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"0'0","reported_seq":11,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.559421+0000","last_change":"2023-09-23T18:05:46.129744+0000","last_active":"2023-09-23T18:05:47.559421+0000","last_peered":"2023-09-23T18:05:47.559421+0000","last_clean":"2023-09-23T18:05:47.559421+0000","last_became_active":"2023-09-23T18:05:46.129427+0000","last_became_peered":"2023-09-23T18:05:46.129427+0000","last_unstale":"2023-09-23T18:05:47.559421+0000","last_undegraded":"2023-09-23T18:05:47.559421+0000","last_fullsized":"2023-09-23T18:05:47.559421+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:08.394345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1195,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886089+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:05:49.886089+0000","last_peered":"2023-09-23T18:05:49.886089+0000","last_clean":"2023-09-23T18:05:49.886089+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:05:49.886089+0000","last_undegraded":"2023-09-23T18:05:49.886089+0000","last_fullsized":"2023-09-23T18:05:49.886089+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":606,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118495+0000","last_change":"2023-09-23T18:04:39.585751+0000","last_active":"2023-09-23T18:05:46.118495+0000","last_peered":"2023-09-23T18:05:46.118495+0000","last_clean":"2023-09-23T18:05:46.118495+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:05:46.118495+0000","last_undegraded":"2023-09-23T18:05:46.118495+0000","last_fullsized":"2023-09-23T18:05:46.118495+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":90,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.776042+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:05:47.776042+0000","last_peered":"2023-09-23T18:05:47.776042+0000","last_clean":"2023-09-23T18:05:47.776042+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:05:47.776042+0000","last_undegraded":"2023-09-23T18:05:47.776042+0000","last_fullsized":"2023-09-23T18:05:47.776042+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":215,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.150032+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:05:47.150032+0000","last_peered":"2023-09-23T18:05:47.150032+0000","last_clean":"2023-09-23T18:05:47.150032+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:05:47.150032+0000","last_undegraded":"2023-09-23T18:05:47.150032+0000","last_fullsized":"2023-09-23T18:05:47.150032+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"0'0","reported_seq":10,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.131585+0000","last_change":"2023-09-23T18:05:46.131311+0000","last_active":"2023-09-23T18:05:46.131585+0000","last_peered":"2023-09-23T18:05:46.131585+0000","last_clean":"2023-09-23T18:05:46.131585+0000","last_became_active":"2023-09-23T18:05:46.131042+0000","last_became_peered":"2023-09-23T18:05:46.131042+0000","last_unstale":"2023-09-23T18:05:46.131585+0000","last_undegraded":"2023-09-23T18:05:46.131585+0000","last_fullsized":"2023-09-23T18:05:46.131585+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:21.768330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":443,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.149762+0000","last_change":"2023-09-23T18:05:15.096283+0000","last_active":"2023-09-23T18:05:47.149762+0000","last_peered":"2023-09-23T18:05:47.149762+0000","last_clean":"2023-09-23T18:05:47.149762+0000","last_became_active":"2023-09-23T18:05:15.095885+0000","last_became_peered":"2023-09-23T18:05:15.095885+0000","last_unstale":"2023-09-23T18:05:47.149762+0000","last_undegraded":"2023-09-23T18:05:47.149762+0000","last_fullsized":"2023-09-23T18:05:47.149762+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":460,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118774+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:05:46.118774+0000","last_peered":"2023-09-23T18:05:46.118774+0000","last_clean":"2023-09-23T18:05:46.118774+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:05:46.118774+0000","last_undegraded":"2023-09-23T18:05:46.118774+0000","last_fullsized":"2023-09-23T18:05:46.118774+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":280,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.119555+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:05:46.119555+0000","last_peered":"2023-09-23T18:05:46.119555+0000","last_clean":"2023-09-23T18:05:46.119555+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:05:46.119555+0000","last_undegraded":"2023-09-23T18:05:46.119555+0000","last_fullsized":"2023-09-23T18:05:46.119555+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":250,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118446+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:05:46.118446+0000","last_peered":"2023-09-23T18:05:46.118446+0000","last_clean":"2023-09-23T18:05:46.118446+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:05:46.118446+0000","last_undegraded":"2023-09-23T18:05:46.118446+0000","last_fullsized":"2023-09-23T18:05:46.118446+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"283'319","reported_seq":330,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:52.687193+0000","last_change":"2023-09-23T18:05:46.127646+0000","last_active":"2023-09-23T18:05:52.687193+0000","last_peered":"2023-09-23T18:05:52.687193+0000","last_clean":"2023-09-23T18:05:52.687193+0000","last_became_active":"2023-09-23T18:05:46.127313+0000","last_became_peered":"2023-09-23T18:05:46.127313+0000","last_unstale":"2023-09-23T18:05:52.687193+0000","last_undegraded":"2023-09-23T18:05:52.687193+0000","last_fullsized":"2023-09-23T18:05:52.687193+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":319,"log_dups_size":0,"ondisk_log_size":319,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:34.513090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20905984,"num_objects":319,"num_object_clones":0,"num_object_copies":957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":319,"num_write_kb":20416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":443,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.150334+0000","last_change":"2023-09-23T18:05:15.085455+0000","last_active":"2023-09-23T18:05:47.150334+0000","last_peered":"2023-09-23T18:05:47.150334+0000","last_clean":"2023-09-23T18:05:47.150334+0000","last_became_active":"2023-09-23T18:05:15.085106+0000","last_became_peered":"2023-09-23T18:05:15.085106+0000","last_unstale":"2023-09-23T18:05:47.150334+0000","last_undegraded":"2023-09-23T18:05:47.150334+0000","last_fullsized":"2023-09-23T18:05:47.150334+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":494,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.119848+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:05:46.119848+0000","last_peered":"2023-09-23T18:05:46.119848+0000","last_clean":"2023-09-23T18:05:46.119848+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:05:46.119848+0000","last_undegraded":"2023-09-23T18:05:46.119848+0000","last_fullsized":"2023-09-23T18:05:46.119848+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":217,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.776508+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:05:47.776508+0000","last_peered":"2023-09-23T18:05:47.776508+0000","last_clean":"2023-09-23T18:05:47.776508+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:05:47.776508+0000","last_undegraded":"2023-09-23T18:05:47.776508+0000","last_fullsized":"2023-09-23T18:05:47.776508+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":248,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.559139+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:05:47.559139+0000","last_peered":"2023-09-23T18:05:47.559139+0000","last_clean":"2023-09-23T18:05:47.559139+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:05:47.559139+0000","last_undegraded":"2023-09-23T18:05:47.559139+0000","last_fullsized":"2023-09-23T18:05:47.559139+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"282'63","reported_seq":75,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:51.198797+0000","last_change":"2023-09-23T18:05:46.122448+0000","last_active":"2023-09-23T18:05:51.198797+0000","last_peered":"2023-09-23T18:05:51.198797+0000","last_clean":"2023-09-23T18:05:51.198797+0000","last_became_active":"2023-09-23T18:05:46.122155+0000","last_became_peered":"2023-09-23T18:05:46.122155+0000","last_unstale":"2023-09-23T18:05:51.198797+0000","last_undegraded":"2023-09-23T18:05:51.198797+0000","last_fullsized":"2023-09-23T18:05:51.198797+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":63,"log_dups_size":0,"ondisk_log_size":63,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:40.972879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4128768,"num_objects":63,"num_object_clones":0,"num_object_copies":189,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":63,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":63,"num_write_kb":4032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":596,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121853+0000","last_change":"2023-09-23T18:05:15.096447+0000","last_active":"2023-09-23T18:05:46.121853+0000","last_peered":"2023-09-23T18:05:46.121853+0000","last_clean":"2023-09-23T18:05:46.121853+0000","last_became_active":"2023-09-23T18:05:15.096059+0000","last_became_peered":"2023-09-23T18:05:15.096059+0000","last_unstale":"2023-09-23T18:05:46.121853+0000","last_undegraded":"2023-09-23T18:05:46.121853+0000","last_fullsized":"2023-09-23T18:05:46.121853+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":128,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775761+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:05:47.775761+0000","last_peered":"2023-09-23T18:05:47.775761+0000","last_clean":"2023-09-23T18:05:47.775761+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:05:47.775761+0000","last_undegraded":"2023-09-23T18:05:47.775761+0000","last_fullsized":"2023-09-23T18:05:47.775761+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":352,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.150245+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:05:47.150245+0000","last_peered":"2023-09-23T18:05:47.150245+0000","last_clean":"2023-09-23T18:05:47.150245+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:05:47.150245+0000","last_undegraded":"2023-09-23T18:05:47.150245+0000","last_fullsized":"2023-09-23T18:05:47.150245+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":132,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775912+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:05:47.775912+0000","last_peered":"2023-09-23T18:05:47.775912+0000","last_clean":"2023-09-23T18:05:47.775912+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:05:47.775912+0000","last_undegraded":"2023-09-23T18:05:47.775912+0000","last_fullsized":"2023-09-23T18:05:47.775912+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"282'54","reported_seq":66,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:51.201323+0000","last_change":"2023-09-23T18:05:46.122766+0000","last_active":"2023-09-23T18:05:51.201323+0000","last_peered":"2023-09-23T18:05:51.201323+0000","last_clean":"2023-09-23T18:05:51.201323+0000","last_became_active":"2023-09-23T18:05:46.122467+0000","last_became_peered":"2023-09-23T18:05:46.122467+0000","last_unstale":"2023-09-23T18:05:51.201323+0000","last_undegraded":"2023-09-23T18:05:51.201323+0000","last_fullsized":"2023-09-23T18:05:51.201323+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":54,"log_dups_size":0,"ondisk_log_size":54,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:35.797941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":3538944,"num_objects":54,"num_object_clones":0,"num_object_copies":162,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":54,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":54,"num_write_kb":3456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":602,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.777146+0000","last_change":"2023-09-23T18:05:15.088351+0000","last_active":"2023-09-23T18:05:47.777146+0000","last_peered":"2023-09-23T18:05:47.777146+0000","last_clean":"2023-09-23T18:05:47.777146+0000","last_became_active":"2023-09-23T18:05:15.087902+0000","last_became_peered":"2023-09-23T18:05:15.087902+0000","last_unstale":"2023-09-23T18:05:47.777146+0000","last_undegraded":"2023-09-23T18:05:47.777146+0000","last_fullsized":"2023-09-23T18:05:47.777146+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:02:50.479420+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":487,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.150171+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:05:47.150171+0000","last_peered":"2023-09-23T18:05:47.150171+0000","last_clean":"2023-09-23T18:05:47.150171+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:05:47.150171+0000","last_undegraded":"2023-09-23T18:05:47.150171+0000","last_fullsized":"2023-09-23T18:05:47.150171+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":307,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118485+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:05:46.118485+0000","last_peered":"2023-09-23T18:05:46.118485+0000","last_clean":"2023-09-23T18:05:46.118485+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:05:46.118485+0000","last_undegraded":"2023-09-23T18:05:46.118485+0000","last_fullsized":"2023-09-23T18:05:46.118485+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":71,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121312+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:05:46.121312+0000","last_peered":"2023-09-23T18:05:46.121312+0000","last_clean":"2023-09-23T18:05:46.121312+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:05:46.121312+0000","last_undegraded":"2023-09-23T18:05:46.121312+0000","last_fullsized":"2023-09-23T18:05:46.121312+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":585,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775551+0000","last_change":"2023-09-23T18:05:15.083744+0000","last_active":"2023-09-23T18:05:47.775551+0000","last_peered":"2023-09-23T18:05:47.775551+0000","last_clean":"2023-09-23T18:05:47.775551+0000","last_became_active":"2023-09-23T18:05:15.083337+0000","last_became_peered":"2023-09-23T18:05:15.083337+0000","last_unstale":"2023-09-23T18:05:47.775551+0000","last_undegraded":"2023-09-23T18:05:47.775551+0000","last_fullsized":"2023-09-23T18:05:47.775551+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":267,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.558520+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:05:47.558520+0000","last_peered":"2023-09-23T18:05:47.558520+0000","last_clean":"2023-09-23T18:05:47.558520+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:05:47.558520+0000","last_undegraded":"2023-09-23T18:05:47.558520+0000","last_fullsized":"2023-09-23T18:05:47.558520+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":295,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.776585+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:05:47.776585+0000","last_peered":"2023-09-23T18:05:47.776585+0000","last_clean":"2023-09-23T18:05:47.776585+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:05:47.776585+0000","last_undegraded":"2023-09-23T18:05:47.776585+0000","last_fullsized":"2023-09-23T18:05:47.776585+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":594,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775918+0000","last_change":"2023-09-23T18:05:15.092746+0000","last_active":"2023-09-23T18:05:47.775918+0000","last_peered":"2023-09-23T18:05:47.775918+0000","last_clean":"2023-09-23T18:05:47.775918+0000","last_became_active":"2023-09-23T18:05:15.092486+0000","last_became_peered":"2023-09-23T18:05:15.092486+0000","last_unstale":"2023-09-23T18:05:47.775918+0000","last_undegraded":"2023-09-23T18:05:47.775918+0000","last_fullsized":"2023-09-23T18:05:47.775918+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":138,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886259+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:05:49.886259+0000","last_peered":"2023-09-23T18:05:49.886259+0000","last_clean":"2023-09-23T18:05:49.886259+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:05:49.886259+0000","last_undegraded":"2023-09-23T18:05:49.886259+0000","last_fullsized":"2023-09-23T18:05:49.886259+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":206,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.123462+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:05:46.123462+0000","last_peered":"2023-09-23T18:05:46.123462+0000","last_clean":"2023-09-23T18:05:46.123462+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:05:46.123462+0000","last_undegraded":"2023-09-23T18:05:46.123462+0000","last_fullsized":"2023-09-23T18:05:46.123462+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":124,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886272+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:05:49.886272+0000","last_peered":"2023-09-23T18:05:49.886272+0000","last_clean":"2023-09-23T18:05:49.886272+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:05:49.886272+0000","last_undegraded":"2023-09-23T18:05:49.886272+0000","last_fullsized":"2023-09-23T18:05:49.886272+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":147,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.776276+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:05:47.776276+0000","last_peered":"2023-09-23T18:05:47.776276+0000","last_clean":"2023-09-23T18:05:47.776276+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:05:47.776276+0000","last_undegraded":"2023-09-23T18:05:47.776276+0000","last_fullsized":"2023-09-23T18:05:47.776276+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":647,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.120222+0000","last_change":"2023-09-23T18:05:15.087831+0000","last_active":"2023-09-23T18:05:46.120222+0000","last_peered":"2023-09-23T18:05:46.120222+0000","last_clean":"2023-09-23T18:05:46.120222+0000","last_became_active":"2023-09-23T18:05:15.087299+0000","last_became_peered":"2023-09-23T18:05:15.087299+0000","last_unstale":"2023-09-23T18:05:46.120222+0000","last_undegraded":"2023-09-23T18:05:46.120222+0000","last_fullsized":"2023-09-23T18:05:46.120222+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:31.678814+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":147,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.776114+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:05:47.776114+0000","last_peered":"2023-09-23T18:05:47.776114+0000","last_clean":"2023-09-23T18:05:47.776114+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:05:47.776114+0000","last_undegraded":"2023-09-23T18:05:47.776114+0000","last_fullsized":"2023-09-23T18:05:47.776114+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":150,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775632+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:05:47.775632+0000","last_peered":"2023-09-23T18:05:47.775632+0000","last_clean":"2023-09-23T18:05:47.775632+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:05:47.775632+0000","last_undegraded":"2023-09-23T18:05:47.775632+0000","last_fullsized":"2023-09-23T18:05:47.775632+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":593,"reported_epoch":281,"state":"active+clean","last_fresh":"2023-09-23T18:05:47.775634+0000","last_change":"2023-09-23T18:05:15.098282+0000","last_active":"2023-09-23T18:05:47.775634+0000","last_peered":"2023-09-23T18:05:47.775634+0000","last_clean":"2023-09-23T18:05:47.775634+0000","last_became_active":"2023-09-23T18:05:15.097779+0000","last_became_peered":"2023-09-23T18:05:15.097779+0000","last_unstale":"2023-09-23T18:05:47.775634+0000","last_undegraded":"2023-09-23T18:05:47.775634+0000","last_fullsized":"2023-09-23T18:05:47.775634+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":260,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.118862+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:05:46.118862+0000","last_peered":"2023-09-23T18:05:46.118862+0000","last_clean":"2023-09-23T18:05:46.118862+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:05:46.118862+0000","last_undegraded":"2023-09-23T18:05:46.118862+0000","last_fullsized":"2023-09-23T18:05:46.118862+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":291,"reported_epoch":280,"state":"active+clean","last_fresh":"2023-09-23T18:05:46.121037+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:05:46.121037+0000","last_peered":"2023-09-23T18:05:46.121037+0000","last_clean":"2023-09-23T18:05:46.121037+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:05:46.121037+0000","last_undegraded":"2023-09-23T18:05:46.121037+0000","last_fullsized":"2023-09-23T18:05:46.121037+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":53280768,"num_objects":813,"num_object_clones":0,"num_object_copies":2439,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":813,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":813,"num_write_kb":52032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":119668736,"data_stored":119668736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":813,"ondisk_log_size":813,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":420413,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":345,"num_read_kb":650,"num_write":169,"num_write_kb":1719,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3721559,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001398,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":116948,"kb_used_data":3340,"kb_used_omap":13,"kb_used_meta":113586,"kb_avail":93628204,"statfs":{"total":95995035648,"available":95875280896,"internally_reserved":0,"allocated":3420160,"data_stored":2358936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":116312168},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":0.76100000000000001,"15min":0.66300000000000003},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.9889999999999999,"5min":6.2770000000000001,"15min":6.2770000000000001},"last":0.69999999999999996},{"interface":"front","average":{"1min":1.266,"5min":1.194,"15min":1.256},"min":{"1min":0.309,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.8209999999999997,"5min":6.641,"15min":6.641},"last":1.6299999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.97699999999999998,"15min":0.77700000000000002},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":7.0309999999999997,"5min":7.1150000000000002,"15min":7.1150000000000002},"last":1.9890000000000001},{"interface":"front","average":{"1min":1.466,"5min":1.1970000000000001,"15min":1.103},"min":{"1min":0.27600000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.9589999999999996,"5min":6.9589999999999996,"15min":6.9589999999999996},"last":2.2410000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.071,"15min":0.98199999999999998},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.6630000000000003,"5min":6.6630000000000003,"15min":6.6630000000000003},"last":1.7110000000000001},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.1100000000000001,"15min":0.91200000000000003},"min":{"1min":0.316,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.9160000000000004,"5min":7.9160000000000004,"15min":7.9160000000000004},"last":1.036}]},{"osd":3,"last update":"Sat Sep 23 18:05:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.472,"15min":1.526},"min":{"1min":0.34599999999999997,"5min":0.316,"15min":0.316},"max":{"1min":7.0579999999999998,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":2.0259999999999998},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.391,"15min":1.1100000000000001},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":15.349,"5min":15.349,"15min":15.349},"last":1.4319999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.073,"15min":1.0720000000000001},"min":{"1min":0.307,"5min":0.245,"15min":0.245},"max":{"1min":6.9729999999999999,"5min":6.9729999999999999,"15min":6.9729999999999999},"last":0.67500000000000004},{"interface":"front","average":{"1min":1.1459999999999999,"5min":0.71399999999999997,"15min":0.58799999999999997},"min":{"1min":0.32200000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0289999999999999,"5min":6.3479999999999999,"15min":6.3479999999999999},"last":0.65900000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.1759999999999999,"15min":1.0609999999999999},"min":{"1min":0.38,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.4749999999999996,"5min":9.5999999999999996,"15min":9.5999999999999996},"last":0.68700000000000006},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.2709999999999999,"15min":1.179},"min":{"1min":0.39400000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.3739999999999997,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.57099999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":0.94599999999999995,"15min":0.78600000000000003},"min":{"1min":0.438,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.2469999999999999,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.747},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.232,"15min":1.161},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.9539999999999997,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":0.64600000000000002}]}]},{"osd":6,"up_from":163,"seq":700079669335,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":733200,"kb_used_data":573000,"kb_used_omap":7,"kb_used_meta":160184,"kb_avail":93011952,"statfs":{"total":95995035648,"available":95244238848,"internally_reserved":0,"allocated":586752000,"data_stored":585702216,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":164029220},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.89900000000000002,"15min":1.2529999999999999},"min":{"1min":0.23100000000000001,"5min":0.224,"15min":0.215},"max":{"1min":1.7,"5min":15.407,"15min":15.407},"last":0.94999999999999996},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.96899999999999997,"15min":2.0470000000000002},"min":{"1min":0.29199999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.5600000000000001,"5min":7.5739999999999998,"15min":13.536},"last":1.276}]},{"osd":1,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.044,"15min":1.7649999999999999},"min":{"1min":0.27300000000000002,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.161,"5min":6.7690000000000001,"15min":14.233000000000001},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.1739999999999999,"15min":1.9650000000000001},"min":{"1min":0.44,"5min":0.317,"15min":0.248},"max":{"1min":3.5830000000000002,"5min":14.789,"15min":14.789},"last":1.341}]},{"osd":2,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.1919999999999999,"15min":2.1030000000000002},"min":{"1min":0.41199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.8410000000000002,"5min":7.173,"15min":11.210000000000001},"last":1.238},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.1439999999999999,"15min":2.4209999999999998},"min":{"1min":0.313,"5min":0.245,"15min":0.245},"max":{"1min":5.4829999999999997,"5min":9.3620000000000001,"15min":11.223000000000001},"last":0.48899999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:05:26 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.097,"15min":1.101},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":4.7539999999999996,"5min":4.7539999999999996,"15min":4.7539999999999996},"last":0.92300000000000004},{"interface":"front","average":{"1min":1.6619999999999999,"5min":1.3260000000000001,"15min":1.2190000000000001},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.484,"5min":7.484,"15min":7.484},"last":1.3049999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0109999999999999,"15min":0.91800000000000004},"min":{"1min":0.34200000000000003,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":5.5069999999999997,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.81299999999999994,"15min":0.59999999999999998},"min":{"1min":0.32300000000000001,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":6.3920000000000003,"5min":6.3920000000000003,"15min":8.3000000000000007},"last":1.351}]},{"osd":5,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":0.91000000000000003,"15min":0.89400000000000002},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.28599999999999998},"max":{"1min":5.4249999999999998,"5min":6.6159999999999997,"15min":11.869},"last":1.329},{"interface":"front","average":{"1min":0.625,"5min":0.95799999999999996,"15min":0.71799999999999997},"min":{"1min":0.377,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.73,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":1.389}]},{"osd":7,"last update":"Sat Sep 23 18:05:31 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.075,"15min":1.0009999999999999},"min":{"1min":0.38400000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.5380000000000003,"5min":5.5380000000000003,"15min":5.5380000000000003},"last":0.55100000000000005},{"interface":"front","average":{"1min":1.647,"5min":1.012,"15min":0.85799999999999998},"min":{"1min":0.40100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.411,"5min":10.411,"15min":10.411},"last":0.80600000000000005}]}]},{"osd":1,"up_from":0,"seq":420906795154,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958680,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":409624,"kb_used_data":7888,"kb_used_omap":7,"kb_used_meta":401720,"kb_avail":93335528,"statfs":{"total":95995035648,"available":95575580672,"internally_reserved":0,"allocated":8077312,"data_stored":6666752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":411362020},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:05:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":1.2909999999999999,"15min":1.2809999999999999},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.17899999999999999},"max":{"1min":10.128,"5min":10.128,"15min":18.154},"last":0.373},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.101,"15min":1.175},"min":{"1min":0.26900000000000002,"5min":0.186,"15min":0.17399999999999999},"max":{"1min":5.5490000000000004,"5min":12.641999999999999,"15min":13.369999999999999},"last":0.53500000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.1459999999999999,"15min":1.446},"min":{"1min":0.26500000000000001,"5min":0.21099999999999999,"15min":0.20499999999999999},"max":{"1min":10.48,"5min":10.48,"15min":20.882000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.288,"15min":1.3420000000000001},"min":{"1min":0.23000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":4.5599999999999996,"5min":12.673,"15min":15.064},"last":0.45500000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:05:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.5649999999999999,"15min":1.5569999999999999},"min":{"1min":0.245,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":11.948,"5min":11.948,"15min":11.948},"last":0.39400000000000002},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.347,"15min":1.2490000000000001},"min":{"1min":0.30599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.9699999999999998,"5min":8.7300000000000004,"15min":8.7300000000000004},"last":0.48699999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.163,"15min":1.226},"min":{"1min":0.32600000000000001,"5min":0.29299999999999998,"15min":0.251},"max":{"1min":4.7510000000000003,"5min":5.4720000000000004,"15min":15.279999999999999},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.2669999999999999,"15min":1.3029999999999999},"min":{"1min":0.379,"5min":0.247,"15min":0.247},"max":{"1min":8.6129999999999995,"5min":10.164999999999999,"15min":18.106999999999999},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":1.3799999999999999,"15min":1.3819999999999999},"min":{"1min":0.40200000000000002,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":8.5730000000000004,"5min":10.194000000000001,"15min":14.720000000000001},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.1519999999999999,"15min":1.1379999999999999},"min":{"1min":0.43099999999999999,"5min":0.254,"15min":0.254},"max":{"1min":5.5170000000000003,"5min":12.702,"15min":12.794},"last":0.63200000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.1459999999999999,"15min":2.056},"min":{"1min":0.38800000000000001,"5min":0.35099999999999998,"15min":0.34599999999999997},"max":{"1min":5.6559999999999997,"5min":5.6559999999999997,"15min":14.956},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.081,"15min":2.181},"min":{"1min":0.48899999999999999,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":2.9969999999999999,"5min":12.596,"15min":12.755000000000001},"last":0.70099999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:05:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.036,"15min":0.95999999999999996},"min":{"1min":0.41999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.4180000000000001,"5min":7.4649999999999999,"15min":7.4649999999999999},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.385,"15min":1.1830000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.468,"5min":5.524,"15min":5.524},"last":0.59899999999999998}]}]},{"osd":2,"up_from":103,"seq":442381631627,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":395132,"kb_used_data":7796,"kb_used_omap":11,"kb_used_meta":387316,"kb_avail":93350020,"statfs":{"total":95995035648,"available":95590420480,"internally_reserved":0,"allocated":7983104,"data_stored":6599410,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":396611959},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.94099999999999995,"15min":1.431},"min":{"1min":0.20799999999999999,"5min":0.187,"15min":0.17199999999999999},"max":{"1min":4.6120000000000001,"5min":12.395,"15min":20.867000000000001},"last":0.312},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.95299999999999996,"15min":1.387},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":4.5519999999999996,"5min":7.6550000000000002,"15min":21.681000000000001},"last":0.41299999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.94699999999999995,"15min":1.4730000000000001},"min":{"1min":0.28299999999999997,"5min":0.215,"15min":0.215},"max":{"1min":5.0140000000000002,"5min":7.2720000000000002,"15min":20.995000000000001},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.504,"5min":0.83399999999999996,"15min":1.2549999999999999},"min":{"1min":0.255,"5min":0.193,"15min":0.193},"max":{"1min":2.0299999999999998,"5min":7.5720000000000001,"15min":15.945},"last":0.33600000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.98899999999999999,"15min":0.85099999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.238,"5min":12.009,"15min":12.009},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.702,"5min":0.996,"15min":0.73899999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.162000000000001,"5min":11.162000000000001,"15min":11.162000000000001},"last":0.378}]},{"osd":4,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.143,"15min":1.2030000000000001},"min":{"1min":0.43099999999999999,"5min":0.28399999999999997,"15min":0.26500000000000001},"max":{"1min":7.6799999999999997,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.83099999999999996,"15min":1.1970000000000001},"min":{"1min":0.38600000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.6310000000000002,"5min":7.5609999999999999,"15min":20.920000000000002},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.88400000000000001,"15min":1.264},"min":{"1min":0.34300000000000003,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":4.5140000000000002,"5min":7.3289999999999997,"15min":20.738},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.25,"15min":1.3580000000000001},"min":{"1min":0.32900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.7140000000000004,"5min":12.103999999999999,"15min":15.496},"last":0.47099999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.22,"15min":1.998},"min":{"1min":0.40899999999999997,"5min":0.307,"15min":0.307},"max":{"1min":7.718,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":0.504},{"interface":"front","average":{"1min":0.876,"5min":1.0169999999999999,"15min":2.2160000000000002},"min":{"1min":0.40300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.5810000000000004,"5min":8.4649999999999999,"15min":20.893000000000001},"last":0.58799999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.3320000000000001,"15min":1.3839999999999999},"min":{"1min":0.42099999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":7.4989999999999997,"5min":12.051,"15min":12.051},"last":0.441},{"interface":"front","average":{"1min":1.766,"5min":1.0009999999999999,"15min":0.755},"min":{"1min":0.45300000000000001,"5min":0.309,"15min":0.309},"max":{"1min":8.5039999999999996,"5min":8.5039999999999996,"15min":8.5039999999999996},"last":0.56699999999999995}]}]},{"osd":3,"up_from":237,"seq":1017907249196,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":217180,"kb_used_data":40212,"kb_used_omap":3,"kb_used_meta":176956,"kb_avail":93527972,"statfs":{"total":95995035648,"available":95772643328,"internally_reserved":0,"allocated":41177088,"data_stored":39599584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":181203688},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":0.91000000000000003,"15min":0.89200000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.5440000000000005,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.0179999999999998,"5min":1.127,"15min":1.097},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.885999999999999,"5min":14.885999999999999,"15min":14.885999999999999},"last":0.40100000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":1.2130000000000001,"15min":1.111},"min":{"1min":0.29799999999999999,"5min":0.245,"15min":0.245},"max":{"1min":21.931999999999999,"5min":21.931999999999999,"15min":21.931999999999999},"last":0.32100000000000001},{"interface":"front","average":{"1min":1.625,"5min":1.01,"15min":0.96699999999999997},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":12.701000000000001,"5min":12.701000000000001,"15min":12.701000000000001},"last":0.371}]},{"osd":2,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":0.94899999999999995,"15min":0.88300000000000001},"min":{"1min":0.32500000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2200000000000006,"5min":8.2200000000000006,"15min":8.2200000000000006},"last":0.51800000000000002},{"interface":"front","average":{"1min":2.46,"5min":1.3280000000000001,"15min":1.2729999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":21.988,"5min":21.988,"15min":21.988},"last":0.48499999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.2509999999999999,"15min":1.153},"min":{"1min":0.375,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":14.935,"5min":14.935,"15min":14.935},"last":0.50600000000000001},{"interface":"front","average":{"1min":2.6600000000000001,"5min":1.3129999999999999,"15min":1.149},"min":{"1min":0.36899999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.786,"5min":12.786,"15min":12.786},"last":0.42999999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":1.3879999999999999,"15min":1.3180000000000001},"min":{"1min":0.48399999999999999,"5min":0.314,"15min":0.314},"max":{"1min":12.743,"5min":12.743,"15min":12.743},"last":0.495},{"interface":"front","average":{"1min":1.827,"5min":1.2629999999999999,"15min":1.208},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.6479999999999997,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.69799999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.125,"15min":1.0980000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.5789999999999997,"5min":7.5789999999999997,"15min":7.5789999999999997},"last":0.72999999999999998},{"interface":"front","average":{"1min":2.5,"5min":1.5720000000000001,"15min":1.51},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":1.625,"15min":1.589},"min":{"1min":0.42599999999999999,"5min":0.377,"15min":0.377},"max":{"1min":12.821,"5min":12.821,"15min":12.821},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.5780000000000001,"15min":1.6060000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":8.6039999999999992,"5min":8.6039999999999992,"15min":8.6039999999999992},"last":0.751}]}]},{"osd":4,"up_from":30,"seq":128849019145,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":248840,"kb_used_data":3072,"kb_used_omap":0,"kb_used_meta":245760,"kb_avail":93496312,"statfs":{"total":95995035648,"available":95740223488,"internally_reserved":0,"allocated":3145728,"data_stored":2088499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":251658240},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.75800000000000001,"15min":0.88700000000000001},"min":{"1min":0.26000000000000001,"5min":0.215,"15min":0.20100000000000001},"max":{"1min":2.2850000000000001,"5min":6.8849999999999998,"15min":14.164},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.0640000000000001,"15min":1.1220000000000001},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":5.7750000000000004,"5min":7.1879999999999997,"15min":11.372},"last":0.433}]},{"osd":1,"last update":"Sat Sep 23 18:05:09 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.204,"15min":1.0880000000000001},"min":{"1min":0.29099999999999998,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":5.9740000000000002,"5min":7.2640000000000002,"15min":10.429},"last":0.496},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.2430000000000001,"15min":1.1439999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":6.7640000000000002,"5min":6.7640000000000002,"15min":10.83},"last":0.46800000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:05:44 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":1.0489999999999999,"15min":1.1559999999999999},"min":{"1min":0.33300000000000002,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":2.1579999999999999,"5min":6.8259999999999996,"15min":11.630000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.169,"5min":1.085,"15min":1.107},"min":{"1min":0.42999999999999999,"5min":0.316,"15min":0.27100000000000002},"max":{"1min":6.609,"5min":6.9299999999999997,"15min":7.5090000000000003},"last":0.55800000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:05:19 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.3620000000000001,"15min":1.3540000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.6750000000000007,"5min":8.6750000000000007,"15min":8.6750000000000007},"last":0.60299999999999998},{"interface":"front","average":{"1min":1.8140000000000001,"5min":1.423,"15min":1.389},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.8170000000000002,"5min":6.165,"15min":6.165},"last":0.58599999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:05:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.95999999999999996,"15min":0.73399999999999999},"min":{"1min":0.24199999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":9.2959999999999994,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.2130000000000001,"15min":0.92500000000000004},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.26100000000000001},"max":{"1min":5.9249999999999998,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.63300000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:05:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":1.0369999999999999,"15min":0.90800000000000003},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":2.1200000000000001,"5min":9.0990000000000002,"15min":9.0990000000000002},"last":0.57299999999999995},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.998,"15min":0.998},"min":{"1min":0.32100000000000001,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":2.4980000000000002,"5min":9.1300000000000008,"15min":10.725},"last":0.61399999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:05:30 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":1.284,"15min":1.0369999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":13.298999999999999,"5min":13.298999999999999,"15min":13.298999999999999},"last":0.68600000000000005},{"interface":"front","average":{"1min":1.7470000000000001,"5min":1.484,"15min":1.3029999999999999},"min":{"1min":0.379,"5min":0.307,"15min":0.307},"max":{"1min":10.192,"5min":10.192,"15min":10.192},"last":0.66700000000000004}]}]},{"osd":5,"up_from":38,"seq":163208757512,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":293140,"kb_used_data":70092,"kb_used_omap":0,"kb_used_meta":223040,"kb_avail":93452012,"statfs":{"total":95995035648,"available":95694860288,"internally_reserved":0,"allocated":71774208,"data_stored":70724707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":228392960},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.88100000000000001,"15min":1.024},"min":{"1min":0.35299999999999998,"5min":0.27300000000000002,"15min":0.217},"max":{"1min":0.85599999999999998,"5min":13.586,"15min":21.376999999999999},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.3400000000000001,"15min":1.24},"min":{"1min":0.32800000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.5190000000000001,"5min":13.574,"15min":21.388999999999999},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.80800000000000005,"15min":0.875},"min":{"1min":0.34999999999999998,"5min":0.26000000000000001,"15min":0.23899999999999999},"max":{"1min":4.6150000000000002,"5min":4.8129999999999997,"15min":10.853999999999999},"last":0.34799999999999998},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.1519999999999999,"15min":1.0569999999999999},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":8.5619999999999994,"5min":8.5619999999999994,"15min":10.206},"last":0.47599999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.1040000000000001,"15min":1.325},"min":{"1min":0.35999999999999999,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":1.5880000000000001,"5min":5.5309999999999997,"15min":38.606999999999999},"last":0.5},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.97199999999999998,"15min":1.226},"min":{"1min":0.36099999999999999,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":2.673,"5min":10.065,"15min":39.670999999999999},"last":0.56699999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:05:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.0289999999999999,"15min":0.85399999999999998},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.593,"5min":8.593,"15min":8.593},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.224,"5min":0.85299999999999998,"15min":0.75900000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.0439999999999996,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":0.41999999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:05:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1.01,"15min":0.79300000000000004},"min":{"1min":0.28699999999999998,"5min":0.24199999999999999,"15min":0.224},"max":{"1min":2.621,"5min":8.6229999999999993,"15min":10.092000000000001},"last":0.313},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.94899999999999995,"15min":0.77000000000000002},"min":{"1min":0.33900000000000002,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":0.97399999999999998,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.378}]},{"osd":6,"last update":"Sat Sep 23 18:05:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.94199999999999995,"15min":1.3560000000000001},"min":{"1min":0.34200000000000003,"5min":0.28299999999999997,"15min":0.27700000000000002},"max":{"1min":1.2370000000000001,"5min":6.5880000000000001,"15min":11.287000000000001},"last":0.46200000000000002},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.873,"15min":1.224},"min":{"1min":0.307,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.6920000000000002,"5min":6.6920000000000002,"15min":8.266},"last":0.51600000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:05:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.298,"15min":1.4630000000000001},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.0999999999999996,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.218,"15min":1.1559999999999999},"min":{"1min":0.42999999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":17.684000000000001,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.434}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":39911424,"data_stored":39911424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":79757312,"data_stored":79757312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:05:54.492 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:05:54.542 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:54 smithi099 ceph-mon[167076]: pgmap v700: 119 pgs: 119 active+clean; 55 MiB data, 2.3 GiB used, 624 GiB / 626 GiB avail; 8.5 MiB/s wr, 135 op/s 2023-09-23T18:05:54.542 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:54 smithi099 ceph-mon[167076]: 1.0 scrub starts 2023-09-23T18:05:54.542 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:54 smithi099 ceph-mon[167076]: 1.0 scrub ok 2023-09-23T18:05:54.542 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:54 smithi099 ceph-mon[167076]: 1.a scrub starts 2023-09-23T18:05:54.542 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:54 smithi099 ceph-mon[167076]: 1.a scrub ok 2023-09-23T18:05:54.576 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:54.745 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:05:54.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:54.755 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:05:54.826 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 4 16 10219 10203 159.39 150.375 0.00900003 0.00626033 2023-09-23T18:05:54.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:54.872 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:05:54.897 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:54.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:54.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:54.962 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:55.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:05:55.111 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:55.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:05:55.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:55.178 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:05:55.251 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:05:55.251 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:05:55.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:05:55.251 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:05:55.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:38884 5.7 5:e88075de:::benchmark_data_smithi055_229117_object38883:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.244864+0000", 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 580.00271865499997, 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.111870281, 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 38884 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.252 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244864+0000", 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244864+0000", 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244864+0000", 2023-09-23T18:05:55.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:55.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244895+0000", 2023-09-23T18:05:55.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1819999999999997e-05 2023-09-23T18:05:55.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244904+0000", 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.524e-06 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.244909+0000", 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7299999999999996e-06 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246498+0000", 2023-09-23T18:05:55.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015898010000000001 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246614+0000", 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011574399999999999 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.246739+0000", 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012482699999999999 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.249667+0000", 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002927972 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.249704+0000", 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7271000000000003e-05 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356593+0000", 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.106889052 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356695+0000", 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000101992 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.356734+0000", 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9084000000000001e-05 2023-09-23T18:05:55.258 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:38888 5.7 5:e84cc5a2:::benchmark_data_smithi055_229117_object38887:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e117)", 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:56:15.248150+0000", 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 579.99943303199996, 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.11199930399999999, 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.259 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 38888 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248150+0000", 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.260 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248150+0000", 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248149+0000", 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248203+0000", 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3788000000000001e-05 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248213+0000", 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0548e-05 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248218+0000", 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7620000000000003e-06 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248262+0000", 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4316000000000001e-05 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248394+0000", 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013165099999999999 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.248541+0000", 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000147482 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.249786+0000", 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001244277 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.251395+0000", 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016097609999999999 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.360065+0000", 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.108669425 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.360114+0000", 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8878000000000002e-05 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:56:15.360149+0000", 2023-09-23T18:05:55.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.54e-05 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:119891 5.f 5:fa4cbfd7:::benchmark_data_smithi055_229117_object14265:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:49.208682+0000", 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 486.03890030899998, 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.102729686, 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.266 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 119891 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208682+0000", 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.267 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208682+0000", 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208681+0000", 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208687+0000", 2023-09-23T18:05:55.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0780000000000001e-06 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208700+0000", 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2958e-05 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208705+0000", 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1549999999999998e-06 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.246600+0000", 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037894272 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.247270+0000", 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00066982499999999998 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.270 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.269068+0000", 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021798858000000001 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.269160+0000", 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1817000000000005e-05 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.269293+0000", 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132445 2023-09-23T18:05:55.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.279028+0000", 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097349380000000003 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.291497+0000", 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012469502 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.309305+0000", 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017807699 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.309366+0000", 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.101e-05 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.311412+0000", 2023-09-23T18:05:55.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020463360000000002 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:119895 5.f 5:f7a66fc1:::benchmark_data_smithi055_229117_object14269:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:49.208732+0000", 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 486.03885111199997, 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.107236875, 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.274 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 119895 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208732+0000", 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208732+0000", 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208731+0000", 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208734+0000", 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3020000000000002e-06 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208739+0000", 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6590000000000004e-06 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.208743+0000", 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5049999999999999e-06 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.247298+0000", 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.038555566999999999 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.248114+0000", 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00081573400000000003 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.275080+0000", 2023-09-23T18:05:55.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.026966034 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.275177+0000", 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7153999999999998e-05 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.275295+0000", 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117398 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.279 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.291456+0000", 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016161406999999999 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.291531+0000", 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5284000000000006e-05 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.314013+0000", 2023-09-23T18:05:55.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022481944 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.314078+0000", 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5141000000000005e-05 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.315968+0000", 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018901160000000001 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.281 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:119901 5.f 5:ff2468ba:::benchmark_data_smithi055_229117_object14275:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e149)", 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:57:49.217244+0000", 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 486.03033848500002, 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.10684498100000001, 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:05:55.282 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 119901 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217244+0000", 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217244+0000", 2023-09-23T18:05:55.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217243+0000", 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217248+0000", 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9840000000000001e-06 2023-09-23T18:05:55.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217262+0000", 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3995000000000001e-05 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.217269+0000", 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6540000000000002e-06 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.248139+0000", 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.030870057999999999 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.248661+0000", 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00052204900000000002 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.282008+0000", 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.033347158000000002 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.282111+0000", 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000103018 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.282254+0000", 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000142965 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.291538+0000", 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0092843200000000004 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.291573+0000", 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4765999999999999e-05 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.322516+0000", 2023-09-23T18:05:55.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.030942648 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.322567+0000", 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0631999999999999e-05 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:57:49.324089+0000", 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015227299999999999 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.289 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.44681.0:125243 5.b e160/110 5:dd4dc99e:::benchmark_data_smithi055_229117_object19617:head v 160'7773, mlcod=160'7769)", 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:26.617777+0000", 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 448.62980582199998, 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.093860759000000002, 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617777+0000", 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617777+0000", 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617776+0000", 2023-09-23T18:05:55.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.617782+0000", 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5339999999999999e-06 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705124+0000", 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.087341910999999994 2023-09-23T18:05:55.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705143+0000", 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9142000000000001e-05 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705213+0000", 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9905000000000002e-05 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.705229+0000", 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6130999999999999e-05 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.711604+0000", 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0063746469999999998 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:26.711638+0000", 2023-09-23T18:05:55.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4147999999999999e-05 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154238 5.f 5:fb40edf7:::benchmark_data_smithi055_229117_object48612:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.836753+0000", 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 406.41082935100002, 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0170112849999999, 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.295 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154238 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836753+0000", 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836753+0000", 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836752+0000", 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836758+0000", 2023-09-23T18:05:55.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2560000000000002e-06 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836767+0000", 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5599999999999999e-06 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836773+0000", 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3879999999999999e-06 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.840698+0000", 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039256660000000004 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.840860+0000", 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016150800000000001 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:05:55.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.840964+0000", 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010398700000000001 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.850606+0000", 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096418819999999992 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.860704+0000", 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010098527 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.870852+0000", 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010147353 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.870862+0000", 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0497e-05 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.853747+0000", 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98288461199999999 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.853765+0000", 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7954000000000001e-05 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154243 5.f 5:f7c86625:::benchmark_data_smithi055_229117_object48617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:05:55.302 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.839762+0000", 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 406.40782041400001, 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0143057070000001, 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154243 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839762+0000", 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839762+0000", 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839761+0000", 2023-09-23T18:05:55.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839770+0000", 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1679999999999993e-06 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839785+0000", 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5633000000000001e-05 2023-09-23T18:05:55.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9209999999999992e-06 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.845356+0000", 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055628429999999996 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.845525+0000", 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000168727 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.845688+0000", 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016269799999999999 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.854335+0000", 2023-09-23T18:05:55.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0086471139999999991 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.860735+0000", 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064005720000000002 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.872057+0000", 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011321743 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.872062+0000", 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3319999999999999e-06 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.854056+0000", 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98199359500000005 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.854068+0000", 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2062000000000001e-05 2023-09-23T18:05:55.309 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33512 6.7 6:ee5c747c:::benchmark_data_smithi055_275963_object33511:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.421493+0000", 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 256.82608938700002, 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0385178110000002, 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.310 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33512 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421493+0000", 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421493+0000", 2023-09-23T18:05:55.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421492+0000", 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421529+0000", 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6989000000000001e-05 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421537+0000", 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0630000000000006e-06 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421542+0000", 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1669999999999998e-06 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421665+0000", 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012245100000000001 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421800+0000", 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000135702 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421965+0000", 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016417600000000001 2023-09-23T18:05:55.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.425828+0000", 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0038633389999999999 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.426124+0000", 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000295581 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.431471+0000", 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0053474300000000002 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.431486+0000", 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5180999999999999e-05 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446442+0000", 2023-09-23T18:05:55.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.014955775 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446456+0000", 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4290999999999999e-05 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.459918+0000", 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.013462211 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460011+0000", 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2694000000000004e-05 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33515 6.7 6:e3dc8fb1:::benchmark_data_smithi055_275963_object33514:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.433835+0000", 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 256.81374793600003, 2023-09-23T18:05:55.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0345420270000001, 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33515 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433835+0000", 2023-09-23T18:05:55.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433835+0000", 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433834+0000", 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433880+0000", 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5352000000000002e-05 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433891+0000", 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1508999999999999e-05 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433897+0000", 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0730000000000001e-06 2023-09-23T18:05:55.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441402+0000", 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075050969999999996 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441413+0000", 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0978999999999999e-05 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446745+0000", 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.005331515 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446751+0000", 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460467+0000", 2023-09-23T18:05:55.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0137163 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460602+0000", 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013485099999999999 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460760+0000", 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015809999999999999 2023-09-23T18:05:55.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.466501+0000", 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057416919999999996 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.466610+0000", 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108761 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.468215+0000", 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016042910000000001 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.468276+0000", 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1511000000000001e-05 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.468377+0000", 2023-09-23T18:05:55.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100777 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33517 6.7 6:e39e0867:::benchmark_data_smithi055_275963_object33516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.439207+0000", 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 256.80837548699998, 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0344513580000001, 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.327 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33517 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439207+0000", 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439207+0000", 2023-09-23T18:05:55.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439206+0000", 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439260+0000", 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3681000000000003e-05 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439273+0000", 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3354e-05 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439281+0000", 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5220000000000001e-06 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441439+0000", 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021585250000000001 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441447+0000", 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7719999999999994e-06 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446765+0000", 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0053182409999999 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446771+0000", 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4070000000000001e-06 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.462292+0000", 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.015521568 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.332 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.462420+0000", 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000128157 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.462587+0000", 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016689499999999999 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.466615+0000", 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040280089999999999 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.468798+0000", 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021825740000000001 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473483+0000", 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046850040000000004 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473546+0000", 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2974999999999999e-05 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473659+0000", 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011288100000000001 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33520 6.7 6:e109f2bc:::benchmark_data_smithi055_275963_object33519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.439287+0000", 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 256.808295864, 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0346285100000001, 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33520 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.336 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439287+0000", 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439287+0000", 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439286+0000", 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439337+0000", 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0813999999999998e-05 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.338 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439346+0000", 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4239999999999993e-06 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439351+0000", 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3070000000000002e-06 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441471+0000", 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021196570000000001 2023-09-23T18:05:55.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441478+0000", 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6179999999999997e-06 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446784+0000", 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.005306155 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446792+0000", 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9149999999999996e-06 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.463643+0000", 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.016850939 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.463765+0000", 2023-09-23T18:05:55.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121725 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.463946+0000", 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018062199999999999 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.466621+0000", 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002675279 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.469013+0000", 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023926289999999998 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473734+0000", 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004720709 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.343 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473798+0000", 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3841000000000001e-05 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473915+0000", 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011738700000000001 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33528 6.7 6:e730599b:::benchmark_data_smithi055_275963_object33527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.439423+0000", 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 256.80815934899999, 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0346796779999998, 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33528 2023-09-23T18:05:55.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439423+0000", 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439423+0000", 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439423+0000", 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439455+0000", 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2558999999999998e-05 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439472+0000", 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6099999999999998e-05 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439477+0000", 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1850000000000003e-06 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441681+0000", 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022039009999999999 2023-09-23T18:05:55.348 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441688+0000", 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0369999999999997e-06 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446808+0000", 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.005120805 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.349 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446813+0000", 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5499999999999996e-06 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.465059+0000", 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0182463390000001 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.465197+0000", 2023-09-23T18:05:55.350 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013815600000000001 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.465330+0000", 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013268399999999999 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.466929+0000", 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001599147 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.351 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.469212+0000", 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022831230000000002 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473963+0000", 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004750354 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.474011+0000", 2023-09-23T18:05:55.352 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8315000000000001e-05 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.474103+0000", 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1954999999999996e-05 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:95849 6.d 6:b0e99437:::benchmark_data_smithi055_275963_object22711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T18:05:55.353 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.214518+0000", 2023-09-23T18:05:55.354 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 147.03306427800001, 2023-09-23T18:05:55.354 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.14248433399999999, 2023-09-23T18:05:55.354 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.354 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.354 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.354 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:05:55.354 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:55.354 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95849 2023-09-23T18:05:55.354 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.354 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214518+0000", 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214518+0000", 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.355 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214518+0000", 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214522+0000", 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7520000000000004e-06 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214531+0000", 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1570000000000006e-06 2023-09-23T18:05:55.356 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214536+0000", 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0289999999999996e-06 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214593+0000", 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7024000000000002e-05 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.357 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214712+0000", 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011810400000000001 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214819+0000", 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000107798 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.217987+0000", 2023-09-23T18:05:55.358 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003167146 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.218620+0000", 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063337599999999995 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356918+0000", 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.138298006 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.359 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356961+0000", 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3210000000000001e-05 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.357003+0000", 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1669999999999999e-05 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.360 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:95851 6.c 6:36e0c06c:::benchmark_data_smithi055_275963_object22713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.220710+0000", 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 147.02687229099999, 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13599963000000001, 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95851 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.361 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220710+0000", 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220710+0000", 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.362 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220710+0000", 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220714+0000", 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0040000000000001e-06 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220722+0000", 2023-09-23T18:05:55.363 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0499999999999992e-06 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220726+0000", 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2169999999999996e-06 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220770+0000", 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3989000000000002e-05 2023-09-23T18:05:55.364 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220853+0000", 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2904000000000004e-05 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,6", 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220924+0000", 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1114999999999999e-05 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.365 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223643+0000", 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027192319999999998 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223709+0000", 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5788000000000001e-05 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223737+0000", 2023-09-23T18:05:55.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7517000000000001e-05 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356586+0000", 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13284983 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356710+0000", 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012371100000000001 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.367 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:95853 6.d 6:b3a07828:::benchmark_data_smithi055_275963_object22715:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.222212+0000", 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 147.02537058600001, 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13501017900000001, 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:05:55.368 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95853 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.222212+0000", 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.222212+0000", 2023-09-23T18:05:55.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.222211+0000", 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.222216+0000", 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1569999999999999e-06 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.222229+0000", 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2303999999999999e-05 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.222235+0000", 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4160000000000001e-06 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.371 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.222277+0000", 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2354000000000001e-05 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.222383+0000", 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000105498 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.222482+0000", 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9216999999999996e-05 2023-09-23T18:05:55.372 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.357015+0000", 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13453288499999999 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.357054+0000", 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8711999999999998e-05 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.373 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.357143+0000", 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9319000000000003e-05 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.357186+0000", 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2404999999999998e-05 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.357222+0000", 2023-09-23T18:05:55.374 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6795000000000003e-05 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.45092.0:95851 6.c e258/238 ondisk, result = 0)", 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.223665+0000", 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 147.02391739699999, 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13305193500000001, 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.375 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223665+0000", 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223665+0000", 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.376 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223665+0000", 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223669+0000", 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0990000000000001e-06 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.377 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223677+0000", 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6920000000000002e-06 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223681+0000", 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0210000000000001e-06 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223728+0000", 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6940000000000001e-05 2023-09-23T18:05:55.378 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223733+0000", 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2480000000000004e-06 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356717+0000", 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13298449800000001 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.379 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:95854 6.7 6:e5102bf3:::benchmark_data_smithi055_275963_object22716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.224604+0000", 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 147.02297889100001, 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.146692287, 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95854 2023-09-23T18:05:55.380 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.224604+0000", 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.224604+0000", 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.381 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.224603+0000", 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.224608+0000", 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9860000000000002e-06 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356545+0000", 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13193690799999999 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356567+0000", 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1766000000000001e-05 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356751+0000", 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018422499999999999 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356910+0000", 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015871900000000001 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.357061+0000", 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015133 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:05:55.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.364897+0000", 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0078364799999999998 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.365385+0000", 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00048802399999999999 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.368392+0000", 2023-09-23T18:05:55.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003006604 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.368446+0000", 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4376000000000003e-05 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.371296+0000", 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002849655 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.386 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.45092.0:95849 6.d e258/248 ondisk, result = 0)", 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.227995+0000", 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 147.019587401, 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.12902124400000001, 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.387 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.227995+0000", 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.227995+0000", 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.227995+0000", 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:55.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.227999+0000", 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9060000000000004e-06 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356552+0000", 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.12855360699999999 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356565+0000", 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2897999999999999e-05 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356904+0000", 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00033923599999999998 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356912+0000", 2023-09-23T18:05:55.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1879999999999996e-06 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.357017+0000", 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104987 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.391 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45092.0:95762 6.e e258/227 6:7a26cd90:::benchmark_data_smithi055_275963_object22624:head v 258'6107, mlcod=217'1539)", 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.233845+0000", 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 147.013737983, 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13611863699999999, 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.233845+0000", 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.233845+0000", 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.233844+0000", 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.233850+0000", 2023-09-23T18:05:55.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0959999999999997e-06 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356591+0000", 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.122740471 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356598+0000", 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4599999999999997e-06 2023-09-23T18:05:55.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356682+0000", 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4319000000000004e-05 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356712+0000", 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9689e-05 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.369927+0000", 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013214681000000001 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.369963+0000", 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.659e-05 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:05:55.396 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:05:55.397 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:05:55.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:55.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:55 smithi055 ceph-mon[179668]: from='client.35754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:05:55.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:55 smithi055 ceph-mon[179668]: 1.5 scrub starts 2023-09-23T18:05:55.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:55 smithi055 ceph-mon[179668]: 1.5 scrub ok 2023-09-23T18:05:55.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:55 smithi055 ceph-mon[179668]: 1.14 scrub starts 2023-09-23T18:05:55.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:55 smithi055 ceph-mon[179668]: 1.14 scrub ok 2023-09-23T18:05:55.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:55 smithi055 ceph-mon[184174]: from='client.35754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:05:55.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:55 smithi055 ceph-mon[184174]: 1.5 scrub starts 2023-09-23T18:05:55.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:55 smithi055 ceph-mon[184174]: 1.5 scrub ok 2023-09-23T18:05:55.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:55 smithi055 ceph-mon[184174]: 1.14 scrub starts 2023-09-23T18:05:55.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:55 smithi055 ceph-mon[184174]: 1.14 scrub ok 2023-09-23T18:05:55.445 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:55 smithi099 ceph-mon[167076]: from='client.35754 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:05:55.445 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:55 smithi099 ceph-mon[167076]: 1.5 scrub starts 2023-09-23T18:05:55.445 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:55 smithi099 ceph-mon[167076]: 1.5 scrub ok 2023-09-23T18:05:55.445 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:55 smithi099 ceph-mon[167076]: 1.14 scrub starts 2023-09-23T18:05:55.445 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:55 smithi099 ceph-mon[167076]: 1.14 scrub ok 2023-09-23T18:05:55.447 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:55.506 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:55.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:55.512 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:55.612 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:55.658 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:55.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:55.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:55.798 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:55.810 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:05:55.828 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 5 16 12476 12460 155.685 141.062 0.00651398 0.00641299 2023-09-23T18:05:55.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:05:55.958 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:56.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:05:56.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:56.030 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:05:56.131 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:56.182 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:56.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:56.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:56.246 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:56.280 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:05:56.285 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:56.320 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:05:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:56 smithi099 ceph-mon[167076]: pgmap v701: 119 pgs: 119 active+clean; 316 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 41 MiB/s wr, 653 op/s 2023-09-23T18:05:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:56 smithi099 ceph-mon[167076]: 1.9 scrub starts 2023-09-23T18:05:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:56 smithi099 ceph-mon[167076]: 1.9 scrub ok 2023-09-23T18:05:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:05:56.374 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:05:56.390 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:56.435 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:05:56.439 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:05:56.439 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:56.440 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:05:56.485 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:05:56.536 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:05:56.541 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:05:56.570 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[179668]: pgmap v701: 119 pgs: 119 active+clean; 316 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 41 MiB/s wr, 653 op/s 2023-09-23T18:05:56.571 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[179668]: 1.9 scrub starts 2023-09-23T18:05:56.571 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[179668]: 1.9 scrub ok 2023-09-23T18:05:56.571 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.571 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.571 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.571 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.571 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[184174]: pgmap v701: 119 pgs: 119 active+clean; 316 MiB data, 3.0 GiB used, 623 GiB / 626 GiB avail; 41 MiB/s wr, 653 op/s 2023-09-23T18:05:56.571 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[184174]: 1.9 scrub starts 2023-09-23T18:05:56.572 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[184174]: 1.9 scrub ok 2023-09-23T18:05:56.572 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.572 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.572 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.572 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:56.578 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:56.587 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:05:56.616 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:05:56.628 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:05:56.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:56.632 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:05:56.632 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:05:56.636 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:05:56.636 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:05:56.636 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:05:56.636 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:05:56.636 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:05:56.636 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:05:56.637 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:05:56.637 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:05:56.733 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:05:56.771 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:56.829 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 6 16 14545 14529 151.288 129.312 0.00305258 0.00660323 2023-09-23T18:05:56.868 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:05:56.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:56.871 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:05:56.972 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:05:57.045 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:57.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/542270738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:57.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[184174]: 1.e scrub starts 2023-09-23T18:05:57.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[184174]: 1.e scrub ok 2023-09-23T18:05:57.240 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[184174]: 1.15 scrub starts 2023-09-23T18:05:57.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[184174]: 1.15 scrub ok 2023-09-23T18:05:57.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:57.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:05:57.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:05:57.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:57.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/542270738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:57.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[179668]: 1.e scrub starts 2023-09-23T18:05:57.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[179668]: 1.e scrub ok 2023-09-23T18:05:57.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[179668]: 1.15 scrub starts 2023-09-23T18:05:57.241 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[179668]: 1.15 scrub ok 2023-09-23T18:05:57.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:57.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:05:57.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:05:57.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:57.408 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:05:57.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:57.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:05:57.492 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:05:57.526 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:05:57.563 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/542270738' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:05:57.563 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:57 smithi099 ceph-mon[167076]: 1.e scrub starts 2023-09-23T18:05:57.563 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:57 smithi099 ceph-mon[167076]: 1.e scrub ok 2023-09-23T18:05:57.563 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:57 smithi099 ceph-mon[167076]: 1.15 scrub starts 2023-09-23T18:05:57.563 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:57 smithi099 ceph-mon[167076]: 1.15 scrub ok 2023-09-23T18:05:57.564 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:57.564 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:05:57.564 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:05:57.564 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:05:57.569 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:57.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:05:57.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:57.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:05:57.725 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:05:57.815 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:57.828 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 7 16 16776 16760 149.594 139.438 0.0125714 0.00667469 2023-09-23T18:05:57.887 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:05:57.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:57.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:05:57.981 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:05:57.981 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:05:57.993 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:05:58.061 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:58.128 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:05:58.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:58.137 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:05:58.237 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:05:58.298 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:58.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:58 smithi099 ceph-mon[167076]: pgmap v702: 119 pgs: 119 active+clean; 603 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T18:05:58.358 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:05:58.360 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:05:58.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:05:58.371 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:05:58.371 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:58 smithi055 ceph-mon[179668]: pgmap v702: 119 pgs: 119 active+clean; 603 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T18:05:58.372 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:58 smithi055 ceph-mon[184174]: pgmap v702: 119 pgs: 119 active+clean; 603 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 75 MiB/s wr, 1.20k op/s 2023-09-23T18:05:58.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:05:58.527 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:05:58.831 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 8 16 18999 18983 148.202 138.938 0.00974081 0.00674084 2023-09-23T18:05:59.278 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:05:59.428 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:05:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:59 smithi099 ceph-mon[167076]: 1.3 scrub starts 2023-09-23T18:05:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:59 smithi099 ceph-mon[167076]: 1.3 scrub ok 2023-09-23T18:05:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:59 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2019428808' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:05:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:59 smithi099 ceph-mon[167076]: 1.d scrub starts 2023-09-23T18:05:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:05:59 smithi099 ceph-mon[167076]: 1.d scrub ok 2023-09-23T18:05:59.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:59 smithi055 ceph-mon[184174]: 1.3 scrub starts 2023-09-23T18:05:59.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:59 smithi055 ceph-mon[184174]: 1.3 scrub ok 2023-09-23T18:05:59.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:59 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2019428808' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:05:59.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:59 smithi055 ceph-mon[184174]: 1.d scrub starts 2023-09-23T18:05:59.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:05:59 smithi055 ceph-mon[184174]: 1.d scrub ok 2023-09-23T18:05:59.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:59 smithi055 ceph-mon[179668]: 1.3 scrub starts 2023-09-23T18:05:59.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:59 smithi055 ceph-mon[179668]: 1.3 scrub ok 2023-09-23T18:05:59.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2019428808' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:05:59.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:59 smithi055 ceph-mon[179668]: 1.d scrub starts 2023-09-23T18:05:59.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:05:59 smithi055 ceph-mon[179668]: 1.d scrub ok 2023-09-23T18:05:59.832 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 9 16 21240 21224 147.282 140.062 0.00732442 0.00678252 2023-09-23T18:06:00.268 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[179668]: pgmap v703: 119 pgs: 119 active+clean; 788 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-09-23T18:06:00.269 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:06:00.269 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:06:00.269 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[179668]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.269 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2019428808' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:00.269 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[179668]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T18:06:00.269 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[179668]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.269 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[179668]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.269 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[179668]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.269 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[179668]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.270 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[184174]: pgmap v703: 119 pgs: 119 active+clean; 788 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-09-23T18:06:00.270 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:06:00.270 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:06:00.270 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[184174]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.270 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2019428808' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:00.270 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[184174]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T18:06:00.270 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[184174]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.270 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[184174]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.271 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[184174]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.271 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-mon[184174]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.310 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:00.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:00.314 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:00.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:06:00.360 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:00.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:00.495 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:00.564 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:06:00.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:00.568 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:00.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:00.568 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:00 smithi099 ceph-mon[167076]: pgmap v703: 119 pgs: 119 active+clean; 788 MiB data, 4.4 GiB used, 621 GiB / 626 GiB avail; 82 MiB/s wr, 1.30k op/s 2023-09-23T18:06:00.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:00 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:06:00.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:00 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:06:00.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:00 smithi099 ceph-mon[167076]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2019428808' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:00.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:00 smithi099 ceph-mon[167076]: osdmap e284: 8 total, 8 up, 7 in 2023-09-23T18:06:00.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:00 smithi099 ceph-mon[167076]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:00 smithi099 ceph-mon[167076]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:00 smithi099 ceph-mon[167076]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:00 smithi099 ceph-mon[167076]: failed to encode map e284 with expected crc 2023-09-23T18:06:00.570 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:06:00.603 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:06:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:06:00] "GET /metrics HTTP/1.1" 200 36444 "" "Prometheus/2.43.0" 2023-09-23T18:06:00.670 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:00.749 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:00.833 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 10 16 23110 23094 144.229 116.875 0.00618831 0.00692662 2023-09-23T18:06:00.918 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:00.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:00.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:01.027 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:01.068 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":704,"stamp":"2023-09-23T18:05:59.261540+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":825796637,"num_objects":12559,"num_object_clones":0,"num_object_copies":37677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12559,"num_whiteouts":0,"num_read":6198,"num_read_kb":12611,"num_write":13562,"num_write_kb":816000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":105,"num_bytes_recovered":3365552,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":18554,"ondisk_log_size":18554,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":362,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4586128,"kb_used_data":2830296,"kb_used_omap":43,"kb_used_meta":1755732,"kb_avail":651629936,"statfs":{"total":671965249536,"available":667269054464,"internally_reserved":0,"allocated":2898223104,"data_stored":2889624624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44337,"internal_metadata":1797870287},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[8,2,6],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":821176397,"num_objects":12532,"num_object_clones":0,"num_object_copies":37596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12532,"num_whiteouts":0,"num_read":-99,"num_read_kb":-185,"num_write":12486,"num_write_kb":801556,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":-362269,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.517720"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":610,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885794+0000","last_change":"2023-09-23T18:05:15.087182+0000","last_active":"2023-09-23T18:05:49.885794+0000","last_peered":"2023-09-23T18:05:49.885794+0000","last_clean":"2023-09-23T18:05:49.885794+0000","last_became_active":"2023-09-23T18:05:15.086678+0000","last_became_peered":"2023-09-23T18:05:15.086678+0000","last_unstale":"2023-09-23T18:05:49.885794+0000","last_undegraded":"2023-09-23T18:05:49.885794+0000","last_fullsized":"2023-09-23T18:05:49.885794+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1262601,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":83,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.919189+0000","last_change":"2023-09-23T18:02:01.781646+0000","last_active":"2023-09-23T18:05:49.919189+0000","last_peered":"2023-09-23T18:05:49.919189+0000","last_clean":"2023-09-23T18:05:49.919189+0000","last_became_active":"2023-09-23T18:02:01.781136+0000","last_became_peered":"2023-09-23T18:02:01.781136+0000","last_unstale":"2023-09-23T18:05:49.919189+0000","last_undegraded":"2023-09-23T18:05:49.919189+0000","last_fullsized":"2023-09-23T18:05:49.919189+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:06.738710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,0],"acting":[4,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":165,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.162385+0000","last_change":"2023-09-23T18:02:18.945629+0000","last_active":"2023-09-23T18:05:49.162385+0000","last_peered":"2023-09-23T18:05:49.162385+0000","last_clean":"2023-09-23T18:05:49.162385+0000","last_became_active":"2023-09-23T18:02:18.945349+0000","last_became_peered":"2023-09-23T18:02:18.945349+0000","last_unstale":"2023-09-23T18:05:49.162385+0000","last_undegraded":"2023-09-23T18:05:49.162385+0000","last_fullsized":"2023-09-23T18:05:49.162385+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:10.864748+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":608,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886187+0000","last_change":"2023-09-23T18:05:15.096602+0000","last_active":"2023-09-23T18:05:49.886187+0000","last_peered":"2023-09-23T18:05:49.886187+0000","last_clean":"2023-09-23T18:05:49.886187+0000","last_became_active":"2023-09-23T18:05:15.096205+0000","last_became_peered":"2023-09-23T18:05:15.096205+0000","last_unstale":"2023-09-23T18:05:49.886187+0000","last_undegraded":"2023-09-23T18:05:49.886187+0000","last_fullsized":"2023-09-23T18:05:49.886187+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:31:18.329681+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":133,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161310+0000","last_change":"2023-09-23T18:02:18.978089+0000","last_active":"2023-09-23T18:05:49.161310+0000","last_peered":"2023-09-23T18:05:49.161310+0000","last_clean":"2023-09-23T18:05:49.161310+0000","last_became_active":"2023-09-23T18:02:18.974696+0000","last_became_peered":"2023-09-23T18:02:18.974696+0000","last_unstale":"2023-09-23T18:05:49.161310+0000","last_undegraded":"2023-09-23T18:05:49.161310+0000","last_fullsized":"2023-09-23T18:05:49.161310+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:23.912278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,6],"acting":[3,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":284,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916333+0000","last_change":"2023-09-23T18:02:47.960145+0000","last_active":"2023-09-23T18:05:49.916333+0000","last_peered":"2023-09-23T18:05:49.916333+0000","last_clean":"2023-09-23T18:05:49.916333+0000","last_became_active":"2023-09-23T18:02:47.959244+0000","last_became_peered":"2023-09-23T18:02:47.959244+0000","last_unstale":"2023-09-23T18:05:49.916333+0000","last_undegraded":"2023-09-23T18:05:49.916333+0000","last_fullsized":"2023-09-23T18:05:49.916333+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:17:53.214424+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":600,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.919261+0000","last_change":"2023-09-23T18:05:15.096807+0000","last_active":"2023-09-23T18:05:49.919261+0000","last_peered":"2023-09-23T18:05:49.919261+0000","last_clean":"2023-09-23T18:05:49.919261+0000","last_became_active":"2023-09-23T18:05:15.096022+0000","last_became_peered":"2023-09-23T18:05:15.096022+0000","last_unstale":"2023-09-23T18:05:49.919261+0000","last_undegraded":"2023-09-23T18:05:49.919261+0000","last_fullsized":"2023-09-23T18:05:49.919261+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:33:32.387626+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":146,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.160947+0000","last_change":"2023-09-23T18:02:19.299374+0000","last_active":"2023-09-23T18:05:49.160947+0000","last_peered":"2023-09-23T18:05:49.160947+0000","last_clean":"2023-09-23T18:05:49.160947+0000","last_became_active":"2023-09-23T18:02:19.298500+0000","last_became_peered":"2023-09-23T18:02:19.298500+0000","last_unstale":"2023-09-23T18:05:49.160947+0000","last_undegraded":"2023-09-23T18:05:49.160947+0000","last_fullsized":"2023-09-23T18:05:49.160947+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:29.080645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":223,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159536+0000","last_change":"2023-09-23T18:02:01.799348+0000","last_active":"2023-09-23T18:05:49.159536+0000","last_peered":"2023-09-23T18:05:49.159536+0000","last_clean":"2023-09-23T18:05:49.159536+0000","last_became_active":"2023-09-23T18:02:01.799136+0000","last_became_peered":"2023-09-23T18:02:01.799136+0000","last_unstale":"2023-09-23T18:05:49.159536+0000","last_undegraded":"2023-09-23T18:05:49.159536+0000","last_fullsized":"2023-09-23T18:05:49.159536+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:54.458731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,0],"acting":[6,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":643,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:56.388716+0000","last_change":"2023-09-23T18:05:56.388682+0000","last_active":"2023-09-23T18:05:56.388716+0000","last_peered":"2023-09-23T18:05:56.388716+0000","last_clean":"2023-09-23T18:05:56.388716+0000","last_became_active":"2023-09-23T18:05:15.086214+0000","last_became_peered":"2023-09-23T18:05:15.086214+0000","last_unstale":"2023-09-23T18:05:56.388716+0000","last_undegraded":"2023-09-23T18:05:56.388716+0000","last_fullsized":"2023-09-23T18:05:56.388716+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:24.425630+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,3],"acting":[7,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":263,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916347+0000","last_change":"2023-09-23T18:02:18.979760+0000","last_active":"2023-09-23T18:05:49.916347+0000","last_peered":"2023-09-23T18:05:49.916347+0000","last_clean":"2023-09-23T18:05:49.916347+0000","last_became_active":"2023-09-23T18:02:18.979482+0000","last_became_peered":"2023-09-23T18:02:18.979482+0000","last_unstale":"2023-09-23T18:05:49.916347+0000","last_undegraded":"2023-09-23T18:05:49.916347+0000","last_fullsized":"2023-09-23T18:05:49.916347+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:00:54.123108+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":140,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.160976+0000","last_change":"2023-09-23T18:02:19.200325+0000","last_active":"2023-09-23T18:05:49.160976+0000","last_peered":"2023-09-23T18:05:49.160976+0000","last_clean":"2023-09-23T18:05:49.160976+0000","last_became_active":"2023-09-23T18:02:18.975795+0000","last_became_peered":"2023-09-23T18:02:18.975795+0000","last_unstale":"2023-09-23T18:05:49.160976+0000","last_undegraded":"2023-09-23T18:05:49.160976+0000","last_fullsized":"2023-09-23T18:05:49.160976+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:43.754725+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,4],"acting":[3,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":591,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.213216+0000","last_change":"2023-09-23T18:05:15.088596+0000","last_active":"2023-09-23T18:05:49.213216+0000","last_peered":"2023-09-23T18:05:49.213216+0000","last_clean":"2023-09-23T18:05:49.213216+0000","last_became_active":"2023-09-23T18:05:15.088205+0000","last_became_peered":"2023-09-23T18:05:15.088205+0000","last_unstale":"2023-09-23T18:05:49.213216+0000","last_undegraded":"2023-09-23T18:05:49.213216+0000","last_fullsized":"2023-09-23T18:05:49.213216+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":265,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886263+0000","last_change":"2023-09-23T18:02:19.025359+0000","last_active":"2023-09-23T18:05:49.886263+0000","last_peered":"2023-09-23T18:05:49.886263+0000","last_clean":"2023-09-23T18:05:49.886263+0000","last_became_active":"2023-09-23T18:02:19.024991+0000","last_became_peered":"2023-09-23T18:02:19.024991+0000","last_unstale":"2023-09-23T18:05:49.886263+0000","last_undegraded":"2023-09-23T18:05:49.886263+0000","last_fullsized":"2023-09-23T18:05:49.886263+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:53:04.685230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":94,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161228+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:05:49.161228+0000","last_peered":"2023-09-23T18:05:49.161228+0000","last_clean":"2023-09-23T18:05:49.161228+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:05:49.161228+0000","last_undegraded":"2023-09-23T18:05:49.161228+0000","last_fullsized":"2023-09-23T18:05:49.161228+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":597,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886503+0000","last_change":"2023-09-23T18:05:15.100844+0000","last_active":"2023-09-23T18:05:49.886503+0000","last_peered":"2023-09-23T18:05:49.886503+0000","last_clean":"2023-09-23T18:05:49.886503+0000","last_became_active":"2023-09-23T18:05:15.100365+0000","last_became_peered":"2023-09-23T18:05:15.100365+0000","last_unstale":"2023-09-23T18:05:49.886503+0000","last_undegraded":"2023-09-23T18:05:49.886503+0000","last_fullsized":"2023-09-23T18:05:49.886503+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":48,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918450+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:05:49.918450+0000","last_peered":"2023-09-23T18:05:49.918450+0000","last_clean":"2023-09-23T18:05:49.918450+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:05:49.918450+0000","last_undegraded":"2023-09-23T18:05:49.918450+0000","last_fullsized":"2023-09-23T18:05:49.918450+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":179,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159163+0000","last_change":"2023-09-23T18:02:01.803351+0000","last_active":"2023-09-23T18:05:49.159163+0000","last_peered":"2023-09-23T18:05:49.159163+0000","last_clean":"2023-09-23T18:05:49.159163+0000","last_became_active":"2023-09-23T18:02:01.803047+0000","last_became_peered":"2023-09-23T18:02:01.803047+0000","last_unstale":"2023-09-23T18:05:49.159163+0000","last_undegraded":"2023-09-23T18:05:49.159163+0000","last_fullsized":"2023-09-23T18:05:49.159163+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:08:19.299028+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,0],"acting":[6,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":604,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916089+0000","last_change":"2023-09-23T18:05:15.096980+0000","last_active":"2023-09-23T18:05:49.916089+0000","last_peered":"2023-09-23T18:05:49.916089+0000","last_clean":"2023-09-23T18:05:49.916089+0000","last_became_active":"2023-09-23T18:05:15.096456+0000","last_became_peered":"2023-09-23T18:05:15.096456+0000","last_unstale":"2023-09-23T18:05:49.916089+0000","last_undegraded":"2023-09-23T18:05:49.916089+0000","last_fullsized":"2023-09-23T18:05:49.916089+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,4],"acting":[0,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":83,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918144+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:05:49.918144+0000","last_peered":"2023-09-23T18:05:49.918144+0000","last_clean":"2023-09-23T18:05:49.918144+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:05:49.918144+0000","last_undegraded":"2023-09-23T18:05:49.918144+0000","last_fullsized":"2023-09-23T18:05:49.918144+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":83,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.919694+0000","last_change":"2023-09-23T18:02:01.771072+0000","last_active":"2023-09-23T18:05:49.919694+0000","last_peered":"2023-09-23T18:05:49.919694+0000","last_clean":"2023-09-23T18:05:49.919694+0000","last_became_active":"2023-09-23T18:02:01.770714+0000","last_became_peered":"2023-09-23T18:02:01.770714+0000","last_unstale":"2023-09-23T18:05:49.919694+0000","last_undegraded":"2023-09-23T18:05:49.919694+0000","last_fullsized":"2023-09-23T18:05:49.919694+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:56.605765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":594,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.915765+0000","last_change":"2023-09-23T18:05:15.095829+0000","last_active":"2023-09-23T18:05:49.915765+0000","last_peered":"2023-09-23T18:05:49.915765+0000","last_clean":"2023-09-23T18:05:49.915765+0000","last_became_active":"2023-09-23T18:05:15.095414+0000","last_became_peered":"2023-09-23T18:05:15.095414+0000","last_unstale":"2023-09-23T18:05:49.915765+0000","last_undegraded":"2023-09-23T18:05:49.915765+0000","last_fullsized":"2023-09-23T18:05:49.915765+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":211,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.919707+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:05:49.919707+0000","last_peered":"2023-09-23T18:05:49.919707+0000","last_clean":"2023-09-23T18:05:49.919707+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:05:49.919707+0000","last_undegraded":"2023-09-23T18:05:49.919707+0000","last_fullsized":"2023-09-23T18:05:49.919707+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":128,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.162019+0000","last_change":"2023-09-23T18:02:18.976075+0000","last_active":"2023-09-23T18:05:49.162019+0000","last_peered":"2023-09-23T18:05:49.162019+0000","last_clean":"2023-09-23T18:05:49.162019+0000","last_became_active":"2023-09-23T18:02:18.975763+0000","last_became_peered":"2023-09-23T18:02:18.975763+0000","last_unstale":"2023-09-23T18:05:49.162019+0000","last_undegraded":"2023-09-23T18:05:49.162019+0000","last_fullsized":"2023-09-23T18:05:49.162019+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:27:38.162189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":600,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885883+0000","last_change":"2023-09-23T18:05:15.091218+0000","last_active":"2023-09-23T18:05:49.885883+0000","last_peered":"2023-09-23T18:05:49.885883+0000","last_clean":"2023-09-23T18:05:49.885883+0000","last_became_active":"2023-09-23T18:05:15.090420+0000","last_became_peered":"2023-09-23T18:05:15.090420+0000","last_unstale":"2023-09-23T18:05:49.885883+0000","last_undegraded":"2023-09-23T18:05:49.885883+0000","last_fullsized":"2023-09-23T18:05:49.885883+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:57.754408+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":282,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916032+0000","last_change":"2023-09-23T18:02:47.959979+0000","last_active":"2023-09-23T18:05:49.916032+0000","last_peered":"2023-09-23T18:05:49.916032+0000","last_clean":"2023-09-23T18:05:49.916032+0000","last_became_active":"2023-09-23T18:02:47.959178+0000","last_became_peered":"2023-09-23T18:02:47.959178+0000","last_unstale":"2023-09-23T18:05:49.916032+0000","last_undegraded":"2023-09-23T18:05:49.916032+0000","last_fullsized":"2023-09-23T18:05:49.916032+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:38.599287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":260,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886364+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:05:49.886364+0000","last_peered":"2023-09-23T18:05:49.886364+0000","last_clean":"2023-09-23T18:05:49.886364+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:05:49.886364+0000","last_undegraded":"2023-09-23T18:05:49.886364+0000","last_fullsized":"2023-09-23T18:05:49.886364+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":585,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161289+0000","last_change":"2023-09-23T18:05:15.097196+0000","last_active":"2023-09-23T18:05:49.161289+0000","last_peered":"2023-09-23T18:05:49.161289+0000","last_clean":"2023-09-23T18:05:49.161289+0000","last_became_active":"2023-09-23T18:05:15.096800+0000","last_became_peered":"2023-09-23T18:05:15.096800+0000","last_unstale":"2023-09-23T18:05:49.161289+0000","last_undegraded":"2023-09-23T18:05:49.161289+0000","last_fullsized":"2023-09-23T18:05:49.161289+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":73,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918151+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:05:49.918151+0000","last_peered":"2023-09-23T18:05:49.918151+0000","last_clean":"2023-09-23T18:05:49.918151+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:05:49.918151+0000","last_undegraded":"2023-09-23T18:05:49.918151+0000","last_fullsized":"2023-09-23T18:05:49.918151+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":273,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916422+0000","last_change":"2023-09-23T18:02:47.960381+0000","last_active":"2023-09-23T18:05:49.916422+0000","last_peered":"2023-09-23T18:05:49.916422+0000","last_clean":"2023-09-23T18:05:49.916422+0000","last_became_active":"2023-09-23T18:02:47.959283+0000","last_became_peered":"2023-09-23T18:02:47.959283+0000","last_unstale":"2023-09-23T18:05:49.916422+0000","last_undegraded":"2023-09-23T18:05:49.916422+0000","last_fullsized":"2023-09-23T18:05:49.916422+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:44:08.695873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":600,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885987+0000","last_change":"2023-09-23T18:05:15.090180+0000","last_active":"2023-09-23T18:05:49.885987+0000","last_peered":"2023-09-23T18:05:49.885987+0000","last_clean":"2023-09-23T18:05:49.885987+0000","last_became_active":"2023-09-23T18:05:15.089754+0000","last_became_peered":"2023-09-23T18:05:15.089754+0000","last_unstale":"2023-09-23T18:05:49.885987+0000","last_undegraded":"2023-09-23T18:05:49.885987+0000","last_fullsized":"2023-09-23T18:05:49.885987+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:06.678700+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":219,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.158765+0000","last_change":"2023-09-23T18:00:52.902741+0000","last_active":"2023-09-23T18:05:49.158765+0000","last_peered":"2023-09-23T18:05:49.158765+0000","last_clean":"2023-09-23T18:05:49.158765+0000","last_became_active":"2023-09-23T18:00:52.902435+0000","last_became_peered":"2023-09-23T18:00:52.902435+0000","last_unstale":"2023-09-23T18:05:49.158765+0000","last_undegraded":"2023-09-23T18:05:49.158765+0000","last_fullsized":"2023-09-23T18:05:49.158765+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:45.911841+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":137,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161439+0000","last_change":"2023-09-23T18:02:47.968872+0000","last_active":"2023-09-23T18:05:49.161439+0000","last_peered":"2023-09-23T18:05:49.161439+0000","last_clean":"2023-09-23T18:05:49.161439+0000","last_became_active":"2023-09-23T18:02:47.968551+0000","last_became_peered":"2023-09-23T18:02:47.968551+0000","last_unstale":"2023-09-23T18:05:49.161439+0000","last_undegraded":"2023-09-23T18:05:49.161439+0000","last_fullsized":"2023-09-23T18:05:49.161439+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:08:54.966088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.b","version":"283'1065","reported_seq":1077,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:57.690965+0000","last_change":"2023-09-23T18:05:46.127510+0000","last_active":"2023-09-23T18:05:57.690965+0000","last_peered":"2023-09-23T18:05:57.690965+0000","last_clean":"2023-09-23T18:05:57.690965+0000","last_became_active":"2023-09-23T18:05:46.127237+0000","last_became_peered":"2023-09-23T18:05:46.127237+0000","last_unstale":"2023-09-23T18:05:57.690965+0000","last_undegraded":"2023-09-23T18:05:57.690965+0000","last_fullsized":"2023-09-23T18:05:57.690965+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":1065,"log_dups_size":0,"ondisk_log_size":1065,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:28:46.020682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69795840,"num_objects":1065,"num_object_clones":0,"num_object_copies":3195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1065,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1065,"num_write_kb":68160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,2],"acting":[5,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":633,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.917990+0000","last_change":"2023-09-23T18:05:15.096714+0000","last_active":"2023-09-23T18:05:49.917990+0000","last_peered":"2023-09-23T18:05:49.917990+0000","last_clean":"2023-09-23T18:05:49.917990+0000","last_became_active":"2023-09-23T18:05:15.096260+0000","last_became_peered":"2023-09-23T18:05:15.096260+0000","last_unstale":"2023-09-23T18:05:49.917990+0000","last_undegraded":"2023-09-23T18:05:49.917990+0000","last_fullsized":"2023-09-23T18:05:49.917990+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":211,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918406+0000","last_change":"2023-09-23T18:01:39.488433+0000","last_active":"2023-09-23T18:05:49.918406+0000","last_peered":"2023-09-23T18:05:49.918406+0000","last_clean":"2023-09-23T18:05:49.918406+0000","last_became_active":"2023-09-23T18:01:39.469581+0000","last_became_peered":"2023-09-23T18:01:39.469581+0000","last_unstale":"2023-09-23T18:05:49.918406+0000","last_undegraded":"2023-09-23T18:05:49.918406+0000","last_fullsized":"2023-09-23T18:05:49.918406+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:49.270390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":44,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":271,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885990+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:05:49.885990+0000","last_peered":"2023-09-23T18:05:49.885990+0000","last_clean":"2023-09-23T18:05:49.885990+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:05:49.885990+0000","last_undegraded":"2023-09-23T18:05:49.885990+0000","last_fullsized":"2023-09-23T18:05:49.885990+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"283'836","reported_seq":849,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:56.184272+0000","last_change":"2023-09-23T18:05:46.126616+0000","last_active":"2023-09-23T18:05:56.184272+0000","last_peered":"2023-09-23T18:05:56.184272+0000","last_clean":"2023-09-23T18:05:56.184272+0000","last_became_active":"2023-09-23T18:05:46.126346+0000","last_became_peered":"2023-09-23T18:05:46.126346+0000","last_unstale":"2023-09-23T18:05:56.184272+0000","last_undegraded":"2023-09-23T18:05:56.184272+0000","last_fullsized":"2023-09-23T18:05:56.184272+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":836,"log_dups_size":0,"ondisk_log_size":836,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:57.668631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54788096,"num_objects":836,"num_object_clones":0,"num_object_copies":2508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":836,"num_write_kb":53504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":621,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161744+0000","last_change":"2023-09-23T18:05:15.083980+0000","last_active":"2023-09-23T18:05:49.161744+0000","last_peered":"2023-09-23T18:05:49.161744+0000","last_clean":"2023-09-23T18:05:49.161744+0000","last_became_active":"2023-09-23T18:05:15.083516+0000","last_became_peered":"2023-09-23T18:05:15.083516+0000","last_unstale":"2023-09-23T18:05:49.161744+0000","last_undegraded":"2023-09-23T18:05:49.161744+0000","last_fullsized":"2023-09-23T18:05:49.161744+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1064488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":226,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918854+0000","last_change":"2023-09-23T18:02:01.807964+0000","last_active":"2023-09-23T18:05:49.918854+0000","last_peered":"2023-09-23T18:05:49.918854+0000","last_clean":"2023-09-23T18:05:49.918854+0000","last_became_active":"2023-09-23T18:02:01.803843+0000","last_became_peered":"2023-09-23T18:02:01.803843+0000","last_unstale":"2023-09-23T18:05:49.918854+0000","last_undegraded":"2023-09-23T18:05:49.918854+0000","last_fullsized":"2023-09-23T18:05:49.918854+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:49.358609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":293,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918306+0000","last_change":"2023-09-23T18:02:18.945712+0000","last_active":"2023-09-23T18:05:49.918306+0000","last_peered":"2023-09-23T18:05:49.918306+0000","last_clean":"2023-09-23T18:05:49.918306+0000","last_became_active":"2023-09-23T18:02:18.945426+0000","last_became_peered":"2023-09-23T18:02:18.945426+0000","last_unstale":"2023-09-23T18:05:49.918306+0000","last_undegraded":"2023-09-23T18:05:49.918306+0000","last_fullsized":"2023-09-23T18:05:49.918306+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:35.270078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"283'578","reported_seq":591,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:54.632749+0000","last_change":"2023-09-23T18:05:46.130561+0000","last_active":"2023-09-23T18:05:54.632749+0000","last_peered":"2023-09-23T18:05:54.632749+0000","last_clean":"2023-09-23T18:05:54.632749+0000","last_became_active":"2023-09-23T18:05:46.130243+0000","last_became_peered":"2023-09-23T18:05:46.130243+0000","last_unstale":"2023-09-23T18:05:54.632749+0000","last_undegraded":"2023-09-23T18:05:54.632749+0000","last_fullsized":"2023-09-23T18:05:54.632749+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":578,"log_dups_size":0,"ondisk_log_size":578,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:29.290251+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":37879808,"num_objects":578,"num_object_clones":0,"num_object_copies":1734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":578,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":578,"num_write_kb":36992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":444,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159677+0000","last_change":"2023-09-23T18:05:15.099144+0000","last_active":"2023-09-23T18:05:49.159677+0000","last_peered":"2023-09-23T18:05:49.159677+0000","last_clean":"2023-09-23T18:05:49.159677+0000","last_became_active":"2023-09-23T18:05:15.098234+0000","last_became_peered":"2023-09-23T18:05:15.098234+0000","last_unstale":"2023-09-23T18:05:49.159677+0000","last_undegraded":"2023-09-23T18:05:49.159677+0000","last_fullsized":"2023-09-23T18:05:49.159677+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":149,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918603+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:05:49.918603+0000","last_peered":"2023-09-23T18:05:49.918603+0000","last_clean":"2023-09-23T18:05:49.918603+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:05:49.918603+0000","last_undegraded":"2023-09-23T18:05:49.918603+0000","last_fullsized":"2023-09-23T18:05:49.918603+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":248,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159082+0000","last_change":"2023-09-23T18:02:47.947455+0000","last_active":"2023-09-23T18:05:49.159082+0000","last_peered":"2023-09-23T18:05:49.159082+0000","last_clean":"2023-09-23T18:05:49.159082+0000","last_became_active":"2023-09-23T18:02:47.947160+0000","last_became_peered":"2023-09-23T18:02:47.947160+0000","last_unstale":"2023-09-23T18:05:49.159082+0000","last_undegraded":"2023-09-23T18:05:49.159082+0000","last_fullsized":"2023-09-23T18:05:49.159082+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:58:38.356978+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"283'560","reported_seq":573,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:54.376469+0000","last_change":"2023-09-23T18:05:46.123859+0000","last_active":"2023-09-23T18:05:54.376469+0000","last_peered":"2023-09-23T18:05:54.376469+0000","last_clean":"2023-09-23T18:05:54.376469+0000","last_became_active":"2023-09-23T18:05:46.123482+0000","last_became_peered":"2023-09-23T18:05:46.123482+0000","last_unstale":"2023-09-23T18:05:54.376469+0000","last_undegraded":"2023-09-23T18:05:54.376469+0000","last_fullsized":"2023-09-23T18:05:54.376469+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":560,"log_dups_size":0,"ondisk_log_size":560,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:50.537709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36700160,"num_objects":560,"num_object_clones":0,"num_object_copies":1680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":560,"num_write_kb":35840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":607,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:56.362788+0000","last_change":"2023-09-23T18:05:56.362746+0000","last_active":"2023-09-23T18:05:56.362788+0000","last_peered":"2023-09-23T18:05:56.362788+0000","last_clean":"2023-09-23T18:05:56.362788+0000","last_became_active":"2023-09-23T18:05:15.102072+0000","last_became_peered":"2023-09-23T18:05:15.102072+0000","last_unstale":"2023-09-23T18:05:56.362788+0000","last_undegraded":"2023-09-23T18:05:56.362788+0000","last_fullsized":"2023-09-23T18:05:56.362788+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.362652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.362652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:33.381515+0000","scrub_duration":0.0052581759999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":259,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886106+0000","last_change":"2023-09-23T18:02:18.954614+0000","last_active":"2023-09-23T18:05:49.886106+0000","last_peered":"2023-09-23T18:05:49.886106+0000","last_clean":"2023-09-23T18:05:49.886106+0000","last_became_active":"2023-09-23T18:02:18.954191+0000","last_became_peered":"2023-09-23T18:02:18.954191+0000","last_unstale":"2023-09-23T18:05:49.886106+0000","last_undegraded":"2023-09-23T18:05:49.886106+0000","last_fullsized":"2023-09-23T18:05:49.886106+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:03.289944+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":285,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916058+0000","last_change":"2023-09-23T18:02:18.935101+0000","last_active":"2023-09-23T18:05:49.916058+0000","last_peered":"2023-09-23T18:05:49.916058+0000","last_clean":"2023-09-23T18:05:49.916058+0000","last_became_active":"2023-09-23T18:02:18.934813+0000","last_became_peered":"2023-09-23T18:02:18.934813+0000","last_unstale":"2023-09-23T18:05:49.916058+0000","last_undegraded":"2023-09-23T18:05:49.916058+0000","last_fullsized":"2023-09-23T18:05:49.916058+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:00.968578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"283'581","reported_seq":594,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:54.607558+0000","last_change":"2023-09-23T18:05:46.128878+0000","last_active":"2023-09-23T18:05:54.607558+0000","last_peered":"2023-09-23T18:05:54.607558+0000","last_clean":"2023-09-23T18:05:54.607558+0000","last_became_active":"2023-09-23T18:05:46.128615+0000","last_became_peered":"2023-09-23T18:05:46.128615+0000","last_unstale":"2023-09-23T18:05:54.607558+0000","last_undegraded":"2023-09-23T18:05:54.607558+0000","last_fullsized":"2023-09-23T18:05:54.607558+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":581,"log_dups_size":0,"ondisk_log_size":581,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:44.175736+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38076416,"num_objects":581,"num_object_clones":0,"num_object_copies":1743,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":581,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":581,"num_write_kb":37184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":635,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:55.369447+0000","last_change":"2023-09-23T18:05:55.369401+0000","last_active":"2023-09-23T18:05:55.369447+0000","last_peered":"2023-09-23T18:05:55.369447+0000","last_clean":"2023-09-23T18:05:55.369447+0000","last_became_active":"2023-09-23T18:05:15.086216+0000","last_became_peered":"2023-09-23T18:05:15.086216+0000","last_unstale":"2023-09-23T18:05:55.369447+0000","last_undegraded":"2023-09-23T18:05:55.369447+0000","last_fullsized":"2023-09-23T18:05:55.369447+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:44.931644+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":171,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161446+0000","last_change":"2023-09-23T18:02:18.943045+0000","last_active":"2023-09-23T18:05:49.161446+0000","last_peered":"2023-09-23T18:05:49.161446+0000","last_clean":"2023-09-23T18:05:49.161446+0000","last_became_active":"2023-09-23T18:02:18.942722+0000","last_became_peered":"2023-09-23T18:02:18.942722+0000","last_unstale":"2023-09-23T18:05:49.161446+0000","last_undegraded":"2023-09-23T18:05:49.161446+0000","last_fullsized":"2023-09-23T18:05:49.161446+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:49.910997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":291,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918221+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:05:49.918221+0000","last_peered":"2023-09-23T18:05:49.918221+0000","last_clean":"2023-09-23T18:05:49.918221+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:05:49.918221+0000","last_undegraded":"2023-09-23T18:05:49.918221+0000","last_fullsized":"2023-09-23T18:05:49.918221+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"283'518","reported_seq":530,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:54.361884+0000","last_change":"2023-09-23T18:05:46.128982+0000","last_active":"2023-09-23T18:05:54.361884+0000","last_peered":"2023-09-23T18:05:54.361884+0000","last_clean":"2023-09-23T18:05:54.361884+0000","last_became_active":"2023-09-23T18:05:46.128696+0000","last_became_peered":"2023-09-23T18:05:46.128696+0000","last_unstale":"2023-09-23T18:05:54.361884+0000","last_undegraded":"2023-09-23T18:05:54.361884+0000","last_fullsized":"2023-09-23T18:05:54.361884+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":518,"log_dups_size":0,"ondisk_log_size":518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:44.112131+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33947648,"num_objects":518,"num_object_clones":0,"num_object_copies":1554,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":518,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":518,"num_write_kb":33152,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":593,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885875+0000","last_change":"2023-09-23T18:05:15.083605+0000","last_active":"2023-09-23T18:05:49.885875+0000","last_peered":"2023-09-23T18:05:49.885875+0000","last_clean":"2023-09-23T18:05:49.885875+0000","last_became_active":"2023-09-23T18:05:15.083184+0000","last_became_peered":"2023-09-23T18:05:15.083184+0000","last_unstale":"2023-09-23T18:05:49.885875+0000","last_undegraded":"2023-09-23T18:05:49.885875+0000","last_fullsized":"2023-09-23T18:05:49.885875+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":211848,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":242,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159238+0000","last_change":"2023-09-23T18:02:18.968765+0000","last_active":"2023-09-23T18:05:49.159238+0000","last_peered":"2023-09-23T18:05:49.159238+0000","last_clean":"2023-09-23T18:05:49.159238+0000","last_became_active":"2023-09-23T18:02:18.968313+0000","last_became_peered":"2023-09-23T18:02:18.968313+0000","last_unstale":"2023-09-23T18:05:49.159238+0000","last_undegraded":"2023-09-23T18:05:49.159238+0000","last_fullsized":"2023-09-23T18:05:49.159238+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:05:36.726186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":83,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918699+0000","last_change":"2023-09-23T18:02:01.772348+0000","last_active":"2023-09-23T18:05:49.918699+0000","last_peered":"2023-09-23T18:05:49.918699+0000","last_clean":"2023-09-23T18:05:49.918699+0000","last_became_active":"2023-09-23T18:02:01.772020+0000","last_became_peered":"2023-09-23T18:02:01.772020+0000","last_unstale":"2023-09-23T18:05:49.918699+0000","last_undegraded":"2023-09-23T18:05:49.918699+0000","last_fullsized":"2023-09-23T18:05:49.918699+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:30.062955+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"283'769","reported_seq":781,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:55.524930+0000","last_change":"2023-09-23T18:05:46.129778+0000","last_active":"2023-09-23T18:05:55.524930+0000","last_peered":"2023-09-23T18:05:55.524930+0000","last_clean":"2023-09-23T18:05:55.524930+0000","last_became_active":"2023-09-23T18:05:46.129527+0000","last_became_peered":"2023-09-23T18:05:46.129527+0000","last_unstale":"2023-09-23T18:05:55.524930+0000","last_undegraded":"2023-09-23T18:05:55.524930+0000","last_fullsized":"2023-09-23T18:05:55.524930+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":769,"log_dups_size":0,"ondisk_log_size":769,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:17.756683+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":50397184,"num_objects":769,"num_object_clones":0,"num_object_copies":2307,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":769,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":769,"num_write_kb":49216,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":604,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.915961+0000","last_change":"2023-09-23T18:05:15.082278+0000","last_active":"2023-09-23T18:05:49.915961+0000","last_peered":"2023-09-23T18:05:49.915961+0000","last_clean":"2023-09-23T18:05:49.915961+0000","last_became_active":"2023-09-23T18:05:15.081901+0000","last_became_peered":"2023-09-23T18:05:15.081901+0000","last_unstale":"2023-09-23T18:05:49.915961+0000","last_undegraded":"2023-09-23T18:05:49.915961+0000","last_fullsized":"2023-09-23T18:05:49.915961+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":26835,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":73,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918194+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:05:49.918194+0000","last_peered":"2023-09-23T18:05:49.918194+0000","last_clean":"2023-09-23T18:05:49.918194+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:05:49.918194+0000","last_undegraded":"2023-09-23T18:05:49.918194+0000","last_fullsized":"2023-09-23T18:05:49.918194+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":293,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.919005+0000","last_change":"2023-09-23T18:02:18.977589+0000","last_active":"2023-09-23T18:05:49.919005+0000","last_peered":"2023-09-23T18:05:49.919005+0000","last_clean":"2023-09-23T18:05:49.919005+0000","last_became_active":"2023-09-23T18:02:18.975269+0000","last_became_peered":"2023-09-23T18:02:18.975269+0000","last_unstale":"2023-09-23T18:05:49.919005+0000","last_undegraded":"2023-09-23T18:05:49.919005+0000","last_fullsized":"2023-09-23T18:05:49.919005+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:43:44.808334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"283'727","reported_seq":739,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:55.533390+0000","last_change":"2023-09-23T18:05:46.123519+0000","last_active":"2023-09-23T18:05:55.533390+0000","last_peered":"2023-09-23T18:05:55.533390+0000","last_clean":"2023-09-23T18:05:55.533390+0000","last_became_active":"2023-09-23T18:05:46.123243+0000","last_became_peered":"2023-09-23T18:05:46.123243+0000","last_unstale":"2023-09-23T18:05:55.533390+0000","last_undegraded":"2023-09-23T18:05:55.533390+0000","last_fullsized":"2023-09-23T18:05:55.533390+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":727,"log_dups_size":0,"ondisk_log_size":727,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:16:44.325296+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":47644672,"num_objects":727,"num_object_clones":0,"num_object_copies":2181,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":727,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":727,"num_write_kb":46528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":599,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:53.510001+0000","last_change":"2023-09-23T18:05:53.509955+0000","last_active":"2023-09-23T18:05:53.510001+0000","last_peered":"2023-09-23T18:05:53.510001+0000","last_clean":"2023-09-23T18:05:53.510001+0000","last_became_active":"2023-09-23T18:05:15.089224+0000","last_became_peered":"2023-09-23T18:05:15.089224+0000","last_unstale":"2023-09-23T18:05:53.510001+0000","last_undegraded":"2023-09-23T18:05:53.510001+0000","last_fullsized":"2023-09-23T18:05:53.510001+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:37.917870+0000","scrub_duration":0.001603043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":129,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161059+0000","last_change":"2023-09-23T18:02:18.941761+0000","last_active":"2023-09-23T18:05:49.161059+0000","last_peered":"2023-09-23T18:05:49.161059+0000","last_clean":"2023-09-23T18:05:49.161059+0000","last_became_active":"2023-09-23T18:02:18.941430+0000","last_became_peered":"2023-09-23T18:02:18.941430+0000","last_unstale":"2023-09-23T18:05:49.161059+0000","last_undegraded":"2023-09-23T18:05:49.161059+0000","last_fullsized":"2023-09-23T18:05:49.161059+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:50:20.929101+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,4],"acting":[3,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":129,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161514+0000","last_change":"2023-09-23T18:02:18.945048+0000","last_active":"2023-09-23T18:05:49.161514+0000","last_peered":"2023-09-23T18:05:49.161514+0000","last_clean":"2023-09-23T18:05:49.161514+0000","last_became_active":"2023-09-23T18:02:18.944784+0000","last_became_peered":"2023-09-23T18:02:18.944784+0000","last_unstale":"2023-09-23T18:05:49.161514+0000","last_undegraded":"2023-09-23T18:05:49.161514+0000","last_fullsized":"2023-09-23T18:05:49.161514+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:20:54.306113+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,7],"acting":[3,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.1","version":"283'1032","reported_seq":1044,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:58.233810+0000","last_change":"2023-09-23T18:05:46.124349+0000","last_active":"2023-09-23T18:05:58.233810+0000","last_peered":"2023-09-23T18:05:58.233810+0000","last_clean":"2023-09-23T18:05:58.233810+0000","last_became_active":"2023-09-23T18:05:46.124081+0000","last_became_peered":"2023-09-23T18:05:46.124081+0000","last_unstale":"2023-09-23T18:05:58.233810+0000","last_undegraded":"2023-09-23T18:05:58.233810+0000","last_fullsized":"2023-09-23T18:05:58.233810+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":1032,"log_dups_size":0,"ondisk_log_size":1032,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:46.818802+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":67633152,"num_objects":1032,"num_object_clones":0,"num_object_copies":3096,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1032,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1032,"num_write_kb":66048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":444,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159521+0000","last_change":"2023-09-23T18:05:15.099360+0000","last_active":"2023-09-23T18:05:49.159521+0000","last_peered":"2023-09-23T18:05:49.159521+0000","last_clean":"2023-09-23T18:05:49.159521+0000","last_became_active":"2023-09-23T18:05:15.098750+0000","last_became_peered":"2023-09-23T18:05:15.098750+0000","last_unstale":"2023-09-23T18:05:49.159521+0000","last_undegraded":"2023-09-23T18:05:49.159521+0000","last_fullsized":"2023-09-23T18:05:49.159521+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":592,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916065+0000","last_change":"2023-09-23T18:02:03.927870+0000","last_active":"2023-09-23T18:05:49.916065+0000","last_peered":"2023-09-23T18:05:49.916065+0000","last_clean":"2023-09-23T18:05:49.916065+0000","last_became_active":"2023-09-23T18:02:03.927420+0000","last_became_peered":"2023-09-23T18:02:03.927420+0000","last_unstale":"2023-09-23T18:05:49.916065+0000","last_undegraded":"2023-09-23T18:05:49.916065+0000","last_fullsized":"2023-09-23T18:05:49.916065+0000","mapping_epoch":229,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":230,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:06.430607+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,2],"acting":[0,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":263,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886448+0000","last_change":"2023-09-23T18:02:19.245738+0000","last_active":"2023-09-23T18:05:49.886448+0000","last_peered":"2023-09-23T18:05:49.886448+0000","last_clean":"2023-09-23T18:05:49.886448+0000","last_became_active":"2023-09-23T18:02:19.245470+0000","last_became_peered":"2023-09-23T18:02:19.245470+0000","last_unstale":"2023-09-23T18:05:49.886448+0000","last_undegraded":"2023-09-23T18:05:49.886448+0000","last_fullsized":"2023-09-23T18:05:49.886448+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:21.849617+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":260,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886133+0000","last_change":"2023-09-23T18:02:01.781610+0000","last_active":"2023-09-23T18:05:49.886133+0000","last_peered":"2023-09-23T18:05:49.886133+0000","last_clean":"2023-09-23T18:05:49.886133+0000","last_became_active":"2023-09-23T18:02:01.781198+0000","last_became_peered":"2023-09-23T18:02:01.781198+0000","last_unstale":"2023-09-23T18:05:49.886133+0000","last_undegraded":"2023-09-23T18:05:49.886133+0000","last_fullsized":"2023-09-23T18:05:49.886133+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:40:23.078867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.7","version":"283'754","reported_seq":766,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:55.501149+0000","last_change":"2023-09-23T18:05:46.129518+0000","last_active":"2023-09-23T18:05:55.501149+0000","last_peered":"2023-09-23T18:05:55.501149+0000","last_clean":"2023-09-23T18:05:55.501149+0000","last_became_active":"2023-09-23T18:05:46.129180+0000","last_became_peered":"2023-09-23T18:05:46.129180+0000","last_unstale":"2023-09-23T18:05:55.501149+0000","last_undegraded":"2023-09-23T18:05:55.501149+0000","last_fullsized":"2023-09-23T18:05:55.501149+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":754,"log_dups_size":0,"ondisk_log_size":754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:27.845759+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":49414144,"num_objects":754,"num_object_clones":0,"num_object_copies":2262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":754,"num_write_kb":48256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":600,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.885834+0000","last_change":"2023-09-23T18:05:15.083813+0000","last_active":"2023-09-23T18:05:49.885834+0000","last_peered":"2023-09-23T18:05:49.885834+0000","last_clean":"2023-09-23T18:05:49.885834+0000","last_became_active":"2023-09-23T18:05:15.083395+0000","last_became_peered":"2023-09-23T18:05:15.083395+0000","last_unstale":"2023-09-23T18:05:49.885834+0000","last_undegraded":"2023-09-23T18:05:49.885834+0000","last_fullsized":"2023-09-23T18:05:49.885834+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:24.402252+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1211,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886092+0000","last_change":"2023-09-23T18:02:04.947841+0000","last_active":"2023-09-23T18:05:49.886092+0000","last_peered":"2023-09-23T18:05:49.886092+0000","last_clean":"2023-09-23T18:05:49.886092+0000","last_became_active":"2023-09-23T18:02:04.947314+0000","last_became_peered":"2023-09-23T18:02:04.947314+0000","last_unstale":"2023-09-23T18:05:49.886092+0000","last_undegraded":"2023-09-23T18:05:49.886092+0000","last_fullsized":"2023-09-23T18:05:49.886092+0000","mapping_epoch":230,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":231,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:14.518573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,0],"acting":[5,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":73,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.919772+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:05:49.919772+0000","last_peered":"2023-09-23T18:05:49.919772+0000","last_clean":"2023-09-23T18:05:49.919772+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:05:49.919772+0000","last_undegraded":"2023-09-23T18:05:49.919772+0000","last_fullsized":"2023-09-23T18:05:49.919772+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":177,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.213317+0000","last_change":"2023-09-23T18:02:18.945551+0000","last_active":"2023-09-23T18:05:49.213317+0000","last_peered":"2023-09-23T18:05:49.213317+0000","last_clean":"2023-09-23T18:05:49.213317+0000","last_became_active":"2023-09-23T18:02:18.944950+0000","last_became_peered":"2023-09-23T18:02:18.944950+0000","last_unstale":"2023-09-23T18:05:49.213317+0000","last_undegraded":"2023-09-23T18:05:49.213317+0000","last_fullsized":"2023-09-23T18:05:49.213317+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:01.206400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,6],"acting":[2,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"7.4","version":"283'586","reported_seq":599,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:54.601679+0000","last_change":"2023-09-23T18:05:46.128848+0000","last_active":"2023-09-23T18:05:54.601679+0000","last_peered":"2023-09-23T18:05:54.601679+0000","last_clean":"2023-09-23T18:05:54.601679+0000","last_became_active":"2023-09-23T18:05:46.128536+0000","last_became_peered":"2023-09-23T18:05:46.128536+0000","last_unstale":"2023-09-23T18:05:54.601679+0000","last_undegraded":"2023-09-23T18:05:54.601679+0000","last_fullsized":"2023-09-23T18:05:54.601679+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":586,"log_dups_size":0,"ondisk_log_size":586,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:01.485297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38404096,"num_objects":586,"num_object_clones":0,"num_object_copies":1758,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":586,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":586,"num_write_kb":37504,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,6],"acting":[0,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":608,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916134+0000","last_change":"2023-09-23T18:05:15.087882+0000","last_active":"2023-09-23T18:05:49.916134+0000","last_peered":"2023-09-23T18:05:49.916134+0000","last_clean":"2023-09-23T18:05:49.916134+0000","last_became_active":"2023-09-23T18:05:15.087524+0000","last_became_peered":"2023-09-23T18:05:15.087524+0000","last_unstale":"2023-09-23T18:05:49.916134+0000","last_undegraded":"2023-09-23T18:05:49.916134+0000","last_fullsized":"2023-09-23T18:05:49.916134+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:55.015941+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":566,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.213076+0000","last_change":"2023-09-23T18:02:17.947473+0000","last_active":"2023-09-23T18:05:49.213076+0000","last_peered":"2023-09-23T18:05:49.213076+0000","last_clean":"2023-09-23T18:05:49.213076+0000","last_became_active":"2023-09-23T18:02:17.946882+0000","last_became_peered":"2023-09-23T18:02:17.946882+0000","last_unstale":"2023-09-23T18:05:49.213076+0000","last_undegraded":"2023-09-23T18:05:49.213076+0000","last_fullsized":"2023-09-23T18:05:49.213076+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:53.918772+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":178,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.213062+0000","last_change":"2023-09-23T18:02:01.780394+0000","last_active":"2023-09-23T18:05:49.213062+0000","last_peered":"2023-09-23T18:05:49.213062+0000","last_clean":"2023-09-23T18:05:49.213062+0000","last_became_active":"2023-09-23T18:02:01.780019+0000","last_became_peered":"2023-09-23T18:02:01.780019+0000","last_unstale":"2023-09-23T18:05:49.213062+0000","last_undegraded":"2023-09-23T18:05:49.213062+0000","last_fullsized":"2023-09-23T18:05:49.213062+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:29:49.105957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,6,0],"acting":[2,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":250,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886440+0000","last_change":"2023-09-23T18:02:01.787461+0000","last_active":"2023-09-23T18:05:49.886440+0000","last_peered":"2023-09-23T18:05:49.886440+0000","last_clean":"2023-09-23T18:05:49.886440+0000","last_became_active":"2023-09-23T18:02:01.787139+0000","last_became_peered":"2023-09-23T18:02:01.787139+0000","last_unstale":"2023-09-23T18:05:49.886440+0000","last_undegraded":"2023-09-23T18:05:49.886440+0000","last_fullsized":"2023-09-23T18:05:49.886440+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:29.515097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.6","version":"283'612","reported_seq":625,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:54.633461+0000","last_change":"2023-09-23T18:05:46.129744+0000","last_active":"2023-09-23T18:05:54.633461+0000","last_peered":"2023-09-23T18:05:54.633461+0000","last_clean":"2023-09-23T18:05:54.633461+0000","last_became_active":"2023-09-23T18:05:46.129427+0000","last_became_peered":"2023-09-23T18:05:46.129427+0000","last_unstale":"2023-09-23T18:05:54.633461+0000","last_undegraded":"2023-09-23T18:05:54.633461+0000","last_fullsized":"2023-09-23T18:05:54.633461+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":612,"log_dups_size":0,"ondisk_log_size":612,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:08.394345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":40108032,"num_objects":612,"num_object_clones":0,"num_object_copies":1836,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":612,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":612,"num_write_kb":39168,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1195,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886089+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:05:49.886089+0000","last_peered":"2023-09-23T18:05:49.886089+0000","last_clean":"2023-09-23T18:05:49.886089+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:05:49.886089+0000","last_undegraded":"2023-09-23T18:05:49.886089+0000","last_fullsized":"2023-09-23T18:05:49.886089+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":618,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:53.385669+0000","last_change":"2023-09-23T18:05:53.385624+0000","last_active":"2023-09-23T18:05:53.385669+0000","last_peered":"2023-09-23T18:05:53.385669+0000","last_clean":"2023-09-23T18:05:53.385669+0000","last_became_active":"2023-09-23T18:02:51.049142+0000","last_became_peered":"2023-09-23T18:02:51.049142+0000","last_unstale":"2023-09-23T18:05:53.385669+0000","last_undegraded":"2023-09-23T18:05:53.385669+0000","last_fullsized":"2023-09-23T18:05:53.385669+0000","mapping_epoch":252,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":253,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:22.047001+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":91,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.163354+0000","last_change":"2023-09-23T18:02:19.022074+0000","last_active":"2023-09-23T18:05:49.163354+0000","last_peered":"2023-09-23T18:05:49.163354+0000","last_clean":"2023-09-23T18:05:49.163354+0000","last_became_active":"2023-09-23T18:02:19.021717+0000","last_became_peered":"2023-09-23T18:02:19.021717+0000","last_unstale":"2023-09-23T18:05:49.163354+0000","last_undegraded":"2023-09-23T18:05:49.163354+0000","last_fullsized":"2023-09-23T18:05:49.163354+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:58:25.270100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":216,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159476+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:05:49.159476+0000","last_peered":"2023-09-23T18:05:49.159476+0000","last_clean":"2023-09-23T18:05:49.159476+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:05:49.159476+0000","last_undegraded":"2023-09-23T18:05:49.159476+0000","last_fullsized":"2023-09-23T18:05:49.159476+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"283'1174","reported_seq":1186,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:58.007846+0000","last_change":"2023-09-23T18:05:46.131311+0000","last_active":"2023-09-23T18:05:58.007846+0000","last_peered":"2023-09-23T18:05:58.007846+0000","last_clean":"2023-09-23T18:05:58.007846+0000","last_became_active":"2023-09-23T18:05:46.131042+0000","last_became_peered":"2023-09-23T18:05:46.131042+0000","last_unstale":"2023-09-23T18:05:58.007846+0000","last_undegraded":"2023-09-23T18:05:58.007846+0000","last_fullsized":"2023-09-23T18:05:58.007846+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":1174,"log_dups_size":0,"ondisk_log_size":1174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:21.768330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76939264,"num_objects":1174,"num_object_clones":0,"num_object_copies":3522,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1174,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1174,"num_write_kb":75136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,0],"acting":[7,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":444,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159122+0000","last_change":"2023-09-23T18:05:15.096283+0000","last_active":"2023-09-23T18:05:49.159122+0000","last_peered":"2023-09-23T18:05:49.159122+0000","last_clean":"2023-09-23T18:05:49.159122+0000","last_became_active":"2023-09-23T18:05:15.095885+0000","last_became_peered":"2023-09-23T18:05:15.095885+0000","last_unstale":"2023-09-23T18:05:49.159122+0000","last_undegraded":"2023-09-23T18:05:49.159122+0000","last_fullsized":"2023-09-23T18:05:49.159122+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:08.260625+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:08.260625+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":462,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.917879+0000","last_change":"2023-09-23T18:02:47.971985+0000","last_active":"2023-09-23T18:05:49.917879+0000","last_peered":"2023-09-23T18:05:49.917879+0000","last_clean":"2023-09-23T18:05:49.917879+0000","last_became_active":"2023-09-23T18:02:18.962941+0000","last_became_peered":"2023-09-23T18:02:18.962941+0000","last_unstale":"2023-09-23T18:05:49.917879+0000","last_undegraded":"2023-09-23T18:05:49.917879+0000","last_fullsized":"2023-09-23T18:05:49.917879+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:37.126710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,3],"acting":[7,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":282,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.915866+0000","last_change":"2023-09-23T18:02:18.948516+0000","last_active":"2023-09-23T18:05:49.915866+0000","last_peered":"2023-09-23T18:05:49.915866+0000","last_clean":"2023-09-23T18:05:49.915866+0000","last_became_active":"2023-09-23T18:02:18.948250+0000","last_became_peered":"2023-09-23T18:02:18.948250+0000","last_unstale":"2023-09-23T18:05:49.915866+0000","last_undegraded":"2023-09-23T18:05:49.915866+0000","last_fullsized":"2023-09-23T18:05:49.915866+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:54.106489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":252,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916090+0000","last_change":"2023-09-23T18:02:01.783199+0000","last_active":"2023-09-23T18:05:49.916090+0000","last_peered":"2023-09-23T18:05:49.916090+0000","last_clean":"2023-09-23T18:05:49.916090+0000","last_became_active":"2023-09-23T18:02:01.782874+0000","last_became_peered":"2023-09-23T18:02:01.782874+0000","last_unstale":"2023-09-23T18:05:49.916090+0000","last_undegraded":"2023-09-23T18:05:49.916090+0000","last_fullsized":"2023-09-23T18:05:49.916090+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:29:00.533156+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,7],"acting":[0,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.0","version":"283'1055","reported_seq":1067,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:57.685845+0000","last_change":"2023-09-23T18:05:46.127646+0000","last_active":"2023-09-23T18:05:57.685845+0000","last_peered":"2023-09-23T18:05:57.685845+0000","last_clean":"2023-09-23T18:05:57.685845+0000","last_became_active":"2023-09-23T18:05:46.127313+0000","last_became_peered":"2023-09-23T18:05:46.127313+0000","last_unstale":"2023-09-23T18:05:57.685845+0000","last_undegraded":"2023-09-23T18:05:57.685845+0000","last_fullsized":"2023-09-23T18:05:57.685845+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":1055,"log_dups_size":0,"ondisk_log_size":1055,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:34.513090+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":69140480,"num_objects":1055,"num_object_clones":0,"num_object_copies":3165,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1055,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1055,"num_write_kb":67520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":444,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159507+0000","last_change":"2023-09-23T18:05:15.085455+0000","last_active":"2023-09-23T18:05:49.159507+0000","last_peered":"2023-09-23T18:05:49.159507+0000","last_clean":"2023-09-23T18:05:49.159507+0000","last_became_active":"2023-09-23T18:05:15.085106+0000","last_became_peered":"2023-09-23T18:05:15.085106+0000","last_unstale":"2023-09-23T18:05:49.159507+0000","last_undegraded":"2023-09-23T18:05:49.159507+0000","last_fullsized":"2023-09-23T18:05:49.159507+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":496,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918542+0000","last_change":"2023-09-23T18:01:39.494877+0000","last_active":"2023-09-23T18:05:49.918542+0000","last_peered":"2023-09-23T18:05:49.918542+0000","last_clean":"2023-09-23T18:05:49.918542+0000","last_became_active":"2023-09-23T18:01:39.461163+0000","last_became_peered":"2023-09-23T18:01:39.461163+0000","last_unstale":"2023-09-23T18:05:49.918542+0000","last_undegraded":"2023-09-23T18:05:49.918542+0000","last_fullsized":"2023-09-23T18:05:49.918542+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:47.295384+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":218,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.213372+0000","last_change":"2023-09-23T18:02:01.787518+0000","last_active":"2023-09-23T18:05:49.213372+0000","last_peered":"2023-09-23T18:05:49.213372+0000","last_clean":"2023-09-23T18:05:49.213372+0000","last_became_active":"2023-09-23T18:02:01.787176+0000","last_became_peered":"2023-09-23T18:02:01.787176+0000","last_unstale":"2023-09-23T18:05:49.213372+0000","last_undegraded":"2023-09-23T18:05:49.213372+0000","last_fullsized":"2023-09-23T18:05:49.213372+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:38:41.415950+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,0],"acting":[2,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":250,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916331+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:05:49.916331+0000","last_peered":"2023-09-23T18:05:49.916331+0000","last_clean":"2023-09-23T18:05:49.916331+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:05:49.916331+0000","last_undegraded":"2023-09-23T18:05:49.916331+0000","last_fullsized":"2023-09-23T18:05:49.916331+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"283'852","reported_seq":865,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:56.200210+0000","last_change":"2023-09-23T18:05:46.122448+0000","last_active":"2023-09-23T18:05:56.200210+0000","last_peered":"2023-09-23T18:05:56.200210+0000","last_clean":"2023-09-23T18:05:56.200210+0000","last_became_active":"2023-09-23T18:05:46.122155+0000","last_became_peered":"2023-09-23T18:05:46.122155+0000","last_unstale":"2023-09-23T18:05:56.200210+0000","last_undegraded":"2023-09-23T18:05:56.200210+0000","last_fullsized":"2023-09-23T18:05:56.200210+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":852,"log_dups_size":0,"ondisk_log_size":852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:40.972879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":55836672,"num_objects":852,"num_object_clones":0,"num_object_copies":2556,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":852,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":852,"num_write_kb":54528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":608,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:54.350998+0000","last_change":"2023-09-23T18:05:54.350959+0000","last_active":"2023-09-23T18:05:54.350998+0000","last_peered":"2023-09-23T18:05:54.350998+0000","last_clean":"2023-09-23T18:05:54.350998+0000","last_became_active":"2023-09-23T18:05:15.096059+0000","last_became_peered":"2023-09-23T18:05:15.096059+0000","last_unstale":"2023-09-23T18:05:54.350998+0000","last_undegraded":"2023-09-23T18:05:54.350998+0000","last_fullsized":"2023-09-23T18:05:54.350998+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:38.722243+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":129,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161485+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:05:49.161485+0000","last_peered":"2023-09-23T18:05:49.161485+0000","last_clean":"2023-09-23T18:05:49.161485+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:05:49.161485+0000","last_undegraded":"2023-09-23T18:05:49.161485+0000","last_fullsized":"2023-09-23T18:05:49.161485+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":353,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159427+0000","last_change":"2023-09-23T18:02:18.981970+0000","last_active":"2023-09-23T18:05:49.159427+0000","last_peered":"2023-09-23T18:05:49.159427+0000","last_clean":"2023-09-23T18:05:49.159427+0000","last_became_active":"2023-09-23T18:02:18.981590+0000","last_became_peered":"2023-09-23T18:02:18.981590+0000","last_unstale":"2023-09-23T18:05:49.159427+0000","last_undegraded":"2023-09-23T18:05:49.159427+0000","last_fullsized":"2023-09-23T18:05:49.159427+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:18.182980+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":133,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161037+0000","last_change":"2023-09-23T18:02:47.957919+0000","last_active":"2023-09-23T18:05:49.161037+0000","last_peered":"2023-09-23T18:05:49.161037+0000","last_clean":"2023-09-23T18:05:49.161037+0000","last_became_active":"2023-09-23T18:02:47.957545+0000","last_became_peered":"2023-09-23T18:02:47.957545+0000","last_unstale":"2023-09-23T18:05:49.161037+0000","last_undegraded":"2023-09-23T18:05:49.161037+0000","last_fullsized":"2023-09-23T18:05:49.161037+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:57.354345+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.2","version":"283'833","reported_seq":846,"reported_epoch":283,"state":"active+clean","last_fresh":"2023-09-23T18:05:56.195343+0000","last_change":"2023-09-23T18:05:46.122766+0000","last_active":"2023-09-23T18:05:56.195343+0000","last_peered":"2023-09-23T18:05:56.195343+0000","last_clean":"2023-09-23T18:05:56.195343+0000","last_became_active":"2023-09-23T18:05:46.122467+0000","last_became_peered":"2023-09-23T18:05:46.122467+0000","last_unstale":"2023-09-23T18:05:56.195343+0000","last_undegraded":"2023-09-23T18:05:56.195343+0000","last_fullsized":"2023-09-23T18:05:56.195343+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":833,"log_dups_size":0,"ondisk_log_size":833,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:04:35.797941+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54591488,"num_objects":833,"num_object_clones":0,"num_object_copies":2499,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":833,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":833,"num_write_kb":53312,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,7],"acting":[3,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":603,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161662+0000","last_change":"2023-09-23T18:05:15.088351+0000","last_active":"2023-09-23T18:05:49.161662+0000","last_peered":"2023-09-23T18:05:49.161662+0000","last_clean":"2023-09-23T18:05:49.161662+0000","last_became_active":"2023-09-23T18:05:15.087902+0000","last_became_peered":"2023-09-23T18:05:15.087902+0000","last_unstale":"2023-09-23T18:05:49.161662+0000","last_undegraded":"2023-09-23T18:05:49.161662+0000","last_fullsized":"2023-09-23T18:05:49.161662+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:02:50.479420+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":488,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.159623+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:05:49.159623+0000","last_peered":"2023-09-23T18:05:49.159623+0000","last_clean":"2023-09-23T18:05:49.159623+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:05:49.159623+0000","last_undegraded":"2023-09-23T18:05:49.159623+0000","last_fullsized":"2023-09-23T18:05:49.159623+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":309,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916229+0000","last_change":"2023-09-23T18:02:47.986869+0000","last_active":"2023-09-23T18:05:49.916229+0000","last_peered":"2023-09-23T18:05:49.916229+0000","last_clean":"2023-09-23T18:05:49.916229+0000","last_became_active":"2023-09-23T18:02:47.970544+0000","last_became_peered":"2023-09-23T18:02:47.970544+0000","last_unstale":"2023-09-23T18:05:49.916229+0000","last_undegraded":"2023-09-23T18:05:49.916229+0000","last_fullsized":"2023-09-23T18:05:49.916229+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:16.974348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":73,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918105+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:05:49.918105+0000","last_peered":"2023-09-23T18:05:49.918105+0000","last_clean":"2023-09-23T18:05:49.918105+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:05:49.918105+0000","last_undegraded":"2023-09-23T18:05:49.918105+0000","last_fullsized":"2023-09-23T18:05:49.918105+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":586,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161029+0000","last_change":"2023-09-23T18:05:15.083744+0000","last_active":"2023-09-23T18:05:49.161029+0000","last_peered":"2023-09-23T18:05:49.161029+0000","last_clean":"2023-09-23T18:05:49.161029+0000","last_became_active":"2023-09-23T18:05:15.083337+0000","last_became_peered":"2023-09-23T18:05:15.083337+0000","last_unstale":"2023-09-23T18:05:49.161029+0000","last_undegraded":"2023-09-23T18:05:49.161029+0000","last_fullsized":"2023-09-23T18:05:49.161029+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":269,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916242+0000","last_change":"2023-09-23T18:01:39.454573+0000","last_active":"2023-09-23T18:05:49.916242+0000","last_peered":"2023-09-23T18:05:49.916242+0000","last_clean":"2023-09-23T18:05:49.916242+0000","last_became_active":"2023-09-23T18:01:39.454270+0000","last_became_peered":"2023-09-23T18:01:39.454270+0000","last_unstale":"2023-09-23T18:05:49.916242+0000","last_undegraded":"2023-09-23T18:05:49.916242+0000","last_fullsized":"2023-09-23T18:05:49.916242+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:53.005749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":296,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.213399+0000","last_change":"2023-09-23T18:02:18.945979+0000","last_active":"2023-09-23T18:05:49.213399+0000","last_peered":"2023-09-23T18:05:49.213399+0000","last_clean":"2023-09-23T18:05:49.213399+0000","last_became_active":"2023-09-23T18:02:18.945721+0000","last_became_peered":"2023-09-23T18:02:18.945721+0000","last_unstale":"2023-09-23T18:05:49.213399+0000","last_undegraded":"2023-09-23T18:05:49.213399+0000","last_fullsized":"2023-09-23T18:05:49.213399+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:12:24.929946+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":595,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.162827+0000","last_change":"2023-09-23T18:05:15.092746+0000","last_active":"2023-09-23T18:05:49.162827+0000","last_peered":"2023-09-23T18:05:49.162827+0000","last_clean":"2023-09-23T18:05:49.162827+0000","last_became_active":"2023-09-23T18:05:15.092486+0000","last_became_peered":"2023-09-23T18:05:15.092486+0000","last_unstale":"2023-09-23T18:05:49.162827+0000","last_undegraded":"2023-09-23T18:05:49.162827+0000","last_fullsized":"2023-09-23T18:05:49.162827+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":138,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886259+0000","last_change":"2023-09-23T18:02:01.777300+0000","last_active":"2023-09-23T18:05:49.886259+0000","last_peered":"2023-09-23T18:05:49.886259+0000","last_clean":"2023-09-23T18:05:49.886259+0000","last_became_active":"2023-09-23T18:02:01.773164+0000","last_became_peered":"2023-09-23T18:02:01.773164+0000","last_unstale":"2023-09-23T18:05:49.886259+0000","last_undegraded":"2023-09-23T18:05:49.886259+0000","last_fullsized":"2023-09-23T18:05:49.886259+0000","mapping_epoch":226,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":227,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:09.608302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":208,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.919911+0000","last_change":"2023-09-23T18:02:18.944686+0000","last_active":"2023-09-23T18:05:49.919911+0000","last_peered":"2023-09-23T18:05:49.919911+0000","last_clean":"2023-09-23T18:05:49.919911+0000","last_became_active":"2023-09-23T18:02:18.944438+0000","last_became_peered":"2023-09-23T18:02:18.944438+0000","last_unstale":"2023-09-23T18:05:49.919911+0000","last_undegraded":"2023-09-23T18:05:49.919911+0000","last_fullsized":"2023-09-23T18:05:49.919911+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:34:11.325350+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":124,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.886272+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:05:49.886272+0000","last_peered":"2023-09-23T18:05:49.886272+0000","last_clean":"2023-09-23T18:05:49.886272+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:05:49.886272+0000","last_undegraded":"2023-09-23T18:05:49.886272+0000","last_fullsized":"2023-09-23T18:05:49.886272+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":148,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.163642+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:05:49.163642+0000","last_peered":"2023-09-23T18:05:49.163642+0000","last_clean":"2023-09-23T18:05:49.163642+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:05:49.163642+0000","last_undegraded":"2023-09-23T18:05:49.163642+0000","last_fullsized":"2023-09-23T18:05:49.163642+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":649,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.918111+0000","last_change":"2023-09-23T18:05:15.087831+0000","last_active":"2023-09-23T18:05:49.918111+0000","last_peered":"2023-09-23T18:05:49.918111+0000","last_clean":"2023-09-23T18:05:49.918111+0000","last_became_active":"2023-09-23T18:05:15.087299+0000","last_became_peered":"2023-09-23T18:05:15.087299+0000","last_unstale":"2023-09-23T18:05:49.918111+0000","last_undegraded":"2023-09-23T18:05:49.918111+0000","last_fullsized":"2023-09-23T18:05:49.918111+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:25:31.678814+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,5],"acting":[7,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":148,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.163026+0000","last_change":"2023-09-23T18:02:17.955616+0000","last_active":"2023-09-23T18:05:49.163026+0000","last_peered":"2023-09-23T18:05:49.163026+0000","last_clean":"2023-09-23T18:05:49.163026+0000","last_became_active":"2023-09-23T18:02:17.955323+0000","last_became_peered":"2023-09-23T18:02:17.955323+0000","last_unstale":"2023-09-23T18:05:49.163026+0000","last_undegraded":"2023-09-23T18:05:49.163026+0000","last_fullsized":"2023-09-23T18:05:49.163026+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:31.008773+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":151,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.161053+0000","last_change":"2023-09-23T18:02:17.948560+0000","last_active":"2023-09-23T18:05:49.161053+0000","last_peered":"2023-09-23T18:05:49.161053+0000","last_clean":"2023-09-23T18:05:49.161053+0000","last_became_active":"2023-09-23T18:02:17.948259+0000","last_became_peered":"2023-09-23T18:02:17.948259+0000","last_unstale":"2023-09-23T18:05:49.161053+0000","last_undegraded":"2023-09-23T18:05:49.161053+0000","last_fullsized":"2023-09-23T18:05:49.161053+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:33.758874+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":594,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.162470+0000","last_change":"2023-09-23T18:05:15.098282+0000","last_active":"2023-09-23T18:05:49.162470+0000","last_peered":"2023-09-23T18:05:49.162470+0000","last_clean":"2023-09-23T18:05:49.162470+0000","last_became_active":"2023-09-23T18:05:15.097779+0000","last_became_peered":"2023-09-23T18:05:15.097779+0000","last_unstale":"2023-09-23T18:05:49.162470+0000","last_undegraded":"2023-09-23T18:05:49.162470+0000","last_fullsized":"2023-09-23T18:05:49.162470+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":262,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.916416+0000","last_change":"2023-09-23T18:00:52.910986+0000","last_active":"2023-09-23T18:05:49.916416+0000","last_peered":"2023-09-23T18:05:49.916416+0000","last_clean":"2023-09-23T18:05:49.916416+0000","last_became_active":"2023-09-23T18:00:52.906426+0000","last_became_peered":"2023-09-23T18:00:52.906426+0000","last_unstale":"2023-09-23T18:05:49.916416+0000","last_undegraded":"2023-09-23T18:05:49.916416+0000","last_fullsized":"2023-09-23T18:05:49.916416+0000","mapping_epoch":199,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":200,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:43.697309+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":293,"reported_epoch":282,"state":"active+clean","last_fresh":"2023-09-23T18:05:49.917765+0000","last_change":"2023-09-23T18:02:18.945090+0000","last_active":"2023-09-23T18:05:49.917765+0000","last_peered":"2023-09-23T18:05:49.917765+0000","last_clean":"2023-09-23T18:05:49.917765+0000","last_became_active":"2023-09-23T18:02:18.944611+0000","last_became_peered":"2023-09-23T18:02:18.944611+0000","last_unstale":"2023-09-23T18:05:49.917765+0000","last_undegraded":"2023-09-23T18:05:49.917765+0000","last_fullsized":"2023-09-23T18:05:49.917765+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:38.364645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":821297152,"num_objects":12532,"num_object_clones":0,"num_object_copies":37596,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":12532,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12532,"num_write_kb":802048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2563637248,"data_stored":2563637248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":12532,"ondisk_log_size":12532,"up":48,"acting":48,"num_store_stats":7},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":6224,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":299658,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":246,"num_read_kb":465,"num_write":123,"num_write_kb":1227,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":3359290,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":57,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001400,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":570980,"kb_used_data":446428,"kb_used_omap":13,"kb_used_meta":124530,"kb_avail":93174172,"statfs":{"total":95995035648,"available":95410352128,"internally_reserved":0,"allocated":457142272,"data_stored":456080694,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":127518824},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":0.76100000000000001,"15min":0.66300000000000003},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":4.9889999999999999,"5min":6.2770000000000001,"15min":6.2770000000000001},"last":6.5309999999999997},{"interface":"front","average":{"1min":1.266,"5min":1.194,"15min":1.256},"min":{"1min":0.309,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":4.8209999999999997,"5min":6.641,"15min":6.641},"last":1.6659999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.026,"5min":0.97699999999999998,"15min":0.77700000000000002},"min":{"1min":0.27100000000000002,"5min":0.252,"15min":0.252},"max":{"1min":7.0309999999999997,"5min":7.1150000000000002,"15min":7.1150000000000002},"last":1.153},{"interface":"front","average":{"1min":1.466,"5min":1.1970000000000001,"15min":1.103},"min":{"1min":0.27600000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.9589999999999996,"5min":6.9589999999999996,"15min":6.9589999999999996},"last":6.4800000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.071,"15min":0.98199999999999998},"min":{"1min":0.28000000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.6630000000000003,"5min":6.6630000000000003,"15min":6.6630000000000003},"last":1.6930000000000001},{"interface":"front","average":{"1min":1.4259999999999999,"5min":1.1100000000000001,"15min":0.91200000000000003},"min":{"1min":0.316,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":7.9160000000000004,"5min":7.9160000000000004,"15min":7.9160000000000004},"last":2.161}]},{"osd":3,"last update":"Sat Sep 23 18:05:21 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.472,"15min":1.526},"min":{"1min":0.34599999999999997,"5min":0.316,"15min":0.316},"max":{"1min":7.0579999999999998,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":1.179},{"interface":"front","average":{"1min":2.4590000000000001,"5min":1.391,"15min":1.1100000000000001},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":15.349,"5min":15.349,"15min":15.349},"last":6.5599999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2070000000000001,"5min":1.073,"15min":1.0720000000000001},"min":{"1min":0.307,"5min":0.245,"15min":0.245},"max":{"1min":6.9729999999999999,"5min":6.9729999999999999,"15min":6.9729999999999999},"last":1.085},{"interface":"front","average":{"1min":1.1459999999999999,"5min":0.71399999999999997,"15min":0.58799999999999997},"min":{"1min":0.32200000000000001,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":5.0289999999999999,"5min":6.3479999999999999,"15min":6.3479999999999999},"last":0.29999999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6930000000000001,"5min":1.1759999999999999,"15min":1.0609999999999999},"min":{"1min":0.38,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":8.4749999999999996,"5min":9.5999999999999996,"15min":9.5999999999999996},"last":1.625},{"interface":"front","average":{"1min":1.6160000000000001,"5min":1.2709999999999999,"15min":1.179},"min":{"1min":0.39400000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.3739999999999997,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":1.1240000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":0.94599999999999995,"15min":0.78600000000000003},"min":{"1min":0.438,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":5.2469999999999999,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.33800000000000002},{"interface":"front","average":{"1min":1.1699999999999999,"5min":1.232,"15min":1.161},"min":{"1min":0.371,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":4.9539999999999997,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":1.585}]}]},{"osd":6,"up_from":163,"seq":700079669336,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":952408,"kb_used_data":776272,"kb_used_omap":7,"kb_used_meta":176120,"kb_avail":92792744,"statfs":{"total":95995035648,"available":95019769856,"internally_reserved":0,"allocated":794902528,"data_stored":793852567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":180347684},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":0.89900000000000002,"15min":1.2529999999999999},"min":{"1min":0.23100000000000001,"5min":0.224,"15min":0.215},"max":{"1min":1.7,"5min":15.407,"15min":15.407},"last":0.24399999999999999},{"interface":"front","average":{"1min":0.77300000000000002,"5min":0.96899999999999997,"15min":2.0470000000000002},"min":{"1min":0.29199999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":3.5600000000000001,"5min":7.5739999999999998,"15min":13.536},"last":1.1120000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.82199999999999995,"5min":1.044,"15min":1.7649999999999999},"min":{"1min":0.27300000000000002,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.161,"5min":6.7690000000000001,"15min":14.233000000000001},"last":0.435},{"interface":"front","average":{"1min":0.82999999999999996,"5min":1.1739999999999999,"15min":1.9650000000000001},"min":{"1min":0.44,"5min":0.317,"15min":0.248},"max":{"1min":3.5830000000000002,"5min":14.789,"15min":14.789},"last":0.90200000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":0.91100000000000003,"5min":1.1919999999999999,"15min":2.1030000000000002},"min":{"1min":0.41199999999999998,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":4.8410000000000002,"5min":7.173,"15min":11.210000000000001},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.2330000000000001,"5min":1.1439999999999999,"15min":2.4209999999999998},"min":{"1min":0.313,"5min":0.245,"15min":0.245},"max":{"1min":5.4829999999999997,"5min":9.3620000000000001,"15min":11.223000000000001},"last":0.374}]},{"osd":3,"last update":"Sat Sep 23 18:05:26 2023","interfaces":[{"interface":"back","average":{"1min":1.365,"5min":1.097,"15min":1.101},"min":{"1min":0.40200000000000002,"5min":0.308,"15min":0.308},"max":{"1min":4.7539999999999996,"5min":4.7539999999999996,"15min":4.7539999999999996},"last":0.85799999999999998},{"interface":"front","average":{"1min":1.6619999999999999,"5min":1.3260000000000001,"15min":1.2190000000000001},"min":{"1min":0.372,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":7.484,"5min":7.484,"15min":7.484},"last":1.8580000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0329999999999999,"5min":1.0109999999999999,"15min":0.91800000000000004},"min":{"1min":0.34200000000000003,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":5.5069999999999997,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.30499999999999999},{"interface":"front","average":{"1min":0.94399999999999995,"5min":0.81299999999999994,"15min":0.59999999999999998},"min":{"1min":0.32300000000000001,"5min":0.248,"15min":0.24099999999999999},"max":{"1min":6.3920000000000003,"5min":6.3920000000000003,"15min":8.3000000000000007},"last":0.35899999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:05:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3779999999999999,"5min":0.91000000000000003,"15min":0.89400000000000002},"min":{"1min":0.35899999999999999,"5min":0.30299999999999999,"15min":0.28599999999999998},"max":{"1min":5.4249999999999998,"5min":6.6159999999999997,"15min":11.869},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.625,"5min":0.95799999999999996,"15min":0.71799999999999997},"min":{"1min":0.377,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":1.73,"5min":5.4240000000000004,"15min":5.4240000000000004},"last":0.39300000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:05:31 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.075,"15min":1.0009999999999999},"min":{"1min":0.38400000000000001,"5min":0.307,"15min":0.307},"max":{"1min":5.5380000000000003,"5min":5.5380000000000003,"15min":5.5380000000000003},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.647,"5min":1.012,"15min":0.85799999999999998},"min":{"1min":0.40100000000000002,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":10.411,"5min":10.411,"15min":10.411},"last":0.41399999999999998}]}]},{"osd":1,"up_from":0,"seq":420906795155,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":0,"up_from":93,"seq":399431958681,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":727724,"kb_used_data":309412,"kb_used_omap":7,"kb_used_meta":418296,"kb_avail":93017428,"statfs":{"total":95995035648,"available":95249846272,"internally_reserved":0,"allocated":316837888,"data_stored":315423462,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":428335844},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:05:13 2023","interfaces":[{"interface":"back","average":{"1min":2.4590000000000001,"5min":1.2909999999999999,"15min":1.2809999999999999},"min":{"1min":0.22600000000000001,"5min":0.20300000000000001,"15min":0.17899999999999999},"max":{"1min":10.128,"5min":10.128,"15min":18.154},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.101,"15min":1.175},"min":{"1min":0.26900000000000002,"5min":0.186,"15min":0.17399999999999999},"max":{"1min":5.5490000000000004,"5min":12.641999999999999,"15min":13.369999999999999},"last":0.36299999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":1.528,"5min":1.1459999999999999,"15min":1.446},"min":{"1min":0.26500000000000001,"5min":0.21099999999999999,"15min":0.20499999999999999},"max":{"1min":10.48,"5min":10.48,"15min":20.882000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.96399999999999997,"5min":1.288,"15min":1.3420000000000001},"min":{"1min":0.23000000000000001,"5min":0.217,"15min":0.217},"max":{"1min":4.5599999999999996,"5min":12.673,"15min":15.064},"last":1.345}]},{"osd":3,"last update":"Sat Sep 23 18:05:21 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.5649999999999999,"15min":1.5569999999999999},"min":{"1min":0.245,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":11.948,"5min":11.948,"15min":11.948},"last":0.52900000000000003},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.347,"15min":1.2490000000000001},"min":{"1min":0.30599999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.9699999999999998,"5min":8.7300000000000004,"15min":8.7300000000000004},"last":4.4050000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93600000000000005,"5min":1.163,"15min":1.226},"min":{"1min":0.32600000000000001,"5min":0.29299999999999998,"15min":0.251},"max":{"1min":4.7510000000000003,"5min":5.4720000000000004,"15min":15.279999999999999},"last":1.2709999999999999},{"interface":"front","average":{"1min":1.8069999999999999,"5min":1.2669999999999999,"15min":1.3029999999999999},"min":{"1min":0.379,"5min":0.247,"15min":0.247},"max":{"1min":8.6129999999999995,"5min":10.164999999999999,"15min":18.106999999999999},"last":0.63700000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0569999999999999,"5min":1.3799999999999999,"15min":1.3819999999999999},"min":{"1min":0.40200000000000002,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":8.5730000000000004,"5min":10.194000000000001,"15min":14.720000000000001},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.1399999999999999,"5min":1.1519999999999999,"15min":1.1379999999999999},"min":{"1min":0.43099999999999999,"5min":0.254,"15min":0.254},"max":{"1min":5.5170000000000003,"5min":12.702,"15min":12.794},"last":0.32600000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":1.0429999999999999,"5min":1.1459999999999999,"15min":2.056},"min":{"1min":0.38800000000000001,"5min":0.35099999999999998,"15min":0.34599999999999997},"max":{"1min":5.6559999999999997,"5min":5.6559999999999997,"15min":14.956},"last":1.4610000000000001},{"interface":"front","average":{"1min":0.92100000000000004,"5min":1.081,"15min":2.181},"min":{"1min":0.48899999999999999,"5min":0.34699999999999998,"15min":0.311},"max":{"1min":2.9969999999999999,"5min":12.596,"15min":12.755000000000001},"last":0.40600000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:05:31 2023","interfaces":[{"interface":"back","average":{"1min":1.1839999999999999,"5min":1.036,"15min":0.95999999999999996},"min":{"1min":0.41999999999999998,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":5.4180000000000001,"5min":7.4649999999999999,"15min":7.4649999999999999},"last":0.432},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.385,"15min":1.1830000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.468,"5min":5.524,"15min":5.524},"last":1.496}]}]},{"osd":2,"up_from":103,"seq":442381631628,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":648656,"kb_used_data":265416,"kb_used_omap":11,"kb_used_meta":383220,"kb_avail":93096496,"statfs":{"total":95995035648,"available":95330811904,"internally_reserved":0,"allocated":271785984,"data_stored":270398424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":392417655},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":0.94099999999999995,"15min":1.431},"min":{"1min":0.20799999999999999,"5min":0.187,"15min":0.17199999999999999},"max":{"1min":4.6120000000000001,"5min":12.395,"15min":20.867000000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.95299999999999996,"15min":1.387},"min":{"1min":0.26000000000000001,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":4.5519999999999996,"5min":7.6550000000000002,"15min":21.681000000000001},"last":0.629}]},{"osd":1,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":0.94699999999999995,"15min":1.4730000000000001},"min":{"1min":0.28299999999999997,"5min":0.215,"15min":0.215},"max":{"1min":5.0140000000000002,"5min":7.2720000000000002,"15min":20.995000000000001},"last":3.1150000000000002},{"interface":"front","average":{"1min":0.504,"5min":0.83399999999999996,"15min":1.2549999999999999},"min":{"1min":0.255,"5min":0.193,"15min":0.193},"max":{"1min":2.0299999999999998,"5min":7.5720000000000001,"15min":15.945},"last":1.95}]},{"osd":3,"last update":"Sat Sep 23 18:05:26 2023","interfaces":[{"interface":"back","average":{"1min":0.80900000000000005,"5min":0.98899999999999999,"15min":0.85099999999999998},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":2.238,"5min":12.009,"15min":12.009},"last":1.919},{"interface":"front","average":{"1min":1.702,"5min":0.996,"15min":0.73899999999999999},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":11.162000000000001,"5min":11.162000000000001,"15min":11.162000000000001},"last":0.38500000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":1.4890000000000001,"5min":1.143,"15min":1.2030000000000001},"min":{"1min":0.43099999999999999,"5min":0.28399999999999997,"15min":0.26500000000000001},"max":{"1min":7.6799999999999997,"5min":9.8960000000000008,"15min":9.8960000000000008},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.95199999999999996,"5min":0.83099999999999996,"15min":1.1970000000000001},"min":{"1min":0.38600000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":4.6310000000000002,"5min":7.5609999999999999,"15min":20.920000000000002},"last":0.66800000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.85099999999999998,"5min":0.88400000000000001,"15min":1.264},"min":{"1min":0.34300000000000003,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":4.5140000000000002,"5min":7.3289999999999997,"15min":20.738},"last":0.73899999999999999},{"interface":"front","average":{"1min":1.0609999999999999,"5min":1.25,"15min":1.3580000000000001},"min":{"1min":0.32900000000000001,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":5.7140000000000004,"5min":12.103999999999999,"15min":15.496},"last":1.857}]},{"osd":6,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3140000000000001,"5min":1.22,"15min":1.998},"min":{"1min":0.40899999999999997,"5min":0.307,"15min":0.307},"max":{"1min":7.718,"5min":9.9480000000000004,"15min":9.9480000000000004},"last":0.51000000000000001},{"interface":"front","average":{"1min":0.876,"5min":1.0169999999999999,"15min":2.2160000000000002},"min":{"1min":0.40300000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.5810000000000004,"5min":8.4649999999999999,"15min":20.893000000000001},"last":0.70699999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:05:33 2023","interfaces":[{"interface":"back","average":{"1min":1.2470000000000001,"5min":1.3320000000000001,"15min":1.3839999999999999},"min":{"1min":0.42099999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":7.4989999999999997,"5min":12.051,"15min":12.051},"last":1.8979999999999999},{"interface":"front","average":{"1min":1.766,"5min":1.0009999999999999,"15min":0.755},"min":{"1min":0.45300000000000001,"5min":0.309,"15min":0.309},"max":{"1min":8.5039999999999996,"5min":8.5039999999999996,"15min":8.5039999999999996},"last":2.766}]}]},{"osd":3,"up_from":237,"seq":1017907249197,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":638628,"kb_used_data":457564,"kb_used_omap":3,"kb_used_meta":181052,"kb_avail":93106524,"statfs":{"total":95995035648,"available":95341080576,"internally_reserved":0,"allocated":468545536,"data_stored":466967855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":185397992},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.4730000000000001,"5min":0.91000000000000003,"15min":0.89200000000000002},"min":{"1min":0.20599999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.5440000000000005,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.54500000000000004},{"interface":"front","average":{"1min":2.0179999999999998,"5min":1.127,"15min":1.097},"min":{"1min":0.27000000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":14.885999999999999,"5min":14.885999999999999,"15min":14.885999999999999},"last":1.214}]},{"osd":1,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.508,"5min":1.2130000000000001,"15min":1.111},"min":{"1min":0.29799999999999999,"5min":0.245,"15min":0.245},"max":{"1min":21.931999999999999,"5min":21.931999999999999,"15min":21.931999999999999},"last":1.01},{"interface":"front","average":{"1min":1.625,"5min":1.01,"15min":0.96699999999999997},"min":{"1min":0.255,"5min":0.255,"15min":0.255},"max":{"1min":12.701000000000001,"5min":12.701000000000001,"15min":12.701000000000001},"last":1.796}]},{"osd":2,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.498,"5min":0.94899999999999995,"15min":0.88300000000000001},"min":{"1min":0.32500000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":8.2200000000000006,"5min":8.2200000000000006,"15min":8.2200000000000006},"last":0.94199999999999995},{"interface":"front","average":{"1min":2.46,"5min":1.3280000000000001,"15min":1.2729999999999999},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":21.988,"5min":21.988,"15min":21.988},"last":0.97899999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2789999999999999,"5min":1.2509999999999999,"15min":1.153},"min":{"1min":0.375,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":14.935,"5min":14.935,"15min":14.935},"last":1.534},{"interface":"front","average":{"1min":2.6600000000000001,"5min":1.3129999999999999,"15min":1.149},"min":{"1min":0.36899999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":12.786,"5min":12.786,"15min":12.786},"last":1.889}]},{"osd":5,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":1.3879999999999999,"15min":1.3180000000000001},"min":{"1min":0.48399999999999999,"5min":0.314,"15min":0.314},"max":{"1min":12.743,"5min":12.743,"15min":12.743},"last":1.7310000000000001},{"interface":"front","average":{"1min":1.827,"5min":1.2629999999999999,"15min":1.208},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":8.6479999999999997,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.75600000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3759999999999999,"5min":1.125,"15min":1.0980000000000001},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":7.5789999999999997,"5min":7.5789999999999997,"15min":7.5789999999999997},"last":2.4399999999999999},{"interface":"front","average":{"1min":2.5,"5min":1.5720000000000001,"15min":1.51},"min":{"1min":0.38100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":14.964,"5min":14.964,"15min":14.964},"last":1.252}]},{"osd":7,"last update":"Sat Sep 23 18:05:20 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":1.625,"15min":1.589},"min":{"1min":0.42599999999999999,"5min":0.377,"15min":0.377},"max":{"1min":12.821,"5min":12.821,"15min":12.821},"last":1.8400000000000001},{"interface":"front","average":{"1min":1.9470000000000001,"5min":1.5780000000000001,"15min":1.6060000000000001},"min":{"1min":0.41899999999999998,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":8.6039999999999992,"5min":8.6039999999999992,"15min":8.6039999999999992},"last":0.78900000000000003}]}]},{"osd":4,"up_from":30,"seq":128849019147,"num_pgs":52,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":559520,"kb_used_data":309016,"kb_used_omap":0,"kb_used_meta":250496,"kb_avail":93185632,"statfs":{"total":95995035648,"available":95422087168,"internally_reserved":0,"allocated":316432384,"data_stored":315374611,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":256507904},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:40 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":0.75800000000000001,"15min":0.88700000000000001},"min":{"1min":0.26000000000000001,"5min":0.215,"15min":0.20100000000000001},"max":{"1min":2.2850000000000001,"5min":6.8849999999999998,"15min":14.164},"last":0.26800000000000002},{"interface":"front","average":{"1min":1.2170000000000001,"5min":1.0640000000000001,"15min":1.1220000000000001},"min":{"1min":0.29399999999999998,"5min":0.24399999999999999,"15min":0.22600000000000001},"max":{"1min":5.7750000000000004,"5min":7.1879999999999997,"15min":11.372},"last":1.804}]},{"osd":1,"last update":"Sat Sep 23 18:05:09 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.204,"15min":1.0880000000000001},"min":{"1min":0.29099999999999998,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":5.9740000000000002,"5min":7.2640000000000002,"15min":10.429},"last":1.099},{"interface":"front","average":{"1min":1.4750000000000001,"5min":1.2430000000000001,"15min":1.1439999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":6.7640000000000002,"5min":6.7640000000000002,"15min":10.83},"last":0.38700000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:05:44 2023","interfaces":[{"interface":"back","average":{"1min":0.76400000000000001,"5min":1.0489999999999999,"15min":1.1559999999999999},"min":{"1min":0.33300000000000002,"5min":0.27100000000000002,"15min":0.26600000000000001},"max":{"1min":2.1579999999999999,"5min":6.8259999999999996,"15min":11.630000000000001},"last":0.314},{"interface":"front","average":{"1min":1.169,"5min":1.085,"15min":1.107},"min":{"1min":0.42999999999999999,"5min":0.316,"15min":0.27100000000000002},"max":{"1min":6.609,"5min":6.9299999999999997,"15min":7.5090000000000003},"last":0.35499999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:05:19 2023","interfaces":[{"interface":"back","average":{"1min":1.968,"5min":1.3620000000000001,"15min":1.3540000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":8.6750000000000007,"5min":8.6750000000000007,"15min":8.6750000000000007},"last":1},{"interface":"front","average":{"1min":1.8140000000000001,"5min":1.423,"15min":1.389},"min":{"1min":0.36699999999999999,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":5.8170000000000002,"5min":6.165,"15min":6.165},"last":1.732}]},{"osd":5,"last update":"Sat Sep 23 18:05:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0980000000000001,"5min":0.95999999999999996,"15min":0.73399999999999999},"min":{"1min":0.24199999999999999,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":9.2959999999999994,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":1.135},{"interface":"front","average":{"1min":1.0820000000000001,"5min":1.2130000000000001,"15min":0.92500000000000004},"min":{"1min":0.30199999999999999,"5min":0.26800000000000002,"15min":0.26100000000000001},"max":{"1min":5.9249999999999998,"5min":11.170999999999999,"15min":11.170999999999999},"last":1.7669999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:05:44 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":1.0369999999999999,"15min":0.90800000000000003},"min":{"1min":0.27000000000000002,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":2.1200000000000001,"5min":9.0990000000000002,"15min":9.0990000000000002},"last":1.0629999999999999},{"interface":"front","average":{"1min":0.74299999999999999,"5min":0.998,"15min":0.998},"min":{"1min":0.32100000000000001,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":2.4980000000000002,"5min":9.1300000000000008,"15min":10.725},"last":1.0329999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:05:30 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":1.284,"15min":1.0369999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":13.298999999999999,"5min":13.298999999999999,"15min":13.298999999999999},"last":3.427},{"interface":"front","average":{"1min":1.7470000000000001,"5min":1.484,"15min":1.3029999999999999},"min":{"1min":0.379,"5min":0.307,"15min":0.307},"max":{"1min":10.192,"5min":10.192,"15min":10.192},"last":2.0110000000000001}]}]},{"osd":5,"up_from":38,"seq":163208757513,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":488212,"kb_used_data":266188,"kb_used_omap":0,"kb_used_meta":222016,"kb_avail":93256940,"statfs":{"total":95995035648,"available":95495106560,"internally_reserved":0,"allocated":272576512,"data_stored":271527011,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":227344384},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.88100000000000001,"15min":1.024},"min":{"1min":0.35299999999999998,"5min":0.27300000000000002,"15min":0.217},"max":{"1min":0.85599999999999998,"5min":13.586,"15min":21.376999999999999},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":1.3400000000000001,"15min":1.24},"min":{"1min":0.32800000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.5190000000000001,"5min":13.574,"15min":21.388999999999999},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:05:13 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.80800000000000005,"15min":0.875},"min":{"1min":0.34999999999999998,"5min":0.26000000000000001,"15min":0.23899999999999999},"max":{"1min":4.6150000000000002,"5min":4.8129999999999997,"15min":10.853999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":1.5960000000000001,"5min":1.1519999999999999,"15min":1.0569999999999999},"min":{"1min":0.30099999999999999,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":8.5619999999999994,"5min":8.5619999999999994,"15min":10.206},"last":0.749}]},{"osd":2,"last update":"Sat Sep 23 18:05:45 2023","interfaces":[{"interface":"back","average":{"1min":0.626,"5min":1.1040000000000001,"15min":1.325},"min":{"1min":0.35999999999999999,"5min":0.26400000000000001,"15min":0.255},"max":{"1min":1.5880000000000001,"5min":5.5309999999999997,"15min":38.606999999999999},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.69699999999999995,"5min":0.97199999999999998,"15min":1.226},"min":{"1min":0.36099999999999999,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":2.673,"5min":10.065,"15min":39.670999999999999},"last":0.70999999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:05:23 2023","interfaces":[{"interface":"back","average":{"1min":1.6599999999999999,"5min":1.0289999999999999,"15min":0.85399999999999998},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":8.593,"5min":8.593,"15min":8.593},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.224,"5min":0.85299999999999998,"15min":0.75900000000000001},"min":{"1min":0.33600000000000002,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":5.0439999999999996,"5min":7.1260000000000003,"15min":7.1260000000000003},"last":0.628}]},{"osd":4,"last update":"Sat Sep 23 18:05:49 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":1.01,"15min":0.79300000000000004},"min":{"1min":0.28699999999999998,"5min":0.24199999999999999,"15min":0.224},"max":{"1min":2.621,"5min":8.6229999999999993,"15min":10.092000000000001},"last":0.318},{"interface":"front","average":{"1min":0.54600000000000004,"5min":0.94899999999999995,"15min":0.77000000000000002},"min":{"1min":0.33900000000000002,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":0.97399999999999998,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.59799999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:05:49 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":0.94199999999999995,"15min":1.3560000000000001},"min":{"1min":0.34200000000000003,"5min":0.28299999999999997,"15min":0.27700000000000002},"max":{"1min":1.2370000000000001,"5min":6.5880000000000001,"15min":11.287000000000001},"last":0.439},{"interface":"front","average":{"1min":0.88400000000000001,"5min":0.873,"15min":1.224},"min":{"1min":0.307,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":6.6920000000000002,"5min":6.6920000000000002,"15min":8.266},"last":0.47899999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:05:32 2023","interfaces":[{"interface":"back","average":{"1min":1.1950000000000001,"5min":1.298,"15min":1.4630000000000001},"min":{"1min":0.40500000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":8.0999999999999996,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.48799999999999999},{"interface":"front","average":{"1min":1.4690000000000001,"5min":1.218,"15min":1.1559999999999999},"min":{"1min":0.42999999999999999,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":17.684000000000001,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.502}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":318242816,"data_stored":318242816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":300089344,"data_stored":300089344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":542441472,"data_stored":542441472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":351469568,"data_stored":351469568,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":272433152,"data_stored":272433152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":289538048,"data_stored":289538048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":489422848,"data_stored":489422848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:06:01.075 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T18:06:01.075 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:06:01.076 INFO:tasks.thrashosds.thrasher:Adding osd 1 2023-09-23T18:06:01.076 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 1 2023-09-23T18:06:01.076 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:01.121 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:01.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:01.126 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:01.142 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:06:01.226 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:01.269 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:01.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:01 smithi099 ceph-mon[167076]: 1.17 scrub starts 2023-09-23T18:06:01.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:01 smithi099 ceph-mon[167076]: 1.17 scrub ok 2023-09-23T18:06:01.373 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:01.377 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:01.378 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:01.478 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:01.545 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:01 smithi055 ceph-mon[184174]: 1.17 scrub starts 2023-09-23T18:06:01.545 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:01 smithi055 ceph-mon[184174]: 1.17 scrub ok 2023-09-23T18:06:01.545 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:01 smithi055 ceph-mon[179668]: 1.17 scrub starts 2023-09-23T18:06:01.546 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:01 smithi055 ceph-mon[179668]: 1.17 scrub ok 2023-09-23T18:06:01.552 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:01.732 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:01.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:01.737 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:01.811 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:06:01.837 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 11 16 25076 25060 142.237 122.875 0.00630975 0.00702355 2023-09-23T18:06:01.838 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:01.875 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:01.943 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:01.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:01.950 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:02.051 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:02.084 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:02.196 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:02.230 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:06:02.261 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:02.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:02.266 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:02.273 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:06:02.312 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:06:02.349 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:06:02.367 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:02.388 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:06:02.432 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:06:02.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[184174]: from='client.55480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:06:02.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[184174]: 1.b scrub starts 2023-09-23T18:06:02.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[184174]: 1.b scrub ok 2023-09-23T18:06:02.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[184174]: 1.7 scrub starts 2023-09-23T18:06:02.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[184174]: 1.7 scrub ok 2023-09-23T18:06:02.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[184174]: pgmap v705: 119 pgs: 119 active+clean; 1.1 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T18:06:02.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3877466746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:02.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[179668]: from='client.55480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:06:02.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[179668]: 1.b scrub starts 2023-09-23T18:06:02.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[179668]: 1.b scrub ok 2023-09-23T18:06:02.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[179668]: 1.7 scrub starts 2023-09-23T18:06:02.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[179668]: 1.7 scrub ok 2023-09-23T18:06:02.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[179668]: pgmap v705: 119 pgs: 119 active+clean; 1.1 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T18:06:02.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3877466746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:02.457 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:02.468 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:06:02.468 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:06:02.476 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:06:02.496 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:02 smithi099 ceph-mon[167076]: from='client.55480 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:06:02.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:02 smithi099 ceph-mon[167076]: 1.b scrub starts 2023-09-23T18:06:02.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:02 smithi099 ceph-mon[167076]: 1.b scrub ok 2023-09-23T18:06:02.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:02 smithi099 ceph-mon[167076]: 1.7 scrub starts 2023-09-23T18:06:02.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:02 smithi099 ceph-mon[167076]: 1.7 scrub ok 2023-09-23T18:06:02.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:02 smithi099 ceph-mon[167076]: pgmap v705: 119 pgs: 119 active+clean; 1.1 GiB data, 5.3 GiB used, 621 GiB / 626 GiB avail; 135 MiB/s wr, 2.16k op/s 2023-09-23T18:06:02.497 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3877466746' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:02.512 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:06:02.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:06:02.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:06:02.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:06:02.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:06:02.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:06:02.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:06:02.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:06:02.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:06:02.584 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:02.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:02.595 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:02.696 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:02.766 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:02.838 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 12 16 26777 26761 139.239 106.312 0.0119255 0.00717448 2023-09-23T18:06:02.872 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:06:02.881 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:02.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:02.888 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:02.989 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:03.060 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:03.127 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:03.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:03.133 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:03.190 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:03.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:03.283 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:03.318 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:03 smithi055 ceph-mon[179668]: 1.11 deep-scrub starts 2023-09-23T18:06:03.318 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:03 smithi055 ceph-mon[179668]: 1.11 deep-scrub ok 2023-09-23T18:06:03.329 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:03.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:03.334 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:03.435 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:03.514 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:03.526 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:06:03.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:03 smithi099 ceph-mon[167076]: 1.11 deep-scrub starts 2023-09-23T18:06:03.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:03 smithi099 ceph-mon[167076]: 1.11 deep-scrub ok 2023-09-23T18:06:03.582 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:03.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:03 smithi055 ceph-mon[184174]: 1.11 deep-scrub starts 2023-09-23T18:06:03.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:03 smithi055 ceph-mon[184174]: 1.11 deep-scrub ok 2023-09-23T18:06:03.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:03.591 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:03.692 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:03.756 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:03.842 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 13 16 28571 28555 137.109 112.125 0.00799061 0.00728796 2023-09-23T18:06:04.121 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:04.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:04.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:04.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:04.354 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.1. 2023-09-23T18:06:04.356 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:04.381 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[184174]: pgmap v706: 119 pgs: 119 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-09-23T18:06:04.381 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[184174]: 1.f scrub starts 2023-09-23T18:06:04.381 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[184174]: 1.f scrub ok 2023-09-23T18:06:04.381 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[184174]: 1.8 scrub starts 2023-09-23T18:06:04.381 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[184174]: 1.8 scrub ok 2023-09-23T18:06:04.382 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[184174]: 1.6 scrub starts 2023-09-23T18:06:04.382 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[184174]: 1.6 scrub ok 2023-09-23T18:06:04.382 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/178474990' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T18:06:04.382 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T18:06:04.382 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:06:04.192+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e284 definitely_dead 0 2023-09-23T18:06:04.382 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[179668]: pgmap v706: 119 pgs: 119 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-09-23T18:06:04.382 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[179668]: 1.f scrub starts 2023-09-23T18:06:04.382 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[179668]: 1.f scrub ok 2023-09-23T18:06:04.383 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[179668]: 1.8 scrub starts 2023-09-23T18:06:04.383 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[179668]: 1.8 scrub ok 2023-09-23T18:06:04.383 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[179668]: 1.6 scrub starts 2023-09-23T18:06:04.383 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[179668]: 1.6 scrub ok 2023-09-23T18:06:04.383 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/178474990' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T18:06:04.383 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:04 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T18:06:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:04 smithi099 ceph-mon[167076]: pgmap v706: 119 pgs: 119 active+clean; 1.3 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 132 MiB/s wr, 2.11k op/s 2023-09-23T18:06:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:04 smithi099 ceph-mon[167076]: 1.f scrub starts 2023-09-23T18:06:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:04 smithi099 ceph-mon[167076]: 1.f scrub ok 2023-09-23T18:06:04.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:04 smithi099 ceph-mon[167076]: 1.8 scrub starts 2023-09-23T18:06:04.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:04 smithi099 ceph-mon[167076]: 1.8 scrub ok 2023-09-23T18:06:04.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:04 smithi099 ceph-mon[167076]: 1.6 scrub starts 2023-09-23T18:06:04.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:04 smithi099 ceph-mon[167076]: 1.6 scrub ok 2023-09-23T18:06:04.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:04 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/178474990' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T18:06:04.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:04 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["1"]}]: dispatch 2023-09-23T18:06:04.711 INFO:tasks.thrashosds.thrasher:Added osd 1 2023-09-23T18:06:04.842 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 14 16 29552 29536 131.701 61.3125 0.00216498 0.00733111 2023-09-23T18:06:05.326 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:06:05.379 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:06:05.678 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[184174]: 1.1e scrub starts 2023-09-23T18:06:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[184174]: 1.1e scrub ok 2023-09-23T18:06:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[184174]: failed to encode map e285 with expected crc 2023-09-23T18:06:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T18:06:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[184174]: osdmap e285: 8 total, 8 up, 8 in 2023-09-23T18:06:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[184174]: pgmap v708: 119 pgs: 6 peering, 113 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 132 MiB/s wr, 2.10k op/s 2023-09-23T18:06:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1986074052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:05.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[179668]: 1.1e scrub starts 2023-09-23T18:06:05.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[179668]: 1.1e scrub ok 2023-09-23T18:06:05.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[179668]: failed to encode map e285 with expected crc 2023-09-23T18:06:05.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T18:06:05.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[179668]: osdmap e285: 8 total, 8 up, 8 in 2023-09-23T18:06:05.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[179668]: pgmap v708: 119 pgs: 6 peering, 113 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 132 MiB/s wr, 2.10k op/s 2023-09-23T18:06:05.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1986074052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:05.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:05 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:05.681 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:05 smithi099 ceph-mon[167076]: 1.1e scrub starts 2023-09-23T18:06:05.682 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:05 smithi099 ceph-mon[167076]: 1.1e scrub ok 2023-09-23T18:06:05.682 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:05 smithi099 ceph-mon[167076]: failed to encode map e285 with expected crc 2023-09-23T18:06:05.682 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:05 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["1"]}]': finished 2023-09-23T18:06:05.682 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:05 smithi099 ceph-mon[167076]: osdmap e285: 8 total, 8 up, 8 in 2023-09-23T18:06:05.682 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:05 smithi099 ceph-mon[167076]: pgmap v708: 119 pgs: 6 peering, 113 active+clean; 1.6 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 132 MiB/s wr, 2.10k op/s 2023-09-23T18:06:05.682 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1986074052' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:05.682 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:05 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:05.683 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:05.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:05.784 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:05.842 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 15 16 29552 29536 122.93 0 - 0.00733111 2023-09-23T18:06:05.853 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:05.889 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:05.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:05.892 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:05.993 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:06.021 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:06.031 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:06.061 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:06.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:06.065 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:06.160 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:06:06.160 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:06:06.160 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:06:06.160 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:06:06.160 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.160 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131500 5.2 5.9981e372 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T18:06:06.160 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.276336+0000", 2023-09-23T18:06:06.160 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 451.88039224300002, 2023-09-23T18:06:06.160 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94858491499999997, 2023-09-23T18:06:06.161 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.161 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:06:06.161 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.161 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.161 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.161 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131500 2023-09-23T18:06:06.161 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.161 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.161 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.161 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.162 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276336+0000", 2023-09-23T18:06:06.162 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.162 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.162 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.162 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.162 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276336+0000", 2023-09-23T18:06:06.162 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.162 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.162 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.162 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.163 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276335+0000", 2023-09-23T18:06:06.163 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:06:06.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.163 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.163 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.163 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276340+0000", 2023-09-23T18:06:06.163 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2310000000000004e-06 2023-09-23T18:06:06.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276348+0000", 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.303e-06 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276356+0000", 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9359999999999999e-06 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.164 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.165 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.165 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276427+0000", 2023-09-23T18:06:06.165 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0314000000000005e-05 2023-09-23T18:06:06.165 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.165 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.165 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.165 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.276438+0000", 2023-09-23T18:06:06.165 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1035e-05 2023-09-23T18:06:06.166 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.166 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.166 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.166 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224903+0000", 2023-09-23T18:06:06.166 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94846568499999995 2023-09-23T18:06:06.166 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.166 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.166 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.166 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224921+0000", 2023-09-23T18:06:06.166 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7461000000000001e-05 2023-09-23T18:06:06.167 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.167 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.167 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.167 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.167 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131508 5.2 5.89ad37e2 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T18:06:06.167 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.286747+0000", 2023-09-23T18:06:06.167 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 451.86998152400002, 2023-09-23T18:06:06.168 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.93821040499999997, 2023-09-23T18:06:06.168 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.168 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:06:06.168 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.168 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.168 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.168 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131508 2023-09-23T18:06:06.168 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.169 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.169 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.169 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.169 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286747+0000", 2023-09-23T18:06:06.169 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.169 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.169 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.169 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.169 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286747+0000", 2023-09-23T18:06:06.169 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286745+0000", 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286752+0000", 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0550000000000002e-06 2023-09-23T18:06:06.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286767+0000", 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4283e-05 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286775+0000", 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1780000000000003e-06 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.171 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.172 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.172 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286834+0000", 2023-09-23T18:06:06.172 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8866999999999997e-05 2023-09-23T18:06:06.172 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.172 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.172 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.172 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.286842+0000", 2023-09-23T18:06:06.172 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7590000000000003e-06 2023-09-23T18:06:06.172 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.172 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224948+0000", 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.938105666 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.224957+0000", 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9469999999999992e-06 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.173 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:131523 5.2 5.551f4f92 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e163)", 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:34.311020+0000", 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 451.84570802399998, 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91399190200000002, 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.174 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 131523 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311020+0000", 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311020+0000", 2023-09-23T18:06:06.175 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311018+0000", 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311028+0000", 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5859999999999994e-06 2023-09-23T18:06:06.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311050+0000", 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2079e-05 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311059+0000", 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9959999999999999e-06 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.177 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311123+0000", 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4539e-05 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:34.311130+0000", 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.917e-06 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.178 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.225002+0000", 2023-09-23T18:06:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91387170399999995 2023-09-23T18:06:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:35.225012+0000", 2023-09-23T18:06:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0132000000000001e-05 2023-09-23T18:06:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.179 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142717 5.9 5:97ac1ed5:::benchmark_data_smithi055_229117_object37091:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.884415+0000", 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 434.27231334499999, 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0500839660000001, 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.180 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142717 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884415+0000", 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884415+0000", 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884413+0000", 2023-09-23T18:06:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884424+0000", 2023-09-23T18:06:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0804e-05 2023-09-23T18:06:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884444+0000", 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0057000000000001e-05 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884453+0000", 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8000000000000004e-06 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884524+0000", 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0729000000000007e-05 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.884537+0000", 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3059e-05 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903080+0000", 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0185432940000001 2023-09-23T18:06:06.184 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903187+0000", 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010691700000000001 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.903271+0000", 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4103000000000004e-05 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.185 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.911979+0000", 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0087084599999999995 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934336+0000", 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022356767999999999 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934371+0000", 2023-09-23T18:06:06.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4693999999999997e-05 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934420+0000", 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8755000000000003e-05 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934499+0000", 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9129999999999996e-05 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.187 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142734 5.9 5:9b1dd87e:::benchmark_data_smithi055_229117_object37108:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.916812+0000", 2023-09-23T18:06:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 434.239916214, 2023-09-23T18:06:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0179610699999999, 2023-09-23T18:06:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.188 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142734 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916812+0000", 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916812+0000", 2023-09-23T18:06:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916810+0000", 2023-09-23T18:06:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916818+0000", 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7209999999999994e-06 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916838+0000", 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9941000000000001e-05 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916846+0000", 2023-09-23T18:06:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5190000000000003e-06 2023-09-23T18:06:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917145+0000", 2023-09-23T18:06:06.192 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00029908799999999998 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917153+0000", 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7700000000000007e-06 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.904847+0000", 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98769368300000004 2023-09-23T18:06:06.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.904960+0000", 2023-09-23T18:06:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112802 2023-09-23T18:06:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.905068+0000", 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010848700000000001 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915793+0000", 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010724450999999999 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934547+0000", 2023-09-23T18:06:06.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018754603000000002 2023-09-23T18:06:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934612+0000", 2023-09-23T18:06:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4272999999999999e-05 2023-09-23T18:06:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934667+0000", 2023-09-23T18:06:06.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4814000000000001e-05 2023-09-23T18:06:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934773+0000", 2023-09-23T18:06:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106517 2023-09-23T18:06:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142738 5.9 5:9dbd7dd5:::benchmark_data_smithi055_229117_object37112:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:06:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 434.23986669200002, 2023-09-23T18:06:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018182892, 2023-09-23T18:06:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.198 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142738 2023-09-23T18:06:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:06:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.199 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916861+0000", 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916864+0000", 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0709999999999999e-06 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916870+0000", 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8780000000000003e-06 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.916882+0000", 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2558000000000001e-05 2023-09-23T18:06:06.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917176+0000", 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00029325599999999999 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.917181+0000", 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6400000000000002e-06 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906621+0000", 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98943947600000004 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906744+0000", 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012348099999999999 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.906869+0000", 2023-09-23T18:06:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012421599999999999 2023-09-23T18:06:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915800+0000", 2023-09-23T18:06:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0089313259999999998 2023-09-23T18:06:06.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934813+0000", 2023-09-23T18:06:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019012940999999998 2023-09-23T18:06:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934959+0000", 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000145921 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934994+0000", 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5707999999999998e-05 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.206 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935044+0000", 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9951000000000003e-05 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142777 5.9 5:91082f54:::benchmark_data_smithi055_229117_object37151:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:51.976270+0000", 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 434.18045805899999, 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.958897997, 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.207 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142777 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976270+0000", 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.208 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976270+0000", 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976269+0000", 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.209 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976276+0000", 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2150000000000004e-06 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976288+0000", 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2092999999999999e-05 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976293+0000", 2023-09-23T18:06:06.210 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1610000000000002e-06 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976338+0000", 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4116000000000002e-05 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:51.976346+0000", 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9200000000000004e-06 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.211 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908348+0000", 2023-09-23T18:06:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.93200213499999995 2023-09-23T18:06:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908468+0000", 2023-09-23T18:06:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120746 2023-09-23T18:06:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.212 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.908590+0000", 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121772 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915806+0000", 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0072161990000000004 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.213 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934909+0000", 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019102811000000001 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935083+0000", 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000173499 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.214 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935114+0000", 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1501999999999998e-05 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935168+0000", 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3947999999999999e-05 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.215 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142802 5.9 5:92d6f126:::benchmark_data_smithi055_229117_object37176:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.030542+0000", 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 434.12618601200001, 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.90473668699999998, 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142802 2023-09-23T18:06:06.216 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030542+0000", 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030542+0000", 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.217 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030541+0000", 2023-09-23T18:06:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030547+0000", 2023-09-23T18:06:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5289999999999997e-06 2023-09-23T18:06:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.218 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030566+0000", 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8630999999999998e-05 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030571+0000", 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0259999999999998e-06 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.219 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030617+0000", 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5439999999999999e-05 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.030623+0000", 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1999999999999999e-06 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910103+0000", 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.87947978999999998 2023-09-23T18:06:06.220 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910251+0000", 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014896700000000001 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.910377+0000", 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125425 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920175+0000", 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097978730000000003 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934939+0000", 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014764019 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935202+0000", 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026282399999999999 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935229+0000", 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7793999999999999e-05 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935279+0000", 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9428999999999997e-05 2023-09-23T18:06:06.223 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142809 5.9 5:9f3ab830:::benchmark_data_smithi055_229117_object37183:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.039995+0000", 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 434.116733076, 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.902436291, 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.224 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142809 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039995+0000", 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039995+0000", 2023-09-23T18:06:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.039994+0000", 2023-09-23T18:06:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040001+0000", 2023-09-23T18:06:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5259999999999996e-06 2023-09-23T18:06:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040017+0000", 2023-09-23T18:06:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5794000000000002e-05 2023-09-23T18:06:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040024+0000", 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3379999999999999e-06 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040079+0000", 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4866e-05 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.040091+0000", 2023-09-23T18:06:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1362000000000001e-05 2023-09-23T18:06:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912030+0000", 2023-09-23T18:06:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.87193905100000002 2023-09-23T18:06:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912160+0000", 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013052100000000001 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.912267+0000", 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010700399999999999 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920180+0000", 2023-09-23T18:06:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079123509999999998 2023-09-23T18:06:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935312+0000", 2023-09-23T18:06:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01513201 2023-09-23T18:06:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942274+0000", 2023-09-23T18:06:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069623669999999997 2023-09-23T18:06:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942331+0000", 2023-09-23T18:06:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7516000000000003e-05 2023-09-23T18:06:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.232 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942431+0000", 2023-09-23T18:06:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9986000000000001e-05 2023-09-23T18:06:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142838 5.9 5:9cf2a92c:::benchmark_data_smithi055_229117_object37212:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.111228+0000", 2023-09-23T18:06:06.233 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 434.04550033999999, 2023-09-23T18:06:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.83141186199999995, 2023-09-23T18:06:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142838 2023-09-23T18:06:06.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111228+0000", 2023-09-23T18:06:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111228+0000", 2023-09-23T18:06:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111227+0000", 2023-09-23T18:06:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:06:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111233+0000", 2023-09-23T18:06:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3160000000000002e-06 2023-09-23T18:06:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111246+0000", 2023-09-23T18:06:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2884000000000001e-05 2023-09-23T18:06:06.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111251+0000", 2023-09-23T18:06:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2360000000000004e-06 2023-09-23T18:06:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111297+0000", 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5881000000000003e-05 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.111307+0000", 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0046e-05 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914054+0000", 2023-09-23T18:06:06.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.80274739399999995 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914133+0000", 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8448999999999994e-05 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.914206+0000", 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3325999999999996e-05 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925627+0000", 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011421277000000001 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935332+0000", 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097046300000000005 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942488+0000", 2023-09-23T18:06:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007155603 2023-09-23T18:06:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942537+0000", 2023-09-23T18:06:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9391000000000002e-05 2023-09-23T18:06:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.942640+0000", 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102618 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142868 5.9 5:93088e3b:::benchmark_data_smithi055_229117_object37242:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.164405+0000", 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 433.992322785, 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.78228625900000004, 2023-09-23T18:06:06.243 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142868 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164405+0000", 2023-09-23T18:06:06.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164405+0000", 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164404+0000", 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164410+0000", 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1349999999999997e-06 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164422+0000", 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1647e-05 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164437+0000", 2023-09-23T18:06:06.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5234e-05 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164485+0000", 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7363e-05 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.164493+0000", 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5429999999999994e-06 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915840+0000", 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.75134721800000004 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915919+0000", 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8091999999999997e-05 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.915992+0000", 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3733999999999996e-05 2023-09-23T18:06:06.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925638+0000", 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096452810000000003 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935357+0000", 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097193910000000008 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946534+0000", 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011177302 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946593+0000", 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8563000000000002e-05 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946692+0000", 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8836e-05 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142887 5.9 5:980e2e89:::benchmark_data_smithi055_229117_object37261:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.211074+0000", 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 433.94565370200002, 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73576580700000005, 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.251 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142887 2023-09-23T18:06:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211074+0000", 2023-09-23T18:06:06.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211074+0000", 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211073+0000", 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211080+0000", 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3629999999999996e-06 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211097+0000", 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6917000000000001e-05 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211103+0000", 2023-09-23T18:06:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2499999999999997e-06 2023-09-23T18:06:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211145+0000", 2023-09-23T18:06:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2466000000000003e-05 2023-09-23T18:06:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211151+0000", 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0410000000000003e-06 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918068+0000", 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70691734399999995 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918196+0000", 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000128081 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.918320+0000", 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012403300000000001 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930458+0000", 2023-09-23T18:06:06.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012137652000000001 2023-09-23T18:06:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.935374+0000", 2023-09-23T18:06:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049156570000000004 2023-09-23T18:06:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946735+0000", 2023-09-23T18:06:06.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011361616999999999 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946767+0000", 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1952999999999997e-05 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946840+0000", 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2910000000000005e-05 2023-09-23T18:06:06.259 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142888 5.9 5:99a82285:::benchmark_data_smithi055_229117_object37262:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 433.94561388699998, 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.73588783199999996, 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.260 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142888 2023-09-23T18:06:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:06:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:06:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211114+0000", 2023-09-23T18:06:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:06.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211117+0000", 2023-09-23T18:06:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4680000000000001e-06 2023-09-23T18:06:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211121+0000", 2023-09-23T18:06:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7610000000000001e-06 2023-09-23T18:06:06.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211124+0000", 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3489999999999999e-06 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211177+0000", 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3133000000000001e-05 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.211184+0000", 2023-09-23T18:06:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6240000000000002e-06 2023-09-23T18:06:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920219+0000", 2023-09-23T18:06:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70903441 2023-09-23T18:06:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920327+0000", 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108477 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.920402+0000", 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5252000000000004e-05 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.266 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930467+0000", 2023-09-23T18:06:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010064867999999999 2023-09-23T18:06:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.937129+0000", 2023-09-23T18:06:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066614200000000004 2023-09-23T18:06:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.267 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946878+0000", 2023-09-23T18:06:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097489120000000002 2023-09-23T18:06:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.946919+0000", 2023-09-23T18:06:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1031000000000001e-05 2023-09-23T18:06:06.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.947002+0000", 2023-09-23T18:06:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3475999999999998e-05 2023-09-23T18:06:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142904 5.9 5:9b2141c3:::benchmark_data_smithi055_229117_object37278:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.244955+0000", 2023-09-23T18:06:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 433.911772826, 2023-09-23T18:06:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.70391956499999997, 2023-09-23T18:06:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.270 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142904 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244955+0000", 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244955+0000", 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244954+0000", 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244960+0000", 2023-09-23T18:06:06.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:06:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244973+0000", 2023-09-23T18:06:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3209e-05 2023-09-23T18:06:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.244978+0000", 2023-09-23T18:06:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1930000000000001e-06 2023-09-23T18:06:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245017+0000", 2023-09-23T18:06:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8452999999999999e-05 2023-09-23T18:06:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245023+0000", 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.409e-06 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.921896+0000", 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67687305900000005 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.921997+0000", 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100753 2023-09-23T18:06:06.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.922064+0000", 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6978999999999997e-05 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930475+0000", 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0084114670000000006 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.940046+0000", 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095709799999999998 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948781+0000", 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0087346990000000003 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948813+0000", 2023-09-23T18:06:06.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1779000000000002e-05 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.948875+0000", 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2073e-05 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.278 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142905 5.9 5:951a98af:::benchmark_data_smithi055_229117_object37279:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 433.91139895700002, 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71295564, 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142905 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245329+0000", 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245332+0000", 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4400000000000001e-06 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245339+0000", 2023-09-23T18:06:06.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4749999999999998e-06 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245343+0000", 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1370000000000004e-06 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245381+0000", 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8396999999999998e-05 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.245386+0000", 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2460000000000003e-06 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923706+0000", 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.67831945699999996 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.283 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923849+0000", 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143669 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.923977+0000", 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012721600000000001 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933852+0000", 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0098750190000000005 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.940069+0000", 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062175399999999997 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958137+0000", 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018067588999999998 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958189+0000", 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1981000000000003e-05 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.958285+0000", 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6036e-05 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142919 5.9 5:9348e633:::benchmark_data_smithi055_229117_object37293:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.270585+0000", 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 433.88614332499998, 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.69084338599999995, 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142919 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.287 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270585+0000", 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270585+0000", 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270583+0000", 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270591+0000", 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2910000000000002e-06 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270604+0000", 2023-09-23T18:06:06.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3349999999999999e-05 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270609+0000", 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1109999999999999e-06 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270662+0000", 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3236000000000003e-05 2023-09-23T18:06:06.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.270673+0000", 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0560999999999999e-05 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925683+0000", 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.65501024500000005 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925803+0000", 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119807 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.925920+0000", 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117419 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933861+0000", 2023-09-23T18:06:06.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079407209999999995 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944507+0000", 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010645846 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961259+0000", 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016751986999999999 2023-09-23T18:06:06.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961315+0000", 2023-09-23T18:06:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6477999999999997e-05 2023-09-23T18:06:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.961428+0000", 2023-09-23T18:06:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011292400000000001 2023-09-23T18:06:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142929 5.9 5:954e3e29:::benchmark_data_smithi055_229117_object37303:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.300689+0000", 2023-09-23T18:06:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 433.85603960999998, 2023-09-23T18:06:06.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.662469262, 2023-09-23T18:06:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142929 2023-09-23T18:06:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300689+0000", 2023-09-23T18:06:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300689+0000", 2023-09-23T18:06:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.297 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300687+0000", 2023-09-23T18:06:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:06:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300694+0000", 2023-09-23T18:06:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0219999999999999e-06 2023-09-23T18:06:06.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300705+0000", 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0886e-05 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300711+0000", 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9490000000000001e-06 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300749+0000", 2023-09-23T18:06:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8077999999999997e-05 2023-09-23T18:06:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.300755+0000", 2023-09-23T18:06:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5389999999999998e-06 2023-09-23T18:06:06.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928440+0000", 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62768513800000003 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928576+0000", 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013626900000000001 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.928683+0000", 2023-09-23T18:06:06.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010661400000000001 2023-09-23T18:06:06.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933946+0000", 2023-09-23T18:06:06.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052633460000000003 2023-09-23T18:06:06.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944539+0000", 2023-09-23T18:06:06.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010592571 2023-09-23T18:06:06.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963005+0000", 2023-09-23T18:06:06.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018466598000000001 2023-09-23T18:06:06.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963057+0000", 2023-09-23T18:06:06.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1529000000000001e-05 2023-09-23T18:06:06.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.305 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963158+0000", 2023-09-23T18:06:06.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010085399999999999 2023-09-23T18:06:06.305 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.305 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.305 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.306 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142941 5.9 5:913fcfd6:::benchmark_data_smithi055_229117_object37315:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.306 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.340683+0000", 2023-09-23T18:06:06.306 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 433.81604527299999, 2023-09-23T18:06:06.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.62261945900000004, 2023-09-23T18:06:06.306 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.306 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.306 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.306 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142941 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340683+0000", 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340683+0000", 2023-09-23T18:06:06.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340682+0000", 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340688+0000", 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.319e-06 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340701+0000", 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3529999999999999e-05 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340708+0000", 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8449999999999997e-06 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340755+0000", 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6428999999999999e-05 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.340762+0000", 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.571e-06 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930510+0000", 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.58974757200000005 2023-09-23T18:06:06.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930615+0000", 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000105213 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.930695+0000", 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0055000000000005e-05 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.933952+0000", 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003257228 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944608+0000", 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010655243 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963204+0000", 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018596950000000001 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963236+0000", 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1894000000000003e-05 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963302+0000", 2023-09-23T18:06:06.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6013999999999995e-05 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:142946 5.9 5:9099fb2d:::benchmark_data_smithi055_229117_object37320:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e168)", 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:52.370819+0000", 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 433.785909183, 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.59265797899999995, 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.314 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 142946 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370819+0000", 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370819+0000", 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370817+0000", 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370824+0000", 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8299999999999998e-06 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370837+0000", 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2716e-05 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370844+0000", 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8789999999999997e-06 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370887+0000", 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2879000000000001e-05 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.370893+0000", 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1120000000000001e-06 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932242+0000", 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.56134879999999998 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932323+0000", 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0871000000000005e-05 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.932392+0000", 2023-09-23T18:06:06.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9295999999999995e-05 2023-09-23T18:06:06.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.934346+0000", 2023-09-23T18:06:06.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019539589999999999 2023-09-23T18:06:06.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.944640+0000", 2023-09-23T18:06:06.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010293754 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963339+0000", 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018699684000000001 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963384+0000", 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4499999999999997e-05 2023-09-23T18:06:06.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:52.963477+0000", 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3245000000000006e-05 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:64827 6.b 6:dd1b4233:::benchmark_data_smithi055_275963_object64826:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e232)", 2023-09-23T18:06:06.322 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:02:10.517572+0000", 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 235.63915629100001, 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.31518728800000001, 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 64827 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:06:06.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.517572+0000", 2023-09-23T18:06:06.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:06:06.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.517572+0000", 2023-09-23T18:06:06.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:06:06.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.517570+0000", 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.517620+0000", 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9679000000000003e-05 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.517633+0000", 2023-09-23T18:06:06.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2833999999999999e-05 2023-09-23T18:06:06.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:06:06.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.517640+0000", 2023-09-23T18:06:06.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.204e-06 2023-09-23T18:06:06.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:06:06.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.802618+0000", 2023-09-23T18:06:06.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28497772799999999 2023-09-23T18:06:06.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:06:06.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.802766+0000", 2023-09-23T18:06:06.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014820600000000001 2023-09-23T18:06:06.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.802937+0000", 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017140999999999999 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.820584+0000", 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017646443000000001 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.820641+0000", 2023-09-23T18:06:06.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7423000000000002e-05 2023-09-23T18:06:06.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:06:06.329 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.830599+0000", 2023-09-23T18:06:06.329 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0099579820000000006 2023-09-23T18:06:06.329 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.329 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.329 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:06:06.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.830661+0000", 2023-09-23T18:06:06.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1587999999999994e-05 2023-09-23T18:06:06.330 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:06:06.330 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:06:06.330 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:06:06.330 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:02:10.832759+0000", 2023-09-23T18:06:06.330 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020983400000000002 2023-09-23T18:06:06.330 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.330 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.330 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.331 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:06:06.331 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:06:06.331 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:06:06.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:06.360 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:06.496 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:06:06.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:06.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:06.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:06:06.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: osdmap e286: 8 total, 8 up, 8 in 2023-09-23T18:06:06.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:06 smithi099 ceph-mon[167076]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:06.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:06.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:06:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: osdmap e286: 8 total, 8 up, 8 in 2023-09-23T18:06:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[179668]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:06.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:06:06.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: osdmap e286: 8 total, 8 up, 8 in 2023-09-23T18:06:06.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.585 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.586 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.586 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.586 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:06 smithi055 ceph-mon[184174]: failed to encode map e286 with expected crc 2023-09-23T18:06:06.654 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:06.717 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:06.779 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:06.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:06.789 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:06.842 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 16 16 29811 29795 116.265 8.09375 0.0249596 0.00859277 2023-09-23T18:06:06.890 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:06.965 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:07.077 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:07.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:07.085 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:07.186 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:07.236 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:06:07.267 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:07.339 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:07.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:07.343 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:07.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:07.481 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:07.530 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:07.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:07.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:07.562 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:06:07.616 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:06:07.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:07.657 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:06:07.668 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:07 smithi099 ceph-mon[167076]: osdmap e287: 8 total, 8 up, 8 in 2023-09-23T18:06:07.668 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:07 smithi099 ceph-mon[167076]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.668 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:07 smithi099 ceph-mon[167076]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.668 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:07 smithi099 ceph-mon[167076]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.668 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:07 smithi099 ceph-mon[167076]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.668 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:07 smithi099 ceph-mon[167076]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.668 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:07.669 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:07 smithi099 ceph-mon[167076]: pgmap v711: 119 pgs: 2 remapped+peering, 6 peering, 111 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.68k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T18:06:07.673 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[179668]: osdmap e287: 8 total, 8 up, 8 in 2023-09-23T18:06:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[179668]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[179668]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[179668]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[179668]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[179668]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:07.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[179668]: pgmap v711: 119 pgs: 2 remapped+peering, 6 peering, 111 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.68k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T18:06:07.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[184174]: osdmap e287: 8 total, 8 up, 8 in 2023-09-23T18:06:07.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[184174]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[184174]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[184174]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[184174]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[184174]: failed to encode map e287 with expected crc 2023-09-23T18:06:07.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:07.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:07 smithi055 ceph-mon[184174]: pgmap v711: 119 pgs: 2 remapped+peering, 6 peering, 111 active+clean; 1.8 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 106 MiB/s wr, 1.68k op/s; 0 B/s, 0 objects/s recovering 2023-09-23T18:06:07.697 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:06:07.734 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:07.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:07.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:07.740 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:06:07.774 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:06:07.786 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:06:07.810 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:06:07.838 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:07.842 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 17 16 31035 31019 113.927 76.5 0.010708 0.00876702 2023-09-23T18:06:07.844 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:06:07.854 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:06:07.854 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:06:07.854 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:06:07.854 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:06:07.855 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:06:07.855 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:06:07.855 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:06:07.855 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:06:07.870 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:07.964 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:07.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:07.968 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:08.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:08.108 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:08.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:08.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:08.202 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:08.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:08.344 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:08.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:08.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:08.442 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:08.543 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:08.547 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:06:08.547 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:06:08.575 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:08 smithi099 ceph-mon[167076]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:08 smithi099 ceph-mon[167076]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:08 smithi099 ceph-mon[167076]: osdmap e288: 8 total, 8 up, 8 in 2023-09-23T18:06:08.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:08 smithi099 ceph-mon[167076]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:08 smithi099 ceph-mon[167076]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:08 smithi099 ceph-mon[167076]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:08 smithi099 ceph-mon[167076]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3203514855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:08.579 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:08.630 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:08.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:08.634 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[179668]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[179668]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[179668]: osdmap e288: 8 total, 8 up, 8 in 2023-09-23T18:06:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[179668]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[179668]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[179668]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[179668]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3203514855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[184174]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[184174]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[184174]: osdmap e288: 8 total, 8 up, 8 in 2023-09-23T18:06:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[184174]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[184174]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[184174]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[184174]: failed to encode map e288 with expected crc 2023-09-23T18:06:08.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3203514855' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:08.684 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:08.735 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:08.812 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:08.842 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 18 16 32259 32243 111.849 76.5 0.00312577 0.00893196 2023-09-23T18:06:08.926 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:06:08.952 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:08.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:08.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:09.062 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:09.126 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:09.190 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:09.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:09.202 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:09.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:09.338 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:09.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:09.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:09.397 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:09.498 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:09.573 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:09.622 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:06:09.712 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:06:09.713 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:06:09.713 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T18:06:09.713 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:06:09.713 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:06:09.713 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T18:06:09.774 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:09 smithi055 ceph-mon[184174]: failed to encode map e289 with expected crc 2023-09-23T18:06:09.774 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:09 smithi055 ceph-mon[184174]: osdmap e289: 8 total, 8 up, 8 in 2023-09-23T18:06:09.774 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:09 smithi055 ceph-mon[184174]: failed to encode map e289 with expected crc 2023-09-23T18:06:09.774 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:09 smithi055 ceph-mon[184174]: failed to encode map e289 with expected crc 2023-09-23T18:06:09.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:09 smithi055 ceph-mon[184174]: pgmap v714: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 12 peering, 103 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 3739/89868 objects degraded (4.161%); 0 B/s, 6 objects/s recovering 2023-09-23T18:06:09.778 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:09 smithi055 ceph-mon[179668]: failed to encode map e289 with expected crc 2023-09-23T18:06:09.778 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:09 smithi055 ceph-mon[179668]: osdmap e289: 8 total, 8 up, 8 in 2023-09-23T18:06:09.778 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:09 smithi055 ceph-mon[179668]: failed to encode map e289 with expected crc 2023-09-23T18:06:09.778 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:09 smithi055 ceph-mon[179668]: failed to encode map e289 with expected crc 2023-09-23T18:06:09.778 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:09 smithi055 ceph-mon[179668]: pgmap v714: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 12 peering, 103 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 3739/89868 objects degraded (4.161%); 0 B/s, 6 objects/s recovering 2023-09-23T18:06:09.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:09 smithi099 ceph-mon[167076]: failed to encode map e289 with expected crc 2023-09-23T18:06:09.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:09 smithi099 ceph-mon[167076]: osdmap e289: 8 total, 8 up, 8 in 2023-09-23T18:06:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:09 smithi099 ceph-mon[167076]: failed to encode map e289 with expected crc 2023-09-23T18:06:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:09 smithi099 ceph-mon[167076]: failed to encode map e289 with expected crc 2023-09-23T18:06:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:09 smithi099 ceph-mon[167076]: pgmap v714: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 2 remapped+peering, 12 peering, 103 active+clean; 1.8 GiB data, 8.1 GiB used, 707 GiB / 715 GiB avail; 53 MiB/s wr, 846 op/s; 3739/89868 objects degraded (4.161%); 0 B/s, 6 objects/s recovering 2023-09-23T18:06:09.844 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 19 16 33517 33501 110.099 78.625 0.0207077 0.00907601 2023-09-23T18:06:10.290 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:10.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:10.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:10.327 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:06:10.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:10.432 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:10.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:10.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:10.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:10.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:10.630 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:10.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:10.681 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:10 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 3739/89868 objects degraded (4.161%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:10.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:10.683 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:10.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:10 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 3739/89868 objects degraded (4.161%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:10.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:10 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 3739/89868 objects degraded (4.161%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:10.684 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:06:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:06:10] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-09-23T18:06:10.784 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:10.843 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:06:10.844291+0000 min lat: 0.00119503 max lat: 2.30701 avg lat: 0.00923529 2023-09-23T18:06:10.844 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:06:10.844 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 20 16 34667 34651 108.189 71.875 0.015117 0.00923529 2023-09-23T18:06:10.859 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:10.922 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:06:10.922 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:06:11.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:06:11.542 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:11.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:11.545 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:11.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:11 smithi099 ceph-mon[167076]: pgmap v715: 119 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 6 peering, 106 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 9327/96450 objects degraded (9.670%); 0 B/s, 4 objects/s recovering 2023-09-23T18:06:11.646 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:11.717 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:11 smithi055 ceph-mon[184174]: pgmap v715: 119 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 6 peering, 106 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 9327/96450 objects degraded (9.670%); 0 B/s, 4 objects/s recovering 2023-09-23T18:06:11.717 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:11 smithi055 ceph-mon[179668]: pgmap v715: 119 pgs: 6 active+recovery_wait+undersized+degraded+remapped, 1 remapped+peering, 6 peering, 106 active+clean; 2.0 GiB data, 8.4 GiB used, 707 GiB / 715 GiB avail; 65 MiB/s wr, 1.03k op/s; 9327/96450 objects degraded (9.670%); 0 B/s, 4 objects/s recovering 2023-09-23T18:06:11.717 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:11.843 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 21 15 35752 35737 106.27 67.875 0.00637078 0.0094013 2023-09-23T18:06:12.022 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:06:12.585 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:12.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:12.594 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:12.695 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:12.792 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:12.846 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 22 16 37031 37015 105.057 79.875 0.0125514 0.00951066 2023-09-23T18:06:12.868 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:12.878 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:12.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:12.883 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:12.983 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:13.067 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:13.069 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:13.200 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:13.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:13.207 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:13.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:13.345 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:13.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:13.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:13.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:13.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:13.538 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:13.849 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 23 16 38048 38032 103.247 63.5625 0.0200909 0.00967875 2023-09-23T18:06:14.029 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:06:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:14 smithi055 ceph-mon[179668]: pgmap v716: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 4 peering, 108 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s; 9327/102003 objects degraded (9.144%); 1580/102003 objects misplaced (1.549%); 2.4 MiB/s, 42 objects/s recovering 2023-09-23T18:06:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4188029358' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:14 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:14.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:14 smithi055 ceph-mon[184174]: pgmap v716: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 4 peering, 108 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s; 9327/102003 objects degraded (9.144%); 1580/102003 objects misplaced (1.549%); 2.4 MiB/s, 42 objects/s recovering 2023-09-23T18:06:14.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4188029358' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:14.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:14 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:14.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:14 smithi099 ceph-mon[167076]: pgmap v716: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 4 peering, 108 active+clean; 2.1 GiB data, 8.8 GiB used, 706 GiB / 715 GiB avail; 50 MiB/s wr, 794 op/s; 9327/102003 objects degraded (9.144%); 1580/102003 objects misplaced (1.549%); 2.4 MiB/s, 42 objects/s recovering 2023-09-23T18:06:14.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4188029358' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:14.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:14 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:14.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:14.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:14.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:14.633 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:14.714 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:14.790 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:14.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:14.795 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:14.852 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 24 16 39165 39149 101.835 69.8125 0.0140977 0.00981182 2023-09-23T18:06:14.896 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:14.967 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:15.077 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:15.085 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:15.089 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:15 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:15.089 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:15 smithi055 ceph-mon[179668]: failed to encode map e290 with expected crc 2023-09-23T18:06:15.089 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:15 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:15.089 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:15 smithi055 ceph-mon[179668]: osdmap e290: 8 total, 8 up, 8 in 2023-09-23T18:06:15.089 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:15 smithi055 ceph-mon[179668]: failed to encode map e290 with expected crc 2023-09-23T18:06:15.090 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:15 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:15.090 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:15 smithi055 ceph-mon[184174]: failed to encode map e290 with expected crc 2023-09-23T18:06:15.090 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:15 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:15.090 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:15 smithi055 ceph-mon[184174]: osdmap e290: 8 total, 8 up, 8 in 2023-09-23T18:06:15.090 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:15 smithi055 ceph-mon[184174]: failed to encode map e290 with expected crc 2023-09-23T18:06:15.160 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:06:15.160 INFO:teuthology.orchestra.run.smithi055.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T18:06:15.160 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:06:15.161 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:06:15.161 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:06:15.182 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:15 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:15.182 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:15 smithi099 ceph-mon[167076]: failed to encode map e290 with expected crc 2023-09-23T18:06:15.182 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:15 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:15.182 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:15 smithi099 ceph-mon[167076]: osdmap e290: 8 total, 8 up, 8 in 2023-09-23T18:06:15.183 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:15 smithi099 ceph-mon[167076]: failed to encode map e290 with expected crc 2023-09-23T18:06:15.187 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:15.289 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:15.455 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:06:15.504 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:06:15.518 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:06:15.549 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:06:15.595 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:06:15.622 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:15.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:15.634 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:15.649 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:06:15.693 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:06:15.729 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:06:15.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:15.770 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:15.773 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:06:15.782 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:06:15.783 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:06:15.783 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:06:15.783 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:06:15.783 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:06:15.783 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:06:15.783 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:06:15.783 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:06:15.819 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:15.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:15.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:15.855 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 25 16 40197 40181 100.342 64.5 0.0217637 0.00996076 2023-09-23T18:06:15.924 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:15.956 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:15.997 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:16.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:16.002 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:16.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:16.146 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:16.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:16.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:16.195 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:16.274 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:16 smithi055 ceph-mon[179668]: pgmap v718: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 9327/110577 objects degraded (8.435%); 1580/110577 objects misplaced (1.429%); 2.1 MiB/s, 37 objects/s recovering 2023-09-23T18:06:16.274 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:16 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 9327/110577 objects degraded (8.435%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:16.274 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3207384105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:16.275 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:16 smithi055 ceph-mon[184174]: pgmap v718: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 9327/110577 objects degraded (8.435%); 1580/110577 objects misplaced (1.429%); 2.1 MiB/s, 37 objects/s recovering 2023-09-23T18:06:16.275 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:16 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 9327/110577 objects degraded (8.435%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:16.275 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3207384105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:16.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:16.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:16 smithi099 ceph-mon[167076]: pgmap v718: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.3 GiB data, 9.4 GiB used, 706 GiB / 715 GiB avail; 68 MiB/s wr, 1.08k op/s; 9327/110577 objects degraded (8.435%); 1580/110577 objects misplaced (1.429%); 2.1 MiB/s, 37 objects/s recovering 2023-09-23T18:06:16.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:16 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 9327/110577 objects degraded (8.435%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:16.325 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3207384105' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:16.328 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:16.366 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:16.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:16.374 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:16.453 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:16.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:16.506 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:16.540 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:06:16.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:16.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:16.552 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:16.653 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:16.683 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:16.724 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:16.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:16.728 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:16.828 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:16.856 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 26 16 41311 41295 99.1503 69.625 0.00911797 0.0100783 2023-09-23T18:06:16.907 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:17.082 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:17.090 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:17.090 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:17.191 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:17.260 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:17.341 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:17.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:17.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:17.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:17.480 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:17.519 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:17.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:17.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:17.627 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:17.656 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:17.698 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:17.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:17.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:17.802 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:17.837 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:17.856 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 27 16 42409 42393 98.0207 68.625 0.0120363 0.0101953 2023-09-23T18:06:17.877 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:17.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:17.881 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:17.932 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:17.983 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:18.055 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:06:18.055 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:42570 7.0 e290/286 7:0f76d0e3:::benchmark_data_smithi055_311528_object42569:head v 290'2723, mlcod=0'0)", 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:18.036488+0000", 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0092222599999999995, 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092558999999999992, 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:18.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.036488+0000", 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.036488+0000", 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.036488+0000", 2023-09-23T18:06:18.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.036532+0000", 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4821999999999999e-05 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.036544+0000", 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1382999999999999e-05 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.036565+0000", 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0808000000000001e-05 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.036626+0000", 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0977999999999998e-05 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:06:18.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.036653+0000", 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7611999999999999e-05 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:42576 7.2 7:4f5b4fe2:::benchmark_data_smithi055_311528_object42575:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e290)", 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:18.037462+0000", 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0082489520000000004, 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0083625060000000005, 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:18.060 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 42576 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.037462+0000", 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.037462+0000", 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.037461+0000", 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:18.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.037485+0000", 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3421000000000002e-05 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.037493+0000", 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.089e-06 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.037499+0000", 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.02e-06 2023-09-23T18:06:18.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040078+0000", 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025791709999999999 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040286+0000", 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020783800000000001 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,2", 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040466+0000", 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000180586 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045302+0000", 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048355259999999997 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:42577 7.7 e290/279 7:ef426e8e:::benchmark_data_smithi055_311528_object42576:head v 290'2721, mlcod=290'2719)", 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:18.040547+0000", 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0051640059999999996, 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053778530000000001, 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040547+0000", 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040547+0000", 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040546+0000", 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040578+0000", 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1714999999999997e-05 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040587+0000", 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2429999999999994e-06 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040593+0000", 2023-09-23T18:06:18.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0510000000000002e-06 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040628+0000", 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5351000000000001e-05 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.040647+0000", 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8725000000000001e-05 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.069 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:42574 7.6 e290/279 7:699e1ea7:::benchmark_data_smithi055_311528_object42573:head v 290'2610, mlcod=290'2608)", 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:18.041126+0000", 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0045847850000000001, 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048882919999999998, 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.070 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041126+0000", 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041126+0000", 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041125+0000", 2023-09-23T18:06:18.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041149+0000", 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3951000000000002e-05 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041160+0000", 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0974e-05 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041167+0000", 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.455e-06 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041301+0000", 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013474599999999999 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041320+0000", 2023-09-23T18:06:18.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8725000000000001e-05 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:42581 7.6 e290/279 7:68fb855f:::benchmark_data_smithi055_311528_object42580:head v 290'2611, mlcod=290'2608)", 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:18.041211+0000", 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0045000140000000001, 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048644200000000004, 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:06:18.074 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041211+0000", 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041211+0000", 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041210+0000", 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041232+0000", 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1435999999999999e-05 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:18.076 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041239+0000", 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1319999999999996e-06 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.041244+0000", 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7609999999999998e-06 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.042853+0000", 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001609709 2023-09-23T18:06:18.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.042875+0000", 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.196e-05 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:42578 7.0 e290/286 7:01b9d232:::benchmark_data_smithi055_311528_object42577:head v 290'2724, mlcod=0'0)", 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:18.044174+0000", 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.001537121, 2023-09-23T18:06:18.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019601179999999998, 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044174+0000", 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044174+0000", 2023-09-23T18:06:18.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044172+0000", 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044221+0000", 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8480999999999999e-05 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044231+0000", 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.007e-05 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044237+0000", 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3439999999999997e-06 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:06:18.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044282+0000", 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5132000000000001e-05 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044308+0000", 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5585e-05 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.082 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:42580 7.0 e290/286 7:09dc7be0:::benchmark_data_smithi055_311528_object42579:head v 290'2725, mlcod=0'0)", 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:18.044250+0000", 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0014604640000000001, 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001962821, 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044250+0000", 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044250+0000", 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044250+0000", 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:18.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044288+0000", 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7934e-05 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044295+0000", 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9140000000000002e-06 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044300+0000", 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9110000000000001e-06 2023-09-23T18:06:18.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045830+0000", 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001530472 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045857+0000", 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7063000000000002e-05 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:18.086 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:42583 7.0 e290/286)", 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:18.044552+0000", 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0011588799999999999, 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001717395, 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044552+0000", 2023-09-23T18:06:18.087 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044552+0000", 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044551+0000", 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044593+0000", 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1338999999999998e-05 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044603+0000", 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0105999999999999e-05 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:18.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.044609+0000", 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6019999999999996e-06 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:42579 7.f e290/279 7:f0fd78c9:::benchmark_data_smithi055_311528_object42578:head v 290'2613, mlcod=290'2611)", 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:18.045035+0000", 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.000675459, 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001289672, 2023-09-23T18:06:18.090 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045035+0000", 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045035+0000", 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:18.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045035+0000", 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045057+0000", 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2067000000000001e-05 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:18.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045067+0000", 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0607e-05 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045074+0000", 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4139999999999999e-06 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045120+0000", 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6554999999999997e-05 2023-09-23T18:06:18.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:18.045140+0000", 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9664999999999999e-05 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 9 2023-09-23T18:06:18.094 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:06:18.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:18.095 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:17 smithi099 ceph-mon[167076]: pgmap v719: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.6 KiB/s rd, 67 MiB/s wr, 1.07k op/s; 9327/116754 objects degraded (7.989%); 1268/116754 objects misplaced (1.086%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T18:06:18.095 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:18.095 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:17 smithi055 ceph-mon[179668]: pgmap v719: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.6 KiB/s rd, 67 MiB/s wr, 1.07k op/s; 9327/116754 objects degraded (7.989%); 1268/116754 objects misplaced (1.086%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T18:06:18.096 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:18.096 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:17 smithi055 ceph-mon[184174]: pgmap v719: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.4 GiB data, 9.8 GiB used, 705 GiB / 715 GiB avail; 1.6 KiB/s rd, 67 MiB/s wr, 1.07k op/s; 9327/116754 objects degraded (7.989%); 1268/116754 objects misplaced (1.086%); 4.2 MiB/s, 67 objects/s recovering 2023-09-23T18:06:18.096 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:18.153 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:18.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:18.162 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:18.262 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:18.276 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:06:18.334 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:18.722 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:18.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:18.729 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:18.778 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:06:18.830 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:18.856 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 28 16 43419 43403 96.7755 63.125 0.014102 0.0103276 2023-09-23T18:06:18.867 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:18.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:18.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:18.938 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:19.038 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:19.069 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:19.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:19.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:19.113 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:19.214 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:19.294 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:19.462 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:19.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:19.476 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:19.577 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:19.609 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:19.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:19.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:19.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:19.680 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:19.751 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:19.754 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:19.780 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:19.818 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:19.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:19.821 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:19.856 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 29 16 44405 44389 95.5634 61.625 0.0258475 0.0104557 2023-09-23T18:06:19.922 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:19.957 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:19.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:20.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:20.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:20.105 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:20.136 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:20.177 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:20.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:20.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:20.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:20.286 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:20 smithi055 ceph-mon[184174]: pgmap v720: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1.3 KiB/s rd, 64 MiB/s wr, 1.03k op/s; 9327/120840 objects degraded (7.718%); 1268/120840 objects misplaced (1.049%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:06:20.286 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:20 smithi055 ceph-mon[179668]: pgmap v720: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1.3 KiB/s rd, 64 MiB/s wr, 1.03k op/s; 9327/120840 objects degraded (7.718%); 1268/120840 objects misplaced (1.049%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:06:20.310 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:20 smithi099 ceph-mon[167076]: pgmap v720: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.5 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1.3 KiB/s rd, 64 MiB/s wr, 1.03k op/s; 9327/120840 objects degraded (7.718%); 1268/120840 objects misplaced (1.049%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:06:20.313 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:20.351 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:20.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:20.356 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:20.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:20.488 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:20.537 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:20.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:20.541 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:20.637 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:06:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:06:20] "GET /metrics HTTP/1.1" 200 36505 "" "Prometheus/2.43.0" 2023-09-23T18:06:20.641 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:20.673 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:20.711 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:06:20.711 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:06:20.711 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T18:06:20.711 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 0.05 2023-09-23T18:06:20.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:20.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:20.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:20.817 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:20.858 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 30 16 45484 45468 94.6223 67.4375 0.0138282 0.0105627 2023-09-23T18:06:20.940 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:21.065 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:21.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:21.097 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:21.112 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:06:21.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/82738483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:21.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:21 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:21.187 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/82738483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:21.188 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:21 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:21.198 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:21.294 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:21.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/82738483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:21.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:21 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:21.554 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:21.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:21.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:21.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:21.694 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:21.738 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:21.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:21.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:21.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:21.858 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 31 16 46471 46455 93.5608 61.6875 0.0105798 0.0106818 2023-09-23T18:06:21.873 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:21.917 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:21.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:21.920 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:22.021 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:22.056 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:22.064 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:06:22.096 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:22.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:22.101 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:22.101 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:06:22.121 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:22.138 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:06:22.182 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:06:22.201 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:22.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:06:22.266 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:06:22.289 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:06:22.289 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:06:22.302 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: pgmap v721: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1.3 KiB/s rd, 69 MiB/s wr, 1.11k op/s; 9327/129681 objects degraded (7.192%); 1268/129681 objects misplaced (0.978%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:06:22.302 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 9327/129681 objects degraded (7.192%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:22.302 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:22.302 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: failed to encode map e291 with expected crc 2023-09-23T18:06:22.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: failed to encode map e291 with expected crc 2023-09-23T18:06:22.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:06:22.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: osdmap e291: 8 total, 8 up, 8 in 2023-09-23T18:06:22.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: failed to encode map e291 with expected crc 2023-09-23T18:06:22.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:22.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:22.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1774731950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:22.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: pgmap v721: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1.3 KiB/s rd, 69 MiB/s wr, 1.11k op/s; 9327/129681 objects degraded (7.192%); 1268/129681 objects misplaced (0.978%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:06:22.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 9327/129681 objects degraded (7.192%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:22.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:22.304 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: failed to encode map e291 with expected crc 2023-09-23T18:06:22.304 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: failed to encode map e291 with expected crc 2023-09-23T18:06:22.304 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:06:22.304 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: osdmap e291: 8 total, 8 up, 8 in 2023-09-23T18:06:22.304 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: failed to encode map e291 with expected crc 2023-09-23T18:06:22.304 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:22.304 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:22.304 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1774731950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:22.306 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:06:22.313 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:22.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: pgmap v721: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 1.3 KiB/s rd, 69 MiB/s wr, 1.11k op/s; 9327/129681 objects degraded (7.192%); 1268/129681 objects misplaced (0.978%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:06:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 9327/129681 objects degraded (7.192%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: failed to encode map e291 with expected crc 2023-09-23T18:06:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: failed to encode map e291 with expected crc 2023-09-23T18:06:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:06:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: osdmap e291: 8 total, 8 up, 8 in 2023-09-23T18:06:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: failed to encode map e291 with expected crc 2023-09-23T18:06:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1774731950' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:22.349 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:06:22.366 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:06:22.366 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:06:22.366 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:06:22.366 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:06:22.366 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:06:22.366 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:06:22.366 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:06:22.366 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:06:22.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:22.506 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:22.506 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:22.607 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:22.701 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:22.858 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 32 16 47418 47402 92.4875 59.1875 0.0178853 0.0108067 2023-09-23T18:06:23.629 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:06:23.678 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:23.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:23.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:23.785 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:06:23.792 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:23.858 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 33 16 48549 48533 91.8273 70.6875 0.0119865 0.010885 2023-09-23T18:06:23.937 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:24.104 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:24.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:24.120 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:24.185 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:24 smithi055 ceph-mon[179668]: pgmap v723: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 9327/134592 objects degraded (6.930%); 954/134592 objects misplaced (0.709%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:06:24.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:24 smithi055 ceph-mon[184174]: pgmap v723: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 9327/134592 objects degraded (6.930%); 954/134592 objects misplaced (0.709%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:06:24.220 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:24.309 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:24.401 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:24.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:24.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:24.507 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:24.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:24 smithi099 ceph-mon[167076]: pgmap v723: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.7 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 72 MiB/s wr, 1.16k op/s; 9327/134592 objects degraded (6.930%); 954/134592 objects misplaced (0.709%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:06:24.592 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:24.625 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:24.674 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:24.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:24.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:24.785 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:24.822 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:24.825 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:24.859 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 34 16 49397 49381 90.6826 53 0.0180514 0.0110219 2023-09-23T18:06:24.865 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:24.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:24.870 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:24.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:25.003 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:25.032 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:06:25.048 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:25.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:25.052 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:25.124 INFO:teuthology.orchestra.run.smithi055.stdout:moved 3 / 357 (0.840336%) 2023-09-23T18:06:25.124 INFO:teuthology.orchestra.run.smithi055.stdout:avg 44.625 2023-09-23T18:06:25.124 INFO:teuthology.orchestra.run.smithi055.stdout:stddev 7.19266 -> 7.56534 (expected baseline 6.24875) 2023-09-23T18:06:25.124 INFO:teuthology.orchestra.run.smithi055.stdout:min osd.2 with 30 -> 29 pgs (0.672269 -> 0.64986 * mean) 2023-09-23T18:06:25.124 INFO:teuthology.orchestra.run.smithi055.stdout:max osd.3 with 53 -> 52 pgs (1.18768 -> 1.16527 * mean) 2023-09-23T18:06:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:06:25.125 INFO:teuthology.orchestra.run.smithi055.stdout:oload 110 2023-09-23T18:06:25.125 INFO:teuthology.orchestra.run.smithi055.stdout:max_change 0.05 2023-09-23T18:06:25.125 INFO:teuthology.orchestra.run.smithi055.stdout:max_change_osds 4 2023-09-23T18:06:25.125 INFO:teuthology.orchestra.run.smithi055.stdout:average_utilization 0.0157 2023-09-23T18:06:25.125 INFO:teuthology.orchestra.run.smithi055.stdout:overload_utilization 0.0173 2023-09-23T18:06:25.125 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 weight 1.0000 -> 0.9500 2023-09-23T18:06:25.125 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 weight 1.0000 -> 0.9500 2023-09-23T18:06:25.125 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 weight 1.0000 -> 0.9500 2023-09-23T18:06:25.153 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:25.184 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:25.190 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:25 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','3':'62260'}"}]: dispatch 2023-09-23T18:06:25.190 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:25 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','3':'62260'}"}]: dispatch 2023-09-23T18:06:25.224 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:25.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:25.227 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:25.328 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:25.382 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:25.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:25 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','3':'62260'}"}]: dispatch 2023-09-23T18:06:25.685 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:25.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:25.689 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:25.789 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:25.816 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:25.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:25.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:25.850 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:25.860 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 35 16 49870 49854 88.9377 29.5625 0.00246365 0.0110569 2023-09-23T18:06:25.951 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:25.981 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:26.023 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T18:06:26.024 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T18:06:26.076 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:26.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:26.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:26.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:26.208 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:26.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:26.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:26.244 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:26.302 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:26 smithi055 ceph-mon[184174]: pgmap v724: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 9327/142161 objects degraded (6.561%); 954/142161 objects misplaced (0.671%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:06:26.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:26 smithi055 ceph-mon[184174]: from='client.45614 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:26.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:26 smithi055 ceph-mon[184174]: failed to encode map e292 with expected crc 2023-09-23T18:06:26.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:26 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','3':'62260'}"}]': finished 2023-09-23T18:06:26.303 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:26 smithi055 ceph-mon[184174]: osdmap e292: 8 total, 8 up, 8 in 2023-09-23T18:06:26.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:26 smithi055 ceph-mon[179668]: pgmap v724: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 9327/142161 objects degraded (6.561%); 954/142161 objects misplaced (0.671%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:06:26.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:26 smithi055 ceph-mon[179668]: from='client.45614 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:26.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:26 smithi055 ceph-mon[179668]: failed to encode map e292 with expected crc 2023-09-23T18:06:26.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:26 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','3':'62260'}"}]': finished 2023-09-23T18:06:26.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:26 smithi055 ceph-mon[179668]: osdmap e292: 8 total, 8 up, 8 in 2023-09-23T18:06:26.327 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:26.345 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:26.371 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:26.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:26.405 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:26 smithi099 ceph-mon[167076]: pgmap v724: 119 pgs: 1 active+recovering+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 2.9 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 66 MiB/s wr, 1.05k op/s; 9327/142161 objects degraded (6.561%); 954/142161 objects misplaced (0.671%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:06:26.405 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:26 smithi099 ceph-mon[167076]: from='client.45614 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:26.405 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:26 smithi099 ceph-mon[167076]: failed to encode map e292 with expected crc 2023-09-23T18:06:26.405 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:26 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'62260','2':'62260','3':'62260'}"}]': finished 2023-09-23T18:06:26.405 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:26 smithi099 ceph-mon[167076]: osdmap e292: 8 total, 8 up, 8 in 2023-09-23T18:06:26.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:26.406 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:26.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:26.534 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:26.554 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:06:26.554 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:06:26.566 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:26.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:26.569 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:26.670 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:26.706 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:26.860 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 36 16 49870 49854 86.4694 0 - 0.0110569 2023-09-23T18:06:26.947 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:26.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:26.955 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:27.056 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:27.084 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:27.119 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:27.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:27.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:27.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:27 smithi055 ceph-mon[179668]: failed to encode map e293 with expected crc 2023-09-23T18:06:27.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:27 smithi055 ceph-mon[179668]: osdmap e293: 8 total, 8 up, 8 in 2023-09-23T18:06:27.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:27.249 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:27.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:27.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:27.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:27.391 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:27.419 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:27 smithi099 ceph-mon[167076]: failed to encode map e293 with expected crc 2023-09-23T18:06:27.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:27 smithi099 ceph-mon[167076]: osdmap e293: 8 total, 8 up, 8 in 2023-09-23T18:06:27.422 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:27.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:27 smithi055 ceph-mon[184174]: failed to encode map e293 with expected crc 2023-09-23T18:06:27.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:27 smithi055 ceph-mon[184174]: osdmap e293: 8 total, 8 up, 8 in 2023-09-23T18:06:27.464 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:27.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:27.468 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:27.517 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:06:27.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:27.601 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:27.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:27.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:27.643 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:27.735 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:27.743 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:27.842 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:27.860 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 37 15 50418 50403 85.0608 17.1562 0.0123383 0.0117492 2023-09-23T18:06:27.919 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:27.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:27.930 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:28.030 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:28.046 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:06:28.062 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:28.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:28.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:28.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:28.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:28.246 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:28.276 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:06:28.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:28.303 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:06:28.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:28.305 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: pgmap v727: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s; 9327/146976 objects degraded (6.346%); 400/146976 objects misplaced (0.272%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T18:06:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 9327/146976 objects degraded (6.346%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: failed to encode map e294 with expected crc 2023-09-23T18:06:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: failed to encode map e294 with expected crc 2023-09-23T18:06:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: osdmap e294: 8 total, 8 up, 8 in 2023-09-23T18:06:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: failed to encode map e294 with expected crc 2023-09-23T18:06:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2769201975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:06:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:28.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:28 smithi099 ceph-mon[167076]: osdmap e295: 8 total, 8 up, 8 in 2023-09-23T18:06:28.334 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:06:28.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: pgmap v727: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s; 9327/146976 objects degraded (6.346%); 400/146976 objects misplaced (0.272%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T18:06:28.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 9327/146976 objects degraded (6.346%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:28.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: failed to encode map e294 with expected crc 2023-09-23T18:06:28.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: failed to encode map e294 with expected crc 2023-09-23T18:06:28.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: osdmap e294: 8 total, 8 up, 8 in 2023-09-23T18:06:28.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: failed to encode map e294 with expected crc 2023-09-23T18:06:28.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2769201975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:28.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:28.345 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:06:28.346 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:28.346 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[184174]: osdmap e295: 8 total, 8 up, 8 in 2023-09-23T18:06:28.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: pgmap v727: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.0 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 60 MiB/s wr, 960 op/s; 9327/146976 objects degraded (6.346%); 400/146976 objects misplaced (0.272%); 9.0 MiB/s, 144 objects/s recovering 2023-09-23T18:06:28.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 9327/146976 objects degraded (6.346%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:28.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: failed to encode map e294 with expected crc 2023-09-23T18:06:28.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: failed to encode map e294 with expected crc 2023-09-23T18:06:28.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: osdmap e294: 8 total, 8 up, 8 in 2023-09-23T18:06:28.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: failed to encode map e294 with expected crc 2023-09-23T18:06:28.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2769201975' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:28.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:28.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:06:28.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:28.347 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:28 smithi055 ceph-mon[179668]: osdmap e295: 8 total, 8 up, 8 in 2023-09-23T18:06:28.361 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:06:28.385 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:06:28.406 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:28.409 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:06:28.432 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:06:28.433 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:28.452 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:06:28.453 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:06:28.453 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:06:28.453 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:06:28.453 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:06:28.453 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:06:28.453 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:06:28.453 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:06:28.453 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:06:28.467 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:28.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:28.470 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:28.571 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:28.603 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:28.860 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 38 16 50830 50814 83.4996 25.6875 0.00274541 0.0117689 2023-09-23T18:06:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:29 smithi055 ceph-mon[184174]: failed to encode map e295 with expected crc 2023-09-23T18:06:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:29 smithi055 ceph-mon[184174]: 1.19 scrub starts 2023-09-23T18:06:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3931256658' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:29.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:29 smithi055 ceph-mon[184174]: osdmap e296: 8 total, 8 up, 8 in 2023-09-23T18:06:29.437 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:29 smithi055 ceph-mon[179668]: failed to encode map e295 with expected crc 2023-09-23T18:06:29.437 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:29 smithi055 ceph-mon[179668]: 1.19 scrub starts 2023-09-23T18:06:29.437 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3931256658' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:29.437 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:29 smithi055 ceph-mon[179668]: osdmap e296: 8 total, 8 up, 8 in 2023-09-23T18:06:29.438 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:29.451 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:29.551 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:29.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:29 smithi099 ceph-mon[167076]: failed to encode map e295 with expected crc 2023-09-23T18:06:29.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:29 smithi099 ceph-mon[167076]: 1.19 scrub starts 2023-09-23T18:06:29.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3931256658' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:29.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:29 smithi099 ceph-mon[167076]: osdmap e296: 8 total, 8 up, 8 in 2023-09-23T18:06:29.636 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:29.861 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 39 16 51744 51728 82.8228 57.125 0.0102122 0.0120687 2023-09-23T18:06:30.431 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:06:30] "GET /metrics HTTP/1.1" 200 36548 "" "Prometheus/2.43.0" 2023-09-23T18:06:30.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-mon[184174]: pgmap v730: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 38 MiB/s wr, 600 op/s; 12378/149373 objects degraded (8.287%); 400/149373 objects misplaced (0.268%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T18:06:30.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-mon[184174]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-mon[184174]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-mon[184174]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-mon[184174]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-mon[179668]: pgmap v730: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 38 MiB/s wr, 600 op/s; 12378/149373 objects degraded (8.287%); 400/149373 objects misplaced (0.268%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T18:06:30.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-mon[179668]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-mon[179668]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-mon[179668]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:30 smithi055 ceph-mon[179668]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.474 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:30.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:30.481 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:30.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:30 smithi099 ceph-mon[167076]: pgmap v730: 119 pgs: 1 active+recovering+undersized+remapped, 1 peering, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.0 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 38 MiB/s wr, 600 op/s; 12378/149373 objects degraded (8.287%); 400/149373 objects misplaced (0.268%); 8.7 MiB/s, 138 objects/s recovering 2023-09-23T18:06:30.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:30 smithi099 ceph-mon[167076]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:30 smithi099 ceph-mon[167076]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:30 smithi099 ceph-mon[167076]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:30 smithi099 ceph-mon[167076]: failed to encode map e296 with expected crc 2023-09-23T18:06:30.582 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:30.655 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:06:30.673 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:30.862 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:06:30.863122+0000 min lat: 0.00119503 max lat: 2.30701 avg lat: 0.0121234 2023-09-23T18:06:30.862 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:06:30.862 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 40 16 52831 52815 82.4483 67.9375 0.00988919 0.0121234 2023-09-23T18:06:31.681 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:31.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:31.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:31.788 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:31.862 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 41 16 53933 53917 82.1174 68.875 0.006348 0.0121719 2023-09-23T18:06:31.874 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:31.909 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:32.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:32 smithi055 ceph-mon[179668]: pgmap v732: 119 pgs: 2 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 38 MiB/s wr, 613 op/s; 10484/153345 objects degraded (6.837%); 2009/153345 objects misplaced (1.310%); 11 MiB/s, 181 objects/s recovering 2023-09-23T18:06:32.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:32 smithi055 ceph-mon[179668]: 1.c scrub starts 2023-09-23T18:06:32.504 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:06:32.554 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:32.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:32.559 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:32.564 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:32 smithi055 ceph-mon[184174]: pgmap v732: 119 pgs: 2 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 38 MiB/s wr, 613 op/s; 10484/153345 objects degraded (6.837%); 2009/153345 objects misplaced (1.310%); 11 MiB/s, 181 objects/s recovering 2023-09-23T18:06:32.564 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:32 smithi055 ceph-mon[184174]: 1.c scrub starts 2023-09-23T18:06:32.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:32 smithi099 ceph-mon[167076]: pgmap v732: 119 pgs: 2 active+recovering+undersized+remapped, 1 peering, 6 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 3.1 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 38 MiB/s wr, 613 op/s; 10484/153345 objects degraded (6.837%); 2009/153345 objects misplaced (1.310%); 11 MiB/s, 181 objects/s recovering 2023-09-23T18:06:32.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:32 smithi099 ceph-mon[167076]: 1.c scrub starts 2023-09-23T18:06:32.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:32.711 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:32.761 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:32.764 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:32.765 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:32.862 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 42 16 54957 54941 81.6862 64 0.013119 0.0122335 2023-09-23T18:06:32.865 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:32.969 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:33.110 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:33.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:33.117 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:33.201 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 10484/153345 objects degraded (6.837%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:33.201 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 10484/153345 objects degraded (6.837%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:33.219 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:33.306 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:33.410 INFO:teuthology.orchestra.run.smithi055.stdout:moved 0 / 357 (0%) 2023-09-23T18:06:33.410 INFO:teuthology.orchestra.run.smithi055.stdout:avg 44.625 2023-09-23T18:06:33.410 INFO:teuthology.orchestra.run.smithi055.stdout:stddev 7.56534 -> 7.56534 (expected baseline 6.24875) 2023-09-23T18:06:33.410 INFO:teuthology.orchestra.run.smithi055.stdout:min osd.2 with 29 -> 29 pgs (0.64986 -> 0.64986 * mean) 2023-09-23T18:06:33.410 INFO:teuthology.orchestra.run.smithi055.stdout:max osd.1 with 53 -> 53 pgs (1.18768 -> 1.18768 * mean) 2023-09-23T18:06:33.410 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:06:33.411 INFO:teuthology.orchestra.run.smithi055.stdout:oload 1000 2023-09-23T18:06:33.411 INFO:teuthology.orchestra.run.smithi055.stdout:max_change 0.05 2023-09-23T18:06:33.411 INFO:teuthology.orchestra.run.smithi055.stdout:max_change_osds 4 2023-09-23T18:06:33.411 INFO:teuthology.orchestra.run.smithi055.stdout:average_utilization 0.0171 2023-09-23T18:06:33.411 INFO:teuthology.orchestra.run.smithi055.stdout:overload_utilization 0.1705 2023-09-23T18:06:33.420 INFO:teuthology.orchestra.run.smithi055.stderr:no change 2023-09-23T18:06:33.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 10484/153345 objects degraded (6.837%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:33.863 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 43 16 55882 55866 81.1313 57.8125 0.0167402 0.0123207 2023-09-23T18:06:33.976 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:33.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:33.981 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:34.051 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:34.082 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:34.096 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T18:06:34.096 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 0.05 2023-09-23T18:06:34.112 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:34.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:34.156 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:34.156 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:34.257 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:34.289 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:34.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:34.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:34.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:34.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:34 smithi055 ceph-mon[184174]: pgmap v733: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s; 10471/156876 objects degraded (6.675%); 1609/156876 objects misplaced (1.026%); 7.1 MiB/s, 116 objects/s recovering 2023-09-23T18:06:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:34 smithi055 ceph-mon[179668]: pgmap v733: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s; 10471/156876 objects degraded (6.675%); 1609/156876 objects misplaced (1.026%); 7.1 MiB/s, 116 objects/s recovering 2023-09-23T18:06:34.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:34.474 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:34 smithi099 ceph-mon[167076]: pgmap v733: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 34 MiB/s wr, 549 op/s; 10471/156876 objects degraded (6.675%); 1609/156876 objects misplaced (1.026%); 7.1 MiB/s, 116 objects/s recovering 2023-09-23T18:06:34.479 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:34.509 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:34.521 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:34.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:34.526 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:34.583 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:06:34.583 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:06:34.583 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.583 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:56489 7.f 7:f199c7cc:::benchmark_data_smithi055_311528_object56488:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T18:06:34.583 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:34.573682+0000", 2023-09-23T18:06:34.583 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0029402569999999999, 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029740510000000001, 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 56489 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:34.584 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573682+0000", 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573682+0000", 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573680+0000", 2023-09-23T18:06:34.585 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573708+0000", 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7194000000000001e-05 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573720+0000", 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2805e-05 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.586 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573729+0000", 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6559999999999998e-06 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573783+0000", 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3934999999999998e-05 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:06:34.587 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573924+0000", 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014087099999999999 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.574062+0000", 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013791300000000001 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.588 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:56495 7.6 7.dce26466 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e296)", 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:34.573750+0000", 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0028722999999999999, 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029896039999999999, 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 56495 2023-09-23T18:06:34.589 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573750+0000", 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573750+0000", 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:34.590 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573749+0000", 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573772+0000", 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2806000000000002e-05 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.591 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573780+0000", 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7549999999999994e-06 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573786+0000", 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2179999999999996e-06 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.592 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:56493 7.5 e296/279)", 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:34.573808+0000", 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0028136680000000001, 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029852720000000002, 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:34.593 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573808+0000", 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573808+0000", 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573808+0000", 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:34.594 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573838+0000", 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0185e-05 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573846+0000", 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9279999999999993e-06 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.595 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.573851+0000", 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2730000000000002e-06 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:56497 7.e e296/279)", 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:34.576593+0000", 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 2.9023e-05, 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00025229899999999999, 2023-09-23T18:06:34.596 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:34.597 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:06:34.597 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:34.597 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.597 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:34.597 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576593+0000", 2023-09-23T18:06:34.597 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:34.597 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.597 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.597 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:34.597 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576593+0000", 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576592+0000", 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576616+0000", 2023-09-23T18:06:34.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3923000000000001e-05 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576629+0000", 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3169e-05 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576637+0000", 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4540000000000001e-06 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:34.599 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:56498 7.e e296/279)", 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:34.576646+0000", 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4294967295.9999766, 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024586299999999997, 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576646+0000", 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576646+0000", 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:34.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576645+0000", 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576667+0000", 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1362999999999999e-05 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576674+0000", 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.379e-06 2023-09-23T18:06:34.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:34.576679+0000", 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5389999999999998e-06 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 5 2023-09-23T18:06:34.603 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:06:34.627 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:34.709 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:34.864 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 44 16 56746 56730 80.5125 54 0.0140642 0.0124146 2023-09-23T18:06:34.902 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:34.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:34.917 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:35.017 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:35.082 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:35.222 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:06:35.252 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:35.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:35.259 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:35.360 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:35.395 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:35.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:35.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:35.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:35.515 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:35 smithi055 ceph-mon[184174]: from='client.55534 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:35.516 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:35 smithi055 ceph-mon[184174]: 1.19 deep-scrub starts 2023-09-23T18:06:35.516 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:35 smithi055 ceph-mon[179668]: from='client.55534 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:35.516 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:35 smithi055 ceph-mon[179668]: 1.19 deep-scrub starts 2023-09-23T18:06:35.542 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:35.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:35 smithi099 ceph-mon[167076]: from='client.55534 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:35.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:35 smithi099 ceph-mon[167076]: 1.19 deep-scrub starts 2023-09-23T18:06:35.654 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:35.864 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 45 16 57654 57638 79.9847 56.75 0.0225635 0.0124963 2023-09-23T18:06:35.932 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:35.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:35.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:36.021 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:36.041 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:36.151 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:36.545 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:36 smithi055 ceph-mon[184174]: pgmap v734: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 10471/164229 objects degraded (6.376%); 1609/164229 objects misplaced (0.980%); 5.6 MiB/s, 91 objects/s recovering 2023-09-23T18:06:36.545 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:36 smithi055 ceph-mon[184174]: 1.c scrub starts 2023-09-23T18:06:36.545 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:36 smithi055 ceph-mon[179668]: pgmap v734: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 10471/164229 objects degraded (6.376%); 1609/164229 objects misplaced (0.980%); 5.6 MiB/s, 91 objects/s recovering 2023-09-23T18:06:36.545 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:36 smithi055 ceph-mon[179668]: 1.c scrub starts 2023-09-23T18:06:36.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:36 smithi099 ceph-mon[167076]: pgmap v734: 119 pgs: 1 active+clean+scrubbing, 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.3 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 47 MiB/s wr, 753 op/s; 10471/164229 objects degraded (6.376%); 1609/164229 objects misplaced (0.980%); 5.6 MiB/s, 91 objects/s recovering 2023-09-23T18:06:36.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:36 smithi099 ceph-mon[167076]: 1.c scrub starts 2023-09-23T18:06:36.617 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:36.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:36.627 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:36.680 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:36.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:36.773 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:36.815 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:36.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:36.818 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:36.864 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 46 16 58519 58503 79.4215 54.0625 0.0128189 0.0125849 2023-09-23T18:06:36.919 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:37.018 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:37.040 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:06:37.115 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:06:37.169 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:06:37.221 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:06:37.262 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:06:37.298 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:06:37.343 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:06:37.389 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:06:37.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:06:37.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:06:37.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:06:37.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:06:37.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:06:37.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:06:37.403 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:06:37.403 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:06:37.490 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:06:37.594 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:37.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[184174]: 1.19 scrub starts 2023-09-23T18:06:37.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:37.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:37.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[184174]: pgmap v735: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 10471/169740 objects degraded (6.169%); 1243/169740 objects misplaced (0.732%); 7.5 MiB/s, 121 objects/s recovering 2023-09-23T18:06:37.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/741690335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:37.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2893275777' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:37.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:37.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:37.606 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:37.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[179668]: 1.19 scrub starts 2023-09-23T18:06:37.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:37.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:37.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[179668]: pgmap v735: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 10471/169740 objects degraded (6.169%); 1243/169740 objects misplaced (0.732%); 7.5 MiB/s, 121 objects/s recovering 2023-09-23T18:06:37.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/741690335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:37.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2893275777' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:37.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:37 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:37.707 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:37.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:37 smithi099 ceph-mon[167076]: 1.19 scrub starts 2023-09-23T18:06:37.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:37.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:37.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:37 smithi099 ceph-mon[167076]: pgmap v735: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 51 MiB/s wr, 822 op/s; 10471/169740 objects degraded (6.169%); 1243/169740 objects misplaced (0.732%); 7.5 MiB/s, 121 objects/s recovering 2023-09-23T18:06:37.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/741690335' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:37.746 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2893275777' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:37.746 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:37 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:06:37.748 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:37.798 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:37.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:37.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:37.864 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 47 16 59390 59374 78.8902 54.4375 0.00859596 0.012671 2023-09-23T18:06:37.903 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:37.999 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:38.473 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:38.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:38.482 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:38.536 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:38.536 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[184174]: failed to encode map e297 with expected crc 2023-09-23T18:06:38.536 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[184174]: failed to encode map e297 with expected crc 2023-09-23T18:06:38.536 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:06:38.536 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[184174]: osdmap e297: 8 total, 8 up, 8 in 2023-09-23T18:06:38.536 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 10471/169740 objects degraded (6.169%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:38.541 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:38.541 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[179668]: failed to encode map e297 with expected crc 2023-09-23T18:06:38.541 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[179668]: failed to encode map e297 with expected crc 2023-09-23T18:06:38.541 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:06:38.541 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[179668]: osdmap e297: 8 total, 8 up, 8 in 2023-09-23T18:06:38.541 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 10471/169740 objects degraded (6.169%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:38.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:38.632 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:38.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:38.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:38.680 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:38.780 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:38.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:38 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:38 smithi099 ceph-mon[167076]: failed to encode map e297 with expected crc 2023-09-23T18:06:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:38 smithi099 ceph-mon[167076]: failed to encode map e297 with expected crc 2023-09-23T18:06:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:06:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:38 smithi099 ceph-mon[167076]: osdmap e297: 8 total, 8 up, 8 in 2023-09-23T18:06:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 10471/169740 objects degraded (6.169%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:38.864 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 48 16 60330 60314 78.4707 58.75 0.0160182 0.0127374 2023-09-23T18:06:38.881 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:39.014 INFO:teuthology.orchestra.run.smithi055.stdout:moved 0 / 357 (0%) 2023-09-23T18:06:39.014 INFO:teuthology.orchestra.run.smithi055.stdout:avg 44.625 2023-09-23T18:06:39.014 INFO:teuthology.orchestra.run.smithi055.stdout:stddev 7.56534 -> 7.56534 (expected baseline 6.24875) 2023-09-23T18:06:39.014 INFO:teuthology.orchestra.run.smithi055.stdout:min osd.2 with 29 -> 29 pgs (0.64986 -> 0.64986 * mean) 2023-09-23T18:06:39.014 INFO:teuthology.orchestra.run.smithi055.stdout:max osd.1 with 53 -> 53 pgs (1.18768 -> 1.18768 * mean) 2023-09-23T18:06:39.014 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:06:39.014 INFO:teuthology.orchestra.run.smithi055.stdout:oload 1000 2023-09-23T18:06:39.014 INFO:teuthology.orchestra.run.smithi055.stdout:max_change 0.05 2023-09-23T18:06:39.015 INFO:teuthology.orchestra.run.smithi055.stdout:max_change_osds 4 2023-09-23T18:06:39.015 INFO:teuthology.orchestra.run.smithi055.stdout:average_utilization 0.0185 2023-09-23T18:06:39.015 INFO:teuthology.orchestra.run.smithi055.stdout:overload_utilization 0.1849 2023-09-23T18:06:39.037 INFO:teuthology.orchestra.run.smithi055.stderr:no change 2023-09-23T18:06:39.058 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:39.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:39.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:39.171 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:39.208 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:39.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:39.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:39.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:39.358 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:39.437 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:39.534 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:06:39.534 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:06:39.562 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:39 smithi055 ceph-mon[184174]: pgmap v737: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s; 10471/173049 objects degraded (6.051%); 1243/173049 objects misplaced (0.718%); 6.7 MiB/s, 109 objects/s recovering 2023-09-23T18:06:39.562 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:39 smithi055 ceph-mon[184174]: from='client.45635 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:39.562 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:39 smithi055 ceph-mon[179668]: pgmap v737: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s; 10471/173049 objects degraded (6.051%); 1243/173049 objects misplaced (0.718%); 6.7 MiB/s, 109 objects/s recovering 2023-09-23T18:06:39.562 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:39 smithi055 ceph-mon[179668]: from='client.45635 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:39.587 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:39.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:39.593 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:39.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:39.719 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T18:06:39.720 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T18:06:39.725 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:39.772 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:39.774 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:39 smithi099 ceph-mon[167076]: pgmap v737: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.5 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 51 MiB/s wr, 810 op/s; 10471/173049 objects degraded (6.051%); 1243/173049 objects misplaced (0.718%); 6.7 MiB/s, 109 objects/s recovering 2023-09-23T18:06:39.774 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:39 smithi099 ceph-mon[167076]: from='client.45635 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:39.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:39.775 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:39.864 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 49 16 61241 61225 78.0315 56.9375 0.0287649 0.0128095 2023-09-23T18:06:39.876 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:39.980 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:06:40.056 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:40.177 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:40.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:40.190 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:40.196 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:06:40.290 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:40.307 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:06:40.412 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:40.443 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:06:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:06:40] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-09-23T18:06:40.532 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:40.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:40.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:40.647 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:40.756 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:40.821 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:40.865 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 50 16 61962 61946 77.3725 45.0625 0.0188634 0.0129175 2023-09-23T18:06:40.949 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:40.956 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:40.958 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:41.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:41.088 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:41.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:41.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:41.140 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:41.241 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:41.303 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:41.605 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:41.844 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:41 smithi055 ceph-mon[184174]: pgmap v738: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 10471/180573 objects degraded (5.799%); 871/180573 objects misplaced (0.482%); 7.1 MiB/s, 115 objects/s recovering 2023-09-23T18:06:41.847 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:41 smithi055 ceph-mon[179668]: pgmap v738: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 10471/180573 objects degraded (5.799%); 871/180573 objects misplaced (0.482%); 7.1 MiB/s, 115 objects/s recovering 2023-09-23T18:06:41.864 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 51 16 62865 62849 76.9618 56.4375 0.0159065 0.0129886 2023-09-23T18:06:42.151 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:42.194 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:42.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:42.206 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:42.307 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:42.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:41 smithi099 ceph-mon[167076]: pgmap v738: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+clean+scrubbing, 5 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 3.7 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 57 MiB/s wr, 906 op/s; 10471/180573 objects degraded (5.799%); 871/180573 objects misplaced (0.482%); 7.1 MiB/s, 115 objects/s recovering 2023-09-23T18:06:42.388 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:42.611 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:42.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:42.617 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:42.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:42.751 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:42.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:42.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:42.805 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:42.864 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 52 16 63749 63733 76.5445 55.25 0.00990136 0.0130575 2023-09-23T18:06:42.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:42.935 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:42.983 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:42.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:42.987 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:43.088 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:43.181 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:43.686 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:43.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:43.697 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:43.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:43.867 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 53 16 64657 64641 76.1679 56.75 0.0194798 0.0131214 2023-09-23T18:06:43.948 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:06:43.948 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:06:43.960 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:44.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:44.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:44.004 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:44.105 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:44.162 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:43 smithi055 ceph-mon[184174]: pgmap v739: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 10471/184731 objects degraded (5.668%); 871/184731 objects misplaced (0.471%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:06:44.162 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 10471/184731 objects degraded (5.668%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:44.162 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:43 smithi055 ceph-mon[179668]: pgmap v739: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 10471/184731 objects degraded (5.668%); 871/184731 objects misplaced (0.471%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:06:44.162 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 10471/184731 objects degraded (5.668%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:44.205 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:44.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:43 smithi099 ceph-mon[167076]: pgmap v739: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 58 MiB/s wr, 927 op/s; 10471/184731 objects degraded (5.668%); 871/184731 objects misplaced (0.471%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:06:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 10471/184731 objects degraded (5.668%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:44.557 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:44.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:44.576 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:44.628 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:06:44.677 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:44.760 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:44.869 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 54 16 65458 65442 75.6819 50.0625 0.0254307 0.0132073 2023-09-23T18:06:44.900 INFO:teuthology.orchestra.run.smithi055.stdout:moved 35 / 357 (9.80392%) 2023-09-23T18:06:44.900 INFO:teuthology.orchestra.run.smithi055.stdout:avg 44.625 2023-09-23T18:06:44.900 INFO:teuthology.orchestra.run.smithi055.stdout:stddev 7.56534 -> 11.2576 (expected baseline 6.24875) 2023-09-23T18:06:44.900 INFO:teuthology.orchestra.run.smithi055.stdout:min osd.2 with 29 -> 31 pgs (0.64986 -> 0.694678 * mean) 2023-09-23T18:06:44.900 INFO:teuthology.orchestra.run.smithi055.stdout:max osd.1 with 53 -> 61 pgs (1.18768 -> 1.36695 * mean) 2023-09-23T18:06:44.900 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:06:44.900 INFO:teuthology.orchestra.run.smithi055.stdout:oload 110 2023-09-23T18:06:44.901 INFO:teuthology.orchestra.run.smithi055.stdout:max_change 3 2023-09-23T18:06:44.901 INFO:teuthology.orchestra.run.smithi055.stdout:max_change_osds 4 2023-09-23T18:06:44.901 INFO:teuthology.orchestra.run.smithi055.stdout:average_utilization 0.0194 2023-09-23T18:06:44.901 INFO:teuthology.orchestra.run.smithi055.stdout:overload_utilization 0.0213 2023-09-23T18:06:44.901 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 weight 0.9500 -> 0.6786 2023-09-23T18:06:44.901 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 weight 0.9500 -> 0.7190 2023-09-23T18:06:44.901 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 weight 0.9500 -> 0.8143 2023-09-23T18:06:44.922 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:44.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:44.932 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:45.030 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:44 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'47121','2':'53364','3':'44472'}"}]: dispatch 2023-09-23T18:06:45.030 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:44 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'47121','2':'53364','3':'44472'}"}]: dispatch 2023-09-23T18:06:45.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:45.059 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:45.151 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:45.152 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:44 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'47121','2':'53364','3':'44472'}"}]: dispatch 2023-09-23T18:06:45.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:45.154 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:45.254 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:45.274 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:06:45.292 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:45.302 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:06:45.331 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:06:45.352 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:06:45.372 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:06:45.391 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:06:45.410 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:06:45.429 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:06:45.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:06:45.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:06:45.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:06:45.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:06:45.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:06:45.431 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:06:45.431 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:06:45.431 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:06:45.739 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:45.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:45.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:45.774 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'reweight-by-utilization'} 2023-09-23T18:06:45.774 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 110 3.0 2023-09-23T18:06:45.843 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:45.869 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 55 16 65482 65466 74.3341 1.5 0.00467016 0.0132113 2023-09-23T18:06:45.886 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:45.908 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:06:45.938 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:45.977 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:45.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:45.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:46.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:46.112 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:46.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[184174]: from='client.55543 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:46.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[184174]: pgmap v740: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s; 10471/191073 objects degraded (5.480%); 871/191073 objects misplaced (0.456%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:06:46.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[184174]: failed to encode map e298 with expected crc 2023-09-23T18:06:46.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[184174]: failed to encode map e298 with expected crc 2023-09-23T18:06:46.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'47121','2':'53364','3':'44472'}"}]': finished 2023-09-23T18:06:46.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[184174]: osdmap e298: 8 total, 8 up, 8 in 2023-09-23T18:06:46.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/618713273' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:46.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:46.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4138355439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:46.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[179668]: from='client.55543 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:46.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[179668]: pgmap v740: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s; 10471/191073 objects degraded (5.480%); 871/191073 objects misplaced (0.456%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:06:46.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[179668]: failed to encode map e298 with expected crc 2023-09-23T18:06:46.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[179668]: failed to encode map e298 with expected crc 2023-09-23T18:06:46.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'47121','2':'53364','3':'44472'}"}]': finished 2023-09-23T18:06:46.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[179668]: osdmap e298: 8 total, 8 up, 8 in 2023-09-23T18:06:46.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/618713273' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:46.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:46.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4138355439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:46.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:46.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:46.148 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:46.249 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:46.291 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:45 smithi099 ceph-mon[167076]: from='client.55543 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:45 smithi099 ceph-mon[167076]: pgmap v740: 119 pgs: 1 active+recovering+undersized+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 56 MiB/s wr, 894 op/s; 10471/191073 objects degraded (5.480%); 871/191073 objects misplaced (0.456%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:06:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:45 smithi099 ceph-mon[167076]: failed to encode map e298 with expected crc 2023-09-23T18:06:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:45 smithi099 ceph-mon[167076]: failed to encode map e298 with expected crc 2023-09-23T18:06:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:45 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'47121','2':'53364','3':'44472'}"}]': finished 2023-09-23T18:06:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:45 smithi099 ceph-mon[167076]: osdmap e298: 8 total, 8 up, 8 in 2023-09-23T18:06:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/618713273' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:45 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:06:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4138355439' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:46.379 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:46.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:46.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:46.483 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:46.509 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:46.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:46.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:46.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:46.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:46.676 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:46.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:46.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:46.714 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:46.815 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:46.843 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:46.869 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 56 16 65482 65466 73.0076 0 - 0.0132113 2023-09-23T18:06:46.930 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:46.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:46.944 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:46.994 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:47.045 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:06:47.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:47.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[184174]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:47.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[184174]: osdmap e299: 8 total, 8 up, 8 in 2023-09-23T18:06:47.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[184174]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[184174]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.128 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[184174]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:47.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[179668]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:47.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[179668]: osdmap e299: 8 total, 8 up, 8 in 2023-09-23T18:06:47.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[179668]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[179668]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:46 smithi055 ceph-mon[179668]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.143 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:47.215 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:06:47.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:47.219 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:06:47.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:46 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:46 smithi099 ceph-mon[167076]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:46 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:06:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:46 smithi099 ceph-mon[167076]: osdmap e299: 8 total, 8 up, 8 in 2023-09-23T18:06:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:46 smithi099 ceph-mon[167076]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:46 smithi099 ceph-mon[167076]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:46 smithi099 ceph-mon[167076]: failed to encode map e299 with expected crc 2023-09-23T18:06:47.345 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:47.380 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:47.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:47.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:47.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:47.510 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:47.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:47.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:47.548 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:47.648 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:47.678 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:06:47.678 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:06:47.678 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65497 7.9 7.be60b6d9 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:47.155606+0000", 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.523180482, 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.52321208100000005, 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65497 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.679 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.155606+0000", 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.155606+0000", 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:47.680 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.155604+0000", 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.155629+0000", 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5182e-05 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.155642+0000", 2023-09-23T18:06:47.681 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3569e-05 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.155651+0000", 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8880000000000003e-06 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.682 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPush(7.1 300/293 [PushOp(7:82dd141d:::benchmark_data_smithi055_311528_object2489:head, version: 282'129, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(7:82dd141d:::benchmark_data_smithi055_311528_object2489:head@282'129, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:46.952234+0000", 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.72655294000000004, 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.72668447800000002, 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:46.952234+0000", 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.683 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:46.952234+0000", 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:46.952233+0000", 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:06:47.684 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:46.952285+0000", 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1214999999999997e-05 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:46.952300+0000", 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5610000000000001e-05 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:46.952318+0000", 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7884e-05 2023-09-23T18:06:47.685 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:47.686 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:06:47.686 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:47.686 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:06:47.686 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:06:47.686 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 2 2023-09-23T18:06:47.686 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:06:47.689 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:47.743 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:47.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:47.749 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:47.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:47.861 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:06:47.869 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 57 16 65503 65487 71.7506 0.65625 0.0139904 0.013482 2023-09-23T18:06:47.875 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:47.909 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:47.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:47.912 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:48.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:48.039 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:48.072 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:48.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:48.076 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:48.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: pgmap v743: 119 pgs: 4 remapped+peering, 1 active+recovering+undersized+remapped, 5 peering, 4 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s; 8635/195057 objects degraded (4.427%); 315/195057 objects misplaced (0.161%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T18:06:48.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:06:48.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: osdmap e300: 8 total, 8 up, 8 in 2023-09-23T18:06:48.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[184174]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: pgmap v743: 119 pgs: 4 remapped+peering, 1 active+recovering+undersized+remapped, 5 peering, 4 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s; 8635/195057 objects degraded (4.427%); 315/195057 objects misplaced (0.161%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.133 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: osdmap e300: 8 total, 8 up, 8 in 2023-09-23T18:06:48.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:47 smithi055 ceph-mon[179668]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:48.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: pgmap v743: 119 pgs: 4 remapped+peering, 1 active+recovering+undersized+remapped, 5 peering, 4 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 57 MiB/s wr, 916 op/s; 8635/195057 objects degraded (4.427%); 315/195057 objects misplaced (0.161%); 7.2 MiB/s, 116 objects/s recovering 2023-09-23T18:06:48.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:06:48.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: osdmap e300: 8 total, 8 up, 8 in 2023-09-23T18:06:48.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.201 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:47 smithi099 ceph-mon[167076]: failed to encode map e300 with expected crc 2023-09-23T18:06:48.202 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:48.236 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:48.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:48.239 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:48.257 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:06:48.340 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:48.366 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:48.398 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:48.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:48.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:48.496 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:06:48.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:48.527 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:48.562 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:48.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:48.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:48.667 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:48.682 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:48.692 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:48.733 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:48.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:48.736 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:48.837 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:48.863 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:48.869 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 58 16 65503 65487 70.5144 0 - 0.013482 2023-09-23T18:06:48.899 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:48.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:48.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:49.003 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:48 smithi099 ceph-mon[167076]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:48 smithi099 ceph-mon[167076]: osdmap e301: 8 total, 8 up, 8 in 2023-09-23T18:06:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:48 smithi099 ceph-mon[167076]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:48 smithi099 ceph-mon[167076]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:48 smithi099 ceph-mon[167076]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 8635/195057 objects degraded (4.427%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:49.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[184174]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[184174]: osdmap e301: 8 total, 8 up, 8 in 2023-09-23T18:06:49.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[184174]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[184174]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[184174]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 8635/195057 objects degraded (4.427%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[179668]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[179668]: osdmap e301: 8 total, 8 up, 8 in 2023-09-23T18:06:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[179668]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[179668]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[179668]: failed to encode map e301 with expected crc 2023-09-23T18:06:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 8635/195057 objects degraded (4.427%), 4 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:49.517 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:49.869 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 59 16 65503 65487 69.32 0 - 0.013482 2023-09-23T18:06:50.023 INFO:teuthology.orchestra.run.smithi055.stdout:moved 32 / 357 (8.96359%) 2023-09-23T18:06:50.023 INFO:teuthology.orchestra.run.smithi055.stdout:avg 44.625 2023-09-23T18:06:50.023 INFO:teuthology.orchestra.run.smithi055.stdout:stddev 11.2576 -> 17.6843 (expected baseline 6.24875) 2023-09-23T18:06:50.023 INFO:teuthology.orchestra.run.smithi055.stdout:min osd.0 with 27 -> 19 pgs (0.605042 -> 0.42577 * mean) 2023-09-23T18:06:50.023 INFO:teuthology.orchestra.run.smithi055.stdout:max osd.1 with 61 -> 67 pgs (1.36695 -> 1.5014 * mean) 2023-09-23T18:06:50.024 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:06:50.024 INFO:teuthology.orchestra.run.smithi055.stdout:oload 110 2023-09-23T18:06:50.024 INFO:teuthology.orchestra.run.smithi055.stdout:max_change 3 2023-09-23T18:06:50.024 INFO:teuthology.orchestra.run.smithi055.stdout:max_change_osds 4 2023-09-23T18:06:50.024 INFO:teuthology.orchestra.run.smithi055.stdout:average_utilization 0.0204 2023-09-23T18:06:50.024 INFO:teuthology.orchestra.run.smithi055.stdout:overload_utilization 0.0224 2023-09-23T18:06:50.024 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 weight 0.6786 -> 0.4893 2023-09-23T18:06:50.024 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 weight 0.7190 -> 0.5444 2023-09-23T18:06:50.024 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 weight 0.8143 -> 0.6924 2023-09-23T18:06:50.108 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:50.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:50.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:50.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:50.214 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[179668]: pgmap v746: 119 pgs: 1 activating+undersized+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 6 peering, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 10 KiB/s rd, 26 MiB/s wr, 425 op/s; 12844/196077 objects degraded (6.550%); 315/196077 objects misplaced (0.161%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T18:06:50.214 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[179668]: failed to encode map e302 with expected crc 2023-09-23T18:06:50.214 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[179668]: osdmap e302: 8 total, 8 up, 8 in 2023-09-23T18:06:50.214 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[179668]: failed to encode map e302 with expected crc 2023-09-23T18:06:50.214 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[179668]: from='client.45653 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:50.214 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'35676','2':'45379','3':'32068'}"}]: dispatch 2023-09-23T18:06:50.215 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[184174]: pgmap v746: 119 pgs: 1 activating+undersized+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 6 peering, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 10 KiB/s rd, 26 MiB/s wr, 425 op/s; 12844/196077 objects degraded (6.550%); 315/196077 objects misplaced (0.161%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T18:06:50.215 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[184174]: failed to encode map e302 with expected crc 2023-09-23T18:06:50.215 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[184174]: osdmap e302: 8 total, 8 up, 8 in 2023-09-23T18:06:50.215 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[184174]: failed to encode map e302 with expected crc 2023-09-23T18:06:50.215 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[184174]: from='client.45653 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:50.215 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:49 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'35676','2':'45379','3':'32068'}"}]: dispatch 2023-09-23T18:06:50.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:49 smithi099 ceph-mon[167076]: pgmap v746: 119 pgs: 1 activating+undersized+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 6 peering, 5 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 10 KiB/s rd, 26 MiB/s wr, 425 op/s; 12844/196077 objects degraded (6.550%); 315/196077 objects misplaced (0.161%); 8.7 MiB/s, 139 objects/s recovering 2023-09-23T18:06:50.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:49 smithi099 ceph-mon[167076]: failed to encode map e302 with expected crc 2023-09-23T18:06:50.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:49 smithi099 ceph-mon[167076]: osdmap e302: 8 total, 8 up, 8 in 2023-09-23T18:06:50.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:49 smithi099 ceph-mon[167076]: failed to encode map e302 with expected crc 2023-09-23T18:06:50.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:49 smithi099 ceph-mon[167076]: from='client.45653 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:06:50.238 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:49 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'0':'35676','2':'45379','3':'32068'}"}]: dispatch 2023-09-23T18:06:50.240 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:50.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:50.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:50.280 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:50.380 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:50.406 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:50.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:50.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:50.442 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:50.543 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:50.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:06:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:06:50] "GET /metrics HTTP/1.1" 200 36565 "" "Prometheus/2.43.0" 2023-09-23T18:06:50.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:06:50.870858+0000 min lat: 0.00119503 max lat: 2.30701 avg lat: 0.013482 2023-09-23T18:06:50.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:06:50.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 60 16 65503 65487 68.1654 0 - 0.013482 2023-09-23T18:06:50.872 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:51.213 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[179668]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.213 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[179668]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.213 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'35676','2':'45379','3':'32068'}"}]': finished 2023-09-23T18:06:51.213 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[179668]: osdmap e303: 8 total, 8 up, 8 in 2023-09-23T18:06:51.213 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[179668]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.213 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[179668]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[184174]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[184174]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'35676','2':'45379','3':'32068'}"}]': finished 2023-09-23T18:06:51.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[184174]: osdmap e303: 8 total, 8 up, 8 in 2023-09-23T18:06:51.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[184174]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.214 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:51 smithi055 ceph-mon[184174]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:51 smithi099 ceph-mon[167076]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:51 smithi099 ceph-mon[167076]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'0':'35676','2':'45379','3':'32068'}"}]': finished 2023-09-23T18:06:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:51 smithi099 ceph-mon[167076]: osdmap e303: 8 total, 8 up, 8 in 2023-09-23T18:06:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:51 smithi099 ceph-mon[167076]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:51 smithi099 ceph-mon[167076]: failed to encode map e303 with expected crc 2023-09-23T18:06:51.422 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:51.443 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:51.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:51.448 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:51.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:51.576 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:51.610 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:51.612 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:51.613 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:51.713 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:51.740 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:51.774 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:51.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:51.784 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:51.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 61 16 65503 65487 67.0486 0 - 0.013482 2023-09-23T18:06:51.885 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:51.911 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:51.952 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:51.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:51.955 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:52.056 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:52.228 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:52 smithi099 ceph-mon[167076]: pgmap v749: 119 pgs: 1 activating+undersized+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 7 peering, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 16 KiB/s rd, 6.5 MiB/s wr, 115 op/s; 21127/196296 objects degraded (10.763%); 3875/196296 objects misplaced (1.974%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T18:06:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:52 smithi099 ceph-mon[167076]: failed to encode map e304 with expected crc 2023-09-23T18:06:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:52 smithi099 ceph-mon[167076]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T18:06:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:52 smithi099 ceph-mon[167076]: failed to encode map e304 with expected crc 2023-09-23T18:06:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:52 smithi099 ceph-mon[167076]: failed to encode map e304 with expected crc 2023-09-23T18:06:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:52.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[179668]: pgmap v749: 119 pgs: 1 activating+undersized+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 7 peering, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 16 KiB/s rd, 6.5 MiB/s wr, 115 op/s; 21127/196296 objects degraded (10.763%); 3875/196296 objects misplaced (1.974%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T18:06:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[179668]: failed to encode map e304 with expected crc 2023-09-23T18:06:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[179668]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T18:06:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[179668]: failed to encode map e304 with expected crc 2023-09-23T18:06:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[179668]: failed to encode map e304 with expected crc 2023-09-23T18:06:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:52.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[184174]: pgmap v749: 119 pgs: 1 activating+undersized+remapped, 4 remapped+peering, 1 active+recovering+undersized+remapped, 7 peering, 8 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 16 KiB/s rd, 6.5 MiB/s wr, 115 op/s; 21127/196296 objects degraded (10.763%); 3875/196296 objects misplaced (1.974%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T18:06:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[184174]: failed to encode map e304 with expected crc 2023-09-23T18:06:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[184174]: osdmap e304: 8 total, 8 up, 8 in 2023-09-23T18:06:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[184174]: failed to encode map e304 with expected crc 2023-09-23T18:06:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[184174]: failed to encode map e304 with expected crc 2023-09-23T18:06:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:06:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:06:52.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 62 16 65503 65487 65.9679 0 - 0.013482 2023-09-23T18:06:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T18:06:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:06:53.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 21127/196296 objects degraded (10.763%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:53.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:53 smithi099 ceph-mon[167076]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T18:06:53.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T18:06:53.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:06:53.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 21127/196296 objects degraded (10.763%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:53.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[179668]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T18:06:53.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: osdmap e305: 8 total, 8 up, 8 in 2023-09-23T18:06:53.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: failed to encode map e305 with expected crc 2023-09-23T18:06:53.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:06:53.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 21127/196296 objects degraded (10.763%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:53.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:53 smithi055 ceph-mon[184174]: osdmap e306: 8 total, 8 up, 8 in 2023-09-23T18:06:53.690 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:53.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:53.695 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:53.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:53.823 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:53.857 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:53.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:53.860 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:53.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 63 16 65503 65487 64.9214 0 - 0.013482 2023-09-23T18:06:53.961 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:53.988 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:54.031 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:54.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:54.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:54.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:54 smithi055 ceph-mon[179668]: pgmap v752: 119 pgs: 5 remapped+peering, 1 active+recovering+undersized+remapped, 7 peering, 4 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.2 MiB/s wr, 22 op/s; 23372/196296 objects degraded (11.907%); 3875/196296 objects misplaced (1.974%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T18:06:54.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:54 smithi055 ceph-mon[179668]: failed to encode map e306 with expected crc 2023-09-23T18:06:54.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:54 smithi055 ceph-mon[179668]: failed to encode map e306 with expected crc 2023-09-23T18:06:54.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:54 smithi055 ceph-mon[179668]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T18:06:54.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:54 smithi099 ceph-mon[167076]: pgmap v752: 119 pgs: 5 remapped+peering, 1 active+recovering+undersized+remapped, 7 peering, 4 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.2 MiB/s wr, 22 op/s; 23372/196296 objects degraded (11.907%); 3875/196296 objects misplaced (1.974%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T18:06:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:54 smithi099 ceph-mon[167076]: failed to encode map e306 with expected crc 2023-09-23T18:06:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:54 smithi099 ceph-mon[167076]: failed to encode map e306 with expected crc 2023-09-23T18:06:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:54 smithi099 ceph-mon[167076]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T18:06:54.136 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:54.167 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:54 smithi055 ceph-mon[184174]: pgmap v752: 119 pgs: 5 remapped+peering, 1 active+recovering+undersized+remapped, 7 peering, 4 active+undersized+remapped, 8 active+recovery_wait+undersized+degraded+remapped, 94 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 5.2 KiB/s rd, 1.2 MiB/s wr, 22 op/s; 23372/196296 objects degraded (11.907%); 3875/196296 objects misplaced (1.974%); 4.5 MiB/s, 73 objects/s recovering 2023-09-23T18:06:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:54 smithi055 ceph-mon[184174]: failed to encode map e306 with expected crc 2023-09-23T18:06:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:54 smithi055 ceph-mon[184174]: failed to encode map e306 with expected crc 2023-09-23T18:06:54.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:54 smithi055 ceph-mon[184174]: osdmap e307: 8 total, 8 up, 8 in 2023-09-23T18:06:54.860 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:54.862 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:54.863 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:54.864 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:54.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 64 16 65503 65487 63.9076 0 - 0.013482 2023-09-23T18:06:54.964 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:54.990 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:55.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:55.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:55.027 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:55.128 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:55.154 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:55.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:55 smithi055 ceph-mon[179668]: failed to encode map e307 with expected crc 2023-09-23T18:06:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:55 smithi055 ceph-mon[179668]: failed to encode map e307 with expected crc 2023-09-23T18:06:55.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:55 smithi055 ceph-mon[184174]: failed to encode map e307 with expected crc 2023-09-23T18:06:55.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:55 smithi055 ceph-mon[184174]: failed to encode map e307 with expected crc 2023-09-23T18:06:55.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:55.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:55.192 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:55.293 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:55.325 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:55.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:55 smithi099 ceph-mon[167076]: failed to encode map e307 with expected crc 2023-09-23T18:06:55.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:55 smithi099 ceph-mon[167076]: failed to encode map e307 with expected crc 2023-09-23T18:06:55.621 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:55.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:55.626 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:55.727 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:55.763 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:55.808 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:55.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:55.811 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:55.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 65 16 65506 65490 62.9279 0.0234375 0.00567033 0.0136492 2023-09-23T18:06:55.911 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:55.939 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:55.985 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:55.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:55.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:56.090 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:06:56.126 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:56.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[184174]: pgmap v755: 119 pgs: 1 active+recovery_wait+degraded, 5 remapped+peering, 9 peering, 4 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 16 KiB/s wr, 0 op/s; 27155/196299 objects degraded (13.833%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:06:56.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[184174]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[184174]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[184174]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[184174]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[184174]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.135 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[184174]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T18:06:56.135 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[184174]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[179668]: pgmap v755: 119 pgs: 1 active+recovery_wait+degraded, 5 remapped+peering, 9 peering, 4 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 16 KiB/s wr, 0 op/s; 27155/196299 objects degraded (13.833%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:06:56.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[179668]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[179668]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[179668]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[179668]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[179668]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[179668]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T18:06:56.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:56 smithi055 ceph-mon[179668]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.280 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:06:56.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:56.285 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:06:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:56 smithi099 ceph-mon[167076]: pgmap v755: 119 pgs: 1 active+recovery_wait+degraded, 5 remapped+peering, 9 peering, 4 active+undersized+remapped, 9 active+recovery_wait+undersized+degraded+remapped, 91 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 16 KiB/s wr, 0 op/s; 27155/196299 objects degraded (13.833%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:06:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:56 smithi099 ceph-mon[167076]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:56 smithi099 ceph-mon[167076]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:56 smithi099 ceph-mon[167076]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:56 smithi099 ceph-mon[167076]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:56 smithi099 ceph-mon[167076]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:56 smithi099 ceph-mon[167076]: osdmap e308: 8 total, 8 up, 8 in 2023-09-23T18:06:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:56 smithi099 ceph-mon[167076]: failed to encode map e308 with expected crc 2023-09-23T18:06:56.369 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:06:56.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:56.395 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:06:56.412 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:56.420 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:06:56.445 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:06:56.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:56.458 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:56.458 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:56.473 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:06:56.497 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:06:56.518 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:06:56.551 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:06:56.552 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:06:56.553 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:06:56.553 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:06:56.553 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:06:56.553 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:06:56.553 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:06:56.553 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:06:56.553 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:06:56.559 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:06:56.606 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:56.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 66 16 65545 65529 62.0119 2.4375 0.00475618 0.0146927 2023-09-23T18:06:56.958 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:06:56.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:56.962 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:06:57.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:57.090 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:57.127 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:57.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:57.131 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:57.232 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:57.258 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:57.293 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:57.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:57.296 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:57.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[184174]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[184174]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[184174]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T18:06:57.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[184174]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[184174]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[184174]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[184174]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3321449475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:57.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[179668]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[179668]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[179668]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T18:06:57.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[179668]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[179668]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[179668]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[179668]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3321449475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:57.396 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:57.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:57 smithi099 ceph-mon[167076]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:57 smithi099 ceph-mon[167076]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:57 smithi099 ceph-mon[167076]: osdmap e309: 8 total, 8 up, 8 in 2023-09-23T18:06:57.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:57 smithi099 ceph-mon[167076]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:57 smithi099 ceph-mon[167076]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:57 smithi099 ceph-mon[167076]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.421 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:57 smithi099 ceph-mon[167076]: failed to encode map e309 with expected crc 2023-09-23T18:06:57.421 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3321449475' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:06:57.423 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:57.457 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:57.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:57.461 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:57.555 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:06:57.555 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:06:57.562 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:57.588 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:57.635 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:57.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:57.638 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:06:57.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:57.765 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:57.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:57.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:57.802 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:57.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 67 16 65545 65529 61.0869 0 - 0.0146927 2023-09-23T18:06:57.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:06:57.929 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:57.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:06:57.969 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:06:57.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:57.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:06:58.026 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:06:58.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:58.104 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: pgmap v758: 119 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 3 peering, 4 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 1.4 MiB/s wr, 22 op/s; 26296/196563 objects degraded (13.378%); 6583/196563 objects misplaced (3.349%); 9.4 MiB/s, 151 objects/s recovering 2023-09-23T18:06:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T18:06:58.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2195869836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 26296/196563 objects degraded (13.378%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:06:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[179668]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T18:06:58.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: pgmap v758: 119 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 3 peering, 4 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 1.4 MiB/s wr, 22 op/s; 26296/196563 objects degraded (13.378%); 6583/196563 objects misplaced (3.349%); 9.4 MiB/s, 151 objects/s recovering 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2195869836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 26296/196563 objects degraded (13.378%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:06:58.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:58 smithi055 ceph-mon[184174]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T18:06:58.197 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:58.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:58.200 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:58.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:06:58.328 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:58.362 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:06:58.364 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: pgmap v758: 119 pgs: 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded, 3 remapped+peering, 3 peering, 4 active+undersized+remapped, 6 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 1.4 MiB/s wr, 22 op/s; 26296/196563 objects degraded (13.378%); 6583/196563 objects misplaced (3.349%); 9.4 MiB/s, 151 objects/s recovering 2023-09-23T18:06:58.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: osdmap e310: 8 total, 8 up, 8 in 2023-09-23T18:06:58.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2195869836' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:58.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:06:58.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:06:58.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: failed to encode map e310 with expected crc 2023-09-23T18:06:58.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 26296/196563 objects degraded (13.378%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:06:58.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:06:58.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:06:58.366 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:58 smithi099 ceph-mon[167076]: osdmap e311: 8 total, 8 up, 8 in 2023-09-23T18:06:58.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:58.367 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:06:58.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:06:58.495 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:58.529 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:06:58.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:58.532 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:06:58.633 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:58.665 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:58.700 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:58.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:58.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:58.732 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:58.804 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:58.834 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:58.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 68 16 65545 65529 60.1891 0 - 0.0146927 2023-09-23T18:06:58.879 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:58.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:58.885 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:58.986 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:06:59.020 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:59.094 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:06:59.094 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:06:59.094 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2023-09-23T18:06:59.095 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:06:59.126 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:06:59.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:59.131 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:06:59.159 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:06:59.205 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:59 smithi055 ceph-mon[184174]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.205 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:59 smithi055 ceph-mon[184174]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.205 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:59 smithi055 ceph-mon[184174]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.205 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:59 smithi055 ceph-mon[184174]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.205 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:06:59 smithi055 ceph-mon[184174]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T18:06:59.205 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:59 smithi055 ceph-mon[179668]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.205 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:59 smithi055 ceph-mon[179668]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.206 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:59 smithi055 ceph-mon[179668]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.206 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:59 smithi055 ceph-mon[179668]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.206 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:06:59 smithi055 ceph-mon[179668]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T18:06:59.232 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:06:59.341 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:06:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:59 smithi099 ceph-mon[167076]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:59 smithi099 ceph-mon[167076]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:59 smithi099 ceph-mon[167076]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:59 smithi099 ceph-mon[167076]: failed to encode map e311 with expected crc 2023-09-23T18:06:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:06:59 smithi099 ceph-mon[167076]: osdmap e312: 8 total, 8 up, 8 in 2023-09-23T18:06:59.870 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:06:59.870 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 69 16 65545 65529 59.3173 0 - 0.0146927 2023-09-23T18:06:59.977 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:06:59.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:06:59.981 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:00.081 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:00.123 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:00.173 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:00.174 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:00 smithi055 ceph-mon[184174]: pgmap v761: 119 pgs: 1 active+recovery_wait+degraded, 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 2 peering, 6 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 1.5 MiB/s wr, 23 op/s; 26296/196587 objects degraded (13.376%); 6583/196587 objects misplaced (3.349%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T18:07:00.174 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:00 smithi055 ceph-mon[184174]: failed to encode map e312 with expected crc 2023-09-23T18:07:00.174 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:00 smithi055 ceph-mon[184174]: failed to encode map e312 with expected crc 2023-09-23T18:07:00.174 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:00 smithi055 ceph-mon[184174]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T18:07:00.174 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:00 smithi055 ceph-mon[179668]: pgmap v761: 119 pgs: 1 active+recovery_wait+degraded, 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 2 peering, 6 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 1.5 MiB/s wr, 23 op/s; 26296/196587 objects degraded (13.376%); 6583/196587 objects misplaced (3.349%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T18:07:00.174 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:00 smithi055 ceph-mon[179668]: failed to encode map e312 with expected crc 2023-09-23T18:07:00.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:00 smithi055 ceph-mon[179668]: failed to encode map e312 with expected crc 2023-09-23T18:07:00.175 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:00 smithi055 ceph-mon[179668]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T18:07:00.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:00.177 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:00.217 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:07:00.217 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:07:00.278 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:00.320 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:00 smithi099 ceph-mon[167076]: pgmap v761: 119 pgs: 1 active+recovery_wait+degraded, 2 activating+undersized+degraded+remapped, 2 active+recovering+undersized+remapped, 3 remapped+peering, 2 peering, 6 active+recovery_wait+undersized+degraded+remapped, 103 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 1.5 MiB/s wr, 23 op/s; 26296/196587 objects degraded (13.376%); 6583/196587 objects misplaced (3.349%); 7.9 MiB/s, 126 objects/s recovering 2023-09-23T18:07:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:00 smithi099 ceph-mon[167076]: failed to encode map e312 with expected crc 2023-09-23T18:07:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:00 smithi099 ceph-mon[167076]: failed to encode map e312 with expected crc 2023-09-23T18:07:00.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:00 smithi099 ceph-mon[167076]: osdmap e313: 8 total, 8 up, 8 in 2023-09-23T18:07:00.373 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:00.392 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:07:00.425 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:07:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:07:00] "GET /metrics HTTP/1.1" 200 36581 "" "Prometheus/2.43.0" 2023-09-23T18:07:00.816 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:00.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:00.820 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:00.871 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 70 16 65545 65529 58.4703 0 - 0.0146927 2023-09-23T18:07:00.921 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:00.950 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:07:00.962 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:01.075 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:01.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:01.079 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:01.179 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:01.220 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:01.273 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:01.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:01.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:01.377 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:01.424 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:01 smithi055 ceph-mon[184174]: failed to encode map e313 with expected crc 2023-09-23T18:07:01.424 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:01 smithi055 ceph-mon[184174]: failed to encode map e313 with expected crc 2023-09-23T18:07:01.424 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:01 smithi055 ceph-mon[184174]: failed to encode map e313 with expected crc 2023-09-23T18:07:01.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:01 smithi055 ceph-mon[179668]: failed to encode map e313 with expected crc 2023-09-23T18:07:01.424 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:01 smithi055 ceph-mon[179668]: failed to encode map e313 with expected crc 2023-09-23T18:07:01.425 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:01 smithi055 ceph-mon[179668]: failed to encode map e313 with expected crc 2023-09-23T18:07:01.426 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:01 smithi099 ceph-mon[167076]: failed to encode map e313 with expected crc 2023-09-23T18:07:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:01 smithi099 ceph-mon[167076]: failed to encode map e313 with expected crc 2023-09-23T18:07:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:01 smithi099 ceph-mon[167076]: failed to encode map e313 with expected crc 2023-09-23T18:07:01.686 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:01.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:01.692 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:01.745 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:07:01.749 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:07:01.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:01.802 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:01.819 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:01.871 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 71 16 65545 65529 57.647 0 - 0.0146927 2023-09-23T18:07:02.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:02.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:02.059 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:02.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:02.187 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:02.226 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:02.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:02.231 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:02.241 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:02.270 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":764,"stamp":"2023-09-23T18:07:00.781391+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4298611711,"num_objects":65552,"num_object_clones":0,"num_object_copies":196656,"num_objects_missing_on_primary":1837,"num_objects_missing":1837,"num_objects_degraded":29023,"num_objects_misplaced":3157,"num_objects_unfound":0,"num_objects_dirty":65552,"num_whiteouts":0,"num_read":6088,"num_read_kb":12404,"num_write":66501,"num_write_kb":4207003,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5168,"num_bytes_recovered":331320004,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":71547,"ondisk_log_size":71547,"up":357,"acting":344,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":392,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15390512,"kb_used_data":13114672,"kb_used_omap":50,"kb_used_meta":2275725,"kb_avail":734570704,"statfs":{"total":767960285184,"available":752200400896,"internally_reserved":0,"allocated":13429424128,"data_stored":13418642678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51828,"internal_metadata":2330342796},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,0,0,0,1,0,1,2,8,4],"upper_bound":32768},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":2031616,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":-1,"num_objects_missing":-1,"num_objects_degraded":2727,"num_objects_misplaced":-3426,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":31,"num_write_kb":1984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":711,"num_bytes_recovered":46596096,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.001126"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":637,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084171+0000","last_change":"2023-09-23T18:05:15.087182+0000","last_active":"2023-09-23T18:06:56.084171+0000","last_peered":"2023-09-23T18:06:56.084171+0000","last_clean":"2023-09-23T18:06:56.084171+0000","last_became_active":"2023-09-23T18:05:15.086678+0000","last_became_peered":"2023-09-23T18:05:15.086678+0000","last_unstale":"2023-09-23T18:06:56.084171+0000","last_undegraded":"2023-09-23T18:06:56.084171+0000","last_fullsized":"2023-09-23T18:06:56.084171+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1262601,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":122,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.097353+0000","last_change":"2023-09-23T18:06:05.377123+0000","last_active":"2023-09-23T18:06:58.097353+0000","last_peered":"2023-09-23T18:06:58.097353+0000","last_clean":"2023-09-23T18:06:58.097353+0000","last_became_active":"2023-09-23T18:06:05.376735+0000","last_became_peered":"2023-09-23T18:06:05.376735+0000","last_unstale":"2023-09-23T18:06:58.097353+0000","last_undegraded":"2023-09-23T18:06:58.097353+0000","last_fullsized":"2023-09-23T18:06:58.097353+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:11.555818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":185,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.088951+0000","last_change":"2023-09-23T18:06:51.071290+0000","last_active":"2023-09-23T18:06:56.088951+0000","last_peered":"2023-09-23T18:06:56.088951+0000","last_clean":"2023-09-23T18:06:56.088951+0000","last_became_active":"2023-09-23T18:06:51.070948+0000","last_became_peered":"2023-09-23T18:06:51.070948+0000","last_unstale":"2023-09-23T18:06:56.088951+0000","last_undegraded":"2023-09-23T18:06:56.088951+0000","last_fullsized":"2023-09-23T18:06:56.088951+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:52.647972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":655,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084845+0000","last_change":"2023-09-23T18:06:54.065593+0000","last_active":"2023-09-23T18:06:56.084845+0000","last_peered":"2023-09-23T18:06:56.084845+0000","last_clean":"2023-09-23T18:06:56.084845+0000","last_became_active":"2023-09-23T18:06:54.065193+0000","last_became_peered":"2023-09-23T18:06:54.065193+0000","last_unstale":"2023-09-23T18:06:56.084845+0000","last_undegraded":"2023-09-23T18:06:56.084845+0000","last_fullsized":"2023-09-23T18:06:56.084845+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:05.792712+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":25,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T18:06:45.908934+0000","last_change":"2023-09-23T18:06:05.370046+0000","last_active":"2023-09-23T18:06:45.908934+0000","last_peered":"2023-09-23T18:06:45.908934+0000","last_clean":"2023-09-23T18:06:45.908934+0000","last_became_active":"2023-09-23T18:06:05.369714+0000","last_became_peered":"2023-09-23T18:06:05.369714+0000","last_unstale":"2023-09-23T18:06:45.908934+0000","last_undegraded":"2023-09-23T18:06:45.908934+0000","last_fullsized":"2023-09-23T18:06:45.908934+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:47.453671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":319,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.096761+0000","last_change":"2023-09-23T18:06:51.070985+0000","last_active":"2023-09-23T18:06:56.096761+0000","last_peered":"2023-09-23T18:06:56.096761+0000","last_clean":"2023-09-23T18:06:56.096761+0000","last_became_active":"2023-09-23T18:06:51.070671+0000","last_became_peered":"2023-09-23T18:06:51.070671+0000","last_unstale":"2023-09-23T18:06:56.096761+0000","last_undegraded":"2023-09-23T18:06:56.096761+0000","last_fullsized":"2023-09-23T18:06:56.096761+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:14.820905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":649,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.097488+0000","last_change":"2023-09-23T18:06:08.439343+0000","last_active":"2023-09-23T18:06:58.097488+0000","last_peered":"2023-09-23T18:06:58.097488+0000","last_clean":"2023-09-23T18:06:58.097488+0000","last_became_active":"2023-09-23T18:06:08.437702+0000","last_became_peered":"2023-09-23T18:06:08.437702+0000","last_unstale":"2023-09-23T18:06:58.097488+0000","last_undegraded":"2023-09-23T18:06:58.097488+0000","last_fullsized":"2023-09-23T18:06:58.097488+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:02.398105+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":188,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.086872+0000","last_change":"2023-09-23T18:06:51.087219+0000","last_active":"2023-09-23T18:06:56.086872+0000","last_peered":"2023-09-23T18:06:56.086872+0000","last_clean":"2023-09-23T18:06:56.086872+0000","last_became_active":"2023-09-23T18:06:51.086885+0000","last_became_peered":"2023-09-23T18:06:51.086885+0000","last_unstale":"2023-09-23T18:06:56.086872+0000","last_undegraded":"2023-09-23T18:06:56.086872+0000","last_fullsized":"2023-09-23T18:06:56.086872+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:16.254186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":263,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.027500+0000","last_change":"2023-09-23T18:06:05.372350+0000","last_active":"2023-09-23T18:06:59.027500+0000","last_peered":"2023-09-23T18:06:59.027500+0000","last_clean":"2023-09-23T18:06:59.027500+0000","last_became_active":"2023-09-23T18:06:05.372026+0000","last_became_peered":"2023-09-23T18:06:05.372026+0000","last_unstale":"2023-09-23T18:06:59.027500+0000","last_undegraded":"2023-09-23T18:06:59.027500+0000","last_fullsized":"2023-09-23T18:06:59.027500+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:37.063301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":690,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.097678+0000","last_change":"2023-09-23T18:06:08.433828+0000","last_active":"2023-09-23T18:06:56.097678+0000","last_peered":"2023-09-23T18:06:56.097678+0000","last_clean":"2023-09-23T18:06:56.097678+0000","last_became_active":"2023-09-23T18:06:08.433445+0000","last_became_peered":"2023-09-23T18:06:08.433445+0000","last_unstale":"2023-09-23T18:06:56.097678+0000","last_undegraded":"2023-09-23T18:06:56.097678+0000","last_fullsized":"2023-09-23T18:06:56.097678+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.816081+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":301,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.031592+0000","last_change":"2023-09-23T18:06:54.195144+0000","last_active":"2023-09-23T18:06:58.031592+0000","last_peered":"2023-09-23T18:06:58.031592+0000","last_clean":"2023-09-23T18:06:58.031592+0000","last_became_active":"2023-09-23T18:06:54.194921+0000","last_became_peered":"2023-09-23T18:06:54.194921+0000","last_unstale":"2023-09-23T18:06:58.031592+0000","last_undegraded":"2023-09-23T18:06:58.031592+0000","last_fullsized":"2023-09-23T18:06:58.031592+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:04.645926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":174,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T18:06:54.048268+0000","last_change":"2023-09-23T18:06:05.364021+0000","last_active":"2023-09-23T18:06:54.048268+0000","last_peered":"2023-09-23T18:06:54.048268+0000","last_clean":"2023-09-23T18:06:54.048268+0000","last_became_active":"2023-09-23T18:06:05.363740+0000","last_became_peered":"2023-09-23T18:06:05.363740+0000","last_unstale":"2023-09-23T18:06:54.048268+0000","last_undegraded":"2023-09-23T18:06:54.048268+0000","last_fullsized":"2023-09-23T18:06:54.048268+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:55.662682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":632,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.039358+0000","last_change":"2023-09-23T18:05:54.888468+0000","last_active":"2023-09-23T18:06:58.039358+0000","last_peered":"2023-09-23T18:06:58.039358+0000","last_clean":"2023-09-23T18:06:58.039358+0000","last_became_active":"2023-09-23T18:05:15.088205+0000","last_became_peered":"2023-09-23T18:05:15.088205+0000","last_unstale":"2023-09-23T18:06:58.039358+0000","last_undegraded":"2023-09-23T18:06:58.039358+0000","last_fullsized":"2023-09-23T18:06:58.039358+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.888321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.888321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:44.454268+0000","scrub_duration":0.0043410219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":302,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084600+0000","last_change":"2023-09-23T18:06:05.371032+0000","last_active":"2023-09-23T18:06:56.084600+0000","last_peered":"2023-09-23T18:06:56.084600+0000","last_clean":"2023-09-23T18:06:56.084600+0000","last_became_active":"2023-09-23T18:06:05.370695+0000","last_became_peered":"2023-09-23T18:06:05.370695+0000","last_unstale":"2023-09-23T18:06:56.084600+0000","last_undegraded":"2023-09-23T18:06:56.084600+0000","last_fullsized":"2023-09-23T18:06:56.084600+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:12.628753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":119,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T18:06:54.047986+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:06:54.047986+0000","last_peered":"2023-09-23T18:06:54.047986+0000","last_clean":"2023-09-23T18:06:54.047986+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:06:54.047986+0000","last_undegraded":"2023-09-23T18:06:54.047986+0000","last_fullsized":"2023-09-23T18:06:54.047986+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":654,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.086207+0000","last_change":"2023-09-23T18:06:54.059837+0000","last_active":"2023-09-23T18:06:56.086207+0000","last_peered":"2023-09-23T18:06:56.086207+0000","last_clean":"2023-09-23T18:06:56.086207+0000","last_became_active":"2023-09-23T18:06:54.059489+0000","last_became_peered":"2023-09-23T18:06:54.059489+0000","last_unstale":"2023-09-23T18:06:56.086207+0000","last_undegraded":"2023-09-23T18:06:56.086207+0000","last_fullsized":"2023-09-23T18:06:56.086207+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.652623+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.652623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:28.333390+0000","scrub_duration":0.00648549,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":75,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.097633+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:06:56.097633+0000","last_peered":"2023-09-23T18:06:56.097633+0000","last_clean":"2023-09-23T18:06:56.097633+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:06:56.097633+0000","last_undegraded":"2023-09-23T18:06:56.097633+0000","last_fullsized":"2023-09-23T18:06:56.097633+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":34,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.089314+0000","last_change":"2023-09-23T18:06:05.363236+0000","last_active":"2023-09-23T18:06:55.089314+0000","last_peered":"2023-09-23T18:06:55.089314+0000","last_clean":"2023-09-23T18:06:55.089314+0000","last_became_active":"2023-09-23T18:06:05.362947+0000","last_became_peered":"2023-09-23T18:06:05.362947+0000","last_unstale":"2023-09-23T18:06:55.089314+0000","last_undegraded":"2023-09-23T18:06:55.089314+0000","last_fullsized":"2023-09-23T18:06:55.089314+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:32.816159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":656,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090473+0000","last_change":"2023-09-23T18:06:48.989986+0000","last_active":"2023-09-23T18:06:55.090473+0000","last_peered":"2023-09-23T18:06:55.090473+0000","last_clean":"2023-09-23T18:06:55.090473+0000","last_became_active":"2023-09-23T18:06:48.989534+0000","last_became_peered":"2023-09-23T18:06:48.989534+0000","last_unstale":"2023-09-23T18:06:55.090473+0000","last_undegraded":"2023-09-23T18:06:55.090473+0000","last_fullsized":"2023-09-23T18:06:55.090473+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":104,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.098172+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:06:58.098172+0000","last_peered":"2023-09-23T18:06:58.098172+0000","last_clean":"2023-09-23T18:06:58.098172+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:06:58.098172+0000","last_undegraded":"2023-09-23T18:06:58.098172+0000","last_fullsized":"2023-09-23T18:06:58.098172+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":122,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.097413+0000","last_change":"2023-09-23T18:06:05.375073+0000","last_active":"2023-09-23T18:06:58.097413+0000","last_peered":"2023-09-23T18:06:58.097413+0000","last_clean":"2023-09-23T18:06:58.097413+0000","last_became_active":"2023-09-23T18:06:05.374752+0000","last_became_peered":"2023-09-23T18:06:05.374752+0000","last_unstale":"2023-09-23T18:06:58.097413+0000","last_undegraded":"2023-09-23T18:06:58.097413+0000","last_fullsized":"2023-09-23T18:06:58.097413+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:09:06.839328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":594,"reported_epoch":310,"state":"remapped+peering","last_fresh":"2023-09-23T18:06:57.116701+0000","last_change":"2023-09-23T18:06:57.116922+0000","last_active":"2023-09-23T18:05:11.035601+0000","last_peered":"2023-09-23T18:02:48.962229+0000","last_clean":"2023-09-23T18:02:45.964322+0000","last_became_active":"2023-09-23T18:02:48.961705+0000","last_became_peered":"2023-09-23T18:02:48.961705+0000","last_unstale":"2023-09-23T18:06:57.116701+0000","last_undegraded":"2023-09-23T18:06:57.116701+0000","last_fullsized":"2023-09-23T18:06:57.116701+0000","mapping_epoch":310,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:01.632666+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:06:01.632666+0000","last_clean_scrub_stamp":"2023-09-23T18:06:01.632666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,1,3,6],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":238,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.097300+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:06:56.097300+0000","last_peered":"2023-09-23T18:06:56.097300+0000","last_clean":"2023-09-23T18:06:56.097300+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:06:56.097300+0000","last_undegraded":"2023-09-23T18:06:56.097300+0000","last_fullsized":"2023-09-23T18:06:56.097300+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":161,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T18:06:53.029039+0000","last_change":"2023-09-23T18:06:05.363898+0000","last_active":"2023-09-23T18:06:53.029039+0000","last_peered":"2023-09-23T18:06:53.029039+0000","last_clean":"2023-09-23T18:06:53.029039+0000","last_became_active":"2023-09-23T18:06:05.363580+0000","last_became_peered":"2023-09-23T18:06:05.363580+0000","last_unstale":"2023-09-23T18:06:53.029039+0000","last_undegraded":"2023-09-23T18:06:53.029039+0000","last_fullsized":"2023-09-23T18:06:53.029039+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:14.913078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":627,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084086+0000","last_change":"2023-09-23T18:05:15.091218+0000","last_active":"2023-09-23T18:06:56.084086+0000","last_peered":"2023-09-23T18:06:56.084086+0000","last_clean":"2023-09-23T18:06:56.084086+0000","last_became_active":"2023-09-23T18:05:15.090420+0000","last_became_peered":"2023-09-23T18:05:15.090420+0000","last_unstale":"2023-09-23T18:06:56.084086+0000","last_undegraded":"2023-09-23T18:06:56.084086+0000","last_fullsized":"2023-09-23T18:06:56.084086+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:57.754408+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":317,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.096496+0000","last_change":"2023-09-23T18:06:51.065050+0000","last_active":"2023-09-23T18:06:56.096496+0000","last_peered":"2023-09-23T18:06:56.096496+0000","last_clean":"2023-09-23T18:06:56.096496+0000","last_became_active":"2023-09-23T18:06:51.064774+0000","last_became_peered":"2023-09-23T18:06:51.064774+0000","last_unstale":"2023-09-23T18:06:56.096496+0000","last_undegraded":"2023-09-23T18:06:56.096496+0000","last_fullsized":"2023-09-23T18:06:56.096496+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:31.508499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":287,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084948+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:06:56.084948+0000","last_peered":"2023-09-23T18:06:56.084948+0000","last_clean":"2023-09-23T18:06:56.084948+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:06:56.084948+0000","last_undegraded":"2023-09-23T18:06:56.084948+0000","last_fullsized":"2023-09-23T18:06:56.084948+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":619,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.089435+0000","last_change":"2023-09-23T18:06:45.920979+0000","last_active":"2023-09-23T18:06:55.089435+0000","last_peered":"2023-09-23T18:06:55.089435+0000","last_clean":"2023-09-23T18:06:55.089435+0000","last_became_active":"2023-09-23T18:06:45.920686+0000","last_became_peered":"2023-09-23T18:06:45.920686+0000","last_unstale":"2023-09-23T18:06:55.089435+0000","last_undegraded":"2023-09-23T18:06:55.089435+0000","last_fullsized":"2023-09-23T18:06:55.089435+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":102,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.100745+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:06:58.100745+0000","last_peered":"2023-09-23T18:06:58.100745+0000","last_clean":"2023-09-23T18:06:58.100745+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:06:58.100745+0000","last_undegraded":"2023-09-23T18:06:58.100745+0000","last_fullsized":"2023-09-23T18:06:58.100745+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":245,"reported_epoch":307,"state":"active+clean","last_fresh":"2023-09-23T18:06:54.048440+0000","last_change":"2023-09-23T18:06:51.125883+0000","last_active":"2023-09-23T18:06:54.048440+0000","last_peered":"2023-09-23T18:06:54.048440+0000","last_clean":"2023-09-23T18:06:54.048440+0000","last_became_active":"2023-09-23T18:06:51.119298+0000","last_became_peered":"2023-09-23T18:06:51.119298+0000","last_unstale":"2023-09-23T18:06:54.048440+0000","last_undegraded":"2023-09-23T18:06:54.048440+0000","last_fullsized":"2023-09-23T18:06:54.048440+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:12.571367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":647,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084332+0000","last_change":"2023-09-23T18:06:08.430910+0000","last_active":"2023-09-23T18:06:56.084332+0000","last_peered":"2023-09-23T18:06:56.084332+0000","last_clean":"2023-09-23T18:06:56.084332+0000","last_became_active":"2023-09-23T18:06:08.430380+0000","last_became_peered":"2023-09-23T18:06:08.430380+0000","last_unstale":"2023-09-23T18:06:56.084332+0000","last_undegraded":"2023-09-23T18:06:56.084332+0000","last_fullsized":"2023-09-23T18:06:56.084332+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:40.113807+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":289,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.026014+0000","last_change":"2023-09-23T18:06:51.128655+0000","last_active":"2023-09-23T18:06:59.026014+0000","last_peered":"2023-09-23T18:06:59.026014+0000","last_clean":"2023-09-23T18:06:59.026014+0000","last_became_active":"2023-09-23T18:06:51.057904+0000","last_became_peered":"2023-09-23T18:06:51.057904+0000","last_unstale":"2023-09-23T18:06:59.026014+0000","last_undegraded":"2023-09-23T18:06:59.026014+0000","last_fullsized":"2023-09-23T18:06:59.026014+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:22.029660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":162,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090691+0000","last_change":"2023-09-23T18:06:45.919283+0000","last_active":"2023-09-23T18:06:55.090691+0000","last_peered":"2023-09-23T18:06:55.090691+0000","last_clean":"2023-09-23T18:06:55.090691+0000","last_became_active":"2023-09-23T18:06:45.918219+0000","last_became_peered":"2023-09-23T18:06:45.918219+0000","last_unstale":"2023-09-23T18:06:55.090691+0000","last_undegraded":"2023-09-23T18:06:55.090691+0000","last_fullsized":"2023-09-23T18:06:55.090691+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:21.242033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.b","version":"309'4135","reported_seq":4188,"reported_epoch":309,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:06:56.834033+0000","last_change":"2023-09-23T18:06:45.927731+0000","last_active":"2023-09-23T18:06:56.834033+0000","last_peered":"2023-09-23T18:06:56.834033+0000","last_clean":"2023-09-23T18:06:04.331781+0000","last_became_active":"2023-09-23T18:06:06.621694+0000","last_became_peered":"2023-09-23T18:06:06.621694+0000","last_unstale":"2023-09-23T18:06:56.834033+0000","last_undegraded":"2023-09-23T18:06:45.926710+0000","last_fullsized":"2023-09-23T18:06:06.376266+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4135,"log_dups_size":0,"ondisk_log_size":4135,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:24.704756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270991360,"num_objects":4135,"num_object_clones":0,"num_object_copies":12405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1860,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4135,"num_write_kb":264640,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":131072,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[5,6],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5,6","objects":1860},{"shards":"5,6","objects":2275}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":690,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.097576+0000","last_change":"2023-09-23T18:06:08.440949+0000","last_active":"2023-09-23T18:06:56.097576+0000","last_peered":"2023-09-23T18:06:56.097576+0000","last_clean":"2023-09-23T18:06:56.097576+0000","last_became_active":"2023-09-23T18:06:08.440543+0000","last_became_peered":"2023-09-23T18:06:08.440543+0000","last_unstale":"2023-09-23T18:06:56.097576+0000","last_undegraded":"2023-09-23T18:06:56.097576+0000","last_fullsized":"2023-09-23T18:06:56.097576+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:39:30.193370+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":252,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.097346+0000","last_change":"2023-09-23T18:06:45.947590+0000","last_active":"2023-09-23T18:06:56.097346+0000","last_peered":"2023-09-23T18:06:56.097346+0000","last_clean":"2023-09-23T18:06:56.097346+0000","last_became_active":"2023-09-23T18:06:45.920061+0000","last_became_peered":"2023-09-23T18:06:45.920061+0000","last_unstale":"2023-09-23T18:06:56.097346+0000","last_undegraded":"2023-09-23T18:06:56.097346+0000","last_fullsized":"2023-09-23T18:06:56.097346+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:31.645928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":298,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084071+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:06:56.084071+0000","last_peered":"2023-09-23T18:06:56.084071+0000","last_clean":"2023-09-23T18:06:56.084071+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:06:56.084071+0000","last_undegraded":"2023-09-23T18:06:56.084071+0000","last_fullsized":"2023-09-23T18:06:56.084071+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"298'4132","reported_seq":4169,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T18:06:53.027384+0000","last_change":"2023-09-23T18:05:46.126616+0000","last_active":"2023-09-23T18:06:53.027384+0000","last_peered":"2023-09-23T18:06:53.027384+0000","last_clean":"2023-09-23T18:06:53.027384+0000","last_became_active":"2023-09-23T18:05:46.126346+0000","last_became_peered":"2023-09-23T18:05:46.126346+0000","last_unstale":"2023-09-23T18:06:53.027384+0000","last_undegraded":"2023-09-23T18:06:53.027384+0000","last_fullsized":"2023-09-23T18:06:53.027384+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4132,"log_dups_size":0,"ondisk_log_size":4132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:57.668631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270794752,"num_objects":4132,"num_object_clones":0,"num_object_copies":12396,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4132,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4132,"num_write_kb":264448,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":633,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:06:58.031923+0000","last_change":"2023-09-23T18:06:57.558911+0000","last_active":"2023-09-23T18:06:58.031923+0000","last_peered":"2023-09-23T18:06:58.031923+0000","last_clean":"2023-09-23T18:02:45.965241+0000","last_became_active":"2023-09-23T18:06:57.558665+0000","last_became_peered":"2023-09-23T18:06:57.558665+0000","last_unstale":"2023-09-23T18:06:58.031923+0000","last_undegraded":"2023-09-23T18:06:57.114912+0000","last_fullsized":"2023-09-23T18:06:57.114581+0000","mapping_epoch":309,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1064488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":267,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.097101+0000","last_change":"2023-09-23T18:06:05.379591+0000","last_active":"2023-09-23T18:06:56.097101+0000","last_peered":"2023-09-23T18:06:56.097101+0000","last_clean":"2023-09-23T18:06:56.097101+0000","last_became_active":"2023-09-23T18:06:05.370176+0000","last_became_peered":"2023-09-23T18:06:05.370176+0000","last_unstale":"2023-09-23T18:06:56.097101+0000","last_undegraded":"2023-09-23T18:06:56.097101+0000","last_fullsized":"2023-09-23T18:06:56.097101+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:08.768138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":342,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.100960+0000","last_change":"2023-09-23T18:06:45.911012+0000","last_active":"2023-09-23T18:06:58.100960+0000","last_peered":"2023-09-23T18:06:58.100960+0000","last_clean":"2023-09-23T18:06:58.100960+0000","last_became_active":"2023-09-23T18:06:45.910722+0000","last_became_peered":"2023-09-23T18:06:45.910722+0000","last_unstale":"2023-09-23T18:06:58.100960+0000","last_undegraded":"2023-09-23T18:06:58.100960+0000","last_fullsized":"2023-09-23T18:06:58.100960+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:19.150719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"297'4036","reported_seq":4076,"reported_epoch":300,"state":"activating+undersized+degraded+remapped","last_fresh":"2023-09-23T18:06:56.268019+0000","last_change":"2023-09-23T18:06:56.329747+0000","last_active":"2023-09-23T18:06:44.901052+0000","last_peered":"2023-09-23T18:06:44.845762+0000","last_clean":"2023-09-23T18:06:04.334743+0000","last_became_active":"2023-09-23T18:06:06.621970+0000","last_became_peered":"2023-09-23T18:06:06.621970+0000","last_unstale":"2023-09-23T18:06:56.268019+0000","last_undegraded":"2023-09-23T18:06:56.268019+0000","last_fullsized":"2023-09-23T18:06:46.926242+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4036,"log_dups_size":0,"ondisk_log_size":4036,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:42.153209+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264503296,"num_objects":4036,"num_object_clones":0,"num_object_copies":12108,"num_objects_missing_on_primary":1836,"num_objects_missing":1836,"num_objects_degraded":4036,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4036,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4036,"num_write_kb":258304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,1,4","objects":2200},{"shards":"0,4,7","objects":1836}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":484,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.027655+0000","last_change":"2023-09-23T18:06:02.805384+0000","last_active":"2023-09-23T18:06:59.027655+0000","last_peered":"2023-09-23T18:06:59.027655+0000","last_clean":"2023-09-23T18:06:59.027655+0000","last_became_active":"2023-09-23T18:05:15.098234+0000","last_became_peered":"2023-09-23T18:05:15.098234+0000","last_unstale":"2023-09-23T18:06:59.027655+0000","last_undegraded":"2023-09-23T18:06:59.027655+0000","last_fullsized":"2023-09-23T18:06:59.027655+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:58.872644+0000","scrub_duration":0.0018893740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":176,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.097735+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:06:56.097735+0000","last_peered":"2023-09-23T18:06:56.097735+0000","last_clean":"2023-09-23T18:06:56.097735+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:06:56.097735+0000","last_undegraded":"2023-09-23T18:06:56.097735+0000","last_fullsized":"2023-09-23T18:06:56.097735+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":288,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.027820+0000","last_change":"2023-09-23T18:06:05.371447+0000","last_active":"2023-09-23T18:06:59.027820+0000","last_peered":"2023-09-23T18:06:59.027820+0000","last_clean":"2023-09-23T18:06:59.027820+0000","last_became_active":"2023-09-23T18:06:05.371065+0000","last_became_peered":"2023-09-23T18:06:05.371065+0000","last_unstale":"2023-09-23T18:06:59.027820+0000","last_undegraded":"2023-09-23T18:06:59.027820+0000","last_fullsized":"2023-09-23T18:06:59.027820+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.025791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"308'4104","reported_seq":4146,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.038604+0000","last_change":"2023-09-23T18:05:46.123859+0000","last_active":"2023-09-23T18:06:58.038604+0000","last_peered":"2023-09-23T18:06:58.038604+0000","last_clean":"2023-09-23T18:06:58.038604+0000","last_became_active":"2023-09-23T18:05:46.123482+0000","last_became_peered":"2023-09-23T18:05:46.123482+0000","last_unstale":"2023-09-23T18:06:58.038604+0000","last_undegraded":"2023-09-23T18:06:58.038604+0000","last_fullsized":"2023-09-23T18:06:58.038604+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4104,"log_dups_size":0,"ondisk_log_size":4104,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:50.537709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":268959744,"num_objects":4104,"num_object_clones":0,"num_object_copies":12312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4104,"num_write_kb":262656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":636,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.097682+0000","last_change":"2023-09-23T18:05:56.362746+0000","last_active":"2023-09-23T18:06:58.097682+0000","last_peered":"2023-09-23T18:06:58.097682+0000","last_clean":"2023-09-23T18:06:58.097682+0000","last_became_active":"2023-09-23T18:05:15.102072+0000","last_became_peered":"2023-09-23T18:05:15.102072+0000","last_unstale":"2023-09-23T18:06:58.097682+0000","last_undegraded":"2023-09-23T18:06:58.097682+0000","last_fullsized":"2023-09-23T18:06:58.097682+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.362652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.362652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:33.381515+0000","scrub_duration":0.0052581759999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":296,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084419+0000","last_change":"2023-09-23T18:06:51.069867+0000","last_active":"2023-09-23T18:06:56.084419+0000","last_peered":"2023-09-23T18:06:56.084419+0000","last_clean":"2023-09-23T18:06:56.084419+0000","last_became_active":"2023-09-23T18:06:51.069523+0000","last_became_peered":"2023-09-23T18:06:51.069523+0000","last_unstale":"2023-09-23T18:06:56.084419+0000","last_undegraded":"2023-09-23T18:06:56.084419+0000","last_fullsized":"2023-09-23T18:06:56.084419+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:40.321844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":324,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.031083+0000","last_change":"2023-09-23T18:06:05.375880+0000","last_active":"2023-09-23T18:06:58.031083+0000","last_peered":"2023-09-23T18:06:58.031083+0000","last_clean":"2023-09-23T18:06:58.031083+0000","last_became_active":"2023-09-23T18:06:05.375150+0000","last_became_peered":"2023-09-23T18:06:05.375150+0000","last_unstale":"2023-09-23T18:06:58.031083+0000","last_undegraded":"2023-09-23T18:06:58.031083+0000","last_fullsized":"2023-09-23T18:06:58.031083+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:36.483694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"297'3991","reported_seq":4027,"reported_epoch":303,"state":"remapped+peering","last_fresh":"2023-09-23T18:06:54.216535+0000","last_change":"2023-09-23T18:06:54.216755+0000","last_active":"2023-09-23T18:06:45.942577+0000","last_peered":"2023-09-23T18:06:44.895531+0000","last_clean":"2023-09-23T18:06:44.895531+0000","last_became_active":"2023-09-23T18:05:46.128615+0000","last_became_peered":"2023-09-23T18:05:46.128615+0000","last_unstale":"2023-09-23T18:06:54.216535+0000","last_undegraded":"2023-09-23T18:06:54.216535+0000","last_fullsized":"2023-09-23T18:06:54.216535+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":3991,"log_dups_size":0,"ondisk_log_size":3991,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:09:38.047259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":261554176,"num_objects":3991,"num_object_clones":0,"num_object_copies":11973,"num_objects_missing_on_primary":1,"num_objects_missing":1,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3991,"num_write_kb":255424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[0,4,5,7],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":682,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.095723+0000","last_change":"2023-09-23T18:06:54.059737+0000","last_active":"2023-09-23T18:06:56.095723+0000","last_peered":"2023-09-23T18:06:56.095723+0000","last_clean":"2023-09-23T18:06:56.095723+0000","last_became_active":"2023-09-23T18:06:54.059375+0000","last_became_peered":"2023-09-23T18:06:54.059375+0000","last_unstale":"2023-09-23T18:06:56.095723+0000","last_undegraded":"2023-09-23T18:06:56.095723+0000","last_fullsized":"2023-09-23T18:06:56.095723+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:12.644423+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":209,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T18:06:53.027199+0000","last_change":"2023-09-23T18:06:45.953975+0000","last_active":"2023-09-23T18:06:53.027199+0000","last_peered":"2023-09-23T18:06:53.027199+0000","last_clean":"2023-09-23T18:06:53.027199+0000","last_became_active":"2023-09-23T18:06:45.922926+0000","last_became_peered":"2023-09-23T18:06:45.922926+0000","last_unstale":"2023-09-23T18:06:53.027199+0000","last_undegraded":"2023-09-23T18:06:53.027199+0000","last_fullsized":"2023-09-23T18:06:53.027199+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:50.592493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":312,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.098093+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:06:58.098093+0000","last_peered":"2023-09-23T18:06:58.098093+0000","last_clean":"2023-09-23T18:06:58.098093+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:06:58.098093+0000","last_undegraded":"2023-09-23T18:06:58.098093+0000","last_fullsized":"2023-09-23T18:06:58.098093+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"300'4052","reported_seq":4106,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:06:58.042878+0000","last_change":"2023-09-23T18:06:51.235091+0000","last_active":"2023-09-23T18:06:58.042878+0000","last_peered":"2023-09-23T18:06:58.042878+0000","last_clean":"2023-09-23T18:06:44.878443+0000","last_became_active":"2023-09-23T18:06:51.229315+0000","last_became_peered":"2023-09-23T18:06:51.229315+0000","last_unstale":"2023-09-23T18:06:58.042878+0000","last_undegraded":"2023-09-23T18:06:51.088029+0000","last_fullsized":"2023-09-23T18:06:51.053926+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4052,"log_dups_size":0,"ondisk_log_size":4052,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:19.575180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265551872,"num_objects":4052,"num_object_clones":0,"num_object_copies":12156,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4052,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4052,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4052,"num_write_kb":259328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":4050},{"shards":"2,3,5","objects":2}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":650,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.085633+0000","last_change":"2023-09-23T18:06:54.065597+0000","last_active":"2023-09-23T18:06:56.085633+0000","last_peered":"2023-09-23T18:06:56.085633+0000","last_clean":"2023-09-23T18:06:56.085633+0000","last_became_active":"2023-09-23T18:06:54.065217+0000","last_became_peered":"2023-09-23T18:06:54.065217+0000","last_unstale":"2023-09-23T18:06:56.085633+0000","last_undegraded":"2023-09-23T18:06:56.085633+0000","last_fullsized":"2023-09-23T18:06:56.085633+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:53.614664+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":282,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.027965+0000","last_change":"2023-09-23T18:06:05.371350+0000","last_active":"2023-09-23T18:06:59.027965+0000","last_peered":"2023-09-23T18:06:59.027965+0000","last_clean":"2023-09-23T18:06:59.027965+0000","last_became_active":"2023-09-23T18:06:05.370877+0000","last_became_peered":"2023-09-23T18:06:05.370877+0000","last_unstale":"2023-09-23T18:06:59.027965+0000","last_undegraded":"2023-09-23T18:06:59.027965+0000","last_fullsized":"2023-09-23T18:06:59.027965+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:58.534355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":122,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.098092+0000","last_change":"2023-09-23T18:06:05.373385+0000","last_active":"2023-09-23T18:06:58.098092+0000","last_peered":"2023-09-23T18:06:58.098092+0000","last_clean":"2023-09-23T18:06:58.098092+0000","last_became_active":"2023-09-23T18:06:05.372982+0000","last_became_peered":"2023-09-23T18:06:05.372982+0000","last_unstale":"2023-09-23T18:06:58.098092+0000","last_undegraded":"2023-09-23T18:06:58.098092+0000","last_fullsized":"2023-09-23T18:06:58.098092+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:48.622115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"308'4155","reported_seq":8010,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.042973+0000","last_change":"2023-09-23T18:06:59.042700+0000","last_active":"2023-09-23T18:06:59.042973+0000","last_peered":"2023-09-23T18:06:59.042973+0000","last_clean":"2023-09-23T18:06:59.042973+0000","last_became_active":"2023-09-23T18:06:59.038205+0000","last_became_peered":"2023-09-23T18:06:59.038205+0000","last_unstale":"2023-09-23T18:06:59.042973+0000","last_undegraded":"2023-09-23T18:06:59.042973+0000","last_fullsized":"2023-09-23T18:06:59.042973+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4155,"log_dups_size":0,"ondisk_log_size":4155,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:40.800800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":272302080,"num_objects":4155,"num_object_clones":0,"num_object_copies":12465,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4155,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4155,"num_write_kb":265920,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1894,"num_bytes_recovered":124125184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":651,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.089529+0000","last_change":"2023-09-23T18:06:08.435018+0000","last_active":"2023-09-23T18:06:55.089529+0000","last_peered":"2023-09-23T18:06:55.089529+0000","last_clean":"2023-09-23T18:06:55.089529+0000","last_became_active":"2023-09-23T18:06:08.434162+0000","last_became_peered":"2023-09-23T18:06:08.434162+0000","last_unstale":"2023-09-23T18:06:55.089529+0000","last_undegraded":"2023-09-23T18:06:55.089529+0000","last_fullsized":"2023-09-23T18:06:55.089529+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:41.703500+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":102,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.100810+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:06:58.100810+0000","last_peered":"2023-09-23T18:06:58.100810+0000","last_clean":"2023-09-23T18:06:58.100810+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:06:58.100810+0000","last_undegraded":"2023-09-23T18:06:58.100810+0000","last_fullsized":"2023-09-23T18:06:58.100810+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":332,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.097708+0000","last_change":"2023-09-23T18:06:05.373305+0000","last_active":"2023-09-23T18:06:58.097708+0000","last_peered":"2023-09-23T18:06:58.097708+0000","last_clean":"2023-09-23T18:06:58.097708+0000","last_became_active":"2023-09-23T18:06:05.372720+0000","last_became_peered":"2023-09-23T18:06:05.372720+0000","last_unstale":"2023-09-23T18:06:58.097708+0000","last_undegraded":"2023-09-23T18:06:58.097708+0000","last_fullsized":"2023-09-23T18:06:58.097708+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:44.934493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"309'4040","reported_seq":5849,"reported_epoch":313,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:07:00.549207+0000","last_change":"2023-09-23T18:06:51.140357+0000","last_active":"2023-09-23T18:07:00.549207+0000","last_peered":"2023-09-23T18:07:00.549207+0000","last_clean":"2023-09-23T18:06:44.837296+0000","last_became_active":"2023-09-23T18:06:47.115056+0000","last_became_peered":"2023-09-23T18:06:47.115056+0000","last_unstale":"2023-09-23T18:07:00.549207+0000","last_undegraded":"2023-09-23T18:07:00.549207+0000","last_fullsized":"2023-09-23T18:06:46.925665+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4040,"log_dups_size":0,"ondisk_log_size":4040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:46.974573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264765440,"num_objects":4040,"num_object_clones":0,"num_object_copies":12120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3157,"num_objects_unfound":0,"num_objects_dirty":4040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4040,"num_write_kb":258560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":878,"num_bytes_recovered":57540608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":3157},{"shards":"6,7","objects":883}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":626,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084656+0000","last_change":"2023-09-23T18:05:53.509955+0000","last_active":"2023-09-23T18:06:56.084656+0000","last_peered":"2023-09-23T18:06:56.084656+0000","last_clean":"2023-09-23T18:06:56.084656+0000","last_became_active":"2023-09-23T18:05:15.089224+0000","last_became_peered":"2023-09-23T18:05:15.089224+0000","last_unstale":"2023-09-23T18:06:56.084656+0000","last_undegraded":"2023-09-23T18:06:56.084656+0000","last_fullsized":"2023-09-23T18:06:56.084656+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:37.917870+0000","scrub_duration":0.001603043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":154,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.088805+0000","last_change":"2023-09-23T18:06:45.904403+0000","last_active":"2023-09-23T18:06:55.088805+0000","last_peered":"2023-09-23T18:06:55.088805+0000","last_clean":"2023-09-23T18:06:55.088805+0000","last_became_active":"2023-09-23T18:06:45.904057+0000","last_became_peered":"2023-09-23T18:06:45.904057+0000","last_unstale":"2023-09-23T18:06:55.088805+0000","last_undegraded":"2023-09-23T18:06:55.088805+0000","last_fullsized":"2023-09-23T18:06:55.088805+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:43.213905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":34,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090611+0000","last_change":"2023-09-23T18:06:05.371661+0000","last_active":"2023-09-23T18:06:55.090611+0000","last_peered":"2023-09-23T18:06:55.090611+0000","last_clean":"2023-09-23T18:06:55.090611+0000","last_became_active":"2023-09-23T18:06:05.370998+0000","last_became_peered":"2023-09-23T18:06:05.370998+0000","last_unstale":"2023-09-23T18:06:55.090611+0000","last_undegraded":"2023-09-23T18:06:55.090611+0000","last_fullsized":"2023-09-23T18:06:55.090611+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:52.262543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"309'4067","reported_seq":4395,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:06:58.100375+0000","last_change":"2023-09-23T18:06:55.920767+0000","last_active":"2023-09-23T18:06:58.100375+0000","last_peered":"2023-09-23T18:06:58.100375+0000","last_clean":"2023-09-23T18:06:25.121665+0000","last_became_active":"2023-09-23T18:06:27.245687+0000","last_became_peered":"2023-09-23T18:06:27.245687+0000","last_unstale":"2023-09-23T18:06:58.100375+0000","last_undegraded":"2023-09-23T18:06:55.918616+0000","last_fullsized":"2023-09-23T18:06:27.154995+0000","mapping_epoch":293,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4067,"log_dups_size":0,"ondisk_log_size":4067,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:36:55.165281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":266534912,"num_objects":4067,"num_object_clones":0,"num_object_copies":12201,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2914,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4067,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4067,"num_write_kb":260288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":137,"num_bytes_recovered":8978432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6],"avail_no_missing":["4","3","6"],"object_location_counts":[{"shards":"3,4,6","objects":2914},{"shards":"4,6","objects":1153}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":504,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.027458+0000","last_change":"2023-09-23T18:06:54.193400+0000","last_active":"2023-09-23T18:06:59.027458+0000","last_peered":"2023-09-23T18:06:59.027458+0000","last_clean":"2023-09-23T18:06:59.027458+0000","last_became_active":"2023-09-23T18:06:54.193019+0000","last_became_peered":"2023-09-23T18:06:54.193019+0000","last_unstale":"2023-09-23T18:06:59.027458+0000","last_undegraded":"2023-09-23T18:06:59.027458+0000","last_fullsized":"2023-09-23T18:06:59.027458+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:48.156749+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":620,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090404+0000","last_change":"2023-09-23T18:06:29.051219+0000","last_active":"2023-09-23T18:06:55.090404+0000","last_peered":"2023-09-23T18:06:55.090404+0000","last_clean":"2023-09-23T18:06:55.090404+0000","last_became_active":"2023-09-23T18:06:29.050570+0000","last_became_peered":"2023-09-23T18:06:29.050570+0000","last_unstale":"2023-09-23T18:06:55.090404+0000","last_undegraded":"2023-09-23T18:06:55.090404+0000","last_fullsized":"2023-09-23T18:06:55.090404+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:17.235584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":300,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.085077+0000","last_change":"2023-09-23T18:06:51.074796+0000","last_active":"2023-09-23T18:06:56.085077+0000","last_peered":"2023-09-23T18:06:56.085077+0000","last_clean":"2023-09-23T18:06:56.085077+0000","last_became_active":"2023-09-23T18:06:51.074525+0000","last_became_peered":"2023-09-23T18:06:51.074525+0000","last_unstale":"2023-09-23T18:06:56.085077+0000","last_undegraded":"2023-09-23T18:06:56.085077+0000","last_fullsized":"2023-09-23T18:06:56.085077+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:48.941608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":34,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.089160+0000","last_change":"2023-09-23T18:06:05.374173+0000","last_active":"2023-09-23T18:06:55.089160+0000","last_peered":"2023-09-23T18:06:55.089160+0000","last_clean":"2023-09-23T18:06:55.089160+0000","last_became_active":"2023-09-23T18:06:05.373835+0000","last_became_peered":"2023-09-23T18:06:05.373835+0000","last_unstale":"2023-09-23T18:06:55.089160+0000","last_undegraded":"2023-09-23T18:06:55.089160+0000","last_fullsized":"2023-09-23T18:06:55.089160+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:25.467488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"300'4220","reported_seq":4263,"reported_epoch":312,"state":"remapped+peering","last_fresh":"2023-09-23T18:06:59.027374+0000","last_change":"2023-09-23T18:06:55.093001+0000","last_active":"2023-09-23T18:06:50.025772+0000","last_peered":"2023-09-23T18:06:48.947638+0000","last_clean":"2023-09-23T18:06:48.947638+0000","last_became_active":"2023-09-23T18:05:46.129180+0000","last_became_peered":"2023-09-23T18:05:46.129180+0000","last_unstale":"2023-09-23T18:06:59.027374+0000","last_undegraded":"2023-09-23T18:06:59.027374+0000","last_fullsized":"2023-09-23T18:06:59.027374+0000","mapping_epoch":308,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4220,"log_dups_size":0,"ondisk_log_size":4220,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:35.648232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276561920,"num_objects":4220,"num_object_clones":0,"num_object_copies":12660,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4220,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4220,"num_write_kb":270080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[3],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":627,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.085153+0000","last_change":"2023-09-23T18:05:15.083813+0000","last_active":"2023-09-23T18:06:56.085153+0000","last_peered":"2023-09-23T18:06:56.085153+0000","last_clean":"2023-09-23T18:06:56.085153+0000","last_became_active":"2023-09-23T18:05:15.083395+0000","last_became_peered":"2023-09-23T18:05:15.083395+0000","last_unstale":"2023-09-23T18:06:56.085153+0000","last_undegraded":"2023-09-23T18:06:56.085153+0000","last_fullsized":"2023-09-23T18:06:56.085153+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:24.402252+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1282,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084506+0000","last_change":"2023-09-23T18:06:08.426828+0000","last_active":"2023-09-23T18:06:56.084506+0000","last_peered":"2023-09-23T18:06:56.084506+0000","last_clean":"2023-09-23T18:06:56.084506+0000","last_became_active":"2023-09-23T18:06:08.426343+0000","last_became_peered":"2023-09-23T18:06:08.426343+0000","last_unstale":"2023-09-23T18:06:56.084506+0000","last_undegraded":"2023-09-23T18:06:56.084506+0000","last_fullsized":"2023-09-23T18:06:56.084506+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:38.924507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":102,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.097506+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:06:58.097506+0000","last_peered":"2023-09-23T18:06:58.097506+0000","last_clean":"2023-09-23T18:06:58.097506+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:06:58.097506+0000","last_undegraded":"2023-09-23T18:06:58.097506+0000","last_fullsized":"2023-09-23T18:06:58.097506+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":43,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090325+0000","last_change":"2023-09-23T18:06:51.088473+0000","last_active":"2023-09-23T18:06:55.090325+0000","last_peered":"2023-09-23T18:06:55.090325+0000","last_clean":"2023-09-23T18:06:55.090325+0000","last_became_active":"2023-09-23T18:06:51.088171+0000","last_became_peered":"2023-09-23T18:06:51.088171+0000","last_unstale":"2023-09-23T18:06:55.090325+0000","last_undegraded":"2023-09-23T18:06:55.090325+0000","last_fullsized":"2023-09-23T18:06:55.090325+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:45.860551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"309'4045","reported_seq":4096,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:06:58.101137+0000","last_change":"2023-09-23T18:06:55.918462+0000","last_active":"2023-09-23T18:06:58.101137+0000","last_peered":"2023-09-23T18:06:58.101137+0000","last_clean":"2023-09-23T18:06:04.313237+0000","last_became_active":"2023-09-23T18:06:55.828485+0000","last_became_peered":"2023-09-23T18:06:55.828485+0000","last_unstale":"2023-09-23T18:06:58.101137+0000","last_undegraded":"2023-09-23T18:06:55.722226+0000","last_fullsized":"2023-09-23T18:06:46.927046+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4045,"log_dups_size":0,"ondisk_log_size":4045,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:24.293178+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265093120,"num_objects":4045,"num_object_clones":0,"num_object_copies":12135,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4034,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4045,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4045,"num_write_kb":258880,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":2188},{"shards":"0,4","objects":1},{"shards":"0,4,6","objects":1844},{"shards":"1,4","objects":12}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":637,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.030619+0000","last_change":"2023-09-23T18:05:15.087882+0000","last_active":"2023-09-23T18:06:58.030619+0000","last_peered":"2023-09-23T18:06:58.030619+0000","last_clean":"2023-09-23T18:06:58.030619+0000","last_became_active":"2023-09-23T18:05:15.087524+0000","last_became_peered":"2023-09-23T18:05:15.087524+0000","last_unstale":"2023-09-23T18:06:58.030619+0000","last_undegraded":"2023-09-23T18:06:58.030619+0000","last_fullsized":"2023-09-23T18:06:58.030619+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:55.015941+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":603,"reported_epoch":311,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:06:58.039043+0000","last_change":"2023-09-23T18:06:56.676920+0000","last_active":"2023-09-23T18:06:58.039043+0000","last_peered":"2023-09-23T18:06:58.039043+0000","last_clean":"2023-09-23T18:06:48.957948+0000","last_became_active":"2023-09-23T18:06:56.676417+0000","last_became_peered":"2023-09-23T18:06:56.676417+0000","last_unstale":"2023-09-23T18:06:58.039043+0000","last_undegraded":"2023-09-23T18:06:56.372665+0000","last_fullsized":"2023-09-23T18:06:58.039043+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.555906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":34,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.089020+0000","last_change":"2023-09-23T18:06:05.358500+0000","last_active":"2023-09-23T18:06:55.089020+0000","last_peered":"2023-09-23T18:06:55.089020+0000","last_clean":"2023-09-23T18:06:55.089020+0000","last_became_active":"2023-09-23T18:06:05.358274+0000","last_became_peered":"2023-09-23T18:06:05.358274+0000","last_unstale":"2023-09-23T18:06:55.089020+0000","last_undegraded":"2023-09-23T18:06:55.089020+0000","last_fullsized":"2023-09-23T18:06:55.089020+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:04.055437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":43,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090764+0000","last_change":"2023-09-23T18:06:45.919995+0000","last_active":"2023-09-23T18:06:55.090764+0000","last_peered":"2023-09-23T18:06:55.090764+0000","last_clean":"2023-09-23T18:06:55.090764+0000","last_became_active":"2023-09-23T18:06:45.919686+0000","last_became_peered":"2023-09-23T18:06:45.919686+0000","last_unstale":"2023-09-23T18:06:55.090764+0000","last_undegraded":"2023-09-23T18:06:55.090764+0000","last_fullsized":"2023-09-23T18:06:55.090764+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:39.113717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"308'4050","reported_seq":4646,"reported_epoch":311,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:06:58.035511+0000","last_change":"2023-09-23T18:06:51.071219+0000","last_active":"2023-09-23T18:06:58.035511+0000","last_peered":"2023-09-23T18:06:58.035511+0000","last_clean":"2023-09-23T18:06:44.864353+0000","last_became_active":"2023-09-23T18:06:47.068859+0000","last_became_peered":"2023-09-23T18:06:47.068859+0000","last_unstale":"2023-09-23T18:06:58.035511+0000","last_undegraded":"2023-09-23T18:06:51.068034+0000","last_fullsized":"2023-09-23T18:06:46.917011+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4050,"log_dups_size":0,"ondisk_log_size":4050,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:56:31.075092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265420800,"num_objects":4050,"num_object_clones":0,"num_object_copies":12150,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3774,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4050,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4050,"num_write_kb":259200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":272,"num_bytes_recovered":17825792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":3774},{"shards":"0,7","objects":276}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1222,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084727+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:06:56.084727+0000","last_peered":"2023-09-23T18:06:56.084727+0000","last_clean":"2023-09-23T18:06:56.084727+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:06:56.084727+0000","last_undegraded":"2023-09-23T18:06:56.084727+0000","last_fullsized":"2023-09-23T18:06:56.084727+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":665,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.095579+0000","last_change":"2023-09-23T18:06:48.957422+0000","last_active":"2023-09-23T18:06:56.095579+0000","last_peered":"2023-09-23T18:06:56.095579+0000","last_clean":"2023-09-23T18:06:56.095579+0000","last_became_active":"2023-09-23T18:06:48.957044+0000","last_became_peered":"2023-09-23T18:06:48.957044+0000","last_unstale":"2023-09-23T18:06:56.095579+0000","last_undegraded":"2023-09-23T18:06:56.095579+0000","last_fullsized":"2023-09-23T18:06:56.095579+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:58.047861+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":142,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084421+0000","last_change":"2023-09-23T18:06:51.057540+0000","last_active":"2023-09-23T18:06:56.084421+0000","last_peered":"2023-09-23T18:06:56.084421+0000","last_clean":"2023-09-23T18:06:56.084421+0000","last_became_active":"2023-09-23T18:06:51.056891+0000","last_became_peered":"2023-09-23T18:06:51.056891+0000","last_unstale":"2023-09-23T18:06:56.084421+0000","last_undegraded":"2023-09-23T18:06:56.084421+0000","last_fullsized":"2023-09-23T18:06:56.084421+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:32.959376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":246,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.026621+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:06:59.026621+0000","last_peered":"2023-09-23T18:06:59.026621+0000","last_clean":"2023-09-23T18:06:59.026621+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:06:59.026621+0000","last_undegraded":"2023-09-23T18:06:59.026621+0000","last_fullsized":"2023-09-23T18:06:59.026621+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"308'4214","reported_seq":4262,"reported_epoch":309,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:06:56.099471+0000","last_change":"2023-09-23T18:06:47.084624+0000","last_active":"2023-09-23T18:06:56.099471+0000","last_peered":"2023-09-23T18:06:56.099471+0000","last_clean":"2023-09-23T18:06:44.892077+0000","last_became_active":"2023-09-23T18:06:47.079259+0000","last_became_peered":"2023-09-23T18:06:47.079259+0000","last_unstale":"2023-09-23T18:06:56.099471+0000","last_undegraded":"2023-09-23T18:06:46.948187+0000","last_fullsized":"2023-09-23T18:06:46.922276+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4214,"log_dups_size":0,"ondisk_log_size":4214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:34:53.735035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276168704,"num_objects":4214,"num_object_clones":0,"num_object_copies":12642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4210,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4214,"num_write_kb":269696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2],"avail_no_missing":["7","0","2"],"object_location_counts":[{"shards":"0,2,7","objects":4210},{"shards":"2,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":457,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.089447+0000","last_change":"2023-09-23T18:06:08.434791+0000","last_active":"2023-09-23T18:06:55.089447+0000","last_peered":"2023-09-23T18:06:55.089447+0000","last_clean":"2023-09-23T18:06:55.089447+0000","last_became_active":"2023-09-23T18:06:08.433718+0000","last_became_peered":"2023-09-23T18:06:08.433718+0000","last_unstale":"2023-09-23T18:06:55.089447+0000","last_undegraded":"2023-09-23T18:06:55.089447+0000","last_fullsized":"2023-09-23T18:06:55.089447+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:57.701252+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:57.701252+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":517,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.095799+0000","last_change":"2023-09-23T18:06:05.403119+0000","last_active":"2023-09-23T18:06:56.095799+0000","last_peered":"2023-09-23T18:06:56.095799+0000","last_clean":"2023-09-23T18:06:56.095799+0000","last_became_active":"2023-09-23T18:06:05.371941+0000","last_became_peered":"2023-09-23T18:06:05.371941+0000","last_unstale":"2023-09-23T18:06:56.095799+0000","last_undegraded":"2023-09-23T18:06:56.095799+0000","last_fullsized":"2023-09-23T18:06:56.095799+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:34:25.525564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":320,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.035063+0000","last_change":"2023-09-23T18:06:45.914529+0000","last_active":"2023-09-23T18:06:58.035063+0000","last_peered":"2023-09-23T18:06:58.035063+0000","last_clean":"2023-09-23T18:06:58.035063+0000","last_became_active":"2023-09-23T18:06:45.914170+0000","last_became_peered":"2023-09-23T18:06:45.914170+0000","last_unstale":"2023-09-23T18:06:58.035063+0000","last_undegraded":"2023-09-23T18:06:58.035063+0000","last_fullsized":"2023-09-23T18:06:58.035063+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:49.492903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":286,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090144+0000","last_change":"2023-09-23T18:06:51.070104+0000","last_active":"2023-09-23T18:06:55.090144+0000","last_peered":"2023-09-23T18:06:55.090144+0000","last_clean":"2023-09-23T18:06:55.090144+0000","last_became_active":"2023-09-23T18:06:51.069771+0000","last_became_peered":"2023-09-23T18:06:51.069771+0000","last_unstale":"2023-09-23T18:06:55.090144+0000","last_undegraded":"2023-09-23T18:06:55.090144+0000","last_fullsized":"2023-09-23T18:06:55.090144+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:49.300491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"309'4125","reported_seq":4185,"reported_epoch":309,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:06:56.850290+0000","last_change":"2023-09-23T18:06:52.197519+0000","last_active":"2023-09-23T18:06:56.850290+0000","last_peered":"2023-09-23T18:06:56.850290+0000","last_clean":"2023-09-23T18:06:04.331883+0000","last_became_active":"2023-09-23T18:06:52.147009+0000","last_became_peered":"2023-09-23T18:06:52.147009+0000","last_unstale":"2023-09-23T18:06:56.850290+0000","last_undegraded":"2023-09-23T18:06:52.136550+0000","last_fullsized":"2023-09-23T18:06:52.052081+0000","mapping_epoch":304,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4125,"log_dups_size":0,"ondisk_log_size":4125,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:05.769984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270336000,"num_objects":4125,"num_object_clones":0,"num_object_copies":12375,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4123,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4125,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4125,"num_write_kb":264000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"1,3,5","objects":2245},{"shards":"1,5","objects":2},{"shards":"2,3,5","objects":1878}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":524,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.027472+0000","last_change":"2023-09-23T18:06:54.055392+0000","last_active":"2023-09-23T18:06:59.027472+0000","last_peered":"2023-09-23T18:06:59.027472+0000","last_clean":"2023-09-23T18:06:59.027472+0000","last_became_active":"2023-09-23T18:06:54.055101+0000","last_became_peered":"2023-09-23T18:06:54.055101+0000","last_unstale":"2023-09-23T18:06:59.027472+0000","last_undegraded":"2023-09-23T18:06:59.027472+0000","last_fullsized":"2023-09-23T18:06:59.027472+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:50.070812+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":531,"reported_epoch":309,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:06:56.095447+0000","last_change":"2023-09-23T18:06:51.071499+0000","last_active":"2023-09-23T18:06:56.095447+0000","last_peered":"2023-09-23T18:06:56.095447+0000","last_clean":"2023-09-23T18:06:48.954757+0000","last_became_active":"2023-09-23T18:06:51.071089+0000","last_became_peered":"2023-09-23T18:06:51.071089+0000","last_unstale":"2023-09-23T18:06:56.095447+0000","last_undegraded":"2023-09-23T18:06:51.054242+0000","last_fullsized":"2023-09-23T18:06:56.095447+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:01.696899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":43,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090246+0000","last_change":"2023-09-23T18:06:51.070131+0000","last_active":"2023-09-23T18:06:55.090246+0000","last_peered":"2023-09-23T18:06:55.090246+0000","last_clean":"2023-09-23T18:06:55.090246+0000","last_became_active":"2023-09-23T18:06:51.069733+0000","last_became_peered":"2023-09-23T18:06:51.069733+0000","last_unstale":"2023-09-23T18:06:55.090246+0000","last_undegraded":"2023-09-23T18:06:55.090246+0000","last_fullsized":"2023-09-23T18:06:55.090246+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:28.050735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":279,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.031157+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:06:58.031157+0000","last_peered":"2023-09-23T18:06:58.031157+0000","last_clean":"2023-09-23T18:06:58.031157+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:06:58.031157+0000","last_undegraded":"2023-09-23T18:06:58.031157+0000","last_fullsized":"2023-09-23T18:06:58.031157+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"300'4119","reported_seq":4156,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T18:06:53.027090+0000","last_change":"2023-09-23T18:05:46.122448+0000","last_active":"2023-09-23T18:06:53.027090+0000","last_peered":"2023-09-23T18:06:53.027090+0000","last_clean":"2023-09-23T18:06:53.027090+0000","last_became_active":"2023-09-23T18:05:46.122155+0000","last_became_peered":"2023-09-23T18:05:46.122155+0000","last_unstale":"2023-09-23T18:06:53.027090+0000","last_undegraded":"2023-09-23T18:06:53.027090+0000","last_fullsized":"2023-09-23T18:06:53.027090+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4119,"log_dups_size":0,"ondisk_log_size":4119,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:40.972879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":269942784,"num_objects":4119,"num_object_clones":0,"num_object_copies":12357,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4119,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4119,"num_write_kb":263616,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":657,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.097931+0000","last_change":"2023-09-23T18:06:48.962388+0000","last_active":"2023-09-23T18:06:58.097931+0000","last_peered":"2023-09-23T18:06:58.097931+0000","last_clean":"2023-09-23T18:06:58.097931+0000","last_became_active":"2023-09-23T18:06:48.962102+0000","last_became_peered":"2023-09-23T18:06:48.962102+0000","last_unstale":"2023-09-23T18:06:58.097931+0000","last_undegraded":"2023-09-23T18:06:58.097931+0000","last_fullsized":"2023-09-23T18:06:58.097931+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.217483+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":147,"reported_epoch":299,"state":"active+clean","last_fresh":"2023-09-23T18:06:45.903499+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:06:45.903499+0000","last_peered":"2023-09-23T18:06:45.903499+0000","last_clean":"2023-09-23T18:06:45.903499+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:06:45.903499+0000","last_undegraded":"2023-09-23T18:06:45.903499+0000","last_fullsized":"2023-09-23T18:06:45.903499+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":49,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090613+0000","last_change":"2023-09-23T18:06:05.398171+0000","last_active":"2023-09-23T18:06:55.090613+0000","last_peered":"2023-09-23T18:06:55.090613+0000","last_clean":"2023-09-23T18:06:55.090613+0000","last_became_active":"2023-09-23T18:06:05.370800+0000","last_became_peered":"2023-09-23T18:06:05.370800+0000","last_unstale":"2023-09-23T18:06:55.090613+0000","last_undegraded":"2023-09-23T18:06:55.090613+0000","last_fullsized":"2023-09-23T18:06:55.090613+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:46.783817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":114,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.095646+0000","last_change":"2023-09-23T18:06:45.908943+0000","last_active":"2023-09-23T18:06:56.095646+0000","last_peered":"2023-09-23T18:06:56.095646+0000","last_clean":"2023-09-23T18:06:56.095646+0000","last_became_active":"2023-09-23T18:06:45.908609+0000","last_became_peered":"2023-09-23T18:06:45.908609+0000","last_unstale":"2023-09-23T18:06:56.095646+0000","last_undegraded":"2023-09-23T18:06:56.095646+0000","last_fullsized":"2023-09-23T18:06:56.095646+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:03:31.275390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"298'4040","reported_seq":7756,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T18:06:53.027519+0000","last_change":"2023-09-23T18:06:29.052454+0000","last_active":"2023-09-23T18:06:53.027519+0000","last_peered":"2023-09-23T18:06:53.027519+0000","last_clean":"2023-09-23T18:06:53.027519+0000","last_became_active":"2023-09-23T18:06:29.046960+0000","last_became_peered":"2023-09-23T18:06:29.046960+0000","last_unstale":"2023-09-23T18:06:53.027519+0000","last_undegraded":"2023-09-23T18:06:53.027519+0000","last_fullsized":"2023-09-23T18:06:53.027519+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4040,"log_dups_size":0,"ondisk_log_size":4040,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:44.191051+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264765440,"num_objects":4040,"num_object_clones":0,"num_object_copies":12120,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4040,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4040,"num_write_kb":258560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1830,"num_bytes_recovered":119930880,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":615,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084338+0000","last_change":"2023-09-23T18:06:54.059843+0000","last_active":"2023-09-23T18:06:56.084338+0000","last_peered":"2023-09-23T18:06:56.084338+0000","last_clean":"2023-09-23T18:06:56.084338+0000","last_became_active":"2023-09-23T18:06:54.059469+0000","last_became_peered":"2023-09-23T18:06:54.059469+0000","last_unstale":"2023-09-23T18:06:56.084338+0000","last_undegraded":"2023-09-23T18:06:56.084338+0000","last_fullsized":"2023-09-23T18:06:56.084338+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:59.157827+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":518,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.027586+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:06:59.027586+0000","last_peered":"2023-09-23T18:06:59.027586+0000","last_clean":"2023-09-23T18:06:59.027586+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:06:59.027586+0000","last_undegraded":"2023-09-23T18:06:59.027586+0000","last_fullsized":"2023-09-23T18:06:59.027586+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":338,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090265+0000","last_change":"2023-09-23T18:06:45.939855+0000","last_active":"2023-09-23T18:06:55.090265+0000","last_peered":"2023-09-23T18:06:55.090265+0000","last_clean":"2023-09-23T18:06:55.090265+0000","last_became_active":"2023-09-23T18:06:45.924278+0000","last_became_peered":"2023-09-23T18:06:45.924278+0000","last_unstale":"2023-09-23T18:06:55.090265+0000","last_undegraded":"2023-09-23T18:06:55.090265+0000","last_fullsized":"2023-09-23T18:06:55.090265+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:43.850375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":102,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.100662+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:06:58.100662+0000","last_peered":"2023-09-23T18:06:58.100662+0000","last_clean":"2023-09-23T18:06:58.100662+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:06:58.100662+0000","last_undegraded":"2023-09-23T18:06:58.100662+0000","last_fullsized":"2023-09-23T18:06:58.100662+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":450,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.026169+0000","last_change":"2023-09-23T18:06:48.961752+0000","last_active":"2023-09-23T18:06:59.026169+0000","last_peered":"2023-09-23T18:06:59.026169+0000","last_clean":"2023-09-23T18:06:59.026169+0000","last_became_active":"2023-09-23T18:06:48.961304+0000","last_became_peered":"2023-09-23T18:06:48.961304+0000","last_unstale":"2023-09-23T18:06:59.026169+0000","last_undegraded":"2023-09-23T18:06:59.026169+0000","last_fullsized":"2023-09-23T18:06:59.026169+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":228,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.097412+0000","last_change":"2023-09-23T18:06:26.148865+0000","last_active":"2023-09-23T18:06:58.097412+0000","last_peered":"2023-09-23T18:06:58.097412+0000","last_clean":"2023-09-23T18:06:58.097412+0000","last_became_active":"2023-09-23T18:06:26.148517+0000","last_became_peered":"2023-09-23T18:06:26.148517+0000","last_unstale":"2023-09-23T18:06:58.097412+0000","last_undegraded":"2023-09-23T18:06:58.097412+0000","last_fullsized":"2023-09-23T18:06:58.097412+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:38.141672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":345,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.916490+0000","last_change":"2023-09-23T18:06:55.916070+0000","last_active":"2023-09-23T18:06:55.916490+0000","last_peered":"2023-09-23T18:06:55.916490+0000","last_clean":"2023-09-23T18:06:55.916490+0000","last_became_active":"2023-09-23T18:06:55.915807+0000","last_became_peered":"2023-09-23T18:06:55.915807+0000","last_unstale":"2023-09-23T18:06:55.916490+0000","last_undegraded":"2023-09-23T18:06:55.916490+0000","last_fullsized":"2023-09-23T18:06:55.916490+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:20.376063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":445,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.027164+0000","last_change":"2023-09-23T18:06:54.055428+0000","last_active":"2023-09-23T18:06:59.027164+0000","last_peered":"2023-09-23T18:06:59.027164+0000","last_clean":"2023-09-23T18:06:59.027164+0000","last_became_active":"2023-09-23T18:06:54.055098+0000","last_became_peered":"2023-09-23T18:06:54.055098+0000","last_unstale":"2023-09-23T18:06:59.027164+0000","last_undegraded":"2023-09-23T18:06:59.027164+0000","last_fullsized":"2023-09-23T18:06:59.027164+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:04.274631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:04.274631+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":179,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084574+0000","last_change":"2023-09-23T18:06:05.382745+0000","last_active":"2023-09-23T18:06:56.084574+0000","last_peered":"2023-09-23T18:06:56.084574+0000","last_clean":"2023-09-23T18:06:56.084574+0000","last_became_active":"2023-09-23T18:06:05.371488+0000","last_became_peered":"2023-09-23T18:06:05.371488+0000","last_unstale":"2023-09-23T18:06:56.084574+0000","last_undegraded":"2023-09-23T18:06:56.084574+0000","last_fullsized":"2023-09-23T18:06:56.084574+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:08.915006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":247,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.097588+0000","last_change":"2023-09-23T18:06:05.375138+0000","last_active":"2023-09-23T18:06:58.097588+0000","last_peered":"2023-09-23T18:06:58.097588+0000","last_clean":"2023-09-23T18:06:58.097588+0000","last_became_active":"2023-09-23T18:06:05.374819+0000","last_became_peered":"2023-09-23T18:06:05.374819+0000","last_unstale":"2023-09-23T18:06:58.097588+0000","last_undegraded":"2023-09-23T18:06:58.097588+0000","last_fullsized":"2023-09-23T18:06:58.097588+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:43.000019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":151,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.084817+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:06:56.084817+0000","last_peered":"2023-09-23T18:06:56.084817+0000","last_clean":"2023-09-23T18:06:56.084817+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:06:56.084817+0000","last_undegraded":"2023-09-23T18:06:56.084817+0000","last_fullsized":"2023-09-23T18:06:56.084817+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":172,"reported_epoch":306,"state":"active+clean","last_fresh":"2023-09-23T18:06:53.027451+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:06:53.027451+0000","last_peered":"2023-09-23T18:06:53.027451+0000","last_clean":"2023-09-23T18:06:53.027451+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:06:53.027451+0000","last_undegraded":"2023-09-23T18:06:53.027451+0000","last_fullsized":"2023-09-23T18:06:53.027451+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":696,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.097109+0000","last_change":"2023-09-23T18:06:08.440953+0000","last_active":"2023-09-23T18:06:56.097109+0000","last_peered":"2023-09-23T18:06:56.097109+0000","last_clean":"2023-09-23T18:06:56.097109+0000","last_became_active":"2023-09-23T18:06:08.440604+0000","last_became_peered":"2023-09-23T18:06:08.440604+0000","last_unstale":"2023-09-23T18:06:56.097109+0000","last_undegraded":"2023-09-23T18:06:56.097109+0000","last_fullsized":"2023-09-23T18:06:56.097109+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:53.274049+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":115,"reported_epoch":312,"state":"active+clean","last_fresh":"2023-09-23T18:06:59.027379+0000","last_change":"2023-09-23T18:06:51.098401+0000","last_active":"2023-09-23T18:06:59.027379+0000","last_peered":"2023-09-23T18:06:59.027379+0000","last_clean":"2023-09-23T18:06:59.027379+0000","last_became_active":"2023-09-23T18:06:51.097959+0000","last_became_peered":"2023-09-23T18:06:51.097959+0000","last_unstale":"2023-09-23T18:06:59.027379+0000","last_undegraded":"2023-09-23T18:06:59.027379+0000","last_fullsized":"2023-09-23T18:06:59.027379+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:16.234414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":205,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.039190+0000","last_change":"2023-09-23T18:06:51.062905+0000","last_active":"2023-09-23T18:06:58.039190+0000","last_peered":"2023-09-23T18:06:58.039190+0000","last_clean":"2023-09-23T18:06:58.039190+0000","last_became_active":"2023-09-23T18:06:51.062578+0000","last_became_peered":"2023-09-23T18:06:51.062578+0000","last_unstale":"2023-09-23T18:06:58.039190+0000","last_undegraded":"2023-09-23T18:06:58.039190+0000","last_fullsized":"2023-09-23T18:06:58.039190+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:25.262818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":629,"reported_epoch":308,"state":"active+clean","last_fresh":"2023-09-23T18:06:55.090138+0000","last_change":"2023-09-23T18:06:48.965770+0000","last_active":"2023-09-23T18:06:55.090138+0000","last_peered":"2023-09-23T18:06:55.090138+0000","last_clean":"2023-09-23T18:06:55.090138+0000","last_became_active":"2023-09-23T18:06:48.965429+0000","last_became_peered":"2023-09-23T18:06:48.965429+0000","last_unstale":"2023-09-23T18:06:55.090138+0000","last_undegraded":"2023-09-23T18:06:55.090138+0000","last_fullsized":"2023-09-23T18:06:55.090138+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":195,"reported_epoch":309,"state":"active+clean","last_fresh":"2023-09-23T18:06:56.086122+0000","last_change":"2023-09-23T18:06:45.908664+0000","last_active":"2023-09-23T18:06:56.086122+0000","last_peered":"2023-09-23T18:06:56.086122+0000","last_clean":"2023-09-23T18:06:56.086122+0000","last_became_active":"2023-09-23T18:06:45.903588+0000","last_became_peered":"2023-09-23T18:06:45.903588+0000","last_unstale":"2023-09-23T18:06:56.086122+0000","last_undegraded":"2023-09-23T18:06:56.086122+0000","last_fullsized":"2023-09-23T18:06:56.086122+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:42.283189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":332,"reported_epoch":311,"state":"active+clean","last_fresh":"2023-09-23T18:06:58.097171+0000","last_change":"2023-09-23T18:06:05.374027+0000","last_active":"2023-09-23T18:06:58.097171+0000","last_peered":"2023-09-23T18:06:58.097171+0000","last_clean":"2023-09-23T18:06:58.097171+0000","last_became_active":"2023-09-23T18:06:05.373670+0000","last_became_peered":"2023-09-23T18:06:05.373670+0000","last_unstale":"2023-09-23T18:06:58.097171+0000","last_undegraded":"2023-09-23T18:06:58.097171+0000","last_fullsized":"2023-09-23T18:06:58.097171+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:01.220735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":16,"stat_sum":{"num_bytes":4294246400,"num_objects":65525,"num_object_clones":0,"num_object_copies":196575,"num_objects_missing_on_primary":1837,"num_objects_missing":1837,"num_objects_degraded":29003,"num_objects_misplaced":3157,"num_objects_unfound":0,"num_objects_dirty":65525,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":65525,"num_write_kb":4193600,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5014,"num_bytes_recovered":328597504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13132431360,"data_stored":13132431360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":65525,"ondisk_log_size":65525,"up":48,"acting":37,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6246,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":165484,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":258,"num_write":69,"num_write_kb":678,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":2716197,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":91,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001412,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1631632,"kb_used_data":1488392,"kb_used_omap":13,"kb_used_meta":143218,"kb_avail":92113520,"statfs":{"total":95995035648,"available":94324244480,"internally_reserved":0,"allocated":1524113408,"data_stored":1522969403,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":146655336},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":0.92900000000000005,"15min":0.71899999999999997},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":9.3499999999999996,"5min":9.3499999999999996,"15min":9.3499999999999996},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.626,"5min":1.0620000000000001,"15min":1.212},"min":{"1min":0.25,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.6659999999999999,"5min":6.641,"15min":6.641},"last":0.30099999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.0429999999999999,"15min":0.79900000000000004},"min":{"1min":0.316,"5min":0.252,"15min":0.252},"max":{"1min":4.6529999999999996,"5min":7.1150000000000002,"15min":7.1150000000000002},"last":0.36599999999999999},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.238,"15min":1.1160000000000001},"min":{"1min":0.33600000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.4800000000000004,"5min":6.9589999999999996,"15min":6.9589999999999996},"last":0.43099999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.089,"15min":0.98799999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":2.742,"5min":6.6630000000000003,"15min":6.6630000000000003},"last":0.33800000000000002},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.1439999999999999,"15min":0.92300000000000004},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.7610000000000001,"5min":7.9160000000000004,"15min":7.9160000000000004},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.357,"5min":1.4330000000000001,"15min":1.5129999999999999},"min":{"1min":0.34100000000000003,"5min":0.316,"15min":0.316},"max":{"1min":4.4169999999999998,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.4299999999999999,"5min":1.4830000000000001,"15min":1.141},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":7.149,"5min":15.349,"15min":15.349},"last":0.51900000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":1.0069999999999999,"15min":1.05},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.944,"5min":6.9729999999999999,"15min":6.9729999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.73799999999999999,"15min":0.59599999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.29,"5min":6.3479999999999999,"15min":6.3479999999999999},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.1539999999999999,"15min":1.054},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.5380000000000003,"5min":9.5999999999999996,"15min":9.5999999999999996},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.80200000000000005,"5min":1.2050000000000001,"15min":1.157},"min":{"1min":0.33600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.359,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.53200000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.97199999999999998,"15min":0.79400000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":3.3780000000000001,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.47499999999999998},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.1659999999999999,"15min":1.139},"min":{"1min":0.32700000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.214,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":0.46000000000000002}]}]},{"osd":6,"up_from":163,"seq":700079669349,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1796204,"kb_used_data":1608420,"kb_used_omap":7,"kb_used_meta":187768,"kb_avail":91948948,"statfs":{"total":95995035648,"available":94155722752,"internally_reserved":0,"allocated":1647022080,"data_stored":1645874404,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":192275236},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,0,0,0,0,0,0,0,0,0,0,1,2],"upper_bound":32768},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:43 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.98299999999999998,"15min":1.2230000000000001},"min":{"1min":0.23400000000000001,"5min":0.224,"15min":0.215},"max":{"1min":2.4300000000000002,"5min":15.407,"15min":15.407},"last":0.45600000000000002},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.2709999999999999,"15min":2.0110000000000001},"min":{"1min":0.309,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.147,"5min":15.147,"15min":15.147},"last":0.51000000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:06:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":1.1579999999999999,"15min":1.7110000000000001},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":6.5309999999999997,"5min":6.7690000000000001,"15min":14.233000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.2729999999999999,"15min":1.8839999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.248},"max":{"1min":3.6899999999999999,"5min":14.789,"15min":14.789},"last":0.47499999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:06:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.276,"15min":2.016},"min":{"1min":0.32800000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.6140000000000008,"5min":8.6140000000000008,"15min":11.210000000000001},"last":0.27600000000000002},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.268,"15min":2.298},"min":{"1min":0.374,"5min":0.245,"15min":0.245},"max":{"1min":8.6300000000000008,"5min":9.3620000000000001,"15min":11.223000000000001},"last":0.40999999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:06:28 2023","interfaces":[{"interface":"back","average":{"1min":1.1020000000000001,"5min":1.0960000000000001,"15min":1.101},"min":{"1min":0.35299999999999998,"5min":0.308,"15min":0.308},"max":{"1min":3.5209999999999999,"5min":4.7539999999999996,"15min":4.7539999999999996},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.1870000000000001,"5min":1.3300000000000001,"15min":1.22},"min":{"1min":0.40600000000000003,"5min":0.32300000000000001,"15min":0.32300000000000001},"max":{"1min":5.2590000000000003,"5min":7.484,"15min":7.484},"last":0.39100000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:06:43 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.90700000000000003,"15min":0.89000000000000001},"min":{"1min":0.254,"5min":0.245,"15min":0.23000000000000001},"max":{"1min":1.085,"5min":5.827,"15min":9.9109999999999996},"last":0.311},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.81699999999999995,"15min":0.61899999999999999},"min":{"1min":0.29899999999999999,"5min":0.29099999999999998,"15min":0.24099999999999999},"max":{"1min":2.202,"5min":6.3920000000000003,"15min":8.3000000000000007},"last":0.52200000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:06:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.93500000000000005,"15min":0.877},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":1.329,"5min":6.6159999999999997,"15min":11.869},"last":0.436},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.95999999999999996,"15min":0.73699999999999999},"min":{"1min":0.33700000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.226,"5min":4.7930000000000001,"15min":5.4240000000000004},"last":0.49199999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:06:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.98599999999999999,"15min":0.97099999999999997},"min":{"1min":0.32700000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.746,"5min":5.5380000000000003,"15min":5.5380000000000003},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.77300000000000002,"5min":1.01,"15min":0.85799999999999998},"min":{"1min":0.34399999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.861,"5min":10.411,"15min":10.411},"last":0.67200000000000004}]}]},{"osd":1,"up_from":98,"seq":420906795166,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1551884,"kb_used_data":1240900,"kb_used_omap":7,"kb_used_meta":310968,"kb_avail":92193268,"statfs":{"total":95995035648,"available":94405906432,"internally_reserved":0,"allocated":1270681600,"data_stored":1269214342,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":318431933},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,1,0,1,2,4,2],"upper_bound":32768},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.77700000000000002,"5min":0.75800000000000001,"15min":0.96299999999999997},"min":{"1min":0.25,"5min":0.20100000000000001,"15min":0.17699999999999999},"max":{"1min":3.1789999999999998,"5min":4.7670000000000003,"15min":15.529999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":5.1779999999999999,"5min":2.0129999999999999,"15min":1.4490000000000001},"min":{"1min":0.27800000000000002,"5min":0.23599999999999999,"15min":0.20899999999999999},"max":{"1min":82.244,"5min":82.244,"15min":82.244},"last":0.36599999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7629999999999999,"5min":1.2789999999999999,"15min":1.4119999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":10.531000000000001,"5min":10.531000000000001,"15min":16.347999999999999},"last":0.504},{"interface":"front","average":{"1min":1.5269999999999999,"5min":0.879,"15min":1.095},"min":{"1min":0.32500000000000001,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":6.3600000000000003,"5min":6.3600000000000003,"15min":11.305999999999999},"last":0.442}]},{"osd":3,"last update":"Sat Sep 23 18:06:25 2023","interfaces":[{"interface":"back","average":{"1min":1.429,"5min":0.96699999999999997,"15min":0.69899999999999995},"min":{"1min":0.39200000000000002,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":6.4429999999999996,"5min":6.6459999999999999,"15min":6.6459999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.5669999999999999,"5min":1.25,"15min":1.0880000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":7.867,"5min":11.125999999999999,"15min":11.125999999999999},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.84599999999999997,"5min":1.0509999999999999,"15min":1.204},"min":{"1min":0.42699999999999999,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.6019999999999999,"5min":8.2639999999999993,"15min":26.568999999999999},"last":0.379},{"interface":"front","average":{"1min":0.94299999999999995,"5min":0.82699999999999996,"15min":1.123},"min":{"1min":0.43099999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":2.9009999999999998,"5min":7.9260000000000002,"15min":26.635000000000002},"last":0.52700000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:06:10 2023","interfaces":[{"interface":"back","average":{"1min":1.0549999999999999,"5min":1.478,"15min":1.2689999999999999},"min":{"1min":0.35299999999999998,"5min":0.33200000000000002,"15min":0.317},"max":{"1min":4.3689999999999998,"5min":11.638999999999999,"15min":11.638999999999999},"last":0.55700000000000005},{"interface":"front","average":{"1min":0.76300000000000001,"5min":1.21,"15min":1.234},"min":{"1min":0.433,"5min":0.36599999999999999,"15min":0.29599999999999999},"max":{"1min":3.6520000000000001,"5min":8.2100000000000009,"15min":9.5519999999999996},"last":0.46700000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:06:48 2023","interfaces":[{"interface":"back","average":{"1min":5.4340000000000002,"5min":2.036,"15min":2.0430000000000001},"min":{"1min":0.438,"5min":0.317,"15min":0.317},"max":{"1min":89.566000000000003,"5min":89.566000000000003,"15min":89.566000000000003},"last":0.41499999999999998},{"interface":"front","average":{"1min":5.8310000000000004,"5min":2.1339999999999999,"15min":1.8009999999999999},"min":{"1min":0.44800000000000001,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":90.608999999999995,"5min":90.608999999999995,"15min":90.608999999999995},"last":0.60199999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:06:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":0.86299999999999999,"15min":0.79000000000000004},"min":{"1min":0.39800000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.6310000000000002,"5min":4.6310000000000002,"15min":4.6310000000000002},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.492,"5min":1.542,"15min":1.732},"min":{"1min":0.45200000000000001,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":5.5839999999999996,"5min":9.5709999999999997,"15min":9.5709999999999997},"last":0.63}]}]},{"osd":0,"up_from":93,"seq":399431958694,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2554316,"kb_used_data":2104004,"kb_used_omap":7,"kb_used_meta":450296,"kb_avail":91190836,"statfs":{"total":95995035648,"available":93379416064,"internally_reserved":0,"allocated":2154500096,"data_stored":2152963174,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":461103844},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:06:14 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.246,"15min":1.3500000000000001},"min":{"1min":0.29699999999999999,"5min":0.20300000000000001,"15min":0.17899999999999999},"max":{"1min":15.441000000000001,"5min":15.441000000000001,"15min":18.154},"last":0.26100000000000001},{"interface":"front","average":{"1min":1.1950000000000001,"5min":0.874,"15min":1.226},"min":{"1min":0.28199999999999997,"5min":0.186,"15min":0.17399999999999999},"max":{"1min":11.220000000000001,"5min":11.220000000000001,"15min":13.369999999999999},"last":0.83199999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.417,"15min":1.54},"min":{"1min":0.311,"5min":0.21099999999999999,"15min":0.20499999999999999},"max":{"1min":10.007999999999999,"5min":10.48,"15min":20.882000000000001},"last":0.29399999999999998},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.3580000000000001,"15min":1.4219999999999999},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.217},"max":{"1min":10.390000000000001,"5min":10.390000000000001,"15min":15.064},"last":0.88400000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:06:22 2023","interfaces":[{"interface":"back","average":{"1min":1.4339999999999999,"5min":1.5409999999999999,"15min":1.5489999999999999},"min":{"1min":0.36499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":4.7359999999999998,"5min":11.948,"15min":11.948},"last":1.0760000000000001},{"interface":"front","average":{"1min":1.9790000000000001,"5min":1.5029999999999999,"15min":1.3009999999999999},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":15.198,"5min":15.198,"15min":15.198},"last":1.0249999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":1.385,"15min":1.2949999999999999},"min":{"1min":0.32100000000000001,"5min":0.29299999999999998,"15min":0.251},"max":{"1min":6.9889999999999999,"5min":6.9889999999999999,"15min":15.279999999999999},"last":1.0429999999999999},{"interface":"front","average":{"1min":2.2400000000000002,"5min":1.613,"15min":1.411},"min":{"1min":0.35299999999999998,"5min":0.247,"15min":0.247},"max":{"1min":15.477,"5min":15.477,"15min":18.106999999999999},"last":0.56499999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.534,"15min":1.4299999999999999},"min":{"1min":0.29799999999999999,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":7.0330000000000004,"5min":10.194000000000001,"15min":14.720000000000001},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.726,"5min":1.238,"15min":1.21},"min":{"1min":0.32600000000000001,"5min":0.254,"15min":0.254},"max":{"1min":10.363,"5min":10.363,"15min":12.794},"last":0.92100000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6910000000000001,"5min":1.3340000000000001,"15min":2.0009999999999999},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.34599999999999997},"max":{"1min":7.056,"5min":7.056,"15min":14.956},"last":1.0600000000000001},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.198,"15min":2.1280000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":10.321,"5min":10.321,"15min":12.755000000000001},"last":0.95599999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:06:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.139,"15min":0.995},"min":{"1min":0.38500000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.3010000000000002,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":0.98499999999999999},{"interface":"front","average":{"1min":1.202,"5min":1.409,"15min":1.1910000000000001},"min":{"1min":0.40600000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.6070000000000002,"5min":5.524,"15min":5.524},"last":1.0920000000000001}]}]},{"osd":2,"up_from":103,"seq":442381631641,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2313040,"kb_used_data":1873864,"kb_used_omap":11,"kb_used_meta":439156,"kb_avail":91432112,"statfs":{"total":95995035648,"available":93626482688,"internally_reserved":0,"allocated":1918836736,"data_stored":1917303412,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":449696119},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":4.4960000000000004,"5min":1.7090000000000001,"15min":1.708},"min":{"1min":0.219,"5min":0.187,"15min":0.17199999999999999},"max":{"1min":36.656999999999996,"5min":36.656999999999996,"15min":36.656999999999996},"last":0.435},{"interface":"front","average":{"1min":3.9820000000000002,"5min":1.5600000000000001,"15min":1.627},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":36.697000000000003,"5min":36.697000000000003,"15min":36.697000000000003},"last":0.47399999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4100000000000001,"5min":1.23,"15min":1.607},"min":{"1min":0.34799999999999998,"5min":0.215,"15min":0.215},"max":{"1min":8.2010000000000005,"5min":8.2010000000000005,"15min":20.995000000000001},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.2320000000000002,"5min":1.1180000000000001,"15min":1.3819999999999999},"min":{"1min":0.252,"5min":0.24199999999999999,"15min":0.193},"max":{"1min":8.3889999999999993,"5min":8.3889999999999993,"15min":15.945},"last":0.35799999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:06:31 2023","interfaces":[{"interface":"back","average":{"1min":2.806,"5min":1.3939999999999999,"15min":0.98599999999999999},"min":{"1min":0.29199999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":37.500999999999998,"5min":37.500999999999998,"15min":37.500999999999998},"last":0.31},{"interface":"front","average":{"1min":2.7370000000000001,"5min":1.4219999999999999,"15min":0.88100000000000001},"min":{"1min":0.372,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":34.915999999999997,"5min":34.915999999999997,"15min":34.915999999999997},"last":0.42099999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.204,"15min":1.278},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":13.016999999999999,"5min":13.016999999999999,"15min":13.016999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.0489999999999999,"15min":1.2769999999999999},"min":{"1min":0.34999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.0820000000000007,"5min":8.0820000000000007,"15min":20.920000000000002},"last":0.54600000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":1.097,"15min":1.345},"min":{"1min":0.32500000000000001,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":8.1760000000000002,"5min":8.1760000000000002,"15min":20.738},"last":0.57599999999999996},{"interface":"front","average":{"1min":2.1259999999999999,"5min":1.4790000000000001,"15min":1.4590000000000001},"min":{"1min":0.47099999999999997,"5min":0.32000000000000001,"15min":0.26700000000000002},"max":{"1min":6.9930000000000003,"5min":12.103999999999999,"15min":15.496},"last":0.45900000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":1.2709999999999999,"15min":1.9610000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":13.065,"5min":13.065,"15min":13.065},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.871,"5min":1.2210000000000001,"15min":2.169},"min":{"1min":0.45900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.4120000000000008,"5min":8.4649999999999999,"15min":20.893000000000001},"last":0.51400000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:06:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.3740000000000001,"15min":1.3979999999999999},"min":{"1min":0.42899999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":6.9500000000000002,"5min":12.051,"15min":12.051},"last":0.39200000000000002},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.181,"15min":0.81499999999999995},"min":{"1min":0.52100000000000002,"5min":0.309,"15min":0.309},"max":{"1min":8.1479999999999997,"5min":8.5039999999999996,"15min":8.5039999999999996},"last":0.79200000000000004}]}]},{"osd":3,"up_from":237,"seq":1017907249208,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2653104,"kb_used_data":2417896,"kb_used_omap":3,"kb_used_meta":235196,"kb_avail":91092048,"statfs":{"total":95995035648,"available":93278257152,"internally_reserved":0,"allocated":2475925504,"data_stored":2474252863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":240841448},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,0,0,0,0,0,0,0,0,3],"upper_bound":16384},"perf_stat":{"commit_latency_ms":4,"apply_latency_ms":4,"commit_latency_ns":4000000,"apply_latency_ns":4000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:20 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.91000000000000003,"15min":0.89200000000000002},"min":{"1min":0.20699999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":8.1579999999999995,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.251},{"interface":"front","average":{"1min":1.087,"5min":1.1279999999999999,"15min":1.097},"min":{"1min":0.33100000000000002,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.1239999999999997,"5min":14.885999999999999,"15min":14.885999999999999},"last":0.44400000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:06:20 2023","interfaces":[{"interface":"back","average":{"1min":1.5009999999999999,"5min":1.3009999999999999,"15min":1.1399999999999999},"min":{"1min":0.32100000000000001,"5min":0.245,"15min":0.245},"max":{"1min":7.0609999999999999,"5min":21.931999999999999,"15min":21.931999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.089,"15min":0.99299999999999999},"min":{"1min":0.25800000000000001,"5min":0.255,"15min":0.255},"max":{"1min":13.631,"5min":13.631,"15min":13.631},"last":0.60499999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:06:20 2023","interfaces":[{"interface":"back","average":{"1min":0.86599999999999999,"5min":0.95199999999999996,"15min":0.88400000000000001},"min":{"1min":0.34899999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":5.9219999999999997,"5min":8.2200000000000006,"15min":8.2200000000000006},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.3069999999999999,"15min":1.266},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":7.2000000000000002,"5min":21.988,"15min":21.988},"last":0.498}]},{"osd":4,"last update":"Sat Sep 23 18:06:20 2023","interfaces":[{"interface":"back","average":{"1min":0.92600000000000005,"5min":1.2150000000000001,"15min":1.141},"min":{"1min":0.311,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.4300000000000002,"5min":14.935,"15min":14.935},"last":0.59199999999999997},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.2789999999999999,"15min":1.1379999999999999},"min":{"1min":0.33400000000000002,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.6819999999999999,"5min":12.786,"15min":12.786},"last":0.51200000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:06:20 2023","interfaces":[{"interface":"back","average":{"1min":1.0409999999999999,"5min":1.3400000000000001,"15min":1.302},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":3.2639999999999998,"5min":12.743,"15min":12.743},"last":0.627},{"interface":"front","average":{"1min":0.97499999999999998,"5min":1.2210000000000001,"15min":1.194},"min":{"1min":0.51100000000000001,"5min":0.315,"15min":0.315},"max":{"1min":5.851,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.47199999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:06:20 2023","interfaces":[{"interface":"back","average":{"1min":1.079,"5min":1.1240000000000001,"15min":1.0980000000000001},"min":{"1min":0.435,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":5.8890000000000002,"5min":7.5789999999999997,"15min":7.5789999999999997},"last":0.53300000000000003},{"interface":"front","average":{"1min":0.93999999999999995,"5min":1.464,"15min":1.474},"min":{"1min":0.41199999999999998,"5min":0.379,"15min":0.379},"max":{"1min":2.4009999999999998,"5min":14.964,"15min":14.964},"last":0.54700000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:06:20 2023","interfaces":[{"interface":"back","average":{"1min":0.98699999999999999,"5min":1.508,"15min":1.55},"min":{"1min":0.39600000000000002,"5min":0.377,"15min":0.377},"max":{"1min":2.7170000000000001,"5min":12.821,"15min":12.821},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.05,"5min":1.464,"15min":1.5680000000000001},"min":{"1min":0.46400000000000002,"5min":0.41899999999999998,"15min":0.41899999999999998},"max":{"1min":5.9530000000000003,"5min":8.6039999999999992,"15min":8.6039999999999992},"last":0.57599999999999996}]}]},{"osd":4,"up_from":30,"seq":128849019158,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1574952,"kb_used_data":1307552,"kb_used_omap":0,"kb_used_meta":267392,"kb_avail":92170200,"statfs":{"total":95995035648,"available":94382284800,"internally_reserved":0,"allocated":1338933248,"data_stored":1337781685,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":273809408},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.8779999999999999,"5min":1.0349999999999999,"15min":0.97799999999999998},"min":{"1min":0.26600000000000001,"5min":0.215,"15min":0.20100000000000001},"max":{"1min":13.375999999999999,"5min":13.375999999999999,"15min":14.164},"last":0.38300000000000001},{"interface":"front","average":{"1min":1.829,"5min":1.3400000000000001,"15min":1.2090000000000001},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":5.7809999999999997,"5min":7.1879999999999997,"15min":11.372},"last":0.35399999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:06:10 2023","interfaces":[{"interface":"back","average":{"1min":0.66000000000000003,"5min":0.95799999999999996,"15min":1.0960000000000001},"min":{"1min":0.35899999999999999,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":2.52,"5min":5.9740000000000002,"15min":10.429},"last":0.317},{"interface":"front","average":{"1min":0.79900000000000004,"5min":1.0700000000000001,"15min":1.1599999999999999},"min":{"1min":0.33900000000000002,"5min":0.27300000000000002,"15min":0.23100000000000001},"max":{"1min":3.456,"5min":6.7640000000000002,"15min":10.83},"last":0.27100000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:06:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.1080000000000001,"15min":1.1850000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26600000000000001},"max":{"1min":4.5060000000000002,"5min":6.8259999999999996,"15min":11.630000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.8660000000000001,"5min":1.2949999999999999,"15min":1.1919999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.27100000000000002},"max":{"1min":13.672000000000001,"5min":13.672000000000001,"15min":13.672000000000001},"last":0.45600000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:06:21 2023","interfaces":[{"interface":"back","average":{"1min":2.754,"5min":1.6419999999999999,"15min":1.448},"min":{"1min":0.42599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":29.154,"5min":29.154,"15min":29.154},"last":0.42899999999999999},{"interface":"front","average":{"1min":2.4180000000000001,"5min":1.6319999999999999,"15min":1.4590000000000001},"min":{"1min":0.40100000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":29.216000000000001,"5min":29.216000000000001,"15min":29.216000000000001},"last":0.505}]},{"osd":5,"last update":"Sat Sep 23 18:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.95799999999999996,"15min":0.73999999999999999},"min":{"1min":0.26800000000000002,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":2.5459999999999998,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.246,"15min":0.93500000000000005},"min":{"1min":0.308,"5min":0.26800000000000002,"15min":0.26100000000000001},"max":{"1min":3.2410000000000001,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.30099999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":1.0469999999999999,"15min":0.89900000000000002},"min":{"1min":0.29599999999999999,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":2.282,"5min":9.0990000000000002,"15min":9.0990000000000002},"last":0.44700000000000001},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.0089999999999999,"15min":0.98399999999999999},"min":{"1min":0.33400000000000002,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":2.335,"5min":9.1300000000000008,"15min":10.725},"last":0.372}]},{"osd":7,"last update":"Sat Sep 23 18:06:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.242,"15min":1.0229999999999999},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":3.427,"5min":13.298999999999999,"15min":13.298999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.3740000000000001,"15min":1.266},"min":{"1min":0.375,"5min":0.307,"15min":0.307},"max":{"1min":2.0110000000000001,"5min":10.192,"15min":10.192},"last":0.40000000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757525,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1315380,"kb_used_data":1073644,"kb_used_omap":0,"kb_used_meta":241728,"kb_avail":92429772,"statfs":{"total":95995035648,"available":94648086528,"internally_reserved":0,"allocated":1099411456,"data_stored":1098283395,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":247529472},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":0.85499999999999998,"15min":1.073},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.217},"max":{"1min":5.4539999999999997,"5min":5.4539999999999997,"15min":21.376999999999999},"last":0.505},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.2589999999999999,"15min":1.2789999999999999},"min":{"1min":0.32000000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.5309999999999997,"5min":8.5099999999999998,"15min":21.388999999999999},"last":0.315}]},{"osd":1,"last update":"Sat Sep 23 18:06:14 2023","interfaces":[{"interface":"back","average":{"1min":0.85899999999999999,"5min":0.75600000000000001,"15min":0.88400000000000001},"min":{"1min":0.33500000000000002,"5min":0.26000000000000001,"15min":0.23899999999999999},"max":{"1min":3.879,"5min":4.6150000000000002,"15min":10.853999999999999},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.83499999999999996,"5min":1.1279999999999999,"15min":1.075},"min":{"1min":0.30299999999999999,"5min":0.26200000000000001,"15min":0.254},"max":{"1min":3.6440000000000001,"5min":8.5619999999999994,"15min":10.206},"last":0.38500000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.145,"15min":1.377},"min":{"1min":0.44600000000000001,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":3.234,"5min":5.5309999999999997,"15min":38.606999999999999},"last":0.59899999999999998},{"interface":"front","average":{"1min":1.395,"5min":1.0820000000000001,"15min":1.2809999999999999},"min":{"1min":0.39900000000000002,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":6.2489999999999997,"5min":10.065,"15min":39.670999999999999},"last":0.45400000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:06:23 2023","interfaces":[{"interface":"back","average":{"1min":1.085,"5min":1.093,"15min":0.875},"min":{"1min":0.39900000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.9780000000000002,"5min":8.593,"15min":8.593},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.653,"5min":1.0409999999999999,"15min":0.82099999999999995},"min":{"1min":0.41999999999999998,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":13.446999999999999,"5min":13.446999999999999,"15min":13.446999999999999},"last":0.56299999999999994}]},{"osd":4,"last update":"Sat Sep 23 18:06:53 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":1.0109999999999999,"15min":0.80500000000000005},"min":{"1min":0.25700000000000001,"5min":0.24199999999999999,"15min":0.224},"max":{"1min":2.7229999999999999,"5min":8.6229999999999993,"15min":10.092000000000001},"last":0.41499999999999998},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.91000000000000003,"15min":0.77100000000000002},"min":{"1min":0.30099999999999999,"5min":0.29499999999999998,"15min":0.23999999999999999},"max":{"1min":1.6850000000000001,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.66700000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:06:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.94499999999999995,"15min":1.2969999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":2.0659999999999998,"5min":6.5880000000000001,"15min":11.287000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.89800000000000002,"15min":1.1830000000000001},"min":{"1min":0.32100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0139999999999998,"5min":6.6920000000000002,"15min":8.266},"last":0.54000000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:06:33 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.133,"15min":1.4079999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.0299999999999998,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.63},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.133,"15min":1.127},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.0760000000000001,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.69899999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2146041856,"data_stored":2146041856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1183514624,"data_stored":1183514624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1911029760,"data_stored":1911029760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2467364864,"data_stored":2467364864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1335427072,"data_stored":1335427072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1095892992,"data_stored":1095892992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1472790528,"data_stored":1472790528,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1520369664,"data_stored":1520369664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:07:02.278 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:07:02.278 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_2 pg_num 26 2023-09-23T18:07:02.278 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:02 smithi055 ceph-mon[184174]: pgmap v764: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 3 remapped+peering, 8 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 496 KiB/s wr, 7 op/s; 29023/196656 objects degraded (14.758%); 3157/196656 objects misplaced (1.605%); 11 MiB/s, 177 objects/s recovering 2023-09-23T18:07:02.278 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:02 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:07:02.279 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:02 smithi055 ceph-mon[179668]: pgmap v764: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 3 remapped+peering, 8 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 496 KiB/s wr, 7 op/s; 29023/196656 objects degraded (14.758%); 3157/196656 objects misplaced (1.605%); 11 MiB/s, 177 objects/s recovering 2023-09-23T18:07:02.279 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:02 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:07:02.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:02.358 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:02.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:02.393 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:02 smithi099 ceph-mon[167076]: pgmap v764: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 activating+undersized+degraded+remapped, 3 remapped+peering, 8 active+recovery_wait+undersized+degraded+remapped, 104 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 496 KiB/s wr, 7 op/s; 29023/196656 objects degraded (14.758%); 3157/196656 objects misplaced (1.605%); 11 MiB/s, 177 objects/s recovering 2023-09-23T18:07:02.393 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:02 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:07:02.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:02.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:02.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:02.521 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:02.557 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:02.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:02.559 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:02.660 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:02.688 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:02.724 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:02.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:02.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:02.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:02.854 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:02.871 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 72 16 65646 65630 56.9344 1.05208 0.00357151 0.0157315 2023-09-23T18:07:02.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:02.893 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:02.893 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:02.980 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:07:02.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:03.003 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:07:03.022 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:03.023 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:07:03.043 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:07:03.065 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:07:03.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:03.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:03.071 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:03.086 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:07:03.106 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:07:03.125 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:07:03.127 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:07:03.127 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:07:03.127 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:07:03.127 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:07:03.127 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:07:03.127 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:07:03.128 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:07:03.128 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:07:03.139 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:03 smithi055 ceph-mon[179668]: from='client.35865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:07:03.139 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/839512298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:03.139 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 29023/196656 objects degraded (14.758%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T18:07:03.139 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:03 smithi055 ceph-mon[184174]: from='client.35865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:07:03.140 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/839512298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:03.140 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 29023/196656 objects degraded (14.758%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T18:07:03.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:03.198 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:03.240 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:03.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:03.243 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:03.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:03 smithi099 ceph-mon[167076]: from='client.35865 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:07:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/839512298' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:03.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 29023/196656 objects degraded (14.758%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T18:07:03.344 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:03.377 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:03.752 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:03.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:03.756 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:03.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:03.871 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 73 16 65646 65630 56.1549 0 - 0.0157315 2023-09-23T18:07:03.883 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:03.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:03.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:03.971 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:04.072 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:04.112 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:04.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:04 smithi055 ceph-mon[179668]: pgmap v765: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 609 KiB/s wr, 9 op/s; 33016/196725 objects degraded (16.783%); 7147/196725 objects misplaced (3.633%); 1/65575 objects unfound (0.002%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T18:07:04.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:04 smithi055 ceph-mon[179668]: Health check failed: 1/65575 objects unfound (0.002%) (OBJECT_UNFOUND) 2023-09-23T18:07:04.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:04 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:07:04.143 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:04 smithi055 ceph-mon[184174]: pgmap v765: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 609 KiB/s wr, 9 op/s; 33016/196725 objects degraded (16.783%); 7147/196725 objects misplaced (3.633%); 1/65575 objects unfound (0.002%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T18:07:04.143 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:04 smithi055 ceph-mon[184174]: Health check failed: 1/65575 objects unfound (0.002%) (OBJECT_UNFOUND) 2023-09-23T18:07:04.143 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:04 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:07:04.219 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:04.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:04.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:04.304 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:04.319 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65733 7.e 7:7c6df98a:::benchmark_data_smithi055_311528_object65732:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:04.287727+0000", 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.022953845, 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022994502999999999, 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:04.320 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65733 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.287727+0000", 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.287727+0000", 2023-09-23T18:07:04.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:04.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.287725+0000", 2023-09-23T18:07:04.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:04.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:04.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.287753+0000", 2023-09-23T18:07:04.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7713999999999999e-05 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.287767+0000", 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3626e-05 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.287775+0000", 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1429999999999998e-06 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.287936+0000", 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016065400000000001 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.288097+0000", 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016160400000000001 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,5", 2023-09-23T18:07:04.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.288283+0000", 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000185274 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.293746+0000", 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0054633759999999998 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301626+0000", 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0078805360000000005 2023-09-23T18:07:04.325 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65737 7.e 7:715e419e:::benchmark_data_smithi055_311528_object65736:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e313)", 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:04.290131+0000", 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.020549866999999999, 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020700252999999998, 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:07:04.326 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65737 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.290131+0000", 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.290131+0000", 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.290130+0000", 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:04.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.290162+0000", 2023-09-23T18:07:04.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1461999999999998e-05 2023-09-23T18:07:04.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:04.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.290171+0000", 2023-09-23T18:07:04.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8030000000000003e-06 2023-09-23T18:07:04.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.290177+0000", 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3040000000000001e-06 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.290642+0000", 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00046530299999999997 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.290791+0000", 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014864500000000001 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,5", 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.290949+0000", 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015786899999999999 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.300873+0000", 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0099238039999999996 2023-09-23T18:07:04.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301666+0000", 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00079393300000000003 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:65741 7.5 e313/299 7:a5423300:::benchmark_data_smithi055_311528_object65740:head v 313'4220, mlcod=0'0)", 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:04.301265+0000", 2023-09-23T18:07:04.332 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0094154029999999993, 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0096549689999999994, 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301265+0000", 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301265+0000", 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301264+0000", 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:04.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301315+0000", 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0593000000000002e-05 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301331+0000", 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5506e-05 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301339+0000", 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5129999999999996e-06 2023-09-23T18:07:04.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.304062+0000", 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027234450000000001 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.304096+0000", 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3417999999999998e-05 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:04.336 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:65731 7.0 e313/304 7:042e8c10:::benchmark_data_smithi055_311528_object65730:head v 313'4141, mlcod=0'0)", 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:04.301390+0000", 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0092910979999999994, 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0095926899999999992, 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301390+0000", 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301390+0000", 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301389+0000", 2023-09-23T18:07:04.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301427+0000", 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7401999999999999e-05 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301435+0000", 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.354e-06 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301440+0000", 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3959999999999997e-06 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301947+0000", 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00050670999999999995 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301976+0000", 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9232999999999999e-05 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:65736 7.0 e313/304 7:02653460:::benchmark_data_smithi055_311528_object65735:head v 313'4142, mlcod=0'0)", 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:04.301455+0000", 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0092261189999999996, 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0095861430000000001, 2023-09-23T18:07:04.341 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:04.342 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:07:04.342 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:04.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:04.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301455+0000", 2023-09-23T18:07:04.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301455+0000", 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301454+0000", 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301491+0000", 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6977000000000002e-05 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301499+0000", 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2219999999999996e-06 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.301504+0000", 2023-09-23T18:07:04.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2399999999999998e-06 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.306060+0000", 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045564760000000003 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.306092+0000", 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1225e-05 2023-09-23T18:07:04.345 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.45557.0:65733 7.e e313/303 ondisk, result = 0)", 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:04.308410+0000", 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0022711929999999999, 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026839120000000001, 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:04.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.308410+0000", 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.308410+0000", 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.308409+0000", 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.308413+0000", 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0709999999999996e-06 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.308421+0000", 2023-09-23T18:07:04.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8499999999999994e-06 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.308427+0000", 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.429e-06 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.310910+0000", 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024838 2023-09-23T18:07:04.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.310921+0000", 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1071e-05 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:65745 7.0 e313/304)", 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:04.310433+0000", 2023-09-23T18:07:04.350 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.00024801499999999998, 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00071122100000000005, 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.310433+0000", 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:04.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.310433+0000", 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.310432+0000", 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.310489+0000", 2023-09-23T18:07:04.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7096999999999999e-05 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.310503+0000", 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4474999999999999e-05 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.310512+0000", 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7649999999999999e-06 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:04.353 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.354 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:04.354 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:07:04.354 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 7 2023-09-23T18:07:04.354 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:07:04.354 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:04.432 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:04 smithi099 ceph-mon[167076]: pgmap v765: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 609 KiB/s wr, 9 op/s; 33016/196725 objects degraded (16.783%); 7147/196725 objects misplaced (3.633%); 1/65575 objects unfound (0.002%); 7.8 MiB/s, 125 objects/s recovering 2023-09-23T18:07:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:04 smithi099 ceph-mon[167076]: Health check failed: 1/65575 objects unfound (0.002%) (OBJECT_UNFOUND) 2023-09-23T18:07:04.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:04 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:07:04.871 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 74 16 65980 65964 55.6784 10.4375 0.00435429 0.0179267 2023-09-23T18:07:04.908 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:04.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:04.914 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:04.941 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:07:05.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:05.049 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:05.092 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:05.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:05.097 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:05.198 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:05.225 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:05.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:05.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:05.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:05.393 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:05.433 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:05.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:05.436 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:05.537 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:05.592 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:05.700 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:05.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:05.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:05.722 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:07:05.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[184174]: pgmap v766: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 965 KiB/s wr, 15 op/s; 33016/196884 objects degraded (16.769%); 7147/196884 objects misplaced (3.630%); 1/65628 objects unfound (0.002%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:07:05.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/244697914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:05.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:05.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:05.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:05.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[179668]: pgmap v766: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 965 KiB/s wr, 15 op/s; 33016/196884 objects degraded (16.769%); 7147/196884 objects misplaced (3.630%); 1/65628 objects unfound (0.002%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:07:05.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/244697914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:05.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:05.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:05.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:05.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:05.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:05 smithi099 ceph-mon[167076]: pgmap v766: 119 pgs: 1 activating+undersized+degraded+remapped, 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 1 remapped+peering, 9 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 965 KiB/s wr, 15 op/s; 33016/196884 objects degraded (16.769%); 7147/196884 objects misplaced (3.630%); 1/65628 objects unfound (0.002%); 6.6 MiB/s, 105 objects/s recovering 2023-09-23T18:07:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/244697914' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:05 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:07:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:07:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:05.834 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:05.871 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:05.872 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 75 16 66167 66151 55.0921 11.6875 0.00930325 0.0180619 2023-09-23T18:07:05.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:05.875 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:05.911 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:07:05.975 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:06.016 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:06.108 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:06.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:06.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:06.207 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:06.212 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:06.264 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:06.347 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:06.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:06.357 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:06.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:06.486 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:06.523 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:06.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:06.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:06.628 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:06.680 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:06.724 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:06.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:06.734 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:06.780 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:07:06.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[184174]: failed to encode map e314 with expected crc 2023-09-23T18:07:06.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:07:06.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[184174]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T18:07:06.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[184174]: failed to encode map e314 with expected crc 2023-09-23T18:07:06.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:06.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:06.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/878841429' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:07:06.781 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:07:06.781 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:07:06.781 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[179668]: failed to encode map e314 with expected crc 2023-09-23T18:07:06.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:07:06.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[179668]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T18:07:06.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[179668]: failed to encode map e314 with expected crc 2023-09-23T18:07:06.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:06.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:06.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/878841429' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:07:06.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:06 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:07:06.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:06 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:07:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:06 smithi099 ceph-mon[167076]: failed to encode map e314 with expected crc 2023-09-23T18:07:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:06 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:07:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:06 smithi099 ceph-mon[167076]: osdmap e314: 8 total, 8 up, 8 in 2023-09-23T18:07:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:06 smithi099 ceph-mon[167076]: failed to encode map e314 with expected crc 2023-09-23T18:07:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:06 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/878841429' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:07:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:06 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:07:06.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:06.869 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:06.871 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 76 16 66406 66390 54.564 14.9375 0.0170027 0.0182543 2023-09-23T18:07:06.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:06.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:06.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:07.007 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:07.016 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:07.052 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:07.143 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:07.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:07.152 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:07.253 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:07.281 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:07.320 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:07.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:07.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:07.425 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:07.428 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 7 pg_num to 26 2023-09-23T18:07:07.452 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:07.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:07.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:07.493 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:07.593 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:07.622 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:07.664 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:07.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:07.669 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:07 smithi055 ceph-mon[184174]: failed to encode map e314 with expected crc 2023-09-23T18:07:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:07 smithi055 ceph-mon[184174]: pgmap v768: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 2.7 MiB/s wr, 42 op/s; 41450/197586 objects degraded (20.978%); 6675/197586 objects misplaced (3.378%); 1/65862 objects unfound (0.002%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T18:07:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:07 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-09-23T18:07:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:07 smithi055 ceph-mon[184174]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T18:07:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:07 smithi055 ceph-mon[179668]: failed to encode map e314 with expected crc 2023-09-23T18:07:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:07 smithi055 ceph-mon[179668]: pgmap v768: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 2.7 MiB/s wr, 42 op/s; 41450/197586 objects degraded (20.978%); 6675/197586 objects misplaced (3.378%); 1/65862 objects unfound (0.002%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T18:07:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:07 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-09-23T18:07:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:07 smithi055 ceph-mon[179668]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T18:07:07.767 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:07:07.767 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:07:07.767 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:07:07.767 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:07:07.767 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:07:07.770 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:07.804 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:07.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:07 smithi099 ceph-mon[167076]: failed to encode map e314 with expected crc 2023-09-23T18:07:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:07 smithi099 ceph-mon[167076]: pgmap v768: 119 pgs: 1 active+recovering+undersized+remapped, 1 activating+undersized+degraded+remapped, 2 active+recovery_wait+degraded, 10 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 2.7 MiB/s wr, 42 op/s; 41450/197586 objects degraded (20.978%); 6675/197586 objects misplaced (3.378%); 1/65862 objects unfound (0.002%); 9.6 MiB/s, 152 objects/s recovering 2023-09-23T18:07:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:07 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "26"}]': finished 2023-09-23T18:07:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:07 smithi099 ceph-mon[167076]: osdmap e315: 8 total, 8 up, 8 in 2023-09-23T18:07:07.871 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 77 16 66619 66603 54.0285 13.3125 0.00673036 0.0184131 2023-09-23T18:07:07.902 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:07.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:07.908 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:08.008 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:08.045 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:08.164 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:07:08.189 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:07:08.338 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:08.342 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:08.342 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:08.444 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:08.495 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:08.545 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:08.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:08.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:08.650 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:08.692 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:08.733 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:08.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:08.736 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:08.781 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:07:08.810 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:07:08.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:08 smithi099 ceph-mon[167076]: failed to encode map e315 with expected crc 2023-09-23T18:07:08.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:08 smithi099 ceph-mon[167076]: failed to encode map e315 with expected crc 2023-09-23T18:07:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 41450/197586 objects degraded (20.978%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:08.834 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:07:08.837 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:08.862 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:07:08.872 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 78 16 66784 66768 53.4683 10.3125 0.00399314 0.0186076 2023-09-23T18:07:08.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:08 smithi055 ceph-mon[184174]: failed to encode map e315 with expected crc 2023-09-23T18:07:08.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:08 smithi055 ceph-mon[184174]: failed to encode map e315 with expected crc 2023-09-23T18:07:08.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 41450/197586 objects degraded (20.978%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:08.872 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:08 smithi055 ceph-mon[179668]: failed to encode map e315 with expected crc 2023-09-23T18:07:08.873 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:08 smithi055 ceph-mon[179668]: failed to encode map e315 with expected crc 2023-09-23T18:07:08.873 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 41450/197586 objects degraded (20.978%), 13 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:08.875 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:08.884 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:07:08.899 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:08.906 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:07:08.919 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:08.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:08.923 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:08.933 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:07:08.934 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:08.964 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:07:08.966 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:07:08.966 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:07:08.966 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:07:08.967 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:07:08.967 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:07:08.967 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:07:08.967 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:07:08.967 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:07:09.024 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:09.058 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:09.286 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:09.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:09.291 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:09.392 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:09.418 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:09.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:09.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:09.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:09.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:09.604 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:09.784 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:09.786 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1157683957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:09.787 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:09 smithi099 ceph-mon[167076]: pgmap v770: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 5.6 MiB/s wr, 89 op/s; 41447/198804 objects degraded (20.848%); 6675/198804 objects misplaced (3.358%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:07:09.787 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:09 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T18:07:09.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:09.788 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:09.876 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 79 16 66933 66917 52.9068 9.3125 0.130386 0.018743 2023-09-23T18:07:09.889 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1157683957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:09 smithi055 ceph-mon[179668]: pgmap v770: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 5.6 MiB/s wr, 89 op/s; 41447/198804 objects degraded (20.848%); 6675/198804 objects misplaced (3.358%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:07:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:09 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T18:07:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1157683957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:09 smithi055 ceph-mon[184174]: pgmap v770: 119 pgs: 1 active+recovering+undersized+remapped, 2 active+recovery_wait+degraded, 11 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 5.6 MiB/s wr, 89 op/s; 41447/198804 objects degraded (20.848%); 6675/198804 objects misplaced (3.358%); 3.7 MiB/s, 59 objects/s recovering 2023-09-23T18:07:09.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:09 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T18:07:10.563 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:10.592 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:07:10] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.43.0" 2023-09-23T18:07:10.602 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:10.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:10.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:10.709 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:10.768 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:10.805 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:10.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:10.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:10.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:10 smithi099 ceph-mon[167076]: Health check cleared: OBJECT_UNFOUND (was: 1/65862 objects unfound (0.002%)) 2023-09-23T18:07:10.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:10 smithi099 ceph-mon[167076]: failed to encode map e316 with expected crc 2023-09-23T18:07:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:10 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T18:07:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:10 smithi099 ceph-mon[167076]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T18:07:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:10 smithi099 ceph-mon[167076]: failed to encode map e316 with expected crc 2023-09-23T18:07:10.876 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:07:10.877298+0000 min lat: 0.00119503 max lat: 19.3768 avg lat: 0.0187431 2023-09-23T18:07:10.876 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:07:10.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 80 16 66934 66918 52.2466 0.0625 0.0255695 0.0187431 2023-09-23T18:07:10.909 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:10.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-mon[184174]: Health check cleared: OBJECT_UNFOUND (was: 1/65862 objects unfound (0.002%)) 2023-09-23T18:07:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-mon[184174]: failed to encode map e316 with expected crc 2023-09-23T18:07:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T18:07:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-mon[184174]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T18:07:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-mon[184174]: failed to encode map e316 with expected crc 2023-09-23T18:07:10.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-mon[179668]: Health check cleared: OBJECT_UNFOUND (was: 1/65862 objects unfound (0.002%)) 2023-09-23T18:07:10.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-mon[179668]: failed to encode map e316 with expected crc 2023-09-23T18:07:10.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T18:07:10.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-mon[179668]: osdmap e316: 8 total, 8 up, 8 in 2023-09-23T18:07:10.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:10 smithi055 ceph-mon[179668]: failed to encode map e316 with expected crc 2023-09-23T18:07:11.054 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:11.535 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:07:11.753 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:11.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:11.757 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:11.858 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:11.876 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 81 16 66935 66919 51.6027 0.0625 1.82579 0.0187701 2023-09-23T18:07:11.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[179668]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[179668]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[179668]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T18:07:11.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[179668]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[179668]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[179668]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[179668]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.887 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[179668]: pgmap v773: 129 pgs: 3 peering, 1 active+recovering+undersized+remapped, 2 remapped+peering, 8 unknown, 3 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 24325/193647 objects degraded (12.562%); 6144/193647 objects misplaced (3.173%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:07:11.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[184174]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[184174]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[184174]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T18:07:11.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[184174]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[184174]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[184174]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[184174]: failed to encode map e317 with expected crc 2023-09-23T18:07:11.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:11 smithi055 ceph-mon[184174]: pgmap v773: 129 pgs: 3 peering, 1 active+recovering+undersized+remapped, 2 remapped+peering, 8 unknown, 3 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 24325/193647 objects degraded (12.562%); 6144/193647 objects misplaced (3.173%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:07:11.889 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:11 smithi099 ceph-mon[167076]: failed to encode map e317 with expected crc 2023-09-23T18:07:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:11 smithi099 ceph-mon[167076]: failed to encode map e317 with expected crc 2023-09-23T18:07:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:11 smithi099 ceph-mon[167076]: osdmap e317: 8 total, 8 up, 8 in 2023-09-23T18:07:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:11 smithi099 ceph-mon[167076]: failed to encode map e317 with expected crc 2023-09-23T18:07:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:11 smithi099 ceph-mon[167076]: failed to encode map e317 with expected crc 2023-09-23T18:07:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:11 smithi099 ceph-mon[167076]: failed to encode map e317 with expected crc 2023-09-23T18:07:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:11 smithi099 ceph-mon[167076]: failed to encode map e317 with expected crc 2023-09-23T18:07:12.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:11 smithi099 ceph-mon[167076]: pgmap v773: 129 pgs: 3 peering, 1 active+recovering+undersized+remapped, 2 remapped+peering, 8 unknown, 3 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 105 active+clean; 3.9 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 24325/193647 objects degraded (12.562%); 6144/193647 objects misplaced (3.173%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:07:12.876 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 82 16 66937 66921 50.9753 0.125 0.00472086 0.0188314 2023-09-23T18:07:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[179668]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[179668]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T18:07:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[179668]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[179668]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[179668]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[179668]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[179668]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[179668]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[184174]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[184174]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T18:07:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[184174]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[184174]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[184174]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[184174]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[184174]: failed to encode map e318 with expected crc 2023-09-23T18:07:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:12 smithi055 ceph-mon[184174]: failed to encode map e318 with expected crc 2023-09-23T18:07:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:12 smithi099 ceph-mon[167076]: failed to encode map e318 with expected crc 2023-09-23T18:07:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:12 smithi099 ceph-mon[167076]: osdmap e318: 8 total, 8 up, 8 in 2023-09-23T18:07:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:12 smithi099 ceph-mon[167076]: failed to encode map e318 with expected crc 2023-09-23T18:07:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:12 smithi099 ceph-mon[167076]: failed to encode map e318 with expected crc 2023-09-23T18:07:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:12 smithi099 ceph-mon[167076]: failed to encode map e318 with expected crc 2023-09-23T18:07:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:12 smithi099 ceph-mon[167076]: failed to encode map e318 with expected crc 2023-09-23T18:07:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:12 smithi099 ceph-mon[167076]: failed to encode map e318 with expected crc 2023-09-23T18:07:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:12 smithi099 ceph-mon[167076]: failed to encode map e318 with expected crc 2023-09-23T18:07:13.354 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:07:13.354 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:07:13.354 INFO:tasks.thrashosds.thrasher:Reweighting osd 4 to 0.9786694738186201 2023-09-23T18:07:13.355 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 0.9786694738186201 2023-09-23T18:07:13.375 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:13.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:13.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:13.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:13.513 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:13.553 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:13.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:13.557 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:13.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:13.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[179668]: failed to encode map e319 with expected crc 2023-09-23T18:07:13.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[179668]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T18:07:13.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[179668]: failed to encode map e319 with expected crc 2023-09-23T18:07:13.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[179668]: pgmap v776: 129 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 2 active+recovering+undersized+remapped, 2 remapped+peering, 4 unknown, 5 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 18206/194331 objects degraded (9.369%); 10112/194331 objects misplaced (5.203%); 11 MiB/s, 170 objects/s recovering 2023-09-23T18:07:13.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[179668]: failed to encode map e319 with expected crc 2023-09-23T18:07:13.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 24325/193647 objects degraded (12.562%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:13.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[184174]: failed to encode map e319 with expected crc 2023-09-23T18:07:13.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[184174]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T18:07:13.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[184174]: failed to encode map e319 with expected crc 2023-09-23T18:07:13.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[184174]: pgmap v776: 129 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 2 active+recovering+undersized+remapped, 2 remapped+peering, 4 unknown, 5 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 18206/194331 objects degraded (9.369%); 10112/194331 objects misplaced (5.203%); 11 MiB/s, 170 objects/s recovering 2023-09-23T18:07:13.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[184174]: failed to encode map e319 with expected crc 2023-09-23T18:07:13.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 24325/193647 objects degraded (12.562%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:13.684 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:13.718 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:13.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:13.721 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:13.822 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:13 smithi099 ceph-mon[167076]: failed to encode map e319 with expected crc 2023-09-23T18:07:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:13 smithi099 ceph-mon[167076]: osdmap e319: 8 total, 8 up, 8 in 2023-09-23T18:07:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:13 smithi099 ceph-mon[167076]: failed to encode map e319 with expected crc 2023-09-23T18:07:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:13 smithi099 ceph-mon[167076]: pgmap v776: 129 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 2 active+recovering+undersized+remapped, 2 remapped+peering, 4 unknown, 5 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 18206/194331 objects degraded (9.369%); 10112/194331 objects misplaced (5.203%); 11 MiB/s, 170 objects/s recovering 2023-09-23T18:07:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:13 smithi099 ceph-mon[167076]: failed to encode map e319 with expected crc 2023-09-23T18:07:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 24325/193647 objects degraded (12.562%), 10 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:13.853 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:13.876 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 83 16 67375 67359 50.6911 27.375 0.00348321 0.0193633 2023-09-23T18:07:14.074 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:14.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:14.077 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:14.178 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:14.209 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:14.341 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:14.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:14.350 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:14.451 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:14.480 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:14.782 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:14 smithi055 ceph-mon[184174]: failed to encode map e320 with expected crc 2023-09-23T18:07:14.782 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:14 smithi055 ceph-mon[184174]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T18:07:14.782 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:14 smithi055 ceph-mon[184174]: failed to encode map e320 with expected crc 2023-09-23T18:07:14.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:14 smithi055 ceph-mon[179668]: failed to encode map e320 with expected crc 2023-09-23T18:07:14.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:14 smithi055 ceph-mon[179668]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T18:07:14.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:14 smithi055 ceph-mon[179668]: failed to encode map e320 with expected crc 2023-09-23T18:07:14.876 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 84 16 67375 67359 50.0879 0 - 0.0193633 2023-09-23T18:07:15.065 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:14 smithi099 ceph-mon[167076]: failed to encode map e320 with expected crc 2023-09-23T18:07:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:14 smithi099 ceph-mon[167076]: osdmap e320: 8 total, 8 up, 8 in 2023-09-23T18:07:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:14 smithi099 ceph-mon[167076]: failed to encode map e320 with expected crc 2023-09-23T18:07:15.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:15.078 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:15.148 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:07:15.148 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:07:15.171 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:15.178 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:15.220 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:15.263 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:15.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:15.267 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:15.368 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:15.394 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:15.427 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:15.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:15.431 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:15.532 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:15.558 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:15.572 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:15.591 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:15.594 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:15.594 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:15.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:07:15.682 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:15 smithi099 ceph-mon[167076]: pgmap v778: 129 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 2 active+recovering+undersized+remapped, 2 remapped+peering, 2 unknown, 7 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 6.2 MiB/s wr, 99 op/s; 18206/194601 objects degraded (9.356%); 10112/194601 objects misplaced (5.196%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:07:15.695 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:15.740 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:15.796 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:15.799 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:15 smithi055 ceph-mon[184174]: pgmap v778: 129 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 2 active+recovering+undersized+remapped, 2 remapped+peering, 2 unknown, 7 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 6.2 MiB/s wr, 99 op/s; 18206/194601 objects degraded (9.356%); 10112/194601 objects misplaced (5.196%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:07:15.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:15 smithi055 ceph-mon[179668]: pgmap v778: 129 pgs: 1 activating+undersized+degraded+remapped, 4 peering, 2 active+recovering+undersized+remapped, 2 remapped+peering, 2 unknown, 7 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 6.2 MiB/s wr, 99 op/s; 18206/194601 objects degraded (9.356%); 10112/194601 objects misplaced (5.196%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:07:15.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:15.801 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:15.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 85 16 67461 67445 49.5621 2.6875 0.00843123 0.0199803 2023-09-23T18:07:15.901 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:15.944 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:16.331 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:16.714 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:07:16.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1093005889' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:07:16.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1093005889' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:07:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1093005889' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:07:16.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 86 16 68064 68048 49.4241 37.6875 0.0038961 0.0202195 2023-09-23T18:07:17.475 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:17.489 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:17.489 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:17.590 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:17.628 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:17.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:17.699 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:17.700 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:17.798 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:07:17.798 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:07:17.801 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:17.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:17 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:17.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:17 smithi099 ceph-mon[167076]: failed to encode map e321 with expected crc 2023-09-23T18:07:17.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1093005889' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:07:17.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:17 smithi099 ceph-mon[167076]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T18:07:17.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:17 smithi099 ceph-mon[167076]: pgmap v780: 129 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 506 op/s; 37312/201510 objects degraded (18.516%); 7958/201510 objects misplaced (3.949%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:07:17.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:17 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:17.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:17 smithi055 ceph-mon[184174]: failed to encode map e321 with expected crc 2023-09-23T18:07:17.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1093005889' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:07:17.855 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:17 smithi055 ceph-mon[184174]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T18:07:17.855 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:17 smithi055 ceph-mon[184174]: pgmap v780: 129 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 506 op/s; 37312/201510 objects degraded (18.516%); 7958/201510 objects misplaced (3.949%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:07:17.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:17 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:17.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:17 smithi055 ceph-mon[179668]: failed to encode map e321 with expected crc 2023-09-23T18:07:17.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1093005889' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:07:17.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:17 smithi055 ceph-mon[179668]: osdmap e321: 8 total, 8 up, 8 in 2023-09-23T18:07:17.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:17 smithi055 ceph-mon[179668]: pgmap v780: 129 pgs: 1 activating+undersized+degraded+remapped, 1 peering, 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 4 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 506 op/s; 37312/201510 objects degraded (18.516%); 7958/201510 objects misplaced (3.949%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:07:17.860 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:17.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 87 16 68904 68888 49.4594 52.5 0.00244933 0.0202106 2023-09-23T18:07:18.243 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:07:18.866 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:18.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:18.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:18.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 88 16 69615 69599 49.4023 44.4375 0.00620599 0.0202174 2023-09-23T18:07:18.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:18 smithi055 ceph-mon[179668]: failed to encode map e321 with expected crc 2023-09-23T18:07:18.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 37312/201510 objects degraded (18.516%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:18.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:18 smithi055 ceph-mon[184174]: failed to encode map e321 with expected crc 2023-09-23T18:07:18.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 37312/201510 objects degraded (18.516%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:18.958 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:18.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:19.011 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:18 smithi099 ceph-mon[167076]: failed to encode map e321 with expected crc 2023-09-23T18:07:19.011 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 37312/201510 objects degraded (18.516%), 19 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:19.014 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:19.108 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:19.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:19.112 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:19.213 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:19.260 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:19.674 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:19 smithi055 ceph-mon[179668]: pgmap v781: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 519 op/s; 39298/203781 objects degraded (19.284%); 7438/203781 objects misplaced (3.650%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T18:07:19.675 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:19 smithi055 ceph-mon[184174]: pgmap v781: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 519 op/s; 39298/203781 objects degraded (19.284%); 7438/203781 objects misplaced (3.650%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T18:07:19.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 89 16 70303 70287 49.3303 43 0.0126858 0.0202533 2023-09-23T18:07:20.029 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:19 smithi099 ceph-mon[167076]: pgmap v781: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 32 MiB/s wr, 519 op/s; 39298/203781 objects degraded (19.284%); 7438/203781 objects misplaced (3.650%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T18:07:20.270 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:07:20.270 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:07:20.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:07:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:07:20] "GET /metrics HTTP/1.1" 200 36667 "" "Prometheus/2.43.0" 2023-09-23T18:07:20.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:07:20.788 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:20.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:20.793 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:20.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 90 8 70960 70952 49.244 41.5625 0.187418 0.0202842 2023-09-23T18:07:20.894 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:20.943 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:21.336 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:21.767 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:21.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:21.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:21.872 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:21.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total time run: 90.1841 2023-09-23T18:07:21.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total writes made: 70960 2023-09-23T18:07:21.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Write size: 65536 2023-09-23T18:07:21.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object size: 65536 2023-09-23T18:07:21.877 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Bandwidth (MB/sec): 49.1772 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Bandwidth: 46.1045 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max bandwidth (MB/sec): 168.312 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min bandwidth (MB/sec): 0 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average IOPS: 786 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev IOPS: 737.702 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max IOPS: 2693 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min IOPS: 0 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average Latency(s): 0.020305 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Latency(s): 0.247504 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max latency(s): 19.3768 2023-09-23T18:07:21.878 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min latency(s): 0.00119503 2023-09-23T18:07:21.884 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T18:07:21.932 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:22.026 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:22.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:22.040 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:22.040 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:21 smithi055 ceph-mon[184174]: pgmap v782: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 39298/208299 objects degraded (18.866%); 7438/208299 objects misplaced (3.571%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T18:07:22.040 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:22.040 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:22.042 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:21 smithi055 ceph-mon[179668]: pgmap v782: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 39298/208299 objects degraded (18.866%); 7438/208299 objects misplaced (3.571%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T18:07:22.042 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:22.043 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:22.126 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:07:22.140 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:22.162 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:07:22.205 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:22.206 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:07:22.242 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:07:22.272 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:07:22.298 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:07:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:21 smithi099 ceph-mon[167076]: pgmap v782: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 36 MiB/s wr, 581 op/s; 39298/208299 objects degraded (18.866%); 7438/208299 objects misplaced (3.571%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T18:07:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:22.337 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:07:22.374 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:07:22.381 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:22.382 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:07:22.382 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:07:22.382 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:07:22.382 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:07:22.383 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:07:22.383 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:07:22.383 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:07:22.383 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:07:22.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:22.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:22.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:22.523 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:22.564 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:22.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:22.568 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:22.593 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:07:22.593 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:07:22.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:07:22.593 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:07:22.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.593 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122858 5.1 5:85c7b7f3:::benchmark_data_smithi055_229117_object17232:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:07:22.593 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821539+0000", 2023-09-23T18:07:22.593 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 539.76836049300005, 2023-09-23T18:07:22.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0618255270000001, 2023-09-23T18:07:22.593 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.594 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.594 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.594 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:22.594 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:22.594 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122858 2023-09-23T18:07:22.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.594 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821539+0000", 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821539+0000", 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821539+0000", 2023-09-23T18:07:22.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821541+0000", 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8899999999999999e-06 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821546+0000", 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2920000000000003e-06 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821550+0000", 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3380000000000001e-06 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821799+0000", 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00024963699999999998 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821802+0000", 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0479999999999999e-06 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.868820+0000", 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.047017925 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.868920+0000", 2023-09-23T18:07:22.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100201 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.868996+0000", 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5976999999999994e-05 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.877506+0000", 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085090640000000002 2023-09-23T18:07:22.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883289+0000", 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057830579999999998 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883316+0000", 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7611999999999999e-05 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883365+0000", 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8773999999999998e-05 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122887 5.1 5:862aed4f:::benchmark_data_smithi055_229117_object17261:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821565+0000", 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 539.76833501600004, 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0619787039999999, 2023-09-23T18:07:22.601 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.602 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.602 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.602 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:22.602 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:22.602 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122887 2023-09-23T18:07:22.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.602 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.602 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.602 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821565+0000", 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821565+0000", 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821565+0000", 2023-09-23T18:07:22.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821567+0000", 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3e-06 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821573+0000", 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6910000000000002e-06 2023-09-23T18:07:22.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821576+0000", 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7440000000000001e-06 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821819+0000", 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000243157 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821822+0000", 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5100000000000001e-06 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.871475+0000", 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0496534580000001 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.871545+0000", 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9819000000000004e-05 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.871625+0000", 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9864999999999995e-05 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.877513+0000", 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058876070000000004 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883402+0000", 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058895689999999999 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883456+0000", 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3618000000000001e-05 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.883544+0000", 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7720999999999996e-05 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:122893 5.1 5:859301a2:::benchmark_data_smithi055_229117_object17267:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e156)", 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:58:22.821588+0000", 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 539.76831176200005, 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.072527035, 2023-09-23T18:07:22.609 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122893 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821588+0000", 2023-09-23T18:07:22.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821588+0000", 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821588+0000", 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:22.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821590+0000", 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0229999999999999e-06 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821596+0000", 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7570000000000001e-06 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821599+0000", 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3179999999999998e-06 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821833+0000", 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000234379 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:22.821836+0000", 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.497e-06 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.873642+0000", 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0518064739999999 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.873706+0000", 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3875999999999996e-05 2023-09-23T18:07:22.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0", 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.873789+0000", 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2707000000000006e-05 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.882910+0000", 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0091208369999999997 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.893965+0000", 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011055684 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.894019+0000", 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3720000000000001e-05 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:58:23.894115+0000", 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6105000000000002e-05 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154240 5.e 5:7f5ef404:::benchmark_data_smithi055_229117_object48614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.838192+0000", 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 493.751707684, 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2567066049999998, 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.617 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154240 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838192+0000", 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838192+0000", 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838191+0000", 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838199+0000", 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1009999999999992e-06 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838217+0000", 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8629000000000001e-05 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838225+0000", 2023-09-23T18:07:22.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5510000000000002e-06 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838294+0000", 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9330000000000002e-05 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838426+0000", 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013151499999999999 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838546+0000", 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120331 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.843064+0000", 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004517928 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.845440+0000", 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00237587 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.862351+0000", 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016911302 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.862359+0000", 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6399999999999997e-06 2023-09-23T18:07:22.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.850984+0000", 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98862541999999998 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.850999+0000", 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4683e-05 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.094798+0000", 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.243799069 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.094899+0000", 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100995 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154257 5.e 5:7493d31c:::benchmark_data_smithi055_229117_object48631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.879851+0000", 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 493.710048911, 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2859593629999999, 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154257 2023-09-23T18:07:22.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879851+0000", 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879851+0000", 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879850+0000", 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879855+0000", 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5829999999999997e-06 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879867+0000", 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1579e-05 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879872+0000", 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8509999999999998e-06 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879916+0000", 2023-09-23T18:07:22.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4608999999999999e-05 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879924+0000", 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8450000000000003e-06 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851029+0000", 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97110422200000002 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851036+0000", 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3780000000000003e-06 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.094924+0000", 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2438876830000001 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.095069+0000", 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014521 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.095710+0000", 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00064142799999999999 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114512+0000", 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018801345000000001 2023-09-23T18:07:22.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114543+0000", 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1266000000000002e-05 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114629+0000", 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6620999999999996e-05 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165607+0000", 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.050977816000000002 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165673+0000", 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5704000000000007e-05 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165810+0000", 2023-09-23T18:07:22.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013739199999999999 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154261 5.e 5:75cfb822:::benchmark_data_smithi055_229117_object48635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.881204+0000", 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 493.70869613000002, 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2848199720000002, 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.635 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154261 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881204+0000", 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881204+0000", 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881203+0000", 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881208+0000", 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1699999999999996e-06 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881221+0000", 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2809000000000001e-05 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881227+0000", 2023-09-23T18:07:22.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7799999999999997e-06 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881298+0000", 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1403999999999996e-05 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881305+0000", 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7789999999999998e-06 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851061+0000", 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96975605099999995 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851068+0000", 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1060000000000002e-06 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.103490+0000", 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2524223880000001 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.103624+0000", 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013401600000000001 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.103729+0000", 2023-09-23T18:07:22.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010425900000000001 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114577+0000", 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010848083999999999 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114604+0000", 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6886999999999999e-05 2023-09-23T18:07:22.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114655+0000", 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1703000000000003e-05 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165828+0000", 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.051172544 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165882+0000", 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4376000000000003e-05 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166024+0000", 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014146100000000001 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154266 5.e 5:70dac101:::benchmark_data_smithi055_229117_object48640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.888147+0000", 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 493.70175310500002, 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.278059764, 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:22.645 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154266 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888147+0000", 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888147+0000", 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888146+0000", 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888152+0000", 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4119999999999998e-06 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888162+0000", 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.679e-06 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888167+0000", 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.775e-06 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888242+0000", 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5483999999999998e-05 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888250+0000", 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.926e-06 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852162+0000", 2023-09-23T18:07:22.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.963912133 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852175+0000", 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2492e-05 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.107232+0000", 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2550570539999999 2023-09-23T18:07:22.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.107391+0000", 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000158949 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.107551+0000", 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0001605 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114820+0000", 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007268666 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.115701+0000", 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00088042600000000002 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.125929+0000", 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010228376000000001 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166033+0000", 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.040103685 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166081+0000", 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8899000000000001e-05 2023-09-23T18:07:22.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166207+0000", 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012513100000000001 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154238 5.f 5:fb40edf7:::benchmark_data_smithi055_229117_object48612:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:07:22.654 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:09.852458+0000", 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 492.73744216199998, 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3442088729999999, 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154238 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852458+0000", 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852458+0000", 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852456+0000", 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852463+0000", 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7150000000000001e-06 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852473+0000", 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0584000000000001e-05 2023-09-23T18:07:22.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852481+0000", 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0229999999999992e-06 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852600+0000", 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119177 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852610+0000", 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3980000000000005e-06 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.145648+0000", 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.29303853 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.145845+0000", 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019688400000000001 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.196602+0000", 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.050756905999999997 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.196667+0000", 2023-09-23T18:07:22.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4517999999999995e-05 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154243 5.f 5:f7c86625:::benchmark_data_smithi055_229117_object48617:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:09.852496+0000", 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 492.73740395599998, 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.360016345, 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.661 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154243 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852496+0000", 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852496+0000", 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852495+0000", 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852499+0000", 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5109999999999999e-06 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852506+0000", 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1690000000000003e-06 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852511+0000", 2023-09-23T18:07:22.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.464e-06 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852635+0000", 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123979 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852642+0000", 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0350000000000004e-06 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.197777+0000", 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.345134979 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.197894+0000", 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116935 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.212448+0000", 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014554414999999999 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.212512+0000", 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4366e-05 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33530 6.6 6:63a1868b:::benchmark_data_smithi055_275963_object33529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:39.452220+0000", 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 343.13767983999998, 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0750561489999999, 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33530 2023-09-23T18:07:22.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452220+0000", 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452220+0000", 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452219+0000", 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452264+0000", 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4159999999999997e-05 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.670 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452272+0000", 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4780000000000008e-06 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452277+0000", 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3499999999999996e-06 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453091+0000", 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00081378099999999997 2023-09-23T18:07:22.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453100+0000", 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0420000000000008e-06 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.451792+0000", 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99869191199999996 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.453176+0000", 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001383965 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.453351+0000", 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017501100000000001 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460176+0000", 2023-09-23T18:07:22.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0068253760000000002 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460632+0000", 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00045508400000000002 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527098+0000", 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.066466833000000003 2023-09-23T18:07:22.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527162+0000", 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3319999999999997e-05 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527276+0000", 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000114576 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.675 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33541 6.6 6:62c48c76:::benchmark_data_smithi055_275963_object33540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 343.13761039899998, 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0751333839999999, 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.676 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33541 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.677 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452289+0000", 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452325+0000", 2023-09-23T18:07:22.678 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5848e-05 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452332+0000", 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5870000000000004e-06 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452337+0000", 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8189999999999999e-06 2023-09-23T18:07:22.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453126+0000", 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00078863000000000002 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453133+0000", 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5630000000000003e-06 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454584+0000", 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0014505659999999 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454710+0000", 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012631 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:07:22.681 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454859+0000", 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014862400000000001 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460661+0000", 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058021690000000002 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.461337+0000", 2023-09-23T18:07:22.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00067573600000000002 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527285+0000", 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.065948602999999995 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527333+0000", 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7329e-05 2023-09-23T18:07:22.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527423+0000", 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0160000000000004e-05 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:99011 6.e 6:75351f29:::benchmark_data_smithi055_275963_object25873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:39.675417+0000", 2023-09-23T18:07:22.684 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 222.91448256999999, 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1206789589999999, 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 99011 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.685 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675417+0000", 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675417+0000", 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675416+0000", 2023-09-23T18:07:22.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:07:22.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.687 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675423+0000", 2023-09-23T18:07:22.687 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9330000000000004e-06 2023-09-23T18:07:22.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.687 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675436+0000", 2023-09-23T18:07:22.687 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2831e-05 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675441+0000", 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2240000000000003e-06 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.746546+0000", 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.071104629000000003 2023-09-23T18:07:22.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.750257+0000", 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037111919999999999 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.793006+0000", 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.042748844000000001 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for rw locks", 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.797725+0000", 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0047187669999999996 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.741846+0000", 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94412087 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.745207+0000", 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003361298 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,5", 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.751189+0000", 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0059818609999999998 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.768299+0000", 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017110071000000001 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.785459+0000", 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017160150999999998 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.786644+0000", 2023-09-23T18:07:22.692 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011847050000000001 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.788542+0000", 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018984799999999999 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.796096+0000", 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075542559999999996 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.693 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:99014 6.e 6:74e9d389:::benchmark_data_smithi055_275963_object25876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:39.742871+0000", 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 222.8470294, 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.066436916, 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.694 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 99014 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.742871+0000", 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.695 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.742871+0000", 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.742869+0000", 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.742875+0000", 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2890000000000003e-06 2023-09-23T18:07:22.696 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.742890+0000", 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4304000000000001e-05 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.742897+0000", 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1940000000000001e-06 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.697 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.772712+0000", 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.029814943999999999 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.776459+0000", 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037466639999999998 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.698 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.813562+0000", 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037103417 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for rw locks", 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.818248+0000", 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004685893 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.768865+0000", 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95061722900000001 2023-09-23T18:07:22.699 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.772477+0000", 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003611852 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,5", 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.778887+0000", 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064096530000000004 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.700 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.796493+0000", 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017606620999999999 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.797656+0000", 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011624319999999999 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.701 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.798853+0000", 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001197201 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.801003+0000", 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021504499999999999 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.809307+0000", 2023-09-23T18:07:22.702 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0083042040000000008 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65460 7.4 7:2d53859f:::benchmark_data_smithi055_311528_object65459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 36.693494573999999, 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9613000379999992, 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:07:22.703 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65460 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.704 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896404+0000", 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896620+0000", 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021651099999999999 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896638+0000", 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8179999999999999e-05 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896646+0000", 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2899999999999997e-06 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896718+0000", 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2587000000000001e-05 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896729+0000", 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1223000000000001e-05 2023-09-23T18:07:22.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.853134+0000", 2023-09-23T18:07:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9564047319999993 2023-09-23T18:07:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.857705+0000", 2023-09-23T18:07:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045713259999999997 2023-09-23T18:07:22.708 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65469 7.4 7:2eb46e41:::benchmark_data_smithi055_311528_object65468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 36.693232922999997, 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 10.043735116000001, 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.709 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65469 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.710 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896666+0000", 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896742+0000", 2023-09-23T18:07:22.711 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5432000000000006e-05 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896755+0000", 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3275000000000001e-05 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896762+0000", 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4760000000000003e-06 2023-09-23T18:07:22.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896860+0000", 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8120000000000002e-05 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896869+0000", 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1409999999999994e-06 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.713 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.714 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.858317+0000", 2023-09-23T18:07:22.714 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9614477719999996 2023-09-23T18:07:22.714 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.714 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.714 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:07:22.714 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.863986+0000", 2023-09-23T18:07:22.714 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0056689319999999998 2023-09-23T18:07:22.714 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.714 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.714 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940290+0000", 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.076304636999999995 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940402+0000", 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111987 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.715 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.716 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65477 7.4 7:23aaada8:::benchmark_data_smithi055_311528_object65476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:07:22.716 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:07:22.716 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 36.693125002000002, 2023-09-23T18:07:22.716 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 10.046314513, 2023-09-23T18:07:22.716 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.716 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.716 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.716 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:22.716 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:22.716 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65477 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.717 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896774+0000", 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896805+0000", 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0916e-05 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.718 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896816+0000", 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0519999999999999e-05 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896820+0000", 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0849999999999999e-06 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.719 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896888+0000", 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8637e-05 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896897+0000", 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6279999999999994e-06 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.864614+0000", 2023-09-23T18:07:22.720 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9677164650000005 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.871623+0000", 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070090059999999999 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.882195+0000", 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010572560999999999 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.721 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.928966+0000", 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.046770763999999999 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940462+0000", 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011496312 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.722 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940512+0000", 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0118999999999999e-05 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940552+0000", 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9535e-05 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943089+0000", 2023-09-23T18:07:22.723 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025376470000000001 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65485 7.4 7:246beb72:::benchmark_data_smithi055_311528_object65484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 35.493733794999997, 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8502435909999999, 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.724 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65485 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096165+0000", 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096281+0000", 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116042 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096295+0000", 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4524e-05 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096301+0000", 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7490000000000004e-06 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.889363+0000", 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7930615480000007 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.896004+0000", 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066411439999999999 2023-09-23T18:07:22.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.906662+0000", 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010657645 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940400+0000", 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.033738490000000003 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943179+0000", 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002778675 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943222+0000", 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2966999999999997e-05 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943292+0000", 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0043999999999995e-05 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946410+0000", 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031180190000000001 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.731 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65486 7.4 7:28b4ee06:::benchmark_data_smithi055_311528_object65485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 35.468483315999997, 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8402319929999997, 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65486 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121415+0000", 2023-09-23T18:07:22.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:07:22.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121466+0000", 2023-09-23T18:07:22.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1007000000000001e-05 2023-09-23T18:07:22.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121481+0000", 2023-09-23T18:07:22.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5287000000000001e-05 2023-09-23T18:07:22.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121489+0000", 2023-09-23T18:07:22.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5010000000000004e-06 2023-09-23T18:07:22.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946571+0000", 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8250813830000006 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946747+0000", 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017653100000000001 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:07:22.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946947+0000", 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019961800000000001 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.954051+0000", 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0071042989999999997 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958525+0000", 2023-09-23T18:07:22.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044738959999999998 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958638+0000", 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113078 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958698+0000", 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0207000000000001e-05 2023-09-23T18:07:22.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961649+0000", 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002950214 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65487 7.4 7:2b3c8c2f:::benchmark_data_smithi055_311528_object65486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:07:22.739 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 35.468399249999997, 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8433001999999998, 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65487 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121500+0000", 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:07:22.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121563+0000", 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3126000000000005e-05 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121579+0000", 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6418000000000001e-05 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121585+0000", 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.575e-06 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949470+0000", 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8278846659999992 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949632+0000", 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016232099999999999 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949768+0000", 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013578700000000001 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.956292+0000", 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065241910000000004 2023-09-23T18:07:22.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958569+0000", 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022767289999999999 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961776+0000", 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032076660000000001 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961847+0000", 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1137999999999996e-05 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964801+0000", 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029533990000000002 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.746 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65498 7.4 7:27d4ee95:::benchmark_data_smithi055_311528_object65497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 35.433889735999998, 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8116268140000003, 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:22.747 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65498 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:07:22.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156009+0000", 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156082+0000", 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3093e-05 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156096+0000", 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4465e-05 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156104+0000", 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4120000000000004e-06 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.951850+0000", 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7957462389999996 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.951998+0000", 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014826300000000001 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:07:22.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.952148+0000", 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000149524 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961673+0000", 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095253839999999996 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961735+0000", 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2036e-05 2023-09-23T18:07:22.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964887+0000", 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003152092 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964960+0000", 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2601999999999994e-05 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:22.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:22.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.967637+0000", 2023-09-23T18:07:22.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026772739999999999 2023-09-23T18:07:22.754 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.754 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.754 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.754 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:22.754 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:22.754 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:07:22.755 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:22.797 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:22.831 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:07:22.884 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.4 to 0.978669 (fa8a) 2023-09-23T18:07:23.002 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:07:23.046 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:23.048 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2461553533' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:23.049 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1066200635' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.9786694738186201}]: dispatch 2023-09-23T18:07:23.049 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2461553533' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:23.049 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1066200635' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.9786694738186201}]: dispatch 2023-09-23T18:07:23.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:23.050 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:23.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:23.177 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2461553533' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:23.177 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1066200635' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 0.9786694738186201}]: dispatch 2023-09-23T18:07:23.180 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:23.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:23.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:23.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:23.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:23.364 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:23.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:23.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:23.412 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:23.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:23.544 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:23.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:23.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:23.605 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:23.706 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:23.739 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:23.756 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:23.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:23.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:23.795 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:23.881 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[184174]: pgmap v783: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s; 39264/211839 objects degraded (18.535%); 7022/211839 objects misplaced (3.315%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:07:23.881 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[184174]: failed to encode map e322 with expected crc 2023-09-23T18:07:23.881 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1066200635' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.9786694738186201}]': finished 2023-09-23T18:07:23.881 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[184174]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T18:07:23.881 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[184174]: failed to encode map e322 with expected crc 2023-09-23T18:07:23.881 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 39264/211839 objects degraded (18.535%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:23.881 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[179668]: pgmap v783: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s; 39264/211839 objects degraded (18.535%); 7022/211839 objects misplaced (3.315%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:07:23.881 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[179668]: failed to encode map e322 with expected crc 2023-09-23T18:07:23.882 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1066200635' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.9786694738186201}]': finished 2023-09-23T18:07:23.882 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[179668]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T18:07:23.882 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[179668]: failed to encode map e322 with expected crc 2023-09-23T18:07:23.882 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 39264/211839 objects degraded (18.535%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:23.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:23.927 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:23.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:23.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:23.979 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:23 smithi099 ceph-mon[167076]: pgmap v783: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 40 MiB/s wr, 647 op/s; 39264/211839 objects degraded (18.535%); 7022/211839 objects misplaced (3.315%); 4.5 MiB/s, 72 objects/s recovering 2023-09-23T18:07:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:23 smithi099 ceph-mon[167076]: failed to encode map e322 with expected crc 2023-09-23T18:07:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:23 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1066200635' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 0.9786694738186201}]': finished 2023-09-23T18:07:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:23 smithi099 ceph-mon[167076]: osdmap e322: 8 total, 8 up, 8 in 2023-09-23T18:07:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:23 smithi099 ceph-mon[167076]: failed to encode map e322 with expected crc 2023-09-23T18:07:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 39264/211839 objects degraded (18.535%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:24.080 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:24.119 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:25.018 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45557.0:72125 7.f e322/303 7:f4bc4207:::benchmark_data_smithi055_311528_object1162:head v 322'4431, mlcod=283'622)", 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:07:25.017684+0000", 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.00018436, 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000208748, 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:25.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:07:25.017684+0000", 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:07:25.017684+0000", 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:07:25.017684+0000", 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:25.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:07:25.017689+0000", 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6280000000000002e-06 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:07:25.017699+0000", 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4909999999999994e-06 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:25.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:07:25.017703+0000", 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1080000000000003e-06 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:07:25.017739+0000", 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5960999999999997e-05 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:07:25.017767+0000", 2023-09-23T18:07:25.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8466000000000002e-05 2023-09-23T18:07:25.023 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:25.023 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:25.023 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:25.023 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:25.023 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:07:25.023 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 1 2023-09-23T18:07:25.023 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:07:25.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:07:26.042 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:26.130 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:26.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:25 smithi055 ceph-mon[179668]: pgmap v785: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 29 MiB/s wr, 483 op/s; 39237/212160 objects degraded (18.494%); 7022/212160 objects misplaced (3.310%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T18:07:26.134 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:25 smithi055 ceph-mon[184174]: pgmap v785: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 29 MiB/s wr, 483 op/s; 39237/212160 objects degraded (18.494%); 7022/212160 objects misplaced (3.310%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T18:07:26.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:26.135 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:26.236 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:26.246 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:25 smithi099 ceph-mon[167076]: pgmap v785: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 29 MiB/s wr, 483 op/s; 39237/212160 objects degraded (18.494%); 7022/212160 objects misplaced (3.310%); 4.0 MiB/s, 64 objects/s recovering 2023-09-23T18:07:26.293 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:26.340 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:26.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:26.345 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:26.446 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:26.491 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:26.587 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:26.589 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:26.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:26.691 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:26.726 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:26.781 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:26.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:26.788 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:26.865 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:26.889 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:26.923 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:26.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:26.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:26.979 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:27.080 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:27.120 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:27.285 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:07:27.286 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:07:27.286 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:07:27.286 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:07:27.286 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:07:27.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:07:27.743 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:27.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:27.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:27.848 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:27.887 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:27.941 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:27.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:27.943 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:28.044 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:28.077 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:28.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:28.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:28.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:27 smithi055 ceph-mon[179668]: pgmap v786: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 450 op/s; 38970/210906 objects degraded (18.477%); 7022/210906 objects misplaced (3.329%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:07:28.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:27 smithi055 ceph-mon[184174]: pgmap v786: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 450 op/s; 38970/210906 objects degraded (18.477%); 7022/210906 objects misplaced (3.329%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:07:28.219 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:28.255 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:28.320 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:28.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:27 smithi099 ceph-mon[167076]: pgmap v786: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 24 MiB/s wr, 450 op/s; 38970/210906 objects degraded (18.477%); 7022/210906 objects misplaced (3.329%); 4.4 MiB/s, 70 objects/s recovering 2023-09-23T18:07:28.338 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:07:28.504 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:28.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:28.512 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:28.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:28.648 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:28.893 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 38970/210906 objects degraded (18.477%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:28.894 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 38970/210906 objects degraded (18.477%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:29.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:29.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:29.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:29.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:29.175 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 38970/210906 objects degraded (18.477%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:29.184 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:29.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:29.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:29.265 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:29.366 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:29.404 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:29.587 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:07:29.587 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:07:29.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154238 5.f 5:fb40edf7:::benchmark_data_smithi055_229117_object48612:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.836753+0000", 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 500.747869713, 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0170112849999999, 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:29.588 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154238 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836753+0000", 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836753+0000", 2023-09-23T18:07:29.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836752+0000", 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836758+0000", 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2560000000000002e-06 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836767+0000", 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5599999999999999e-06 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.836773+0000", 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3879999999999999e-06 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.591 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.840698+0000", 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039256660000000004 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.840860+0000", 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016150800000000001 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.840964+0000", 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010398700000000001 2023-09-23T18:07:29.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.850606+0000", 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096418819999999992 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.860704+0000", 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010098527 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.593 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.870852+0000", 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010147353 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.870862+0000", 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0497e-05 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.853747+0000", 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98288461199999999 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.853765+0000", 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7954000000000001e-05 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154243 5.f 5:f7c86625:::benchmark_data_smithi055_229117_object48617:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.839762+0000", 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 500.744860776, 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0143057070000001, 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154243 2023-09-23T18:07:29.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839762+0000", 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839762+0000", 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839761+0000", 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839770+0000", 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1679999999999993e-06 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839785+0000", 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5633000000000001e-05 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9209999999999992e-06 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.845356+0000", 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055628429999999996 2023-09-23T18:07:29.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.845525+0000", 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000168727 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.845688+0000", 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016269799999999999 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.854335+0000", 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0086471139999999991 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.860735+0000", 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064005720000000002 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.872057+0000", 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011321743 2023-09-23T18:07:29.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.872062+0000", 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3319999999999999e-06 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.854056+0000", 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98199359500000005 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.602 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.854068+0000", 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2062000000000001e-05 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33512 6.7 6:ee5c747c:::benchmark_data_smithi055_275963_object33511:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.421493+0000", 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 351.16312974900001, 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0385178110000002, 2023-09-23T18:07:29.603 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33512 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421493+0000", 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421493+0000", 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421492+0000", 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421529+0000", 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6989000000000001e-05 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421537+0000", 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0630000000000006e-06 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421542+0000", 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1669999999999998e-06 2023-09-23T18:07:29.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421665+0000", 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012245100000000001 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421800+0000", 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000135702 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.421965+0000", 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016417600000000001 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.425828+0000", 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0038633389999999999 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.426124+0000", 2023-09-23T18:07:29.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000295581 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.431471+0000", 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0053474300000000002 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.431486+0000", 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5180999999999999e-05 2023-09-23T18:07:29.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446442+0000", 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.014955775 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446456+0000", 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4290999999999999e-05 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.459918+0000", 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.013462211 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460011+0000", 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2694000000000004e-05 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33515 6.7 6:e3dc8fb1:::benchmark_data_smithi055_275963_object33514:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.433835+0000", 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 351.15078829800001, 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0345420270000001, 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33515 2023-09-23T18:07:29.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433835+0000", 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433835+0000", 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433834+0000", 2023-09-23T18:07:29.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:29.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433880+0000", 2023-09-23T18:07:29.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5352000000000002e-05 2023-09-23T18:07:29.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433891+0000", 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1508999999999999e-05 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.433897+0000", 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0730000000000001e-06 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441402+0000", 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075050969999999996 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441413+0000", 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0978999999999999e-05 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446745+0000", 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.005331515 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446751+0000", 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460467+0000", 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0137163 2023-09-23T18:07:29.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460602+0000", 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013485099999999999 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460760+0000", 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015809999999999999 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.466501+0000", 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057416919999999996 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.466610+0000", 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108761 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.468215+0000", 2023-09-23T18:07:29.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016042910000000001 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.468276+0000", 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1511000000000001e-05 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.468377+0000", 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100777 2023-09-23T18:07:29.620 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33517 6.7 6:e39e0867:::benchmark_data_smithi055_275963_object33516:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.439207+0000", 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 351.14541584900002, 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0344513580000001, 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.621 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33517 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439207+0000", 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439207+0000", 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439206+0000", 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439260+0000", 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3681000000000003e-05 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439273+0000", 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3354e-05 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439281+0000", 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5220000000000001e-06 2023-09-23T18:07:29.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441439+0000", 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021585250000000001 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441447+0000", 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7719999999999994e-06 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446765+0000", 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0053182409999999 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446771+0000", 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4070000000000001e-06 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.626 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.462292+0000", 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.015521568 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.462420+0000", 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000128157 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.462587+0000", 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016689499999999999 2023-09-23T18:07:29.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.466615+0000", 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040280089999999999 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.468798+0000", 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021825740000000001 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473483+0000", 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046850040000000004 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473546+0000", 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2974999999999999e-05 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473659+0000", 2023-09-23T18:07:29.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011288100000000001 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33520 6.7 6:e109f2bc:::benchmark_data_smithi055_275963_object33519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.439287+0000", 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 351.14533622599998, 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0346285100000001, 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.630 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33520 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439287+0000", 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439287+0000", 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439286+0000", 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439337+0000", 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0813999999999998e-05 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439346+0000", 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4239999999999993e-06 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439351+0000", 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3070000000000002e-06 2023-09-23T18:07:29.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441471+0000", 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021196570000000001 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441478+0000", 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6179999999999997e-06 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446784+0000", 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.005306155 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446792+0000", 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9149999999999996e-06 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.463643+0000", 2023-09-23T18:07:29.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.016850939 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.463765+0000", 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121725 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.463946+0000", 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018062199999999999 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.466621+0000", 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002675279 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.469013+0000", 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023926289999999998 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473734+0000", 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004720709 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473798+0000", 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3841000000000001e-05 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473915+0000", 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011738700000000001 2023-09-23T18:07:29.638 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33528 6.7 6:e730599b:::benchmark_data_smithi055_275963_object33527:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.439423+0000", 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 351.14519971099998, 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0346796779999998, 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.639 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33528 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439423+0000", 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439423+0000", 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439423+0000", 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439455+0000", 2023-09-23T18:07:29.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2558999999999998e-05 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439472+0000", 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6099999999999998e-05 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.439477+0000", 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1850000000000003e-06 2023-09-23T18:07:29.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441681+0000", 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022039009999999999 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441688+0000", 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0369999999999997e-06 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446808+0000", 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.005120805 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.446813+0000", 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5499999999999996e-06 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.465059+0000", 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0182463390000001 2023-09-23T18:07:29.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.465197+0000", 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013815600000000001 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.465330+0000", 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013268399999999999 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.466929+0000", 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001599147 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.469212+0000", 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022831230000000002 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.473963+0000", 2023-09-23T18:07:29.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004750354 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.474011+0000", 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8315000000000001e-05 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.474103+0000", 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1954999999999996e-05 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.647 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:95849 6.d 6:b0e99437:::benchmark_data_smithi055_275963_object22711:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.214518+0000", 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 241.37010463999999, 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.14248433399999999, 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:29.648 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95849 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214518+0000", 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214518+0000", 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214518+0000", 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214522+0000", 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7520000000000004e-06 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214531+0000", 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1570000000000006e-06 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214536+0000", 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0289999999999996e-06 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214593+0000", 2023-09-23T18:07:29.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7024000000000002e-05 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214712+0000", 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011810400000000001 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.214819+0000", 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000107798 2023-09-23T18:07:29.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.217987+0000", 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003167146 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.218620+0000", 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063337599999999995 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356918+0000", 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.138298006 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356961+0000", 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3210000000000001e-05 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.357003+0000", 2023-09-23T18:07:29.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1669999999999999e-05 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:95851 6.c 6:36e0c06c:::benchmark_data_smithi055_275963_object22713:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.220710+0000", 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 241.363912653, 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13599963000000001, 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.655 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95851 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220710+0000", 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220710+0000", 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220710+0000", 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220714+0000", 2023-09-23T18:07:29.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0040000000000001e-06 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220722+0000", 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0499999999999992e-06 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220726+0000", 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2169999999999996e-06 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220770+0000", 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3989000000000002e-05 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220853+0000", 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2904000000000004e-05 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,6", 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.220924+0000", 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1114999999999999e-05 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223643+0000", 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027192319999999998 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223709+0000", 2023-09-23T18:07:29.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5788000000000001e-05 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.223737+0000", 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7517000000000001e-05 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356586+0000", 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13284983 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.661 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356710+0000", 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012371100000000001 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:95854 6.7 6:e5102bf3:::benchmark_data_smithi055_275963_object22716:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e258)", 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.224604+0000", 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 241.36001925299999, 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.146692287, 2023-09-23T18:07:29.662 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 95854 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.224604+0000", 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.224604+0000", 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.224603+0000", 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.224608+0000", 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9860000000000002e-06 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356545+0000", 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13193690799999999 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356567+0000", 2023-09-23T18:07:29.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1766000000000001e-05 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356751+0000", 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018422499999999999 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356910+0000", 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015871900000000001 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.357061+0000", 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015133 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.364897+0000", 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0078364799999999998 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.365385+0000", 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00048802399999999999 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.368392+0000", 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003006604 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.368446+0000", 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4376000000000003e-05 2023-09-23T18:07:29.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.371296+0000", 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002849655 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45092.0:95762 6.e e258/227 6:7a26cd90:::benchmark_data_smithi055_275963_object22624:head v 258'6107, mlcod=217'1539)", 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:28.233845+0000", 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 241.35077834500001, 2023-09-23T18:07:29.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13611863699999999, 2023-09-23T18:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.233845+0000", 2023-09-23T18:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.670 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.233845+0000", 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.233844+0000", 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.233850+0000", 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0959999999999997e-06 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356591+0000", 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.122740471 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356598+0000", 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4599999999999997e-06 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356682+0000", 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4319000000000004e-05 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.356712+0000", 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9689e-05 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.369927+0000", 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013214681000000001 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:28.369963+0000", 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.659e-05 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.674 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:29530 7.0 7:0e6f1cbd:::benchmark_data_smithi055_311528_object29529:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:04.366682+0000", 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 85.217940670000004, 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2685781020000002, 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29530 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.675 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366682+0000", 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366682+0000", 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.676 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366681+0000", 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366710+0000", 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8829000000000001e-05 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366722+0000", 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1983e-05 2023-09-23T18:07:29.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366730+0000", 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1960000000000008e-06 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366787+0000", 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7119e-05 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.678 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366798+0000", 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1513e-05 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356024+0000", 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98922533499999998 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356038+0000", 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4138000000000001e-05 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.626480+0000", 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.270442158 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.626641+0000", 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000160686 2023-09-23T18:07:29.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.626815+0000", 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000174122 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.632597+0000", 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057818310000000003 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.681 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.633733+0000", 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001136733 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.633800+0000", 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6983999999999994e-05 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.682 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.633864+0000", 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3874000000000005e-05 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.635261+0000", 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013961850000000001 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.683 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:29531 7.0 7:0426d0e9:::benchmark_data_smithi055_311528_object29530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:04.366741+0000", 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 85.217882458000005, 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2742705650000001, 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29531 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.684 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366741+0000", 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366741+0000", 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.685 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366740+0000", 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366787+0000", 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7206999999999997e-05 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366795+0000", 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.61e-06 2023-09-23T18:07:29.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366800+0000", 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7369999999999997e-06 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366848+0000", 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8473999999999997e-05 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.366858+0000", 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0308e-05 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.357399+0000", 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99054061400000004 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.357409+0000", 2023-09-23T18:07:29.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0053e-05 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.629618+0000", 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.272208827 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.629790+0000", 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017243400000000001 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.629953+0000", 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016290599999999999 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.633690+0000", 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037368430000000001 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.639004+0000", 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0053140000000000001 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.639864+0000", 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00085939000000000002 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.639916+0000", 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2942999999999998e-05 2023-09-23T18:07:29.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.641011+0000", 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010947140000000001 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:29538 7.0 7:06c48aad:::benchmark_data_smithi055_311528_object29537:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:04.373735+0000", 2023-09-23T18:07:29.692 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 85.210887870999997, 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2687814780000002, 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29538 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.693 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.373735+0000", 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.373735+0000", 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.373728+0000", 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999924 2023-09-23T18:07:29.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.373792+0000", 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3825999999999995e-05 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.373809+0000", 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7323999999999999e-05 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.695 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.373818+0000", 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5810000000000003e-06 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.373877+0000", 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8386e-05 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.696 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.373884+0000", 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8179999999999998e-06 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.357434+0000", 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.983550064 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.357440+0000", 2023-09-23T18:07:29.697 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7599999999999997e-06 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.631160+0000", 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2737196690000001 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.631289+0000", 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012912500000000001 2023-09-23T18:07:29.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.631459+0000", 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016959599999999999 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.633741+0000", 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022825520000000002 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.699 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.700 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.641256+0000", 2023-09-23T18:07:29.700 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075140739999999999 2023-09-23T18:07:29.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.700 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.700 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.700 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.641520+0000", 2023-09-23T18:07:29.700 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026432499999999999 2023-09-23T18:07:29.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.700 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.700 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.701 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.641567+0000", 2023-09-23T18:07:29.701 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7559999999999999e-05 2023-09-23T18:07:29.701 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.701 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.701 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.701 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.642517+0000", 2023-09-23T18:07:29.701 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00094930899999999996 2023-09-23T18:07:29.701 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.701 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.701 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:29548 7.0 7:04ea1c25:::benchmark_data_smithi055_311528_object29547:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:04.381280+0000", 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 85.203342829999997, 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2684034130000001, 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:29.702 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29548 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381280+0000", 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381280+0000", 2023-09-23T18:07:29.703 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381279+0000", 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381349+0000", 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0049000000000006e-05 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.704 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381368+0000", 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9510000000000001e-05 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381375+0000", 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.516e-06 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381427+0000", 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1659999999999997e-05 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381435+0000", 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0360000000000006e-06 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.357461+0000", 2023-09-23T18:07:29.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97602654899999997 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.357468+0000", 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5579999999999997e-06 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.632627+0000", 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.275159524 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.707 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.632754+0000", 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012710999999999999 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.632862+0000", 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108045 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.637741+0000", 2023-09-23T18:07:29.708 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048785510000000001 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.641333+0000", 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0035922660000000002 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.648158+0000", 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0068249809999999999 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.709 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.710 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.710 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.648238+0000", 2023-09-23T18:07:29.710 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9402999999999999e-05 2023-09-23T18:07:29.710 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.710 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.710 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.710 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.649684+0000", 2023-09-23T18:07:29.710 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001446105 2023-09-23T18:07:29.710 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.710 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.711 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.711 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.711 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.711 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:29549 7.0 7:061624d5:::benchmark_data_smithi055_311528_object29548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:07:29.711 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:04.381464+0000", 2023-09-23T18:07:29.711 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 85.203158838999997, 2023-09-23T18:07:29.711 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2690686179999999, 2023-09-23T18:07:29.711 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.711 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.711 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29549 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381464+0000", 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.712 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381464+0000", 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381464+0000", 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.713 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381535+0000", 2023-09-23T18:07:29.714 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1785999999999993e-05 2023-09-23T18:07:29.714 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.714 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.714 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.715 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381549+0000", 2023-09-23T18:07:29.715 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3134e-05 2023-09-23T18:07:29.715 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.715 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.715 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.715 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381561+0000", 2023-09-23T18:07:29.715 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2763e-05 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381622+0000", 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0486999999999999e-05 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.381632+0000", 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6479999999999998e-06 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.716 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.357489+0000", 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97585741400000003 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.357495+0000", 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3929999999999996e-06 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.717 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.635302+0000", 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2778069409999999 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.635434+0000", 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013183300000000001 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.635536+0000", 2023-09-23T18:07:29.718 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000101888 2023-09-23T18:07:29.719 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.719 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.719 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.719 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.637746+0000", 2023-09-23T18:07:29.719 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002210376 2023-09-23T18:07:29.719 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.719 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.719 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.719 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.646776+0000", 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0090297229999999999 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.649725+0000", 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029488769999999999 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.649784+0000", 2023-09-23T18:07:29.720 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8754e-05 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.650533+0000", 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000749159 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:29550 7.0 7:0a41286c:::benchmark_data_smithi055_311528_object29549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:07:29.721 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:04.382276+0000", 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 85.202347153000005, 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2692957370000002, 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29550 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.722 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.382276+0000", 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.382276+0000", 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.382275+0000", 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.382355+0000", 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9249e-05 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.382367+0000", 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2716e-05 2023-09-23T18:07:29.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.382372+0000", 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.651e-06 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.382410+0000", 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8501999999999998e-05 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.725 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:04.382418+0000", 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.537e-06 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.357517+0000", 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97509880000000004 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.357523+0000", 2023-09-23T18:07:29.726 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6100000000000002e-06 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.636588+0000", 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.279064411 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.636677+0000", 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8906000000000006e-05 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.636785+0000", 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010850599999999999 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.637874+0000", 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010884359999999999 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.646820+0000", 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0089460749999999995 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.650638+0000", 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0038179070000000002 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.650696+0000", 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8180000000000002e-05 2023-09-23T18:07:29.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.651572+0000", 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00087589999999999999 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:29509 7.b 7:dc782a25:::benchmark_data_smithi055_311528_object29508:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T18:07:29.730 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:05.354609+0000", 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 84.230013662999994, 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2702194819999999, 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29509 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.354609+0000", 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.354609+0000", 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.354608+0000", 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:07:29.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356017+0000", 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014090400000000001 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356029+0000", 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2014000000000001e-05 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356038+0000", 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9530000000000005e-06 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.359552+0000", 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0035145459999999999 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.359573+0000", 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0681000000000001e-05 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.624724+0000", 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.265151138 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.624829+0000", 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104916 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.735 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:29514 7.b 7:d81719d6:::benchmark_data_smithi055_311528_object29513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:05.356065+0000", 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 84.228557851000005, 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.280434026, 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.736 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29514 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356065+0000", 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356065+0000", 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356064+0000", 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356194+0000", 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000129722 2023-09-23T18:07:29.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356204+0000", 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0319000000000001e-05 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356210+0000", 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2360000000000001e-06 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.359603+0000", 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033929519999999999 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.359610+0000", 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1040000000000001e-06 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.626038+0000", 2023-09-23T18:07:29.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.266427695 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.626978+0000", 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00094050100000000005 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.636426+0000", 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0094477459999999999 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.636499+0000", 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3044000000000001e-05 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:29522 7.b 7:d86057d0:::benchmark_data_smithi055_311528_object29521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e286)", 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:05.356217+0000", 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 84.228406027000005, 2023-09-23T18:07:29.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.28119428, 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29522 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356217+0000", 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356217+0000", 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356216+0000", 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356277+0000", 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0452999999999999e-05 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356285+0000", 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3450000000000006e-06 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.356291+0000", 2023-09-23T18:07:29.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3399999999999997e-06 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.359628+0000", 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033379630000000002 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:05.359633+0000", 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.977e-06 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.627108+0000", 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.267474153 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.627247+0000", 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013978499999999999 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:07:29.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.627383+0000", 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013554499999999999 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.628925+0000", 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015422750000000001 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.631197+0000", 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022718320000000001 2023-09-23T18:07:29.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.636541+0000", 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0053443930000000002 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.636604+0000", 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2102999999999998e-05 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:07:29.749 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:07:29.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:06.637411+0000", 2023-09-23T18:07:29.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000807806 2023-09-23T18:07:29.750 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.750 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.750 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.750 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:07:29.750 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:07:29.750 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:07:30.006 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:07:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:30 smithi099 ceph-mon[167076]: pgmap v787: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 484 op/s; 38558/207612 objects degraded (18.572%); 6562/207612 objects misplaced (3.161%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T18:07:30.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:30 smithi055 ceph-mon[179668]: pgmap v787: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 484 op/s; 38558/207612 objects degraded (18.572%); 6562/207612 objects misplaced (3.161%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T18:07:30.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:30 smithi055 ceph-mon[184174]: pgmap v787: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 19 MiB/s wr, 484 op/s; 38558/207612 objects degraded (18.572%); 6562/207612 objects misplaced (3.161%); 6.8 MiB/s, 108 objects/s recovering 2023-09-23T18:07:30.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:07:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:07:30] "GET /metrics HTTP/1.1" 200 36662 "" "Prometheus/2.43.0" 2023-09-23T18:07:30.709 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:31.319 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:31.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:31.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:31.426 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:31.466 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:31.511 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:31.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:31.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:31.623 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:31.711 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:07:31.711 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:07:31.711 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T18:07:31.711 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:07:31.725 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:31.762 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:32.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:32.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:32.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:32.121 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:32.153 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:32.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:32 smithi055 ceph-mon[179668]: pgmap v788: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 9.7 MiB/s wr, 440 op/s; 38080/204402 objects degraded (18.630%); 6562/204402 objects misplaced (3.210%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:07:32.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:32 smithi055 ceph-mon[184174]: pgmap v788: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 9.7 MiB/s wr, 440 op/s; 38080/204402 objects degraded (18.630%); 6562/204402 objects misplaced (3.210%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:07:32.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:32 smithi099 ceph-mon[167076]: pgmap v788: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.2 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 9.7 MiB/s wr, 440 op/s; 38080/204402 objects degraded (18.630%); 6562/204402 objects misplaced (3.210%); 8.9 MiB/s, 141 objects/s recovering 2023-09-23T18:07:32.960 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:33.082 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:07:33.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1227790053' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:33.183 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:33 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:33.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 38080/204402 objects degraded (18.630%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:33.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1227790053' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:33.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:33 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:33.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 38080/204402 objects degraded (18.630%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:33.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:33 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1227790053' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:33.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:33 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:33.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 38080/204402 objects degraded (18.630%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:33.860 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:07:33.887 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:07:33.913 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:07:33.945 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:07:33.979 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:07:34.007 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:07:34.039 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:07:34.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:34.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:34.055 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:34.066 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:07:34.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:07:34.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:07:34.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:07:34.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:07:34.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:07:34.068 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:07:34.069 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:07:34.069 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:07:34.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[184174]: pgmap v789: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 2.0 MiB/s wr, 409 op/s; 37804/201447 objects degraded (18.766%); 6086/201447 objects misplaced (3.021%); 11 MiB/s, 169 objects/s recovering 2023-09-23T18:07:34.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:34.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[184174]: failed to encode map e323 with expected crc 2023-09-23T18:07:34.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:07:34.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[184174]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T18:07:34.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[184174]: failed to encode map e323 with expected crc 2023-09-23T18:07:34.094 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[184174]: failed to encode map e323 with expected crc 2023-09-23T18:07:34.094 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1057729172' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:34.094 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[179668]: pgmap v789: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 2.0 MiB/s wr, 409 op/s; 37804/201447 objects degraded (18.766%); 6086/201447 objects misplaced (3.021%); 11 MiB/s, 169 objects/s recovering 2023-09-23T18:07:34.094 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:34.094 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[179668]: failed to encode map e323 with expected crc 2023-09-23T18:07:34.094 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:07:34.094 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[179668]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T18:07:34.094 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[179668]: failed to encode map e323 with expected crc 2023-09-23T18:07:34.094 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[179668]: failed to encode map e323 with expected crc 2023-09-23T18:07:34.095 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1057729172' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:34.156 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:34.204 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:34 smithi099 ceph-mon[167076]: pgmap v789: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 2.0 MiB/s wr, 409 op/s; 37804/201447 objects degraded (18.766%); 6086/201447 objects misplaced (3.021%); 11 MiB/s, 169 objects/s recovering 2023-09-23T18:07:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:34 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:34 smithi099 ceph-mon[167076]: failed to encode map e323 with expected crc 2023-09-23T18:07:34.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:34 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:07:34.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:34 smithi099 ceph-mon[167076]: osdmap e323: 8 total, 8 up, 8 in 2023-09-23T18:07:34.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:34 smithi099 ceph-mon[167076]: failed to encode map e323 with expected crc 2023-09-23T18:07:34.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:34 smithi099 ceph-mon[167076]: failed to encode map e323 with expected crc 2023-09-23T18:07:34.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1057729172' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:34.349 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:34.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:34.356 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:34.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:34.489 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:34.918 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:34.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:34.924 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:35.025 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:35.061 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:35.075 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:07:35.077 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:07:35.104 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:35.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:35.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:35.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:35.240 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:35.248 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:07:35.248 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:07:35.284 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:35.288 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:35.288 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:35.389 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:35.430 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:35.442 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":791,"stamp":"2023-09-23T18:07:34.794669+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4377123863,"num_objects":66751,"num_object_clones":0,"num_object_copies":200253,"num_objects_missing_on_primary":1576,"num_objects_missing":1576,"num_objects_degraded":37651,"num_objects_misplaced":6086,"num_objects_unfound":0,"num_objects_dirty":66751,"num_whiteouts":0,"num_read":6089,"num_read_kb":12405,"num_write":76174,"num_write_kb":4554844,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8843,"num_bytes_recovered":572164804,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":123096,"ondisk_log_size":123096,"up":387,"acting":380,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":425,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":15790200,"kb_used_data":13378552,"kb_used_omap":50,"kb_used_meta":2411469,"kb_avail":734171016,"statfs":{"total":767960285184,"available":751791120384,"internally_reserved":0,"allocated":13699637248,"data_stored":13688392072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51828,"internal_metadata":2469344652},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1,3,3,0,2,2,3,6],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-260177896,"num_objects":-3969,"num_object_clones":0,"num_object_copies":-11907,"num_objects_missing_on_primary":-247,"num_objects_missing":-247,"num_objects_degraded":-1586,"num_objects_misplaced":-936,"num_objects_unfound":0,"num_objects_dirty":-3969,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4129,"num_write_kb":5057,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1823,"num_bytes_recovered":119472128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003912"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":649,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681628+0000","last_change":"2023-09-23T18:05:15.087182+0000","last_active":"2023-09-23T18:07:16.681628+0000","last_peered":"2023-09-23T18:07:16.681628+0000","last_clean":"2023-09-23T18:07:16.681628+0000","last_became_active":"2023-09-23T18:05:15.086678+0000","last_became_peered":"2023-09-23T18:05:15.086678+0000","last_unstale":"2023-09-23T18:07:16.681628+0000","last_undegraded":"2023-09-23T18:07:16.681628+0000","last_fullsized":"2023-09-23T18:07:16.681628+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1262601,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":132,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684694+0000","last_change":"2023-09-23T18:06:05.377123+0000","last_active":"2023-09-23T18:07:16.684694+0000","last_peered":"2023-09-23T18:07:16.684694+0000","last_clean":"2023-09-23T18:07:16.684694+0000","last_became_active":"2023-09-23T18:06:05.376735+0000","last_became_peered":"2023-09-23T18:06:05.376735+0000","last_unstale":"2023-09-23T18:07:16.684694+0000","last_undegraded":"2023-09-23T18:07:16.684694+0000","last_fullsized":"2023-09-23T18:07:16.684694+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:11.555818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":198,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.683804+0000","last_change":"2023-09-23T18:06:51.071290+0000","last_active":"2023-09-23T18:07:16.683804+0000","last_peered":"2023-09-23T18:07:16.683804+0000","last_clean":"2023-09-23T18:07:16.683804+0000","last_became_active":"2023-09-23T18:06:51.070948+0000","last_became_peered":"2023-09-23T18:06:51.070948+0000","last_unstale":"2023-09-23T18:07:16.683804+0000","last_undegraded":"2023-09-23T18:07:16.683804+0000","last_fullsized":"2023-09-23T18:07:16.683804+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:52.647972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":667,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684814+0000","last_change":"2023-09-23T18:06:54.065593+0000","last_active":"2023-09-23T18:07:16.684814+0000","last_peered":"2023-09-23T18:07:16.684814+0000","last_clean":"2023-09-23T18:07:16.684814+0000","last_became_active":"2023-09-23T18:06:54.065193+0000","last_became_peered":"2023-09-23T18:06:54.065193+0000","last_unstale":"2023-09-23T18:07:16.684814+0000","last_undegraded":"2023-09-23T18:07:16.684814+0000","last_fullsized":"2023-09-23T18:07:16.684814+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:05.792712+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":37,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.724221+0000","last_change":"2023-09-23T18:06:05.370046+0000","last_active":"2023-09-23T18:07:16.724221+0000","last_peered":"2023-09-23T18:07:16.724221+0000","last_clean":"2023-09-23T18:07:16.724221+0000","last_became_active":"2023-09-23T18:06:05.369714+0000","last_became_peered":"2023-09-23T18:06:05.369714+0000","last_unstale":"2023-09-23T18:07:16.724221+0000","last_undegraded":"2023-09-23T18:07:16.724221+0000","last_fullsized":"2023-09-23T18:07:16.724221+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:47.453671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":331,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.688781+0000","last_change":"2023-09-23T18:06:51.070985+0000","last_active":"2023-09-23T18:07:16.688781+0000","last_peered":"2023-09-23T18:07:16.688781+0000","last_clean":"2023-09-23T18:07:16.688781+0000","last_became_active":"2023-09-23T18:06:51.070671+0000","last_became_peered":"2023-09-23T18:06:51.070671+0000","last_unstale":"2023-09-23T18:07:16.688781+0000","last_undegraded":"2023-09-23T18:07:16.688781+0000","last_fullsized":"2023-09-23T18:07:16.688781+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:14.820905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":659,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684829+0000","last_change":"2023-09-23T18:06:08.439343+0000","last_active":"2023-09-23T18:07:16.684829+0000","last_peered":"2023-09-23T18:07:16.684829+0000","last_clean":"2023-09-23T18:07:16.684829+0000","last_became_active":"2023-09-23T18:06:08.437702+0000","last_became_peered":"2023-09-23T18:06:08.437702+0000","last_unstale":"2023-09-23T18:07:16.684829+0000","last_undegraded":"2023-09-23T18:07:16.684829+0000","last_fullsized":"2023-09-23T18:07:16.684829+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:02.398105+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":201,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681814+0000","last_change":"2023-09-23T18:06:51.087219+0000","last_active":"2023-09-23T18:07:16.681814+0000","last_peered":"2023-09-23T18:07:16.681814+0000","last_clean":"2023-09-23T18:07:16.681814+0000","last_became_active":"2023-09-23T18:06:51.086885+0000","last_became_peered":"2023-09-23T18:06:51.086885+0000","last_unstale":"2023-09-23T18:07:16.681814+0000","last_undegraded":"2023-09-23T18:07:16.681814+0000","last_fullsized":"2023-09-23T18:07:16.681814+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:16.254186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":272,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.679697+0000","last_change":"2023-09-23T18:06:05.372350+0000","last_active":"2023-09-23T18:07:16.679697+0000","last_peered":"2023-09-23T18:07:16.679697+0000","last_clean":"2023-09-23T18:07:16.679697+0000","last_became_active":"2023-09-23T18:06:05.372026+0000","last_became_peered":"2023-09-23T18:06:05.372026+0000","last_unstale":"2023-09-23T18:07:16.679697+0000","last_undegraded":"2023-09-23T18:07:16.679697+0000","last_fullsized":"2023-09-23T18:07:16.679697+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:37.063301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"322'4490","reported_seq":4552,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T18:07:31.663357+0000","last_change":"2023-09-23T18:07:10.644350+0000","last_active":"2023-09-23T18:07:31.663357+0000","last_peered":"2023-09-23T18:07:31.663357+0000","last_clean":"2023-09-23T18:07:31.663357+0000","last_became_active":"2023-09-23T18:07:10.642673+0000","last_became_peered":"2023-09-23T18:07:10.642673+0000","last_unstale":"2023-09-23T18:07:31.663357+0000","last_undegraded":"2023-09-23T18:07:31.663357+0000","last_fullsized":"2023-09-23T18:07:31.663357+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4490,"log_dups_size":0,"ondisk_log_size":4490,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135921664,"num_objects":2074,"num_object_clones":0,"num_object_copies":6222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2394,"num_write_kb":142976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":702,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.688083+0000","last_change":"2023-09-23T18:06:08.433828+0000","last_active":"2023-09-23T18:07:16.688083+0000","last_peered":"2023-09-23T18:07:16.688083+0000","last_clean":"2023-09-23T18:07:16.688083+0000","last_became_active":"2023-09-23T18:06:08.433445+0000","last_became_peered":"2023-09-23T18:06:08.433445+0000","last_unstale":"2023-09-23T18:07:16.688083+0000","last_undegraded":"2023-09-23T18:07:16.688083+0000","last_fullsized":"2023-09-23T18:07:16.688083+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.816081+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":311,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.691260+0000","last_change":"2023-09-23T18:06:54.195144+0000","last_active":"2023-09-23T18:07:16.691260+0000","last_peered":"2023-09-23T18:07:16.691260+0000","last_clean":"2023-09-23T18:07:16.691260+0000","last_became_active":"2023-09-23T18:06:54.194921+0000","last_became_peered":"2023-09-23T18:06:54.194921+0000","last_unstale":"2023-09-23T18:07:16.691260+0000","last_undegraded":"2023-09-23T18:07:16.691260+0000","last_fullsized":"2023-09-23T18:07:16.691260+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:04.645926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":189,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684537+0000","last_change":"2023-09-23T18:06:05.364021+0000","last_active":"2023-09-23T18:07:16.684537+0000","last_peered":"2023-09-23T18:07:16.684537+0000","last_clean":"2023-09-23T18:07:16.684537+0000","last_became_active":"2023-09-23T18:06:05.363740+0000","last_became_peered":"2023-09-23T18:06:05.363740+0000","last_unstale":"2023-09-23T18:07:16.684537+0000","last_undegraded":"2023-09-23T18:07:16.684537+0000","last_fullsized":"2023-09-23T18:07:16.684537+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:55.662682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"322'4400","reported_seq":8141,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T18:07:31.692263+0000","last_change":"2023-09-23T18:07:10.654923+0000","last_active":"2023-09-23T18:07:31.692263+0000","last_peered":"2023-09-23T18:07:31.692263+0000","last_clean":"2023-09-23T18:07:31.692263+0000","last_became_active":"2023-09-23T18:07:10.653290+0000","last_became_peered":"2023-09-23T18:07:10.653290+0000","last_unstale":"2023-09-23T18:07:31.692263+0000","last_undegraded":"2023-09-23T18:07:31.692263+0000","last_fullsized":"2023-09-23T18:07:31.692263+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4400,"log_dups_size":0,"ondisk_log_size":4400,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135004160,"num_objects":2060,"num_object_clones":0,"num_object_copies":6180,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2060,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2336,"num_write_kb":140672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":915,"num_bytes_recovered":59965440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":644,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T18:07:33.065603+0000","last_change":"2023-09-23T18:05:54.888468+0000","last_active":"2023-09-23T18:07:33.065603+0000","last_peered":"2023-09-23T18:07:33.065603+0000","last_clean":"2023-09-23T18:07:33.065603+0000","last_became_active":"2023-09-23T18:05:15.088205+0000","last_became_peered":"2023-09-23T18:05:15.088205+0000","last_unstale":"2023-09-23T18:07:33.065603+0000","last_undegraded":"2023-09-23T18:07:33.065603+0000","last_fullsized":"2023-09-23T18:07:33.065603+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.888321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.888321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:44.454268+0000","scrub_duration":0.0043410219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":314,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.682751+0000","last_change":"2023-09-23T18:06:05.371032+0000","last_active":"2023-09-23T18:07:16.682751+0000","last_peered":"2023-09-23T18:07:16.682751+0000","last_clean":"2023-09-23T18:07:16.682751+0000","last_became_active":"2023-09-23T18:06:05.370695+0000","last_became_peered":"2023-09-23T18:06:05.370695+0000","last_unstale":"2023-09-23T18:07:16.682751+0000","last_undegraded":"2023-09-23T18:07:16.682751+0000","last_fullsized":"2023-09-23T18:07:16.682751+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:12.628753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":134,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684821+0000","last_change":"2023-09-23T18:02:17.956754+0000","last_active":"2023-09-23T18:07:16.684821+0000","last_peered":"2023-09-23T18:07:16.684821+0000","last_clean":"2023-09-23T18:07:16.684821+0000","last_became_active":"2023-09-23T18:02:17.956264+0000","last_became_peered":"2023-09-23T18:02:17.956264+0000","last_unstale":"2023-09-23T18:07:16.684821+0000","last_undegraded":"2023-09-23T18:07:16.684821+0000","last_fullsized":"2023-09-23T18:07:16.684821+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:04:04.728785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.11","version":"322'4407","reported_seq":4827,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:30.711562+0000","last_change":"2023-09-23T18:07:12.880502+0000","last_active":"2023-09-23T18:07:30.711562+0000","last_peered":"2023-09-23T18:07:30.711562+0000","last_clean":"2023-09-23T18:06:25.121665+0000","last_became_active":"2023-09-23T18:07:12.878060+0000","last_became_peered":"2023-09-23T18:07:12.878060+0000","last_unstale":"2023-09-23T18:07:30.711562+0000","last_undegraded":"2023-09-23T18:07:12.230591+0000","last_fullsized":"2023-09-23T18:07:30.711562+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4407,"log_dups_size":0,"ondisk_log_size":4407,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137920512,"num_objects":2105,"num_object_clones":0,"num_object_copies":6315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1418,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2327,"num_write_kb":141792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":106,"num_bytes_recovered":6914048,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":["4","3","6"],"object_location_counts":[{"shards":"1,4,6","objects":687},{"shards":"3,4,6","objects":1418}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":666,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.683178+0000","last_change":"2023-09-23T18:06:54.059837+0000","last_active":"2023-09-23T18:07:16.683178+0000","last_peered":"2023-09-23T18:07:16.683178+0000","last_clean":"2023-09-23T18:07:16.683178+0000","last_became_active":"2023-09-23T18:06:54.059489+0000","last_became_peered":"2023-09-23T18:06:54.059489+0000","last_unstale":"2023-09-23T18:07:16.683178+0000","last_undegraded":"2023-09-23T18:07:16.683178+0000","last_fullsized":"2023-09-23T18:07:16.683178+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.652623+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.652623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:28.333390+0000","scrub_duration":0.00648549,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":87,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.688037+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:07:16.688037+0000","last_peered":"2023-09-23T18:07:16.688037+0000","last_clean":"2023-09-23T18:07:16.688037+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:07:16.688037+0000","last_undegraded":"2023-09-23T18:07:16.688037+0000","last_fullsized":"2023-09-23T18:07:16.688037+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":47,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681407+0000","last_change":"2023-09-23T18:06:05.363236+0000","last_active":"2023-09-23T18:07:16.681407+0000","last_peered":"2023-09-23T18:07:16.681407+0000","last_clean":"2023-09-23T18:07:16.681407+0000","last_became_active":"2023-09-23T18:06:05.362947+0000","last_became_peered":"2023-09-23T18:06:05.362947+0000","last_unstale":"2023-09-23T18:07:16.681407+0000","last_undegraded":"2023-09-23T18:07:16.681407+0000","last_fullsized":"2023-09-23T18:07:16.681407+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:32.816159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"322'4512","reported_seq":4902,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:32.634695+0000","last_change":"2023-09-23T18:07:10.752220+0000","last_active":"2023-09-23T18:07:32.634695+0000","last_peered":"2023-09-23T18:07:32.634695+0000","last_clean":"2023-09-23T18:06:04.331883+0000","last_became_active":"2023-09-23T18:07:10.724852+0000","last_became_peered":"2023-09-23T18:07:10.724852+0000","last_unstale":"2023-09-23T18:07:32.634695+0000","last_undegraded":"2023-09-23T18:07:10.718485+0000","last_fullsized":"2023-09-23T18:07:32.634695+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4512,"log_dups_size":0,"ondisk_log_size":4512,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136970240,"num_objects":2090,"num_object_clones":0,"num_object_copies":6270,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1915,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2090,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2400,"num_write_kb":143680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":155,"num_bytes_recovered":10158080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"1,2,5","objects":175},{"shards":"1,3,5","objects":1111},{"shards":"2,3,5","objects":804}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":669,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.683015+0000","last_change":"2023-09-23T18:06:48.989986+0000","last_active":"2023-09-23T18:07:16.683015+0000","last_peered":"2023-09-23T18:07:16.683015+0000","last_clean":"2023-09-23T18:07:16.683015+0000","last_became_active":"2023-09-23T18:06:48.989534+0000","last_became_peered":"2023-09-23T18:06:48.989534+0000","last_unstale":"2023-09-23T18:07:16.683015+0000","last_undegraded":"2023-09-23T18:07:16.683015+0000","last_fullsized":"2023-09-23T18:07:16.683015+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":114,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.687637+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:07:16.687637+0000","last_peered":"2023-09-23T18:07:16.687637+0000","last_clean":"2023-09-23T18:07:16.687637+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:07:16.687637+0000","last_undegraded":"2023-09-23T18:07:16.687637+0000","last_fullsized":"2023-09-23T18:07:16.687637+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":132,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.687091+0000","last_change":"2023-09-23T18:06:05.375073+0000","last_active":"2023-09-23T18:07:16.687091+0000","last_peered":"2023-09-23T18:07:16.687091+0000","last_clean":"2023-09-23T18:07:16.687091+0000","last_became_active":"2023-09-23T18:06:05.374752+0000","last_became_peered":"2023-09-23T18:06:05.374752+0000","last_unstale":"2023-09-23T18:07:16.687091+0000","last_undegraded":"2023-09-23T18:07:16.687091+0000","last_fullsized":"2023-09-23T18:07:16.687091+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:09:06.839328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"322'4512","reported_seq":4938,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:30.416813+0000","last_change":"2023-09-23T18:07:10.793174+0000","last_active":"2023-09-23T18:07:30.416813+0000","last_peered":"2023-09-23T18:07:30.416813+0000","last_clean":"2023-09-23T18:06:48.947638+0000","last_became_active":"2023-09-23T18:07:10.789937+0000","last_became_peered":"2023-09-23T18:07:10.789937+0000","last_unstale":"2023-09-23T18:07:30.416813+0000","last_undegraded":"2023-09-23T18:07:10.745107+0000","last_fullsized":"2023-09-23T18:07:30.416813+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4512,"log_dups_size":0,"ondisk_log_size":4512,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":139624448,"num_objects":2131,"num_object_clones":0,"num_object_copies":6393,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3924,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2131,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2365,"num_write_kb":143840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":119,"num_bytes_recovered":7766016,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":["6","0","3"],"object_location_counts":[{"shards":"0,3,6","objects":1962},{"shards":"1,6,7","objects":169}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":622,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.680895+0000","last_change":"2023-09-23T18:07:00.038224+0000","last_active":"2023-09-23T18:07:16.680895+0000","last_peered":"2023-09-23T18:07:16.680895+0000","last_clean":"2023-09-23T18:07:16.680895+0000","last_became_active":"2023-09-23T18:07:00.037771+0000","last_became_peered":"2023-09-23T18:07:00.037771+0000","last_unstale":"2023-09-23T18:07:16.680895+0000","last_undegraded":"2023-09-23T18:07:16.680895+0000","last_fullsized":"2023-09-23T18:07:16.680895+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:01.632666+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:06:01.632666+0000","last_clean_scrub_stamp":"2023-09-23T18:06:01.632666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":250,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.685255+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:07:16.685255+0000","last_peered":"2023-09-23T18:07:16.685255+0000","last_clean":"2023-09-23T18:07:16.685255+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:07:16.685255+0000","last_undegraded":"2023-09-23T18:07:16.685255+0000","last_fullsized":"2023-09-23T18:07:16.685255+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":177,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.685078+0000","last_change":"2023-09-23T18:06:05.363898+0000","last_active":"2023-09-23T18:07:16.685078+0000","last_peered":"2023-09-23T18:07:16.685078+0000","last_clean":"2023-09-23T18:07:16.685078+0000","last_became_active":"2023-09-23T18:06:05.363580+0000","last_became_peered":"2023-09-23T18:06:05.363580+0000","last_unstale":"2023-09-23T18:07:16.685078+0000","last_undegraded":"2023-09-23T18:07:16.685078+0000","last_fullsized":"2023-09-23T18:07:16.685078+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:14:14.913078+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.16","version":"322'4341","reported_seq":4955,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:29.668326+0000","last_change":"2023-09-23T18:07:10.673514+0000","last_active":"2023-09-23T18:07:29.668326+0000","last_peered":"2023-09-23T18:07:29.668326+0000","last_clean":"2023-09-23T18:06:44.864353+0000","last_became_active":"2023-09-23T18:07:10.670934+0000","last_became_peered":"2023-09-23T18:07:10.670934+0000","last_unstale":"2023-09-23T18:07:29.668326+0000","last_undegraded":"2023-09-23T18:07:10.647655+0000","last_fullsized":"2023-09-23T18:07:29.668326+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4341,"log_dups_size":0,"ondisk_log_size":4341,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135725056,"num_objects":2071,"num_object_clones":0,"num_object_copies":6213,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1855,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2071,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2275,"num_write_kb":139072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":136,"num_bytes_recovered":8912896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":1855},{"shards":"0,5,7","objects":216}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":639,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681458+0000","last_change":"2023-09-23T18:05:15.091218+0000","last_active":"2023-09-23T18:07:16.681458+0000","last_peered":"2023-09-23T18:07:16.681458+0000","last_clean":"2023-09-23T18:07:16.681458+0000","last_became_active":"2023-09-23T18:05:15.090420+0000","last_became_peered":"2023-09-23T18:05:15.090420+0000","last_unstale":"2023-09-23T18:07:16.681458+0000","last_undegraded":"2023-09-23T18:07:16.681458+0000","last_fullsized":"2023-09-23T18:07:16.681458+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:57.754408+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":329,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684652+0000","last_change":"2023-09-23T18:06:51.065050+0000","last_active":"2023-09-23T18:07:16.684652+0000","last_peered":"2023-09-23T18:07:16.684652+0000","last_clean":"2023-09-23T18:07:16.684652+0000","last_became_active":"2023-09-23T18:06:51.064774+0000","last_became_peered":"2023-09-23T18:06:51.064774+0000","last_unstale":"2023-09-23T18:07:16.684652+0000","last_undegraded":"2023-09-23T18:07:16.684652+0000","last_fullsized":"2023-09-23T18:07:16.684652+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:31.508499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":299,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.682140+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:07:16.682140+0000","last_peered":"2023-09-23T18:07:16.682140+0000","last_clean":"2023-09-23T18:07:16.682140+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:07:16.682140+0000","last_undegraded":"2023-09-23T18:07:16.682140+0000","last_fullsized":"2023-09-23T18:07:16.682140+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"322'4596","reported_seq":4972,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:32.939359+0000","last_change":"2023-09-23T18:07:10.711949+0000","last_active":"2023-09-23T18:07:32.939359+0000","last_peered":"2023-09-23T18:07:32.939359+0000","last_clean":"2023-09-23T18:06:44.892077+0000","last_became_active":"2023-09-23T18:07:10.707811+0000","last_became_peered":"2023-09-23T18:07:10.707811+0000","last_unstale":"2023-09-23T18:07:32.939359+0000","last_undegraded":"2023-09-23T18:07:10.673033+0000","last_fullsized":"2023-09-23T18:07:32.939359+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4596,"log_dups_size":0,"ondisk_log_size":4596,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140148736,"num_objects":2139,"num_object_clones":0,"num_object_copies":6417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1985,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2447,"num_write_kb":146720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":154,"num_bytes_recovered":10092544,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":["7","0","2"],"object_location_counts":[{"shards":"0,2,7","objects":1985},{"shards":"1,2,7","objects":154}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":632,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684108+0000","last_change":"2023-09-23T18:06:45.920979+0000","last_active":"2023-09-23T18:07:16.684108+0000","last_peered":"2023-09-23T18:07:16.684108+0000","last_clean":"2023-09-23T18:07:16.684108+0000","last_became_active":"2023-09-23T18:06:45.920686+0000","last_became_peered":"2023-09-23T18:06:45.920686+0000","last_unstale":"2023-09-23T18:07:16.684108+0000","last_undegraded":"2023-09-23T18:07:16.684108+0000","last_fullsized":"2023-09-23T18:07:16.684108+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":112,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.688447+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:07:16.688447+0000","last_peered":"2023-09-23T18:07:16.688447+0000","last_clean":"2023-09-23T18:07:16.688447+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:07:16.688447+0000","last_undegraded":"2023-09-23T18:07:16.688447+0000","last_fullsized":"2023-09-23T18:07:16.688447+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":260,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684968+0000","last_change":"2023-09-23T18:06:51.125883+0000","last_active":"2023-09-23T18:07:16.684968+0000","last_peered":"2023-09-23T18:07:16.684968+0000","last_clean":"2023-09-23T18:07:16.684968+0000","last_became_active":"2023-09-23T18:06:51.119298+0000","last_became_peered":"2023-09-23T18:06:51.119298+0000","last_unstale":"2023-09-23T18:07:16.684968+0000","last_undegraded":"2023-09-23T18:07:16.684968+0000","last_fullsized":"2023-09-23T18:07:16.684968+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:12.571367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.14","version":"322'4375","reported_seq":4662,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:30.743493+0000","last_change":"2023-09-23T18:07:10.747071+0000","last_active":"2023-09-23T18:07:30.743493+0000","last_peered":"2023-09-23T18:07:30.743493+0000","last_clean":"2023-09-23T18:06:04.313237+0000","last_became_active":"2023-09-23T18:07:10.742490+0000","last_became_peered":"2023-09-23T18:07:10.742490+0000","last_unstale":"2023-09-23T18:07:30.743493+0000","last_undegraded":"2023-09-23T18:07:10.708526+0000","last_fullsized":"2023-09-23T18:07:30.743493+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4375,"log_dups_size":0,"ondisk_log_size":4375,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137101312,"num_objects":2092,"num_object_clones":0,"num_object_copies":6276,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1925,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2092,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2310,"num_write_kb":140864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":110,"num_bytes_recovered":7176192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":1094},{"shards":"0,4","objects":1},{"shards":"0,4,6","objects":829},{"shards":"1,4,6","objects":168}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":659,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681112+0000","last_change":"2023-09-23T18:06:08.430910+0000","last_active":"2023-09-23T18:07:16.681112+0000","last_peered":"2023-09-23T18:07:16.681112+0000","last_clean":"2023-09-23T18:07:16.681112+0000","last_became_active":"2023-09-23T18:06:08.430380+0000","last_became_peered":"2023-09-23T18:06:08.430380+0000","last_unstale":"2023-09-23T18:07:16.681112+0000","last_undegraded":"2023-09-23T18:07:16.681112+0000","last_fullsized":"2023-09-23T18:07:16.681112+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:40.113807+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":298,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.675274+0000","last_change":"2023-09-23T18:06:51.128655+0000","last_active":"2023-09-23T18:07:16.675274+0000","last_peered":"2023-09-23T18:07:16.675274+0000","last_clean":"2023-09-23T18:07:16.675274+0000","last_became_active":"2023-09-23T18:06:51.057904+0000","last_became_peered":"2023-09-23T18:06:51.057904+0000","last_unstale":"2023-09-23T18:07:16.675274+0000","last_undegraded":"2023-09-23T18:07:16.675274+0000","last_fullsized":"2023-09-23T18:07:16.675274+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:22.029660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":175,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684178+0000","last_change":"2023-09-23T18:06:45.919283+0000","last_active":"2023-09-23T18:07:16.684178+0000","last_peered":"2023-09-23T18:07:16.684178+0000","last_clean":"2023-09-23T18:07:16.684178+0000","last_became_active":"2023-09-23T18:06:45.918219+0000","last_became_peered":"2023-09-23T18:06:45.918219+0000","last_unstale":"2023-09-23T18:07:16.684178+0000","last_undegraded":"2023-09-23T18:07:16.684178+0000","last_fullsized":"2023-09-23T18:07:16.684178+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:21.242033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.b","version":"322'4760","reported_seq":5114,"reported_epoch":322,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:07:32.552308+0000","last_change":"2023-09-23T18:07:10.754157+0000","last_active":"2023-09-23T18:07:32.552308+0000","last_peered":"2023-09-23T18:07:32.552308+0000","last_clean":"2023-09-23T18:06:04.331781+0000","last_became_active":"2023-09-23T18:06:06.621694+0000","last_became_peered":"2023-09-23T18:06:06.621694+0000","last_unstale":"2023-09-23T18:07:32.552308+0000","last_undegraded":"2023-09-23T18:07:10.753028+0000","last_fullsized":"2023-09-23T18:06:06.376266+0000","mapping_epoch":286,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4760,"log_dups_size":0,"ondisk_log_size":4760,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:24.704756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":271843328,"num_objects":4148,"num_object_clones":0,"num_object_copies":12444,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1717,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4148,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4760,"num_write_kb":285056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":9502720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[5,6],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5,6","objects":1555},{"shards":"5,6","objects":2431}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":702,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.687980+0000","last_change":"2023-09-23T18:06:08.440949+0000","last_active":"2023-09-23T18:07:16.687980+0000","last_peered":"2023-09-23T18:07:16.687980+0000","last_clean":"2023-09-23T18:07:16.687980+0000","last_became_active":"2023-09-23T18:06:08.440543+0000","last_became_peered":"2023-09-23T18:06:08.440543+0000","last_unstale":"2023-09-23T18:07:16.687980+0000","last_undegraded":"2023-09-23T18:07:16.687980+0000","last_fullsized":"2023-09-23T18:07:16.687980+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:39:30.193370+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":264,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.686521+0000","last_change":"2023-09-23T18:06:45.947590+0000","last_active":"2023-09-23T18:07:16.686521+0000","last_peered":"2023-09-23T18:07:16.686521+0000","last_clean":"2023-09-23T18:07:16.686521+0000","last_became_active":"2023-09-23T18:06:45.920061+0000","last_became_peered":"2023-09-23T18:06:45.920061+0000","last_unstale":"2023-09-23T18:07:16.686521+0000","last_undegraded":"2023-09-23T18:07:16.686521+0000","last_fullsized":"2023-09-23T18:07:16.686521+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:31.645928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":310,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681385+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:07:16.681385+0000","last_peered":"2023-09-23T18:07:16.681385+0000","last_clean":"2023-09-23T18:07:16.681385+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:07:16.681385+0000","last_undegraded":"2023-09-23T18:07:16.681385+0000","last_fullsized":"2023-09-23T18:07:16.681385+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"322'4786","reported_seq":4838,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T18:07:31.919773+0000","last_change":"2023-09-23T18:05:46.126616+0000","last_active":"2023-09-23T18:07:31.919773+0000","last_peered":"2023-09-23T18:07:31.919773+0000","last_clean":"2023-09-23T18:07:31.919773+0000","last_became_active":"2023-09-23T18:05:46.126346+0000","last_became_peered":"2023-09-23T18:05:46.126346+0000","last_unstale":"2023-09-23T18:07:31.919773+0000","last_undegraded":"2023-09-23T18:07:31.919773+0000","last_fullsized":"2023-09-23T18:07:31.919773+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4786,"log_dups_size":0,"ondisk_log_size":4786,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:50:57.668631+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":277348352,"num_objects":4232,"num_object_clones":0,"num_object_copies":12696,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4232,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4786,"num_write_kb":288576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,5],"acting":[3,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":643,"reported_epoch":321,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:07:16.690610+0000","last_change":"2023-09-23T18:06:57.558911+0000","last_active":"2023-09-23T18:07:16.690610+0000","last_peered":"2023-09-23T18:07:16.690610+0000","last_clean":"2023-09-23T18:02:45.965241+0000","last_became_active":"2023-09-23T18:06:57.558665+0000","last_became_peered":"2023-09-23T18:06:57.558665+0000","last_unstale":"2023-09-23T18:07:16.690610+0000","last_undegraded":"2023-09-23T18:06:57.114912+0000","last_fullsized":"2023-09-23T18:06:57.114581+0000","mapping_epoch":309,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":1064488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6],"avail_no_missing":["0","3","6"],"object_location_counts":[{"shards":"0,3,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":279,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.689125+0000","last_change":"2023-09-23T18:06:05.379591+0000","last_active":"2023-09-23T18:07:16.689125+0000","last_peered":"2023-09-23T18:07:16.689125+0000","last_clean":"2023-09-23T18:07:16.689125+0000","last_became_active":"2023-09-23T18:06:05.370176+0000","last_became_peered":"2023-09-23T18:06:05.370176+0000","last_unstale":"2023-09-23T18:07:16.689125+0000","last_undegraded":"2023-09-23T18:07:16.689125+0000","last_fullsized":"2023-09-23T18:07:16.689125+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:08.768138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":352,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.688693+0000","last_change":"2023-09-23T18:06:45.911012+0000","last_active":"2023-09-23T18:07:16.688693+0000","last_peered":"2023-09-23T18:07:16.688693+0000","last_clean":"2023-09-23T18:07:16.688693+0000","last_became_active":"2023-09-23T18:06:45.910722+0000","last_became_peered":"2023-09-23T18:06:45.910722+0000","last_unstale":"2023-09-23T18:07:16.688693+0000","last_undegraded":"2023-09-23T18:07:16.688693+0000","last_fullsized":"2023-09-23T18:07:16.688693+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:19.150719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"322'4383","reported_seq":4683,"reported_epoch":322,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:07:32.353087+0000","last_change":"2023-09-23T18:07:12.877579+0000","last_active":"2023-09-23T18:07:32.353087+0000","last_peered":"2023-09-23T18:07:32.353087+0000","last_clean":"2023-09-23T18:06:04.334743+0000","last_became_active":"2023-09-23T18:07:12.854124+0000","last_became_peered":"2023-09-23T18:07:12.854124+0000","last_unstale":"2023-09-23T18:07:32.353087+0000","last_undegraded":"2023-09-23T18:07:12.196594+0000","last_fullsized":"2023-09-23T18:07:10.610402+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4383,"log_dups_size":0,"ondisk_log_size":4383,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136904704,"num_objects":2089,"num_object_clones":0,"num_object_copies":6267,"num_objects_missing_on_primary":820,"num_objects_missing":820,"num_objects_degraded":1930,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2317,"num_write_kb":140992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":114,"num_bytes_recovered":7471104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1110},{"shards":"0,1,4,7","objects":6},{"shards":"0,4,7","objects":820},{"shards":"1,4","objects":153}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":493,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.679898+0000","last_change":"2023-09-23T18:06:02.805384+0000","last_active":"2023-09-23T18:07:16.679898+0000","last_peered":"2023-09-23T18:07:16.679898+0000","last_clean":"2023-09-23T18:07:16.679898+0000","last_became_active":"2023-09-23T18:05:15.098234+0000","last_became_peered":"2023-09-23T18:05:15.098234+0000","last_unstale":"2023-09-23T18:07:16.679898+0000","last_undegraded":"2023-09-23T18:07:16.679898+0000","last_fullsized":"2023-09-23T18:07:16.679898+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:58.872644+0000","scrub_duration":0.0018893740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":188,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.688297+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:07:16.688297+0000","last_peered":"2023-09-23T18:07:16.688297+0000","last_clean":"2023-09-23T18:07:16.688297+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:07:16.688297+0000","last_undegraded":"2023-09-23T18:07:16.688297+0000","last_fullsized":"2023-09-23T18:07:16.688297+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":297,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.675849+0000","last_change":"2023-09-23T18:06:05.371447+0000","last_active":"2023-09-23T18:07:16.675849+0000","last_peered":"2023-09-23T18:07:16.675849+0000","last_clean":"2023-09-23T18:07:16.675849+0000","last_became_active":"2023-09-23T18:06:05.371065+0000","last_became_peered":"2023-09-23T18:06:05.371065+0000","last_unstale":"2023-09-23T18:07:16.675849+0000","last_undegraded":"2023-09-23T18:07:16.675849+0000","last_fullsized":"2023-09-23T18:07:16.675849+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.025791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"323'4471","reported_seq":4534,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T18:07:33.951598+0000","last_change":"2023-09-23T18:07:10.632010+0000","last_active":"2023-09-23T18:07:33.951598+0000","last_peered":"2023-09-23T18:07:33.951598+0000","last_clean":"2023-09-23T18:07:33.951598+0000","last_became_active":"2023-09-23T18:07:10.629989+0000","last_became_peered":"2023-09-23T18:07:10.629989+0000","last_unstale":"2023-09-23T18:07:33.951598+0000","last_undegraded":"2023-09-23T18:07:33.951598+0000","last_fullsized":"2023-09-23T18:07:33.951598+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4471,"log_dups_size":0,"ondisk_log_size":4471,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135102464,"num_objects":2061,"num_object_clones":0,"num_object_copies":6183,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2061,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2379,"num_write_kb":142112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":646,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.685772+0000","last_change":"2023-09-23T18:05:56.362746+0000","last_active":"2023-09-23T18:07:16.685772+0000","last_peered":"2023-09-23T18:07:16.685772+0000","last_clean":"2023-09-23T18:07:16.685772+0000","last_became_active":"2023-09-23T18:05:15.102072+0000","last_became_peered":"2023-09-23T18:05:15.102072+0000","last_unstale":"2023-09-23T18:07:16.685772+0000","last_undegraded":"2023-09-23T18:07:16.685772+0000","last_fullsized":"2023-09-23T18:07:16.685772+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.362652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.362652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:33.381515+0000","scrub_duration":0.0052581759999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":308,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681761+0000","last_change":"2023-09-23T18:06:51.069867+0000","last_active":"2023-09-23T18:07:16.681761+0000","last_peered":"2023-09-23T18:07:16.681761+0000","last_clean":"2023-09-23T18:07:16.681761+0000","last_became_active":"2023-09-23T18:06:51.069523+0000","last_became_peered":"2023-09-23T18:06:51.069523+0000","last_unstale":"2023-09-23T18:07:16.681761+0000","last_undegraded":"2023-09-23T18:07:16.681761+0000","last_fullsized":"2023-09-23T18:07:16.681761+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:40.321844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":334,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.690807+0000","last_change":"2023-09-23T18:06:05.375880+0000","last_active":"2023-09-23T18:07:16.690807+0000","last_peered":"2023-09-23T18:07:16.690807+0000","last_clean":"2023-09-23T18:07:16.690807+0000","last_became_active":"2023-09-23T18:06:05.375150+0000","last_became_peered":"2023-09-23T18:06:05.375150+0000","last_unstale":"2023-09-23T18:07:16.690807+0000","last_undegraded":"2023-09-23T18:07:16.690807+0000","last_fullsized":"2023-09-23T18:07:16.690807+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:36.483694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"322'4605","reported_seq":7507,"reported_epoch":322,"state":"active+recovering+undersized+remapped","last_fresh":"2023-09-23T18:07:31.927101+0000","last_change":"2023-09-23T18:07:11.699032+0000","last_active":"2023-09-23T18:07:31.927101+0000","last_peered":"2023-09-23T18:07:31.927101+0000","last_clean":"2023-09-23T18:06:44.895531+0000","last_became_active":"2023-09-23T18:07:04.021246+0000","last_became_peered":"2023-09-23T18:07:04.021246+0000","last_unstale":"2023-09-23T18:07:31.927101+0000","last_undegraded":"2023-09-23T18:07:31.927101+0000","last_fullsized":"2023-09-23T18:06:54.218590+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4605,"log_dups_size":0,"ondisk_log_size":4605,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:30.190068+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":267976704,"num_objects":4089,"num_object_clones":0,"num_object_copies":12267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6086,"num_objects_unfound":0,"num_objects_dirty":4089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4605,"num_write_kb":278208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":948,"num_bytes_recovered":62128128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[3,7],"avail_no_missing":["3","0","7"],"object_location_counts":[{"shards":"0,3,7","objects":3043},{"shards":"3,7","objects":1046}],"blocked_by":[],"up_primary":4,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":694,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.685662+0000","last_change":"2023-09-23T18:06:54.059737+0000","last_active":"2023-09-23T18:07:16.685662+0000","last_peered":"2023-09-23T18:07:16.685662+0000","last_clean":"2023-09-23T18:07:16.685662+0000","last_became_active":"2023-09-23T18:06:54.059375+0000","last_became_peered":"2023-09-23T18:06:54.059375+0000","last_unstale":"2023-09-23T18:07:16.685662+0000","last_undegraded":"2023-09-23T18:07:16.685662+0000","last_fullsized":"2023-09-23T18:07:16.685662+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:12.644423+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":224,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.685268+0000","last_change":"2023-09-23T18:06:45.953975+0000","last_active":"2023-09-23T18:07:16.685268+0000","last_peered":"2023-09-23T18:07:16.685268+0000","last_clean":"2023-09-23T18:07:16.685268+0000","last_became_active":"2023-09-23T18:06:45.922926+0000","last_became_peered":"2023-09-23T18:06:45.922926+0000","last_unstale":"2023-09-23T18:07:16.685268+0000","last_undegraded":"2023-09-23T18:07:16.685268+0000","last_fullsized":"2023-09-23T18:07:16.685268+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:59:50.592493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":322,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.687486+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:07:16.687486+0000","last_peered":"2023-09-23T18:07:16.687486+0000","last_clean":"2023-09-23T18:07:16.687486+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:07:16.687486+0000","last_undegraded":"2023-09-23T18:07:16.687486+0000","last_fullsized":"2023-09-23T18:07:16.687486+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"323'4649","reported_seq":4714,"reported_epoch":323,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:07:34.347117+0000","last_change":"2023-09-23T18:06:51.235091+0000","last_active":"2023-09-23T18:07:34.347117+0000","last_peered":"2023-09-23T18:07:34.347117+0000","last_clean":"2023-09-23T18:06:44.878443+0000","last_became_active":"2023-09-23T18:06:51.229315+0000","last_became_peered":"2023-09-23T18:06:51.229315+0000","last_unstale":"2023-09-23T18:07:34.347117+0000","last_undegraded":"2023-09-23T18:06:51.088029+0000","last_fullsized":"2023-09-23T18:06:51.053926+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4649,"log_dups_size":0,"ondisk_log_size":4649,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:11:19.575180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":265879552,"num_objects":4057,"num_object_clones":0,"num_object_copies":12171,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4052,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4057,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4649,"num_write_kb":278592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,1],"acting":[2,5],"avail_no_missing":["2","0","5"],"object_location_counts":[{"shards":"0,2,5","objects":3754},{"shards":"2,3,5","objects":2},{"shards":"2,5","objects":5}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":662,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681537+0000","last_change":"2023-09-23T18:06:54.065597+0000","last_active":"2023-09-23T18:07:16.681537+0000","last_peered":"2023-09-23T18:07:16.681537+0000","last_clean":"2023-09-23T18:07:16.681537+0000","last_became_active":"2023-09-23T18:06:54.065217+0000","last_became_peered":"2023-09-23T18:06:54.065217+0000","last_unstale":"2023-09-23T18:07:16.681537+0000","last_undegraded":"2023-09-23T18:07:16.681537+0000","last_fullsized":"2023-09-23T18:07:16.681537+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:53.614664+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":291,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.675993+0000","last_change":"2023-09-23T18:06:05.371350+0000","last_active":"2023-09-23T18:07:16.675993+0000","last_peered":"2023-09-23T18:07:16.675993+0000","last_clean":"2023-09-23T18:07:16.675993+0000","last_became_active":"2023-09-23T18:06:05.370877+0000","last_became_peered":"2023-09-23T18:06:05.370877+0000","last_unstale":"2023-09-23T18:07:16.675993+0000","last_undegraded":"2023-09-23T18:07:16.675993+0000","last_fullsized":"2023-09-23T18:07:16.675993+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:58.534355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":132,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.685377+0000","last_change":"2023-09-23T18:06:05.373385+0000","last_active":"2023-09-23T18:07:16.685377+0000","last_peered":"2023-09-23T18:07:16.685377+0000","last_clean":"2023-09-23T18:07:16.685377+0000","last_became_active":"2023-09-23T18:06:05.372982+0000","last_became_peered":"2023-09-23T18:06:05.372982+0000","last_unstale":"2023-09-23T18:07:16.685377+0000","last_undegraded":"2023-09-23T18:07:16.685377+0000","last_fullsized":"2023-09-23T18:07:16.685377+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:48.622115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"322'4700","reported_seq":8565,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T18:07:30.486913+0000","last_change":"2023-09-23T18:06:59.042700+0000","last_active":"2023-09-23T18:07:30.486913+0000","last_peered":"2023-09-23T18:07:30.486913+0000","last_clean":"2023-09-23T18:07:30.486913+0000","last_became_active":"2023-09-23T18:06:59.038205+0000","last_became_peered":"2023-09-23T18:06:59.038205+0000","last_unstale":"2023-09-23T18:07:30.486913+0000","last_undegraded":"2023-09-23T18:07:30.486913+0000","last_fullsized":"2023-09-23T18:07:30.486913+0000","mapping_epoch":311,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4700,"log_dups_size":0,"ondisk_log_size":4700,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:40.800800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":278069272,"num_objects":4244,"num_object_clones":0,"num_object_copies":12732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4244,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":4700,"num_write_kb":286145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1894,"num_bytes_recovered":124125184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":664,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684202+0000","last_change":"2023-09-23T18:06:08.435018+0000","last_active":"2023-09-23T18:07:16.684202+0000","last_peered":"2023-09-23T18:07:16.684202+0000","last_clean":"2023-09-23T18:07:16.684202+0000","last_became_active":"2023-09-23T18:06:08.434162+0000","last_became_peered":"2023-09-23T18:06:08.434162+0000","last_unstale":"2023-09-23T18:07:16.684202+0000","last_undegraded":"2023-09-23T18:07:16.684202+0000","last_fullsized":"2023-09-23T18:07:16.684202+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:41.703500+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":112,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.688489+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:07:16.688489+0000","last_peered":"2023-09-23T18:07:16.688489+0000","last_clean":"2023-09-23T18:07:16.688489+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:07:16.688489+0000","last_undegraded":"2023-09-23T18:07:16.688489+0000","last_fullsized":"2023-09-23T18:07:16.688489+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":342,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.685903+0000","last_change":"2023-09-23T18:06:05.373305+0000","last_active":"2023-09-23T18:07:16.685903+0000","last_peered":"2023-09-23T18:07:16.685903+0000","last_clean":"2023-09-23T18:07:16.685903+0000","last_became_active":"2023-09-23T18:06:05.372720+0000","last_became_peered":"2023-09-23T18:06:05.372720+0000","last_unstale":"2023-09-23T18:07:16.685903+0000","last_undegraded":"2023-09-23T18:07:16.685903+0000","last_fullsized":"2023-09-23T18:07:16.685903+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:44.934493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"322'4611","reported_seq":8777,"reported_epoch":322,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:07:30.555968+0000","last_change":"2023-09-23T18:07:11.687041+0000","last_active":"2023-09-23T18:07:30.555968+0000","last_peered":"2023-09-23T18:07:30.555968+0000","last_clean":"2023-09-23T18:06:44.837296+0000","last_became_active":"2023-09-23T18:06:47.115056+0000","last_became_peered":"2023-09-23T18:06:47.115056+0000","last_unstale":"2023-09-23T18:07:30.555968+0000","last_undegraded":"2023-09-23T18:07:11.685252+0000","last_fullsized":"2023-09-23T18:06:46.925665+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4611,"log_dups_size":0,"ondisk_log_size":4611,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:46.974573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":270991360,"num_objects":4135,"num_object_clones":0,"num_object_copies":12405,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1984,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4135,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4611,"num_write_kb":279872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2051,"num_bytes_recovered":134414336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":1984},{"shards":"6,7","objects":2151}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":638,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681379+0000","last_change":"2023-09-23T18:05:53.509955+0000","last_active":"2023-09-23T18:07:16.681379+0000","last_peered":"2023-09-23T18:07:16.681379+0000","last_clean":"2023-09-23T18:07:16.681379+0000","last_became_active":"2023-09-23T18:05:15.089224+0000","last_became_peered":"2023-09-23T18:05:15.089224+0000","last_unstale":"2023-09-23T18:07:16.681379+0000","last_undegraded":"2023-09-23T18:07:16.681379+0000","last_fullsized":"2023-09-23T18:07:16.681379+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:37.917870+0000","scrub_duration":0.001603043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":167,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681748+0000","last_change":"2023-09-23T18:06:45.904403+0000","last_active":"2023-09-23T18:07:16.681748+0000","last_peered":"2023-09-23T18:07:16.681748+0000","last_clean":"2023-09-23T18:07:16.681748+0000","last_became_active":"2023-09-23T18:06:45.904057+0000","last_became_peered":"2023-09-23T18:06:45.904057+0000","last_unstale":"2023-09-23T18:07:16.681748+0000","last_undegraded":"2023-09-23T18:07:16.681748+0000","last_fullsized":"2023-09-23T18:07:16.681748+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:43.213905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":47,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684094+0000","last_change":"2023-09-23T18:06:05.371661+0000","last_active":"2023-09-23T18:07:16.684094+0000","last_peered":"2023-09-23T18:07:16.684094+0000","last_clean":"2023-09-23T18:07:16.684094+0000","last_became_active":"2023-09-23T18:06:05.370998+0000","last_became_peered":"2023-09-23T18:06:05.370998+0000","last_unstale":"2023-09-23T18:07:16.684094+0000","last_undegraded":"2023-09-23T18:07:16.684094+0000","last_fullsized":"2023-09-23T18:07:16.684094+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:52.262543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"322'4373","reported_seq":4780,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:30.761590+0000","last_change":"2023-09-23T18:07:12.880468+0000","last_active":"2023-09-23T18:07:30.761590+0000","last_peered":"2023-09-23T18:07:30.761590+0000","last_clean":"2023-09-23T18:06:25.121665+0000","last_became_active":"2023-09-23T18:07:12.877916+0000","last_became_peered":"2023-09-23T18:07:12.877916+0000","last_unstale":"2023-09-23T18:07:30.761590+0000","last_undegraded":"2023-09-23T18:07:12.235381+0000","last_fullsized":"2023-09-23T18:07:30.761590+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4373,"log_dups_size":0,"ondisk_log_size":4373,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":138313728,"num_objects":2110,"num_object_clones":0,"num_object_copies":6330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1430,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2292,"num_write_kb":140896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":97,"num_bytes_recovered":6389760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":["4","3","6"],"object_location_counts":[{"shards":"1,4,6","objects":680},{"shards":"3,4,6","objects":1430}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":513,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.679629+0000","last_change":"2023-09-23T18:06:54.193400+0000","last_active":"2023-09-23T18:07:16.679629+0000","last_peered":"2023-09-23T18:07:16.679629+0000","last_clean":"2023-09-23T18:07:16.679629+0000","last_became_active":"2023-09-23T18:06:54.193019+0000","last_became_peered":"2023-09-23T18:06:54.193019+0000","last_unstale":"2023-09-23T18:07:16.679629+0000","last_undegraded":"2023-09-23T18:07:16.679629+0000","last_fullsized":"2023-09-23T18:07:16.679629+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:48.156749+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":633,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.682941+0000","last_change":"2023-09-23T18:06:29.051219+0000","last_active":"2023-09-23T18:07:16.682941+0000","last_peered":"2023-09-23T18:07:16.682941+0000","last_clean":"2023-09-23T18:07:16.682941+0000","last_became_active":"2023-09-23T18:06:29.050570+0000","last_became_peered":"2023-09-23T18:06:29.050570+0000","last_unstale":"2023-09-23T18:07:16.682941+0000","last_undegraded":"2023-09-23T18:07:16.682941+0000","last_fullsized":"2023-09-23T18:07:16.682941+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:17.235584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":312,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.682340+0000","last_change":"2023-09-23T18:06:51.074796+0000","last_active":"2023-09-23T18:07:16.682340+0000","last_peered":"2023-09-23T18:07:16.682340+0000","last_clean":"2023-09-23T18:07:16.682340+0000","last_became_active":"2023-09-23T18:06:51.074525+0000","last_became_peered":"2023-09-23T18:06:51.074525+0000","last_unstale":"2023-09-23T18:07:16.682340+0000","last_undegraded":"2023-09-23T18:07:16.682340+0000","last_fullsized":"2023-09-23T18:07:16.682340+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:48.941608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":47,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681280+0000","last_change":"2023-09-23T18:06:05.374173+0000","last_active":"2023-09-23T18:07:16.681280+0000","last_peered":"2023-09-23T18:07:16.681280+0000","last_clean":"2023-09-23T18:07:16.681280+0000","last_became_active":"2023-09-23T18:06:05.373835+0000","last_became_peered":"2023-09-23T18:06:05.373835+0000","last_unstale":"2023-09-23T18:07:16.681280+0000","last_undegraded":"2023-09-23T18:07:16.681280+0000","last_fullsized":"2023-09-23T18:07:16.681280+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:25.467488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"322'4531","reported_seq":4945,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:30.554887+0000","last_change":"2023-09-23T18:07:11.678464+0000","last_active":"2023-09-23T18:07:30.554887+0000","last_peered":"2023-09-23T18:07:30.554887+0000","last_clean":"2023-09-23T18:06:48.947638+0000","last_became_active":"2023-09-23T18:07:11.672631+0000","last_became_peered":"2023-09-23T18:07:11.672631+0000","last_unstale":"2023-09-23T18:07:30.554887+0000","last_undegraded":"2023-09-23T18:07:11.667350+0000","last_fullsized":"2023-09-23T18:07:30.554887+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4531,"log_dups_size":0,"ondisk_log_size":4531,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":141524992,"num_objects":2159,"num_object_clones":0,"num_object_copies":6477,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4052,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2159,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2383,"num_write_kb":145376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":113,"num_bytes_recovered":7438336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":["6","0","3"],"object_location_counts":[{"shards":"0,3,6","objects":2026},{"shards":"1,6,7","objects":133}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":639,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.680711+0000","last_change":"2023-09-23T18:05:15.083813+0000","last_active":"2023-09-23T18:07:16.680711+0000","last_peered":"2023-09-23T18:07:16.680711+0000","last_clean":"2023-09-23T18:07:16.680711+0000","last_became_active":"2023-09-23T18:05:15.083395+0000","last_became_peered":"2023-09-23T18:05:15.083395+0000","last_unstale":"2023-09-23T18:07:16.680711+0000","last_undegraded":"2023-09-23T18:07:16.680711+0000","last_fullsized":"2023-09-23T18:07:16.680711+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:24.402252+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1294,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681256+0000","last_change":"2023-09-23T18:06:08.426828+0000","last_active":"2023-09-23T18:07:16.681256+0000","last_peered":"2023-09-23T18:07:16.681256+0000","last_clean":"2023-09-23T18:07:16.681256+0000","last_became_active":"2023-09-23T18:06:08.426343+0000","last_became_peered":"2023-09-23T18:06:08.426343+0000","last_unstale":"2023-09-23T18:07:16.681256+0000","last_undegraded":"2023-09-23T18:07:16.681256+0000","last_fullsized":"2023-09-23T18:07:16.681256+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:38.924507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":112,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.687160+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:07:16.687160+0000","last_peered":"2023-09-23T18:07:16.687160+0000","last_clean":"2023-09-23T18:07:16.687160+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:07:16.687160+0000","last_undegraded":"2023-09-23T18:07:16.687160+0000","last_fullsized":"2023-09-23T18:07:16.687160+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":56,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.682858+0000","last_change":"2023-09-23T18:06:51.088473+0000","last_active":"2023-09-23T18:07:16.682858+0000","last_peered":"2023-09-23T18:07:16.682858+0000","last_clean":"2023-09-23T18:07:16.682858+0000","last_became_active":"2023-09-23T18:06:51.088171+0000","last_became_peered":"2023-09-23T18:06:51.088171+0000","last_unstale":"2023-09-23T18:07:16.682858+0000","last_undegraded":"2023-09-23T18:07:16.682858+0000","last_fullsized":"2023-09-23T18:07:16.682858+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:45.860551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"322'4362","reported_seq":4657,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:30.771758+0000","last_change":"2023-09-23T18:07:11.696361+0000","last_active":"2023-09-23T18:07:30.771758+0000","last_peered":"2023-09-23T18:07:30.771758+0000","last_clean":"2023-09-23T18:06:04.313237+0000","last_became_active":"2023-09-23T18:07:11.692837+0000","last_became_peered":"2023-09-23T18:07:11.692837+0000","last_unstale":"2023-09-23T18:07:30.771758+0000","last_undegraded":"2023-09-23T18:07:11.667257+0000","last_fullsized":"2023-09-23T18:07:30.771758+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4362,"log_dups_size":0,"ondisk_log_size":4362,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":135987200,"num_objects":2075,"num_object_clones":0,"num_object_copies":6225,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1888,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2075,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2297,"num_write_kb":139904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":112,"num_bytes_recovered":7372800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":["4","0"],"object_location_counts":[{"shards":"0,1,4","objects":1094},{"shards":"0,4,6","objects":794},{"shards":"1,4,6","objects":187}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":648,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:19.868202+0000","last_change":"2023-09-23T18:05:15.087882+0000","last_active":"2023-09-23T18:07:19.868202+0000","last_peered":"2023-09-23T18:07:19.868202+0000","last_clean":"2023-09-23T18:07:19.868202+0000","last_became_active":"2023-09-23T18:05:15.087524+0000","last_became_peered":"2023-09-23T18:05:15.087524+0000","last_unstale":"2023-09-23T18:07:19.868202+0000","last_undegraded":"2023-09-23T18:07:19.868202+0000","last_fullsized":"2023-09-23T18:07:19.868202+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:55.015941+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":614,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:22.878833+0000","last_change":"2023-09-23T18:06:56.676920+0000","last_active":"2023-09-23T18:07:22.878833+0000","last_peered":"2023-09-23T18:07:22.878833+0000","last_clean":"2023-09-23T18:06:48.957948+0000","last_became_active":"2023-09-23T18:06:56.676417+0000","last_became_peered":"2023-09-23T18:06:56.676417+0000","last_unstale":"2023-09-23T18:07:22.878833+0000","last_undegraded":"2023-09-23T18:06:56.372665+0000","last_fullsized":"2023-09-23T18:07:22.878833+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:40.555906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":["2","0","3"],"object_location_counts":[{"shards":"0,2,3","objects":1}],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":47,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681921+0000","last_change":"2023-09-23T18:06:05.358500+0000","last_active":"2023-09-23T18:07:16.681921+0000","last_peered":"2023-09-23T18:07:16.681921+0000","last_clean":"2023-09-23T18:07:16.681921+0000","last_became_active":"2023-09-23T18:06:05.358274+0000","last_became_peered":"2023-09-23T18:06:05.358274+0000","last_unstale":"2023-09-23T18:07:16.681921+0000","last_undegraded":"2023-09-23T18:07:16.681921+0000","last_fullsized":"2023-09-23T18:07:16.681921+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:04.055437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":56,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684263+0000","last_change":"2023-09-23T18:06:45.919995+0000","last_active":"2023-09-23T18:07:16.684263+0000","last_peered":"2023-09-23T18:07:16.684263+0000","last_clean":"2023-09-23T18:07:16.684263+0000","last_became_active":"2023-09-23T18:06:45.919686+0000","last_became_peered":"2023-09-23T18:06:45.919686+0000","last_unstale":"2023-09-23T18:07:16.684263+0000","last_undegraded":"2023-09-23T18:07:16.684263+0000","last_fullsized":"2023-09-23T18:07:16.684263+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:39.113717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"322'4353","reported_seq":4968,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:29.836349+0000","last_change":"2023-09-23T18:07:11.654839+0000","last_active":"2023-09-23T18:07:29.836349+0000","last_peered":"2023-09-23T18:07:29.836349+0000","last_clean":"2023-09-23T18:06:44.864353+0000","last_became_active":"2023-09-23T18:07:11.651208+0000","last_became_peered":"2023-09-23T18:07:11.651208+0000","last_unstale":"2023-09-23T18:07:29.836349+0000","last_undegraded":"2023-09-23T18:07:11.622540+0000","last_fullsized":"2023-09-23T18:07:29.836349+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4353,"log_dups_size":0,"ondisk_log_size":4353,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137428992,"num_objects":2097,"num_object_clones":0,"num_object_copies":6291,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1919,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2097,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2287,"num_write_kb":140288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":136,"num_bytes_recovered":8912896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":1919},{"shards":"0,5,7","objects":178}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1234,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684684+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:07:16.684684+0000","last_peered":"2023-09-23T18:07:16.684684+0000","last_clean":"2023-09-23T18:07:16.684684+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:07:16.684684+0000","last_undegraded":"2023-09-23T18:07:16.684684+0000","last_fullsized":"2023-09-23T18:07:16.684684+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":677,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.685011+0000","last_change":"2023-09-23T18:06:48.957422+0000","last_active":"2023-09-23T18:07:16.685011+0000","last_peered":"2023-09-23T18:07:16.685011+0000","last_clean":"2023-09-23T18:07:16.685011+0000","last_became_active":"2023-09-23T18:06:48.957044+0000","last_became_peered":"2023-09-23T18:06:48.957044+0000","last_unstale":"2023-09-23T18:07:16.685011+0000","last_undegraded":"2023-09-23T18:07:16.685011+0000","last_fullsized":"2023-09-23T18:07:16.685011+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:58.047861+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":154,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681184+0000","last_change":"2023-09-23T18:06:51.057540+0000","last_active":"2023-09-23T18:07:16.681184+0000","last_peered":"2023-09-23T18:07:16.681184+0000","last_clean":"2023-09-23T18:07:16.681184+0000","last_became_active":"2023-09-23T18:06:51.056891+0000","last_became_peered":"2023-09-23T18:06:51.056891+0000","last_unstale":"2023-09-23T18:07:16.681184+0000","last_undegraded":"2023-09-23T18:07:16.681184+0000","last_fullsized":"2023-09-23T18:07:16.681184+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:32.959376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":255,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.680659+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:07:16.680659+0000","last_peered":"2023-09-23T18:07:16.680659+0000","last_clean":"2023-09-23T18:07:16.680659+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:07:16.680659+0000","last_undegraded":"2023-09-23T18:07:16.680659+0000","last_fullsized":"2023-09-23T18:07:16.680659+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"322'4595","reported_seq":5030,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:32.841890+0000","last_change":"2023-09-23T18:07:11.666722+0000","last_active":"2023-09-23T18:07:32.841890+0000","last_peered":"2023-09-23T18:07:32.841890+0000","last_clean":"2023-09-23T18:06:44.892077+0000","last_became_active":"2023-09-23T18:07:11.642881+0000","last_became_peered":"2023-09-23T18:07:11.642881+0000","last_unstale":"2023-09-23T18:07:32.841890+0000","last_undegraded":"2023-09-23T18:07:11.636195+0000","last_fullsized":"2023-09-23T18:07:32.841890+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4595,"log_dups_size":0,"ondisk_log_size":4595,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136413184,"num_objects":2081,"num_object_clones":0,"num_object_copies":6243,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1889,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2081,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2445,"num_write_kb":144864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":182,"num_bytes_recovered":11927552,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":["7","0","2"],"object_location_counts":[{"shards":"0,2,7","objects":1889},{"shards":"1,2,7","objects":192}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":470,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684025+0000","last_change":"2023-09-23T18:06:08.434791+0000","last_active":"2023-09-23T18:07:16.684025+0000","last_peered":"2023-09-23T18:07:16.684025+0000","last_clean":"2023-09-23T18:07:16.684025+0000","last_became_active":"2023-09-23T18:06:08.433718+0000","last_became_peered":"2023-09-23T18:06:08.433718+0000","last_unstale":"2023-09-23T18:07:16.684025+0000","last_undegraded":"2023-09-23T18:07:16.684025+0000","last_fullsized":"2023-09-23T18:07:16.684025+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:57.701252+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:57.701252+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":529,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.685093+0000","last_change":"2023-09-23T18:06:05.403119+0000","last_active":"2023-09-23T18:07:16.685093+0000","last_peered":"2023-09-23T18:07:16.685093+0000","last_clean":"2023-09-23T18:07:16.685093+0000","last_became_active":"2023-09-23T18:06:05.371941+0000","last_became_peered":"2023-09-23T18:06:05.371941+0000","last_unstale":"2023-09-23T18:07:16.685093+0000","last_undegraded":"2023-09-23T18:07:16.685093+0000","last_fullsized":"2023-09-23T18:07:16.685093+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:34:25.525564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":330,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.690678+0000","last_change":"2023-09-23T18:06:45.914529+0000","last_active":"2023-09-23T18:07:16.690678+0000","last_peered":"2023-09-23T18:07:16.690678+0000","last_clean":"2023-09-23T18:07:16.690678+0000","last_became_active":"2023-09-23T18:06:45.914170+0000","last_became_peered":"2023-09-23T18:06:45.914170+0000","last_unstale":"2023-09-23T18:07:16.690678+0000","last_undegraded":"2023-09-23T18:07:16.690678+0000","last_fullsized":"2023-09-23T18:07:16.690678+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:49.492903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":299,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.682737+0000","last_change":"2023-09-23T18:06:51.070104+0000","last_active":"2023-09-23T18:07:16.682737+0000","last_peered":"2023-09-23T18:07:16.682737+0000","last_clean":"2023-09-23T18:07:16.682737+0000","last_became_active":"2023-09-23T18:06:51.069771+0000","last_became_peered":"2023-09-23T18:06:51.069771+0000","last_unstale":"2023-09-23T18:07:16.682737+0000","last_undegraded":"2023-09-23T18:07:16.682737+0000","last_fullsized":"2023-09-23T18:07:16.682737+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:49.300491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"322'4518","reported_seq":4903,"reported_epoch":322,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:32.626466+0000","last_change":"2023-09-23T18:07:11.704447+0000","last_active":"2023-09-23T18:07:32.626466+0000","last_peered":"2023-09-23T18:07:32.626466+0000","last_clean":"2023-09-23T18:06:04.331883+0000","last_became_active":"2023-09-23T18:07:11.677873+0000","last_became_peered":"2023-09-23T18:07:11.677873+0000","last_unstale":"2023-09-23T18:07:32.626466+0000","last_undegraded":"2023-09-23T18:07:11.672793+0000","last_fullsized":"2023-09-23T18:07:32.626466+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4518,"log_dups_size":0,"ondisk_log_size":4518,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":137887744,"num_objects":2104,"num_object_clones":0,"num_object_copies":6312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1902,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2406,"num_write_kb":144320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":151,"num_bytes_recovered":9895936,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"1,2,5","objects":202},{"shards":"1,3,5","objects":1134},{"shards":"2,3,5","objects":768}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":533,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.677104+0000","last_change":"2023-09-23T18:06:54.055392+0000","last_active":"2023-09-23T18:07:16.677104+0000","last_peered":"2023-09-23T18:07:16.677104+0000","last_clean":"2023-09-23T18:07:16.677104+0000","last_became_active":"2023-09-23T18:06:54.055101+0000","last_became_peered":"2023-09-23T18:06:54.055101+0000","last_unstale":"2023-09-23T18:07:16.677104+0000","last_undegraded":"2023-09-23T18:07:16.677104+0000","last_fullsized":"2023-09-23T18:07:16.677104+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:50.070812+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":543,"reported_epoch":321,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:07:16.688159+0000","last_change":"2023-09-23T18:06:51.071499+0000","last_active":"2023-09-23T18:07:16.688159+0000","last_peered":"2023-09-23T18:07:16.688159+0000","last_clean":"2023-09-23T18:06:48.954757+0000","last_became_active":"2023-09-23T18:06:51.071089+0000","last_became_peered":"2023-09-23T18:06:51.071089+0000","last_unstale":"2023-09-23T18:07:16.688159+0000","last_undegraded":"2023-09-23T18:06:51.054242+0000","last_fullsized":"2023-09-23T18:07:16.688159+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:01.696899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":56,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.682782+0000","last_change":"2023-09-23T18:06:51.070131+0000","last_active":"2023-09-23T18:07:16.682782+0000","last_peered":"2023-09-23T18:07:16.682782+0000","last_clean":"2023-09-23T18:07:16.682782+0000","last_became_active":"2023-09-23T18:06:51.069733+0000","last_became_peered":"2023-09-23T18:06:51.069733+0000","last_unstale":"2023-09-23T18:07:16.682782+0000","last_undegraded":"2023-09-23T18:07:16.682782+0000","last_fullsized":"2023-09-23T18:07:16.682782+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:28.050735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":289,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.690984+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:07:16.690984+0000","last_peered":"2023-09-23T18:07:16.690984+0000","last_clean":"2023-09-23T18:07:16.690984+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:07:16.690984+0000","last_undegraded":"2023-09-23T18:07:16.690984+0000","last_fullsized":"2023-09-23T18:07:16.690984+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"322'4471","reported_seq":4533,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T18:07:31.848756+0000","last_change":"2023-09-23T18:07:10.620752+0000","last_active":"2023-09-23T18:07:31.848756+0000","last_peered":"2023-09-23T18:07:31.848756+0000","last_clean":"2023-09-23T18:07:31.848756+0000","last_became_active":"2023-09-23T18:07:10.618491+0000","last_became_peered":"2023-09-23T18:07:10.618491+0000","last_unstale":"2023-09-23T18:07:31.848756+0000","last_undegraded":"2023-09-23T18:07:31.848756+0000","last_fullsized":"2023-09-23T18:07:31.848756+0000","mapping_epoch":279,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4471,"log_dups_size":0,"ondisk_log_size":4471,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136773632,"num_objects":2087,"num_object_clones":0,"num_object_copies":6261,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2087,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2375,"num_write_kb":142784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,4],"acting":[3,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":667,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.685117+0000","last_change":"2023-09-23T18:06:48.962388+0000","last_active":"2023-09-23T18:07:16.685117+0000","last_peered":"2023-09-23T18:07:16.685117+0000","last_clean":"2023-09-23T18:07:16.685117+0000","last_became_active":"2023-09-23T18:06:48.962102+0000","last_became_peered":"2023-09-23T18:06:48.962102+0000","last_unstale":"2023-09-23T18:07:16.685117+0000","last_undegraded":"2023-09-23T18:07:16.685117+0000","last_fullsized":"2023-09-23T18:07:16.685117+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.217483+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":157,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.695589+0000","last_change":"2023-09-23T18:02:17.949063+0000","last_active":"2023-09-23T18:07:16.695589+0000","last_peered":"2023-09-23T18:07:16.695589+0000","last_clean":"2023-09-23T18:07:16.695589+0000","last_became_active":"2023-09-23T18:02:17.946291+0000","last_became_peered":"2023-09-23T18:02:17.946291+0000","last_unstale":"2023-09-23T18:07:16.695589+0000","last_undegraded":"2023-09-23T18:07:16.695589+0000","last_fullsized":"2023-09-23T18:07:16.695589+0000","mapping_epoch":237,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:27:29.585264+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":62,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.683096+0000","last_change":"2023-09-23T18:06:05.398171+0000","last_active":"2023-09-23T18:07:16.683096+0000","last_peered":"2023-09-23T18:07:16.683096+0000","last_clean":"2023-09-23T18:07:16.683096+0000","last_became_active":"2023-09-23T18:06:05.370800+0000","last_became_peered":"2023-09-23T18:06:05.370800+0000","last_unstale":"2023-09-23T18:07:16.683096+0000","last_undegraded":"2023-09-23T18:07:16.683096+0000","last_fullsized":"2023-09-23T18:07:16.683096+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:46.783817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":126,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684945+0000","last_change":"2023-09-23T18:06:45.908943+0000","last_active":"2023-09-23T18:07:16.684945+0000","last_peered":"2023-09-23T18:07:16.684945+0000","last_clean":"2023-09-23T18:07:16.684945+0000","last_became_active":"2023-09-23T18:06:45.908609+0000","last_became_peered":"2023-09-23T18:06:45.908609+0000","last_unstale":"2023-09-23T18:07:16.684945+0000","last_undegraded":"2023-09-23T18:07:16.684945+0000","last_fullsized":"2023-09-23T18:07:16.684945+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:03:31.275390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"322'4413","reported_seq":8154,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T18:07:31.919957+0000","last_change":"2023-09-23T18:07:10.628117+0000","last_active":"2023-09-23T18:07:31.919957+0000","last_peered":"2023-09-23T18:07:31.919957+0000","last_clean":"2023-09-23T18:07:31.919957+0000","last_became_active":"2023-09-23T18:07:10.626078+0000","last_became_peered":"2023-09-23T18:07:10.626078+0000","last_unstale":"2023-09-23T18:07:31.919957+0000","last_undegraded":"2023-09-23T18:07:31.919957+0000","last_fullsized":"2023-09-23T18:07:31.919957+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4413,"log_dups_size":0,"ondisk_log_size":4413,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134938624,"num_objects":2059,"num_object_clones":0,"num_object_copies":6177,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2059,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2349,"num_write_kb":141056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":915,"num_bytes_recovered":59965440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":627,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681703+0000","last_change":"2023-09-23T18:06:54.059843+0000","last_active":"2023-09-23T18:07:16.681703+0000","last_peered":"2023-09-23T18:07:16.681703+0000","last_clean":"2023-09-23T18:07:16.681703+0000","last_became_active":"2023-09-23T18:06:54.059469+0000","last_became_peered":"2023-09-23T18:06:54.059469+0000","last_unstale":"2023-09-23T18:07:16.681703+0000","last_undegraded":"2023-09-23T18:07:16.681703+0000","last_fullsized":"2023-09-23T18:07:16.681703+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:59.157827+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":527,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.679824+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:07:16.679824+0000","last_peered":"2023-09-23T18:07:16.679824+0000","last_clean":"2023-09-23T18:07:16.679824+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:07:16.679824+0000","last_undegraded":"2023-09-23T18:07:16.679824+0000","last_fullsized":"2023-09-23T18:07:16.679824+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":351,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.682819+0000","last_change":"2023-09-23T18:06:45.939855+0000","last_active":"2023-09-23T18:07:16.682819+0000","last_peered":"2023-09-23T18:07:16.682819+0000","last_clean":"2023-09-23T18:07:16.682819+0000","last_became_active":"2023-09-23T18:06:45.924278+0000","last_became_peered":"2023-09-23T18:06:45.924278+0000","last_unstale":"2023-09-23T18:07:16.682819+0000","last_undegraded":"2023-09-23T18:07:16.682819+0000","last_fullsized":"2023-09-23T18:07:16.682819+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:43.850375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":112,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.688399+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:07:16.688399+0000","last_peered":"2023-09-23T18:07:16.688399+0000","last_clean":"2023-09-23T18:07:16.688399+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:07:16.688399+0000","last_undegraded":"2023-09-23T18:07:16.688399+0000","last_fullsized":"2023-09-23T18:07:16.688399+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":459,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.675458+0000","last_change":"2023-09-23T18:06:48.961752+0000","last_active":"2023-09-23T18:07:16.675458+0000","last_peered":"2023-09-23T18:07:16.675458+0000","last_clean":"2023-09-23T18:07:16.675458+0000","last_became_active":"2023-09-23T18:06:48.961304+0000","last_became_peered":"2023-09-23T18:06:48.961304+0000","last_unstale":"2023-09-23T18:07:16.675458+0000","last_undegraded":"2023-09-23T18:07:16.675458+0000","last_fullsized":"2023-09-23T18:07:16.675458+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":238,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.687023+0000","last_change":"2023-09-23T18:06:26.148865+0000","last_active":"2023-09-23T18:07:16.687023+0000","last_peered":"2023-09-23T18:07:16.687023+0000","last_clean":"2023-09-23T18:07:16.687023+0000","last_became_active":"2023-09-23T18:06:26.148517+0000","last_became_peered":"2023-09-23T18:06:26.148517+0000","last_unstale":"2023-09-23T18:07:16.687023+0000","last_undegraded":"2023-09-23T18:07:16.687023+0000","last_fullsized":"2023-09-23T18:07:16.687023+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:38.141672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":358,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.682694+0000","last_change":"2023-09-23T18:06:55.916070+0000","last_active":"2023-09-23T18:07:16.682694+0000","last_peered":"2023-09-23T18:07:16.682694+0000","last_clean":"2023-09-23T18:07:16.682694+0000","last_became_active":"2023-09-23T18:06:55.915807+0000","last_became_peered":"2023-09-23T18:06:55.915807+0000","last_unstale":"2023-09-23T18:07:16.682694+0000","last_undegraded":"2023-09-23T18:07:16.682694+0000","last_fullsized":"2023-09-23T18:07:16.682694+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:20.376063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.18","version":"323'4476","reported_seq":4539,"reported_epoch":323,"state":"active+clean","last_fresh":"2023-09-23T18:07:33.991143+0000","last_change":"2023-09-23T18:07:10.670625+0000","last_active":"2023-09-23T18:07:33.991143+0000","last_peered":"2023-09-23T18:07:33.991143+0000","last_clean":"2023-09-23T18:07:33.991143+0000","last_became_active":"2023-09-23T18:07:10.669408+0000","last_became_peered":"2023-09-23T18:07:10.669408+0000","last_unstale":"2023-09-23T18:07:33.991143+0000","last_undegraded":"2023-09-23T18:07:33.991143+0000","last_fullsized":"2023-09-23T18:07:33.991143+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":317,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4476,"log_dups_size":0,"ondisk_log_size":4476,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":136871936,"num_objects":2089,"num_object_clones":0,"num_object_copies":6267,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2089,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2385,"num_write_kb":143136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":454,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.676902+0000","last_change":"2023-09-23T18:06:54.055428+0000","last_active":"2023-09-23T18:07:16.676902+0000","last_peered":"2023-09-23T18:07:16.676902+0000","last_clean":"2023-09-23T18:07:16.676902+0000","last_became_active":"2023-09-23T18:06:54.055098+0000","last_became_peered":"2023-09-23T18:06:54.055098+0000","last_unstale":"2023-09-23T18:07:16.676902+0000","last_undegraded":"2023-09-23T18:07:16.676902+0000","last_fullsized":"2023-09-23T18:07:16.676902+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:04.274631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:04.274631+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":191,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.681846+0000","last_change":"2023-09-23T18:06:05.382745+0000","last_active":"2023-09-23T18:07:16.681846+0000","last_peered":"2023-09-23T18:07:16.681846+0000","last_clean":"2023-09-23T18:07:16.681846+0000","last_became_active":"2023-09-23T18:06:05.371488+0000","last_became_peered":"2023-09-23T18:06:05.371488+0000","last_unstale":"2023-09-23T18:07:16.681846+0000","last_undegraded":"2023-09-23T18:07:16.681846+0000","last_fullsized":"2023-09-23T18:07:16.681846+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:08.915006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":257,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.687327+0000","last_change":"2023-09-23T18:06:05.375138+0000","last_active":"2023-09-23T18:07:16.687327+0000","last_peered":"2023-09-23T18:07:16.687327+0000","last_clean":"2023-09-23T18:07:16.687327+0000","last_became_active":"2023-09-23T18:06:05.374819+0000","last_became_peered":"2023-09-23T18:06:05.374819+0000","last_unstale":"2023-09-23T18:07:16.687327+0000","last_undegraded":"2023-09-23T18:07:16.687327+0000","last_fullsized":"2023-09-23T18:07:16.687327+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:43.000019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"322'4384","reported_seq":4727,"reported_epoch":322,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:07:32.142364+0000","last_change":"2023-09-23T18:07:15.657831+0000","last_active":"2023-09-23T18:07:32.142364+0000","last_peered":"2023-09-23T18:07:32.142364+0000","last_clean":"2023-09-23T18:06:04.334743+0000","last_became_active":"2023-09-23T18:07:15.614815+0000","last_became_peered":"2023-09-23T18:07:15.614815+0000","last_unstale":"2023-09-23T18:07:32.142364+0000","last_undegraded":"2023-09-23T18:07:15.006367+0000","last_fullsized":"2023-09-23T18:07:13.635726+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4384,"log_dups_size":0,"ondisk_log_size":4384,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":134086656,"num_objects":2046,"num_object_clones":0,"num_object_copies":6138,"num_objects_missing_on_primary":756,"num_objects_missing":756,"num_objects_degraded":1846,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2046,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":2318,"num_write_kb":139648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":136,"num_bytes_recovered":8912896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1090},{"shards":"0,1,4,7","objects":4},{"shards":"0,4,7","objects":756},{"shards":"1,4","objects":196}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":163,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.682022+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:07:16.682022+0000","last_peered":"2023-09-23T18:07:16.682022+0000","last_clean":"2023-09-23T18:07:16.682022+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:07:16.682022+0000","last_undegraded":"2023-09-23T18:07:16.682022+0000","last_fullsized":"2023-09-23T18:07:16.682022+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":188,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:21.943405+0000","last_change":"2023-09-23T18:02:19.032009+0000","last_active":"2023-09-23T18:07:21.943405+0000","last_peered":"2023-09-23T18:07:21.943405+0000","last_clean":"2023-09-23T18:07:21.943405+0000","last_became_active":"2023-09-23T18:02:19.031646+0000","last_became_peered":"2023-09-23T18:02:19.031646+0000","last_unstale":"2023-09-23T18:07:21.943405+0000","last_undegraded":"2023-09-23T18:07:21.943405+0000","last_fullsized":"2023-09-23T18:07:21.943405+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:26:32.390672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":708,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.690195+0000","last_change":"2023-09-23T18:06:08.440953+0000","last_active":"2023-09-23T18:07:16.690195+0000","last_peered":"2023-09-23T18:07:16.690195+0000","last_clean":"2023-09-23T18:07:16.690195+0000","last_became_active":"2023-09-23T18:06:08.440604+0000","last_became_peered":"2023-09-23T18:06:08.440604+0000","last_unstale":"2023-09-23T18:07:16.690195+0000","last_undegraded":"2023-09-23T18:07:16.690195+0000","last_fullsized":"2023-09-23T18:07:16.690195+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:53.274049+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":124,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.677043+0000","last_change":"2023-09-23T18:06:51.098401+0000","last_active":"2023-09-23T18:07:16.677043+0000","last_peered":"2023-09-23T18:07:16.677043+0000","last_clean":"2023-09-23T18:07:16.677043+0000","last_became_active":"2023-09-23T18:06:51.097959+0000","last_became_peered":"2023-09-23T18:06:51.097959+0000","last_unstale":"2023-09-23T18:07:16.677043+0000","last_undegraded":"2023-09-23T18:07:16.677043+0000","last_fullsized":"2023-09-23T18:07:16.677043+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:16.234414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":216,"reported_epoch":322,"state":"active+clean","last_fresh":"2023-09-23T18:07:22.878984+0000","last_change":"2023-09-23T18:06:51.062905+0000","last_active":"2023-09-23T18:07:22.878984+0000","last_peered":"2023-09-23T18:07:22.878984+0000","last_clean":"2023-09-23T18:07:22.878984+0000","last_became_active":"2023-09-23T18:06:51.062578+0000","last_became_peered":"2023-09-23T18:06:51.062578+0000","last_unstale":"2023-09-23T18:07:22.878984+0000","last_undegraded":"2023-09-23T18:07:22.878984+0000","last_fullsized":"2023-09-23T18:07:22.878984+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:25.262818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":642,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.683889+0000","last_change":"2023-09-23T18:06:48.965770+0000","last_active":"2023-09-23T18:07:16.683889+0000","last_peered":"2023-09-23T18:07:16.683889+0000","last_clean":"2023-09-23T18:07:16.683889+0000","last_became_active":"2023-09-23T18:06:48.965429+0000","last_became_peered":"2023-09-23T18:06:48.965429+0000","last_unstale":"2023-09-23T18:07:16.683889+0000","last_undegraded":"2023-09-23T18:07:16.683889+0000","last_fullsized":"2023-09-23T18:07:16.683889+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":207,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.683089+0000","last_change":"2023-09-23T18:06:45.908664+0000","last_active":"2023-09-23T18:07:16.683089+0000","last_peered":"2023-09-23T18:07:16.683089+0000","last_clean":"2023-09-23T18:07:16.683089+0000","last_became_active":"2023-09-23T18:06:45.903588+0000","last_became_peered":"2023-09-23T18:06:45.903588+0000","last_unstale":"2023-09-23T18:07:16.683089+0000","last_undegraded":"2023-09-23T18:07:16.683089+0000","last_fullsized":"2023-09-23T18:07:16.683089+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:42.283189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":342,"reported_epoch":321,"state":"active+clean","last_fresh":"2023-09-23T18:07:16.684257+0000","last_change":"2023-09-23T18:06:05.374027+0000","last_active":"2023-09-23T18:07:16.684257+0000","last_peered":"2023-09-23T18:07:16.684257+0000","last_clean":"2023-09-23T18:07:16.684257+0000","last_became_active":"2023-09-23T18:06:05.373670+0000","last_became_peered":"2023-09-23T18:06:05.373670+0000","last_unstale":"2023-09-23T18:07:16.684257+0000","last_undegraded":"2023-09-23T18:07:16.684257+0000","last_fullsized":"2023-09-23T18:07:16.684257+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:01.220735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":4372758552,"num_objects":66724,"num_object_clones":0,"num_object_copies":200172,"num_objects_missing_on_primary":1576,"num_objects_missing":1576,"num_objects_degraded":37631,"num_objects_misplaced":6086,"num_objects_unfound":0,"num_objects_dirty":66724,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":75198,"num_write_kb":4541441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8689,"num_bytes_recovered":569442304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13632745472,"data_stored":13632733256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":117074,"ondisk_log_size":117074,"up":78,"acting":72,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6246,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":165484,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":136,"num_read_kb":258,"num_write":69,"num_write_kb":678,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11,"num_bytes_recovered":2716197,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001419,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1655128,"kb_used_data":1506960,"kb_used_omap":13,"kb_used_meta":148146,"kb_avail":92090024,"statfs":{"total":95995035648,"available":94300184576,"internally_reserved":0,"allocated":1543127040,"data_stored":1541943236,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":151701608},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":0.92900000000000005,"15min":0.71899999999999997},"min":{"1min":0.217,"5min":0.217,"15min":0.217},"max":{"1min":9.3499999999999996,"5min":9.3499999999999996,"15min":9.3499999999999996},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.626,"5min":1.0620000000000001,"15min":1.212},"min":{"1min":0.25,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":1.6659999999999999,"5min":6.641,"15min":6.641},"last":0.32100000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0109999999999999,"5min":1.0429999999999999,"15min":0.79900000000000004},"min":{"1min":0.316,"5min":0.252,"15min":0.252},"max":{"1min":4.6529999999999996,"5min":7.1150000000000002,"15min":7.1150000000000002},"last":0.33100000000000002},{"interface":"front","average":{"1min":1.2589999999999999,"5min":1.238,"15min":1.1160000000000001},"min":{"1min":0.33600000000000002,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":6.4800000000000004,"5min":6.9589999999999996,"15min":6.9589999999999996},"last":0.254}]},{"osd":2,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.089,"15min":0.98799999999999999},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":2.742,"5min":6.6630000000000003,"15min":6.6630000000000003},"last":5.048},{"interface":"front","average":{"1min":0.98099999999999998,"5min":1.1439999999999999,"15min":0.92300000000000004},"min":{"1min":0.307,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":3.7610000000000001,"5min":7.9160000000000004,"15min":7.9160000000000004},"last":5.1829999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:07:25 2023","interfaces":[{"interface":"back","average":{"1min":1.1599999999999999,"5min":1.3540000000000001,"15min":1.4870000000000001},"min":{"1min":0.35299999999999998,"5min":0.316,"15min":0.316},"max":{"1min":4.758,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":0.44},{"interface":"front","average":{"1min":1.1579999999999999,"5min":1.5209999999999999,"15min":1.153},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":4.1150000000000002,"5min":15.349,"15min":15.349},"last":0.46000000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":0.74099999999999999,"5min":1.0069999999999999,"15min":1.05},"min":{"1min":0.27800000000000002,"5min":0.245,"15min":0.245},"max":{"1min":3.944,"5min":6.9729999999999999,"15min":6.9729999999999999},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.64700000000000002,"5min":0.73799999999999999,"15min":0.59599999999999997},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.29,"5min":6.3479999999999999,"15min":6.3479999999999999},"last":0.39000000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":0.89600000000000002,"5min":1.1539999999999999,"15min":1.054},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":4.5380000000000003,"5min":9.5999999999999996,"15min":9.5999999999999996},"last":0.34799999999999998},{"interface":"front","average":{"1min":0.80200000000000005,"5min":1.2050000000000001,"15min":1.157},"min":{"1min":0.33600000000000002,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.359,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.371}]},{"osd":6,"last update":"Sat Sep 23 18:06:35 2023","interfaces":[{"interface":"back","average":{"1min":0.83799999999999997,"5min":0.97199999999999998,"15min":0.79400000000000004},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":3.3780000000000001,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.79600000000000004,"5min":1.1659999999999999,"15min":1.139},"min":{"1min":0.32700000000000001,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":2.214,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":0.40500000000000003}]}]},{"osd":6,"up_from":163,"seq":700079669355,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1700500,"kb_used_data":1486860,"kb_used_omap":7,"kb_used_meta":213624,"kb_avail":92044652,"statfs":{"total":95995035648,"available":94253723648,"internally_reserved":0,"allocated":1522544640,"data_stored":1521364839,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":218751780},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:43 2023","interfaces":[{"interface":"back","average":{"1min":0.89500000000000002,"5min":0.98299999999999998,"15min":1.2230000000000001},"min":{"1min":0.23400000000000001,"5min":0.224,"15min":0.215},"max":{"1min":2.4300000000000002,"5min":15.407,"15min":15.407},"last":0.251},{"interface":"front","average":{"1min":2.1619999999999999,"5min":1.2709999999999999,"15min":2.0110000000000001},"min":{"1min":0.309,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.147,"5min":15.147,"15min":15.147},"last":0.66800000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:06:43 2023","interfaces":[{"interface":"back","average":{"1min":1.3280000000000001,"5min":1.1579999999999999,"15min":1.7110000000000001},"min":{"1min":0.27200000000000002,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":6.5309999999999997,"5min":6.7690000000000001,"15min":14.233000000000001},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.2729999999999999,"15min":1.8839999999999999},"min":{"1min":0.26000000000000001,"5min":0.26000000000000001,"15min":0.248},"max":{"1min":3.6899999999999999,"5min":14.789,"15min":14.789},"last":0.73599999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:06:43 2023","interfaces":[{"interface":"back","average":{"1min":1.2030000000000001,"5min":1.276,"15min":2.016},"min":{"1min":0.32800000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":8.6140000000000008,"5min":8.6140000000000008,"15min":11.210000000000001},"last":3.6669999999999998},{"interface":"front","average":{"1min":1.2090000000000001,"5min":1.268,"15min":2.298},"min":{"1min":0.374,"5min":0.245,"15min":0.245},"max":{"1min":8.6300000000000008,"5min":9.3620000000000001,"15min":11.223000000000001},"last":3.6819999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":1.151,"15min":1.119},"min":{"1min":0.313,"5min":0.308,"15min":0.308},"max":{"1min":4.5890000000000004,"5min":4.7539999999999996,"15min":4.7539999999999996},"last":0.89800000000000002},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.282,"15min":1.204},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.5979999999999999,"5min":7.484,"15min":7.484},"last":1.371}]},{"osd":4,"last update":"Sat Sep 23 18:06:43 2023","interfaces":[{"interface":"back","average":{"1min":0.44400000000000001,"5min":0.90700000000000003,"15min":0.89000000000000001},"min":{"1min":0.254,"5min":0.245,"15min":0.23000000000000001},"max":{"1min":1.085,"5min":5.827,"15min":9.9109999999999996},"last":0.308},{"interface":"front","average":{"1min":0.67900000000000005,"5min":0.81699999999999995,"15min":0.61899999999999999},"min":{"1min":0.29899999999999999,"5min":0.29099999999999998,"15min":0.24099999999999999},"max":{"1min":2.202,"5min":6.3920000000000003,"15min":8.3000000000000007},"last":0.36699999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:06:43 2023","interfaces":[{"interface":"back","average":{"1min":0.54100000000000004,"5min":0.93500000000000005,"15min":0.877},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":1.329,"5min":6.6159999999999997,"15min":11.869},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.79800000000000004,"5min":0.95999999999999996,"15min":0.73699999999999999},"min":{"1min":0.33700000000000002,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":2.226,"5min":4.7930000000000001,"15min":5.4240000000000004},"last":0.433}]},{"osd":7,"last update":"Sat Sep 23 18:06:32 2023","interfaces":[{"interface":"back","average":{"1min":0.51900000000000002,"5min":0.98599999999999999,"15min":0.97099999999999997},"min":{"1min":0.32700000000000001,"5min":0.307,"15min":0.307},"max":{"1min":0.746,"5min":5.5380000000000003,"15min":5.5380000000000003},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.77300000000000002,"5min":1.01,"15min":0.85799999999999998},"min":{"1min":0.34399999999999997,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":1.861,"5min":10.411,"15min":10.411},"last":0.40799999999999997}]}]},{"osd":1,"up_from":98,"seq":420906795173,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1707324,"kb_used_data":1346228,"kb_used_omap":7,"kb_used_meta":361016,"kb_avail":92037828,"statfs":{"total":95995035648,"available":94246735872,"internally_reserved":0,"allocated":1378537472,"data_stored":1376976558,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":369681085},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,0,0,1,2,3,6],"upper_bound":1024},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.3979999999999999,"5min":0.81399999999999995,"15min":1.03},"min":{"1min":0.182,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":7.0359999999999996,"5min":7.0359999999999996,"15min":15.529999999999999},"last":2.153},{"interface":"front","average":{"1min":1.696,"5min":2.1419999999999999,"15min":1.5349999999999999},"min":{"1min":0.20100000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.5510000000000002,"5min":82.244,"15min":82.244},"last":0.39800000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.7629999999999999,"5min":1.2789999999999999,"15min":1.4119999999999999},"min":{"1min":0.22,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":10.531000000000001,"5min":10.531000000000001,"15min":16.347999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.5269999999999999,"5min":0.879,"15min":1.095},"min":{"1min":0.32500000000000001,"5min":0.23599999999999999,"15min":0.21299999999999999},"max":{"1min":6.3600000000000003,"5min":6.3600000000000003,"15min":11.305999999999999},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.1679999999999999,"15min":0.76600000000000001},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.1479999999999997,"5min":7.1479999999999997,"15min":7.1479999999999997},"last":0.311},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.2729999999999999,"15min":1.0960000000000001},"min":{"1min":0.28000000000000003,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.9640000000000004,"5min":11.125999999999999,"15min":11.125999999999999},"last":0.68100000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.0940000000000001,"5min":1.0469999999999999,"15min":1.2310000000000001},"min":{"1min":0.34499999999999997,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":3.4390000000000001,"5min":8.2639999999999993,"15min":26.568999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.2869999999999999,"5min":0.95699999999999996,"15min":1.163},"min":{"1min":0.307,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":7.0640000000000001,"5min":7.9260000000000002,"15min":26.635000000000002},"last":0.59599999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:07:16 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.5369999999999999,"15min":1.319},"min":{"1min":0.35599999999999998,"5min":0.33200000000000002,"15min":0.317},"max":{"1min":7.6260000000000003,"5min":11.638999999999999,"15min":11.638999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.2330000000000001,"15min":1.2669999999999999},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":5.3600000000000003,"5min":8.2100000000000009,"15min":9.5519999999999996},"last":0.438}]},{"osd":6,"last update":"Sat Sep 23 18:06:48 2023","interfaces":[{"interface":"back","average":{"1min":5.4340000000000002,"5min":2.036,"15min":2.0430000000000001},"min":{"1min":0.438,"5min":0.317,"15min":0.317},"max":{"1min":89.566000000000003,"5min":89.566000000000003,"15min":89.566000000000003},"last":0.46100000000000002},{"interface":"front","average":{"1min":5.8310000000000004,"5min":2.1339999999999999,"15min":1.8009999999999999},"min":{"1min":0.44800000000000001,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":90.608999999999995,"5min":90.608999999999995,"15min":90.608999999999995},"last":0.57099999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:06:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2350000000000001,"5min":0.86299999999999999,"15min":0.79000000000000004},"min":{"1min":0.39800000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":4.6310000000000002,"5min":4.6310000000000002,"15min":4.6310000000000002},"last":0.63600000000000001},{"interface":"front","average":{"1min":1.492,"5min":1.542,"15min":1.732},"min":{"1min":0.45200000000000001,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":5.5839999999999996,"5min":9.5709999999999997,"15min":9.5709999999999997},"last":0.64900000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958700,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2575400,"kb_used_data":2120992,"kb_used_omap":7,"kb_used_meta":454392,"kb_avail":91169752,"statfs":{"total":95995035648,"available":93357826048,"internally_reserved":0,"allocated":2171895808,"data_stored":2170292578,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":465298148},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:07:19 2023","interfaces":[{"interface":"back","average":{"1min":2.0270000000000001,"5min":1.4770000000000001,"15min":1.4610000000000001},"min":{"1min":0.26100000000000001,"5min":0.20300000000000001,"15min":0.17899999999999999},"max":{"1min":15.836,"5min":15.836,"15min":18.154},"last":0.22},{"interface":"front","average":{"1min":1.895,"5min":1.1499999999999999,"15min":1.3240000000000001},"min":{"1min":0.29399999999999998,"5min":0.26800000000000002,"15min":0.17399999999999999},"max":{"1min":14.468999999999999,"5min":14.468999999999999,"15min":14.468999999999999},"last":3.677}]},{"osd":2,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.417,"15min":1.54},"min":{"1min":0.311,"5min":0.21099999999999999,"15min":0.20499999999999999},"max":{"1min":10.007999999999999,"5min":10.48,"15min":20.882000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.5600000000000001,"5min":1.3580000000000001,"15min":1.4219999999999999},"min":{"1min":0.24399999999999999,"5min":0.217,"15min":0.217},"max":{"1min":10.390000000000001,"5min":10.390000000000001,"15min":15.064},"last":3.7229999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":2.1440000000000001,"5min":1.659,"15min":1.5880000000000001},"min":{"1min":0.29199999999999998,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":12.779999999999999,"5min":12.779999999999999,"15min":12.779999999999999},"last":0.496},{"interface":"front","average":{"1min":1.506,"5min":1.5640000000000001,"15min":1.3220000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":8.4730000000000008,"5min":15.198,"15min":15.198},"last":0.29599999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":1.385,"15min":1.2949999999999999},"min":{"1min":0.32100000000000001,"5min":0.29299999999999998,"15min":0.251},"max":{"1min":6.9889999999999999,"5min":6.9889999999999999,"15min":15.279999999999999},"last":0.379},{"interface":"front","average":{"1min":2.2400000000000002,"5min":1.613,"15min":1.411},"min":{"1min":0.35299999999999998,"5min":0.247,"15min":0.247},"max":{"1min":15.477,"5min":15.477,"15min":18.106999999999999},"last":0.47399999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.363,"5min":1.534,"15min":1.4299999999999999},"min":{"1min":0.29799999999999999,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":7.0330000000000004,"5min":10.194000000000001,"15min":14.720000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":1.726,"5min":1.238,"15min":1.21},"min":{"1min":0.32600000000000001,"5min":0.254,"15min":0.254},"max":{"1min":10.363,"5min":10.363,"15min":12.794},"last":3.766}]},{"osd":6,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.6910000000000001,"5min":1.3340000000000001,"15min":2.0009999999999999},"min":{"1min":0.35199999999999998,"5min":0.35099999999999998,"15min":0.34599999999999997},"max":{"1min":7.056,"5min":7.056,"15min":14.956},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.198,"15min":2.1280000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":10.321,"5min":10.321,"15min":12.755000000000001},"last":3.835}]},{"osd":7,"last update":"Sat Sep 23 18:06:31 2023","interfaces":[{"interface":"back","average":{"1min":1.4390000000000001,"5min":1.139,"15min":0.995},"min":{"1min":0.38500000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":9.3010000000000002,"5min":9.3010000000000002,"15min":9.3010000000000002},"last":3.8029999999999999},{"interface":"front","average":{"1min":1.202,"5min":1.409,"15min":1.1910000000000001},"min":{"1min":0.40600000000000003,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":4.6070000000000002,"5min":5.524,"15min":5.524},"last":0.46300000000000002}]}]},{"osd":2,"up_from":103,"seq":442381631648,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2415396,"kb_used_data":1966364,"kb_used_omap":11,"kb_used_meta":449012,"kb_avail":91329756,"statfs":{"total":95995035648,"available":93521670144,"internally_reserved":0,"allocated":2013556736,"data_stored":2011964498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":459788663},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":4.4960000000000004,"5min":1.7090000000000001,"15min":1.708},"min":{"1min":0.219,"5min":0.187,"15min":0.17199999999999999},"max":{"1min":36.656999999999996,"5min":36.656999999999996,"15min":36.656999999999996},"last":0.36299999999999999},{"interface":"front","average":{"1min":3.9820000000000002,"5min":1.5600000000000001,"15min":1.627},"min":{"1min":0.252,"5min":0.24299999999999999,"15min":0.215},"max":{"1min":36.697000000000003,"5min":36.697000000000003,"15min":36.697000000000003},"last":0.50700000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":2.4100000000000001,"5min":1.23,"15min":1.607},"min":{"1min":0.34799999999999998,"5min":0.215,"15min":0.215},"max":{"1min":8.2010000000000005,"5min":8.2010000000000005,"15min":20.995000000000001},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.2320000000000002,"5min":1.1180000000000001,"15min":1.3819999999999999},"min":{"1min":0.252,"5min":0.24199999999999999,"15min":0.193},"max":{"1min":8.3889999999999993,"5min":8.3889999999999993,"15min":15.945},"last":0.49399999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:07:31 2023","interfaces":[{"interface":"back","average":{"1min":2.5539999999999998,"5min":1.7490000000000001,"15min":1.1040000000000001},"min":{"1min":0.30199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":9.3469999999999995,"5min":37.500999999999998,"15min":37.500999999999998},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.4489999999999998,"5min":1.7889999999999999,"15min":1.0029999999999999},"min":{"1min":0.28299999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.315,"5min":34.915999999999997,"15min":34.915999999999997},"last":1.2629999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7370000000000001,"5min":1.204,"15min":1.278},"min":{"1min":0.311,"5min":0.29099999999999998,"15min":0.26500000000000001},"max":{"1min":13.016999999999999,"5min":13.016999999999999,"15min":13.016999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.8089999999999999,"5min":1.0489999999999999,"15min":1.2769999999999999},"min":{"1min":0.34999999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":8.0820000000000007,"5min":8.0820000000000007,"15min":20.920000000000002},"last":0.57399999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.847,"5min":1.097,"15min":1.345},"min":{"1min":0.32500000000000001,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":8.1760000000000002,"5min":8.1760000000000002,"15min":20.738},"last":0.624},{"interface":"front","average":{"1min":2.1259999999999999,"5min":1.4790000000000001,"15min":1.4590000000000001},"min":{"1min":0.47099999999999997,"5min":0.32000000000000001,"15min":0.26700000000000002},"max":{"1min":6.9930000000000003,"5min":12.103999999999999,"15min":15.496},"last":0.39000000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:06:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7609999999999999,"5min":1.2709999999999999,"15min":1.9610000000000001},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":13.065,"5min":13.065,"15min":13.065},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.871,"5min":1.2210000000000001,"15min":2.169},"min":{"1min":0.45900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":8.4120000000000008,"5min":8.4649999999999999,"15min":20.893000000000001},"last":0.66700000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:06:33 2023","interfaces":[{"interface":"back","average":{"1min":1.6220000000000001,"5min":1.3740000000000001,"15min":1.3979999999999999},"min":{"1min":0.42899999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":6.9500000000000002,"5min":12.051,"15min":12.051},"last":0.58699999999999997},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.181,"15min":0.81499999999999995},"min":{"1min":0.52100000000000002,"5min":0.309,"15min":0.309},"max":{"1min":8.1479999999999997,"5min":8.5039999999999996,"15min":8.5039999999999996},"last":0.65000000000000002}]}]},{"osd":3,"up_from":237,"seq":1017907249215,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2625700,"kb_used_data":2376540,"kb_used_omap":3,"kb_used_meta":249148,"kb_avail":91119452,"statfs":{"total":95995035648,"available":93306318848,"internally_reserved":0,"allocated":2433576960,"data_stored":2431807039,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":255128296},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.874,"15min":0.88},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.532,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":2.306},{"interface":"front","average":{"1min":1.9279999999999999,"5min":1.2969999999999999,"15min":1.153},"min":{"1min":0.22,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.7329999999999997,"5min":14.885999999999999,"15min":14.885999999999999},"last":0.80100000000000005}]},{"osd":1,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":1.486,"15min":1.202},"min":{"1min":0.34699999999999998,"5min":0.245,"15min":0.245},"max":{"1min":7.9619999999999997,"5min":21.931999999999999,"15min":21.931999999999999},"last":0.94599999999999995},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.085,"15min":0.99199999999999999},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.3109999999999999,"5min":13.631,"15min":13.631},"last":2.1219999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.90400000000000003,"15min":0.86799999999999999},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.004,"5min":8.2200000000000006,"15min":8.2200000000000006},"last":2.3959999999999999},{"interface":"front","average":{"1min":1.8400000000000001,"5min":1.4259999999999999,"15min":1.306},"min":{"1min":0.33900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.0990000000000002,"5min":21.988,"15min":21.988},"last":0.83399999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":1.347,"15min":1.1850000000000001},"min":{"1min":0.379,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.8959999999999999,"5min":14.935,"15min":14.935},"last":0.71999999999999997},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.2569999999999999,"15min":1.131},"min":{"1min":0.34499999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.3540000000000001,"5min":12.786,"15min":12.786},"last":0.82199999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.2749999999999999,"15min":1.2809999999999999},"min":{"1min":0.38700000000000001,"5min":0.307,"15min":0.307},"max":{"1min":6.3879999999999999,"5min":12.743,"15min":12.743},"last":0.751},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.1319999999999999,"15min":1.1639999999999999},"min":{"1min":0.36699999999999999,"5min":0.315,"15min":0.315},"max":{"1min":2.9470000000000001,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":2.3530000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.0600000000000001,"15min":1.0760000000000001},"min":{"1min":0.40899999999999997,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":2.972,"5min":7.5789999999999997,"15min":7.5789999999999997},"last":2.3759999999999999},{"interface":"front","average":{"1min":1.714,"5min":1.5109999999999999,"15min":1.49},"min":{"1min":0.432,"5min":0.379,"15min":0.379},"max":{"1min":7.9349999999999996,"5min":14.964,"15min":14.964},"last":0.76200000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.397,"15min":1.5129999999999999},"min":{"1min":0.42199999999999999,"5min":0.377,"15min":0.377},"max":{"1min":6.4189999999999996,"5min":12.821,"15min":12.821},"last":0.78100000000000003},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.3089999999999999,"15min":1.516},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":2.911,"5min":8.6039999999999992,"15min":8.6039999999999992},"last":3.5190000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019164,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1696088,"kb_used_data":1416272,"kb_used_omap":0,"kb_used_meta":279808,"kb_avail":92049064,"statfs":{"total":95995035648,"available":94258241536,"internally_reserved":0,"allocated":1450262528,"data_stored":1449078987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":286523392},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,3,0,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:41 2023","interfaces":[{"interface":"back","average":{"1min":1.8779999999999999,"5min":1.0349999999999999,"15min":0.97799999999999998},"min":{"1min":0.26600000000000001,"5min":0.215,"15min":0.20100000000000001},"max":{"1min":13.375999999999999,"5min":13.375999999999999,"15min":14.164},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.829,"5min":1.3400000000000001,"15min":1.2090000000000001},"min":{"1min":0.28999999999999998,"5min":0.253,"15min":0.22600000000000001},"max":{"1min":5.7809999999999997,"5min":7.1879999999999997,"15min":11.372},"last":0.35899999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:07:11 2023","interfaces":[{"interface":"back","average":{"1min":1.506,"5min":1.147,"15min":1.161},"min":{"1min":0.27900000000000003,"5min":0.26400000000000001,"15min":0.249},"max":{"1min":7.6059999999999999,"5min":7.6059999999999999,"15min":10.429},"last":0.45900000000000002},{"interface":"front","average":{"1min":1.081,"5min":1.0780000000000001,"15min":1.1950000000000001},"min":{"1min":0.27100000000000002,"5min":0.27100000000000002,"15min":0.23100000000000001},"max":{"1min":3.4129999999999998,"5min":6.7640000000000002,"15min":10.83},"last":0.52200000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:06:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0680000000000001,"5min":1.1080000000000001,"15min":1.1850000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.26600000000000001},"max":{"1min":4.5060000000000002,"5min":6.8259999999999996,"15min":11.630000000000001},"last":0.41699999999999998},{"interface":"front","average":{"1min":1.8660000000000001,"5min":1.2949999999999999,"15min":1.1919999999999999},"min":{"1min":0.314,"5min":0.314,"15min":0.27100000000000002},"max":{"1min":13.672000000000001,"5min":13.672000000000001,"15min":13.672000000000001},"last":0.46899999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:07:24 2023","interfaces":[{"interface":"back","average":{"1min":1.8700000000000001,"5min":1.746,"15min":1.482},"min":{"1min":0.36599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":13.462999999999999,"5min":29.154,"15min":29.154},"last":0.495},{"interface":"front","average":{"1min":1.7569999999999999,"5min":1.7090000000000001,"15min":1.4850000000000001},"min":{"1min":0.34799999999999998,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":12.938000000000001,"5min":29.216000000000001,"15min":29.216000000000001},"last":0.40400000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.95799999999999996,"15min":0.73999999999999999},"min":{"1min":0.26800000000000002,"5min":0.23799999999999999,"15min":0.224},"max":{"1min":2.5459999999999998,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.246,"15min":0.93500000000000005},"min":{"1min":0.308,"5min":0.26800000000000002,"15min":0.26100000000000001},"max":{"1min":3.2410000000000001,"5min":11.170999999999999,"15min":11.170999999999999},"last":0.317}]},{"osd":6,"last update":"Sat Sep 23 18:06:49 2023","interfaces":[{"interface":"back","average":{"1min":0.64700000000000002,"5min":1.0469999999999999,"15min":0.89900000000000002},"min":{"1min":0.29599999999999999,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":2.282,"5min":9.0990000000000002,"15min":9.0990000000000002},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.0089999999999999,"15min":0.98399999999999999},"min":{"1min":0.33400000000000002,"5min":0.28999999999999998,"15min":0.28399999999999997},"max":{"1min":2.335,"5min":9.1300000000000008,"15min":10.725},"last":0.42299999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:06:32 2023","interfaces":[{"interface":"back","average":{"1min":0.70599999999999996,"5min":1.242,"15min":1.0229999999999999},"min":{"1min":0.32600000000000001,"5min":0.314,"15min":0.314},"max":{"1min":3.427,"5min":13.298999999999999,"15min":13.298999999999999},"last":2.4489999999999998},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.3740000000000001,"15min":1.266},"min":{"1min":0.375,"5min":0.307,"15min":0.307},"max":{"1min":2.0110000000000001,"5min":10.192,"15min":10.192},"last":0.5}]}]},{"osd":5,"up_from":38,"seq":163208757532,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1414664,"kb_used_data":1158336,"kb_used_omap":0,"kb_used_meta":256320,"kb_avail":92330488,"statfs":{"total":95995035648,"available":94546419712,"internally_reserved":0,"allocated":1186136064,"data_stored":1184964337,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":262471680},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3200000000000001,"5min":0.85499999999999998,"15min":1.073},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.217},"max":{"1min":5.4539999999999997,"5min":5.4539999999999997,"15min":21.376999999999999},"last":1.3500000000000001},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.2589999999999999,"15min":1.2789999999999999},"min":{"1min":0.32000000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":5.5309999999999997,"5min":8.5099999999999998,"15min":21.388999999999999},"last":1.363}]},{"osd":1,"last update":"Sat Sep 23 18:07:15 2023","interfaces":[{"interface":"back","average":{"1min":1.0669999999999999,"5min":0.81899999999999995,"15min":0.90800000000000003},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":7.6479999999999997,"5min":7.6479999999999997,"15min":10.853999999999999},"last":1.444},{"interface":"front","average":{"1min":1.054,"5min":1.1220000000000001,"15min":1.107},"min":{"1min":0.30099999999999999,"5min":0.28000000000000003,"15min":0.254},"max":{"1min":8.9749999999999996,"5min":8.9749999999999996,"15min":10.206},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:06:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.145,"15min":1.377},"min":{"1min":0.44600000000000001,"5min":0.28999999999999998,"15min":0.255},"max":{"1min":3.234,"5min":5.5309999999999997,"15min":38.606999999999999},"last":1.5620000000000001},{"interface":"front","average":{"1min":1.395,"5min":1.0820000000000001,"15min":1.2809999999999999},"min":{"1min":0.39900000000000002,"5min":0.308,"15min":0.30099999999999999},"max":{"1min":6.2489999999999997,"5min":10.065,"15min":39.670999999999999},"last":0.47299999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.097,"15min":0.877},"min":{"1min":0.35499999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.6440000000000001,"5min":8.593,"15min":8.593},"last":3.3660000000000001},{"interface":"front","average":{"1min":0.77000000000000002,"5min":1.0529999999999999,"15min":0.82499999999999996},"min":{"1min":0.40899999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.1459999999999999,"5min":13.446999999999999,"15min":13.446999999999999},"last":3.3220000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:06:53 2023","interfaces":[{"interface":"back","average":{"1min":0.68600000000000005,"5min":1.0109999999999999,"15min":0.80500000000000005},"min":{"1min":0.25700000000000001,"5min":0.24199999999999999,"15min":0.224},"max":{"1min":2.7229999999999999,"5min":8.6229999999999993,"15min":10.092000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.91000000000000003,"15min":0.77100000000000002},"min":{"1min":0.30099999999999999,"5min":0.29499999999999998,"15min":0.23999999999999999},"max":{"1min":1.6850000000000001,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.39800000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:06:53 2023","interfaces":[{"interface":"back","average":{"1min":0.70799999999999996,"5min":0.94499999999999995,"15min":1.2969999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":2.0659999999999998,"5min":6.5880000000000001,"15min":11.287000000000001},"last":0.439},{"interface":"front","average":{"1min":0.79900000000000004,"5min":0.89800000000000002,"15min":1.1830000000000001},"min":{"1min":0.32100000000000001,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":2.0139999999999998,"5min":6.6920000000000002,"15min":8.266},"last":0.51000000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:06:33 2023","interfaces":[{"interface":"back","average":{"1min":0.72199999999999998,"5min":1.133,"15min":1.4079999999999999},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":2.0299999999999998,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.58699999999999997},{"interface":"front","average":{"1min":0.69999999999999996,"5min":1.133,"15min":1.127},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":2.0760000000000001,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.54300000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2160988160,"data_stored":2160984088,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1371344896,"data_stored":1371340824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":2003566592,"data_stored":2003566592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2423324672,"data_stored":2423324672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1444151296,"data_stored":1444151296,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1180565504,"data_stored":1180565504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1512968192,"data_stored":1512964120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1535836160,"data_stored":1535836160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:07:35.451 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 31 2023-09-23T18:07:35.496 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:35.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:35.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:35.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:35.637 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:35.681 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:35.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:35.686 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:35.699 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:07:35.787 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:35.872 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:35.895 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:07:35.928 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:35.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:35.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:36.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:36.051 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:36 smithi099 ceph-mon[167076]: pgmap v791: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 506 KiB/s wr, 412 op/s; 37651/200253 objects degraded (18.802%); 6086/200253 objects misplaced (3.039%); 11 MiB/s, 182 objects/s recovering 2023-09-23T18:07:36.052 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:36 smithi099 ceph-mon[167076]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:07:36.071 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:36.117 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:36.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:36.121 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:36.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:36 smithi055 ceph-mon[184174]: pgmap v791: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 506 KiB/s wr, 412 op/s; 37651/200253 objects degraded (18.802%); 6086/200253 objects misplaced (3.039%); 11 MiB/s, 182 objects/s recovering 2023-09-23T18:07:36.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:36 smithi055 ceph-mon[184174]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:07:36.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:36 smithi055 ceph-mon[179668]: pgmap v791: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.1 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 102 B/s rd, 506 KiB/s wr, 412 op/s; 37651/200253 objects degraded (18.802%); 6086/200253 objects misplaced (3.039%); 11 MiB/s, 182 objects/s recovering 2023-09-23T18:07:36.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:36 smithi055 ceph-mon[179668]: from='client.35925 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:07:36.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:36.252 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:36.283 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:36.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:36.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:36.305 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:36.406 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:36.422 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:36.472 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:36.478 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:07:36.589 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:36.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:36.601 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:36.701 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:36.759 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:36.833 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:36.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:36.838 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:36.939 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:36.992 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:37.054 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:37.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:37.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:37.159 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:37.204 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:37.266 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:37.271 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:37.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:37.273 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:37.373 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:37.420 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:37.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:37.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:37.521 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:37.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:37.527 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:37.598 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:37.598 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:37.628 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:37.700 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:37.732 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:07:37.732 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:07:37.824 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:37.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:37.829 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:37.929 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:37.989 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:38.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:38 smithi055 ceph-mon[179668]: pgmap v792: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 37086/198057 objects degraded (18.725%); 6086/198057 objects misplaced (3.073%); 13 MiB/s, 207 objects/s recovering 2023-09-23T18:07:38.112 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 37086/198057 objects degraded (18.725%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:38.112 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:38 smithi055 ceph-mon[184174]: pgmap v792: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 37086/198057 objects degraded (18.725%); 6086/198057 objects misplaced (3.073%); 13 MiB/s, 207 objects/s recovering 2023-09-23T18:07:38.112 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 37086/198057 objects degraded (18.725%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:38.112 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:38.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:38.118 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:38.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:07:38.134 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:38 smithi099 ceph-mon[167076]: pgmap v792: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 37086/198057 objects degraded (18.725%); 6086/198057 objects misplaced (3.073%); 13 MiB/s, 207 objects/s recovering 2023-09-23T18:07:38.134 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 37086/198057 objects degraded (18.725%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:38.219 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:38.280 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:38.435 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:38.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:38.443 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:38.543 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:38.581 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:38.623 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:38.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:38.628 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:38.729 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:38.756 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:38.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:38.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:38.794 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:38.801 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:38.895 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:38.924 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:39.062 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:39.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:39.070 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:39.107 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 1 pgp_num to 31 2023-09-23T18:07:39.170 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:39.172 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3598124057' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T18:07:39.173 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3598124057' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T18:07:39.218 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3598124057' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]: dispatch 2023-09-23T18:07:39.258 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:39.319 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:39.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:39.327 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:39.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:39.458 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:39.574 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:39.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:39.580 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:39.680 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:39.713 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:39.825 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:39.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:39.829 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:39.930 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:39.970 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:39.971 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:07:40.009 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:07:40.025 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:40.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:40.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:40.048 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:07:40.083 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:07:40.119 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:07:40.120 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:07:40.132 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:40.155 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:07:40.158 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:07:40.158 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:07:40.164 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:40 smithi099 ceph-mon[167076]: pgmap v793: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 36710/195036 objects degraded (18.822%); 5650/195036 objects misplaced (2.897%); 13 MiB/s, 202 objects/s recovering 2023-09-23T18:07:40.164 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:40 smithi099 ceph-mon[167076]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.164 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:40 smithi099 ceph-mon[167076]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.164 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:40 smithi099 ceph-mon[167076]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.164 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:40 smithi099 ceph-mon[167076]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.164 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3598124057' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T18:07:40.164 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:40 smithi099 ceph-mon[167076]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T18:07:40.165 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1813220224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:07:40.165 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:40 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:07:40.165 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/352812895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:40.170 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:40.195 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:07:40.209 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:40.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:40.213 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:40.225 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:07:40.227 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:07:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:07:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:07:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:07:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:07:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:07:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:07:40.228 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:07:40.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:40.342 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:40.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[184174]: pgmap v793: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 36710/195036 objects degraded (18.822%); 5650/195036 objects misplaced (2.897%); 13 MiB/s, 202 objects/s recovering 2023-09-23T18:07:40.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[184174]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[184174]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[184174]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[184174]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3598124057' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T18:07:40.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[184174]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T18:07:40.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1813220224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:07:40.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:07:40.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/352812895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:40.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[179668]: pgmap v793: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.0 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 419 op/s; 36710/195036 objects degraded (18.822%); 5650/195036 objects misplaced (2.897%); 13 MiB/s, 202 objects/s recovering 2023-09-23T18:07:40.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[179668]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[179668]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[179668]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[179668]: failed to encode map e324 with expected crc 2023-09-23T18:07:40.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3598124057' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "31"}]': finished 2023-09-23T18:07:40.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[179668]: osdmap e324: 8 total, 8 up, 8 in 2023-09-23T18:07:40.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1813220224' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:07:40.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:07:40.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/352812895' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:40.662 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:40.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:40.666 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:40.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:07:40.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:07:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:07:40] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.43.0" 2023-09-23T18:07:40.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:40.796 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:40.832 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:40.835 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:40.835 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:40.936 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:40.974 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:41.021 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:41.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:41.025 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:41.126 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:41.181 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:41.224 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:41.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:41.232 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:41.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:41.362 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:41 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:41.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:41 smithi099 ceph-mon[167076]: failed to encode map e325 with expected crc 2023-09-23T18:07:41.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:41 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:07:41.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:41 smithi099 ceph-mon[167076]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T18:07:41.363 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:41 smithi099 ceph-mon[167076]: failed to encode map e325 with expected crc 2023-09-23T18:07:41.365 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:41.389 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:41.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:41.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:41.416 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:41 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:41 smithi055 ceph-mon[184174]: failed to encode map e325 with expected crc 2023-09-23T18:07:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:41 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:07:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:41 smithi055 ceph-mon[184174]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T18:07:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:41 smithi055 ceph-mon[184174]: failed to encode map e325 with expected crc 2023-09-23T18:07:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:41 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:41 smithi055 ceph-mon[179668]: failed to encode map e325 with expected crc 2023-09-23T18:07:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:41 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:07:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:41 smithi055 ceph-mon[179668]: osdmap e325: 8 total, 8 up, 8 in 2023-09-23T18:07:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:41 smithi055 ceph-mon[179668]: failed to encode map e325 with expected crc 2023-09-23T18:07:41.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:41.545 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:41.655 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:41.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:41.661 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:41.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:41.797 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:41.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:41.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:41.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:41.949 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:41.994 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:42.037 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:42.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:42.045 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:42.146 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:42.183 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:42.232 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:42.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:42.237 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:42.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:42 smithi099 ceph-mon[167076]: pgmap v796: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 393 op/s; 36047/191844 objects degraded (18.790%); 5650/191844 objects misplaced (2.945%); 13 MiB/s, 202 objects/s recovering 2023-09-23T18:07:42.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:42.370 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:42.416 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:42.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:42.422 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:42.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:42 smithi055 ceph-mon[184174]: pgmap v796: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 393 op/s; 36047/191844 objects degraded (18.790%); 5650/191844 objects misplaced (2.945%); 13 MiB/s, 202 objects/s recovering 2023-09-23T18:07:42.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:42 smithi055 ceph-mon[179668]: pgmap v796: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 393 op/s; 36047/191844 objects degraded (18.790%); 5650/191844 objects misplaced (2.945%); 13 MiB/s, 202 objects/s recovering 2023-09-23T18:07:42.522 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:42.554 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:42.696 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:42.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:42.703 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:42.730 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:07:42.804 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:42.835 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:07:42.843 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:42.904 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:42.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:42.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:42.922 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:07:42.922 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:07:42.922 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:07:42.922 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:07:43.011 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:43.042 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:43.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 36047/191844 objects degraded (18.790%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 36047/191844 objects degraded (18.790%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:43.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 36047/191844 objects degraded (18.790%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:43.532 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:43.556 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:43.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:43.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:43.727 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:43.787 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:07:43.827 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:43.861 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:43.964 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:43.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:43.969 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:44.069 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:44.096 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:44.136 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:44.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:44.140 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:44.241 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:44.276 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:44.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:44.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:44.322 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:44.393 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:44 smithi055 ceph-mon[179668]: pgmap v797: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 35800/189258 objects degraded (18.916%); 5154/189258 objects misplaced (2.723%); 16 MiB/s, 257 objects/s recovering 2023-09-23T18:07:44.393 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:44 smithi055 ceph-mon[184174]: pgmap v797: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 35800/189258 objects degraded (18.916%); 5154/189258 objects misplaced (2.723%); 16 MiB/s, 257 objects/s recovering 2023-09-23T18:07:44.423 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:44.466 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:44.496 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:44.561 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:44.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:44.573 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:44.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:44 smithi099 ceph-mon[167076]: pgmap v797: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.9 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 35800/189258 objects degraded (18.916%); 5154/189258 objects misplaced (2.723%); 16 MiB/s, 257 objects/s recovering 2023-09-23T18:07:44.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:44.708 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:07:44.708 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:07:44.708 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T18:07:44.708 INFO:tasks.thrashosds.thrasher:Setting osd 3 primary_affinity to 0.352858 2023-09-23T18:07:44.709 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 0.3528578048630462 2023-09-23T18:07:44.733 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:44.783 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:44.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:44.787 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:44.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:44.924 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:44.968 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:44.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:44.972 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:45.073 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:45.144 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:45.203 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:45.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:45.209 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:45.309 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:45.378 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:45.441 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:45.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:45.445 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:45.546 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:45.609 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:45.717 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:45.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:45.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:45.832 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:45.834 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:07:45.835 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:07:45.835 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:07:45.835 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:07:45.835 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:07:45.908 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:45.959 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:45.985 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:45.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:45.991 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:46.027 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:07:46.066 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:07:46.092 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:46.108 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:07:46.145 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:46.151 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:07:46.181 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:07:46.202 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:07:46.259 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:07:46.316 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:07:46.328 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:46 smithi055 ceph-mon[179668]: pgmap v798: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 35669/188115 objects degraded (18.961%); 5154/188115 objects misplaced (2.740%); 14 MiB/s, 219 objects/s recovering 2023-09-23T18:07:46.328 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1589299787' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:46.329 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1733870455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:46.329 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:46 smithi055 ceph-mon[184174]: pgmap v798: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 35669/188115 objects degraded (18.961%); 5154/188115 objects misplaced (2.740%); 14 MiB/s, 219 objects/s recovering 2023-09-23T18:07:46.329 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1589299787' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:46.329 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1733870455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:46.370 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:07:46.390 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:07:46.390 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:07:46.390 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:07:46.390 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:07:46.390 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:07:46.390 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:07:46.390 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:07:46.390 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:07:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:46 smithi099 ceph-mon[167076]: pgmap v798: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.8 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 414 op/s; 35669/188115 objects degraded (18.961%); 5154/188115 objects misplaced (2.740%); 14 MiB/s, 219 objects/s recovering 2023-09-23T18:07:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1589299787' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:46.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1733870455' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:46.938 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:46.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:07:46.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:46.942 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:47.043 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:47.077 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:47.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:47.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:47.135 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:47.236 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:47.280 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:47 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:07:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:47 smithi099 ceph-mon[167076]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1589299787' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:07:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:47 smithi099 ceph-mon[167076]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T18:07:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:47 smithi099 ceph-mon[167076]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:47 smithi099 ceph-mon[167076]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:47 smithi099 ceph-mon[167076]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.329 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:47.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:47.336 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:07:47.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[179668]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1589299787' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:07:47.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[179668]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T18:07:47.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[179668]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[179668]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[179668]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:07:47.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[184174]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1589299787' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:07:47.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[184174]: osdmap e326: 8 total, 8 up, 8 in 2023-09-23T18:07:47.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[184174]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[184174]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:47 smithi055 ceph-mon[184174]: failed to encode map e326 with expected crc 2023-09-23T18:07:47.437 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:47.479 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:47.532 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:47.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:47.537 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:47.604 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:47.638 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:47.701 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:47.765 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:47.769 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:47.769 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:47.870 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:47.906 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:48.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:48.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:48.062 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:48.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:48.201 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.3 primary-affinity to 0.352858 (8231242) 2023-09-23T18:07:48.206 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:48.470 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:48 smithi099 ceph-mon[167076]: pgmap v800: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 35007/185751 objects degraded (18.846%); 5154/185751 objects misplaced (2.775%); 14 MiB/s, 217 objects/s recovering 2023-09-23T18:07:48.470 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:48 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/143649946' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3528578048630462}]: dispatch 2023-09-23T18:07:48.470 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:48 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3528578048630462}]: dispatch 2023-09-23T18:07:48.470 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 35007/185751 objects degraded (18.846%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:48.475 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:48 smithi055 ceph-mon[179668]: pgmap v800: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 35007/185751 objects degraded (18.846%); 5154/185751 objects misplaced (2.775%); 14 MiB/s, 217 objects/s recovering 2023-09-23T18:07:48.475 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:48 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/143649946' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3528578048630462}]: dispatch 2023-09-23T18:07:48.475 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:48 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3528578048630462}]: dispatch 2023-09-23T18:07:48.475 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 35007/185751 objects degraded (18.846%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:48.476 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:48 smithi055 ceph-mon[184174]: pgmap v800: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 401 op/s; 35007/185751 objects degraded (18.846%); 5154/185751 objects misplaced (2.775%); 14 MiB/s, 217 objects/s recovering 2023-09-23T18:07:48.476 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:48 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/143649946' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3528578048630462}]: dispatch 2023-09-23T18:07:48.476 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:48 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3528578048630462}]: dispatch 2023-09-23T18:07:48.476 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 35007/185751 objects degraded (18.846%), 20 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:07:48.483 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:48.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:48.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:48.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:48.616 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:48.736 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:07:48.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:48.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:48.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:48.888 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:48.917 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:48.961 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:48.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:48.964 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:49.065 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:49.094 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:49.108 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:07:49.109 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:07:49.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:49.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:49.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:49.150 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:07:49.234 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:49.377 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:49.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:49 smithi055 ceph-mon[179668]: failed to encode map e327 with expected crc 2023-09-23T18:07:49.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:49 smithi055 ceph-mon[179668]: failed to encode map e327 with expected crc 2023-09-23T18:07:49.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:49 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3528578048630462}]': finished 2023-09-23T18:07:49.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:49 smithi055 ceph-mon[179668]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T18:07:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:49 smithi055 ceph-mon[184174]: failed to encode map e327 with expected crc 2023-09-23T18:07:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:49 smithi055 ceph-mon[184174]: failed to encode map e327 with expected crc 2023-09-23T18:07:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:49 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3528578048630462}]': finished 2023-09-23T18:07:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:49 smithi055 ceph-mon[184174]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T18:07:49.442 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:49.445 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:49.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:49.447 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:49.466 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:49 smithi099 ceph-mon[167076]: failed to encode map e327 with expected crc 2023-09-23T18:07:49.466 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:49 smithi099 ceph-mon[167076]: failed to encode map e327 with expected crc 2023-09-23T18:07:49.466 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:49 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 0.3528578048630462}]': finished 2023-09-23T18:07:49.466 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:49 smithi099 ceph-mon[167076]: osdmap e327: 8 total, 8 up, 8 in 2023-09-23T18:07:49.490 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:07:49.548 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:49.642 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:49.767 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:49.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:49.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:49.877 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:49.928 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:49.973 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:49.980 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:49.980 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:50.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:50.111 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:50.156 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:50.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:50.160 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:50.261 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:50.311 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:50.367 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:50.392 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:07:50] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.43.0" 2023-09-23T18:07:50.393 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[179668]: pgmap v802: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 384 op/s; 34607/182619 objects degraded (18.950%); 4638/182619 objects misplaced (2.540%); 14 MiB/s, 224 objects/s recovering 2023-09-23T18:07:50.393 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[179668]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.393 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[179668]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.393 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[179668]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.393 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[179668]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T18:07:50.393 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[179668]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.394 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[179668]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.394 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[179668]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.394 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[179668]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.395 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[184174]: pgmap v802: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 384 op/s; 34607/182619 objects degraded (18.950%); 4638/182619 objects misplaced (2.540%); 14 MiB/s, 224 objects/s recovering 2023-09-23T18:07:50.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[184174]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[184174]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[184174]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[184174]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T18:07:50.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[184174]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[184174]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[184174]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:50 smithi055 ceph-mon[184174]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.406 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:50.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:50.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:50.517 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:50.550 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:50 smithi099 ceph-mon[167076]: pgmap v802: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 384 op/s; 34607/182619 objects degraded (18.950%); 4638/182619 objects misplaced (2.540%); 14 MiB/s, 224 objects/s recovering 2023-09-23T18:07:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:50 smithi099 ceph-mon[167076]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:50 smithi099 ceph-mon[167076]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:50 smithi099 ceph-mon[167076]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:50 smithi099 ceph-mon[167076]: osdmap e328: 8 total, 8 up, 8 in 2023-09-23T18:07:50.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:50 smithi099 ceph-mon[167076]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:50 smithi099 ceph-mon[167076]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:50 smithi099 ceph-mon[167076]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:50 smithi099 ceph-mon[167076]: failed to encode map e328 with expected crc 2023-09-23T18:07:50.700 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:50.743 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:50.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:50.760 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:50.860 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:50.889 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:50.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:50.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:50.929 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:51.030 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:51.057 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:51.091 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:51.094 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:51.094 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:51.195 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:51.225 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:51.264 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:51.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:51.268 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:51.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:51.401 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:51.444 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:51.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:51.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:51.550 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:51.582 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:51.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:51.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:51.621 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:51.721 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:51.803 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:51.884 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:51.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:51.888 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:51.989 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:52.070 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:52.315 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:07:52.316 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:07:52.318 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:52.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:52.323 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:52.424 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:52.454 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:07:52.492 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:52 smithi055 ceph-mon[184174]: pgmap v804: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 34064/179745 objects degraded (18.951%); 4638/179745 objects misplaced (2.580%); 17 MiB/s, 267 objects/s recovering 2023-09-23T18:07:52.493 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:52.493 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:52.493 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/206378845' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:07:52.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:52 smithi055 ceph-mon[179668]: pgmap v804: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 34064/179745 objects degraded (18.951%); 4638/179745 objects misplaced (2.580%); 17 MiB/s, 267 objects/s recovering 2023-09-23T18:07:52.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:52.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:52.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/206378845' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:07:52.514 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:52 smithi099 ceph-mon[167076]: pgmap v804: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.7 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 464 op/s; 34064/179745 objects degraded (18.951%); 4638/179745 objects misplaced (2.580%); 17 MiB/s, 267 objects/s recovering 2023-09-23T18:07:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:07:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:07:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/206378845' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:07:52.615 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:07:52.655 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:07:52.695 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:07:52.724 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:07:52.733 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:52.740 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:07:52.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:52.742 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:52.747 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:07:52.779 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:07:52.801 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:07:52.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:07:52.833 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:07:52.834 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:07:52.834 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:07:52.834 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:07:52.834 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:07:52.834 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:07:52.834 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:07:52.834 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:07:52.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:52.871 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:52.904 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:52.907 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:52.907 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:53.008 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:53.062 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:53.108 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:53.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:53.113 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:53.214 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:53.254 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:53.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:53.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/206378845' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:07:53.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[184174]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T18:07:53.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[184174]: failed to encode map e329 with expected crc 2023-09-23T18:07:53.332 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4152577874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:53.333 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 34064/179745 objects degraded (18.951%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:07:53.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:53.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/206378845' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:07:53.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[179668]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T18:07:53.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[179668]: failed to encode map e329 with expected crc 2023-09-23T18:07:53.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4152577874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:53.333 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 34064/179745 objects degraded (18.951%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:07:53.360 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:53.364 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:53.365 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:53.465 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:53.491 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:53 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:07:53.491 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/206378845' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:07:53.492 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:53 smithi099 ceph-mon[167076]: osdmap e329: 8 total, 8 up, 8 in 2023-09-23T18:07:53.492 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:53 smithi099 ceph-mon[167076]: failed to encode map e329 with expected crc 2023-09-23T18:07:53.492 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4152577874' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:53.492 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 34064/179745 objects degraded (18.951%), 20 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:07:53.494 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:53.536 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:53.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:53.540 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:53.576 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:53.641 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:53.679 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:53.688 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:07:53.689 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:07:53.689 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T18:07:53.689 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:07:53.689 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:07:53.689 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T18:07:53.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:53.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:53.726 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:53.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:53.866 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:53.917 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:53.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:53.921 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:54.022 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:54.063 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:54.321 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:54 smithi055 ceph-mon[184174]: failed to encode map e329 with expected crc 2023-09-23T18:07:54.321 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:54 smithi055 ceph-mon[184174]: pgmap v806: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 33852/178236 objects degraded (18.993%); 3994/178236 objects misplaced (2.241%); 17 MiB/s, 270 objects/s recovering 2023-09-23T18:07:54.322 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:54 smithi055 ceph-mon[179668]: failed to encode map e329 with expected crc 2023-09-23T18:07:54.322 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:54 smithi055 ceph-mon[179668]: pgmap v806: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 33852/178236 objects degraded (18.993%); 3994/178236 objects misplaced (2.241%); 17 MiB/s, 270 objects/s recovering 2023-09-23T18:07:54.359 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:54.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:54.363 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:54.369 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:07:54.464 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:54.505 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:54.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:54 smithi099 ceph-mon[167076]: failed to encode map e329 with expected crc 2023-09-23T18:07:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:54 smithi099 ceph-mon[167076]: pgmap v806: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 417 op/s; 33852/178236 objects degraded (18.993%); 3994/178236 objects misplaced (2.241%); 17 MiB/s, 270 objects/s recovering 2023-09-23T18:07:54.668 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:54.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:54.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:54.773 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:07:54.822 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:54.933 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:07:54.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:54.942 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:07:55.018 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:07:55.043 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:55.074 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:55.207 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:07:55.207 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:07:55.207 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:07:55.207 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:07:55.207 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:07:55.360 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:07:55.627 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:07:55.902 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:56.127 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:56.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:56.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:56.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:56.283 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:56.344 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:56.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:56.350 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:56.393 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:56.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:56 smithi055 ceph-mon[179668]: pgmap v807: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 33753/176709 objects degraded (19.101%); 3994/176709 objects misplaced (2.260%); 10 MiB/s, 160 objects/s recovering 2023-09-23T18:07:56.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:56 smithi055 ceph-mon[184174]: pgmap v807: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 33753/176709 objects degraded (19.101%); 3994/176709 objects misplaced (2.260%); 10 MiB/s, 160 objects/s recovering 2023-09-23T18:07:56.432 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:56 smithi099 ceph-mon[167076]: pgmap v807: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 297 op/s; 33753/176709 objects degraded (19.101%); 3994/176709 objects misplaced (2.260%); 10 MiB/s, 160 objects/s recovering 2023-09-23T18:07:56.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:07:56.485 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:56.592 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:07:56.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:56.600 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:07:56.701 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:56.706 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:07:56.706 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "" 2023-09-23T18:07:56.706 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:07:56.755 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:07:56.799 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:56.929 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:56.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:56.937 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:57.038 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:57.113 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:57.433 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:57.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:57.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:57.539 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:07:57.586 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:57.802 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:07:57.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:57.812 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:07:57.913 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:58.010 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:58.112 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:58.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:58.123 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:58.224 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:58.258 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:58.300 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:58.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:58.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:58.405 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:07:58.413 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:58 smithi055 ceph-mon[184174]: pgmap v808: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 33194/174804 objects degraded (18.989%); 3994/174804 objects misplaced (2.285%); 12 MiB/s, 187 objects/s recovering 2023-09-23T18:07:58.413 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 33194/174804 objects degraded (18.989%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:07:58.414 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:58 smithi055 ceph-mon[179668]: pgmap v808: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 33194/174804 objects degraded (18.989%); 3994/174804 objects misplaced (2.285%); 12 MiB/s, 187 objects/s recovering 2023-09-23T18:07:58.414 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 33194/174804 objects degraded (18.989%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:07:58.432 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:58.470 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:07:58.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:58.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:07:58.493 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:07:58.493 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:07:58.493 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:07:58.493 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:07:58.494 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.494 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:67505 6.c 6.b5479fac (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T18:07:58.494 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:17.174047+0000", 2023-09-23T18:07:58.494 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 341.31637716799997, 2023-09-23T18:07:58.494 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75978317699999998, 2023-09-23T18:07:58.494 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.495 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:07:58.495 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.495 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:07:58.495 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:07:58.495 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67505 2023-09-23T18:07:58.496 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.496 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.496 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.496 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.496 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174047+0000", 2023-09-23T18:07:58.497 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.497 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.497 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.497 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174047+0000", 2023-09-23T18:07:58.497 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.498 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.498 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.498 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174045+0000", 2023-09-23T18:07:58.498 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.498 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.499 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.499 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.499 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174088+0000", 2023-09-23T18:07:58.499 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2586000000000002e-05 2023-09-23T18:07:58.499 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.500 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.500 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.500 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174099+0000", 2023-09-23T18:07:58.500 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0993e-05 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174112+0000", 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2860000000000001e-05 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174191+0000", 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9517000000000005e-05 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174204+0000", 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2486999999999999e-05 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.933818+0000", 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75961363900000001 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.502 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.933830+0000", 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2374999999999999e-05 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:3953 7.3 7:c9c6e60b:::benchmark_data_smithi055_311528_object3952:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:05:52.348950+0000", 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 126.14147429099999, 2023-09-23T18:07:58.503 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10760905599999999, 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 3953 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.504 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.348950+0000", 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.348950+0000", 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.348948+0000", 2023-09-23T18:07:58.505 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.506 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.506 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.506 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.506 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.348979+0000", 2023-09-23T18:07:58.506 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1390000000000003e-05 2023-09-23T18:07:58.506 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.506 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.506 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.506 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.348998+0000", 2023-09-23T18:07:58.506 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8124000000000001e-05 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349005+0000", 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.96e-06 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.351476+0000", 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024704919999999999 2023-09-23T18:07:58.507 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.351603+0000", 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000127317 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.351737+0000", 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000133201 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.508 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.353794+0000", 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020570829999999999 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359295+0000", 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055012220000000001 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.509 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456453+0000", 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.097158142000000003 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456515+0000", 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2411999999999998e-05 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.510 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456559+0000", 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3337999999999997e-05 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:3956 7.3 7:c6320996:::benchmark_data_smithi055_311528_object3955:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:05:52.349077+0000", 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 126.141347014, 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.107628167, 2023-09-23T18:07:58.511 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 3956 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349077+0000", 2023-09-23T18:07:58.512 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349077+0000", 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349076+0000", 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:58.513 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.514 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.514 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.514 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349136+0000", 2023-09-23T18:07:58.514 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9446e-05 2023-09-23T18:07:58.514 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.514 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.514 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.514 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349144+0000", 2023-09-23T18:07:58.514 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.174e-06 2023-09-23T18:07:58.514 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349150+0000", 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.541e-06 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.352162+0000", 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030124069999999999 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.515 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.516 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.516 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.352304+0000", 2023-09-23T18:07:58.516 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014179700000000001 2023-09-23T18:07:58.516 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.516 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.516 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:07:58.516 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.352447+0000", 2023-09-23T18:07:58.516 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000143139 2023-09-23T18:07:58.516 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.516 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359286+0000", 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0068389870000000004 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.362398+0000", 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031117509999999998 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.517 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.518 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456607+0000", 2023-09-23T18:07:58.518 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.094209698999999994 2023-09-23T18:07:58.518 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.518 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.518 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.518 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456650+0000", 2023-09-23T18:07:58.518 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2546999999999998e-05 2023-09-23T18:07:58.518 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.518 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.518 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456705+0000", 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5192000000000003e-05 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:3970 7.3 7:c65e77dc:::benchmark_data_smithi055_311528_object3969:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:05:52.357400+0000", 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 126.133023713, 2023-09-23T18:07:58.519 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.099444679999999994, 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 3970 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.520 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357400+0000", 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357400+0000", 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357400+0000", 2023-09-23T18:07:58.521 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357451+0000", 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0998000000000002e-05 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357459+0000", 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6680000000000007e-06 2023-09-23T18:07:58.522 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357465+0000", 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0979999999999999e-06 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359320+0000", 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00185439 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.523 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359465+0000", 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000145544 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359613+0000", 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014714700000000001 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.524 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.362696+0000", 2023-09-23T18:07:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030835200000000002 2023-09-23T18:07:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.363052+0000", 2023-09-23T18:07:58.525 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00035548200000000001 2023-09-23T18:07:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456758+0000", 2023-09-23T18:07:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.093706809000000002 2023-09-23T18:07:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.526 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456807+0000", 2023-09-23T18:07:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8826000000000003e-05 2023-09-23T18:07:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456845+0000", 2023-09-23T18:07:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7718e-05 2023-09-23T18:07:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.527 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.528 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.528 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.528 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:3972 7.3 7:c3137e10:::benchmark_data_smithi055_311528_object3971:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-09-23T18:07:58.528 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:05:52.359357+0000", 2023-09-23T18:07:58.528 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 126.131067184, 2023-09-23T18:07:58.528 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.097698785999999996, 2023-09-23T18:07:58.528 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.528 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 3972 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359357+0000", 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.529 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359357+0000", 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359356+0000", 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.530 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359379+0000", 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2860000000000001e-05 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359389+0000", 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0392e-05 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.531 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359397+0000", 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5730000000000002e-06 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.361819+0000", 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024226870000000002 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.361944+0000", 2023-09-23T18:07:58.532 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000124731 2023-09-23T18:07:58.533 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.533 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.533 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:07:58.533 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.362076+0000", 2023-09-23T18:07:58.533 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013221899999999999 2023-09-23T18:07:58.533 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.533 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.533 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.533 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.363154+0000", 2023-09-23T18:07:58.533 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001077756 2023-09-23T18:07:58.534 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.534 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.534 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.534 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.363441+0000", 2023-09-23T18:07:58.534 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00028649699999999998 2023-09-23T18:07:58.534 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.534 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.534 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.534 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456933+0000", 2023-09-23T18:07:58.534 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.093492254999999996 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.457017+0000", 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3906000000000006e-05 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.457056+0000", 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8714999999999998e-05 2023-09-23T18:07:58.535 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29533 7.2 7:46d5e94b:::benchmark_data_smithi055_311528_object29532:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.367301+0000", 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 114.123122662, 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2571953860000002, 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.536 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29533 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367301+0000", 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.537 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.538 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.538 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367301+0000", 2023-09-23T18:07:58.538 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.538 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.538 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.538 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.538 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367300+0000", 2023-09-23T18:07:58.538 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.538 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.538 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367324+0000", 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4797999999999999e-05 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367337+0000", 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2553e-05 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.539 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367345+0000", 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4850000000000003e-06 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.370630+0000", 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0032851439999999998 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:07:58.540 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.370641+0000", 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1644000000000001e-05 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.346322+0000", 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97568077600000003 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.346335+0000", 2023-09-23T18:07:58.541 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2475e-05 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.616610+0000", 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.270275923 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.616787+0000", 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000176718 2023-09-23T18:07:58.542 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.543 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.543 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,2", 2023-09-23T18:07:58.543 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.616979+0000", 2023-09-23T18:07:58.543 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019225199999999999 2023-09-23T18:07:58.543 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.543 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.543 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.543 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.619865+0000", 2023-09-23T18:07:58.543 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002885084 2023-09-23T18:07:58.543 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.544 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.544 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.544 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.620879+0000", 2023-09-23T18:07:58.544 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0010146090000000001 2023-09-23T18:07:58.544 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.544 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.544 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.544 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.622841+0000", 2023-09-23T18:07:58.544 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001961824 2023-09-23T18:07:58.544 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.622881+0000", 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0451999999999998e-05 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.624497+0000", 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016152269999999999 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.545 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.546 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.546 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.546 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:50814 7.2 7:41fb1492:::benchmark_data_smithi055_311528_object50813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T18:07:58.546 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:28.375669+0000", 2023-09-23T18:07:58.546 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 90.114754927000007, 2023-09-23T18:07:58.546 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.71295601600000003, 2023-09-23T18:07:58.546 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.546 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.546 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 50814 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375669+0000", 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.547 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375669+0000", 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375667+0000", 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375692+0000", 2023-09-23T18:07:58.548 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4368000000000001e-05 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375708+0000", 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6643999999999999e-05 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375717+0000", 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1179999999999998e-06 2023-09-23T18:07:58.549 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375765+0000", 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7469999999999998e-05 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375773+0000", 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2619999999999998e-06 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.550 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.079328+0000", 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.70355522800000003 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.079462+0000", 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000133351 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.551 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,2", 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.079595+0000", 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013323999999999999 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.085875+0000", 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0062797399999999998 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.552 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088199+0000", 2023-09-23T18:07:58.553 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023238709999999999 2023-09-23T18:07:58.553 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.553 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.553 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.553 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088530+0000", 2023-09-23T18:07:58.553 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00033098300000000003 2023-09-23T18:07:58.553 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.553 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.553 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.553 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088578+0000", 2023-09-23T18:07:58.554 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8789000000000003e-05 2023-09-23T18:07:58.554 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.554 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.554 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.554 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088625+0000", 2023-09-23T18:07:58.554 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6600000000000001e-05 2023-09-23T18:07:58.554 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.554 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.554 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.554 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:50830 7.2 7:466e5603:::benchmark_data_smithi055_311528_object50829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:28.429072+0000", 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 90.061351746, 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.65978882900000002, 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.555 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 50830 2023-09-23T18:07:58.556 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.556 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.556 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.556 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.556 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429072+0000", 2023-09-23T18:07:58.556 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.556 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.556 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.556 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429072+0000", 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429071+0000", 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.557 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429093+0000", 2023-09-23T18:07:58.558 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2030000000000001e-05 2023-09-23T18:07:58.558 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.558 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.558 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.558 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429105+0000", 2023-09-23T18:07:58.558 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2473000000000001e-05 2023-09-23T18:07:58.558 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.558 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.558 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.558 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429111+0000", 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7810000000000002e-06 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429156+0000", 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4366000000000002e-05 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429163+0000", 2023-09-23T18:07:58.559 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6790000000000005e-06 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.083673+0000", 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.65450945999999999 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.083790+0000", 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011707899999999999 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.560 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,2", 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.083920+0000", 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012990399999999999 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.087641+0000", 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037213620000000002 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.561 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088229+0000", 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000588325 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088785+0000", 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00055543900000000002 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.562 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088827+0000", 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1804999999999997e-05 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088861+0000", 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4329999999999998e-05 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.563 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:65495 7.7 e300/279)", 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:47.154630+0000", 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 71.335793976000005, 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.062563752, 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154630+0000", 2023-09-23T18:07:58.564 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154630+0000", 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154628+0000", 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.565 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154773+0000", 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014484099999999999 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154789+0000", 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5288999999999999e-05 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.566 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154798+0000", 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6330000000000008e-06 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217166+0000", 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0623673030000003 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.567 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217194+0000", 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8147999999999999e-05 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:65500 7.7 e300/279)", 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:47.168272+0000", 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 71.322152106999994, 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0489593309999998, 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.568 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168272+0000", 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168272+0000", 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.569 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.570 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.570 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.570 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168271+0000", 2023-09-23T18:07:58.570 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.570 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.570 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.570 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.570 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168313+0000", 2023-09-23T18:07:58.570 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1974000000000002e-05 2023-09-23T18:07:58.570 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.571 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.571 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.571 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168330+0000", 2023-09-23T18:07:58.571 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7615999999999999e-05 2023-09-23T18:07:58.571 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.571 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.571 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.571 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168339+0000", 2023-09-23T18:07:58.571 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5709999999999998e-06 2023-09-23T18:07:58.571 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.572 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.572 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.572 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217215+0000", 2023-09-23T18:07:58.572 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0488766289999996 2023-09-23T18:07:58.572 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.572 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.572 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.572 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217231+0000", 2023-09-23T18:07:58.572 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5775999999999999e-05 2023-09-23T18:07:58.572 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:65503 7.7 e300/279)", 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:47.202135+0000", 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 71.288288774999998, 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0151491789999998, 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.573 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202135+0000", 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202135+0000", 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.574 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202134+0000", 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202195+0000", 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1135000000000003e-05 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.575 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202210+0000", 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4632e-05 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202218+0000", 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6489999999999997e-06 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217265+0000", 2023-09-23T18:07:58.576 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0150468220000004 2023-09-23T18:07:58.577 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.577 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.577 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.577 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217284+0000", 2023-09-23T18:07:58.577 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9029999999999999e-05 2023-09-23T18:07:58.577 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.577 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.577 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.577 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.577 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.578 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65507 7.f 7:f2a3a65c:::benchmark_data_smithi055_311528_object65506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T18:07:58.578 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:55.876474+0000", 2023-09-23T18:07:58.578 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 62.613949531999999, 2023-09-23T18:07:58.578 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3220000350000003, 2023-09-23T18:07:58.578 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.578 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.578 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.578 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65507 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876474+0000", 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.579 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.580 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876474+0000", 2023-09-23T18:07:58.580 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.580 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.580 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.580 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.580 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876473+0000", 2023-09-23T18:07:58.580 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.580 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.580 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.580 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876504+0000", 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0627999999999999e-05 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876521+0000", 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7286e-05 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.581 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876530+0000", 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0680000000000003e-06 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.167045+0000", 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2905153990000002 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.167224+0000", 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017898499999999999 2023-09-23T18:07:58.582 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.167406+0000", 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018159 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.174760+0000", 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0073541259999999999 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.583 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.584 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.584 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177828+0000", 2023-09-23T18:07:58.584 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030683960000000001 2023-09-23T18:07:58.584 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.584 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.584 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.584 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.191991+0000", 2023-09-23T18:07:58.584 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014162492 2023-09-23T18:07:58.584 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.584 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.196568+0000", 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045769340000000004 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.196607+0000", 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8658999999999997e-05 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.585 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.586 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198474+0000", 2023-09-23T18:07:58.586 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001867884 2023-09-23T18:07:58.586 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.586 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.586 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.586 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.586 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.586 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65522 7.f 7:f4ce2e57:::benchmark_data_smithi055_311528_object65521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T18:07:58.586 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:55.973972+0000", 2023-09-23T18:07:58.586 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 62.516452180000002, 2023-09-23T18:07:58.587 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2270217859999999, 2023-09-23T18:07:58.587 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.587 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.587 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.587 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.587 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.587 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65522 2023-09-23T18:07:58.587 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.587 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.587 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.973972+0000", 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.973972+0000", 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.588 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.973969+0000", 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.974001+0000", 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1111000000000001e-05 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.974022+0000", 2023-09-23T18:07:58.589 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1293000000000001e-05 2023-09-23T18:07:58.590 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.590 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.590 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.590 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.974037+0000", 2023-09-23T18:07:58.590 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4935e-05 2023-09-23T18:07:58.590 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.590 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.590 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.590 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.169722+0000", 2023-09-23T18:07:58.591 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1956848210000004 2023-09-23T18:07:58.591 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.591 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.591 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.591 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.169899+0000", 2023-09-23T18:07:58.591 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000177849 2023-09-23T18:07:58.591 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.591 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.591 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:07:58.591 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.170109+0000", 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000209667 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177841+0000", 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0077319750000000003 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177878+0000", 2023-09-23T18:07:58.592 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6709999999999999e-05 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198539+0000", 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020661536000000001 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198675+0000", 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013579400000000001 2023-09-23T18:07:58.593 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.594 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.594 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.594 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198720+0000", 2023-09-23T18:07:58.594 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5111999999999998e-05 2023-09-23T18:07:58.594 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.594 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.594 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.594 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.200994+0000", 2023-09-23T18:07:58.594 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022732540000000002 2023-09-23T18:07:58.594 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65531 7.f 7:fb49bebd:::benchmark_data_smithi055_311528_object65530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:55.993528+0000", 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 62.496895635999998, 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2096306049999992, 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.595 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65531 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993528+0000", 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.596 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.597 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.597 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993528+0000", 2023-09-23T18:07:58.597 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.597 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.597 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.597 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.597 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993526+0000", 2023-09-23T18:07:58.597 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:07:58.597 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.598 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.598 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.598 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993552+0000", 2023-09-23T18:07:58.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5392e-05 2023-09-23T18:07:58.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.598 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.598 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.598 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993568+0000", 2023-09-23T18:07:58.598 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6087e-05 2023-09-23T18:07:58.598 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993577+0000", 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8240000000000006e-06 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.172136+0000", 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1785597140000004 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.599 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.172295+0000", 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015823599999999999 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.172472+0000", 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000176918 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.600 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177885+0000", 2023-09-23T18:07:58.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0054133879999999999 2023-09-23T18:07:58.601 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.601 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.601 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.601 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177931+0000", 2023-09-23T18:07:58.601 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5782000000000003e-05 2023-09-23T18:07:58.601 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.601 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.601 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198584+0000", 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020652818999999999 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.201065+0000", 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024819600000000001 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.602 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.201131+0000", 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5105999999999996e-05 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.203159+0000", 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020283129999999999 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.603 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:75464 7.f 7:f021c787:::benchmark_data_smithi055_311528_object4501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.273115+0000", 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 26.217308767999999, 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.26578997100000001, 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 75464 2023-09-23T18:07:58.604 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273115+0000", 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273115+0000", 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.605 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.606 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.606 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273114+0000", 2023-09-23T18:07:58.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.606 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.606 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.606 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.606 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273122+0000", 2023-09-23T18:07:58.606 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.875e-06 2023-09-23T18:07:58.606 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273136+0000", 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4139e-05 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273148+0000", 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2351000000000001e-05 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.607 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.274461+0000", 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013133260000000001 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.274628+0000", 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016638499999999999 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.608 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.274782+0000", 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000154518 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.282769+0000", 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0079868630000000003 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.282877+0000", 2023-09-23T18:07:58.609 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010753599999999999 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.289020+0000", 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0061434560000000003 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536667+0000", 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.247647165 2023-09-23T18:07:58.610 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536722+0000", 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4858000000000002e-05 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.538905+0000", 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021829219999999999 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.611 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:75467 7.f 7:f8a917d7:::benchmark_data_smithi055_311528_object4504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.276954+0000", 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 26.213469956000001, 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.263980031, 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.612 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 75467 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276954+0000", 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.613 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276954+0000", 2023-09-23T18:07:58.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276952+0000", 2023-09-23T18:07:58.614 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.614 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.614 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.614 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.614 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276960+0000", 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8660000000000006e-06 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276974+0000", 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4287000000000001e-05 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276983+0000", 2023-09-23T18:07:58.615 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2609999999999993e-06 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.277455+0000", 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00047249100000000001 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.277596+0000", 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014098499999999999 2023-09-23T18:07:58.616 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.277684+0000", 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7489000000000002e-05 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.282810+0000", 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0051259110000000004 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.617 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.282885+0000", 2023-09-23T18:07:58.618 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5488000000000006e-05 2023-09-23T18:07:58.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.618 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.618 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536591+0000", 2023-09-23T18:07:58.618 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25370599599999999 2023-09-23T18:07:58.618 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.618 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.538972+0000", 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023805570000000002 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.539010+0000", 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8217999999999999e-05 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.619 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.540934+0000", 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001924133 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.45557.0:75467 7.f e322/303 ondisk, result = 0)", 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.284876+0000", 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 26.205548027999999, 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25172437600000003, 2023-09-23T18:07:58.620 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284876+0000", 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284876+0000", 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.621 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.622 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284876+0000", 2023-09-23T18:07:58.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:58.622 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.622 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.622 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.622 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284878+0000", 2023-09-23T18:07:58.622 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7190000000000001e-06 2023-09-23T18:07:58.622 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284885+0000", 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4320000000000004e-06 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284889+0000", 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.549e-06 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.623 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536571+0000", 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25168132100000001 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536583+0000", 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2312e-05 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536600+0000", 2023-09-23T18:07:58.624 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.73e-05 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.45557.0:75464 7.f e322/303 ondisk, result = 0)", 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.284917+0000", 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 26.205507257000001, 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.254001488, 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.625 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284917+0000", 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284917+0000", 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.626 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.627 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.627 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.627 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284916+0000", 2023-09-23T18:07:58.627 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:58.627 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.627 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.627 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.627 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284920+0000", 2023-09-23T18:07:58.627 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0180000000000003e-06 2023-09-23T18:07:58.627 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284929+0000", 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2150000000000001e-06 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284934+0000", 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.339e-06 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.628 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536651+0000", 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.251717315 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536660+0000", 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8939999999999999e-06 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.629 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.538918+0000", 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022580959999999998 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.45557.0:75467 7.f e322/303 ondisk, result = 0)", 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.284939+0000", 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 26.205484849000001, 2023-09-23T18:07:58.630 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25600794700000001, 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284939+0000", 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284939+0000", 2023-09-23T18:07:58.631 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284939+0000", 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284941+0000", 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5270000000000001e-06 2023-09-23T18:07:58.632 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284947+0000", 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7939999999999999e-06 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284952+0000", 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4379999999999997e-06 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.633 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.538960+0000", 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25400828800000003 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.538967+0000", 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1999999999999997e-06 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.634 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.540947+0000", 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019799459999999998 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:75477 7.a 7:5834a874:::benchmark_data_smithi055_311528_object4514:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.290788+0000", 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 26.199635578999999, 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25928344599999997, 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:07:58.635 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 75477 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.290788+0000", 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.636 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.637 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.637 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:07:58.637 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.290788+0000", 2023-09-23T18:07:58.637 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:07:58.637 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.637 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.637 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:07:58.637 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.290787+0000", 2023-09-23T18:07:58.637 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:07:58.637 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.638 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.638 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:07:58.638 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.290795+0000", 2023-09-23T18:07:58.638 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0700000000000007e-06 2023-09-23T18:07:58.638 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.638 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.638 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:07:58.638 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536543+0000", 2023-09-23T18:07:58.638 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.24574849800000001 2023-09-23T18:07:58.638 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.639 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:07:58.639 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536561+0000", 2023-09-23T18:07:58.639 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7600000000000001e-05 2023-09-23T18:07:58.639 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.639 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.639 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:07:58.639 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536612+0000", 2023-09-23T18:07:58.639 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0654e-05 2023-09-23T18:07:58.639 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.640 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.640 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:07:58.640 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536743+0000", 2023-09-23T18:07:58.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013162800000000001 2023-09-23T18:07:58.640 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.640 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.640 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5", 2023-09-23T18:07:58.640 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536851+0000", 2023-09-23T18:07:58.640 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010739999999999999 2023-09-23T18:07:58.640 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.544832+0000", 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.007981485 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.549935+0000", 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0051031009999999996 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.641 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.549994+0000", 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8411e-05 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.550035+0000", 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1658e-05 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:07:58.642 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:07:58.643 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.550072+0000", 2023-09-23T18:07:58.643 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6570999999999999e-05 2023-09-23T18:07:58.643 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.643 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.643 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.643 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:07:58.643 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:07:58.643 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:07:58.643 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:07:58.644 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:58 smithi099 ceph-mon[167076]: pgmap v808: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.6 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 325 op/s; 33194/174804 objects degraded (18.989%); 3994/174804 objects misplaced (2.285%); 12 MiB/s, 187 objects/s recovering 2023-09-23T18:07:58.645 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 33194/174804 objects degraded (18.989%), 20 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:07:58.703 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:58.734 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:07:58.775 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:07:58.804 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:07:58.834 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:07:58.862 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:07:58.889 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:07:58.903 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:07:58.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:58.908 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:07:58.916 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:07:58.945 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:07:58.950 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:07:58.952 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:07:58.952 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:07:58.952 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:07:58.952 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:07:58.952 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:07:58.952 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:07:58.952 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:07:58.953 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:07:59.009 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:59.058 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:59.117 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:59.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:59.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:07:59.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3858024728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:59.245 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:59 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:59.245 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:07:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3217972482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:59.261 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:59.269 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:07:59.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:07:59.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:59.311 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:07:59.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:07:59.449 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:59.491 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:07:59.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:59.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:07:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:59 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3858024728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:59 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:07:59 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3217972482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:59.597 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:07:59.657 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:59 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3858024728' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:59.657 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:59 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:07:59.657 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:07:59 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3217972482' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:07:59.661 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:07:59.863 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:07:59.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:07:59.871 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:07:59.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:00.001 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:00.039 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:00.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:00.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:00.143 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:00.188 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:00.237 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:00.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:00.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:00.250 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:00 smithi099 ceph-mon[167076]: pgmap v809: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 32759/172347 objects degraded (19.008%); 3358/172347 objects misplaced (1.948%); 15 MiB/s, 234 objects/s recovering 2023-09-23T18:08:00.250 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:00 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:00.250 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:00 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:00.250 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:00 smithi099 ceph-mon[167076]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T18:08:00.251 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:00 smithi099 ceph-mon[167076]: failed to encode map e330 with expected crc 2023-09-23T18:08:00.251 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:00 smithi099 ceph-mon[167076]: failed to encode map e330 with expected crc 2023-09-23T18:08:00.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:00.380 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:00.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:08:00] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.43.0" 2023-09-23T18:08:00.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[184174]: pgmap v809: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 32759/172347 objects degraded (19.008%); 3358/172347 objects misplaced (1.948%); 15 MiB/s, 234 objects/s recovering 2023-09-23T18:08:00.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:00.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[184174]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T18:08:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[184174]: failed to encode map e330 with expected crc 2023-09-23T18:08:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[184174]: failed to encode map e330 with expected crc 2023-09-23T18:08:00.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[179668]: pgmap v809: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.5 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 356 op/s; 32759/172347 objects degraded (19.008%); 3358/172347 objects misplaced (1.948%); 15 MiB/s, 234 objects/s recovering 2023-09-23T18:08:00.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:00.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:00.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[179668]: osdmap e330: 8 total, 8 up, 8 in 2023-09-23T18:08:00.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[179668]: failed to encode map e330 with expected crc 2023-09-23T18:08:00.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:00 smithi055 ceph-mon[179668]: failed to encode map e330 with expected crc 2023-09-23T18:08:00.432 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:00.434 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:00.435 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:00.536 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:00.567 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:01.598 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:01.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:01.602 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:01.703 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:01.752 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:01.812 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:01.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:01.818 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:01.919 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:01.967 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:02.026 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:02.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:02.033 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:02.088 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:08:02.089 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:08:02.089 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T18:08:02.089 INFO:tasks.thrashosds.thrasher:Setting osd 2 primary_affinity to 0.000000 2023-09-23T18:08:02.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 0 2023-09-23T18:08:02.134 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:02.193 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:02.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:02 smithi099 ceph-mon[167076]: pgmap v811: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 32064/168177 objects degraded (19.066%); 3358/168177 objects misplaced (1.997%); 15 MiB/s, 245 objects/s recovering 2023-09-23T18:08:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:02 smithi055 ceph-mon[184174]: pgmap v811: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 32064/168177 objects degraded (19.066%); 3358/168177 objects misplaced (1.997%); 15 MiB/s, 245 objects/s recovering 2023-09-23T18:08:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:02 smithi055 ceph-mon[179668]: pgmap v811: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 448 op/s; 32064/168177 objects degraded (19.066%); 3358/168177 objects misplaced (1.997%); 15 MiB/s, 245 objects/s recovering 2023-09-23T18:08:03.320 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:03.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:03.324 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:03.425 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:03.482 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:03.528 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:03.541 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:03.546 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:03.547 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:03.547 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 32064/168177 objects degraded (19.066%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:03.547 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 32064/168177 objects degraded (19.066%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:03.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 32064/168177 objects degraded (19.066%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:03.600 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:03.648 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:03.681 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:03.703 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:08:04.419 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:04.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:04.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:04.526 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:04.572 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:04 smithi055 ceph-mon[184174]: pgmap v812: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 31785/166053 objects degraded (19.141%); 2626/166053 objects misplaced (1.581%); 16 MiB/s, 248 objects/s recovering 2023-09-23T18:08:04.573 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:04 smithi055 ceph-mon[179668]: pgmap v812: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 31785/166053 objects degraded (19.141%); 2626/166053 objects misplaced (1.581%); 16 MiB/s, 248 objects/s recovering 2023-09-23T18:08:04.575 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:04.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:04 smithi099 ceph-mon[167076]: pgmap v812: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.4 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 405 op/s; 31785/166053 objects degraded (19.141%); 2626/166053 objects misplaced (1.581%); 16 MiB/s, 248 objects/s recovering 2023-09-23T18:08:04.688 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:04.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:04.692 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:04.793 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:04.845 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:05.073 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:05.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:05.076 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:05.177 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:05.333 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:05.369 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:08:05.480 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:05.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:05.483 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:05.584 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:05.616 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:05.657 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:05.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:05.660 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:05.761 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:05.846 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:05.981 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:05.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:05.988 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:06.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:06.149 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:06.279 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:06.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:06.286 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:06.387 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:06.441 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:06.500 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:06.531 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:06.533 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:06 smithi055 ceph-mon[184174]: pgmap v813: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 31634/163839 objects degraded (19.308%); 2626/163839 objects misplaced (1.603%); 16 MiB/s, 253 objects/s recovering 2023-09-23T18:08:06.533 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:08:06.535 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:06 smithi055 ceph-mon[179668]: pgmap v813: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 31634/163839 objects degraded (19.308%); 2626/163839 objects misplaced (1.603%); 16 MiB/s, 253 objects/s recovering 2023-09-23T18:08:06.535 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:08:06.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:06.537 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:06.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:06 smithi099 ceph-mon[167076]: pgmap v813: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.3 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 428 op/s; 31634/163839 objects degraded (19.308%); 2626/163839 objects misplaced (1.603%); 16 MiB/s, 253 objects/s recovering 2023-09-23T18:08:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:08:06.638 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:06.670 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:06.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:06.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:06.720 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:06.816 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:06.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:06.855 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:06.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:06.898 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:06.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:06.999 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:07.089 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:07.412 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:07.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:07.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:07.522 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:07.582 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:07.674 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:08:07.674 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:08:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:07 smithi055 ceph-mon[184174]: pgmap v814: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 30884/161106 objects degraded (19.170%); 2626/161106 objects misplaced (1.630%); 17 MiB/s, 269 objects/s recovering 2023-09-23T18:08:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:07 smithi055 ceph-mon[179668]: pgmap v814: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 30884/161106 objects degraded (19.170%); 2626/161106 objects misplaced (1.630%); 17 MiB/s, 269 objects/s recovering 2023-09-23T18:08:07.742 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:07.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:07.747 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:07 smithi099 ceph-mon[167076]: pgmap v814: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.3 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 456 op/s; 30884/161106 objects degraded (19.170%); 2626/161106 objects misplaced (1.630%); 17 MiB/s, 269 objects/s recovering 2023-09-23T18:08:07.847 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:07.889 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:07.932 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:07.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:07.937 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:08.037 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:08.140 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:08.227 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:08.232 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:08.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:08.334 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:08.392 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 30884/161106 objects degraded (19.170%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 30884/161106 objects degraded (19.170%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 30884/161106 objects degraded (19.170%), 20 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:08.849 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:08:08.865 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:08.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:08.873 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:08.974 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:09.040 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:09.165 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:09.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:09.169 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:09.270 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:09.352 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:09.491 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.2 primary-affinity to 0 (802) 2023-09-23T18:08:09.566 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:09.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:09.574 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:09.575 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/743870229' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-09-23T18:08:09.575 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:09 smithi055 ceph-mon[184174]: pgmap v815: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 30471/158580 objects degraded (19.215%); 1790/158580 objects misplaced (1.129%); 17 MiB/s, 277 objects/s recovering 2023-09-23T18:08:09.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/743870229' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-09-23T18:08:09.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:09 smithi055 ceph-mon[179668]: pgmap v815: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 30471/158580 objects degraded (19.215%); 1790/158580 objects misplaced (1.129%); 17 MiB/s, 277 objects/s recovering 2023-09-23T18:08:09.671 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:08:09.675 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:09.692 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:08:09.704 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:09.723 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:08:09.745 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:09.747 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:08:09.748 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/743870229' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]: dispatch 2023-09-23T18:08:09.748 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:09 smithi099 ceph-mon[167076]: pgmap v815: 129 pgs: 1 active+recovering+undersized+remapped, 14 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 458 op/s; 30471/158580 objects degraded (19.215%); 1790/158580 objects misplaced (1.129%); 17 MiB/s, 277 objects/s recovering 2023-09-23T18:08:09.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:09.749 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:09.771 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:08:09.798 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:08:09.827 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:08:09.850 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:09.859 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:08:09.862 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:08:09.862 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:08:09.862 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:08:09.862 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:08:09.862 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:08:09.862 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:08:09.863 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:08:09.863 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:08:09.889 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:09.977 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:09.981 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:09.982 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:10.068 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:10.083 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:10.115 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:10.328 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:10.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:10.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:10.426 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:08:10] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.43.0" 2023-09-23T18:08:10.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:10.459 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:10.494 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:08:10.508 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:10.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:10.515 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:10.616 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[184174]: failed to encode map e331 with expected crc 2023-09-23T18:08:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/743870229' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-09-23T18:08:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[184174]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T18:08:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[184174]: failed to encode map e331 with expected crc 2023-09-23T18:08:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3670568782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4290781936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:08:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[179668]: failed to encode map e331 with expected crc 2023-09-23T18:08:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/743870229' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-09-23T18:08:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[179668]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T18:08:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[179668]: failed to encode map e331 with expected crc 2023-09-23T18:08:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3670568782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4290781936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:08:10.688 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:10.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:10 smithi099 ceph-mon[167076]: failed to encode map e331 with expected crc 2023-09-23T18:08:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/743870229' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 0.0}]': finished 2023-09-23T18:08:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:10 smithi099 ceph-mon[167076]: osdmap e331: 8 total, 8 up, 8 in 2023-09-23T18:08:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:10 smithi099 ceph-mon[167076]: failed to encode map e331 with expected crc 2023-09-23T18:08:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3670568782' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4290781936' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:08:11.386 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:11.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:11.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:11.492 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:11.531 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:11.575 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:11.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:11.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:11.681 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:11.724 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:11.801 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:11.809 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:08:11.809 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:08:11.809 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:08:11.810 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:08:11.810 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:08:11.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:11.811 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:11.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:11.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4290781936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:08:11.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T18:08:11.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: pgmap v818: 129 pgs: 2 peering, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 29837/154716 objects degraded (19.285%); 1790/154716 objects misplaced (1.157%); 16 MiB/s, 261 objects/s recovering 2023-09-23T18:08:11.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:11.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:11.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:11.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4290781936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:08:11.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T18:08:11.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: pgmap v818: 129 pgs: 2 peering, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 29837/154716 objects degraded (19.285%); 1790/154716 objects misplaced (1.157%); 16 MiB/s, 261 objects/s recovering 2023-09-23T18:08:11.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:11.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:11 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4290781936' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:08:11.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: osdmap e332: 8 total, 8 up, 8 in 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: failed to encode map e332 with expected crc 2023-09-23T18:08:11.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: pgmap v818: 129 pgs: 2 peering, 1 active+recovering+undersized+remapped, 13 active+recovery_wait+degraded, 6 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.2 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 472 op/s; 29837/154716 objects degraded (19.285%); 1790/154716 objects misplaced (1.157%); 16 MiB/s, 261 objects/s recovering 2023-09-23T18:08:11.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:11.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:11 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:11.912 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:11.989 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:12.265 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:12.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:12.282 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:12.287 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:08:12.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:12.415 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:12.427 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:08:12.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:12.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:12.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:12.559 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:12.590 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:12.633 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:12.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:12.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:12.691 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:08:12.691 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[179668]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.691 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[179668]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.691 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[179668]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T18:08:12.691 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[179668]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.692 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[179668]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:08:12.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[184174]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[184174]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[184174]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T18:08:12.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[184174]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.692 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:12 smithi055 ceph-mon[184174]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.738 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:12.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:12 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:08:12.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:12 smithi099 ceph-mon[167076]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:12 smithi099 ceph-mon[167076]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:12 smithi099 ceph-mon[167076]: osdmap e333: 8 total, 8 up, 8 in 2023-09-23T18:08:12.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:12 smithi099 ceph-mon[167076]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:12 smithi099 ceph-mon[167076]: failed to encode map e333 with expected crc 2023-09-23T18:08:12.769 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:12.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:12.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:12.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:12.918 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:13.001 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:13.077 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:13.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:13.082 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:13.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:13.227 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:13.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:13.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:13.284 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:13.324 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:13.385 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:13.389 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:13.462 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:13.495 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:08:13.547 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:13.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:13.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:13.659 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:13.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:13 smithi055 ceph-mon[184174]: pgmap v820: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 14 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 29598/153048 objects degraded (19.339%); 556/153048 objects misplaced (0.363%); 23 MiB/s, 365 objects/s recovering 2023-09-23T18:08:13.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 29837/154716 objects degraded (19.285%), 19 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:13.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:13.720 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:13.721 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:13 smithi055 ceph-mon[179668]: pgmap v820: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 14 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 29598/153048 objects degraded (19.339%); 556/153048 objects misplaced (0.363%); 23 MiB/s, 365 objects/s recovering 2023-09-23T18:08:13.721 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 29837/154716 objects degraded (19.285%), 19 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:13.721 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:13.721 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:13.752 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:13.827 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 18:08:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=cleanup t=2023-09-23T18:08:13.702418537Z level=info msg="Completed cleanup jobs" duration=43.38253ms 2023-09-23T18:08:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:13 smithi099 ceph-mon[167076]: pgmap v820: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 14 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 447 op/s; 29598/153048 objects degraded (19.339%); 556/153048 objects misplaced (0.363%); 23 MiB/s, 365 objects/s recovering 2023-09-23T18:08:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 29837/154716 objects degraded (19.285%), 19 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:13.863 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:13.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:13.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:13.971 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:14.027 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:14.094 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:14.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:14.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:14.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:14.235 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:14.283 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:14.287 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:14.288 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:14.388 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:14.435 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:14.619 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:14.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:14.623 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:14.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:14.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:14.624 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:14.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:14.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:14.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:14.724 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:14.759 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:14.767 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:14.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:08:14.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:08:14.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:14.893 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:14.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:14.898 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:14.999 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:15.040 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:08:15.040 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:08:15.040 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T18:08:15.040 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:08:15.040 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:08:15.040 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T18:08:15.041 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:15.082 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:15.086 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:15.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:15.218 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:15.262 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:15.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:15.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:15.276 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154235 5.2 5:44f142c1:::benchmark_data_smithi055_229117_object48609:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.835816+0000", 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 546.437630818, 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.120395545, 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:08:15.277 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154235 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835816+0000", 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835816+0000", 2023-09-23T18:08:15.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835815+0000", 2023-09-23T18:08:15.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:15.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835819+0000", 2023-09-23T18:08:15.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7720000000000001e-06 2023-09-23T18:08:15.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835827+0000", 2023-09-23T18:08:15.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9359999999999999e-06 2023-09-23T18:08:15.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835835+0000", 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3760000000000002e-06 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839941+0000", 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041069510000000002 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840066+0000", 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000124762 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840214+0000", 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014806500000000001 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.845845+0000", 2023-09-23T18:08:15.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0056306910000000002 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.856676+0000", 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010831341 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.856687+0000", 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0695e-05 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956153+0000", 2023-09-23T18:08:15.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0994657109999999 2023-09-23T18:08:15.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956212+0000", 2023-09-23T18:08:15.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8913000000000003e-05 2023-09-23T18:08:15.285 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.285 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.286 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.286 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.286 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.286 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154236 5.2 5:46f3ddda:::benchmark_data_smithi055_229117_object48610:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:08:15.286 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.835841+0000", 2023-09-23T18:08:15.286 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 546.43760617700002, 2023-09-23T18:08:15.286 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1205958920000001, 2023-09-23T18:08:15.286 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.286 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:08:15.286 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154236 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835841+0000", 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835841+0000", 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835840+0000", 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835843+0000", 2023-09-23T18:08:15.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.638e-06 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835849+0000", 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9150000000000001e-06 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.835854+0000", 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5650000000000001e-06 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.842866+0000", 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0070116919999999999 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.842987+0000", 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120715 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:08:15.290 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.843124+0000", 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137155 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.846319+0000", 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031950939999999999 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.858461+0000", 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012142101000000001 2023-09-23T18:08:15.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.858472+0000", 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0736e-05 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956386+0000", 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.097913626 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.956437+0000", 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1063000000000002e-05 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154274 5.5 5:aa5b5b0c:::benchmark_data_smithi055_229117_object48648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.105657+0000", 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 544.16778947499995, 2023-09-23T18:08:15.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.20535718, 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154274 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105657+0000", 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105657+0000", 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105656+0000", 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:15.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105663+0000", 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.742e-06 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105675+0000", 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1868000000000001e-05 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105683+0000", 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9410000000000007e-06 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298209+0000", 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.19252627899999999 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298292+0000", 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3452e-05 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.297 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.298686+0000", 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00039359900000000001 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306095+0000", 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0074089589999999997 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.298 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310633+0000", 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045383070000000001 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310919+0000", 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000285803 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310969+0000", 2023-09-23T18:08:15.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0439999999999998e-05 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311015+0000", 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5250999999999998e-05 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154280 5.5 5:a78a277e:::benchmark_data_smithi055_229117_object48654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:08:15.300 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.166102+0000", 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 544.10734459699995, 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14501488900000001, 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154280 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166102+0000", 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166102+0000", 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166101+0000", 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:15.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166109+0000", 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5660000000000001e-06 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166122+0000", 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3269e-05 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.166128+0000", 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5640000000000002e-06 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303566+0000", 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13743744199999999 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303721+0000", 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000155585 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.303856+0000", 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000134177 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310596+0000", 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0067407539999999998 2023-09-23T18:08:15.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310889+0000", 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00029301300000000001 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311049+0000", 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016005699999999999 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311086+0000", 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6881999999999997e-05 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.311117+0000", 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0936000000000003e-05 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.307 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154282 5.5 5:a1cbca1c:::benchmark_data_smithi055_229117_object48656:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.171172+0000", 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 544.10227463800004, 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14593202499999999, 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154282 2023-09-23T18:08:15.308 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171172+0000", 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171172+0000", 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171170+0000", 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171181+0000", 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.064e-05 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171212+0000", 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1433999999999998e-05 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.171223+0000", 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0265000000000001e-05 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306130+0000", 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13490687900000001 2023-09-23T18:08:15.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306257+0000", 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012784 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.306375+0000", 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011726999999999999 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.310640+0000", 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00426495 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.314542+0000", 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039025510000000002 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317023+0000", 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024812549999999999 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.313 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317068+0000", 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4632999999999998e-05 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317104+0000", 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6188e-05 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154290 5.5 5:a8c62379:::benchmark_data_smithi055_229117_object48664:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e177)", 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:11.196937+0000", 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 544.07650967999996, 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.12027204499999999, 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154290 2023-09-23T18:08:15.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196937+0000", 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196937+0000", 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196936+0000", 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196943+0000", 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.306e-06 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196960+0000", 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7173e-05 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.196969+0000", 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7390000000000005e-06 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308177+0000", 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11120793499999999 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308311+0000", 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013440899999999999 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.308441+0000", 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000129677 2023-09-23T18:08:15.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.312252+0000", 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038113700000000001 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.314577+0000", 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023246730000000002 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317137+0000", 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025597609999999998 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317173+0000", 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6371000000000001e-05 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.317209+0000", 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6191e-05 2023-09-23T18:08:15.321 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161327 5.e 5:7b0ada0d:::benchmark_data_smithi055_229117_object55701:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852567+0000", 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 532.42087984099999, 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0633735559999999, 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.322 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161327 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852567+0000", 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852567+0000", 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852565+0000", 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852572+0000", 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3729999999999998e-06 2023-09-23T18:08:15.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852585+0000", 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3426e-05 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852592+0000", 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2830000000000004e-06 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852658+0000", 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5399000000000002e-05 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op must wait for map", 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852668+0000", 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7110000000000007e-06 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872062+0000", 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019394650999999999 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872073+0000", 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0607e-05 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.885379+0000", 2023-09-23T18:08:15.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0133066719999999 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.887726+0000", 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023469929999999999 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.887957+0000", 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00023082800000000001 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915266+0000", 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027308517000000001 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915340+0000", 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4178999999999996e-05 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915403+0000", 2023-09-23T18:08:15.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2842000000000006e-05 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915811+0000", 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00040861899999999999 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915853+0000", 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1768999999999999e-05 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915941+0000", 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7410000000000005e-05 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161356 5.e 5:729afca1:::benchmark_data_smithi055_229117_object55730:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 532.42084192000004, 2023-09-23T18:08:15.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0635528459999999, 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161356 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852605+0000", 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:15.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852607+0000", 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9570000000000001e-06 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852613+0000", 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1199999999999999e-06 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852616+0000", 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4319999999999999e-06 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852880+0000", 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026363200000000003 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852889+0000", 2023-09-23T18:08:15.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8529999999999998e-06 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872097+0000", 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019208026999999999 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872104+0000", 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5739999999999998e-06 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.905797+0000", 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0336926310000001 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.905945+0000", 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000147899 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:08:15.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.906095+0000", 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015013200000000001 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915327+0000", 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092321869999999993 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915351+0000", 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.376e-05 2023-09-23T18:08:15.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915452+0000", 2023-09-23T18:08:15.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000101004 2023-09-23T18:08:15.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915997+0000", 2023-09-23T18:08:15.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00054548699999999999 2023-09-23T18:08:15.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916057+0000", 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9496000000000001e-05 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916158+0000", 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000100995 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.340 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161361 5.e 5:727196ef:::benchmark_data_smithi055_229117_object55735:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852804+0000", 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 532.42064298699995, 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0635737650000001, 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.341 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161361 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852804+0000", 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852804+0000", 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852803+0000", 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852808+0000", 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3579999999999996e-06 2023-09-23T18:08:15.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852820+0000", 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2822e-05 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852828+0000", 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9980000000000003e-06 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111282 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852947+0000", 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3749999999999997e-06 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872127+0000", 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019180169 2023-09-23T18:08:15.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872134+0000", 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.776e-06 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908641+0000", 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.036506683 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908722+0000", 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1464999999999993e-05 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.908805+0000", 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3060999999999996e-05 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915361+0000", 2023-09-23T18:08:15.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00655603 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915491+0000", 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012961299999999999 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915586+0000", 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5175999999999998e-05 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916217+0000", 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00063121599999999998 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916266+0000", 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8649000000000001e-05 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.916378+0000", 2023-09-23T18:08:15.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111749 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161400 5.e 5:7bd12518:::benchmark_data_smithi055_229117_object55774:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852914+0000", 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 532.42053323699997, 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.066386922, 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.350 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161400 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852914+0000", 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852914+0000", 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852913+0000", 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852917+0000", 2023-09-23T18:08:15.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1459999999999998e-06 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852927+0000", 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4700000000000008e-06 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852934+0000", 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8630000000000002e-06 2023-09-23T18:08:15.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852982+0000", 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8585999999999999e-05 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852990+0000", 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3989999999999998e-06 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872158+0000", 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019168266 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872163+0000", 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9620000000000001e-06 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.910791+0000", 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0386283650000001 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.910919+0000", 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012744899999999999 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.911051+0000", 2023-09-23T18:08:15.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013257500000000001 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915372+0000", 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004320618 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915538+0000", 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016641900000000001 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915724+0000", 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018572200000000001 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919091+0000", 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033675419999999998 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919170+0000", 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8464e-05 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919301+0000", 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013067499999999999 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:161452 5.e 5:7e9dcc4b:::benchmark_data_smithi055_229117_object55826:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e180)", 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 532.42050703899997, 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0665552599999999, 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161452 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.360 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852940+0000", 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852939+0000", 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852943+0000", 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1070000000000001e-06 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852949+0000", 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7900000000000002e-06 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.852955+0000", 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7760000000000003e-06 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.853019+0000", 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3619000000000002e-05 2023-09-23T18:08:15.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.853023+0000", 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1899999999999997e-06 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872216+0000", 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019192753 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:22.872224+0000", 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2779999999999993e-06 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913001+0000", 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.040776715 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913126+0000", 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012584599999999999 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4,5", 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.913244+0000", 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000117377 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915463+0000", 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022194480000000002 2023-09-23T18:08:15.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915650+0000", 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018667299999999999 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.915759+0000", 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109165 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919339+0000", 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035801880000000002 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919386+0000", 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6295999999999999e-05 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:23.919495+0000", 2023-09-23T18:08:15.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109508 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206358 5.2 5:46c0880b:::benchmark_data_smithi055_229117_object100732:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.895804+0000", 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 459.37764302900001, 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.119344292, 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.369 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206358 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895804+0000", 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895804+0000", 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895803+0000", 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895807+0000", 2023-09-23T18:08:15.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4759999999999998e-06 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895814+0000", 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.781e-06 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895819+0000", 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5269999999999998e-06 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.899325+0000", 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003505883 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.899449+0000", 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000123726 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:08:15.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.899575+0000", 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000126308 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.904891+0000", 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053159310000000003 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.904948+0000", 2023-09-23T18:08:15.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6798000000000001e-05 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.014957+0000", 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.110008884 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.014989+0000", 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2298999999999997e-05 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015038+0000", 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8914e-05 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015148+0000", 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000110263 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206362 5.2 5:4058eb16:::benchmark_data_smithi055_229117_object100736:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 459.377615564, 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.119556432, 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206362 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895831+0000", 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895834+0000", 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.943e-06 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895840+0000", 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4489999999999996e-06 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.895846+0000", 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4999999999999999e-06 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.902010+0000", 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0061640469999999998 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.902147+0000", 2023-09-23T18:08:15.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000137324 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.902297+0000", 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014995699999999999 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.904939+0000", 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026419070000000002 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015191+0000", 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11025235799999999 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015223+0000", 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1665000000000003e-05 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015245+0000", 2023-09-23T18:08:15.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2456e-05 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015286+0000", 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1009e-05 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.015388+0000", 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000101327 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.383 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:206368 5.2 5:4617325c:::benchmark_data_smithi055_229117_object100742:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e194)", 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:00:35.905040+0000", 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 459.36840683600002, 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.117245135, 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:15.384 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 206368 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905040+0000", 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905040+0000", 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905038+0000", 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905046+0000", 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2590000000000003e-06 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905057+0000", 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1406999999999999e-05 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905065+0000", 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9300000000000003e-06 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905107+0000", 2023-09-23T18:08:15.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1492000000000002e-05 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:35.905267+0000", 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016003600000000001 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5,6", 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.012294+0000", 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10702790199999999 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.016113+0000", 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038184 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.016149+0000", 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6520999999999998e-05 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.016543+0000", 2023-09-23T18:08:15.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00039407299999999999 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.022111+0000", 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055670880000000004 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.022163+0000", 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2528999999999998e-05 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:00:36.022285+0000", 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012215699999999999 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29520 7.4 7:2e3eff57:::benchmark_data_smithi055_311528_object29519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.342145+0000", 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 130.931301566, 2023-09-23T18:08:15.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.279446246, 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29520 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342145+0000", 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342145+0000", 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342144+0000", 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:15.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342167+0000", 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2671e-05 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342179+0000", 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2537999999999999e-05 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342199+0000", 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9791000000000001e-05 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342244+0000", 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5379e-05 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342363+0000", 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119085 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342477+0000", 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011398 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.344360+0000", 2023-09-23T18:08:15.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018828829999999999 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.351598+0000", 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072381310000000001 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.351613+0000", 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5082999999999999e-05 2023-09-23T18:08:15.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361553+0000", 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0099392469999999 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361566+0000", 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3742e-05 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.621482+0000", 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.259916043 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.621592+0000", 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109143 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29523 7.4 7:218b7f51:::benchmark_data_smithi055_311528_object29522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.362687+0000", 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 130.91076013399999, 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2720197689999999, 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:15.400 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29523 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362687+0000", 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362687+0000", 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362685+0000", 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362714+0000", 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9023999999999999e-05 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362730+0000", 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6053999999999999e-05 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362738+0000", 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7570000000000004e-06 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.363432+0000", 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00069362000000000004 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.363443+0000", 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1031e-05 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361597+0000", 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99815435299999999 2023-09-23T18:08:15.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361605+0000", 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7270000000000007e-06 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.623050+0000", 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2614453999999999 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.623261+0000", 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021100599999999999 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.623463+0000", 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020170799999999999 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626400+0000", 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00293728 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.628125+0000", 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001725266 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.633086+0000", 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049610030000000003 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.633146+0000", 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0000000000000002e-05 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.634707+0000", 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001560077 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29540 7.9 7:9dcdb66e:::benchmark_data_smithi055_311528_object29539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.375098+0000", 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 130.89834931199999, 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2623359889999999, 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29540 2023-09-23T18:08:15.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375098+0000", 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375098+0000", 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375096+0000", 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375120+0000", 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3737e-05 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375132+0000", 2023-09-23T18:08:15.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2211999999999999e-05 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375140+0000", 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6350000000000006e-06 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375200+0000", 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.067e-05 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375211+0000", 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0248000000000001e-05 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356527+0000", 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98131692100000001 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356539+0000", 2023-09-23T18:08:15.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1392e-05 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625642+0000", 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2691035230000001 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625784+0000", 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014192400000000001 2023-09-23T18:08:15.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625926+0000", 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000141607 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629334+0000", 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034082079999999998 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.630984+0000", 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00164963 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635790+0000", 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048063300000000002 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635850+0000", 2023-09-23T18:08:15.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9979000000000003e-05 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637434+0000", 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001583552 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29542 7.9 7:9a322fbf:::benchmark_data_smithi055_311528_object29541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.376211+0000", 2023-09-23T18:08:15.417 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 130.89723553499999, 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2627142660000001, 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29542 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376211+0000", 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376211+0000", 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376210+0000", 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376231+0000", 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1126000000000001e-05 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376248+0000", 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6461e-05 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376256+0000", 2023-09-23T18:08:15.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5590000000000006e-06 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.378137+0000", 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001880928 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.378147+0000", 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9010000000000006e-06 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356782+0000", 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97863468399999998 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356790+0000", 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5560000000000008e-06 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629412+0000", 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.272622081 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629587+0000", 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000174432 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629697+0000", 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109821 2023-09-23T18:08:15.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.631159+0000", 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014629059999999999 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.631648+0000", 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00048873199999999997 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637552+0000", 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0059042740000000002 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637614+0000", 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1674000000000006e-05 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.638926+0000", 2023-09-23T18:08:15.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013115170000000001 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29546 7.4 7:2ffa6bd1:::benchmark_data_smithi055_311528_object29545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.379888+0000", 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 130.893559332, 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2559961080000002, 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.426 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29546 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379888+0000", 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379888+0000", 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379886+0000", 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379920+0000", 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4292999999999998e-05 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379934+0000", 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4307999999999999e-05 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379943+0000", 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7010000000000002e-06 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379996+0000", 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2849000000000002e-05 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.380004+0000", 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5900000000000002e-06 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361629+0000", 2023-09-23T18:08:15.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.981625623 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.364509+0000", 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028797689999999999 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.624800+0000", 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.260291504 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.624960+0000", 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000159739 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625123+0000", 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016252700000000001 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.628149+0000", 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00302673 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629738+0000", 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001588922 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.634757+0000", 2023-09-23T18:08:15.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050189459999999998 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.634816+0000", 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8294000000000001e-05 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635884+0000", 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001068102 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.434 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29552 7.4 7:2f4e28dc:::benchmark_data_smithi055_311528_object29551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.385790+0000", 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 130.887656556, 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2515060980000001, 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:15.435 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29552 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385790+0000", 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385790+0000", 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:15.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385789+0000", 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385809+0000", 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0041e-05 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385821+0000", 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1803e-05 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385829+0000", 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4889999999999995e-06 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385881+0000", 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1408e-05 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385890+0000", 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2089999999999994e-06 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.364654+0000", 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97876412000000002 2023-09-23T18:08:15.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.364667+0000", 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2456999999999999e-05 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626449+0000", 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2617827670000001 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:15.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626582+0000", 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013226399999999999 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626686+0000", 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104701 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629538+0000", 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028518670000000001 2023-09-23T18:08:15.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629790+0000", 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00025145700000000002 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635940+0000", 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0061504029999999996 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635986+0000", 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6304000000000003e-05 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637296+0000", 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013099990000000001 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:15.443 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:15.444 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:08:15.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:15.473 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:15.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:15.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:15.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:15.595 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:08:15.619 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:15.620 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:08:15.650 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:08:15.685 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:15.686 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:08:15.725 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:08:15.771 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:08:15.801 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:08:15.813 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:08:15.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:15 smithi099 ceph-mon[167076]: failed to encode map e334 with expected crc 2023-09-23T18:08:15.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:15 smithi099 ceph-mon[167076]: failed to encode map e334 with expected crc 2023-09-23T18:08:15.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:15 smithi099 ceph-mon[167076]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T18:08:15.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:15 smithi099 ceph-mon[167076]: pgmap v822: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 14 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 273 op/s; 29512/152604 objects degraded (19.339%); 556/152604 objects misplaced (0.364%); 17 MiB/s, 270 objects/s recovering 2023-09-23T18:08:15.845 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:08:15.847 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:08:15.847 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:08:15.847 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:08:15.847 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:08:15.847 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:08:15.848 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:08:15.848 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:08:15.848 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:08:15.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:15 smithi055 ceph-mon[184174]: failed to encode map e334 with expected crc 2023-09-23T18:08:15.855 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:15 smithi055 ceph-mon[184174]: failed to encode map e334 with expected crc 2023-09-23T18:08:15.855 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:15 smithi055 ceph-mon[184174]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T18:08:15.855 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:15 smithi055 ceph-mon[184174]: pgmap v822: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 14 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 273 op/s; 29512/152604 objects degraded (19.339%); 556/152604 objects misplaced (0.364%); 17 MiB/s, 270 objects/s recovering 2023-09-23T18:08:15.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:15 smithi055 ceph-mon[179668]: failed to encode map e334 with expected crc 2023-09-23T18:08:15.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:15 smithi055 ceph-mon[179668]: failed to encode map e334 with expected crc 2023-09-23T18:08:15.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:15 smithi055 ceph-mon[179668]: osdmap e334: 8 total, 8 up, 8 in 2023-09-23T18:08:15.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:15 smithi055 ceph-mon[179668]: pgmap v822: 129 pgs: 1 active+recovering+undersized+remapped, 2 peering, 14 active+recovery_wait+degraded, 5 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 5.2 KiB/s rd, 14 KiB/s wr, 273 op/s; 29512/152604 objects degraded (19.339%); 556/152604 objects misplaced (0.364%); 17 MiB/s, 270 objects/s recovering 2023-09-23T18:08:16.213 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:16.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:16.229 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:16.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:16.376 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:16.452 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:16.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:16.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:16.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:16.566 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:16.595 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:16.633 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:16.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:16.637 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:16.638 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:16 smithi099 ceph-mon[167076]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.638 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:16 smithi099 ceph-mon[167076]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T18:08:16.638 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:16 smithi099 ceph-mon[167076]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.638 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:16 smithi099 ceph-mon[167076]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.638 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:16 smithi099 ceph-mon[167076]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.638 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/207765376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:16.732 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:16.739 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:16.792 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[184174]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.792 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[184174]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T18:08:16.792 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[184174]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.792 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[184174]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.793 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[184174]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.793 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/207765376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:16.796 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[179668]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.796 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[179668]: osdmap e335: 8 total, 8 up, 8 in 2023-09-23T18:08:16.797 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[179668]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.797 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[179668]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.797 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[179668]: failed to encode map e335 with expected crc 2023-09-23T18:08:16.797 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/207765376' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:16.803 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:16.928 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:16.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:16.933 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:17.034 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:17.078 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:17.127 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:17.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:17.132 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:17.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:17.269 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:17.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:17.321 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:17.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:17.422 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:17.454 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:17.495 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:17.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:17.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:17.602 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:17.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:17 smithi055 ceph-mon[184174]: pgmap v824: 129 pgs: 1 peering, 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 263 op/s; 26104/150042 objects degraded (17.398%); 2622/150042 objects misplaced (1.748%); 19 MiB/s, 296 objects/s recovering 2023-09-23T18:08:17.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:17 smithi055 ceph-mon[179668]: pgmap v824: 129 pgs: 1 peering, 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 263 op/s; 26104/150042 objects degraded (17.398%); 2622/150042 objects misplaced (1.748%); 19 MiB/s, 296 objects/s recovering 2023-09-23T18:08:17.659 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:17.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:17 smithi099 ceph-mon[167076]: pgmap v824: 129 pgs: 1 peering, 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.1 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 263 op/s; 26104/150042 objects degraded (17.398%); 2622/150042 objects misplaced (1.748%); 19 MiB/s, 296 objects/s recovering 2023-09-23T18:08:17.920 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:17.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:17.927 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:18.028 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:18.057 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:18.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:18.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:18.105 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:18.206 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:18.242 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:18.255 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:08:18.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:18.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:18.283 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:18.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:18.413 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:18.432 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:08:18.433 INFO:teuthology.orchestra.run.smithi055.stdout: "success": "" 2023-09-23T18:08:18.433 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:08:18.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:18.457 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:18.558 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:18.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:18 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T18:08:18.628 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:18.668 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:08:18.668 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:08:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:18 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T18:08:18.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 26104/150042 objects degraded (17.398%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:18.854 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:18 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1874855495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:18.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 26104/150042 objects degraded (17.398%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:18.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:18 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1874855495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:18.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:18 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T18:08:18.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 26104/150042 objects degraded (17.398%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:18.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:18 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1874855495' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:19.083 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:08:19.097 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:19.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:19.102 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:19.159 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:08:19.202 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:19.286 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:19.416 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:19.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:19.419 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:19.505 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:19.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:19.560 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:19.601 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:19.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:19.605 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:19.706 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:19.732 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:19 smithi055 ceph-mon[179668]: pgmap v825: 129 pgs: 1 peering, 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3.8 KiB/s rd, 10 KiB/s wr, 380 op/s; 25513/146433 objects degraded (17.423%); 2622/146433 objects misplaced (1.791%); 20 MiB/s, 324 objects/s recovering 2023-09-23T18:08:19.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:19 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:08:19.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:19 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1874855495' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:19.733 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:19 smithi055 ceph-mon[179668]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T18:08:19.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:19 smithi055 ceph-mon[184174]: pgmap v825: 129 pgs: 1 peering, 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3.8 KiB/s rd, 10 KiB/s wr, 380 op/s; 25513/146433 objects degraded (17.423%); 2622/146433 objects misplaced (1.791%); 20 MiB/s, 324 objects/s recovering 2023-09-23T18:08:19.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:19 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:08:19.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:19 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1874855495' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:19.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:19 smithi055 ceph-mon[184174]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T18:08:19.744 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:19.802 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:19.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:19.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:19.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:19 smithi099 ceph-mon[167076]: pgmap v825: 129 pgs: 1 peering, 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 109 active+clean; 3.0 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3.8 KiB/s rd, 10 KiB/s wr, 380 op/s; 25513/146433 objects degraded (17.423%); 2622/146433 objects misplaced (1.791%); 20 MiB/s, 324 objects/s recovering 2023-09-23T18:08:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:19 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:08:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:19 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1874855495' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:19 smithi099 ceph-mon[167076]: osdmap e336: 8 total, 8 up, 8 in 2023-09-23T18:08:19.907 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:19.950 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:20.332 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:20.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:20.339 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:20.347 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:20.441 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:20.487 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:20.591 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:08:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:08:20] "GET /metrics HTTP/1.1" 200 36686 "" "Prometheus/2.43.0" 2023-09-23T18:08:20.591 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:20 smithi055 ceph-mon[184174]: failed to encode map e336 with expected crc 2023-09-23T18:08:20.591 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:20 smithi055 ceph-mon[184174]: failed to encode map e336 with expected crc 2023-09-23T18:08:20.770 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:20.774 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:20.775 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:20.876 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:20.906 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:20 smithi099 ceph-mon[167076]: failed to encode map e336 with expected crc 2023-09-23T18:08:20.907 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:20 smithi099 ceph-mon[167076]: failed to encode map e336 with expected crc 2023-09-23T18:08:20.907 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:20.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:20 smithi055 ceph-mon[179668]: failed to encode map e336 with expected crc 2023-09-23T18:08:20.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:20 smithi055 ceph-mon[179668]: failed to encode map e336 with expected crc 2023-09-23T18:08:20.948 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:20.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:20.951 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:21.017 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:08:21.046 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:08:21.053 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:21.073 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:08:21.095 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:21.104 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:08:21.137 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:08:21.168 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:08:21.344 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:08:21.344 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:08:21.345 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:08:21.345 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:08:21.345 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:08:21.345 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:08:21.345 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:08:21.345 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:08:21.345 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:08:21.345 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:08:21.470 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:08:21.920 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:21 smithi055 ceph-mon[184174]: pgmap v827: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 24958/142851 objects degraded (17.471%); 1538/142851 objects misplaced (1.077%); 25 MiB/s, 392 objects/s recovering 2023-09-23T18:08:21.920 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3935712071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:21.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:21.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:21.921 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:21 smithi055 ceph-mon[179668]: pgmap v827: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 24958/142851 objects degraded (17.471%); 1538/142851 objects misplaced (1.077%); 25 MiB/s, 392 objects/s recovering 2023-09-23T18:08:21.921 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3935712071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:21.921 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:21.921 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:21.963 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:21.970 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:21.971 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:22.072 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:21 smithi099 ceph-mon[167076]: pgmap v827: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 517 op/s; 24958/142851 objects degraded (17.471%); 1538/142851 objects misplaced (1.077%); 25 MiB/s, 392 objects/s recovering 2023-09-23T18:08:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3935712071' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:22.106 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:22.250 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:22.255 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:22.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:22.356 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:22.394 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:22.442 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:22.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:22.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:22.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:22.581 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:22.597 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:22.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:22.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:22.646 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:22.691 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:08:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:08:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:08:22.691 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:08:22.692 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:08:22.746 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:22.799 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:22.854 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:22.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:22.861 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:22.961 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:23.007 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:23.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:23.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:23.063 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:23.100 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:08:23.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:23.202 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:23.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:23.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:23.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:23.354 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:23.418 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:23.593 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:23.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:23.598 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:23.699 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:23.759 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:23.869 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:23.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:23.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:23.963 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:08:23.976 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:24.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:23 smithi055 ceph-mon[179668]: pgmap v828: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 24479/140142 objects degraded (17.467%); 1538/140142 objects misplaced (1.097%); 23 MiB/s, 367 objects/s recovering 2023-09-23T18:08:24.007 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 24479/140142 objects degraded (17.467%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:24.007 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:23 smithi055 ceph-mon[184174]: pgmap v828: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 24479/140142 objects degraded (17.467%); 1538/140142 objects misplaced (1.097%); 23 MiB/s, 367 objects/s recovering 2023-09-23T18:08:24.007 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 24479/140142 objects degraded (17.467%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:24.077 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:24.148 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:08:24.148 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:08:24.148 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_2 2023-09-23T18:08:24.149 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:08:24.237 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:24.237 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:24.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:24.252 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:23 smithi099 ceph-mon[167076]: pgmap v828: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.9 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 519 op/s; 24479/140142 objects degraded (17.467%); 1538/140142 objects misplaced (1.097%); 23 MiB/s, 367 objects/s recovering 2023-09-23T18:08:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 24479/140142 objects degraded (17.467%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:24.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:24.388 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:24.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:24.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:24.453 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:24.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:24.589 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:24.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:24.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:24.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:24.742 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:24.807 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:24.912 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:08:25.066 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:25.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:25.071 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:25.072 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1355177322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:08:25.072 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:24 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:08:25.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1355177322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:08:25.072 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:24 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:08:25.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1355177322' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:08:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:24 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:08:25.172 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:25.213 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:25.367 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:25.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:25.414 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:25.414 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:25.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:25.545 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:25.589 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:25.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:25.593 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:25.694 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:25.701 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:25.736 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:25.796 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:25.805 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:25.806 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:25.907 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:25.915 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:08:25.915 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:08:25.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:08:25.915 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:08:25.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.915 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33540 6.e 6:757bf288:::benchmark_data_smithi055_275963_object33539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:08:25.915 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.437994+0000", 2023-09-23T18:08:25.915 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 406.47314510199999, 2023-09-23T18:08:25.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.108388157, 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33540 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:25.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.437994+0000", 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.437994+0000", 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.437991+0000", 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999967 2023-09-23T18:08:25.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438029+0000", 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.824e-05 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438067+0000", 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8182999999999997e-05 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438088+0000", 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0129999999999999e-05 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440561+0000", 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024736430000000002 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440572+0000", 2023-09-23T18:08:25.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0878e-05 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.534316+0000", 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.093743492 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.534461+0000", 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014496100000000001 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.534632+0000", 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017151100000000001 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.539612+0000", 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049804710000000002 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:25.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.543049+0000", 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003436588 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.544731+0000", 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016814320000000001 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.544791+0000", 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0006000000000001e-05 2023-09-23T18:08:25.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546382+0000", 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001591765 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33544 6.e 6:7c426a47:::benchmark_data_smithi055_275963_object33543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.438104+0000", 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 406.473034977, 2023-09-23T18:08:25.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.109720906, 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33544 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438104+0000", 2023-09-23T18:08:25.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438104+0000", 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438095+0000", 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999905 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438139+0000", 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3952999999999997e-05 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438945+0000", 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00080640499999999995 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:25.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438968+0000", 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3285000000000002e-05 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440588+0000", 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001619896 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440592+0000", 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8349999999999997e-06 2023-09-23T18:08:25.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.537672+0000", 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0970796060000001 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.537831+0000", 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015879199999999999 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.537997+0000", 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016669100000000001 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546403+0000", 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0084053589999999994 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546451+0000", 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8115000000000003e-05 2023-09-23T18:08:25.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546557+0000", 2023-09-23T18:08:25.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010645300000000001 2023-09-23T18:08:25.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:25.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546603+0000", 2023-09-23T18:08:25.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6140000000000002e-05 2023-09-23T18:08:25.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547825+0000", 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001221845 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33549 6.e 6:7eeaa6a3:::benchmark_data_smithi055_275963_object33548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.438977+0000", 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 406.472162368, 2023-09-23T18:08:25.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.110446166, 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33549 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:25.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438977+0000", 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438977+0000", 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438976+0000", 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:25.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439029+0000", 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3094999999999999e-05 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439039+0000", 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6029999999999994e-06 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439047+0000", 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5599999999999994e-06 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440614+0000", 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015667210000000001 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440620+0000", 2023-09-23T18:08:25.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7189999999999998e-06 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.539732+0000", 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0991117319999999 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.539867+0000", 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000135091 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.540051+0000", 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018418899999999999 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546410+0000", 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063587649999999997 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547897+0000", 2023-09-23T18:08:25.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014873009999999999 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547939+0000", 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2076999999999997e-05 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547993+0000", 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4109e-05 2023-09-23T18:08:25.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.549423+0000", 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001429865 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33551 6.e 6:7ac7b330:::benchmark_data_smithi055_275963_object33550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.439054+0000", 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 406.47208532100001, 2023-09-23T18:08:25.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.113266128, 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33551 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439054+0000", 2023-09-23T18:08:25.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439054+0000", 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439053+0000", 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439099+0000", 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5565999999999997e-05 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439108+0000", 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.789e-06 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:25.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439114+0000", 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.567e-06 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440644+0000", 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001530127 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440649+0000", 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.211e-06 2023-09-23T18:08:25.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.541379+0000", 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.100730271 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.541553+0000", 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017443600000000001 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.541772+0000", 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021913 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546460+0000", 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004687241 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.549485+0000", 2023-09-23T18:08:25.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030253210000000001 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.551089+0000", 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016042529999999999 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.551163+0000", 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3696999999999996e-05 2023-09-23T18:08:25.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.552320+0000", 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00115711 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:71682 7.9 7:94c03b51:::benchmark_data_smithi055_311528_object719:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:25.947 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:23.853495+0000", 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 62.057644074000002, 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.56297644700000005, 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 71682 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:23.853495+0000", 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:23.853495+0000", 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:23.853493+0000", 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:25.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:25.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:23.853504+0000", 2023-09-23T18:08:25.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0875e-05 2023-09-23T18:08:25.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:25.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:23.853520+0000", 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6195e-05 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:23.853531+0000", 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1202999999999999e-05 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:23.876080+0000", 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022548807000000001 2023-09-23T18:08:25.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:23.879355+0000", 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0032745869999999998 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:23.911631+0000", 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032276623999999997 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:23.915584+0000", 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039526159999999999 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.378052+0000", 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.46246814800000002 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.382930+0000", 2023-09-23T18:08:25.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048777600000000001 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.390365+0000", 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.007435194 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.403822+0000", 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013457075000000001 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.405029+0000", 2023-09-23T18:08:25.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0012068560000000001 2023-09-23T18:08:25.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:25.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.406484+0000", 2023-09-23T18:08:25.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014546159999999999 2023-09-23T18:08:25.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.408533+0000", 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020494189999999998 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.416472+0000", 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0079385559999999994 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:71919 7.19 7:9b0f5c72:::benchmark_data_smithi055_311528_object956:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:24.595250+0000", 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.315889429000002, 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.58977086400000001, 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 71919 2023-09-23T18:08:25.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.595250+0000", 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.595250+0000", 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.595249+0000", 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.595254+0000", 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9309999999999996e-06 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:25.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.595266+0000", 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.185e-05 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.595272+0000", 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.835e-06 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.616848+0000", 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021575454000000001 2023-09-23T18:08:25.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.621513+0000", 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0046650750000000003 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.638986+0000", 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017473282 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.642209+0000", 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0032233090000000002 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.136680+0000", 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.494470409 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:25.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.139963+0000", 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0032834629999999999 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.145647+0000", 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0056836899999999999 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.156015+0000", 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010368359000000001 2023-09-23T18:08:25.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.156812+0000", 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00079722099999999998 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.175708+0000", 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018895149999999999 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.177855+0000", 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021476440000000002 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.185021+0000", 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0071654570000000001 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:72035 7.19 7:9ef2df1e:::benchmark_data_smithi055_311528_object1072:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:24.858212+0000", 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 61.052927717000003, 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.578065261, 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:25.968 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 72035 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.858212+0000", 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.858212+0000", 2023-09-23T18:08:25.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:25.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.858210+0000", 2023-09-23T18:08:25.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:25.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:25.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.858216+0000", 2023-09-23T18:08:25.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5419999999999996e-06 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.858230+0000", 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3995999999999999e-05 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.858238+0000", 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6519999999999995e-06 2023-09-23T18:08:25.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.880402+0000", 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022163756 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.884026+0000", 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036238519999999999 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.924823+0000", 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040797158 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:24.928438+0000", 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003614688 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.398049+0000", 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.46961165300000002 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.400941+0000", 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028914240000000001 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.405776+0000", 2023-09-23T18:08:25.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048349220000000002 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.410524+0000", 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004748174 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.411333+0000", 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00080875800000000002 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.428528+0000", 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017195502000000001 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.429901+0000", 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013729230000000001 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.436277+0000", 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063757739999999999 2023-09-23T18:08:25.976 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:72239 7.19 7:9d98312d:::benchmark_data_smithi055_311528_object1276:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:25.227122+0000", 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 60.684017118, 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.58176607599999997, 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:25.977 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 72239 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.227122+0000", 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:25.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.227122+0000", 2023-09-23T18:08:25.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:25.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.227120+0000", 2023-09-23T18:08:25.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:25.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:25.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.227127+0000", 2023-09-23T18:08:25.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2710000000000003e-06 2023-09-23T18:08:25.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:25.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.227142+0000", 2023-09-23T18:08:25.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4248e-05 2023-09-23T18:08:25.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.227153+0000", 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0981e-05 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.329246+0000", 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10209321 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.332475+0000", 2023-09-23T18:08:25.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0032294060000000002 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.348058+0000", 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01558299 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.351680+0000", 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036219149999999999 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.768612+0000", 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.41693183499999997 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.771115+0000", 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002503281 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.776882+0000", 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0057667150000000004 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.800014+0000", 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.023131685999999999 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:25.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.800366+0000", 2023-09-23T18:08:25.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00035271899999999998 2023-09-23T18:08:25.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.801376+0000", 2023-09-23T18:08:25.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001009806 2023-09-23T18:08:25.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.803020+0000", 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016433489999999999 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.808888+0000", 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0058686750000000003 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:72277 7.19 7:9a8b3116:::benchmark_data_smithi055_311528_object1314:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:25.307165+0000", 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 60.603974209, 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.62060767999999999, 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 72277 2023-09-23T18:08:25.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.307165+0000", 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.307165+0000", 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.307163+0000", 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.307171+0000", 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9389999999999997e-06 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:25.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.307193+0000", 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.16e-05 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.307202+0000", 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2210000000000003e-06 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.366911+0000", 2023-09-23T18:08:25.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.059708872000000003 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.370535+0000", 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036241089999999999 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.391676+0000", 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021140833000000001 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.394640+0000", 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029643600000000001 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.905373+0000", 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.51073308100000003 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.907786+0000", 2023-09-23T18:08:25.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002412651 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.913297+0000", 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055110300000000001 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.918479+0000", 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0051818109999999997 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.919549+0000", 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001070372 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.920291+0000", 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00074158600000000002 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:25.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.921894+0000", 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016029880000000001 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:25.927773+0000", 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0058789489999999996 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:72752 7.9 7:927c3682:::benchmark_data_smithi055_311528_object1789:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:26.140392+0000", 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 59.770746897000002, 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.56058439199999999, 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 72752 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:25.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.140392+0000", 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.140392+0000", 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:25.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.140390+0000", 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.140398+0000", 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4590000000000001e-06 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.140417+0000", 2023-09-23T18:08:25.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9680000000000001e-05 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.140426+0000", 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2230000000000007e-06 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.146817+0000", 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063911089999999999 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:25.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.150545+0000", 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037286260000000001 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.172061+0000", 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021515754000000002 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.176217+0000", 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041555050000000003 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.666765+0000", 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.49054820300000002 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.669350+0000", 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025853030000000002 2023-09-23T18:08:26.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.674713+0000", 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053630170000000003 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.679963+0000", 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0052497029999999997 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.681110+0000", 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011476469999999999 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.691331+0000", 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010220306 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.693294+0000", 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019634069999999999 2023-09-23T18:08:26.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.700977+0000", 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.007682553 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:72759 7.9 7:9666d0d0:::benchmark_data_smithi055_311528_object1796:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:26.155321+0000", 2023-09-23T18:08:26.004 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 59.755818275999999, 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.599399927, 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 72759 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.155321+0000", 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.155321+0000", 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:26.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.155319+0000", 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.155326+0000", 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8040000000000004e-06 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.155339+0000", 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2969999999999999e-05 2023-09-23T18:08:26.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:26.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.155346+0000", 2023-09-23T18:08:26.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.878e-06 2023-09-23T18:08:26.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.189216+0000", 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.033870824000000001 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.192930+0000", 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037137070000000001 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.211300+0000", 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018369901000000001 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.214564+0000", 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0032643669999999998 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.701420+0000", 2023-09-23T18:08:26.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.48685552700000001 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.704237+0000", 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002817169 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.710923+0000", 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0066854619999999997 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.742656+0000", 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031733259999999999 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.744033+0000", 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001376976 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.745402+0000", 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001369419 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.747188+0000", 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0017856580000000001 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.754721+0000", 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0075329899999999998 2023-09-23T18:08:26.013 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:72774 7.9 7:92bcea14:::benchmark_data_smithi055_311528_object1811:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:26.178577+0000", 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 59.732561885999999, 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.58771137900000003, 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:26.014 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 72774 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.178577+0000", 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.015 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:26.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.178577+0000", 2023-09-23T18:08:26.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:26.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.178576+0000", 2023-09-23T18:08:26.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:26.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.178584+0000", 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9570000000000002e-06 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.178600+0000", 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6362e-05 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.178609+0000", 2023-09-23T18:08:26.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0189999999999995e-06 2023-09-23T18:08:26.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.194854+0000", 2023-09-23T18:08:26.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016245033999999998 2023-09-23T18:08:26.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.198676+0000", 2023-09-23T18:08:26.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038219030000000002 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.228423+0000", 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.029746788 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.231946+0000", 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00352338 2023-09-23T18:08:26.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.724870+0000", 2023-09-23T18:08:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.492924427 2023-09-23T18:08:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.728847+0000", 2023-09-23T18:08:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039763480000000002 2023-09-23T18:08:26.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.736906+0000", 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0080594020000000002 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.755978+0000", 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019072154000000001 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.756375+0000", 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00039630900000000002 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.757960+0000", 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015852660000000001 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.759722+0000", 2023-09-23T18:08:26.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0017616940000000001 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.766289+0000", 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0065671310000000004 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:72942 7.19 7:9d3e86b0:::benchmark_data_smithi055_311528_object1979:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:26.023 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:26.541893+0000", 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 59.369246326999999, 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.56402074000000002, 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 72942 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.541893+0000", 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.541893+0000", 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.541891+0000", 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:26.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:26.026 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.541898+0000", 2023-09-23T18:08:26.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9290000000000001e-06 2023-09-23T18:08:26.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:26.026 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.541911+0000", 2023-09-23T18:08:26.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2931e-05 2023-09-23T18:08:26.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.541920+0000", 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7290000000000006e-06 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.585165+0000", 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043244965000000003 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.588601+0000", 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003435876 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.619670+0000", 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031068603 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.623252+0000", 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035821609999999999 2023-09-23T18:08:26.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.074852+0000", 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.45160031 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.077845+0000", 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029927159999999999 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.083449+0000", 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0056044459999999999 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.097042+0000", 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013592488 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.097300+0000", 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00025801699999999998 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.098257+0000", 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00095731699999999996 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.099856+0000", 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001598822 2023-09-23T18:08:26.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.105914+0000", 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060578569999999998 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:72945 7.19 7:9b88581a:::benchmark_data_smithi055_311528_object1982:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:26.551400+0000", 2023-09-23T18:08:26.032 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 59.359739603999998, 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.61315037100000003, 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 72945 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:26.033 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.551400+0000", 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.551400+0000", 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.551398+0000", 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:26.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.551406+0000", 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5739999999999998e-06 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.551418+0000", 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2320000000000001e-05 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.551428+0000", 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.057e-05 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.588897+0000", 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.037468992999999999 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.591723+0000", 2023-09-23T18:08:26.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028251159999999999 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.635384+0000", 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043661830999999998 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.639100+0000", 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037156250000000002 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.133338+0000", 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.49423774300000001 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.135823+0000", 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002484722 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.141104+0000", 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0052809880000000003 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.154558+0000", 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013454247000000001 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.155597+0000", 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0010388319999999999 2023-09-23T18:08:26.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.156705+0000", 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001108167 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.158510+0000", 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018049209999999999 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.164550+0000", 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060403189999999997 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:72956 7.19 7:9f2f3ece:::benchmark_data_smithi055_311528_object1993:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:26.641436+0000", 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 59.269703028000002, 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.56956525300000005, 2023-09-23T18:08:26.041 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 72956 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.641436+0000", 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.042 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.641436+0000", 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.641435+0000", 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.641442+0000", 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9480000000000002e-06 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.641455+0000", 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2979000000000001e-05 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:26.044 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.641462+0000", 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1199999999999996e-06 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.650717+0000", 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092548119999999994 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.653573+0000", 2023-09-23T18:08:26.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00285659 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.682013+0000", 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028439942999999999 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:26.685103+0000", 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030897939999999999 2023-09-23T18:08:26.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.176126+0000", 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.491022813 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.178608+0000", 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024819830000000001 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.047 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.184457+0000", 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0058493870000000002 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.201778+0000", 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017320703999999999 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.048 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.203153+0000", 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00137525 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.204364+0000", 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0012109499999999999 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.205656+0000", 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001291798 2023-09-23T18:08:26.049 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.211001+0000", 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053456479999999997 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:73399 7.19 7:9f76dd02:::benchmark_data_smithi055_311528_object2436:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:26.050 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:27.653732+0000", 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 58.257407596, 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.57187848399999996, 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 73399 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.051 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.653732+0000", 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.653732+0000", 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.653730+0000", 2023-09-23T18:08:26.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.653736+0000", 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4709999999999998e-06 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.653747+0000", 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1557e-05 2023-09-23T18:08:26.053 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.653754+0000", 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6200000000000001e-06 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.692418+0000", 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038664397000000003 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.054 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.696195+0000", 2023-09-23T18:08:26.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003776883 2023-09-23T18:08:26.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.740689+0000", 2023-09-23T18:08:26.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.044493515999999997 2023-09-23T18:08:26.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.744701+0000", 2023-09-23T18:08:26.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040122059999999999 2023-09-23T18:08:26.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.172544+0000", 2023-09-23T18:08:26.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.42784314099999998 2023-09-23T18:08:26.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.174955+0000", 2023-09-23T18:08:26.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024110479999999998 2023-09-23T18:08:26.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.180616+0000", 2023-09-23T18:08:26.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0056604130000000004 2023-09-23T18:08:26.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.185921+0000", 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053054349999999998 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.186687+0000", 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000765754 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.219039+0000", 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032352417000000001 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.220389+0000", 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013499370000000001 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.225610+0000", 2023-09-23T18:08:26.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0052209869999999999 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:73413 7.9 7:93bdda68:::benchmark_data_smithi055_311528_object2450:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:27.682580+0000", 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 58.228559044999997, 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.57530552499999998, 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:26.060 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 73413 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.682580+0000", 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.682580+0000", 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.682578+0000", 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.682586+0000", 2023-09-23T18:08:26.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9189999999999999e-06 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.682603+0000", 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6348e-05 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.682611+0000", 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3110000000000006e-06 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.700774+0000", 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018163065999999999 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.705611+0000", 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048371009999999999 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.736697+0000", 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031085590999999999 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:27.739870+0000", 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031734760000000002 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.189151+0000", 2023-09-23T18:08:26.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.44928138299999998 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.191748+0000", 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025967310000000001 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.196951+0000", 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0052024599999999999 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.219503+0000", 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022552075000000001 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.220648+0000", 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001144946 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.248762+0000", 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028113920000000001 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.250679+0000", 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001917766 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.257886+0000", 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072063470000000001 2023-09-23T18:08:26.068 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:73754 7.19 7:999dfb10:::benchmark_data_smithi055_311528_object2791:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:28.636738+0000", 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 57.274400739999997, 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.56482160599999998, 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:26.069 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 73754 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.636738+0000", 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:26.070 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.636738+0000", 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.636737+0000", 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.636745+0000", 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0129999999999993e-06 2023-09-23T18:08:26.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.636762+0000", 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7578e-05 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.636772+0000", 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.679e-06 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.653895+0000", 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017123276 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.657467+0000", 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035721009999999998 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.686576+0000", 2023-09-23T18:08:26.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.029108230999999998 2023-09-23T18:08:26.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:28.690019+0000", 2023-09-23T18:08:26.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034437059999999999 2023-09-23T18:08:26.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:29.151207+0000", 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.46118814000000002 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:29.153633+0000", 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024255230000000002 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:29.158906+0000", 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.005273036 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:29.163946+0000", 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050403310000000003 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:29.165122+0000", 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001176056 2023-09-23T18:08:26.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:29.193391+0000", 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028268103999999999 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:29.195510+0000", 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021197260000000002 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:29.201560+0000", 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060498339999999996 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:83245 7.19 7:9f649a88:::benchmark_data_smithi055_311528_object12282:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:52.699771+0000", 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 33.211368116000003, 2023-09-23T18:08:26.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.570757548, 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 83245 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.699771+0000", 2023-09-23T18:08:26.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.699771+0000", 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.699770+0000", 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:08:26.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.699775+0000", 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.101e-06 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.699787+0000", 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1803e-05 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.699794+0000", 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4490000000000002e-06 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.745127+0000", 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.045332861000000002 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.747626+0000", 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024985469999999998 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.760679+0000", 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013053555999999999 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.764470+0000", 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037908479999999999 2023-09-23T18:08:26.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.230181+0000", 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.46571045599999999 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.232273+0000", 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020922409999999999 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.237018+0000", 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004744786 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.260471+0000", 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.023453076 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.261286+0000", 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00081485900000000003 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.263517+0000", 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022313070000000001 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.265047+0000", 2023-09-23T18:08:26.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015303789999999999 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.270529+0000", 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0054813329999999997 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:83275 7.9 7:9001bba7:::benchmark_data_smithi055_311528_object12312:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:52.906249+0000", 2023-09-23T18:08:26.087 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 33.004889906000002, 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.59401608699999997, 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 83275 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.906249+0000", 2023-09-23T18:08:26.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.906249+0000", 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.906248+0000", 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.906255+0000", 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3300000000000001e-06 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.906269+0000", 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3589e-05 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.906275+0000", 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5710000000000003e-06 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.920043+0000", 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01376787 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.922997+0000", 2023-09-23T18:08:26.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029536359999999999 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.940093+0000", 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017096712 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:52.943635+0000", 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035419570000000001 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.472713+0000", 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.52907776200000001 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.475739+0000", 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003026262 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:08:26.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.481491+0000", 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.005751847 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.492210+0000", 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010718947 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.493164+0000", 2023-09-23T18:08:26.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00095410899999999997 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.493479+0000", 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00031448499999999998 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.494972+0000", 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014933349999999999 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:26.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:26.096 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:26.096 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:53.500265+0000", 2023-09-23T18:08:26.096 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0052934059999999996 2023-09-23T18:08:26.096 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.096 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.096 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.096 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:26.096 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:26.096 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:08:26.097 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:26.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[184174]: pgmap v829: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 24324/139164 objects degraded (17.479%); 1538/139164 objects misplaced (1.105%); 20 MiB/s, 324 objects/s recovering 2023-09-23T18:08:26.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:26.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[184174]: failed to encode map e337 with expected crc 2023-09-23T18:08:26.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[184174]: failed to encode map e337 with expected crc 2023-09-23T18:08:26.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:08:26.098 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[184174]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T18:08:26.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[179668]: pgmap v829: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 24324/139164 objects degraded (17.479%); 1538/139164 objects misplaced (1.105%); 20 MiB/s, 324 objects/s recovering 2023-09-23T18:08:26.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:26.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[179668]: failed to encode map e337 with expected crc 2023-09-23T18:08:26.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[179668]: failed to encode map e337 with expected crc 2023-09-23T18:08:26.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:08:26.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:25 smithi055 ceph-mon[179668]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T18:08:26.292 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:26.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:26.299 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:25 smithi099 ceph-mon[167076]: pgmap v829: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.8 GiB data, 13 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 473 op/s; 24324/139164 objects degraded (17.479%); 1538/139164 objects misplaced (1.105%); 20 MiB/s, 324 objects/s recovering 2023-09-23T18:08:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:25 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:25 smithi099 ceph-mon[167076]: failed to encode map e337 with expected crc 2023-09-23T18:08:26.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:25 smithi099 ceph-mon[167076]: failed to encode map e337 with expected crc 2023-09-23T18:08:26.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:25 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:08:26.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:25 smithi099 ceph-mon[167076]: osdmap e337: 8 total, 8 up, 8 in 2023-09-23T18:08:26.401 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:26.457 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:27.433 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:08:27.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:27 smithi055 ceph-mon[179668]: pgmap v831: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 23686/135450 objects degraded (17.487%); 166/135450 objects misplaced (0.123%); 23 MiB/s, 362 objects/s recovering 2023-09-23T18:08:28.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:27 smithi055 ceph-mon[184174]: pgmap v831: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 23686/135450 objects degraded (17.487%); 166/135450 objects misplaced (0.123%); 23 MiB/s, 362 objects/s recovering 2023-09-23T18:08:28.206 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:28.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:28.214 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:28.303 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:08:28.315 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:28.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:27 smithi099 ceph-mon[167076]: pgmap v831: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 457 op/s; 23686/135450 objects degraded (17.487%); 166/135450 objects misplaced (0.123%); 23 MiB/s, 362 objects/s recovering 2023-09-23T18:08:28.404 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 23686/135450 objects degraded (17.487%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:29.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 23686/135450 objects degraded (17.487%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:29.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 23686/135450 objects degraded (17.487%), 18 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:29.783 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:29.790 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:29.791 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:29.891 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:29.949 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:30 smithi099 ceph-mon[167076]: pgmap v832: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 23013/131475 objects degraded (17.504%); 166/131475 objects misplaced (0.126%); 23 MiB/s, 367 objects/s recovering 2023-09-23T18:08:30.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:30 smithi055 ceph-mon[184174]: pgmap v832: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 23013/131475 objects degraded (17.504%); 166/131475 objects misplaced (0.126%); 23 MiB/s, 367 objects/s recovering 2023-09-23T18:08:30.361 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:30 smithi055 ceph-mon[179668]: pgmap v832: 129 pgs: 14 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 2.7 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 511 op/s; 23013/131475 objects degraded (17.504%); 166/131475 objects misplaced (0.126%); 23 MiB/s, 367 objects/s recovering 2023-09-23T18:08:30.673 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:08:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:08:30] "GET /metrics HTTP/1.1" 200 36691 "" "Prometheus/2.43.0" 2023-09-23T18:08:30.699 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:30.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:30.705 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:30.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:30.875 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:30.930 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:30.938 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:30.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:30.941 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:31.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:31.096 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:31.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:31.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:31.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:31.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:31.340 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:31.408 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:31.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:31.413 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:31.514 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:31.572 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:31.762 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:31.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:31.767 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:31.868 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:31.879 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:31.879 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:08:31.917 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:08:31.953 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:31.962 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:08:32.004 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:08:32.048 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:08:32.088 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:08:32.133 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:08:32.200 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:32.201 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:08:32.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:32.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:32.225 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:08:32.225 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:08:32.225 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:08:32.225 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:08:32.225 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:08:32.226 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:08:32.226 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:08:32.226 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:08:32.285 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:32 smithi055 ceph-mon[184174]: pgmap v833: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 21120/126960 objects degraded (16.635%); 738/126960 objects misplaced (0.581%); 23 MiB/s, 368 objects/s recovering 2023-09-23T18:08:32.286 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:32 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/253999490' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:32.286 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:32 smithi055 ceph-mon[179668]: pgmap v833: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 21120/126960 objects degraded (16.635%); 738/126960 objects misplaced (0.581%); 23 MiB/s, 368 objects/s recovering 2023-09-23T18:08:32.286 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:32 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/253999490' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:32.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:32.342 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:32 smithi099 ceph-mon[167076]: pgmap v833: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.6 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 529 op/s; 21120/126960 objects degraded (16.635%); 738/126960 objects misplaced (0.581%); 23 MiB/s, 368 objects/s recovering 2023-09-23T18:08:32.343 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:32 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/253999490' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:32.345 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:32.388 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:32.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:32.394 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:32.495 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:32.587 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:32.693 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:08:32.702 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:08:32.809 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:32.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:32.815 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:32.916 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:32.974 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:33.235 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:33.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:33.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 21120/126960 objects degraded (16.635%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:33.243 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 21120/126960 objects degraded (16.635%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:33.244 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:33.292 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":833,"stamp":"2023-09-23T18:08:30.824157+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2776031737,"num_objects":42320,"num_object_clones":0,"num_object_copies":126960,"num_objects_missing_on_primary":112,"num_objects_missing":112,"num_objects_degraded":21120,"num_objects_misplaced":738,"num_objects_unfound":0,"num_objects_dirty":42320,"num_whiteouts":0,"num_read":6104,"num_read_kb":12433,"num_write":100612,"num_write_kb":4554917,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24838,"num_bytes_recovered":1620838165,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":142432,"ondisk_log_size":142432,"up":387,"acting":383,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":421,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":12710452,"kb_used_data":10112308,"kb_used_omap":50,"kb_used_meta":2598029,"kb_avail":737250764,"statfs":{"total":767960285184,"available":754944782336,"internally_reserved":0,"allocated":10355003392,"data_stored":10343368384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51828,"internal_metadata":2660382092},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,3,1,2,4,2,0,1,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-347144192,"num_objects":-5297,"num_object_clones":0,"num_object_copies":-15891,"num_objects_missing_on_primary":-343,"num_objects_missing":-343,"num_objects_degraded":-3838,"num_objects_misplaced":-800,"num_objects_unfound":0,"num_objects_dirty":-5297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5297,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3690,"num_bytes_recovered":241827840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004089"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":664,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089635+0000","last_change":"2023-09-23T18:05:15.087182+0000","last_active":"2023-09-23T18:08:19.089635+0000","last_peered":"2023-09-23T18:08:19.089635+0000","last_clean":"2023-09-23T18:08:19.089635+0000","last_became_active":"2023-09-23T18:05:15.086678+0000","last_became_peered":"2023-09-23T18:05:15.086678+0000","last_unstale":"2023-09-23T18:08:19.089635+0000","last_undegraded":"2023-09-23T18:08:19.089635+0000","last_fullsized":"2023-09-23T18:08:19.089635+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1262601,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":147,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.086430+0000","last_change":"2023-09-23T18:06:05.377123+0000","last_active":"2023-09-23T18:08:19.086430+0000","last_peered":"2023-09-23T18:08:19.086430+0000","last_clean":"2023-09-23T18:08:19.086430+0000","last_became_active":"2023-09-23T18:06:05.376735+0000","last_became_peered":"2023-09-23T18:06:05.376735+0000","last_unstale":"2023-09-23T18:08:19.086430+0000","last_undegraded":"2023-09-23T18:08:19.086430+0000","last_fullsized":"2023-09-23T18:08:19.086430+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:11.555818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":213,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.090482+0000","last_change":"2023-09-23T18:06:51.071290+0000","last_active":"2023-09-23T18:08:19.090482+0000","last_peered":"2023-09-23T18:08:19.090482+0000","last_clean":"2023-09-23T18:08:19.090482+0000","last_became_active":"2023-09-23T18:06:51.070948+0000","last_became_peered":"2023-09-23T18:06:51.070948+0000","last_unstale":"2023-09-23T18:08:19.090482+0000","last_undegraded":"2023-09-23T18:08:19.090482+0000","last_fullsized":"2023-09-23T18:08:19.090482+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:52.647972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":682,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.090454+0000","last_change":"2023-09-23T18:06:54.065593+0000","last_active":"2023-09-23T18:08:19.090454+0000","last_peered":"2023-09-23T18:08:19.090454+0000","last_clean":"2023-09-23T18:08:19.090454+0000","last_became_active":"2023-09-23T18:06:54.065193+0000","last_became_peered":"2023-09-23T18:06:54.065193+0000","last_unstale":"2023-09-23T18:08:19.090454+0000","last_undegraded":"2023-09-23T18:08:19.090454+0000","last_fullsized":"2023-09-23T18:08:19.090454+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:05.792712+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":52,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.098762+0000","last_change":"2023-09-23T18:06:05.370046+0000","last_active":"2023-09-23T18:08:19.098762+0000","last_peered":"2023-09-23T18:08:19.098762+0000","last_clean":"2023-09-23T18:08:19.098762+0000","last_became_active":"2023-09-23T18:06:05.369714+0000","last_became_peered":"2023-09-23T18:06:05.369714+0000","last_unstale":"2023-09-23T18:08:19.098762+0000","last_undegraded":"2023-09-23T18:08:19.098762+0000","last_fullsized":"2023-09-23T18:08:19.098762+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:47.453671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":346,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.081274+0000","last_change":"2023-09-23T18:06:51.070985+0000","last_active":"2023-09-23T18:08:19.081274+0000","last_peered":"2023-09-23T18:08:19.081274+0000","last_clean":"2023-09-23T18:08:19.081274+0000","last_became_active":"2023-09-23T18:06:51.070671+0000","last_became_peered":"2023-09-23T18:06:51.070671+0000","last_unstale":"2023-09-23T18:08:19.081274+0000","last_undegraded":"2023-09-23T18:08:19.081274+0000","last_fullsized":"2023-09-23T18:08:19.081274+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:14.820905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":674,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.086571+0000","last_change":"2023-09-23T18:06:08.439343+0000","last_active":"2023-09-23T18:08:19.086571+0000","last_peered":"2023-09-23T18:08:19.086571+0000","last_clean":"2023-09-23T18:08:19.086571+0000","last_became_active":"2023-09-23T18:06:08.437702+0000","last_became_peered":"2023-09-23T18:06:08.437702+0000","last_unstale":"2023-09-23T18:08:19.086571+0000","last_undegraded":"2023-09-23T18:08:19.086571+0000","last_fullsized":"2023-09-23T18:08:19.086571+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:02.398105+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":216,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087900+0000","last_change":"2023-09-23T18:06:51.087219+0000","last_active":"2023-09-23T18:08:19.087900+0000","last_peered":"2023-09-23T18:08:19.087900+0000","last_clean":"2023-09-23T18:08:19.087900+0000","last_became_active":"2023-09-23T18:06:51.086885+0000","last_became_peered":"2023-09-23T18:06:51.086885+0000","last_unstale":"2023-09-23T18:08:19.087900+0000","last_undegraded":"2023-09-23T18:08:19.087900+0000","last_fullsized":"2023-09-23T18:08:19.087900+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:16.254186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":287,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.094242+0000","last_change":"2023-09-23T18:06:05.372350+0000","last_active":"2023-09-23T18:08:19.094242+0000","last_peered":"2023-09-23T18:08:19.094242+0000","last_clean":"2023-09-23T18:08:19.094242+0000","last_became_active":"2023-09-23T18:06:05.372026+0000","last_became_peered":"2023-09-23T18:06:05.372026+0000","last_unstale":"2023-09-23T18:08:19.094242+0000","last_undegraded":"2023-09-23T18:08:19.094242+0000","last_fullsized":"2023-09-23T18:08:19.094242+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:37.063301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.13","version":"337'5258","reported_seq":5352,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:30.777464+0000","last_change":"2023-09-23T18:08:10.525503+0000","last_active":"2023-09-23T18:08:30.777464+0000","last_peered":"2023-09-23T18:08:30.777464+0000","last_clean":"2023-09-23T18:08:30.777464+0000","last_became_active":"2023-09-23T18:08:10.516929+0000","last_became_peered":"2023-09-23T18:08:10.516929+0000","last_unstale":"2023-09-23T18:08:30.777464+0000","last_undegraded":"2023-09-23T18:08:30.777464+0000","last_fullsized":"2023-09-23T18:08:30.777464+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5258,"log_dups_size":0,"ondisk_log_size":5258,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85590016,"num_objects":1306,"num_object_clones":0,"num_object_copies":3918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1306,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3162,"num_write_kb":142976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":717,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.081278+0000","last_change":"2023-09-23T18:06:08.433828+0000","last_active":"2023-09-23T18:08:19.081278+0000","last_peered":"2023-09-23T18:08:19.081278+0000","last_clean":"2023-09-23T18:08:19.081278+0000","last_became_active":"2023-09-23T18:06:08.433445+0000","last_became_peered":"2023-09-23T18:06:08.433445+0000","last_unstale":"2023-09-23T18:08:19.081278+0000","last_undegraded":"2023-09-23T18:08:19.081278+0000","last_fullsized":"2023-09-23T18:08:19.081278+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.816081+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":327,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.136714+0000","last_change":"2023-09-23T18:06:54.195144+0000","last_active":"2023-09-23T18:08:19.136714+0000","last_peered":"2023-09-23T18:08:19.136714+0000","last_clean":"2023-09-23T18:08:19.136714+0000","last_became_active":"2023-09-23T18:06:54.194921+0000","last_became_peered":"2023-09-23T18:06:54.194921+0000","last_unstale":"2023-09-23T18:08:19.136714+0000","last_undegraded":"2023-09-23T18:08:19.136714+0000","last_fullsized":"2023-09-23T18:08:19.136714+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:04.645926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,6],"acting":[0,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":204,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.096325+0000","last_change":"2023-09-23T18:06:05.364021+0000","last_active":"2023-09-23T18:08:19.096325+0000","last_peered":"2023-09-23T18:08:19.096325+0000","last_clean":"2023-09-23T18:08:19.096325+0000","last_became_active":"2023-09-23T18:06:05.363740+0000","last_became_peered":"2023-09-23T18:06:05.363740+0000","last_unstale":"2023-09-23T18:08:19.096325+0000","last_undegraded":"2023-09-23T18:08:19.096325+0000","last_fullsized":"2023-09-23T18:08:19.096325+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:55.662682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.12","version":"337'5125","reported_seq":8889,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:27.538517+0000","last_change":"2023-09-23T18:07:49.252511+0000","last_active":"2023-09-23T18:08:27.538517+0000","last_peered":"2023-09-23T18:08:27.538517+0000","last_clean":"2023-09-23T18:08:27.538517+0000","last_became_active":"2023-09-23T18:07:49.239551+0000","last_became_peered":"2023-09-23T18:07:49.239551+0000","last_unstale":"2023-09-23T18:08:27.538517+0000","last_undegraded":"2023-09-23T18:08:27.538517+0000","last_fullsized":"2023-09-23T18:08:27.538517+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5125,"log_dups_size":0,"ondisk_log_size":5125,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87490560,"num_objects":1335,"num_object_clones":0,"num_object_copies":4005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3061,"num_write_kb":140672,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":915,"num_bytes_recovered":59965440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":615,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.081728+0000","last_change":"2023-09-23T18:08:10.518125+0000","last_active":"2023-09-23T18:08:19.081728+0000","last_peered":"2023-09-23T18:08:19.081728+0000","last_clean":"2023-09-23T18:08:19.081728+0000","last_became_active":"2023-09-23T18:08:10.517840+0000","last_became_peered":"2023-09-23T18:08:10.517840+0000","last_unstale":"2023-09-23T18:08:19.081728+0000","last_undegraded":"2023-09-23T18:08:19.081728+0000","last_fullsized":"2023-09-23T18:08:19.081728+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.888321+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.888321+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":162897,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":329,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.092774+0000","last_change":"2023-09-23T18:06:05.371032+0000","last_active":"2023-09-23T18:08:19.092774+0000","last_peered":"2023-09-23T18:08:19.092774+0000","last_clean":"2023-09-23T18:08:19.092774+0000","last_became_active":"2023-09-23T18:06:05.370695+0000","last_became_peered":"2023-09-23T18:06:05.370695+0000","last_unstale":"2023-09-23T18:08:19.092774+0000","last_undegraded":"2023-09-23T18:08:19.092774+0000","last_fullsized":"2023-09-23T18:08:19.092774+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:12.628753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":122,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.091775+0000","last_change":"2023-09-23T18:07:49.223713+0000","last_active":"2023-09-23T18:08:19.091775+0000","last_peered":"2023-09-23T18:08:19.091775+0000","last_clean":"2023-09-23T18:08:19.091775+0000","last_became_active":"2023-09-23T18:07:49.223409+0000","last_became_peered":"2023-09-23T18:07:49.223409+0000","last_unstale":"2023-09-23T18:08:19.091775+0000","last_undegraded":"2023-09-23T18:08:19.091775+0000","last_fullsized":"2023-09-23T18:08:19.091775+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:12.387318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.11","version":"337'5195","reported_seq":7206,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:30.768792+0000","last_change":"2023-09-23T18:07:12.880502+0000","last_active":"2023-09-23T18:08:30.768792+0000","last_peered":"2023-09-23T18:08:30.768792+0000","last_clean":"2023-09-23T18:06:25.121665+0000","last_became_active":"2023-09-23T18:07:12.878060+0000","last_became_peered":"2023-09-23T18:07:12.878060+0000","last_unstale":"2023-09-23T18:08:30.768792+0000","last_undegraded":"2023-09-23T18:07:12.230591+0000","last_fullsized":"2023-09-23T18:08:30.768792+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5195,"log_dups_size":0,"ondisk_log_size":5195,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86278144,"num_objects":1317,"num_object_clones":0,"num_object_copies":3951,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1317,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3115,"num_write_kb":141792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":894,"num_bytes_recovered":58556416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":["4","3","6"],"object_location_counts":[{"shards":"1,4,6","objects":687},{"shards":"3,4,6","objects":630}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":681,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089960+0000","last_change":"2023-09-23T18:06:54.059837+0000","last_active":"2023-09-23T18:08:19.089960+0000","last_peered":"2023-09-23T18:08:19.089960+0000","last_clean":"2023-09-23T18:08:19.089960+0000","last_became_active":"2023-09-23T18:06:54.059489+0000","last_became_peered":"2023-09-23T18:06:54.059489+0000","last_unstale":"2023-09-23T18:08:19.089960+0000","last_undegraded":"2023-09-23T18:08:19.089960+0000","last_fullsized":"2023-09-23T18:08:19.089960+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.652623+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.652623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:28.333390+0000","scrub_duration":0.00648549,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":102,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.081200+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:08:19.081200+0000","last_peered":"2023-09-23T18:08:19.081200+0000","last_clean":"2023-09-23T18:08:19.081200+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:08:19.081200+0000","last_undegraded":"2023-09-23T18:08:19.081200+0000","last_fullsized":"2023-09-23T18:08:19.081200+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":63,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:24.900242+0000","last_change":"2023-09-23T18:06:05.363236+0000","last_active":"2023-09-23T18:08:24.900242+0000","last_peered":"2023-09-23T18:08:24.900242+0000","last_clean":"2023-09-23T18:08:24.900242+0000","last_became_active":"2023-09-23T18:06:05.362947+0000","last_became_peered":"2023-09-23T18:06:05.362947+0000","last_unstale":"2023-09-23T18:08:24.900242+0000","last_undegraded":"2023-09-23T18:08:24.900242+0000","last_fullsized":"2023-09-23T18:08:24.900242+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:32.816159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.10","version":"337'5264","reported_seq":7173,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:27.676962+0000","last_change":"2023-09-23T18:07:10.752220+0000","last_active":"2023-09-23T18:08:27.676962+0000","last_peered":"2023-09-23T18:08:27.676962+0000","last_clean":"2023-09-23T18:06:04.331883+0000","last_became_active":"2023-09-23T18:07:10.724852+0000","last_became_peered":"2023-09-23T18:07:10.724852+0000","last_unstale":"2023-09-23T18:08:27.676962+0000","last_undegraded":"2023-09-23T18:07:10.718485+0000","last_fullsized":"2023-09-23T18:08:27.676962+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5264,"log_dups_size":0,"ondisk_log_size":5264,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87687168,"num_objects":1338,"num_object_clones":0,"num_object_copies":4014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1163,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1338,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3152,"num_write_kb":143680,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":907,"num_bytes_recovered":59441152,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"1,2,5","objects":175},{"shards":"1,3,5","objects":1111},{"shards":"2,3,5","objects":52}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":684,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089437+0000","last_change":"2023-09-23T18:06:48.989986+0000","last_active":"2023-09-23T18:08:19.089437+0000","last_peered":"2023-09-23T18:08:19.089437+0000","last_clean":"2023-09-23T18:08:19.089437+0000","last_became_active":"2023-09-23T18:06:48.989534+0000","last_became_peered":"2023-09-23T18:06:48.989534+0000","last_unstale":"2023-09-23T18:08:19.089437+0000","last_undegraded":"2023-09-23T18:08:19.089437+0000","last_fullsized":"2023-09-23T18:08:19.089437+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":129,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089491+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:08:19.089491+0000","last_peered":"2023-09-23T18:08:19.089491+0000","last_clean":"2023-09-23T18:08:19.089491+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:08:19.089491+0000","last_undegraded":"2023-09-23T18:08:19.089491+0000","last_fullsized":"2023-09-23T18:08:19.089491+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":147,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087418+0000","last_change":"2023-09-23T18:06:05.375073+0000","last_active":"2023-09-23T18:08:19.087418+0000","last_peered":"2023-09-23T18:08:19.087418+0000","last_clean":"2023-09-23T18:08:19.087418+0000","last_became_active":"2023-09-23T18:06:05.374752+0000","last_became_peered":"2023-09-23T18:06:05.374752+0000","last_unstale":"2023-09-23T18:08:19.087418+0000","last_undegraded":"2023-09-23T18:08:19.087418+0000","last_fullsized":"2023-09-23T18:08:19.087418+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:09:06.839328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.17","version":"337'5335","reported_seq":8245,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:30.515642+0000","last_change":"2023-09-23T18:07:10.793174+0000","last_active":"2023-09-23T18:08:30.515642+0000","last_peered":"2023-09-23T18:08:30.515642+0000","last_clean":"2023-09-23T18:06:48.947638+0000","last_became_active":"2023-09-23T18:07:10.789937+0000","last_became_peered":"2023-09-23T18:07:10.789937+0000","last_unstale":"2023-09-23T18:08:30.515642+0000","last_undegraded":"2023-09-23T18:07:10.745107+0000","last_fullsized":"2023-09-23T18:08:30.515642+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5335,"log_dups_size":0,"ondisk_log_size":5335,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85688320,"num_objects":1308,"num_object_clones":0,"num_object_copies":3924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2278,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3188,"num_write_kb":143840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":942,"num_bytes_recovered":61702144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":["6","0","3"],"object_location_counts":[{"shards":"0,3,6","objects":1139},{"shards":"1,6,7","objects":169}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":637,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089346+0000","last_change":"2023-09-23T18:07:00.038224+0000","last_active":"2023-09-23T18:08:19.089346+0000","last_peered":"2023-09-23T18:08:19.089346+0000","last_clean":"2023-09-23T18:08:19.089346+0000","last_became_active":"2023-09-23T18:07:00.037771+0000","last_became_peered":"2023-09-23T18:07:00.037771+0000","last_unstale":"2023-09-23T18:08:19.089346+0000","last_undegraded":"2023-09-23T18:08:19.089346+0000","last_fullsized":"2023-09-23T18:08:19.089346+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:01.632666+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:06:01.632666+0000","last_clean_scrub_stamp":"2023-09-23T18:06:01.632666+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":265,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.080904+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:08:19.080904+0000","last_peered":"2023-09-23T18:08:19.080904+0000","last_clean":"2023-09-23T18:08:19.080904+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:08:19.080904+0000","last_undegraded":"2023-09-23T18:08:19.080904+0000","last_fullsized":"2023-09-23T18:08:19.080904+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":152,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089092+0000","last_change":"2023-09-23T18:07:49.229808+0000","last_active":"2023-09-23T18:08:19.089092+0000","last_peered":"2023-09-23T18:08:19.089092+0000","last_clean":"2023-09-23T18:08:19.089092+0000","last_became_active":"2023-09-23T18:07:49.229526+0000","last_became_peered":"2023-09-23T18:07:49.229526+0000","last_unstale":"2023-09-23T18:08:19.089092+0000","last_undegraded":"2023-09-23T18:08:19.089092+0000","last_fullsized":"2023-09-23T18:08:19.089092+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:01.224432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.16","version":"337'5090","reported_seq":7954,"reported_epoch":337,"state":"active+recovering","last_fresh":"2023-09-23T18:08:29.881266+0000","last_change":"2023-09-23T18:08:26.106603+0000","last_active":"2023-09-23T18:08:29.881266+0000","last_peered":"2023-09-23T18:08:29.881266+0000","last_clean":"2023-09-23T18:06:44.864353+0000","last_became_active":"2023-09-23T18:07:10.670934+0000","last_became_peered":"2023-09-23T18:07:10.670934+0000","last_unstale":"2023-09-23T18:08:29.881266+0000","last_undegraded":"2023-09-23T18:08:29.881266+0000","last_fullsized":"2023-09-23T18:08:29.881266+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5090,"log_dups_size":0,"ondisk_log_size":5090,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86638592,"num_objects":1322,"num_object_clones":0,"num_object_copies":3966,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":738,"num_objects_unfound":0,"num_objects_dirty":1322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3024,"num_write_kb":139072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1253,"num_bytes_recovered":82116608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":738},{"shards":"0,5,7","objects":584}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":654,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089462+0000","last_change":"2023-09-23T18:05:15.091218+0000","last_active":"2023-09-23T18:08:19.089462+0000","last_peered":"2023-09-23T18:08:19.089462+0000","last_clean":"2023-09-23T18:08:19.089462+0000","last_became_active":"2023-09-23T18:05:15.090420+0000","last_became_peered":"2023-09-23T18:05:15.090420+0000","last_unstale":"2023-09-23T18:08:19.089462+0000","last_undegraded":"2023-09-23T18:08:19.089462+0000","last_fullsized":"2023-09-23T18:08:19.089462+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:57.754408+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":344,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.080754+0000","last_change":"2023-09-23T18:06:51.065050+0000","last_active":"2023-09-23T18:08:19.080754+0000","last_peered":"2023-09-23T18:08:19.080754+0000","last_clean":"2023-09-23T18:08:19.080754+0000","last_became_active":"2023-09-23T18:06:51.064774+0000","last_became_peered":"2023-09-23T18:06:51.064774+0000","last_unstale":"2023-09-23T18:08:19.080754+0000","last_undegraded":"2023-09-23T18:08:19.080754+0000","last_fullsized":"2023-09-23T18:08:19.080754+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:31.508499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":314,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089949+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:08:19.089949+0000","last_peered":"2023-09-23T18:08:19.089949+0000","last_clean":"2023-09-23T18:08:19.089949+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:08:19.089949+0000","last_undegraded":"2023-09-23T18:08:19.089949+0000","last_fullsized":"2023-09-23T18:08:19.089949+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.15","version":"337'5377","reported_seq":7330,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:27.972359+0000","last_change":"2023-09-23T18:07:10.711949+0000","last_active":"2023-09-23T18:08:27.972359+0000","last_peered":"2023-09-23T18:08:27.972359+0000","last_clean":"2023-09-23T18:06:44.892077+0000","last_became_active":"2023-09-23T18:07:10.707811+0000","last_became_peered":"2023-09-23T18:07:10.707811+0000","last_unstale":"2023-09-23T18:08:27.972359+0000","last_undegraded":"2023-09-23T18:07:10.673033+0000","last_fullsized":"2023-09-23T18:08:27.972359+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5377,"log_dups_size":0,"ondisk_log_size":5377,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88965120,"num_objects":1358,"num_object_clones":0,"num_object_copies":4074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1204,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3228,"num_write_kb":146720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":935,"num_bytes_recovered":61276160,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":["7","0","2"],"object_location_counts":[{"shards":"0,2,7","objects":1204},{"shards":"1,2,7","objects":154}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":647,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.088203+0000","last_change":"2023-09-23T18:06:45.920979+0000","last_active":"2023-09-23T18:08:19.088203+0000","last_peered":"2023-09-23T18:08:19.088203+0000","last_clean":"2023-09-23T18:08:19.088203+0000","last_became_active":"2023-09-23T18:06:45.920686+0000","last_became_peered":"2023-09-23T18:06:45.920686+0000","last_unstale":"2023-09-23T18:08:19.088203+0000","last_undegraded":"2023-09-23T18:08:19.088203+0000","last_fullsized":"2023-09-23T18:08:19.088203+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":127,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087125+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:08:19.087125+0000","last_peered":"2023-09-23T18:08:19.087125+0000","last_clean":"2023-09-23T18:08:19.087125+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:08:19.087125+0000","last_undegraded":"2023-09-23T18:08:19.087125+0000","last_fullsized":"2023-09-23T18:08:19.087125+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":275,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.097101+0000","last_change":"2023-09-23T18:06:51.125883+0000","last_active":"2023-09-23T18:08:19.097101+0000","last_peered":"2023-09-23T18:08:19.097101+0000","last_clean":"2023-09-23T18:08:19.097101+0000","last_became_active":"2023-09-23T18:06:51.119298+0000","last_became_peered":"2023-09-23T18:06:51.119298+0000","last_unstale":"2023-09-23T18:08:19.097101+0000","last_undegraded":"2023-09-23T18:08:19.097101+0000","last_fullsized":"2023-09-23T18:08:19.097101+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:12.571367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"7.14","version":"337'5207","reported_seq":7174,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:30.678700+0000","last_change":"2023-09-23T18:07:10.747071+0000","last_active":"2023-09-23T18:08:30.678700+0000","last_peered":"2023-09-23T18:08:30.678700+0000","last_clean":"2023-09-23T18:06:04.313237+0000","last_became_active":"2023-09-23T18:07:10.742490+0000","last_became_peered":"2023-09-23T18:07:10.742490+0000","last_unstale":"2023-09-23T18:08:30.678700+0000","last_undegraded":"2023-09-23T18:07:10.708526+0000","last_fullsized":"2023-09-23T18:08:30.678700+0000","mapping_epoch":316,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5207,"log_dups_size":0,"ondisk_log_size":5207,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82575360,"num_objects":1260,"num_object_clones":0,"num_object_copies":3780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1092,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3142,"num_write_kb":140864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":942,"num_bytes_recovered":61702144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1092},{"shards":"1,4,6","objects":168}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":674,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089376+0000","last_change":"2023-09-23T18:06:08.430910+0000","last_active":"2023-09-23T18:08:19.089376+0000","last_peered":"2023-09-23T18:08:19.089376+0000","last_clean":"2023-09-23T18:08:19.089376+0000","last_became_active":"2023-09-23T18:06:08.430380+0000","last_became_peered":"2023-09-23T18:06:08.430380+0000","last_unstale":"2023-09-23T18:08:19.089376+0000","last_undegraded":"2023-09-23T18:08:19.089376+0000","last_fullsized":"2023-09-23T18:08:19.089376+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:40.113807+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":313,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.090818+0000","last_change":"2023-09-23T18:06:51.128655+0000","last_active":"2023-09-23T18:08:19.090818+0000","last_peered":"2023-09-23T18:08:19.090818+0000","last_clean":"2023-09-23T18:08:19.090818+0000","last_became_active":"2023-09-23T18:06:51.057904+0000","last_became_peered":"2023-09-23T18:06:51.057904+0000","last_unstale":"2023-09-23T18:08:19.090818+0000","last_undegraded":"2023-09-23T18:08:19.090818+0000","last_fullsized":"2023-09-23T18:08:19.090818+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:22.029660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":190,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.090368+0000","last_change":"2023-09-23T18:06:45.919283+0000","last_active":"2023-09-23T18:08:19.090368+0000","last_peered":"2023-09-23T18:08:19.090368+0000","last_clean":"2023-09-23T18:08:19.090368+0000","last_became_active":"2023-09-23T18:06:45.918219+0000","last_became_peered":"2023-09-23T18:06:45.918219+0000","last_unstale":"2023-09-23T18:08:19.090368+0000","last_undegraded":"2023-09-23T18:08:19.090368+0000","last_fullsized":"2023-09-23T18:08:19.090368+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:21.242033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.b","version":"337'6199","reported_seq":6568,"reported_epoch":337,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:08:27.683448+0000","last_change":"2023-09-23T18:07:10.754157+0000","last_active":"2023-09-23T18:08:27.683448+0000","last_peered":"2023-09-23T18:08:27.683448+0000","last_clean":"2023-09-23T18:06:04.331781+0000","last_became_active":"2023-09-23T18:06:06.621694+0000","last_became_peered":"2023-09-23T18:06:06.621694+0000","last_unstale":"2023-09-23T18:08:27.683448+0000","last_undegraded":"2023-09-23T18:07:10.753028+0000","last_fullsized":"2023-09-23T18:06:06.376266+0000","mapping_epoch":286,"log_start":"283'500","ondisk_log_start":"283'500","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5699,"log_dups_size":0,"ondisk_log_size":5699,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:55:24.704756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":177537024,"num_objects":2709,"num_object_clones":0,"num_object_copies":8127,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1717,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2709,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6199,"num_write_kb":285056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":145,"num_bytes_recovered":9502720,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[5,6],"avail_no_missing":["5","2","6"],"object_location_counts":[{"shards":"2,5,6","objects":116},{"shards":"5,6","objects":992}],"blocked_by":[],"up_primary":1,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":717,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.081114+0000","last_change":"2023-09-23T18:06:08.440949+0000","last_active":"2023-09-23T18:08:19.081114+0000","last_peered":"2023-09-23T18:08:19.081114+0000","last_clean":"2023-09-23T18:08:19.081114+0000","last_became_active":"2023-09-23T18:06:08.440543+0000","last_became_peered":"2023-09-23T18:06:08.440543+0000","last_unstale":"2023-09-23T18:08:19.081114+0000","last_undegraded":"2023-09-23T18:08:19.081114+0000","last_fullsized":"2023-09-23T18:08:19.081114+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:39:30.193370+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":279,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.085449+0000","last_change":"2023-09-23T18:06:45.947590+0000","last_active":"2023-09-23T18:08:19.085449+0000","last_peered":"2023-09-23T18:08:19.085449+0000","last_clean":"2023-09-23T18:08:19.085449+0000","last_became_active":"2023-09-23T18:06:45.920061+0000","last_became_peered":"2023-09-23T18:06:45.920061+0000","last_unstale":"2023-09-23T18:08:19.085449+0000","last_undegraded":"2023-09-23T18:08:19.085449+0000","last_fullsized":"2023-09-23T18:08:19.085449+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:31.645928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":325,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089377+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:08:19.089377+0000","last_peered":"2023-09-23T18:08:19.089377+0000","last_clean":"2023-09-23T18:08:19.089377+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:08:19.089377+0000","last_undegraded":"2023-09-23T18:08:19.089377+0000","last_fullsized":"2023-09-23T18:08:19.089377+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"7.a","version":"337'6288","reported_seq":6371,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:27.660846+0000","last_change":"2023-09-23T18:08:10.522358+0000","last_active":"2023-09-23T18:08:27.660846+0000","last_peered":"2023-09-23T18:08:27.660846+0000","last_clean":"2023-09-23T18:08:27.660846+0000","last_became_active":"2023-09-23T18:08:10.505420+0000","last_became_peered":"2023-09-23T18:08:10.505420+0000","last_unstale":"2023-09-23T18:08:27.660846+0000","last_undegraded":"2023-09-23T18:08:27.660846+0000","last_fullsized":"2023-09-23T18:08:27.660846+0000","mapping_epoch":331,"log_start":"283'619","ondisk_log_start":"283'619","created":279,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5669,"log_dups_size":0,"ondisk_log_size":5669,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:01.639710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":178913280,"num_objects":2730,"num_object_clones":0,"num_object_copies":8190,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2730,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6288,"num_write_kb":288576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":671,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.135898+0000","last_change":"2023-09-23T18:08:15.579325+0000","last_active":"2023-09-23T18:08:19.135898+0000","last_peered":"2023-09-23T18:08:19.135898+0000","last_clean":"2023-09-23T18:08:19.135898+0000","last_became_active":"2023-09-23T18:08:15.578866+0000","last_became_peered":"2023-09-23T18:08:15.578866+0000","last_unstale":"2023-09-23T18:08:19.135898+0000","last_undegraded":"2023-09-23T18:08:19.135898+0000","last_fullsized":"2023-09-23T18:08:19.135898+0000","mapping_epoch":334,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":335,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1392168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":294,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.081593+0000","last_change":"2023-09-23T18:06:05.379591+0000","last_active":"2023-09-23T18:08:19.081593+0000","last_peered":"2023-09-23T18:08:19.081593+0000","last_clean":"2023-09-23T18:08:19.081593+0000","last_became_active":"2023-09-23T18:06:05.370176+0000","last_became_peered":"2023-09-23T18:06:05.370176+0000","last_unstale":"2023-09-23T18:08:19.081593+0000","last_undegraded":"2023-09-23T18:08:19.081593+0000","last_fullsized":"2023-09-23T18:08:19.081593+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:08.768138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":367,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087320+0000","last_change":"2023-09-23T18:06:45.911012+0000","last_active":"2023-09-23T18:08:19.087320+0000","last_peered":"2023-09-23T18:08:19.087320+0000","last_clean":"2023-09-23T18:08:19.087320+0000","last_became_active":"2023-09-23T18:06:45.910722+0000","last_became_peered":"2023-09-23T18:06:45.910722+0000","last_unstale":"2023-09-23T18:08:19.087320+0000","last_undegraded":"2023-09-23T18:08:19.087320+0000","last_fullsized":"2023-09-23T18:08:19.087320+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:19.150719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.9","version":"337'5150","reported_seq":6994,"reported_epoch":337,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:08:27.571174+0000","last_change":"2023-09-23T18:07:12.877579+0000","last_active":"2023-09-23T18:08:27.571174+0000","last_peered":"2023-09-23T18:08:27.571174+0000","last_clean":"2023-09-23T18:06:04.334743+0000","last_became_active":"2023-09-23T18:07:12.854124+0000","last_became_peered":"2023-09-23T18:07:12.854124+0000","last_unstale":"2023-09-23T18:08:27.571174+0000","last_undegraded":"2023-09-23T18:07:12.196594+0000","last_fullsized":"2023-09-23T18:07:10.610402+0000","mapping_epoch":299,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5150,"log_dups_size":0,"ondisk_log_size":5150,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86638592,"num_objects":1322,"num_object_clones":0,"num_object_copies":3966,"num_objects_missing_on_primary":58,"num_objects_missing":58,"num_objects_degraded":1168,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1322,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3084,"num_write_kb":140992,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":881,"num_bytes_recovered":57737216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1110},{"shards":"0,1,4,7","objects":1},{"shards":"0,4,7","objects":58},{"shards":"1,4","objects":153}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":508,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.094371+0000","last_change":"2023-09-23T18:06:02.805384+0000","last_active":"2023-09-23T18:08:19.094371+0000","last_peered":"2023-09-23T18:08:19.094371+0000","last_clean":"2023-09-23T18:08:19.094371+0000","last_became_active":"2023-09-23T18:05:15.098234+0000","last_became_peered":"2023-09-23T18:05:15.098234+0000","last_unstale":"2023-09-23T18:08:19.094371+0000","last_undegraded":"2023-09-23T18:08:19.094371+0000","last_fullsized":"2023-09-23T18:08:19.094371+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:58.872644+0000","scrub_duration":0.0018893740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":203,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.081513+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:08:19.081513+0000","last_peered":"2023-09-23T18:08:19.081513+0000","last_clean":"2023-09-23T18:08:19.081513+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:08:19.081513+0000","last_undegraded":"2023-09-23T18:08:19.081513+0000","last_fullsized":"2023-09-23T18:08:19.081513+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":312,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.091466+0000","last_change":"2023-09-23T18:06:05.371447+0000","last_active":"2023-09-23T18:08:19.091466+0000","last_peered":"2023-09-23T18:08:19.091466+0000","last_clean":"2023-09-23T18:08:19.091466+0000","last_became_active":"2023-09-23T18:06:05.371065+0000","last_became_peered":"2023-09-23T18:06:05.371065+0000","last_unstale":"2023-09-23T18:08:19.091466+0000","last_undegraded":"2023-09-23T18:08:19.091466+0000","last_fullsized":"2023-09-23T18:08:19.091466+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.025791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.8","version":"337'5188","reported_seq":5275,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:27.951095+0000","last_change":"2023-09-23T18:08:10.521036+0000","last_active":"2023-09-23T18:08:27.951095+0000","last_peered":"2023-09-23T18:08:27.951095+0000","last_clean":"2023-09-23T18:08:27.951095+0000","last_became_active":"2023-09-23T18:08:10.509931+0000","last_became_peered":"2023-09-23T18:08:10.509931+0000","last_unstale":"2023-09-23T18:08:27.951095+0000","last_undegraded":"2023-09-23T18:08:27.951095+0000","last_fullsized":"2023-09-23T18:08:27.951095+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5188,"log_dups_size":0,"ondisk_log_size":5188,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":88113152,"num_objects":1344,"num_object_clones":0,"num_object_copies":4032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1344,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3096,"num_write_kb":142112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":661,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087050+0000","last_change":"2023-09-23T18:05:56.362746+0000","last_active":"2023-09-23T18:08:19.087050+0000","last_peered":"2023-09-23T18:08:19.087050+0000","last_clean":"2023-09-23T18:08:19.087050+0000","last_became_active":"2023-09-23T18:05:15.102072+0000","last_became_peered":"2023-09-23T18:05:15.102072+0000","last_unstale":"2023-09-23T18:08:19.087050+0000","last_undegraded":"2023-09-23T18:08:19.087050+0000","last_fullsized":"2023-09-23T18:08:19.087050+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.362652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.362652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:33.381515+0000","scrub_duration":0.0052581759999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":323,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089405+0000","last_change":"2023-09-23T18:06:51.069867+0000","last_active":"2023-09-23T18:08:19.089405+0000","last_peered":"2023-09-23T18:08:19.089405+0000","last_clean":"2023-09-23T18:08:19.089405+0000","last_became_active":"2023-09-23T18:06:51.069523+0000","last_became_peered":"2023-09-23T18:06:51.069523+0000","last_unstale":"2023-09-23T18:08:19.089405+0000","last_undegraded":"2023-09-23T18:08:19.089405+0000","last_fullsized":"2023-09-23T18:08:19.089405+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:40.321844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":349,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.135398+0000","last_change":"2023-09-23T18:06:05.375880+0000","last_active":"2023-09-23T18:08:19.135398+0000","last_peered":"2023-09-23T18:08:19.135398+0000","last_clean":"2023-09-23T18:08:19.135398+0000","last_became_active":"2023-09-23T18:06:05.375150+0000","last_became_peered":"2023-09-23T18:06:05.375150+0000","last_unstale":"2023-09-23T18:08:19.135398+0000","last_undegraded":"2023-09-23T18:08:19.135398+0000","last_fullsized":"2023-09-23T18:08:19.135398+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:36.483694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.f","version":"337'6165","reported_seq":18224,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:30.774643+0000","last_change":"2023-09-23T18:08:15.604271+0000","last_active":"2023-09-23T18:08:30.774643+0000","last_peered":"2023-09-23T18:08:30.774643+0000","last_clean":"2023-09-23T18:08:30.774643+0000","last_became_active":"2023-09-23T18:08:15.581890+0000","last_became_peered":"2023-09-23T18:08:15.581890+0000","last_unstale":"2023-09-23T18:08:30.774643+0000","last_undegraded":"2023-09-23T18:08:30.774643+0000","last_fullsized":"2023-09-23T18:08:30.774643+0000","mapping_epoch":334,"log_start":"284'1476","ondisk_log_start":"284'1476","created":279,"last_epoch_clean":335,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":4689,"log_dups_size":0,"ondisk_log_size":4689,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:14.498809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":165740544,"num_objects":2529,"num_object_clones":0,"num_object_copies":7587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6165,"num_write_kb":278208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3991,"num_bytes_recovered":261554176,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":709,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.080630+0000","last_change":"2023-09-23T18:06:54.059737+0000","last_active":"2023-09-23T18:08:19.080630+0000","last_peered":"2023-09-23T18:08:19.080630+0000","last_clean":"2023-09-23T18:08:19.080630+0000","last_became_active":"2023-09-23T18:06:54.059375+0000","last_became_peered":"2023-09-23T18:06:54.059375+0000","last_unstale":"2023-09-23T18:08:19.080630+0000","last_undegraded":"2023-09-23T18:08:19.080630+0000","last_fullsized":"2023-09-23T18:08:19.080630+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:12.644423+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":123,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087577+0000","last_change":"2023-09-23T18:07:49.227201+0000","last_active":"2023-09-23T18:08:19.087577+0000","last_peered":"2023-09-23T18:08:19.087577+0000","last_clean":"2023-09-23T18:08:19.087577+0000","last_became_active":"2023-09-23T18:07:49.226878+0000","last_became_peered":"2023-09-23T18:07:49.226878+0000","last_unstale":"2023-09-23T18:08:19.087577+0000","last_undegraded":"2023-09-23T18:08:19.087577+0000","last_fullsized":"2023-09-23T18:08:19.087577+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:08.994668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":337,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089404+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:08:19.089404+0000","last_peered":"2023-09-23T18:08:19.089404+0000","last_clean":"2023-09-23T18:08:19.089404+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:08:19.089404+0000","last_undegraded":"2023-09-23T18:08:19.089404+0000","last_fullsized":"2023-09-23T18:08:19.089404+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.e","version":"337'6043","reported_seq":7164,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:27.677998+0000","last_change":"2023-09-23T18:08:11.585423+0000","last_active":"2023-09-23T18:08:27.677998+0000","last_peered":"2023-09-23T18:08:27.677998+0000","last_clean":"2023-09-23T18:06:44.878443+0000","last_became_active":"2023-09-23T18:08:11.532496+0000","last_became_peered":"2023-09-23T18:08:11.532496+0000","last_unstale":"2023-09-23T18:08:27.677998+0000","last_undegraded":"2023-09-23T18:08:11.519228+0000","last_fullsized":"2023-09-23T18:08:27.677998+0000","mapping_epoch":332,"log_start":"283'300","ondisk_log_start":"283'300","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5743,"log_dups_size":0,"ondisk_log_size":5743,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:02:07.530865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174522368,"num_objects":2663,"num_object_clones":0,"num_object_copies":7989,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3534,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2663,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6043,"num_write_kb":278592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":518,"num_bytes_recovered":33947648,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":["5","0","2"],"object_location_counts":[{"shards":"0,2,5","objects":2360},{"shards":"1,2,5","objects":301},{"shards":"2,3,5","objects":2}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":677,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089544+0000","last_change":"2023-09-23T18:06:54.065597+0000","last_active":"2023-09-23T18:08:19.089544+0000","last_peered":"2023-09-23T18:08:19.089544+0000","last_clean":"2023-09-23T18:08:19.089544+0000","last_became_active":"2023-09-23T18:06:54.065217+0000","last_became_peered":"2023-09-23T18:06:54.065217+0000","last_unstale":"2023-09-23T18:08:19.089544+0000","last_undegraded":"2023-09-23T18:08:19.089544+0000","last_fullsized":"2023-09-23T18:08:19.089544+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:53.614664+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":306,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.091728+0000","last_change":"2023-09-23T18:06:05.371350+0000","last_active":"2023-09-23T18:08:19.091728+0000","last_peered":"2023-09-23T18:08:19.091728+0000","last_clean":"2023-09-23T18:08:19.091728+0000","last_became_active":"2023-09-23T18:06:05.370877+0000","last_became_peered":"2023-09-23T18:06:05.370877+0000","last_unstale":"2023-09-23T18:08:19.091728+0000","last_undegraded":"2023-09-23T18:08:19.091728+0000","last_fullsized":"2023-09-23T18:08:19.091728+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:58.534355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":147,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.086700+0000","last_change":"2023-09-23T18:06:05.373385+0000","last_active":"2023-09-23T18:08:19.086700+0000","last_peered":"2023-09-23T18:08:19.086700+0000","last_clean":"2023-09-23T18:08:19.086700+0000","last_became_active":"2023-09-23T18:06:05.372982+0000","last_became_peered":"2023-09-23T18:06:05.372982+0000","last_unstale":"2023-09-23T18:08:19.086700+0000","last_undegraded":"2023-09-23T18:08:19.086700+0000","last_fullsized":"2023-09-23T18:08:19.086700+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:48.622115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.d","version":"337'6366","reported_seq":10246,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:30.528068+0000","last_change":"2023-09-23T18:06:59.042700+0000","last_active":"2023-09-23T18:08:30.528068+0000","last_peered":"2023-09-23T18:08:30.528068+0000","last_clean":"2023-09-23T18:08:30.528068+0000","last_became_active":"2023-09-23T18:06:59.038205+0000","last_became_peered":"2023-09-23T18:06:59.038205+0000","last_unstale":"2023-09-23T18:08:30.528068+0000","last_undegraded":"2023-09-23T18:08:30.528068+0000","last_fullsized":"2023-09-23T18:08:30.528068+0000","mapping_epoch":311,"log_start":"283'1200","ondisk_log_start":"283'1200","created":279,"last_epoch_clean":312,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5166,"log_dups_size":0,"ondisk_log_size":5166,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:40.800800+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168886296,"num_objects":2578,"num_object_clones":0,"num_object_copies":7734,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2578,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6366,"num_write_kb":286145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1894,"num_bytes_recovered":124125184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":680,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:22.572861+0000","last_change":"2023-09-23T18:06:08.435018+0000","last_active":"2023-09-23T18:08:22.572861+0000","last_peered":"2023-09-23T18:08:22.572861+0000","last_clean":"2023-09-23T18:08:22.572861+0000","last_became_active":"2023-09-23T18:06:08.434162+0000","last_became_peered":"2023-09-23T18:06:08.434162+0000","last_unstale":"2023-09-23T18:08:22.572861+0000","last_undegraded":"2023-09-23T18:08:22.572861+0000","last_fullsized":"2023-09-23T18:08:22.572861+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:41.703500+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":127,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087144+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:08:19.087144+0000","last_peered":"2023-09-23T18:08:19.087144+0000","last_clean":"2023-09-23T18:08:19.087144+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:08:19.087144+0000","last_undegraded":"2023-09-23T18:08:19.087144+0000","last_fullsized":"2023-09-23T18:08:19.087144+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":357,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087174+0000","last_change":"2023-09-23T18:06:05.373305+0000","last_active":"2023-09-23T18:08:19.087174+0000","last_peered":"2023-09-23T18:08:19.087174+0000","last_clean":"2023-09-23T18:08:19.087174+0000","last_became_active":"2023-09-23T18:06:05.372720+0000","last_became_peered":"2023-09-23T18:06:05.372720+0000","last_unstale":"2023-09-23T18:08:19.087174+0000","last_undegraded":"2023-09-23T18:08:19.087174+0000","last_fullsized":"2023-09-23T18:08:19.087174+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:44.934493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.c","version":"337'6174","reported_seq":10355,"reported_epoch":337,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:08:30.578264+0000","last_change":"2023-09-23T18:07:11.687041+0000","last_active":"2023-09-23T18:08:30.578264+0000","last_peered":"2023-09-23T18:08:30.578264+0000","last_clean":"2023-09-23T18:06:44.837296+0000","last_became_active":"2023-09-23T18:06:47.115056+0000","last_became_peered":"2023-09-23T18:06:47.115056+0000","last_unstale":"2023-09-23T18:08:30.578264+0000","last_undegraded":"2023-09-23T18:07:11.685252+0000","last_fullsized":"2023-09-23T18:06:46.925665+0000","mapping_epoch":299,"log_start":"283'1000","ondisk_log_start":"283'1000","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5174,"log_dups_size":0,"ondisk_log_size":5174,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:08:46.974573+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":168558592,"num_objects":2572,"num_object_clones":0,"num_object_copies":7716,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1984,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2572,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6174,"num_write_kb":279872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2051,"num_bytes_recovered":134414336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7],"avail_no_missing":["6","3","7"],"object_location_counts":[{"shards":"3,6,7","objects":1984},{"shards":"6,7","objects":588}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":653,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089686+0000","last_change":"2023-09-23T18:05:53.509955+0000","last_active":"2023-09-23T18:08:19.089686+0000","last_peered":"2023-09-23T18:08:19.089686+0000","last_clean":"2023-09-23T18:08:19.089686+0000","last_became_active":"2023-09-23T18:05:15.089224+0000","last_became_peered":"2023-09-23T18:05:15.089224+0000","last_unstale":"2023-09-23T18:08:19.089686+0000","last_undegraded":"2023-09-23T18:08:19.089686+0000","last_fullsized":"2023-09-23T18:08:19.089686+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:37.917870+0000","scrub_duration":0.001603043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":182,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087809+0000","last_change":"2023-09-23T18:06:45.904403+0000","last_active":"2023-09-23T18:08:19.087809+0000","last_peered":"2023-09-23T18:08:19.087809+0000","last_clean":"2023-09-23T18:08:19.087809+0000","last_became_active":"2023-09-23T18:06:45.904057+0000","last_became_peered":"2023-09-23T18:06:45.904057+0000","last_unstale":"2023-09-23T18:08:19.087809+0000","last_undegraded":"2023-09-23T18:08:19.087809+0000","last_fullsized":"2023-09-23T18:08:19.087809+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:43.213905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":63,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:22.573036+0000","last_change":"2023-09-23T18:06:05.371661+0000","last_active":"2023-09-23T18:08:22.573036+0000","last_peered":"2023-09-23T18:08:22.573036+0000","last_clean":"2023-09-23T18:08:22.573036+0000","last_became_active":"2023-09-23T18:06:05.370998+0000","last_became_peered":"2023-09-23T18:06:05.370998+0000","last_unstale":"2023-09-23T18:08:22.573036+0000","last_undegraded":"2023-09-23T18:08:22.573036+0000","last_fullsized":"2023-09-23T18:08:22.573036+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:52.262543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.1","version":"337'5174","reported_seq":7196,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:30.649667+0000","last_change":"2023-09-23T18:07:12.880468+0000","last_active":"2023-09-23T18:08:30.649667+0000","last_peered":"2023-09-23T18:08:30.649667+0000","last_clean":"2023-09-23T18:06:25.121665+0000","last_became_active":"2023-09-23T18:07:12.877916+0000","last_became_peered":"2023-09-23T18:07:12.877916+0000","last_unstale":"2023-09-23T18:08:30.649667+0000","last_undegraded":"2023-09-23T18:07:12.235381+0000","last_fullsized":"2023-09-23T18:08:30.649667+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5174,"log_dups_size":0,"ondisk_log_size":5174,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85819392,"num_objects":1309,"num_object_clones":0,"num_object_copies":3927,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":630,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1309,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3093,"num_write_kb":140896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":897,"num_bytes_recovered":58818560,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":["4","3","6"],"object_location_counts":[{"shards":"1,4,6","objects":679},{"shards":"3,4,6","objects":630}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":528,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.094180+0000","last_change":"2023-09-23T18:06:54.193400+0000","last_active":"2023-09-23T18:08:19.094180+0000","last_peered":"2023-09-23T18:08:19.094180+0000","last_clean":"2023-09-23T18:08:19.094180+0000","last_became_active":"2023-09-23T18:06:54.193019+0000","last_became_peered":"2023-09-23T18:06:54.193019+0000","last_unstale":"2023-09-23T18:08:19.094180+0000","last_undegraded":"2023-09-23T18:08:19.094180+0000","last_fullsized":"2023-09-23T18:08:19.094180+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:48.156749+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":648,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089408+0000","last_change":"2023-09-23T18:06:29.051219+0000","last_active":"2023-09-23T18:08:19.089408+0000","last_peered":"2023-09-23T18:08:19.089408+0000","last_clean":"2023-09-23T18:08:19.089408+0000","last_became_active":"2023-09-23T18:06:29.050570+0000","last_became_peered":"2023-09-23T18:06:29.050570+0000","last_unstale":"2023-09-23T18:08:19.089408+0000","last_undegraded":"2023-09-23T18:08:19.089408+0000","last_fullsized":"2023-09-23T18:08:19.089408+0000","mapping_epoch":295,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":296,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:14:17.235584+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":327,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.090066+0000","last_change":"2023-09-23T18:06:51.074796+0000","last_active":"2023-09-23T18:08:19.090066+0000","last_peered":"2023-09-23T18:08:19.090066+0000","last_clean":"2023-09-23T18:08:19.090066+0000","last_became_active":"2023-09-23T18:06:51.074525+0000","last_became_peered":"2023-09-23T18:06:51.074525+0000","last_unstale":"2023-09-23T18:08:19.090066+0000","last_undegraded":"2023-09-23T18:08:19.090066+0000","last_fullsized":"2023-09-23T18:08:19.090066+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:48.941608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":63,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.088350+0000","last_change":"2023-09-23T18:06:05.374173+0000","last_active":"2023-09-23T18:08:19.088350+0000","last_peered":"2023-09-23T18:08:19.088350+0000","last_clean":"2023-09-23T18:08:19.088350+0000","last_became_active":"2023-09-23T18:06:05.373835+0000","last_became_peered":"2023-09-23T18:06:05.373835+0000","last_unstale":"2023-09-23T18:08:19.088350+0000","last_undegraded":"2023-09-23T18:08:19.088350+0000","last_fullsized":"2023-09-23T18:08:19.088350+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:25.467488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.7","version":"337'5365","reported_seq":11876,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:30.572375+0000","last_change":"2023-09-23T18:08:26.099444+0000","last_active":"2023-09-23T18:08:30.572375+0000","last_peered":"2023-09-23T18:08:30.572375+0000","last_clean":"2023-09-23T18:08:30.572375+0000","last_became_active":"2023-09-23T18:07:11.672631+0000","last_became_peered":"2023-09-23T18:07:11.672631+0000","last_unstale":"2023-09-23T18:08:30.572375+0000","last_undegraded":"2023-09-23T18:08:30.572375+0000","last_fullsized":"2023-09-23T18:08:30.572375+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":337,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5365,"log_dups_size":0,"ondisk_log_size":5365,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86867968,"num_objects":1325,"num_object_clones":0,"num_object_copies":3975,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1325,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3217,"num_write_kb":145376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2139,"num_bytes_recovered":140214272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":654,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089783+0000","last_change":"2023-09-23T18:05:15.083813+0000","last_active":"2023-09-23T18:08:19.089783+0000","last_peered":"2023-09-23T18:08:19.089783+0000","last_clean":"2023-09-23T18:08:19.089783+0000","last_became_active":"2023-09-23T18:05:15.083395+0000","last_became_peered":"2023-09-23T18:05:15.083395+0000","last_unstale":"2023-09-23T18:08:19.089783+0000","last_undegraded":"2023-09-23T18:08:19.089783+0000","last_fullsized":"2023-09-23T18:08:19.089783+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:13:24.402252+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1309,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089533+0000","last_change":"2023-09-23T18:06:08.426828+0000","last_active":"2023-09-23T18:08:19.089533+0000","last_peered":"2023-09-23T18:08:19.089533+0000","last_clean":"2023-09-23T18:08:19.089533+0000","last_became_active":"2023-09-23T18:06:08.426343+0000","last_became_peered":"2023-09-23T18:06:08.426343+0000","last_unstale":"2023-09-23T18:08:19.089533+0000","last_undegraded":"2023-09-23T18:08:19.089533+0000","last_fullsized":"2023-09-23T18:08:19.089533+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:38.924507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":127,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087538+0000","last_change":"2023-09-23T18:01:39.453113+0000","last_active":"2023-09-23T18:08:19.087538+0000","last_peered":"2023-09-23T18:08:19.087538+0000","last_clean":"2023-09-23T18:08:19.087538+0000","last_became_active":"2023-09-23T18:01:39.452757+0000","last_became_peered":"2023-09-23T18:01:39.452757+0000","last_unstale":"2023-09-23T18:08:19.087538+0000","last_undegraded":"2023-09-23T18:08:19.087538+0000","last_fullsized":"2023-09-23T18:08:19.087538+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:12.045422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":72,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089337+0000","last_change":"2023-09-23T18:06:51.088473+0000","last_active":"2023-09-23T18:08:19.089337+0000","last_peered":"2023-09-23T18:08:19.089337+0000","last_clean":"2023-09-23T18:08:19.089337+0000","last_became_active":"2023-09-23T18:06:51.088171+0000","last_became_peered":"2023-09-23T18:06:51.088171+0000","last_unstale":"2023-09-23T18:08:19.089337+0000","last_undegraded":"2023-09-23T18:08:19.089337+0000","last_fullsized":"2023-09-23T18:08:19.089337+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:45.860551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.4","version":"337'5160","reported_seq":7065,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:30.737432+0000","last_change":"2023-09-23T18:07:11.696361+0000","last_active":"2023-09-23T18:08:30.737432+0000","last_peered":"2023-09-23T18:08:30.737432+0000","last_clean":"2023-09-23T18:06:04.313237+0000","last_became_active":"2023-09-23T18:07:11.692837+0000","last_became_peered":"2023-09-23T18:07:11.692837+0000","last_unstale":"2023-09-23T18:08:30.737432+0000","last_undegraded":"2023-09-23T18:07:11.667257+0000","last_fullsized":"2023-09-23T18:08:30.737432+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5160,"log_dups_size":0,"ondisk_log_size":5160,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83689472,"num_objects":1277,"num_object_clones":0,"num_object_copies":3831,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1091,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1277,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3095,"num_write_kb":139904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":909,"num_bytes_recovered":59604992,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":["4","0","1"],"object_location_counts":[{"shards":"0,1,4","objects":1091},{"shards":"1,4,6","objects":186}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":663,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.135421+0000","last_change":"2023-09-23T18:05:15.087882+0000","last_active":"2023-09-23T18:08:19.135421+0000","last_peered":"2023-09-23T18:08:19.135421+0000","last_clean":"2023-09-23T18:08:19.135421+0000","last_became_active":"2023-09-23T18:05:15.087524+0000","last_became_peered":"2023-09-23T18:05:15.087524+0000","last_unstale":"2023-09-23T18:08:19.135421+0000","last_undegraded":"2023-09-23T18:08:19.135421+0000","last_fullsized":"2023-09-23T18:08:19.135421+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:55.015941+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":327,"reported_epoch":336,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:19.136548+0000","last_change":"2023-09-23T18:08:10.503358+0000","last_active":"2023-09-23T18:08:19.136548+0000","last_peered":"2023-09-23T18:08:19.136548+0000","last_clean":"2023-09-23T17:47:00.620059+0000","last_became_active":"2023-09-23T18:08:10.502784+0000","last_became_peered":"2023-09-23T18:08:10.502784+0000","last_unstale":"2023-09-23T18:08:19.136548+0000","last_undegraded":"2023-09-23T18:08:10.483617+0000","last_fullsized":"2023-09-23T18:08:19.136548+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":238,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:19.871991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":["0","2","3"],"object_location_counts":[{"shards":"0,2,3","objects":1}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":63,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.088042+0000","last_change":"2023-09-23T18:06:05.358500+0000","last_active":"2023-09-23T18:08:19.088042+0000","last_peered":"2023-09-23T18:08:19.088042+0000","last_clean":"2023-09-23T18:08:19.088042+0000","last_became_active":"2023-09-23T18:06:05.358274+0000","last_became_peered":"2023-09-23T18:06:05.358274+0000","last_unstale":"2023-09-23T18:08:19.088042+0000","last_undegraded":"2023-09-23T18:08:19.088042+0000","last_fullsized":"2023-09-23T18:08:19.088042+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:04.055437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":72,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.090396+0000","last_change":"2023-09-23T18:06:45.919995+0000","last_active":"2023-09-23T18:08:19.090396+0000","last_peered":"2023-09-23T18:08:19.090396+0000","last_clean":"2023-09-23T18:08:19.090396+0000","last_became_active":"2023-09-23T18:06:45.919686+0000","last_became_peered":"2023-09-23T18:06:45.919686+0000","last_unstale":"2023-09-23T18:08:19.090396+0000","last_undegraded":"2023-09-23T18:08:19.090396+0000","last_fullsized":"2023-09-23T18:08:19.090396+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:39.113717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.6","version":"337'5185","reported_seq":7402,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:29.765097+0000","last_change":"2023-09-23T18:07:11.654839+0000","last_active":"2023-09-23T18:08:29.765097+0000","last_peered":"2023-09-23T18:08:29.765097+0000","last_clean":"2023-09-23T18:06:44.864353+0000","last_became_active":"2023-09-23T18:07:11.651208+0000","last_became_peered":"2023-09-23T18:07:11.651208+0000","last_unstale":"2023-09-23T18:08:29.765097+0000","last_undegraded":"2023-09-23T18:07:11.622540+0000","last_fullsized":"2023-09-23T18:08:29.765097+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5185,"log_dups_size":0,"ondisk_log_size":5185,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82903040,"num_objects":1265,"num_object_clones":0,"num_object_copies":3795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1126,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3119,"num_write_kb":140288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":929,"num_bytes_recovered":60882944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,5],"acting":[0,7,5],"avail_no_missing":["0","3","7"],"object_location_counts":[{"shards":"0,3,7","objects":1126},{"shards":"0,5,7","objects":139}],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1249,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.090326+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:08:19.090326+0000","last_peered":"2023-09-23T18:08:19.090326+0000","last_clean":"2023-09-23T18:08:19.090326+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:08:19.090326+0000","last_undegraded":"2023-09-23T18:08:19.090326+0000","last_fullsized":"2023-09-23T18:08:19.090326+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":692,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.080237+0000","last_change":"2023-09-23T18:06:48.957422+0000","last_active":"2023-09-23T18:08:19.080237+0000","last_peered":"2023-09-23T18:08:19.080237+0000","last_clean":"2023-09-23T18:08:19.080237+0000","last_became_active":"2023-09-23T18:06:48.957044+0000","last_became_peered":"2023-09-23T18:06:48.957044+0000","last_unstale":"2023-09-23T18:08:19.080237+0000","last_undegraded":"2023-09-23T18:08:19.080237+0000","last_fullsized":"2023-09-23T18:08:19.080237+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:58.047861+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":169,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089461+0000","last_change":"2023-09-23T18:06:51.057540+0000","last_active":"2023-09-23T18:08:19.089461+0000","last_peered":"2023-09-23T18:08:19.089461+0000","last_clean":"2023-09-23T18:08:19.089461+0000","last_became_active":"2023-09-23T18:06:51.056891+0000","last_became_peered":"2023-09-23T18:06:51.056891+0000","last_unstale":"2023-09-23T18:08:19.089461+0000","last_undegraded":"2023-09-23T18:08:19.089461+0000","last_fullsized":"2023-09-23T18:08:19.089461+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:32.959376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":270,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.092485+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:08:19.092485+0000","last_peered":"2023-09-23T18:08:19.092485+0000","last_clean":"2023-09-23T18:08:19.092485+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:08:19.092485+0000","last_undegraded":"2023-09-23T18:08:19.092485+0000","last_fullsized":"2023-09-23T18:08:19.092485+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"7.5","version":"337'5318","reported_seq":7214,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:27.992918+0000","last_change":"2023-09-23T18:07:11.666722+0000","last_active":"2023-09-23T18:08:27.992918+0000","last_peered":"2023-09-23T18:08:27.992918+0000","last_clean":"2023-09-23T18:06:44.892077+0000","last_became_active":"2023-09-23T18:07:11.642881+0000","last_became_peered":"2023-09-23T18:07:11.642881+0000","last_unstale":"2023-09-23T18:08:27.992918+0000","last_undegraded":"2023-09-23T18:07:11.636195+0000","last_fullsized":"2023-09-23T18:08:27.992918+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5318,"log_dups_size":0,"ondisk_log_size":5318,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":89030656,"num_objects":1358,"num_object_clones":0,"num_object_copies":4074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1166,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3168,"num_write_kb":144864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":905,"num_bytes_recovered":59310080,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":["7","0","2"],"object_location_counts":[{"shards":"0,2,7","objects":1166},{"shards":"1,2,7","objects":192}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":486,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.088141+0000","last_change":"2023-09-23T18:06:08.434791+0000","last_active":"2023-09-23T18:08:19.088141+0000","last_peered":"2023-09-23T18:08:19.088141+0000","last_clean":"2023-09-23T18:08:19.088141+0000","last_became_active":"2023-09-23T18:06:08.433718+0000","last_became_peered":"2023-09-23T18:06:08.433718+0000","last_unstale":"2023-09-23T18:08:19.088141+0000","last_undegraded":"2023-09-23T18:08:19.088141+0000","last_fullsized":"2023-09-23T18:08:19.088141+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:57.701252+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:57.701252+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":14,"num_read_kb":27,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":544,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.080476+0000","last_change":"2023-09-23T18:06:05.403119+0000","last_active":"2023-09-23T18:08:19.080476+0000","last_peered":"2023-09-23T18:08:19.080476+0000","last_clean":"2023-09-23T18:08:19.080476+0000","last_became_active":"2023-09-23T18:06:05.371941+0000","last_became_peered":"2023-09-23T18:06:05.371941+0000","last_unstale":"2023-09-23T18:08:19.080476+0000","last_undegraded":"2023-09-23T18:08:19.080476+0000","last_fullsized":"2023-09-23T18:08:19.080476+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:34:25.525564+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":346,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.136494+0000","last_change":"2023-09-23T18:06:45.914529+0000","last_active":"2023-09-23T18:08:19.136494+0000","last_peered":"2023-09-23T18:08:19.136494+0000","last_clean":"2023-09-23T18:08:19.136494+0000","last_became_active":"2023-09-23T18:06:45.914170+0000","last_became_peered":"2023-09-23T18:06:45.914170+0000","last_unstale":"2023-09-23T18:08:19.136494+0000","last_undegraded":"2023-09-23T18:08:19.136494+0000","last_fullsized":"2023-09-23T18:08:19.136494+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:49.492903+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,5],"acting":[0,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":314,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089142+0000","last_change":"2023-09-23T18:06:51.070104+0000","last_active":"2023-09-23T18:08:19.089142+0000","last_peered":"2023-09-23T18:08:19.089142+0000","last_clean":"2023-09-23T18:08:19.089142+0000","last_became_active":"2023-09-23T18:06:51.069771+0000","last_became_peered":"2023-09-23T18:06:51.069771+0000","last_unstale":"2023-09-23T18:08:19.089142+0000","last_undegraded":"2023-09-23T18:08:19.089142+0000","last_fullsized":"2023-09-23T18:08:19.089142+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:49.300491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.0","version":"337'5246","reported_seq":7102,"reported_epoch":337,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:27.702692+0000","last_change":"2023-09-23T18:07:11.704447+0000","last_active":"2023-09-23T18:08:27.702692+0000","last_peered":"2023-09-23T18:08:27.702692+0000","last_clean":"2023-09-23T18:06:04.331883+0000","last_became_active":"2023-09-23T18:07:11.677873+0000","last_became_peered":"2023-09-23T18:07:11.677873+0000","last_unstale":"2023-09-23T18:08:27.702692+0000","last_undegraded":"2023-09-23T18:07:11.672793+0000","last_fullsized":"2023-09-23T18:08:27.702692+0000","mapping_epoch":317,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5246,"log_dups_size":0,"ondisk_log_size":5246,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90177536,"num_objects":1376,"num_object_clones":0,"num_object_copies":4128,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1174,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1376,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3134,"num_write_kb":144320,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":879,"num_bytes_recovered":57606144,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":["5","3"],"object_location_counts":[{"shards":"1,2,5","objects":202},{"shards":"1,3,5","objects":1134},{"shards":"2,3,5","objects":40}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":548,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.092171+0000","last_change":"2023-09-23T18:06:54.055392+0000","last_active":"2023-09-23T18:08:19.092171+0000","last_peered":"2023-09-23T18:08:19.092171+0000","last_clean":"2023-09-23T18:08:19.092171+0000","last_became_active":"2023-09-23T18:06:54.055101+0000","last_became_peered":"2023-09-23T18:06:54.055101+0000","last_unstale":"2023-09-23T18:08:19.092171+0000","last_undegraded":"2023-09-23T18:08:19.092171+0000","last_fullsized":"2023-09-23T18:08:19.092171+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:50.070812+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":558,"reported_epoch":336,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:08:19.081372+0000","last_change":"2023-09-23T18:06:51.071499+0000","last_active":"2023-09-23T18:08:19.081372+0000","last_peered":"2023-09-23T18:08:19.081372+0000","last_clean":"2023-09-23T18:06:48.954757+0000","last_became_active":"2023-09-23T18:06:51.071089+0000","last_became_peered":"2023-09-23T18:06:51.071089+0000","last_unstale":"2023-09-23T18:08:19.081372+0000","last_undegraded":"2023-09-23T18:06:51.054242+0000","last_fullsized":"2023-09-23T18:08:19.081372+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:01.696899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":72,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089266+0000","last_change":"2023-09-23T18:06:51.070131+0000","last_active":"2023-09-23T18:08:19.089266+0000","last_peered":"2023-09-23T18:08:19.089266+0000","last_clean":"2023-09-23T18:08:19.089266+0000","last_became_active":"2023-09-23T18:06:51.069733+0000","last_became_peered":"2023-09-23T18:06:51.069733+0000","last_unstale":"2023-09-23T18:08:19.089266+0000","last_undegraded":"2023-09-23T18:08:19.089266+0000","last_fullsized":"2023-09-23T18:08:19.089266+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:28.050735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":305,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.135493+0000","last_change":"2023-09-23T18:01:26.017382+0000","last_active":"2023-09-23T18:08:19.135493+0000","last_peered":"2023-09-23T18:08:19.135493+0000","last_clean":"2023-09-23T18:08:19.135493+0000","last_became_active":"2023-09-23T18:01:26.016890+0000","last_became_peered":"2023-09-23T18:01:26.016890+0000","last_unstale":"2023-09-23T18:08:19.135493+0000","last_undegraded":"2023-09-23T18:08:19.135493+0000","last_fullsized":"2023-09-23T18:08:19.135493+0000","mapping_epoch":215,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":216,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:25.668653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"7.3","version":"337'5238","reported_seq":5332,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:30.621532+0000","last_change":"2023-09-23T18:08:10.514743+0000","last_active":"2023-09-23T18:08:30.621532+0000","last_peered":"2023-09-23T18:08:30.621532+0000","last_clean":"2023-09-23T18:08:30.621532+0000","last_became_active":"2023-09-23T18:08:10.506710+0000","last_became_peered":"2023-09-23T18:08:10.506710+0000","last_unstale":"2023-09-23T18:08:30.621532+0000","last_undegraded":"2023-09-23T18:08:30.621532+0000","last_fullsized":"2023-09-23T18:08:30.621532+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5238,"log_dups_size":0,"ondisk_log_size":5238,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":86507520,"num_objects":1320,"num_object_clones":0,"num_object_copies":3960,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1320,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3142,"num_write_kb":142784,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":682,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.086442+0000","last_change":"2023-09-23T18:06:48.962388+0000","last_active":"2023-09-23T18:08:19.086442+0000","last_peered":"2023-09-23T18:08:19.086442+0000","last_clean":"2023-09-23T18:08:19.086442+0000","last_became_active":"2023-09-23T18:06:48.962102+0000","last_became_peered":"2023-09-23T18:06:48.962102+0000","last_unstale":"2023-09-23T18:08:19.086442+0000","last_undegraded":"2023-09-23T18:08:19.086442+0000","last_fullsized":"2023-09-23T18:08:19.086442+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.217483+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":82,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.085306+0000","last_change":"2023-09-23T18:07:49.227192+0000","last_active":"2023-09-23T18:08:19.085306+0000","last_peered":"2023-09-23T18:08:19.085306+0000","last_clean":"2023-09-23T18:08:19.085306+0000","last_became_active":"2023-09-23T18:07:49.226805+0000","last_became_peered":"2023-09-23T18:07:49.226805+0000","last_unstale":"2023-09-23T18:08:19.085306+0000","last_undegraded":"2023-09-23T18:08:19.085306+0000","last_fullsized":"2023-09-23T18:08:19.085306+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:04:59.910186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":78,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089521+0000","last_change":"2023-09-23T18:06:05.398171+0000","last_active":"2023-09-23T18:08:19.089521+0000","last_peered":"2023-09-23T18:08:19.089521+0000","last_clean":"2023-09-23T18:08:19.089521+0000","last_became_active":"2023-09-23T18:06:05.370800+0000","last_became_peered":"2023-09-23T18:06:05.370800+0000","last_unstale":"2023-09-23T18:08:19.089521+0000","last_undegraded":"2023-09-23T18:08:19.089521+0000","last_fullsized":"2023-09-23T18:08:19.089521+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:46.783817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":141,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.080308+0000","last_change":"2023-09-23T18:06:45.908943+0000","last_active":"2023-09-23T18:08:19.080308+0000","last_peered":"2023-09-23T18:08:19.080308+0000","last_clean":"2023-09-23T18:08:19.080308+0000","last_became_active":"2023-09-23T18:06:45.908609+0000","last_became_peered":"2023-09-23T18:06:45.908609+0000","last_unstale":"2023-09-23T18:08:19.080308+0000","last_undegraded":"2023-09-23T18:08:19.080308+0000","last_fullsized":"2023-09-23T18:08:19.080308+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:03:31.275390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"7.2","version":"337'5139","reported_seq":8903,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:27.550581+0000","last_change":"2023-09-23T18:07:49.240379+0000","last_active":"2023-09-23T18:08:27.550581+0000","last_peered":"2023-09-23T18:08:27.550581+0000","last_clean":"2023-09-23T18:08:27.550581+0000","last_became_active":"2023-09-23T18:07:49.235518+0000","last_became_peered":"2023-09-23T18:07:49.235518+0000","last_unstale":"2023-09-23T18:08:27.550581+0000","last_undegraded":"2023-09-23T18:08:27.550581+0000","last_fullsized":"2023-09-23T18:08:27.550581+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":279,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5139,"log_dups_size":0,"ondisk_log_size":5139,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87359488,"num_objects":1333,"num_object_clones":0,"num_object_copies":3999,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1333,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3075,"num_write_kb":141056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":915,"num_bytes_recovered":59965440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,2],"acting":[1,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":642,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089321+0000","last_change":"2023-09-23T18:06:54.059843+0000","last_active":"2023-09-23T18:08:19.089321+0000","last_peered":"2023-09-23T18:08:19.089321+0000","last_clean":"2023-09-23T18:08:19.089321+0000","last_became_active":"2023-09-23T18:06:54.059469+0000","last_became_peered":"2023-09-23T18:06:54.059469+0000","last_unstale":"2023-09-23T18:08:19.089321+0000","last_undegraded":"2023-09-23T18:08:19.089321+0000","last_fullsized":"2023-09-23T18:08:19.089321+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:59.157827+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":542,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.094326+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:08:19.094326+0000","last_peered":"2023-09-23T18:08:19.094326+0000","last_clean":"2023-09-23T18:08:19.094326+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:08:19.094326+0000","last_undegraded":"2023-09-23T18:08:19.094326+0000","last_fullsized":"2023-09-23T18:08:19.094326+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":366,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089245+0000","last_change":"2023-09-23T18:06:45.939855+0000","last_active":"2023-09-23T18:08:19.089245+0000","last_peered":"2023-09-23T18:08:19.089245+0000","last_clean":"2023-09-23T18:08:19.089245+0000","last_became_active":"2023-09-23T18:06:45.924278+0000","last_became_peered":"2023-09-23T18:06:45.924278+0000","last_unstale":"2023-09-23T18:08:19.089245+0000","last_undegraded":"2023-09-23T18:08:19.089245+0000","last_fullsized":"2023-09-23T18:08:19.089245+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:43.850375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":127,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087065+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:08:19.087065+0000","last_peered":"2023-09-23T18:08:19.087065+0000","last_clean":"2023-09-23T18:08:19.087065+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:08:19.087065+0000","last_undegraded":"2023-09-23T18:08:19.087065+0000","last_fullsized":"2023-09-23T18:08:19.087065+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":474,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.091315+0000","last_change":"2023-09-23T18:06:48.961752+0000","last_active":"2023-09-23T18:08:19.091315+0000","last_peered":"2023-09-23T18:08:19.091315+0000","last_clean":"2023-09-23T18:08:19.091315+0000","last_became_active":"2023-09-23T18:06:48.961304+0000","last_became_peered":"2023-09-23T18:06:48.961304+0000","last_unstale":"2023-09-23T18:08:19.091315+0000","last_undegraded":"2023-09-23T18:08:19.091315+0000","last_fullsized":"2023-09-23T18:08:19.091315+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":253,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087412+0000","last_change":"2023-09-23T18:06:26.148865+0000","last_active":"2023-09-23T18:08:19.087412+0000","last_peered":"2023-09-23T18:08:19.087412+0000","last_clean":"2023-09-23T18:08:19.087412+0000","last_became_active":"2023-09-23T18:06:26.148517+0000","last_became_peered":"2023-09-23T18:06:26.148517+0000","last_unstale":"2023-09-23T18:08:19.087412+0000","last_undegraded":"2023-09-23T18:08:19.087412+0000","last_fullsized":"2023-09-23T18:08:19.087412+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:38.141672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":373,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089168+0000","last_change":"2023-09-23T18:06:55.916070+0000","last_active":"2023-09-23T18:08:19.089168+0000","last_peered":"2023-09-23T18:08:19.089168+0000","last_clean":"2023-09-23T18:08:19.089168+0000","last_became_active":"2023-09-23T18:06:55.915807+0000","last_became_peered":"2023-09-23T18:06:55.915807+0000","last_unstale":"2023-09-23T18:08:19.089168+0000","last_undegraded":"2023-09-23T18:08:19.089168+0000","last_fullsized":"2023-09-23T18:08:19.089168+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:20.376063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"7.18","version":"337'5167","reported_seq":5252,"reported_epoch":337,"state":"active+clean","last_fresh":"2023-09-23T18:08:27.847937+0000","last_change":"2023-09-23T18:08:10.527785+0000","last_active":"2023-09-23T18:08:27.847937+0000","last_peered":"2023-09-23T18:08:27.847937+0000","last_clean":"2023-09-23T18:08:27.847937+0000","last_became_active":"2023-09-23T18:08:10.518124+0000","last_became_peered":"2023-09-23T18:08:10.518124+0000","last_unstale":"2023-09-23T18:08:27.847937+0000","last_undegraded":"2023-09-23T18:08:27.847937+0000","last_fullsized":"2023-09-23T18:08:27.847937+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5167,"log_dups_size":0,"ondisk_log_size":5167,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":91586560,"num_objects":1398,"num_object_clones":0,"num_object_copies":4194,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1398,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3076,"num_write_kb":143136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":469,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.092245+0000","last_change":"2023-09-23T18:06:54.055428+0000","last_active":"2023-09-23T18:08:19.092245+0000","last_peered":"2023-09-23T18:08:19.092245+0000","last_clean":"2023-09-23T18:08:19.092245+0000","last_became_active":"2023-09-23T18:06:54.055098+0000","last_became_peered":"2023-09-23T18:06:54.055098+0000","last_unstale":"2023-09-23T18:08:19.092245+0000","last_undegraded":"2023-09-23T18:08:19.092245+0000","last_fullsized":"2023-09-23T18:08:19.092245+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:04.274631+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:04.274631+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":206,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089569+0000","last_change":"2023-09-23T18:06:05.382745+0000","last_active":"2023-09-23T18:08:19.089569+0000","last_peered":"2023-09-23T18:08:19.089569+0000","last_clean":"2023-09-23T18:08:19.089569+0000","last_became_active":"2023-09-23T18:06:05.371488+0000","last_became_peered":"2023-09-23T18:06:05.371488+0000","last_unstale":"2023-09-23T18:08:19.089569+0000","last_undegraded":"2023-09-23T18:08:19.089569+0000","last_fullsized":"2023-09-23T18:08:19.089569+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:08.915006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":272,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.087632+0000","last_change":"2023-09-23T18:06:05.375138+0000","last_active":"2023-09-23T18:08:19.087632+0000","last_peered":"2023-09-23T18:08:19.087632+0000","last_clean":"2023-09-23T18:08:19.087632+0000","last_became_active":"2023-09-23T18:06:05.374819+0000","last_became_peered":"2023-09-23T18:06:05.374819+0000","last_unstale":"2023-09-23T18:08:19.087632+0000","last_undegraded":"2023-09-23T18:08:19.087632+0000","last_fullsized":"2023-09-23T18:08:19.087632+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:43.000019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"7.19","version":"337'5089","reported_seq":6854,"reported_epoch":337,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:08:27.594673+0000","last_change":"2023-09-23T18:07:15.657831+0000","last_active":"2023-09-23T18:08:27.594673+0000","last_peered":"2023-09-23T18:08:27.594673+0000","last_clean":"2023-09-23T18:06:04.334743+0000","last_became_active":"2023-09-23T18:07:15.614815+0000","last_became_peered":"2023-09-23T18:07:15.614815+0000","last_unstale":"2023-09-23T18:08:27.594673+0000","last_undegraded":"2023-09-23T18:07:15.006367+0000","last_fullsized":"2023-09-23T18:07:13.635726+0000","mapping_epoch":319,"log_start":"0'0","ondisk_log_start":"0'0","created":316,"last_epoch_clean":280,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:05:45.094891+0000","last_clean_scrub_stamp":"2023-09-23T18:05:45.094891+0000","objects_scrubbed":0,"log_size":5089,"log_dups_size":0,"ondisk_log_size":5089,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":87883776,"num_objects":1341,"num_object_clones":0,"num_object_copies":4023,"num_objects_missing_on_primary":54,"num_objects_missing":54,"num_objects_degraded":1144,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1341,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3023,"num_write_kb":139648,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":842,"num_bytes_recovered":55181312,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4],"avail_no_missing":["0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1090},{"shards":"0,4,7","objects":54},{"shards":"1,4","objects":197}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":178,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089825+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:08:19.089825+0000","last_peered":"2023-09-23T18:08:19.089825+0000","last_clean":"2023-09-23T18:08:19.089825+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:08:19.089825+0000","last_undegraded":"2023-09-23T18:08:19.089825+0000","last_fullsized":"2023-09-23T18:08:19.089825+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":186,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.135587+0000","last_change":"2023-09-23T18:07:49.223892+0000","last_active":"2023-09-23T18:08:19.135587+0000","last_peered":"2023-09-23T18:08:19.135587+0000","last_clean":"2023-09-23T18:08:19.135587+0000","last_became_active":"2023-09-23T18:07:49.223601+0000","last_became_peered":"2023-09-23T18:07:49.223601+0000","last_unstale":"2023-09-23T18:08:19.135587+0000","last_undegraded":"2023-09-23T18:08:19.135587+0000","last_fullsized":"2023-09-23T18:08:19.135587+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:30.995982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":723,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.081852+0000","last_change":"2023-09-23T18:06:08.440953+0000","last_active":"2023-09-23T18:08:19.081852+0000","last_peered":"2023-09-23T18:08:19.081852+0000","last_clean":"2023-09-23T18:08:19.081852+0000","last_became_active":"2023-09-23T18:06:08.440604+0000","last_became_peered":"2023-09-23T18:06:08.440604+0000","last_unstale":"2023-09-23T18:08:19.081852+0000","last_undegraded":"2023-09-23T18:08:19.081852+0000","last_fullsized":"2023-09-23T18:08:19.081852+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:53.274049+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":139,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.092087+0000","last_change":"2023-09-23T18:06:51.098401+0000","last_active":"2023-09-23T18:08:19.092087+0000","last_peered":"2023-09-23T18:08:19.092087+0000","last_clean":"2023-09-23T18:08:19.092087+0000","last_became_active":"2023-09-23T18:06:51.097959+0000","last_became_peered":"2023-09-23T18:06:51.097959+0000","last_unstale":"2023-09-23T18:08:19.092087+0000","last_undegraded":"2023-09-23T18:08:19.092087+0000","last_fullsized":"2023-09-23T18:08:19.092087+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:16.234414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":204,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.136402+0000","last_change":"2023-09-23T18:08:10.499513+0000","last_active":"2023-09-23T18:08:19.136402+0000","last_peered":"2023-09-23T18:08:19.136402+0000","last_clean":"2023-09-23T18:08:19.136402+0000","last_became_active":"2023-09-23T18:08:10.499130+0000","last_became_peered":"2023-09-23T18:08:10.499130+0000","last_unstale":"2023-09-23T18:08:19.136402+0000","last_undegraded":"2023-09-23T18:08:19.136402+0000","last_fullsized":"2023-09-23T18:08:19.136402+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:31.556596+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,5],"acting":[0,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":657,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.090529+0000","last_change":"2023-09-23T18:06:48.965770+0000","last_active":"2023-09-23T18:08:19.090529+0000","last_peered":"2023-09-23T18:08:19.090529+0000","last_clean":"2023-09-23T18:08:19.090529+0000","last_became_active":"2023-09-23T18:06:48.965429+0000","last_became_peered":"2023-09-23T18:06:48.965429+0000","last_unstale":"2023-09-23T18:08:19.090529+0000","last_undegraded":"2023-09-23T18:08:19.090529+0000","last_fullsized":"2023-09-23T18:08:19.090529+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":222,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.089874+0000","last_change":"2023-09-23T18:06:45.908664+0000","last_active":"2023-09-23T18:08:19.089874+0000","last_peered":"2023-09-23T18:08:19.089874+0000","last_clean":"2023-09-23T18:08:19.089874+0000","last_became_active":"2023-09-23T18:06:45.903588+0000","last_became_peered":"2023-09-23T18:06:45.903588+0000","last_unstale":"2023-09-23T18:08:19.089874+0000","last_undegraded":"2023-09-23T18:08:19.089874+0000","last_fullsized":"2023-09-23T18:08:19.089874+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:42.283189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":357,"reported_epoch":336,"state":"active+clean","last_fresh":"2023-09-23T18:08:19.086169+0000","last_change":"2023-09-23T18:06:05.374027+0000","last_active":"2023-09-23T18:08:19.086169+0000","last_peered":"2023-09-23T18:08:19.086169+0000","last_clean":"2023-09-23T18:08:19.086169+0000","last_became_active":"2023-09-23T18:06:05.373670+0000","last_became_peered":"2023-09-23T18:06:05.373670+0000","last_unstale":"2023-09-23T18:08:19.086169+0000","last_undegraded":"2023-09-23T18:08:19.086169+0000","last_fullsized":"2023-09-23T18:08:19.086169+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:01.220735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":7,"num_pg":26,"stat_sum":{"num_bytes":2771648536,"num_objects":42293,"num_object_clones":0,"num_object_copies":126879,"num_objects_missing_on_primary":112,"num_objects_missing":112,"num_objects_degraded":21101,"num_objects_misplaced":738,"num_objects_unfound":0,"num_objects_dirty":42293,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":99629,"num_write_kb":4541441,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24683,"num_bytes_recovered":1617625088,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10278875136,"data_stored":10278862920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":136410,"ondisk_log_size":136410,"up":78,"acting":74,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":6246,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":183374,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":151,"num_read_kb":286,"num_write":76,"num_write_kb":751,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":3206774,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":94,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001430,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1199092,"kb_used_data":1032684,"kb_used_omap":13,"kb_used_meta":166386,"kb_avail":92546060,"statfs":{"total":95995035648,"available":94767165440,"internally_reserved":0,"allocated":1057468416,"data_stored":1056247732,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":170379368},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,0,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:07:37 2023","interfaces":[{"interface":"back","average":{"1min":0.81499999999999995,"5min":0.96899999999999997,"15min":0.73299999999999998},"min":{"1min":0.24299999999999999,"5min":0.217,"15min":0.217},"max":{"1min":6.694,"5min":9.3499999999999996,"15min":9.3499999999999996},"last":3.4220000000000002},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.91600000000000004,"15min":1.163},"min":{"1min":0.30099999999999999,"5min":0.25,"15min":0.23000000000000001},"max":{"1min":1.4810000000000001,"5min":6.641,"15min":6.641},"last":0.36599999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.181,"15min":0.84499999999999997},"min":{"1min":0.27300000000000002,"5min":0.26600000000000001,"15min":0.252},"max":{"1min":9.4909999999999997,"5min":9.4909999999999997,"15min":9.4909999999999997},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.47499999999999998,"5min":1.1220000000000001,"15min":1.0780000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.679,"5min":6.9589999999999996,"15min":6.9589999999999996},"last":0.879}]},{"osd":2,"last update":"Sat Sep 23 18:07:37 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.1180000000000001,"15min":0.998},"min":{"1min":0.28899999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":5.048,"5min":6.6630000000000003,"15min":6.6630000000000003},"last":0.40400000000000003},{"interface":"front","average":{"1min":1.167,"5min":1.214,"15min":0.94599999999999995},"min":{"1min":0.41299999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.1829999999999998,"5min":7.9160000000000004,"15min":7.9160000000000004},"last":1.1519999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:08:27 2023","interfaces":[{"interface":"back","average":{"1min":1.105,"5min":1.264,"15min":1.4570000000000001},"min":{"1min":0.39900000000000002,"5min":0.316,"15min":0.316},"max":{"1min":4.2069999999999999,"5min":7.6070000000000002,"15min":7.6070000000000002},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.73999999999999999,"5min":1.4750000000000001,"15min":1.1379999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":1.8640000000000001,"5min":15.349,"15min":15.349},"last":0.97299999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:07:37 2023","interfaces":[{"interface":"back","average":{"1min":0.39600000000000002,"5min":0.872,"15min":1.0049999999999999},"min":{"1min":0.28699999999999998,"5min":0.27700000000000002,"15min":0.245},"max":{"1min":0.54500000000000004,"5min":6.9729999999999999,"15min":6.9729999999999999},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.74199999999999999,"15min":0.59799999999999998},"min":{"1min":0.311,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":1.651,"5min":6.3479999999999999,"15min":6.3479999999999999},"last":0.92600000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:07:37 2023","interfaces":[{"interface":"back","average":{"1min":0.58799999999999997,"5min":1.071,"15min":1.026},"min":{"1min":0.308,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":1.5209999999999999,"5min":9.5999999999999996,"15min":9.5999999999999996},"last":0.42899999999999999},{"interface":"front","average":{"1min":0.5,"5min":1.079,"15min":1.115},"min":{"1min":0.371,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":0.81699999999999995,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.39100000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:07:37 2023","interfaces":[{"interface":"back","average":{"1min":0.60199999999999998,"5min":0.95199999999999996,"15min":0.78800000000000003},"min":{"1min":0.35699999999999998,"5min":0.308,"15min":0.308},"max":{"1min":2.7879999999999998,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.95099999999999996},{"interface":"front","average":{"1min":0.625,"5min":1.0660000000000001,"15min":1.1060000000000001},"min":{"1min":0.35999999999999999,"5min":0.32000000000000001,"15min":0.30399999999999999},"max":{"1min":1.819,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":0.57599999999999996}]}]},{"osd":6,"up_from":163,"seq":700079669367,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1077744,"kb_used_data":863208,"kb_used_omap":7,"kb_used_meta":214520,"kb_avail":92667408,"statfs":{"total":95995035648,"available":94891425792,"internally_reserved":0,"allocated":883924992,"data_stored":882708311,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":219669284},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.38800000000000001,"5min":0.89900000000000002,"15min":1.159},"min":{"1min":0.251,"5min":0.23100000000000001,"15min":0.215},"max":{"1min":0.998,"5min":15.407,"15min":15.407},"last":0.45000000000000001},{"interface":"front","average":{"1min":0.56100000000000005,"5min":1.1850000000000001,"15min":1.8680000000000001},"min":{"1min":0.314,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":1.6739999999999999,"5min":15.147,"15min":15.147},"last":0.53300000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.746,"5min":1.1259999999999999,"15min":1.6180000000000001},"min":{"1min":0.28199999999999997,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":3.915,"5min":6.7690000000000001,"15min":14.233000000000001},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.89700000000000002,"5min":1.2509999999999999,"15min":1.7829999999999999},"min":{"1min":0.34000000000000002,"5min":0.26000000000000001,"15min":0.248},"max":{"1min":2.4990000000000001,"5min":14.789,"15min":14.789},"last":0.32800000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.1870000000000001,"15min":1.901},"min":{"1min":0.27600000000000002,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6669999999999998,"5min":8.6140000000000008,"15min":11.210000000000001},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.73499999999999999,"5min":1.2030000000000001,"15min":2.145},"min":{"1min":0.32800000000000001,"5min":0.245,"15min":0.245},"max":{"1min":3.6819999999999999,"5min":9.3620000000000001,"15min":11.223000000000001},"last":1.6060000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:07:30 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":1.151,"15min":1.119},"min":{"1min":0.313,"5min":0.308,"15min":0.308},"max":{"1min":4.5890000000000004,"5min":4.7539999999999996,"15min":4.7539999999999996},"last":0.54800000000000004},{"interface":"front","average":{"1min":0.92600000000000005,"5min":1.282,"15min":1.204},"min":{"1min":0.31900000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":3.5979999999999999,"5min":7.484,"15min":7.484},"last":0.37}]},{"osd":4,"last update":"Sat Sep 23 18:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.88700000000000001,"5min":0.85299999999999998,"15min":0.89200000000000002},"min":{"1min":0.29799999999999999,"5min":0.254,"15min":0.23000000000000001},"max":{"1min":8.0280000000000005,"5min":8.0280000000000005,"15min":9.9109999999999996},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.0549999999999999,"5min":0.875,"15min":0.66400000000000003},"min":{"1min":0.33500000000000002,"5min":0.29899999999999999,"15min":0.24099999999999999},"max":{"1min":8.1159999999999997,"5min":8.1159999999999997,"15min":8.3000000000000007},"last":0.51800000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.83099999999999996,"15min":0.85599999999999998},"min":{"1min":0.29799999999999999,"5min":0.29299999999999998,"15min":0.28599999999999998},"max":{"1min":0.72999999999999998,"5min":6.6159999999999997,"15min":11.869},"last":1.571},{"interface":"front","average":{"1min":0.46000000000000002,"5min":0.81000000000000005,"15min":0.73299999999999998},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":0.59099999999999997,"5min":4.7930000000000001,"15min":5.4240000000000004},"last":0.85699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.59099999999999997,"5min":0.91200000000000003,"15min":0.94599999999999995},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.093,"5min":5.5380000000000003,"15min":5.5380000000000003},"last":0.41999999999999998},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.98499999999999999,"15min":0.84899999999999998},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.395,"5min":10.411,"15min":10.411},"last":0.65900000000000003}]}]},{"osd":1,"up_from":98,"seq":420906795184,"num_pgs":74,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1695596,"kb_used_data":1239076,"kb_used_omap":7,"kb_used_meta":456504,"kb_avail":92049556,"statfs":{"total":95995035648,"available":94258745344,"internally_reserved":0,"allocated":1268813824,"data_stored":1267197249,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":467460797},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,0,2,0,0,1,7],"upper_bound":512},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.891,"5min":1.117,"15min":1.1299999999999999},"min":{"1min":0.219,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":9.0120000000000005,"5min":9.0120000000000005,"15min":15.529999999999999},"last":0.38400000000000001},{"interface":"front","average":{"1min":1.125,"5min":2.1829999999999998,"15min":1.5840000000000001},"min":{"1min":0.223,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":3.0960000000000001,"5min":82.244,"15min":82.244},"last":0.28399999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:07:50 2023","interfaces":[{"interface":"back","average":{"1min":0.88500000000000001,"5min":1.2629999999999999,"15min":1.4410000000000001},"min":{"1min":0.27000000000000002,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":6.0300000000000002,"5min":10.531000000000001,"15min":16.347999999999999},"last":0.315},{"interface":"front","average":{"1min":1.165,"5min":0.98399999999999999,"15min":1.1439999999999999},"min":{"1min":0.28999999999999998,"5min":0.28399999999999997,"15min":0.21299999999999999},"max":{"1min":7.0940000000000003,"5min":7.0940000000000003,"15min":11.305999999999999},"last":0.45100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:07:29 2023","interfaces":[{"interface":"back","average":{"1min":1.575,"5min":1.1679999999999999,"15min":0.76600000000000001},"min":{"1min":0.311,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":7.1479999999999997,"5min":7.1479999999999997,"15min":7.1479999999999997},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.1220000000000001,"5min":1.2729999999999999,"15min":1.0960000000000001},"min":{"1min":0.28000000000000003,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.9640000000000004,"5min":11.125999999999999,"15min":11.125999999999999},"last":0.33800000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.341,"5min":1.1619999999999999,"15min":1.2749999999999999},"min":{"1min":0.376,"5min":0.28199999999999997,"15min":0.28199999999999997},"max":{"1min":8.3919999999999995,"5min":8.3919999999999995,"15min":26.568999999999999},"last":0.437},{"interface":"front","average":{"1min":1.1719999999999999,"5min":1.071,"15min":1.196},"min":{"1min":0.35899999999999999,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":6.0670000000000002,"5min":7.9260000000000002,"15min":26.635000000000002},"last":0.65800000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:08:17 2023","interfaces":[{"interface":"back","average":{"1min":1.6020000000000001,"5min":1.597,"15min":1.3779999999999999},"min":{"1min":0.42199999999999999,"5min":0.34300000000000003,"15min":0.317},"max":{"1min":4.9969999999999999,"5min":11.638999999999999,"15min":11.638999999999999},"last":0.48599999999999999},{"interface":"front","average":{"1min":1.5269999999999999,"5min":1.3480000000000001,"15min":1.321},"min":{"1min":0.41599999999999998,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":8.4309999999999992,"5min":8.4309999999999992,"15min":9.5519999999999996},"last":0.40500000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:07:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":2.0550000000000002,"15min":1.978},"min":{"1min":0.39800000000000002,"5min":0.377,"15min":0.317},"max":{"1min":8.3339999999999996,"5min":89.566000000000003,"15min":89.566000000000003},"last":2.3959999999999999},{"interface":"front","average":{"1min":1.022,"5min":2.1019999999999999,"15min":1.7529999999999999},"min":{"1min":0.38,"5min":0.371,"15min":0.34699999999999998},"max":{"1min":6.0860000000000003,"5min":90.608999999999995,"15min":90.608999999999995},"last":2.2970000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:07:40 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":0.96599999999999997,"15min":0.82399999999999995},"min":{"1min":0.42999999999999999,"5min":0.39400000000000002,"15min":0.33900000000000002},"max":{"1min":7.1219999999999999,"5min":7.1219999999999999,"15min":7.1219999999999999},"last":2.1909999999999998},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.4159999999999999,"15min":1.6899999999999999},"min":{"1min":0.45200000000000001,"5min":0.45000000000000001,"15min":0.40999999999999998},"max":{"1min":6.1210000000000004,"5min":9.5709999999999997,"15min":9.5709999999999997},"last":0.53100000000000003}]}]},{"osd":0,"up_from":93,"seq":399431958712,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2405276,"kb_used_data":1914772,"kb_used_omap":7,"kb_used_meta":490488,"kb_avail":91339876,"statfs":{"total":95995035648,"available":93532033024,"internally_reserved":0,"allocated":1960726528,"data_stored":1959063439,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":502260452},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:08:20 2023","interfaces":[{"interface":"back","average":{"1min":1.869,"5min":1.6479999999999999,"15min":1.5609999999999999},"min":{"1min":0.217,"5min":0.20699999999999999,"15min":0.17899999999999999},"max":{"1min":9.3249999999999993,"5min":15.836,"15min":18.154},"last":0.316},{"interface":"front","average":{"1min":1.3620000000000001,"5min":1.262,"15min":1.387},"min":{"1min":0.251,"5min":0.251,"15min":0.17399999999999999},"max":{"1min":6.399,"5min":14.468999999999999,"15min":14.468999999999999},"last":0.246}]},{"osd":2,"last update":"Sat Sep 23 18:07:46 2023","interfaces":[{"interface":"back","average":{"1min":0.999,"5min":1.391,"15min":1.583},"min":{"1min":0.29399999999999998,"5min":0.21099999999999999,"15min":0.20499999999999999},"max":{"1min":7.6299999999999999,"5min":10.48,"15min":20.882000000000001},"last":0.71599999999999997},{"interface":"front","average":{"1min":1.4399999999999999,"5min":1.399,"15min":1.4930000000000001},"min":{"1min":0.27900000000000003,"5min":0.22500000000000001,"15min":0.217},"max":{"1min":7.7400000000000002,"5min":10.390000000000001,"15min":15.064},"last":0.33400000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:08:25 2023","interfaces":[{"interface":"back","average":{"1min":1.4419999999999999,"5min":1.637,"15min":1.581},"min":{"1min":0.27900000000000003,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":5.2229999999999999,"5min":12.779999999999999,"15min":12.779999999999999},"last":0.29299999999999998},{"interface":"front","average":{"1min":1.53,"5min":1.6299999999999999,"15min":1.3440000000000001},"min":{"1min":0.29599999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.7190000000000003,"5min":15.198,"15min":15.198},"last":0.80100000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:07:46 2023","interfaces":[{"interface":"back","average":{"1min":1.244,"5min":1.4890000000000001,"15min":1.3360000000000001},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.251},"max":{"1min":8.4109999999999996,"5min":8.4109999999999996,"15min":15.279999999999999},"last":0.34699999999999998},{"interface":"front","average":{"1min":1.105,"5min":1.5860000000000001,"15min":1.4430000000000001},"min":{"1min":0.36299999999999999,"5min":0.247,"15min":0.247},"max":{"1min":7.6500000000000004,"5min":15.477,"15min":18.106999999999999},"last":0.40100000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:07:46 2023","interfaces":[{"interface":"back","average":{"1min":1.4359999999999999,"5min":1.599,"15min":1.484},"min":{"1min":0.32800000000000001,"5min":0.29299999999999998,"15min":0.28399999999999997},"max":{"1min":7.6660000000000004,"5min":10.194000000000001,"15min":14.720000000000001},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.4370000000000001,"5min":1.3680000000000001,"15min":1.2629999999999999},"min":{"1min":0.44400000000000001,"5min":0.32600000000000001,"15min":0.254},"max":{"1min":7.7990000000000004,"5min":10.363,"15min":12.794},"last":0.42699999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:07:46 2023","interfaces":[{"interface":"back","average":{"1min":4.407,"5min":1.9950000000000001,"15min":2.1269999999999998},"min":{"1min":0.38700000000000001,"5min":0.35099999999999998,"15min":0.34599999999999997},"max":{"1min":58.457999999999998,"5min":58.457999999999998,"15min":58.457999999999998},"last":0.41299999999999998},{"interface":"front","average":{"1min":4.2809999999999997,"5min":1.885,"15min":2.2290000000000001},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":58.354999999999997,"5min":58.354999999999997,"15min":58.354999999999997},"last":0.379}]},{"osd":7,"last update":"Sat Sep 23 18:07:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":1.23,"15min":1.0249999999999999},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":10.416,"5min":10.416,"15min":10.416},"last":0.46899999999999997},{"interface":"front","average":{"1min":1.859,"5min":1.5640000000000001,"15min":1.2430000000000001},"min":{"1min":0.46300000000000002,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":8.4990000000000006,"5min":8.4990000000000006,"15min":8.4990000000000006},"last":0.47599999999999998}]}]},{"osd":2,"up_from":103,"seq":442381631659,"num_pgs":39,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1661708,"kb_used_data":1216772,"kb_used_omap":11,"kb_used_meta":444916,"kb_avail":92083444,"statfs":{"total":95995035648,"available":94293446656,"internally_reserved":0,"allocated":1245974528,"data_stored":1244326863,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":455594359},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:07:47 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.8420000000000001,"15min":1.774},"min":{"1min":0.217,"5min":0.19800000000000001,"15min":0.17199999999999999},"max":{"1min":12.710000000000001,"5min":36.656999999999996,"15min":36.656999999999996},"last":0.42199999999999999},{"interface":"front","average":{"1min":1.0069999999999999,"5min":1.6499999999999999,"15min":1.6679999999999999},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.215},"max":{"1min":5.7699999999999996,"5min":36.697000000000003,"15min":36.697000000000003},"last":0.45800000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:07:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.2749999999999999,"15min":1.6519999999999999},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.215},"max":{"1min":5.6159999999999997,"5min":8.2010000000000005,"15min":20.995000000000001},"last":0.49399999999999999},{"interface":"front","average":{"1min":1.256,"5min":1.2190000000000001,"15min":1.444},"min":{"1min":0.26000000000000001,"5min":0.252,"15min":0.193},"max":{"1min":5.843,"5min":8.3889999999999993,"15min":15.945},"last":0.29999999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:07:31 2023","interfaces":[{"interface":"back","average":{"1min":2.5539999999999998,"5min":1.7490000000000001,"15min":1.1040000000000001},"min":{"1min":0.30199999999999999,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":9.3469999999999995,"5min":37.500999999999998,"15min":37.500999999999998},"last":0.40400000000000003},{"interface":"front","average":{"1min":2.4489999999999998,"5min":1.7889999999999999,"15min":1.0029999999999999},"min":{"1min":0.28299999999999997,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":13.315,"5min":34.915999999999997,"15min":34.915999999999997},"last":0.374}]},{"osd":4,"last update":"Sat Sep 23 18:07:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6839999999999999,"5min":1.367,"15min":1.3500000000000001},"min":{"1min":0.372,"5min":0.311,"15min":0.26500000000000001},"max":{"1min":7.21,"5min":13.016999999999999,"15min":13.016999999999999},"last":1.167},{"interface":"front","average":{"1min":1.2509999999999999,"5min":1.2070000000000001,"15min":1.3200000000000001},"min":{"1min":0.36299999999999999,"5min":0.314,"15min":0.26700000000000002},"max":{"1min":5.5629999999999997,"5min":8.0820000000000007,"15min":20.920000000000002},"last":0.68000000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:07:47 2023","interfaces":[{"interface":"back","average":{"1min":1.2150000000000001,"5min":1.214,"15min":1.385},"min":{"1min":0.34300000000000003,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":5.649,"5min":8.1760000000000002,"15min":20.738},"last":0.71499999999999997},{"interface":"front","average":{"1min":1.165,"5min":1.5029999999999999,"15min":1.4970000000000001},"min":{"1min":0.373,"5min":0.32000000000000001,"15min":0.26700000000000002},"max":{"1min":5.9969999999999999,"5min":12.103999999999999,"15min":15.496},"last":0.33600000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:07:47 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.3959999999999999,"15min":1.9159999999999999},"min":{"1min":0.40600000000000003,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":7.1740000000000004,"5min":13.065,"15min":13.065},"last":0.437},{"interface":"front","average":{"1min":1.115,"5min":1.3100000000000001,"15min":2.0710000000000002},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":5.6820000000000004,"5min":8.4649999999999999,"15min":20.893000000000001},"last":1.4910000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:07:36 2023","interfaces":[{"interface":"back","average":{"1min":1.492,"5min":1.3899999999999999,"15min":1.4039999999999999},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.9480000000000004,"5min":12.051,"15min":12.051},"last":1.2509999999999999},{"interface":"front","average":{"1min":1.5009999999999999,"5min":1.355,"15min":0.873},"min":{"1min":0.503,"5min":0.40699999999999997,"15min":0.309},"max":{"1min":6.9279999999999999,"5min":8.5039999999999996,"15min":8.5039999999999996},"last":0.79300000000000004}]}]},{"osd":3,"up_from":237,"seq":1017907249226,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2207012,"kb_used_data":1953756,"kb_used_omap":3,"kb_used_meta":253244,"kb_avail":91538140,"statfs":{"total":95995035648,"available":93735055360,"internally_reserved":0,"allocated":2000646144,"data_stored":1998804858,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":259322600},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":0.70499999999999996,"5min":0.874,"15min":0.88},"min":{"1min":0.245,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":3.532,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.84599999999999997},{"interface":"front","average":{"1min":1.9279999999999999,"5min":1.2969999999999999,"15min":1.153},"min":{"1min":0.22,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":7.7329999999999997,"5min":14.885999999999999,"15min":14.885999999999999},"last":1.329}]},{"osd":1,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":1.9870000000000001,"5min":1.486,"15min":1.202},"min":{"1min":0.34699999999999998,"5min":0.245,"15min":0.245},"max":{"1min":7.9619999999999997,"5min":21.931999999999999,"15min":21.931999999999999},"last":2.2309999999999999},{"interface":"front","average":{"1min":0.92500000000000004,"5min":1.085,"15min":0.99199999999999999},"min":{"1min":0.28000000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.3109999999999999,"5min":13.631,"15min":13.631},"last":1.452}]},{"osd":2,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.90400000000000003,"15min":0.86799999999999999},"min":{"1min":0.29099999999999998,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":3.004,"5min":8.2200000000000006,"15min":8.2200000000000006},"last":0.97099999999999997},{"interface":"front","average":{"1min":1.8400000000000001,"5min":1.4259999999999999,"15min":1.306},"min":{"1min":0.33900000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.0990000000000002,"5min":21.988,"15min":21.988},"last":0.498}]},{"osd":4,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":1.766,"5min":1.347,"15min":1.1850000000000001},"min":{"1min":0.379,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":7.8959999999999999,"5min":14.935,"15min":14.935},"last":37.231000000000002},{"interface":"front","average":{"1min":0.95999999999999996,"5min":1.2569999999999999,"15min":1.131},"min":{"1min":0.34499999999999997,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.3540000000000001,"5min":12.786,"15min":12.786},"last":37.381}]},{"osd":5,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":0.96099999999999997,"5min":1.2749999999999999,"15min":1.2809999999999999},"min":{"1min":0.38700000000000001,"5min":0.307,"15min":0.307},"max":{"1min":6.3879999999999999,"5min":12.743,"15min":12.743},"last":1.278},{"interface":"front","average":{"1min":0.73399999999999999,"5min":1.1319999999999999,"15min":1.1639999999999999},"min":{"1min":0.36699999999999999,"5min":0.315,"15min":0.315},"max":{"1min":2.9470000000000001,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.94699999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":0.76700000000000002,"5min":1.0600000000000001,"15min":1.0760000000000001},"min":{"1min":0.40899999999999997,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":2.972,"5min":7.5789999999999997,"15min":7.5789999999999997},"last":0.89400000000000002},{"interface":"front","average":{"1min":1.714,"5min":1.5109999999999999,"15min":1.49},"min":{"1min":0.432,"5min":0.379,"15min":0.379},"max":{"1min":7.9349999999999996,"5min":14.964,"15min":14.964},"last":0.53300000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:07:22 2023","interfaces":[{"interface":"back","average":{"1min":1.0169999999999999,"5min":1.397,"15min":1.5129999999999999},"min":{"1min":0.42199999999999999,"5min":0.377,"15min":0.377},"max":{"1min":6.4189999999999996,"5min":12.821,"15min":12.821},"last":1.3160000000000001},{"interface":"front","average":{"1min":0.84399999999999997,"5min":1.3089999999999999,"15min":1.516},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":2.911,"5min":8.6039999999999992,"15min":8.6039999999999992},"last":0.92300000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019176,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1295412,"kb_used_data":998892,"kb_used_omap":0,"kb_used_meta":296512,"kb_avail":92449740,"statfs":{"total":95995035648,"available":94668533760,"internally_reserved":0,"allocated":1022865408,"data_stored":1021644987,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":303628288},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,1,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:07:43 2023","interfaces":[{"interface":"back","average":{"1min":0.79000000000000004,"5min":1.0429999999999999,"15min":0.996},"min":{"1min":0.26900000000000002,"5min":0.215,"15min":0.20100000000000001},"max":{"1min":5.5940000000000003,"5min":13.375999999999999,"15min":14.164},"last":0.34399999999999997},{"interface":"front","average":{"1min":0.66400000000000003,"5min":1.2430000000000001,"15min":1.218},"min":{"1min":0.29099999999999998,"5min":0.25700000000000001,"15min":0.22600000000000001},"max":{"1min":2.6419999999999999,"5min":7.1879999999999997,"15min":11.372},"last":0.48399999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:08:13 2023","interfaces":[{"interface":"back","average":{"1min":1.2549999999999999,"5min":1.1539999999999999,"15min":1.21},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":3.5310000000000001,"5min":7.6059999999999999,"15min":10.429},"last":0.58999999999999997},{"interface":"front","average":{"1min":1.2709999999999999,"5min":1.0960000000000001,"15min":1.2430000000000001},"min":{"1min":0.29099999999999998,"5min":0.27100000000000002,"15min":0.23100000000000001},"max":{"1min":5.5460000000000003,"5min":6.7640000000000002,"15min":10.83},"last":0.38700000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:07:51 2023","interfaces":[{"interface":"back","average":{"1min":1.006,"5min":1.0740000000000001,"15min":1.21},"min":{"1min":0.29199999999999998,"5min":0.28799999999999998,"15min":0.26600000000000001},"max":{"1min":5.5780000000000003,"5min":6.8259999999999996,"15min":11.630000000000001},"last":0.44},{"interface":"front","average":{"1min":1.069,"5min":1.2929999999999999,"15min":1.224},"min":{"1min":0.40100000000000002,"5min":0.314,"15min":0.27100000000000002},"max":{"1min":5.6459999999999999,"5min":13.672000000000001,"15min":13.672000000000001},"last":0.46500000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:08:26 2023","interfaces":[{"interface":"back","average":{"1min":1.6890000000000001,"5min":1.8140000000000001,"15min":1.5049999999999999},"min":{"1min":0.39200000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.6669999999999998,"5min":29.154,"15min":29.154},"last":0.41299999999999998},{"interface":"front","average":{"1min":1.829,"5min":1.8,"15min":1.5149999999999999},"min":{"1min":0.34699999999999998,"5min":0.34699999999999998,"15min":0.34200000000000003},"max":{"1min":5.7009999999999996,"5min":29.216000000000001,"15min":29.216000000000001},"last":0.60499999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:07:51 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":0.95699999999999996,"15min":0.76100000000000001},"min":{"1min":0.247,"5min":0.24199999999999999,"15min":0.224},"max":{"1min":7.5099999999999998,"5min":9.9819999999999993,"15min":9.9819999999999993},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.45100000000000001,"5min":1.1240000000000001,"15min":0.93200000000000005},"min":{"1min":0.29999999999999999,"5min":0.28799999999999998,"15min":0.26100000000000001},"max":{"1min":1.5269999999999999,"5min":11.170999999999999,"15min":11.170999999999999},"last":1.038}]},{"osd":6,"last update":"Sat Sep 23 18:07:51 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.94099999999999995,"15min":0.88500000000000001},"min":{"1min":0.29399999999999998,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":3.129,"5min":9.0990000000000002,"15min":9.0990000000000002},"last":0.62},{"interface":"front","average":{"1min":0.44900000000000001,"5min":0.89500000000000002,"15min":0.95399999999999996},"min":{"1min":0.312,"5min":0.312,"15min":0.28399999999999997},"max":{"1min":0.94899999999999995,"5min":9.1300000000000008,"15min":10.725},"last":0.66400000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:07:36 2023","interfaces":[{"interface":"back","average":{"1min":0.71499999999999997,"5min":1.202,"15min":1.01},"min":{"1min":0.34899999999999998,"5min":0.314,"15min":0.314},"max":{"1min":3.2189999999999999,"5min":13.298999999999999,"15min":13.298999999999999},"last":0.88800000000000001},{"interface":"front","average":{"1min":0.63200000000000001,"5min":1.258,"15min":1.228},"min":{"1min":0.36599999999999999,"5min":0.35799999999999998,"15min":0.307},"max":{"1min":3.367,"5min":10.192,"15min":10.192},"last":0.85199999999999998}]}]},{"osd":5,"up_from":38,"seq":163208757543,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1168612,"kb_used_data":893148,"kb_used_omap":0,"kb_used_meta":275456,"kb_avail":92576540,"statfs":{"total":95995035648,"available":94798376960,"internally_reserved":0,"allocated":914583552,"data_stored":913374945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":282066944},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:07:51 2023","interfaces":[{"interface":"back","average":{"1min":0.44900000000000001,"5min":0.75,"15min":1.0649999999999999},"min":{"1min":0.249,"5min":0.24399999999999999,"15min":0.217},"max":{"1min":1.3500000000000001,"5min":5.4539999999999997,"15min":21.376999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.95799999999999996,"15min":1.2729999999999999},"min":{"1min":0.312,"5min":0.28399999999999997,"15min":0.22900000000000001},"max":{"1min":1.363,"5min":8.5099999999999998,"15min":21.388999999999999},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:08:18 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.0269999999999999,"15min":0.96299999999999997},"min":{"1min":0.20799999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":10.547000000000001,"5min":10.547000000000001,"15min":10.853999999999999},"last":0.23899999999999999},{"interface":"front","average":{"1min":1.087,"5min":1.129,"15min":1.1419999999999999},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.399,"5min":8.9749999999999996,"15min":10.206},"last":0.44500000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:07:51 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.88400000000000001,"15min":1.3740000000000001},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":1.5620000000000001,"5min":5.5309999999999997,"15min":38.606999999999999},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.47499999999999998,"5min":1.022,"15min":1.2749999999999999},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":1.319,"5min":10.065,"15min":39.670999999999999},"last":0.316}]},{"osd":3,"last update":"Sat Sep 23 18:07:27 2023","interfaces":[{"interface":"back","average":{"1min":0.78700000000000003,"5min":1.097,"15min":0.877},"min":{"1min":0.35499999999999998,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":4.6440000000000001,"5min":8.593,"15min":8.593},"last":9.2309999999999999},{"interface":"front","average":{"1min":0.77000000000000002,"5min":1.0529999999999999,"15min":0.82499999999999996},"min":{"1min":0.40899999999999997,"5min":0.33600000000000002,"15min":0.33600000000000002},"max":{"1min":2.1459999999999999,"5min":13.446999999999999,"15min":13.446999999999999},"last":3.109}]},{"osd":4,"last update":"Sat Sep 23 18:07:58 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.76300000000000001,"15min":0.80000000000000004},"min":{"1min":0.28899999999999998,"5min":0.254,"15min":0.224},"max":{"1min":0.52500000000000002,"5min":8.6229999999999993,"15min":10.092000000000001},"last":0.29799999999999999},{"interface":"front","average":{"1min":0.45500000000000002,"5min":0.71899999999999997,"15min":0.76700000000000002},"min":{"1min":0.33200000000000002,"5min":0.29499999999999998,"15min":0.23999999999999999},"max":{"1min":0.66700000000000004,"5min":6.6379999999999999,"15min":6.6379999999999999},"last":0.36399999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:07:58 2023","interfaces":[{"interface":"back","average":{"1min":0.50700000000000001,"5min":0.74199999999999999,"15min":1.2250000000000001},"min":{"1min":0.34599999999999997,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":0.77300000000000002,"5min":4.673,"15min":11.287000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":0.48599999999999999,"5min":0.79400000000000004,"15min":1.121},"min":{"1min":0.30399999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.78900000000000003,"5min":6.6920000000000002,"15min":8.266},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:07:34 2023","interfaces":[{"interface":"back","average":{"1min":0.504,"5min":0.92500000000000004,"15min":1.339},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.63,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.50600000000000001,"5min":1.0089999999999999,"15min":1.0860000000000001},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.69899999999999995,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.503}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1948913664,"data_stored":1948909592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1259999232,"data_stored":1259995160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1235943424,"data_stored":1235943424,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1989214208,"data_stored":1989214208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1013972992,"data_stored":1013972992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":909508608,"data_stored":909508608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":876023808,"data_stored":876019736,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":7,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1045299200,"data_stored":1045299200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:08:33.301 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:08:33.302 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_2 pg_num 36 2023-09-23T18:08:33.345 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:33.410 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:33.471 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:33.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:33.483 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 21120/126960 objects degraded (16.635%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:33.583 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:33.669 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:33.760 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:33.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:33.767 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:33.868 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:33.939 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:34.008 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:34.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:34.012 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:34.113 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:34.177 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:34.189 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:34 smithi055 ceph-mon[179668]: from='client.45794 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:08:34.189 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:34 smithi055 ceph-mon[179668]: pgmap v834: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 20580/123000 objects degraded (16.732%); 738/123000 objects misplaced (0.600%); 23 MiB/s, 374 objects/s recovering 2023-09-23T18:08:34.190 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:34 smithi055 ceph-mon[184174]: from='client.45794 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:08:34.190 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:34 smithi055 ceph-mon[184174]: pgmap v834: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 20580/123000 objects degraded (16.732%); 738/123000 objects misplaced (0.600%); 23 MiB/s, 374 objects/s recovering 2023-09-23T18:08:34.281 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:34.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:34.297 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:34.324 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:08:34.324 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:08:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:34 smithi099 ceph-mon[167076]: from='client.45794 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:08:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:34 smithi099 ceph-mon[167076]: pgmap v834: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 571 op/s; 20580/123000 objects degraded (16.732%); 738/123000 objects misplaced (0.600%); 23 MiB/s, 374 objects/s recovering 2023-09-23T18:08:34.372 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:34.398 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:34.488 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:34.701 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:34.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:34.713 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:34.718 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:08:34.735 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:08:34.814 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:34.919 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:34.988 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:34.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:34.997 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:35.097 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:35.152 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:08:35.190 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:35.295 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:35 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1988551418' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:35.295 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:35 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:35.296 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:35 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1988551418' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:35.296 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:35 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:35.434 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:35.442 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:35.442 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:35.543 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:35.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:35 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1988551418' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:35.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:35 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:35.591 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:35.710 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:35.723 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:35.724 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:35.825 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:35.905 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:35.947 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:35.968 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:35.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:35.973 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:36.053 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:36.075 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:36.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[184174]: pgmap v835: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 20348/121587 objects degraded (16.735%); 738/121587 objects misplaced (0.607%); 24 MiB/s, 382 objects/s recovering 2023-09-23T18:08:36.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:36.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[184174]: failed to encode map e338 with expected crc 2023-09-23T18:08:36.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[184174]: failed to encode map e338 with expected crc 2023-09-23T18:08:36.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:36.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[184174]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T18:08:36.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[179668]: pgmap v835: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 20348/121587 objects degraded (16.735%); 738/121587 objects misplaced (0.607%); 24 MiB/s, 382 objects/s recovering 2023-09-23T18:08:36.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:36.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[179668]: failed to encode map e338 with expected crc 2023-09-23T18:08:36.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[179668]: failed to encode map e338 with expected crc 2023-09-23T18:08:36.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:36.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:36 smithi055 ceph-mon[179668]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T18:08:36.159 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:36.287 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:36.300 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:36.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:36.401 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:36.462 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:36 smithi099 ceph-mon[167076]: pgmap v835: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.5 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 0 B/s wr, 585 op/s; 20348/121587 objects degraded (16.735%); 738/121587 objects misplaced (0.607%); 24 MiB/s, 382 objects/s recovering 2023-09-23T18:08:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:36 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:36 smithi099 ceph-mon[167076]: failed to encode map e338 with expected crc 2023-09-23T18:08:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:36 smithi099 ceph-mon[167076]: failed to encode map e338 with expected crc 2023-09-23T18:08:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:36 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:36 smithi099 ceph-mon[167076]: osdmap e338: 8 total, 8 up, 8 in 2023-09-23T18:08:36.616 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:36.627 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:36.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:36.729 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:36.825 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:37.108 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:37.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:37.114 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:37.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:37.299 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:37.359 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:37.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:37.371 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:37.451 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 7 pg_num to 36 2023-09-23T18:08:37.472 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:37.547 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:37.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1663810311' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T18:08:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:37 smithi099 ceph-mon[167076]: pgmap v837: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 19489/115377 objects degraded (16.892%); 342/115377 objects misplaced (0.296%); 23 MiB/s, 372 objects/s recovering 2023-09-23T18:08:37.638 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:37.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:37.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1663810311' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T18:08:37.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:37 smithi055 ceph-mon[184174]: pgmap v837: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 19489/115377 objects degraded (16.892%); 342/115377 objects misplaced (0.296%); 23 MiB/s, 372 objects/s recovering 2023-09-23T18:08:37.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:37.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:37.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1663810311' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]: dispatch 2023-09-23T18:08:37.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:37 smithi055 ceph-mon[179668]: pgmap v837: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.3 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 668 op/s; 19489/115377 objects degraded (16.892%); 342/115377 objects misplaced (0.296%); 23 MiB/s, 372 objects/s recovering 2023-09-23T18:08:37.645 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:37.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:37.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:37.752 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:37.820 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:37.839 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:08:37.878 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:37.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:37.886 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:37.987 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:38.024 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:08:38.024 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:08:38.061 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:38.497 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:08:38.498 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:38.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:38.511 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:38.612 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:38.645 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:38.646 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:08:38.665 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:08:38.697 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:38.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:38.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:38.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:38 smithi099 ceph-mon[167076]: failed to encode map e339 with expected crc 2023-09-23T18:08:38.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:38 smithi099 ceph-mon[167076]: failed to encode map e339 with expected crc 2023-09-23T18:08:38.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1663810311' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]': finished 2023-09-23T18:08:38.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:38 smithi099 ceph-mon[167076]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T18:08:38.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19489/115377 objects degraded (16.892%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:38.718 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:08:38.751 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:08:38.791 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:08:38.799 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:38.801 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:38.825 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:08:38.853 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:38.867 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:08:38.902 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:08:38.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:08:38.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:08:38.904 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:08:38.905 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:08:38.905 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:08:38.905 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:08:38.905 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:08:38.905 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:08:38.909 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:38.916 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:38.917 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:38.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:38 smithi055 ceph-mon[184174]: failed to encode map e339 with expected crc 2023-09-23T18:08:38.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:38 smithi055 ceph-mon[184174]: failed to encode map e339 with expected crc 2023-09-23T18:08:38.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1663810311' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]': finished 2023-09-23T18:08:38.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:38 smithi055 ceph-mon[184174]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T18:08:38.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19489/115377 objects degraded (16.892%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:38.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:38 smithi055 ceph-mon[179668]: failed to encode map e339 with expected crc 2023-09-23T18:08:38.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:38 smithi055 ceph-mon[179668]: failed to encode map e339 with expected crc 2023-09-23T18:08:38.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1663810311' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num", "val": "36"}]': finished 2023-09-23T18:08:38.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:38 smithi055 ceph-mon[179668]: osdmap e339: 8 total, 8 up, 8 in 2023-09-23T18:08:38.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19489/115377 objects degraded (16.892%), 17 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:39.017 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:39.068 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:39.125 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:39.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:39.131 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:39.231 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:39.285 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:39.349 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:39.357 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:39.358 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:39.451 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2483726536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:39.451 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:39 smithi055 ceph-mon[179668]: pgmap v839: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 18673/108801 objects degraded (17.163%); 342/108801 objects misplaced (0.314%); 21 MiB/s, 340 objects/s recovering 2023-09-23T18:08:39.451 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:39 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T18:08:39.459 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:39.740 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2483726536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:39.740 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:39 smithi055 ceph-mon[184174]: pgmap v839: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 18673/108801 objects degraded (17.163%); 342/108801 objects misplaced (0.314%); 21 MiB/s, 340 objects/s recovering 2023-09-23T18:08:39.740 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:39 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T18:08:39.745 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:39.794 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:39.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:39.798 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2483726536' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:39 smithi099 ceph-mon[167076]: pgmap v839: 129 pgs: 13 active+recovery_wait+degraded, 1 active+recovering, 4 active+recovery_wait+undersized+degraded+remapped, 111 active+clean; 2.2 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 755 op/s; 18673/108801 objects degraded (17.163%); 342/108801 objects misplaced (0.314%); 21 MiB/s, 340 objects/s recovering 2023-09-23T18:08:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:39 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]: dispatch 2023-09-23T18:08:39.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:39.992 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:40.096 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:40.127 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:40.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:40.130 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:40.231 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:40.267 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:40.308 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:40.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:40.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:40.413 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:40.439 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:40.492 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:40.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:40.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:40.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:40.650 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:40.672 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-mon[184174]: failed to encode map e340 with expected crc 2023-09-23T18:08:40.673 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T18:08:40.673 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-mon[184174]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T18:08:40.673 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-mon[184174]: failed to encode map e340 with expected crc 2023-09-23T18:08:40.673 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-mon[184174]: failed to encode map e340 with expected crc 2023-09-23T18:08:40.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-mon[179668]: failed to encode map e340 with expected crc 2023-09-23T18:08:40.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T18:08:40.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-mon[179668]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T18:08:40.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-mon[179668]: failed to encode map e340 with expected crc 2023-09-23T18:08:40.673 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-mon[179668]: failed to encode map e340 with expected crc 2023-09-23T18:08:40.674 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:08:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:08:40] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.43.0" 2023-09-23T18:08:40.695 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:40.699 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:40 smithi099 ceph-mon[167076]: failed to encode map e340 with expected crc 2023-09-23T18:08:40.699 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:40 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_2", "var": "pg_num_actual", "val": "36"}]': finished 2023-09-23T18:08:40.699 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:40 smithi099 ceph-mon[167076]: osdmap e340: 8 total, 8 up, 8 in 2023-09-23T18:08:40.699 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:40 smithi099 ceph-mon[167076]: failed to encode map e340 with expected crc 2023-09-23T18:08:40.699 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:40 smithi099 ceph-mon[167076]: failed to encode map e340 with expected crc 2023-09-23T18:08:40.700 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:40.700 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:40.801 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:40.832 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:40.871 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:40.874 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:40.874 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:40.975 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:41.001 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:41.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:41.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:41.036 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:41.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:41.164 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:41.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:41.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:41.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:41.301 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:41.327 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:08:41.327 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:08:41.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:08:41.327 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:08:41.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.328 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154239 5.d 5:b6c40944:::benchmark_data_smithi055_229117_object48613:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:08:41.328 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:08:41.328 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 572.48465867799996, 2023-09-23T18:08:41.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5202187069999997, 2023-09-23T18:08:41.328 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.328 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:08:41.328 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.328 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:41.328 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154239 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839793+0000", 2023-09-23T18:08:41.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839792+0000", 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839799+0000", 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2969999999999998e-06 2023-09-23T18:08:41.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839815+0000", 2023-09-23T18:08:41.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5458e-05 2023-09-23T18:08:41.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839823+0000", 2023-09-23T18:08:41.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.833e-06 2023-09-23T18:08:41.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.839864+0000", 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0865999999999998e-05 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840002+0000", 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013775199999999999 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5", 2023-09-23T18:08:41.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.840128+0000", 2023-09-23T18:08:41.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012573200000000001 2023-09-23T18:08:41.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.845223+0000", 2023-09-23T18:08:41.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050947299999999996 2023-09-23T18:08:41.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.854611+0000", 2023-09-23T18:08:41.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093881209999999993 2023-09-23T18:08:41.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.854622+0000", 2023-09-23T18:08:41.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1011e-05 2023-09-23T18:08:41.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851119+0000", 2023-09-23T18:08:41.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.996496979 2023-09-23T18:08:41.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851132+0000", 2023-09-23T18:08:41.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.312e-05 2023-09-23T18:08:41.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.340349+0000", 2023-09-23T18:08:41.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4892177549999999 2023-09-23T18:08:41.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:08:41.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.340443+0000", 2023-09-23T18:08:41.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3108000000000004e-05 2023-09-23T18:08:41.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.359910+0000", 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.019467288999999999 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360012+0000", 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000102093 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.337 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154248 5.d 5:bb80aea8:::benchmark_data_smithi055_229117_object48622:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.855620+0000", 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 572.46883235300004, 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5213527490000001, 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.338 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:41.339 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:41.339 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154248 2023-09-23T18:08:41.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.341 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.341 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855620+0000", 2023-09-23T18:08:41.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855620+0000", 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855618+0000", 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855624+0000", 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3399999999999997e-06 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855643+0000", 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9772999999999999e-05 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855653+0000", 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4870000000000008e-06 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855688+0000", 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5308999999999997e-05 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.855697+0000", 2023-09-23T18:08:41.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6910000000000003e-06 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851160+0000", 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99546285999999995 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851169+0000", 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8920000000000006e-06 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.341897+0000", 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4907282319999999 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.342015+0000", 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118402 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.342112+0000", 2023-09-23T18:08:41.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6391000000000006e-05 2023-09-23T18:08:41.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356673+0000", 2023-09-23T18:08:41.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014561579999999999 2023-09-23T18:08:41.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356774+0000", 2023-09-23T18:08:41.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010047699999999999 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.375006+0000", 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018232224000000002 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.375055+0000", 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9345999999999998e-05 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.376972+0000", 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001916966 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154254 5.4 5:2eea31d3:::benchmark_data_smithi055_229117_object48628:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.876460+0000", 2023-09-23T18:08:41.349 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 572.44799161799995, 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2290530139999998, 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154254 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876460+0000", 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876460+0000", 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876458+0000", 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876469+0000", 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0913999999999999e-05 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876489+0000", 2023-09-23T18:08:41.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9649000000000001e-05 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876500+0000", 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1600000000000001e-05 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876561+0000", 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0857999999999999e-05 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876573+0000", 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1836e-05 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851144+0000", 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97457099800000002 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851155+0000", 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1548e-05 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.095633+0000", 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.244477501 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.095782+0000", 2023-09-23T18:08:41.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014866200000000001 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.095879+0000", 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7288999999999996e-05 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100460+0000", 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045811979999999999 2023-09-23T18:08:41.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100470+0000", 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6609999999999995e-06 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105323+0000", 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048527759999999996 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105356+0000", 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3546e-05 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105407+0000", 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1372000000000002e-05 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105513+0000", 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010586699999999999 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154255 5.d 5:bf0b5457:::benchmark_data_smithi055_229117_object48629:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.876519+0000", 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 572.44793312900003, 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.514717928, 2023-09-23T18:08:41.359 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154255 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876519+0000", 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876519+0000", 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876518+0000", 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876522+0000", 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0609999999999997e-06 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876531+0000", 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2300000000000008e-06 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876537+0000", 2023-09-23T18:08:41.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5570000000000001e-06 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876590+0000", 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2528000000000003e-05 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.876600+0000", 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8800000000000003e-06 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851204+0000", 2023-09-23T18:08:41.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97460424499999998 2023-09-23T18:08:41.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851212+0000", 2023-09-23T18:08:41.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2460000000000003e-06 2023-09-23T18:08:41.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.346812+0000", 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4955998619999999 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.346965+0000", 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015338600000000001 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:08:41.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.347122+0000", 2023-09-23T18:08:41.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000156537 2023-09-23T18:08:41.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356744+0000", 2023-09-23T18:08:41.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0096223920000000004 2023-09-23T18:08:41.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.356784+0000", 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9984000000000002e-05 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.389719+0000", 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032934915000000002 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.389760+0000", 2023-09-23T18:08:41.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0655999999999998e-05 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391237+0000", 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014769970000000001 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.368 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154260 5.d 5:b55d3ddc:::benchmark_data_smithi055_229117_object48634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.880860+0000", 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 572.44359183799997, 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5120832480000002, 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154260 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.369 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880860+0000", 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880860+0000", 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880859+0000", 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880866+0000", 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2790000000000001e-06 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880882+0000", 2023-09-23T18:08:41.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5529999999999999e-05 2023-09-23T18:08:41.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880890+0000", 2023-09-23T18:08:41.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.602e-06 2023-09-23T18:08:41.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880936+0000", 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5847000000000003e-05 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.880946+0000", 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8519999999999999e-06 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851234+0000", 2023-09-23T18:08:41.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97028760300000005 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851240+0000", 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5690000000000001e-06 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.351422+0000", 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.500181193 2023-09-23T18:08:41.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.351601+0000", 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017931699999999999 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.351779+0000", 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017792700000000001 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360041+0000", 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082620220000000008 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360048+0000", 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4649999999999997e-06 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391364+0000", 2023-09-23T18:08:41.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031316043000000002 2023-09-23T18:08:41.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391424+0000", 2023-09-23T18:08:41.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0053000000000003e-05 2023-09-23T18:08:41.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.392943+0000", 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001518997 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154262 5.d 5:bdaa66ea:::benchmark_data_smithi055_229117_object48636:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.881965+0000", 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 572.44248694600003, 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.526328737, 2023-09-23T18:08:41.378 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154262 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881965+0000", 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881965+0000", 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881963+0000", 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881971+0000", 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8699999999999992e-06 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.881990+0000", 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9173000000000001e-05 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.882000+0000", 2023-09-23T18:08:41.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4210000000000001e-06 2023-09-23T18:08:41.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.882051+0000", 2023-09-23T18:08:41.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1026999999999998e-05 2023-09-23T18:08:41.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.882062+0000", 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1463e-05 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851261+0000", 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96919898400000004 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851295+0000", 2023-09-23T18:08:41.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3628999999999999e-05 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360083+0000", 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5087881159999998 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360255+0000", 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017206400000000001 2023-09-23T18:08:41.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.360415+0000", 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015982 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.374968+0000", 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014553857999999999 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391256+0000", 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016287340000000001 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.406335+0000", 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015079231 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.406390+0000", 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4511e-05 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.408294+0000", 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001904177 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.387 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154263 5.b 5:dbd24d9d:::benchmark_data_smithi055_229117_object48637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.883292+0000", 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 572.44115952699997, 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0983152249999999, 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154263 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883292+0000", 2023-09-23T18:08:41.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883292+0000", 2023-09-23T18:08:41.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883291+0000", 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883300+0000", 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6999999999999997e-06 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883317+0000", 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7048999999999999e-05 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883327+0000", 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0108999999999999e-05 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883392+0000", 2023-09-23T18:08:41.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4751000000000004e-05 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.883402+0000", 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0149000000000001e-05 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.854589+0000", 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97118778699999997 2023-09-23T18:08:41.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.854601+0000", 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1292e-05 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.958490+0000", 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.103889399 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.958651+0000", 2023-09-23T18:08:41.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016055600000000001 2023-09-23T18:08:41.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:08:41.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.958761+0000", 2023-09-23T18:08:41.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011042799999999999 2023-09-23T18:08:41.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.968153+0000", 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093921730000000002 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981288+0000", 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01313507 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981455+0000", 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000166551 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981509+0000", 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4163999999999998e-05 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:10.981608+0000", 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8573000000000006e-05 2023-09-23T18:08:41.396 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154267 5.4 5:27d4bd8e:::benchmark_data_smithi055_229117_object48641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:08.891154+0000", 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 572.43329782000001, 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2238628070000002, 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.397 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.398 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:41.398 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:41.398 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154267 2023-09-23T18:08:41.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.398 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891154+0000", 2023-09-23T18:08:41.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891154+0000", 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891152+0000", 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891163+0000", 2023-09-23T18:08:41.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0482999999999999e-05 2023-09-23T18:08:41.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891191+0000", 2023-09-23T18:08:41.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8411e-05 2023-09-23T18:08:41.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891202+0000", 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0841e-05 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891258+0000", 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6165999999999997e-05 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:08.891269+0000", 2023-09-23T18:08:41.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0572e-05 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851195+0000", 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95992611100000003 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.851204+0000", 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.645e-06 2023-09-23T18:08:41.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100511+0000", 2023-09-23T18:08:41.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.249306255 2023-09-23T18:08:41.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100638+0000", 2023-09-23T18:08:41.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012729000000000001 2023-09-23T18:08:41.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.100765+0000", 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012738699999999999 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105586+0000", 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048209419999999999 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.105597+0000", 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0890000000000001e-05 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.114801+0000", 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092037030000000006 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.114844+0000", 2023-09-23T18:08:41.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3130999999999998e-05 2023-09-23T18:08:41.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.114898+0000", 2023-09-23T18:08:41.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4320999999999997e-05 2023-09-23T18:08:41.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.115017+0000", 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118578 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154270 5.d 5:b225da75:::benchmark_data_smithi055_229117_object48644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:09.964328+0000", 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 571.36012353900003, 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4456077330000001, 2023-09-23T18:08:41.407 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.408 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.408 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.408 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:41.408 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:41.408 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154270 2023-09-23T18:08:41.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.408 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964328+0000", 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964328+0000", 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964327+0000", 2023-09-23T18:08:41.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:41.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964335+0000", 2023-09-23T18:08:41.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9510000000000006e-06 2023-09-23T18:08:41.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964352+0000", 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7091e-05 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964361+0000", 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3870000000000001e-06 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964414+0000", 2023-09-23T18:08:41.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2559999999999998e-05 2023-09-23T18:08:41.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964421+0000", 2023-09-23T18:08:41.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0230000000000004e-06 2023-09-23T18:08:41.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.365357+0000", 2023-09-23T18:08:41.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.400936304 2023-09-23T18:08:41.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.365487+0000", 2023-09-23T18:08:41.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013022199999999999 2023-09-23T18:08:41.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:08:41.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.365592+0000", 2023-09-23T18:08:41.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104739 2023-09-23T18:08:41.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.389646+0000", 2023-09-23T18:08:41.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024053990000000001 2023-09-23T18:08:41.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.391266+0000", 2023-09-23T18:08:41.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016202969999999999 2023-09-23T18:08:41.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.408353+0000", 2023-09-23T18:08:41.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017086086 2023-09-23T18:08:41.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.408410+0000", 2023-09-23T18:08:41.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7760999999999998e-05 2023-09-23T18:08:41.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.409936+0000", 2023-09-23T18:08:41.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015258559999999999 2023-09-23T18:08:41.416 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.416 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.417 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.417 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.44681.0:154271 5.6 5:6eca1f68:::benchmark_data_smithi055_229117_object48645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:08:41.417 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T17:59:09.964509+0000", 2023-09-23T18:08:41.417 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 571.35994270100002, 2023-09-23T18:08:41.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.263656766, 2023-09-23T18:08:41.417 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.417 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.418 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.418 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.44681", 2023-09-23T18:08:41.418 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:41.418 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 154271 2023-09-23T18:08:41.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.418 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964509+0000", 2023-09-23T18:08:41.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964509+0000", 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964508+0000", 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:08:41.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964515+0000", 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3600000000000001e-06 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964530+0000", 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5153e-05 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964538+0000", 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1119999999999996e-06 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964594+0000", 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5926e-05 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:09.964603+0000", 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2220000000000008e-06 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.215843+0000", 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2512403320000001 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.215990+0000", 2023-09-23T18:08:41.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000147095 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.216730+0000", 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00073915700000000005 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225069+0000", 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0083390189999999996 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225111+0000", 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2026e-05 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225280+0000", 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000169524 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.225348+0000", 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8232000000000007e-05 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T17:59:11.228166+0000", 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028175650000000002 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33521 6.2 6:4a04cd74:::benchmark_data_smithi055_275963_object33520:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.422794+0000", 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 422.90165839100001, 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.013332479, 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:41.426 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33521 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422794+0000", 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422794+0000", 2023-09-23T18:08:41.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422792+0000", 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422842+0000", 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0219999999999997e-05 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422852+0000", 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4700000000000008e-06 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422857+0000", 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5779999999999998e-06 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.422908+0000", 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0387999999999999e-05 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423038+0000", 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013043899999999999 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:08:41.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423199+0000", 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016039900000000001 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.425419+0000", 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022201949999999999 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448248+0000", 2023-09-23T18:08:41.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022829682 2023-09-23T18:08:41.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448261+0000", 2023-09-23T18:08:41.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2745000000000001e-05 2023-09-23T18:08:41.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.436111+0000", 2023-09-23T18:08:41.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98784968799999995 2023-09-23T18:08:41.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.436126+0000", 2023-09-23T18:08:41.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5163e-05 2023-09-23T18:08:41.433 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.433 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.433 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.434 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33523 6.f 6:f5fa26bf:::benchmark_data_smithi055_275963_object33522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e218)", 2023-09-23T18:08:41.434 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.423670+0000", 2023-09-23T18:08:41.434 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 422.90078212899999, 2023-09-23T18:08:41.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.050894467, 2023-09-23T18:08:41.434 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.434 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:08:41.434 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.434 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:08:41.434 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:41.435 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33523 2023-09-23T18:08:41.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.435 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.435 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.435 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.435 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423670+0000", 2023-09-23T18:08:41.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.435 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.435 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423670+0000", 2023-09-23T18:08:41.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423669+0000", 2023-09-23T18:08:41.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:08:41.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423707+0000", 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8553999999999997e-05 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423723+0000", 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5512e-05 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423732+0000", 2023-09-23T18:08:41.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6130000000000004e-06 2023-09-23T18:08:41.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423786+0000", 2023-09-23T18:08:41.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4255999999999997e-05 2023-09-23T18:08:41.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.423953+0000", 2023-09-23T18:08:41.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016769800000000001 2023-09-23T18:08:41.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:08:41.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.424125+0000", 2023-09-23T18:08:41.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000171409 2023-09-23T18:08:41.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.425758+0000", 2023-09-23T18:08:41.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016331780000000001 2023-09-23T18:08:41.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448278+0000", 2023-09-23T18:08:41.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022520401999999998 2023-09-23T18:08:41.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.448286+0000", 2023-09-23T18:08:41.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4089999999999997e-06 2023-09-23T18:08:41.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433253+0000", 2023-09-23T18:08:41.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98496738100000003 2023-09-23T18:08:41.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433268+0000", 2023-09-23T18:08:41.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4986e-05 2023-09-23T18:08:41.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.463950+0000", 2023-09-23T18:08:41.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0306814449999999 2023-09-23T18:08:41.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:08:41.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465116+0000", 2023-09-23T18:08:41.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011660500000000001 2023-09-23T18:08:41.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474508+0000", 2023-09-23T18:08:41.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093924479999999994 2023-09-23T18:08:41.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474564+0000", 2023-09-23T18:08:41.444 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6112000000000001e-05 2023-09-23T18:08:41.444 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.444 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.444 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.444 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33545 6.f 6:f8f54476:::benchmark_data_smithi055_275963_object33544:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:08:41.445 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.452774+0000", 2023-09-23T18:08:41.445 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 422.87167829399999, 2023-09-23T18:08:41.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0213147949999999, 2023-09-23T18:08:41.445 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.445 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.445 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.445 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:08:41.445 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:41.445 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33545 2023-09-23T18:08:41.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452774+0000", 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452774+0000", 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452773+0000", 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452804+0000", 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1013000000000003e-05 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.448 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452817+0000", 2023-09-23T18:08:41.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3525e-05 2023-09-23T18:08:41.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.448 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.448 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.448 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452825+0000", 2023-09-23T18:08:41.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5809999999999999e-06 2023-09-23T18:08:41.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.448 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.448 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452879+0000", 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4762999999999997e-05 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.452893+0000", 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3278999999999999e-05 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433299+0000", 2023-09-23T18:08:41.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98040629000000001 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433308+0000", 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6859999999999995e-06 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465523+0000", 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0322154109999999 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.450 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465714+0000", 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000190941 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.465951+0000", 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00023701199999999999 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.451 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.470811+0000", 2023-09-23T18:08:41.452 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004859693 2023-09-23T18:08:41.452 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.452 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.452 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.452 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.470872+0000", 2023-09-23T18:08:41.452 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1622000000000001e-05 2023-09-23T18:08:41.452 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.452 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.452 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.452 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.473905+0000", 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030324229999999998 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.473979+0000", 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4177999999999994e-05 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474088+0000", 2023-09-23T18:08:41.453 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010944500000000001 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33550 6.f 6:fa5a6c38:::benchmark_data_smithi055_275963_object33549:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.456197+0000", 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 422.86825502900001, 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0180980869999998, 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.454 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33550 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456197+0000", 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.455 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456197+0000", 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456195+0000", 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.456 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.457 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456221+0000", 2023-09-23T18:08:41.457 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5984000000000001e-05 2023-09-23T18:08:41.457 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.457 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.457 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.457 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456241+0000", 2023-09-23T18:08:41.457 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0132999999999998e-05 2023-09-23T18:08:41.457 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.457 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.458 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.458 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456251+0000", 2023-09-23T18:08:41.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4180000000000003e-06 2023-09-23T18:08:41.458 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.458 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.458 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.458 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456325+0000", 2023-09-23T18:08:41.458 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4190000000000006e-05 2023-09-23T18:08:41.458 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.456341+0000", 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5846e-05 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433338+0000", 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97699677799999995 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.459 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.460 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.460 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433347+0000", 2023-09-23T18:08:41.460 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2590000000000006e-06 2023-09-23T18:08:41.460 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.460 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.460 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.460 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.467560+0000", 2023-09-23T18:08:41.460 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.034212833 2023-09-23T18:08:41.460 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.460 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.467689+0000", 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012873499999999999 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.467798+0000", 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010996400000000001 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.461 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.470822+0000", 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030237990000000002 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.471707+0000", 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00088451899999999995 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474135+0000", 2023-09-23T18:08:41.462 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002428655 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474192+0000", 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6406999999999997e-05 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474295+0000", 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000103137 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.463 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33562 6.f 6:f588a2ef:::benchmark_data_smithi055_275963_object33561:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:38.476003+0000", 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 422.84844859100002, 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9984609899999999, 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:08:41.464 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33562 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476003+0000", 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476003+0000", 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.465 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476002+0000", 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476021+0000", 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8847000000000001e-05 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.466 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.467 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.467 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476034+0000", 2023-09-23T18:08:41.467 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2411e-05 2023-09-23T18:08:41.467 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.467 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.467 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.467 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476039+0000", 2023-09-23T18:08:41.467 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5709999999999997e-06 2023-09-23T18:08:41.467 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.468 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.468 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.468 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476088+0000", 2023-09-23T18:08:41.468 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8936e-05 2023-09-23T18:08:41.468 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.468 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.468 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.468 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:38.476097+0000", 2023-09-23T18:08:41.468 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3600000000000002e-06 2023-09-23T18:08:41.468 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.469 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.469 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.469 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433567+0000", 2023-09-23T18:08:41.469 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95746954799999995 2023-09-23T18:08:41.469 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.469 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.469 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.469 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.433577+0000", 2023-09-23T18:08:41.469 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0359e-05 2023-09-23T18:08:41.469 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.470 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.470 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.470 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.469156+0000", 2023-09-23T18:08:41.470 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0355791350000001 2023-09-23T18:08:41.470 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.470 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.470 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.470 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.469366+0000", 2023-09-23T18:08:41.470 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000209228 2023-09-23T18:08:41.470 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.469586+0000", 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00022075999999999999 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.471026+0000", 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001439582 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.471 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.472 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.471737+0000", 2023-09-23T18:08:41.472 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00071104700000000002 2023-09-23T18:08:41.472 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.472 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.472 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.472 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474335+0000", 2023-09-23T18:08:41.472 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025979190000000002 2023-09-23T18:08:41.472 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.472 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.472 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474377+0000", 2023-09-23T18:08:41.473 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1820000000000003e-05 2023-09-23T18:08:41.473 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.473 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.473 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.473 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.474464+0000", 2023-09-23T18:08:41.473 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7562e-05 2023-09-23T18:08:41.473 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.473 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.473 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.474 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.474 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.474 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:84427 7.13 7:cb0230ba:::benchmark_data_smithi055_311528_object13464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-09-23T18:08:41.474 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:55.875336+0000", 2023-09-23T18:08:41.474 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 45.449116246000003, 2023-09-23T18:08:41.474 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.079786512000000004, 2023-09-23T18:08:41.474 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.474 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.474 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.474 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:41.475 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:41.475 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 84427 2023-09-23T18:08:41.475 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.475 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.475 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.475 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.475 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875336+0000", 2023-09-23T18:08:41.475 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.475 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.475 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875336+0000", 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875335+0000", 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.476 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.477 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875338+0000", 2023-09-23T18:08:41.477 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2179999999999999e-06 2023-09-23T18:08:41.477 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.477 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.477 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.477 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875344+0000", 2023-09-23T18:08:41.477 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8830000000000003e-06 2023-09-23T18:08:41.477 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.477 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.478 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.478 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875349+0000", 2023-09-23T18:08:41.478 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1229999999999999e-06 2023-09-23T18:08:41.478 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.478 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.478 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.478 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.876247+0000", 2023-09-23T18:08:41.478 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00089710500000000004 2023-09-23T18:08:41.478 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.478 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.876381+0000", 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000134797 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.876487+0000", 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000105522 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:08:41.479 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.879537+0000", 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030497250000000001 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.879732+0000", 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000195477 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.955002+0000", 2023-09-23T18:08:41.480 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.075269427 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.955061+0000", 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.94e-05 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.955122+0000", 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1279999999999996e-05 2023-09-23T18:08:41.481 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.482 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.482 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.482 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.482 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.482 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:90637 7.e 7.1c1cc12e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T18:08:41.482 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:09.733301+0000", 2023-09-23T18:08:41.482 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 31.5911507, 2023-09-23T18:08:41.482 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75068017799999998, 2023-09-23T18:08:41.482 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.482 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 90637 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733301+0000", 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.483 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.484 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.484 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733301+0000", 2023-09-23T18:08:41.484 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.484 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.484 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.484 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.484 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733299+0000", 2023-09-23T18:08:41.484 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:08:41.484 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.485 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.485 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.485 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733310+0000", 2023-09-23T18:08:41.485 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0934000000000001e-05 2023-09-23T18:08:41.485 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.485 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.485 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.485 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733335+0000", 2023-09-23T18:08:41.485 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4669999999999999e-05 2023-09-23T18:08:41.485 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733347+0000", 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1987999999999999e-05 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733414+0000", 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7006000000000002e-05 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.486 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.487 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.487 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733425+0000", 2023-09-23T18:08:41.487 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1722999999999999e-05 2023-09-23T18:08:41.487 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.487 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.487 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.487 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:10.483970+0000", 2023-09-23T18:08:41.487 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75054469899999998 2023-09-23T18:08:41.487 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.487 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.488 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.488 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:10.483981+0000", 2023-09-23T18:08:41.488 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1402e-05 2023-09-23T18:08:41.488 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.488 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.488 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.488 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.488 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.488 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:90667 7.e 7.a9db13be (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T18:08:41.489 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:09.836569+0000", 2023-09-23T18:08:41.489 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 31.487882635999998, 2023-09-23T18:08:41.489 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.64744713499999995, 2023-09-23T18:08:41.489 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.489 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:08:41.489 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:08:41.489 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:08:41.489 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:41.489 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 90667 2023-09-23T18:08:41.490 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.490 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.490 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.490 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.490 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836569+0000", 2023-09-23T18:08:41.490 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.490 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.490 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.490 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.491 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836569+0000", 2023-09-23T18:08:41.491 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.491 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.491 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.491 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.491 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836568+0000", 2023-09-23T18:08:41.491 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:41.491 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.491 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.492 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.492 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836575+0000", 2023-09-23T18:08:41.492 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7150000000000001e-06 2023-09-23T18:08:41.492 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.492 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.492 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.492 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836589+0000", 2023-09-23T18:08:41.492 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4613999999999999e-05 2023-09-23T18:08:41.492 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.493 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.493 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.493 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836598+0000", 2023-09-23T18:08:41.493 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4859999999999997e-06 2023-09-23T18:08:41.493 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.493 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.493 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.493 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836663+0000", 2023-09-23T18:08:41.493 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5655999999999996e-05 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836679+0000", 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5449000000000001e-05 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:10.484006+0000", 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.64732701599999998 2023-09-23T18:08:41.494 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:10.484016+0000", 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0664e-05 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:108085 7.8 e340/331 7:101c9370:::benchmark_data_smithi055_311528_object37122:head v 340'5490, mlcod=339'5488)", 2023-09-23T18:08:41.495 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:39.569707+0000", 2023-09-23T18:08:41.496 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.754745392, 2023-09-23T18:08:41.496 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14917005999999999, 2023-09-23T18:08:41.496 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.496 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.496 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.496 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.496 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.496 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569707+0000", 2023-09-23T18:08:41.496 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.496 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.497 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.497 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569707+0000", 2023-09-23T18:08:41.497 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.497 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.497 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.497 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.497 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569704+0000", 2023-09-23T18:08:41.497 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:41.498 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.498 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.498 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569722+0000", 2023-09-23T18:08:41.498 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.747e-05 2023-09-23T18:08:41.498 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.498 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.498 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.498 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569744+0000", 2023-09-23T18:08:41.498 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1849e-05 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569756+0000", 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.185e-05 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.617599+0000", 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047843760999999999 2023-09-23T18:08:41.499 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.500 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.500 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.500 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.617637+0000", 2023-09-23T18:08:41.500 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7209999999999998e-05 2023-09-23T18:08:41.500 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.500 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.500 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.500 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.718825+0000", 2023-09-23T18:08:41.500 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.101188404 2023-09-23T18:08:41.501 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.501 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.501 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.718877+0000", 2023-09-23T18:08:41.501 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1604000000000003e-05 2023-09-23T18:08:41.501 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.501 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.501 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.501 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.501 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:108088 7.13 e340/331 7:c8e7817b:::benchmark_data_smithi055_311528_object37125:head v 340'5502, mlcod=339'5500)", 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:39.646388+0000", 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.6780643309999999, 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10060771, 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646388+0000", 2023-09-23T18:08:41.502 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646388+0000", 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646381+0000", 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999933 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.503 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646401+0000", 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0454000000000001e-05 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646421+0000", 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9341999999999999e-05 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646435+0000", 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.384e-05 2023-09-23T18:08:41.504 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.742550+0000", 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.096115361999999996 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.742574+0000", 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3747000000000002e-05 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.505 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.746950+0000", 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0043766830000000001 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.746995+0000", 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4990999999999997e-05 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:08:41.506 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:08:41.507 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:08:41.507 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:08:41.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:41.508 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:41.509 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:41.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:41.509 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:08:41.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:41.662 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: osdmap e341: 8 total, 8 up, 8 in 2023-09-23T18:08:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2224566483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:08:41.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[184174]: pgmap v842: 139 pgs: 2 peering, 1 remapped+peering, 5 unknown, 13 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 16547/103695 objects degraded (15.957%); 26 MiB/s, 422 objects/s recovering 2023-09-23T18:08:41.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: osdmap e341: 8 total, 8 up, 8 in 2023-09-23T18:08:41.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2224566483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:08:41.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:41 smithi055 ceph-mon[179668]: pgmap v842: 139 pgs: 2 peering, 1 remapped+peering, 5 unknown, 13 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 16547/103695 objects degraded (15.957%); 26 MiB/s, 422 objects/s recovering 2023-09-23T18:08:41.710 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:41.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:08:41.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:41.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:41.797 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.797 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: osdmap e341: 8 total, 8 up, 8 in 2023-09-23T18:08:41.797 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.797 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.797 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.797 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.797 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.798 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.798 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.798 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.798 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: failed to encode map e341 with expected crc 2023-09-23T18:08:41.798 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2224566483' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:08:41.798 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:41 smithi099 ceph-mon[167076]: pgmap v842: 139 pgs: 2 peering, 1 remapped+peering, 5 unknown, 13 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 997 op/s; 16547/103695 objects degraded (15.957%); 26 MiB/s, 422 objects/s recovering 2023-09-23T18:08:41.817 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:41.866 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:41.918 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:41.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:41.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:42.025 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:42.069 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:42.118 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:42.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:42.121 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:42.222 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:42.255 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:42.297 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:42.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:42.300 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:42.314 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:42.401 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:42.441 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:42.482 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:42.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:42.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:42.520 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:42 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:42.520 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:42 smithi055 ceph-mon[179668]: failed to encode map e342 with expected crc 2023-09-23T18:08:42.521 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:42 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2224566483' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:08:42.521 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:42 smithi055 ceph-mon[179668]: osdmap e342: 8 total, 8 up, 8 in 2023-09-23T18:08:42.521 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:42 smithi055 ceph-mon[179668]: failed to encode map e342 with expected crc 2023-09-23T18:08:42.521 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:42 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:42.521 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:42 smithi055 ceph-mon[184174]: failed to encode map e342 with expected crc 2023-09-23T18:08:42.521 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:42 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2224566483' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:08:42.522 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:42 smithi055 ceph-mon[184174]: osdmap e342: 8 total, 8 up, 8 in 2023-09-23T18:08:42.522 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:42 smithi055 ceph-mon[184174]: failed to encode map e342 with expected crc 2023-09-23T18:08:42.544 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:08:42.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:42.661 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:42.778 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:42.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:42.786 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:42.786 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:42 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:42.786 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:42 smithi099 ceph-mon[167076]: failed to encode map e342 with expected crc 2023-09-23T18:08:42.786 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:42 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2224566483' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:08:42.786 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:42 smithi099 ceph-mon[167076]: osdmap e342: 8 total, 8 up, 8 in 2023-09-23T18:08:42.786 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:42 smithi099 ceph-mon[167076]: failed to encode map e342 with expected crc 2023-09-23T18:08:42.886 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:42.942 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:43.082 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:43.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:43.089 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:43.098 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:08:43.098 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:08:43.099 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T18:08:43.099 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 0.05 2023-09-23T18:08:43.190 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:43.254 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:43.320 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:43.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:43.326 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:43.427 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:43.490 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:43.554 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:43.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:43.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:43.659 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:43 smithi055 ceph-mon[179668]: failed to encode map e343 with expected crc 2023-09-23T18:08:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:43 smithi055 ceph-mon[179668]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T18:08:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:43 smithi055 ceph-mon[179668]: pgmap v845: 139 pgs: 2 peering, 1 remapped+peering, 17 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 15318/100671 objects degraded (15.216%); 237/100671 objects misplaced (0.235%); 31 MiB/s, 493 objects/s recovering 2023-09-23T18:08:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16547/103695 objects degraded (15.957%), 16 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:43 smithi055 ceph-mon[184174]: failed to encode map e343 with expected crc 2023-09-23T18:08:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:43 smithi055 ceph-mon[184174]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T18:08:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:43 smithi055 ceph-mon[184174]: pgmap v845: 139 pgs: 2 peering, 1 remapped+peering, 17 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 15318/100671 objects degraded (15.216%); 237/100671 objects misplaced (0.235%); 31 MiB/s, 493 objects/s recovering 2023-09-23T18:08:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16547/103695 objects degraded (15.957%), 16 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:43.725 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:43.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:43 smithi099 ceph-mon[167076]: failed to encode map e343 with expected crc 2023-09-23T18:08:43.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:43 smithi099 ceph-mon[167076]: osdmap e343: 8 total, 8 up, 8 in 2023-09-23T18:08:43.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:43 smithi099 ceph-mon[167076]: pgmap v845: 139 pgs: 2 peering, 1 remapped+peering, 17 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 2.1 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 677 op/s; 15318/100671 objects degraded (15.216%); 237/100671 objects misplaced (0.235%); 31 MiB/s, 493 objects/s recovering 2023-09-23T18:08:43.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16547/103695 objects degraded (15.957%), 16 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:43.728 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:08:43.774 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:08:43.790 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:43.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:43.799 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:43.809 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:08:43.844 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:08:43.879 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:08:43.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:43.910 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:08:43.941 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:43.941 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:08:43.953 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:08:43.974 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:43.983 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:08:43.989 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:08:43.989 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:08:43.989 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:08:43.989 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:08:43.989 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:08:43.990 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:08:43.990 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:08:43.990 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:08:44.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:44.056 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:08:44.057 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:08:44.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:44.060 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:44.161 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:44.233 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:44.459 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:44.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:44.472 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:44.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:08:44.573 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:44.623 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:44.657 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/188431928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:44.658 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/188431928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:44.732 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:44.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:44.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/188431928' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:44.838 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:44.926 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:44.995 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:44.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:44.999 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:45.099 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:45.126 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:45.171 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:45.288 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:45.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:45.294 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:45.313 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:08:45.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:08:45.449 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:45.561 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:45 smithi055 ceph-mon[184174]: pgmap v846: 139 pgs: 2 peering, 1 remapped+peering, 17 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 15150/99645 objects degraded (15.204%); 237/99645 objects misplaced (0.238%); 25 MiB/s, 397 objects/s recovering 2023-09-23T18:08:45.562 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:45 smithi055 ceph-mon[179668]: pgmap v846: 139 pgs: 2 peering, 1 remapped+peering, 17 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 15150/99645 objects degraded (15.204%); 237/99645 objects misplaced (0.238%); 25 MiB/s, 397 objects/s recovering 2023-09-23T18:08:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:45 smithi099 ceph-mon[167076]: pgmap v846: 139 pgs: 2 peering, 1 remapped+peering, 17 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 2.0 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 0 B/s wr, 566 op/s; 15150/99645 objects degraded (15.204%); 237/99645 objects misplaced (0.238%); 25 MiB/s, 397 objects/s recovering 2023-09-23T18:08:45.864 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:08:45.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:45.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:08:45.973 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:46.037 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:46.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:46.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:46.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:46.420 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:46.469 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:08:46.486 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:46.525 INFO:teuthology.orchestra.run.smithi055.stdout:moved 0 / 417 (0%) 2023-09-23T18:08:46.525 INFO:teuthology.orchestra.run.smithi055.stdout:avg 52.125 2023-09-23T18:08:46.525 INFO:teuthology.orchestra.run.smithi055.stdout:stddev 19.8396 -> 19.8396 (expected baseline 6.75347) 2023-09-23T18:08:46.525 INFO:teuthology.orchestra.run.smithi055.stdout:min osd.0 with 21 -> 21 pgs (0.402878 -> 0.402878 * mean) 2023-09-23T18:08:46.525 INFO:teuthology.orchestra.run.smithi055.stdout:max osd.1 with 80 -> 80 pgs (1.53477 -> 1.53477 * mean) 2023-09-23T18:08:46.525 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:08:46.526 INFO:teuthology.orchestra.run.smithi055.stdout:oload 1000 2023-09-23T18:08:46.526 INFO:teuthology.orchestra.run.smithi055.stdout:max_change 0.05 2023-09-23T18:08:46.526 INFO:teuthology.orchestra.run.smithi055.stdout:max_change_osds 4 2023-09-23T18:08:46.526 INFO:teuthology.orchestra.run.smithi055.stdout:average_utilization 0.0151 2023-09-23T18:08:46.526 INFO:teuthology.orchestra.run.smithi055.stdout:overload_utilization 0.1514 2023-09-23T18:08:46.526 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4 weight 0.9787 -> 1.0000 2023-09-23T18:08:46.528 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:46 smithi099 ceph-mon[167076]: from='client.45815 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:46.529 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:46 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'65536'}"}]: dispatch 2023-09-23T18:08:46.571 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:46.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:46.579 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:46.679 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:46.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:46 smithi055 ceph-mon[179668]: from='client.45815 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:46.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:46 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'65536'}"}]: dispatch 2023-09-23T18:08:46.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:46 smithi055 ceph-mon[184174]: from='client.45815 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:46.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:46 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd reweightn", "weights": "{'4':'65536'}"}]: dispatch 2023-09-23T18:08:46.748 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:46.926 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:46.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:46.931 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:46.944 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T18:08:46.945 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 3.0 2023-09-23T18:08:47.032 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:47.082 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:47.125 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:47.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:47.128 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:47.196 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:47.222 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:08:47.222 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:08:47.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:47.263 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:47.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:47.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:47.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:47.463 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:47.493 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:47.550 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:08:47.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:08:47.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:47 smithi099 ceph-mon[167076]: failed to encode map e344 with expected crc 2023-09-23T18:08:47.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'65536'}"}]': finished 2023-09-23T18:08:47.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:47 smithi099 ceph-mon[167076]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T18:08:47.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:47 smithi099 ceph-mon[167076]: failed to encode map e344 with expected crc 2023-09-23T18:08:47.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:47 smithi099 ceph-mon[167076]: failed to encode map e344 with expected crc 2023-09-23T18:08:47.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:47 smithi099 ceph-mon[167076]: pgmap v848: 139 pgs: 18 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 15490/93387 objects degraded (16.587%); 237/93387 objects misplaced (0.254%); 21 MiB/s, 343 objects/s recovering 2023-09-23T18:08:47.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3882107907' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:47.860 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[184174]: failed to encode map e344 with expected crc 2023-09-23T18:08:47.860 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'65536'}"}]': finished 2023-09-23T18:08:47.860 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[184174]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T18:08:47.860 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[184174]: failed to encode map e344 with expected crc 2023-09-23T18:08:47.861 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[184174]: failed to encode map e344 with expected crc 2023-09-23T18:08:47.861 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[184174]: pgmap v848: 139 pgs: 18 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 15490/93387 objects degraded (16.587%); 237/93387 objects misplaced (0.254%); 21 MiB/s, 343 objects/s recovering 2023-09-23T18:08:47.861 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3882107907' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:47.861 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[179668]: failed to encode map e344 with expected crc 2023-09-23T18:08:47.861 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd reweightn", "weights": "{'4':'65536'}"}]': finished 2023-09-23T18:08:47.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[179668]: osdmap e344: 8 total, 8 up, 8 in 2023-09-23T18:08:47.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[179668]: failed to encode map e344 with expected crc 2023-09-23T18:08:47.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[179668]: failed to encode map e344 with expected crc 2023-09-23T18:08:47.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[179668]: pgmap v848: 139 pgs: 18 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 117 active+clean; 1.9 GiB data, 11 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 572 op/s; 15490/93387 objects degraded (16.587%); 237/93387 objects misplaced (0.254%); 21 MiB/s, 343 objects/s recovering 2023-09-23T18:08:47.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3882107907' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:08:48.098 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:48.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:48.103 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:48.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:48.282 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:48.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:48.354 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:48.355 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:48.404 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:08:48.456 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:48.541 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:48.590 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:48.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:48.596 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:48.697 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:48.708 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:48.769 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:08:48.770 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[184174]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.770 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[184174]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.770 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3882107907' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:48.770 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[184174]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T18:08:48.770 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[184174]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.770 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[184174]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.770 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15490/93387 objects degraded (16.587%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:48.770 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:08:48.771 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[179668]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.771 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[179668]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.771 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3882107907' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:48.771 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[179668]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T18:08:48.771 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[179668]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.771 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[179668]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.771 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15490/93387 objects degraded (16.587%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:48.792 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:48 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:08:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:48 smithi099 ceph-mon[167076]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:48 smithi099 ceph-mon[167076]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:48 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3882107907' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:08:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:48 smithi099 ceph-mon[167076]: osdmap e345: 8 total, 8 up, 8 in 2023-09-23T18:08:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:48 smithi099 ceph-mon[167076]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:48 smithi099 ceph-mon[167076]: failed to encode map e345 with expected crc 2023-09-23T18:08:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15490/93387 objects degraded (16.587%), 22 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:08:48.876 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:48.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:48.880 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:48.981 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:49.052 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:49.197 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:49.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:49.205 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:49.306 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:49.374 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:49.423 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:08:49.436 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:49.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:49.445 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:49.455 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:08:49.490 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:08:49.524 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:08:49.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:49.574 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:08:49.592 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:08:49.622 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:08:49.628 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:49.661 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:08:49.663 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:08:49.663 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:08:49.663 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:08:49.663 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:08:49.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:08:49.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:08:49.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:08:49.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:08:49.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:49 smithi055 ceph-mon[179668]: pgmap v850: 139 pgs: 17 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 14369/86286 objects degraded (16.653%); 31 MiB/s, 537 objects/s recovering 2023-09-23T18:08:49.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3164432608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:49.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:49 smithi055 ceph-mon[184174]: pgmap v850: 139 pgs: 17 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 14369/86286 objects degraded (16.653%); 31 MiB/s, 537 objects/s recovering 2023-09-23T18:08:49.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3164432608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:49.726 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:49.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:49.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:49.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:49 smithi099 ceph-mon[167076]: pgmap v850: 139 pgs: 17 active+recovery_wait+degraded, 3 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.8 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 912 op/s; 14369/86286 objects degraded (16.653%); 31 MiB/s, 537 objects/s recovering 2023-09-23T18:08:49.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3164432608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:49.835 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:49.850 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:08:49.851 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:08:49.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:08:49.851 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:08:49.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.851 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154240 5.e 5:7f5ef404:::benchmark_data_smithi055_229117_object48614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e174)", 2023-09-23T18:08:49.851 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.838192+0000", 2023-09-23T18:08:49.851 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 581.01041826599999, 2023-09-23T18:08:49.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2567066049999998, 2023-09-23T18:08:49.851 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.851 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154240 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838192+0000", 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838192+0000", 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838191+0000", 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838199+0000", 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1009999999999992e-06 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838217+0000", 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8629000000000001e-05 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838225+0000", 2023-09-23T18:08:49.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5510000000000002e-06 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838294+0000", 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9330000000000002e-05 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838426+0000", 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013151499999999999 2023-09-23T18:08:49.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:08:49.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.838546+0000", 2023-09-23T18:08:49.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120331 2023-09-23T18:08:49.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.843064+0000", 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004517928 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.845440+0000", 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00237587 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.862351+0000", 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016911302 2023-09-23T18:08:49.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.862359+0000", 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6399999999999997e-06 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.850984+0000", 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98862541999999998 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.850999+0000", 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4683e-05 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.094798+0000", 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.243799069 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.094899+0000", 2023-09-23T18:08:49.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100995 2023-09-23T18:08:49.861 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.861 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.861 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.861 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154257 5.e 5:7493d31c:::benchmark_data_smithi055_229117_object48631:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:08:49.861 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.879851+0000", 2023-09-23T18:08:49.861 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 580.96875949299999, 2023-09-23T18:08:49.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2859593629999999, 2023-09-23T18:08:49.862 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.862 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.862 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.862 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:08:49.862 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:49.862 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154257 2023-09-23T18:08:49.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.862 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879851+0000", 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879851+0000", 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879850+0000", 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879855+0000", 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5829999999999997e-06 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879867+0000", 2023-09-23T18:08:49.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1579e-05 2023-09-23T18:08:49.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879872+0000", 2023-09-23T18:08:49.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8509999999999998e-06 2023-09-23T18:08:49.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879916+0000", 2023-09-23T18:08:49.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4608999999999999e-05 2023-09-23T18:08:49.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.879924+0000", 2023-09-23T18:08:49.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8450000000000003e-06 2023-09-23T18:08:49.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851029+0000", 2023-09-23T18:08:49.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97110422200000002 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851036+0000", 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3780000000000003e-06 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.094924+0000", 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2438876830000001 2023-09-23T18:08:49.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.095069+0000", 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014521 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.095710+0000", 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00064142799999999999 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114512+0000", 2023-09-23T18:08:49.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018801345000000001 2023-09-23T18:08:49.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114543+0000", 2023-09-23T18:08:49.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1266000000000002e-05 2023-09-23T18:08:49.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114629+0000", 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6620999999999996e-05 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165607+0000", 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.050977816000000002 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165673+0000", 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5704000000000007e-05 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165810+0000", 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013739199999999999 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.872 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154261 5.e 5:75cfb822:::benchmark_data_smithi055_229117_object48635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.881204+0000", 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 580.96740671199996, 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2848199720000002, 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:08:49.873 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154261 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881204+0000", 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881204+0000", 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881203+0000", 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881208+0000", 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1699999999999996e-06 2023-09-23T18:08:49.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881221+0000", 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2809000000000001e-05 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881227+0000", 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7799999999999997e-06 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881298+0000", 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1403999999999996e-05 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.881305+0000", 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7789999999999998e-06 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851061+0000", 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96975605099999995 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.851068+0000", 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1060000000000002e-06 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.103490+0000", 2023-09-23T18:08:49.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2524223880000001 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.103624+0000", 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013401600000000001 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.103729+0000", 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010425900000000001 2023-09-23T18:08:49.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114577+0000", 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010848083999999999 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114604+0000", 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6886999999999999e-05 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114655+0000", 2023-09-23T18:08:49.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1703000000000003e-05 2023-09-23T18:08:49.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165828+0000", 2023-09-23T18:08:49.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.051172544 2023-09-23T18:08:49.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.165882+0000", 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4376000000000003e-05 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166024+0000", 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014146100000000001 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154266 5.e 5:70dac101:::benchmark_data_smithi055_229117_object48640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e175)", 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:08.888147+0000", 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 580.96046368700001, 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.278059764, 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:49.883 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154266 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888147+0000", 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888147+0000", 2023-09-23T18:08:49.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888146+0000", 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888152+0000", 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4119999999999998e-06 2023-09-23T18:08:49.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888162+0000", 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.679e-06 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888167+0000", 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.775e-06 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888242+0000", 2023-09-23T18:08:49.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5483999999999998e-05 2023-09-23T18:08:49.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:08.888250+0000", 2023-09-23T18:08:49.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.926e-06 2023-09-23T18:08:49.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852162+0000", 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.963912133 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852175+0000", 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2492e-05 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.107232+0000", 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2550570539999999 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.107391+0000", 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000158949 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.107551+0000", 2023-09-23T18:08:49.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0001605 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.114820+0000", 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007268666 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.115701+0000", 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00088042600000000002 2023-09-23T18:08:49.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.125929+0000", 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010228376000000001 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166033+0000", 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.040103685 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166081+0000", 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8899000000000001e-05 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.166207+0000", 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012513100000000001 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.892 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.893 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154238 5.f 5:fb40edf7:::benchmark_data_smithi055_229117_object48612:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:08:49.893 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:09.852458+0000", 2023-09-23T18:08:49.893 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 579.99615274400003, 2023-09-23T18:08:49.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3442088729999999, 2023-09-23T18:08:49.893 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.893 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:08:49.893 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.893 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154238 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852458+0000", 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852458+0000", 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852456+0000", 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852463+0000", 2023-09-23T18:08:49.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7150000000000001e-06 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852473+0000", 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0584000000000001e-05 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852481+0000", 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0229999999999992e-06 2023-09-23T18:08:49.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852600+0000", 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119177 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852610+0000", 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3980000000000005e-06 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.145648+0000", 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.29303853 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.145845+0000", 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019688400000000001 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.196602+0000", 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.050756905999999997 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.196667+0000", 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4517999999999995e-05 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.44681.0:154243 5.f 5:f7c86625:::benchmark_data_smithi055_229117_object48617:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e176)", 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T17:59:09.852496+0000", 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 579.99611453800003, 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.360016345, 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.44681", 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/2026121912", 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 154243 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.900 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852496+0000", 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852496+0000", 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852495+0000", 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852499+0000", 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5109999999999999e-06 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852506+0000", 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1690000000000003e-06 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852511+0000", 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.464e-06 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852635+0000", 2023-09-23T18:08:49.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123979 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:09.852642+0000", 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0350000000000004e-06 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.197777+0000", 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.345134979 2023-09-23T18:08:49.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.197894+0000", 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116935 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.212448+0000", 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014554414999999999 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T17:59:11.212512+0000", 2023-09-23T18:08:49.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4366e-05 2023-09-23T18:08:49.906 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.906 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.906 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.906 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33541 6.6 6:62c48c76:::benchmark_data_smithi055_275963_object33540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:08:49.906 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 430.39632098099997, 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0751333839999999, 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33541 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:08:49.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:08:49.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452289+0000", 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452325+0000", 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5848e-05 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452332+0000", 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5870000000000004e-06 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452337+0000", 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8189999999999999e-06 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453126+0000", 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00078863000000000002 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453133+0000", 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5630000000000003e-06 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454584+0000", 2023-09-23T18:08:49.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0014505659999999 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454710+0000", 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012631 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454859+0000", 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014862400000000001 2023-09-23T18:08:49.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460661+0000", 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058021690000000002 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.461337+0000", 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00067573600000000002 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527285+0000", 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.065948602999999995 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527333+0000", 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7329e-05 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527423+0000", 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0160000000000004e-05 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:99011 6.e 6:75351f29:::benchmark_data_smithi055_275963_object25873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:39.675417+0000", 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 310.17319315200001, 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1206789589999999, 2023-09-23T18:08:49.915 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 99011 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675417+0000", 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675417+0000", 2023-09-23T18:08:49.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675416+0000", 2023-09-23T18:08:49.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:08:49.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675423+0000", 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9330000000000004e-06 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675436+0000", 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2831e-05 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675441+0000", 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2240000000000003e-06 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.746546+0000", 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.071104629000000003 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.750257+0000", 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037111919999999999 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.793006+0000", 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.042748844000000001 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for rw locks", 2023-09-23T18:08:49.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.797725+0000", 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0047187669999999996 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.741846+0000", 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94412087 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.745207+0000", 2023-09-23T18:08:49.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003361298 2023-09-23T18:08:49.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,5", 2023-09-23T18:08:49.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.751189+0000", 2023-09-23T18:08:49.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0059818609999999998 2023-09-23T18:08:49.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.768299+0000", 2023-09-23T18:08:49.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017110071000000001 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.785459+0000", 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017160150999999998 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.786644+0000", 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011847050000000001 2023-09-23T18:08:49.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.788542+0000", 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018984799999999999 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.796096+0000", 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075542559999999996 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.924 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.925 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.925 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65460 7.4 7:2d53859f:::benchmark_data_smithi055_311528_object65459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:08:49.925 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:08:49.925 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 123.95220515600001, 2023-09-23T18:08:49.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9613000379999992, 2023-09-23T18:08:49.925 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.925 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:08:49.925 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:49.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:49.926 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65460 2023-09-23T18:08:49.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.926 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:08:49.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896404+0000", 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896620+0000", 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021651099999999999 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896638+0000", 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8179999999999999e-05 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896646+0000", 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2899999999999997e-06 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896718+0000", 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2587000000000001e-05 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896729+0000", 2023-09-23T18:08:49.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1223000000000001e-05 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.853134+0000", 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9564047319999993 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.857705+0000", 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045713259999999997 2023-09-23T18:08:49.930 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65469 7.4 7:2eb46e41:::benchmark_data_smithi055_311528_object65468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 123.951943505, 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 10.043735116000001, 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:08:49.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65469 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896666+0000", 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896742+0000", 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5432000000000006e-05 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896755+0000", 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3275000000000001e-05 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896762+0000", 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4760000000000003e-06 2023-09-23T18:08:49.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896860+0000", 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8120000000000002e-05 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896869+0000", 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1409999999999994e-06 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.858317+0000", 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9614477719999996 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.863986+0000", 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0056689319999999998 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940290+0000", 2023-09-23T18:08:49.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.076304636999999995 2023-09-23T18:08:49.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940402+0000", 2023-09-23T18:08:49.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111987 2023-09-23T18:08:49.937 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.937 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.937 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65477 7.4 7:23aaada8:::benchmark_data_smithi055_311528_object65476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 123.95183558399999, 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 10.046314513, 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:49.938 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65477 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896774+0000", 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896805+0000", 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0916e-05 2023-09-23T18:08:49.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896816+0000", 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0519999999999999e-05 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896820+0000", 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0849999999999999e-06 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896888+0000", 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8637e-05 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896897+0000", 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6279999999999994e-06 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.864614+0000", 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9677164650000005 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.871623+0000", 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070090059999999999 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.882195+0000", 2023-09-23T18:08:49.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010572560999999999 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.928966+0000", 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.046770763999999999 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940462+0000", 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011496312 2023-09-23T18:08:49.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940512+0000", 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0118999999999999e-05 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940552+0000", 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9535e-05 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943089+0000", 2023-09-23T18:08:49.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025376470000000001 2023-09-23T18:08:49.946 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.946 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.946 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.946 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65485 7.4 7:246beb72:::benchmark_data_smithi055_311528_object65484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:08:49.946 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:08:49.946 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 122.752444377, 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8502435909999999, 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65485 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096165+0000", 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096281+0000", 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116042 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096295+0000", 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4524e-05 2023-09-23T18:08:49.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096301+0000", 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7490000000000004e-06 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.889363+0000", 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7930615480000007 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.896004+0000", 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066411439999999999 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.906662+0000", 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010657645 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940400+0000", 2023-09-23T18:08:49.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.033738490000000003 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943179+0000", 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002778675 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943222+0000", 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2966999999999997e-05 2023-09-23T18:08:49.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943292+0000", 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0043999999999995e-05 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946410+0000", 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031180190000000001 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.953 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65486 7.4 7:28b4ee06:::benchmark_data_smithi055_311528_object65485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 122.727193898, 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8402319929999997, 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:49.954 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65486 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121415+0000", 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121466+0000", 2023-09-23T18:08:49.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1007000000000001e-05 2023-09-23T18:08:49.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121481+0000", 2023-09-23T18:08:49.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5287000000000001e-05 2023-09-23T18:08:49.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121489+0000", 2023-09-23T18:08:49.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5010000000000004e-06 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946571+0000", 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8250813830000006 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946747+0000", 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017653100000000001 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946947+0000", 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019961800000000001 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.954051+0000", 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0071042989999999997 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958525+0000", 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044738959999999998 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958638+0000", 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113078 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958698+0000", 2023-09-23T18:08:49.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0207000000000001e-05 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961649+0000", 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002950214 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65487 7.4 7:2b3c8c2f:::benchmark_data_smithi055_311528_object65486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:08:49.961 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 122.727109832, 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8433001999999998, 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65487 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121500+0000", 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121563+0000", 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3126000000000005e-05 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121579+0000", 2023-09-23T18:08:49.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6418000000000001e-05 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121585+0000", 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.575e-06 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949470+0000", 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8278846659999992 2023-09-23T18:08:49.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949632+0000", 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016232099999999999 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949768+0000", 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013578700000000001 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.956292+0000", 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065241910000000004 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958569+0000", 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022767289999999999 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961776+0000", 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032076660000000001 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961847+0000", 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1137999999999996e-05 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964801+0000", 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029533990000000002 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65498 7.4 7:27d4ee95:::benchmark_data_smithi055_311528_object65497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 122.692600318, 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8116268140000003, 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.969 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65498 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156009+0000", 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156082+0000", 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3093e-05 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156096+0000", 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4465e-05 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156104+0000", 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4120000000000004e-06 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.951850+0000", 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7957462389999996 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.951998+0000", 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014826300000000001 2023-09-23T18:08:49.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.952148+0000", 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000149524 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961673+0000", 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095253839999999996 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961735+0000", 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2036e-05 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964887+0000", 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003152092 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964960+0000", 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2601999999999994e-05 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.967637+0000", 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026772739999999999 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108071 7.21 7:85ca81b0:::benchmark_data_smithi055_311528_object37108:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 10.303985737, 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9624768260000001, 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:49.977 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108071 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544624+0000", 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544628+0000", 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.698e-06 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544634+0000", 2023-09-23T18:08:49.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3849999999999999e-06 2023-09-23T18:08:49.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544638+0000", 2023-09-23T18:08:49.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7560000000000001e-06 2023-09-23T18:08:49.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581968+0000", 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037330020999999998 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581975+0000", 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1470000000000004e-06 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487605+0000", 2023-09-23T18:08:49.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9056295090000002 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487615+0000", 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0036e-05 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502365+0000", 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0147500899999999 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502491+0000", 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125716 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502677+0000", 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018593700000000001 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504929+0000", 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022522219999999999 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504942+0000", 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2636999999999999e-05 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506803+0000", 2023-09-23T18:08:49.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018613709999999999 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506851+0000", 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8578000000000001e-05 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507102+0000", 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025014900000000002 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.985 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108080 7.21 7:85d5fb8e:::benchmark_data_smithi055_311528_object37117:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 10.30395427, 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.962840377, 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.986 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108080 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544658+0000", 2023-09-23T18:08:49.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9589999999999998e-06 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544663+0000", 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8069999999999999e-06 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544667+0000", 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1790000000000001e-06 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581992+0000", 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037325061999999999 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581997+0000", 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5480000000000003e-06 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487637+0000", 2023-09-23T18:08:49.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9056401429999998 2023-09-23T18:08:49.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:49.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487642+0000", 2023-09-23T18:08:49.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:08:49.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.503810+0000", 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.01616774 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.503955+0000", 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000145291 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504156+0000", 2023-09-23T18:08:49.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020058200000000001 2023-09-23T18:08:49.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507170+0000", 2023-09-23T18:08:49.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003013511 2023-09-23T18:08:49.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:49.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507182+0000", 2023-09-23T18:08:49.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.218e-05 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507234+0000", 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2349000000000003e-05 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507288+0000", 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4307e-05 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507497+0000", 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000208233 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108094 7.1 7:819d9c99:::benchmark_data_smithi055_311528_object37131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:08:49.995 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 10.080594788000001, 2023-09-23T18:08:49.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7316361810000001, 2023-09-23T18:08:49.996 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:49.996 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:49.996 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:49.996 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:49.996 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:49.996 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108094 2023-09-23T18:08:49.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.996 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:49.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:49.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:08:49.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:49.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:08:49.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:49.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:49.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768015+0000", 2023-09-23T18:08:49.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:08:49.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:49.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768019+0000", 2023-09-23T18:08:49.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3989999999999997e-06 2023-09-23T18:08:49.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768032+0000", 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2707e-05 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768037+0000", 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2490000000000001e-06 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:49.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:50.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768081+0000", 2023-09-23T18:08:50.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3421000000000003e-05 2023-09-23T18:08:50.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:50.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768093+0000", 2023-09-23T18:08:50.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1989e-05 2023-09-23T18:08:50.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.479295+0000", 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71120236800000003 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.479313+0000", 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8434e-05 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492409+0000", 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.013095748 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492585+0000", 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000176007 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:08:50.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492874+0000", 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00028859600000000001 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.497854+0000", 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049804380000000002 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499314+0000", 2023-09-23T18:08:50.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014593449999999999 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499349+0000", 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5802e-05 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499422+0000", 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2278000000000002e-05 2023-09-23T18:08:50.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499652+0000", 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00023028400000000001 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108097 7.21 7:87bce1cb:::benchmark_data_smithi055_311528_object37134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:08:50.005 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 9.3513537969999998, 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0105963070000001, 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108097 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497255+0000", 2023-09-23T18:08:50.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497262+0000", 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.029e-06 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497275+0000", 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3349e-05 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497284+0000", 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3639999999999999e-06 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497329+0000", 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5105000000000003e-05 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497335+0000", 2023-09-23T18:08:50.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8989999999999998e-06 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487661+0000", 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99032663499999996 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487667+0000", 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.446e-06 2023-09-23T18:08:50.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504977+0000", 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.017310079 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505119+0000", 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000142707 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:08:50.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505315+0000", 2023-09-23T18:08:50.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019584499999999999 2023-09-23T18:08:50.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:50.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507193+0000", 2023-09-23T18:08:50.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001877213 2023-09-23T18:08:50.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:50.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507527+0000", 2023-09-23T18:08:50.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00033465300000000002 2023-09-23T18:08:50.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:50.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507614+0000", 2023-09-23T18:08:50.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7287000000000006e-05 2023-09-23T18:08:50.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507647+0000", 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2481999999999998e-05 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507853+0000", 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000206092 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:50.014 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108108 7.21 7:860e36b6:::benchmark_data_smithi055_311528_object37145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 9.3003612180000008, 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.963000675, 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:50.015 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108108 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:50.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548248+0000", 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548257+0000", 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1549999999999996e-06 2023-09-23T18:08:50.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548278+0000", 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1229e-05 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548286+0000", 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0150000000000003e-06 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548337+0000", 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1131999999999998e-05 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548346+0000", 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8070000000000006e-06 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487685+0000", 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.939339378 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487691+0000", 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7100000000000004e-06 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505973+0000", 2023-09-23T18:08:50.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018282141 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506088+0000", 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011443299999999999 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506199+0000", 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011132 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507536+0000", 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013369 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507578+0000", 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1971000000000002e-05 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:50.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.510980+0000", 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034023870000000002 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.511029+0000", 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9064999999999999e-05 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.511250+0000", 2023-09-23T18:08:50.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022076600000000001 2023-09-23T18:08:50.024 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:50.024 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:50.024 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:50.024 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:50.024 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:50.024 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:08:50.025 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:50.025 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:50.025 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:50.026 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:50.127 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:50.195 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:50.208 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:08:50.269 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:50.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:50.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:50.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:08:50.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:50.460 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:50.521 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:50.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:50.526 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:50.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:50.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:08:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:08:50] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.43.0" 2023-09-23T18:08:50.702 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:50.774 INFO:teuthology.orchestra.run.smithi055.stdout:moved 21 / 417 (5.03597%) 2023-09-23T18:08:50.774 INFO:teuthology.orchestra.run.smithi055.stdout:avg 52.125 2023-09-23T18:08:50.774 INFO:teuthology.orchestra.run.smithi055.stdout:stddev 19.8396 -> 25.9684 (expected baseline 6.75347) 2023-09-23T18:08:50.774 INFO:teuthology.orchestra.run.smithi055.stdout:min osd.0 with 21 -> 8 pgs (0.402878 -> 0.153477 * mean) 2023-09-23T18:08:50.775 INFO:teuthology.orchestra.run.smithi055.stdout:max osd.1 with 80 -> 88 pgs (1.53477 -> 1.68825 * mean) 2023-09-23T18:08:50.777 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:08:50.777 INFO:teuthology.orchestra.run.smithi055.stdout:oload 110 2023-09-23T18:08:50.777 INFO:teuthology.orchestra.run.smithi055.stdout:max_change 3 2023-09-23T18:08:50.777 INFO:teuthology.orchestra.run.smithi055.stdout:max_change_osds 4 2023-09-23T18:08:50.777 INFO:teuthology.orchestra.run.smithi055.stdout:average_utilization 0.0144 2023-09-23T18:08:50.777 INFO:teuthology.orchestra.run.smithi055.stdout:overload_utilization 0.0158 2023-09-23T18:08:50.777 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 weight 0.5444 -> 0.3167 2023-09-23T18:08:50.778 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 weight 0.4893 -> 0.3428 2023-09-23T18:08:50.778 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:50.779 INFO:teuthology.orchestra.run.smithi055.stderr:no change 2023-09-23T18:08:50.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:50.788 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:50.889 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:50.945 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:51.107 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:08:51.115 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:08:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T18:08:50.771+0000 7f3f79c94700 -1 mgr.server reply reply (2) No such file or directory no change 2023-09-23T18:08:51.132 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:51.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:51.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:51.142 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:51.169 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '3.0', 'overage': '1000', 'type': 'test-reweight-by-utilization'} 2023-09-23T18:08:51.170 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 1000 3.0 2023-09-23T18:08:51.238 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:51.284 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:51.407 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:51.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:51.416 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:51.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:51.604 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:51.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:51 smithi099 ceph-mon[167076]: from='client.36057 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:51.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:51 smithi099 ceph-mon[167076]: pgmap v851: 139 pgs: 15 active+recovery_wait+degraded, 2 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 11975/79173 objects degraded (15.125%); 461/79173 objects misplaced (0.582%); 31 MiB/s, 524 objects/s recovering 2023-09-23T18:08:51.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:51.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:52.096 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:08:52.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:52.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:52.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:52.157 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:51 smithi055 ceph-mon[184174]: from='client.36057 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:52.157 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:51 smithi055 ceph-mon[184174]: pgmap v851: 139 pgs: 15 active+recovery_wait+degraded, 2 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 11975/79173 objects degraded (15.125%); 461/79173 objects misplaced (0.582%); 31 MiB/s, 524 objects/s recovering 2023-09-23T18:08:52.157 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:52.157 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:52.158 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:51 smithi055 ceph-mon[179668]: from='client.36057 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:52.158 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:51 smithi055 ceph-mon[179668]: pgmap v851: 139 pgs: 15 active+recovery_wait+degraded, 2 active+recovering, 3 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.6 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 0 B/s wr, 895 op/s; 11975/79173 objects degraded (15.125%); 461/79173 objects misplaced (0.582%); 31 MiB/s, 524 objects/s recovering 2023-09-23T18:08:52.159 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:08:52.159 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:08:52.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:52.261 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:52.299 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:52.350 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:52.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:52.358 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:52.459 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:52.528 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:52.590 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:52.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:52.595 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:52.696 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:52.788 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:53.421 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:53.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:53.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:53.533 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:53.576 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:53.705 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:53.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:53.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:53.764 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:53.816 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:53.883 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:53.965 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:08:53.981 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:53 smithi055 ceph-mon[184174]: pgmap v852: 139 pgs: 14 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 10238/73206 objects degraded (13.985%); 1467/73206 objects misplaced (2.004%); 35 MiB/s, 594 objects/s recovering 2023-09-23T18:08:53.981 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 10238/73206 objects degraded (13.985%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:08:53.981 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1170537167' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:08:53.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:53.982 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:08:53.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:53 smithi055 ceph-mon[179668]: pgmap v852: 139 pgs: 14 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 10238/73206 objects degraded (13.985%); 1467/73206 objects misplaced (2.004%); 35 MiB/s, 594 objects/s recovering 2023-09-23T18:08:53.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 10238/73206 objects degraded (13.985%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:08:53.983 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1170537167' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:08:54.084 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:54.107 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:08:54.154 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:54.156 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:53 smithi099 ceph-mon[167076]: pgmap v852: 139 pgs: 14 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.5 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.10k op/s; 10238/73206 objects degraded (13.985%); 1467/73206 objects misplaced (2.004%); 35 MiB/s, 594 objects/s recovering 2023-09-23T18:08:54.157 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 10238/73206 objects degraded (13.985%), 17 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:08:54.157 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1170537167' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:08:54.243 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:54.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:54.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:54.351 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:54.408 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:54.475 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:54.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:54.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:54.582 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:54.651 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:54.723 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:54.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:54.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:54.776 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:08:54.776 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:08:54.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.776 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:121550 7.e 7:71da2675:::benchmark_data_smithi055_311528_object50587:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-09-23T18:08:54.776 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:54.759098+0000", 2023-09-23T18:08:54.777 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.014550284, 2023-09-23T18:08:54.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014581691000000001, 2023-09-23T18:08:54.777 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:54.777 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:08:54.777 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:54.777 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:54.777 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:54.777 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121550 2023-09-23T18:08:54.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.777 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.759098+0000", 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.759098+0000", 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.759096+0000", 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.759101+0000", 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.741e-06 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:54.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.759114+0000", 2023-09-23T18:08:54.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2622999999999999e-05 2023-09-23T18:08:54.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:54.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.759119+0000", 2023-09-23T18:08:54.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7299999999999996e-06 2023-09-23T18:08:54.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.759162+0000", 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3282000000000003e-05 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.759794+0000", 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063178900000000001 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.767301+0000", 2023-09-23T18:08:54.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075070609999999998 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.767355+0000", 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4787000000000002e-05 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,2", 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.767634+0000", 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000278807 2023-09-23T18:08:54.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.771645+0000", 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040111729999999998 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.771664+0000", 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8318e-05 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:54.783 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.784 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:121562 7.20 7:07315108:::benchmark_data_smithi055_311528_object50599:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-09-23T18:08:54.784 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:54.768100+0000", 2023-09-23T18:08:54.784 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0055475849999999998, 2023-09-23T18:08:54.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005687966, 2023-09-23T18:08:54.784 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:54.784 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:08:54.784 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121562 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.768100+0000", 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.768100+0000", 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.768098+0000", 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:54.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.768106+0000", 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0709999999999995e-06 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.768122+0000", 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5736999999999999e-05 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.768130+0000", 2023-09-23T18:08:54.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8140000000000001e-06 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.768181+0000", 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0689999999999997e-05 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.768862+0000", 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00068166799999999997 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.788 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:121563 7.20 7:066c2d9a:::benchmark_data_smithi055_311528_object50600:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:54.771158+0000", 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0024899840000000002, 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027014949999999999, 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:54.789 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121563 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.771158+0000", 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:54.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.771158+0000", 2023-09-23T18:08:54.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:54.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.771156+0000", 2023-09-23T18:08:54.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:54.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.771163+0000", 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5089999999999999e-06 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.771177+0000", 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4521999999999999e-05 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.771183+0000", 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.541e-06 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.771239+0000", 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6159999999999998e-05 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:08:54.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773564+0000", 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023251610000000001 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "MOSDPGPushReply(7.20 346/340 [PushReplyOp(7:07315108:::benchmark_data_smithi055_311528_object50599:head)])", 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:54.773401+0000", 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.000246529, 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00052541399999999998, 2023-09-23T18:08:54.794 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773401+0000", 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773401+0000", 2023-09-23T18:08:54.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773401+0000", 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773411+0000", 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0169e-05 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773428+0000", 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7421000000000001e-05 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773435+0000", 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5379999999999999e-06 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:54.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773593+0000", 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015817399999999999 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "MOSDPGPushReply(7.10 346/316 [PushReplyOp(7:0bedbc74:::benchmark_data_smithi055_311528_object55763:head)])", 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:54.773445+0000", 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.00020287699999999999, 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00053450399999999999, 2023-09-23T18:08:54.798 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773445+0000", 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773445+0000", 2023-09-23T18:08:54.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:54.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773445+0000", 2023-09-23T18:08:54.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:54.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:54.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773448+0000", 2023-09-23T18:08:54.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7829999999999999e-06 2023-09-23T18:08:54.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:54.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773454+0000", 2023-09-23T18:08:54.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9479999999999996e-06 2023-09-23T18:08:54.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773459+0000", 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4709999999999998e-06 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:121569 7.a 7:517f10aa:::benchmark_data_smithi055_311528_object50606:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:54.773590+0000", 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 5.8072999999999998e-05, 2023-09-23T18:08:54.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00043924900000000001, 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121569 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:54.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773590+0000", 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773590+0000", 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773588+0000", 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:08:54.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773595+0000", 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0339999999999999e-06 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773608+0000", 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2387e-05 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773614+0000", 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5570000000000001e-06 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773679+0000", 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4430999999999993e-05 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:54.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773790+0000", 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110696 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:121570 7.0 7.895983c0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e346)", 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:54.773628+0000", 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 1.9695e-05, 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000464974, 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:54.807 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121570 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773628+0000", 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773628+0000", 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773628+0000", 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:54.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773631+0000", 2023-09-23T18:08:54.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8389999999999998e-06 2023-09-23T18:08:54.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:54.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773637+0000", 2023-09-23T18:08:54.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0850000000000002e-06 2023-09-23T18:08:54.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773641+0000", 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9559999999999999e-06 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.45557.0:121550 7.e e346/340 ondisk, result = 0)", 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:54.773660+0000", 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 4294967295.9999881, 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00048630599999999999, 2023-09-23T18:08:54.811 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773660+0000", 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773660+0000", 2023-09-23T18:08:54.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773659+0000", 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773664+0000", 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4429999999999996e-06 2023-09-23T18:08:54.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:08:54.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773675+0000", 2023-09-23T18:08:54.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0882e-05 2023-09-23T18:08:54.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:08:54.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773681+0000", 2023-09-23T18:08:54.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5030000000000002e-06 2023-09-23T18:08:54.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773852+0000", 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000170908 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:54.773861+0000", 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5720000000000003e-06 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.815 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:08:54.816 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.816 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:08:54.816 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:08:54.816 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 8 2023-09-23T18:08:54.816 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:08:54.827 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:54.900 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:55.013 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:55.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:55.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:55.054 INFO:teuthology.orchestra.run.smithi055.stdout:moved 1 / 417 (0.239808%) 2023-09-23T18:08:55.054 INFO:teuthology.orchestra.run.smithi055.stdout:avg 52.125 2023-09-23T18:08:55.054 INFO:teuthology.orchestra.run.smithi055.stdout:stddev 19.8396 -> 19.7765 (expected baseline 6.75347) 2023-09-23T18:08:55.054 INFO:teuthology.orchestra.run.smithi055.stdout:min osd.0 with 21 -> 21 pgs (0.402878 -> 0.402878 * mean) 2023-09-23T18:08:55.054 INFO:teuthology.orchestra.run.smithi055.stdout:max osd.1 with 80 -> 80 pgs (1.53477 -> 1.53477 * mean) 2023-09-23T18:08:55.054 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:08:55.054 INFO:teuthology.orchestra.run.smithi055.stdout:oload 1000 2023-09-23T18:08:55.054 INFO:teuthology.orchestra.run.smithi055.stdout:max_change 3 2023-09-23T18:08:55.055 INFO:teuthology.orchestra.run.smithi055.stdout:max_change_osds 4 2023-09-23T18:08:55.055 INFO:teuthology.orchestra.run.smithi055.stdout:average_utilization 0.0132 2023-09-23T18:08:55.055 INFO:teuthology.orchestra.run.smithi055.stdout:overload_utilization 0.1322 2023-09-23T18:08:55.055 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 weight 0.6924 -> 0.7169 2023-09-23T18:08:55.069 INFO:teuthology.orchestra.run.smithi055.stderr:no change 2023-09-23T18:08:55.123 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:55.181 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:08:55 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T18:08:55.045+0000 7f3f79c94700 -1 mgr.server reply reply (1) Operation not permitted no change 2023-09-23T18:08:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:54 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1170537167' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:08:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:54 smithi055 ceph-mon[184174]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T18:08:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:54 smithi055 ceph-mon[184174]: failed to encode map e346 with expected crc 2023-09-23T18:08:55.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:54 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:55.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1170537167' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:08:55.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:54 smithi055 ceph-mon[179668]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T18:08:55.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:54 smithi055 ceph-mon[179668]: failed to encode map e346 with expected crc 2023-09-23T18:08:55.191 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:54 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:08:55.192 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1170537167' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:08:55.192 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:54 smithi099 ceph-mon[167076]: osdmap e346: 8 total, 8 up, 8 in 2023-09-23T18:08:55.192 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:54 smithi099 ceph-mon[167076]: failed to encode map e346 with expected crc 2023-09-23T18:08:55.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:08:55.210 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:55.417 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:55.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:55.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:55.526 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:55.564 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '0.05', 'overage': '110', 'type': 'test-reweight-by-utilization'} 2023-09-23T18:08:55.564 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd test-reweight-by-utilization 110 0.05 2023-09-23T18:08:55.592 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:55.650 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:55.657 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:55.658 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:55.759 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:55.877 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:55.926 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:55 smithi099 ceph-mon[167076]: failed to encode map e346 with expected crc 2023-09-23T18:08:55.926 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:55 smithi099 ceph-mon[167076]: pgmap v854: 139 pgs: 14 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 9900/71184 objects degraded (13.908%); 1467/71184 objects misplaced (2.061%); 31 MiB/s, 524 objects/s recovering 2023-09-23T18:08:55.926 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:55 smithi099 ceph-mon[167076]: from='client.36069 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:55.959 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:55.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:55.967 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:56.007 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:08:56.019 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:08:56.047 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:08:56.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:56.090 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:08:56.118 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:56.129 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:08:56.167 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:08:56.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:55 smithi055 ceph-mon[179668]: failed to encode map e346 with expected crc 2023-09-23T18:08:56.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:55 smithi055 ceph-mon[179668]: pgmap v854: 139 pgs: 14 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 9900/71184 objects degraded (13.908%); 1467/71184 objects misplaced (2.061%); 31 MiB/s, 524 objects/s recovering 2023-09-23T18:08:56.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:55 smithi055 ceph-mon[179668]: from='client.36069 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:56.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:55 smithi055 ceph-mon[184174]: failed to encode map e346 with expected crc 2023-09-23T18:08:56.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:55 smithi055 ceph-mon[184174]: pgmap v854: 139 pgs: 14 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 3 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 119 active+clean; 1.5 GiB data, 9.5 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 924 op/s; 9900/71184 objects degraded (13.908%); 1467/71184 objects misplaced (2.061%); 31 MiB/s, 524 objects/s recovering 2023-09-23T18:08:56.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:55 smithi055 ceph-mon[184174]: from='client.36069 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 1000, "max_change": 3.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:56.182 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:56.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:56.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:56.203 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:08:56.245 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:08:56.270 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:08:56.276 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:08:56.276 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:08:56.276 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:08:56.276 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:08:56.276 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:08:56.276 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:08:56.276 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:08:56.276 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:08:56.292 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:56.334 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:56.353 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:56.416 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:56.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:56.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:56.527 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:56.595 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:56.920 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:56.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:56.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:57.029 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:08:57.030 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:08:57.081 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:57.141 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:08:57.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:57.146 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/646832829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:57.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:08:57.152 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/646832829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:57.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/646832829' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:08:57.247 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:08:57.302 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:57.366 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:08:57.375 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:57.375 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:08:57.476 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:57.531 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:57.555 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:08:57.555 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:08:57.672 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:57.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:57.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:57.779 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:08:57.855 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:57.922 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:08:57.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:57.935 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:08:57.986 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:08:58.036 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:08:58.085 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:58.113 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:08:58.148 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:08:58.153 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:57 smithi055 ceph-mon[184174]: pgmap v855: 139 pgs: 13 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 8933/63018 objects degraded (14.175%); 1006/63018 objects misplaced (1.596%); 35 MiB/s, 585 objects/s recovering 2023-09-23T18:08:58.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:57 smithi055 ceph-mon[179668]: pgmap v855: 139 pgs: 13 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 8933/63018 objects degraded (14.175%); 1006/63018 objects misplaced (1.596%); 35 MiB/s, 585 objects/s recovering 2023-09-23T18:08:58.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:58.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:08:58.255 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:58.310 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:58.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:57 smithi099 ceph-mon[167076]: pgmap v855: 139 pgs: 13 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 121 active+clean; 1.3 GiB data, 9.2 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 8933/63018 objects degraded (14.175%); 1006/63018 objects misplaced (1.596%); 35 MiB/s, 585 objects/s recovering 2023-09-23T18:08:58.343 INFO:teuthology.orchestra.run.smithi055.stdout:moved 6 / 417 (1.43885%) 2023-09-23T18:08:58.343 INFO:teuthology.orchestra.run.smithi055.stdout:avg 52.125 2023-09-23T18:08:58.343 INFO:teuthology.orchestra.run.smithi055.stdout:stddev 19.8396 -> 21.6762 (expected baseline 6.75347) 2023-09-23T18:08:58.343 INFO:teuthology.orchestra.run.smithi055.stdout:min osd.0 with 21 -> 16 pgs (0.402878 -> 0.306954 * mean) 2023-09-23T18:08:58.343 INFO:teuthology.orchestra.run.smithi055.stdout:max osd.1 with 80 -> 83 pgs (1.53477 -> 1.59233 * mean) 2023-09-23T18:08:58.343 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:08:58.343 INFO:teuthology.orchestra.run.smithi055.stdout:oload 110 2023-09-23T18:08:58.344 INFO:teuthology.orchestra.run.smithi055.stdout:max_change 0.05 2023-09-23T18:08:58.344 INFO:teuthology.orchestra.run.smithi055.stdout:max_change_osds 4 2023-09-23T18:08:58.344 INFO:teuthology.orchestra.run.smithi055.stdout:average_utilization 0.0129 2023-09-23T18:08:58.344 INFO:teuthology.orchestra.run.smithi055.stdout:overload_utilization 0.0141 2023-09-23T18:08:58.344 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0 weight 0.5444 -> 0.4944 2023-09-23T18:08:58.344 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3 weight 0.4893 -> 0.4393 2023-09-23T18:08:58.344 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2 weight 0.6924 -> 0.6971 2023-09-23T18:08:58.373 INFO:teuthology.orchestra.run.smithi055.stderr:no change 2023-09-23T18:08:58.383 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:58.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:58.388 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:58.436 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:08:58 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T18:08:58.342+0000 7f3f79c94700 -1 mgr.server reply reply (3) No such process no change 2023-09-23T18:08:58.489 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:08:58.522 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:58.725 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:08:58.799 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:08:59.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 8933/63018 objects degraded (14.175%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:08:59.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:08:58 smithi055 ceph-mon[184174]: from='client.36078 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:59.093 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 8933/63018 objects degraded (14.175%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:08:59.094 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:08:58 smithi055 ceph-mon[179668]: from='client.36078 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:59.275 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 8933/63018 objects degraded (14.175%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:08:59.275 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:08:58 smithi099 ceph-mon[167076]: from='client.36078 -' entity='client.admin' cmd=[{"prefix": "osd test-reweight-by-utilization", "oload": 110, "max_change": 0.05, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:08:59.368 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:08:59.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:59.373 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:08:59.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:08:59.522 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:08:59.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:08:59.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:08:59.656 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:08:59.725 INFO:tasks.thrashosds.thrasher:Reweighting by: {'max_change': '1.0', 'overage': '1000', 'type': 'reweight-by-utilization'} 2023-09-23T18:08:59.725 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight-by-utilization 1000 1.0 2023-09-23T18:08:59.757 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:08:59.817 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:00.097 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:00.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:00.103 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:00.204 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:00.259 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:00.268 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:00.281 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:09:00.281 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:09:00.306 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:00 smithi099 ceph-mon[167076]: pgmap v856: 139 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 8058/54891 objects degraded (14.680%); 4/54891 objects misplaced (0.007%); 37 MiB/s, 596 objects/s recovering 2023-09-23T18:09:00.359 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:09:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:09:00] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.43.0" 2023-09-23T18:09:00.360 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:00 smithi055 ceph-mon[179668]: pgmap v856: 139 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 8058/54891 objects degraded (14.680%); 4/54891 objects misplaced (0.007%); 37 MiB/s, 596 objects/s recovering 2023-09-23T18:09:00.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:00 smithi055 ceph-mon[184174]: pgmap v856: 139 pgs: 13 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 1.1 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 8058/54891 objects degraded (14.680%); 4/54891 objects misplaced (0.007%); 37 MiB/s, 596 objects/s recovering 2023-09-23T18:09:00.466 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:00.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:00.473 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:00.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:00.646 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:00.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:09:00.705 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:00.712 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:00.713 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:00.814 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:00.893 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:00.906 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:01.182 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:01.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:01.188 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:01.289 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:01.340 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:01.455 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:09:01.474 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:01.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:01.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:01.582 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:01.638 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:01.710 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:01.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:01.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:01.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:01.889 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:01.902 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:09:01.935 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:09:01.964 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:01.965 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:09:01.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:01.973 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:02.001 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:09:02.035 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:09:02.067 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:09:02.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:02.106 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:09:02.125 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:02.132 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:09:02.149 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:09:02.161 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:09:02.161 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:09:02.161 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:09:02.161 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:09:02.162 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:09:02.162 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:09:02.162 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:09:02.162 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:09:02.287 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:02.293 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:02 smithi099 ceph-mon[167076]: pgmap v857: 139 pgs: 13 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 970 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 7162/46452 objects degraded (15.418%); 4/46452 objects misplaced (0.009%); 31 MiB/s, 490 objects/s recovering 2023-09-23T18:09:02.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/379022957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:02.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:02 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:02.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4116018480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:02.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:02.295 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:02.396 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:02.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:02 smithi055 ceph-mon[184174]: pgmap v857: 139 pgs: 13 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 970 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 7162/46452 objects degraded (15.418%); 4/46452 objects misplaced (0.009%); 31 MiB/s, 490 objects/s recovering 2023-09-23T18:09:02.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/379022957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:02.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:02 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:02.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4116018480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:02 smithi055 ceph-mon[179668]: pgmap v857: 139 pgs: 13 active+recovery_wait+degraded, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 970 MiB data, 8.3 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.09k op/s; 7162/46452 objects degraded (15.418%); 4/46452 objects misplaced (0.009%); 31 MiB/s, 490 objects/s recovering 2023-09-23T18:09:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/379022957' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:02 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:02.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4116018480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:02.469 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:02.692 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:02.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:02.698 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:02.798 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:02.855 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:02.915 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:02.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:02.921 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:02.968 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:09:02.968 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:09:02.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.968 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:130655 7.c 7:355f0497:::benchmark_data_smithi055_311528_object59692:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2023-09-23T18:09:02.968 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:02.945839+0000", 2023-09-23T18:09:02.969 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.021231943999999999, 2023-09-23T18:09:02.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021261265000000001, 2023-09-23T18:09:02.969 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:09:02.969 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:09:02.969 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:09:02.969 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:09:02.969 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:09:02.969 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 130655 2023-09-23T18:09:02.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.969 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:09:02.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:09:02.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.945839+0000", 2023-09-23T18:09:02.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:02.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:09:02.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.945839+0000", 2023-09-23T18:09:02.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:02.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:09:02.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.945838+0000", 2023-09-23T18:09:02.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:09:02.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:09:02.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.945842+0000", 2023-09-23T18:09:02.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.692e-06 2023-09-23T18:09:02.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:09:02.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.945852+0000", 2023-09-23T18:09:02.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9429999999999995e-06 2023-09-23T18:09:02.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:09:02.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.945857+0000", 2023-09-23T18:09:02.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6500000000000001e-06 2023-09-23T18:09:02.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:09:02.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.948537+0000", 2023-09-23T18:09:02.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026796789999999999 2023-09-23T18:09:02.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:09:02.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.949211+0000", 2023-09-23T18:09:02.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000674419 2023-09-23T18:09:02.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:09:02.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.960146+0000", 2023-09-23T18:09:02.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010934569999999999 2023-09-23T18:09:02.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:09:02.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.960237+0000", 2023-09-23T18:09:02.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1273999999999994e-05 2023-09-23T18:09:02.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:09:02.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.960608+0000", 2023-09-23T18:09:02.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000371202 2023-09-23T18:09:02.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:09:02.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964367+0000", 2023-09-23T18:09:02.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003758431 2023-09-23T18:09:02.976 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:02.976 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:09:02.977 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:02.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.977 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:130664 7.1c 7:3ede4898:::benchmark_data_smithi055_311528_object59701:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2023-09-23T18:09:02.977 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:02.951004+0000", 2023-09-23T18:09:02.977 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.016066649999999998, 2023-09-23T18:09:02.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016194938999999998, 2023-09-23T18:09:02.977 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:09:02.978 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:09:02.978 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:09:02.978 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:09:02.978 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:09:02.978 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 130664 2023-09-23T18:09:02.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.978 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:09:02.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:09:02.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.951004+0000", 2023-09-23T18:09:02.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:02.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:09:02.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.951004+0000", 2023-09-23T18:09:02.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:02.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:09:02.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.951003+0000", 2023-09-23T18:09:02.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:09:02.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:09:02.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.951009+0000", 2023-09-23T18:09:02.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.981e-06 2023-09-23T18:09:02.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:09:02.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.951023+0000", 2023-09-23T18:09:02.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4473e-05 2023-09-23T18:09:02.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:09:02.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.951030+0000", 2023-09-23T18:09:02.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5880000000000003e-06 2023-09-23T18:09:02.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:09:02.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.951070+0000", 2023-09-23T18:09:02.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0447000000000001e-05 2023-09-23T18:09:02.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:09:02.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.953349+0000", 2023-09-23T18:09:02.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002279316 2023-09-23T18:09:02.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:09:02.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.958076+0000", 2023-09-23T18:09:02.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0047261919999999997 2023-09-23T18:09:02.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:09:02.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.958168+0000", 2023-09-23T18:09:02.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2184999999999999e-05 2023-09-23T18:09:02.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:09:02.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.958507+0000", 2023-09-23T18:09:02.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00033949199999999999 2023-09-23T18:09:02.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:09:02.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964364+0000", 2023-09-23T18:09:02.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058564649999999999 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964400+0000", 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6514999999999999e-05 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.985 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:130668 7.c 7:30546c14:::benchmark_data_smithi055_311528_object59705:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e347)", 2023-09-23T18:09:02.986 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:02.962232+0000", 2023-09-23T18:09:02.986 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0048385240000000003, 2023-09-23T18:09:02.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0050630179999999999, 2023-09-23T18:09:02.986 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:09:02.986 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:09:02.986 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:09:02.986 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:09:02.986 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:09:02.986 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 130668 2023-09-23T18:09:02.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.987 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:09:02.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:09:02.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.962232+0000", 2023-09-23T18:09:02.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:02.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:09:02.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.962232+0000", 2023-09-23T18:09:02.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:02.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:09:02.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.962231+0000", 2023-09-23T18:09:02.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:09:02.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:09:02.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.962238+0000", 2023-09-23T18:09:02.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3830000000000003e-06 2023-09-23T18:09:02.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:09:02.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.962251+0000", 2023-09-23T18:09:02.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3037000000000001e-05 2023-09-23T18:09:02.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:09:02.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.962258+0000", 2023-09-23T18:09:02.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6819999999999997e-06 2023-09-23T18:09:02.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:09:02.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964390+0000", 2023-09-23T18:09:02.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021327239999999999 2023-09-23T18:09:02.991 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:02.991 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:09:02.991 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:02.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.991 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.45557.0:130655 7.c e347/341 ondisk, result = 0)", 2023-09-23T18:09:02.991 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:02.963047+0000", 2023-09-23T18:09:02.991 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0040236209999999998, 2023-09-23T18:09:02.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0043067169999999998, 2023-09-23T18:09:02.991 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:09:02.992 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:09:02.992 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:09:02.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:09:02.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.963047+0000", 2023-09-23T18:09:02.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:02.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:09:02.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.963047+0000", 2023-09-23T18:09:02.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:02.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:09:02.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.963046+0000", 2023-09-23T18:09:02.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:09:02.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:09:02.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.963051+0000", 2023-09-23T18:09:02.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8359999999999999e-06 2023-09-23T18:09:02.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:09:02.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.963063+0000", 2023-09-23T18:09:02.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1793e-05 2023-09-23T18:09:02.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:09:02.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.963068+0000", 2023-09-23T18:09:02.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4480000000000002e-06 2023-09-23T18:09:02.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:09:02.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964354+0000", 2023-09-23T18:09:02.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001285149 2023-09-23T18:09:02.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:09:02.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964360+0000", 2023-09-23T18:09:02.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.64e-06 2023-09-23T18:09:02.996 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:02.996 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:09:02.996 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:02.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.997 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.45557.0:130655 7.c e347/341)", 2023-09-23T18:09:02.997 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:02.964551+0000", 2023-09-23T18:09:02.997 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0025196950000000002, 2023-09-23T18:09:02.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00285768, 2023-09-23T18:09:02.997 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:09:02.997 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:09:02.997 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:09:02.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:09:02.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964551+0000", 2023-09-23T18:09:02.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:02.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:09:02.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964551+0000", 2023-09-23T18:09:02.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:02.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:09:02.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964550+0000", 2023-09-23T18:09:02.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:09:02.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:02.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:02.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:09:02.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964564+0000", 2023-09-23T18:09:02.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.379e-05 2023-09-23T18:09:02.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:09:03.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964576+0000", 2023-09-23T18:09:03.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1464999999999999e-05 2023-09-23T18:09:03.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:09:03.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964581+0000", 2023-09-23T18:09:03.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8629999999999996e-06 2023-09-23T18:09:03.000 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.001 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:09:03.001 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.001 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.45557.0:130664 7.1c e347/340)", 2023-09-23T18:09:03.001 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:02.964590+0000", 2023-09-23T18:09:03.001 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0024804150000000001, 2023-09-23T18:09:03.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002862477, 2023-09-23T18:09:03.001 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:09:03.001 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:09:03.002 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:09:03.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:09:03.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964590+0000", 2023-09-23T18:09:03.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:03.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:09:03.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964590+0000", 2023-09-23T18:09:03.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:03.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:09:03.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964590+0000", 2023-09-23T18:09:03.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:09:03.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:09:03.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964593+0000", 2023-09-23T18:09:03.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.723e-06 2023-09-23T18:09:03.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:09:03.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964599+0000", 2023-09-23T18:09:03.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9610000000000002e-06 2023-09-23T18:09:03.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:09:03.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964602+0000", 2023-09-23T18:09:03.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.326e-06 2023-09-23T18:09:03.005 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.005 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:09:03.005 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.005 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45557.0:130678 7.11 e347/316 7:88d83827:::benchmark_data_smithi055_311528_object59715:head v 347'6159, mlcod=347'6157)", 2023-09-23T18:09:03.005 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:02.964696+0000", 2023-09-23T18:09:03.006 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0023745070000000001, 2023-09-23T18:09:03.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002804946, 2023-09-23T18:09:03.006 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:09:03.006 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:09:03.006 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:09:03.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:09:03.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964696+0000", 2023-09-23T18:09:03.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:03.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:09:03.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964696+0000", 2023-09-23T18:09:03.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:03.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:09:03.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964695+0000", 2023-09-23T18:09:03.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:09:03.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:09:03.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964702+0000", 2023-09-23T18:09:03.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9770000000000003e-06 2023-09-23T18:09:03.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:09:03.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964712+0000", 2023-09-23T18:09:03.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3249999999999997e-06 2023-09-23T18:09:03.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:09:03.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964717+0000", 2023-09-23T18:09:03.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6459999999999998e-06 2023-09-23T18:09:03.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:09:03.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964762+0000", 2023-09-23T18:09:03.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4651000000000003e-05 2023-09-23T18:09:03.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:09:03.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964780+0000", 2023-09-23T18:09:03.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8029999999999998e-05 2023-09-23T18:09:03.010 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.010 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:09:03.011 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.011 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45557.0:130673 7.b e347/341 7:d793c0ac:::benchmark_data_smithi055_311528_object59710:head v 347'7510, mlcod=322'4597)", 2023-09-23T18:09:03.011 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:02.964890+0000", 2023-09-23T18:09:03.011 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0021808029999999998, 2023-09-23T18:09:03.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026847799999999999, 2023-09-23T18:09:03.011 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:09:03.011 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:09:03.011 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:09:03.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:09:03.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964890+0000", 2023-09-23T18:09:03.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:03.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:09:03.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964890+0000", 2023-09-23T18:09:03.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:03.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:09:03.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964889+0000", 2023-09-23T18:09:03.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:09:03.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:09:03.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964895+0000", 2023-09-23T18:09:03.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8220000000000003e-06 2023-09-23T18:09:03.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:09:03.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964903+0000", 2023-09-23T18:09:03.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4589999999999998e-06 2023-09-23T18:09:03.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:09:03.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964908+0000", 2023-09-23T18:09:03.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7299999999999996e-06 2023-09-23T18:09:03.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:09:03.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964943+0000", 2023-09-23T18:09:03.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5178000000000001e-05 2023-09-23T18:09:03.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:09:03.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.964963+0000", 2023-09-23T18:09:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9789999999999999e-05 2023-09-23T18:09:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:09:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45557.0:130662 7.4 e347/317)", 2023-09-23T18:09:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:02.965800+0000", 2023-09-23T18:09:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0012710099999999999, 2023-09-23T18:09:03.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001847439, 2023-09-23T18:09:03.017 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:09:03.017 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:09:03.017 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:09:03.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:09:03.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.965800+0000", 2023-09-23T18:09:03.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:03.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:09:03.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.965800+0000", 2023-09-23T18:09:03.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:03.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:09:03.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.965799+0000", 2023-09-23T18:09:03.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:09:03.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:09:03.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.965806+0000", 2023-09-23T18:09:03.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0679999999999999e-06 2023-09-23T18:09:03.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:09:03.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.965816+0000", 2023-09-23T18:09:03.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0308e-05 2023-09-23T18:09:03.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:09:03.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.965822+0000", 2023-09-23T18:09:03.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.733e-06 2023-09-23T18:09:03.020 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.020 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:09:03.020 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.020 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45557.0:130676 7.b e347/341)", 2023-09-23T18:09:03.020 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:02.967634+0000", 2023-09-23T18:09:03.021 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 4294967295.9994364, 2023-09-23T18:09:03.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7012000000000001e-05, 2023-09-23T18:09:03.021 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:09:03.021 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:09:03.021 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:09:03.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:09:03.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.967634+0000", 2023-09-23T18:09:03.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:03.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:09:03.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.967634+0000", 2023-09-23T18:09:03.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:09:03.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:09:03.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.967634+0000", 2023-09-23T18:09:03.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:09:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:09:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.967641+0000", 2023-09-23T18:09:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1400000000000002e-06 2023-09-23T18:09:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:09:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.967657+0000", 2023-09-23T18:09:03.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5906e-05 2023-09-23T18:09:03.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:09:03.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:09:03.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:09:03.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:02.967662+0000", 2023-09-23T18:09:03.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6570000000000002e-06 2023-09-23T18:09:03.024 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.024 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:09:03.024 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.024 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:09:03.024 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:09:03.025 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 10 2023-09-23T18:09:03.025 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:09:03.025 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:03.106 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:03.173 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:03.177 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:03.178 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:03.279 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:03.307 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:03 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:03.307 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:03 smithi099 ceph-mon[167076]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.308 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:03 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:03.308 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:03 smithi099 ceph-mon[167076]: osdmap e347: 8 total, 8 up, 8 in 2023-09-23T18:09:03.308 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:03 smithi099 ceph-mon[167076]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.308 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:03 smithi099 ceph-mon[167076]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.308 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:03 smithi099 ceph-mon[167076]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.308 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7162/46452 objects degraded (15.418%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:09:03.308 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:03 smithi099 ceph-mon[167076]: osdmap e348: 8 total, 8 up, 8 in 2023-09-23T18:09:03.331 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:03.391 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:03.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:03.398 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:03.403 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:09:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[179668]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[179668]: osdmap e347: 8 total, 8 up, 8 in 2023-09-23T18:09:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[179668]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[179668]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[179668]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7162/46452 objects degraded (15.418%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:09:03.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[179668]: osdmap e348: 8 total, 8 up, 8 in 2023-09-23T18:09:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[184174]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[184174]: osdmap e347: 8 total, 8 up, 8 in 2023-09-23T18:09:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[184174]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[184174]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[184174]: failed to encode map e347 with expected crc 2023-09-23T18:09:03.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7162/46452 objects degraded (15.418%), 16 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:09:03.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:03 smithi055 ceph-mon[184174]: osdmap e348: 8 total, 8 up, 8 in 2023-09-23T18:09:03.438 INFO:teuthology.orchestra.run.smithi055.stdout:moved 0 / 417 (0%) 2023-09-23T18:09:03.438 INFO:teuthology.orchestra.run.smithi055.stdout:avg 52.125 2023-09-23T18:09:03.438 INFO:teuthology.orchestra.run.smithi055.stdout:stddev 19.8396 -> 19.8396 (expected baseline 6.75347) 2023-09-23T18:09:03.438 INFO:teuthology.orchestra.run.smithi055.stdout:min osd.0 with 21 -> 21 pgs (0.402878 -> 0.402878 * mean) 2023-09-23T18:09:03.438 INFO:teuthology.orchestra.run.smithi055.stdout:max osd.1 with 80 -> 80 pgs (1.53477 -> 1.53477 * mean) 2023-09-23T18:09:03.438 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:09:03.438 INFO:teuthology.orchestra.run.smithi055.stdout:oload 1000 2023-09-23T18:09:03.438 INFO:teuthology.orchestra.run.smithi055.stdout:max_change 1 2023-09-23T18:09:03.438 INFO:teuthology.orchestra.run.smithi055.stdout:max_change_osds 4 2023-09-23T18:09:03.439 INFO:teuthology.orchestra.run.smithi055.stdout:average_utilization 0.0110 2023-09-23T18:09:03.439 INFO:teuthology.orchestra.run.smithi055.stdout:overload_utilization 0.1097 2023-09-23T18:09:03.443 INFO:teuthology.orchestra.run.smithi055.stderr:no change 2023-09-23T18:09:03.499 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:03.533 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:03.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:03.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:03.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:03.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:03.703 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:03.737 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:03.740 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:03.740 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:03.841 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:03.872 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:03.912 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:03.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:03.915 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:04.015 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:04.045 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:04.082 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:09:04.082 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:04.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:04.086 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:04.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:04.238 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:04.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:04 smithi055 ceph-mon[179668]: pgmap v859: 139 pgs: 12 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 838 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 5218/40128 objects degraded (13.003%); 31 MiB/s, 569 objects/s recovering 2023-09-23T18:09:04.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:04 smithi055 ceph-mon[179668]: failed to encode map e348 with expected crc 2023-09-23T18:09:04.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:04 smithi055 ceph-mon[179668]: osdmap e349: 8 total, 8 up, 8 in 2023-09-23T18:09:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:04 smithi055 ceph-mon[184174]: pgmap v859: 139 pgs: 12 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 838 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 5218/40128 objects degraded (13.003%); 31 MiB/s, 569 objects/s recovering 2023-09-23T18:09:04.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:04 smithi055 ceph-mon[184174]: failed to encode map e348 with expected crc 2023-09-23T18:09:04.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:04 smithi055 ceph-mon[184174]: osdmap e349: 8 total, 8 up, 8 in 2023-09-23T18:09:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:04 smithi099 ceph-mon[167076]: pgmap v859: 139 pgs: 12 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 838 MiB data, 8.0 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 1.18k op/s; 5218/40128 objects degraded (13.003%); 31 MiB/s, 569 objects/s recovering 2023-09-23T18:09:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:04 smithi099 ceph-mon[167076]: failed to encode map e348 with expected crc 2023-09-23T18:09:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:04 smithi099 ceph-mon[167076]: osdmap e349: 8 total, 8 up, 8 in 2023-09-23T18:09:04.735 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:09:05.143 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:05 smithi055 ceph-mon[184174]: from='client.45851 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:09:05.143 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:05 smithi055 ceph-mon[184174]: failed to encode map e349 with expected crc 2023-09-23T18:09:05.143 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:05 smithi055 ceph-mon[184174]: failed to encode map e349 with expected crc 2023-09-23T18:09:05.143 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:05 smithi055 ceph-mon[184174]: failed to encode map e349 with expected crc 2023-09-23T18:09:05.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:05 smithi055 ceph-mon[179668]: from='client.45851 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:09:05.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:05 smithi055 ceph-mon[179668]: failed to encode map e349 with expected crc 2023-09-23T18:09:05.144 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:05 smithi055 ceph-mon[179668]: failed to encode map e349 with expected crc 2023-09-23T18:09:05.144 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:05 smithi055 ceph-mon[179668]: failed to encode map e349 with expected crc 2023-09-23T18:09:05.147 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:09:05.251 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:05.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:05.257 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:05.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:05.435 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:05 smithi099 ceph-mon[167076]: from='client.45851 -' entity='client.admin' cmd=[{"prefix": "osd reweight-by-utilization", "oload": 1000, "max_change": 1.0, "target": ["mon-mgr", ""]}]: dispatch 2023-09-23T18:09:05.436 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:05 smithi099 ceph-mon[167076]: failed to encode map e349 with expected crc 2023-09-23T18:09:05.436 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:05 smithi099 ceph-mon[167076]: failed to encode map e349 with expected crc 2023-09-23T18:09:05.436 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:05 smithi099 ceph-mon[167076]: failed to encode map e349 with expected crc 2023-09-23T18:09:05.440 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:05.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:05.500 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:05.576 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:05.600 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:05.651 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:05.920 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:05.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:05.929 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:06.030 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:06.090 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:06.107 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:06.153 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:06 smithi055 ceph-mon[179668]: pgmap v862: 139 pgs: 12 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 794 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 4861/38001 objects degraded (12.792%); 17 MiB/s, 369 objects/s recovering 2023-09-23T18:09:06.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:06.168 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:06.169 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:06.169 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:06 smithi099 ceph-mon[167076]: pgmap v862: 139 pgs: 12 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 794 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 4861/38001 objects degraded (12.792%); 17 MiB/s, 369 objects/s recovering 2023-09-23T18:09:06.270 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:06.337 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:06.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:06 smithi055 ceph-mon[184174]: pgmap v862: 139 pgs: 12 active+recovery_wait+degraded, 1 active+recovering, 1 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 122 active+clean; 794 MiB data, 7.7 GiB used, 707 GiB / 715 GiB avail; 0 B/s wr, 937 op/s; 4861/38001 objects degraded (12.792%); 17 MiB/s, 369 objects/s recovering 2023-09-23T18:09:06.572 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:06.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:06.581 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:06.682 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:06.748 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:06.820 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:06.825 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:06.825 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:06.926 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:06.994 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:07.227 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:07.231 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:07.232 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:07.258 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:09:07.258 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:09:07.258 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:09:07.258 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:09:07.258 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:09:07.333 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:07.381 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:07.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:07.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:07.447 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:07.547 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:07.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:07.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:07.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:07 smithi099 ceph-mon[167076]: pgmap v863: 139 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 639 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 881 op/s; 3638/30573 objects degraded (11.899%); 7/30573 objects misplaced (0.023%); 18 MiB/s, 391 objects/s recovering 2023-09-23T18:09:07.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:07.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:07.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:07 smithi055 ceph-mon[179668]: pgmap v863: 139 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 639 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 881 op/s; 3638/30573 objects degraded (11.899%); 7/30573 objects misplaced (0.023%); 18 MiB/s, 391 objects/s recovering 2023-09-23T18:09:07.625 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:07.625 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:07.626 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:07 smithi055 ceph-mon[184174]: pgmap v863: 139 pgs: 10 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering, 1 active+undersized+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 123 active+clean; 639 MiB data, 7.5 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 881 op/s; 3638/30573 objects degraded (11.899%); 7/30573 objects misplaced (0.023%); 18 MiB/s, 391 objects/s recovering 2023-09-23T18:09:07.636 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:07.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:09:07.715 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:07.721 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:07.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:07.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:07.878 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:07.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:07.958 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:07.958 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:07.991 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:09:08.024 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:09:08.057 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:09:08.059 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:08.092 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:09:08.119 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:08.129 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:09:08.162 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:09:08.173 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:08.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:08.179 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:08.206 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:09:08.242 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:09:08.245 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:09:08.245 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:09:08.245 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:09:08.246 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:09:08.246 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:09:08.246 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:09:08.246 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:09:08.246 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:09:08.280 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:08.356 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:08.416 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:09:08.462 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:09:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1247767756' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:08 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1741210569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 3638/30573 objects degraded (11.899%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:09:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1247767756' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:08 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1741210569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 3638/30573 objects degraded (11.899%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:09:08.719 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1247767756' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:08.720 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:08 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:08.720 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1741210569' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:08.720 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 3638/30573 objects degraded (11.899%), 12 pgs degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:09:08.827 INFO:tasks.thrashosds.thrasher:in_osds: [0, 2, 3, 6, 7, 5, 4, 1] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:09:08.827 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:09:08.827 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [0, 2, 3, 6, 7, 5, 4, 1] 2023-09-23T18:09:08.827 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 0 2023-09-23T18:09:08.839 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:08.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:08.848 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:08.949 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:09.039 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:09.108 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:09.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:09.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:09.221 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:09.277 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:09.342 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:09.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:09.347 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:09.448 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:09.502 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:09.567 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:09.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:09.572 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:09.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:09.722 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:09.731 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:09 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:09.731 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:09 smithi099 ceph-mon[167076]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.731 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:09 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:09:09.731 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:09 smithi099 ceph-mon[167076]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T18:09:09.732 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:09 smithi099 ceph-mon[167076]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.732 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:09 smithi099 ceph-mon[167076]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.732 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:09 smithi099 ceph-mon[167076]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.732 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:09 smithi099 ceph-mon[167076]: pgmap v865: 139 pgs: 8 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 483 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 1995/23079 objects degraded (8.644%); 7/23079 objects misplaced (0.030%); 21 MiB/s, 544 objects/s recovering 2023-09-23T18:09:09.737 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:09.806 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:09.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:09.815 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:09.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:09.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:09.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[184174]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:09:09.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[184174]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T18:09:09.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[184174]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[184174]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[184174]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[184174]: pgmap v865: 139 pgs: 8 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 483 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 1995/23079 objects degraded (8.644%); 7/23079 objects misplaced (0.030%); 21 MiB/s, 544 objects/s recovering 2023-09-23T18:09:09.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:09.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[179668]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:09:09.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[179668]: osdmap e350: 8 total, 8 up, 8 in 2023-09-23T18:09:09.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[179668]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[179668]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[179668]: failed to encode map e350 with expected crc 2023-09-23T18:09:09.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:09 smithi055 ceph-mon[179668]: pgmap v865: 139 pgs: 8 active+recovery_wait+degraded, 1 active+recovery_wait, 2 active+recovering, 2 active+recovery_wait+undersized+degraded+remapped, 126 active+clean; 483 MiB data, 6.9 GiB used, 708 GiB / 715 GiB avail; 0 B/s wr, 946 op/s; 1995/23079 objects degraded (8.644%); 7/23079 objects misplaced (0.030%); 21 MiB/s, 544 objects/s recovering 2023-09-23T18:09:09.989 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:09.997 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:09:09.997 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:09:10.062 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:10.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:10.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:10.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:10.225 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:10.377 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:10.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:10.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:10.471 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:09:10.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:10.592 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:10.678 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:10.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:09:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:09:10] "GET /metrics HTTP/1.1" 200 36611 "" "Prometheus/2.43.0" 2023-09-23T18:09:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:10 smithi055 ceph-mon[184174]: 7.16 scrub starts 2023-09-23T18:09:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:10 smithi055 ceph-mon[184174]: 7.15 scrub starts 2023-09-23T18:09:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:10 smithi055 ceph-mon[179668]: 7.16 scrub starts 2023-09-23T18:09:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:10 smithi055 ceph-mon[179668]: 7.15 scrub starts 2023-09-23T18:09:10.682 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:10.682 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:10.682 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:10 smithi099 ceph-mon[167076]: 7.16 scrub starts 2023-09-23T18:09:10.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:10 smithi099 ceph-mon[167076]: 7.15 scrub starts 2023-09-23T18:09:10.783 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:10.876 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:10.942 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:10.947 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:09:10.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:10.949 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:11.039 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:09:11.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:11.107 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:11.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:11.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:11.158 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:11.168 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:09:11.261 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:11.362 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:11.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:11 smithi055 ceph-mon[179668]: pgmap v866: 139 pgs: 5 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering, 131 active+clean; 289 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 1483/13737 objects degraded (10.796%); 15 MiB/s, 455 objects/s recovering 2023-09-23T18:09:11.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:11 smithi055 ceph-mon[184174]: pgmap v866: 139 pgs: 5 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering, 131 active+clean; 289 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 1483/13737 objects degraded (10.796%); 15 MiB/s, 455 objects/s recovering 2023-09-23T18:09:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:11 smithi099 ceph-mon[167076]: pgmap v866: 139 pgs: 5 active+recovery_wait+degraded, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovering, 131 active+clean; 289 MiB data, 6.4 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.04k op/s; 1483/13737 objects degraded (10.796%); 15 MiB/s, 455 objects/s recovering 2023-09-23T18:09:12.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:12 smithi099 ceph-mon[167076]: failed to encode map e351 with expected crc 2023-09-23T18:09:12.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:12 smithi099 ceph-mon[167076]: failed to encode map e351 with expected crc 2023-09-23T18:09:12.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:12 smithi099 ceph-mon[167076]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T18:09:12.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:12 smithi099 ceph-mon[167076]: failed to encode map e351 with expected crc 2023-09-23T18:09:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:12 smithi055 ceph-mon[184174]: failed to encode map e351 with expected crc 2023-09-23T18:09:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:12 smithi055 ceph-mon[184174]: failed to encode map e351 with expected crc 2023-09-23T18:09:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:12 smithi055 ceph-mon[184174]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T18:09:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:12 smithi055 ceph-mon[184174]: failed to encode map e351 with expected crc 2023-09-23T18:09:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:12 smithi055 ceph-mon[179668]: failed to encode map e351 with expected crc 2023-09-23T18:09:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:12 smithi055 ceph-mon[179668]: failed to encode map e351 with expected crc 2023-09-23T18:09:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:12 smithi055 ceph-mon[179668]: osdmap e351: 8 total, 8 up, 8 in 2023-09-23T18:09:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:12 smithi055 ceph-mon[179668]: failed to encode map e351 with expected crc 2023-09-23T18:09:13.056 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Removed 70960 objects 2023-09-23T18:09:13.057 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Clean up completed and total clean up time :111.172 2023-09-23T18:09:13.154 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:13.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:13.158 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:13.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:13.288 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:13 smithi099 ceph-mon[167076]: failed to encode map e352 with expected crc 2023-09-23T18:09:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:13 smithi099 ceph-mon[167076]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T18:09:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:13 smithi099 ceph-mon[167076]: failed to encode map e352 with expected crc 2023-09-23T18:09:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:13 smithi099 ceph-mon[167076]: 7.5 scrub starts 2023-09-23T18:09:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:13 smithi099 ceph-mon[167076]: pgmap v869: 139 pgs: 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 134 active+clean; 153 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.31k op/s; 822/7212 objects degraded (11.398%); 12 MiB/s, 604 objects/s recovering 2023-09-23T18:09:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 1483/13737 objects degraded (10.796%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:09:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[179668]: failed to encode map e352 with expected crc 2023-09-23T18:09:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[179668]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T18:09:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[179668]: failed to encode map e352 with expected crc 2023-09-23T18:09:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[179668]: 7.5 scrub starts 2023-09-23T18:09:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[179668]: pgmap v869: 139 pgs: 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 134 active+clean; 153 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.31k op/s; 822/7212 objects degraded (11.398%); 12 MiB/s, 604 objects/s recovering 2023-09-23T18:09:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 1483/13737 objects degraded (10.796%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:09:13.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[184174]: failed to encode map e352 with expected crc 2023-09-23T18:09:13.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[184174]: osdmap e352: 8 total, 8 up, 8 in 2023-09-23T18:09:13.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[184174]: failed to encode map e352 with expected crc 2023-09-23T18:09:13.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[184174]: 7.5 scrub starts 2023-09-23T18:09:13.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[184174]: pgmap v869: 139 pgs: 2 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 134 active+clean; 153 MiB data, 6.0 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 1.31k op/s; 822/7212 objects degraded (11.398%); 12 MiB/s, 604 objects/s recovering 2023-09-23T18:09:13.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 1483/13737 objects degraded (10.796%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:09:14.164 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:14.509 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:14.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:14.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:14.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:14.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:14.717 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:14.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:14 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:14.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:14 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:14.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:14 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:09:15.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:15.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:15.519 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:15.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:15.653 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:15.774 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:15.777 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:15 smithi099 ceph-mon[167076]: failed to encode map e353 with expected crc 2023-09-23T18:09:15.777 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:15 smithi099 ceph-mon[167076]: failed to encode map e353 with expected crc 2023-09-23T18:09:15.777 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:15 smithi099 ceph-mon[167076]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T18:09:15.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:15 smithi099 ceph-mon[167076]: 1.19 scrub starts 2023-09-23T18:09:15.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:15 smithi099 ceph-mon[167076]: 1.19 scrub ok 2023-09-23T18:09:15.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:15 smithi099 ceph-mon[167076]: pgmap v871: 139 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 135 active+clean; 107 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 805/5001 objects degraded (16.097%); 3.9 MiB/s, 262 objects/s recovering 2023-09-23T18:09:15.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:15.779 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:15.847 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[179668]: failed to encode map e353 with expected crc 2023-09-23T18:09:15.848 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[179668]: failed to encode map e353 with expected crc 2023-09-23T18:09:15.848 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[179668]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T18:09:15.848 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[179668]: 1.19 scrub starts 2023-09-23T18:09:15.848 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[179668]: 1.19 scrub ok 2023-09-23T18:09:15.848 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[179668]: pgmap v871: 139 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 135 active+clean; 107 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 805/5001 objects degraded (16.097%); 3.9 MiB/s, 262 objects/s recovering 2023-09-23T18:09:15.848 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:09:15.753+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e354 definitely_dead 0 2023-09-23T18:09:15.848 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[184174]: failed to encode map e353 with expected crc 2023-09-23T18:09:15.848 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[184174]: failed to encode map e353 with expected crc 2023-09-23T18:09:15.848 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[184174]: osdmap e353: 8 total, 8 up, 8 in 2023-09-23T18:09:15.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[184174]: 1.19 scrub starts 2023-09-23T18:09:15.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[184174]: 1.19 scrub ok 2023-09-23T18:09:15.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:15 smithi055 ceph-mon[184174]: pgmap v871: 139 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 135 active+clean; 107 MiB data, 5.7 GiB used, 710 GiB / 715 GiB avail; 0 B/s wr, 1.02k op/s; 805/5001 objects degraded (16.097%); 3.9 MiB/s, 262 objects/s recovering 2023-09-23T18:09:15.879 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:15.910 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:16.100 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:16.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:16.104 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:16.205 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:16.238 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:16.378 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:16.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:16.383 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:16.484 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:16.512 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:16.548 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:16.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:16.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:16.576 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.0. 2023-09-23T18:09:16.652 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:16.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[184174]: 7.6 scrub starts 2023-09-23T18:09:16.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[184174]: 7.6 scrub ok 2023-09-23T18:09:16.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[184174]: 7.14 scrub starts 2023-09-23T18:09:16.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[184174]: 7.14 scrub ok 2023-09-23T18:09:16.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[184174]: failed to encode map e354 with expected crc 2023-09-23T18:09:16.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[184174]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T18:09:16.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[184174]: failed to encode map e354 with expected crc 2023-09-23T18:09:16.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1665271247' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T18:09:16.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T18:09:16.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[184174]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T18:09:16.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[179668]: 7.6 scrub starts 2023-09-23T18:09:16.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[179668]: 7.6 scrub ok 2023-09-23T18:09:16.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[179668]: 7.14 scrub starts 2023-09-23T18:09:16.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[179668]: 7.14 scrub ok 2023-09-23T18:09:16.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[179668]: failed to encode map e354 with expected crc 2023-09-23T18:09:16.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[179668]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T18:09:16.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[179668]: failed to encode map e354 with expected crc 2023-09-23T18:09:16.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1665271247' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T18:09:16.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T18:09:16.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:16 smithi055 ceph-mon[179668]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T18:09:16.688 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:09:16.688 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:09:16.688 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:16.770 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:09:16.793 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:09:16.809 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:16 smithi099 ceph-mon[167076]: 7.6 scrub starts 2023-09-23T18:09:16.809 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:16 smithi099 ceph-mon[167076]: 7.6 scrub ok 2023-09-23T18:09:16.809 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:16 smithi099 ceph-mon[167076]: 7.14 scrub starts 2023-09-23T18:09:16.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:16 smithi099 ceph-mon[167076]: 7.14 scrub ok 2023-09-23T18:09:16.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:16 smithi099 ceph-mon[167076]: failed to encode map e354 with expected crc 2023-09-23T18:09:16.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:16 smithi099 ceph-mon[167076]: osdmap e354: 8 total, 8 up, 8 in 2023-09-23T18:09:16.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:16 smithi099 ceph-mon[167076]: failed to encode map e354 with expected crc 2023-09-23T18:09:16.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1665271247' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T18:09:16.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:16 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T18:09:16.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:16 smithi099 ceph-mon[167076]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T18:09:16.815 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:09:16.842 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:09:16.863 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:09:16.881 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:09:16.901 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:09:16.920 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:09:16.921 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:09:16.922 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:09:16.922 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:09:16.922 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:09:16.922 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:09:16.922 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:09:16.922 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:09:16.922 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:09:17.026 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:17.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:17.030 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:17.051 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:09:17.131 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:17.161 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:17.562 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:17.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:17.572 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:17.673 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:17.701 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:17.727 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:09:17.735 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:17.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:17.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: 7.4 scrub starts 2023-09-23T18:09:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: 7.4 scrub ok 2023-09-23T18:09:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: 7.8 scrub starts 2023-09-23T18:09:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: 7.8 scrub ok 2023-09-23T18:09:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T18:09:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: 1.1e scrub starts 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: 1.1e scrub ok 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: 1.3 scrub starts 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: 1.3 scrub ok 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: 1.18 scrub starts 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: 1.18 scrub ok 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/416733097' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:17.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:17 smithi099 ceph-mon[167076]: pgmap v874: 139 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 137 active+clean; 31 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 599/1350 objects degraded (44.370%) 2023-09-23T18:09:17.840 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:17.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: 7.4 scrub starts 2023-09-23T18:09:17.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: 7.4 scrub ok 2023-09-23T18:09:17.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: 7.8 scrub starts 2023-09-23T18:09:17.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: 7.8 scrub ok 2023-09-23T18:09:17.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T18:09:17.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T18:09:17.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: 1.1e scrub starts 2023-09-23T18:09:17.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: 1.1e scrub ok 2023-09-23T18:09:17.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: 1.3 scrub starts 2023-09-23T18:09:17.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: 1.3 scrub ok 2023-09-23T18:09:17.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: 1.18 scrub starts 2023-09-23T18:09:17.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: 1.18 scrub ok 2023-09-23T18:09:17.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/416733097' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:17.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[179668]: pgmap v874: 139 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 137 active+clean; 31 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 599/1350 objects degraded (44.370%) 2023-09-23T18:09:17.868 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: 7.4 scrub starts 2023-09-23T18:09:17.868 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: 7.4 scrub ok 2023-09-23T18:09:17.868 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: 7.8 scrub starts 2023-09-23T18:09:17.868 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: 7.8 scrub ok 2023-09-23T18:09:17.868 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T18:09:17.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: osdmap e355: 8 total, 8 up, 7 in 2023-09-23T18:09:17.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: failed to encode map e355 with expected crc 2023-09-23T18:09:17.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: 1.1e scrub starts 2023-09-23T18:09:17.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: 1.1e scrub ok 2023-09-23T18:09:17.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: 1.3 scrub starts 2023-09-23T18:09:17.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: 1.3 scrub ok 2023-09-23T18:09:17.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: 1.18 scrub starts 2023-09-23T18:09:17.870 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: 1.18 scrub ok 2023-09-23T18:09:17.870 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/416733097' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:17.870 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:17 smithi055 ceph-mon[184174]: pgmap v874: 139 pgs: 1 active+recovery_wait+degraded, 1 active+recovering, 137 active+clean; 31 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail; 599/1350 objects degraded (44.370%) 2023-09-23T18:09:17.870 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:17.903 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:17.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:17.906 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:18.007 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:18.035 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:18.068 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:18.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:18.071 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:18.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:18.205 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:18.218 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_2 2023-09-23T18:09:18.219 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_2 unique_pool_2 --yes-i-really-really-mean-it 2023-09-23T18:09:18.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:18.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:18.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:18.399 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:18.429 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:18.464 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:18.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:18.467 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:18.568 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[179668]: failed to encode map e356 with expected crc 2023-09-23T18:09:18.568 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[179668]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T18:09:18.569 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:18.605 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:09:18.614 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:18.672 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:18.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:18.686 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:18.787 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:18.815 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:18.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:18 smithi099 ceph-mon[167076]: failed to encode map e356 with expected crc 2023-09-23T18:09:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:18 smithi099 ceph-mon[167076]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T18:09:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:18 smithi099 ceph-mon[167076]: failed to encode map e356 with expected crc 2023-09-23T18:09:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:18 smithi099 ceph-mon[167076]: 1.1d scrub starts 2023-09-23T18:09:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:18 smithi099 ceph-mon[167076]: 1.1d scrub ok 2023-09-23T18:09:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:18 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2821523778' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:18 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 599/1350 objects degraded (44.370%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:09:18.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[179668]: failed to encode map e356 with expected crc 2023-09-23T18:09:18.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[179668]: 1.1d scrub starts 2023-09-23T18:09:18.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[179668]: 1.1d scrub ok 2023-09-23T18:09:18.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2821523778' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:18.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:18.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 599/1350 objects degraded (44.370%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:09:18.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[184174]: failed to encode map e356 with expected crc 2023-09-23T18:09:18.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[184174]: osdmap e356: 8 total, 8 up, 7 in 2023-09-23T18:09:18.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[184174]: failed to encode map e356 with expected crc 2023-09-23T18:09:18.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[184174]: 1.1d scrub starts 2023-09-23T18:09:18.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[184174]: 1.1d scrub ok 2023-09-23T18:09:18.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2821523778' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:18.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:18.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 599/1350 objects degraded (44.370%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:09:18.980 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:18.996 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:19.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:19.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:19.105 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:19.145 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:19.157 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:09:19.157 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:09:19.157 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:09:19.157 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:09:19.157 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:09:19.227 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:19.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:19.230 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:19.331 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:19.362 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:19.528 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:09:19.570 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:19.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:19.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:19.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:09:19.675 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:09:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: 7.7 scrub starts 2023-09-23T18:09:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: 7.7 scrub ok 2023-09-23T18:09:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T18:09:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: 7.1d starting backfill to osd.3 from (0'0,0'0] MAX to 352'7888 2023-09-23T18:09:19.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: 7.d starting backfill to osd.3 from (0'0,0'0] MAX to 352'7889 2023-09-23T18:09:19.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: 1.11 deep-scrub starts 2023-09-23T18:09:19.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[184174]: pgmap v877: 139 pgs: 7 remapped+peering, 132 active+clean; 23 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:19.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:09:19.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: 7.7 scrub starts 2023-09-23T18:09:19.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: 7.7 scrub ok 2023-09-23T18:09:19.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:19.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T18:09:19.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: 7.1d starting backfill to osd.3 from (0'0,0'0] MAX to 352'7888 2023-09-23T18:09:19.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: 7.d starting backfill to osd.3 from (0'0,0'0] MAX to 352'7889 2023-09-23T18:09:19.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: 1.11 deep-scrub starts 2023-09-23T18:09:19.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:19 smithi055 ceph-mon[179668]: pgmap v877: 139 pgs: 7 remapped+peering, 132 active+clean; 23 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:19.706 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:19.793 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:19.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:19.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:09:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: 7.7 scrub starts 2023-09-23T18:09:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: 7.7 scrub ok 2023-09-23T18:09:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: osdmap e357: 8 total, 8 up, 7 in 2023-09-23T18:09:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: failed to encode map e357 with expected crc 2023-09-23T18:09:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: 7.1d starting backfill to osd.3 from (0'0,0'0] MAX to 352'7888 2023-09-23T18:09:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: 7.d starting backfill to osd.3 from (0'0,0'0] MAX to 352'7889 2023-09-23T18:09:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: 1.11 deep-scrub starts 2023-09-23T18:09:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:19 smithi099 ceph-mon[167076]: pgmap v877: 139 pgs: 7 remapped+peering, 132 active+clean; 23 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:19.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:19.924 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:20.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:20.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:20.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:20.321 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:20.359 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:20.391 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:20.402 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:20.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:20.407 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:20.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:09:20] "GET /metrics HTTP/1.1" 200 36611 "" "Prometheus/2.43.0" 2023-09-23T18:09:20.509 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:20.544 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:20.587 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:20.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:20.591 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:20.692 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:20.732 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:20.747 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:09:20.821 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 599/1350 objects degraded (44.370%), 1 pg degraded) 2023-09-23T18:09:20.821 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:09:20.821 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.821 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T18:09:20.821 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.821 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.821 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.822 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.822 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.822 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.822 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.822 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[184174]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.822 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 599/1350 objects degraded (44.370%), 1 pg degraded) 2023-09-23T18:09:20.822 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:09:20.822 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.822 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T18:09:20.823 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.823 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.823 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.823 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.823 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.823 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.823 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.823 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:20 smithi055 ceph-mon[179668]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 599/1350 objects degraded (44.370%), 1 pg degraded) 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: osdmap e358: 8 total, 8 up, 7 in 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:20 smithi099 ceph-mon[167076]: failed to encode map e358 with expected crc 2023-09-23T18:09:20.984 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:20.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:20.988 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:21.019 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:09:21.088 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:21.121 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:21.208 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:21.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:21.211 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:21.311 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:21.347 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:21.582 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:21.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:21.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:21.644 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_2' removed 2023-09-23T18:09:21.688 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:21.718 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:21.829 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:09:21.832 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:21.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: failed to encode map e359 with expected crc 2023-09-23T18:09:21.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: failed to encode map e359 with expected crc 2023-09-23T18:09:21.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: osdmap e359: 8 total, 8 up, 7 in 2023-09-23T18:09:21.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: failed to encode map e359 with expected crc 2023-09-23T18:09:21.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: failed to encode map e359 with expected crc 2023-09-23T18:09:21.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: failed to encode map e359 with expected crc 2023-09-23T18:09:21.836 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/848318232' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:09:21.836 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:09:21.836 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: pgmap v880: 139 pgs: 2 active, 7 remapped+peering, 130 active+clean; 21 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:21.836 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:21.836 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:21.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:21.837 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:21.837 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: failed to encode map e359 with expected crc 2023-09-23T18:09:21.837 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: failed to encode map e359 with expected crc 2023-09-23T18:09:21.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: osdmap e359: 8 total, 8 up, 7 in 2023-09-23T18:09:21.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: failed to encode map e359 with expected crc 2023-09-23T18:09:21.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: failed to encode map e359 with expected crc 2023-09-23T18:09:21.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: failed to encode map e359 with expected crc 2023-09-23T18:09:21.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/848318232' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:09:21.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:09:21.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: pgmap v880: 139 pgs: 2 active, 7 remapped+peering, 130 active+clean; 21 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:21.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:21.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:21.860 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:09:21.882 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:09:21.905 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:09:21.926 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:09:21.938 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:21.946 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:09:21.969 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:09:21.970 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:21.994 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:09:21.996 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:09:21.996 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:09:21.996 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:09:21.996 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:09:21.996 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:09:21.996 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:09:21.996 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:09:21.997 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:09:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: failed to encode map e359 with expected crc 2023-09-23T18:09:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: failed to encode map e359 with expected crc 2023-09-23T18:09:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: osdmap e359: 8 total, 8 up, 7 in 2023-09-23T18:09:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: failed to encode map e359 with expected crc 2023-09-23T18:09:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: failed to encode map e359 with expected crc 2023-09-23T18:09:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: failed to encode map e359 with expected crc 2023-09-23T18:09:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/848318232' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:09:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:09:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: pgmap v880: 139 pgs: 2 active, 7 remapped+peering, 130 active+clean; 21 MiB data, 3.3 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:22.104 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:22.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:22.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:22.125 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:22.181 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T18:09:22.182 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T18:09:22.182 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_3 2023-09-23T18:09:22.183 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_3 16 2023-09-23T18:09:22.209 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 6, 7, 5, 4, 1] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:09:22.209 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:09:22.209 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T18:09:22.210 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:09:22.210 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:09:22.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T18:09:22.220 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:22.254 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:22.370 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:22.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:22.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:22.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:22.503 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:22.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[179668]: 7.17 scrub starts 2023-09-23T18:09:22.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[179668]: failed to encode map e360 with expected crc 2023-09-23T18:09:22.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:09:22.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[179668]: osdmap e360: 8 total, 8 up, 7 in 2023-09-23T18:09:22.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3800938171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:22.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:22.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:22.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[184174]: 7.17 scrub starts 2023-09-23T18:09:22.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[184174]: failed to encode map e360 with expected crc 2023-09-23T18:09:22.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:09:22.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[184174]: osdmap e360: 8 total, 8 up, 7 in 2023-09-23T18:09:22.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3800938171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:22.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:22.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:22 smithi099 ceph-mon[167076]: 7.17 scrub starts 2023-09-23T18:09:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:22 smithi099 ceph-mon[167076]: failed to encode map e360 with expected crc 2023-09-23T18:09:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:22 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_2", "pool2": "unique_pool_2", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:09:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:22 smithi099 ceph-mon[167076]: osdmap e360: 8 total, 8 up, 7 in 2023-09-23T18:09:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3800938171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:22.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:23.061 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:23.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:23.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:23.744 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:23.845 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:23.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:23 smithi055 ceph-mon[184174]: failed to encode map e360 with expected crc 2023-09-23T18:09:23.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:23 smithi055 ceph-mon[184174]: pgmap v882: 103 pgs: 103 active+clean; 4.3 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:23.873 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:23 smithi055 ceph-mon[179668]: failed to encode map e360 with expected crc 2023-09-23T18:09:23.873 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:23 smithi055 ceph-mon[179668]: pgmap v882: 103 pgs: 103 active+clean; 4.3 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:23.880 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:23.915 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:23.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:23.918 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:24.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:24.045 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:23 smithi099 ceph-mon[167076]: failed to encode map e360 with expected crc 2023-09-23T18:09:24.045 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:23 smithi099 ceph-mon[167076]: pgmap v882: 103 pgs: 103 active+clean; 4.3 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:24.048 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:24.485 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:09:24.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:24.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:24.833 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:24.934 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:24.961 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:25.035 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_3' created 2023-09-23T18:09:25.036 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:09:25.094 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:25.177 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:09:25.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:25 smithi099 ceph-mon[167076]: 1.14 scrub starts 2023-09-23T18:09:25.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:25 smithi099 ceph-mon[167076]: 1.14 scrub ok 2023-09-23T18:09:25.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:25 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2679910142' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:25.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:25 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:25.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:25 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2005227255' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:09:25.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:25 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:09:25.378 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[184174]: 1.14 scrub starts 2023-09-23T18:09:25.378 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[184174]: 1.14 scrub ok 2023-09-23T18:09:25.378 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2679910142' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:25.378 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:25.378 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2005227255' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:09:25.378 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:09:25.378 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[179668]: 1.14 scrub starts 2023-09-23T18:09:25.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[179668]: 1.14 scrub ok 2023-09-23T18:09:25.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2679910142' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:25.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:25.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2005227255' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:09:25.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:25 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]: dispatch 2023-09-23T18:09:25.487 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_3 rados --yes-i-really-mean-it || true 2023-09-23T18:09:26.200 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:26.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:26.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:26.204 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:26.231 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[179668]: pgmap v883: 103 pgs: 103 active+clean; 4.3 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:26.231 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[179668]: 1.11 deep-scrub starts 2023-09-23T18:09:26.231 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[179668]: 1.11 deep-scrub ok 2023-09-23T18:09:26.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:26.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[179668]: failed to encode map e361 with expected crc 2023-09-23T18:09:26.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:09:26.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T18:09:26.232 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[179668]: osdmap e361: 8 total, 8 up, 7 in 2023-09-23T18:09:26.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[184174]: pgmap v883: 103 pgs: 103 active+clean; 4.3 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:26.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[184174]: 1.11 deep-scrub starts 2023-09-23T18:09:26.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[184174]: 1.11 deep-scrub ok 2023-09-23T18:09:26.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:26.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[184174]: failed to encode map e361 with expected crc 2023-09-23T18:09:26.233 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:09:26.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T18:09:26.234 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:26 smithi055 ceph-mon[184174]: osdmap e361: 8 total, 8 up, 7 in 2023-09-23T18:09:26.258 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:09:26.258 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:09:26.276 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:26 smithi099 ceph-mon[167076]: pgmap v883: 103 pgs: 103 active+clean; 4.3 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:26.276 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:26 smithi099 ceph-mon[167076]: 1.11 deep-scrub starts 2023-09-23T18:09:26.276 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:26 smithi099 ceph-mon[167076]: 1.11 deep-scrub ok 2023-09-23T18:09:26.276 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:26 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:26.276 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:26 smithi099 ceph-mon[167076]: failed to encode map e361 with expected crc 2023-09-23T18:09:26.276 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:26 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:09:26.276 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:26 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_3", "pg_num": 16}]': finished 2023-09-23T18:09:26.276 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:26 smithi099 ceph-mon[167076]: osdmap e361: 8 total, 8 up, 7 in 2023-09-23T18:09:26.305 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:26.337 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:26.478 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:26.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:26.481 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:26.582 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:26.615 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:26.715 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:09:26.972 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:09:26.972 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:09:26.973 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:09:26.973 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:09:26.973 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:26.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:26.977 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:26.985 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:09:27.004 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:09:27.023 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:09:27.044 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:09:27.046 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:09:27.046 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:09:27.046 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:09:27.046 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:09:27.046 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:09:27.046 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:09:27.046 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:09:27.046 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:09:27.078 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:27.107 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:27.146 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:27.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:27.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:27.236 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.236 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: osdmap e362: 8 total, 8 up, 7 in 2023-09-23T18:09:27.236 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.236 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:27.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:27.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:27.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:27.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:27.237 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3249209666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:27.252 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:27.284 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:27.320 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:27.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:27.323 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:27.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: osdmap e362: 8 total, 8 up, 7 in 2023-09-23T18:09:27.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.397 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:27.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:27.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:27.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:27.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:27.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3249209666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:27.398 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.398 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: osdmap e362: 8 total, 8 up, 7 in 2023-09-23T18:09:27.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: failed to encode map e362 with expected crc 2023-09-23T18:09:27.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:27.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:27.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:09:27.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:09:27.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:27.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3249209666' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:27.400 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:09:27.400 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "" 2023-09-23T18:09:27.400 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:09:27.424 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:27.454 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:27.661 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:27.724 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:09:27.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:27.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:27.751 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:27.851 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:27.889 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:27.933 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:27.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:27.938 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:28.039 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:28.067 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:28.108 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:28.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:28.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:28.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:28.240 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:28.273 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:28.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:28.276 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:28 smithi099 ceph-mon[167076]: pgmap v886: 119 pgs: 6 creating+peering, 10 unknown, 103 active+clean; 4.3 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:28 smithi099 ceph-mon[167076]: failed to encode map e363 with expected crc 2023-09-23T18:09:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:28 smithi099 ceph-mon[167076]: osdmap e363: 8 total, 8 up, 7 in 2023-09-23T18:09:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:28 smithi099 ceph-mon[167076]: failed to encode map e363 with expected crc 2023-09-23T18:09:28.378 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:28.397 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:28 smithi055 ceph-mon[179668]: pgmap v886: 119 pgs: 6 creating+peering, 10 unknown, 103 active+clean; 4.3 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:28.398 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:28 smithi055 ceph-mon[179668]: failed to encode map e363 with expected crc 2023-09-23T18:09:28.398 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:28 smithi055 ceph-mon[179668]: osdmap e363: 8 total, 8 up, 7 in 2023-09-23T18:09:28.398 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:28 smithi055 ceph-mon[179668]: failed to encode map e363 with expected crc 2023-09-23T18:09:28.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:28 smithi055 ceph-mon[184174]: pgmap v886: 119 pgs: 6 creating+peering, 10 unknown, 103 active+clean; 4.3 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:28.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:28 smithi055 ceph-mon[184174]: failed to encode map e363 with expected crc 2023-09-23T18:09:28.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:28 smithi055 ceph-mon[184174]: osdmap e363: 8 total, 8 up, 7 in 2023-09-23T18:09:28.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:28 smithi055 ceph-mon[184174]: failed to encode map e363 with expected crc 2023-09-23T18:09:28.409 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:28.499 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:28.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:28.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:28.585 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:28.603 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:28.636 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:28.677 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:28.679 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:28.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:28.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:28.808 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:28.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:28.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:28.844 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:28.881 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:09:28.881 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:09:28.881 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:09:28.881 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:09:28.945 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:28.977 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:29.056 INFO:teuthology.orchestra.run.smithi055.stderr:enabled application 'rados' on pool 'unique_pool_3' 2023-09-23T18:09:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/871913750' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:09:29.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/871913750' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:09:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/871913750' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:09:29.289 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:29.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:29.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:29.330 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:09:29.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:29.422 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:29.452 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:09:29.454 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:29.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:29.456 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:29.557 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:29.586 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:29.725 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:29.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:29.730 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:29.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:29.858 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:29.891 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:29.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:29.894 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:29.995 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:30.022 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:30.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:30.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:30.057 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:30.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:30 smithi055 ceph-mon[179668]: pgmap v888: 119 pgs: 6 creating+peering, 113 active+clean; 4.3 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:30 smithi055 ceph-mon[179668]: failed to encode map e364 with expected crc 2023-09-23T18:09:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/871913750' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:09:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:30 smithi055 ceph-mon[179668]: osdmap e364: 8 total, 8 up, 7 in 2023-09-23T18:09:30.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:30 smithi055 ceph-mon[184174]: pgmap v888: 119 pgs: 6 creating+peering, 113 active+clean; 4.3 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:30 smithi055 ceph-mon[184174]: failed to encode map e364 with expected crc 2023-09-23T18:09:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/871913750' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:09:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:30 smithi055 ceph-mon[184174]: osdmap e364: 8 total, 8 up, 7 in 2023-09-23T18:09:30.181 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:30 smithi099 ceph-mon[167076]: pgmap v888: 119 pgs: 6 creating+peering, 113 active+clean; 4.3 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:30.181 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:30 smithi099 ceph-mon[167076]: failed to encode map e364 with expected crc 2023-09-23T18:09:30.181 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/871913750' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_3", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:09:30.182 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:30 smithi099 ceph-mon[167076]: osdmap e364: 8 total, 8 up, 7 in 2023-09-23T18:09:30.184 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:30.215 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:30.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:30.218 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:30.319 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:30.350 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:30.385 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:30.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:30.388 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:30.390 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:30.431 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:30.489 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:30.517 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:30.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:30.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:30.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:30.585 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:09:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:09:30] "GET /metrics HTTP/1.1" 200 36601 "" "Prometheus/2.43.0" 2023-09-23T18:09:30.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:30.680 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:30.713 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:30.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:30.716 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:30.812 DEBUG:teuthology.orchestra.run.smithi055:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_3 bench 90 write' 2023-09-23T18:09:30.814 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T18:09:30.814 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T18:09:30.817 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:30.845 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:30.879 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:30.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:30.882 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:30.983 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:30.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:hints = 1 2023-09-23T18:09:30.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T18:09:30.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object prefix: benchmark_data_smithi055_346788 2023-09-23T18:09:30.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:09:30.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 0 0 0 0 0 0 - 0 2023-09-23T18:09:31.036 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:31.083 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:09:31.192 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:31.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:31.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:31.299 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:31.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2081207450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:31.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:31 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:31.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2081207450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:31.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:31 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:31.346 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2081207450' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:31.346 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:31 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:31.351 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:31.659 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:31.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:31.666 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:31.767 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:31.822 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:31.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:31.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:31.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:31.977 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:31.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 1 16 3617 3601 225.051 225.062 0.0036966 0.00442411 2023-09-23T18:09:32.024 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:32.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:32.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:32.092 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:32.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:32.242 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:32.265 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:09:32.299 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:32.300 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:09:32.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:32.304 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:32.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:32 smithi099 ceph-mon[167076]: pgmap v890: 119 pgs: 119 active+clean; 4.3 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:32.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:32 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:32.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:32 smithi099 ceph-mon[167076]: failed to encode map e365 with expected crc 2023-09-23T18:09:32.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:32 smithi099 ceph-mon[167076]: failed to encode map e365 with expected crc 2023-09-23T18:09:32.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:32 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:32.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:32 smithi099 ceph-mon[167076]: osdmap e365: 8 total, 8 up, 7 in 2023-09-23T18:09:32.333 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:09:32.371 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:09:32.405 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:32.417 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:09:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[179668]: pgmap v890: 119 pgs: 119 active+clean; 4.3 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[179668]: failed to encode map e365 with expected crc 2023-09-23T18:09:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[179668]: failed to encode map e365 with expected crc 2023-09-23T18:09:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:32.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[179668]: osdmap e365: 8 total, 8 up, 7 in 2023-09-23T18:09:32.431 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:09:32.431 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:09:32.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[184174]: pgmap v890: 119 pgs: 119 active+clean; 4.3 MiB data, 3.1 GiB used, 623 GiB / 626 GiB avail 2023-09-23T18:09:32.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:32.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[184174]: failed to encode map e365 with expected crc 2023-09-23T18:09:32.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[184174]: failed to encode map e365 with expected crc 2023-09-23T18:09:32.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:32.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:32 smithi055 ceph-mon[184174]: osdmap e365: 8 total, 8 up, 7 in 2023-09-23T18:09:32.452 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:09:32.459 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:32.486 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:09:32.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:32.523 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:09:32.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:32.524 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:32.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:09:32.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:09:32.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:09:32.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:09:32.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:09:32.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:09:32.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:09:32.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:09:32.625 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:32.680 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:32.732 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 6, 7, 5, 4, 1] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:09:32.733 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:09:32.733 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T18:09:32.733 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:09:32.787 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:32.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:32.801 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:32.825 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:09:32.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:32.953 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:32.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 2 16 6786 6770 211.548 198.062 0.00625571 0.00471421 2023-09-23T18:09:33.015 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:33.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:33.020 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:33.084 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2415927796' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:33.121 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:33.210 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:33.270 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:33.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:33.274 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:33.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:33 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2415927796' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:33.375 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:33.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2415927796' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:33.440 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:33.496 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:33.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:33.502 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:33.580 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:33.603 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:33.650 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:33.726 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:33.728 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:33.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:33.730 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:33.831 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:33.878 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:33.896 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:09:33.936 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:33.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:33.947 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:33.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 3 16 9777 9761 203.328 186.938 0.00225378 0.00491221 2023-09-23T18:09:34.048 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:34.128 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:34.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:34.349 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:34 smithi099 ceph-mon[167076]: pgmap v892: 119 pgs: 119 active+clean; 209 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 34 MiB/s wr, 542 op/s 2023-09-23T18:09:34.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:34.350 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:34.371 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:34 smithi055 ceph-mon[179668]: pgmap v892: 119 pgs: 119 active+clean; 209 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 34 MiB/s wr, 542 op/s 2023-09-23T18:09:34.371 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:34 smithi055 ceph-mon[184174]: pgmap v892: 119 pgs: 119 active+clean; 209 MiB data, 3.2 GiB used, 623 GiB / 626 GiB avail; 34 MiB/s wr, 542 op/s 2023-09-23T18:09:34.451 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:34.512 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:34.568 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:34.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:34.573 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:34.674 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:34.723 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:34.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 4 16 12580 12564 196.283 175.188 0.00937355 0.00508724 2023-09-23T18:09:35.075 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:35.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:35.080 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:35.123 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:09:35.182 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:35.222 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:35.275 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:35.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:35.279 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:35.279 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:35.380 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:35.465 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:35.475 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:09:35.481 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:09:35.628 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:35.637 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:35.637 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:35.738 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:35.797 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:35.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 5 16 14839 14823 185.261 141.188 0.00189831 0.00539094 2023-09-23T18:09:36.127 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:09:36.127 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:09:36.156 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:36.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:36.162 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:36.202 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:36.263 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:36.317 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:36.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:36 smithi055 ceph-mon[179668]: pgmap v893: 119 pgs: 119 active+clean; 236 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 30 MiB/s wr, 470 op/s 2023-09-23T18:09:36.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:36 smithi055 ceph-mon[184174]: pgmap v893: 119 pgs: 119 active+clean; 236 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 30 MiB/s wr, 470 op/s 2023-09-23T18:09:36.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:36 smithi099 ceph-mon[167076]: pgmap v893: 119 pgs: 119 active+clean; 236 MiB data, 3.5 GiB used, 622 GiB / 626 GiB avail; 30 MiB/s wr, 470 op/s 2023-09-23T18:09:36.993 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 6 16 17210 17194 179.081 148.188 0.00854113 0.00557949 2023-09-23T18:09:37.162 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:37.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:37.168 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:37.256 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":893,"stamp":"2023-09-23T18:09:34.861230+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":247248112,"num_objects":3731,"num_object_clones":0,"num_object_copies":11193,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3731,"num_whiteouts":0,"num_read":5985,"num_read_kb":12209,"num_write":4628,"num_write_kb":249946,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":184,"num_bytes_recovered":4649044,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":9726,"ondisk_log_size":9726,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":373,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":3694320,"kb_used_data":1210616,"kb_used_omap":43,"kb_used_meta":2483604,"kb_avail":652521744,"statfs":{"total":671965249536,"available":668182265856,"internally_reserved":0,"allocated":1239670784,"data_stored":1229485846,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":44376,"internal_metadata":2543211176},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":242700619,"num_objects":3703,"num_object_clones":0,"num_object_copies":11109,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3703,"num_whiteouts":0,"num_read":-38,"num_read_kb":-70,"num_write":3686,"num_write_kb":236873,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-2,"num_bytes_recovered":-243402,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.826623"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":702,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521703+0000","last_change":"2023-09-23T18:09:16.712699+0000","last_active":"2023-09-23T18:09:29.521703+0000","last_peered":"2023-09-23T18:09:29.521703+0000","last_clean":"2023-09-23T18:09:29.521703+0000","last_became_active":"2023-09-23T18:05:15.086678+0000","last_became_peered":"2023-09-23T18:05:15.086678+0000","last_unstale":"2023-09-23T18:09:29.521703+0000","last_undegraded":"2023-09-23T18:09:29.521703+0000","last_fullsized":"2023-09-23T18:09:29.521703+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.712627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.712627+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:45.803427+0000","scrub_duration":0.0024013070000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":174,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420299+0000","last_change":"2023-09-23T18:06:05.377123+0000","last_active":"2023-09-23T18:09:28.420299+0000","last_peered":"2023-09-23T18:09:28.420299+0000","last_clean":"2023-09-23T18:09:28.420299+0000","last_became_active":"2023-09-23T18:06:05.376735+0000","last_became_peered":"2023-09-23T18:06:05.376735+0000","last_unstale":"2023-09-23T18:09:28.420299+0000","last_undegraded":"2023-09-23T18:09:28.420299+0000","last_fullsized":"2023-09-23T18:09:28.420299+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:11.555818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":242,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521330+0000","last_change":"2023-09-23T18:06:51.071290+0000","last_active":"2023-09-23T18:09:29.521330+0000","last_peered":"2023-09-23T18:09:29.521330+0000","last_clean":"2023-09-23T18:09:29.521330+0000","last_became_active":"2023-09-23T18:06:51.070948+0000","last_became_peered":"2023-09-23T18:06:51.070948+0000","last_unstale":"2023-09-23T18:09:29.521330+0000","last_undegraded":"2023-09-23T18:09:29.521330+0000","last_fullsized":"2023-09-23T18:09:29.521330+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:52.647972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":710,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521736+0000","last_change":"2023-09-23T18:06:54.065593+0000","last_active":"2023-09-23T18:09:29.521736+0000","last_peered":"2023-09-23T18:09:29.521736+0000","last_clean":"2023-09-23T18:09:29.521736+0000","last_became_active":"2023-09-23T18:06:54.065193+0000","last_became_peered":"2023-09-23T18:06:54.065193+0000","last_unstale":"2023-09-23T18:09:29.521736+0000","last_undegraded":"2023-09-23T18:09:29.521736+0000","last_fullsized":"2023-09-23T18:09:29.521736+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:05.792712+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,2],"acting":[5,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":81,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521801+0000","last_change":"2023-09-23T18:06:05.370046+0000","last_active":"2023-09-23T18:09:29.521801+0000","last_peered":"2023-09-23T18:09:29.521801+0000","last_clean":"2023-09-23T18:09:29.521801+0000","last_became_active":"2023-09-23T18:06:05.369714+0000","last_became_peered":"2023-09-23T18:06:05.369714+0000","last_unstale":"2023-09-23T18:09:29.521801+0000","last_undegraded":"2023-09-23T18:09:29.521801+0000","last_fullsized":"2023-09-23T18:09:29.521801+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:47.453671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":374,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522242+0000","last_change":"2023-09-23T18:06:51.070985+0000","last_active":"2023-09-23T18:09:29.522242+0000","last_peered":"2023-09-23T18:09:29.522242+0000","last_clean":"2023-09-23T18:09:29.522242+0000","last_became_active":"2023-09-23T18:06:51.070671+0000","last_became_peered":"2023-09-23T18:06:51.070671+0000","last_unstale":"2023-09-23T18:09:29.522242+0000","last_undegraded":"2023-09-23T18:09:29.522242+0000","last_fullsized":"2023-09-23T18:09:29.522242+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:14.820905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":701,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420371+0000","last_change":"2023-09-23T18:06:08.439343+0000","last_active":"2023-09-23T18:09:28.420371+0000","last_peered":"2023-09-23T18:09:28.420371+0000","last_clean":"2023-09-23T18:09:28.420371+0000","last_became_active":"2023-09-23T18:06:08.437702+0000","last_became_peered":"2023-09-23T18:06:08.437702+0000","last_unstale":"2023-09-23T18:09:28.420371+0000","last_undegraded":"2023-09-23T18:09:28.420371+0000","last_fullsized":"2023-09-23T18:09:28.420371+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:02.398105+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":245,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521086+0000","last_change":"2023-09-23T18:06:51.087219+0000","last_active":"2023-09-23T18:09:29.521086+0000","last_peered":"2023-09-23T18:09:29.521086+0000","last_clean":"2023-09-23T18:09:29.521086+0000","last_became_active":"2023-09-23T18:06:51.086885+0000","last_became_peered":"2023-09-23T18:06:51.086885+0000","last_unstale":"2023-09-23T18:09:29.521086+0000","last_undegraded":"2023-09-23T18:09:29.521086+0000","last_fullsized":"2023-09-23T18:09:29.521086+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:16.254186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":314,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037570+0000","last_change":"2023-09-23T18:06:05.372350+0000","last_active":"2023-09-23T18:09:27.037570+0000","last_peered":"2023-09-23T18:09:27.037570+0000","last_clean":"2023-09-23T18:09:27.037570+0000","last_became_active":"2023-09-23T18:06:05.372026+0000","last_became_peered":"2023-09-23T18:06:05.372026+0000","last_unstale":"2023-09-23T18:09:27.037570+0000","last_undegraded":"2023-09-23T18:09:27.037570+0000","last_fullsized":"2023-09-23T18:09:27.037570+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:37.063301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":745,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522071+0000","last_change":"2023-09-23T18:06:08.433828+0000","last_active":"2023-09-23T18:09:29.522071+0000","last_peered":"2023-09-23T18:09:29.522071+0000","last_clean":"2023-09-23T18:09:29.522071+0000","last_became_active":"2023-09-23T18:06:08.433445+0000","last_became_peered":"2023-09-23T18:06:08.433445+0000","last_unstale":"2023-09-23T18:09:29.522071+0000","last_undegraded":"2023-09-23T18:09:29.522071+0000","last_fullsized":"2023-09-23T18:09:29.522071+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:49.816081+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":309,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.523059+0000","last_change":"2023-09-23T18:09:17.597665+0000","last_active":"2023-09-23T18:09:29.523059+0000","last_peered":"2023-09-23T18:09:29.523059+0000","last_clean":"2023-09-23T18:09:29.523059+0000","last_became_active":"2023-09-23T18:09:17.597324+0000","last_became_peered":"2023-09-23T18:09:17.597324+0000","last_unstale":"2023-09-23T18:09:29.523059+0000","last_undegraded":"2023-09-23T18:09:29.523059+0000","last_fullsized":"2023-09-23T18:09:29.523059+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:59.415179+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":233,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.052733+0000","last_change":"2023-09-23T18:06:05.364021+0000","last_active":"2023-09-23T18:09:29.052733+0000","last_peered":"2023-09-23T18:09:29.052733+0000","last_clean":"2023-09-23T18:09:29.052733+0000","last_became_active":"2023-09-23T18:06:05.363740+0000","last_became_peered":"2023-09-23T18:06:05.363740+0000","last_unstale":"2023-09-23T18:09:29.052733+0000","last_undegraded":"2023-09-23T18:09:29.052733+0000","last_fullsized":"2023-09-23T18:09:29.052733+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:55.662682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":653,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521528+0000","last_change":"2023-09-23T18:09:23.589835+0000","last_active":"2023-09-23T18:09:29.521528+0000","last_peered":"2023-09-23T18:09:29.521528+0000","last_clean":"2023-09-23T18:09:29.521528+0000","last_became_active":"2023-09-23T18:08:10.517840+0000","last_became_peered":"2023-09-23T18:08:10.517840+0000","last_unstale":"2023-09-23T18:09:29.521528+0000","last_undegraded":"2023-09-23T18:09:29.521528+0000","last_fullsized":"2023-09-23T18:09:29.521528+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:26.034507+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":357,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522798+0000","last_change":"2023-09-23T18:06:05.371032+0000","last_active":"2023-09-23T18:09:29.522798+0000","last_peered":"2023-09-23T18:09:29.522798+0000","last_clean":"2023-09-23T18:09:29.522798+0000","last_became_active":"2023-09-23T18:06:05.370695+0000","last_became_peered":"2023-09-23T18:06:05.370695+0000","last_unstale":"2023-09-23T18:09:29.522798+0000","last_undegraded":"2023-09-23T18:09:29.522798+0000","last_fullsized":"2023-09-23T18:09:29.522798+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:12.628753+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":149,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037712+0000","last_change":"2023-09-23T18:07:49.223713+0000","last_active":"2023-09-23T18:09:27.037712+0000","last_peered":"2023-09-23T18:09:27.037712+0000","last_clean":"2023-09-23T18:09:27.037712+0000","last_became_active":"2023-09-23T18:07:49.223409+0000","last_became_peered":"2023-09-23T18:07:49.223409+0000","last_unstale":"2023-09-23T18:09:27.037712+0000","last_undegraded":"2023-09-23T18:09:27.037712+0000","last_fullsized":"2023-09-23T18:09:27.037712+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:12.387318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":709,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.523111+0000","last_change":"2023-09-23T18:06:54.059837+0000","last_active":"2023-09-23T18:09:29.523111+0000","last_peered":"2023-09-23T18:09:29.523111+0000","last_clean":"2023-09-23T18:09:29.523111+0000","last_became_active":"2023-09-23T18:06:54.059489+0000","last_became_peered":"2023-09-23T18:06:54.059489+0000","last_unstale":"2023-09-23T18:09:29.523111+0000","last_undegraded":"2023-09-23T18:09:29.523111+0000","last_fullsized":"2023-09-23T18:09:29.523111+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.652623+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.652623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:28.333390+0000","scrub_duration":0.00648549,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":130,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521986+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:09:29.521986+0000","last_peered":"2023-09-23T18:09:29.521986+0000","last_clean":"2023-09-23T18:09:29.521986+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:09:29.521986+0000","last_undegraded":"2023-09-23T18:09:29.521986+0000","last_fullsized":"2023-09-23T18:09:29.521986+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":91,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521464+0000","last_change":"2023-09-23T18:06:05.363236+0000","last_active":"2023-09-23T18:09:29.521464+0000","last_peered":"2023-09-23T18:09:29.521464+0000","last_clean":"2023-09-23T18:09:29.521464+0000","last_became_active":"2023-09-23T18:06:05.362947+0000","last_became_peered":"2023-09-23T18:06:05.362947+0000","last_unstale":"2023-09-23T18:09:29.521464+0000","last_undegraded":"2023-09-23T18:09:29.521464+0000","last_fullsized":"2023-09-23T18:09:29.521464+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:32.816159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":730,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521641+0000","last_change":"2023-09-23T18:09:20.622230+0000","last_active":"2023-09-23T18:09:29.521641+0000","last_peered":"2023-09-23T18:09:29.521641+0000","last_clean":"2023-09-23T18:09:29.521641+0000","last_became_active":"2023-09-23T18:09:20.621845+0000","last_became_peered":"2023-09-23T18:09:20.621845+0000","last_unstale":"2023-09-23T18:09:29.521641+0000","last_undegraded":"2023-09-23T18:09:29.521641+0000","last_fullsized":"2023-09-23T18:09:29.521641+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":156,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.419666+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:09:28.419666+0000","last_peered":"2023-09-23T18:09:28.419666+0000","last_clean":"2023-09-23T18:09:28.419666+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:09:28.419666+0000","last_undegraded":"2023-09-23T18:09:28.419666+0000","last_fullsized":"2023-09-23T18:09:28.419666+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":174,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420883+0000","last_change":"2023-09-23T18:06:05.375073+0000","last_active":"2023-09-23T18:09:28.420883+0000","last_peered":"2023-09-23T18:09:28.420883+0000","last_clean":"2023-09-23T18:09:28.420883+0000","last_became_active":"2023-09-23T18:06:05.374752+0000","last_became_peered":"2023-09-23T18:06:05.374752+0000","last_unstale":"2023-09-23T18:09:28.420883+0000","last_undegraded":"2023-09-23T18:09:28.420883+0000","last_fullsized":"2023-09-23T18:09:28.420883+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:09:06.839328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":678,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522214+0000","last_change":"2023-09-23T18:09:24.869654+0000","last_active":"2023-09-23T18:09:29.522214+0000","last_peered":"2023-09-23T18:09:29.522214+0000","last_clean":"2023-09-23T18:09:29.522214+0000","last_became_active":"2023-09-23T18:07:00.037771+0000","last_became_peered":"2023-09-23T18:07:00.037771+0000","last_unstale":"2023-09-23T18:09:29.522214+0000","last_undegraded":"2023-09-23T18:09:29.522214+0000","last_fullsized":"2023-09-23T18:09:29.522214+0000","mapping_epoch":312,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.869576+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:46.164137+0000","scrub_duration":0.001769587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":293,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521239+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:09:29.521239+0000","last_peered":"2023-09-23T18:09:29.521239+0000","last_clean":"2023-09-23T18:09:29.521239+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:09:29.521239+0000","last_undegraded":"2023-09-23T18:09:29.521239+0000","last_fullsized":"2023-09-23T18:09:29.521239+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":180,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.520979+0000","last_change":"2023-09-23T18:07:49.229808+0000","last_active":"2023-09-23T18:09:29.520979+0000","last_peered":"2023-09-23T18:09:29.520979+0000","last_clean":"2023-09-23T18:09:29.520979+0000","last_became_active":"2023-09-23T18:07:49.229526+0000","last_became_peered":"2023-09-23T18:07:49.229526+0000","last_unstale":"2023-09-23T18:09:29.520979+0000","last_undegraded":"2023-09-23T18:09:29.520979+0000","last_fullsized":"2023-09-23T18:09:29.520979+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:01.224432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":682,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521412+0000","last_change":"2023-09-23T18:05:15.091218+0000","last_active":"2023-09-23T18:09:29.521412+0000","last_peered":"2023-09-23T18:09:29.521412+0000","last_clean":"2023-09-23T18:09:29.521412+0000","last_became_active":"2023-09-23T18:05:15.090420+0000","last_became_peered":"2023-09-23T18:05:15.090420+0000","last_unstale":"2023-09-23T18:09:29.521412+0000","last_undegraded":"2023-09-23T18:09:29.521412+0000","last_fullsized":"2023-09-23T18:09:29.521412+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:28:57.754408+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":372,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.520935+0000","last_change":"2023-09-23T18:06:51.065050+0000","last_active":"2023-09-23T18:09:29.520935+0000","last_peered":"2023-09-23T18:09:29.520935+0000","last_clean":"2023-09-23T18:09:29.520935+0000","last_became_active":"2023-09-23T18:06:51.064774+0000","last_became_peered":"2023-09-23T18:06:51.064774+0000","last_unstale":"2023-09-23T18:09:29.520935+0000","last_undegraded":"2023-09-23T18:09:29.520935+0000","last_fullsized":"2023-09-23T18:09:29.520935+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:31.508499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":342,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522918+0000","last_change":"2023-09-23T18:02:19.245928+0000","last_active":"2023-09-23T18:09:29.522918+0000","last_peered":"2023-09-23T18:09:29.522918+0000","last_clean":"2023-09-23T18:09:29.522918+0000","last_became_active":"2023-09-23T18:02:19.245647+0000","last_became_peered":"2023-09-23T18:02:19.245647+0000","last_unstale":"2023-09-23T18:09:29.522918+0000","last_undegraded":"2023-09-23T18:09:29.522918+0000","last_fullsized":"2023-09-23T18:09:29.522918+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:04:45.876201+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":693,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522117+0000","last_change":"2023-09-23T18:09:20.618243+0000","last_active":"2023-09-23T18:09:29.522117+0000","last_peered":"2023-09-23T18:09:29.522117+0000","last_clean":"2023-09-23T18:09:29.522117+0000","last_became_active":"2023-09-23T18:09:20.617859+0000","last_became_peered":"2023-09-23T18:09:20.617859+0000","last_unstale":"2023-09-23T18:09:29.522117+0000","last_undegraded":"2023-09-23T18:09:29.522117+0000","last_fullsized":"2023-09-23T18:09:29.522117+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":5,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17890,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":15,"num_read_kb":28,"num_write":7,"num_write_kb":73,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":53670,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":154,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420222+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:09:28.420222+0000","last_peered":"2023-09-23T18:09:28.420222+0000","last_clean":"2023-09-23T18:09:28.420222+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:09:28.420222+0000","last_undegraded":"2023-09-23T18:09:28.420222+0000","last_fullsized":"2023-09-23T18:09:28.420222+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":304,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.052951+0000","last_change":"2023-09-23T18:06:51.125883+0000","last_active":"2023-09-23T18:09:29.052951+0000","last_peered":"2023-09-23T18:09:29.052951+0000","last_clean":"2023-09-23T18:09:29.052951+0000","last_became_active":"2023-09-23T18:06:51.119298+0000","last_became_peered":"2023-09-23T18:06:51.119298+0000","last_unstale":"2023-09-23T18:09:29.052951+0000","last_undegraded":"2023-09-23T18:09:29.052951+0000","last_fullsized":"2023-09-23T18:09:29.052951+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:12.571367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":702,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521673+0000","last_change":"2023-09-23T18:06:08.430910+0000","last_active":"2023-09-23T18:09:29.521673+0000","last_peered":"2023-09-23T18:09:29.521673+0000","last_clean":"2023-09-23T18:09:29.521673+0000","last_became_active":"2023-09-23T18:06:08.430380+0000","last_became_peered":"2023-09-23T18:06:08.430380+0000","last_unstale":"2023-09-23T18:09:29.521673+0000","last_undegraded":"2023-09-23T18:09:29.521673+0000","last_fullsized":"2023-09-23T18:09:29.521673+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:40.113807+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":340,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.036960+0000","last_change":"2023-09-23T18:06:51.128655+0000","last_active":"2023-09-23T18:09:27.036960+0000","last_peered":"2023-09-23T18:09:27.036960+0000","last_clean":"2023-09-23T18:09:27.036960+0000","last_became_active":"2023-09-23T18:06:51.057904+0000","last_became_peered":"2023-09-23T18:06:51.057904+0000","last_unstale":"2023-09-23T18:09:27.036960+0000","last_undegraded":"2023-09-23T18:09:27.036960+0000","last_fullsized":"2023-09-23T18:09:27.036960+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:22.029660+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,2],"acting":[6,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":219,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521924+0000","last_change":"2023-09-23T18:06:45.919283+0000","last_active":"2023-09-23T18:09:29.521924+0000","last_peered":"2023-09-23T18:09:29.521924+0000","last_clean":"2023-09-23T18:09:29.521924+0000","last_became_active":"2023-09-23T18:06:45.918219+0000","last_became_peered":"2023-09-23T18:06:45.918219+0000","last_unstale":"2023-09-23T18:09:29.521924+0000","last_undegraded":"2023-09-23T18:09:29.521924+0000","last_fullsized":"2023-09-23T18:09:29.521924+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:21.242033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.4","version":"365'354","reported_seq":366,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T18:09:32.741319+0000","last_change":"2023-09-23T18:09:26.040060+0000","last_active":"2023-09-23T18:09:32.741319+0000","last_peered":"2023-09-23T18:09:32.741319+0000","last_clean":"2023-09-23T18:09:32.741319+0000","last_became_active":"2023-09-23T18:09:26.039778+0000","last_became_peered":"2023-09-23T18:09:26.039778+0000","last_unstale":"2023-09-23T18:09:32.741319+0000","last_undegraded":"2023-09-23T18:09:32.741319+0000","last_fullsized":"2023-09-23T18:09:32.741319+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":354,"log_dups_size":0,"ondisk_log_size":354,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:53:59.827893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23199744,"num_objects":354,"num_object_clones":0,"num_object_copies":1062,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":354,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":354,"num_write_kb":22656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":745,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521891+0000","last_change":"2023-09-23T18:06:08.440949+0000","last_active":"2023-09-23T18:09:29.521891+0000","last_peered":"2023-09-23T18:09:29.521891+0000","last_clean":"2023-09-23T18:09:29.521891+0000","last_became_active":"2023-09-23T18:06:08.440543+0000","last_became_peered":"2023-09-23T18:06:08.440543+0000","last_unstale":"2023-09-23T18:09:29.521891+0000","last_undegraded":"2023-09-23T18:09:29.521891+0000","last_fullsized":"2023-09-23T18:09:29.521891+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:39:30.193370+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":307,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522891+0000","last_change":"2023-09-23T18:06:45.947590+0000","last_active":"2023-09-23T18:09:29.522891+0000","last_peered":"2023-09-23T18:09:29.522891+0000","last_clean":"2023-09-23T18:09:29.522891+0000","last_became_active":"2023-09-23T18:06:45.920061+0000","last_became_peered":"2023-09-23T18:06:45.920061+0000","last_unstale":"2023-09-23T18:09:29.522891+0000","last_undegraded":"2023-09-23T18:09:29.522891+0000","last_fullsized":"2023-09-23T18:09:29.522891+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:31.645928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":353,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521225+0000","last_change":"2023-09-23T18:02:47.956429+0000","last_active":"2023-09-23T18:09:29.521225+0000","last_peered":"2023-09-23T18:09:29.521225+0000","last_clean":"2023-09-23T18:09:29.521225+0000","last_became_active":"2023-09-23T18:02:47.956104+0000","last_became_peered":"2023-09-23T18:02:47.956104+0000","last_unstale":"2023-09-23T18:09:29.521225+0000","last_undegraded":"2023-09-23T18:09:29.521225+0000","last_fullsized":"2023-09-23T18:09:29.521225+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:28.620088+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"0'0","reported_seq":11,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037357+0000","last_change":"2023-09-23T18:09:26.035691+0000","last_active":"2023-09-23T18:09:27.037357+0000","last_peered":"2023-09-23T18:09:27.037357+0000","last_clean":"2023-09-23T18:09:27.037357+0000","last_became_active":"2023-09-23T18:09:26.035411+0000","last_became_peered":"2023-09-23T18:09:26.035411+0000","last_unstale":"2023-09-23T18:09:27.037357+0000","last_undegraded":"2023-09-23T18:09:27.037357+0000","last_fullsized":"2023-09-23T18:09:27.037357+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:51.634243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":448,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037637+0000","last_change":"2023-09-23T18:09:20.624452+0000","last_active":"2023-09-23T18:09:27.037637+0000","last_peered":"2023-09-23T18:09:27.037637+0000","last_clean":"2023-09-23T18:09:27.037637+0000","last_became_active":"2023-09-23T18:09:20.624059+0000","last_became_peered":"2023-09-23T18:09:20.624059+0000","last_unstale":"2023-09-23T18:09:27.037637+0000","last_undegraded":"2023-09-23T18:09:27.037637+0000","last_fullsized":"2023-09-23T18:09:27.037637+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":4,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T17:43:05.158540+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":354515,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":322,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522419+0000","last_change":"2023-09-23T18:06:05.379591+0000","last_active":"2023-09-23T18:09:29.522419+0000","last_peered":"2023-09-23T18:09:29.522419+0000","last_clean":"2023-09-23T18:09:29.522419+0000","last_became_active":"2023-09-23T18:06:05.370176+0000","last_became_peered":"2023-09-23T18:06:05.370176+0000","last_unstale":"2023-09-23T18:09:29.522419+0000","last_undegraded":"2023-09-23T18:09:29.522419+0000","last_fullsized":"2023-09-23T18:09:29.522419+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:08.768138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":394,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420518+0000","last_change":"2023-09-23T18:06:45.911012+0000","last_active":"2023-09-23T18:09:28.420518+0000","last_peered":"2023-09-23T18:09:28.420518+0000","last_clean":"2023-09-23T18:09:28.420518+0000","last_became_active":"2023-09-23T18:06:45.910722+0000","last_became_peered":"2023-09-23T18:06:45.910722+0000","last_unstale":"2023-09-23T18:09:28.420518+0000","last_undegraded":"2023-09-23T18:09:28.420518+0000","last_fullsized":"2023-09-23T18:09:28.420518+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:19.150719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"0'0","reported_seq":11,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037308+0000","last_change":"2023-09-23T18:09:26.033831+0000","last_active":"2023-09-23T18:09:27.037308+0000","last_peered":"2023-09-23T18:09:27.037308+0000","last_clean":"2023-09-23T18:09:27.037308+0000","last_became_active":"2023-09-23T18:09:26.033509+0000","last_became_peered":"2023-09-23T18:09:26.033509+0000","last_unstale":"2023-09-23T18:09:27.037308+0000","last_undegraded":"2023-09-23T18:09:27.037308+0000","last_fullsized":"2023-09-23T18:09:27.037308+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:13.557685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":535,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037664+0000","last_change":"2023-09-23T18:06:02.805384+0000","last_active":"2023-09-23T18:09:27.037664+0000","last_peered":"2023-09-23T18:09:27.037664+0000","last_clean":"2023-09-23T18:09:27.037664+0000","last_became_active":"2023-09-23T18:05:15.098234+0000","last_became_peered":"2023-09-23T18:05:15.098234+0000","last_unstale":"2023-09-23T18:09:27.037664+0000","last_undegraded":"2023-09-23T18:09:27.037664+0000","last_fullsized":"2023-09-23T18:09:27.037664+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:58.872644+0000","scrub_duration":0.0018893740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":231,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522239+0000","last_change":"2023-09-23T18:01:24.999629+0000","last_active":"2023-09-23T18:09:29.522239+0000","last_peered":"2023-09-23T18:09:29.522239+0000","last_clean":"2023-09-23T18:09:29.522239+0000","last_became_active":"2023-09-23T18:01:24.999324+0000","last_became_peered":"2023-09-23T18:01:24.999324+0000","last_unstale":"2023-09-23T18:09:29.522239+0000","last_undegraded":"2023-09-23T18:09:29.522239+0000","last_fullsized":"2023-09-23T18:09:29.522239+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:28.538461+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":339,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037180+0000","last_change":"2023-09-23T18:06:05.371447+0000","last_active":"2023-09-23T18:09:27.037180+0000","last_peered":"2023-09-23T18:09:27.037180+0000","last_clean":"2023-09-23T18:09:27.037180+0000","last_became_active":"2023-09-23T18:06:05.371065+0000","last_became_peered":"2023-09-23T18:06:05.371065+0000","last_unstale":"2023-09-23T18:09:27.037180+0000","last_undegraded":"2023-09-23T18:09:27.037180+0000","last_fullsized":"2023-09-23T18:09:27.037180+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.025791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"365'367","reported_seq":380,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T18:09:32.682008+0000","last_change":"2023-09-23T18:09:26.507113+0000","last_active":"2023-09-23T18:09:32.682008+0000","last_peered":"2023-09-23T18:09:32.682008+0000","last_clean":"2023-09-23T18:09:32.682008+0000","last_became_active":"2023-09-23T18:09:26.506876+0000","last_became_peered":"2023-09-23T18:09:26.506876+0000","last_unstale":"2023-09-23T18:09:32.682008+0000","last_undegraded":"2023-09-23T18:09:32.682008+0000","last_fullsized":"2023-09-23T18:09:32.682008+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":367,"log_dups_size":0,"ondisk_log_size":367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:54.009952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24051712,"num_objects":367,"num_object_clones":0,"num_object_copies":1101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":367,"num_write_kb":23488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":688,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.421303+0000","last_change":"2023-09-23T18:05:56.362746+0000","last_active":"2023-09-23T18:09:28.421303+0000","last_peered":"2023-09-23T18:09:28.421303+0000","last_clean":"2023-09-23T18:09:28.421303+0000","last_became_active":"2023-09-23T18:05:15.102072+0000","last_became_peered":"2023-09-23T18:05:15.102072+0000","last_unstale":"2023-09-23T18:09:28.421303+0000","last_undegraded":"2023-09-23T18:09:28.421303+0000","last_fullsized":"2023-09-23T18:09:28.421303+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.362652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.362652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:33.381515+0000","scrub_duration":0.0052581759999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":351,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522749+0000","last_change":"2023-09-23T18:06:51.069867+0000","last_active":"2023-09-23T18:09:29.522749+0000","last_peered":"2023-09-23T18:09:29.522749+0000","last_clean":"2023-09-23T18:09:29.522749+0000","last_became_active":"2023-09-23T18:06:51.069523+0000","last_became_peered":"2023-09-23T18:06:51.069523+0000","last_unstale":"2023-09-23T18:09:29.522749+0000","last_undegraded":"2023-09-23T18:09:29.522749+0000","last_fullsized":"2023-09-23T18:09:29.522749+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:40.321844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":309,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522028+0000","last_change":"2023-09-23T18:09:17.600213+0000","last_active":"2023-09-23T18:09:29.522028+0000","last_peered":"2023-09-23T18:09:29.522028+0000","last_clean":"2023-09-23T18:09:29.522028+0000","last_became_active":"2023-09-23T18:09:17.599866+0000","last_became_peered":"2023-09-23T18:09:17.599866+0000","last_unstale":"2023-09-23T18:09:29.522028+0000","last_undegraded":"2023-09-23T18:09:29.522028+0000","last_fullsized":"2023-09-23T18:09:29.522028+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:02:00.057700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"365'353","reported_seq":365,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T18:09:32.733496+0000","last_change":"2023-09-23T18:09:26.506653+0000","last_active":"2023-09-23T18:09:32.733496+0000","last_peered":"2023-09-23T18:09:32.733496+0000","last_clean":"2023-09-23T18:09:32.733496+0000","last_became_active":"2023-09-23T18:09:26.506328+0000","last_became_peered":"2023-09-23T18:09:26.506328+0000","last_unstale":"2023-09-23T18:09:32.733496+0000","last_undegraded":"2023-09-23T18:09:32.733496+0000","last_fullsized":"2023-09-23T18:09:32.733496+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":353,"log_dups_size":0,"ondisk_log_size":353,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:15.313982+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23134208,"num_objects":353,"num_object_clones":0,"num_object_copies":1059,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":353,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":353,"num_write_kb":22592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":737,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.520869+0000","last_change":"2023-09-23T18:06:54.059737+0000","last_active":"2023-09-23T18:09:29.520869+0000","last_peered":"2023-09-23T18:09:29.520869+0000","last_clean":"2023-09-23T18:09:29.520869+0000","last_became_active":"2023-09-23T18:06:54.059375+0000","last_became_peered":"2023-09-23T18:06:54.059375+0000","last_unstale":"2023-09-23T18:09:29.520869+0000","last_undegraded":"2023-09-23T18:09:29.520869+0000","last_fullsized":"2023-09-23T18:09:29.520869+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:04:12.644423+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":150,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.421099+0000","last_change":"2023-09-23T18:07:49.227201+0000","last_active":"2023-09-23T18:09:28.421099+0000","last_peered":"2023-09-23T18:09:28.421099+0000","last_clean":"2023-09-23T18:09:28.421099+0000","last_became_active":"2023-09-23T18:07:49.226878+0000","last_became_peered":"2023-09-23T18:07:49.226878+0000","last_unstale":"2023-09-23T18:09:28.421099+0000","last_undegraded":"2023-09-23T18:09:28.421099+0000","last_fullsized":"2023-09-23T18:09:28.421099+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:08.994668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":364,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.419666+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:09:28.419666+0000","last_peered":"2023-09-23T18:09:28.419666+0000","last_clean":"2023-09-23T18:09:28.419666+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:09:28.419666+0000","last_undegraded":"2023-09-23T18:09:28.419666+0000","last_fullsized":"2023-09-23T18:09:28.419666+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"365'352","reported_seq":365,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T18:09:32.678221+0000","last_change":"2023-09-23T18:09:26.039617+0000","last_active":"2023-09-23T18:09:32.678221+0000","last_peered":"2023-09-23T18:09:32.678221+0000","last_clean":"2023-09-23T18:09:32.678221+0000","last_became_active":"2023-09-23T18:09:26.039343+0000","last_became_peered":"2023-09-23T18:09:26.039343+0000","last_unstale":"2023-09-23T18:09:32.678221+0000","last_undegraded":"2023-09-23T18:09:32.678221+0000","last_fullsized":"2023-09-23T18:09:32.678221+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":352,"log_dups_size":0,"ondisk_log_size":352,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:33:03.222625+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23068672,"num_objects":352,"num_object_clones":0,"num_object_copies":1056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":352,"num_write_kb":22528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":705,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521509+0000","last_change":"2023-09-23T18:06:54.065597+0000","last_active":"2023-09-23T18:09:29.521509+0000","last_peered":"2023-09-23T18:09:29.521509+0000","last_clean":"2023-09-23T18:09:29.521509+0000","last_became_active":"2023-09-23T18:06:54.065217+0000","last_became_peered":"2023-09-23T18:06:54.065217+0000","last_unstale":"2023-09-23T18:09:29.521509+0000","last_undegraded":"2023-09-23T18:09:29.521509+0000","last_fullsized":"2023-09-23T18:09:29.521509+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:28:53.614664+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":333,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037327+0000","last_change":"2023-09-23T18:06:05.371350+0000","last_active":"2023-09-23T18:09:27.037327+0000","last_peered":"2023-09-23T18:09:27.037327+0000","last_clean":"2023-09-23T18:09:27.037327+0000","last_became_active":"2023-09-23T18:06:05.370877+0000","last_became_peered":"2023-09-23T18:06:05.370877+0000","last_unstale":"2023-09-23T18:09:27.037327+0000","last_undegraded":"2023-09-23T18:09:27.037327+0000","last_fullsized":"2023-09-23T18:09:27.037327+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:58.534355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":174,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.421043+0000","last_change":"2023-09-23T18:06:05.373385+0000","last_active":"2023-09-23T18:09:28.421043+0000","last_peered":"2023-09-23T18:09:28.421043+0000","last_clean":"2023-09-23T18:09:28.421043+0000","last_became_active":"2023-09-23T18:06:05.372982+0000","last_became_peered":"2023-09-23T18:06:05.372982+0000","last_unstale":"2023-09-23T18:09:28.421043+0000","last_undegraded":"2023-09-23T18:09:28.421043+0000","last_fullsized":"2023-09-23T18:09:28.421043+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:48.622115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"0'0","reported_seq":11,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420392+0000","last_change":"2023-09-23T18:09:26.039044+0000","last_active":"2023-09-23T18:09:28.420392+0000","last_peered":"2023-09-23T18:09:28.420392+0000","last_clean":"2023-09-23T18:09:28.420392+0000","last_became_active":"2023-09-23T18:09:26.038724+0000","last_became_peered":"2023-09-23T18:09:26.038724+0000","last_unstale":"2023-09-23T18:09:28.420392+0000","last_undegraded":"2023-09-23T18:09:28.420392+0000","last_fullsized":"2023-09-23T18:09:28.420392+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:19.254109+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":725,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522123+0000","last_change":"2023-09-23T18:09:20.617988+0000","last_active":"2023-09-23T18:09:29.522123+0000","last_peered":"2023-09-23T18:09:29.522123+0000","last_clean":"2023-09-23T18:09:29.522123+0000","last_became_active":"2023-09-23T18:09:20.617737+0000","last_became_peered":"2023-09-23T18:09:20.617737+0000","last_unstale":"2023-09-23T18:09:29.522123+0000","last_undegraded":"2023-09-23T18:09:29.522123+0000","last_fullsized":"2023-09-23T18:09:29.522123+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:06.411037+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,2],"acting":[1,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":154,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420304+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:09:28.420304+0000","last_peered":"2023-09-23T18:09:28.420304+0000","last_clean":"2023-09-23T18:09:28.420304+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:09:28.420304+0000","last_undegraded":"2023-09-23T18:09:28.420304+0000","last_fullsized":"2023-09-23T18:09:28.420304+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":384,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.421416+0000","last_change":"2023-09-23T18:06:05.373305+0000","last_active":"2023-09-23T18:09:28.421416+0000","last_peered":"2023-09-23T18:09:28.421416+0000","last_clean":"2023-09-23T18:09:28.421416+0000","last_became_active":"2023-09-23T18:06:05.372720+0000","last_became_peered":"2023-09-23T18:06:05.372720+0000","last_unstale":"2023-09-23T18:09:28.421416+0000","last_undegraded":"2023-09-23T18:09:28.421416+0000","last_fullsized":"2023-09-23T18:09:28.421416+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:44.934493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"365'355","reported_seq":368,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T18:09:32.664387+0000","last_change":"2023-09-23T18:09:26.037473+0000","last_active":"2023-09-23T18:09:32.664387+0000","last_peered":"2023-09-23T18:09:32.664387+0000","last_clean":"2023-09-23T18:09:32.664387+0000","last_became_active":"2023-09-23T18:09:26.037180+0000","last_became_peered":"2023-09-23T18:09:26.037180+0000","last_unstale":"2023-09-23T18:09:32.664387+0000","last_undegraded":"2023-09-23T18:09:32.664387+0000","last_fullsized":"2023-09-23T18:09:32.664387+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":355,"log_dups_size":0,"ondisk_log_size":355,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:29.949339+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23265280,"num_objects":355,"num_object_clones":0,"num_object_copies":1065,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":355,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":355,"num_write_kb":22720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":681,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521976+0000","last_change":"2023-09-23T18:05:53.509955+0000","last_active":"2023-09-23T18:09:29.521976+0000","last_peered":"2023-09-23T18:09:29.521976+0000","last_clean":"2023-09-23T18:09:29.521976+0000","last_became_active":"2023-09-23T18:05:15.089224+0000","last_became_peered":"2023-09-23T18:05:15.089224+0000","last_unstale":"2023-09-23T18:09:29.521976+0000","last_undegraded":"2023-09-23T18:09:29.521976+0000","last_fullsized":"2023-09-23T18:09:29.521976+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:37.917870+0000","scrub_duration":0.001603043,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":211,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521315+0000","last_change":"2023-09-23T18:06:45.904403+0000","last_active":"2023-09-23T18:09:29.521315+0000","last_peered":"2023-09-23T18:09:29.521315+0000","last_clean":"2023-09-23T18:09:29.521315+0000","last_became_active":"2023-09-23T18:06:45.904057+0000","last_became_peered":"2023-09-23T18:06:45.904057+0000","last_unstale":"2023-09-23T18:09:29.521315+0000","last_undegraded":"2023-09-23T18:09:29.521315+0000","last_fullsized":"2023-09-23T18:09:29.521315+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:43.213905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":91,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521879+0000","last_change":"2023-09-23T18:06:05.371661+0000","last_active":"2023-09-23T18:09:29.521879+0000","last_peered":"2023-09-23T18:09:29.521879+0000","last_clean":"2023-09-23T18:09:29.521879+0000","last_became_active":"2023-09-23T18:06:05.370998+0000","last_became_peered":"2023-09-23T18:06:05.370998+0000","last_unstale":"2023-09-23T18:09:29.521879+0000","last_undegraded":"2023-09-23T18:09:29.521879+0000","last_fullsized":"2023-09-23T18:09:29.521879+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:52.262543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"0'0","reported_seq":11,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.421226+0000","last_change":"2023-09-23T18:09:26.033882+0000","last_active":"2023-09-23T18:09:28.421226+0000","last_peered":"2023-09-23T18:09:28.421226+0000","last_clean":"2023-09-23T18:09:28.421226+0000","last_became_active":"2023-09-23T18:09:26.033595+0000","last_became_peered":"2023-09-23T18:09:26.033595+0000","last_unstale":"2023-09-23T18:09:28.421226+0000","last_undegraded":"2023-09-23T18:09:28.421226+0000","last_fullsized":"2023-09-23T18:09:28.421226+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:47.811446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":555,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037574+0000","last_change":"2023-09-23T18:06:54.193400+0000","last_active":"2023-09-23T18:09:27.037574+0000","last_peered":"2023-09-23T18:09:27.037574+0000","last_clean":"2023-09-23T18:09:27.037574+0000","last_became_active":"2023-09-23T18:06:54.193019+0000","last_became_peered":"2023-09-23T18:06:54.193019+0000","last_unstale":"2023-09-23T18:09:27.037574+0000","last_undegraded":"2023-09-23T18:09:27.037574+0000","last_fullsized":"2023-09-23T18:09:27.037574+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:48.156749+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":722,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521199+0000","last_change":"2023-09-23T18:09:20.620128+0000","last_active":"2023-09-23T18:09:29.521199+0000","last_peered":"2023-09-23T18:09:29.521199+0000","last_clean":"2023-09-23T18:09:29.521199+0000","last_became_active":"2023-09-23T18:09:20.619448+0000","last_became_peered":"2023-09-23T18:09:20.619448+0000","last_unstale":"2023-09-23T18:09:29.521199+0000","last_undegraded":"2023-09-23T18:09:29.521199+0000","last_fullsized":"2023-09-23T18:09:29.521199+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:10.514164+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":39,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":355,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522774+0000","last_change":"2023-09-23T18:06:51.074796+0000","last_active":"2023-09-23T18:09:29.522774+0000","last_peered":"2023-09-23T18:09:29.522774+0000","last_clean":"2023-09-23T18:09:29.522774+0000","last_became_active":"2023-09-23T18:06:51.074525+0000","last_became_peered":"2023-09-23T18:06:51.074525+0000","last_unstale":"2023-09-23T18:09:29.522774+0000","last_undegraded":"2023-09-23T18:09:29.522774+0000","last_fullsized":"2023-09-23T18:09:29.522774+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:48.941608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":91,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521385+0000","last_change":"2023-09-23T18:06:05.374173+0000","last_active":"2023-09-23T18:09:29.521385+0000","last_peered":"2023-09-23T18:09:29.521385+0000","last_clean":"2023-09-23T18:09:29.521385+0000","last_became_active":"2023-09-23T18:06:05.373835+0000","last_became_peered":"2023-09-23T18:06:05.373835+0000","last_unstale":"2023-09-23T18:09:29.521385+0000","last_undegraded":"2023-09-23T18:09:29.521385+0000","last_fullsized":"2023-09-23T18:09:29.521385+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:25.467488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"365'367","reported_seq":379,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T18:09:32.737709+0000","last_change":"2023-09-23T18:09:26.506753+0000","last_active":"2023-09-23T18:09:32.737709+0000","last_peered":"2023-09-23T18:09:32.737709+0000","last_clean":"2023-09-23T18:09:32.737709+0000","last_became_active":"2023-09-23T18:09:26.506424+0000","last_became_peered":"2023-09-23T18:09:26.506424+0000","last_unstale":"2023-09-23T18:09:32.737709+0000","last_undegraded":"2023-09-23T18:09:32.737709+0000","last_fullsized":"2023-09-23T18:09:32.737709+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":367,"log_dups_size":0,"ondisk_log_size":367,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:19.740572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24051712,"num_objects":367,"num_object_clones":0,"num_object_copies":1101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":367,"num_write_kb":23488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[5,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":702,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522406+0000","last_change":"2023-09-23T18:09:20.622229+0000","last_active":"2023-09-23T18:09:29.522406+0000","last_peered":"2023-09-23T18:09:29.522406+0000","last_clean":"2023-09-23T18:09:29.522406+0000","last_became_active":"2023-09-23T18:09:20.621888+0000","last_became_peered":"2023-09-23T18:09:20.621888+0000","last_unstale":"2023-09-23T18:09:29.522406+0000","last_undegraded":"2023-09-23T18:09:29.522406+0000","last_fullsized":"2023-09-23T18:09:29.522406+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:46.413980+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,4],"acting":[5,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1337,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521848+0000","last_change":"2023-09-23T18:06:08.426828+0000","last_active":"2023-09-23T18:09:29.521848+0000","last_peered":"2023-09-23T18:09:29.521848+0000","last_clean":"2023-09-23T18:09:29.521848+0000","last_became_active":"2023-09-23T18:06:08.426343+0000","last_became_peered":"2023-09-23T18:06:08.426343+0000","last_unstale":"2023-09-23T18:09:29.521848+0000","last_undegraded":"2023-09-23T18:09:29.521848+0000","last_fullsized":"2023-09-23T18:09:29.521848+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:38.924507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":164,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420959+0000","last_change":"2023-09-23T18:09:17.600475+0000","last_active":"2023-09-23T18:09:28.420959+0000","last_peered":"2023-09-23T18:09:28.420959+0000","last_clean":"2023-09-23T18:09:28.420959+0000","last_became_active":"2023-09-23T18:09:17.600166+0000","last_became_peered":"2023-09-23T18:09:17.600166+0000","last_unstale":"2023-09-23T18:09:28.420959+0000","last_undegraded":"2023-09-23T18:09:28.420959+0000","last_fullsized":"2023-09-23T18:09:28.420959+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:00.861734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":100,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521090+0000","last_change":"2023-09-23T18:06:51.088473+0000","last_active":"2023-09-23T18:09:29.521090+0000","last_peered":"2023-09-23T18:09:29.521090+0000","last_clean":"2023-09-23T18:09:29.521090+0000","last_became_active":"2023-09-23T18:06:51.088171+0000","last_became_peered":"2023-09-23T18:06:51.088171+0000","last_unstale":"2023-09-23T18:09:29.521090+0000","last_undegraded":"2023-09-23T18:09:29.521090+0000","last_fullsized":"2023-09-23T18:09:29.521090+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:45.860551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"0'0","reported_seq":11,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420466+0000","last_change":"2023-09-23T18:09:26.034398+0000","last_active":"2023-09-23T18:09:28.420466+0000","last_peered":"2023-09-23T18:09:28.420466+0000","last_clean":"2023-09-23T18:09:28.420466+0000","last_became_active":"2023-09-23T18:09:26.034124+0000","last_became_peered":"2023-09-23T18:09:26.034124+0000","last_unstale":"2023-09-23T18:09:28.420466+0000","last_undegraded":"2023-09-23T18:09:28.420466+0000","last_fullsized":"2023-09-23T18:09:28.420466+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:24.809053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":615,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420458+0000","last_change":"2023-09-23T18:09:20.623551+0000","last_active":"2023-09-23T18:09:28.420458+0000","last_peered":"2023-09-23T18:09:28.420458+0000","last_clean":"2023-09-23T18:09:28.420458+0000","last_became_active":"2023-09-23T18:09:20.623244+0000","last_became_peered":"2023-09-23T18:09:20.623244+0000","last_unstale":"2023-09-23T18:09:28.420458+0000","last_undegraded":"2023-09-23T18:09:28.420458+0000","last_fullsized":"2023-09-23T18:09:28.420458+0000","mapping_epoch":358,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":359,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:42.090222+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,5],"acting":[4,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":640,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521480+0000","last_change":"2023-09-23T18:09:17.626340+0000","last_active":"2023-09-23T18:09:29.521480+0000","last_peered":"2023-09-23T18:09:29.521480+0000","last_clean":"2023-09-23T18:09:29.521480+0000","last_became_active":"2023-09-23T18:09:17.592375+0000","last_became_peered":"2023-09-23T18:09:17.592375+0000","last_unstale":"2023-09-23T18:09:29.521480+0000","last_undegraded":"2023-09-23T18:09:29.521480+0000","last_fullsized":"2023-09-23T18:09:29.521480+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:36:41.585281+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":91,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521255+0000","last_change":"2023-09-23T18:06:05.358500+0000","last_active":"2023-09-23T18:09:29.521255+0000","last_peered":"2023-09-23T18:09:29.521255+0000","last_clean":"2023-09-23T18:09:29.521255+0000","last_became_active":"2023-09-23T18:06:05.358274+0000","last_became_peered":"2023-09-23T18:06:05.358274+0000","last_unstale":"2023-09-23T18:09:29.521255+0000","last_undegraded":"2023-09-23T18:09:29.521255+0000","last_fullsized":"2023-09-23T18:09:29.521255+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:04.055437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":100,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522023+0000","last_change":"2023-09-23T18:06:45.919995+0000","last_active":"2023-09-23T18:09:29.522023+0000","last_peered":"2023-09-23T18:09:29.522023+0000","last_clean":"2023-09-23T18:09:29.522023+0000","last_became_active":"2023-09-23T18:06:45.919686+0000","last_became_peered":"2023-09-23T18:06:45.919686+0000","last_unstale":"2023-09-23T18:09:29.522023+0000","last_undegraded":"2023-09-23T18:09:29.522023+0000","last_fullsized":"2023-09-23T18:09:29.522023+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:39.113717+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"0'0","reported_seq":11,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037715+0000","last_change":"2023-09-23T18:09:26.506592+0000","last_active":"2023-09-23T18:09:27.037715+0000","last_peered":"2023-09-23T18:09:27.037715+0000","last_clean":"2023-09-23T18:09:27.037715+0000","last_became_active":"2023-09-23T18:09:26.506284+0000","last_became_peered":"2023-09-23T18:09:26.506284+0000","last_unstale":"2023-09-23T18:09:27.037715+0000","last_undegraded":"2023-09-23T18:09:27.037715+0000","last_fullsized":"2023-09-23T18:09:27.037715+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:39:27.126122+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1277,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521646+0000","last_change":"2023-09-23T18:01:25.002404+0000","last_active":"2023-09-23T18:09:29.521646+0000","last_peered":"2023-09-23T18:09:29.521646+0000","last_clean":"2023-09-23T18:09:29.521646+0000","last_became_active":"2023-09-23T18:01:25.001889+0000","last_became_peered":"2023-09-23T18:01:25.001889+0000","last_unstale":"2023-09-23T18:09:29.521646+0000","last_undegraded":"2023-09-23T18:09:29.521646+0000","last_fullsized":"2023-09-23T18:09:29.521646+0000","mapping_epoch":214,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:35:19.334314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":720,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.520435+0000","last_change":"2023-09-23T18:06:48.957422+0000","last_active":"2023-09-23T18:09:29.520435+0000","last_peered":"2023-09-23T18:09:29.520435+0000","last_clean":"2023-09-23T18:09:29.520435+0000","last_became_active":"2023-09-23T18:06:48.957044+0000","last_became_peered":"2023-09-23T18:06:48.957044+0000","last_unstale":"2023-09-23T18:09:29.520435+0000","last_undegraded":"2023-09-23T18:09:29.520435+0000","last_fullsized":"2023-09-23T18:09:29.520435+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:58.047861+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":197,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521763+0000","last_change":"2023-09-23T18:06:51.057540+0000","last_active":"2023-09-23T18:09:29.521763+0000","last_peered":"2023-09-23T18:09:29.521763+0000","last_clean":"2023-09-23T18:09:29.521763+0000","last_became_active":"2023-09-23T18:06:51.056891+0000","last_became_peered":"2023-09-23T18:06:51.056891+0000","last_unstale":"2023-09-23T18:09:29.521763+0000","last_undegraded":"2023-09-23T18:09:29.521763+0000","last_fullsized":"2023-09-23T18:09:29.521763+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:32.959376+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":297,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037403+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:09:27.037403+0000","last_peered":"2023-09-23T18:09:27.037403+0000","last_clean":"2023-09-23T18:09:27.037403+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:09:27.037403+0000","last_undegraded":"2023-09-23T18:09:27.037403+0000","last_fullsized":"2023-09-23T18:09:27.037403+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"365'374","reported_seq":387,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T18:09:32.669999+0000","last_change":"2023-09-23T18:09:26.037157+0000","last_active":"2023-09-23T18:09:32.669999+0000","last_peered":"2023-09-23T18:09:32.669999+0000","last_clean":"2023-09-23T18:09:32.669999+0000","last_became_active":"2023-09-23T18:09:26.036830+0000","last_became_peered":"2023-09-23T18:09:26.036830+0000","last_unstale":"2023-09-23T18:09:32.669999+0000","last_undegraded":"2023-09-23T18:09:32.669999+0000","last_fullsized":"2023-09-23T18:09:32.669999+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":374,"log_dups_size":0,"ondisk_log_size":374,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:16:27.049316+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":24510464,"num_objects":374,"num_object_clones":0,"num_object_copies":1122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":374,"num_write_kb":23936,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":526,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522039+0000","last_change":"2023-09-23T18:09:16.643471+0000","last_active":"2023-09-23T18:09:29.522039+0000","last_peered":"2023-09-23T18:09:29.522039+0000","last_clean":"2023-09-23T18:09:29.522039+0000","last_became_active":"2023-09-23T18:06:08.433718+0000","last_became_peered":"2023-09-23T18:06:08.433718+0000","last_unstale":"2023-09-23T18:09:29.522039+0000","last_undegraded":"2023-09-23T18:09:29.522039+0000","last_fullsized":"2023-09-23T18:09:29.522039+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:37.624632+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":600,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.520662+0000","last_change":"2023-09-23T18:09:17.609194+0000","last_active":"2023-09-23T18:09:29.520662+0000","last_peered":"2023-09-23T18:09:29.520662+0000","last_clean":"2023-09-23T18:09:29.520662+0000","last_became_active":"2023-09-23T18:09:17.596933+0000","last_became_peered":"2023-09-23T18:09:17.596933+0000","last_unstale":"2023-09-23T18:09:29.520662+0000","last_undegraded":"2023-09-23T18:09:29.520662+0000","last_fullsized":"2023-09-23T18:09:29.520662+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:32.587721+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":319,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420387+0000","last_change":"2023-09-23T18:09:17.589599+0000","last_active":"2023-09-23T18:09:28.420387+0000","last_peered":"2023-09-23T18:09:28.420387+0000","last_clean":"2023-09-23T18:09:28.420387+0000","last_became_active":"2023-09-23T18:09:17.589271+0000","last_became_peered":"2023-09-23T18:09:17.589271+0000","last_unstale":"2023-09-23T18:09:28.420387+0000","last_undegraded":"2023-09-23T18:09:28.420387+0000","last_fullsized":"2023-09-23T18:09:28.420387+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:57.518394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,7],"acting":[4,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":343,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521484+0000","last_change":"2023-09-23T18:06:51.070104+0000","last_active":"2023-09-23T18:09:29.521484+0000","last_peered":"2023-09-23T18:09:29.521484+0000","last_clean":"2023-09-23T18:09:29.521484+0000","last_became_active":"2023-09-23T18:06:51.069771+0000","last_became_peered":"2023-09-23T18:06:51.069771+0000","last_unstale":"2023-09-23T18:09:29.521484+0000","last_undegraded":"2023-09-23T18:09:29.521484+0000","last_fullsized":"2023-09-23T18:09:29.521484+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:49.300491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"365'364","reported_seq":377,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T18:09:32.667397+0000","last_change":"2023-09-23T18:09:26.507006+0000","last_active":"2023-09-23T18:09:32.667397+0000","last_peered":"2023-09-23T18:09:32.667397+0000","last_clean":"2023-09-23T18:09:32.667397+0000","last_became_active":"2023-09-23T18:09:26.506732+0000","last_became_peered":"2023-09-23T18:09:26.506732+0000","last_unstale":"2023-09-23T18:09:32.667397+0000","last_undegraded":"2023-09-23T18:09:32.667397+0000","last_fullsized":"2023-09-23T18:09:32.667397+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":364,"log_dups_size":0,"ondisk_log_size":364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:01.574367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":23855104,"num_objects":364,"num_object_clones":0,"num_object_copies":1092,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":364,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":364,"num_write_kb":23296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":575,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037514+0000","last_change":"2023-09-23T18:06:54.055392+0000","last_active":"2023-09-23T18:09:27.037514+0000","last_peered":"2023-09-23T18:09:27.037514+0000","last_clean":"2023-09-23T18:09:27.037514+0000","last_became_active":"2023-09-23T18:06:54.055101+0000","last_became_peered":"2023-09-23T18:06:54.055101+0000","last_unstale":"2023-09-23T18:09:27.037514+0000","last_undegraded":"2023-09-23T18:09:27.037514+0000","last_fullsized":"2023-09-23T18:09:27.037514+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:50.070812+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":604,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522153+0000","last_change":"2023-09-23T18:08:46.401461+0000","last_active":"2023-09-23T18:09:29.522153+0000","last_peered":"2023-09-23T18:09:29.522153+0000","last_clean":"2023-09-23T18:09:29.522153+0000","last_became_active":"2023-09-23T18:06:51.071089+0000","last_became_peered":"2023-09-23T18:06:51.071089+0000","last_unstale":"2023-09-23T18:09:29.522153+0000","last_undegraded":"2023-09-23T18:09:29.522153+0000","last_fullsized":"2023-09-23T18:09:29.522153+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:01.696899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":100,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521083+0000","last_change":"2023-09-23T18:06:51.070131+0000","last_active":"2023-09-23T18:09:29.521083+0000","last_peered":"2023-09-23T18:09:29.521083+0000","last_clean":"2023-09-23T18:09:29.521083+0000","last_became_active":"2023-09-23T18:06:51.069733+0000","last_became_peered":"2023-09-23T18:06:51.069733+0000","last_unstale":"2023-09-23T18:09:29.521083+0000","last_undegraded":"2023-09-23T18:09:29.521083+0000","last_fullsized":"2023-09-23T18:09:29.521083+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:14:28.050735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":194,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521480+0000","last_change":"2023-09-23T18:09:17.601340+0000","last_active":"2023-09-23T18:09:29.521480+0000","last_peered":"2023-09-23T18:09:29.521480+0000","last_clean":"2023-09-23T18:09:29.521480+0000","last_became_active":"2023-09-23T18:09:17.600875+0000","last_became_peered":"2023-09-23T18:09:17.600875+0000","last_unstale":"2023-09-23T18:09:29.521480+0000","last_undegraded":"2023-09-23T18:09:29.521480+0000","last_fullsized":"2023-09-23T18:09:29.521480+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:14.788149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"365'430","reported_seq":442,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T18:09:33.035303+0000","last_change":"2023-09-23T18:09:26.506571+0000","last_active":"2023-09-23T18:09:33.035303+0000","last_peered":"2023-09-23T18:09:33.035303+0000","last_clean":"2023-09-23T18:09:33.035303+0000","last_became_active":"2023-09-23T18:09:26.506199+0000","last_became_peered":"2023-09-23T18:09:26.506199+0000","last_unstale":"2023-09-23T18:09:33.035303+0000","last_undegraded":"2023-09-23T18:09:33.035303+0000","last_fullsized":"2023-09-23T18:09:33.035303+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":430,"log_dups_size":0,"ondisk_log_size":430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:32:12.800777+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":28180480,"num_objects":430,"num_object_clones":0,"num_object_copies":1290,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":430,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":430,"num_write_kb":27520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":709,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420865+0000","last_change":"2023-09-23T18:06:48.962388+0000","last_active":"2023-09-23T18:09:28.420865+0000","last_peered":"2023-09-23T18:09:28.420865+0000","last_clean":"2023-09-23T18:09:28.420865+0000","last_became_active":"2023-09-23T18:06:48.962102+0000","last_became_peered":"2023-09-23T18:06:48.962102+0000","last_unstale":"2023-09-23T18:09:28.420865+0000","last_undegraded":"2023-09-23T18:09:28.420865+0000","last_fullsized":"2023-09-23T18:09:28.420865+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:00.217483+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":124,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522778+0000","last_change":"2023-09-23T18:09:17.594648+0000","last_active":"2023-09-23T18:09:29.522778+0000","last_peered":"2023-09-23T18:09:29.522778+0000","last_clean":"2023-09-23T18:09:29.522778+0000","last_became_active":"2023-09-23T18:09:17.590075+0000","last_became_peered":"2023-09-23T18:09:17.590075+0000","last_unstale":"2023-09-23T18:09:29.522778+0000","last_undegraded":"2023-09-23T18:09:29.522778+0000","last_fullsized":"2023-09-23T18:09:29.522778+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:35:48.372553+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":106,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521741+0000","last_change":"2023-09-23T18:06:05.398171+0000","last_active":"2023-09-23T18:09:29.521741+0000","last_peered":"2023-09-23T18:09:29.521741+0000","last_clean":"2023-09-23T18:09:29.521741+0000","last_became_active":"2023-09-23T18:06:05.370800+0000","last_became_peered":"2023-09-23T18:06:05.370800+0000","last_unstale":"2023-09-23T18:09:29.521741+0000","last_undegraded":"2023-09-23T18:09:29.521741+0000","last_fullsized":"2023-09-23T18:09:29.521741+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:46.783817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":169,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.520475+0000","last_change":"2023-09-23T18:06:45.908943+0000","last_active":"2023-09-23T18:09:29.520475+0000","last_peered":"2023-09-23T18:09:29.520475+0000","last_clean":"2023-09-23T18:09:29.520475+0000","last_became_active":"2023-09-23T18:06:45.908609+0000","last_became_peered":"2023-09-23T18:06:45.908609+0000","last_unstale":"2023-09-23T18:09:29.520475+0000","last_undegraded":"2023-09-23T18:09:29.520475+0000","last_fullsized":"2023-09-23T18:09:29.520475+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:03:31.275390+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"365'388","reported_seq":400,"reported_epoch":365,"state":"active+clean","last_fresh":"2023-09-23T18:09:32.737641+0000","last_change":"2023-09-23T18:09:26.039720+0000","last_active":"2023-09-23T18:09:32.737641+0000","last_peered":"2023-09-23T18:09:32.737641+0000","last_clean":"2023-09-23T18:09:32.737641+0000","last_became_active":"2023-09-23T18:09:26.039428+0000","last_became_peered":"2023-09-23T18:09:26.039428+0000","last_unstale":"2023-09-23T18:09:32.737641+0000","last_undegraded":"2023-09-23T18:09:32.737641+0000","last_fullsized":"2023-09-23T18:09:32.737641+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":388,"log_dups_size":0,"ondisk_log_size":388,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:11:02.046256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":25427968,"num_objects":388,"num_object_clones":0,"num_object_copies":1164,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":388,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":388,"num_write_kb":24832,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,2],"acting":[5,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":670,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522670+0000","last_change":"2023-09-23T18:06:54.059843+0000","last_active":"2023-09-23T18:09:29.522670+0000","last_peered":"2023-09-23T18:09:29.522670+0000","last_clean":"2023-09-23T18:09:29.522670+0000","last_became_active":"2023-09-23T18:06:54.059469+0000","last_became_peered":"2023-09-23T18:06:54.059469+0000","last_unstale":"2023-09-23T18:09:29.522670+0000","last_undegraded":"2023-09-23T18:09:29.522670+0000","last_fullsized":"2023-09-23T18:09:29.522670+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:59.157827+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":569,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037632+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:09:27.037632+0000","last_peered":"2023-09-23T18:09:27.037632+0000","last_clean":"2023-09-23T18:09:27.037632+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:09:27.037632+0000","last_undegraded":"2023-09-23T18:09:27.037632+0000","last_fullsized":"2023-09-23T18:09:27.037632+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":395,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521520+0000","last_change":"2023-09-23T18:06:45.939855+0000","last_active":"2023-09-23T18:09:29.521520+0000","last_peered":"2023-09-23T18:09:29.521520+0000","last_clean":"2023-09-23T18:09:29.521520+0000","last_became_active":"2023-09-23T18:06:45.924278+0000","last_became_peered":"2023-09-23T18:06:45.924278+0000","last_unstale":"2023-09-23T18:09:29.521520+0000","last_undegraded":"2023-09-23T18:09:29.521520+0000","last_fullsized":"2023-09-23T18:09:29.521520+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:43.850375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":154,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420140+0000","last_change":"2023-09-23T18:01:39.461463+0000","last_active":"2023-09-23T18:09:28.420140+0000","last_peered":"2023-09-23T18:09:28.420140+0000","last_clean":"2023-09-23T18:09:28.420140+0000","last_became_active":"2023-09-23T18:01:39.461168+0000","last_became_peered":"2023-09-23T18:01:39.461168+0000","last_unstale":"2023-09-23T18:09:28.420140+0000","last_undegraded":"2023-09-23T18:09:28.420140+0000","last_fullsized":"2023-09-23T18:09:28.420140+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:59:15.521958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":511,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037102+0000","last_change":"2023-09-23T18:09:14.580901+0000","last_active":"2023-09-23T18:09:27.037102+0000","last_peered":"2023-09-23T18:09:27.037102+0000","last_clean":"2023-09-23T18:09:27.037102+0000","last_became_active":"2023-09-23T18:06:48.961304+0000","last_became_peered":"2023-09-23T18:06:48.961304+0000","last_unstale":"2023-09-23T18:09:27.037102+0000","last_undegraded":"2023-09-23T18:09:27.037102+0000","last_fullsized":"2023-09-23T18:09:27.037102+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:14.580819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:14.580819+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:04.095372+0000","scrub_duration":0.003313033,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":280,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420804+0000","last_change":"2023-09-23T18:06:26.148865+0000","last_active":"2023-09-23T18:09:28.420804+0000","last_peered":"2023-09-23T18:09:28.420804+0000","last_clean":"2023-09-23T18:09:28.420804+0000","last_became_active":"2023-09-23T18:06:26.148517+0000","last_became_peered":"2023-09-23T18:06:26.148517+0000","last_unstale":"2023-09-23T18:09:28.420804+0000","last_undegraded":"2023-09-23T18:09:28.420804+0000","last_fullsized":"2023-09-23T18:09:28.420804+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:38.141672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":402,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521035+0000","last_change":"2023-09-23T18:06:55.916070+0000","last_active":"2023-09-23T18:09:29.521035+0000","last_peered":"2023-09-23T18:09:29.521035+0000","last_clean":"2023-09-23T18:09:29.521035+0000","last_became_active":"2023-09-23T18:06:55.915807+0000","last_became_peered":"2023-09-23T18:06:55.915807+0000","last_unstale":"2023-09-23T18:09:29.521035+0000","last_undegraded":"2023-09-23T18:09:29.521035+0000","last_fullsized":"2023-09-23T18:09:29.521035+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:20.376063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":506,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037458+0000","last_change":"2023-09-23T18:09:16.604488+0000","last_active":"2023-09-23T18:09:27.037458+0000","last_peered":"2023-09-23T18:09:27.037458+0000","last_clean":"2023-09-23T18:09:27.037458+0000","last_became_active":"2023-09-23T18:06:54.055098+0000","last_became_peered":"2023-09-23T18:06:54.055098+0000","last_unstale":"2023-09-23T18:09:27.037458+0000","last_undegraded":"2023-09-23T18:09:27.037458+0000","last_fullsized":"2023-09-23T18:09:27.037458+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.604423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.604423+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:28.598882+0000","scrub_duration":0.0014924669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":234,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522801+0000","last_change":"2023-09-23T18:06:05.382745+0000","last_active":"2023-09-23T18:09:29.522801+0000","last_peered":"2023-09-23T18:09:29.522801+0000","last_clean":"2023-09-23T18:09:29.522801+0000","last_became_active":"2023-09-23T18:06:05.371488+0000","last_became_peered":"2023-09-23T18:06:05.371488+0000","last_unstale":"2023-09-23T18:09:29.522801+0000","last_undegraded":"2023-09-23T18:09:29.522801+0000","last_fullsized":"2023-09-23T18:09:29.522801+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:05:08.915006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":299,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.421178+0000","last_change":"2023-09-23T18:06:05.375138+0000","last_active":"2023-09-23T18:09:28.421178+0000","last_peered":"2023-09-23T18:09:28.421178+0000","last_clean":"2023-09-23T18:09:28.421178+0000","last_became_active":"2023-09-23T18:06:05.374819+0000","last_became_peered":"2023-09-23T18:06:05.374819+0000","last_unstale":"2023-09-23T18:09:28.421178+0000","last_undegraded":"2023-09-23T18:09:28.421178+0000","last_fullsized":"2023-09-23T18:09:28.421178+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:43.000019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":206,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.523136+0000","last_change":"2023-09-23T18:01:39.482904+0000","last_active":"2023-09-23T18:09:29.523136+0000","last_peered":"2023-09-23T18:09:29.523136+0000","last_clean":"2023-09-23T18:09:29.523136+0000","last_became_active":"2023-09-23T18:01:39.460389+0000","last_became_peered":"2023-09-23T18:01:39.460389+0000","last_unstale":"2023-09-23T18:09:29.523136+0000","last_undegraded":"2023-09-23T18:09:29.523136+0000","last_fullsized":"2023-09-23T18:09:29.523136+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:23:21.402372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":188,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.420283+0000","last_change":"2023-09-23T18:09:17.585898+0000","last_active":"2023-09-23T18:09:28.420283+0000","last_peered":"2023-09-23T18:09:28.420283+0000","last_clean":"2023-09-23T18:09:28.420283+0000","last_became_active":"2023-09-23T18:09:17.585571+0000","last_became_peered":"2023-09-23T18:09:17.585571+0000","last_unstale":"2023-09-23T18:09:28.420283+0000","last_undegraded":"2023-09-23T18:09:28.420283+0000","last_fullsized":"2023-09-23T18:09:28.420283+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:58.967072+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":751,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521673+0000","last_change":"2023-09-23T18:06:08.440953+0000","last_active":"2023-09-23T18:09:29.521673+0000","last_peered":"2023-09-23T18:09:29.521673+0000","last_clean":"2023-09-23T18:09:29.521673+0000","last_became_active":"2023-09-23T18:06:08.440604+0000","last_became_peered":"2023-09-23T18:06:08.440604+0000","last_unstale":"2023-09-23T18:09:29.521673+0000","last_undegraded":"2023-09-23T18:09:29.521673+0000","last_fullsized":"2023-09-23T18:09:29.521673+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:53.274049+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":166,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:27.037465+0000","last_change":"2023-09-23T18:06:51.098401+0000","last_active":"2023-09-23T18:09:27.037465+0000","last_peered":"2023-09-23T18:09:27.037465+0000","last_clean":"2023-09-23T18:09:27.037465+0000","last_became_active":"2023-09-23T18:06:51.097959+0000","last_became_peered":"2023-09-23T18:06:51.097959+0000","last_unstale":"2023-09-23T18:09:27.037465+0000","last_undegraded":"2023-09-23T18:09:27.037465+0000","last_fullsized":"2023-09-23T18:09:27.037465+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:16.234414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":212,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.522487+0000","last_change":"2023-09-23T18:09:17.597665+0000","last_active":"2023-09-23T18:09:29.522487+0000","last_peered":"2023-09-23T18:09:29.522487+0000","last_clean":"2023-09-23T18:09:29.522487+0000","last_became_active":"2023-09-23T18:09:17.597341+0000","last_became_peered":"2023-09-23T18:09:17.597341+0000","last_unstale":"2023-09-23T18:09:29.522487+0000","last_undegraded":"2023-09-23T18:09:29.522487+0000","last_fullsized":"2023-09-23T18:09:29.522487+0000","mapping_epoch":355,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":356,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:36:45.067091+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":698,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.521384+0000","last_change":"2023-09-23T18:09:17.636095+0000","last_active":"2023-09-23T18:09:29.521384+0000","last_peered":"2023-09-23T18:09:29.521384+0000","last_clean":"2023-09-23T18:09:29.521384+0000","last_became_active":"2023-09-23T18:06:48.965429+0000","last_became_peered":"2023-09-23T18:06:48.965429+0000","last_unstale":"2023-09-23T18:09:29.521384+0000","last_undegraded":"2023-09-23T18:09:29.521384+0000","last_fullsized":"2023-09-23T18:09:29.521384+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:17.635974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:17.635974+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:06.792253+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":250,"reported_epoch":364,"state":"active+clean","last_fresh":"2023-09-23T18:09:29.523095+0000","last_change":"2023-09-23T18:06:45.908664+0000","last_active":"2023-09-23T18:09:29.523095+0000","last_peered":"2023-09-23T18:09:29.523095+0000","last_clean":"2023-09-23T18:09:29.523095+0000","last_became_active":"2023-09-23T18:06:45.903588+0000","last_became_peered":"2023-09-23T18:06:45.903588+0000","last_unstale":"2023-09-23T18:09:29.523095+0000","last_undegraded":"2023-09-23T18:09:29.523095+0000","last_fullsized":"2023-09-23T18:09:29.523095+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:42.283189+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":384,"reported_epoch":363,"state":"active+clean","last_fresh":"2023-09-23T18:09:28.419907+0000","last_change":"2023-09-23T18:06:05.374027+0000","last_active":"2023-09-23T18:09:28.419907+0000","last_peered":"2023-09-23T18:09:28.419907+0000","last_clean":"2023-09-23T18:09:28.419907+0000","last_became_active":"2023-09-23T18:06:05.373670+0000","last_became_peered":"2023-09-23T18:06:05.373670+0000","last_unstale":"2023-09-23T18:09:28.419907+0000","last_undegraded":"2023-09-23T18:09:28.419907+0000","last_fullsized":"2023-09-23T18:09:28.419907+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:01.220735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":49,"num_bytes_recovered":6246,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":242745344,"num_objects":3704,"num_object_clones":0,"num_object_copies":11112,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3704,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3704,"num_write_kb":237056,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":881065984,"data_stored":881065984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":3704,"ondisk_log_size":3704,"up":48,"acting":48,"num_store_stats":7},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":302941,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":33,"num_read_kb":63,"num_write":17,"num_write_kb":165,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":448437,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":134,"num_bytes_recovered":4194361,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001443,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":372832,"kb_used_data":165208,"kb_used_omap":13,"kb_used_meta":207602,"kb_avail":93372320,"statfs":{"total":95995035648,"available":95613255680,"internally_reserved":0,"allocated":169172992,"data_stored":167937070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":212584552},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.1559999999999999,"5min":1.083,"15min":0.76900000000000002},"min":{"1min":0.23999999999999999,"5min":0.217,"15min":0.217},"max":{"1min":4.1109999999999998,"5min":9.3499999999999996,"15min":9.3499999999999996},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.79100000000000004,"5min":0.878,"15min":1.1299999999999999},"min":{"1min":0.27100000000000002,"5min":0.25,"15min":0.23000000000000001},"max":{"1min":3.0209999999999999,"5min":6.641,"15min":6.641},"last":0.40899999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.4299999999999999,"5min":1.2929999999999999,"15min":0.89500000000000002},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.5179999999999998,"5min":9.4909999999999997,"15min":9.4909999999999997},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.2250000000000001,"5min":1.1459999999999999,"15min":1.089},"min":{"1min":0.314,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":5.0309999999999997,"5min":6.9589999999999996,"15min":6.9589999999999996},"last":0.501}]},{"osd":2,"last update":"Sat Sep 23 18:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3500000000000001,"5min":1.1799999999999999,"15min":1.0249999999999999},"min":{"1min":0.28199999999999997,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":4.6529999999999996,"5min":6.6630000000000003,"15min":6.6630000000000003},"last":0.44},{"interface":"front","average":{"1min":1.339,"5min":1.25,"15min":0.98199999999999998},"min":{"1min":0.38400000000000001,"5min":0.30399999999999999,"15min":0.29399999999999998},"max":{"1min":4.5940000000000003,"5min":7.9160000000000004,"15min":7.9160000000000004},"last":0.44900000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:09:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.373,"15min":1.474},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.8710000000000004,"5min":7.0579999999999998,"15min":7.6070000000000002},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.5,"5min":1.4570000000000001,"15min":1.173},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.7889999999999997,"5min":15.349,"15min":15.349},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:08:41 2023","interfaces":[{"interface":"back","average":{"1min":0.88400000000000001,"5min":0.89200000000000002,"15min":0.99199999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.245},"max":{"1min":3.1200000000000001,"5min":6.9729999999999999,"15min":6.9729999999999999},"last":0.34499999999999997},{"interface":"front","average":{"1min":0.73399999999999999,"5min":0.76300000000000001,"15min":0.61099999999999999},"min":{"1min":0.32200000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":2.0880000000000001,"5min":6.3479999999999999,"15min":6.3479999999999999},"last":0.46500000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0660000000000001,"5min":1.141,"15min":1.03},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.125,"5min":9.5999999999999996,"15min":9.5999999999999996},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.84999999999999998,"5min":1.075,"15min":1.0960000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0859999999999999,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":0.53400000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:08:41 2023","interfaces":[{"interface":"back","average":{"1min":0.82999999999999996,"5min":0.98399999999999999,"15min":0.79600000000000004},"min":{"1min":0.33900000000000002,"5min":0.308,"15min":0.308},"max":{"1min":2.448,"5min":6.3730000000000002,"15min":6.3730000000000002},"last":0.37},{"interface":"front","average":{"1min":0.76700000000000002,"5min":1.0740000000000001,"15min":1.0820000000000001},"min":{"1min":0.312,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":2.4140000000000001,"5min":9.5519999999999996,"15min":9.5519999999999996},"last":0.47899999999999998}]}]},{"osd":6,"up_from":163,"seq":700079669379,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252112,"kb_used_data":8328,"kb_used_omap":7,"kb_used_meta":243768,"kb_avail":93493040,"statfs":{"total":95995035648,"available":95736872960,"internally_reserved":0,"allocated":8527872,"data_stored":7300202,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":249619236},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1220000000000001,"5min":0.80700000000000005,"15min":1.143},"min":{"1min":0.23899999999999999,"5min":0.23100000000000001,"15min":0.215},"max":{"1min":8.4019999999999992,"5min":8.4019999999999992,"15min":15.407},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.3480000000000001,"5min":1.2869999999999999,"15min":1.778},"min":{"1min":0.312,"5min":0.27100000000000002,"15min":0.23100000000000001},"max":{"1min":7.6779999999999999,"5min":15.147,"15min":15.147},"last":0.44600000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":0.94399999999999995,"5min":1.0149999999999999,"15min":1.538},"min":{"1min":0.26800000000000002,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":2.8650000000000002,"5min":6.5309999999999997,"15min":14.233000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":1.2549999999999999,"5min":1.121,"15min":1.7050000000000001},"min":{"1min":0.26300000000000001,"5min":0.26000000000000001,"15min":0.248},"max":{"1min":4.0449999999999999,"5min":5.2809999999999997,"15min":14.789},"last":0.48799999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.145,"15min":1.804},"min":{"1min":0.29799999999999999,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.2570000000000001,"5min":8.6140000000000008,"15min":11.210000000000001},"last":0.50900000000000001},{"interface":"front","average":{"1min":1.0589999999999999,"5min":1.181,"15min":2.012},"min":{"1min":0.248,"5min":0.248,"15min":0.245},"max":{"1min":3.2189999999999999,"5min":9.3620000000000001,"15min":11.223000000000001},"last":0.46800000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:08:30 2023","interfaces":[{"interface":"back","average":{"1min":1.258,"5min":1.181,"15min":1.129},"min":{"1min":0.40200000000000002,"5min":0.313,"15min":0.308},"max":{"1min":3.5920000000000001,"5min":4.7539999999999996,"15min":4.7539999999999996},"last":0.377},{"interface":"front","average":{"1min":1.788,"5min":1.407,"15min":1.246},"min":{"1min":0.32400000000000001,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":7.0250000000000004,"5min":7.484,"15min":7.484},"last":0.54500000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":0.86699999999999999,"15min":0.88700000000000001},"min":{"1min":0.27100000000000002,"5min":0.254,"15min":0.23000000000000001},"max":{"1min":3.3530000000000002,"5min":8.0280000000000005,"15min":9.9109999999999996},"last":0.33900000000000002},{"interface":"front","average":{"1min":1.119,"5min":0.96499999999999997,"15min":0.71199999999999997},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.24099999999999999},"max":{"1min":3.976,"5min":8.1159999999999997,"15min":8.3000000000000007},"last":0.434}]},{"osd":5,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":0.99299999999999999,"15min":0.89400000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7219999999999995,"5min":9.7219999999999995,"15min":11.869},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.5289999999999999,"5min":0.92200000000000004,"15min":0.80100000000000005},"min":{"1min":0.30599999999999999,"5min":0.251,"15min":0.251},"max":{"1min":10.436,"5min":10.436,"15min":10.436},"last":0.40899999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:08:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.80100000000000005,"15min":0.92800000000000005},"min":{"1min":0.34699999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":2.532,"5min":5.5380000000000003,"15min":5.5380000000000003},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.95999999999999996,"15min":0.84399999999999997},"min":{"1min":0.36699999999999999,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.0630000000000002,"5min":10.411,"15min":10.411},"last":0.501}]}]},{"osd":1,"up_from":98,"seq":420906795197,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":670920,"kb_used_data":88064,"kb_used_omap":7,"kb_used_meta":582840,"kb_avail":93074232,"statfs":{"total":95995035648,"available":95308013568,"internally_reserved":0,"allocated":90177536,"data_stored":88501464,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":596828861},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.413,"15min":1.2390000000000001},"min":{"1min":0.184,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":14.567,"5min":14.567,"15min":15.529999999999999},"last":1.1319999999999999},{"interface":"front","average":{"1min":1.4099999999999999,"5min":2.3100000000000001,"15min":1.651},"min":{"1min":0.25,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.774,"5min":82.244,"15min":82.244},"last":0.94599999999999995}]},{"osd":2,"last update":"Sat Sep 23 18:08:50 2023","interfaces":[{"interface":"back","average":{"1min":2.5649999999999999,"5min":1.5920000000000001,"15min":1.5820000000000001},"min":{"1min":0.29499999999999998,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":7.2270000000000003,"5min":10.531000000000001,"15min":16.347999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":2.0179999999999998,"5min":1.27,"15min":1.2490000000000001},"min":{"1min":0.309,"5min":0.28499999999999998,"15min":0.21299999999999999},"max":{"1min":7.641,"5min":7.641,"15min":11.305999999999999},"last":0.59999999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:08:32 2023","interfaces":[{"interface":"back","average":{"1min":2.4140000000000001,"5min":1.538,"15min":0.89000000000000001},"min":{"1min":0.41899999999999998,"5min":0.27100000000000002,"15min":0.27100000000000002},"max":{"1min":11.164,"5min":11.164,"15min":11.164},"last":1.204},{"interface":"front","average":{"1min":1.383,"5min":1.3480000000000001,"15min":1.121},"min":{"1min":0.33800000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":5.7960000000000003,"5min":11.125999999999999,"15min":11.125999999999999},"last":1.1160000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":1.3700000000000001,"15min":1.339},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":4.806,"5min":8.3919999999999995,"15min":26.568999999999999},"last":1.415},{"interface":"front","average":{"1min":2.141,"5min":1.4019999999999999,"15min":1.294},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":6.6280000000000001,"5min":7.9260000000000002,"15min":26.635000000000002},"last":0.38100000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.7869999999999999,"15min":1.4379999999999999},"min":{"1min":0.38900000000000001,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":6.7489999999999997,"5min":11.638999999999999,"15min":11.638999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.579,"15min":1.393},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":6.5609999999999999,"5min":8.4309999999999992,"15min":9.5519999999999996},"last":1.296}]},{"osd":6,"last update":"Sat Sep 23 18:08:50 2023","interfaces":[{"interface":"back","average":{"1min":1.8420000000000001,"5min":2.262,"15min":1.95},"min":{"1min":0.372,"5min":0.372,"15min":0.317},"max":{"1min":6.5339999999999998,"5min":89.566000000000003,"15min":89.566000000000003},"last":1.181},{"interface":"front","average":{"1min":1.927,"5min":2.3010000000000002,"15min":1.7649999999999999},"min":{"1min":0.40899999999999997,"5min":0.38,"15min":0.34699999999999998},"max":{"1min":6.6219999999999999,"5min":90.608999999999995,"15min":90.608999999999995},"last":1.1679999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:08:41 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":1.177,"15min":0.89200000000000002},"min":{"1min":0.46800000000000003,"5min":0.39400000000000002,"15min":0.33900000000000002},"max":{"1min":6.6740000000000004,"5min":7.1219999999999999,"15min":7.1219999999999999},"last":1.0489999999999999},{"interface":"front","average":{"1min":1.645,"5min":1.462,"15min":1.677},"min":{"1min":0.496,"5min":0.45000000000000001,"15min":0.40999999999999998},"max":{"1min":6.7220000000000004,"5min":9.5709999999999997,"15min":9.5709999999999997},"last":2.1539999999999999}]}]},{"osd":0,"up_from":0,"seq":399431958724,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":2,"up_from":103,"seq":442381631672,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":738788,"kb_used_data":191324,"kb_used_omap":11,"kb_used_meta":547444,"kb_avail":93006364,"statfs":{"total":95995035648,"available":95238516736,"internally_reserved":0,"allocated":195915776,"data_stored":194212446,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":560583031},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3799999999999999,"5min":2.0150000000000001,"15min":1.843},"min":{"1min":0.23300000000000001,"5min":0.20799999999999999,"15min":0.17199999999999999},"max":{"1min":11.173,"5min":36.656999999999996,"15min":36.656999999999996},"last":0.501},{"interface":"front","average":{"1min":1.52,"5min":1.7789999999999999,"15min":1.744},"min":{"1min":0.24099999999999999,"5min":0.23599999999999999,"15min":0.215},"max":{"1min":11.284000000000001,"5min":36.697000000000003,"15min":36.697000000000003},"last":0.69599999999999995}]},{"osd":1,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.7989999999999999,"5min":1.4490000000000001,"15min":1.7450000000000001},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.215},"max":{"1min":11.244999999999999,"5min":11.244999999999999,"15min":20.995000000000001},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.129,"5min":1.268,"15min":1.4970000000000001},"min":{"1min":0.29999999999999999,"5min":0.252,"15min":0.193},"max":{"1min":3.3340000000000001,"5min":8.3889999999999993,"15min":15.945},"last":0.57199999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:08:36 2023","interfaces":[{"interface":"back","average":{"1min":1.343,"5min":1.861,"15min":1.141},"min":{"1min":0.309,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.9119999999999999,"5min":37.500999999999998,"15min":37.500999999999998},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.827,"5min":2.032,"15min":1.0840000000000001},"min":{"1min":0.374,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":7.4900000000000002,"5min":34.915999999999997,"15min":34.915999999999997},"last":1.514}]},{"osd":4,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.129,"5min":1.425,"15min":1.385},"min":{"1min":0.433,"5min":0.311,"15min":0.26500000000000001},"max":{"1min":2.6309999999999998,"5min":13.016999999999999,"15min":13.016999999999999},"last":1.3109999999999999},{"interface":"front","average":{"1min":1.161,"5min":1.3149999999999999,"15min":1.3560000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26700000000000002},"max":{"1min":7.1230000000000002,"5min":8.0820000000000007,"15min":20.920000000000002},"last":0.53000000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.2969999999999999,"15min":1.4219999999999999},"min":{"1min":0.33000000000000002,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":7.2389999999999999,"5min":8.1760000000000002,"15min":20.738},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.167,"5min":1.5069999999999999,"15min":1.534},"min":{"1min":0.33600000000000002,"5min":0.32900000000000001,"15min":0.26700000000000002},"max":{"1min":3.1499999999999999,"5min":12.103999999999999,"15min":15.496},"last":0.61699999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.0720000000000001,"5min":1.423,"15min":1.8320000000000001},"min":{"1min":0.432,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.452,"5min":13.065,"15min":13.065},"last":1.454},{"interface":"front","average":{"1min":1.873,"5min":1.528,"15min":2.0249999999999999},"min":{"1min":0.42099999999999999,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":11.122999999999999,"5min":11.122999999999999,"15min":20.893000000000001},"last":2.3929999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:08:36 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.417,"15min":1.387},"min":{"1min":0.39200000000000002,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":2.6989999999999998,"5min":12.051,"15min":12.051},"last":1.0009999999999999},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.5669999999999999,"15min":0.95399999999999996},"min":{"1min":0.45100000000000001,"5min":0.437,"15min":0.309},"max":{"1min":11.212999999999999,"5min":11.212999999999999,"15min":11.212999999999999},"last":2.4319999999999999}]}]},{"osd":3,"up_from":237,"seq":1017907249239,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":900268,"kb_used_data":606820,"kb_used_omap":3,"kb_used_meta":293436,"kb_avail":92844884,"statfs":{"total":95995035648,"available":95073161216,"internally_reserved":0,"allocated":621383680,"data_stored":619501063,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":300479208},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.008,"15min":0.89400000000000002},"min":{"1min":0.24399999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.5679999999999996,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.38500000000000001},{"interface":"front","average":{"1min":1.004,"5min":1.532,"15min":1.1850000000000001},"min":{"1min":0.23799999999999999,"5min":0.22,"15min":0.20699999999999999},"max":{"1min":9.1999999999999993,"5min":14.885999999999999,"15min":14.885999999999999},"last":0.51400000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.887,"15min":1.29},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.245},"max":{"1min":14.241,"5min":21.931999999999999,"15min":21.931999999999999},"last":0.48199999999999998},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.3140000000000001,"15min":1.0449999999999999},"min":{"1min":0.29199999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.008,"5min":13.631,"15min":13.631},"last":0.67300000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.0329999999999999,"15min":0.90100000000000002},"min":{"1min":0.29099999999999998,"5min":0.247,"15min":0.22600000000000001},"max":{"1min":4.5899999999999999,"5min":8.2200000000000006,"15min":8.2200000000000006},"last":0.46700000000000003},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.669,"15min":1.333},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.0090000000000003,"5min":21.988,"15min":21.988},"last":0.55700000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.885,"15min":1.335},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.29299999999999998},"max":{"1min":7.5540000000000003,"5min":37.231000000000002,"15min":37.231000000000002},"last":0.64200000000000002},{"interface":"front","average":{"1min":1.246,"5min":1.819,"15min":1.294},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":4.673,"5min":37.381,"15min":37.381},"last":0.72599999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.4570000000000001,"15min":1.3069999999999999},"min":{"1min":0.34699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":9.3170000000000002,"5min":12.743,"15min":12.743},"last":0.752},{"interface":"front","average":{"1min":1.099,"5min":1.1279999999999999,"15min":1.147},"min":{"1min":0.376,"5min":0.315,"15min":0.315},"max":{"1min":4.601,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.70199999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":2.3290000000000002,"15min":1.4930000000000001},"min":{"1min":0.41199999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.657,"5min":115.54000000000001,"15min":115.54000000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.101,"5min":2.6560000000000001,"15min":1.835},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":7.5860000000000003,"5min":115.691,"15min":115.691},"last":0.71199999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":1.4950000000000001,"15min":1.512},"min":{"1min":0.501,"5min":0.39600000000000002,"15min":0.377},"max":{"1min":5.5179999999999998,"5min":12.821,"15min":12.821},"last":0.77200000000000002},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.405,"15min":1.512},"min":{"1min":0.442,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.6159999999999997,"5min":8.6039999999999992,"15min":8.6039999999999992},"last":0.627}]}]},{"osd":4,"up_from":30,"seq":128849019188,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":313048,"kb_used_data":4240,"kb_used_omap":0,"kb_used_meta":308800,"kb_avail":93432104,"statfs":{"total":95995035648,"available":95674474496,"internally_reserved":0,"allocated":4341760,"data_stored":3106278,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":316211200},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:08:44 2023","interfaces":[{"interface":"back","average":{"1min":1.1359999999999999,"5min":1.151,"15min":1.038},"min":{"1min":0.26100000000000001,"5min":0.215,"15min":0.20100000000000001},"max":{"1min":4.0279999999999996,"5min":13.375999999999999,"15min":14.164},"last":0.45100000000000001},{"interface":"front","average":{"1min":1.032,"5min":1.1599999999999999,"15min":1.252},"min":{"1min":0.28799999999999998,"5min":0.25700000000000001,"15min":0.22600000000000001},"max":{"1min":5.1070000000000002,"5min":6.1989999999999998,"15min":11.372},"last":0.59599999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:09:15 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":1.298,"15min":1.26},"min":{"1min":0.25800000000000001,"5min":0.249,"15min":0.249},"max":{"1min":4.2949999999999999,"5min":7.6059999999999999,"15min":10.429},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.7130000000000001,"5min":1.2669999999999999,"15min":1.3200000000000001},"min":{"1min":0.27600000000000002,"5min":0.27100000000000002,"15min":0.23100000000000001},"max":{"1min":5.5129999999999999,"5min":6.7640000000000002,"15min":10.83},"last":0.56899999999999995}]},{"osd":2,"last update":"Sat Sep 23 18:08:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.1459999999999999,"15min":1.256},"min":{"1min":0.31900000000000001,"5min":0.28799999999999998,"15min":0.26600000000000001},"max":{"1min":5.5640000000000001,"5min":6.8259999999999996,"15min":11.630000000000001},"last":0.64800000000000002},{"interface":"front","average":{"1min":1.798,"5min":1.452,"15min":1.3049999999999999},"min":{"1min":0.32100000000000001,"5min":0.314,"15min":0.27100000000000002},"max":{"1min":4.7699999999999996,"5min":13.672000000000001,"15min":13.672000000000001},"last":0.72499999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:09:28 2023","interfaces":[{"interface":"back","average":{"1min":3.141,"5min":2.2839999999999998,"15min":1.6240000000000001},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":28.754999999999999,"5min":29.154,"15min":29.154},"last":0.68400000000000005},{"interface":"front","average":{"1min":2.645,"5min":2.0920000000000001,"15min":1.6000000000000001},"min":{"1min":0.36699999999999999,"5min":0.34699999999999998,"15min":0.34200000000000003},"max":{"1min":29.213000000000001,"5min":29.216000000000001,"15min":29.216000000000001},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:08:51 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.036,"15min":0.81799999999999995},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.5279999999999996,"5min":9.2959999999999994,"15min":9.9819999999999993},"last":0.61799999999999999},{"interface":"front","average":{"1min":1.2649999999999999,"5min":1.1299999999999999,"15min":0.98199999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":9.1609999999999996,"5min":9.1999999999999993,"15min":11.170999999999999},"last":0.50900000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:08:51 2023","interfaces":[{"interface":"back","average":{"1min":0.93700000000000006,"5min":0.97099999999999997,"15min":0.89600000000000002},"min":{"1min":0.313,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":5.8310000000000004,"5min":9.0990000000000002,"15min":9.0990000000000002},"last":0.66300000000000003},{"interface":"front","average":{"1min":1.198,"5min":0.94899999999999995,"15min":0.97399999999999998},"min":{"1min":0.314,"5min":0.312,"15min":0.28399999999999997},"max":{"1min":4.9740000000000002,"5min":9.1300000000000008,"15min":10.725},"last":0.70199999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:08:37 2023","interfaces":[{"interface":"back","average":{"1min":0.80300000000000005,"5min":1.113,"15min":1.002},"min":{"1min":0.35199999999999998,"5min":0.314,"15min":0.314},"max":{"1min":3.7370000000000001,"5min":13.298999999999999,"15min":13.298999999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":0.70799999999999996,"5min":1.1080000000000001,"15min":1.194},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.307},"max":{"1min":1.617,"5min":10.192,"15min":10.192},"last":0.71499999999999997}]}]},{"osd":5,"up_from":38,"seq":163208757556,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":446352,"kb_used_data":146632,"kb_used_omap":0,"kb_used_meta":299712,"kb_avail":93298800,"statfs":{"total":95995035648,"available":95537971200,"internally_reserved":0,"allocated":150151168,"data_stored":148927323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":306905088},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.77900000000000003,"15min":1.069},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.217},"max":{"1min":2.5649999999999999,"5min":5.4539999999999997,"15min":21.376999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.94799999999999995,"15min":1.3},"min":{"1min":0.249,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":3.5910000000000002,"5min":8.5099999999999998,"15min":21.388999999999999},"last":0.40600000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.038,"15min":0.96699999999999997},"min":{"1min":0.23899999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.6349999999999998,"5min":10.547000000000001,"15min":10.853999999999999},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.1299999999999999,"15min":1.1759999999999999},"min":{"1min":0.26600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.625,"5min":8.9749999999999996,"15min":10.206},"last":0.34200000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.002,"15min":1.4119999999999999},"min":{"1min":0.38600000000000001,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":5.0570000000000004,"5min":5.5309999999999997,"15min":38.606999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.0549999999999999,"15min":1.302},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6000000000000001,"5min":10.065,"15min":39.670999999999999},"last":0.38100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":1.266,"15min":0.96099999999999997},"min":{"1min":0.32200000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6499999999999999,"5min":9.2309999999999999,"15min":9.2309999999999999},"last":0.437},{"interface":"front","average":{"1min":1.6930000000000001,"5min":1.3779999999999999,"15min":0.94699999999999995},"min":{"1min":0.35599999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.362,"5min":13.446999999999999,"15min":13.446999999999999},"last":0.46100000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:08:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.79800000000000004,"15min":0.83699999999999997},"min":{"1min":0.29799999999999999,"5min":0.25700000000000001,"15min":0.224},"max":{"1min":3.7149999999999999,"5min":8.6229999999999993,"15min":10.092000000000001},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.94699999999999995,"15min":0.83999999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.23999999999999999},"max":{"1min":9.1750000000000007,"5min":9.1750000000000007,"15min":9.1750000000000007},"last":0.312}]},{"osd":6,"last update":"Sat Sep 23 18:08:59 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":0.86299999999999999,"15min":1.1930000000000001},"min":{"1min":0.314,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":6.032,"5min":6.032,"15min":11.287000000000001},"last":0.36099999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.872,"15min":1.0880000000000001},"min":{"1min":0.33100000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.6709999999999998,"5min":6.6920000000000002,"15min":8.266},"last":0.41499999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:08:37 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.89400000000000002,"15min":1.282},"min":{"1min":0.40799999999999997,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.464,"5min":8.8510000000000009,"15min":8.8510000000000009},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.68400000000000005,"5min":0.97199999999999998,"15min":1.0569999999999999},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":3.5830000000000002,"5min":17.684000000000001,"15min":17.684000000000001},"last":0.51100000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":164626432,"data_stored":164626432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":179044352,"data_stored":179044352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":92078080,"data_stored":92078080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":243924992,"data_stored":243924992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":201392128,"data_stored":201392128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:09:37.264 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:09:37.265 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 41 2023-09-23T18:09:37.269 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:37.322 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:37.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:37 smithi055 ceph-mon[179668]: from='client.36180 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:09:37.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:37.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:37.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:37.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:37 smithi055 ceph-mon[184174]: from='client.36180 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:09:37.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:37.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:37.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:37.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:37 smithi099 ceph-mon[167076]: from='client.36180 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:09:37.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:37.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:37.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:37.682 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:37.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:37.687 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:37.787 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:37.898 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:37.954 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:37.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:37.958 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:37.994 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 7 16 19651 19635 175.29 152.562 0.00547161 0.00570018 2023-09-23T18:09:38.059 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:38.112 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:38.325 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:38.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:38 smithi099 ceph-mon[167076]: pgmap v894: 119 pgs: 119 active+clean; 570 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T18:09:38.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:38.330 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:38.420 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:38 smithi055 ceph-mon[179668]: pgmap v894: 119 pgs: 119 active+clean; 570 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T18:09:38.420 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:38 smithi055 ceph-mon[184174]: pgmap v894: 119 pgs: 119 active+clean; 570 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 71 MiB/s wr, 1.13k op/s 2023-09-23T18:09:38.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:38.476 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:38.481 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:09:38.542 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:38.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:38.549 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:38.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:38.693 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:38.753 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:38.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:38.759 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:38.859 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:38.930 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:38.978 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:38.997 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 8 16 21838 21822 170.387 136.688 0.0133858 0.00586108 2023-09-23T18:09:39.015 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:39.021 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:39.021 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:39.122 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:39.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1603507034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:39.145 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:09:39.177 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:39.191 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:09:39.233 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:09:39.275 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:09:39.312 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:09:39.344 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:09:39.344 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:39.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:39.349 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:39.377 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:09:39.413 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:09:39.423 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:09:39.423 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:09:39.423 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:09:39.423 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:09:39.423 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:09:39.423 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:09:39.423 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:09:39.423 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:09:39.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1603507034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:39.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:39.500 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1603507034' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:39.504 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:39.516 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:39.563 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:39.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:39.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:39.667 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:39.717 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:39.997 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 9 16 23938 23922 166.039 131.25 0.00580918 0.00601662 2023-09-23T18:09:40.032 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:40.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:40.040 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:40.141 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:40.158 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:09:40.203 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:40.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:40 smithi055 ceph-mon[184174]: pgmap v895: 119 pgs: 119 active+clean; 1.1 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T18:09:40.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1038518070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:40.358 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:40 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:40.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:40 smithi055 ceph-mon[179668]: pgmap v895: 119 pgs: 119 active+clean; 1.1 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T18:09:40.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1038518070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:40.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:40 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:40 smithi099 ceph-mon[167076]: pgmap v895: 119 pgs: 119 active+clean; 1.1 GiB data, 5.9 GiB used, 620 GiB / 626 GiB avail; 110 MiB/s wr, 1.76k op/s 2023-09-23T18:09:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1038518070' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:40 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:09:40.641 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:40.645 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:09:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:09:40] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-09-23T18:09:40.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:40.647 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:40.748 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:40.806 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:40.870 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:40.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:40.877 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:40.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:40.997 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 10 16 25798 25782 161.061 116.25 0.00537147 0.00620315 2023-09-23T18:09:41.033 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:41.089 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:41.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:41.100 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:41.200 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:41.274 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:41.314 INFO:teuthology.orchestra.run.smithi055.stderr:Error EINVAL: specified pg_num 41 < pg_num_max 32 2023-09-23T18:09:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:41 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:41 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:09:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:41 smithi055 ceph-mon[184174]: osdmap e366: 8 total, 8 up, 7 in 2023-09-23T18:09:41.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:41 smithi055 ceph-mon[184174]: failed to encode map e366 with expected crc 2023-09-23T18:09:41.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:41 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:41.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:41 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:09:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:41 smithi055 ceph-mon[179668]: osdmap e366: 8 total, 8 up, 7 in 2023-09-23T18:09:41.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:41 smithi055 ceph-mon[179668]: failed to encode map e366 with expected crc 2023-09-23T18:09:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:41 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:41 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:09:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:41 smithi099 ceph-mon[167076]: osdmap e366: 8 total, 8 up, 7 in 2023-09-23T18:09:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:41 smithi099 ceph-mon[167076]: failed to encode map e366 with expected crc 2023-09-23T18:09:41.667 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:41.672 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:41.672 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:41.727 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T18:09:41.773 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:41.829 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:41.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:41.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:41.897 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:41.990 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:09:41.991 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:09:41.991 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:09:41.991 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:09:41.997 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 11 16 27743 27727 157.471 121.562 0.00513321 0.00634607 2023-09-23T18:09:41.998 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:42.048 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:42.073 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:09:42.113 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:42.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:42.121 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:42.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:42.280 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:42.346 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:42.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:42.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:42.421 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:42 smithi055 ceph-mon[184174]: pgmap v897: 119 pgs: 119 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-09-23T18:09:42.422 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:42 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2397909307' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:09:42.422 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:42 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:09:42.422 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:42 smithi055 ceph-mon[179668]: pgmap v897: 119 pgs: 119 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-09-23T18:09:42.422 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:42 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2397909307' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:09:42.422 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:42 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:09:42.442 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:09:42.450 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:42.558 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:42.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:42 smithi099 ceph-mon[167076]: pgmap v897: 119 pgs: 119 active+clean; 1.3 GiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 136 MiB/s wr, 2.17k op/s 2023-09-23T18:09:42.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:42 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2397909307' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:09:42.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:42 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "41"}]: dispatch 2023-09-23T18:09:42.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:42.612 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:42.612 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:42.713 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:42.732 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:09:42.768 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:42.828 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:42.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:42.835 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:42.866 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:42.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:42.997 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 12 16 29554 29538 153.781 113.188 0.00574555 0.00649662 2023-09-23T18:09:43.000 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:43.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:43.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:43.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:43.156 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:43 smithi055 ceph-mon[179668]: 1.c scrub starts 2023-09-23T18:09:43.156 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:43 smithi055 ceph-mon[179668]: 1.c scrub ok 2023-09-23T18:09:43.173 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:43.214 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:43 smithi099 ceph-mon[167076]: 1.c scrub starts 2023-09-23T18:09:43.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:43 smithi099 ceph-mon[167076]: 1.c scrub ok 2023-09-23T18:09:43.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:43 smithi055 ceph-mon[184174]: 1.c scrub starts 2023-09-23T18:09:43.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:43 smithi055 ceph-mon[184174]: 1.c scrub ok 2023-09-23T18:09:43.906 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:43.909 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:43.910 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:43.997 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 13 16 31518 31502 151.394 122.75 0.00802426 0.00659961 2023-09-23T18:09:44.011 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:44.068 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:44.102 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:44.131 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:44.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:44.138 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:44.238 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:44.290 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:44.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:44 smithi099 ceph-mon[167076]: pgmap v898: 119 pgs: 119 active+clean; 1.7 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-09-23T18:09:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:44 smithi099 ceph-mon[167076]: 1.16 scrub starts 2023-09-23T18:09:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:44 smithi099 ceph-mon[167076]: 1.16 scrub ok 2023-09-23T18:09:44.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:44 smithi055 ceph-mon[184174]: pgmap v898: 119 pgs: 119 active+clean; 1.7 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-09-23T18:09:44.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:44 smithi055 ceph-mon[184174]: 1.16 scrub starts 2023-09-23T18:09:44.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:44 smithi055 ceph-mon[184174]: 1.16 scrub ok 2023-09-23T18:09:44.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:44 smithi055 ceph-mon[179668]: pgmap v898: 119 pgs: 119 active+clean; 1.7 GiB data, 7.3 GiB used, 618 GiB / 626 GiB avail; 148 MiB/s wr, 2.37k op/s 2023-09-23T18:09:44.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:44 smithi055 ceph-mon[179668]: 1.16 scrub starts 2023-09-23T18:09:44.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:44 smithi055 ceph-mon[179668]: 1.16 scrub ok 2023-09-23T18:09:44.998 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 14 16 33313 33297 148.593 112.188 0.0047248 0.00672517 2023-09-23T18:09:45.044 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:45.176 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:45.181 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:45.181 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:45.241 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:45 smithi055 ceph-mon[184174]: 1.13 scrub starts 2023-09-23T18:09:45.242 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:45 smithi055 ceph-mon[184174]: 1.13 scrub ok 2023-09-23T18:09:45.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:45 smithi055 ceph-mon[179668]: 1.13 scrub starts 2023-09-23T18:09:45.242 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:45 smithi055 ceph-mon[179668]: 1.13 scrub ok 2023-09-23T18:09:45.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:45.346 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:45.406 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:45.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:45.414 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:45.515 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:45.576 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:45.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:45 smithi099 ceph-mon[167076]: 1.13 scrub starts 2023-09-23T18:09:45.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:45 smithi099 ceph-mon[167076]: 1.13 scrub ok 2023-09-23T18:09:45.833 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:45.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:45.838 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:45.939 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:45.990 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:45.998 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 15 16 35107 35091 146.162 112.125 0.0102035 0.00683523 2023-09-23T18:09:46.065 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:46.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:46.073 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:46.174 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:46.234 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:46 smithi099 ceph-mon[167076]: pgmap v899: 119 pgs: 119 active+clean; 1.7 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-09-23T18:09:46.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:46 smithi055 ceph-mon[184174]: pgmap v899: 119 pgs: 119 active+clean; 1.7 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-09-23T18:09:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:46 smithi055 ceph-mon[179668]: pgmap v899: 119 pgs: 119 active+clean; 1.7 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 149 MiB/s wr, 2.39k op/s 2023-09-23T18:09:46.589 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:46.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:46.597 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:46.612 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:09:46.649 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:09:46.685 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:09:46.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:09:46.717 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:09:46.728 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 6, 7, 5, 4, 1] out_osds: [0] dead_osds: [] live_osds: [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:09:46.728 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:09:46.729 INFO:tasks.thrashosds.thrasher:Killing osd 5, live_osds are [0, 1, 2, 4, 5, 6, 7, 3] 2023-09-23T18:09:46.729 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T18:09:46.729 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T18:09:46.754 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:46.756 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:09:46.795 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:09:46.820 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:09:46.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:46.823 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:09:46.836 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:09:46.874 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:09:46.884 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:09:46.884 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:09:46.884 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:09:46.884 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:09:46.884 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:09:46.885 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:09:46.885 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:09:46.885 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:09:46.906 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:09:46.907 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:09:46.924 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:46.998 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 16 16 36783 36767 143.574 104.75 0.0107177 0.00695796 2023-09-23T18:09:47.006 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:09:46 smithi099 systemd[1]: Stopping Ceph osd.5 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:09:47.014 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:47.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:47.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:47.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:47.210 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:47.267 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/86629605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:47.275 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:47.347 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:47.352 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:47.353 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:47.453 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:47.497 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/86629605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:47.498 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/86629605' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:47.519 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:47.550 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:09:47.577 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:09:47 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[112818]: 2023-09-23T18:09:47.308+0000 7f3a47e39700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:09:47.577 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:09:47 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[112818]: 2023-09-23T18:09:47.308+0000 7f3a47e39700 -1 osd.5 366 *** Got signal Terminated *** 2023-09-23T18:09:47.577 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:09:47 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[112818]: 2023-09-23T18:09:47.308+0000 7f3a47e39700 -1 osd.5 366 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:09:47.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:47.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:47.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:47.689 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:47.742 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:47.801 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:47.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:47.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:47.907 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:47.960 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:47.998 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 17 16 38434 38418 141.198 103.188 0.00652546 0.00707815 2023-09-23T18:09:48.012 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:48.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:48.017 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:48.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:48.180 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:48.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:48.249 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:09:48.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:48.255 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:48.275 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:48.355 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:48.383 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:48.547 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:48 smithi055 ceph-mon[179668]: pgmap v900: 119 pgs: 119 active+clean; 1.9 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T18:09:48.547 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:48 smithi055 ceph-mon[179668]: osd.5 marked itself down and dead 2023-09-23T18:09:48.547 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:48 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/716612198' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:48.547 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:48 smithi055 ceph-mon[184174]: pgmap v900: 119 pgs: 119 active+clean; 1.9 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T18:09:48.547 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:48 smithi055 ceph-mon[184174]: osd.5 marked itself down and dead 2023-09-23T18:09:48.547 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:48 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/716612198' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:48.564 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:48.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:48.568 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:48.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:48 smithi099 ceph-mon[167076]: pgmap v900: 119 pgs: 119 active+clean; 1.9 GiB data, 8.3 GiB used, 618 GiB / 626 GiB avail; 138 MiB/s wr, 2.21k op/s 2023-09-23T18:09:48.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:48 smithi099 ceph-mon[167076]: osd.5 marked itself down and dead 2023-09-23T18:09:48.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:48 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/716612198' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:09:48.668 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:48.698 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:48.783 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:48.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:48.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:48.888 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:48.917 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:48.952 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:48.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:48.955 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:48.998 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 18 16 38843 38827 134.775 25.5625 0.00517547 0.0071071 2023-09-23T18:09:49.056 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:49.086 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:49.309 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:49.313 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:09:49 smithi099 bash[240606]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5 2023-09-23T18:09:49.313 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:49 smithi099 ceph-mon[167076]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:09:49.314 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:49 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:09:49.314 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:49 smithi099 ceph-mon[167076]: failed to encode map e367 with expected crc 2023-09-23T18:09:49.314 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:49 smithi099 ceph-mon[167076]: failed to encode map e367 with expected crc 2023-09-23T18:09:49.314 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/716612198' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:49.314 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:49 smithi099 ceph-mon[167076]: osdmap e367: 8 total, 7 up, 7 in 2023-09-23T18:09:49.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:49.315 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:49.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:49.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[179668]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:09:49.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:09:49.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[179668]: failed to encode map e367 with expected crc 2023-09-23T18:09:49.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[179668]: failed to encode map e367 with expected crc 2023-09-23T18:09:49.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/716612198' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:49.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[179668]: osdmap e367: 8 total, 7 up, 7 in 2023-09-23T18:09:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[184174]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:09:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:09:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[184174]: failed to encode map e367 with expected crc 2023-09-23T18:09:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[184174]: failed to encode map e367 with expected crc 2023-09-23T18:09:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/716612198' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:09:49.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:49 smithi055 ceph-mon[184174]: osdmap e367: 8 total, 7 up, 7 in 2023-09-23T18:09:49.496 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:49.535 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:09:49.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:49.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:49.575 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:49.676 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:49.725 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:49.789 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:49.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:49.796 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:49.896 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:49.960 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:49.969 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:09:49.969 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:09:49.998 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 19 16 40225 40209 132.225 86.375 0.0124335 0.00755858 2023-09-23T18:09:50.024 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:50.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:50.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:50.134 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:50.197 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:50.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[179668]: pgmap v902: 119 pgs: 28 stale+active+clean, 91 active+clean; 2.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 140 MiB/s wr, 2.23k op/s 2023-09-23T18:09:50.236 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[179668]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[184174]: pgmap v902: 119 pgs: 28 stale+active+clean, 91 active+clean; 2.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 140 MiB/s wr, 2.23k op/s 2023-09-23T18:09:50.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[184174]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.236 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[184174]: osdmap e368: 8 total, 7 up, 7 in 2023-09-23T18:09:50.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[184174]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[184174]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[184174]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[184174]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:50.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:50.267 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:50.306 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:09:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:50 smithi099 ceph-mon[167076]: pgmap v902: 119 pgs: 28 stale+active+clean, 91 active+clean; 2.2 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 140 MiB/s wr, 2.23k op/s 2023-09-23T18:09:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:50 smithi099 ceph-mon[167076]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:50 smithi099 ceph-mon[167076]: osdmap e368: 8 total, 7 up, 7 in 2023-09-23T18:09:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:50 smithi099 ceph-mon[167076]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:50 smithi099 ceph-mon[167076]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:50 smithi099 ceph-mon[167076]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:50 smithi099 ceph-mon[167076]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.368 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:50.439 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:50.491 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:50.495 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:50.507 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:09:50] "GET /metrics HTTP/1.1" 200 36592 "" "Prometheus/2.43.0" 2023-09-23T18:09:50.508 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:50.508 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:50.509 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[179668]: osdmap e368: 8 total, 7 up, 7 in 2023-09-23T18:09:50.509 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[179668]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.509 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[179668]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.509 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[179668]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.509 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:50 smithi055 ceph-mon[179668]: failed to encode map e368 with expected crc 2023-09-23T18:09:50.572 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:09:50.609 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:50.721 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:50.811 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:50.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:50.819 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:50.920 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:50.989 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:50.998 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:09:50.999642+0000 min lat: 0.00131331 max lat: 1.03702 avg lat: 0.00763012 2023-09-23T18:09:50.998 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:09:50.999 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 20 16 41934 41918 130.954 106.812 0.0142539 0.00763012 2023-09-23T18:09:51.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T18:09:51.059 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:09:51.059 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T18:09:51.327 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:51.337 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:51 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 7706/118608 objects degraded (6.497%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:09:51.338 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:51 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 7706/118608 objects degraded (6.497%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:09:51.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:51.339 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:51.439 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:51.441 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:51.508 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:51.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:51 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 7706/118608 objects degraded (6.497%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:09:51.628 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:51.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:51.634 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:51.710 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:51.735 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:51.809 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:51.873 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:51.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:51.883 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:51.981 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:09:51.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:51.999 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 21 16 43759 43743 130.147 114.062 0.00645909 0.0076796 2023-09-23T18:09:52.020 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:09:52.023 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:52.056 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:09:52.083 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:52.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:52.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:52.090 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:09:52.128 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:09:52.164 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:09:52.191 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:52.205 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:09:52.205 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:09:52.205 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:09:52.205 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:09:52.205 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:09:52.205 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:09:52.206 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:09:52.206 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:09:52.206 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:09:52.280 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:52.349 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:52.353 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:52.354 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:52.455 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:52.510 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:52 smithi055 ceph-mon[179668]: pgmap v904: 119 pgs: 8 active+undersized+degraded, 16 active+undersized, 15 stale+active+clean, 80 active+clean; 2.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s; 7706/118608 objects degraded (6.497%) 2023-09-23T18:09:52.511 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:52.511 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:52.511 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4234129748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:52.511 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:52 smithi055 ceph-mon[184174]: pgmap v904: 119 pgs: 8 active+undersized+degraded, 16 active+undersized, 15 stale+active+clean, 80 active+clean; 2.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s; 7706/118608 objects degraded (6.497%) 2023-09-23T18:09:52.511 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:52.511 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:52.511 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4234129748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:52.517 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:52.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:52 smithi099 ceph-mon[167076]: pgmap v904: 119 pgs: 8 active+undersized+degraded, 16 active+undersized, 15 stale+active+clean, 80 active+clean; 2.4 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 98 MiB/s wr, 1.56k op/s; 7706/118608 objects degraded (6.497%) 2023-09-23T18:09:52.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:09:52.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:09:52.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4234129748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:09:52.879 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:09:52.879 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:09:52.999 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 22 16 45629 45613 129.543 116.875 0.00673202 0.00771436 2023-09-23T18:09:53.159 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:53.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:53.165 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:53.266 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:53.338 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:53.412 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:53.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:53.420 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:53.521 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:53.567 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:53.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:53.624 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:53.624 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:53.704 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:09:53.704 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:09:53.704 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.704 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:46905 8.f 8:f7ebd34a:::benchmark_data_smithi055_346788_object46904:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T18:09:53.704 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:53.694012+0000", 2023-09-23T18:09:53.704 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0053048740000000002, 2023-09-23T18:09:53.704 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053391030000000004, 2023-09-23T18:09:53.704 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:09:53.704 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 46905 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.694012+0000", 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:09:53.705 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.694012+0000", 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.694011+0000", 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.706 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.694038+0000", 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7047999999999999e-05 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.694052+0000", 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4217e-05 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:09:53.707 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.694058+0000", 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7780000000000004e-06 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.694251+0000", 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019351700000000001 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.694442+0000", 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019097600000000001 2023-09-23T18:09:53.708 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.694592+0000", 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014933700000000001 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.698740+0000", 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041479589999999997 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.709 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.698747+0000", 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9229999999999996e-06 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:46909 8.7 8:e37042d8:::benchmark_data_smithi055_346788_object46908:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e368)", 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:53.696434+0000", 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0028831669999999998, 2023-09-23T18:09:53.710 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030168270000000001, 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 46909 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696434+0000", 2023-09-23T18:09:53.711 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696434+0000", 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696432+0000", 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.712 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696459+0000", 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6800000000000001e-05 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696471+0000", 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1844999999999999e-05 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:09:53.713 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696479+0000", 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7489999999999998e-06 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696695+0000", 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021665500000000001 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696839+0000", 2023-09-23T18:09:53.714 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000143201 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,6", 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696977+0000", 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013800499999999999 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.699170+0000", 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021931049999999999 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:09:53.715 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:46908 8.b e368/361 8:d9bd3294:::benchmark_data_smithi055_346788_object46907:head v 368'2918, mlcod=368'2915)", 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:53.696470+0000", 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0028471030000000001, 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003062099, 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.716 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696470+0000", 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696470+0000", 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696469+0000", 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:09:53.717 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696524+0000", 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5279999999999999e-05 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696541+0000", 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6634e-05 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.718 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696547+0000", 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8850000000000004e-06 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696611+0000", 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4344000000000006e-05 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.696631+0000", 2023-09-23T18:09:53.719 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9562e-05 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:46912 8.b e368/361 8:d2227633:::benchmark_data_smithi055_346788_object46911:head v 368'2919, mlcod=368'2915)", 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:53.698305+0000", 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.00101169, 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0012887479999999999, 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:09:53.720 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.698305+0000", 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.698305+0000", 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.721 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.698305+0000", 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.698337+0000", 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2472999999999999e-05 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:09:53.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.698344+0000", 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5490000000000003e-06 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.698348+0000", 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4649999999999996e-06 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.698395+0000", 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7138999999999997e-05 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:09:53.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:09:53.724 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:09:53.724 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:53.698413+0000", 2023-09-23T18:09:53.724 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7734999999999999e-05 2023-09-23T18:09:53.724 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:09:53.724 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:09:53.724 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:09:53.724 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:09:53.724 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:09:53.724 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 4 2023-09-23T18:09:53.724 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:09:53.725 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:09:53.761 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:53.812 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:09:53.820 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:53.820 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:09:53.921 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:53.980 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:53.999 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 23 16 47446 47430 128.847 113.562 0.00753441 0.0077574 2023-09-23T18:09:54.099 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:54.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:54.107 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:54.159 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:09:54.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:54.266 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:54.311 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:54.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:54.316 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:54.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:54 smithi055 ceph-mon[179668]: pgmap v905: 119 pgs: 15 active+undersized+degraded, 28 active+undersized, 8 stale+active+clean, 68 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 5.1 KiB/s rd, 117 MiB/s wr, 1.87k op/s; 21659/127812 objects degraded (16.946%) 2023-09-23T18:09:54.417 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:54.473 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:54.542 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:54.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:54.550 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:54.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:54 smithi099 ceph-mon[167076]: pgmap v905: 119 pgs: 15 active+undersized+degraded, 28 active+undersized, 8 stale+active+clean, 68 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 5.1 KiB/s rd, 117 MiB/s wr, 1.87k op/s; 21659/127812 objects degraded (16.946%) 2023-09-23T18:09:54.651 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:54.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:54 smithi055 ceph-mon[184174]: pgmap v905: 119 pgs: 15 active+undersized+degraded, 28 active+undersized, 8 stale+active+clean, 68 active+clean; 2.6 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 5.1 KiB/s rd, 117 MiB/s wr, 1.87k op/s; 21659/127812 objects degraded (16.946%) 2023-09-23T18:09:54.719 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:54.785 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:54.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:54.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:54.880 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:09:54.890 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:54.994 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:55.005 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 24 16 49166 49150 127.957 107.5 0.0136046 0.00780956 2023-09-23T18:09:55.020 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:55.144 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:55.149 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:55.150 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:55.251 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:55.304 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:55.381 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:09:55.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3842778584' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:55.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:55 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:55.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3842778584' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:55.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:55 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:55.819 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:55.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3842778584' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:55.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:55 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:09:55.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:55.830 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:55.931 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:55.986 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:56.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 25 15 50807 50792 126.943 102.625 0.0102667 0.0078726 2023-09-23T18:09:56.059 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 6, 7, 5, 4, 1] out_osds: [0] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:09:56.060 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:09:56.060 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-09-23T18:09:56.060 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 0 2023-09-23T18:09:56.199 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:56.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:56.204 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:56.305 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:09:56.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[179668]: pgmap v906: 119 pgs: 20 active+undersized+degraded, 38 active+undersized, 61 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 12 KiB/s rd, 97 MiB/s wr, 1.55k op/s; 27470/130686 objects degraded (21.020%); 119 KiB/s, 0 objects/s recovering 2023-09-23T18:09:56.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:56.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[179668]: failed to encode map e369 with expected crc 2023-09-23T18:09:56.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:09:56.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[179668]: osdmap e369: 8 total, 7 up, 7 in 2023-09-23T18:09:56.357 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[179668]: failed to encode map e369 with expected crc 2023-09-23T18:09:56.375 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:56.606 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:09:56.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:56.617 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:09:56.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[184174]: pgmap v906: 119 pgs: 20 active+undersized+degraded, 38 active+undersized, 61 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 12 KiB/s rd, 97 MiB/s wr, 1.55k op/s; 27470/130686 objects degraded (21.020%); 119 KiB/s, 0 objects/s recovering 2023-09-23T18:09:56.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:56.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[184174]: failed to encode map e369 with expected crc 2023-09-23T18:09:56.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:09:56.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[184174]: osdmap e369: 8 total, 7 up, 7 in 2023-09-23T18:09:56.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:56 smithi055 ceph-mon[184174]: failed to encode map e369 with expected crc 2023-09-23T18:09:56.695 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:56.718 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:56.772 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:56 smithi099 ceph-mon[167076]: pgmap v906: 119 pgs: 20 active+undersized+degraded, 38 active+undersized, 61 active+clean; 2.7 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 12 KiB/s rd, 97 MiB/s wr, 1.55k op/s; 27470/130686 objects degraded (21.020%); 119 KiB/s, 0 objects/s recovering 2023-09-23T18:09:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:56 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:09:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:56 smithi099 ceph-mon[167076]: failed to encode map e369 with expected crc 2023-09-23T18:09:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:56 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:09:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:56 smithi099 ceph-mon[167076]: osdmap e369: 8 total, 7 up, 7 in 2023-09-23T18:09:56.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:56 smithi099 ceph-mon[167076]: failed to encode map e369 with expected crc 2023-09-23T18:09:57.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 26 16 52519 52503 126.173 106.938 0.00966092 0.00792091 2023-09-23T18:09:57.005 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:09:57.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:57.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:09:57.113 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:57.156 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:57.200 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:57.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:57.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:57.226 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:57.327 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:57.394 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:57.410 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:57 smithi099 ceph-mon[167076]: pgmap v908: 119 pgs: 20 active+undersized+degraded, 38 active+undersized, 61 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 18 KiB/s rd, 83 MiB/s wr, 1.34k op/s; 29114/142176 objects degraded (20.477%); 130 KiB/s, 0 objects/s recovering 2023-09-23T18:09:57.410 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:09:57 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 29114/142176 objects degraded (20.477%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T18:09:57.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:57 smithi055 ceph-mon[179668]: pgmap v908: 119 pgs: 20 active+undersized+degraded, 38 active+undersized, 61 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 18 KiB/s rd, 83 MiB/s wr, 1.34k op/s; 29114/142176 objects degraded (20.477%); 130 KiB/s, 0 objects/s recovering 2023-09-23T18:09:57.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:57 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 29114/142176 objects degraded (20.477%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T18:09:57.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:57 smithi055 ceph-mon[184174]: pgmap v908: 119 pgs: 20 active+undersized+degraded, 38 active+undersized, 61 active+clean; 2.9 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 18 KiB/s rd, 83 MiB/s wr, 1.34k op/s; 29114/142176 objects degraded (20.477%); 130 KiB/s, 0 objects/s recovering 2023-09-23T18:09:57.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:09:57 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 29114/142176 objects degraded (20.477%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T18:09:57.656 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:57.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:57.663 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:57.764 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:57.813 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:57.862 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:57.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:57.868 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:57.935 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:09:57.936 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:09:57.936 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:09:57.936 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:09:57.936 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:09:57.968 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:58.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 27 16 54166 54150 125.313 102.938 0.0162289 0.00797555 2023-09-23T18:09:58.005 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:58.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:58.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:58.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:58.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:09:58.210 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:58.260 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:09:58.266 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:58.266 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:09:58.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:58.409 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:58.454 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:09:58.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:58.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:58.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:58.572 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:09:58.621 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:58.666 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:09:58.681 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:09:58.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:58.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:09:58.792 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:58.903 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:59.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 28 16 55703 55687 124.268 96.0625 0.00977056 0.00804159 2023-09-23T18:09:59.120 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:59.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:59.124 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:59.225 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:09:59.265 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:09:59.309 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:09:59.337 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:59.347 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:09:59.354 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:59.396 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:09:59.414 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:09:59.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:59.425 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:09:59.437 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:09:59.480 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:09:59.519 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:09:59.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:09:59.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:09:59.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:09:59.519 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:09:59.520 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:09:59.520 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:09:59.520 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:09:59.520 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:09:59.526 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:09:59.591 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:09:59.747 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:09:59.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:09:59.752 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:09:59.852 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:09:59.875 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:09:59.922 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:00.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 29 16 57006 56990 122.791 81.4375 0.00647097 0.00813964 2023-09-23T18:10:00.140 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:09:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:09:59.928+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e369 definitely_dead 0 2023-09-23T18:10:00.141 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pgmap v909: 119 pgs: 20 active+undersized+degraded, 38 active+undersized, 61 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 15 KiB/s rd, 100 MiB/s wr, 1.61k op/s; 32866/156654 objects degraded (20.980%); 108 KiB/s, 0 objects/s recovering 2023-09-23T18:10:00.141 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3312145806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:00.141 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/514851899' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T18:10:00.141 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T18:10:00.141 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 32866/156654 objects degraded (20.980%), 20 pgs degraded 2023-09-23T18:10:00.141 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T18:10:00.141 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T18:10:00.141 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: osd.5 (root=default,host=smithi099) is down 2023-09-23T18:10:00.141 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: [WRN] PG_DEGRADED: Degraded data redundancy: 32866/156654 objects degraded (20.980%), 20 pgs degraded 2023-09-23T18:10:00.141 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 1.c is active+undersized+degraded, acting [6,4] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 1.11 is active+undersized+degraded, acting [1,6] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 2.2 is active+undersized+degraded, acting [1,6] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 2.3 is active+undersized+degraded, acting [7,2] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 3.7 is active+undersized+degraded, acting [7,3] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 3.d is active+undersized+degraded, acting [7,6] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 3.10 is active+undersized+degraded, acting [6,2] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 3.1c is active+undersized+degraded, acting [4,1] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 3.1d is active+undersized+degraded, acting [4,6] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 3.1f is active+undersized+degraded, acting [6,4] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 8.0 is active+undersized+degraded, acting [1,3] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 8.1 is active+undersized+degraded, acting [1,2] 2023-09-23T18:10:00.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 8.2 is active+undersized+degraded, acting [4,6] 2023-09-23T18:10:00.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 8.3 is active+undersized+degraded, acting [1,6] 2023-09-23T18:10:00.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 8.4 is active+undersized+degraded, acting [4,6] 2023-09-23T18:10:00.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 8.8 is active+undersized+degraded, acting [3,7] 2023-09-23T18:10:00.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 8.9 is active+undersized+degraded, acting [6,3] 2023-09-23T18:10:00.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 8.a is active+undersized+degraded, acting [1,2] 2023-09-23T18:10:00.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 8.c is active+undersized+degraded, acting [7,3] 2023-09-23T18:10:00.143 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[179668]: pg 8.d is active+undersized+degraded, acting [7,2] 2023-09-23T18:10:00.149 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.0. 2023-09-23T18:10:00.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:10:00] "GET /metrics HTTP/1.1" 200 36605 "" "Prometheus/2.43.0" 2023-09-23T18:10:00.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pgmap v909: 119 pgs: 20 active+undersized+degraded, 38 active+undersized, 61 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 15 KiB/s rd, 100 MiB/s wr, 1.61k op/s; 32866/156654 objects degraded (20.980%); 108 KiB/s, 0 objects/s recovering 2023-09-23T18:10:00.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3312145806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:00.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/514851899' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 32866/156654 objects degraded (20.980%), 20 pgs degraded 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: osd.5 (root=default,host=smithi099) is down 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: [WRN] PG_DEGRADED: Degraded data redundancy: 32866/156654 objects degraded (20.980%), 20 pgs degraded 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 1.c is active+undersized+degraded, acting [6,4] 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 1.11 is active+undersized+degraded, acting [1,6] 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 2.2 is active+undersized+degraded, acting [1,6] 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 2.3 is active+undersized+degraded, acting [7,2] 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 3.7 is active+undersized+degraded, acting [7,3] 2023-09-23T18:10:00.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 3.d is active+undersized+degraded, acting [7,6] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 3.10 is active+undersized+degraded, acting [6,2] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 3.1c is active+undersized+degraded, acting [4,1] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 3.1d is active+undersized+degraded, acting [4,6] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 3.1f is active+undersized+degraded, acting [6,4] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 8.0 is active+undersized+degraded, acting [1,3] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 8.1 is active+undersized+degraded, acting [1,2] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 8.2 is active+undersized+degraded, acting [4,6] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 8.3 is active+undersized+degraded, acting [1,6] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 8.4 is active+undersized+degraded, acting [4,6] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 8.8 is active+undersized+degraded, acting [3,7] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 8.9 is active+undersized+degraded, acting [6,3] 2023-09-23T18:10:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 8.a is active+undersized+degraded, acting [1,2] 2023-09-23T18:10:00.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 8.c is active+undersized+degraded, acting [7,3] 2023-09-23T18:10:00.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:00 smithi055 ceph-mon[184174]: pg 8.d is active+undersized+degraded, acting [7,2] 2023-09-23T18:10:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pgmap v909: 119 pgs: 20 active+undersized+degraded, 38 active+undersized, 61 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 15 KiB/s rd, 100 MiB/s wr, 1.61k op/s; 32866/156654 objects degraded (20.980%); 108 KiB/s, 0 objects/s recovering 2023-09-23T18:10:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3312145806' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/514851899' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T18:10:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: Health detail: HEALTH_WARN noscrub flag(s) set; 1 osds down; Degraded data redundancy: 32866/156654 objects degraded (20.980%), 20 pgs degraded 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: [WRN] OSDMAP_FLAGS: noscrub flag(s) set 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: [WRN] OSD_DOWN: 1 osds down 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: osd.5 (root=default,host=smithi099) is down 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: [WRN] PG_DEGRADED: Degraded data redundancy: 32866/156654 objects degraded (20.980%), 20 pgs degraded 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 1.c is active+undersized+degraded, acting [6,4] 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 1.11 is active+undersized+degraded, acting [1,6] 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 2.2 is active+undersized+degraded, acting [1,6] 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 2.3 is active+undersized+degraded, acting [7,2] 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 3.7 is active+undersized+degraded, acting [7,3] 2023-09-23T18:10:00.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 3.d is active+undersized+degraded, acting [7,6] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 3.10 is active+undersized+degraded, acting [6,2] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 3.1c is active+undersized+degraded, acting [4,1] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 3.1d is active+undersized+degraded, acting [4,6] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 3.1f is active+undersized+degraded, acting [6,4] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 8.0 is active+undersized+degraded, acting [1,3] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 8.1 is active+undersized+degraded, acting [1,2] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 8.2 is active+undersized+degraded, acting [4,6] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 8.3 is active+undersized+degraded, acting [1,6] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 8.4 is active+undersized+degraded, acting [4,6] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 8.8 is active+undersized+degraded, acting [3,7] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 8.9 is active+undersized+degraded, acting [6,3] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 8.a is active+undersized+degraded, acting [1,2] 2023-09-23T18:10:00.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 8.c is active+undersized+degraded, acting [7,3] 2023-09-23T18:10:00.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:00 smithi099 ceph-mon[167076]: pg 8.d is active+undersized+degraded, acting [7,2] 2023-09-23T18:10:00.657 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:00.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:00.661 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:00.715 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:10:00.716 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:10:00.762 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:00.791 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:00.825 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:00.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:00.828 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:00.929 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:00.955 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:00.987 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:00.990 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:00.990 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:01.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 30 16 57285 57269 119.28 17.4375 0.0024698 0.00814308 2023-09-23T18:10:01.085 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-09-23T18:10:01.091 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:01.123 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:01.175 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:01.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:01.179 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:01.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:01 smithi055 ceph-mon[184174]: failed to encode map e370 with expected crc 2023-09-23T18:10:01.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:01 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T18:10:01.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:01 smithi055 ceph-mon[184174]: osdmap e370: 8 total, 7 up, 8 in 2023-09-23T18:10:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:01 smithi055 ceph-mon[179668]: failed to encode map e370 with expected crc 2023-09-23T18:10:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:01 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T18:10:01.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:01 smithi055 ceph-mon[179668]: osdmap e370: 8 total, 7 up, 8 in 2023-09-23T18:10:01.280 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:01.310 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:01 smithi099 ceph-mon[167076]: failed to encode map e370 with expected crc 2023-09-23T18:10:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:01 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T18:10:01.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:01 smithi099 ceph-mon[167076]: osdmap e370: 8 total, 7 up, 8 in 2023-09-23T18:10:01.844 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:10:01.844 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:10:01.862 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:01.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:01.865 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:01.966 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:01.994 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:02.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 31 16 57285 57269 115.433 0 - 0.00814308 2023-09-23T18:10:02.228 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:02.233 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:02.233 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:02.334 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:02.375 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:02.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:02.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:02 smithi099 ceph-mon[167076]: pgmap v911: 119 pgs: 2 peering, 19 active+undersized+degraded, 38 active+undersized, 60 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 5.2 KiB/s rd, 95 MiB/s wr, 1.53k op/s; 30679/164403 objects degraded (18.661%); 90 KiB/s, 0 objects/s recovering 2023-09-23T18:10:02.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:02 smithi099 ceph-mon[167076]: failed to encode map e371 with expected crc 2023-09-23T18:10:02.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:02 smithi099 ceph-mon[167076]: osdmap e371: 8 total, 7 up, 8 in 2023-09-23T18:10:02.420 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:02 smithi099 ceph-mon[167076]: failed to encode map e371 with expected crc 2023-09-23T18:10:02.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:02.421 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:02 smithi055 ceph-mon[184174]: pgmap v911: 119 pgs: 2 peering, 19 active+undersized+degraded, 38 active+undersized, 60 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 5.2 KiB/s rd, 95 MiB/s wr, 1.53k op/s; 30679/164403 objects degraded (18.661%); 90 KiB/s, 0 objects/s recovering 2023-09-23T18:10:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:02 smithi055 ceph-mon[184174]: failed to encode map e371 with expected crc 2023-09-23T18:10:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:02 smithi055 ceph-mon[184174]: osdmap e371: 8 total, 7 up, 8 in 2023-09-23T18:10:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:02 smithi055 ceph-mon[184174]: failed to encode map e371 with expected crc 2023-09-23T18:10:02.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:02 smithi055 ceph-mon[179668]: pgmap v911: 119 pgs: 2 peering, 19 active+undersized+degraded, 38 active+undersized, 60 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 5.2 KiB/s rd, 95 MiB/s wr, 1.53k op/s; 30679/164403 objects degraded (18.661%); 90 KiB/s, 0 objects/s recovering 2023-09-23T18:10:02.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:02 smithi055 ceph-mon[179668]: failed to encode map e371 with expected crc 2023-09-23T18:10:02.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:02 smithi055 ceph-mon[179668]: osdmap e371: 8 total, 7 up, 8 in 2023-09-23T18:10:02.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:02 smithi055 ceph-mon[179668]: failed to encode map e371 with expected crc 2023-09-23T18:10:02.522 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:02.592 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:02.716 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:10:03.000 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 32 16 58007 57991 113.235 22.5625 0.0123869 0.00882444 2023-09-23T18:10:03.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:03 smithi055 ceph-mon[179668]: failed to encode map e372 with expected crc 2023-09-23T18:10:03.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:03 smithi055 ceph-mon[179668]: osdmap e372: 8 total, 7 up, 8 in 2023-09-23T18:10:03.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 30679/164403 objects degraded (18.661%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:03.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:03 smithi055 ceph-mon[179668]: osdmap e373: 8 total, 7 up, 8 in 2023-09-23T18:10:03.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:03 smithi055 ceph-mon[184174]: failed to encode map e372 with expected crc 2023-09-23T18:10:03.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:03 smithi055 ceph-mon[184174]: osdmap e372: 8 total, 7 up, 8 in 2023-09-23T18:10:03.434 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 30679/164403 objects degraded (18.661%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:03.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:03 smithi055 ceph-mon[184174]: osdmap e373: 8 total, 7 up, 8 in 2023-09-23T18:10:03.501 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:03.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:03.507 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:03.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:03 smithi099 ceph-mon[167076]: failed to encode map e372 with expected crc 2023-09-23T18:10:03.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:03 smithi099 ceph-mon[167076]: osdmap e372: 8 total, 7 up, 8 in 2023-09-23T18:10:03.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 30679/164403 objects degraded (18.661%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:03.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:03 smithi099 ceph-mon[167076]: osdmap e373: 8 total, 7 up, 8 in 2023-09-23T18:10:03.608 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:03.680 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:03.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:03.928 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:03.928 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:04.001 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 33 15 59169 59154 112.006 72.6875 0.00841333 0.00892269 2023-09-23T18:10:04.029 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:04.095 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:04.165 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:04.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:04.183 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:04.198 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:10:04.284 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:04.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[179668]: pgmap v914: 119 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 18 active+undersized+degraded, 35 active+undersized, 59 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 31971/169926 objects degraded (18.815%); 139 KiB/s, 4 objects/s recovering 2023-09-23T18:10:04.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[179668]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[179668]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[179668]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[179668]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[179668]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[179668]: osdmap e374: 8 total, 7 up, 8 in 2023-09-23T18:10:04.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[184174]: pgmap v914: 119 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 18 active+undersized+degraded, 35 active+undersized, 59 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 31971/169926 objects degraded (18.815%); 139 KiB/s, 4 objects/s recovering 2023-09-23T18:10:04.353 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[184174]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.353 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[184174]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.353 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[184174]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.353 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[184174]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.353 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[184174]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.353 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:04 smithi055 ceph-mon[184174]: osdmap e374: 8 total, 7 up, 8 in 2023-09-23T18:10:04.354 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:04.414 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:04.439 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:04.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:04.448 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:04.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:04 smithi099 ceph-mon[167076]: pgmap v914: 119 pgs: 2 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 18 active+undersized+degraded, 35 active+undersized, 59 active+clean; 3.5 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 96 MiB/s wr, 1.54k op/s; 31971/169926 objects degraded (18.815%); 139 KiB/s, 4 objects/s recovering 2023-09-23T18:10:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:04 smithi099 ceph-mon[167076]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:04 smithi099 ceph-mon[167076]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:04 smithi099 ceph-mon[167076]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:04 smithi099 ceph-mon[167076]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:04 smithi099 ceph-mon[167076]: failed to encode map e373 with expected crc 2023-09-23T18:10:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:04 smithi099 ceph-mon[167076]: osdmap e374: 8 total, 7 up, 8 in 2023-09-23T18:10:04.584 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:04.628 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:04.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:04.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:04.734 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:04.828 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:04.906 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:04.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:04.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:05.002 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 34 16 60077 60061 110.376 56.6875 0.00748168 0.00905382 2023-09-23T18:10:05.013 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:05.113 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:05.251 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:10:05.279 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:05 smithi055 ceph-mon[179668]: failed to encode map e374 with expected crc 2023-09-23T18:10:05.279 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:05 smithi055 ceph-mon[179668]: failed to encode map e374 with expected crc 2023-09-23T18:10:05.279 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2235430945' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:05.279 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:05 smithi055 ceph-mon[184174]: failed to encode map e374 with expected crc 2023-09-23T18:10:05.279 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:05 smithi055 ceph-mon[184174]: failed to encode map e374 with expected crc 2023-09-23T18:10:05.279 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2235430945' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:05.502 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:05.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:05 smithi099 ceph-mon[167076]: failed to encode map e374 with expected crc 2023-09-23T18:10:05.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:05 smithi099 ceph-mon[167076]: failed to encode map e374 with expected crc 2023-09-23T18:10:05.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2235430945' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:05.602 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:05.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:05.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:05.710 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:05.785 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:05.885 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:05.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:05.892 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:05.993 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:06.005 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 35 16 61127 61111 109.088 65.625 0.0103372 0.00916025 2023-09-23T18:10:06.063 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:06.086 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 6, 7, 5, 4, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:10:06.086 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:10:06.087 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2023-09-23T18:10:06.087 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:10:06.204 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:06.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:06.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:06.314 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:06.417 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:06.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:06 smithi055 ceph-mon[179668]: pgmap v917: 119 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 17 active+undersized+degraded, 35 active+undersized, 59 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s; 32443/172953 objects degraded (18.758%); 3625/172953 objects misplaced (2.096%); 208 KiB/s, 8 objects/s recovering 2023-09-23T18:10:06.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:06 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:06.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:06 smithi055 ceph-mon[179668]: failed to encode map e375 with expected crc 2023-09-23T18:10:06.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:06 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2235430945' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:06.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:06 smithi055 ceph-mon[179668]: osdmap e375: 8 total, 7 up, 8 in 2023-09-23T18:10:06.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:06 smithi055 ceph-mon[184174]: pgmap v917: 119 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 17 active+undersized+degraded, 35 active+undersized, 59 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s; 32443/172953 objects degraded (18.758%); 3625/172953 objects misplaced (2.096%); 208 KiB/s, 8 objects/s recovering 2023-09-23T18:10:06.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:06 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:06.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:06 smithi055 ceph-mon[184174]: failed to encode map e375 with expected crc 2023-09-23T18:10:06.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:06 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2235430945' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:06.433 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:06 smithi055 ceph-mon[184174]: osdmap e375: 8 total, 7 up, 8 in 2023-09-23T18:10:06.473 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:10:06.550 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:06.551 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:10:06.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:06.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:06 smithi099 ceph-mon[167076]: pgmap v917: 119 pgs: 1 active+recovery_wait+remapped, 2 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 17 active+undersized+degraded, 35 active+undersized, 59 active+clean; 3.5 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 45 MiB/s wr, 712 op/s; 32443/172953 objects degraded (18.758%); 3625/172953 objects misplaced (2.096%); 208 KiB/s, 8 objects/s recovering 2023-09-23T18:10:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:06 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:06 smithi099 ceph-mon[167076]: failed to encode map e375 with expected crc 2023-09-23T18:10:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:06 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2235430945' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:06 smithi099 ceph-mon[167076]: osdmap e375: 8 total, 7 up, 8 in 2023-09-23T18:10:06.599 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:10:06.634 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:10:06.658 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:06.674 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:10:06.724 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:10:06.727 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:06.769 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:10:06.769 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:10:06.769 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:10:06.769 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:10:06.770 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:10:06.770 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:10:06.770 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:10:06.770 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:10:06.770 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:10:06.808 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:06.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:06.815 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:06.916 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:06.953 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:06.990 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:06.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:06.993 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:07.005 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 36 16 61999 61983 107.572 54.5 0.00553832 0.00929036 2023-09-23T18:10:07.094 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:07.181 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:07.193 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:10:07.193 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:10:07.237 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:07.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:07 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4100409175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:07.258 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:07.258 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:07.304 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:07.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:07.312 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:07.412 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:07 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4100409175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:07.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:07.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:07 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4100409175' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:07.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:07.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:07.493 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:07.617 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:07.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:07.623 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:07.661 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:10:07.661 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:10:07.661 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.661 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.45923.0:62648 8.6 e375/371)", 2023-09-23T18:10:07.661 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:07.659781+0000", 2023-09-23T18:10:07.661 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.00073600399999999996, 2023-09-23T18:10:07.661 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00075495599999999999, 2023-09-23T18:10:07.661 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:07.661 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.659781+0000", 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.659781+0000", 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:07.662 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.659781+0000", 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.659785+0000", 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3250000000000001e-06 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:07.663 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.659802+0000", 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6569e-05 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.659810+0000", 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9300000000000003e-06 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:62651 8.a e375/367 8:5576b7bf:::benchmark_data_smithi055_346788_object62650:head v 375'3964, mlcod=375'3962)", 2023-09-23T18:10:07.664 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:07.660099+0000", 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.00041882099999999999, 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00047266099999999998, 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.660099+0000", 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.665 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.660099+0000", 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.660098+0000", 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.666 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.660127+0000", 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8663999999999999e-05 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.660137+0000", 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9629999999999993e-06 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:07.667 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.660143+0000", 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7510000000000002e-06 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.660222+0000", 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9134000000000005e-05 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.660249+0000", 2023-09-23T18:10:07.668 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6502000000000001e-05 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:62648 8.6 8:6dce3df4:::benchmark_data_smithi055_346788_object62647:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e375)", 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:07.652939+0000", 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0075781570000000003, 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0076745379999999998, 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:07.669 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:10:07.670 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:07.670 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:10:07.670 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:10:07.670 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 62648 2023-09-23T18:10:07.670 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.670 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:07.670 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.670 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:07.670 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.652939+0000", 2023-09-23T18:10:07.670 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.652939+0000", 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.652937+0000", 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:10:07.671 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.652964+0000", 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7081999999999999e-05 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.652986+0000", 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1991999999999999e-05 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.672 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.652997+0000", 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0492999999999999e-05 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.653609+0000", 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00061205400000000001 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.673 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:07.674 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.653774+0000", 2023-09-23T18:10:07.674 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016492500000000001 2023-09-23T18:10:07.674 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.674 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.674 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,6,7", 2023-09-23T18:10:07.674 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.653907+0000", 2023-09-23T18:10:07.674 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013357500000000001 2023-09-23T18:10:07.674 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.674 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.674 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:07.675 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.658458+0000", 2023-09-23T18:10:07.675 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045508329999999998 2023-09-23T18:10:07.675 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:07.675 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:07.676 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:07.676 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.658498+0000", 2023-09-23T18:10:07.676 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0024999999999997e-05 2023-09-23T18:10:07.676 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:07.676 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:07.676 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:07.676 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:07.676 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:10:07.676 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 3 2023-09-23T18:10:07.676 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:10:07.724 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:07.756 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:07.800 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:07.804 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:10:07.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:07.808 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:07.908 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:07.941 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:07.990 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:07.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:07.994 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:08.005 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 37 16 62986 62970 106.332 61.6875 0.0189864 0.00939899 2023-09-23T18:10:08.094 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:08.125 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:08.164 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:08.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:08.167 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:08.269 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:08.337 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:08.366 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:10:08.430 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:08.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:08.433 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:08.534 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:08.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:08 smithi099 ceph-mon[167076]: pgmap v919: 119 pgs: 5 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 16 active+undersized+degraded, 28 active+undersized, 63 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3.0 KiB/s rd, 51 MiB/s wr, 825 op/s; 36438/176937 objects degraded (20.594%); 7367/176937 objects misplaced (4.164%); 1.3 MiB/s, 10 objects/s recovering 2023-09-23T18:10:08.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 36438/176937 objects degraded (20.594%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:08.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:08 smithi055 ceph-mon[179668]: pgmap v919: 119 pgs: 5 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 16 active+undersized+degraded, 28 active+undersized, 63 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3.0 KiB/s rd, 51 MiB/s wr, 825 op/s; 36438/176937 objects degraded (20.594%); 7367/176937 objects misplaced (4.164%); 1.3 MiB/s, 10 objects/s recovering 2023-09-23T18:10:08.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 36438/176937 objects degraded (20.594%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:08.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:08 smithi055 ceph-mon[184174]: pgmap v919: 119 pgs: 5 active+clean+remapped, 1 active+recovery_wait+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 4 active+undersized+remapped, 16 active+undersized+degraded, 28 active+undersized, 63 active+clean; 3.6 GiB data, 13 GiB used, 702 GiB / 715 GiB avail; 3.0 KiB/s rd, 51 MiB/s wr, 825 op/s; 36438/176937 objects degraded (20.594%); 7367/176937 objects misplaced (4.164%); 1.3 MiB/s, 10 objects/s recovering 2023-09-23T18:10:08.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 36438/176937 objects degraded (20.594%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:08.640 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:08.837 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:08.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:08.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:08.950 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:08.998 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:09.008 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 38 16 63910 63894 105.046 57.75 0.0128368 0.00951135 2023-09-23T18:10:09.030 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:09.096 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:09.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:09.110 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:09.195 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:10:09.211 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:09.330 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:09.526 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:10:09.541 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:10:09.727 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:09.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:09.743 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:09.844 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:09.900 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:10.011 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 39 16 64868 64852 103.884 59.875 0.00682042 0.0096219 2023-09-23T18:10:10.135 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:10.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:10.151 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:10.157 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:10.253 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:10.319 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:10.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:10:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:10:10] "GET /metrics HTTP/1.1" 200 36662 "" "Prometheus/2.43.0" 2023-09-23T18:10:10.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:10 smithi055 ceph-mon[184174]: pgmap v920: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 2.3 KiB/s rd, 57 MiB/s wr, 917 op/s; 38138/186411 objects degraded (20.459%); 7367/186411 objects misplaced (3.952%); 2.8 MiB/s, 33 objects/s recovering 2023-09-23T18:10:10.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:10 smithi055 ceph-mon[179668]: pgmap v920: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 2.3 KiB/s rd, 57 MiB/s wr, 917 op/s; 38138/186411 objects degraded (20.459%); 7367/186411 objects misplaced (3.952%); 2.8 MiB/s, 33 objects/s recovering 2023-09-23T18:10:10.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:10 smithi099 ceph-mon[167076]: pgmap v920: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 3.8 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 2.3 KiB/s rd, 57 MiB/s wr, 917 op/s; 38138/186411 objects degraded (20.459%); 7367/186411 objects misplaced (3.952%); 2.8 MiB/s, 33 objects/s recovering 2023-09-23T18:10:10.683 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:10.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:10.694 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:10.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:10.840 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:10.890 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:10.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:10.896 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:10.996 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:11.010 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:10:11.011235+0000 min lat: 0.00131331 max lat: 2.26605 avg lat: 0.00970105 2023-09-23T18:10:11.010 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:10:11.010 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 40 16 65973 65957 103.012 69.0625 0.0189601 0.00970105 2023-09-23T18:10:11.062 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:11.294 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:11 smithi055 ceph-mon[179668]: from='client.45980 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:10:11.379 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":920,"stamp":"2023-09-23T18:10:08.876421+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4074743654,"num_objects":62137,"num_object_clones":0,"num_object_copies":186411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38138,"num_objects_misplaced":7367,"num_objects_unfound":0,"num_objects_dirty":62137,"num_whiteouts":0,"num_read":6037,"num_read_kb":12306,"num_write":63059,"num_write_kb":3988186,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":402,"num_bytes_recovered":23438976,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":68132,"ondisk_log_size":68132,"up":306,"acting":311,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":390,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":14489972,"kb_used_data":11295156,"kb_used_omap":50,"kb_used_meta":3194701,"kb_avail":735471244,"statfs":{"total":767960285184,"available":753122553856,"internally_reserved":0,"allocated":11566239744,"data_stored":11554128521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51828,"internal_metadata":3271374220},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[6,0,4,2,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":359867121,"num_objects":5495,"num_object_clones":0,"num_object_copies":16485,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6167,"num_objects_misplaced":7367,"num_objects_unfound":0,"num_objects_dirty":5495,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":5500,"num_write_kb":351781,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":204,"num_bytes_recovered":17317416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.002166"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":613,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.135251+0000","last_change":"2023-09-23T18:09:49.260251+0000","last_active":"2023-09-23T18:10:04.135251+0000","last_peered":"2023-09-23T18:10:04.135251+0000","last_clean":"2023-09-23T18:05:06.478917+0000","last_became_active":"2023-09-23T18:09:49.260251+0000","last_became_peered":"2023-09-23T18:09:49.260251+0000","last_unstale":"2023-09-23T18:10:04.135251+0000","last_undegraded":"2023-09-23T18:10:04.135251+0000","last_fullsized":"2023-09-23T18:09:49.251327+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.712627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.712627+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":737225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":185,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131169+0000","last_change":"2023-09-23T18:06:05.377123+0000","last_active":"2023-09-23T18:10:04.131169+0000","last_peered":"2023-09-23T18:10:04.131169+0000","last_clean":"2023-09-23T18:10:04.131169+0000","last_became_active":"2023-09-23T18:06:05.376735+0000","last_became_peered":"2023-09-23T18:06:05.376735+0000","last_unstale":"2023-09-23T18:10:04.131169+0000","last_undegraded":"2023-09-23T18:10:04.131169+0000","last_fullsized":"2023-09-23T18:10:04.131169+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:11.555818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":252,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.151125+0000","last_change":"2023-09-23T18:06:51.071290+0000","last_active":"2023-09-23T18:10:04.151125+0000","last_peered":"2023-09-23T18:10:04.151125+0000","last_clean":"2023-09-23T18:10:04.151125+0000","last_became_active":"2023-09-23T18:06:51.070948+0000","last_became_peered":"2023-09-23T18:06:51.070948+0000","last_unstale":"2023-09-23T18:10:04.151125+0000","last_undegraded":"2023-09-23T18:10:04.151125+0000","last_fullsized":"2023-09-23T18:10:04.151125+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:52.647972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":606,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.132165+0000","last_change":"2023-09-23T18:09:49.244257+0000","last_active":"2023-09-23T18:10:04.132165+0000","last_peered":"2023-09-23T18:10:04.132165+0000","last_clean":"2023-09-23T18:02:45.965198+0000","last_became_active":"2023-09-23T18:09:49.244257+0000","last_became_peered":"2023-09-23T18:09:49.244257+0000","last_unstale":"2023-09-23T18:10:04.132165+0000","last_undegraded":"2023-09-23T18:10:04.132165+0000","last_fullsized":"2023-09-23T18:09:49.242161+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:34.927458+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":91,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.160568+0000","last_change":"2023-09-23T18:06:05.370046+0000","last_active":"2023-09-23T18:10:04.160568+0000","last_peered":"2023-09-23T18:10:04.160568+0000","last_clean":"2023-09-23T18:10:04.160568+0000","last_became_active":"2023-09-23T18:06:05.369714+0000","last_became_peered":"2023-09-23T18:06:05.369714+0000","last_unstale":"2023-09-23T18:10:04.160568+0000","last_undegraded":"2023-09-23T18:10:04.160568+0000","last_fullsized":"2023-09-23T18:10:04.160568+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:47.453671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":391,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.135120+0000","last_change":"2023-09-23T18:09:49.264642+0000","last_active":"2023-09-23T18:10:04.135120+0000","last_peered":"2023-09-23T18:10:04.135120+0000","last_clean":"2023-09-23T18:09:40.155891+0000","last_became_active":"2023-09-23T18:09:49.264642+0000","last_became_peered":"2023-09-23T18:09:49.264642+0000","last_unstale":"2023-09-23T18:10:04.135120+0000","last_undegraded":"2023-09-23T18:10:04.135120+0000","last_fullsized":"2023-09-23T18:09:49.251516+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:29.396482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":712,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131481+0000","last_change":"2023-09-23T18:06:08.439343+0000","last_active":"2023-09-23T18:10:04.131481+0000","last_peered":"2023-09-23T18:10:04.131481+0000","last_clean":"2023-09-23T18:10:04.131481+0000","last_became_active":"2023-09-23T18:06:08.437702+0000","last_became_peered":"2023-09-23T18:06:08.437702+0000","last_unstale":"2023-09-23T18:10:04.131481+0000","last_undegraded":"2023-09-23T18:10:04.131481+0000","last_fullsized":"2023-09-23T18:10:04.131481+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:02.398105+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":255,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.151390+0000","last_change":"2023-09-23T18:06:51.087219+0000","last_active":"2023-09-23T18:10:04.151390+0000","last_peered":"2023-09-23T18:10:04.151390+0000","last_clean":"2023-09-23T18:10:04.151390+0000","last_became_active":"2023-09-23T18:06:51.086885+0000","last_became_peered":"2023-09-23T18:06:51.086885+0000","last_unstale":"2023-09-23T18:10:04.151390+0000","last_undegraded":"2023-09-23T18:10:04.151390+0000","last_fullsized":"2023-09-23T18:10:04.151390+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:16.254186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":332,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.122763+0000","last_change":"2023-09-23T18:09:49.249873+0000","last_active":"2023-09-23T18:10:04.122763+0000","last_peered":"2023-09-23T18:10:04.122763+0000","last_clean":"2023-09-23T18:09:40.149572+0000","last_became_active":"2023-09-23T18:09:49.249873+0000","last_became_peered":"2023-09-23T18:09:49.249873+0000","last_unstale":"2023-09-23T18:10:04.122763+0000","last_undegraded":"2023-09-23T18:10:04.122763+0000","last_fullsized":"2023-09-23T18:09:49.240762+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:52.086856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":762,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.135624+0000","last_change":"2023-09-23T18:09:49.267065+0000","last_active":"2023-09-23T18:10:04.135624+0000","last_peered":"2023-09-23T18:10:04.135624+0000","last_clean":"2023-09-23T18:09:40.155011+0000","last_became_active":"2023-09-23T18:09:49.267065+0000","last_became_peered":"2023-09-23T18:09:49.267065+0000","last_unstale":"2023-09-23T18:10:04.135624+0000","last_undegraded":"2023-09-23T18:10:04.135624+0000","last_fullsized":"2023-09-23T18:09:49.257376+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:52.249937+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":15,"reported_epoch":374,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:04.131862+0000","last_change":"2023-09-23T18:10:02.191390+0000","last_active":"2023-09-23T18:10:04.131862+0000","last_peered":"2023-09-23T18:10:04.131862+0000","last_clean":"2023-09-23T18:10:04.131862+0000","last_became_active":"2023-09-23T18:10:02.190942+0000","last_became_peered":"2023-09-23T18:10:02.190942+0000","last_unstale":"2023-09-23T18:10:04.131862+0000","last_undegraded":"2023-09-23T18:10:04.131862+0000","last_fullsized":"2023-09-23T18:10:04.131862+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.670277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":243,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.127805+0000","last_change":"2023-09-23T18:06:05.364021+0000","last_active":"2023-09-23T18:10:04.127805+0000","last_peered":"2023-09-23T18:10:04.127805+0000","last_clean":"2023-09-23T18:10:04.127805+0000","last_became_active":"2023-09-23T18:06:05.363740+0000","last_became_peered":"2023-09-23T18:06:05.363740+0000","last_unstale":"2023-09-23T18:10:04.127805+0000","last_undegraded":"2023-09-23T18:10:04.127805+0000","last_fullsized":"2023-09-23T18:10:04.127805+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:55.662682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":663,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.134796+0000","last_change":"2023-09-23T18:09:23.589835+0000","last_active":"2023-09-23T18:10:04.134796+0000","last_peered":"2023-09-23T18:10:04.134796+0000","last_clean":"2023-09-23T18:10:04.134796+0000","last_became_active":"2023-09-23T18:08:10.517840+0000","last_became_peered":"2023-09-23T18:08:10.517840+0000","last_unstale":"2023-09-23T18:10:04.134796+0000","last_undegraded":"2023-09-23T18:10:04.134796+0000","last_fullsized":"2023-09-23T18:10:04.134796+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:26.034507+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":286,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.135337+0000","last_change":"2023-09-23T18:09:49.264696+0000","last_active":"2023-09-23T18:10:04.135337+0000","last_peered":"2023-09-23T18:10:04.135337+0000","last_clean":"2023-09-23T18:05:59.285835+0000","last_became_active":"2023-09-23T18:09:49.264696+0000","last_became_peered":"2023-09-23T18:09:49.264696+0000","last_unstale":"2023-09-23T18:10:04.135337+0000","last_undegraded":"2023-09-23T18:10:04.135337+0000","last_fullsized":"2023-09-23T18:09:49.251195+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:32.680036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":160,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.121919+0000","last_change":"2023-09-23T18:07:49.223713+0000","last_active":"2023-09-23T18:10:04.121919+0000","last_peered":"2023-09-23T18:10:04.121919+0000","last_clean":"2023-09-23T18:10:04.121919+0000","last_became_active":"2023-09-23T18:07:49.223409+0000","last_became_peered":"2023-09-23T18:07:49.223409+0000","last_unstale":"2023-09-23T18:10:04.121919+0000","last_undegraded":"2023-09-23T18:10:04.121919+0000","last_fullsized":"2023-09-23T18:10:04.121919+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:12.387318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":430,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.122828+0000","last_change":"2023-09-23T18:09:49.254157+0000","last_active":"2023-09-23T18:10:04.122828+0000","last_peered":"2023-09-23T18:10:04.122828+0000","last_clean":"2023-09-23T17:49:06.430604+0000","last_became_active":"2023-09-23T18:09:49.254157+0000","last_became_peered":"2023-09-23T18:09:49.254157+0000","last_unstale":"2023-09-23T18:10:04.122828+0000","last_undegraded":"2023-09-23T18:10:04.122828+0000","last_fullsized":"2023-09-23T18:09:49.240876+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.652623+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.652623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":140,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.135506+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:10:04.135506+0000","last_peered":"2023-09-23T18:10:04.135506+0000","last_clean":"2023-09-23T18:10:04.135506+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:10:04.135506+0000","last_undegraded":"2023-09-23T18:10:04.135506+0000","last_fullsized":"2023-09-23T18:10:04.135506+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":101,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.160955+0000","last_change":"2023-09-23T18:06:05.363236+0000","last_active":"2023-09-23T18:10:04.160955+0000","last_peered":"2023-09-23T18:10:04.160955+0000","last_clean":"2023-09-23T18:10:04.160955+0000","last_became_active":"2023-09-23T18:06:05.362947+0000","last_became_peered":"2023-09-23T18:06:05.362947+0000","last_unstale":"2023-09-23T18:10:04.160955+0000","last_undegraded":"2023-09-23T18:10:04.160955+0000","last_fullsized":"2023-09-23T18:10:04.160955+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:32.816159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":775,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.160468+0000","last_change":"2023-09-23T18:10:04.160097+0000","last_active":"2023-09-23T18:10:04.160468+0000","last_peered":"2023-09-23T18:10:04.160468+0000","last_clean":"2023-09-23T18:10:04.160468+0000","last_became_active":"2023-09-23T18:10:04.159754+0000","last_became_peered":"2023-09-23T18:10:04.159754+0000","last_unstale":"2023-09-23T18:10:04.160468+0000","last_undegraded":"2023-09-23T18:10:04.160468+0000","last_fullsized":"2023-09-23T18:10:04.160468+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:09:59.467151+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":167,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131301+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:10:04.131301+0000","last_peered":"2023-09-23T18:10:04.131301+0000","last_clean":"2023-09-23T18:10:04.131301+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:10:04.131301+0000","last_undegraded":"2023-09-23T18:10:04.131301+0000","last_fullsized":"2023-09-23T18:10:04.131301+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":185,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.132628+0000","last_change":"2023-09-23T18:06:05.375073+0000","last_active":"2023-09-23T18:10:04.132628+0000","last_peered":"2023-09-23T18:10:04.132628+0000","last_clean":"2023-09-23T18:10:04.132628+0000","last_became_active":"2023-09-23T18:06:05.374752+0000","last_became_peered":"2023-09-23T18:06:05.374752+0000","last_unstale":"2023-09-23T18:10:04.132628+0000","last_undegraded":"2023-09-23T18:10:04.132628+0000","last_fullsized":"2023-09-23T18:10:04.132628+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:09:06.839328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":610,"reported_epoch":374,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:04.157141+0000","last_change":"2023-09-23T18:09:49.245471+0000","last_active":"2023-09-23T18:10:04.157141+0000","last_peered":"2023-09-23T18:10:04.157141+0000","last_clean":"2023-09-23T18:02:45.964322+0000","last_became_active":"2023-09-23T18:09:49.245471+0000","last_became_peered":"2023-09-23T18:09:49.245471+0000","last_unstale":"2023-09-23T18:10:04.157141+0000","last_undegraded":"2023-09-23T18:09:49.238416+0000","last_fullsized":"2023-09-23T18:09:49.238297+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.869576+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":303,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.135040+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:10:04.135040+0000","last_peered":"2023-09-23T18:10:04.135040+0000","last_clean":"2023-09-23T18:10:04.135040+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:10:04.135040+0000","last_undegraded":"2023-09-23T18:10:04.135040+0000","last_fullsized":"2023-09-23T18:10:04.135040+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":190,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.150163+0000","last_change":"2023-09-23T18:07:49.229808+0000","last_active":"2023-09-23T18:10:04.150163+0000","last_peered":"2023-09-23T18:10:04.150163+0000","last_clean":"2023-09-23T18:10:04.150163+0000","last_became_active":"2023-09-23T18:07:49.229526+0000","last_became_peered":"2023-09-23T18:07:49.229526+0000","last_unstale":"2023-09-23T18:10:04.150163+0000","last_undegraded":"2023-09-23T18:10:04.150163+0000","last_fullsized":"2023-09-23T18:10:04.150163+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:01.224432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":622,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.135224+0000","last_change":"2023-09-23T18:09:49.260631+0000","last_active":"2023-09-23T18:10:04.135224+0000","last_peered":"2023-09-23T18:10:04.135224+0000","last_clean":"2023-09-23T18:05:06.478813+0000","last_became_active":"2023-09-23T18:09:49.260631+0000","last_became_peered":"2023-09-23T18:09:49.260631+0000","last_unstale":"2023-09-23T18:10:04.135224+0000","last_undegraded":"2023-09-23T18:10:04.135224+0000","last_fullsized":"2023-09-23T18:09:49.251470+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:23.252461+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":382,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.134882+0000","last_change":"2023-09-23T18:06:51.065050+0000","last_active":"2023-09-23T18:10:04.134882+0000","last_peered":"2023-09-23T18:10:04.134882+0000","last_clean":"2023-09-23T18:10:04.134882+0000","last_became_active":"2023-09-23T18:06:51.064774+0000","last_became_peered":"2023-09-23T18:06:51.064774+0000","last_unstale":"2023-09-23T18:10:04.134882+0000","last_undegraded":"2023-09-23T18:10:04.134882+0000","last_fullsized":"2023-09-23T18:10:04.134882+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:31.508499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":224,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.136061+0000","last_change":"2023-09-23T18:09:49.260925+0000","last_active":"2023-09-23T18:10:04.136061+0000","last_peered":"2023-09-23T18:10:04.136061+0000","last_clean":"2023-09-23T18:02:16.054597+0000","last_became_active":"2023-09-23T18:09:49.260925+0000","last_became_peered":"2023-09-23T18:09:49.260925+0000","last_unstale":"2023-09-23T18:10:04.136061+0000","last_undegraded":"2023-09-23T18:10:04.136061+0000","last_fullsized":"2023-09-23T18:09:49.256792+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:07.524952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":738,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.177601+0000","last_change":"2023-09-23T18:10:04.171274+0000","last_active":"2023-09-23T18:10:04.177601+0000","last_peered":"2023-09-23T18:10:04.177601+0000","last_clean":"2023-09-23T18:10:04.177601+0000","last_became_active":"2023-09-23T18:10:04.166434+0000","last_became_peered":"2023-09-23T18:10:04.166434+0000","last_unstale":"2023-09-23T18:10:04.177601+0000","last_undegraded":"2023-09-23T18:10:04.177601+0000","last_fullsized":"2023-09-23T18:10:04.177601+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:30.091293+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":165,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131066+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:10:04.131066+0000","last_peered":"2023-09-23T18:10:04.131066+0000","last_clean":"2023-09-23T18:10:04.131066+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:10:04.131066+0000","last_undegraded":"2023-09-23T18:10:04.131066+0000","last_fullsized":"2023-09-23T18:10:04.131066+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":314,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.126973+0000","last_change":"2023-09-23T18:06:51.125883+0000","last_active":"2023-09-23T18:10:04.126973+0000","last_peered":"2023-09-23T18:10:04.126973+0000","last_clean":"2023-09-23T18:10:04.126973+0000","last_became_active":"2023-09-23T18:06:51.119298+0000","last_became_peered":"2023-09-23T18:06:51.119298+0000","last_unstale":"2023-09-23T18:10:04.126973+0000","last_undegraded":"2023-09-23T18:10:04.126973+0000","last_fullsized":"2023-09-23T18:10:04.126973+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:12.571367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":623,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.160712+0000","last_change":"2023-09-23T18:09:49.248584+0000","last_active":"2023-09-23T18:10:04.160712+0000","last_peered":"2023-09-23T18:10:04.160712+0000","last_clean":"2023-09-23T18:05:59.276387+0000","last_became_active":"2023-09-23T18:09:49.248584+0000","last_became_peered":"2023-09-23T18:09:49.248584+0000","last_unstale":"2023-09-23T18:10:04.160712+0000","last_undegraded":"2023-09-23T18:10:04.160712+0000","last_fullsized":"2023-09-23T18:09:49.239167+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:29.998743+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":358,"reported_epoch":374,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:04.121847+0000","last_change":"2023-09-23T18:09:49.247743+0000","last_active":"2023-09-23T18:10:04.121847+0000","last_peered":"2023-09-23T18:10:04.121847+0000","last_clean":"2023-09-23T18:09:40.148528+0000","last_became_active":"2023-09-23T18:09:49.247743+0000","last_became_peered":"2023-09-23T18:09:49.247743+0000","last_unstale":"2023-09-23T18:10:04.121847+0000","last_undegraded":"2023-09-23T18:09:49.239512+0000","last_fullsized":"2023-09-23T18:09:49.239401+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:54:53.674393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":229,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.152944+0000","last_change":"2023-09-23T18:06:45.919283+0000","last_active":"2023-09-23T18:10:04.152944+0000","last_peered":"2023-09-23T18:10:04.152944+0000","last_clean":"2023-09-23T18:10:04.152944+0000","last_became_active":"2023-09-23T18:06:45.918219+0000","last_became_peered":"2023-09-23T18:06:45.918219+0000","last_unstale":"2023-09-23T18:10:04.152944+0000","last_undegraded":"2023-09-23T18:10:04.152944+0000","last_fullsized":"2023-09-23T18:10:04.152944+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:21.242033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.4","version":"375'3792","reported_seq":3819,"reported_epoch":375,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:05.797935+0000","last_change":"2023-09-23T18:09:49.246955+0000","last_active":"2023-09-23T18:10:05.797935+0000","last_peered":"2023-09-23T18:10:05.797935+0000","last_clean":"2023-09-23T18:09:48.184212+0000","last_became_active":"2023-09-23T18:09:49.246955+0000","last_became_peered":"2023-09-23T18:09:49.246955+0000","last_unstale":"2023-09-23T18:10:05.797935+0000","last_undegraded":"2023-09-23T18:09:49.243456+0000","last_fullsized":"2023-09-23T18:09:49.243334+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3792,"log_dups_size":0,"ondisk_log_size":3792,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:44.207061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248512512,"num_objects":3792,"num_object_clones":0,"num_object_copies":11376,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3792,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3792,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3792,"num_write_kb":242688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":3792}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":755,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.135242+0000","last_change":"2023-09-23T18:06:08.440949+0000","last_active":"2023-09-23T18:10:04.135242+0000","last_peered":"2023-09-23T18:10:04.135242+0000","last_clean":"2023-09-23T18:10:04.135242+0000","last_became_active":"2023-09-23T18:06:08.440543+0000","last_became_peered":"2023-09-23T18:06:08.440543+0000","last_unstale":"2023-09-23T18:10:04.135242+0000","last_undegraded":"2023-09-23T18:10:04.135242+0000","last_fullsized":"2023-09-23T18:10:04.135242+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:39:30.193370+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":317,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.135591+0000","last_change":"2023-09-23T18:06:45.947590+0000","last_active":"2023-09-23T18:10:04.135591+0000","last_peered":"2023-09-23T18:10:04.135591+0000","last_clean":"2023-09-23T18:10:04.135591+0000","last_became_active":"2023-09-23T18:06:45.920061+0000","last_became_peered":"2023-09-23T18:06:45.920061+0000","last_unstale":"2023-09-23T18:10:04.135591+0000","last_undegraded":"2023-09-23T18:10:04.135591+0000","last_fullsized":"2023-09-23T18:10:04.135591+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:31.645928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":246,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.134578+0000","last_change":"2023-09-23T18:09:49.260845+0000","last_active":"2023-09-23T18:10:04.134578+0000","last_peered":"2023-09-23T18:10:04.134578+0000","last_clean":"2023-09-23T18:02:45.953891+0000","last_became_active":"2023-09-23T18:09:49.260845+0000","last_became_peered":"2023-09-23T18:09:49.260845+0000","last_unstale":"2023-09-23T18:10:04.134578+0000","last_undegraded":"2023-09-23T18:10:04.134578+0000","last_fullsized":"2023-09-23T18:09:49.252007+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:35.249760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"375'3736","reported_seq":3758,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T18:10:05.610906+0000","last_change":"2023-09-23T18:09:26.035691+0000","last_active":"2023-09-23T18:10:05.610906+0000","last_peered":"2023-09-23T18:10:05.610906+0000","last_clean":"2023-09-23T18:10:05.610906+0000","last_became_active":"2023-09-23T18:09:26.035411+0000","last_became_peered":"2023-09-23T18:09:26.035411+0000","last_unstale":"2023-09-23T18:10:05.610906+0000","last_undegraded":"2023-09-23T18:10:05.610906+0000","last_fullsized":"2023-09-23T18:10:05.610906+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3736,"log_dups_size":0,"ondisk_log_size":3736,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:51.634243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":244842496,"num_objects":3736,"num_object_clones":0,"num_object_copies":11208,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3736,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3736,"num_write_kb":239104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":641,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.137382+0000","last_change":"2023-09-23T18:10:04.137099+0000","last_active":"2023-09-23T18:10:04.137382+0000","last_peered":"2023-09-23T18:10:04.137382+0000","last_clean":"2023-09-23T18:10:04.137382+0000","last_became_active":"2023-09-23T18:10:04.136723+0000","last_became_peered":"2023-09-23T18:10:04.136723+0000","last_unstale":"2023-09-23T18:10:04.137382+0000","last_undegraded":"2023-09-23T18:10:04.137382+0000","last_fullsized":"2023-09-23T18:10:04.137382+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:41.510146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:41.510146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1392168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":332,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.135397+0000","last_change":"2023-09-23T18:06:05.379591+0000","last_active":"2023-09-23T18:10:04.135397+0000","last_peered":"2023-09-23T18:10:04.135397+0000","last_clean":"2023-09-23T18:10:04.135397+0000","last_became_active":"2023-09-23T18:06:05.370176+0000","last_became_peered":"2023-09-23T18:06:05.370176+0000","last_unstale":"2023-09-23T18:10:04.135397+0000","last_undegraded":"2023-09-23T18:10:04.135397+0000","last_fullsized":"2023-09-23T18:10:04.135397+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:08.768138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":405,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131259+0000","last_change":"2023-09-23T18:06:45.911012+0000","last_active":"2023-09-23T18:10:04.131259+0000","last_peered":"2023-09-23T18:10:04.131259+0000","last_clean":"2023-09-23T18:10:04.131259+0000","last_became_active":"2023-09-23T18:06:45.910722+0000","last_became_peered":"2023-09-23T18:06:45.910722+0000","last_unstale":"2023-09-23T18:10:04.131259+0000","last_undegraded":"2023-09-23T18:10:04.131259+0000","last_fullsized":"2023-09-23T18:10:04.131259+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:19.150719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"375'4043","reported_seq":4071,"reported_epoch":375,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T18:10:08.572871+0000","last_change":"2023-09-23T18:10:02.341560+0000","last_active":"2023-09-23T18:10:08.572871+0000","last_peered":"2023-09-23T18:10:08.572871+0000","last_clean":"2023-09-23T18:10:00.095783+0000","last_became_active":"2023-09-23T18:10:02.330146+0000","last_became_peered":"2023-09-23T18:10:02.330146+0000","last_unstale":"2023-09-23T18:10:08.572871+0000","last_undegraded":"2023-09-23T18:10:08.572871+0000","last_fullsized":"2023-09-23T18:10:08.572871+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4043,"log_dups_size":0,"ondisk_log_size":4043,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:09.375692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":264962048,"num_objects":4043,"num_object_clones":0,"num_object_copies":12129,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3625,"num_objects_unfound":0,"num_objects_dirty":4043,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4043,"num_write_kb":258752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[2,6,7],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":418}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":553,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.122935+0000","last_change":"2023-09-23T18:09:49.245759+0000","last_active":"2023-09-23T18:10:04.122935+0000","last_peered":"2023-09-23T18:10:04.122935+0000","last_clean":"2023-09-23T18:09:40.149639+0000","last_became_active":"2023-09-23T18:09:49.245759+0000","last_became_peered":"2023-09-23T18:09:49.245759+0000","last_unstale":"2023-09-23T18:10:04.122935+0000","last_undegraded":"2023-09-23T18:10:04.122935+0000","last_fullsized":"2023-09-23T18:09:49.241138+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:29.659220+0000","scrub_duration":0.0018893740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":248,"reported_epoch":374,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:04.135877+0000","last_change":"2023-09-23T18:09:49.260821+0000","last_active":"2023-09-23T18:10:04.135877+0000","last_peered":"2023-09-23T18:10:04.135877+0000","last_clean":"2023-09-23T18:09:40.155190+0000","last_became_active":"2023-09-23T18:09:49.260821+0000","last_became_peered":"2023-09-23T18:09:49.260821+0000","last_unstale":"2023-09-23T18:10:04.135877+0000","last_undegraded":"2023-09-23T18:09:49.257202+0000","last_fullsized":"2023-09-23T18:09:49.257118+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:05.023383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":350,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.122128+0000","last_change":"2023-09-23T18:06:05.371447+0000","last_active":"2023-09-23T18:10:04.122128+0000","last_peered":"2023-09-23T18:10:04.122128+0000","last_clean":"2023-09-23T18:10:04.122128+0000","last_became_active":"2023-09-23T18:06:05.371065+0000","last_became_peered":"2023-09-23T18:06:05.371065+0000","last_unstale":"2023-09-23T18:10:04.122128+0000","last_undegraded":"2023-09-23T18:10:04.122128+0000","last_fullsized":"2023-09-23T18:10:04.122128+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.025791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"375'3919","reported_seq":3942,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T18:10:07.684771+0000","last_change":"2023-09-23T18:09:26.507113+0000","last_active":"2023-09-23T18:10:07.684771+0000","last_peered":"2023-09-23T18:10:07.684771+0000","last_clean":"2023-09-23T18:10:07.684771+0000","last_became_active":"2023-09-23T18:09:26.506876+0000","last_became_peered":"2023-09-23T18:09:26.506876+0000","last_unstale":"2023-09-23T18:10:07.684771+0000","last_undegraded":"2023-09-23T18:10:07.684771+0000","last_fullsized":"2023-09-23T18:10:07.684771+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3919,"log_dups_size":0,"ondisk_log_size":3919,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:54.009952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256835584,"num_objects":3919,"num_object_clones":0,"num_object_copies":11757,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3919,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3919,"num_write_kb":250816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":699,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131879+0000","last_change":"2023-09-23T18:05:56.362746+0000","last_active":"2023-09-23T18:10:04.131879+0000","last_peered":"2023-09-23T18:10:04.131879+0000","last_clean":"2023-09-23T18:10:04.131879+0000","last_became_active":"2023-09-23T18:05:15.102072+0000","last_became_peered":"2023-09-23T18:05:15.102072+0000","last_unstale":"2023-09-23T18:10:04.131879+0000","last_undegraded":"2023-09-23T18:10:04.131879+0000","last_fullsized":"2023-09-23T18:10:04.131879+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.362652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.362652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:33.381515+0000","scrub_duration":0.0052581759999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":298,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.122388+0000","last_change":"2023-09-23T18:09:49.253688+0000","last_active":"2023-09-23T18:10:04.122388+0000","last_peered":"2023-09-23T18:10:04.122388+0000","last_clean":"2023-09-23T18:06:48.956525+0000","last_became_active":"2023-09-23T18:09:49.253688+0000","last_became_peered":"2023-09-23T18:09:49.253688+0000","last_unstale":"2023-09-23T18:10:04.122388+0000","last_undegraded":"2023-09-23T18:10:04.122388+0000","last_fullsized":"2023-09-23T18:09:49.240299+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:58.227257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":14,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.133772+0000","last_change":"2023-09-23T18:10:01.176492+0000","last_active":"2023-09-23T18:10:04.133772+0000","last_peered":"2023-09-23T18:10:04.133772+0000","last_clean":"2023-09-23T18:10:04.133772+0000","last_became_active":"2023-09-23T18:10:01.176155+0000","last_became_peered":"2023-09-23T18:10:01.176155+0000","last_unstale":"2023-09-23T18:10:04.133772+0000","last_undegraded":"2023-09-23T18:10:04.133772+0000","last_fullsized":"2023-09-23T18:10:04.133772+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:53.737654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"375'3923","reported_seq":3948,"reported_epoch":375,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:07.636698+0000","last_change":"2023-09-23T18:09:49.245179+0000","last_active":"2023-09-23T18:10:07.636698+0000","last_peered":"2023-09-23T18:10:07.636698+0000","last_clean":"2023-09-23T18:09:48.233578+0000","last_became_active":"2023-09-23T18:09:49.245179+0000","last_became_peered":"2023-09-23T18:09:49.245179+0000","last_unstale":"2023-09-23T18:10:07.636698+0000","last_undegraded":"2023-09-23T18:09:49.238600+0000","last_fullsized":"2023-09-23T18:09:49.238472+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3923,"log_dups_size":0,"ondisk_log_size":3923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:29.489220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257097728,"num_objects":3923,"num_object_clones":0,"num_object_copies":11769,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3923,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3923,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3923,"num_write_kb":251072,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":3923}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":754,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.134974+0000","last_change":"2023-09-23T18:09:49.250289+0000","last_active":"2023-09-23T18:10:04.134974+0000","last_peered":"2023-09-23T18:10:04.134974+0000","last_clean":"2023-09-23T18:09:40.154309+0000","last_became_active":"2023-09-23T18:09:49.250289+0000","last_became_peered":"2023-09-23T18:09:49.250289+0000","last_unstale":"2023-09-23T18:10:04.134974+0000","last_undegraded":"2023-09-23T18:10:04.134974+0000","last_fullsized":"2023-09-23T18:09:49.248499+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:54.767780+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":161,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.132733+0000","last_change":"2023-09-23T18:07:49.227201+0000","last_active":"2023-09-23T18:10:04.132733+0000","last_peered":"2023-09-23T18:10:04.132733+0000","last_clean":"2023-09-23T18:10:04.132733+0000","last_became_active":"2023-09-23T18:07:49.226878+0000","last_became_peered":"2023-09-23T18:07:49.226878+0000","last_unstale":"2023-09-23T18:10:04.132733+0000","last_undegraded":"2023-09-23T18:10:04.132733+0000","last_fullsized":"2023-09-23T18:10:04.132733+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:08.994668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":375,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131267+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:10:04.131267+0000","last_peered":"2023-09-23T18:10:04.131267+0000","last_clean":"2023-09-23T18:10:04.131267+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:10:04.131267+0000","last_undegraded":"2023-09-23T18:10:04.131267+0000","last_fullsized":"2023-09-23T18:10:04.131267+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"375'3910","reported_seq":4330,"reported_epoch":375,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:10:07.697005+0000","last_change":"2023-09-23T18:10:02.349030+0000","last_active":"2023-09-23T18:10:07.697005+0000","last_peered":"2023-09-23T18:10:07.697005+0000","last_clean":"2023-09-23T18:09:48.228392+0000","last_became_active":"2023-09-23T18:10:02.311652+0000","last_became_peered":"2023-09-23T18:10:02.311652+0000","last_unstale":"2023-09-23T18:10:07.697005+0000","last_undegraded":"2023-09-23T18:10:02.208436+0000","last_fullsized":"2023-09-23T18:10:02.183430+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3910,"log_dups_size":0,"ondisk_log_size":3910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:41.860294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256245760,"num_objects":3910,"num_object_clones":0,"num_object_copies":11730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3388,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3910,"num_write_kb":250240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":192,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":522}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":645,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.121733+0000","last_change":"2023-09-23T18:09:49.247368+0000","last_active":"2023-09-23T18:10:04.121733+0000","last_peered":"2023-09-23T18:10:04.121733+0000","last_clean":"2023-09-23T18:06:48.958041+0000","last_became_active":"2023-09-23T18:09:49.247368+0000","last_became_peered":"2023-09-23T18:09:49.247368+0000","last_unstale":"2023-09-23T18:10:04.121733+0000","last_undegraded":"2023-09-23T18:10:04.121733+0000","last_fullsized":"2023-09-23T18:09:49.239209+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:34.776285+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":344,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.122586+0000","last_change":"2023-09-23T18:06:05.371350+0000","last_active":"2023-09-23T18:10:04.122586+0000","last_peered":"2023-09-23T18:10:04.122586+0000","last_clean":"2023-09-23T18:10:04.122586+0000","last_became_active":"2023-09-23T18:06:05.370877+0000","last_became_peered":"2023-09-23T18:06:05.370877+0000","last_unstale":"2023-09-23T18:10:04.122586+0000","last_undegraded":"2023-09-23T18:10:04.122586+0000","last_fullsized":"2023-09-23T18:10:04.122586+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:58.534355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":185,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131941+0000","last_change":"2023-09-23T18:06:05.373385+0000","last_active":"2023-09-23T18:10:04.131941+0000","last_peered":"2023-09-23T18:10:04.131941+0000","last_clean":"2023-09-23T18:10:04.131941+0000","last_became_active":"2023-09-23T18:06:05.372982+0000","last_became_peered":"2023-09-23T18:06:05.372982+0000","last_unstale":"2023-09-23T18:10:04.131941+0000","last_undegraded":"2023-09-23T18:10:04.131941+0000","last_fullsized":"2023-09-23T18:10:04.131941+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:48.622115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"375'3742","reported_seq":3781,"reported_epoch":375,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:10:05.772704+0000","last_change":"2023-09-23T18:10:02.341385+0000","last_active":"2023-09-23T18:10:05.772704+0000","last_peered":"2023-09-23T18:10:05.772704+0000","last_clean":"2023-09-23T18:09:48.231362+0000","last_became_active":"2023-09-23T18:10:02.333199+0000","last_became_peered":"2023-09-23T18:10:02.333199+0000","last_unstale":"2023-09-23T18:10:05.772704+0000","last_undegraded":"2023-09-23T18:10:02.199212+0000","last_fullsized":"2023-09-23T18:10:02.178517+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3742,"log_dups_size":0,"ondisk_log_size":3742,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:35.880677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":245235712,"num_objects":3742,"num_object_clones":0,"num_object_copies":11226,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3527,"num_objects_misplaced":3742,"num_objects_unfound":0,"num_objects_dirty":3742,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3742,"num_write_kb":239488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":215}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":758,"reported_epoch":374,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:04.177625+0000","last_change":"2023-09-23T18:10:04.176981+0000","last_active":"2023-09-23T18:10:04.177625+0000","last_peered":"2023-09-23T18:10:04.177625+0000","last_clean":"2023-09-23T18:10:04.177625+0000","last_became_active":"2023-09-23T18:10:04.176481+0000","last_became_peered":"2023-09-23T18:10:04.176481+0000","last_unstale":"2023-09-23T18:10:04.177625+0000","last_undegraded":"2023-09-23T18:10:04.177625+0000","last_fullsized":"2023-09-23T18:10:04.177625+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:12.991004+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,2],"avail_no_missing":["1","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":165,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131179+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:10:04.131179+0000","last_peered":"2023-09-23T18:10:04.131179+0000","last_clean":"2023-09-23T18:10:04.131179+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:10:04.131179+0000","last_undegraded":"2023-09-23T18:10:04.131179+0000","last_fullsized":"2023-09-23T18:10:04.131179+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":395,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131966+0000","last_change":"2023-09-23T18:06:05.373305+0000","last_active":"2023-09-23T18:10:04.131966+0000","last_peered":"2023-09-23T18:10:04.131966+0000","last_clean":"2023-09-23T18:10:04.131966+0000","last_became_active":"2023-09-23T18:06:05.372720+0000","last_became_peered":"2023-09-23T18:06:05.372720+0000","last_unstale":"2023-09-23T18:10:04.131966+0000","last_undegraded":"2023-09-23T18:10:04.131966+0000","last_fullsized":"2023-09-23T18:10:04.131966+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:44.934493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"375'3966","reported_seq":3999,"reported_epoch":375,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:07.696452+0000","last_change":"2023-09-23T18:09:49.272932+0000","last_active":"2023-09-23T18:10:07.696452+0000","last_peered":"2023-09-23T18:10:07.696452+0000","last_clean":"2023-09-23T18:09:48.219569+0000","last_became_active":"2023-09-23T18:09:49.258387+0000","last_became_peered":"2023-09-23T18:09:49.258387+0000","last_unstale":"2023-09-23T18:10:07.696452+0000","last_undegraded":"2023-09-23T18:09:49.243174+0000","last_fullsized":"2023-09-23T18:09:49.240692+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3966,"log_dups_size":0,"ondisk_log_size":3966,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:55.945872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259915776,"num_objects":3966,"num_object_clones":0,"num_object_copies":11898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3966,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3966,"num_write_kb":253824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":3966}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":430,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.121997+0000","last_change":"2023-09-23T18:09:49.255810+0000","last_active":"2023-09-23T18:10:04.121997+0000","last_peered":"2023-09-23T18:10:04.121997+0000","last_clean":"2023-09-23T17:49:06.430604+0000","last_became_active":"2023-09-23T18:09:49.255810+0000","last_became_peered":"2023-09-23T18:09:49.255810+0000","last_unstale":"2023-09-23T18:10:04.121997+0000","last_undegraded":"2023-09-23T18:10:04.121997+0000","last_fullsized":"2023-09-23T18:09:49.239753+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":221,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.151305+0000","last_change":"2023-09-23T18:06:45.904403+0000","last_active":"2023-09-23T18:10:04.151305+0000","last_peered":"2023-09-23T18:10:04.151305+0000","last_clean":"2023-09-23T18:10:04.151305+0000","last_became_active":"2023-09-23T18:06:45.904057+0000","last_became_peered":"2023-09-23T18:06:45.904057+0000","last_unstale":"2023-09-23T18:10:04.151305+0000","last_undegraded":"2023-09-23T18:10:04.151305+0000","last_fullsized":"2023-09-23T18:10:04.151305+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:43.213905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":101,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.152875+0000","last_change":"2023-09-23T18:06:05.371661+0000","last_active":"2023-09-23T18:10:04.152875+0000","last_peered":"2023-09-23T18:10:04.152875+0000","last_clean":"2023-09-23T18:10:04.152875+0000","last_became_active":"2023-09-23T18:06:05.370998+0000","last_became_peered":"2023-09-23T18:06:05.370998+0000","last_unstale":"2023-09-23T18:10:04.152875+0000","last_undegraded":"2023-09-23T18:10:04.152875+0000","last_fullsized":"2023-09-23T18:10:04.152875+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:52.262543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"375'3818","reported_seq":3840,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T18:10:05.772063+0000","last_change":"2023-09-23T18:09:26.033882+0000","last_active":"2023-09-23T18:10:05.772063+0000","last_peered":"2023-09-23T18:10:05.772063+0000","last_clean":"2023-09-23T18:10:05.772063+0000","last_became_active":"2023-09-23T18:09:26.033595+0000","last_became_peered":"2023-09-23T18:09:26.033595+0000","last_unstale":"2023-09-23T18:10:05.772063+0000","last_undegraded":"2023-09-23T18:10:05.772063+0000","last_fullsized":"2023-09-23T18:10:05.772063+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3818,"log_dups_size":0,"ondisk_log_size":3818,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:47.811446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250216448,"num_objects":3818,"num_object_clones":0,"num_object_copies":11454,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3818,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3818,"num_write_kb":244352,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":573,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.122677+0000","last_change":"2023-09-23T18:09:49.254468+0000","last_active":"2023-09-23T18:10:04.122677+0000","last_peered":"2023-09-23T18:10:04.122677+0000","last_clean":"2023-09-23T18:09:40.149522+0000","last_became_active":"2023-09-23T18:09:49.254468+0000","last_became_peered":"2023-09-23T18:09:49.254468+0000","last_unstale":"2023-09-23T18:10:04.122677+0000","last_undegraded":"2023-09-23T18:10:04.122677+0000","last_fullsized":"2023-09-23T18:09:49.240580+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:11.928170+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":777,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.178066+0000","last_change":"2023-09-23T18:10:04.168459+0000","last_active":"2023-09-23T18:10:04.178066+0000","last_peered":"2023-09-23T18:10:04.178066+0000","last_clean":"2023-09-23T18:10:04.178066+0000","last_became_active":"2023-09-23T18:10:04.167807+0000","last_became_peered":"2023-09-23T18:10:04.167807+0000","last_unstale":"2023-09-23T18:10:04.178066+0000","last_undegraded":"2023-09-23T18:10:04.178066+0000","last_fullsized":"2023-09-23T18:10:04.178066+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:40.519034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":302,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.122299+0000","last_change":"2023-09-23T18:09:49.247765+0000","last_active":"2023-09-23T18:10:04.122299+0000","last_peered":"2023-09-23T18:10:04.122299+0000","last_clean":"2023-09-23T18:06:48.956978+0000","last_became_active":"2023-09-23T18:09:49.247765+0000","last_became_peered":"2023-09-23T18:09:49.247765+0000","last_unstale":"2023-09-23T18:10:04.122299+0000","last_undegraded":"2023-09-23T18:10:04.122299+0000","last_fullsized":"2023-09-23T18:09:49.240659+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:22.681242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":107,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.160890+0000","last_change":"2023-09-23T18:09:49.249900+0000","last_active":"2023-09-23T18:10:04.160890+0000","last_peered":"2023-09-23T18:10:04.160890+0000","last_clean":"2023-09-23T18:09:40.164966+0000","last_became_active":"2023-09-23T18:09:49.249900+0000","last_became_peered":"2023-09-23T18:09:49.249900+0000","last_unstale":"2023-09-23T18:10:04.160890+0000","last_undegraded":"2023-09-23T18:10:04.160890+0000","last_fullsized":"2023-09-23T18:09:49.239398+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:02.030814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"375'3825","reported_seq":3857,"reported_epoch":375,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:06.973856+0000","last_change":"2023-09-23T18:09:49.273902+0000","last_active":"2023-09-23T18:10:06.973856+0000","last_peered":"2023-09-23T18:10:06.973856+0000","last_clean":"2023-09-23T18:09:48.238829+0000","last_became_active":"2023-09-23T18:09:49.264471+0000","last_became_peered":"2023-09-23T18:09:49.264471+0000","last_unstale":"2023-09-23T18:10:06.973856+0000","last_undegraded":"2023-09-23T18:09:49.241143+0000","last_fullsized":"2023-09-23T18:09:49.235824+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3825,"log_dups_size":0,"ondisk_log_size":3825,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:01.252100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250675200,"num_objects":3825,"num_object_clones":0,"num_object_copies":11475,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3825,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3825,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3825,"num_write_kb":244800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":3825}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":596,"reported_epoch":374,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:04.147359+0000","last_change":"2023-09-23T18:10:04.146853+0000","last_active":"2023-09-23T18:10:04.147359+0000","last_peered":"2023-09-23T18:10:04.147359+0000","last_clean":"2023-09-23T18:10:04.147359+0000","last_became_active":"2023-09-23T18:10:04.146326+0000","last_became_peered":"2023-09-23T18:10:04.146326+0000","last_unstale":"2023-09-23T18:10:04.147359+0000","last_undegraded":"2023-09-23T18:10:04.147359+0000","last_fullsized":"2023-09-23T18:10:04.147359+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:39.103607+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1234,"reported_epoch":374,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:04.155512+0000","last_change":"2023-09-23T18:09:49.245522+0000","last_active":"2023-09-23T18:10:04.155512+0000","last_peered":"2023-09-23T18:10:04.155512+0000","last_clean":"2023-09-23T18:05:59.276534+0000","last_became_active":"2023-09-23T18:09:49.245522+0000","last_became_peered":"2023-09-23T18:09:49.245522+0000","last_unstale":"2023-09-23T18:10:04.155512+0000","last_undegraded":"2023-09-23T18:09:49.238958+0000","last_fullsized":"2023-09-23T18:09:49.238829+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:46.845650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":185,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.132656+0000","last_change":"2023-09-23T18:10:01.168325+0000","last_active":"2023-09-23T18:10:04.132656+0000","last_peered":"2023-09-23T18:10:04.132656+0000","last_clean":"2023-09-23T18:10:04.132656+0000","last_became_active":"2023-09-23T18:10:01.168019+0000","last_became_peered":"2023-09-23T18:10:01.168019+0000","last_unstale":"2023-09-23T18:10:04.132656+0000","last_undegraded":"2023-09-23T18:10:04.132656+0000","last_fullsized":"2023-09-23T18:10:04.132656+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:06.229925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":110,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.150551+0000","last_change":"2023-09-23T18:06:51.088473+0000","last_active":"2023-09-23T18:10:04.150551+0000","last_peered":"2023-09-23T18:10:04.150551+0000","last_clean":"2023-09-23T18:10:04.150551+0000","last_became_active":"2023-09-23T18:06:51.088171+0000","last_became_peered":"2023-09-23T18:06:51.088171+0000","last_unstale":"2023-09-23T18:10:04.150551+0000","last_undegraded":"2023-09-23T18:10:04.150551+0000","last_fullsized":"2023-09-23T18:10:04.150551+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:45.860551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"375'3754","reported_seq":3775,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T18:10:05.798770+0000","last_change":"2023-09-23T18:09:26.034398+0000","last_active":"2023-09-23T18:10:05.798770+0000","last_peered":"2023-09-23T18:10:05.798770+0000","last_clean":"2023-09-23T18:10:05.798770+0000","last_became_active":"2023-09-23T18:09:26.034124+0000","last_became_peered":"2023-09-23T18:09:26.034124+0000","last_unstale":"2023-09-23T18:10:05.798770+0000","last_undegraded":"2023-09-23T18:10:05.798770+0000","last_fullsized":"2023-09-23T18:10:05.798770+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3754,"log_dups_size":0,"ondisk_log_size":3754,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:24.809053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246022144,"num_objects":3754,"num_object_clones":0,"num_object_copies":11262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3754,"num_write_kb":240256,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":643,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.140799+0000","last_change":"2023-09-23T18:10:04.140020+0000","last_active":"2023-09-23T18:10:04.140799+0000","last_peered":"2023-09-23T18:10:04.140799+0000","last_clean":"2023-09-23T18:10:04.140799+0000","last_became_active":"2023-09-23T18:10:04.139695+0000","last_became_peered":"2023-09-23T18:10:04.139695+0000","last_unstale":"2023-09-23T18:10:04.140799+0000","last_undegraded":"2023-09-23T18:10:04.140799+0000","last_fullsized":"2023-09-23T18:10:04.140799+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:40.787527+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":40,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.133875+0000","last_change":"2023-09-23T18:10:01.223543+0000","last_active":"2023-09-23T18:10:04.133875+0000","last_peered":"2023-09-23T18:10:04.133875+0000","last_clean":"2023-09-23T18:10:04.133875+0000","last_became_active":"2023-09-23T18:10:01.176897+0000","last_became_peered":"2023-09-23T18:10:01.176897+0000","last_unstale":"2023-09-23T18:10:04.133875+0000","last_undegraded":"2023-09-23T18:10:04.133875+0000","last_fullsized":"2023-09-23T18:10:04.133875+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:03:34.483871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":102,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T18:10:05.238995+0000","last_change":"2023-09-23T18:06:05.358500+0000","last_active":"2023-09-23T18:10:05.238995+0000","last_peered":"2023-09-23T18:10:05.238995+0000","last_clean":"2023-09-23T18:10:05.238995+0000","last_became_active":"2023-09-23T18:06:05.358274+0000","last_became_peered":"2023-09-23T18:06:05.358274+0000","last_unstale":"2023-09-23T18:10:05.238995+0000","last_undegraded":"2023-09-23T18:10:05.238995+0000","last_fullsized":"2023-09-23T18:10:05.238995+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:04.055437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":116,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.153085+0000","last_change":"2023-09-23T18:09:49.257163+0000","last_active":"2023-09-23T18:10:04.153085+0000","last_peered":"2023-09-23T18:10:04.153085+0000","last_clean":"2023-09-23T18:09:40.168372+0000","last_became_active":"2023-09-23T18:09:49.257163+0000","last_became_peered":"2023-09-23T18:09:49.257163+0000","last_unstale":"2023-09-23T18:10:04.153085+0000","last_undegraded":"2023-09-23T18:10:04.153085+0000","last_fullsized":"2023-09-23T18:09:49.240945+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:14.517993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"375'3831","reported_seq":3860,"reported_epoch":375,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:05.633692+0000","last_change":"2023-09-23T18:09:49.245077+0000","last_active":"2023-09-23T18:10:05.633692+0000","last_peered":"2023-09-23T18:10:05.633692+0000","last_clean":"2023-09-23T18:09:48.227889+0000","last_became_active":"2023-09-23T18:09:49.245077+0000","last_became_peered":"2023-09-23T18:09:49.245077+0000","last_unstale":"2023-09-23T18:10:05.633692+0000","last_undegraded":"2023-09-23T18:09:49.239224+0000","last_fullsized":"2023-09-23T18:09:49.239102+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3831,"log_dups_size":0,"ondisk_log_size":3831,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:54.675527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":251068416,"num_objects":3831,"num_object_clones":0,"num_object_copies":11493,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3831,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3831,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3831,"num_write_kb":245184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":3831}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":998,"reported_epoch":374,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:04.134926+0000","last_change":"2023-09-23T18:09:49.260151+0000","last_active":"2023-09-23T18:10:04.134926+0000","last_peered":"2023-09-23T18:10:04.134926+0000","last_clean":"2023-09-23T17:47:00.624503+0000","last_became_active":"2023-09-23T18:09:49.260151+0000","last_became_peered":"2023-09-23T18:09:49.260151+0000","last_unstale":"2023-09-23T18:10:04.134926+0000","last_undegraded":"2023-09-23T18:09:49.255776+0000","last_fullsized":"2023-09-23T18:09:49.255629+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:31.486599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":730,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.134618+0000","last_change":"2023-09-23T18:06:48.957422+0000","last_active":"2023-09-23T18:10:04.134618+0000","last_peered":"2023-09-23T18:10:04.134618+0000","last_clean":"2023-09-23T18:10:04.134618+0000","last_became_active":"2023-09-23T18:06:48.957044+0000","last_became_peered":"2023-09-23T18:06:48.957044+0000","last_unstale":"2023-09-23T18:10:04.134618+0000","last_undegraded":"2023-09-23T18:10:04.134618+0000","last_fullsized":"2023-09-23T18:10:04.134618+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:58.047861+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":144,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.122294+0000","last_change":"2023-09-23T18:09:49.245056+0000","last_active":"2023-09-23T18:10:04.122294+0000","last_peered":"2023-09-23T18:10:04.122294+0000","last_clean":"2023-09-23T17:55:56.841167+0000","last_became_active":"2023-09-23T18:09:49.245056+0000","last_became_peered":"2023-09-23T18:09:49.245056+0000","last_unstale":"2023-09-23T18:10:04.122294+0000","last_undegraded":"2023-09-23T18:10:04.122294+0000","last_fullsized":"2023-09-23T18:09:49.240277+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:39.548152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":308,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.123013+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:10:04.123013+0000","last_peered":"2023-09-23T18:10:04.123013+0000","last_clean":"2023-09-23T18:10:04.123013+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:10:04.123013+0000","last_undegraded":"2023-09-23T18:10:04.123013+0000","last_fullsized":"2023-09-23T18:10:04.123013+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"375'3967","reported_seq":3996,"reported_epoch":375,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:07.705457+0000","last_change":"2023-09-23T18:09:49.246060+0000","last_active":"2023-09-23T18:10:07.705457+0000","last_peered":"2023-09-23T18:10:07.705457+0000","last_clean":"2023-09-23T18:09:48.219556+0000","last_became_active":"2023-09-23T18:09:49.246060+0000","last_became_peered":"2023-09-23T18:09:49.246060+0000","last_unstale":"2023-09-23T18:10:07.705457+0000","last_undegraded":"2023-09-23T18:09:49.238965+0000","last_fullsized":"2023-09-23T18:09:49.238836+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3967,"log_dups_size":0,"ondisk_log_size":3967,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:32.216226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":259981312,"num_objects":3967,"num_object_clones":0,"num_object_copies":11901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3967,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3967,"num_write_kb":253888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":3967}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":536,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.153897+0000","last_change":"2023-09-23T18:09:16.643471+0000","last_active":"2023-09-23T18:10:04.153897+0000","last_peered":"2023-09-23T18:10:04.153897+0000","last_clean":"2023-09-23T18:10:04.153897+0000","last_became_active":"2023-09-23T18:06:08.433718+0000","last_became_peered":"2023-09-23T18:06:08.433718+0000","last_unstale":"2023-09-23T18:10:04.153897+0000","last_undegraded":"2023-09-23T18:10:04.153897+0000","last_fullsized":"2023-09-23T18:10:04.153897+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:37.624632+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":638,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.135303+0000","last_change":"2023-09-23T18:10:01.230791+0000","last_active":"2023-09-23T18:10:04.135303+0000","last_peered":"2023-09-23T18:10:04.135303+0000","last_clean":"2023-09-23T18:10:04.135303+0000","last_became_active":"2023-09-23T18:10:01.170909+0000","last_became_peered":"2023-09-23T18:10:01.170909+0000","last_unstale":"2023-09-23T18:10:04.135303+0000","last_undegraded":"2023-09-23T18:10:04.135303+0000","last_fullsized":"2023-09-23T18:10:04.135303+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:08.759640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":15,"reported_epoch":374,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:04.133416+0000","last_change":"2023-09-23T18:10:02.189623+0000","last_active":"2023-09-23T18:10:04.133416+0000","last_peered":"2023-09-23T18:10:04.133416+0000","last_clean":"2023-09-23T18:10:04.133416+0000","last_became_active":"2023-09-23T18:10:02.189234+0000","last_became_peered":"2023-09-23T18:10:02.189234+0000","last_unstale":"2023-09-23T18:10:04.133416+0000","last_undegraded":"2023-09-23T18:10:04.133416+0000","last_fullsized":"2023-09-23T18:10:04.133416+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:18.377869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":359,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.150168+0000","last_change":"2023-09-23T18:09:49.263629+0000","last_active":"2023-09-23T18:10:04.150168+0000","last_peered":"2023-09-23T18:10:04.150168+0000","last_clean":"2023-09-23T18:09:40.167801+0000","last_became_active":"2023-09-23T18:09:49.263629+0000","last_became_peered":"2023-09-23T18:09:49.263629+0000","last_unstale":"2023-09-23T18:10:04.150168+0000","last_undegraded":"2023-09-23T18:10:04.150168+0000","last_fullsized":"2023-09-23T18:09:49.239463+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:25.971337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"375'3977","reported_seq":4000,"reported_epoch":375,"state":"active+clean","last_fresh":"2023-09-23T18:10:07.706065+0000","last_change":"2023-09-23T18:09:26.507006+0000","last_active":"2023-09-23T18:10:07.706065+0000","last_peered":"2023-09-23T18:10:07.706065+0000","last_clean":"2023-09-23T18:10:07.706065+0000","last_became_active":"2023-09-23T18:09:26.506732+0000","last_became_peered":"2023-09-23T18:09:26.506732+0000","last_unstale":"2023-09-23T18:10:07.706065+0000","last_undegraded":"2023-09-23T18:10:07.706065+0000","last_fullsized":"2023-09-23T18:10:07.706065+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3977,"log_dups_size":0,"ondisk_log_size":3977,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:01.574367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260636672,"num_objects":3977,"num_object_clones":0,"num_object_copies":11931,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3977,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3977,"num_write_kb":254528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":586,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.122803+0000","last_change":"2023-09-23T18:06:54.055392+0000","last_active":"2023-09-23T18:10:04.122803+0000","last_peered":"2023-09-23T18:10:04.122803+0000","last_clean":"2023-09-23T18:10:04.122803+0000","last_became_active":"2023-09-23T18:06:54.055101+0000","last_became_peered":"2023-09-23T18:06:54.055101+0000","last_unstale":"2023-09-23T18:10:04.122803+0000","last_undegraded":"2023-09-23T18:10:04.122803+0000","last_fullsized":"2023-09-23T18:10:04.122803+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:50.070812+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":614,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.135772+0000","last_change":"2023-09-23T18:08:46.401461+0000","last_active":"2023-09-23T18:10:04.135772+0000","last_peered":"2023-09-23T18:10:04.135772+0000","last_clean":"2023-09-23T18:10:04.135772+0000","last_became_active":"2023-09-23T18:06:51.071089+0000","last_became_peered":"2023-09-23T18:06:51.071089+0000","last_unstale":"2023-09-23T18:10:04.135772+0000","last_undegraded":"2023-09-23T18:10:04.135772+0000","last_fullsized":"2023-09-23T18:10:04.135772+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:01.696899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":116,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.150426+0000","last_change":"2023-09-23T18:09:49.247536+0000","last_active":"2023-09-23T18:10:04.150426+0000","last_peered":"2023-09-23T18:10:04.150426+0000","last_clean":"2023-09-23T18:09:40.168110+0000","last_became_active":"2023-09-23T18:09:49.247536+0000","last_became_peered":"2023-09-23T18:09:49.247536+0000","last_unstale":"2023-09-23T18:10:04.150426+0000","last_undegraded":"2023-09-23T18:10:04.150426+0000","last_fullsized":"2023-09-23T18:09:49.239753+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:23.414385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":15,"reported_epoch":374,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:04.133649+0000","last_change":"2023-09-23T18:10:02.191332+0000","last_active":"2023-09-23T18:10:04.133649+0000","last_peered":"2023-09-23T18:10:04.133649+0000","last_clean":"2023-09-23T18:10:04.133649+0000","last_became_active":"2023-09-23T18:10:02.190947+0000","last_became_peered":"2023-09-23T18:10:02.190947+0000","last_unstale":"2023-09-23T18:10:04.133649+0000","last_undegraded":"2023-09-23T18:10:04.133649+0000","last_fullsized":"2023-09-23T18:10:04.133649+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:03:14.070652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"375'3907","reported_seq":3936,"reported_epoch":375,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:08.057521+0000","last_change":"2023-09-23T18:09:49.253852+0000","last_active":"2023-09-23T18:10:08.057521+0000","last_peered":"2023-09-23T18:10:08.057521+0000","last_clean":"2023-09-23T18:09:48.216272+0000","last_became_active":"2023-09-23T18:09:49.253852+0000","last_became_peered":"2023-09-23T18:09:49.253852+0000","last_unstale":"2023-09-23T18:10:08.057521+0000","last_undegraded":"2023-09-23T18:09:49.249506+0000","last_fullsized":"2023-09-23T18:09:49.249391+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3907,"log_dups_size":0,"ondisk_log_size":3907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:10.981898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":256049152,"num_objects":3907,"num_object_clones":0,"num_object_copies":11721,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3907,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3907,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3907,"num_write_kb":250048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":3907}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":727,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.131449+0000","last_change":"2023-09-23T18:09:49.245223+0000","last_active":"2023-09-23T18:10:04.131449+0000","last_peered":"2023-09-23T18:10:04.131449+0000","last_clean":"2023-09-23T18:09:40.159105+0000","last_became_active":"2023-09-23T18:09:49.245223+0000","last_became_peered":"2023-09-23T18:09:49.245223+0000","last_unstale":"2023-09-23T18:10:04.131449+0000","last_undegraded":"2023-09-23T18:10:04.131449+0000","last_fullsized":"2023-09-23T18:09:49.241984+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:36:00.210097+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":155,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.135427+0000","last_change":"2023-09-23T18:10:01.179787+0000","last_active":"2023-09-23T18:10:04.135427+0000","last_peered":"2023-09-23T18:10:04.135427+0000","last_clean":"2023-09-23T18:10:04.135427+0000","last_became_active":"2023-09-23T18:10:01.174679+0000","last_became_peered":"2023-09-23T18:10:01.174679+0000","last_unstale":"2023-09-23T18:10:04.135427+0000","last_undegraded":"2023-09-23T18:10:04.135427+0000","last_fullsized":"2023-09-23T18:10:04.135427+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:27.491151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":116,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.150600+0000","last_change":"2023-09-23T18:06:05.398171+0000","last_active":"2023-09-23T18:10:04.150600+0000","last_peered":"2023-09-23T18:10:04.150600+0000","last_clean":"2023-09-23T18:10:04.150600+0000","last_became_active":"2023-09-23T18:06:05.370800+0000","last_became_peered":"2023-09-23T18:06:05.370800+0000","last_unstale":"2023-09-23T18:10:04.150600+0000","last_undegraded":"2023-09-23T18:10:04.150600+0000","last_fullsized":"2023-09-23T18:10:04.150600+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:46.783817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":186,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.134723+0000","last_change":"2023-09-23T18:09:49.265848+0000","last_active":"2023-09-23T18:10:04.134723+0000","last_peered":"2023-09-23T18:10:04.134723+0000","last_clean":"2023-09-23T18:09:40.154366+0000","last_became_active":"2023-09-23T18:09:49.265848+0000","last_became_peered":"2023-09-23T18:09:49.265848+0000","last_unstale":"2023-09-23T18:10:04.134723+0000","last_undegraded":"2023-09-23T18:10:04.134723+0000","last_fullsized":"2023-09-23T18:09:49.251764+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:23.255571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"375'4000","reported_seq":4027,"reported_epoch":375,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:08.023349+0000","last_change":"2023-09-23T18:09:49.264734+0000","last_active":"2023-09-23T18:10:08.023349+0000","last_peered":"2023-09-23T18:10:08.023349+0000","last_clean":"2023-09-23T18:09:48.206765+0000","last_became_active":"2023-09-23T18:09:49.264734+0000","last_became_peered":"2023-09-23T18:09:49.264734+0000","last_unstale":"2023-09-23T18:10:08.023349+0000","last_undegraded":"2023-09-23T18:09:49.257799+0000","last_fullsized":"2023-09-23T18:09:49.257700+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4000,"log_dups_size":0,"ondisk_log_size":4000,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:31.109407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262144000,"num_objects":4000,"num_object_clones":0,"num_object_copies":12000,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4000,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4000,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4000,"num_write_kb":256000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":4000}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":627,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.134902+0000","last_change":"2023-09-23T18:09:49.258198+0000","last_active":"2023-09-23T18:10:04.134902+0000","last_peered":"2023-09-23T18:10:04.134902+0000","last_clean":"2023-09-23T18:05:06.479280+0000","last_became_active":"2023-09-23T18:09:49.258198+0000","last_became_peered":"2023-09-23T18:09:49.258198+0000","last_unstale":"2023-09-23T18:10:04.134902+0000","last_undegraded":"2023-09-23T18:10:04.134902+0000","last_fullsized":"2023-09-23T18:09:49.249014+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:05.551792+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":580,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.122875+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:10:04.122875+0000","last_peered":"2023-09-23T18:10:04.122875+0000","last_clean":"2023-09-23T18:10:04.122875+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:10:04.122875+0000","last_undegraded":"2023-09-23T18:10:04.122875+0000","last_fullsized":"2023-09-23T18:10:04.122875+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":405,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.150307+0000","last_change":"2023-09-23T18:06:45.939855+0000","last_active":"2023-09-23T18:10:04.150307+0000","last_peered":"2023-09-23T18:10:04.150307+0000","last_clean":"2023-09-23T18:10:04.150307+0000","last_became_active":"2023-09-23T18:06:45.924278+0000","last_became_peered":"2023-09-23T18:06:45.924278+0000","last_unstale":"2023-09-23T18:10:04.150307+0000","last_undegraded":"2023-09-23T18:10:04.150307+0000","last_fullsized":"2023-09-23T18:10:04.150307+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:43.850375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":172,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.131039+0000","last_change":"2023-09-23T18:09:49.245496+0000","last_active":"2023-09-23T18:10:04.131039+0000","last_peered":"2023-09-23T18:10:04.131039+0000","last_clean":"2023-09-23T18:09:40.158239+0000","last_became_active":"2023-09-23T18:09:49.245496+0000","last_became_peered":"2023-09-23T18:09:49.245496+0000","last_unstale":"2023-09-23T18:10:04.131039+0000","last_undegraded":"2023-09-23T18:10:04.131039+0000","last_fullsized":"2023-09-23T18:09:49.241846+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:14:46.672379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":522,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.121842+0000","last_change":"2023-09-23T18:09:14.580901+0000","last_active":"2023-09-23T18:10:04.121842+0000","last_peered":"2023-09-23T18:10:04.121842+0000","last_clean":"2023-09-23T18:10:04.121842+0000","last_became_active":"2023-09-23T18:06:48.961304+0000","last_became_peered":"2023-09-23T18:06:48.961304+0000","last_unstale":"2023-09-23T18:10:04.121842+0000","last_undegraded":"2023-09-23T18:10:04.121842+0000","last_fullsized":"2023-09-23T18:10:04.121842+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:14.580819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:14.580819+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:04.095372+0000","scrub_duration":0.003313033,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":291,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.132542+0000","last_change":"2023-09-23T18:06:26.148865+0000","last_active":"2023-09-23T18:10:04.132542+0000","last_peered":"2023-09-23T18:10:04.132542+0000","last_clean":"2023-09-23T18:10:04.132542+0000","last_became_active":"2023-09-23T18:06:26.148517+0000","last_became_peered":"2023-09-23T18:06:26.148517+0000","last_unstale":"2023-09-23T18:10:04.132542+0000","last_undegraded":"2023-09-23T18:10:04.132542+0000","last_fullsized":"2023-09-23T18:10:04.132542+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:38.141672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":418,"reported_epoch":374,"state":"active+undersized","last_fresh":"2023-09-23T18:10:04.150280+0000","last_change":"2023-09-23T18:09:49.248926+0000","last_active":"2023-09-23T18:10:04.150280+0000","last_peered":"2023-09-23T18:10:04.150280+0000","last_clean":"2023-09-23T18:09:40.167682+0000","last_became_active":"2023-09-23T18:09:49.248926+0000","last_became_peered":"2023-09-23T18:09:49.248926+0000","last_unstale":"2023-09-23T18:10:04.150280+0000","last_undegraded":"2023-09-23T18:10:04.150280+0000","last_fullsized":"2023-09-23T18:09:49.239548+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:18.664900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":517,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.122484+0000","last_change":"2023-09-23T18:09:16.604488+0000","last_active":"2023-09-23T18:10:04.122484+0000","last_peered":"2023-09-23T18:10:04.122484+0000","last_clean":"2023-09-23T18:10:04.122484+0000","last_became_active":"2023-09-23T18:06:54.055098+0000","last_became_peered":"2023-09-23T18:06:54.055098+0000","last_unstale":"2023-09-23T18:10:04.122484+0000","last_undegraded":"2023-09-23T18:10:04.122484+0000","last_fullsized":"2023-09-23T18:10:04.122484+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.604423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.604423+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:28.598882+0000","scrub_duration":0.0014924669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":65,"reported_epoch":374,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:04.131385+0000","last_change":"2023-09-23T18:09:49.254308+0000","last_active":"2023-09-23T18:10:04.131385+0000","last_peered":"2023-09-23T18:10:04.131385+0000","last_clean":"2023-09-23T18:01:36.146652+0000","last_became_active":"2023-09-23T18:09:49.254308+0000","last_became_peered":"2023-09-23T18:09:49.254308+0000","last_unstale":"2023-09-23T18:10:04.131385+0000","last_undegraded":"2023-09-23T18:09:49.242698+0000","last_fullsized":"2023-09-23T18:09:49.242588+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:35.825451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":310,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.132753+0000","last_change":"2023-09-23T18:06:05.375138+0000","last_active":"2023-09-23T18:10:04.132753+0000","last_peered":"2023-09-23T18:10:04.132753+0000","last_clean":"2023-09-23T18:10:04.132753+0000","last_became_active":"2023-09-23T18:06:05.374819+0000","last_became_peered":"2023-09-23T18:06:05.374819+0000","last_unstale":"2023-09-23T18:10:04.132753+0000","last_undegraded":"2023-09-23T18:10:04.132753+0000","last_fullsized":"2023-09-23T18:10:04.132753+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:43.000019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":65,"reported_epoch":374,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:04.131668+0000","last_change":"2023-09-23T18:09:49.246801+0000","last_active":"2023-09-23T18:10:04.131668+0000","last_peered":"2023-09-23T18:10:04.131668+0000","last_clean":"2023-09-23T18:01:36.147388+0000","last_became_active":"2023-09-23T18:09:49.246801+0000","last_became_peered":"2023-09-23T18:09:49.246801+0000","last_unstale":"2023-09-23T18:10:04.131668+0000","last_undegraded":"2023-09-23T18:09:49.243160+0000","last_fullsized":"2023-09-23T18:09:49.241934+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:36.925478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":14,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.131456+0000","last_change":"2023-09-23T18:10:01.178038+0000","last_active":"2023-09-23T18:10:04.131456+0000","last_peered":"2023-09-23T18:10:04.131456+0000","last_clean":"2023-09-23T18:10:04.131456+0000","last_became_active":"2023-09-23T18:10:01.177675+0000","last_became_peered":"2023-09-23T18:10:01.177675+0000","last_unstale":"2023-09-23T18:10:04.131456+0000","last_undegraded":"2023-09-23T18:10:04.131456+0000","last_fullsized":"2023-09-23T18:10:04.131456+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:24.038395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":761,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.134936+0000","last_change":"2023-09-23T18:06:08.440953+0000","last_active":"2023-09-23T18:10:04.134936+0000","last_peered":"2023-09-23T18:10:04.134936+0000","last_clean":"2023-09-23T18:10:04.134936+0000","last_became_active":"2023-09-23T18:06:08.440604+0000","last_became_peered":"2023-09-23T18:06:08.440604+0000","last_unstale":"2023-09-23T18:10:04.134936+0000","last_undegraded":"2023-09-23T18:10:04.134936+0000","last_fullsized":"2023-09-23T18:10:04.134936+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:53.274049+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":177,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.122713+0000","last_change":"2023-09-23T18:06:51.098401+0000","last_active":"2023-09-23T18:10:04.122713+0000","last_peered":"2023-09-23T18:10:04.122713+0000","last_clean":"2023-09-23T18:10:04.122713+0000","last_became_active":"2023-09-23T18:06:51.097959+0000","last_became_peered":"2023-09-23T18:06:51.097959+0000","last_unstale":"2023-09-23T18:10:04.122713+0000","last_undegraded":"2023-09-23T18:10:04.122713+0000","last_fullsized":"2023-09-23T18:10:04.122713+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:16.234414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":15,"reported_epoch":374,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:04.133272+0000","last_change":"2023-09-23T18:10:02.307514+0000","last_active":"2023-09-23T18:10:04.133272+0000","last_peered":"2023-09-23T18:10:04.133272+0000","last_clean":"2023-09-23T18:10:04.133272+0000","last_became_active":"2023-09-23T18:10:02.307036+0000","last_became_peered":"2023-09-23T18:10:02.307036+0000","last_unstale":"2023-09-23T18:10:04.133272+0000","last_undegraded":"2023-09-23T18:10:04.133272+0000","last_fullsized":"2023-09-23T18:10:04.133272+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:19.665837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,1],"avail_no_missing":["0","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":708,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.151176+0000","last_change":"2023-09-23T18:09:17.636095+0000","last_active":"2023-09-23T18:10:04.151176+0000","last_peered":"2023-09-23T18:10:04.151176+0000","last_clean":"2023-09-23T18:10:04.151176+0000","last_became_active":"2023-09-23T18:06:48.965429+0000","last_became_peered":"2023-09-23T18:06:48.965429+0000","last_unstale":"2023-09-23T18:10:04.151176+0000","last_undegraded":"2023-09-23T18:10:04.151176+0000","last_fullsized":"2023-09-23T18:10:04.151176+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:17.635974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:17.635974+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:06.792253+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":184,"reported_epoch":374,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:04.122506+0000","last_change":"2023-09-23T18:09:49.245379+0000","last_active":"2023-09-23T18:10:04.122506+0000","last_peered":"2023-09-23T18:10:04.122506+0000","last_clean":"2023-09-23T18:00:42.557373+0000","last_became_active":"2023-09-23T18:09:49.245379+0000","last_became_peered":"2023-09-23T18:09:49.245379+0000","last_unstale":"2023-09-23T18:10:04.122506+0000","last_undegraded":"2023-09-23T18:09:49.240279+0000","last_fullsized":"2023-09-23T18:09:49.240171+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:29.708404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":2}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":395,"reported_epoch":374,"state":"active+clean","last_fresh":"2023-09-23T18:10:04.130880+0000","last_change":"2023-09-23T18:06:05.374027+0000","last_active":"2023-09-23T18:10:04.130880+0000","last_peered":"2023-09-23T18:10:04.130880+0000","last_clean":"2023-09-23T18:10:04.130880+0000","last_became_active":"2023-09-23T18:06:05.373670+0000","last_became_peered":"2023-09-23T18:06:05.373670+0000","last_unstale":"2023-09-23T18:10:04.130880+0000","last_undegraded":"2023-09-23T18:10:04.130880+0000","last_fullsized":"2023-09-23T18:10:04.130880+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:01.220735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":84,"acting":86,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":5692,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":86,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":4070440960,"num_objects":62110,"num_object_clones":0,"num_object_copies":186330,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":38126,"num_objects_misplaced":7367,"num_objects_unfound":0,"num_objects_dirty":62110,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":62110,"num_write_kb":3975040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":194,"num_bytes_recovered":12713984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11262492672,"data_stored":11262492672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":62110,"ondisk_log_size":62110,"up":39,"acting":38,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":102867,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":85,"num_read_kb":160,"num_write":42,"num_write_kb":421,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2330654,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":77,"acting":79,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":22,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001450,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1985412,"kb_used_data":1750524,"kb_used_omap":13,"kb_used_meta":234866,"kb_avail":91759740,"statfs":{"total":95995035648,"available":93961973760,"internally_reserved":0,"allocated":1792536576,"data_stored":1791280123,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":240502888},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,1,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.1080000000000001,"15min":0.78700000000000003},"min":{"1min":0.254,"5min":0.217,"15min":0.217},"max":{"1min":3.1930000000000001,"5min":9.3499999999999996,"15min":9.3499999999999996},"last":0.214},{"interface":"front","average":{"1min":1.1930000000000001,"5min":0.88600000000000001,"15min":1.1240000000000001},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.23000000000000001},"max":{"1min":3.8159999999999998,"5min":4.8209999999999997,"15min":6.641},"last":2.0329999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.2430000000000001,"15min":0.94199999999999995},"min":{"1min":0.38900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.1840000000000002,"5min":9.4909999999999997,"15min":9.4909999999999997},"last":2.6110000000000002},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.0940000000000001,"15min":1.0880000000000001},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.1549999999999998,"5min":6.9589999999999996,"15min":6.9589999999999996},"last":1.1990000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.2370000000000001,"15min":1.034},"min":{"1min":0.35899999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.4039999999999999,"5min":6.6630000000000003,"15min":6.6630000000000003},"last":4.5860000000000003},{"interface":"front","average":{"1min":1.325,"5min":1.2470000000000001,"15min":1.016},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.29399999999999998},"max":{"1min":3.653,"5min":7.9160000000000004,"15min":7.9160000000000004},"last":4.9459999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:09:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.373,"15min":1.474},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.8710000000000004,"5min":7.0579999999999998,"15min":7.6070000000000002},"last":1.9379999999999999},{"interface":"front","average":{"1min":1.5,"5min":1.4570000000000001,"15min":1.173},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.7889999999999997,"5min":15.349,"15min":15.349},"last":0.33300000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":0.88600000000000001,"15min":1.0009999999999999},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.245},"max":{"1min":4.4669999999999996,"5min":6.9729999999999999,"15min":6.9729999999999999},"last":0.38800000000000001},{"interface":"front","average":{"1min":1.4330000000000001,"5min":0.90000000000000002,"15min":0.67200000000000004},"min":{"1min":0.31900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.1059999999999999,"5min":5.1059999999999999,"15min":6.3479999999999999},"last":0.30399999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.0860000000000001,"15min":1.042},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.6280000000000001,"5min":8.4749999999999996,"15min":9.5999999999999996},"last":3.2149999999999999},{"interface":"front","average":{"1min":1.127,"5min":0.97899999999999998,"15min":1.095},"min":{"1min":0.35899999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.2839999999999998,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":3.1720000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":1.032,"15min":0.84199999999999997},"min":{"1min":0.37,"5min":0.308,"15min":0.308},"max":{"1min":6.782,"5min":6.782,"15min":6.782},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.1120000000000001,"5min":0.89400000000000002,"15min":1.081},"min":{"1min":0.35299999999999998,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":3.1280000000000001,"5min":4.9539999999999997,"15min":9.5519999999999996},"last":0.40500000000000003}]}]},{"osd":6,"up_from":163,"seq":700079669386,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1973308,"kb_used_data":1700084,"kb_used_omap":7,"kb_used_meta":273208,"kb_avail":91771844,"statfs":{"total":95995035648,"available":93974368256,"internally_reserved":0,"allocated":1740886016,"data_stored":1739637638,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":279765796},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":0.999,"15min":1.181},"min":{"1min":0.34899999999999998,"5min":0.23100000000000001,"15min":0.215},"max":{"1min":6.3550000000000004,"5min":8.4019999999999992,"15min":15.407},"last":0.42799999999999999},{"interface":"front","average":{"1min":1.349,"5min":1.238,"15min":1.6879999999999999},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.23100000000000001},"max":{"1min":4.1550000000000002,"5min":15.147,"15min":15.147},"last":0.26300000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":1.1379999999999999,"15min":1.5189999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":6.4240000000000004,"5min":6.5309999999999997,"15min":14.233000000000001},"last":2.4950000000000001},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.0700000000000001,"15min":1.6200000000000001},"min":{"1min":0.30099999999999999,"5min":0.26000000000000001,"15min":0.248},"max":{"1min":4.194,"5min":4.194,"15min":14.789},"last":0.29599999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.718,"5min":1.137,"15min":1.75},"min":{"1min":0.39800000000000002,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":6.9989999999999997,"5min":8.6140000000000008,"15min":11.210000000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.8879999999999999,"5min":1.224,"15min":1.9359999999999999},"min":{"1min":0.40200000000000002,"5min":0.248,"15min":0.245},"max":{"1min":4.7220000000000004,"5min":8.6300000000000008,"15min":11.223000000000001},"last":0.39200000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:09:31 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":1.401,"15min":1.1830000000000001},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.308},"max":{"1min":8.5449999999999999,"5min":8.5449999999999999,"15min":8.5449999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":1.236,"5min":1.359,"15min":1.2509999999999999},"min":{"1min":0.42099999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.2699999999999996,"5min":7.484,"15min":7.484},"last":2.7290000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.484,"5min":0.92500000000000004,"15min":0.92900000000000005},"min":{"1min":0.28399999999999997,"5min":0.254,"15min":0.23000000000000001},"max":{"1min":4.6749999999999998,"5min":8.0280000000000005,"15min":9.9109999999999996},"last":0.311},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.056,"15min":0.78500000000000003},"min":{"1min":0.35099999999999998,"5min":0.27200000000000002,"15min":0.24099999999999999},"max":{"1min":6.2949999999999999,"5min":8.1159999999999997,"15min":8.3000000000000007},"last":0.84099999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":0.99299999999999999,"15min":0.89400000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7219999999999995,"5min":9.7219999999999995,"15min":11.869},"last":1.681},{"interface":"front","average":{"1min":1.5289999999999999,"5min":0.92200000000000004,"15min":0.80100000000000005},"min":{"1min":0.30599999999999999,"5min":0.251,"15min":0.251},"max":{"1min":10.436,"5min":10.436,"15min":10.436},"last":1.746}]},{"osd":7,"last update":"Sat Sep 23 18:09:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":1.0229999999999999,"15min":1.0029999999999999},"min":{"1min":0.378,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.460000000000001,"5min":13.460000000000001,"15min":13.460000000000001},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.917,"5min":1.139,"15min":0.92000000000000004},"min":{"1min":0.41999999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.1470000000000002,"5min":10.411,"15min":10.411},"last":0.36699999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795204,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2385540,"kb_used_data":1766588,"kb_used_omap":7,"kb_used_meta":618936,"kb_avail":91359612,"statfs":{"total":95995035648,"available":93552242688,"internally_reserved":0,"allocated":1808986112,"data_stored":1807267584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":633791165},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.413,"15min":1.2390000000000001},"min":{"1min":0.184,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":14.567,"5min":14.567,"15min":15.529999999999999},"last":2.0859999999999999},{"interface":"front","average":{"1min":1.4099999999999999,"5min":2.3100000000000001,"15min":1.651},"min":{"1min":0.25,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.774,"5min":82.244,"15min":82.244},"last":0.26100000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.407,"15min":1.613},"min":{"1min":0.31,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":2.6549999999999998,"5min":10.531000000000001,"15min":16.347999999999999},"last":0.29699999999999999},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.2569999999999999,"15min":1.286},"min":{"1min":0.245,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":4.6890000000000001,"5min":7.641,"15min":11.305999999999999},"last":3.4009999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.655,"15min":0.92800000000000005},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":4.8140000000000001,"5min":11.164,"15min":11.164},"last":2.1459999999999999},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.468,"15min":1.131},"min":{"1min":0.317,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.6950000000000003,"5min":11.125999999999999,"15min":11.125999999999999},"last":0.32700000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":1.3700000000000001,"15min":1.339},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":4.806,"5min":8.3919999999999995,"15min":26.568999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":2.141,"5min":1.4019999999999999,"15min":1.294},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":6.6280000000000001,"5min":7.9260000000000002,"15min":26.635000000000002},"last":2.1219999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.7869999999999999,"15min":1.4379999999999999},"min":{"1min":0.38900000000000001,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":6.7489999999999997,"5min":11.638999999999999,"15min":11.638999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.579,"15min":1.393},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":6.5609999999999999,"5min":8.4309999999999992,"15min":9.5519999999999996},"last":1.962}]},{"osd":6,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":2.1339999999999999,"15min":1.895},"min":{"1min":0.40300000000000002,"5min":0.372,"15min":0.317},"max":{"1min":4.7690000000000001,"5min":89.566000000000003,"15min":89.566000000000003},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.032,"5min":2.1200000000000001,"15min":1.7170000000000001},"min":{"1min":0.42499999999999999,"5min":0.38,"15min":0.34699999999999998},"max":{"1min":3.1600000000000001,"5min":90.608999999999995,"15min":90.608999999999995},"last":0.50900000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":1.2749999999999999,"15min":0.93000000000000005},"min":{"1min":0.45300000000000001,"5min":0.39800000000000002,"15min":0.33900000000000002},"max":{"1min":4.7590000000000003,"5min":7.1219999999999999,"15min":7.1219999999999999},"last":2.1699999999999999},{"interface":"front","average":{"1min":1.194,"5min":1.4179999999999999,"15min":1.635},"min":{"1min":0.45600000000000002,"5min":0.45000000000000001,"15min":0.40999999999999998},"max":{"1min":3.4969999999999999,"5min":7.306,"15min":9.5709999999999997},"last":0.48999999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958731,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":850080,"kb_used_data":333336,"kb_used_omap":7,"kb_used_meta":516728,"kb_avail":92895072,"statfs":{"total":95995035648,"available":95124553728,"internally_reserved":0,"allocated":341336064,"data_stored":339581600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":529130212},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:09:21 2023","interfaces":[{"interface":"back","average":{"1min":1.351,"5min":1.823,"15min":1.6259999999999999},"min":{"1min":0.219,"5min":0.217,"15min":0.17899999999999999},"max":{"1min":9.2599999999999998,"5min":15.836,"15min":18.154},"last":3.8029999999999999},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.3759999999999999,"15min":1.431},"min":{"1min":0.246,"5min":0.246,"15min":0.17399999999999999},"max":{"1min":4.4009999999999998,"5min":14.468999999999999,"15min":14.468999999999999},"last":1.784}]},{"osd":2,"last update":"Sat Sep 23 18:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.4059999999999999,"15min":1.716},"min":{"1min":0.25700000000000001,"5min":0.24399999999999999,"15min":0.20499999999999999},"max":{"1min":3.7909999999999999,"5min":10.48,"15min":20.882000000000001},"last":3.6160000000000001},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.325,"15min":1.6220000000000001},"min":{"1min":0.33300000000000002,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":4.4249999999999998,"5min":10.84,"15min":15.064},"last":1.6719999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:09:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.7769999999999999,"15min":1.583},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.20499999999999999},"max":{"1min":9.8390000000000004,"5min":12.779999999999999,"15min":12.779999999999999},"last":2.484},{"interface":"front","average":{"1min":1.575,"5min":1.613,"15min":1.369},"min":{"1min":0.373,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.5019999999999998,"5min":15.198,"15min":15.198},"last":2.4460000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.3169999999999999,"15min":1.4330000000000001},"min":{"1min":0.44400000000000001,"5min":0.32100000000000001,"15min":0.251},"max":{"1min":3.782,"5min":8.4109999999999996,"15min":15.279999999999999},"last":2.6520000000000001},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.5740000000000001,"15min":1.542},"min":{"1min":0.371,"5min":0.34999999999999998,"15min":0.247},"max":{"1min":3.7629999999999999,"5min":15.477,"15min":18.106999999999999},"last":3.649}]},{"osd":5,"last update":"Sat Sep 23 18:08:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.7230000000000001,"15min":1.5600000000000001},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":9.2010000000000005,"5min":10.194000000000001,"15min":14.720000000000001},"last":1.478},{"interface":"front","average":{"1min":1.409,"5min":1.512,"15min":1.3140000000000001},"min":{"1min":0.33300000000000002,"5min":0.32600000000000001,"15min":0.254},"max":{"1min":5.1230000000000002,"5min":10.363,"15min":12.794},"last":2.1160000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":2.0489999999999999,"15min":1.998},"min":{"1min":0.38900000000000001,"5min":0.35199999999999998,"15min":0.34599999999999997},"max":{"1min":4.2489999999999997,"5min":58.457999999999998,"15min":58.457999999999998},"last":2.6850000000000001},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.9690000000000001,"15min":2.04},"min":{"1min":0.49299999999999999,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":4.3470000000000004,"5min":58.354999999999997,"15min":58.354999999999997},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:09:35 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.4359999999999999,"15min":1.1140000000000001},"min":{"1min":0.41599999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.4790000000000001,"5min":10.416,"15min":10.416},"last":0.52700000000000002},{"interface":"front","average":{"1min":1.548,"5min":1.536,"15min":1.3089999999999999},"min":{"1min":0.443,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.8760000000000003,"5min":8.923,"15min":8.923},"last":2.7250000000000001}]}]},{"osd":2,"up_from":103,"seq":442381631679,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1867988,"kb_used_data":1288524,"kb_used_omap":11,"kb_used_meta":579444,"kb_avail":91877164,"statfs":{"total":95995035648,"available":94082215936,"internally_reserved":0,"allocated":1319448576,"data_stored":1317713394,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":593351031},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,0,3,2,2,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.8240000000000001,"15min":1.9079999999999999},"min":{"1min":0.22,"5min":0.20799999999999999,"15min":0.17199999999999999},"max":{"1min":11.946,"5min":36.656999999999996,"15min":36.656999999999996},"last":5.9829999999999997},{"interface":"front","average":{"1min":1.393,"5min":1.7410000000000001,"15min":1.8109999999999999},"min":{"1min":0.26000000000000001,"5min":0.23599999999999999,"15min":0.215},"max":{"1min":11.978,"5min":36.697000000000003,"15min":36.697000000000003},"last":11.045}]},{"osd":1,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.5549999999999999,"15min":1.839},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.215},"max":{"1min":12.106,"5min":12.106,"15min":20.995000000000001},"last":6.0259999999999998},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.2350000000000001,"15min":1.5449999999999999},"min":{"1min":0.28399999999999997,"5min":0.252,"15min":0.193},"max":{"1min":5.7300000000000004,"5min":8.3889999999999993,"15min":15.945},"last":8.8409999999999993}]},{"osd":3,"last update":"Sat Sep 23 18:09:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":1.764,"15min":1.1759999999999999},"min":{"1min":0.313,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5960000000000001,"5min":37.500999999999998,"15min":37.500999999999998},"last":3.5910000000000002},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.95,"15min":1.113},"min":{"1min":0.29199999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.6239999999999997,"5min":34.915999999999997,"15min":34.915999999999997},"last":3.1379999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.4239999999999999,"15min":1.417},"min":{"1min":0.31900000000000001,"5min":0.311,"15min":0.26500000000000001},"max":{"1min":5.0880000000000001,"5min":13.016999999999999,"15min":13.016999999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":1.7350000000000001,"5min":1.381,"15min":1.431},"min":{"1min":0.33300000000000002,"5min":0.28699999999999998,"15min":0.26700000000000002},"max":{"1min":12.084,"5min":12.084,"15min":20.920000000000002},"last":5.931}]},{"osd":5,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.2969999999999999,"15min":1.4219999999999999},"min":{"1min":0.33000000000000002,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":7.2389999999999999,"5min":8.1760000000000002,"15min":20.738},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.167,"5min":1.5069999999999999,"15min":1.534},"min":{"1min":0.33600000000000002,"5min":0.32900000000000001,"15min":0.26700000000000002},"max":{"1min":3.1499999999999999,"5min":12.103999999999999,"15min":15.496},"last":5.774}]},{"osd":6,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.3779999999999999,"15min":1.7509999999999999},"min":{"1min":0.38600000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1100000000000003,"5min":13.065,"15min":13.065},"last":0.30499999999999999},{"interface":"front","average":{"1min":1.897,"5min":1.526,"15min":1.9790000000000001},"min":{"1min":0.46700000000000003,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":12.032999999999999,"5min":12.032999999999999,"15min":20.893000000000001},"last":5.8920000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:09:36 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.3899999999999999,"15min":1.3879999999999999},"min":{"1min":0.42699999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.6449999999999996,"5min":7.4989999999999997,"15min":12.051},"last":3.5590000000000002},{"interface":"front","average":{"1min":2.0390000000000001,"5min":1.7370000000000001,"15min":1.048},"min":{"1min":0.48399999999999999,"5min":0.45100000000000001,"15min":0.309},"max":{"1min":12.058,"5min":12.058,"15min":12.058},"last":5.9589999999999996}]}]},{"osd":3,"up_from":237,"seq":1017907249246,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1834528,"kb_used_data":1531224,"kb_used_omap":3,"kb_used_meta":303292,"kb_avail":91910624,"statfs":{"total":95995035648,"available":94116478976,"internally_reserved":0,"allocated":1567973376,"data_stored":1566055370,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":310571752},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.008,"15min":0.89400000000000002},"min":{"1min":0.24399999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.5679999999999996,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":3.605},{"interface":"front","average":{"1min":1.004,"5min":1.532,"15min":1.1850000000000001},"min":{"1min":0.23799999999999999,"5min":0.22,"15min":0.20699999999999999},"max":{"1min":9.1999999999999993,"5min":14.885999999999999,"15min":14.885999999999999},"last":0.36199999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.887,"15min":1.29},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.245},"max":{"1min":14.241,"5min":21.931999999999999,"15min":21.931999999999999},"last":0.70199999999999996},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.3140000000000001,"15min":1.0449999999999999},"min":{"1min":0.29199999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.008,"5min":13.631,"15min":13.631},"last":4.3540000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.0329999999999999,"15min":0.90100000000000002},"min":{"1min":0.29099999999999998,"5min":0.247,"15min":0.22600000000000001},"max":{"1min":4.5899999999999999,"5min":8.2200000000000006,"15min":8.2200000000000006},"last":0.74099999999999999},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.669,"15min":1.333},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.0090000000000003,"5min":21.988,"15min":21.988},"last":0.75700000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.885,"15min":1.335},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.29299999999999998},"max":{"1min":7.5540000000000003,"5min":37.231000000000002,"15min":37.231000000000002},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.246,"5min":1.819,"15min":1.294},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":4.673,"5min":37.381,"15min":37.381},"last":0.53500000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.4570000000000001,"15min":1.3069999999999999},"min":{"1min":0.34699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":9.3170000000000002,"5min":12.743,"15min":12.743},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.1279999999999999,"15min":1.147},"min":{"1min":0.376,"5min":0.315,"15min":0.315},"max":{"1min":4.601,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.435}]},{"osd":6,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":2.3290000000000002,"15min":1.4930000000000001},"min":{"1min":0.41199999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.657,"5min":115.54000000000001,"15min":115.54000000000001},"last":0.498},{"interface":"front","average":{"1min":1.101,"5min":2.6560000000000001,"15min":1.835},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":7.5860000000000003,"5min":115.691,"15min":115.691},"last":0.39900000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":1.4950000000000001,"15min":1.512},"min":{"1min":0.501,"5min":0.39600000000000002,"15min":0.377},"max":{"1min":5.5179999999999998,"5min":12.821,"15min":12.821},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.405,"15min":1.512},"min":{"1min":0.442,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.6159999999999997,"5min":8.6039999999999992,"15min":8.6039999999999992},"last":0.47299999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019195,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1791108,"kb_used_data":1447036,"kb_used_omap":0,"kb_used_meta":344064,"kb_avail":91954044,"statfs":{"total":95995035648,"available":94160941056,"internally_reserved":0,"allocated":1481764864,"data_stored":1480508674,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":352321536},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.1060000000000001,"15min":1.0669999999999999},"min":{"1min":0.27600000000000002,"5min":0.26000000000000001,"15min":0.20100000000000001},"max":{"1min":6.5090000000000003,"5min":13.375999999999999,"15min":14.164},"last":0.23699999999999999},{"interface":"front","average":{"1min":1.369,"5min":1.222,"15min":1.3080000000000001},"min":{"1min":0.30099999999999999,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":7.5609999999999999,"5min":7.5609999999999999,"15min":11.372},"last":2.3759999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:09:15 2023","interfaces":[{"interface":"back","average":{"1min":1.29,"5min":1.298,"15min":1.26},"min":{"1min":0.25800000000000001,"5min":0.249,"15min":0.249},"max":{"1min":4.2949999999999999,"5min":7.6059999999999999,"15min":10.429},"last":2.2010000000000001},{"interface":"front","average":{"1min":1.7130000000000001,"5min":1.2669999999999999,"15min":1.3200000000000001},"min":{"1min":0.27600000000000002,"5min":0.27100000000000002,"15min":0.23100000000000001},"max":{"1min":5.5129999999999999,"5min":6.7640000000000002,"15min":10.83},"last":0.26800000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":1.431,"15min":1.4139999999999999},"min":{"1min":0.38200000000000001,"5min":0.28799999999999998,"15min":0.26600000000000001},"max":{"1min":32.887999999999998,"5min":32.887999999999998,"15min":32.887999999999998},"last":4.71},{"interface":"front","average":{"1min":2.661,"5min":1.712,"15min":1.444},"min":{"1min":0.34899999999999998,"5min":0.314,"15min":0.27100000000000002},"max":{"1min":29.103999999999999,"5min":29.103999999999999,"15min":29.103999999999999},"last":1.5580000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:09:28 2023","interfaces":[{"interface":"back","average":{"1min":3.141,"5min":2.2839999999999998,"15min":1.6240000000000001},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":28.754999999999999,"5min":29.154,"15min":29.154},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.645,"5min":2.0920000000000001,"15min":1.6000000000000001},"min":{"1min":0.36699999999999999,"5min":0.34699999999999998,"15min":0.34200000000000003},"max":{"1min":29.213000000000001,"5min":29.216000000000001,"15min":29.216000000000001},"last":5.0990000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:08:51 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.036,"15min":0.81799999999999995},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.5279999999999996,"5min":9.2959999999999994,"15min":9.9819999999999993},"last":4.6429999999999998},{"interface":"front","average":{"1min":1.2649999999999999,"5min":1.1299999999999999,"15min":0.98199999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":9.1609999999999996,"5min":9.1999999999999993,"15min":11.170999999999999},"last":0.94799999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":0.78000000000000003,"15min":0.91700000000000004},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":4.3410000000000002,"5min":5.8310000000000004,"15min":9.0990000000000002},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.84999999999999998,"15min":0.99199999999999999},"min":{"1min":0.41899999999999998,"5min":0.312,"15min":0.28399999999999997},"max":{"1min":3.4329999999999998,"5min":4.9740000000000002,"15min":10.725},"last":0.38900000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:09:40 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.159,"15min":1.0569999999999999},"min":{"1min":0.437,"5min":0.314,"15min":0.314},"max":{"1min":9.2509999999999994,"5min":13.298999999999999,"15min":13.298999999999999},"last":0.373},{"interface":"front","average":{"1min":1.345,"5min":1.0189999999999999,"15min":1.2030000000000001},"min":{"1min":0.40000000000000002,"5min":0.32400000000000001,"15min":0.307},"max":{"1min":3.7810000000000001,"5min":10.192,"15min":10.192},"last":0.51900000000000002}]}]},{"osd":5,"up_from":38,"seq":163208757559,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1802008,"kb_used_data":1477840,"kb_used_omap":0,"kb_used_meta":324160,"kb_avail":91943144,"statfs":{"total":95995035648,"available":94149779456,"internally_reserved":0,"allocated":1513308160,"data_stored":1512084138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331939840},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.77900000000000003,"15min":1.069},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.217},"max":{"1min":2.5649999999999999,"5min":5.4539999999999997,"15min":21.376999999999999},"last":1.3560000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.94799999999999995,"15min":1.3},"min":{"1min":0.249,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":3.5910000000000002,"5min":8.5099999999999998,"15min":21.388999999999999},"last":3.379}]},{"osd":1,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.038,"15min":0.96699999999999997},"min":{"1min":0.23899999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.6349999999999998,"5min":10.547000000000001,"15min":10.853999999999999},"last":1.385},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.1299999999999999,"15min":1.1759999999999999},"min":{"1min":0.26600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.625,"5min":8.9749999999999996,"15min":10.206},"last":3.4239999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.002,"15min":1.4119999999999999},"min":{"1min":0.38600000000000001,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":5.0570000000000004,"5min":5.5309999999999997,"15min":38.606999999999999},"last":1.962},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.0549999999999999,"15min":1.302},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6000000000000001,"5min":10.065,"15min":39.670999999999999},"last":1.335}]},{"osd":3,"last update":"Sat Sep 23 18:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":1.266,"15min":0.96099999999999997},"min":{"1min":0.32200000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6499999999999999,"5min":9.2309999999999999,"15min":9.2309999999999999},"last":5.194},{"interface":"front","average":{"1min":1.6930000000000001,"5min":1.3779999999999999,"15min":0.94699999999999995},"min":{"1min":0.35599999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.362,"5min":13.446999999999999,"15min":13.446999999999999},"last":1.988}]},{"osd":4,"last update":"Sat Sep 23 18:08:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.79800000000000004,"15min":0.83699999999999997},"min":{"1min":0.29799999999999999,"5min":0.25700000000000001,"15min":0.224},"max":{"1min":3.7149999999999999,"5min":8.6229999999999993,"15min":10.092000000000001},"last":6.8600000000000003},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.94699999999999995,"15min":0.83999999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.23999999999999999},"max":{"1min":9.1750000000000007,"5min":9.1750000000000007,"15min":9.1750000000000007},"last":2.1890000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:08:59 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":0.86299999999999999,"15min":1.1930000000000001},"min":{"1min":0.314,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":6.032,"5min":6.032,"15min":11.287000000000001},"last":1.9279999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.872,"15min":1.0880000000000001},"min":{"1min":0.33100000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.6709999999999998,"5min":6.6920000000000002,"15min":8.266},"last":1.3089999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:09:38 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":0.99299999999999999,"15min":1.302},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.1849999999999996,"5min":8.0999999999999996,"15min":8.8510000000000009},"last":2.7679999999999998},{"interface":"front","average":{"1min":1.752,"5min":1.022,"15min":1.099},"min":{"1min":0.46000000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.2400000000000002,"5min":17.684000000000001,"15min":17.684000000000001},"last":6.6449999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":39124992,"data_stored":39124992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1802829824,"data_stored":1802829824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1313341440,"data_stored":1313341440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1524498432,"data_stored":1524498432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1489108992,"data_stored":1489108992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1551040512,"data_stored":1551040512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1740177408,"data_stored":1740177408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1802371072,"data_stored":1802371072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:10:11.388 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pgp_num 16 2023-09-23T18:10:11.388 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:11.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:11.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:11.492 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:11.548 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:11 smithi055 ceph-mon[184174]: from='client.45980 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:10:11.564 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:11.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:11 smithi099 ceph-mon[167076]: from='client.45980 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:10:11.694 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:11.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:11.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:11.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:11.840 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:11.877 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:11.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:11.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:11.886 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:11.987 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:12.018 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 41 16 66904 66888 101.912 58.1875 0.016691 0.00980706 2023-09-23T18:10:12.078 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:12.153 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:12.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:12.162 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:12.262 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:12.296 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:12.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:12.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:12.339 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:12.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:12 smithi055 ceph-mon[179668]: pgmap v921: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 1.8 KiB/s rd, 54 MiB/s wr, 869 op/s; 38701/191610 objects degraded (20.198%); 7668/191610 objects misplaced (4.002%); 2.1 MiB/s, 25 objects/s recovering 2023-09-23T18:10:12.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:12 smithi055 ceph-mon[184174]: pgmap v921: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 1.8 KiB/s rd, 54 MiB/s wr, 869 op/s; 38701/191610 objects degraded (20.198%); 7668/191610 objects misplaced (4.002%); 2.1 MiB/s, 25 objects/s recovering 2023-09-23T18:10:12.439 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:12.504 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:12.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:12 smithi099 ceph-mon[167076]: pgmap v921: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 3.9 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 1.8 KiB/s rd, 54 MiB/s wr, 869 op/s; 38701/191610 objects degraded (20.198%); 7668/191610 objects misplaced (4.002%); 2.1 MiB/s, 25 objects/s recovering 2023-09-23T18:10:12.577 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:12.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:12.587 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:12.687 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:12.719 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:12.764 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:12.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:12.767 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:12.868 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:12.956 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:13.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 42 16 67803 67787 100.812 56.1875 0.0110925 0.00991443 2023-09-23T18:10:13.125 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:10:13.125 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:10:13.125 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:10:13.125 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:10:13.126 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:10:13.263 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:13.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:13.271 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:13.372 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:13.424 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:13.488 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:13.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 38701/191610 objects degraded (20.198%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:13 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1295842004' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:13.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:13 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:13.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 38701/191610 objects degraded (20.198%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:13.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:13 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1295842004' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:13.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:13 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:13.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 38701/191610 objects degraded (20.198%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:13.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:13 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1295842004' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:13.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:13 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:13.703 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:13.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:13.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:13.810 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:13.830 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:10:13.852 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:13.901 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:13.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:13.905 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:14.005 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:14.019 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 43 16 68820 68804 99.9429 63.5625 0.0199766 0.0100003 2023-09-23T18:10:14.124 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:14.142 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:10:14.236 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:14.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:14.247 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:14.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:14.383 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:14.398 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:10:14.422 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:14.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:14.425 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:14.454 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:10:14.488 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:10:14.526 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:14.530 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:10:14.561 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:14.574 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:10:14.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:14 smithi099 ceph-mon[167076]: pgmap v922: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 1.6 KiB/s rd, 59 MiB/s wr, 950 op/s; 39730/197913 objects degraded (20.074%); 7668/197913 objects misplaced (3.874%); 3.2 MiB/s, 42 objects/s recovering 2023-09-23T18:10:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:14 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:14 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:10:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:14 smithi099 ceph-mon[167076]: osdmap e376: 8 total, 7 up, 8 in 2023-09-23T18:10:14.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:14.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:14.614 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:14.619 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:10:14.671 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:10:14.671 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:10:14.672 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:10:14.672 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:10:14.672 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:10:14.672 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:10:14.672 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:10:14.672 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:10:14.672 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:10:14.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:14 smithi055 ceph-mon[179668]: pgmap v922: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 1.6 KiB/s rd, 59 MiB/s wr, 950 op/s; 39730/197913 objects degraded (20.074%); 7668/197913 objects misplaced (3.874%); 3.2 MiB/s, 42 objects/s recovering 2023-09-23T18:10:14.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:14 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:14.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:14 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:10:14.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:14 smithi055 ceph-mon[179668]: osdmap e376: 8 total, 7 up, 8 in 2023-09-23T18:10:14.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:14 smithi055 ceph-mon[184174]: pgmap v922: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.0 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 1.6 KiB/s rd, 59 MiB/s wr, 950 op/s; 39730/197913 objects degraded (20.074%); 7668/197913 objects misplaced (3.874%); 3.2 MiB/s, 42 objects/s recovering 2023-09-23T18:10:14.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:14 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:14.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:14 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:10:14.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:14 smithi055 ceph-mon[184174]: osdmap e376: 8 total, 7 up, 8 in 2023-09-23T18:10:14.715 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:14.751 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:14.802 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:14.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:14.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:14.908 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:14.985 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:15.020 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 44 16 69700 69684 98.9219 55 0.00695588 0.0101034 2023-09-23T18:10:15.252 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:15.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:15.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:15.363 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:15.402 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:15.448 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:15.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:15.457 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:15.511 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:10:15.512 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:15 smithi055 ceph-mon[179668]: failed to encode map e376 with expected crc 2023-09-23T18:10:15.512 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:15 smithi055 ceph-mon[179668]: failed to encode map e376 with expected crc 2023-09-23T18:10:15.512 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4138135735' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:15.512 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:15 smithi055 ceph-mon[184174]: failed to encode map e376 with expected crc 2023-09-23T18:10:15.512 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:15 smithi055 ceph-mon[184174]: failed to encode map e376 with expected crc 2023-09-23T18:10:15.512 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4138135735' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:15.513 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:10:15.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:15.567 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:15.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:15 smithi099 ceph-mon[167076]: failed to encode map e376 with expected crc 2023-09-23T18:10:15.598 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:15 smithi099 ceph-mon[167076]: failed to encode map e376 with expected crc 2023-09-23T18:10:15.599 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4138135735' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:15.599 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:15.644 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:15.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:15.651 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:15.751 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:15.830 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:16.013 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:16.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:16.019 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:16.020 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 45 16 70753 70737 98.1863 65.8125 0.00566252 0.010179 2023-09-23T18:10:16.120 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:16.207 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:16.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:16 smithi055 ceph-mon[184174]: pgmap v924: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 40332/200646 objects degraded (20.101%); 7668/200646 objects misplaced (3.822%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:10:16.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:16 smithi055 ceph-mon[179668]: pgmap v924: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 40332/200646 objects degraded (20.101%); 7668/200646 objects misplaced (3.822%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:10:16.779 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:10:16.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:16 smithi099 ceph-mon[167076]: pgmap v924: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.1 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 40332/200646 objects degraded (20.101%); 7668/200646 objects misplaced (3.822%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:10:17.020 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 46 16 71758 71742 97.4176 62.8125 0.00441915 0.0102581 2023-09-23T18:10:17.328 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:17.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:17.338 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:17.421 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 8 pgp_num to 16 2023-09-23T18:10:17.439 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:17.514 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:10:17.518 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:17.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/237227777' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:10:17.599 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:17 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:10:17.599 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/237227777' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:10:17.599 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:17 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:10:17.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/237227777' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:10:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:17 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:10:17.853 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:17.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:17.860 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:17.961 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:17.991 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:18.022 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 47 16 72683 72667 96.5709 57.8125 0.0209215 0.0103495 2023-09-23T18:10:18.032 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:18.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:18.036 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:18.137 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:18.211 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:18.281 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:18.336 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:18.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:18.346 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:18.417 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:10:18.417 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:10:18.447 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:18.518 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:18 smithi055 ceph-mon[179668]: pgmap v925: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 40942/206259 objects degraded (19.850%); 7980/206259 objects misplaced (3.869%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:10:18.519 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:18 smithi055 ceph-mon[179668]: failed to encode map e377 with expected crc 2023-09-23T18:10:18.519 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:18 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:10:18.519 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:18 smithi055 ceph-mon[179668]: osdmap e377: 8 total, 7 up, 8 in 2023-09-23T18:10:18.519 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 40942/206259 objects degraded (19.850%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:18.519 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:18 smithi055 ceph-mon[184174]: pgmap v925: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 40942/206259 objects degraded (19.850%); 7980/206259 objects misplaced (3.869%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:10:18.519 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:18 smithi055 ceph-mon[184174]: failed to encode map e377 with expected crc 2023-09-23T18:10:18.519 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:18 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:10:18.520 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:18 smithi055 ceph-mon[184174]: osdmap e377: 8 total, 7 up, 8 in 2023-09-23T18:10:18.520 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 40942/206259 objects degraded (19.850%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:18.525 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:18.824 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:18.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:18 smithi099 ceph-mon[167076]: pgmap v925: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.2 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 40942/206259 objects degraded (19.850%); 7980/206259 objects misplaced (3.869%); 2.2 MiB/s, 35 objects/s recovering 2023-09-23T18:10:18.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:18 smithi099 ceph-mon[167076]: failed to encode map e377 with expected crc 2023-09-23T18:10:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:18 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:10:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:18 smithi099 ceph-mon[167076]: osdmap e377: 8 total, 7 up, 8 in 2023-09-23T18:10:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 40942/206259 objects degraded (19.850%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:18.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:18.831 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:18.931 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:18.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:10:19.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 48 16 73604 73588 95.7552 57.5625 0.0147562 0.0104369 2023-09-23T18:10:19.064 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:19.302 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:19.303 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:19.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:19.310 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:19.410 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:19.450 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:19.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:19.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:19.521 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:19.622 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:19.664 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:10:19.718 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:19.790 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:19.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:19.806 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:19.907 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:20.014 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:20.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 49 16 74569 74553 95.0321 60.3125 0.0200233 0.0105159 2023-09-23T18:10:20.231 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:20.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:20.236 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:20.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:20.380 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:20.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:20.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:20.436 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:20.513 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:10:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:10:20] "GET /metrics HTTP/1.1" 200 36662 "" "Prometheus/2.43.0" 2023-09-23T18:10:20.514 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:20 smithi055 ceph-mon[184174]: pgmap v927: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 42615/215289 objects degraded (19.794%); 7980/215289 objects misplaced (3.707%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T18:10:20.514 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:20 smithi055 ceph-mon[179668]: pgmap v927: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 42615/215289 objects degraded (19.794%); 7980/215289 objects misplaced (3.707%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T18:10:20.537 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:20 smithi099 ceph-mon[167076]: pgmap v927: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.4 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 62 MiB/s wr, 985 op/s; 42615/215289 objects degraded (19.794%); 7980/215289 objects misplaced (3.707%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T18:10:20.609 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:20.679 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:20.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:20.685 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:20.786 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:20.829 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:20.882 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:20.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:20.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:20.969 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:10:20.969 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:10:20.987 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:21.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 50 16 75524 75508 94.3255 59.6875 0.0166383 0.0105953 2023-09-23T18:10:21.097 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:21.174 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:10:21.197 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:21.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:21.203 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:21.208 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:10:21.242 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:10:21.289 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:10:21.303 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:21.329 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:10:21.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:10:21.373 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:10:21.375 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:21.425 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:10:21.426 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:10:21.426 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:10:21.426 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:10:21.426 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:10:21.426 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:10:21.426 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:10:21.426 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:10:21.426 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:10:21.436 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:21.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:21.443 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:21.469 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:10:21.545 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:21.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4012044111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:21.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2700659017' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:21.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:21 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:21.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4012044111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:21.620 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2700659017' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:21.620 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:21 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:21.623 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:21.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4012044111' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:21.831 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2700659017' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:21.831 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:21 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:21.983 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:10:22.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 51 16 76472 76456 93.638 59.25 0.0207047 0.0106743 2023-09-23T18:10:22.078 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:22.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:22.096 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:22.145 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:10:22.145 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:10:22.196 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:22.260 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:22.307 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:22.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:22.315 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:22.415 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[179668]: pgmap v928: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 43220/220647 objects degraded (19.588%); 8270/220647 objects misplaced (3.748%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:10:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:10:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[179668]: failed to encode map e378 with expected crc 2023-09-23T18:10:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[179668]: failed to encode map e378 with expected crc 2023-09-23T18:10:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:22.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[179668]: osdmap e378: 8 total, 7 up, 8 in 2023-09-23T18:10:22.496 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:22.561 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:22 smithi099 ceph-mon[167076]: pgmap v928: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 43220/220647 objects degraded (19.588%); 8270/220647 objects misplaced (3.748%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:10:22.562 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:22 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:10:22.562 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:22.562 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:22 smithi099 ceph-mon[167076]: failed to encode map e378 with expected crc 2023-09-23T18:10:22.562 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:22 smithi099 ceph-mon[167076]: failed to encode map e378 with expected crc 2023-09-23T18:10:22.562 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:22 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:22.562 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:22 smithi099 ceph-mon[167076]: osdmap e378: 8 total, 7 up, 8 in 2023-09-23T18:10:22.577 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:22.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:22.584 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:22.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[184174]: pgmap v928: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 59 MiB/s wr, 946 op/s; 43220/220647 objects degraded (19.588%); 8270/220647 objects misplaced (3.748%); 1.5 MiB/s, 23 objects/s recovering 2023-09-23T18:10:22.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:10:22.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:22.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[184174]: failed to encode map e378 with expected crc 2023-09-23T18:10:22.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[184174]: failed to encode map e378 with expected crc 2023-09-23T18:10:22.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:22.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:22 smithi055 ceph-mon[184174]: osdmap e378: 8 total, 7 up, 8 in 2023-09-23T18:10:22.685 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:22.766 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:22.849 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:22.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:22.857 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:22.938 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 6, 7, 5, 4, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:10:22.938 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:10:22.938 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T18:10:22.938 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:10:22.958 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:23.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 52 16 77512 77496 93.0874 65 0.0126606 0.0107368 2023-09-23T18:10:23.070 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:23.166 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:23.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:23.186 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:23.286 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:23.339 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:23.380 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:10:23.380 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:10:23.380 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:10:23.380 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:10:23.381 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:10:23.402 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:23.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:23.415 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:23.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:23.557 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:23.774 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:23.777 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:23 smithi099 ceph-mon[167076]: pgmap v930: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 44215/226848 objects degraded (19.491%); 8270/226848 objects misplaced (3.646%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:23.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 43220/220647 objects degraded (19.588%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:23.778 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:23.779 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:23.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:10:23.839 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:23.881 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:23.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:23 smithi055 ceph-mon[184174]: pgmap v930: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 44215/226848 objects degraded (19.491%); 8270/226848 objects misplaced (3.646%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:23.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 43220/220647 objects degraded (19.588%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:23.942 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:23 smithi055 ceph-mon[179668]: pgmap v930: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.6 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 68 MiB/s wr, 1.09k op/s; 44215/226848 objects degraded (19.491%); 8270/226848 objects misplaced (3.646%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:23.942 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 43220/220647 objects degraded (19.588%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:23.957 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:24.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 53 16 78478 78462 92.4704 60.375 0.00374969 0.0108068 2023-09-23T18:10:24.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:10:24.147 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:10:24.489 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:10:25.024 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 54 16 79359 79343 91.7779 55.0625 0.0133924 0.0108896 2023-09-23T18:10:25.055 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:25.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:25.060 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:25.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:25.200 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:25.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:25.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:25.259 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:25.360 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:25.424 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:25.554 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:25.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:25.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:25.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:25.739 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:25.775 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:10:25.775 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:10:25.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:10:25.775 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:10:25.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.775 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33541 6.6 6:62c48c76:::benchmark_data_smithi055_275963_object33540:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 526.31699519599999, 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0751333839999999, 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33541 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452290+0000", 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452289+0000", 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452325+0000", 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5848e-05 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452332+0000", 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5870000000000004e-06 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.452337+0000", 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8189999999999999e-06 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453126+0000", 2023-09-23T18:10:25.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00078863000000000002 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.453133+0000", 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5630000000000003e-06 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454584+0000", 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0014505659999999 2023-09-23T18:10:25.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454710+0000", 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012631 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.454859+0000", 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014862400000000001 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.460661+0000", 2023-09-23T18:10:25.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058021690000000002 2023-09-23T18:10:25.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.461337+0000", 2023-09-23T18:10:25.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00067573600000000002 2023-09-23T18:10:25.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527285+0000", 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.065948602999999995 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527333+0000", 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7329e-05 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:40.527423+0000", 2023-09-23T18:10:25.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0160000000000004e-05 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:99011 6.e 6:75351f29:::benchmark_data_smithi055_275963_object25873:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e260)", 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:03:39.675417+0000", 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 406.09386736699997, 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1206789589999999, 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.784 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 99011 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675417+0000", 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675417+0000", 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675416+0000", 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675423+0000", 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9330000000000004e-06 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675436+0000", 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2831e-05 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.675441+0000", 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2240000000000003e-06 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.746546+0000", 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.071104629000000003 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.750257+0000", 2023-09-23T18:10:25.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037111919999999999 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.793006+0000", 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.042748844000000001 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for rw locks", 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:39.797725+0000", 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0047187669999999996 2023-09-23T18:10:25.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.741846+0000", 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94412087 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.745207+0000", 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003361298 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,5", 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.751189+0000", 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0059818609999999998 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.768299+0000", 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017110071000000001 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.785459+0000", 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017160150999999998 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.786644+0000", 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011847050000000001 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.788542+0000", 2023-09-23T18:10:25.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018984799999999999 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:03:40.796096+0000", 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075542559999999996 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.793 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65460 7.4 7:2d53859f:::benchmark_data_smithi055_311528_object65459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 219.87287937100001, 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9613000379999992, 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65460 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.794 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:10:25.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:10:25.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896404+0000", 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896620+0000", 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021651099999999999 2023-09-23T18:10:25.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896638+0000", 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8179999999999999e-05 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896646+0000", 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2899999999999997e-06 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896718+0000", 2023-09-23T18:10:25.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2587000000000001e-05 2023-09-23T18:10:25.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896729+0000", 2023-09-23T18:10:25.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1223000000000001e-05 2023-09-23T18:10:25.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.853134+0000", 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9564047319999993 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.857705+0000", 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045713259999999997 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.799 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65469 7.4 7:2eb46e41:::benchmark_data_smithi055_311528_object65468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 219.87261771999999, 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 10.043735116000001, 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.800 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65469 2023-09-23T18:10:25.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.801 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:10:25.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896666+0000", 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896742+0000", 2023-09-23T18:10:25.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5432000000000006e-05 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896755+0000", 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3275000000000001e-05 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896762+0000", 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4760000000000003e-06 2023-09-23T18:10:25.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896860+0000", 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8120000000000002e-05 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896869+0000", 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1409999999999994e-06 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.858317+0000", 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9614477719999996 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.863986+0000", 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0056689319999999998 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940290+0000", 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.076304636999999995 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940402+0000", 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111987 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65477 7.4 7:23aaada8:::benchmark_data_smithi055_311528_object65476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 219.872509799, 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 10.046314513, 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.807 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65477 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896774+0000", 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896805+0000", 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0916e-05 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896816+0000", 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0519999999999999e-05 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896820+0000", 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0849999999999999e-06 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896888+0000", 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8637e-05 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896897+0000", 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6279999999999994e-06 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.864614+0000", 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9677164650000005 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.871623+0000", 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070090059999999999 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.882195+0000", 2023-09-23T18:10:25.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010572560999999999 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.928966+0000", 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.046770763999999999 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940462+0000", 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011496312 2023-09-23T18:10:25.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940512+0000", 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0118999999999999e-05 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940552+0000", 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9535e-05 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943089+0000", 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025376470000000001 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65485 7.4 7:246beb72:::benchmark_data_smithi055_311528_object65484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:10:25.815 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 218.67311859200001, 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8502435909999999, 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65485 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:10:25.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:10:25.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096165+0000", 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096281+0000", 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116042 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096295+0000", 2023-09-23T18:10:25.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4524e-05 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096301+0000", 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7490000000000004e-06 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.889363+0000", 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7930615480000007 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.896004+0000", 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066411439999999999 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.906662+0000", 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010657645 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940400+0000", 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.033738490000000003 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943179+0000", 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002778675 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943222+0000", 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2966999999999997e-05 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943292+0000", 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0043999999999995e-05 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946410+0000", 2023-09-23T18:10:25.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031180190000000001 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65486 7.4 7:28b4ee06:::benchmark_data_smithi055_311528_object65485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 218.64786811299999, 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8402319929999997, 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.823 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65486 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121415+0000", 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121466+0000", 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1007000000000001e-05 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121481+0000", 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5287000000000001e-05 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121489+0000", 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5010000000000004e-06 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946571+0000", 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8250813830000006 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946747+0000", 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017653100000000001 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946947+0000", 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019961800000000001 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.954051+0000", 2023-09-23T18:10:25.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0071042989999999997 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958525+0000", 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044738959999999998 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958638+0000", 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113078 2023-09-23T18:10:25.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958698+0000", 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0207000000000001e-05 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961649+0000", 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002950214 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.830 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65487 7.4 7:2b3c8c2f:::benchmark_data_smithi055_311528_object65486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 218.64778404699999, 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8433001999999998, 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.831 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65487 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:10:25.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121500+0000", 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121563+0000", 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3126000000000005e-05 2023-09-23T18:10:25.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121579+0000", 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6418000000000001e-05 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121585+0000", 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.575e-06 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949470+0000", 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8278846659999992 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949632+0000", 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016232099999999999 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949768+0000", 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013578700000000001 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.956292+0000", 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065241910000000004 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958569+0000", 2023-09-23T18:10:25.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022767289999999999 2023-09-23T18:10:25.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961776+0000", 2023-09-23T18:10:25.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032076660000000001 2023-09-23T18:10:25.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961847+0000", 2023-09-23T18:10:25.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1137999999999996e-05 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964801+0000", 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029533990000000002 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.838 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65498 7.4 7:27d4ee95:::benchmark_data_smithi055_311528_object65497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 218.61327453300001, 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8116268140000003, 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65498 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156009+0000", 2023-09-23T18:10:25.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:25.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156082+0000", 2023-09-23T18:10:25.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3093e-05 2023-09-23T18:10:25.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156096+0000", 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4465e-05 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156104+0000", 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4120000000000004e-06 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.951850+0000", 2023-09-23T18:10:25.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7957462389999996 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.951998+0000", 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014826300000000001 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.952148+0000", 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000149524 2023-09-23T18:10:25.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961673+0000", 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095253839999999996 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961735+0000", 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2036e-05 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964887+0000", 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003152092 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964960+0000", 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2601999999999994e-05 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.967637+0000", 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026772739999999999 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108071 7.21 7:85ca81b0:::benchmark_data_smithi055_311528_object37108:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 106.224659952, 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9624768260000001, 2023-09-23T18:10:25.846 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108071 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:10:25.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544624+0000", 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544628+0000", 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.698e-06 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544634+0000", 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3849999999999999e-06 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544638+0000", 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7560000000000001e-06 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581968+0000", 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037330020999999998 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581975+0000", 2023-09-23T18:10:25.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1470000000000004e-06 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487605+0000", 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9056295090000002 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487615+0000", 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0036e-05 2023-09-23T18:10:25.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502365+0000", 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0147500899999999 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502491+0000", 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125716 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502677+0000", 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018593700000000001 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504929+0000", 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022522219999999999 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504942+0000", 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2636999999999999e-05 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506803+0000", 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018613709999999999 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506851+0000", 2023-09-23T18:10:25.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8578000000000001e-05 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507102+0000", 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025014900000000002 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.855 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108080 7.21 7:85d5fb8e:::benchmark_data_smithi055_311528_object37117:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 106.224628485, 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.962840377, 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108080 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.856 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544658+0000", 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9589999999999998e-06 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544663+0000", 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8069999999999999e-06 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544667+0000", 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1790000000000001e-06 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581992+0000", 2023-09-23T18:10:25.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037325061999999999 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581997+0000", 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5480000000000003e-06 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487637+0000", 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9056401429999998 2023-09-23T18:10:25.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487642+0000", 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.503810+0000", 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.01616774 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.503955+0000", 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000145291 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504156+0000", 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020058200000000001 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507170+0000", 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003013511 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507182+0000", 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.218e-05 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507234+0000", 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2349000000000003e-05 2023-09-23T18:10:25.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507288+0000", 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4307e-05 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507497+0000", 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000208233 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.864 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108094 7.1 7:819d9c99:::benchmark_data_smithi055_311528_object37131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 106.001269003, 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7316361810000001, 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.865 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108094 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768015+0000", 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768019+0000", 2023-09-23T18:10:25.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3989999999999997e-06 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768032+0000", 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2707e-05 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768037+0000", 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2490000000000001e-06 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768081+0000", 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3421000000000003e-05 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768093+0000", 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1989e-05 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.479295+0000", 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71120236800000003 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.479313+0000", 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8434e-05 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492409+0000", 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.013095748 2023-09-23T18:10:25.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492585+0000", 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000176007 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492874+0000", 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00028859600000000001 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.497854+0000", 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049804380000000002 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499314+0000", 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014593449999999999 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499349+0000", 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5802e-05 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499422+0000", 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2278000000000002e-05 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499652+0000", 2023-09-23T18:10:25.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00023028400000000001 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108097 7.21 7:87bce1cb:::benchmark_data_smithi055_311528_object37134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.27202801200001, 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0105963070000001, 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.874 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108097 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497255+0000", 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497262+0000", 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.029e-06 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497275+0000", 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3349e-05 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497284+0000", 2023-09-23T18:10:25.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3639999999999999e-06 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497329+0000", 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5105000000000003e-05 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497335+0000", 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8989999999999998e-06 2023-09-23T18:10:25.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487661+0000", 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99032663499999996 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487667+0000", 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.446e-06 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504977+0000", 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.017310079 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505119+0000", 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000142707 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505315+0000", 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019584499999999999 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507193+0000", 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001877213 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507527+0000", 2023-09-23T18:10:25.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00033465300000000002 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507614+0000", 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7287000000000006e-05 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507647+0000", 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2481999999999998e-05 2023-09-23T18:10:25.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507853+0000", 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000206092 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108108 7.21 7:860e36b6:::benchmark_data_smithi055_311528_object37145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:10:25.883 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.221035433, 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.963000675, 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108108 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548248+0000", 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548257+0000", 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1549999999999996e-06 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548278+0000", 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1229e-05 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548286+0000", 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0150000000000003e-06 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548337+0000", 2023-09-23T18:10:25.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1131999999999998e-05 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548346+0000", 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8070000000000006e-06 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487685+0000", 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.939339378 2023-09-23T18:10:25.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487691+0000", 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7100000000000004e-06 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505973+0000", 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018282141 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506088+0000", 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011443299999999999 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506199+0000", 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011132 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507536+0000", 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013369 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507578+0000", 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1971000000000002e-05 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.510980+0000", 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034023870000000002 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.511029+0000", 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9064999999999999e-05 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.511250+0000", 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022076600000000001 2023-09-23T18:10:25.892 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:24660 8.2 8:400557a5:::benchmark_data_smithi055_346788_object24659:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e366)", 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:40.362638+0000", 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 45.406646569999999, 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.103550143, 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.893 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.894 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:10:25.894 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:10:25.894 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 24660 2023-09-23T18:10:25.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.895 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.362638+0000", 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.362638+0000", 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.362637+0000", 2023-09-23T18:10:25.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.362744+0000", 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106267 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.362753+0000", 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3950000000000007e-06 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.362758+0000", 2023-09-23T18:10:25.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6589999999999998e-06 2023-09-23T18:10:25.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.362793+0000", 2023-09-23T18:10:25.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5655000000000003e-05 2023-09-23T18:10:25.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.362873+0000", 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9789000000000005e-05 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.362951+0000", 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7688000000000006e-05 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.364183+0000", 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012317560000000001 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.364804+0000", 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00062141900000000001 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.466095+0000", 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.101291282 2023-09-23T18:10:25.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.466150+0000", 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4790999999999998e-05 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:40.466188+0000", 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8316999999999999e-05 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.901 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:49959 8.b 8:dbef8ad1:::benchmark_data_smithi055_346788_object49958:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e369)", 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:09:55.457642+0000", 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 30.311642846000002, 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.092962488999999995, 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.902 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:10:25.903 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:10:25.903 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 49959 2023-09-23T18:10:25.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.903 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.457642+0000", 2023-09-23T18:10:25.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.457642+0000", 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.457641+0000", 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.457688+0000", 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6739000000000001e-05 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.457693+0000", 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3809999999999998e-06 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.457697+0000", 2023-09-23T18:10:25.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.304e-06 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.460820+0000", 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031229220000000002 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.460911+0000", 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1055000000000001e-05 2023-09-23T18:10:25.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.461004+0000", 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3467000000000005e-05 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.462177+0000", 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011725489999999999 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.462415+0000", 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00023796000000000001 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.550502+0000", 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.088087724000000006 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.550550+0000", 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7919e-05 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:09:55.550604+0000", 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4001e-05 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.909 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:78449 8.b 8:d8a38403:::benchmark_data_smithi055_346788_object78448:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:10:23.987954+0000", 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 1.7813307040000002, 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.117048414, 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 78449 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.910 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.987954+0000", 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.987954+0000", 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.987953+0000", 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.988023+0000", 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9720000000000003e-05 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.988037+0000", 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3976e-05 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.988042+0000", 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.101e-06 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.988092+0000", 2023-09-23T18:10:25.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.075e-05 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.988264+0000", 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000171714 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.988421+0000", 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015695600000000001 2023-09-23T18:10:25.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.991312+0000", 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028910730000000001 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:23.993979+0000", 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026667359999999998 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.104930+0000", 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.11095113199999999 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.104973+0000", 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2642999999999998e-05 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.105002+0000", 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9739000000000001e-05 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:78460 8.b 8:d86291f4:::benchmark_data_smithi055_346788_object78459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:10:24.000966+0000", 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 1.7683184619999999, 2023-09-23T18:10:25.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.115200594, 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 78460 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.000966+0000", 2023-09-23T18:10:25.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.000966+0000", 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.000965+0000", 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001023+0000", 2023-09-23T18:10:25.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7277000000000001e-05 2023-09-23T18:10:25.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001035+0000", 2023-09-23T18:10:25.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2408999999999999e-05 2023-09-23T18:10:25.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001041+0000", 2023-09-23T18:10:25.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2389999999999999e-06 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001088+0000", 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6987000000000003e-05 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001214+0000", 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012590999999999999 2023-09-23T18:10:25.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001363+0000", 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000148474 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.006597+0000", 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052342040000000001 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.006606+0000", 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8890000000000008e-06 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.116097+0000", 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.109491544 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.116137+0000", 2023-09-23T18:10:25.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9675000000000002e-05 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.116167+0000", 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9974000000000001e-05 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:78464 8.b 8:d89f6373:::benchmark_data_smithi055_346788_object78463:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T18:10:25.925 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:10:24.001358+0000", 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 1.767926383, 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.114900393, 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 78464 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001358+0000", 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001358+0000", 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001358+0000", 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001437+0000", 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9506999999999996e-05 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001447+0000", 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5060000000000001e-06 2023-09-23T18:10:25.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.001453+0000", 2023-09-23T18:10:25.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6139999999999997e-06 2023-09-23T18:10:25.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.004026+0000", 2023-09-23T18:10:25.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025720769999999999 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.004148+0000", 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012219800000000001 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.004266+0000", 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000118647 2023-09-23T18:10:25.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.006640+0000", 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002374101 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.006671+0000", 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0188999999999998e-05 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.116206+0000", 2023-09-23T18:10:25.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.10953497099999999 2023-09-23T18:10:25.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.116237+0000", 2023-09-23T18:10:25.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1201000000000002e-05 2023-09-23T18:10:25.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:24.116259+0000", 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1937e-05 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:79773 8.b 8:d48bf6ff:::benchmark_data_smithi055_346788_object79772:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e378)", 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:10:25.425600+0000", 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.343684608, 2023-09-23T18:10:25.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13671467000000001, 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 79773 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:25.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.425600+0000", 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.425600+0000", 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.425600+0000", 2023-09-23T18:10:25.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.425655+0000", 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5631999999999999e-05 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.425663+0000", 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1149999999999994e-06 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.425668+0000", 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8930000000000004e-06 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.425718+0000", 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9416000000000003e-05 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:25.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.425829+0000", 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111673 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,7", 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.425970+0000", 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014019100000000001 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.428988+0000", 2023-09-23T18:10:25.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030186890000000002 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.431356+0000", 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002367882 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.562230+0000", 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.130873718 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.562277+0000", 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7237999999999997e-05 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:25.562315+0000", 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7731000000000002e-05 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.940 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:25.941 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:25.941 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:10:25.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:25.942 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:25.942 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:26.029 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 55 16 80297 80281 91.1677 58.625 0.0221482 0.0109628 2023-09-23T18:10:26.043 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:25 smithi099 ceph-mon[167076]: pgmap v931: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 44850/229692 objects degraded (19.526%); 8270/229692 objects misplaced (3.600%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:26.087 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:25 smithi055 ceph-mon[179668]: pgmap v931: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 44850/229692 objects degraded (19.526%); 8270/229692 objects misplaced (3.600%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:26.087 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:25 smithi055 ceph-mon[184174]: pgmap v931: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.7 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 976 op/s; 44850/229692 objects degraded (19.526%); 8270/229692 objects misplaced (3.600%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:26.118 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:26.182 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:10:26.240 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:26.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:26.246 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:26.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:26.388 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:26.398 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:26.452 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:26.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:26.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:26.556 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:26.651 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:26.684 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:26.760 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:26.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:26.770 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:26.871 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:26.890 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:10:26.924 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:27.029 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 56 16 81176 81160 90.521 54.9375 0.0105839 0.0110396 2023-09-23T18:10:27.041 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:27.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:27.047 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:27.147 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:27.266 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:27.615 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:27.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:27.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:27.722 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:27.799 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:28.031 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 57 16 82074 82058 89.9153 56.125 0.0143519 0.0111167 2023-09-23T18:10:28.162 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:28.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:28.179 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:28.254 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:27 smithi055 ceph-mon[184174]: pgmap v932: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s; 45397/234972 objects degraded (19.320%); 8585/234972 objects misplaced (3.654%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T18:10:28.254 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:28.255 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:27 smithi055 ceph-mon[179668]: pgmap v932: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s; 45397/234972 objects degraded (19.320%); 8585/234972 objects misplaced (3.654%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T18:10:28.255 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:28.280 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:27 smithi099 ceph-mon[167076]: pgmap v932: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 4.8 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 61 MiB/s wr, 975 op/s; 45397/234972 objects degraded (19.320%); 8585/234972 objects misplaced (3.654%); 2.6 MiB/s, 40 objects/s recovering 2023-09-23T18:10:28.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:10:28.339 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:28.365 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:10:28.366 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:10:28.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:28.366 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45923.0:82337 8.6 e378/371 8:68f9b867:::benchmark_data_smithi055_346788_object82336:head v 378'5212, mlcod=0'0)", 2023-09-23T18:10:28.366 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:10:28.364029+0000", 2023-09-23T18:10:28.366 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0010371989999999999, 2023-09-23T18:10:28.366 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010677200000000001, 2023-09-23T18:10:28.366 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:28.366 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:10:28.366 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:28.366 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:28.364029+0000", 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:28.364029+0000", 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:28.367 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:28.364029+0000", 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:28.364094+0000", 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5426999999999995e-05 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:28.364110+0000", 2023-09-23T18:10:28.368 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5447999999999998e-05 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:28.364116+0000", 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8190000000000003e-06 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:28.364176+0000", 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9160000000000003e-05 2023-09-23T18:10:28.369 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:28.370 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:28.370 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:28.370 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:28.364199+0000", 2023-09-23T18:10:28.370 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3098000000000001e-05 2023-09-23T18:10:28.370 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:28.370 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:28.370 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:28.370 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:28.370 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:10:28.370 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 1 2023-09-23T18:10:28.371 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:10:28.375 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:10:28.377 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:10:28.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:28.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:28.408 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:28.509 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:28.587 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:28.837 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":932,"stamp":"2023-09-23T18:10:26.888141+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5135574886,"num_objects":78324,"num_object_clones":0,"num_object_copies":234972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":45397,"num_objects_misplaced":8585,"num_objects_unfound":0,"num_objects_dirty":78324,"num_whiteouts":0,"num_read":6037,"num_read_kb":12306,"num_write":79246,"num_write_kb":5024154,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":968,"num_bytes_recovered":60532352,"num_keys_recovered":20,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82519,"ondisk_log_size":82519,"up":306,"acting":311,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":374,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":16936136,"kb_used_data":13668552,"kb_used_omap":50,"kb_used_meta":3267469,"kb_avail":733025080,"statfs":{"total":767960285184,"available":750617681920,"internally_reserved":0,"allocated":13996597248,"data_stored":13984387841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":51828,"internal_metadata":3345888652},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,4,2,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":607256576,"num_objects":9266,"num_object_clones":0,"num_object_copies":27798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4150,"num_objects_misplaced":605,"num_objects_unfound":0,"num_objects_dirty":9266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9266,"num_write_kb":593024,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":388,"num_bytes_recovered":25427968,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.502040"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":616,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.401620+0000","last_change":"2023-09-23T18:09:49.260251+0000","last_active":"2023-09-23T18:10:17.401620+0000","last_peered":"2023-09-23T18:10:17.401620+0000","last_clean":"2023-09-23T18:05:06.478917+0000","last_became_active":"2023-09-23T18:09:49.260251+0000","last_became_peered":"2023-09-23T18:09:49.260251+0000","last_unstale":"2023-09-23T18:10:17.401620+0000","last_undegraded":"2023-09-23T18:10:17.401620+0000","last_fullsized":"2023-09-23T18:09:49.251327+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.712627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.712627+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":737225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":188,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399480+0000","last_change":"2023-09-23T18:06:05.377123+0000","last_active":"2023-09-23T18:10:17.399480+0000","last_peered":"2023-09-23T18:10:17.399480+0000","last_clean":"2023-09-23T18:10:17.399480+0000","last_became_active":"2023-09-23T18:06:05.376735+0000","last_became_peered":"2023-09-23T18:06:05.376735+0000","last_unstale":"2023-09-23T18:10:17.399480+0000","last_undegraded":"2023-09-23T18:10:17.399480+0000","last_fullsized":"2023-09-23T18:10:17.399480+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:11.555818+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":255,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.398934+0000","last_change":"2023-09-23T18:06:51.071290+0000","last_active":"2023-09-23T18:10:17.398934+0000","last_peered":"2023-09-23T18:10:17.398934+0000","last_clean":"2023-09-23T18:10:17.398934+0000","last_became_active":"2023-09-23T18:06:51.070948+0000","last_became_peered":"2023-09-23T18:06:51.070948+0000","last_unstale":"2023-09-23T18:10:17.398934+0000","last_undegraded":"2023-09-23T18:10:17.398934+0000","last_fullsized":"2023-09-23T18:10:17.398934+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:52.647972+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":609,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.398596+0000","last_change":"2023-09-23T18:09:49.244257+0000","last_active":"2023-09-23T18:10:17.398596+0000","last_peered":"2023-09-23T18:10:17.398596+0000","last_clean":"2023-09-23T18:02:45.965198+0000","last_became_active":"2023-09-23T18:09:49.244257+0000","last_became_peered":"2023-09-23T18:09:49.244257+0000","last_unstale":"2023-09-23T18:10:17.398596+0000","last_undegraded":"2023-09-23T18:10:17.398596+0000","last_fullsized":"2023-09-23T18:09:49.242161+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:34.927458+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2],"acting":[4,2],"avail_no_missing":["4","2"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":94,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.396350+0000","last_change":"2023-09-23T18:06:05.370046+0000","last_active":"2023-09-23T18:10:17.396350+0000","last_peered":"2023-09-23T18:10:17.396350+0000","last_clean":"2023-09-23T18:10:17.396350+0000","last_became_active":"2023-09-23T18:06:05.369714+0000","last_became_peered":"2023-09-23T18:06:05.369714+0000","last_unstale":"2023-09-23T18:10:17.396350+0000","last_undegraded":"2023-09-23T18:10:17.396350+0000","last_fullsized":"2023-09-23T18:10:17.396350+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:47.453671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":394,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.401516+0000","last_change":"2023-09-23T18:09:49.264642+0000","last_active":"2023-09-23T18:10:17.401516+0000","last_peered":"2023-09-23T18:10:17.401516+0000","last_clean":"2023-09-23T18:09:40.155891+0000","last_became_active":"2023-09-23T18:09:49.264642+0000","last_became_peered":"2023-09-23T18:09:49.264642+0000","last_unstale":"2023-09-23T18:10:17.401516+0000","last_undegraded":"2023-09-23T18:10:17.401516+0000","last_fullsized":"2023-09-23T18:09:49.251516+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:29.396482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":715,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.401677+0000","last_change":"2023-09-23T18:06:08.439343+0000","last_active":"2023-09-23T18:10:17.401677+0000","last_peered":"2023-09-23T18:10:17.401677+0000","last_clean":"2023-09-23T18:10:17.401677+0000","last_became_active":"2023-09-23T18:06:08.437702+0000","last_became_peered":"2023-09-23T18:06:08.437702+0000","last_unstale":"2023-09-23T18:10:17.401677+0000","last_undegraded":"2023-09-23T18:10:17.401677+0000","last_fullsized":"2023-09-23T18:10:17.401677+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:02.398105+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":258,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.395713+0000","last_change":"2023-09-23T18:06:51.087219+0000","last_active":"2023-09-23T18:10:17.395713+0000","last_peered":"2023-09-23T18:10:17.395713+0000","last_clean":"2023-09-23T18:10:17.395713+0000","last_became_active":"2023-09-23T18:06:51.086885+0000","last_became_peered":"2023-09-23T18:06:51.086885+0000","last_unstale":"2023-09-23T18:10:17.395713+0000","last_undegraded":"2023-09-23T18:10:17.395713+0000","last_fullsized":"2023-09-23T18:10:17.395713+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:16.254186+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":335,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.393244+0000","last_change":"2023-09-23T18:09:49.249873+0000","last_active":"2023-09-23T18:10:17.393244+0000","last_peered":"2023-09-23T18:10:17.393244+0000","last_clean":"2023-09-23T18:09:40.149572+0000","last_became_active":"2023-09-23T18:09:49.249873+0000","last_became_peered":"2023-09-23T18:09:49.249873+0000","last_unstale":"2023-09-23T18:10:17.393244+0000","last_undegraded":"2023-09-23T18:10:17.393244+0000","last_fullsized":"2023-09-23T18:09:49.240762+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:52.086856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":765,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.401372+0000","last_change":"2023-09-23T18:09:49.267065+0000","last_active":"2023-09-23T18:10:17.401372+0000","last_peered":"2023-09-23T18:10:17.401372+0000","last_clean":"2023-09-23T18:09:40.155011+0000","last_became_active":"2023-09-23T18:09:49.267065+0000","last_became_peered":"2023-09-23T18:09:49.267065+0000","last_unstale":"2023-09-23T18:10:17.401372+0000","last_undegraded":"2023-09-23T18:10:17.401372+0000","last_fullsized":"2023-09-23T18:09:49.257376+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:52.249937+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":19,"reported_epoch":377,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:17.412775+0000","last_change":"2023-09-23T18:10:02.191390+0000","last_active":"2023-09-23T18:10:17.412775+0000","last_peered":"2023-09-23T18:10:17.412775+0000","last_clean":"2023-09-23T18:10:17.412775+0000","last_became_active":"2023-09-23T18:10:02.190942+0000","last_became_peered":"2023-09-23T18:10:02.190942+0000","last_unstale":"2023-09-23T18:10:17.412775+0000","last_undegraded":"2023-09-23T18:10:17.412775+0000","last_fullsized":"2023-09-23T18:10:17.412775+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:45.670277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6],"acting":[0,6,1],"avail_no_missing":["0","1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":246,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.414731+0000","last_change":"2023-09-23T18:06:05.364021+0000","last_active":"2023-09-23T18:10:17.414731+0000","last_peered":"2023-09-23T18:10:17.414731+0000","last_clean":"2023-09-23T18:10:17.414731+0000","last_became_active":"2023-09-23T18:06:05.363740+0000","last_became_peered":"2023-09-23T18:06:05.363740+0000","last_unstale":"2023-09-23T18:10:17.414731+0000","last_undegraded":"2023-09-23T18:10:17.414731+0000","last_fullsized":"2023-09-23T18:10:17.414731+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:55.662682+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":666,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.400838+0000","last_change":"2023-09-23T18:09:23.589835+0000","last_active":"2023-09-23T18:10:17.400838+0000","last_peered":"2023-09-23T18:10:17.400838+0000","last_clean":"2023-09-23T18:10:17.400838+0000","last_became_active":"2023-09-23T18:08:10.517840+0000","last_became_peered":"2023-09-23T18:08:10.517840+0000","last_unstale":"2023-09-23T18:10:17.400838+0000","last_undegraded":"2023-09-23T18:10:17.400838+0000","last_fullsized":"2023-09-23T18:10:17.400838+0000","mapping_epoch":331,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":332,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:26.034507+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":289,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.401848+0000","last_change":"2023-09-23T18:09:49.264696+0000","last_active":"2023-09-23T18:10:17.401848+0000","last_peered":"2023-09-23T18:10:17.401848+0000","last_clean":"2023-09-23T18:05:59.285835+0000","last_became_active":"2023-09-23T18:09:49.264696+0000","last_became_peered":"2023-09-23T18:09:49.264696+0000","last_unstale":"2023-09-23T18:10:17.401848+0000","last_undegraded":"2023-09-23T18:10:17.401848+0000","last_fullsized":"2023-09-23T18:09:49.251195+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:32.680036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":163,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.392642+0000","last_change":"2023-09-23T18:07:49.223713+0000","last_active":"2023-09-23T18:10:17.392642+0000","last_peered":"2023-09-23T18:10:17.392642+0000","last_clean":"2023-09-23T18:10:17.392642+0000","last_became_active":"2023-09-23T18:07:49.223409+0000","last_became_peered":"2023-09-23T18:07:49.223409+0000","last_unstale":"2023-09-23T18:10:17.392642+0000","last_undegraded":"2023-09-23T18:10:17.392642+0000","last_fullsized":"2023-09-23T18:10:17.392642+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:12.387318+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":433,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.393346+0000","last_change":"2023-09-23T18:09:49.254157+0000","last_active":"2023-09-23T18:10:17.393346+0000","last_peered":"2023-09-23T18:10:17.393346+0000","last_clean":"2023-09-23T17:49:06.430604+0000","last_became_active":"2023-09-23T18:09:49.254157+0000","last_became_peered":"2023-09-23T18:09:49.254157+0000","last_unstale":"2023-09-23T18:10:17.393346+0000","last_undegraded":"2023-09-23T18:10:17.393346+0000","last_fullsized":"2023-09-23T18:09:49.240876+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.652623+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.652623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":143,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.401306+0000","last_change":"2023-09-23T18:02:47.962597+0000","last_active":"2023-09-23T18:10:17.401306+0000","last_peered":"2023-09-23T18:10:17.401306+0000","last_clean":"2023-09-23T18:10:17.401306+0000","last_became_active":"2023-09-23T18:02:47.952953+0000","last_became_peered":"2023-09-23T18:02:47.952953+0000","last_unstale":"2023-09-23T18:10:17.401306+0000","last_undegraded":"2023-09-23T18:10:17.401306+0000","last_fullsized":"2023-09-23T18:10:17.401306+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:09.331039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":270,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":104,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.400848+0000","last_change":"2023-09-23T18:06:05.363236+0000","last_active":"2023-09-23T18:10:17.400848+0000","last_peered":"2023-09-23T18:10:17.400848+0000","last_clean":"2023-09-23T18:10:17.400848+0000","last_became_active":"2023-09-23T18:06:05.362947+0000","last_became_peered":"2023-09-23T18:06:05.362947+0000","last_unstale":"2023-09-23T18:10:17.400848+0000","last_undegraded":"2023-09-23T18:10:17.400848+0000","last_fullsized":"2023-09-23T18:10:17.400848+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:32.816159+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":778,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399537+0000","last_change":"2023-09-23T18:10:04.160097+0000","last_active":"2023-09-23T18:10:17.399537+0000","last_peered":"2023-09-23T18:10:17.399537+0000","last_clean":"2023-09-23T18:10:17.399537+0000","last_became_active":"2023-09-23T18:10:04.159754+0000","last_became_peered":"2023-09-23T18:10:04.159754+0000","last_unstale":"2023-09-23T18:10:17.399537+0000","last_undegraded":"2023-09-23T18:10:17.399537+0000","last_fullsized":"2023-09-23T18:10:17.399537+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:09:59.467151+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":170,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399671+0000","last_change":"2023-09-23T18:01:39.481237+0000","last_active":"2023-09-23T18:10:17.399671+0000","last_peered":"2023-09-23T18:10:17.399671+0000","last_clean":"2023-09-23T18:10:17.399671+0000","last_became_active":"2023-09-23T18:01:39.458107+0000","last_became_peered":"2023-09-23T18:01:39.458107+0000","last_unstale":"2023-09-23T18:10:17.399671+0000","last_undegraded":"2023-09-23T18:10:17.399671+0000","last_fullsized":"2023-09-23T18:10:17.399671+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:05.108857+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":188,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.398998+0000","last_change":"2023-09-23T18:06:05.375073+0000","last_active":"2023-09-23T18:10:17.398998+0000","last_peered":"2023-09-23T18:10:17.398998+0000","last_clean":"2023-09-23T18:10:17.398998+0000","last_became_active":"2023-09-23T18:06:05.374752+0000","last_became_peered":"2023-09-23T18:06:05.374752+0000","last_unstale":"2023-09-23T18:10:17.398998+0000","last_undegraded":"2023-09-23T18:10:17.398998+0000","last_fullsized":"2023-09-23T18:10:17.398998+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:09:06.839328+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":613,"reported_epoch":377,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:17.396155+0000","last_change":"2023-09-23T18:09:49.245471+0000","last_active":"2023-09-23T18:10:17.396155+0000","last_peered":"2023-09-23T18:10:17.396155+0000","last_clean":"2023-09-23T18:02:45.964322+0000","last_became_active":"2023-09-23T18:09:49.245471+0000","last_became_peered":"2023-09-23T18:09:49.245471+0000","last_unstale":"2023-09-23T18:10:17.396155+0000","last_undegraded":"2023-09-23T18:09:49.238416+0000","last_fullsized":"2023-09-23T18:09:49.238297+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.869576+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":306,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.400795+0000","last_change":"2023-09-23T18:01:39.470540+0000","last_active":"2023-09-23T18:10:17.400795+0000","last_peered":"2023-09-23T18:10:17.400795+0000","last_clean":"2023-09-23T18:10:17.400795+0000","last_became_active":"2023-09-23T18:01:39.454288+0000","last_became_peered":"2023-09-23T18:01:39.454288+0000","last_unstale":"2023-09-23T18:10:17.400795+0000","last_undegraded":"2023-09-23T18:10:17.400795+0000","last_fullsized":"2023-09-23T18:10:17.400795+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:03.945540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":68,"num_keys_recovered":20,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":194,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.398509+0000","last_change":"2023-09-23T18:07:49.229808+0000","last_active":"2023-09-23T18:10:17.398509+0000","last_peered":"2023-09-23T18:10:17.398509+0000","last_clean":"2023-09-23T18:10:17.398509+0000","last_became_active":"2023-09-23T18:07:49.229526+0000","last_became_peered":"2023-09-23T18:07:49.229526+0000","last_unstale":"2023-09-23T18:10:17.398509+0000","last_undegraded":"2023-09-23T18:10:17.398509+0000","last_fullsized":"2023-09-23T18:10:17.398509+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:46:01.224432+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":625,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.401461+0000","last_change":"2023-09-23T18:09:49.260631+0000","last_active":"2023-09-23T18:10:17.401461+0000","last_peered":"2023-09-23T18:10:17.401461+0000","last_clean":"2023-09-23T18:05:06.478813+0000","last_became_active":"2023-09-23T18:09:49.260631+0000","last_became_peered":"2023-09-23T18:09:49.260631+0000","last_unstale":"2023-09-23T18:10:17.401461+0000","last_undegraded":"2023-09-23T18:10:17.401461+0000","last_fullsized":"2023-09-23T18:09:49.251470+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:23.252461+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":385,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.400931+0000","last_change":"2023-09-23T18:06:51.065050+0000","last_active":"2023-09-23T18:10:17.400931+0000","last_peered":"2023-09-23T18:10:17.400931+0000","last_clean":"2023-09-23T18:10:17.400931+0000","last_became_active":"2023-09-23T18:06:51.064774+0000","last_became_peered":"2023-09-23T18:06:51.064774+0000","last_unstale":"2023-09-23T18:10:17.400931+0000","last_undegraded":"2023-09-23T18:10:17.400931+0000","last_fullsized":"2023-09-23T18:10:17.400931+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:44:31.508499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":227,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.401589+0000","last_change":"2023-09-23T18:09:49.260925+0000","last_active":"2023-09-23T18:10:17.401589+0000","last_peered":"2023-09-23T18:10:17.401589+0000","last_clean":"2023-09-23T18:02:16.054597+0000","last_became_active":"2023-09-23T18:09:49.260925+0000","last_became_peered":"2023-09-23T18:09:49.260925+0000","last_unstale":"2023-09-23T18:10:17.401589+0000","last_undegraded":"2023-09-23T18:10:17.401589+0000","last_fullsized":"2023-09-23T18:09:49.256792+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:59:07.524952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":741,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.402056+0000","last_change":"2023-09-23T18:10:04.171274+0000","last_active":"2023-09-23T18:10:17.402056+0000","last_peered":"2023-09-23T18:10:17.402056+0000","last_clean":"2023-09-23T18:10:17.402056+0000","last_became_active":"2023-09-23T18:10:04.166434+0000","last_became_peered":"2023-09-23T18:10:04.166434+0000","last_unstale":"2023-09-23T18:10:17.402056+0000","last_undegraded":"2023-09-23T18:10:17.402056+0000","last_fullsized":"2023-09-23T18:10:17.402056+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:30.091293+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":168,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.400314+0000","last_change":"2023-09-23T18:01:39.461311+0000","last_active":"2023-09-23T18:10:17.400314+0000","last_peered":"2023-09-23T18:10:17.400314+0000","last_clean":"2023-09-23T18:10:17.400314+0000","last_became_active":"2023-09-23T18:01:39.459608+0000","last_became_peered":"2023-09-23T18:01:39.459608+0000","last_unstale":"2023-09-23T18:10:17.400314+0000","last_undegraded":"2023-09-23T18:10:17.400314+0000","last_fullsized":"2023-09-23T18:10:17.400314+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:55:32.254373+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":317,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.415013+0000","last_change":"2023-09-23T18:06:51.125883+0000","last_active":"2023-09-23T18:10:17.415013+0000","last_peered":"2023-09-23T18:10:17.415013+0000","last_clean":"2023-09-23T18:10:17.415013+0000","last_became_active":"2023-09-23T18:06:51.119298+0000","last_became_peered":"2023-09-23T18:06:51.119298+0000","last_unstale":"2023-09-23T18:10:17.415013+0000","last_undegraded":"2023-09-23T18:10:17.415013+0000","last_fullsized":"2023-09-23T18:10:17.415013+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:12.571367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":626,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.400665+0000","last_change":"2023-09-23T18:09:49.248584+0000","last_active":"2023-09-23T18:10:17.400665+0000","last_peered":"2023-09-23T18:10:17.400665+0000","last_clean":"2023-09-23T18:05:59.276387+0000","last_became_active":"2023-09-23T18:09:49.248584+0000","last_became_peered":"2023-09-23T18:09:49.248584+0000","last_unstale":"2023-09-23T18:10:17.400665+0000","last_undegraded":"2023-09-23T18:10:17.400665+0000","last_fullsized":"2023-09-23T18:09:49.239167+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:29.998743+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":361,"reported_epoch":377,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:17.392544+0000","last_change":"2023-09-23T18:09:49.247743+0000","last_active":"2023-09-23T18:10:17.392544+0000","last_peered":"2023-09-23T18:10:17.392544+0000","last_clean":"2023-09-23T18:09:40.148528+0000","last_became_active":"2023-09-23T18:09:49.247743+0000","last_became_peered":"2023-09-23T18:09:49.247743+0000","last_unstale":"2023-09-23T18:10:17.392544+0000","last_undegraded":"2023-09-23T18:09:49.239512+0000","last_fullsized":"2023-09-23T18:09:49.239401+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:54:53.674393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[{"shards":"2,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":232,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399436+0000","last_change":"2023-09-23T18:06:45.919283+0000","last_active":"2023-09-23T18:10:17.399436+0000","last_peered":"2023-09-23T18:10:17.399436+0000","last_clean":"2023-09-23T18:10:17.399436+0000","last_became_active":"2023-09-23T18:06:45.918219+0000","last_became_peered":"2023-09-23T18:06:45.918219+0000","last_unstale":"2023-09-23T18:10:17.399436+0000","last_undegraded":"2023-09-23T18:10:17.399436+0000","last_fullsized":"2023-09-23T18:10:17.399436+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:21.242033+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.4","version":"378'4924","reported_seq":4954,"reported_epoch":378,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:25.825780+0000","last_change":"2023-09-23T18:09:49.246955+0000","last_active":"2023-09-23T18:10:25.825780+0000","last_peered":"2023-09-23T18:10:25.825780+0000","last_clean":"2023-09-23T18:09:48.184212+0000","last_became_active":"2023-09-23T18:09:49.246955+0000","last_became_peered":"2023-09-23T18:09:49.246955+0000","last_unstale":"2023-09-23T18:10:25.825780+0000","last_undegraded":"2023-09-23T18:09:49.243456+0000","last_fullsized":"2023-09-23T18:09:49.243334+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4924,"log_dups_size":0,"ondisk_log_size":4924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:49:44.207061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322699264,"num_objects":4924,"num_object_clones":0,"num_object_copies":14772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4924,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4924,"num_write_kb":315136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":4924}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":758,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.401096+0000","last_change":"2023-09-23T18:06:08.440949+0000","last_active":"2023-09-23T18:10:17.401096+0000","last_peered":"2023-09-23T18:10:17.401096+0000","last_clean":"2023-09-23T18:10:17.401096+0000","last_became_active":"2023-09-23T18:06:08.440543+0000","last_became_peered":"2023-09-23T18:06:08.440543+0000","last_unstale":"2023-09-23T18:10:17.401096+0000","last_undegraded":"2023-09-23T18:10:17.401096+0000","last_fullsized":"2023-09-23T18:10:17.401096+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:39:30.193370+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":320,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.401225+0000","last_change":"2023-09-23T18:06:45.947590+0000","last_active":"2023-09-23T18:10:17.401225+0000","last_peered":"2023-09-23T18:10:17.401225+0000","last_clean":"2023-09-23T18:10:17.401225+0000","last_became_active":"2023-09-23T18:06:45.920061+0000","last_became_peered":"2023-09-23T18:06:45.920061+0000","last_unstale":"2023-09-23T18:10:17.401225+0000","last_undegraded":"2023-09-23T18:10:17.401225+0000","last_fullsized":"2023-09-23T18:10:17.401225+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:31.645928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":249,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.401317+0000","last_change":"2023-09-23T18:09:49.260845+0000","last_active":"2023-09-23T18:10:17.401317+0000","last_peered":"2023-09-23T18:10:17.401317+0000","last_clean":"2023-09-23T18:02:45.953891+0000","last_became_active":"2023-09-23T18:09:49.260845+0000","last_became_peered":"2023-09-23T18:09:49.260845+0000","last_unstale":"2023-09-23T18:10:17.401317+0000","last_undegraded":"2023-09-23T18:10:17.401317+0000","last_fullsized":"2023-09-23T18:09:49.252007+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:35.249760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"378'4936","reported_seq":4961,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T18:10:25.622039+0000","last_change":"2023-09-23T18:09:26.035691+0000","last_active":"2023-09-23T18:10:25.622039+0000","last_peered":"2023-09-23T18:10:25.622039+0000","last_clean":"2023-09-23T18:10:25.622039+0000","last_became_active":"2023-09-23T18:09:26.035411+0000","last_became_peered":"2023-09-23T18:09:26.035411+0000","last_unstale":"2023-09-23T18:10:25.622039+0000","last_undegraded":"2023-09-23T18:10:25.622039+0000","last_fullsized":"2023-09-23T18:10:25.622039+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4936,"log_dups_size":0,"ondisk_log_size":4936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:51.634243+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323485696,"num_objects":4936,"num_object_clones":0,"num_object_copies":14808,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4936,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4936,"num_write_kb":315904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":645,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.412517+0000","last_change":"2023-09-23T18:10:04.137099+0000","last_active":"2023-09-23T18:10:17.412517+0000","last_peered":"2023-09-23T18:10:17.412517+0000","last_clean":"2023-09-23T18:10:17.412517+0000","last_became_active":"2023-09-23T18:10:04.136723+0000","last_became_peered":"2023-09-23T18:10:04.136723+0000","last_unstale":"2023-09-23T18:10:17.412517+0000","last_undegraded":"2023-09-23T18:10:17.412517+0000","last_fullsized":"2023-09-23T18:10:17.412517+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:41.510146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:41.510146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1392168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,4],"acting":[0,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":335,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.402011+0000","last_change":"2023-09-23T18:06:05.379591+0000","last_active":"2023-09-23T18:10:17.402011+0000","last_peered":"2023-09-23T18:10:17.402011+0000","last_clean":"2023-09-23T18:10:17.402011+0000","last_became_active":"2023-09-23T18:06:05.370176+0000","last_became_peered":"2023-09-23T18:06:05.370176+0000","last_unstale":"2023-09-23T18:10:17.402011+0000","last_undegraded":"2023-09-23T18:10:17.402011+0000","last_fullsized":"2023-09-23T18:10:17.402011+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:08.768138+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":408,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.400426+0000","last_change":"2023-09-23T18:06:45.911012+0000","last_active":"2023-09-23T18:10:17.400426+0000","last_peered":"2023-09-23T18:10:17.400426+0000","last_clean":"2023-09-23T18:10:17.400426+0000","last_became_active":"2023-09-23T18:06:45.910722+0000","last_became_peered":"2023-09-23T18:06:45.910722+0000","last_unstale":"2023-09-23T18:10:17.400426+0000","last_undegraded":"2023-09-23T18:10:17.400426+0000","last_fullsized":"2023-09-23T18:10:17.400426+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:19.150719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"378'4956","reported_seq":4987,"reported_epoch":378,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T18:10:23.571787+0000","last_change":"2023-09-23T18:10:02.341560+0000","last_active":"2023-09-23T18:10:23.571787+0000","last_peered":"2023-09-23T18:10:23.571787+0000","last_clean":"2023-09-23T18:10:00.095783+0000","last_became_active":"2023-09-23T18:10:02.330146+0000","last_became_peered":"2023-09-23T18:10:02.330146+0000","last_unstale":"2023-09-23T18:10:23.571787+0000","last_undegraded":"2023-09-23T18:10:23.571787+0000","last_fullsized":"2023-09-23T18:10:23.571787+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4956,"log_dups_size":0,"ondisk_log_size":4956,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:09.375692+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":324796416,"num_objects":4956,"num_object_clones":0,"num_object_copies":14868,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":3625,"num_objects_unfound":0,"num_objects_dirty":4956,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4956,"num_write_kb":317184,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,7],"acting":[2,6,7],"avail_no_missing":["2","6","7"],"object_location_counts":[{"shards":"2,6,7","objects":1331}],"blocked_by":[],"up_primary":0,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":556,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.393499+0000","last_change":"2023-09-23T18:09:49.245759+0000","last_active":"2023-09-23T18:10:17.393499+0000","last_peered":"2023-09-23T18:10:17.393499+0000","last_clean":"2023-09-23T18:09:40.149639+0000","last_became_active":"2023-09-23T18:09:49.245759+0000","last_became_peered":"2023-09-23T18:09:49.245759+0000","last_unstale":"2023-09-23T18:10:17.393499+0000","last_undegraded":"2023-09-23T18:10:17.393499+0000","last_fullsized":"2023-09-23T18:09:49.241138+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:31:29.659220+0000","scrub_duration":0.0018893740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":251,"reported_epoch":377,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:17.401509+0000","last_change":"2023-09-23T18:09:49.260821+0000","last_active":"2023-09-23T18:10:17.401509+0000","last_peered":"2023-09-23T18:10:17.401509+0000","last_clean":"2023-09-23T18:09:40.155190+0000","last_became_active":"2023-09-23T18:09:49.260821+0000","last_became_peered":"2023-09-23T18:09:49.260821+0000","last_unstale":"2023-09-23T18:10:17.401509+0000","last_undegraded":"2023-09-23T18:09:49.257202+0000","last_fullsized":"2023-09-23T18:09:49.257118+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:05.023383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":353,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.395643+0000","last_change":"2023-09-23T18:06:05.371447+0000","last_active":"2023-09-23T18:10:17.395643+0000","last_peered":"2023-09-23T18:10:17.395643+0000","last_clean":"2023-09-23T18:10:17.395643+0000","last_became_active":"2023-09-23T18:06:05.371065+0000","last_became_peered":"2023-09-23T18:06:05.371065+0000","last_unstale":"2023-09-23T18:10:17.395643+0000","last_undegraded":"2023-09-23T18:10:17.395643+0000","last_fullsized":"2023-09-23T18:10:17.395643+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.025791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"378'4806","reported_seq":4832,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T18:10:22.698226+0000","last_change":"2023-09-23T18:09:26.507113+0000","last_active":"2023-09-23T18:10:22.698226+0000","last_peered":"2023-09-23T18:10:22.698226+0000","last_clean":"2023-09-23T18:10:22.698226+0000","last_became_active":"2023-09-23T18:09:26.506876+0000","last_became_peered":"2023-09-23T18:09:26.506876+0000","last_unstale":"2023-09-23T18:10:22.698226+0000","last_undegraded":"2023-09-23T18:10:22.698226+0000","last_fullsized":"2023-09-23T18:10:22.698226+0000","mapping_epoch":361,"log_start":"365'300","ondisk_log_start":"365'300","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4506,"log_dups_size":0,"ondisk_log_size":4506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:03:54.009952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314966016,"num_objects":4806,"num_object_clones":0,"num_object_copies":14418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4806,"num_write_kb":307584,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":702,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.400054+0000","last_change":"2023-09-23T18:05:56.362746+0000","last_active":"2023-09-23T18:10:17.400054+0000","last_peered":"2023-09-23T18:10:17.400054+0000","last_clean":"2023-09-23T18:10:17.400054+0000","last_became_active":"2023-09-23T18:05:15.102072+0000","last_became_peered":"2023-09-23T18:05:15.102072+0000","last_unstale":"2023-09-23T18:10:17.400054+0000","last_undegraded":"2023-09-23T18:10:17.400054+0000","last_fullsized":"2023-09-23T18:10:17.400054+0000","mapping_epoch":277,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.362652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.362652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:31:33.381515+0000","scrub_duration":0.0052581759999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":301,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.393273+0000","last_change":"2023-09-23T18:09:49.253688+0000","last_active":"2023-09-23T18:10:17.393273+0000","last_peered":"2023-09-23T18:10:17.393273+0000","last_clean":"2023-09-23T18:06:48.956525+0000","last_became_active":"2023-09-23T18:09:49.253688+0000","last_became_peered":"2023-09-23T18:09:49.253688+0000","last_unstale":"2023-09-23T18:10:17.393273+0000","last_undegraded":"2023-09-23T18:10:17.393273+0000","last_fullsized":"2023-09-23T18:09:49.240299+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:58.227257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":18,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.412371+0000","last_change":"2023-09-23T18:10:01.176492+0000","last_active":"2023-09-23T18:10:17.412371+0000","last_peered":"2023-09-23T18:10:17.412371+0000","last_clean":"2023-09-23T18:10:17.412371+0000","last_became_active":"2023-09-23T18:10:01.176155+0000","last_became_peered":"2023-09-23T18:10:01.176155+0000","last_unstale":"2023-09-23T18:10:17.412371+0000","last_undegraded":"2023-09-23T18:10:17.412371+0000","last_fullsized":"2023-09-23T18:10:17.412371+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:03:53.737654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.0","version":"378'4873","reported_seq":4901,"reported_epoch":378,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:22.704522+0000","last_change":"2023-09-23T18:09:49.245179+0000","last_active":"2023-09-23T18:10:22.704522+0000","last_peered":"2023-09-23T18:10:22.704522+0000","last_clean":"2023-09-23T18:09:48.233578+0000","last_became_active":"2023-09-23T18:09:49.245179+0000","last_became_peered":"2023-09-23T18:09:49.245179+0000","last_unstale":"2023-09-23T18:10:22.704522+0000","last_undegraded":"2023-09-23T18:09:49.238600+0000","last_fullsized":"2023-09-23T18:09:49.238472+0000","mapping_epoch":367,"log_start":"365'300","ondisk_log_start":"365'300","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4573,"log_dups_size":0,"ondisk_log_size":4573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:29.489220+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319356928,"num_objects":4873,"num_object_clones":0,"num_object_copies":14619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4873,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4873,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4873,"num_write_kb":311872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":4873}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":757,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.400625+0000","last_change":"2023-09-23T18:09:49.250289+0000","last_active":"2023-09-23T18:10:17.400625+0000","last_peered":"2023-09-23T18:10:17.400625+0000","last_clean":"2023-09-23T18:09:40.154309+0000","last_became_active":"2023-09-23T18:09:49.250289+0000","last_became_peered":"2023-09-23T18:09:49.250289+0000","last_unstale":"2023-09-23T18:10:17.400625+0000","last_undegraded":"2023-09-23T18:10:17.400625+0000","last_fullsized":"2023-09-23T18:09:49.248499+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:54.767780+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":164,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399211+0000","last_change":"2023-09-23T18:07:49.227201+0000","last_active":"2023-09-23T18:10:17.399211+0000","last_peered":"2023-09-23T18:10:17.399211+0000","last_clean":"2023-09-23T18:10:17.399211+0000","last_became_active":"2023-09-23T18:07:49.226878+0000","last_became_peered":"2023-09-23T18:07:49.226878+0000","last_unstale":"2023-09-23T18:10:17.399211+0000","last_undegraded":"2023-09-23T18:10:17.399211+0000","last_fullsized":"2023-09-23T18:10:17.399211+0000","mapping_epoch":327,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":328,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:08.994668+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":378,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399580+0000","last_change":"2023-09-23T18:02:18.944326+0000","last_active":"2023-09-23T18:10:17.399580+0000","last_peered":"2023-09-23T18:10:17.399580+0000","last_clean":"2023-09-23T18:10:17.399580+0000","last_became_active":"2023-09-23T18:02:18.944031+0000","last_became_peered":"2023-09-23T18:02:18.944031+0000","last_unstale":"2023-09-23T18:10:17.399580+0000","last_undegraded":"2023-09-23T18:10:17.399580+0000","last_fullsized":"2023-09-23T18:10:17.399580+0000","mapping_epoch":238,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":239,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:25:27.545191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"378'4805","reported_seq":6360,"reported_epoch":378,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:10:22.715926+0000","last_change":"2023-09-23T18:10:02.349030+0000","last_active":"2023-09-23T18:10:22.715926+0000","last_peered":"2023-09-23T18:10:22.715926+0000","last_clean":"2023-09-23T18:09:48.228392+0000","last_became_active":"2023-09-23T18:10:02.311652+0000","last_became_peered":"2023-09-23T18:10:02.311652+0000","last_unstale":"2023-09-23T18:10:22.715926+0000","last_undegraded":"2023-09-23T18:10:02.208436+0000","last_fullsized":"2023-09-23T18:10:02.183430+0000","mapping_epoch":371,"log_start":"365'300","ondisk_log_start":"365'300","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4505,"log_dups_size":0,"ondisk_log_size":4505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:41.860294+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314900480,"num_objects":4805,"num_object_clones":0,"num_object_copies":14415,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2822,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4805,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4805,"num_write_kb":307520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":758,"num_bytes_recovered":49676288,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":1983}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":648,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.392476+0000","last_change":"2023-09-23T18:09:49.247368+0000","last_active":"2023-09-23T18:10:17.392476+0000","last_peered":"2023-09-23T18:10:17.392476+0000","last_clean":"2023-09-23T18:06:48.958041+0000","last_became_active":"2023-09-23T18:09:49.247368+0000","last_became_peered":"2023-09-23T18:09:49.247368+0000","last_unstale":"2023-09-23T18:10:17.392476+0000","last_undegraded":"2023-09-23T18:10:17.392476+0000","last_fullsized":"2023-09-23T18:09:49.239209+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:37:34.776285+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":347,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.395961+0000","last_change":"2023-09-23T18:06:05.371350+0000","last_active":"2023-09-23T18:10:17.395961+0000","last_peered":"2023-09-23T18:10:17.395961+0000","last_clean":"2023-09-23T18:10:17.395961+0000","last_became_active":"2023-09-23T18:06:05.370877+0000","last_became_peered":"2023-09-23T18:06:05.370877+0000","last_unstale":"2023-09-23T18:10:17.395961+0000","last_undegraded":"2023-09-23T18:10:17.395961+0000","last_fullsized":"2023-09-23T18:10:17.395961+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:58.534355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":188,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399805+0000","last_change":"2023-09-23T18:06:05.373385+0000","last_active":"2023-09-23T18:10:17.399805+0000","last_peered":"2023-09-23T18:10:17.399805+0000","last_clean":"2023-09-23T18:10:17.399805+0000","last_became_active":"2023-09-23T18:06:05.372982+0000","last_became_peered":"2023-09-23T18:06:05.372982+0000","last_unstale":"2023-09-23T18:10:17.399805+0000","last_undegraded":"2023-09-23T18:10:17.399805+0000","last_fullsized":"2023-09-23T18:10:17.399805+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:48.622115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"378'4960","reported_seq":5002,"reported_epoch":378,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:10:25.790295+0000","last_change":"2023-09-23T18:10:02.341385+0000","last_active":"2023-09-23T18:10:25.790295+0000","last_peered":"2023-09-23T18:10:25.790295+0000","last_clean":"2023-09-23T18:09:48.231362+0000","last_became_active":"2023-09-23T18:10:02.333199+0000","last_became_peered":"2023-09-23T18:10:02.333199+0000","last_unstale":"2023-09-23T18:10:25.790295+0000","last_undegraded":"2023-09-23T18:10:02.199212+0000","last_fullsized":"2023-09-23T18:10:02.178517+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4960,"log_dups_size":0,"ondisk_log_size":4960,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:07:35.880677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325058560,"num_objects":4960,"num_object_clones":0,"num_object_copies":14880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3527,"num_objects_misplaced":4960,"num_objects_unfound":0,"num_objects_dirty":4960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4960,"num_write_kb":317440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":1433}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":761,"reported_epoch":377,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:17.402135+0000","last_change":"2023-09-23T18:10:04.176981+0000","last_active":"2023-09-23T18:10:17.402135+0000","last_peered":"2023-09-23T18:10:17.402135+0000","last_clean":"2023-09-23T18:10:17.402135+0000","last_became_active":"2023-09-23T18:10:04.176481+0000","last_became_peered":"2023-09-23T18:10:04.176481+0000","last_unstale":"2023-09-23T18:10:17.402135+0000","last_undegraded":"2023-09-23T18:10:17.402135+0000","last_fullsized":"2023-09-23T18:10:17.402135+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:12.991004+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,2],"avail_no_missing":["1","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":168,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.400361+0000","last_change":"2023-09-23T18:01:39.461473+0000","last_active":"2023-09-23T18:10:17.400361+0000","last_peered":"2023-09-23T18:10:17.400361+0000","last_clean":"2023-09-23T18:10:17.400361+0000","last_became_active":"2023-09-23T18:01:39.461046+0000","last_became_peered":"2023-09-23T18:01:39.461046+0000","last_unstale":"2023-09-23T18:10:17.400361+0000","last_undegraded":"2023-09-23T18:10:17.400361+0000","last_fullsized":"2023-09-23T18:10:17.400361+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:33.907007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":398,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.400115+0000","last_change":"2023-09-23T18:06:05.373305+0000","last_active":"2023-09-23T18:10:17.400115+0000","last_peered":"2023-09-23T18:10:17.400115+0000","last_clean":"2023-09-23T18:10:17.400115+0000","last_became_active":"2023-09-23T18:06:05.372720+0000","last_became_peered":"2023-09-23T18:06:05.372720+0000","last_unstale":"2023-09-23T18:10:17.400115+0000","last_undegraded":"2023-09-23T18:10:17.400115+0000","last_fullsized":"2023-09-23T18:10:17.400115+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:26:44.934493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"378'4876","reported_seq":4912,"reported_epoch":378,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:22.714548+0000","last_change":"2023-09-23T18:09:49.272932+0000","last_active":"2023-09-23T18:10:22.714548+0000","last_peered":"2023-09-23T18:10:22.714548+0000","last_clean":"2023-09-23T18:09:48.219569+0000","last_became_active":"2023-09-23T18:09:49.258387+0000","last_became_peered":"2023-09-23T18:09:49.258387+0000","last_unstale":"2023-09-23T18:10:22.714548+0000","last_undegraded":"2023-09-23T18:09:49.243174+0000","last_fullsized":"2023-09-23T18:09:49.240692+0000","mapping_epoch":367,"log_start":"365'300","ondisk_log_start":"365'300","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4576,"log_dups_size":0,"ondisk_log_size":4576,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:55.945872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319553536,"num_objects":4876,"num_object_clones":0,"num_object_copies":14628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4876,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4876,"num_write_kb":312064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":4876}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":433,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.395469+0000","last_change":"2023-09-23T18:09:49.255810+0000","last_active":"2023-09-23T18:10:17.395469+0000","last_peered":"2023-09-23T18:10:17.395469+0000","last_clean":"2023-09-23T17:49:06.430604+0000","last_became_active":"2023-09-23T18:09:49.255810+0000","last_became_peered":"2023-09-23T18:09:49.255810+0000","last_unstale":"2023-09-23T18:10:17.395469+0000","last_undegraded":"2023-09-23T18:10:17.395469+0000","last_fullsized":"2023-09-23T18:09:49.239753+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":224,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.395622+0000","last_change":"2023-09-23T18:06:45.904403+0000","last_active":"2023-09-23T18:10:17.395622+0000","last_peered":"2023-09-23T18:10:17.395622+0000","last_clean":"2023-09-23T18:10:17.395622+0000","last_became_active":"2023-09-23T18:06:45.904057+0000","last_became_peered":"2023-09-23T18:06:45.904057+0000","last_unstale":"2023-09-23T18:10:17.395622+0000","last_undegraded":"2023-09-23T18:10:17.395622+0000","last_fullsized":"2023-09-23T18:10:17.395622+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:43.213905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":104,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399359+0000","last_change":"2023-09-23T18:06:05.371661+0000","last_active":"2023-09-23T18:10:17.399359+0000","last_peered":"2023-09-23T18:10:17.399359+0000","last_clean":"2023-09-23T18:10:17.399359+0000","last_became_active":"2023-09-23T18:06:05.370998+0000","last_became_peered":"2023-09-23T18:06:05.370998+0000","last_unstale":"2023-09-23T18:10:17.399359+0000","last_undegraded":"2023-09-23T18:10:17.399359+0000","last_fullsized":"2023-09-23T18:10:17.399359+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:52.262543+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"378'5044","reported_seq":5069,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T18:10:25.795576+0000","last_change":"2023-09-23T18:09:26.033882+0000","last_active":"2023-09-23T18:10:25.795576+0000","last_peered":"2023-09-23T18:10:25.795576+0000","last_clean":"2023-09-23T18:10:25.795576+0000","last_became_active":"2023-09-23T18:09:26.033595+0000","last_became_peered":"2023-09-23T18:09:26.033595+0000","last_unstale":"2023-09-23T18:10:25.795576+0000","last_undegraded":"2023-09-23T18:10:25.795576+0000","last_fullsized":"2023-09-23T18:10:25.795576+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":5044,"log_dups_size":0,"ondisk_log_size":5044,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:47.811446+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":330563584,"num_objects":5044,"num_object_clones":0,"num_object_copies":15132,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5044,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5044,"num_write_kb":322816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":576,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.393160+0000","last_change":"2023-09-23T18:09:49.254468+0000","last_active":"2023-09-23T18:10:17.393160+0000","last_peered":"2023-09-23T18:10:17.393160+0000","last_clean":"2023-09-23T18:09:40.149522+0000","last_became_active":"2023-09-23T18:09:49.254468+0000","last_became_peered":"2023-09-23T18:09:49.254468+0000","last_unstale":"2023-09-23T18:10:17.393160+0000","last_undegraded":"2023-09-23T18:10:17.393160+0000","last_fullsized":"2023-09-23T18:09:49.240580+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:11.928170+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":780,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.398908+0000","last_change":"2023-09-23T18:10:04.168459+0000","last_active":"2023-09-23T18:10:17.398908+0000","last_peered":"2023-09-23T18:10:17.398908+0000","last_clean":"2023-09-23T18:10:17.398908+0000","last_became_active":"2023-09-23T18:10:04.167807+0000","last_became_peered":"2023-09-23T18:10:04.167807+0000","last_unstale":"2023-09-23T18:10:17.398908+0000","last_undegraded":"2023-09-23T18:10:17.398908+0000","last_fullsized":"2023-09-23T18:10:17.398908+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:40.519034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":305,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.393481+0000","last_change":"2023-09-23T18:09:49.247765+0000","last_active":"2023-09-23T18:10:17.393481+0000","last_peered":"2023-09-23T18:10:17.393481+0000","last_clean":"2023-09-23T18:06:48.956978+0000","last_became_active":"2023-09-23T18:09:49.247765+0000","last_became_peered":"2023-09-23T18:09:49.247765+0000","last_unstale":"2023-09-23T18:10:17.393481+0000","last_undegraded":"2023-09-23T18:10:17.393481+0000","last_fullsized":"2023-09-23T18:09:49.240659+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:49:22.681242+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2],"acting":[6,2],"avail_no_missing":["6","2"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":111,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.728273+0000","last_change":"2023-09-23T18:09:49.249900+0000","last_active":"2023-09-23T18:10:17.728273+0000","last_peered":"2023-09-23T18:10:17.728273+0000","last_clean":"2023-09-23T18:09:40.164966+0000","last_became_active":"2023-09-23T18:09:49.249900+0000","last_became_peered":"2023-09-23T18:09:49.249900+0000","last_unstale":"2023-09-23T18:10:17.728273+0000","last_undegraded":"2023-09-23T18:10:17.728273+0000","last_fullsized":"2023-09-23T18:09:49.239398+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:02.030814+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"378'4720","reported_seq":4755,"reported_epoch":378,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:21.994562+0000","last_change":"2023-09-23T18:09:49.273902+0000","last_active":"2023-09-23T18:10:21.994562+0000","last_peered":"2023-09-23T18:10:21.994562+0000","last_clean":"2023-09-23T18:09:48.238829+0000","last_became_active":"2023-09-23T18:09:49.264471+0000","last_became_peered":"2023-09-23T18:09:49.264471+0000","last_unstale":"2023-09-23T18:10:21.994562+0000","last_undegraded":"2023-09-23T18:09:49.241143+0000","last_fullsized":"2023-09-23T18:09:49.235824+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4720,"log_dups_size":0,"ondisk_log_size":4720,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:01.252100+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":309329920,"num_objects":4720,"num_object_clones":0,"num_object_copies":14160,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4720,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4720,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4720,"num_write_kb":302080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":4720}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":600,"reported_epoch":377,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:17.416147+0000","last_change":"2023-09-23T18:10:04.146853+0000","last_active":"2023-09-23T18:10:17.416147+0000","last_peered":"2023-09-23T18:10:17.416147+0000","last_clean":"2023-09-23T18:10:17.416147+0000","last_became_active":"2023-09-23T18:10:04.146326+0000","last_became_peered":"2023-09-23T18:10:04.146326+0000","last_unstale":"2023-09-23T18:10:17.416147+0000","last_undegraded":"2023-09-23T18:10:17.416147+0000","last_fullsized":"2023-09-23T18:10:17.416147+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:39.103607+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,4],"avail_no_missing":["0","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1237,"reported_epoch":377,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:17.400532+0000","last_change":"2023-09-23T18:09:49.245522+0000","last_active":"2023-09-23T18:10:17.400532+0000","last_peered":"2023-09-23T18:10:17.400532+0000","last_clean":"2023-09-23T18:05:59.276534+0000","last_became_active":"2023-09-23T18:09:49.245522+0000","last_became_peered":"2023-09-23T18:09:49.245522+0000","last_unstale":"2023-09-23T18:10:17.400532+0000","last_undegraded":"2023-09-23T18:09:49.238958+0000","last_fullsized":"2023-09-23T18:09:49.238829+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:46.845650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":188,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399080+0000","last_change":"2023-09-23T18:10:01.168325+0000","last_active":"2023-09-23T18:10:17.399080+0000","last_peered":"2023-09-23T18:10:17.399080+0000","last_clean":"2023-09-23T18:10:17.399080+0000","last_became_active":"2023-09-23T18:10:01.168019+0000","last_became_peered":"2023-09-23T18:10:01.168019+0000","last_unstale":"2023-09-23T18:10:17.399080+0000","last_undegraded":"2023-09-23T18:10:17.399080+0000","last_fullsized":"2023-09-23T18:10:17.399080+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:06.229925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":114,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.728446+0000","last_change":"2023-09-23T18:06:51.088473+0000","last_active":"2023-09-23T18:10:17.728446+0000","last_peered":"2023-09-23T18:10:17.728446+0000","last_clean":"2023-09-23T18:10:17.728446+0000","last_became_active":"2023-09-23T18:06:51.088171+0000","last_became_peered":"2023-09-23T18:06:51.088171+0000","last_unstale":"2023-09-23T18:10:17.728446+0000","last_undegraded":"2023-09-23T18:10:17.728446+0000","last_fullsized":"2023-09-23T18:10:17.728446+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:45.860551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"378'4935","reported_seq":4960,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T18:10:25.779442+0000","last_change":"2023-09-23T18:09:26.034398+0000","last_active":"2023-09-23T18:10:25.779442+0000","last_peered":"2023-09-23T18:10:25.779442+0000","last_clean":"2023-09-23T18:10:25.779442+0000","last_became_active":"2023-09-23T18:09:26.034124+0000","last_became_peered":"2023-09-23T18:09:26.034124+0000","last_unstale":"2023-09-23T18:10:25.779442+0000","last_undegraded":"2023-09-23T18:10:25.779442+0000","last_fullsized":"2023-09-23T18:10:25.779442+0000","mapping_epoch":361,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4935,"log_dups_size":0,"ondisk_log_size":4935,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:36:24.809053+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323420160,"num_objects":4935,"num_object_clones":0,"num_object_copies":14805,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4935,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4935,"num_write_kb":315840,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":647,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.418605+0000","last_change":"2023-09-23T18:10:04.140020+0000","last_active":"2023-09-23T18:10:17.418605+0000","last_peered":"2023-09-23T18:10:17.418605+0000","last_clean":"2023-09-23T18:10:17.418605+0000","last_became_active":"2023-09-23T18:10:04.139695+0000","last_became_peered":"2023-09-23T18:10:04.139695+0000","last_unstale":"2023-09-23T18:10:17.418605+0000","last_undegraded":"2023-09-23T18:10:17.418605+0000","last_fullsized":"2023-09-23T18:10:17.418605+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:21:40.787527+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,4],"acting":[0,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":44,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.416282+0000","last_change":"2023-09-23T18:10:01.223543+0000","last_active":"2023-09-23T18:10:17.416282+0000","last_peered":"2023-09-23T18:10:17.416282+0000","last_clean":"2023-09-23T18:10:17.416282+0000","last_became_active":"2023-09-23T18:10:01.176897+0000","last_became_peered":"2023-09-23T18:10:01.176897+0000","last_unstale":"2023-09-23T18:10:17.416282+0000","last_undegraded":"2023-09-23T18:10:17.416282+0000","last_fullsized":"2023-09-23T18:10:17.416282+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:03:34.483871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2,1],"acting":[0,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":105,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.395851+0000","last_change":"2023-09-23T18:06:05.358500+0000","last_active":"2023-09-23T18:10:17.395851+0000","last_peered":"2023-09-23T18:10:17.395851+0000","last_clean":"2023-09-23T18:10:17.395851+0000","last_became_active":"2023-09-23T18:06:05.358274+0000","last_became_peered":"2023-09-23T18:06:05.358274+0000","last_unstale":"2023-09-23T18:10:17.395851+0000","last_undegraded":"2023-09-23T18:10:17.395851+0000","last_fullsized":"2023-09-23T18:10:17.395851+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:04.055437+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":119,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.399580+0000","last_change":"2023-09-23T18:09:49.257163+0000","last_active":"2023-09-23T18:10:17.399580+0000","last_peered":"2023-09-23T18:10:17.399580+0000","last_clean":"2023-09-23T18:09:40.168372+0000","last_became_active":"2023-09-23T18:09:49.257163+0000","last_became_peered":"2023-09-23T18:09:49.257163+0000","last_unstale":"2023-09-23T18:10:17.399580+0000","last_undegraded":"2023-09-23T18:10:17.399580+0000","last_fullsized":"2023-09-23T18:09:49.240945+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:14.517993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"378'5024","reported_seq":5056,"reported_epoch":378,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:25.636998+0000","last_change":"2023-09-23T18:09:49.245077+0000","last_active":"2023-09-23T18:10:25.636998+0000","last_peered":"2023-09-23T18:10:25.636998+0000","last_clean":"2023-09-23T18:09:48.227889+0000","last_became_active":"2023-09-23T18:09:49.245077+0000","last_became_peered":"2023-09-23T18:09:49.245077+0000","last_unstale":"2023-09-23T18:10:25.636998+0000","last_undegraded":"2023-09-23T18:09:49.239224+0000","last_fullsized":"2023-09-23T18:09:49.239102+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":5024,"log_dups_size":0,"ondisk_log_size":5024,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:54.675527+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329252864,"num_objects":5024,"num_object_clones":0,"num_object_copies":15072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5024,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5024,"num_write_kb":321536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":5024}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1001,"reported_epoch":377,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:17.400801+0000","last_change":"2023-09-23T18:09:49.260151+0000","last_active":"2023-09-23T18:10:17.400801+0000","last_peered":"2023-09-23T18:10:17.400801+0000","last_clean":"2023-09-23T17:47:00.624503+0000","last_became_active":"2023-09-23T18:09:49.260151+0000","last_became_peered":"2023-09-23T18:09:49.260151+0000","last_unstale":"2023-09-23T18:10:17.400801+0000","last_undegraded":"2023-09-23T18:09:49.255776+0000","last_fullsized":"2023-09-23T18:09:49.255629+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:41:31.486599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":733,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.400549+0000","last_change":"2023-09-23T18:06:48.957422+0000","last_active":"2023-09-23T18:10:17.400549+0000","last_peered":"2023-09-23T18:10:17.400549+0000","last_clean":"2023-09-23T18:10:17.400549+0000","last_became_active":"2023-09-23T18:06:48.957044+0000","last_became_peered":"2023-09-23T18:06:48.957044+0000","last_unstale":"2023-09-23T18:10:17.400549+0000","last_undegraded":"2023-09-23T18:10:17.400549+0000","last_fullsized":"2023-09-23T18:10:17.400549+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:58.047861+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":147,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.395845+0000","last_change":"2023-09-23T18:09:49.245056+0000","last_active":"2023-09-23T18:10:17.395845+0000","last_peered":"2023-09-23T18:10:17.395845+0000","last_clean":"2023-09-23T17:55:56.841167+0000","last_became_active":"2023-09-23T18:09:49.245056+0000","last_became_peered":"2023-09-23T18:09:49.245056+0000","last_unstale":"2023-09-23T18:10:17.395845+0000","last_undegraded":"2023-09-23T18:10:17.395845+0000","last_fullsized":"2023-09-23T18:09:49.240277+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:39.548152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":311,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.393658+0000","last_change":"2023-09-23T18:01:39.456371+0000","last_active":"2023-09-23T18:10:17.393658+0000","last_peered":"2023-09-23T18:10:17.393658+0000","last_clean":"2023-09-23T18:10:17.393658+0000","last_became_active":"2023-09-23T18:01:39.455939+0000","last_became_peered":"2023-09-23T18:01:39.455939+0000","last_unstale":"2023-09-23T18:10:17.393658+0000","last_undegraded":"2023-09-23T18:10:17.393658+0000","last_fullsized":"2023-09-23T18:10:17.393658+0000","mapping_epoch":219,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:43.946658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"378'4859","reported_seq":4891,"reported_epoch":378,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:22.719017+0000","last_change":"2023-09-23T18:09:49.246060+0000","last_active":"2023-09-23T18:10:22.719017+0000","last_peered":"2023-09-23T18:10:22.719017+0000","last_clean":"2023-09-23T18:09:48.219556+0000","last_became_active":"2023-09-23T18:09:49.246060+0000","last_became_peered":"2023-09-23T18:09:49.246060+0000","last_unstale":"2023-09-23T18:10:22.719017+0000","last_undegraded":"2023-09-23T18:09:49.238965+0000","last_fullsized":"2023-09-23T18:09:49.238836+0000","mapping_epoch":367,"log_start":"365'300","ondisk_log_start":"365'300","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4559,"log_dups_size":0,"ondisk_log_size":4559,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:29:32.216226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318439424,"num_objects":4859,"num_object_clones":0,"num_object_copies":14577,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4859,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4859,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4859,"num_write_kb":310976,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":4859}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":540,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T18:10:21.461879+0000","last_change":"2023-09-23T18:09:16.643471+0000","last_active":"2023-09-23T18:10:21.461879+0000","last_peered":"2023-09-23T18:10:21.461879+0000","last_clean":"2023-09-23T18:10:21.461879+0000","last_became_active":"2023-09-23T18:06:08.433718+0000","last_became_peered":"2023-09-23T18:06:08.433718+0000","last_unstale":"2023-09-23T18:10:21.461879+0000","last_undegraded":"2023-09-23T18:10:21.461879+0000","last_fullsized":"2023-09-23T18:10:21.461879+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:37.624632+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":641,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.401752+0000","last_change":"2023-09-23T18:10:01.230791+0000","last_active":"2023-09-23T18:10:17.401752+0000","last_peered":"2023-09-23T18:10:17.401752+0000","last_clean":"2023-09-23T18:10:17.401752+0000","last_became_active":"2023-09-23T18:10:01.170909+0000","last_became_peered":"2023-09-23T18:10:01.170909+0000","last_unstale":"2023-09-23T18:10:17.401752+0000","last_undegraded":"2023-09-23T18:10:17.401752+0000","last_fullsized":"2023-09-23T18:10:17.401752+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:08.759640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":19,"reported_epoch":377,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:17.416021+0000","last_change":"2023-09-23T18:10:02.189623+0000","last_active":"2023-09-23T18:10:17.416021+0000","last_peered":"2023-09-23T18:10:17.416021+0000","last_clean":"2023-09-23T18:10:17.416021+0000","last_became_active":"2023-09-23T18:10:02.189234+0000","last_became_peered":"2023-09-23T18:10:02.189234+0000","last_unstale":"2023-09-23T18:10:17.416021+0000","last_undegraded":"2023-09-23T18:10:17.416021+0000","last_fullsized":"2023-09-23T18:10:17.416021+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:18.377869+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4],"acting":[0,4,7],"avail_no_missing":["0","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":362,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.399239+0000","last_change":"2023-09-23T18:09:49.263629+0000","last_active":"2023-09-23T18:10:17.399239+0000","last_peered":"2023-09-23T18:10:17.399239+0000","last_clean":"2023-09-23T18:09:40.167801+0000","last_became_active":"2023-09-23T18:09:49.263629+0000","last_became_peered":"2023-09-23T18:09:49.263629+0000","last_unstale":"2023-09-23T18:10:17.399239+0000","last_undegraded":"2023-09-23T18:10:17.399239+0000","last_fullsized":"2023-09-23T18:09:49.239463+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:25.971337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"378'4819","reported_seq":4844,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T18:10:22.728371+0000","last_change":"2023-09-23T18:09:26.507006+0000","last_active":"2023-09-23T18:10:22.728371+0000","last_peered":"2023-09-23T18:10:22.728371+0000","last_clean":"2023-09-23T18:10:22.728371+0000","last_became_active":"2023-09-23T18:09:26.506732+0000","last_became_peered":"2023-09-23T18:09:26.506732+0000","last_unstale":"2023-09-23T18:10:22.728371+0000","last_undegraded":"2023-09-23T18:10:22.728371+0000","last_fullsized":"2023-09-23T18:10:22.728371+0000","mapping_epoch":361,"log_start":"365'300","ondisk_log_start":"365'300","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4519,"log_dups_size":0,"ondisk_log_size":4519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:01.574367+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315817984,"num_objects":4819,"num_object_clones":0,"num_object_copies":14457,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4819,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4819,"num_write_kb":308416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":589,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.393158+0000","last_change":"2023-09-23T18:06:54.055392+0000","last_active":"2023-09-23T18:10:17.393158+0000","last_peered":"2023-09-23T18:10:17.393158+0000","last_clean":"2023-09-23T18:10:17.393158+0000","last_became_active":"2023-09-23T18:06:54.055101+0000","last_became_peered":"2023-09-23T18:06:54.055101+0000","last_unstale":"2023-09-23T18:10:17.393158+0000","last_undegraded":"2023-09-23T18:10:17.393158+0000","last_fullsized":"2023-09-23T18:10:17.393158+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:50.070812+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":617,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.401393+0000","last_change":"2023-09-23T18:08:46.401461+0000","last_active":"2023-09-23T18:10:17.401393+0000","last_peered":"2023-09-23T18:10:17.401393+0000","last_clean":"2023-09-23T18:10:17.401393+0000","last_became_active":"2023-09-23T18:06:51.071089+0000","last_became_peered":"2023-09-23T18:06:51.071089+0000","last_unstale":"2023-09-23T18:10:17.401393+0000","last_undegraded":"2023-09-23T18:10:17.401393+0000","last_fullsized":"2023-09-23T18:10:17.401393+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":343,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:01.696899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":120,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.728396+0000","last_change":"2023-09-23T18:09:49.247536+0000","last_active":"2023-09-23T18:10:17.728396+0000","last_peered":"2023-09-23T18:10:17.728396+0000","last_clean":"2023-09-23T18:09:40.168110+0000","last_became_active":"2023-09-23T18:09:49.247536+0000","last_became_peered":"2023-09-23T18:09:49.247536+0000","last_unstale":"2023-09-23T18:10:17.728396+0000","last_undegraded":"2023-09-23T18:10:17.728396+0000","last_fullsized":"2023-09-23T18:09:49.239753+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:23.414385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":19,"reported_epoch":377,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:17.412265+0000","last_change":"2023-09-23T18:10:02.191332+0000","last_active":"2023-09-23T18:10:17.412265+0000","last_peered":"2023-09-23T18:10:17.412265+0000","last_clean":"2023-09-23T18:10:17.412265+0000","last_became_active":"2023-09-23T18:10:02.190947+0000","last_became_peered":"2023-09-23T18:10:02.190947+0000","last_unstale":"2023-09-23T18:10:17.412265+0000","last_undegraded":"2023-09-23T18:10:17.412265+0000","last_fullsized":"2023-09-23T18:10:17.412265+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:03:14.070652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"378'4876","reported_seq":4908,"reported_epoch":378,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:23.073045+0000","last_change":"2023-09-23T18:09:49.253852+0000","last_active":"2023-09-23T18:10:23.073045+0000","last_peered":"2023-09-23T18:10:23.073045+0000","last_clean":"2023-09-23T18:09:48.216272+0000","last_became_active":"2023-09-23T18:09:49.253852+0000","last_became_peered":"2023-09-23T18:09:49.253852+0000","last_unstale":"2023-09-23T18:10:23.073045+0000","last_undegraded":"2023-09-23T18:09:49.249506+0000","last_fullsized":"2023-09-23T18:09:49.249391+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4876,"log_dups_size":0,"ondisk_log_size":4876,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:10.981898+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319553536,"num_objects":4876,"num_object_clones":0,"num_object_copies":14628,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4876,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4876,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4876,"num_write_kb":312064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":4876}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":730,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.399035+0000","last_change":"2023-09-23T18:09:49.245223+0000","last_active":"2023-09-23T18:10:17.399035+0000","last_peered":"2023-09-23T18:10:17.399035+0000","last_clean":"2023-09-23T18:09:40.159105+0000","last_became_active":"2023-09-23T18:09:49.245223+0000","last_became_peered":"2023-09-23T18:09:49.245223+0000","last_unstale":"2023-09-23T18:10:17.399035+0000","last_undegraded":"2023-09-23T18:10:17.399035+0000","last_fullsized":"2023-09-23T18:09:49.241984+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:36:00.210097+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":158,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.401098+0000","last_change":"2023-09-23T18:10:01.179787+0000","last_active":"2023-09-23T18:10:17.401098+0000","last_peered":"2023-09-23T18:10:17.401098+0000","last_clean":"2023-09-23T18:10:17.401098+0000","last_became_active":"2023-09-23T18:10:01.174679+0000","last_became_peered":"2023-09-23T18:10:01.174679+0000","last_unstale":"2023-09-23T18:10:17.401098+0000","last_undegraded":"2023-09-23T18:10:17.401098+0000","last_fullsized":"2023-09-23T18:10:17.401098+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:27.491151+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":120,"reported_epoch":378,"state":"active+clean","last_fresh":"2023-09-23T18:10:21.462693+0000","last_change":"2023-09-23T18:06:05.398171+0000","last_active":"2023-09-23T18:10:21.462693+0000","last_peered":"2023-09-23T18:10:21.462693+0000","last_clean":"2023-09-23T18:10:21.462693+0000","last_became_active":"2023-09-23T18:06:05.370800+0000","last_became_peered":"2023-09-23T18:06:05.370800+0000","last_unstale":"2023-09-23T18:10:21.462693+0000","last_undegraded":"2023-09-23T18:10:21.462693+0000","last_fullsized":"2023-09-23T18:10:21.462693+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:59:46.783817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":189,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.400689+0000","last_change":"2023-09-23T18:09:49.265848+0000","last_active":"2023-09-23T18:10:17.400689+0000","last_peered":"2023-09-23T18:10:17.400689+0000","last_clean":"2023-09-23T18:09:40.154366+0000","last_became_active":"2023-09-23T18:09:49.265848+0000","last_became_peered":"2023-09-23T18:09:49.265848+0000","last_unstale":"2023-09-23T18:10:17.400689+0000","last_undegraded":"2023-09-23T18:10:17.400689+0000","last_fullsized":"2023-09-23T18:09:49.251764+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:23.255571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"378'4884","reported_seq":4914,"reported_epoch":378,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:23.007304+0000","last_change":"2023-09-23T18:09:49.264734+0000","last_active":"2023-09-23T18:10:23.007304+0000","last_peered":"2023-09-23T18:10:23.007304+0000","last_clean":"2023-09-23T18:09:48.206765+0000","last_became_active":"2023-09-23T18:09:49.264734+0000","last_became_peered":"2023-09-23T18:09:49.264734+0000","last_unstale":"2023-09-23T18:10:23.007304+0000","last_undegraded":"2023-09-23T18:09:49.257799+0000","last_fullsized":"2023-09-23T18:09:49.257700+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4884,"log_dups_size":0,"ondisk_log_size":4884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:31.109407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":320077824,"num_objects":4884,"num_object_clones":0,"num_object_copies":14652,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4884,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4884,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4884,"num_write_kb":312576,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":4884}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":630,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.400943+0000","last_change":"2023-09-23T18:09:49.258198+0000","last_active":"2023-09-23T18:10:17.400943+0000","last_peered":"2023-09-23T18:10:17.400943+0000","last_clean":"2023-09-23T18:05:06.479280+0000","last_became_active":"2023-09-23T18:09:49.258198+0000","last_became_peered":"2023-09-23T18:09:49.258198+0000","last_unstale":"2023-09-23T18:10:17.400943+0000","last_undegraded":"2023-09-23T18:10:17.400943+0000","last_fullsized":"2023-09-23T18:09:49.249014+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:05.551792+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":583,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.393445+0000","last_change":"2023-09-23T18:02:47.949907+0000","last_active":"2023-09-23T18:10:17.393445+0000","last_peered":"2023-09-23T18:10:17.393445+0000","last_clean":"2023-09-23T18:10:17.393445+0000","last_became_active":"2023-09-23T18:02:47.949336+0000","last_became_peered":"2023-09-23T18:02:47.949336+0000","last_unstale":"2023-09-23T18:10:17.393445+0000","last_undegraded":"2023-09-23T18:10:17.393445+0000","last_fullsized":"2023-09-23T18:10:17.393445+0000","mapping_epoch":249,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:50.572835+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":408,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399358+0000","last_change":"2023-09-23T18:06:45.939855+0000","last_active":"2023-09-23T18:10:17.399358+0000","last_peered":"2023-09-23T18:10:17.399358+0000","last_clean":"2023-09-23T18:10:17.399358+0000","last_became_active":"2023-09-23T18:06:45.924278+0000","last_became_peered":"2023-09-23T18:06:45.924278+0000","last_unstale":"2023-09-23T18:10:17.399358+0000","last_undegraded":"2023-09-23T18:10:17.399358+0000","last_fullsized":"2023-09-23T18:10:17.399358+0000","mapping_epoch":298,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:33:43.850375+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":175,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.400217+0000","last_change":"2023-09-23T18:09:49.245496+0000","last_active":"2023-09-23T18:10:17.400217+0000","last_peered":"2023-09-23T18:10:17.400217+0000","last_clean":"2023-09-23T18:09:40.158239+0000","last_became_active":"2023-09-23T18:09:49.245496+0000","last_became_peered":"2023-09-23T18:09:49.245496+0000","last_unstale":"2023-09-23T18:10:17.400217+0000","last_undegraded":"2023-09-23T18:10:17.400217+0000","last_fullsized":"2023-09-23T18:09:49.241846+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:14:46.672379+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":525,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.392583+0000","last_change":"2023-09-23T18:09:14.580901+0000","last_active":"2023-09-23T18:10:17.392583+0000","last_peered":"2023-09-23T18:10:17.392583+0000","last_clean":"2023-09-23T18:10:17.392583+0000","last_became_active":"2023-09-23T18:06:48.961304+0000","last_became_peered":"2023-09-23T18:06:48.961304+0000","last_unstale":"2023-09-23T18:10:17.392583+0000","last_undegraded":"2023-09-23T18:10:17.392583+0000","last_fullsized":"2023-09-23T18:10:17.392583+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:14.580819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:14.580819+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:50:04.095372+0000","scrub_duration":0.003313033,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":294,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.398920+0000","last_change":"2023-09-23T18:06:26.148865+0000","last_active":"2023-09-23T18:10:17.398920+0000","last_peered":"2023-09-23T18:10:17.398920+0000","last_clean":"2023-09-23T18:10:17.398920+0000","last_became_active":"2023-09-23T18:06:26.148517+0000","last_became_peered":"2023-09-23T18:06:26.148517+0000","last_unstale":"2023-09-23T18:10:17.398920+0000","last_undegraded":"2023-09-23T18:10:17.398920+0000","last_fullsized":"2023-09-23T18:10:17.398920+0000","mapping_epoch":292,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":293,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:38.141672+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":421,"reported_epoch":377,"state":"active+undersized","last_fresh":"2023-09-23T18:10:17.398584+0000","last_change":"2023-09-23T18:09:49.248926+0000","last_active":"2023-09-23T18:10:17.398584+0000","last_peered":"2023-09-23T18:10:17.398584+0000","last_clean":"2023-09-23T18:09:40.167682+0000","last_became_active":"2023-09-23T18:09:49.248926+0000","last_became_peered":"2023-09-23T18:09:49.248926+0000","last_unstale":"2023-09-23T18:10:17.398584+0000","last_undegraded":"2023-09-23T18:10:17.398584+0000","last_fullsized":"2023-09-23T18:09:49.239548+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":308,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:18.664900+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":520,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.392885+0000","last_change":"2023-09-23T18:09:16.604488+0000","last_active":"2023-09-23T18:10:17.392885+0000","last_peered":"2023-09-23T18:10:17.392885+0000","last_clean":"2023-09-23T18:10:17.392885+0000","last_became_active":"2023-09-23T18:06:54.055098+0000","last_became_peered":"2023-09-23T18:06:54.055098+0000","last_unstale":"2023-09-23T18:10:17.392885+0000","last_undegraded":"2023-09-23T18:10:17.392885+0000","last_fullsized":"2023-09-23T18:10:17.392885+0000","mapping_epoch":306,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.604423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.604423+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:45:28.598882+0000","scrub_duration":0.0014924669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":68,"reported_epoch":377,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:17.399729+0000","last_change":"2023-09-23T18:09:49.254308+0000","last_active":"2023-09-23T18:10:17.399729+0000","last_peered":"2023-09-23T18:10:17.399729+0000","last_clean":"2023-09-23T18:01:36.146652+0000","last_became_active":"2023-09-23T18:09:49.254308+0000","last_became_peered":"2023-09-23T18:09:49.254308+0000","last_unstale":"2023-09-23T18:10:17.399729+0000","last_undegraded":"2023-09-23T18:09:49.242698+0000","last_fullsized":"2023-09-23T18:09:49.242588+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:34:35.825451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":313,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399286+0000","last_change":"2023-09-23T18:06:05.375138+0000","last_active":"2023-09-23T18:10:17.399286+0000","last_peered":"2023-09-23T18:10:17.399286+0000","last_clean":"2023-09-23T18:10:17.399286+0000","last_became_active":"2023-09-23T18:06:05.374819+0000","last_became_peered":"2023-09-23T18:06:05.374819+0000","last_unstale":"2023-09-23T18:10:17.399286+0000","last_undegraded":"2023-09-23T18:10:17.399286+0000","last_fullsized":"2023-09-23T18:10:17.399286+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:58:43.000019+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":68,"reported_epoch":377,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:17.399617+0000","last_change":"2023-09-23T18:09:49.246801+0000","last_active":"2023-09-23T18:10:17.399617+0000","last_peered":"2023-09-23T18:10:17.399617+0000","last_clean":"2023-09-23T18:01:36.147388+0000","last_became_active":"2023-09-23T18:09:49.246801+0000","last_became_peered":"2023-09-23T18:09:49.246801+0000","last_unstale":"2023-09-23T18:10:17.399617+0000","last_undegraded":"2023-09-23T18:09:49.243160+0000","last_fullsized":"2023-09-23T18:09:49.241934+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:44:36.925478+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":1}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":18,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.418661+0000","last_change":"2023-09-23T18:10:01.178038+0000","last_active":"2023-09-23T18:10:17.418661+0000","last_peered":"2023-09-23T18:10:17.418661+0000","last_clean":"2023-09-23T18:10:17.418661+0000","last_became_active":"2023-09-23T18:10:01.177675+0000","last_became_peered":"2023-09-23T18:10:01.177675+0000","last_unstale":"2023-09-23T18:10:17.418661+0000","last_undegraded":"2023-09-23T18:10:17.418661+0000","last_fullsized":"2023-09-23T18:10:17.418661+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:24.038395+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":764,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.401055+0000","last_change":"2023-09-23T18:06:08.440953+0000","last_active":"2023-09-23T18:10:17.401055+0000","last_peered":"2023-09-23T18:10:17.401055+0000","last_clean":"2023-09-23T18:10:17.401055+0000","last_became_active":"2023-09-23T18:06:08.440604+0000","last_became_peered":"2023-09-23T18:06:08.440604+0000","last_unstale":"2023-09-23T18:10:17.401055+0000","last_undegraded":"2023-09-23T18:10:17.401055+0000","last_fullsized":"2023-09-23T18:10:17.401055+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:53.274049+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":180,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.393083+0000","last_change":"2023-09-23T18:06:51.098401+0000","last_active":"2023-09-23T18:10:17.393083+0000","last_peered":"2023-09-23T18:10:17.393083+0000","last_clean":"2023-09-23T18:10:17.393083+0000","last_became_active":"2023-09-23T18:06:51.097959+0000","last_became_peered":"2023-09-23T18:06:51.097959+0000","last_unstale":"2023-09-23T18:10:17.393083+0000","last_undegraded":"2023-09-23T18:10:17.393083+0000","last_fullsized":"2023-09-23T18:10:17.393083+0000","mapping_epoch":303,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:56:16.234414+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":19,"reported_epoch":377,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:10:17.415895+0000","last_change":"2023-09-23T18:10:02.307514+0000","last_active":"2023-09-23T18:10:17.415895+0000","last_peered":"2023-09-23T18:10:17.415895+0000","last_clean":"2023-09-23T18:10:17.415895+0000","last_became_active":"2023-09-23T18:10:02.307036+0000","last_became_peered":"2023-09-23T18:10:02.307036+0000","last_unstale":"2023-09-23T18:10:17.415895+0000","last_undegraded":"2023-09-23T18:10:17.415895+0000","last_fullsized":"2023-09-23T18:10:17.415895+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:19.665837+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,2],"acting":[0,2,1],"avail_no_missing":["0","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":711,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399011+0000","last_change":"2023-09-23T18:09:17.636095+0000","last_active":"2023-09-23T18:10:17.399011+0000","last_peered":"2023-09-23T18:10:17.399011+0000","last_clean":"2023-09-23T18:10:17.399011+0000","last_became_active":"2023-09-23T18:06:48.965429+0000","last_became_peered":"2023-09-23T18:06:48.965429+0000","last_unstale":"2023-09-23T18:10:17.399011+0000","last_undegraded":"2023-09-23T18:10:17.399011+0000","last_fullsized":"2023-09-23T18:10:17.399011+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:17.635974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:17.635974+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:06.792253+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":187,"reported_epoch":377,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:10:17.393192+0000","last_change":"2023-09-23T18:09:49.245379+0000","last_active":"2023-09-23T18:10:17.393192+0000","last_peered":"2023-09-23T18:10:17.393192+0000","last_clean":"2023-09-23T18:00:42.557373+0000","last_became_active":"2023-09-23T18:09:49.245379+0000","last_became_peered":"2023-09-23T18:09:49.245379+0000","last_unstale":"2023-09-23T18:10:17.393192+0000","last_undegraded":"2023-09-23T18:09:49.240279+0000","last_fullsized":"2023-09-23T18:09:49.240171+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:29.708404+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":2}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":398,"reported_epoch":377,"state":"active+clean","last_fresh":"2023-09-23T18:10:17.399967+0000","last_change":"2023-09-23T18:06:05.374027+0000","last_active":"2023-09-23T18:10:17.399967+0000","last_peered":"2023-09-23T18:10:17.399967+0000","last_clean":"2023-09-23T18:10:17.399967+0000","last_became_active":"2023-09-23T18:06:05.373670+0000","last_became_peered":"2023-09-23T18:06:05.373670+0000","last_unstale":"2023-09-23T18:10:17.399967+0000","last_undegraded":"2023-09-23T18:10:17.399967+0000","last_fullsized":"2023-09-23T18:10:17.399967+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:01.220735+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":84,"acting":86,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":44,"num_bytes_recovered":5692,"num_keys_recovered":20,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":84,"acting":86,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5131272192,"num_objects":78297,"num_object_clones":0,"num_object_copies":234891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":45385,"num_objects_misplaced":8585,"num_objects_unfound":0,"num_objects_dirty":78297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":78297,"num_write_kb":5011008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":760,"num_bytes_recovered":49807360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14049804288,"data_stored":14049804288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76497,"ondisk_log_size":76497,"up":39,"acting":38,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":102867,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":85,"num_read_kb":160,"num_write":42,"num_write_kb":421,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2330654,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":77,"acting":79,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":156,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":22,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001453,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2407528,"kb_used_data":2160160,"kb_used_omap":13,"kb_used_meta":247346,"kb_avail":91337624,"statfs":{"total":95995035648,"available":93529726976,"internally_reserved":0,"allocated":2212003840,"data_stored":2210735435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":253282408},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":0.89700000000000002,"5min":1.1080000000000001,"15min":0.78700000000000003},"min":{"1min":0.254,"5min":0.217,"15min":0.217},"max":{"1min":3.1930000000000001,"5min":9.3499999999999996,"15min":9.3499999999999996},"last":2.069},{"interface":"front","average":{"1min":1.1930000000000001,"5min":0.88600000000000001,"15min":1.1240000000000001},"min":{"1min":0.28399999999999997,"5min":0.25,"15min":0.23000000000000001},"max":{"1min":3.8159999999999998,"5min":4.8209999999999997,"15min":6.641},"last":3.052}]},{"osd":1,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.383,"5min":1.2430000000000001,"15min":0.94199999999999995},"min":{"1min":0.38900000000000001,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.1840000000000002,"5min":9.4909999999999997,"15min":9.4909999999999997},"last":1.754},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.0940000000000001,"15min":1.0880000000000001},"min":{"1min":0.25700000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":3.1549999999999998,"5min":6.9589999999999996,"15min":6.9589999999999996},"last":2.7629999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.0740000000000001,"5min":1.2370000000000001,"15min":1.034},"min":{"1min":0.35899999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":3.4039999999999999,"5min":6.6630000000000003,"15min":6.6630000000000003},"last":3.1680000000000001},{"interface":"front","average":{"1min":1.325,"5min":1.2470000000000001,"15min":1.016},"min":{"1min":0.33400000000000002,"5min":0.307,"15min":0.29399999999999998},"max":{"1min":3.653,"5min":7.9160000000000004,"15min":7.9160000000000004},"last":1.6279999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:09:28 2023","interfaces":[{"interface":"back","average":{"1min":1.8069999999999999,"5min":1.373,"15min":1.474},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.8710000000000004,"5min":7.0579999999999998,"15min":7.6070000000000002},"last":3.8050000000000002},{"interface":"front","average":{"1min":1.5,"5min":1.4570000000000001,"15min":1.173},"min":{"1min":0.317,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.7889999999999997,"5min":15.349,"15min":15.349},"last":1.944}]},{"osd":4,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.204,"5min":0.88600000000000001,"15min":1.0009999999999999},"min":{"1min":0.27500000000000002,"5min":0.25800000000000001,"15min":0.245},"max":{"1min":4.4669999999999996,"5min":6.9729999999999999,"15min":6.9729999999999999},"last":0.30599999999999999},{"interface":"front","average":{"1min":1.4330000000000001,"5min":0.90000000000000002,"15min":0.67200000000000004},"min":{"1min":0.31900000000000001,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":5.1059999999999999,"5min":5.1059999999999999,"15min":6.3479999999999999},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.0860000000000001,"15min":1.042},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.6280000000000001,"5min":8.4749999999999996,"15min":9.5999999999999996},"last":3.2149999999999999},{"interface":"front","average":{"1min":1.127,"5min":0.97899999999999998,"15min":1.095},"min":{"1min":0.35899999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.2839999999999998,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":3.1720000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.4019999999999999,"5min":1.032,"15min":0.84199999999999997},"min":{"1min":0.37,"5min":0.308,"15min":0.308},"max":{"1min":6.782,"5min":6.782,"15min":6.782},"last":0.32800000000000001},{"interface":"front","average":{"1min":1.1120000000000001,"5min":0.89400000000000002,"15min":1.081},"min":{"1min":0.35299999999999998,"5min":0.312,"15min":0.30399999999999999},"max":{"1min":3.1280000000000001,"5min":4.9539999999999997,"15min":9.5519999999999996},"last":0.317}]}]},{"osd":6,"up_from":163,"seq":700079669390,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2514784,"kb_used_data":2234008,"kb_used_omap":7,"kb_used_meta":280760,"kb_avail":91230368,"statfs":{"total":95995035648,"available":93419896832,"internally_reserved":0,"allocated":2287624192,"data_stored":2286363862,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":287499044},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":0.999,"15min":1.181},"min":{"1min":0.34899999999999998,"5min":0.23100000000000001,"15min":0.215},"max":{"1min":6.3550000000000004,"5min":8.4019999999999992,"15min":15.407},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.349,"5min":1.238,"15min":1.6879999999999999},"min":{"1min":0.34100000000000003,"5min":0.29199999999999998,"15min":0.23100000000000001},"max":{"1min":4.1550000000000002,"5min":15.147,"15min":15.147},"last":5.8339999999999996}]},{"osd":1,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8500000000000001,"5min":1.1379999999999999,"15min":1.5189999999999999},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":6.4240000000000004,"5min":6.5309999999999997,"15min":14.233000000000001},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.1519999999999999,"5min":1.0700000000000001,"15min":1.6200000000000001},"min":{"1min":0.30099999999999999,"5min":0.26000000000000001,"15min":0.248},"max":{"1min":4.194,"5min":4.194,"15min":14.789},"last":6.0289999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.718,"5min":1.137,"15min":1.75},"min":{"1min":0.39800000000000002,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":6.9989999999999997,"5min":8.6140000000000008,"15min":11.210000000000001},"last":5.8789999999999996},{"interface":"front","average":{"1min":1.8879999999999999,"5min":1.224,"15min":1.9359999999999999},"min":{"1min":0.40200000000000002,"5min":0.248,"15min":0.245},"max":{"1min":4.7220000000000004,"5min":8.6300000000000008,"15min":11.223000000000001},"last":2.8919999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:09:31 2023","interfaces":[{"interface":"back","average":{"1min":1.9079999999999999,"5min":1.401,"15min":1.1830000000000001},"min":{"1min":0.36399999999999999,"5min":0.313,"15min":0.308},"max":{"1min":8.5449999999999999,"5min":8.5449999999999999,"15min":8.5449999999999999},"last":2.919},{"interface":"front","average":{"1min":1.236,"5min":1.359,"15min":1.2509999999999999},"min":{"1min":0.42099999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":4.2699999999999996,"5min":7.484,"15min":7.484},"last":6.0590000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.484,"5min":0.92500000000000004,"15min":0.92900000000000005},"min":{"1min":0.28399999999999997,"5min":0.254,"15min":0.23000000000000001},"max":{"1min":4.6749999999999998,"5min":8.0280000000000005,"15min":9.9109999999999996},"last":2.8319999999999999},{"interface":"front","average":{"1min":1.4830000000000001,"5min":1.056,"15min":0.78500000000000003},"min":{"1min":0.35099999999999998,"5min":0.27200000000000002,"15min":0.24099999999999999},"max":{"1min":6.2949999999999999,"5min":8.1159999999999997,"15min":8.3000000000000007},"last":3.2010000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":0.99299999999999999,"15min":0.89400000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7219999999999995,"5min":9.7219999999999995,"15min":11.869},"last":1.681},{"interface":"front","average":{"1min":1.5289999999999999,"5min":0.92200000000000004,"15min":0.80100000000000005},"min":{"1min":0.30599999999999999,"5min":0.251,"15min":0.251},"max":{"1min":10.436,"5min":10.436,"15min":10.436},"last":1.746}]},{"osd":7,"last update":"Sat Sep 23 18:09:37 2023","interfaces":[{"interface":"back","average":{"1min":2.0880000000000001,"5min":1.0229999999999999,"15min":1.0029999999999999},"min":{"1min":0.378,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":13.460000000000001,"5min":13.460000000000001,"15min":13.460000000000001},"last":2.8620000000000001},{"interface":"front","average":{"1min":1.917,"5min":1.139,"15min":0.92000000000000004},"min":{"1min":0.41999999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.1470000000000002,"5min":10.411,"15min":10.411},"last":0.46200000000000002}]}]},{"osd":1,"up_from":98,"seq":420906795207,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2803564,"kb_used_data":2166564,"kb_used_omap":7,"kb_used_meta":636984,"kb_avail":90941588,"statfs":{"total":95995035648,"available":93124186112,"internally_reserved":0,"allocated":2218561536,"data_stored":2216827544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":652272317},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":2.0129999999999999,"5min":1.413,"15min":1.2390000000000001},"min":{"1min":0.184,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":14.567,"5min":14.567,"15min":15.529999999999999},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.4099999999999999,"5min":2.3100000000000001,"15min":1.651},"min":{"1min":0.25,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":6.774,"5min":82.244,"15min":82.244},"last":10.992000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":0.92100000000000004,"5min":1.407,"15min":1.613},"min":{"1min":0.31,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":2.6549999999999998,"5min":10.531000000000001,"15min":16.347999999999999},"last":12.717000000000001},{"interface":"front","average":{"1min":0.99399999999999999,"5min":1.2569999999999999,"15min":1.286},"min":{"1min":0.245,"5min":0.245,"15min":0.21299999999999999},"max":{"1min":4.6890000000000001,"5min":7.641,"15min":11.305999999999999},"last":0.48699999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:09:34 2023","interfaces":[{"interface":"back","average":{"1min":1.137,"5min":1.655,"15min":0.92800000000000005},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":4.8140000000000001,"5min":11.164,"15min":11.164},"last":2.1030000000000002},{"interface":"front","average":{"1min":1.1559999999999999,"5min":1.468,"15min":1.131},"min":{"1min":0.317,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":6.6950000000000003,"5min":11.125999999999999,"15min":11.125999999999999},"last":10.141}]},{"osd":4,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.637,"5min":1.3700000000000001,"15min":1.339},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.28199999999999997},"max":{"1min":4.806,"5min":8.3919999999999995,"15min":26.568999999999999},"last":0.83899999999999997},{"interface":"front","average":{"1min":2.141,"5min":1.4019999999999999,"15min":1.294},"min":{"1min":0.29999999999999999,"5min":0.28599999999999998,"15min":0.27000000000000002},"max":{"1min":6.6280000000000001,"5min":7.9260000000000002,"15min":26.635000000000002},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.7869999999999999,"15min":1.4379999999999999},"min":{"1min":0.38900000000000001,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":6.7489999999999997,"5min":11.638999999999999,"15min":11.638999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.579,"15min":1.393},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":6.5609999999999999,"5min":8.4309999999999992,"15min":9.5519999999999996},"last":1.962}]},{"osd":6,"last update":"Sat Sep 23 18:09:52 2023","interfaces":[{"interface":"back","average":{"1min":1.4470000000000001,"5min":2.1339999999999999,"15min":1.895},"min":{"1min":0.40300000000000002,"5min":0.372,"15min":0.317},"max":{"1min":4.7690000000000001,"5min":89.566000000000003,"15min":89.566000000000003},"last":0.88700000000000001},{"interface":"front","average":{"1min":1.032,"5min":2.1200000000000001,"15min":1.7170000000000001},"min":{"1min":0.42499999999999999,"5min":0.38,"15min":0.34699999999999998},"max":{"1min":3.1600000000000001,"5min":90.608999999999995,"15min":90.608999999999995},"last":10.063000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:09:44 2023","interfaces":[{"interface":"back","average":{"1min":1.327,"5min":1.2749999999999999,"15min":0.93000000000000005},"min":{"1min":0.45300000000000001,"5min":0.39800000000000002,"15min":0.33900000000000002},"max":{"1min":4.7590000000000003,"5min":7.1219999999999999,"15min":7.1219999999999999},"last":0.72399999999999998},{"interface":"front","average":{"1min":1.194,"5min":1.4179999999999999,"15min":1.635},"min":{"1min":0.45600000000000002,"5min":0.45000000000000001,"15min":0.40999999999999998},"max":{"1min":3.4969999999999999,"5min":7.306,"15min":9.5709999999999997},"last":10.109}]}]},{"osd":0,"up_from":93,"seq":399431958735,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":834256,"kb_used_data":317512,"kb_used_omap":7,"kb_used_meta":516728,"kb_avail":92910896,"statfs":{"total":95995035648,"available":95140757504,"internally_reserved":0,"allocated":325132288,"data_stored":323361888,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":529130212},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:10:24 2023","interfaces":[{"interface":"back","average":{"1min":1.7290000000000001,"5min":1.677,"15min":1.7170000000000001},"min":{"1min":0.252,"5min":0.217,"15min":0.17899999999999999},"max":{"1min":8.4920000000000009,"5min":15.836,"15min":18.154},"last":4.0179999999999998},{"interface":"front","average":{"1min":1.6739999999999999,"5min":1.4419999999999999,"15min":1.514},"min":{"1min":0.25800000000000001,"5min":0.246,"15min":0.17399999999999999},"max":{"1min":5.2750000000000004,"5min":14.468999999999999,"15min":14.468999999999999},"last":0.25800000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:09:48 2023","interfaces":[{"interface":"back","average":{"1min":0.84899999999999998,"5min":1.4059999999999999,"15min":1.716},"min":{"1min":0.25700000000000001,"5min":0.24399999999999999,"15min":0.20499999999999999},"max":{"1min":3.7909999999999999,"5min":10.48,"15min":20.882000000000001},"last":0.86499999999999999},{"interface":"front","average":{"1min":0.97099999999999997,"5min":1.325,"15min":1.6220000000000001},"min":{"1min":0.33300000000000002,"5min":0.23000000000000001,"15min":0.217},"max":{"1min":4.4249999999999998,"5min":10.84,"15min":15.064},"last":0.56200000000000006}]},{"osd":3,"last update":"Sat Sep 23 18:09:27 2023","interfaces":[{"interface":"back","average":{"1min":1.5880000000000001,"5min":1.7769999999999999,"15min":1.583},"min":{"1min":0.25600000000000001,"5min":0.245,"15min":0.20499999999999999},"max":{"1min":9.8390000000000004,"5min":12.779999999999999,"15min":12.779999999999999},"last":6.21},{"interface":"front","average":{"1min":1.575,"5min":1.613,"15min":1.369},"min":{"1min":0.373,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.5019999999999998,"5min":15.198,"15min":15.198},"last":6.2990000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.268,"5min":1.3169999999999999,"15min":1.4330000000000001},"min":{"1min":0.44400000000000001,"5min":0.32100000000000001,"15min":0.251},"max":{"1min":3.782,"5min":8.4109999999999996,"15min":15.279999999999999},"last":6.3680000000000003},{"interface":"front","average":{"1min":0.82599999999999996,"5min":1.5740000000000001,"15min":1.542},"min":{"1min":0.371,"5min":0.34999999999999998,"15min":0.247},"max":{"1min":3.7629999999999999,"5min":15.477,"15min":18.106999999999999},"last":0.90700000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:08:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.7230000000000001,"15min":1.5600000000000001},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":9.2010000000000005,"5min":10.194000000000001,"15min":14.720000000000001},"last":1.478},{"interface":"front","average":{"1min":1.409,"5min":1.512,"15min":1.3140000000000001},"min":{"1min":0.33300000000000002,"5min":0.32600000000000001,"15min":0.254},"max":{"1min":5.1230000000000002,"5min":10.363,"15min":12.794},"last":2.1160000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:09:48 2023","interfaces":[{"interface":"back","average":{"1min":1.4530000000000001,"5min":2.0489999999999999,"15min":1.998},"min":{"1min":0.38900000000000001,"5min":0.35199999999999998,"15min":0.34599999999999997},"max":{"1min":4.2489999999999997,"5min":58.457999999999998,"15min":58.457999999999998},"last":6.2569999999999997},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.9690000000000001,"15min":2.04},"min":{"1min":0.49299999999999999,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":4.3470000000000004,"5min":58.354999999999997,"15min":58.354999999999997},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:09:35 2023","interfaces":[{"interface":"back","average":{"1min":1.635,"5min":1.4359999999999999,"15min":1.1140000000000001},"min":{"1min":0.41599999999999998,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":7.4790000000000001,"5min":10.416,"15min":10.416},"last":3.7610000000000001},{"interface":"front","average":{"1min":1.548,"5min":1.536,"15min":1.3089999999999999},"min":{"1min":0.443,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":5.8760000000000003,"5min":8.923,"15min":8.923},"last":6.3330000000000002}]}]},{"osd":2,"up_from":103,"seq":442381631682,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2150012,"kb_used_data":1574644,"kb_used_omap":11,"kb_used_meta":575348,"kb_avail":91595140,"statfs":{"total":95995035648,"available":93793423360,"internally_reserved":0,"allocated":1612435456,"data_stored":1610684810,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":589156727},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3149999999999999,"5min":1.8240000000000001,"15min":1.9079999999999999},"min":{"1min":0.22,"5min":0.20799999999999999,"15min":0.17199999999999999},"max":{"1min":11.946,"5min":36.656999999999996,"15min":36.656999999999996},"last":2.9390000000000001},{"interface":"front","average":{"1min":1.393,"5min":1.7410000000000001,"15min":1.8109999999999999},"min":{"1min":0.26000000000000001,"5min":0.23599999999999999,"15min":0.215},"max":{"1min":11.978,"5min":36.697000000000003,"15min":36.697000000000003},"last":2.8820000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.8160000000000001,"5min":1.5549999999999999,"15min":1.839},"min":{"1min":0.26500000000000001,"5min":0.23599999999999999,"15min":0.215},"max":{"1min":12.106,"5min":12.106,"15min":20.995000000000001},"last":3.0739999999999998},{"interface":"front","average":{"1min":1.0569999999999999,"5min":1.2350000000000001,"15min":1.5449999999999999},"min":{"1min":0.28399999999999997,"5min":0.252,"15min":0.193},"max":{"1min":5.7300000000000004,"5min":8.3889999999999993,"15min":15.945},"last":3.4470000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:09:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":1.764,"15min":1.1759999999999999},"min":{"1min":0.313,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":5.5960000000000001,"5min":37.500999999999998,"15min":37.500999999999998},"last":3.4180000000000001},{"interface":"front","average":{"1min":1.0369999999999999,"5min":1.95,"15min":1.113},"min":{"1min":0.29199999999999998,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":6.6239999999999997,"5min":34.915999999999997,"15min":34.915999999999997},"last":5.5330000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.083,"5min":1.4239999999999999,"15min":1.417},"min":{"1min":0.31900000000000001,"5min":0.311,"15min":0.26500000000000001},"max":{"1min":5.0880000000000001,"5min":13.016999999999999,"15min":13.016999999999999},"last":3.3940000000000001},{"interface":"front","average":{"1min":1.7350000000000001,"5min":1.381,"15min":1.431},"min":{"1min":0.33300000000000002,"5min":0.28699999999999998,"15min":0.26700000000000002},"max":{"1min":12.084,"5min":12.084,"15min":20.920000000000002},"last":2.9140000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.1890000000000001,"5min":1.2969999999999999,"15min":1.4219999999999999},"min":{"1min":0.33000000000000002,"5min":0.32100000000000001,"15min":0.315},"max":{"1min":7.2389999999999999,"5min":8.1760000000000002,"15min":20.738},"last":0.85699999999999998},{"interface":"front","average":{"1min":1.167,"5min":1.5069999999999999,"15min":1.534},"min":{"1min":0.33600000000000002,"5min":0.32900000000000001,"15min":0.26700000000000002},"max":{"1min":3.1499999999999999,"5min":12.103999999999999,"15min":15.496},"last":5.774}]},{"osd":6,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.103,"5min":1.3779999999999999,"15min":1.7509999999999999},"min":{"1min":0.38600000000000001,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":5.1100000000000003,"5min":13.065,"15min":13.065},"last":3.3439999999999999},{"interface":"front","average":{"1min":1.897,"5min":1.526,"15min":1.9790000000000001},"min":{"1min":0.46700000000000003,"5min":0.38300000000000001,"15min":0.33900000000000002},"max":{"1min":12.032999999999999,"5min":12.032999999999999,"15min":20.893000000000001},"last":2.992}]},{"osd":7,"last update":"Sat Sep 23 18:09:36 2023","interfaces":[{"interface":"back","average":{"1min":1.425,"5min":1.3899999999999999,"15min":1.3879999999999999},"min":{"1min":0.42699999999999999,"5min":0.39200000000000002,"15min":0.39200000000000002},"max":{"1min":5.6449999999999996,"5min":7.4989999999999997,"15min":12.051},"last":3.375},{"interface":"front","average":{"1min":2.0390000000000001,"5min":1.7370000000000001,"15min":1.048},"min":{"1min":0.48399999999999999,"5min":0.45100000000000001,"15min":0.309},"max":{"1min":12.058,"5min":12.058,"15min":12.058},"last":2.9700000000000002}]}]},{"osd":3,"up_from":237,"seq":1017907249249,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2154184,"kb_used_data":1832832,"kb_used_omap":3,"kb_used_meta":321340,"kb_avail":91590968,"statfs":{"total":95995035648,"available":93789151232,"internally_reserved":0,"allocated":1876819968,"data_stored":1874886498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":329052904},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.008,"15min":0.89400000000000002},"min":{"1min":0.24399999999999999,"5min":0.20599999999999999,"15min":0.20599999999999999},"max":{"1min":4.5679999999999996,"5min":8.5440000000000005,"15min":8.5440000000000005},"last":0.34200000000000003},{"interface":"front","average":{"1min":1.004,"5min":1.532,"15min":1.1850000000000001},"min":{"1min":0.23799999999999999,"5min":0.22,"15min":0.20699999999999999},"max":{"1min":9.1999999999999993,"5min":14.885999999999999,"15min":14.885999999999999},"last":1.9059999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.982,"5min":1.887,"15min":1.29},"min":{"1min":0.26700000000000002,"5min":0.25900000000000001,"15min":0.245},"max":{"1min":14.241,"5min":21.931999999999999,"15min":21.931999999999999},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.3959999999999999,"5min":1.3140000000000001,"15min":1.0449999999999999},"min":{"1min":0.29199999999999998,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":5.008,"5min":13.631,"15min":13.631},"last":0.48899999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.0329999999999999,"15min":0.90100000000000002},"min":{"1min":0.29099999999999998,"5min":0.247,"15min":0.22600000000000001},"max":{"1min":4.5899999999999999,"5min":8.2200000000000006,"15min":8.2200000000000006},"last":0.435},{"interface":"front","average":{"1min":1.1890000000000001,"5min":1.669,"15min":1.333},"min":{"1min":0.30499999999999999,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":8.0090000000000003,"5min":21.988,"15min":21.988},"last":0.40999999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.885,"15min":1.335},"min":{"1min":0.33800000000000002,"5min":0.309,"15min":0.29299999999999998},"max":{"1min":7.5540000000000003,"5min":37.231000000000002,"15min":37.231000000000002},"last":0.46400000000000002},{"interface":"front","average":{"1min":1.246,"5min":1.819,"15min":1.294},"min":{"1min":0.38300000000000001,"5min":0.33400000000000002,"15min":0.30499999999999999},"max":{"1min":4.673,"5min":37.381,"15min":37.381},"last":0.44900000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.4570000000000001,"15min":1.3069999999999999},"min":{"1min":0.34699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":9.3170000000000002,"5min":12.743,"15min":12.743},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.1279999999999999,"15min":1.147},"min":{"1min":0.376,"5min":0.315,"15min":0.315},"max":{"1min":4.601,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.435}]},{"osd":6,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":2.3290000000000002,"15min":1.4930000000000001},"min":{"1min":0.41199999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.657,"5min":115.54000000000001,"15min":115.54000000000001},"last":0.55200000000000005},{"interface":"front","average":{"1min":1.101,"5min":2.6560000000000001,"15min":1.835},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":7.5860000000000003,"5min":115.691,"15min":115.691},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.7110000000000001,"5min":1.4950000000000001,"15min":1.512},"min":{"1min":0.501,"5min":0.39600000000000002,"15min":0.377},"max":{"1min":5.5179999999999998,"5min":12.821,"15min":12.821},"last":0.52000000000000002},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.405,"15min":1.512},"min":{"1min":0.442,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":5.6159999999999997,"5min":8.6039999999999992,"15min":8.6039999999999992},"last":0.60999999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019199,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2269800,"kb_used_data":1904992,"kb_used_omap":0,"kb_used_meta":364800,"kb_avail":91475352,"statfs":{"total":95995035648,"available":93670760448,"internally_reserved":0,"allocated":1950711808,"data_stored":1949443666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":373555200},"hb_peers":[0,1,2,3,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3],"upper_bound":4},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:09:45 2023","interfaces":[{"interface":"back","average":{"1min":0.95999999999999996,"5min":1.1060000000000001,"15min":1.0669999999999999},"min":{"1min":0.27600000000000002,"5min":0.26000000000000001,"15min":0.20100000000000001},"max":{"1min":6.5090000000000003,"5min":13.375999999999999,"15min":14.164},"last":9.2759999999999998},{"interface":"front","average":{"1min":1.369,"5min":1.222,"15min":1.3080000000000001},"min":{"1min":0.30099999999999999,"5min":0.28799999999999998,"15min":0.22600000000000001},"max":{"1min":7.5609999999999999,"5min":7.5609999999999999,"15min":11.372},"last":8.3490000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:10:18 2023","interfaces":[{"interface":"back","average":{"1min":3.9870000000000001,"5min":1.7390000000000001,"15min":1.4910000000000001},"min":{"1min":0.371,"5min":0.249,"15min":0.249},"max":{"1min":42.929000000000002,"5min":42.929000000000002,"15min":42.929000000000002},"last":2.4060000000000001},{"interface":"front","average":{"1min":3.4630000000000001,"5min":1.665,"15min":1.5129999999999999},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.23100000000000001},"max":{"1min":43.494999999999997,"5min":43.494999999999997,"15min":43.494999999999997},"last":2.5680000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":1.431,"15min":1.4139999999999999},"min":{"1min":0.38200000000000001,"5min":0.28799999999999998,"15min":0.26600000000000001},"max":{"1min":32.887999999999998,"5min":32.887999999999998,"15min":32.887999999999998},"last":2.6099999999999999},{"interface":"front","average":{"1min":2.661,"5min":1.712,"15min":1.444},"min":{"1min":0.34899999999999998,"5min":0.314,"15min":0.27100000000000002},"max":{"1min":29.103999999999999,"5min":29.103999999999999,"15min":29.103999999999999},"last":2.6299999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:09:28 2023","interfaces":[{"interface":"back","average":{"1min":3.141,"5min":2.2839999999999998,"15min":1.6240000000000001},"min":{"1min":0.37,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":28.754999999999999,"5min":29.154,"15min":29.154},"last":2.5310000000000001},{"interface":"front","average":{"1min":2.645,"5min":2.0920000000000001,"15min":1.6000000000000001},"min":{"1min":0.36699999999999999,"5min":0.34699999999999998,"15min":0.34200000000000003},"max":{"1min":29.213000000000001,"5min":29.216000000000001,"15min":29.216000000000001},"last":9.343}]},{"osd":5,"last update":"Sat Sep 23 18:08:51 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.036,"15min":0.81799999999999995},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.5279999999999996,"5min":9.2959999999999994,"15min":9.9819999999999993},"last":4.6429999999999998},{"interface":"front","average":{"1min":1.2649999999999999,"5min":1.1299999999999999,"15min":0.98199999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":9.1609999999999996,"5min":9.1999999999999993,"15min":11.170999999999999},"last":0.94799999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:09:51 2023","interfaces":[{"interface":"back","average":{"1min":1.087,"5min":0.78000000000000003,"15min":0.91700000000000004},"min":{"1min":0.315,"5min":0.27000000000000002,"15min":0.23599999999999999},"max":{"1min":4.3410000000000002,"5min":5.8310000000000004,"15min":9.0990000000000002},"last":2.3740000000000001},{"interface":"front","average":{"1min":1.1719999999999999,"5min":0.84999999999999998,"15min":0.99199999999999999},"min":{"1min":0.41899999999999998,"5min":0.312,"15min":0.28399999999999997},"max":{"1min":3.4329999999999998,"5min":4.9740000000000002,"15min":10.725},"last":2.4340000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:09:40 2023","interfaces":[{"interface":"back","average":{"1min":1.7270000000000001,"5min":1.159,"15min":1.0569999999999999},"min":{"1min":0.437,"5min":0.314,"15min":0.314},"max":{"1min":9.2509999999999994,"5min":13.298999999999999,"15min":13.298999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.345,"5min":1.0189999999999999,"15min":1.2030000000000001},"min":{"1min":0.40000000000000002,"5min":0.32400000000000001,"15min":0.307},"max":{"1min":3.7810000000000001,"5min":10.192,"15min":10.192},"last":2.266}]}]},{"osd":5,"up_from":38,"seq":163208757559,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1802008,"kb_used_data":1477840,"kb_used_omap":0,"kb_used_meta":324160,"kb_avail":91943144,"statfs":{"total":95995035648,"available":94149779456,"internally_reserved":0,"allocated":1513308160,"data_stored":1512084138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331939840},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.77900000000000003,"15min":1.069},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.217},"max":{"1min":2.5649999999999999,"5min":5.4539999999999997,"15min":21.376999999999999},"last":1.3560000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.94799999999999995,"15min":1.3},"min":{"1min":0.249,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":3.5910000000000002,"5min":8.5099999999999998,"15min":21.388999999999999},"last":3.379}]},{"osd":1,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.038,"15min":0.96699999999999997},"min":{"1min":0.23899999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.6349999999999998,"5min":10.547000000000001,"15min":10.853999999999999},"last":1.385},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.1299999999999999,"15min":1.1759999999999999},"min":{"1min":0.26600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.625,"5min":8.9749999999999996,"15min":10.206},"last":3.4239999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.002,"15min":1.4119999999999999},"min":{"1min":0.38600000000000001,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":5.0570000000000004,"5min":5.5309999999999997,"15min":38.606999999999999},"last":1.962},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.0549999999999999,"15min":1.302},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6000000000000001,"5min":10.065,"15min":39.670999999999999},"last":1.335}]},{"osd":3,"last update":"Sat Sep 23 18:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":1.266,"15min":0.96099999999999997},"min":{"1min":0.32200000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6499999999999999,"5min":9.2309999999999999,"15min":9.2309999999999999},"last":5.194},{"interface":"front","average":{"1min":1.6930000000000001,"5min":1.3779999999999999,"15min":0.94699999999999995},"min":{"1min":0.35599999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.362,"5min":13.446999999999999,"15min":13.446999999999999},"last":1.988}]},{"osd":4,"last update":"Sat Sep 23 18:08:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.79800000000000004,"15min":0.83699999999999997},"min":{"1min":0.29799999999999999,"5min":0.25700000000000001,"15min":0.224},"max":{"1min":3.7149999999999999,"5min":8.6229999999999993,"15min":10.092000000000001},"last":6.8600000000000003},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.94699999999999995,"15min":0.83999999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.23999999999999999},"max":{"1min":9.1750000000000007,"5min":9.1750000000000007,"15min":9.1750000000000007},"last":2.1890000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:08:59 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":0.86299999999999999,"15min":1.1930000000000001},"min":{"1min":0.314,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":6.032,"5min":6.032,"15min":11.287000000000001},"last":1.9279999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.872,"15min":1.0880000000000001},"min":{"1min":0.33100000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.6709999999999998,"5min":6.6920000000000002,"15min":8.266},"last":1.3089999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:09:38 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":0.99299999999999999,"15min":1.302},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.1849999999999996,"5min":8.0999999999999996,"15min":8.8510000000000009},"last":2.7679999999999998},{"interface":"front","average":{"1min":1.752,"5min":1.022,"15min":1.099},"min":{"1min":0.46000000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.2400000000000002,"5min":17.684000000000001,"15min":17.684000000000001},"last":6.6449999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327090176,"data_stored":327090176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2214920192,"data_stored":2214920192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1609629696,"data_stored":1609629696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1879638016,"data_stored":1879638016,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1954611200,"data_stored":1954611200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1551040512,"data_stored":1551040512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2288910336,"data_stored":2288910336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2223964160,"data_stored":2223964160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:10:28.845 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 41 2023-09-23T18:10:28.865 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:28.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:28.876 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:28.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:10:28.977 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:29.031 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 58 16 82868 82852 89.221 49.625 0.0261364 0.0112019 2023-09-23T18:10:29.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 45397/234972 objects degraded (19.320%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:29.099 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:28 smithi055 ceph-mon[179668]: from='client.36252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:10:29.099 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 45397/234972 objects degraded (19.320%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:29.099 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:28 smithi055 ceph-mon[184174]: from='client.36252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:10:29.108 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:29.194 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:29.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:29.202 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:29.302 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:29.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 45397/234972 objects degraded (19.320%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:29.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:28 smithi099 ceph-mon[167076]: from='client.36252 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:10:29.402 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:29.513 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:10:29.550 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:10:29.551 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:29.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:29.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:29.595 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:10:29.653 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:10:29.656 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:29.687 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:10:29.697 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:10:29.734 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:10:29.764 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:29.776 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:10:29.776 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:10:29.776 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:10:29.776 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:10:29.776 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:10:29.776 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:10:29.776 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:10:29.776 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:10:29.777 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:10:29.835 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:29.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:29.845 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:29.945 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:29.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:29 smithi055 ceph-mon[179668]: pgmap v933: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 46926/243543 objects degraded (19.268%); 8585/243543 objects misplaced (3.525%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:10:29.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3172245701' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:10:29.994 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2082228957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:30.008 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:30.040 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 59 16 83663 83647 88.5376 49.6875 0.0243935 0.0112889 2023-09-23T18:10:30.048 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:10:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:10:30.226 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:10:30.242 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:30.250 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:30.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:29 smithi055 ceph-mon[184174]: pgmap v933: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 46926/243543 objects degraded (19.268%); 8585/243543 objects misplaced (3.525%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:10:30.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3172245701' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:10:30.250 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2082228957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:30.251 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:30.298 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:30.303 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:29 smithi099 ceph-mon[167076]: pgmap v933: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.0 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 59 MiB/s wr, 941 op/s; 46926/243543 objects degraded (19.268%); 8585/243543 objects misplaced (3.525%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:10:30.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3172245701' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:10:30.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2082228957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:30.352 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:30.431 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:30.604 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:30.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:30.612 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:10:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:10:30] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.43.0" 2023-09-23T18:10:30.612 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:30.714 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:30.794 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:30.875 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:30.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:30.882 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:30.983 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:31.040 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:10:31.041295+0000 min lat: 0.00131331 max lat: 2.26605 avg lat: 0.0113543 2023-09-23T18:10:31.040 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:10:31.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 60 16 84591 84575 88.0289 58 0.0132948 0.0113543 2023-09-23T18:10:31.041 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:31.155 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:10:31.156 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:10:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:31 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3172245701' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:10:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:31 smithi055 ceph-mon[179668]: osdmap e379: 8 total, 7 up, 8 in 2023-09-23T18:10:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:31 smithi055 ceph-mon[179668]: failed to encode map e379 with expected crc 2023-09-23T18:10:31.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:31 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:31.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3172245701' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:10:31.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:31 smithi055 ceph-mon[184174]: osdmap e379: 8 total, 7 up, 8 in 2023-09-23T18:10:31.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:31 smithi055 ceph-mon[184174]: failed to encode map e379 with expected crc 2023-09-23T18:10:31.182 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:31 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:31.182 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3172245701' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:10:31.182 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:31 smithi099 ceph-mon[167076]: osdmap e379: 8 total, 7 up, 8 in 2023-09-23T18:10:31.183 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:31 smithi099 ceph-mon[167076]: failed to encode map e379 with expected crc 2023-09-23T18:10:31.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:10:32.040 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 61 16 85538 85522 87.5563 59.1875 0.0105847 0.011416 2023-09-23T18:10:32.227 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:10:32.267 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:32.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:32.277 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:32.277 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:32 smithi055 ceph-mon[179668]: pgmap v935: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 47431/248391 objects degraded (19.095%); 8844/248391 objects misplaced (3.561%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:10:32.278 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:32 smithi055 ceph-mon[184174]: pgmap v935: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 47431/248391 objects degraded (19.095%); 8844/248391 objects misplaced (3.561%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:10:32.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:32 smithi099 ceph-mon[167076]: pgmap v935: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.1 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 61 MiB/s wr, 977 op/s; 47431/248391 objects degraded (19.095%); 8844/248391 objects misplaced (3.561%); 2.6 MiB/s, 42 objects/s recovering 2023-09-23T18:10:32.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:32.448 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:32.495 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:32.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:32.499 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:32.600 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:32.633 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:10:32.647 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:32.675 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:10:32.742 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:32.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:32.746 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:32.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:32.888 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:32.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:32.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:32.983 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:33.040 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 62 16 86381 86365 86.9942 52.6875 0.0342038 0.0114892 2023-09-23T18:10:33.084 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:33.202 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:33.258 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:33.294 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:33.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:33.304 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:33.405 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:33.499 INFO:teuthology.orchestra.run.smithi055.stderr:Error EINVAL: specified pgp_num 41 > pg_num 31 2023-09-23T18:10:33.511 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:33.607 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:33.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:33.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:33.721 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:33.781 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:33.950 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:33.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:33.957 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:34.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 63 16 87189 87173 86.415 50.5 0.00952327 0.0115662 2023-09-23T18:10:34.059 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:34.117 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:34 smithi055 ceph-mon[184174]: pgmap v936: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 48279/253992 objects degraded (19.008%); 8844/253992 objects misplaced (3.482%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:10:34.117 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:34 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 48279/253992 objects degraded (19.008%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:34.117 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3543608099' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T18:10:34.117 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:34 smithi055 ceph-mon[179668]: pgmap v936: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 48279/253992 objects degraded (19.008%); 8844/253992 objects misplaced (3.482%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:10:34.118 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:34 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 48279/253992 objects degraded (19.008%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:34.118 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3543608099' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T18:10:34.126 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:34.178 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:10:34.178 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:10:34.195 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:34.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:34.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:34.209 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:34 smithi099 ceph-mon[167076]: pgmap v936: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 57 MiB/s wr, 904 op/s; 48279/253992 objects degraded (19.008%); 8844/253992 objects misplaced (3.482%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:10:34.209 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:34 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 48279/253992 objects degraded (19.008%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:34.209 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3543608099' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "41"}]: dispatch 2023-09-23T18:10:34.304 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:34.379 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:34.496 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:34.512 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:34.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:34.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:34.536 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T18:10:34.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:10:34.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:34.666 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:34.720 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:34.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:34.735 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:34.835 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:34.914 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:35.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 64 16 88057 88041 85.9128 54.25 0.0144981 0.0116346 2023-09-23T18:10:35.074 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:35.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:35.081 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:35.182 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:35.235 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:35.266 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:35 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:35.266 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:35 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:35.287 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:35.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:35.293 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:35.324 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:10:35.394 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:35.430 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:35.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:35.495 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:35 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:35.495 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:35 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:35.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:35.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:35.555 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:10:35.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:35 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:35.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:35 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:35.594 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:10:35.598 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:35.638 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:10:35.644 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:35.675 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:10:35.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:35.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:35.699 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:35.720 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:10:35.774 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:10:35.800 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:35.816 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:10:35.817 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:10:35.817 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:10:35.817 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:10:35.817 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:10:35.817 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:10:35.817 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:10:35.817 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:10:35.817 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:10:35.899 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:36.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 65 16 88934 88918 85.4345 54.8125 0.0130238 0.0116999 2023-09-23T18:10:36.041 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:36.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:36.056 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:36.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:36.197 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:36.244 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:36.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:36.249 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:36.256 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:10:36.256 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:10:36.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:36.389 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:36 smithi099 ceph-mon[167076]: pgmap v937: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s; 48851/256521 objects degraded (19.044%); 8844/256521 objects misplaced (3.448%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:10:36.389 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3137599113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:36.389 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:36.389 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:36.393 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:36.492 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:36.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:36.497 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:36.524 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:36 smithi055 ceph-mon[179668]: pgmap v937: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s; 48851/256521 objects degraded (19.044%); 8844/256521 objects misplaced (3.448%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:10:36.525 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3137599113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:36.525 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:36.525 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:36.525 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:36 smithi055 ceph-mon[184174]: pgmap v937: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.2 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 56 MiB/s wr, 893 op/s; 48851/256521 objects degraded (19.044%); 8844/256521 objects misplaced (3.448%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:10:36.525 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3137599113' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:36.525 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:36.526 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:36.598 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:36.694 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:36.728 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:10:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:10:36.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45923.0:89499 8.8 e379/367 8:19dbcb3d:::benchmark_data_smithi055_346788_object89498:head v 379'5551, mlcod=379'5549)", 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:10:36.721839+0000", 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0067323979999999997, 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0067519190000000003, 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.721839+0000", 2023-09-23T18:10:36.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.721839+0000", 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.721839+0000", 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:36.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.721884+0000", 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5191999999999997e-05 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.721899+0000", 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5224e-05 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.721906+0000", 2023-09-23T18:10:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4559999999999997e-06 2023-09-23T18:10:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.721956+0000", 2023-09-23T18:10:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0022000000000003e-05 2023-09-23T18:10:36.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.721979+0000", 2023-09-23T18:10:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2840000000000002e-05 2023-09-23T18:10:36.733 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45923.0:89501 8.8 e379/367 8:1ffd35fd:::benchmark_data_smithi055_346788_object89500:head v 379'5552, mlcod=379'5549)", 2023-09-23T18:10:36.734 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:10:36.724199+0000", 2023-09-23T18:10:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0043728550000000001, 2023-09-23T18:10:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044326310000000002, 2023-09-23T18:10:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:10:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.724199+0000", 2023-09-23T18:10:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.724199+0000", 2023-09-23T18:10:36.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.724198+0000", 2023-09-23T18:10:36.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.724259+0000", 2023-09-23T18:10:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0548999999999999e-05 2023-09-23T18:10:36.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.724273+0000", 2023-09-23T18:10:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4358999999999999e-05 2023-09-23T18:10:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.724279+0000", 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.648e-06 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.724914+0000", 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063551899999999997 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.724937+0000", 2023-09-23T18:10:36.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3141000000000001e-05 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.45923.0:89502 8.8 e379/367 8:14362a0a:::benchmark_data_smithi055_346788_object89501:head v 379'5553, mlcod=379'5549)", 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:10:36.725968+0000", 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0026038160000000001, 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026979970000000002, 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:36.741 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.725968+0000", 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.725968+0000", 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:36.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.725967+0000", 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.726006+0000", 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8841000000000003e-05 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.726019+0000", 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3348e-05 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.726025+0000", 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.468e-06 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.727200+0000", 2023-09-23T18:10:36.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011756340000000001 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:10:36.727217+0000", 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6682999999999999e-05 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:10:36.745 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 3 2023-09-23T18:10:36.746 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:10:36.783 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:36.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:36.788 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:36.889 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:36.992 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:37.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 66 16 89770 89754 84.932 52.25 0.0295602 0.0117688 2023-09-23T18:10:37.089 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:37.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:37.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:37.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:10:37.200 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:37.265 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:37.317 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:37.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:37.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:37.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:37.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3863106139' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:37.318 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:37.318 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:37.318 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:37.318 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:37.319 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:37.319 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3863106139' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:37.319 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:37 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:37.330 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:37.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:37.335 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:37.336 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:37.336 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:10:37.336 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:10:37.336 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:10:37.336 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3863106139' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:37.336 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:37 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:37.436 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:37.490 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:37.560 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:37.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:37.568 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:37.669 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:37.723 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:37.780 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:37.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:37.787 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:37.879 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:10:37.888 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:37.970 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:38.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 67 16 90745 90729 84.574 60.9375 0.0210557 0.011818 2023-09-23T18:10:38.041 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:38.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:38.046 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:38.137 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:10:38.148 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:38.236 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:38.257 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:10:38.305 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:38.309 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:38.310 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:38.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:38 smithi099 ceph-mon[167076]: pgmap v938: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 49391/261480 objects degraded (18.889%); 9132/261480 objects misplaced (3.492%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:10:38.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:38 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:38.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:38.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:38 smithi099 ceph-mon[167076]: osdmap e380: 8 total, 7 up, 8 in 2023-09-23T18:10:38.398 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 49391/261480 objects degraded (18.889%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:38.411 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:38.511 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:38.593 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:38.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:38 smithi055 ceph-mon[179668]: pgmap v938: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 49391/261480 objects degraded (18.889%); 9132/261480 objects misplaced (3.492%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:10:38.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:38 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:38.601 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:38.601 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:38 smithi055 ceph-mon[179668]: osdmap e380: 8 total, 7 up, 8 in 2023-09-23T18:10:38.601 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 49391/261480 objects degraded (18.889%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:38.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:38.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:38.602 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:38 smithi055 ceph-mon[184174]: pgmap v938: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.3 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 883 op/s; 49391/261480 objects degraded (18.889%); 9132/261480 objects misplaced (3.492%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:10:38.602 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:38 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:38.602 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:38.603 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:38 smithi055 ceph-mon[184174]: osdmap e380: 8 total, 7 up, 8 in 2023-09-23T18:10:38.603 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 49391/261480 objects degraded (18.889%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:38.702 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:38.789 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:39.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 68 16 91704 91688 84.2118 59.9375 0.00967032 0.0118691 2023-09-23T18:10:39.314 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:10:39.314 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:10:39.314 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:10:39.314 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:10:39.314 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:10:39.365 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:39.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:39.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:39.473 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:39.537 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 6, 7, 5, 4, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:10:39.537 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:10:39.537 INFO:tasks.thrashosds.thrasher:inject_pause on osd.7 2023-09-23T18:10:39.537 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:10:39.537 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:10:39.538 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 config set filestore_inject_stall 3 2023-09-23T18:10:39.551 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:39.588 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:39 smithi099 ceph-mon[167076]: failed to encode map e380 with expected crc 2023-09-23T18:10:39.588 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:39 smithi099 ceph-mon[167076]: failed to encode map e380 with expected crc 2023-09-23T18:10:39.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:39 smithi055 ceph-mon[184174]: failed to encode map e380 with expected crc 2023-09-23T18:10:39.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:39 smithi055 ceph-mon[184174]: failed to encode map e380 with expected crc 2023-09-23T18:10:39.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:39 smithi055 ceph-mon[179668]: failed to encode map e380 with expected crc 2023-09-23T18:10:39.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:39 smithi055 ceph-mon[179668]: failed to encode map e380 with expected crc 2023-09-23T18:10:39.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:10:40.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 69 16 92752 92736 83.9407 65.5 0.0142102 0.0119092 2023-09-23T18:10:40.305 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:10:40.615 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:40 smithi099 ceph-mon[167076]: pgmap v940: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 50813/269637 objects degraded (18.845%); 9132/269637 objects misplaced (3.387%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:10:40.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:10:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:10:40] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.43.0" 2023-09-23T18:10:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:40 smithi055 ceph-mon[179668]: pgmap v940: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 50813/269637 objects degraded (18.845%); 9132/269637 objects misplaced (3.387%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:10:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:40 smithi055 ceph-mon[184174]: pgmap v940: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.5 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 61 MiB/s wr, 978 op/s; 50813/269637 objects degraded (18.845%); 9132/269637 objects misplaced (3.387%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:10:40.768 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:10:41.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 70 16 93759 93743 83.6407 62.9375 0.0184573 0.0119516 2023-09-23T18:10:41.104 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:41.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:41.108 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:41.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:41.270 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:41.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:41.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:41.362 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:41.463 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:41.556 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:41.574 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:41.647 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:41.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:41.651 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:41.688 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:10:41.688 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T18:10:41.688 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:10:41.752 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:41.829 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:42.041 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 71 16 94784 94768 83.3651 64.0625 0.00970738 0.0119915 2023-09-23T18:10:42.372 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:42.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:42.377 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:42.458 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:42 smithi099 ceph-mon[167076]: pgmap v941: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 51422/274989 objects degraded (18.700%); 9429/274989 objects misplaced (3.429%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T18:10:42.478 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:42.556 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:42.619 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:42.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:42.628 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:42.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:42 smithi055 ceph-mon[184174]: pgmap v941: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 51422/274989 objects degraded (18.700%); 9429/274989 objects misplaced (3.429%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T18:10:42.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:42 smithi055 ceph-mon[179668]: pgmap v941: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.6 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 55 MiB/s wr, 886 op/s; 51422/274989 objects degraded (18.700%); 9429/274989 objects misplaced (3.429%); 2.7 MiB/s, 43 objects/s recovering 2023-09-23T18:10:42.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:42.732 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:10:42.732 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:10:42.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:10:42.732 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:5 6.5 6:aa422d9f:::benchmark_data_smithi055_275963_object4:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.962868+0000", 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 558.76587021199998, 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0413645090000001, 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 5 2023-09-23T18:10:42.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.734 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962868+0000", 2023-09-23T18:10:42.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962868+0000", 2023-09-23T18:10:42.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962867+0000", 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962943+0000", 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6361999999999998e-05 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962956+0000", 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2972999999999999e-05 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962965+0000", 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5830000000000007e-06 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985717+0000", 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022752456000000001 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985726+0000", 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8119999999999997e-06 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:24.999795+0000", 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.014069508 2023-09-23T18:10:42.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:24.999895+0000", 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9772999999999995e-05 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.000018+0000", 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000122621 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001650+0000", 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016320659999999999 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004113+0000", 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024632030000000002 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004166+0000", 2023-09-23T18:10:42.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3303000000000001e-05 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004233+0000", 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6569000000000006e-05 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:26 6.5 6:a2aea16e:::benchmark_data_smithi055_275963_object25:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.962976+0000", 2023-09-23T18:10:42.740 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 558.76576284500004, 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041444284, 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 26 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962976+0000", 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962976+0000", 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.962975+0000", 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963048+0000", 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3052000000000005e-05 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963057+0000", 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8589999999999994e-06 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963062+0000", 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1019999999999996e-06 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985743+0000", 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022680881 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985746+0000", 2023-09-23T18:10:42.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6199999999999999e-06 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.000289+0000", 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.014543376 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.000397+0000", 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010802300000000001 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:10:42.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.000542+0000", 2023-09-23T18:10:42.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014492899999999999 2023-09-23T18:10:42.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001656+0000", 2023-09-23T18:10:42.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011136539999999999 2023-09-23T18:10:42.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004279+0000", 2023-09-23T18:10:42.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002623776 2023-09-23T18:10:42.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004330+0000", 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0961999999999998e-05 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004420+0000", 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9585000000000004e-05 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:38 6.5 6:ab4b060d:::benchmark_data_smithi055_275963_object37:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.963071+0000", 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 558.76566754700002, 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041474614, 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 38 2023-09-23T18:10:42.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963071+0000", 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963071+0000", 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963071+0000", 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963129+0000", 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8417e-05 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963149+0000", 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9789999999999999e-05 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963154+0000", 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3279999999999996e-06 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985756+0000", 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022602048999999999 2023-09-23T18:10:42.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985758+0000", 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2630000000000002e-06 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001146+0000", 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0153872100000001 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001256+0000", 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110223 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001395+0000", 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013967299999999999 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003007+0000", 2023-09-23T18:10:42.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016115529999999999 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004466+0000", 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001459115 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004493+0000", 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6808000000000002e-05 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004546+0000", 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2677999999999999e-05 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:58 6.5 6:a3f61190:::benchmark_data_smithi055_275963_object57:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.963160+0000", 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 558.76557894200005, 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041575398, 2023-09-23T18:10:42.760 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 58 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963160+0000", 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963160+0000", 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963159+0000", 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963250+0000", 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0525999999999994e-05 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963259+0000", 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5030000000000003e-06 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963265+0000", 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8640000000000001e-06 2023-09-23T18:10:42.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985768+0000", 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022502696999999999 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985770+0000", 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5100000000000001e-06 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.764 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001682+0000", 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.015911507 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001784+0000", 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010179900000000001 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.001893+0000", 2023-09-23T18:10:42.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010975400000000001 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003018+0000", 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011243480000000001 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004604+0000", 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015860099999999999 2023-09-23T18:10:42.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004651+0000", 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6944999999999999e-05 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004735+0000", 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4363000000000005e-05 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:76 6.5 6:a6e8b6c3:::benchmark_data_smithi055_275963_object75:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.963283+0000", 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 558.76545579100002, 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0416376890000001, 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.768 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 76 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963283+0000", 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963283+0000", 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963282+0000", 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963364+0000", 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1633999999999998e-05 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963374+0000", 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0370000000000001e-05 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963379+0000", 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7890000000000002e-06 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985780+0000", 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022400929 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985782+0000", 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1179999999999999e-06 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002089+0000", 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0163073250000001 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002170+0000", 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0921999999999995e-05 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002280+0000", 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000109159 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.773 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003824+0000", 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001544754 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004786+0000", 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00096193199999999998 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004835+0000", 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8804000000000002e-05 2023-09-23T18:10:42.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004920+0000", 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5459999999999998e-05 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:105 6.5 6:a7b1533e:::benchmark_data_smithi055_275963_object104:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:10:42.775 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.963385+0000", 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 558.76535383299995, 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041704696, 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 105 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.776 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963385+0000", 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963385+0000", 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963384+0000", 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963433+0000", 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8285000000000003e-05 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963439+0000", 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.844e-06 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963446+0000", 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4529999999999999e-06 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985792+0000", 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022346594000000001 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985795+0000", 2023-09-23T18:10:42.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1670000000000002e-06 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002533+0000", 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0167385369999999 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002654+0000", 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120804 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.002793+0000", 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000139457 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004027+0000", 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012339440000000001 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004957+0000", 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00092994099999999999 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.005009+0000", 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1399e-05 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.005089+0000", 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0758000000000002e-05 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.782 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:115 6.5 6:aa1b9814:::benchmark_data_smithi055_275963_object114:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e214)", 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:23.963452+0000", 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 558.76528671899996, 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.042225293, 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.783 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 115 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963452+0000", 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963452+0000", 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963451+0000", 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963497+0000", 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5408999999999998e-05 2023-09-23T18:10:42.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963504+0000", 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6429999999999998e-06 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.963509+0000", 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6189999999999999e-06 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985803+0000", 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022294333 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:23.985806+0000", 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1299999999999999e-06 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003043+0000", 2023-09-23T18:10:42.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0172373260000001 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003474+0000", 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00043128399999999999 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 6", 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.003618+0000", 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143957 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.004071+0000", 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00045290599999999999 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.005592+0000", 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001521002 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.005625+0000", 2023-09-23T18:10:42.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2938999999999998e-05 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.005677+0000", 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2154000000000002e-05 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:127 6.6 6:6d064c04:::benchmark_data_smithi055_275963_object126:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.790 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.008537+0000", 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.72020160700004, 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0229015530000001, 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 127 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008537+0000", 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008537+0000", 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008536+0000", 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:42.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008806+0000", 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00027006399999999998 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008818+0000", 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1547e-05 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008824+0000", 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3230000000000003e-06 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008874+0000", 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9767e-05 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.008882+0000", 2023-09-23T18:10:42.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8529999999999992e-06 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.020896+0000", 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0120143370000001 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.021056+0000", 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015974199999999999 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.021277+0000", 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000221208 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024530+0000", 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032529759999999999 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031109+0000", 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065789000000000004 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031241+0000", 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132788 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031301+0000", 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9379999999999997e-05 2023-09-23T18:10:42.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031439+0000", 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000137658 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:137 6.6 6:63c48133:::benchmark_data_smithi055_275963_object136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.011129+0000", 2023-09-23T18:10:42.798 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.71760989300003, 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0205278870000001, 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 137 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011129+0000", 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011129+0000", 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011128+0000", 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011153+0000", 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4885e-05 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011161+0000", 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8529999999999992e-06 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011167+0000", 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8449999999999999e-06 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011214+0000", 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7255000000000001e-05 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.011222+0000", 2023-09-23T18:10:42.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5499999999999997e-06 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.021598+0000", 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0103761330000001 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.021729+0000", 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000131092 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.021866+0000", 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013756400000000001 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024537+0000", 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026707839999999998 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031133+0000", 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065955650000000003 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031479+0000", 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00034648799999999999 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031534+0000", 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4576000000000001e-05 2023-09-23T18:10:42.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031657+0000", 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000122741 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:229 6.6 6:6dbd3e64:::benchmark_data_smithi055_275963_object228:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.036295+0000", 2023-09-23T18:10:42.806 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.69244314599996, 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99556675400000005, 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 229 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036295+0000", 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036295+0000", 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036295+0000", 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036379+0000", 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4479000000000002e-05 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036387+0000", 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7559999999999999e-06 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036392+0000", 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.843e-06 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036451+0000", 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8856e-05 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.036460+0000", 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8740000000000001e-06 2023-09-23T18:10:42.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.022147+0000", 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98568734999999996 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.022259+0000", 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112061 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.022422+0000", 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000163053 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024544+0000", 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002121805 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031152+0000", 2023-09-23T18:10:42.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006608137 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031694+0000", 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00054246299999999997 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031742+0000", 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7777e-05 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031862+0000", 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119978 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:234 6.6 6:6b65f21e:::benchmark_data_smithi055_275963_object233:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.038246+0000", 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.69049220299996, 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.993822388, 2023-09-23T18:10:42.814 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 234 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038246+0000", 2023-09-23T18:10:42.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038246+0000", 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038245+0000", 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038352+0000", 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106671 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038363+0000", 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1026000000000001e-05 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038369+0000", 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8320000000000002e-06 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038416+0000", 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7846000000000002e-05 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.038424+0000", 2023-09-23T18:10:42.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7430000000000002e-06 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.022680+0000", 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98425543400000004 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.022807+0000", 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012723699999999999 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.023021+0000", 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021391499999999999 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025104+0000", 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002082815 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031170+0000", 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0060661270000000002 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031901+0000", 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00073105900000000005 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031949+0000", 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8347000000000003e-05 2023-09-23T18:10:42.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032069+0000", 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119639 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:265 6.6 6:6dcd9c47:::benchmark_data_smithi055_275963_object264:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.046473+0000", 2023-09-23T18:10:42.822 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.68226570000002, 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98580278600000004, 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 265 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.823 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046473+0000", 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046473+0000", 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046472+0000", 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:42.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046502+0000", 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0365000000000002e-05 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046511+0000", 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1659999999999994e-06 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046515+0000", 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.549e-06 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046582+0000", 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6396999999999995e-05 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.046590+0000", 2023-09-23T18:10:42.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6680000000000007e-06 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.023319+0000", 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97672869900000003 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.023420+0000", 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010121299999999999 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.023591+0000", 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000170567 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025110+0000", 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001519187 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031189+0000", 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0060787430000000002 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032109+0000", 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00092012799999999998 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032157+0000", 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7997000000000001e-05 2023-09-23T18:10:42.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032276+0000", 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000118878 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:267 6.6 6:6e7c4e87:::benchmark_data_smithi055_275963_object266:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.048471+0000", 2023-09-23T18:10:42.830 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.680267878, 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98400787999999995, 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 267 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.831 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048471+0000", 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048471+0000", 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048469+0000", 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:42.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048537+0000", 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7155000000000003e-05 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048553+0000", 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6660999999999999e-05 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048568+0000", 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4876e-05 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048630+0000", 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1891999999999996e-05 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048639+0000", 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.482e-06 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.023970+0000", 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97533058399999994 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024119+0000", 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014856900000000001 2023-09-23T18:10:42.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024293+0000", 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017458900000000001 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025610+0000", 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001316582 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031206+0000", 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055965299999999997 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032312+0000", 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011053180000000001 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032359+0000", 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7657000000000001e-05 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032479+0000", 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011932 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.838 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:270 6.6 6:6138c5d1:::benchmark_data_smithi055_275963_object269:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.048591+0000", 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.68014711000001, 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98409610000000003, 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 270 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048591+0000", 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048591+0000", 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048591+0000", 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048664+0000", 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3314999999999999e-05 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048675+0000", 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1086e-05 2023-09-23T18:10:42.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048684+0000", 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0869999999999996e-06 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048723+0000", 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9069999999999997e-05 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.048728+0000", 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8930000000000004e-06 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024585+0000", 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97585701199999997 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024710+0000", 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012514899999999999 2023-09-23T18:10:42.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.024881+0000", 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017054400000000001 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.028592+0000", 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0037112740000000001 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.031224+0000", 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002631965 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032515+0000", 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001290802 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032569+0000", 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3838999999999997e-05 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032688+0000", 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011871600000000001 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.846 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:272 6.6 6:6b2a1235:::benchmark_data_smithi055_275963_object271:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.049079+0000", 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.67965956199998, 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98390465400000005, 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 272 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049079+0000", 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049079+0000", 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049078+0000", 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049166+0000", 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7336999999999994e-05 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049186+0000", 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.012e-05 2023-09-23T18:10:42.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049192+0000", 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5060000000000001e-06 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049242+0000", 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9203000000000003e-05 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.049251+0000", 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.38e-06 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025142+0000", 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97589093699999996 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025265+0000", 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012347899999999999 2023-09-23T18:10:42.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025399+0000", 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013394200000000001 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.029874+0000", 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044745369999999998 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032725+0000", 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028511719999999999 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032820+0000", 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5140000000000001e-05 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032868+0000", 2023-09-23T18:10:42.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8102999999999997e-05 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032984+0000", 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115351 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.854 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:301 6.6 6:6e0688bd:::benchmark_data_smithi055_275963_object300:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.056725+0000", 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.67201338400002, 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97644475600000002, 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 301 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056725+0000", 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056725+0000", 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056724+0000", 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056770+0000", 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6489000000000002e-05 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056786+0000", 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5571000000000001e-05 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056792+0000", 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4160000000000001e-06 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056850+0000", 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7812000000000001e-05 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.056862+0000", 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1591e-05 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025643+0000", 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96878142499999997 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025748+0000", 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010444 2023-09-23T18:10:42.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.025882+0000", 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013407499999999999 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.029881+0000", 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039991810000000001 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032758+0000", 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002876773 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033018+0000", 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000260012 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033063+0000", 2023-09-23T18:10:42.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4845000000000002e-05 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033170+0000", 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010732500000000001 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.862 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:334 6.6 6:60a7b3fd:::benchmark_data_smithi055_275963_object333:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.065215+0000", 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.66352366900003, 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96813921400000003, 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 334 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065215+0000", 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065215+0000", 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065214+0000", 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065277+0000", 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3297000000000001e-05 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065289+0000", 2023-09-23T18:10:42.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1198e-05 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065293+0000", 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6489999999999999e-06 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065343+0000", 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9872e-05 2023-09-23T18:10:42.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.065351+0000", 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4460000000000004e-06 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.026148+0000", 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96079755200000005 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.026296+0000", 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014780800000000001 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.026438+0000", 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014171600000000001 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.029888+0000", 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034506150000000002 2023-09-23T18:10:42.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.032789+0000", 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002900953 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033203+0000", 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00041394499999999998 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033246+0000", 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2812000000000003e-05 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033354+0000", 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010816500000000001 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:359 6.6 6:65e2c22f:::benchmark_data_smithi055_275963_object358:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e215)", 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:25.072910+0000", 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 557.65582842699996, 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96068453799999998, 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 359 2023-09-23T18:10:42.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.072910+0000", 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.072910+0000", 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.072909+0000", 2023-09-23T18:10:42.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:10:42.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.072984+0000", 2023-09-23T18:10:42.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4961000000000003e-05 2023-09-23T18:10:42.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.072996+0000", 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1915e-05 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.073001+0000", 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8210000000000001e-06 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.073067+0000", 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6765000000000002e-05 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:25.073077+0000", 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4739999999999994e-06 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.026727+0000", 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95364974300000005 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.026838+0000", 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011121299999999999 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.027005+0000", 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016734700000000001 2023-09-23T18:10:42.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.029895+0000", 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028893489999999998 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033387+0000", 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034925830000000001 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033444+0000", 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6764000000000001e-05 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033485+0000", 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1313999999999998e-05 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:26.033595+0000", 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010945100000000001 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33530 6.6 6.d16185c6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.440253+0000", 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 544.288486003, 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.003888519, 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.879 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33530 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440253+0000", 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440253+0000", 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440252+0000", 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440284+0000", 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2190000000000002e-05 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440291+0000", 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0439999999999998e-06 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.440296+0000", 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2699999999999998e-06 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441796+0000", 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015002279999999999 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.441806+0000", 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.825e-06 2023-09-23T18:10:42.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.444126+0000", 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0023203279999999 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.444141+0000", 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5159e-05 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.884 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45092.0:33541 6.6 6.6e312346 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e219)", 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:01:38.450570+0000", 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 544.27816858200003, 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99379889600000004, 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45092", 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:42.885 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 33541 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450570+0000", 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450570+0000", 2023-09-23T18:10:42.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450569+0000", 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450621+0000", 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2008000000000001e-05 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450632+0000", 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1651000000000001e-05 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450638+0000", 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9839999999999998e-06 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450690+0000", 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1693999999999997e-05 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:38.450699+0000", 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8140000000000007e-06 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.444300+0000", 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99360119899999999 2023-09-23T18:10:42.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:10:42.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:10:42.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:10:42.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:01:39.444369+0000", 2023-09-23T18:10:42.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9006999999999998e-05 2023-09-23T18:10:42.890 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.890 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.890 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.890 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:10:42.890 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:10:42.890 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:10:42.891 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:43.045 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 72 16 95829 95813 83.1125 65.3125 0.0172465 0.0120274 2023-09-23T18:10:43.047 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:43.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:43.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:43.118 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:10:43.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:43.185 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:10:43.194 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:43.239 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:43.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:43.243 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:43.344 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:43.425 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 51422/274989 objects degraded (18.700%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:43.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 51422/274989 objects degraded (18.700%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:43.453 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:43.555 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:43.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:43.559 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:43.660 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:43.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 51422/274989 objects degraded (18.700%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:43.834 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:43.919 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:43.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:43.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:44.027 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:44.044 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 73 16 96746 96730 82.7587 57.3125 0.0120237 0.0120775 2023-09-23T18:10:44.059 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:44.103 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:44.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:44.107 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:44.208 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:44.292 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:44.381 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:44.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:44.391 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:44.492 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:44.531 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:44.573 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:44.582 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:44.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:44.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:44.670 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:44 smithi055 ceph-mon[179668]: pgmap v942: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 52385/281535 objects degraded (18.607%); 9429/281535 objects misplaced (3.349%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:44.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:44 smithi055 ceph-mon[184174]: pgmap v942: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 52385/281535 objects degraded (18.607%); 9429/281535 objects misplaced (3.349%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:44.673 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:44.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:44.732 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:44 smithi099 ceph-mon[167076]: pgmap v942: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.7 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 57 MiB/s wr, 917 op/s; 52385/281535 objects degraded (18.607%); 9429/281535 objects misplaced (3.349%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:44.739 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:44.778 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:44.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:44.783 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:44.883 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:44.962 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:45.051 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 74 16 97633 97617 82.3815 55.4375 0.0200499 0.0121327 2023-09-23T18:10:45.142 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:45.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:45.146 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:45.180 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:10:45.247 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:45.259 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:10:45.298 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:10:45.331 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:45.343 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:10:45.380 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:10:45.424 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:10:45.469 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:10:45.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:10:45.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:10:45.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:10:45.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:10:45.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:10:45.470 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:10:45.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:10:45.470 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:10:45.617 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/528769712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:45.617 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/528769712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:45.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/528769712' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:46.051 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 75 16 98542 98526 82.0408 56.8125 0.00850421 0.0121843 2023-09-23T18:10:46.087 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:46.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:46.096 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:46.197 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:46.264 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:46.322 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:10:46.322 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:10:46.407 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:46.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:46.418 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:46.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:46.550 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:46.598 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:46.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:46.603 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:46.616 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:46 smithi055 ceph-mon[184174]: pgmap v943: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 53023/284484 objects degraded (18.638%); 9429/284484 objects misplaced (3.314%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:46.618 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:46 smithi055 ceph-mon[179668]: pgmap v943: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 53023/284484 objects degraded (18.638%); 9429/284484 objects misplaced (3.314%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:46.704 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:46.785 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:46.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:46 smithi099 ceph-mon[167076]: pgmap v943: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.8 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 931 op/s; 53023/284484 objects degraded (18.638%); 9429/284484 objects misplaced (3.314%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:46.865 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:46.879 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:46.880 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:46.980 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:47.027 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:10:47.027 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:10:47.058 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 76 16 99449 99433 81.7043 56.6875 0.0205652 0.0122348 2023-09-23T18:10:47.061 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:47.238 INFO:tasks.thrashosds.thrasher:in_osds: [2, 3, 6, 7, 5, 4, 1, 0] out_osds: [] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:10:47.238 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:10:47.238 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [2, 3, 6, 7, 5, 4, 1, 0] 2023-09-23T18:10:47.239 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 2 2023-09-23T18:10:47.397 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:47.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:47.532 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:47.533 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:10:47.533 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:10:47.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:47.678 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:47.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:47.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:47 smithi099 ceph-mon[167076]: pgmap v944: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 53684/290031 objects degraded (18.510%); 9710/290031 objects misplaced (3.348%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:47.726 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2088363529' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:47.726 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:47 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:47.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:47.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:47.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:47.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:47 smithi055 ceph-mon[179668]: pgmap v944: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 53684/290031 objects degraded (18.510%); 9710/290031 objects misplaced (3.348%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:47.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2088363529' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:47.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:47 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:47.867 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:47 smithi055 ceph-mon[184174]: pgmap v944: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 5.9 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 59 MiB/s wr, 951 op/s; 53684/290031 objects degraded (18.510%); 9710/290031 objects misplaced (3.348%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:10:47.867 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2088363529' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:47.867 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:47 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:10:47.879 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:47.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:47.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:47.930 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:48.030 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:48.060 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 77 16 100352 100336 81.37 56.4375 0.0240149 0.0122846 2023-09-23T18:10:48.151 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:48.196 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:48.271 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:48.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:48.277 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:48.324 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:10:48.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:48.388 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:48.420 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:48.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:48.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:48.465 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:48.565 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:10:48.597 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:48.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:10:48.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:48.641 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:10:48.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:48 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:48.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:48 smithi055 ceph-mon[184174]: failed to encode map e381 with expected crc 2023-09-23T18:10:48.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:48 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:10:48.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:48 smithi055 ceph-mon[184174]: osdmap e381: 8 total, 7 up, 8 in 2023-09-23T18:10:48.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 53684/290031 objects degraded (18.510%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:48.687 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:48 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:48.688 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:48 smithi055 ceph-mon[179668]: failed to encode map e381 with expected crc 2023-09-23T18:10:48.688 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:48 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:10:48.688 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:48 smithi055 ceph-mon[179668]: osdmap e381: 8 total, 7 up, 8 in 2023-09-23T18:10:48.688 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 53684/290031 objects degraded (18.510%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:48.742 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:48.781 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:48 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:10:48.781 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:48 smithi099 ceph-mon[167076]: failed to encode map e381 with expected crc 2023-09-23T18:10:48.781 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:48 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:10:48.781 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:48 smithi099 ceph-mon[167076]: osdmap e381: 8 total, 7 up, 8 in 2023-09-23T18:10:48.781 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 53684/290031 objects degraded (18.510%), 18 pgs degraded (PG_DEGRADED) 2023-09-23T18:10:48.785 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:48.834 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:48.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:48.839 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:48.939 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:48.981 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:49.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:49.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:49.034 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:49.060 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 78 16 101249 101233 81.0458 56.0625 0.00924003 0.0123343 2023-09-23T18:10:49.135 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:49.204 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:49.394 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:49.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:49.400 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:49.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:49.545 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:49.593 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:49.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:49.602 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:49.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:49 smithi055 ceph-mon[184174]: pgmap v946: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 55084/298317 objects degraded (18.465%); 9710/298317 objects misplaced (3.255%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:10:49.669 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:49 smithi055 ceph-mon[179668]: pgmap v946: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 55084/298317 objects degraded (18.465%); 9710/298317 objects misplaced (3.255%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:10:49.703 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:49.826 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:49.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:49 smithi099 ceph-mon[167076]: pgmap v946: 119 pgs: 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 16 active+undersized+degraded, 28 active+undersized, 66 active+clean; 6.1 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 60 MiB/s wr, 955 op/s; 55084/298317 objects degraded (18.465%); 9710/298317 objects misplaced (3.255%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:10:49.883 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:49.893 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:10:49.910 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:49.915 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:49.916 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:50.017 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:50.060 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 79 16 102207 102191 80.7779 59.875 0.0239376 0.012375 2023-09-23T18:10:50.124 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:50.365 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:50.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:50.373 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:50.474 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:50.507 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:50.539 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:10:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:10:50] "GET /metrics HTTP/1.1" 200 36645 "" "Prometheus/2.43.0" 2023-09-23T18:10:50.547 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:50.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:50.551 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:50.652 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:50.744 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:50.777 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:50 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2947196771' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T18:10:50.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:50 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T18:10:50.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:50 smithi099 ceph-mon[167076]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T18:10:50.831 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:10:50.718+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e381 definitely_dead 0 2023-09-23T18:10:50.831 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:50 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2947196771' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T18:10:50.831 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:50 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T18:10:50.832 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:50 smithi055 ceph-mon[179668]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T18:10:50.832 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:50 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2947196771' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T18:10:50.832 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:50 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T18:10:50.832 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:50 smithi055 ceph-mon[184174]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T18:10:50.832 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.2. 2023-09-23T18:10:50.953 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:10:50.953 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:10:51.060 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:10:51.061501+0000 min lat: 0.00131331 max lat: 2.26605 avg lat: 0.0124223 2023-09-23T18:10:51.060 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:10:51.061 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 80 16 102812 102796 80.2412 37.8125 0.0034043 0.0124223 2023-09-23T18:10:51.215 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:51.217 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:51.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:51.319 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:51.350 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:51.390 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:51.393 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:51.394 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:51.479 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:51.495 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:51.525 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:51.881 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:10:51.955 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:51.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:51.961 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:51.998 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:10:52.018 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:10:52.038 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:10:52.060 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 81 16 102812 102796 79.2513 0 - 0.0124223 2023-09-23T18:10:52.062 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:52.064 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:10:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:51 smithi099 ceph-mon[167076]: failed to encode map e382 with expected crc 2023-09-23T18:10:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:51 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T18:10:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:51 smithi099 ceph-mon[167076]: osdmap e382: 8 total, 7 up, 7 in 2023-09-23T18:10:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:51 smithi099 ceph-mon[167076]: pgmap v948: 119 pgs: 5 peering, 1 active+recovery_wait+remapped, 6 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 27 active+undersized, 62 active+clean; 6.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s; 58513/303414 objects degraded (19.285%); 7151/303414 objects misplaced (2.357%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:10:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:52.089 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:10:52.090 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:51 smithi055 ceph-mon[184174]: failed to encode map e382 with expected crc 2023-09-23T18:10:52.090 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:51 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T18:10:52.091 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:51 smithi055 ceph-mon[184174]: osdmap e382: 8 total, 7 up, 7 in 2023-09-23T18:10:52.091 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:51 smithi055 ceph-mon[184174]: pgmap v948: 119 pgs: 5 peering, 1 active+recovery_wait+remapped, 6 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 27 active+undersized, 62 active+clean; 6.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s; 58513/303414 objects degraded (19.285%); 7151/303414 objects misplaced (2.357%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:10:52.091 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:52.091 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:51 smithi055 ceph-mon[179668]: failed to encode map e382 with expected crc 2023-09-23T18:10:52.091 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:51 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T18:10:52.091 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:51 smithi055 ceph-mon[179668]: osdmap e382: 8 total, 7 up, 7 in 2023-09-23T18:10:52.091 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:51 smithi055 ceph-mon[179668]: pgmap v948: 119 pgs: 5 peering, 1 active+recovery_wait+remapped, 6 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 16 active+undersized+degraded, 27 active+undersized, 62 active+clean; 6.2 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 57 MiB/s wr, 911 op/s; 58513/303414 objects degraded (19.285%); 7151/303414 objects misplaced (2.357%); 2.1 MiB/s, 33 objects/s recovering 2023-09-23T18:10:52.091 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:10:52.094 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:52.113 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:10:52.135 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:52.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:52.139 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:52.139 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:10:52.140 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:10:52.140 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:10:52.140 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:10:52.140 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:10:52.140 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:10:52.140 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:10:52.140 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:10:52.140 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:10:52.240 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:52.286 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:52.321 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:52.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:52.324 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:52.425 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:52.454 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:52.491 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:52.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:52.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:52.510 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:10:52.511 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:10:52.597 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:52.629 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:52.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:52.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:52.668 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:52.769 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:52.801 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:52.858 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:52.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:52.864 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:52.965 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:10:53.010 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:53.052 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:53.064 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 82 16 102871 102855 78.3271 1.84375 0.011734 0.0126504 2023-09-23T18:10:53.065 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:10:53.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:53.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:10:53.070 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:52 smithi055 ceph-mon[184174]: failed to encode map e383 with expected crc 2023-09-23T18:10:53.071 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:52 smithi055 ceph-mon[184174]: osdmap e383: 8 total, 7 up, 7 in 2023-09-23T18:10:53.071 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:52 smithi055 ceph-mon[184174]: failed to encode map e383 with expected crc 2023-09-23T18:10:53.071 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:52 smithi055 ceph-mon[184174]: failed to encode map e383 with expected crc 2023-09-23T18:10:53.071 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2063304387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:53.071 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:52 smithi055 ceph-mon[179668]: failed to encode map e383 with expected crc 2023-09-23T18:10:53.071 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:52 smithi055 ceph-mon[179668]: osdmap e383: 8 total, 7 up, 7 in 2023-09-23T18:10:53.071 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:52 smithi055 ceph-mon[179668]: failed to encode map e383 with expected crc 2023-09-23T18:10:53.071 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:52 smithi055 ceph-mon[179668]: failed to encode map e383 with expected crc 2023-09-23T18:10:53.071 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2063304387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:53.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:52 smithi099 ceph-mon[167076]: failed to encode map e383 with expected crc 2023-09-23T18:10:53.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:52 smithi099 ceph-mon[167076]: osdmap e383: 8 total, 7 up, 7 in 2023-09-23T18:10:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:52 smithi099 ceph-mon[167076]: failed to encode map e383 with expected crc 2023-09-23T18:10:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:52 smithi099 ceph-mon[167076]: failed to encode map e383 with expected crc 2023-09-23T18:10:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2063304387' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:10:53.171 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:53.251 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:53.371 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:53.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:53.380 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:53.481 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:53.519 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:53.575 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:53.578 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:53.579 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:53.680 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:53.759 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:53.936 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:53.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:53.949 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:54.049 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:54.064 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 83 16 103729 103713 78.0297 53.625 0.0222552 0.0128104 2023-09-23T18:10:54.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: failed to encode map e384 with expected crc 2023-09-23T18:10:54.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: osdmap e384: 8 total, 7 up, 7 in 2023-09-23T18:10:54.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: failed to encode map e384 with expected crc 2023-09-23T18:10:54.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 381'6546 2023-09-23T18:10:54.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: 8.e starting backfill to osd.1 from (0'0,0'0] MAX to 381'6451 2023-09-23T18:10:54.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 381'6432 2023-09-23T18:10:54.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: pgmap v951: 119 pgs: 5 remapped+peering, 5 peering, 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized+degraded, 27 active+undersized, 59 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 62 MiB/s wr, 989 op/s; 51167/307125 objects degraded (16.660%); 7151/307125 objects misplaced (2.328%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T18:10:54.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 58513/303414 objects degraded (19.285%), 18 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T18:10:54.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: osdmap e385: 8 total, 7 up, 7 in 2023-09-23T18:10:54.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.131 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[179668]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: failed to encode map e384 with expected crc 2023-09-23T18:10:54.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: osdmap e384: 8 total, 7 up, 7 in 2023-09-23T18:10:54.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: failed to encode map e384 with expected crc 2023-09-23T18:10:54.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 381'6546 2023-09-23T18:10:54.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: 8.e starting backfill to osd.1 from (0'0,0'0] MAX to 381'6451 2023-09-23T18:10:54.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 381'6432 2023-09-23T18:10:54.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: pgmap v951: 119 pgs: 5 remapped+peering, 5 peering, 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized+degraded, 27 active+undersized, 59 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 62 MiB/s wr, 989 op/s; 51167/307125 objects degraded (16.660%); 7151/307125 objects misplaced (2.328%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T18:10:54.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 58513/303414 objects degraded (19.285%), 18 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T18:10:54.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: osdmap e385: 8 total, 7 up, 7 in 2023-09-23T18:10:54.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:53 smithi055 ceph-mon[184174]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.135 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:54.175 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:10:54.210 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:54.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:54.216 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:54.317 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:54.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: failed to encode map e384 with expected crc 2023-09-23T18:10:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: osdmap e384: 8 total, 7 up, 7 in 2023-09-23T18:10:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: failed to encode map e384 with expected crc 2023-09-23T18:10:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: 8.d starting backfill to osd.6 from (0'0,0'0] MAX to 381'6546 2023-09-23T18:10:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: 8.e starting backfill to osd.1 from (0'0,0'0] MAX to 381'6451 2023-09-23T18:10:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: 8.a starting backfill to osd.3 from (0'0,0'0] MAX to 381'6432 2023-09-23T18:10:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: pgmap v951: 119 pgs: 5 remapped+peering, 5 peering, 1 active+recovery_wait+remapped, 6 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 15 active+undersized+degraded, 27 active+undersized, 59 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 62 MiB/s wr, 989 op/s; 51167/307125 objects degraded (16.660%); 7151/307125 objects misplaced (2.328%); 4.8 MiB/s, 76 objects/s recovering 2023-09-23T18:10:54.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 58513/303414 objects degraded (19.285%), 18 pgs degraded, 43 pgs undersized (PG_DEGRADED) 2023-09-23T18:10:54.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: osdmap e385: 8 total, 7 up, 7 in 2023-09-23T18:10:54.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:53 smithi099 ceph-mon[167076]: failed to encode map e385 with expected crc 2023-09-23T18:10:54.401 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:54.466 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:54.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:54.473 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:54.511 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:10:54.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:10:54.625 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:54.673 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:10:54.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:54.677 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:10:54.778 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:10:54.871 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:55.066 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 84 16 104671 104655 77.8003 58.875 0.0244677 0.0128493 2023-09-23T18:10:55.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:54 smithi099 ceph-mon[167076]: failed to encode map e385 with expected crc 2023-09-23T18:10:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3338045168' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:54 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:54 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:10:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:54 smithi099 ceph-mon[167076]: failed to encode map e386 with expected crc 2023-09-23T18:10:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:54 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:54 smithi099 ceph-mon[167076]: osdmap e386: 8 total, 7 up, 7 in 2023-09-23T18:10:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[179668]: failed to encode map e385 with expected crc 2023-09-23T18:10:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3338045168' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:10:55.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[179668]: failed to encode map e386 with expected crc 2023-09-23T18:10:55.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:55.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[179668]: osdmap e386: 8 total, 7 up, 7 in 2023-09-23T18:10:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[184174]: failed to encode map e385 with expected crc 2023-09-23T18:10:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3338045168' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:10:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:10:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[184174]: failed to encode map e386 with expected crc 2023-09-23T18:10:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:10:55.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:54 smithi055 ceph-mon[184174]: osdmap e386: 8 total, 7 up, 7 in 2023-09-23T18:10:55.717 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:10:55.717 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:10:55.717 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:10:55.717 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:10:55.717 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.717 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:67505 6.c 6.b5479fac (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e237)", 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:02:17.174047+0000", 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 518.42084959600004, 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75978317699999998, 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67505 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.718 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174047+0000", 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174047+0000", 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.719 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174045+0000", 2023-09-23T18:10:55.720 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.720 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.720 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.720 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.720 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174088+0000", 2023-09-23T18:10:55.720 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2586000000000002e-05 2023-09-23T18:10:55.720 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.720 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.721 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.721 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174099+0000", 2023-09-23T18:10:55.721 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0993e-05 2023-09-23T18:10:55.721 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.721 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.721 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.721 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174112+0000", 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2860000000000001e-05 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174191+0000", 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9517000000000005e-05 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.174204+0000", 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2486999999999999e-05 2023-09-23T18:10:55.722 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.933818+0000", 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75961363900000001 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:02:17.933830+0000", 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2374999999999999e-05 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.723 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:3953 7.3 7:c9c6e60b:::benchmark_data_smithi055_311528_object3952:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:05:52.348950+0000", 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 303.24594671900002, 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10760905599999999, 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.724 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 3953 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.348950+0000", 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.725 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.348950+0000", 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.348948+0000", 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.348979+0000", 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1390000000000003e-05 2023-09-23T18:10:55.726 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.348998+0000", 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8124000000000001e-05 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349005+0000", 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.96e-06 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.727 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.351476+0000", 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024704919999999999 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.351603+0000", 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000127317 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:10:55.728 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.351737+0000", 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000133201 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.353794+0000", 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020570829999999999 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359295+0000", 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055012220000000001 2023-09-23T18:10:55.729 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456453+0000", 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.097158142000000003 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456515+0000", 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2411999999999998e-05 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.730 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456559+0000", 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3337999999999997e-05 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:3956 7.3 7:c6320996:::benchmark_data_smithi055_311528_object3955:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:05:52.349077+0000", 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 303.24581944200003, 2023-09-23T18:10:55.731 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.107628167, 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 3956 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.732 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349077+0000", 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349077+0000", 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349076+0000", 2023-09-23T18:10:55.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349136+0000", 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9446e-05 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349144+0000", 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.174e-06 2023-09-23T18:10:55.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.349150+0000", 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.541e-06 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.352162+0000", 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030124069999999999 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.735 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.352304+0000", 2023-09-23T18:10:55.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014179700000000001 2023-09-23T18:10:55.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:10:55.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.352447+0000", 2023-09-23T18:10:55.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000143139 2023-09-23T18:10:55.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359286+0000", 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0068389870000000004 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.362398+0000", 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0031117509999999998 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456607+0000", 2023-09-23T18:10:55.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.094209698999999994 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456650+0000", 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2546999999999998e-05 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456705+0000", 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5192000000000003e-05 2023-09-23T18:10:55.738 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:3970 7.3 7:c65e77dc:::benchmark_data_smithi055_311528_object3969:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e283)", 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:05:52.357400+0000", 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 303.23749614100001, 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.099444679999999994, 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.739 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 3970 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357400+0000", 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.740 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357400+0000", 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357400+0000", 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.741 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357451+0000", 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0998000000000002e-05 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357459+0000", 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6680000000000007e-06 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.357465+0000", 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0979999999999999e-06 2023-09-23T18:10:55.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359320+0000", 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00185439 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359465+0000", 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000145544 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.359613+0000", 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014714700000000001 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.362696+0000", 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030835200000000002 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.744 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.363052+0000", 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00035548200000000001 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456758+0000", 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.093706809000000002 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456807+0000", 2023-09-23T18:10:55.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8826000000000003e-05 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:05:52.456845+0000", 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7718e-05 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.746 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29533 7.2 7:46d5e94b:::benchmark_data_smithi055_311528_object29532:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.367301+0000", 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 291.22759509000002, 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2571953860000002, 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29533 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.747 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367301+0000", 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367301+0000", 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367300+0000", 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367324+0000", 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4797999999999999e-05 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367337+0000", 2023-09-23T18:10:55.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2553e-05 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.367345+0000", 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4850000000000003e-06 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.370630+0000", 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0032851439999999998 2023-09-23T18:10:55.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.370641+0000", 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1644000000000001e-05 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.346322+0000", 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97568077600000003 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.346335+0000", 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2475e-05 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.616610+0000", 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.270275923 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.752 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.616787+0000", 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000176718 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,2", 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.616979+0000", 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019225199999999999 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.619865+0000", 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002885084 2023-09-23T18:10:55.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.620879+0000", 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0010146090000000001 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.622841+0000", 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001961824 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.622881+0000", 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0451999999999998e-05 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.624497+0000", 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016152269999999999 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.755 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:50814 7.2 7:41fb1492:::benchmark_data_smithi055_311528_object50813:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:28.375669+0000", 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 267.21922735499999, 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.71295601600000003, 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.756 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 50814 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375669+0000", 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375669+0000", 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375667+0000", 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375692+0000", 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4368000000000001e-05 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375708+0000", 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6643999999999999e-05 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375717+0000", 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1179999999999998e-06 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375765+0000", 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7469999999999998e-05 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.375773+0000", 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2619999999999998e-06 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.079328+0000", 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.70355522800000003 2023-09-23T18:10:55.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.079462+0000", 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000133351 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,2", 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.079595+0000", 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013323999999999999 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.761 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.085875+0000", 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0062797399999999998 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088199+0000", 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023238709999999999 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088530+0000", 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00033098300000000003 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088578+0000", 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8789000000000003e-05 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088625+0000", 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6600000000000001e-05 2023-09-23T18:10:55.763 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:50830 7.2 7:466e5603:::benchmark_data_smithi055_311528_object50829:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e295)", 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:28.429072+0000", 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 267.16582417400002, 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.65978882900000002, 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.764 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 50830 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429072+0000", 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.765 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429072+0000", 2023-09-23T18:10:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429071+0000", 2023-09-23T18:10:55.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429093+0000", 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2030000000000001e-05 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429105+0000", 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2473000000000001e-05 2023-09-23T18:10:55.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429111+0000", 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7810000000000002e-06 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429156+0000", 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4366000000000002e-05 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:28.429163+0000", 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6790000000000005e-06 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.083673+0000", 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.65450945999999999 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.083790+0000", 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011707899999999999 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,2", 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.083920+0000", 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012990399999999999 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.770 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.087641+0000", 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037213620000000002 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088229+0000", 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000588325 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088785+0000", 2023-09-23T18:10:55.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00055543900000000002 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088827+0000", 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1804999999999997e-05 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:29.088861+0000", 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4329999999999998e-05 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.772 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:65495 7.7 e300/279)", 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:47.154630+0000", 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 248.440266404, 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.062563752, 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.774 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154630+0000", 2023-09-23T18:10:55.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.774 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.774 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154630+0000", 2023-09-23T18:10:55.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.774 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154628+0000", 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154773+0000", 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014484099999999999 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154789+0000", 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5288999999999999e-05 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.154798+0000", 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6330000000000008e-06 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217166+0000", 2023-09-23T18:10:55.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0623673030000003 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217194+0000", 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8147999999999999e-05 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:65500 7.7 e300/279)", 2023-09-23T18:10:55.777 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:47.168272+0000", 2023-09-23T18:10:55.778 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 248.426624535, 2023-09-23T18:10:55.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0489593309999998, 2023-09-23T18:10:55.778 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.778 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:10:55.778 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.778 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168272+0000", 2023-09-23T18:10:55.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168272+0000", 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168271+0000", 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168313+0000", 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1974000000000002e-05 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168330+0000", 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7615999999999999e-05 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.168339+0000", 2023-09-23T18:10:55.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5709999999999998e-06 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217215+0000", 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0488766289999996 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217231+0000", 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5775999999999999e-05 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.781 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:65503 7.7 e300/279)", 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:47.202135+0000", 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 248.39276120299999, 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0151491789999998, 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202135+0000", 2023-09-23T18:10:55.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202135+0000", 2023-09-23T18:10:55.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202134+0000", 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202195+0000", 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1135000000000003e-05 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.784 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.785 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202210+0000", 2023-09-23T18:10:55.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4632e-05 2023-09-23T18:10:55.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.785 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:47.202218+0000", 2023-09-23T18:10:55.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6489999999999997e-06 2023-09-23T18:10:55.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217265+0000", 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0150468220000004 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:54.217284+0000", 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9029999999999999e-05 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65507 7.f 7:f2a3a65c:::benchmark_data_smithi055_311528_object65506:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:55.876474+0000", 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 239.71842196, 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3220000350000003, 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65507 2023-09-23T18:10:55.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876474+0000", 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876474+0000", 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876473+0000", 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876504+0000", 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0627999999999999e-05 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876521+0000", 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7286e-05 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.876530+0000", 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0680000000000003e-06 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.167045+0000", 2023-09-23T18:10:55.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2905153990000002 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.167224+0000", 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017898499999999999 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.167406+0000", 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018159 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.174760+0000", 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0073541259999999999 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177828+0000", 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030683960000000001 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.191991+0000", 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014162492 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.196568+0000", 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045769340000000004 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.196607+0000", 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8658999999999997e-05 2023-09-23T18:10:55.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198474+0000", 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001867884 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65522 7.f 7:f4ce2e57:::benchmark_data_smithi055_311528_object65521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T18:10:55.794 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:55.973972+0000", 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 239.620924608, 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2270217859999999, 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65522 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.973972+0000", 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.973972+0000", 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.973969+0000", 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.974001+0000", 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1111000000000001e-05 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.974022+0000", 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1293000000000001e-05 2023-09-23T18:10:55.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.974037+0000", 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4935e-05 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.169722+0000", 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1956848210000004 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.169899+0000", 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000177849 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.170109+0000", 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000209667 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.799 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177841+0000", 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0077319750000000003 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177878+0000", 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6709999999999999e-05 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198539+0000", 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020661536000000001 2023-09-23T18:10:55.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198675+0000", 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013579400000000001 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198720+0000", 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5111999999999998e-05 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.200994+0000", 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022732540000000002 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65531 7.f 7:fb49bebd:::benchmark_data_smithi055_311528_object65530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:55.993528+0000", 2023-09-23T18:10:55.802 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 239.60136806400001, 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2096306049999992, 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65531 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993528+0000", 2023-09-23T18:10:55.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.804 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993528+0000", 2023-09-23T18:10:55.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993526+0000", 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993552+0000", 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5392e-05 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993568+0000", 2023-09-23T18:10:55.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6087e-05 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993577+0000", 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8240000000000006e-06 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.172136+0000", 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1785597140000004 2023-09-23T18:10:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.172295+0000", 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015823599999999999 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.172472+0000", 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000176918 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177885+0000", 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0054133879999999999 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177931+0000", 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5782000000000003e-05 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198584+0000", 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020652818999999999 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.201065+0000", 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024819600000000001 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.201131+0000", 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5105999999999996e-05 2023-09-23T18:10:55.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.203159+0000", 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020283129999999999 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:75464 7.f 7:f021c787:::benchmark_data_smithi055_311528_object4501:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.273115+0000", 2023-09-23T18:10:55.810 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 203.32178119599999, 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.26578997100000001, 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 75464 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273115+0000", 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273115+0000", 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273114+0000", 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273122+0000", 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.875e-06 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273136+0000", 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4139e-05 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.273148+0000", 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2351000000000001e-05 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.274461+0000", 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013133260000000001 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.274628+0000", 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016638499999999999 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.274782+0000", 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000154518 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.282769+0000", 2023-09-23T18:10:55.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0079868630000000003 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.282877+0000", 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010753599999999999 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.289020+0000", 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0061434560000000003 2023-09-23T18:10:55.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536667+0000", 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.247647165 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536722+0000", 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4858000000000002e-05 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.538905+0000", 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021829219999999999 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:75467 7.f 7:f8a917d7:::benchmark_data_smithi055_311528_object4504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.276954+0000", 2023-09-23T18:10:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 203.31794238399999, 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.263980031, 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 75467 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276954+0000", 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276954+0000", 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276952+0000", 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276960+0000", 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8660000000000006e-06 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276974+0000", 2023-09-23T18:10:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4287000000000001e-05 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.276983+0000", 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2609999999999993e-06 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.277455+0000", 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00047249100000000001 2023-09-23T18:10:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.277596+0000", 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014098499999999999 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.277684+0000", 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7489000000000002e-05 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.282810+0000", 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0051259110000000004 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.282885+0000", 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5488000000000006e-05 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536591+0000", 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25370599599999999 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.538972+0000", 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023805570000000002 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.539010+0000", 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8217999999999999e-05 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.540934+0000", 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001924133 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.45557.0:75467 7.f e322/303 ondisk, result = 0)", 2023-09-23T18:10:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.284876+0000", 2023-09-23T18:10:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 203.31002045599999, 2023-09-23T18:10:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25172437600000003, 2023-09-23T18:10:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:10:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284876+0000", 2023-09-23T18:10:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284876+0000", 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284876+0000", 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284878+0000", 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7190000000000001e-06 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284885+0000", 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4320000000000004e-06 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284889+0000", 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.549e-06 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536571+0000", 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25168132100000001 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536583+0000", 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2312e-05 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536600+0000", 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.73e-05 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.45557.0:75464 7.f e322/303 ondisk, result = 0)", 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.284917+0000", 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 203.309979685, 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.254001488, 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284917+0000", 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284917+0000", 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284916+0000", 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284920+0000", 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0180000000000003e-06 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284929+0000", 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2150000000000001e-06 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284934+0000", 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.339e-06 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536651+0000", 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.251717315 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536660+0000", 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8939999999999999e-06 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.538918+0000", 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022580959999999998 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.45557.0:75467 7.f e322/303 ondisk, result = 0)", 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.284939+0000", 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 203.309957277, 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25600794700000001, 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284939+0000", 2023-09-23T18:10:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284939+0000", 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284939+0000", 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:10:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284941+0000", 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5270000000000001e-06 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284947+0000", 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7939999999999999e-06 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.284952+0000", 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4379999999999997e-06 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.538960+0000", 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25400828800000003 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.538967+0000", 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1999999999999997e-06 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.540947+0000", 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019799459999999998 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:75477 7.a 7:5834a874:::benchmark_data_smithi055_311528_object4514:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e322)", 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:32.290788+0000", 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 203.304108007, 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25928344599999997, 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 75477 2023-09-23T18:10:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.290788+0000", 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.290788+0000", 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.290787+0000", 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.290795+0000", 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0700000000000007e-06 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536543+0000", 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.24574849800000001 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536561+0000", 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7600000000000001e-05 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536612+0000", 2023-09-23T18:10:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0654e-05 2023-09-23T18:10:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536743+0000", 2023-09-23T18:10:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013162800000000001 2023-09-23T18:10:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,5", 2023-09-23T18:10:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.536851+0000", 2023-09-23T18:10:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010739999999999999 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.544832+0000", 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.007981485 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.549935+0000", 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0051031009999999996 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.549994+0000", 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8411e-05 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.550035+0000", 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1658e-05 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:32.550072+0000", 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6570999999999999e-05 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:108068 7.3 e339/331 7:c37c53b9:::benchmark_data_smithi055_311528_object37105:head v 339'5460, mlcod=339'5458)", 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:39.460048+0000", 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 136.134848439, 2023-09-23T18:10:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13192663199999999, 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.460048+0000", 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.460048+0000", 2023-09-23T18:10:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.460047+0000", 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.460056+0000", 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6600000000000007e-06 2023-09-23T18:10:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.460075+0000", 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8643000000000001e-05 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.460087+0000", 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2347000000000001e-05 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.460145+0000", 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7954000000000001e-05 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.460171+0000", 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5414000000000001e-05 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.591975+0000", 2023-09-23T18:10:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.131803845 2023-09-23T18:10:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:10:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:10:55.854 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:10:55.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:55.855 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:10:55.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:10:55.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:55 smithi055 ceph-mon[184174]: failed to encode map e386 with expected crc 2023-09-23T18:10:55.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:55 smithi055 ceph-mon[184174]: pgmap v954: 119 pgs: 5 remapped+peering, 5 peering, 1 active+recovering+remapped, 1 active+undersized+degraded+remapped, 7 active+clean+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 57 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 27 MiB/s wr, 428 op/s; 51505/308562 objects degraded (16.692%); 13595/308562 objects misplaced (4.406%); 4.2 MiB/s, 2 keys/s, 70 objects/s recovering 2023-09-23T18:10:55.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:55 smithi055 ceph-mon[179668]: failed to encode map e386 with expected crc 2023-09-23T18:10:55.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:55 smithi055 ceph-mon[179668]: pgmap v954: 119 pgs: 5 remapped+peering, 5 peering, 1 active+recovering+remapped, 1 active+undersized+degraded+remapped, 7 active+clean+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 57 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 27 MiB/s wr, 428 op/s; 51505/308562 objects degraded (16.692%); 13595/308562 objects misplaced (4.406%); 4.2 MiB/s, 2 keys/s, 70 objects/s recovering 2023-09-23T18:10:55.956 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:56.022 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:56.066 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 85 16 105715 105699 77.6527 65.25 0.010045 0.0128738 2023-09-23T18:10:56.255 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:56.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:56.263 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:56.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:55 smithi099 ceph-mon[167076]: failed to encode map e386 with expected crc 2023-09-23T18:10:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:55 smithi099 ceph-mon[167076]: pgmap v954: 119 pgs: 5 remapped+peering, 5 peering, 1 active+recovering+remapped, 1 active+undersized+degraded+remapped, 7 active+clean+remapped, 2 active+undersized+remapped, 1 active+recovering+undersized+degraded+remapped, 13 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 26 active+undersized, 57 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 27 MiB/s wr, 428 op/s; 51505/308562 objects degraded (16.692%); 13595/308562 objects misplaced (4.406%); 4.2 MiB/s, 2 keys/s, 70 objects/s recovering 2023-09-23T18:10:56.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:56.382 INFO:tasks.thrashosds.thrasher:in_osds: [3, 6, 7, 5, 4, 1, 0] out_osds: [2] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:10:56.382 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:10:56.383 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [3, 6, 7, 5, 4, 1, 0] 2023-09-23T18:10:56.383 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T18:10:56.406 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:56.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:56.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:56.470 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:56.571 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:56.663 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:57.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 86 16 106715 106699 77.4765 62.5 0.0181182 0.0129028 2023-09-23T18:10:58.066 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 87 16 107786 107770 77.3554 66.9375 0.0282577 0.0129227 2023-09-23T18:10:58.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:57 smithi099 ceph-mon[167076]: pgmap v955: 119 pgs: 5 remapped+peering, 1 active+recovering+remapped, 1 active+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 12 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 60 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 33 MiB/s wr, 530 op/s; 48890/311538 objects degraded (15.693%); 23207/311538 objects misplaced (7.449%); 5.6 MiB/s, 1 keys/s, 84 objects/s recovering 2023-09-23T18:10:58.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:57 smithi055 ceph-mon[184174]: pgmap v955: 119 pgs: 5 remapped+peering, 1 active+recovering+remapped, 1 active+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 12 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 60 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 33 MiB/s wr, 530 op/s; 48890/311538 objects degraded (15.693%); 23207/311538 objects misplaced (7.449%); 5.6 MiB/s, 1 keys/s, 84 objects/s recovering 2023-09-23T18:10:58.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:57 smithi055 ceph-mon[179668]: pgmap v955: 119 pgs: 5 remapped+peering, 1 active+recovering+remapped, 1 active+undersized+degraded+remapped, 12 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+undersized+remapped, 12 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 60 active+clean; 6.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 33 MiB/s wr, 530 op/s; 48890/311538 objects degraded (15.693%); 23207/311538 objects misplaced (7.449%); 5.6 MiB/s, 1 keys/s, 84 objects/s recovering 2023-09-23T18:10:58.954 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:10:58.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:58.966 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:10:58.994 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:10:59.066 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:10:59.067 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 88 16 108800 108784 77.1966 63.375 0.0235492 0.012948 2023-09-23T18:10:59.179 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:59.182 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:59.231 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:10:59.254 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:10:59.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:59.263 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:10:59.266 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:10:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 48890/311538 objects degraded (15.693%), 15 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-09-23T18:10:59.273 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:10:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 48890/311538 objects degraded (15.693%), 15 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-09-23T18:10:59.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:10:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 48890/311538 objects degraded (15.693%), 15 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-09-23T18:10:59.364 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:10:59.396 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:10:59.430 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:10:59.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:10:59.434 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:10:59.535 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:10:59.602 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:00.068 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 89 16 109704 109688 76.9625 56.5 0.0074867 0.0129873 2023-09-23T18:11:00.113 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:00.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:00.125 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:00.226 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:00.262 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:00.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:00 smithi055 ceph-mon[179668]: pgmap v956: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s; 58599/320244 objects degraded (18.298%); 36772/320244 objects misplaced (11.482%); 4.7 MiB/s, 1 keys/s, 70 objects/s recovering 2023-09-23T18:11:00.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:00 smithi055 ceph-mon[184174]: pgmap v956: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s; 58599/320244 objects degraded (18.298%); 36772/320244 objects misplaced (11.482%); 4.7 MiB/s, 1 keys/s, 70 objects/s recovering 2023-09-23T18:11:00.436 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:11:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:11:00] "GET /metrics HTTP/1.1" 200 36632 "" "Prometheus/2.43.0" 2023-09-23T18:11:00.581 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:00 smithi099 ceph-mon[167076]: pgmap v956: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.5 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 45 MiB/s wr, 718 op/s; 58599/320244 objects degraded (18.298%); 36772/320244 objects misplaced (11.482%); 4.7 MiB/s, 1 keys/s, 70 objects/s recovering 2023-09-23T18:11:00.628 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:11:01.045 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:01.048 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:01.048 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:01.068 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total time run: 90.0297 2023-09-23T18:11:01.068 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total writes made: 110589 2023-09-23T18:11:01.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Write size: 65536 2023-09-23T18:11:01.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object size: 65536 2023-09-23T18:11:01.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Bandwidth (MB/sec): 76.7726 2023-09-23T18:11:01.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Bandwidth: 40.7616 2023-09-23T18:11:01.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max bandwidth (MB/sec): 225.062 2023-09-23T18:11:01.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min bandwidth (MB/sec): 0 2023-09-23T18:11:01.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average IOPS: 1228 2023-09-23T18:11:01.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev IOPS: 652.196 2023-09-23T18:11:01.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max IOPS: 3601 2023-09-23T18:11:01.069 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min IOPS: 0 2023-09-23T18:11:01.070 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average Latency(s): 0.0130201 2023-09-23T18:11:01.070 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Latency(s): 0.0398624 2023-09-23T18:11:01.070 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max latency(s): 2.26992 2023-09-23T18:11:01.070 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min latency(s): 0.00131331 2023-09-23T18:11:01.074 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T18:11:01.149 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:01.182 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:01.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:01.228 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:01.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:01.355 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:01.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:01.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:01.414 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:01.514 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:01.545 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:01.587 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:01.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:01.592 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:01.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:01.725 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:01.769 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:01.775 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:01.776 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:01.877 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:01.917 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:01.978 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:02.224 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:02.229 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:02.229 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:02.330 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:02.368 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:02.417 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:02 smithi055 ceph-mon[179668]: pgmap v957: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 781 op/s; 59195/325905 objects degraded (18.163%); 37417/325905 objects misplaced (11.481%); 3.5 MiB/s, 1 keys/s, 53 objects/s recovering 2023-09-23T18:11:02.417 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:02 smithi055 ceph-mon[184174]: pgmap v957: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 781 op/s; 59195/325905 objects degraded (18.163%); 37417/325905 objects misplaced (11.481%); 3.5 MiB/s, 1 keys/s, 53 objects/s recovering 2023-09-23T18:11:02.422 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:02.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:02.431 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:02.437 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:02 smithi099 ceph-mon[167076]: pgmap v957: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.6 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 49 MiB/s wr, 781 op/s; 59195/325905 objects degraded (18.163%); 37417/325905 objects misplaced (11.481%); 3.5 MiB/s, 1 keys/s, 53 objects/s recovering 2023-09-23T18:11:02.531 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:02.592 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:02.755 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:03.009 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:03.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:03.015 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:03.116 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:03.214 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:03.301 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 59195/325905 objects degraded (18.163%), 16 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:03.301 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 59195/325905 objects degraded (18.163%), 16 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:03.503 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:03.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:03.510 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:03.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 59195/325905 objects degraded (18.163%), 16 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:03.610 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:03.710 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:03.800 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:11:03.812 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:03.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:03.831 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:03.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:11:03.870 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:11:03.908 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:11:03.931 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:03.951 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:11:03.990 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:11:04.027 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:11:04.028 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:11:04.028 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:11:04.028 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:11:04.028 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:11:04.028 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:11:04.028 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:11:04.028 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:11:04.028 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:11:04.029 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:04.163 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:04.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:04.177 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:04.255 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:11:04.255 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:11:04.278 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:04.353 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:04 smithi055 ceph-mon[184174]: pgmap v958: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 817 op/s; 59702/328251 objects degraded (18.188%); 37651/328251 objects misplaced (11.470%); 2.9 MiB/s, 42 objects/s recovering 2023-09-23T18:11:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:04 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/581424171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:04.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:04 smithi055 ceph-mon[179668]: pgmap v958: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 817 op/s; 59702/328251 objects degraded (18.188%); 37651/328251 objects misplaced (11.470%); 2.9 MiB/s, 42 objects/s recovering 2023-09-23T18:11:04.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:04 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/581424171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:04.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:11:04.302+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e386 definitely_dead 0 2023-09-23T18:11:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:04 smithi099 ceph-mon[167076]: pgmap v958: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 47 MiB/s wr, 817 op/s; 59702/328251 objects degraded (18.188%); 37651/328251 objects misplaced (11.470%); 2.9 MiB/s, 42 objects/s recovering 2023-09-23T18:11:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:04 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/581424171' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:04.693 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:04.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:04.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:04.798 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:04.871 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:04.926 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:04.931 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:04.931 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:04.954 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:11:04.955 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:11:04.978 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:11:05.032 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:05.160 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:05.230 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.3. 2023-09-23T18:11:05.255 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:05.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:05.262 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:05.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:05.371 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2659489540' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:11:05.371 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:05 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:11:05.371 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:05 smithi055 ceph-mon[184174]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T18:11:05.371 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2659489540' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:11:05.371 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:05 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:11:05.372 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:05 smithi055 ceph-mon[179668]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T18:11:05.388 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:05.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:05.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:05.424 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:05.525 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:05.562 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:05.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2659489540' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:11:05.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:05 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:11:05.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:05 smithi099 ceph-mon[167076]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T18:11:05.825 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:05.827 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:05.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:05.929 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:05.958 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:05.991 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:05.995 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:05.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:05.999 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:06.099 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:06.129 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:06.223 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:06.228 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:11:06.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:06.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:06.336 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:06.378 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:06 smithi099 ceph-mon[167076]: pgmap v959: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 95 B/s rd, 45 MiB/s wr, 813 op/s; 59808/328512 objects degraded (18.206%); 37567/328512 objects misplaced (11.436%); 2.9 MiB/s, 42 objects/s recovering 2023-09-23T18:11:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:06 smithi099 ceph-mon[167076]: failed to encode map e387 with expected crc 2023-09-23T18:11:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:06 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T18:11:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:06 smithi099 ceph-mon[167076]: osdmap e387: 8 total, 7 up, 6 in 2023-09-23T18:11:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:06 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2370137974' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:06 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:06.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[179668]: pgmap v959: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 95 B/s rd, 45 MiB/s wr, 813 op/s; 59808/328512 objects degraded (18.206%); 37567/328512 objects misplaced (11.436%); 2.9 MiB/s, 42 objects/s recovering 2023-09-23T18:11:06.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[179668]: failed to encode map e387 with expected crc 2023-09-23T18:11:06.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T18:11:06.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[179668]: osdmap e387: 8 total, 7 up, 6 in 2023-09-23T18:11:06.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2370137974' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:06.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:06.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[184174]: pgmap v959: 119 pgs: 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 12 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 61 active+clean; 6.7 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 95 B/s rd, 45 MiB/s wr, 813 op/s; 59808/328512 objects degraded (18.206%); 37567/328512 objects misplaced (11.436%); 2.9 MiB/s, 42 objects/s recovering 2023-09-23T18:11:06.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[184174]: failed to encode map e387 with expected crc 2023-09-23T18:11:06.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T18:11:06.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[184174]: osdmap e387: 8 total, 7 up, 6 in 2023-09-23T18:11:06.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2370137974' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:06.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:06 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:06.739 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:06.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:06.743 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:06.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:06.871 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:06.908 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:06.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:06.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:06.956 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:11:07.011 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:07.049 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:07.143 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:07.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:07.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:07.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:07.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[179668]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[179668]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:11:07.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[179668]: osdmap e388: 8 total, 7 up, 6 in 2023-09-23T18:11:07.243 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[179668]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[179668]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[179668]: 8.a starting backfill to osd.4 from (0'0,0'0] MAX to 386'7207 2023-09-23T18:11:07.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:07.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:07.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[184174]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[184174]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:11:07.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[184174]: osdmap e388: 8 total, 7 up, 6 in 2023-09-23T18:11:07.244 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[184174]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[184174]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[184174]: 8.a starting backfill to osd.4 from (0'0,0'0] MAX to 386'7207 2023-09-23T18:11:07.245 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:07.249 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:07.294 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:07.333 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:07.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:07.338 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:07.439 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:07.478 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:11:07.478 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:11:07.495 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:07 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:07 smithi099 ceph-mon[167076]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:07 smithi099 ceph-mon[167076]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:07 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:11:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:07 smithi099 ceph-mon[167076]: osdmap e388: 8 total, 7 up, 6 in 2023-09-23T18:11:07.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:07 smithi099 ceph-mon[167076]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:07 smithi099 ceph-mon[167076]: failed to encode map e388 with expected crc 2023-09-23T18:11:07.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:07 smithi099 ceph-mon[167076]: 8.a starting backfill to osd.4 from (0'0,0'0] MAX to 386'7207 2023-09-23T18:11:07.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:07.636 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:07.645 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:07.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:07.746 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:07.804 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:07.861 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:07.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:07.871 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:07.934 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:07.972 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:07.995 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:11:08.031 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:08.142 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:08.154 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:08.154 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:08.255 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:08.300 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:08.337 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: pgmap v962: 119 pgs: 7 peering, 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 127 B/s rd, 30 MiB/s wr, 786 op/s; 52659/324576 objects degraded (16.224%); 37146/324576 objects misplaced (11.444%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T18:11:08.337 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: failed to encode map e389 with expected crc 2023-09-23T18:11:08.337 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: osdmap e389: 8 total, 7 up, 6 in 2023-09-23T18:11:08.337 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: failed to encode map e389 with expected crc 2023-09-23T18:11:08.337 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: 8.9 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7214 2023-09-23T18:11:08.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 386'7119 2023-09-23T18:11:08.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: failed to encode map e389 with expected crc 2023-09-23T18:11:08.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: 8.7 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7097 2023-09-23T18:11:08.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: 8.f starting backfill to osd.6 from (0'0,0'0] MAX to 386'7087 2023-09-23T18:11:08.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7130 2023-09-23T18:11:08.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 52659/324576 objects degraded (16.224%), 15 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:08.338 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[184174]: osdmap e390: 8 total, 7 up, 6 in 2023-09-23T18:11:08.338 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: pgmap v962: 119 pgs: 7 peering, 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 127 B/s rd, 30 MiB/s wr, 786 op/s; 52659/324576 objects degraded (16.224%); 37146/324576 objects misplaced (11.444%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T18:11:08.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: failed to encode map e389 with expected crc 2023-09-23T18:11:08.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: osdmap e389: 8 total, 7 up, 6 in 2023-09-23T18:11:08.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: failed to encode map e389 with expected crc 2023-09-23T18:11:08.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: 8.9 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7214 2023-09-23T18:11:08.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 386'7119 2023-09-23T18:11:08.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: failed to encode map e389 with expected crc 2023-09-23T18:11:08.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: 8.7 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7097 2023-09-23T18:11:08.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: 8.f starting backfill to osd.6 from (0'0,0'0] MAX to 386'7087 2023-09-23T18:11:08.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7130 2023-09-23T18:11:08.339 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 52659/324576 objects degraded (16.224%), 15 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:08.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:08 smithi055 ceph-mon[179668]: osdmap e390: 8 total, 7 up, 6 in 2023-09-23T18:11:08.374 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:08.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:08.381 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:08.482 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:08.555 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:08.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: pgmap v962: 119 pgs: 7 peering, 1 active+recovering+remapped, 17 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 23 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.6 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 127 B/s rd, 30 MiB/s wr, 786 op/s; 52659/324576 objects degraded (16.224%); 37146/324576 objects misplaced (11.444%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: failed to encode map e389 with expected crc 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: osdmap e389: 8 total, 7 up, 6 in 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: failed to encode map e389 with expected crc 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: 8.9 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7214 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 386'7119 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: failed to encode map e389 with expected crc 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: 8.7 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7097 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: 8.f starting backfill to osd.6 from (0'0,0'0] MAX to 386'7087 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: 8.0 starting backfill to osd.4 from (0'0,0'0] MAX to 386'7130 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 52659/324576 objects degraded (16.224%), 15 pgs degraded, 35 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:08 smithi099 ceph-mon[167076]: osdmap e390: 8 total, 7 up, 6 in 2023-09-23T18:11:08.756 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:08.779 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:08.780 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:08.788 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:11:08.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:08.918 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:08.972 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:08.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:08.975 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:09.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:09.089 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:09.114 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:09.171 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:09.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:09.177 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:09.277 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:09.337 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:09.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:09 smithi055 ceph-mon[184174]: failed to encode map e390 with expected crc 2023-09-23T18:11:09.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:09 smithi055 ceph-mon[184174]: failed to encode map e390 with expected crc 2023-09-23T18:11:09.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:09 smithi055 ceph-mon[184174]: failed to encode map e390 with expected crc 2023-09-23T18:11:09.455 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:09 smithi055 ceph-mon[184174]: osdmap e391: 8 total, 7 up, 6 in 2023-09-23T18:11:09.455 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:09 smithi055 ceph-mon[179668]: failed to encode map e390 with expected crc 2023-09-23T18:11:09.455 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:09 smithi055 ceph-mon[179668]: failed to encode map e390 with expected crc 2023-09-23T18:11:09.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:09 smithi055 ceph-mon[179668]: failed to encode map e390 with expected crc 2023-09-23T18:11:09.456 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:09 smithi055 ceph-mon[179668]: osdmap e391: 8 total, 7 up, 6 in 2023-09-23T18:11:09.515 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:09.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:09.520 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:09.560 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:11:09.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:09 smithi099 ceph-mon[167076]: failed to encode map e390 with expected crc 2023-09-23T18:11:09.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:09 smithi099 ceph-mon[167076]: failed to encode map e390 with expected crc 2023-09-23T18:11:09.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:09 smithi099 ceph-mon[167076]: failed to encode map e390 with expected crc 2023-09-23T18:11:09.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:09 smithi099 ceph-mon[167076]: osdmap e391: 8 total, 7 up, 6 in 2023-09-23T18:11:09.594 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:11:09.621 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:09.631 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:11:09.674 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:11:09.699 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:09.711 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:11:09.760 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:11:09.818 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:11:09.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:11:09.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:11:09.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:11:09.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:11:09.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:11:09.819 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:11:09.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:11:09.819 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:11:09.829 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:09.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:09.839 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:09.940 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:10.017 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:10.314 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:10.323 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:10.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:10.332 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:10.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:11:10] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.43.0" 2023-09-23T18:11:10.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[184174]: pgmap v965: 119 pgs: 1 remapped+peering, 7 peering, 2 active+recovery_wait+remapped, 20 active+clean+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 50 active+clean; 6.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 992 KiB/s wr, 762 op/s; 45025/319725 objects degraded (14.082%); 62930/319725 objects misplaced (19.683%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T18:11:10.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[184174]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[184174]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[184174]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[184174]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2318891456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:10.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[179668]: pgmap v965: 119 pgs: 1 remapped+peering, 7 peering, 2 active+recovery_wait+remapped, 20 active+clean+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 50 active+clean; 6.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 992 KiB/s wr, 762 op/s; 45025/319725 objects degraded (14.082%); 62930/319725 objects misplaced (19.683%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T18:11:10.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[179668]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[179668]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[179668]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[179668]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2318891456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:10.433 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:10.510 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:10.546 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:11:10.546 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:11:10.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:10 smithi099 ceph-mon[167076]: pgmap v965: 119 pgs: 1 remapped+peering, 7 peering, 2 active+recovery_wait+remapped, 20 active+clean+remapped, 1 active+undersized+remapped, 2 active+remapped+backfill_wait, 2 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 50 active+clean; 6.5 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 992 KiB/s wr, 762 op/s; 45025/319725 objects degraded (14.082%); 62930/319725 objects misplaced (19.683%); 2.1 MiB/s, 34 objects/s recovering 2023-09-23T18:11:10.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:10 smithi099 ceph-mon[167076]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:10 smithi099 ceph-mon[167076]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:10 smithi099 ceph-mon[167076]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:10 smithi099 ceph-mon[167076]: failed to encode map e391 with expected crc 2023-09-23T18:11:10.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2318891456' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:10.670 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:10.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:10.682 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:10.783 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:10.859 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:10.915 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 4, 1, 0] out_osds: [2, 3] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:11:10.916 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:11:10.916 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T18:11:10.916 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T18:11:11.137 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:11.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:11.148 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:11.249 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:11.260 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29520 7.4 7:2e3eff57:::benchmark_data_smithi055_311528_object29519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.342145+0000", 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 306.91239389700002, 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.279446246, 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:11.261 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29520 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342145+0000", 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342145+0000", 2023-09-23T18:11:11.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342144+0000", 2023-09-23T18:11:11.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:11.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342167+0000", 2023-09-23T18:11:11.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2671e-05 2023-09-23T18:11:11.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342179+0000", 2023-09-23T18:11:11.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2537999999999999e-05 2023-09-23T18:11:11.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342199+0000", 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9791000000000001e-05 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342244+0000", 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5379e-05 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342363+0000", 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119085 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342477+0000", 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011398 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.344360+0000", 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018828829999999999 2023-09-23T18:11:11.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.351598+0000", 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072381310000000001 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.351613+0000", 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5082999999999999e-05 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361553+0000", 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0099392469999999 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361566+0000", 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3742e-05 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.621482+0000", 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.259916043 2023-09-23T18:11:11.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.621592+0000", 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109143 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29523 7.4 7:218b7f51:::benchmark_data_smithi055_311528_object29522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.362687+0000", 2023-09-23T18:11:11.271 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 306.891852465, 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2720197689999999, 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29523 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362687+0000", 2023-09-23T18:11:11.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362687+0000", 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362685+0000", 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:11.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362714+0000", 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9023999999999999e-05 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362730+0000", 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6053999999999999e-05 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362738+0000", 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7570000000000004e-06 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.363432+0000", 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00069362000000000004 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.363443+0000", 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1031e-05 2023-09-23T18:11:11.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361597+0000", 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99815435299999999 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361605+0000", 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7270000000000007e-06 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.623050+0000", 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2614453999999999 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.623261+0000", 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021100599999999999 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:11:11.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.623463+0000", 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020170799999999999 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626400+0000", 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00293728 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.628125+0000", 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001725266 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.633086+0000", 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049610030000000003 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.633146+0000", 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0000000000000002e-05 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.634707+0000", 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001560077 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29540 7.9 7:9dcdb66e:::benchmark_data_smithi055_311528_object29539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.375098+0000", 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 306.87944164300001, 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2623359889999999, 2023-09-23T18:11:11.280 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29540 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375098+0000", 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375098+0000", 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375096+0000", 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375120+0000", 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3737e-05 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375132+0000", 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2211999999999999e-05 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375140+0000", 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6350000000000006e-06 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375200+0000", 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.067e-05 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375211+0000", 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0248000000000001e-05 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356527+0000", 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98131692100000001 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356539+0000", 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1392e-05 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625642+0000", 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2691035230000001 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625784+0000", 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014192400000000001 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625926+0000", 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000141607 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.286 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629334+0000", 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034082079999999998 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.630984+0000", 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00164963 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635790+0000", 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048063300000000002 2023-09-23T18:11:11.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635850+0000", 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9979000000000003e-05 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637434+0000", 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001583552 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29542 7.9 7:9a322fbf:::benchmark_data_smithi055_311528_object29541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.376211+0000", 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 306.87832786600001, 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2627142660000001, 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29542 2023-09-23T18:11:11.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376211+0000", 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376211+0000", 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376210+0000", 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376231+0000", 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1126000000000001e-05 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376248+0000", 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6461e-05 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376256+0000", 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5590000000000006e-06 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.378137+0000", 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001880928 2023-09-23T18:11:11.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.378147+0000", 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9010000000000006e-06 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356782+0000", 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97863468399999998 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356790+0000", 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5560000000000008e-06 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629412+0000", 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.272622081 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629587+0000", 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000174432 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629697+0000", 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109821 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.631159+0000", 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014629059999999999 2023-09-23T18:11:11.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.631648+0000", 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00048873199999999997 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637552+0000", 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0059042740000000002 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.297 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.297 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637614+0000", 2023-09-23T18:11:11.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1674000000000006e-05 2023-09-23T18:11:11.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.297 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.297 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.297 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.638926+0000", 2023-09-23T18:11:11.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013115170000000001 2023-09-23T18:11:11.297 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.297 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.298 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.298 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.298 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29546 7.4 7:2ffa6bd1:::benchmark_data_smithi055_311528_object29545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:11:11.298 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.379888+0000", 2023-09-23T18:11:11.298 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 306.87465166300001, 2023-09-23T18:11:11.298 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2559961080000002, 2023-09-23T18:11:11.298 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.298 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29546 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379888+0000", 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379888+0000", 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379886+0000", 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379920+0000", 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4292999999999998e-05 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379934+0000", 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4307999999999999e-05 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379943+0000", 2023-09-23T18:11:11.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7010000000000002e-06 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379996+0000", 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2849000000000002e-05 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.380004+0000", 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5900000000000002e-06 2023-09-23T18:11:11.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361629+0000", 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.981625623 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.364509+0000", 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028797689999999999 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.624800+0000", 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.260291504 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.624960+0000", 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000159739 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:11:11.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625123+0000", 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016252700000000001 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.628149+0000", 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00302673 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629738+0000", 2023-09-23T18:11:11.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001588922 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.634757+0000", 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050189459999999998 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.634816+0000", 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8294000000000001e-05 2023-09-23T18:11:11.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635884+0000", 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001068102 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29552 7.4 7:2f4e28dc:::benchmark_data_smithi055_311528_object29551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.385790+0000", 2023-09-23T18:11:11.307 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 306.86874888699998, 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2515060980000001, 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29552 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.308 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385790+0000", 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385790+0000", 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385789+0000", 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:11.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385809+0000", 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0041e-05 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385821+0000", 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1803e-05 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385829+0000", 2023-09-23T18:11:11.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4889999999999995e-06 2023-09-23T18:11:11.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385881+0000", 2023-09-23T18:11:11.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1408e-05 2023-09-23T18:11:11.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385890+0000", 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2089999999999994e-06 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.364654+0000", 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97876412000000002 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:11.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.364667+0000", 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2456999999999999e-05 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626449+0000", 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2617827670000001 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626582+0000", 2023-09-23T18:11:11.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013226399999999999 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626686+0000", 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104701 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629538+0000", 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028518670000000001 2023-09-23T18:11:11.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:11.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629790+0000", 2023-09-23T18:11:11.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00025145700000000002 2023-09-23T18:11:11.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635940+0000", 2023-09-23T18:11:11.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0061504029999999996 2023-09-23T18:11:11.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635986+0000", 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6304000000000003e-05 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637296+0000", 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013099990000000001 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.316 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:108098 7.6 7:63f0b014:::benchmark_data_smithi055_311528_object37135:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:40.505253+0000", 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 150.74928593000001, 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13445985899999999, 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:11.317 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108098 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505253+0000", 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505253+0000", 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505251+0000", 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505261+0000", 2023-09-23T18:11:11.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4450000000000002e-06 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505282+0000", 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1251999999999999e-05 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505290+0000", 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2879999999999993e-06 2023-09-23T18:11:11.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505343+0000", 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2034999999999999e-05 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.506478+0000", 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011350469999999999 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.624282+0000", 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.117804585 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.624374+0000", 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1632000000000006e-05 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:11:11.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.624475+0000", 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010101400000000001 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.626282+0000", 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018073939999999999 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.628754+0000", 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002471829 2023-09-23T18:11:11.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.639295+0000", 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010541404000000001 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.639338+0000", 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2814e-05 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.639713+0000", 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000374948 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:62831 8.6 e375/371 8:6ff862bb:::benchmark_data_smithi055_346788_object62830:head v 375'4001, mlcod=0'0)", 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:07.849801+0000", 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 63.404738041000002, 2023-09-23T18:11:11.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.042918612000000002, 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.849801+0000", 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.849801+0000", 2023-09-23T18:11:11.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.849800+0000", 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.849859+0000", 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9011000000000001e-05 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.849874+0000", 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4922e-05 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.849882+0000", 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4880000000000007e-06 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.849933+0000", 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0875000000000003e-05 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.849962+0000", 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9181999999999999e-05 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.892676+0000", 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.042713416999999997 2023-09-23T18:11:11.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:07.892720+0000", 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3914e-05 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:79318 8.1 e378/371 8:83825118:::benchmark_data_smithi055_346788_object79317:head v 378'4954, mlcod=365'844)", 2023-09-23T18:11:11.330 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:24.986670+0000", 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 46.267869075999997, 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.044327062, 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:24.986670+0000", 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:24.986670+0000", 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:24.986669+0000", 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:24.986694+0000", 2023-09-23T18:11:11.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.423e-05 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:24.986703+0000", 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8300000000000008e-06 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:24.986709+0000", 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8649999999999998e-06 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:24.988394+0000", 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016842719999999999 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:24.988421+0000", 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7413999999999999e-05 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.334 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:25.030946+0000", 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.042524582999999998 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:25.030997+0000", 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1638999999999998e-05 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:88522 8.2 e379/371 8:4b2b0f84:::benchmark_data_smithi055_346788_object88521:head v 379'5492, mlcod=0'0)", 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:35.572982+0000", 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 35.681557499999997, 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.050153703000000001, 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.572982+0000", 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.572982+0000", 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.572981+0000", 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.573023+0000", 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.244e-05 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.573042+0000", 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8419000000000001e-05 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.573051+0000", 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4639999999999995e-06 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575760+0000", 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027086300000000001 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575785+0000", 2023-09-23T18:11:11.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5405000000000002e-05 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623088+0000", 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047302960999999998 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623135+0000", 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7342000000000002e-05 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.340 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.341 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.341 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:88524 8.2 e379/371 8:46f9590d:::benchmark_data_smithi055_346788_object88523:head v 379'5493, mlcod=0'0)", 2023-09-23T18:11:11.341 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:35.575910+0000", 2023-09-23T18:11:11.341 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 35.678629286000003, 2023-09-23T18:11:11.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047258545999999998, 2023-09-23T18:11:11.341 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.341 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.341 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575910+0000", 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575910+0000", 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575909+0000", 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575956+0000", 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6783000000000003e-05 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575968+0000", 2023-09-23T18:11:11.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2165e-05 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575975+0000", 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5469999999999999e-06 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.581529+0000", 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055535189999999998 2023-09-23T18:11:11.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.581561+0000", 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1767000000000003e-05 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623141+0000", 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.041580060000000002 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623169+0000", 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7665e-05 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPush(8.6 386/371 [PushOp(8:6c728b0a:::benchmark_data_smithi055_346788_object2082:head, version: 365'134, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:6c728b0a:::benchmark_data_smithi055_346788_object2082:head@365'134, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 16.902906765000001, 2023-09-23T18:11:11.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.060643765000000002, 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351669+0000", 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6786999999999999e-05 2023-09-23T18:11:11.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351689+0000", 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9692000000000001e-05 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351702+0000", 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3709e-05 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.408798+0000", 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.057095414999999997 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.408806+0000", 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5909999999999996e-06 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.412276+0000", 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034699190000000001 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:104012 8.6 e386/371 8:6f916fc8:::benchmark_data_smithi055_346788_object104011:head v 386'6585, mlcod=365'133)", 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:54.387490+0000", 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 16.867049059999999, 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047785543, 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.351 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387490+0000", 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387490+0000", 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387489+0000", 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387519+0000", 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9485e-05 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387539+0000", 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0528999999999999e-05 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387548+0000", 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2240000000000001e-06 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.396833+0000", 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092844009999999994 2023-09-23T18:11:11.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.396858+0000", 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5072e-05 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.435227+0000", 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038369551000000002 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.435276+0000", 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8396000000000002e-05 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:104072 8.6 e386/371 8:6c4db58a:::benchmark_data_smithi055_346788_object104071:head v 386'6591, mlcod=365'134)", 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:54.468379+0000", 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 16.786159752, 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043526678999999999, 2023-09-23T18:11:11.356 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.468379+0000", 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.468379+0000", 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.468379+0000", 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.468403+0000", 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3444000000000001e-05 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.468409+0000", 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4629999999999998e-06 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.468414+0000", 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2469999999999999e-06 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.480802+0000", 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012387176999999999 2023-09-23T18:11:11.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.480819+0000", 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7214000000000002e-05 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.511854+0000", 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.031035126999999999 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.511906+0000", 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2305999999999997e-05 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:105386 8.6 e386/371 8:677d6cd5:::benchmark_data_smithi055_346788_object105385:head v 386'6659, mlcod=365'174)", 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:55.758588+0000", 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 15.495951721000001, 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043583523999999998, 2023-09-23T18:11:11.361 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.758588+0000", 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.758588+0000", 2023-09-23T18:11:11.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.758587+0000", 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.758616+0000", 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8818000000000001e-05 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.758632+0000", 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6548000000000001e-05 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.758641+0000", 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4530000000000002e-06 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.762612+0000", 2023-09-23T18:11:11.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039716049999999996 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.762639+0000", 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6618000000000002e-05 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.802122+0000", 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.039482903 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.802171+0000", 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9406000000000001e-05 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:105389 8.6 e386/371 8:617cba55:::benchmark_data_smithi055_346788_object105388:head v 386'6660, mlcod=365'174)", 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:55.764711+0000", 2023-09-23T18:11:11.366 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 15.48982792, 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.049308314999999998, 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764711+0000", 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764711+0000", 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764710+0000", 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764741+0000", 2023-09-23T18:11:11.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0073000000000002e-05 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764755+0000", 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4827999999999999e-05 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764764+0000", 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.738e-06 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.769484+0000", 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047194649999999999 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.769511+0000", 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7542000000000002e-05 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.813963+0000", 2023-09-23T18:11:11.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.044451549 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.814020+0000", 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7012999999999998e-05 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:105393 8.6 e386/371 8:6f69194b:::benchmark_data_smithi055_346788_object105392:head v 386'6661, mlcod=365'174)", 2023-09-23T18:11:11.371 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:55.768467+0000", 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 15.486072675999999, 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.045600396000000001, 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.768467+0000", 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.768467+0000", 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.768466+0000", 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.768492+0000", 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6103000000000001e-05 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.768513+0000", 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1056e-05 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.768521+0000", 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4610000000000008e-06 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.775765+0000", 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072438049999999999 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.775800+0000", 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5150000000000001e-05 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.814036+0000", 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038236033000000003 2023-09-23T18:11:11.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.814067+0000", 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0818000000000002e-05 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:109643 8.6 e386/371 8:6b4a303b:::benchmark_data_smithi055_346788_object109642:head v 386'6916, mlcod=365'308)", 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 11.235263781, 2023-09-23T18:11:11.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.046387045000000002, 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:11:11.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019295+0000", 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9360000000000001e-05 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019301+0000", 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7490000000000001e-06 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019305+0000", 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8700000000000002e-06 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.027156+0000", 2023-09-23T18:11:11.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0078509070000000007 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.027176+0000", 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.037e-05 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.065607+0000", 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038430756000000003 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.065663+0000", 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5334000000000002e-05 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:110379 8.6 e386/371 8:623139ad:::benchmark_data_smithi055_346788_object110378:head v 386'6966, mlcod=365'331)", 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:00.777299+0000", 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 10.477240223999999, 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.045922075999999999, 2023-09-23T18:11:11.381 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.777299+0000", 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.777299+0000", 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.777298+0000", 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.777330+0000", 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2230000000000001e-05 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.777347+0000", 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7169000000000001e-05 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.777356+0000", 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6549999999999993e-06 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.779852+0000", 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024960730000000001 2023-09-23T18:11:11.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.779886+0000", 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3745999999999998e-05 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.823152+0000", 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043266211999999998 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.823221+0000", 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9004000000000005e-05 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:113041 8.2 e386/371 8:485708a9:::benchmark_data_smithi055_346788_object2449:head v 386'7026, mlcod=0'0)", 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:03.726635+0000", 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 7.5279044129999999, 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043800011999999999, 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:11.386 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.726635+0000", 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.726635+0000", 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.726634+0000", 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.726647+0000", 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2954e-05 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.726659+0000", 2023-09-23T18:11:11.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2745000000000001e-05 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.726668+0000", 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7390000000000005e-06 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.741472+0000", 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014804292 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.741502+0000", 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9237000000000001e-05 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.770388+0000", 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.028886882999999999 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:11.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.770435+0000", 2023-09-23T18:11:11.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6412000000000002e-05 2023-09-23T18:11:11.391 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.391 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.391 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.391 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:11.391 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:11.391 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:11:11.391 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:12.547 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:11:12.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:12 smithi099 ceph-mon[167076]: pgmap v967: 119 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 2 active+recovery_wait+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 54440/316545 objects degraded (17.198%); 64784/316545 objects misplaced (20.466%); 19 MiB/s, 303 objects/s recovering 2023-09-23T18:11:12.597 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:12 smithi055 ceph-mon[179668]: pgmap v967: 119 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 2 active+recovery_wait+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 54440/316545 objects degraded (17.198%); 64784/316545 objects misplaced (20.466%); 19 MiB/s, 303 objects/s recovering 2023-09-23T18:11:12.597 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:12 smithi055 ceph-mon[184174]: pgmap v967: 119 pgs: 1 active+remapped+backfilling, 1 remapped+peering, 2 active+recovery_wait+remapped, 21 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.4 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 568 op/s; 54440/316545 objects degraded (17.198%); 64784/316545 objects misplaced (20.466%); 19 MiB/s, 303 objects/s recovering 2023-09-23T18:11:13.269 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:13.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:13.277 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:13.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:13.380 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:11:13.429 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:13.475 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:13.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:13.480 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:13.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 54440/316545 objects degraded (17.198%), 16 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:13.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:13.611 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:13.649 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:13.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:13.654 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:13.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 54440/316545 objects degraded (17.198%), 16 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:13.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 54440/316545 objects degraded (17.198%), 16 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:13.754 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:13.845 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:13.916 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:13.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:13.925 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:13.950 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:14.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:14.068 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:14.117 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:14.125 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:14.125 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:14.126 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:14.226 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:14.302 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:14.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:14 smithi099 ceph-mon[167076]: pgmap v968: 119 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 21 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 60131/311331 objects degraded (19.314%); 70134/311331 objects misplaced (22.527%); 16 MiB/s, 251 objects/s recovering 2023-09-23T18:11:14.392 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:14.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:14.400 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:14.498 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:14 smithi055 ceph-mon[179668]: pgmap v968: 119 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 21 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 60131/311331 objects degraded (19.314%); 70134/311331 objects misplaced (22.527%); 16 MiB/s, 251 objects/s recovering 2023-09-23T18:11:14.498 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:14 smithi055 ceph-mon[184174]: pgmap v968: 119 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 21 active+clean+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 1 active+undersized+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 735 op/s; 60131/311331 objects degraded (19.314%); 70134/311331 objects misplaced (22.527%); 16 MiB/s, 251 objects/s recovering 2023-09-23T18:11:14.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:14.557 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:14.565 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:14.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:14.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:14.611 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:14.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:14.767 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:14.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:14.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:14.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:14.930 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:14.984 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:15.092 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:15.099 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:15.100 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:15.201 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:15.261 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:15.333 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:15.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:15.340 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:15.353 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:11:15.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2105301335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:15.398 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:15 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:15.398 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2105301335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:15.398 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:15 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:15.441 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:15.551 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:15.674 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:15.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:15.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:15.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:15.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2105301335' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:15.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:15 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:15.860 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:15.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:15.929 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:15.930 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:16.030 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:16.129 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:16.209 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:16.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:16.222 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:16.272 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:11:16.299 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:11:16.323 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:16.353 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:11:16.390 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:11:16.405 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:16.420 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:11:16.452 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:11:16.488 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:11:16.489 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:11:16.489 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:11:16.489 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:11:16.489 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:11:16.489 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:11:16.489 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:11:16.489 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:11:16.489 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:11:16.593 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:16.596 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[179668]: pgmap v969: 119 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 22 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 59588/308865 objects degraded (19.293%); 69591/308865 objects misplaced (22.531%); 12 MiB/s, 196 objects/s recovering 2023-09-23T18:11:16.597 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:16.597 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[179668]: failed to encode map e392 with expected crc 2023-09-23T18:11:16.597 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:11:16.597 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[179668]: osdmap e392: 8 total, 7 up, 6 in 2023-09-23T18:11:16.597 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3037310635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:16.601 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[184174]: pgmap v969: 119 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 22 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 59588/308865 objects degraded (19.293%); 69591/308865 objects misplaced (22.531%); 12 MiB/s, 196 objects/s recovering 2023-09-23T18:11:16.601 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:16.601 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[184174]: failed to encode map e392 with expected crc 2023-09-23T18:11:16.601 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:11:16.601 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[184174]: osdmap e392: 8 total, 7 up, 6 in 2023-09-23T18:11:16.601 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3037310635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:16.613 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:16.614 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:16.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:16.752 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:16 smithi099 ceph-mon[167076]: pgmap v969: 119 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 22 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.3 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 658 op/s; 59588/308865 objects degraded (19.293%); 69591/308865 objects misplaced (22.531%); 12 MiB/s, 196 objects/s recovering 2023-09-23T18:11:16.752 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:16 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:16.752 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:16 smithi099 ceph-mon[167076]: failed to encode map e392 with expected crc 2023-09-23T18:11:16.752 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:16 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:11:16.752 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:16 smithi099 ceph-mon[167076]: osdmap e392: 8 total, 7 up, 6 in 2023-09-23T18:11:16.752 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3037310635' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:16.754 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:16.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:16.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:16.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:16.912 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:16.936 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:11:16.937 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:11:17.001 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:17.124 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:17.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:17.133 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:17.169 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:17 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:11:17.123+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e392 definitely_dead 0 2023-09-23T18:11:17.234 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:17.305 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:17.378 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:17.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:17.386 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:17.395 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.3. 2023-09-23T18:11:17.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/765543149' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:11:17.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:17 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:11:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/765543149' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:11:17.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:17 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:11:17.465 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:11:17.465 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:11:17.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:17.514 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:17.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:17.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:17.554 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:17.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:17.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/765543149' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:11:17.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:17 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:11:17.682 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:17.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:17.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:17.724 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:17.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:17.853 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:17.888 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T18:11:17.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:17.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:17.895 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:17.965 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:11:17.996 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:18.041 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:18.553 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:11:18.555 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:18.555 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:18.555 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:18.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[179668]: pgmap v971: 119 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 22 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 58786/304074 objects degraded (19.333%); 68249/304074 objects misplaced (22.445%); 17 MiB/s, 277 objects/s recovering 2023-09-23T18:11:18.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[179668]: failed to encode map e393 with expected crc 2023-09-23T18:11:18.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T18:11:18.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[179668]: osdmap e393: 8 total, 7 up, 7 in 2023-09-23T18:11:18.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 58786/304074 objects degraded (19.333%), 17 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:18.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[179668]: 8.a starting backfill to osd.3 from (368'2966,386'7206] MIN to 392'7699 2023-09-23T18:11:18.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[179668]: osdmap e394: 8 total, 7 up, 7 in 2023-09-23T18:11:18.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[184174]: pgmap v971: 119 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 22 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 58786/304074 objects degraded (19.333%); 68249/304074 objects misplaced (22.445%); 17 MiB/s, 277 objects/s recovering 2023-09-23T18:11:18.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[184174]: failed to encode map e393 with expected crc 2023-09-23T18:11:18.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T18:11:18.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[184174]: osdmap e393: 8 total, 7 up, 7 in 2023-09-23T18:11:18.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 58786/304074 objects degraded (19.333%), 17 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:18.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[184174]: 8.a starting backfill to osd.3 from (368'2966,386'7206] MIN to 392'7699 2023-09-23T18:11:18.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:18 smithi055 ceph-mon[184174]: osdmap e394: 8 total, 7 up, 7 in 2023-09-23T18:11:18.653 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:18.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:18.682 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:18 smithi099 ceph-mon[167076]: pgmap v971: 119 pgs: 1 active+remapped+backfilling, 2 active+recovery_wait+remapped, 22 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+remapped+backfill_wait, 2 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 55 active+clean; 6.2 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 651 op/s; 58786/304074 objects degraded (19.333%); 68249/304074 objects misplaced (22.445%); 17 MiB/s, 277 objects/s recovering 2023-09-23T18:11:18.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:18 smithi099 ceph-mon[167076]: failed to encode map e393 with expected crc 2023-09-23T18:11:18.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:18 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T18:11:18.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:18 smithi099 ceph-mon[167076]: osdmap e393: 8 total, 7 up, 7 in 2023-09-23T18:11:18.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 58786/304074 objects degraded (19.333%), 17 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:18.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:18 smithi099 ceph-mon[167076]: 8.a starting backfill to osd.3 from (368'2966,386'7206] MIN to 392'7699 2023-09-23T18:11:18.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:18 smithi099 ceph-mon[167076]: osdmap e394: 8 total, 7 up, 7 in 2023-09-23T18:11:18.685 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:18.723 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:18.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:18.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:18.827 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:18.864 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:18.938 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:11:19.375 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:19 smithi055 ceph-mon[184174]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.375 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:19 smithi055 ceph-mon[184174]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:19 smithi055 ceph-mon[184174]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:19 smithi055 ceph-mon[184174]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.376 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:19 smithi055 ceph-mon[184174]: osdmap e395: 8 total, 7 up, 7 in 2023-09-23T18:11:19.376 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:19 smithi055 ceph-mon[179668]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.376 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:19 smithi055 ceph-mon[179668]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.376 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:19 smithi055 ceph-mon[179668]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.376 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:19 smithi055 ceph-mon[179668]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.376 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:19 smithi055 ceph-mon[179668]: osdmap e395: 8 total, 7 up, 7 in 2023-09-23T18:11:19.713 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:19.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:19.720 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:19.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:19.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:19 smithi099 ceph-mon[167076]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:19 smithi099 ceph-mon[167076]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:19 smithi099 ceph-mon[167076]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:19 smithi099 ceph-mon[167076]: failed to encode map e394 with expected crc 2023-09-23T18:11:19.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:19 smithi099 ceph-mon[167076]: osdmap e395: 8 total, 7 up, 7 in 2023-09-23T18:11:19.857 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:19.904 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:19.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:19.911 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:20.011 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:20.054 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:20.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:20.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:20.107 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:20.208 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:20.258 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:20.582 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:20.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:20.592 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:20.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:20 smithi055 ceph-mon[184174]: pgmap v974: 119 pgs: 1 active+recovery_wait+remapped, 6 peering, 8 remapped+peering, 1 active+remapped+backfilling, 17 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 6.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 766 op/s; 38978/297534 objects degraded (13.100%); 35892/297534 objects misplaced (12.063%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T18:11:20.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:20 smithi055 ceph-mon[184174]: failed to encode map e395 with expected crc 2023-09-23T18:11:20.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:20 smithi055 ceph-mon[184174]: failed to encode map e395 with expected crc 2023-09-23T18:11:20.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:20 smithi055 ceph-mon[179668]: pgmap v974: 119 pgs: 1 active+recovery_wait+remapped, 6 peering, 8 remapped+peering, 1 active+remapped+backfilling, 17 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 6.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 766 op/s; 38978/297534 objects degraded (13.100%); 35892/297534 objects misplaced (12.063%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T18:11:20.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:20 smithi055 ceph-mon[179668]: failed to encode map e395 with expected crc 2023-09-23T18:11:20.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:20 smithi055 ceph-mon[179668]: failed to encode map e395 with expected crc 2023-09-23T18:11:20.596 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:11:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:11:20] "GET /metrics HTTP/1.1" 200 36615 "" "Prometheus/2.43.0" 2023-09-23T18:11:20.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:20.732 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:20 smithi099 ceph-mon[167076]: pgmap v974: 119 pgs: 1 active+recovery_wait+remapped, 6 peering, 8 remapped+peering, 1 active+remapped+backfilling, 17 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 8 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 52 active+clean; 6.1 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 766 op/s; 38978/297534 objects degraded (13.100%); 35892/297534 objects misplaced (12.063%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T18:11:20.732 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:20 smithi099 ceph-mon[167076]: failed to encode map e395 with expected crc 2023-09-23T18:11:20.732 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:20 smithi099 ceph-mon[167076]: failed to encode map e395 with expected crc 2023-09-23T18:11:20.736 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:20.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:20.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:20.795 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:20.896 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:21.020 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:21.242 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:21.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:21.257 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:21.300 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:21.305 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:21.358 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:21.421 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:21.613 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:21.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:21.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:21.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:21.752 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:21.796 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:21.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:21.800 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:21.901 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:21.953 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:22.253 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:22.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:22.262 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:22.362 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:22.426 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:11:22.427 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:11:22.444 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:22 smithi055 ceph-mon[179668]: pgmap v976: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 peering, 8 remapped+peering, 16 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 53 active+clean; 6.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 843 op/s; 38532/294723 objects degraded (13.074%); 27577/294723 objects misplaced (9.357%); 33 MiB/s, 522 objects/s recovering 2023-09-23T18:11:22.445 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:22.445 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:22 smithi055 ceph-mon[184174]: pgmap v976: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 peering, 8 remapped+peering, 16 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 53 active+clean; 6.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 843 op/s; 38532/294723 objects degraded (13.074%); 27577/294723 objects misplaced (9.357%); 33 MiB/s, 522 objects/s recovering 2023-09-23T18:11:22.445 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:22.451 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:22.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:22 smithi099 ceph-mon[167076]: pgmap v976: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 6 peering, 8 remapped+peering, 16 active+clean+remapped, 2 active+recovering+undersized+degraded+remapped, 9 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovery_wait+degraded+remapped, 53 active+clean; 6.0 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 843 op/s; 38532/294723 objects degraded (13.074%); 27577/294723 objects misplaced (9.357%); 33 MiB/s, 522 objects/s recovering 2023-09-23T18:11:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:22.881 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:22.887 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:22.888 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:22.888 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 4, 1, 0, 3] out_osds: [2] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:11:22.888 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:11:22.888 INFO:tasks.thrashosds.thrasher:Removing osd 4, in_osds are: [6, 7, 5, 4, 1, 0, 3] 2023-09-23T18:11:22.889 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 4 2023-09-23T18:11:22.986 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:11:22.989 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:23.124 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:23.262 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:23.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:23.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:23.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:23.431 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:23.476 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:23.482 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:23.519 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:11:23.566 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 38532/294723 objects degraded (13.074%), 13 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:23.566 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 38532/294723 objects degraded (13.074%), 13 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:23.567 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:11:23.583 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:23.600 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:11:23.636 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:11:23.640 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:23.664 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:11:23.702 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:11:23.737 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:11:23.738 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:11:23.738 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:11:23.738 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:11:23.738 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:11:23.738 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:11:23.738 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:11:23.739 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:11:23.739 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:11:23.757 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:23.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:23.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:23.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 38532/294723 objects degraded (13.074%), 13 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:23.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:23.912 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:23.922 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:23.961 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:23.964 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:23.964 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:24.065 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:24.136 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:24.201 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:11:24.202 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:11:24.292 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:24.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:24.297 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:24.297 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:24.398 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:24.450 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:11:24.460 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:24 smithi055 ceph-mon[184174]: pgmap v977: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 3 peering, 3 remapped+peering, 16 active+clean+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 5.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 49865/291153 objects degraded (17.127%); 26956/291153 objects misplaced (9.258%); 28 MiB/s, 443 objects/s recovering 2023-09-23T18:11:24.460 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/266263451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:24.460 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2691923108' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:11:24.460 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:24 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:11:24.460 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:24 smithi055 ceph-mon[179668]: pgmap v977: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 3 peering, 3 remapped+peering, 16 active+clean+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 5.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 49865/291153 objects degraded (17.127%); 26956/291153 objects misplaced (9.258%); 28 MiB/s, 443 objects/s recovering 2023-09-23T18:11:24.460 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/266263451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:24.460 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2691923108' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:11:24.460 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:24 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:11:24.461 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:24.519 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:24.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:24.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:24.624 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:24.674 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:24.733 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:24.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:24.739 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:24 smithi099 ceph-mon[167076]: pgmap v977: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 3 peering, 3 remapped+peering, 16 active+clean+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 10 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 58 active+clean; 5.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 717 op/s; 49865/291153 objects degraded (17.127%); 26956/291153 objects misplaced (9.258%); 28 MiB/s, 443 objects/s recovering 2023-09-23T18:11:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/266263451' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2691923108' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:11:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:24 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:11:24.839 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:24.876 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:24.924 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:24.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:24.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:25.029 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:25.105 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:25.508 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:25.514 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:25.515 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:25.615 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:25.659 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:25.701 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:25.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:25.704 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:25.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:25 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:25.732 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:25 smithi055 ceph-mon[184174]: failed to encode map e396 with expected crc 2023-09-23T18:11:25.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:25 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:11:25.733 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:25 smithi055 ceph-mon[184174]: osdmap e396: 8 total, 7 up, 7 in 2023-09-23T18:11:25.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:25 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:25.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:25 smithi055 ceph-mon[179668]: failed to encode map e396 with expected crc 2023-09-23T18:11:25.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:25 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:11:25.734 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:25 smithi055 ceph-mon[179668]: osdmap e396: 8 total, 7 up, 7 in 2023-09-23T18:11:25.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:25.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:25 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:25.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:25 smithi099 ceph-mon[167076]: failed to encode map e396 with expected crc 2023-09-23T18:11:25.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:25 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:11:25.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:25 smithi099 ceph-mon[167076]: osdmap e396: 8 total, 7 up, 7 in 2023-09-23T18:11:25.868 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:25.925 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:25.928 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:25.928 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:26.029 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:26.076 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:26.203 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:11:26.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:26 smithi099 ceph-mon[167076]: pgmap v979: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 19 active+clean+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 5.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 55597/289560 objects degraded (19.201%); 26766/289560 objects misplaced (9.244%); 24 MiB/s, 378 objects/s recovering 2023-09-23T18:11:26.890 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:26.901 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:26 smithi055 ceph-mon[179668]: pgmap v979: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 19 active+clean+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 5.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 55597/289560 objects degraded (19.201%); 26766/289560 objects misplaced (9.244%); 24 MiB/s, 378 objects/s recovering 2023-09-23T18:11:26.902 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:26 smithi055 ceph-mon[184174]: pgmap v979: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 19 active+clean+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 5.9 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 420 op/s; 55597/289560 objects degraded (19.201%); 26766/289560 objects misplaced (9.244%); 24 MiB/s, 378 objects/s recovering 2023-09-23T18:11:26.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:26.903 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:27.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:27.047 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:27.108 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:27.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:27.114 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:27.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:27.264 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:27.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:27.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:27.318 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:27.419 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:27.498 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:27.539 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:27 smithi055 ceph-mon[184174]: pgmap v980: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 19 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 54941/285591 objects degraded (19.238%); 25443/285591 objects misplaced (8.909%); 30 MiB/s, 474 objects/s recovering 2023-09-23T18:11:27.539 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:27 smithi055 ceph-mon[179668]: pgmap v980: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 19 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 54941/285591 objects degraded (19.238%); 25443/285591 objects misplaced (8.909%); 30 MiB/s, 474 objects/s recovering 2023-09-23T18:11:27.566 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:27.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:27.575 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:27.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:27 smithi099 ceph-mon[167076]: pgmap v980: 119 pgs: 1 active+remapped+backfilling, 1 active+recovery_wait+remapped, 19 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 497 op/s; 54941/285591 objects degraded (19.238%); 25443/285591 objects misplaced (8.909%); 30 MiB/s, 474 objects/s recovering 2023-09-23T18:11:27.675 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:27.733 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:11:27.747 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:27.823 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:11:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:11:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:11:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:11:27.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33540 6.e 6:757bf288:::benchmark_data_smithi055_275963_object33539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.437994+0000", 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 588.38062325600004, 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.108388157, 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33540 2023-09-23T18:11:27.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.437994+0000", 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.437994+0000", 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.437991+0000", 2023-09-23T18:11:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999967 2023-09-23T18:11:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438029+0000", 2023-09-23T18:11:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.824e-05 2023-09-23T18:11:27.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438067+0000", 2023-09-23T18:11:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8182999999999997e-05 2023-09-23T18:11:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438088+0000", 2023-09-23T18:11:27.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0129999999999999e-05 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440561+0000", 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024736430000000002 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440572+0000", 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0878e-05 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.534316+0000", 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.093743492 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.534461+0000", 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014496100000000001 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:11:27.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.534632+0000", 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017151100000000001 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.539612+0000", 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049804710000000002 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.543049+0000", 2023-09-23T18:11:27.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003436588 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.544731+0000", 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016814320000000001 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.544791+0000", 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0006000000000001e-05 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546382+0000", 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001591765 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33544 6.e 6:7c426a47:::benchmark_data_smithi055_275963_object33543:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.438104+0000", 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 588.38051313100004, 2023-09-23T18:11:27.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.109720906, 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33544 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438104+0000", 2023-09-23T18:11:27.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438104+0000", 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438095+0000", 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999905 2023-09-23T18:11:27.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438139+0000", 2023-09-23T18:11:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3952999999999997e-05 2023-09-23T18:11:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438945+0000", 2023-09-23T18:11:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00080640499999999995 2023-09-23T18:11:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438968+0000", 2023-09-23T18:11:27.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3285000000000002e-05 2023-09-23T18:11:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440588+0000", 2023-09-23T18:11:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001619896 2023-09-23T18:11:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440592+0000", 2023-09-23T18:11:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8349999999999997e-06 2023-09-23T18:11:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.537672+0000", 2023-09-23T18:11:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0970796060000001 2023-09-23T18:11:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.537831+0000", 2023-09-23T18:11:27.841 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015879199999999999 2023-09-23T18:11:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:11:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.537997+0000", 2023-09-23T18:11:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016669100000000001 2023-09-23T18:11:27.842 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546403+0000", 2023-09-23T18:11:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0084053589999999994 2023-09-23T18:11:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546451+0000", 2023-09-23T18:11:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8115000000000003e-05 2023-09-23T18:11:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.844 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546557+0000", 2023-09-23T18:11:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010645300000000001 2023-09-23T18:11:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546603+0000", 2023-09-23T18:11:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6140000000000002e-05 2023-09-23T18:11:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547825+0000", 2023-09-23T18:11:27.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001221845 2023-09-23T18:11:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.847 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33549 6.e 6:7eeaa6a3:::benchmark_data_smithi055_275963_object33548:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:11:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.438977+0000", 2023-09-23T18:11:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 588.37964052200005, 2023-09-23T18:11:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.110446166, 2023-09-23T18:11:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.848 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:11:27.849 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:11:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33549 2023-09-23T18:11:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438977+0000", 2023-09-23T18:11:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438977+0000", 2023-09-23T18:11:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.438976+0000", 2023-09-23T18:11:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439029+0000", 2023-09-23T18:11:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3094999999999999e-05 2023-09-23T18:11:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439039+0000", 2023-09-23T18:11:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6029999999999994e-06 2023-09-23T18:11:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.854 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439047+0000", 2023-09-23T18:11:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5599999999999994e-06 2023-09-23T18:11:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440614+0000", 2023-09-23T18:11:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015667210000000001 2023-09-23T18:11:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440620+0000", 2023-09-23T18:11:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7189999999999998e-06 2023-09-23T18:11:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.539732+0000", 2023-09-23T18:11:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0991117319999999 2023-09-23T18:11:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.856 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.539867+0000", 2023-09-23T18:11:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000135091 2023-09-23T18:11:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:11:27.857 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.540051+0000", 2023-09-23T18:11:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018418899999999999 2023-09-23T18:11:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.858 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546410+0000", 2023-09-23T18:11:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063587649999999997 2023-09-23T18:11:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.859 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547897+0000", 2023-09-23T18:11:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014873009999999999 2023-09-23T18:11:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.860 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547939+0000", 2023-09-23T18:11:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2076999999999997e-05 2023-09-23T18:11:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.547993+0000", 2023-09-23T18:11:27.861 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4109e-05 2023-09-23T18:11:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.549423+0000", 2023-09-23T18:11:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001429865 2023-09-23T18:11:27.862 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45092.0:33551 6.e 6:7ac7b330:::benchmark_data_smithi055_275963_object33550:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e220)", 2023-09-23T18:11:27.863 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:01:39.439054+0000", 2023-09-23T18:11:27.864 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 588.37956347500005, 2023-09-23T18:11:27.864 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.113266128, 2023-09-23T18:11:27.864 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.864 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.864 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45092", 2023-09-23T18:11:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/881972110", 2023-09-23T18:11:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 33551 2023-09-23T18:11:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.865 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439054+0000", 2023-09-23T18:11:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.866 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439054+0000", 2023-09-23T18:11:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439053+0000", 2023-09-23T18:11:27.867 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439099+0000", 2023-09-23T18:11:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5565999999999997e-05 2023-09-23T18:11:27.868 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439108+0000", 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.789e-06 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.439114+0000", 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.567e-06 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.869 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440644+0000", 2023-09-23T18:11:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001530127 2023-09-23T18:11:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:39.440649+0000", 2023-09-23T18:11:27.870 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.211e-06 2023-09-23T18:11:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.541379+0000", 2023-09-23T18:11:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.100730271 2023-09-23T18:11:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.541553+0000", 2023-09-23T18:11:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017443600000000001 2023-09-23T18:11:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,4", 2023-09-23T18:11:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.541772+0000", 2023-09-23T18:11:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021913 2023-09-23T18:11:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.546460+0000", 2023-09-23T18:11:27.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004687241 2023-09-23T18:11:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.549485+0000", 2023-09-23T18:11:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030253210000000001 2023-09-23T18:11:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.551089+0000", 2023-09-23T18:11:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016042529999999999 2023-09-23T18:11:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.551163+0000", 2023-09-23T18:11:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3696999999999996e-05 2023-09-23T18:11:27.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:01:40.552320+0000", 2023-09-23T18:11:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00115711 2023-09-23T18:11:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.876 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:108086 7.22 7:46c8def9:::benchmark_data_smithi055_311528_object37123:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:11:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:39.639458+0000", 2023-09-23T18:11:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 168.17915949900001, 2023-09-23T18:11:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.86476655800000002, 2023-09-23T18:11:27.877 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108086 2023-09-23T18:11:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.878 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639458+0000", 2023-09-23T18:11:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639458+0000", 2023-09-23T18:11:27.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639455+0000", 2023-09-23T18:11:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999971 2023-09-23T18:11:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639469+0000", 2023-09-23T18:11:27.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4528999999999999e-05 2023-09-23T18:11:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639496+0000", 2023-09-23T18:11:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6537e-05 2023-09-23T18:11:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.881 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639510+0000", 2023-09-23T18:11:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3509999999999999e-05 2023-09-23T18:11:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.780408+0000", 2023-09-23T18:11:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14089864299999999 2023-09-23T18:11:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.780419+0000", 2023-09-23T18:11:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0786e-05 2023-09-23T18:11:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.500214+0000", 2023-09-23T18:11:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.719795454 2023-09-23T18:11:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.500334+0000", 2023-09-23T18:11:27.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119627 2023-09-23T18:11:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,3", 2023-09-23T18:11:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.500459+0000", 2023-09-23T18:11:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012500400000000001 2023-09-23T18:11:27.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.501601+0000", 2023-09-23T18:11:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011421140000000001 2023-09-23T18:11:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.502716+0000", 2023-09-23T18:11:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001114959 2023-09-23T18:11:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.504110+0000", 2023-09-23T18:11:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013935799999999999 2023-09-23T18:11:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.504161+0000", 2023-09-23T18:11:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1282999999999997e-05 2023-09-23T18:11:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.504224+0000", 2023-09-23T18:11:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3471000000000003e-05 2023-09-23T18:11:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.890 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:108123 7.b 7:d79fe6bb:::benchmark_data_smithi055_311528_object37160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:40.575125+0000", 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 167.243492667, 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95218377899999995, 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.891 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108123 2023-09-23T18:11:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575125+0000", 2023-09-23T18:11:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575125+0000", 2023-09-23T18:11:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575123+0000", 2023-09-23T18:11:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:11:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575129+0000", 2023-09-23T18:11:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0599999999999996e-06 2023-09-23T18:11:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575144+0000", 2023-09-23T18:11:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4541e-05 2023-09-23T18:11:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575152+0000", 2023-09-23T18:11:27.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1429999999999998e-06 2023-09-23T18:11:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575229+0000", 2023-09-23T18:11:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7212999999999994e-05 2023-09-23T18:11:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575241+0000", 2023-09-23T18:11:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2505e-05 2023-09-23T18:11:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.520155+0000", 2023-09-23T18:11:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.94491320499999998 2023-09-23T18:11:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.520539+0000", 2023-09-23T18:11:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000384736 2023-09-23T18:11:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.898 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:11:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.521133+0000", 2023-09-23T18:11:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00059405799999999996 2023-09-23T18:11:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.524812+0000", 2023-09-23T18:11:27.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036780530000000001 2023-09-23T18:11:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.526787+0000", 2023-09-23T18:11:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019753050000000001 2023-09-23T18:11:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.526873+0000", 2023-09-23T18:11:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6248000000000005e-05 2023-09-23T18:11:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.526994+0000", 2023-09-23T18:11:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120562 2023-09-23T18:11:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.527308+0000", 2023-09-23T18:11:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00031482999999999999 2023-09-23T18:11:27.902 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:108185 7.b 7:d2330501:::benchmark_data_smithi055_311528_object37222:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:11:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:40.683688+0000", 2023-09-23T18:11:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 167.13492912699999, 2023-09-23T18:11:27.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.84651908899999995, 2023-09-23T18:11:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108185 2023-09-23T18:11:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.683688+0000", 2023-09-23T18:11:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.683688+0000", 2023-09-23T18:11:27.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.683687+0000", 2023-09-23T18:11:27.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:27.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.683694+0000", 2023-09-23T18:11:27.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5980000000000002e-06 2023-09-23T18:11:27.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.683707+0000", 2023-09-23T18:11:27.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2993000000000001e-05 2023-09-23T18:11:27.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.683714+0000", 2023-09-23T18:11:27.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6960000000000005e-06 2023-09-23T18:11:27.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.683758+0000", 2023-09-23T18:11:27.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4211000000000001e-05 2023-09-23T18:11:27.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.683766+0000", 2023-09-23T18:11:27.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5780000000000001e-06 2023-09-23T18:11:27.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.522932+0000", 2023-09-23T18:11:27.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.83916644200000001 2023-09-23T18:11:27.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.523173+0000", 2023-09-23T18:11:27.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024053200000000001 2023-09-23T18:11:27.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:11:27.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.523599+0000", 2023-09-23T18:11:27.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00042600100000000002 2023-09-23T18:11:27.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.527382+0000", 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037832880000000001 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.527406+0000", 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3770000000000001e-05 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.529754+0000", 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00234768 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.529865+0000", 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011110700000000001 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.530207+0000", 2023-09-23T18:11:27.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00034253499999999999 2023-09-23T18:11:27.915 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.915 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.915 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.915 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:108248 7.b 7:d150079a:::benchmark_data_smithi055_311528_object37285:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:11:27.915 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:40.757291+0000", 2023-09-23T18:11:27.915 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 167.06132688100001, 2023-09-23T18:11:27.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.77352319300000005, 2023-09-23T18:11:27.916 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.916 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.916 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.916 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.916 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.916 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108248 2023-09-23T18:11:27.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.917 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.757291+0000", 2023-09-23T18:11:27.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.757291+0000", 2023-09-23T18:11:27.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.757290+0000", 2023-09-23T18:11:27.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:27.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.757295+0000", 2023-09-23T18:11:27.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3290000000000001e-06 2023-09-23T18:11:27.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.757327+0000", 2023-09-23T18:11:27.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1458000000000003e-05 2023-09-23T18:11:27.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.757342+0000", 2023-09-23T18:11:27.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5276000000000001e-05 2023-09-23T18:11:27.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.757454+0000", 2023-09-23T18:11:27.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111637 2023-09-23T18:11:27.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.757464+0000", 2023-09-23T18:11:27.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.075e-05 2023-09-23T18:11:27.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.524889+0000", 2023-09-23T18:11:27.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.76742455200000004 2023-09-23T18:11:27.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.525217+0000", 2023-09-23T18:11:27.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00032863899999999998 2023-09-23T18:11:27.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.525624+0000", 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00040606799999999999 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.527470+0000", 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001846037 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.529643+0000", 2023-09-23T18:11:27.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021735639999999998 2023-09-23T18:11:27.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.530306+0000", 2023-09-23T18:11:27.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00066284700000000005 2023-09-23T18:11:27.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.530435+0000", 2023-09-23T18:11:27.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012907000000000001 2023-09-23T18:11:27.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.927 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.530814+0000", 2023-09-23T18:11:27.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00037866400000000002 2023-09-23T18:11:27.928 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.928 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.928 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.928 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130809 7.19 7:9fc2251d:::benchmark_data_smithi055_311528_object59846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:27.928 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.096744+0000", 2023-09-23T18:11:27.929 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.72187316, 2023-09-23T18:11:27.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.038223272, 2023-09-23T18:11:27.929 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.929 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.929 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.929 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.930 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.930 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130809 2023-09-23T18:11:27.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.930 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096744+0000", 2023-09-23T18:11:27.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096744+0000", 2023-09-23T18:11:27.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096743+0000", 2023-09-23T18:11:27.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:27.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096750+0000", 2023-09-23T18:11:27.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5590000000000002e-06 2023-09-23T18:11:27.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096759+0000", 2023-09-23T18:11:27.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6590000000000002e-06 2023-09-23T18:11:27.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096766+0000", 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6499999999999999e-06 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.097072+0000", 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00030662300000000001 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.097082+0000", 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5710000000000004e-06 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.127052+0000", 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.029969822 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.127174+0000", 2023-09-23T18:11:27.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000122481 2023-09-23T18:11:27.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:27.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.127480+0000", 2023-09-23T18:11:27.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00030583699999999999 2023-09-23T18:11:27.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134732+0000", 2023-09-23T18:11:27.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072513999999999999 2023-09-23T18:11:27.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134740+0000", 2023-09-23T18:11:27.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3990000000000004e-06 2023-09-23T18:11:27.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134861+0000", 2023-09-23T18:11:27.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000121248 2023-09-23T18:11:27.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134922+0000", 2023-09-23T18:11:27.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1042000000000003e-05 2023-09-23T18:11:27.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134967+0000", 2023-09-23T18:11:27.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5229999999999999e-05 2023-09-23T18:11:27.940 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.941 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.941 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.941 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130810 7.19 7:9eaf2462:::benchmark_data_smithi055_311528_object59847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:27.941 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.096843+0000", 2023-09-23T18:11:27.941 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.72177399500001, 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.038332319, 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130810 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096843+0000", 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096843+0000", 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096843+0000", 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:27.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096847+0000", 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4139999999999996e-06 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096857+0000", 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4029999999999996e-06 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096863+0000", 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7070000000000003e-06 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.097102+0000", 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000238958 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.097110+0000", 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3579999999999997e-06 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.129315+0000", 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.032205713 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.129414+0000", 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8196999999999995e-05 2023-09-23T18:11:27.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.129683+0000", 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026979100000000002 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134985+0000", 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053020360000000004 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135052+0000", 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6750999999999998e-05 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135093+0000", 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1121999999999997e-05 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135145+0000", 2023-09-23T18:11:27.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1456000000000003e-05 2023-09-23T18:11:27.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135176+0000", 2023-09-23T18:11:27.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1028000000000002e-05 2023-09-23T18:11:27.949 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.949 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.949 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130818 7.19 7:99afb0dd:::benchmark_data_smithi055_311528_object59855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.108760+0000", 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.709857269, 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0266103639999999, 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130818 2023-09-23T18:11:27.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108760+0000", 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108760+0000", 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108758+0000", 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108766+0000", 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5859999999999999e-06 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108780+0000", 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4324e-05 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108789+0000", 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7849999999999997e-06 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108958+0000", 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000168937 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108970+0000", 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2383e-05 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.131100+0000", 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0221301270000001 2023-09-23T18:11:27.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.131236+0000", 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013559800000000001 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.131498+0000", 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026210699999999998 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134993+0000", 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034951940000000001 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135222+0000", 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00022855 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135265+0000", 2023-09-23T18:11:27.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3387999999999999e-05 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135323+0000", 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8149999999999997e-05 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135370+0000", 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7070000000000002e-05 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.957 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130851 7.19 7:99593c89:::benchmark_data_smithi055_311528_object59888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.149542+0000", 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.669075158, 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98596146699999998, 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.958 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130851 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149542+0000", 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149542+0000", 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149541+0000", 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149547+0000", 2023-09-23T18:11:27.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8769999999999998e-06 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149555+0000", 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4030000000000007e-06 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149563+0000", 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5510000000000002e-06 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.151045+0000", 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001481818 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.151052+0000", 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1679999999999998e-06 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.132864+0000", 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.981811824 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.132970+0000", 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000106037 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.133254+0000", 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00028440900000000001 2023-09-23T18:11:27.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135198+0000", 2023-09-23T18:11:27.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00194392 2023-09-23T18:11:27.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135423+0000", 2023-09-23T18:11:27.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000224711 2023-09-23T18:11:27.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135456+0000", 2023-09-23T18:11:27.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3707000000000001e-05 2023-09-23T18:11:27.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135482+0000", 2023-09-23T18:11:27.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5802000000000001e-05 2023-09-23T18:11:27.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135504+0000", 2023-09-23T18:11:27.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1605000000000001e-05 2023-09-23T18:11:27.966 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.966 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.966 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.966 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130859 7.19 7:9cd3c3cf:::benchmark_data_smithi055_311528_object59896:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:27.967 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.160780+0000", 2023-09-23T18:11:27.967 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.657837223, 2023-09-23T18:11:27.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98387569699999999, 2023-09-23T18:11:27.967 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.967 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.967 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.967 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.967 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.967 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130859 2023-09-23T18:11:27.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160780+0000", 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160780+0000", 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160779+0000", 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160786+0000", 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8090000000000004e-06 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160797+0000", 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1953999999999999e-05 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160806+0000", 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2989999999999997e-06 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160869+0000", 2023-09-23T18:11:27.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3701999999999994e-05 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160896+0000", 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6448000000000002e-05 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135538+0000", 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97464208399999996 2023-09-23T18:11:27.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135632+0000", 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3869000000000005e-05 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135828+0000", 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000196205 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.140042+0000", 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0042135619999999997 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144498+0000", 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044563279999999999 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144556+0000", 2023-09-23T18:11:27.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8196000000000003e-05 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144616+0000", 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9828999999999999e-05 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144656+0000", 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9838e-05 2023-09-23T18:11:27.974 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130860 7.19 7:9f617653:::benchmark_data_smithi055_311528_object59897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.160819+0000", 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.65779845399999, 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98406092999999994, 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.975 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130860 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160819+0000", 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160819+0000", 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160818+0000", 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160822+0000", 2023-09-23T18:11:27.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3390000000000001e-06 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160829+0000", 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7130000000000005e-06 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160835+0000", 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9710000000000001e-06 2023-09-23T18:11:27.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160932+0000", 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6945000000000001e-05 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160942+0000", 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2029999999999998e-06 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.137518+0000", 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97657627899999999 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.137637+0000", 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119502 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.137906+0000", 2023-09-23T18:11:27.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026830400000000003 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.142645+0000", 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047394070000000002 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144709+0000", 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020640020000000001 2023-09-23T18:11:27.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144784+0000", 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5236999999999998e-05 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144836+0000", 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1625999999999997e-05 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144880+0000", 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3964999999999997e-05 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130895 7.19 7:9b882bdd:::benchmark_data_smithi055_311528_object59932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.192324+0000", 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.62629322199999, 2023-09-23T18:11:27.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95269991499999995, 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130895 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192324+0000", 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192324+0000", 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192323+0000", 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:27.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192329+0000", 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8669999999999999e-06 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192340+0000", 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1095e-05 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192345+0000", 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5330000000000002e-06 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192385+0000", 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9743999999999998e-05 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192395+0000", 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0068000000000001e-05 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.138986+0000", 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.94659121700000004 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.139056+0000", 2023-09-23T18:11:27.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9802999999999996e-05 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.139230+0000", 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017401100000000001 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144510+0000", 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0052801319999999999 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144750+0000", 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024012999999999999 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144963+0000", 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000212189 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144990+0000", 2023-09-23T18:11:27.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7562000000000001e-05 2023-09-23T18:11:27.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:27.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145024+0000", 2023-09-23T18:11:27.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3930000000000002e-05 2023-09-23T18:11:27.991 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.991 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:27.991 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:27.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130917 7.19 7:99576fc8:::benchmark_data_smithi055_311528_object59954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.210519+0000", 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.608098641, 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.93460085299999995, 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130917 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.992 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210519+0000", 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210519+0000", 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210518+0000", 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210524+0000", 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6829999999999996e-06 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210544+0000", 2023-09-23T18:11:27.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0341999999999999e-05 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210552+0000", 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5799999999999992e-06 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210613+0000", 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0959999999999999e-05 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210628+0000", 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4586999999999999e-05 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.140077+0000", 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.929449148 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:27.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.140177+0000", 2023-09-23T18:11:27.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000100169 2023-09-23T18:11:27.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:27.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.140402+0000", 2023-09-23T18:11:27.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000224243 2023-09-23T18:11:27.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:27.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144518+0000", 2023-09-23T18:11:27.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041164879999999997 2023-09-23T18:11:27.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144927+0000", 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000408565 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145047+0000", 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012084200000000001 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:27.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145082+0000", 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5000999999999999e-05 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145120+0000", 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7154999999999999e-05 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130975 7.19 7:9b428e91:::benchmark_data_smithi055_311528_object60012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.264723+0000", 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.55389447600001, 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.88050636100000002, 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130975 2023-09-23T18:11:28.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264723+0000", 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264723+0000", 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:28.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264721+0000", 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264728+0000", 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8340000000000001e-06 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264741+0000", 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2666999999999999e-05 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264748+0000", 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.311e-06 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:28.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264805+0000", 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6986e-05 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264816+0000", 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1268e-05 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.141359+0000", 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.87654296300000001 2023-09-23T18:11:28.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.141441+0000", 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1876e-05 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.141611+0000", 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000169727 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144666+0000", 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030556799999999999 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145144+0000", 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00047729100000000002 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:28.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145160+0000", 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6844999999999999e-05 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145200+0000", 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9512999999999999e-05 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145229+0000", 2023-09-23T18:11:28.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9258e-05 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130994 7.19 7:9a8395d6:::benchmark_data_smithi055_311528_object60031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.285435+0000", 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.53318214999999, 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.86362094700000003, 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:28.009 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130994 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285435+0000", 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285435+0000", 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285434+0000", 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285440+0000", 2023-09-23T18:11:28.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8250000000000001e-06 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285452+0000", 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.188e-05 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285459+0000", 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5379999999999997e-06 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285513+0000", 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4119000000000001e-05 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285521+0000", 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.072e-06 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:28.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.142673+0000", 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.85715205299999997 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.142795+0000", 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012132499999999999 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.143009+0000", 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000214572 2023-09-23T18:11:28.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144717+0000", 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001707339 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145278+0000", 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00056135099999999997 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.148998+0000", 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037199619999999998 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.149030+0000", 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2348000000000003e-05 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.149056+0000", 2023-09-23T18:11:28.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5958000000000001e-05 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:131060 7.19 7:9daf985b:::benchmark_data_smithi055_311528_object60097:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.366148+0000", 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.45246970599999, 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.78629320400000002, 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:28.017 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 131060 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.366148+0000", 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.366148+0000", 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.366147+0000", 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.366152+0000", 2023-09-23T18:11:28.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4060000000000004e-06 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.366161+0000", 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9400000000000008e-06 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.366166+0000", 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3129999999999998e-06 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.366234+0000", 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8058999999999993e-05 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.366245+0000", 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0903999999999999e-05 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:28.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145306+0000", 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.77906028500000002 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145417+0000", 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011138599999999999 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145636+0000", 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021898600000000001 2023-09-23T18:11:28.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.152248+0000", 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0066116609999999996 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.152260+0000", 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2575000000000001e-05 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.152355+0000", 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4825000000000001e-05 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.152403+0000", 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7759999999999997e-05 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.152441+0000", 2023-09-23T18:11:28.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8019999999999999e-05 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:131078 7.19 7:98476dbc:::benchmark_data_smithi055_311528_object60115:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.390551+0000", 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 144.428065887, 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.79658990799999996, 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:28.025 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 131078 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.390551+0000", 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.390551+0000", 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.390550+0000", 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:28.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.390557+0000", 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5969999999999997e-06 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.390567+0000", 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0875e-05 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.390575+0000", 2023-09-23T18:11:28.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.345e-06 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.390615+0000", 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9941000000000003e-05 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.390626+0000", 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1362999999999999e-05 2023-09-23T18:11:28.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.146990+0000", 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75636372299999999 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.147099+0000", 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000108733 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.147359+0000", 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026064399999999998 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.152268+0000", 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049090130000000003 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.152321+0000", 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3038000000000003e-05 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.187051+0000", 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.034729479000000001 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:28.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.187102+0000", 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1551000000000002e-05 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.187141+0000", 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9150000000000003e-05 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:28.033 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:11:28.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:28.035 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:28.035 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:28.035 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:28.136 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:28.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:27 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:11:27.922+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e396 definitely_dead 0 2023-09-23T18:11:28.190 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:28.241 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:28.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:28.250 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:28.317 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:11:28.351 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:28.441 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:28.508 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:28.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:28.511 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:28.540 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.4. 2023-09-23T18:11:28.611 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:28.666 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:28.798 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:28 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2796501935' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:11:28.798 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:28 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:11:28.799 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:28 smithi055 ceph-mon[184174]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T18:11:28.799 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 54941/285591 objects degraded (19.238%), 16 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:28.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:28 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2796501935' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:11:28.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:28 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:11:28.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:28 smithi055 ceph-mon[179668]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T18:11:28.800 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 54941/285591 objects degraded (19.238%), 16 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:28.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:28 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2796501935' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:11:28.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:28 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["4"]}]: dispatch 2023-09-23T18:11:28.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:28 smithi099 ceph-mon[167076]: Client client.admin marked osd.4 out, while it was still marked up 2023-09-23T18:11:28.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 54941/285591 objects degraded (19.238%), 16 pgs degraded, 30 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:29.078 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:29.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:29.081 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:29.182 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:29.219 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:29.320 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:29.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:29.326 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:29.427 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:29.467 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:29.543 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:29.551 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:29.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:29.557 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:29.557 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:29.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:29 smithi055 ceph-mon[179668]: failed to encode map e397 with expected crc 2023-09-23T18:11:29.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:29 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T18:11:29.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:29 smithi055 ceph-mon[179668]: osdmap e397: 8 total, 7 up, 6 in 2023-09-23T18:11:29.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:29 smithi055 ceph-mon[179668]: pgmap v982: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 19 active+clean+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 5.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 650 op/s; 53640/279111 objects degraded (19.218%); 24789/279111 objects misplaced (8.881%); 16 MiB/s, 258 objects/s recovering 2023-09-23T18:11:29.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:29 smithi055 ceph-mon[184174]: failed to encode map e397 with expected crc 2023-09-23T18:11:29.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:29 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T18:11:29.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:29 smithi055 ceph-mon[184174]: osdmap e397: 8 total, 7 up, 6 in 2023-09-23T18:11:29.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:29 smithi055 ceph-mon[184174]: pgmap v982: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 19 active+clean+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 5.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 650 op/s; 53640/279111 objects degraded (19.218%); 24789/279111 objects misplaced (8.881%); 16 MiB/s, 258 objects/s recovering 2023-09-23T18:11:29.659 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:29.691 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:29.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:29 smithi099 ceph-mon[167076]: failed to encode map e397 with expected crc 2023-09-23T18:11:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:29 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["4"]}]': finished 2023-09-23T18:11:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:29 smithi099 ceph-mon[167076]: osdmap e397: 8 total, 7 up, 6 in 2023-09-23T18:11:29.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:29 smithi099 ceph-mon[167076]: pgmap v982: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 19 active+clean+remapped, 1 active+recovery_wait, 1 active+recovering+undersized+degraded+remapped, 11 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 21 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 60 active+clean; 5.7 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 650 op/s; 53640/279111 objects degraded (19.218%); 24789/279111 objects misplaced (8.881%); 16 MiB/s, 258 objects/s recovering 2023-09-23T18:11:29.860 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:29.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:29.864 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:29.881 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:11:29.900 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:11:29.920 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:11:29.943 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:11:29.963 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:11:29.965 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:29.985 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:11:29.997 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:30.015 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:11:30.016 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:11:30.016 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:11:30.016 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:11:30.016 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:11:30.016 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:11:30.016 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:11:30.016 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:11:30.017 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:11:30.091 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:30.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:30.096 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:30.197 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:30.226 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:30.641 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:30.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[179668]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[179668]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[179668]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[179668]: osdmap e398: 8 total, 7 up, 6 in 2023-09-23T18:11:30.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[179668]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1100106751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:30.651 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:11:30] "GET /metrics HTTP/1.1" 200 36633 "" "Prometheus/2.43.0" 2023-09-23T18:11:30.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[184174]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[184174]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.652 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[184174]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.652 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[184174]: osdmap e398: 8 total, 7 up, 6 in 2023-09-23T18:11:30.652 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[184174]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.652 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1100106751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:30.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:30.653 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:30.707 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:11:30.707 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:11:30.754 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:30.824 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:30.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:30 smithi099 ceph-mon[167076]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:30 smithi099 ceph-mon[167076]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:30 smithi099 ceph-mon[167076]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:30 smithi099 ceph-mon[167076]: osdmap e398: 8 total, 7 up, 6 in 2023-09-23T18:11:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:30 smithi099 ceph-mon[167076]: failed to encode map e398 with expected crc 2023-09-23T18:11:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1100106751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:30.902 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:30.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:30.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:31.018 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:31.078 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:31.305 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:31.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:31.310 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:31.411 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:31.528 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:31.700 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:31.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:31.706 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:31.807 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:31.815 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: osdmap e399: 8 total, 7 up, 6 in 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: 8.4 starting backfill to osd.3 from (0'0,0'0] MAX to 396'8032 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: 8.b starting backfill to osd.6 from (0'0,0'0] MAX to 396'7998 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: 8.5 starting backfill to osd.0 from (0'0,0'0] MAX to 396'8066 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: 8.e continuing backfill to osd.1 from (372'3597,396'8094] 8:7b7c7e01:::benchmark_data_smithi055_346788_object57885:head to 396'8094 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: 8.e starting backfill to osd.3 from (0'0,0'0] MAX to 396'8094 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: 8.f starting backfill to osd.6 from (0'0,0'0] MAX to 396'8032 2023-09-23T18:11:31.816 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[179668]: pgmap v985: 119 pgs: 5 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 22 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 1 active+recovery_wait, 8 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 51 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 3.3 KiB/s rd, 9.2 KiB/s wr, 665 op/s; 49403/277629 objects degraded (17.795%); 39911/277629 objects misplaced (14.376%); 18 MiB/s, 279 objects/s recovering 2023-09-23T18:11:31.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: osdmap e399: 8 total, 7 up, 6 in 2023-09-23T18:11:31.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: 8.4 starting backfill to osd.3 from (0'0,0'0] MAX to 396'8032 2023-09-23T18:11:31.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: 8.b starting backfill to osd.6 from (0'0,0'0] MAX to 396'7998 2023-09-23T18:11:31.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: 8.5 starting backfill to osd.0 from (0'0,0'0] MAX to 396'8066 2023-09-23T18:11:31.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: 8.e continuing backfill to osd.1 from (372'3597,396'8094] 8:7b7c7e01:::benchmark_data_smithi055_346788_object57885:head to 396'8094 2023-09-23T18:11:31.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: 8.e starting backfill to osd.3 from (0'0,0'0] MAX to 396'8094 2023-09-23T18:11:31.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: 8.f starting backfill to osd.6 from (0'0,0'0] MAX to 396'8032 2023-09-23T18:11:31.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:31 smithi055 ceph-mon[184174]: pgmap v985: 119 pgs: 5 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 22 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 1 active+recovery_wait, 8 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 51 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 3.3 KiB/s rd, 9.2 KiB/s wr, 665 op/s; 49403/277629 objects degraded (17.795%); 39911/277629 objects misplaced (14.376%); 18 MiB/s, 279 objects/s recovering 2023-09-23T18:11:31.820 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:11:31.820 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:11:31.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: osdmap e399: 8 total, 7 up, 6 in 2023-09-23T18:11:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: 8.4 starting backfill to osd.3 from (0'0,0'0] MAX to 396'8032 2023-09-23T18:11:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: 8.b starting backfill to osd.6 from (0'0,0'0] MAX to 396'7998 2023-09-23T18:11:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: 8.5 starting backfill to osd.0 from (0'0,0'0] MAX to 396'8066 2023-09-23T18:11:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: failed to encode map e399 with expected crc 2023-09-23T18:11:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: 8.e continuing backfill to osd.1 from (372'3597,396'8094] 8:7b7c7e01:::benchmark_data_smithi055_346788_object57885:head to 396'8094 2023-09-23T18:11:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: 8.e starting backfill to osd.3 from (0'0,0'0] MAX to 396'8094 2023-09-23T18:11:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: 8.f starting backfill to osd.6 from (0'0,0'0] MAX to 396'8032 2023-09-23T18:11:31.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:31 smithi099 ceph-mon[167076]: pgmap v985: 119 pgs: 5 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 22 active+clean+remapped, 1 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 1 active+recovery_wait, 8 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 51 active+clean; 5.7 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 3.3 KiB/s rd, 9.2 KiB/s wr, 665 op/s; 49403/277629 objects degraded (17.795%); 39911/277629 objects misplaced (14.376%); 18 MiB/s, 279 objects/s recovering 2023-09-23T18:11:31.870 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:32.263 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:32.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:32.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:32.368 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:11:32.371 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:32.534 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:32.609 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:32.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:32.614 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:32.708 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:11:32.715 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:32.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:32 smithi055 ceph-mon[179668]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:32 smithi055 ceph-mon[179668]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:32 smithi055 ceph-mon[179668]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:32 smithi055 ceph-mon[179668]: osdmap e400: 8 total, 7 up, 6 in 2023-09-23T18:11:32.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:32 smithi055 ceph-mon[179668]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.783 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:32 smithi055 ceph-mon[184174]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.783 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:32 smithi055 ceph-mon[184174]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.783 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:32 smithi055 ceph-mon[184174]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.783 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:32 smithi055 ceph-mon[184174]: osdmap e400: 8 total, 7 up, 6 in 2023-09-23T18:11:32.783 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:32 smithi055 ceph-mon[184174]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:32 smithi099 ceph-mon[167076]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:32 smithi099 ceph-mon[167076]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:32 smithi099 ceph-mon[167076]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:32 smithi099 ceph-mon[167076]: osdmap e400: 8 total, 7 up, 6 in 2023-09-23T18:11:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:32 smithi099 ceph-mon[167076]: failed to encode map e400 with expected crc 2023-09-23T18:11:32.861 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:32.928 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:32.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:32.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:33.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:33.073 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:33.123 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:33.130 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:33.131 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:33.218 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:33.232 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:33.316 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:33.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2735790904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:33.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[179668]: failed to encode map e401 with expected crc 2023-09-23T18:11:33.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[179668]: osdmap e401: 8 total, 7 up, 6 in 2023-09-23T18:11:33.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[179668]: failed to encode map e401 with expected crc 2023-09-23T18:11:33.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[179668]: failed to encode map e401 with expected crc 2023-09-23T18:11:33.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[179668]: pgmap v988: 119 pgs: 5 remapped+peering, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+remapped+backfilling, 33 active+clean+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 39 active+clean; 5.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 5.0 KiB/s rd, 14 KiB/s wr, 448 op/s; 48318/273777 objects degraded (17.649%); 50699/273777 objects misplaced (18.518%); 15 MiB/s, 234 objects/s recovering 2023-09-23T18:11:33.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 49403/277629 objects degraded (17.795%), 16 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:33.634 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:11:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:33 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2735790904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:33 smithi099 ceph-mon[167076]: failed to encode map e401 with expected crc 2023-09-23T18:11:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:33 smithi099 ceph-mon[167076]: osdmap e401: 8 total, 7 up, 6 in 2023-09-23T18:11:33.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:33 smithi099 ceph-mon[167076]: failed to encode map e401 with expected crc 2023-09-23T18:11:33.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:33 smithi099 ceph-mon[167076]: failed to encode map e401 with expected crc 2023-09-23T18:11:33.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:33 smithi099 ceph-mon[167076]: pgmap v988: 119 pgs: 5 remapped+peering, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+remapped+backfilling, 33 active+clean+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 39 active+clean; 5.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 5.0 KiB/s rd, 14 KiB/s wr, 448 op/s; 48318/273777 objects degraded (17.649%); 50699/273777 objects misplaced (18.518%); 15 MiB/s, 234 objects/s recovering 2023-09-23T18:11:33.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 49403/277629 objects degraded (17.795%), 16 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:33.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2735790904' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:33.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[184174]: failed to encode map e401 with expected crc 2023-09-23T18:11:33.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[184174]: osdmap e401: 8 total, 7 up, 6 in 2023-09-23T18:11:33.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[184174]: failed to encode map e401 with expected crc 2023-09-23T18:11:33.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[184174]: failed to encode map e401 with expected crc 2023-09-23T18:11:33.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[184174]: pgmap v988: 119 pgs: 5 remapped+peering, 2 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 1 active+remapped+backfilling, 33 active+clean+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 39 active+clean; 5.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 5.0 KiB/s rd, 14 KiB/s wr, 448 op/s; 48318/273777 objects degraded (17.649%); 50699/273777 objects misplaced (18.518%); 15 MiB/s, 234 objects/s recovering 2023-09-23T18:11:33.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 49403/277629 objects degraded (17.795%), 16 pgs degraded, 22 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:34.049 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 3] out_osds: [2, 4] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:11:34.049 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:11:34.050 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T18:11:34.050 INFO:tasks.thrashosds.thrasher:Setting osd 0 primary_affinity to 0.000000 2023-09-23T18:11:34.050 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 0 2023-09-23T18:11:34.345 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:34.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:34.349 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:34.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:34.480 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:34.534 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:34.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:34.541 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:34.642 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:34.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:34 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:11:34.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:34 smithi055 ceph-mon[184174]: failed to encode map e402 with expected crc 2023-09-23T18:11:34.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2735790904' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:11:34.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:34 smithi055 ceph-mon[184174]: osdmap e402: 8 total, 7 up, 6 in 2023-09-23T18:11:34.655 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:34 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:11:34.655 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:34 smithi055 ceph-mon[179668]: failed to encode map e402 with expected crc 2023-09-23T18:11:34.655 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2735790904' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:11:34.655 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:34 smithi055 ceph-mon[179668]: osdmap e402: 8 total, 7 up, 6 in 2023-09-23T18:11:34.690 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:34.751 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:34.757 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:34 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:11:34.757 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:34 smithi099 ceph-mon[167076]: failed to encode map e402 with expected crc 2023-09-23T18:11:34.757 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2735790904' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:11:34.757 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:34 smithi099 ceph-mon[167076]: osdmap e402: 8 total, 7 up, 6 in 2023-09-23T18:11:34.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:34.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:34.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:34.895 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:34.947 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:34.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:34.951 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:35.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:35.111 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:35.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:35.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:35.172 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:35.273 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:35.336 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:35.611 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:35.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:35.615 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:35.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:35.748 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:35.797 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:35.800 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:35.801 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:35.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:35 smithi055 ceph-mon[184174]: pgmap v990: 119 pgs: 5 remapped+peering, 3 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 37 active+clean+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 9.3 KiB/s rd, 25 KiB/s wr, 567 op/s; 47430/271737 objects degraded (17.454%); 55622/271737 objects misplaced (20.469%); 29 MiB/s, 2 keys/s, 466 objects/s recovering 2023-09-23T18:11:35.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:35 smithi055 ceph-mon[179668]: pgmap v990: 119 pgs: 5 remapped+peering, 3 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 37 active+clean+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 9.3 KiB/s rd, 25 KiB/s wr, 567 op/s; 47430/271737 objects degraded (17.454%); 55622/271737 objects misplaced (20.469%); 29 MiB/s, 2 keys/s, 466 objects/s recovering 2023-09-23T18:11:35.901 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:35.963 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:35 smithi099 ceph-mon[167076]: pgmap v990: 119 pgs: 5 remapped+peering, 3 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 37 active+clean+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+remapped, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 9.3 KiB/s rd, 25 KiB/s wr, 567 op/s; 47430/271737 objects degraded (17.454%); 55622/271737 objects misplaced (20.469%); 29 MiB/s, 2 keys/s, 466 objects/s recovering 2023-09-23T18:11:36.104 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:36.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:36.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:36.121 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:36.159 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:36.215 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:36.258 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:36.387 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:36.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:36.394 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:36.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:36.533 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:36.576 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:36.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:36.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:36.682 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:36.729 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:36.768 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:36.772 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:36.773 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:36.873 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:36.940 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:37.131 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:37.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:37.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:37.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:37.265 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:11:37.265 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:11:37.277 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:37.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:37.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:37.324 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:37.425 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:37.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:37.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:37.488 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:11:37.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:37.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:37.489 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:11:37.491 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:37.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:11:37.708 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:11:37.730 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:37.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:37.737 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:37.838 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:37.872 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:37.919 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:37.922 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:37.923 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:38.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:38.051 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:38.093 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:38.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:38.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:38.134 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:11:38.197 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:38.302 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:38.395 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:38.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:38.401 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:38.502 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:38.584 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:38.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:38 smithi055 ceph-mon[179668]: pgmap v991: 119 pgs: 3 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 46 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 506 op/s; 46892/268548 objects degraded (17.461%); 54820/268548 objects misplaced (20.413%); 19 MiB/s, 1 keys/s, 307 objects/s recovering 2023-09-23T18:11:38.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 46892/268548 objects degraded (17.461%), 19 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:38.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:38 smithi055 ceph-mon[184174]: pgmap v991: 119 pgs: 3 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 46 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 506 op/s; 46892/268548 objects degraded (17.461%); 54820/268548 objects misplaced (20.413%); 19 MiB/s, 1 keys/s, 307 objects/s recovering 2023-09-23T18:11:38.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 46892/268548 objects degraded (17.461%), 19 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:38.697 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:38.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:38.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:38.810 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:38 smithi099 ceph-mon[167076]: pgmap v991: 119 pgs: 3 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 46 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.5 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 506 op/s; 46892/268548 objects degraded (17.461%); 54820/268548 objects misplaced (20.413%); 19 MiB/s, 1 keys/s, 307 objects/s recovering 2023-09-23T18:11:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 46892/268548 objects degraded (17.461%), 19 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:38.883 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:11:38.906 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:38.908 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:38.940 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:11:38.982 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:11:39.014 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:11:39.057 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:11:39.077 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:39.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:39.084 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:39.084 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:39.101 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:11:39.152 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:11:39.152 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:11:39.152 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:11:39.153 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:11:39.153 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:11:39.153 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:11:39.153 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:11:39.153 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:11:39.153 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:11:39.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:39.218 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:39.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:39.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:39.267 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:39.368 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:39.410 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:39.469 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:39.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:39.475 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:39.476 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3442437382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:39.476 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2874126662' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T18:11:39.476 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:39 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T18:11:39.498 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.0 primary-affinity to 0 (802) 2023-09-23T18:11:39.577 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:39.614 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:39.647 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:11:39.648 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:11:39.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:39.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:39.721 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:39.741 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3442437382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:39.742 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2874126662' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T18:11:39.742 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:39 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T18:11:39.742 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3442437382' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:39.742 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2874126662' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T18:11:39.742 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:39 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]: dispatch 2023-09-23T18:11:39.822 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:39.864 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:40.130 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:40.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:40.133 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:40.234 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:40.264 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:40.302 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:40.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:40.305 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:40.406 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:40.438 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:40.476 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:40.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:40.484 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:40.585 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:40.616 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:40.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:11:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:11:40] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.43.0" 2023-09-23T18:11:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:40 smithi055 ceph-mon[184174]: pgmap v992: 119 pgs: 3 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 46 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 2.9 KiB/s rd, 7.5 KiB/s wr, 828 op/s; 45391/259380 objects degraded (17.500%); 53219/259380 objects misplaced (20.518%); 17 MiB/s, 1 keys/s, 273 objects/s recovering 2023-09-23T18:11:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:40 smithi055 ceph-mon[184174]: failed to encode map e403 with expected crc 2023-09-23T18:11:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:40 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T18:11:40.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:40 smithi055 ceph-mon[184174]: osdmap e403: 8 total, 7 up, 6 in 2023-09-23T18:11:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:40 smithi055 ceph-mon[179668]: pgmap v992: 119 pgs: 3 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 46 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 2.9 KiB/s rd, 7.5 KiB/s wr, 828 op/s; 45391/259380 objects degraded (17.500%); 53219/259380 objects misplaced (20.518%); 17 MiB/s, 1 keys/s, 273 objects/s recovering 2023-09-23T18:11:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:40 smithi055 ceph-mon[179668]: failed to encode map e403 with expected crc 2023-09-23T18:11:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:40 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T18:11:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:40 smithi055 ceph-mon[179668]: osdmap e403: 8 total, 7 up, 6 in 2023-09-23T18:11:40.705 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:40.708 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:40 smithi099 ceph-mon[167076]: pgmap v992: 119 pgs: 3 active+recovery_wait+degraded, 2 active+recovery_wait+remapped, 46 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.3 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 2.9 KiB/s rd, 7.5 KiB/s wr, 828 op/s; 45391/259380 objects degraded (17.500%); 53219/259380 objects misplaced (20.518%); 17 MiB/s, 1 keys/s, 273 objects/s recovering 2023-09-23T18:11:40.708 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:40 smithi099 ceph-mon[167076]: failed to encode map e403 with expected crc 2023-09-23T18:11:40.708 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:40 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 0.0}]': finished 2023-09-23T18:11:40.708 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:40 smithi099 ceph-mon[167076]: osdmap e403: 8 total, 7 up, 6 in 2023-09-23T18:11:40.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:40.709 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:40.810 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:40.844 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:41.596 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:41.606 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:41.607 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:41.648 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:11:41.708 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:41.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[184174]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[184174]: osdmap e404: 8 total, 7 up, 6 in 2023-09-23T18:11:41.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[184174]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[184174]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[184174]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.776 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[184174]: pgmap v995: 119 pgs: 1 peering, 8 remapped+peering, 3 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 39 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 677 op/s; 43176/257310 objects degraded (16.780%); 44327/257310 objects misplaced (17.227%); 1.3 MiB/s, 22 objects/s recovering 2023-09-23T18:11:41.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[179668]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[179668]: osdmap e404: 8 total, 7 up, 6 in 2023-09-23T18:11:41.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[179668]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[179668]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[179668]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.776 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:41 smithi055 ceph-mon[179668]: pgmap v995: 119 pgs: 1 peering, 8 remapped+peering, 3 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 39 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 677 op/s; 43176/257310 objects degraded (16.780%); 44327/257310 objects misplaced (17.227%); 1.3 MiB/s, 22 objects/s recovering 2023-09-23T18:11:41.783 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:41 smithi099 ceph-mon[167076]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:41 smithi099 ceph-mon[167076]: osdmap e404: 8 total, 7 up, 6 in 2023-09-23T18:11:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:41 smithi099 ceph-mon[167076]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:41 smithi099 ceph-mon[167076]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:41 smithi099 ceph-mon[167076]: failed to encode map e404 with expected crc 2023-09-23T18:11:41.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:41 smithi099 ceph-mon[167076]: pgmap v995: 119 pgs: 1 peering, 8 remapped+peering, 3 active+recovery_wait+degraded, 1 active+recovery_wait+remapped, 39 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 677 op/s; 43176/257310 objects degraded (16.780%); 44327/257310 objects misplaced (17.227%); 1.3 MiB/s, 22 objects/s recovering 2023-09-23T18:11:42.573 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:42.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:42.580 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:42.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:42.738 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:42.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:42.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:42.813 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:42.813 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:42 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:11:42.813 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:42 smithi099 ceph-mon[167076]: failed to encode map e405 with expected crc 2023-09-23T18:11:42.813 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:42 smithi099 ceph-mon[167076]: failed to encode map e405 with expected crc 2023-09-23T18:11:42.813 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:42 smithi099 ceph-mon[167076]: osdmap e405: 8 total, 7 up, 6 in 2023-09-23T18:11:42.813 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:42 smithi099 ceph-mon[167076]: failed to encode map e405 with expected crc 2023-09-23T18:11:42.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:42 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:11:42.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:42 smithi055 ceph-mon[184174]: failed to encode map e405 with expected crc 2023-09-23T18:11:42.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:42 smithi055 ceph-mon[184174]: failed to encode map e405 with expected crc 2023-09-23T18:11:42.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:42 smithi055 ceph-mon[184174]: osdmap e405: 8 total, 7 up, 6 in 2023-09-23T18:11:42.859 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:42 smithi055 ceph-mon[184174]: failed to encode map e405 with expected crc 2023-09-23T18:11:42.860 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:42 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:11:42.860 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:42 smithi055 ceph-mon[179668]: failed to encode map e405 with expected crc 2023-09-23T18:11:42.860 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:42 smithi055 ceph-mon[179668]: failed to encode map e405 with expected crc 2023-09-23T18:11:42.860 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:42 smithi055 ceph-mon[179668]: osdmap e405: 8 total, 7 up, 6 in 2023-09-23T18:11:42.860 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:42 smithi055 ceph-mon[179668]: failed to encode map e405 with expected crc 2023-09-23T18:11:42.913 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:11:42.914 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:11:42.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:11:42.914 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:11:42.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.914 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:84427 7.13 7:cb0230ba:::benchmark_data_smithi055_311528_object13464:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e329)", 2023-09-23T18:11:42.914 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:07:55.875336+0000", 2023-09-23T18:11:42.914 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 227.03376515900001, 2023-09-23T18:11:42.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.079786512000000004, 2023-09-23T18:11:42.914 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 84427 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875336+0000", 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875336+0000", 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875335+0000", 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875338+0000", 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2179999999999999e-06 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875344+0000", 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8830000000000003e-06 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.875349+0000", 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1229999999999999e-06 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.876247+0000", 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00089710500000000004 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.876381+0000", 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000134797 2023-09-23T18:11:42.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:11:42.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.876487+0000", 2023-09-23T18:11:42.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000105522 2023-09-23T18:11:42.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:42.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.879537+0000", 2023-09-23T18:11:42.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030497250000000001 2023-09-23T18:11:42.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.879732+0000", 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000195477 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.955002+0000", 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.075269427 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.955061+0000", 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.94e-05 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:55.955122+0000", 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1279999999999996e-05 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:90637 7.e 7.1c1cc12e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:09.733301+0000", 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 213.17579961300001, 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75068017799999998, 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 90637 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.922 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733301+0000", 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733301+0000", 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733299+0000", 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733310+0000", 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0934000000000001e-05 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733335+0000", 2023-09-23T18:11:42.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4669999999999999e-05 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733347+0000", 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1987999999999999e-05 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733414+0000", 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7006000000000002e-05 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.733425+0000", 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1722999999999999e-05 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:10.483970+0000", 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.75054469899999998 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:10.483981+0000", 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1402e-05 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:90667 7.e 7.a9db13be (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e331)", 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:09.836569+0000", 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 213.07253154899999, 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.64744713499999995, 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:11:42.927 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 90667 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836569+0000", 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836569+0000", 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836568+0000", 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836575+0000", 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7150000000000001e-06 2023-09-23T18:11:42.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836589+0000", 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4613999999999999e-05 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836598+0000", 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4859999999999997e-06 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836663+0000", 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5655999999999996e-05 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:09.836679+0000", 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5449000000000001e-05 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:10.484006+0000", 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.64732701599999998 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:10.484016+0000", 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0664e-05 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:108085 7.8 e340/331 7:101c9370:::benchmark_data_smithi055_311528_object37122:head v 340'5490, mlcod=339'5488)", 2023-09-23T18:11:42.932 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:39.569707+0000", 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 183.33939430500001, 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14917005999999999, 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569707+0000", 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569707+0000", 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569704+0000", 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569722+0000", 2023-09-23T18:11:42.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.747e-05 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569744+0000", 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1849e-05 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.569756+0000", 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.185e-05 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.617599+0000", 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047843760999999999 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.617637+0000", 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7209999999999998e-05 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.718825+0000", 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.101188404 2023-09-23T18:11:42.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.718877+0000", 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1604000000000003e-05 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45557.0:108088 7.13 e340/331 7:c8e7817b:::benchmark_data_smithi055_311528_object37125:head v 340'5502, mlcod=339'5500)", 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:39.646388+0000", 2023-09-23T18:11:42.937 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 183.262713244, 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10060771, 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646388+0000", 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646388+0000", 2023-09-23T18:11:42.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646381+0000", 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999933 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646401+0000", 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0454000000000001e-05 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646421+0000", 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9341999999999999e-05 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.646435+0000", 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.384e-05 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.742550+0000", 2023-09-23T18:11:42.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.096115361999999996 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.742574+0000", 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3747000000000002e-05 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.746950+0000", 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0043766830000000001 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.746995+0000", 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4990999999999997e-05 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:103986 8.6 8:68622a95:::benchmark_data_smithi055_346788_object103985:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:54.339463+0000", 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 48.569637383, 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.068684608999999994, 2023-09-23T18:11:42.942 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 103986 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.339463+0000", 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.339463+0000", 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.339462+0000", 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.339490+0000", 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8693000000000001e-05 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.339503+0000", 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3213e-05 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.339517+0000", 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.31e-05 2023-09-23T18:11:42.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.350797+0000", 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011280930999999999 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351057+0000", 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00025955500000000002 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,6,7", 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351316+0000", 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00025920799999999998 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.355336+0000", 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040199290000000002 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.358596+0000", 2023-09-23T18:11:42.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0032594189999999999 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.359563+0000", 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00096705300000000005 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.405578+0000", 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.046014942000000003 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.405634+0000", 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6560000000000001e-05 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.408148+0000", 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025139429999999998 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:110785 8.6 8:69ea1899:::benchmark_data_smithi055_346788_object193:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:01.282352+0000", 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 41.626748606, 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.069086563000000004, 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 110785 2023-09-23T18:11:42.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.282352+0000", 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.282352+0000", 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.282351+0000", 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.282356+0000", 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5480000000000001e-06 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.282365+0000", 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0769999999999997e-06 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.282371+0000", 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0800000000000002e-06 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.289334+0000", 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0069627719999999999 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.289494+0000", 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015968499999999999 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,6,7", 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.289610+0000", 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011573700000000001 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.298207+0000", 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0085971699999999995 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.299976+0000", 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0017692059999999999 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.309015+0000", 2023-09-23T18:11:42.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0090394210000000006 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.309154+0000", 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013854500000000001 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.309218+0000", 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4055999999999998e-05 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:01.351439+0000", 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.042220884 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:111976 8.6 8:6c5dd4bb:::benchmark_data_smithi055_346788_object1384:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:02.534223+0000", 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 40.374878281999997, 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.069634276999999994, 2023-09-23T18:11:42.957 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 111976 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.534223+0000", 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.534223+0000", 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.534222+0000", 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.534226+0000", 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8089999999999999e-06 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.534233+0000", 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5179999999999998e-06 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.534239+0000", 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5130000000000004e-06 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.568028+0000", 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.033789701999999998 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.568240+0000", 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000211774 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,6,7", 2023-09-23T18:11:42.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.568386+0000", 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014574299999999999 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.579215+0000", 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01082935 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.579291+0000", 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5600999999999997e-05 2023-09-23T18:11:42.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.587442+0000", 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0081514819999999998 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.601526+0000", 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014083101000000001 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.601580+0000", 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4954999999999998e-05 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.603857+0000", 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002276372 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:111977 8.6 8:66f19a3e:::benchmark_data_smithi055_346788_object1385:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:02.537429+0000", 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 40.371671630000002, 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.068677530000000001, 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 111977 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.965 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.537429+0000", 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.537429+0000", 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.537428+0000", 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.537435+0000", 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9659999999999999e-06 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.537447+0000", 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1939e-05 2023-09-23T18:11:42.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.537454+0000", 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7160000000000003e-06 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.572447+0000", 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.034992585999999999 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.572583+0000", 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013564999999999999 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,6,7", 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.572712+0000", 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012968100000000001 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.579332+0000", 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0066197060000000004 2023-09-23T18:11:42.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.587498+0000", 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0081657880000000002 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.589846+0000", 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023480350000000001 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.603937+0000", 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014091027000000001 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.603995+0000", 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7676000000000001e-05 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:02.606107+0000", 2023-09-23T18:11:42.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021122509999999999 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:112985 8.6 8:6817c5c1:::benchmark_data_smithi055_346788_object2393:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e386)", 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:03.659493+0000", 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 39.249608000999999, 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.087886767000000005, 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.972 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 112985 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.659493+0000", 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.659493+0000", 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.659492+0000", 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.659496+0000", 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0750000000000002e-06 2023-09-23T18:11:42.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.659501+0000", 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5729999999999998e-06 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.659505+0000", 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1130000000000003e-06 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.681207+0000", 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021701654000000001 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.681343+0000", 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000136181 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,6,7", 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.681479+0000", 2023-09-23T18:11:42.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000135523 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.693889+0000", 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012410711 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.694924+0000", 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0010346050000000001 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.706864+0000", 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011940493999999999 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.741638+0000", 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.034773169 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.741692+0000", 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4598000000000001e-05 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:03.747380+0000", 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0056874350000000002 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.979 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:136078 8.e e402/398 8:7d936bc7:::benchmark_data_smithi055_346788_object25486:head v 402'8509, mlcod=402'8506)", 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:39.490956+0000", 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 3.4181447120000001, 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.133625839, 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.490956+0000", 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.490956+0000", 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.490955+0000", 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.490964+0000", 2023-09-23T18:11:42.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0520000000000007e-06 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.490982+0000", 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7995999999999998e-05 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.490992+0000", 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0017000000000001e-05 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.501932+0000", 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010939717999999999 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.501955+0000", 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3110000000000001e-05 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.624530+0000", 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.122574736 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.624582+0000", 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2231000000000002e-05 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:136086 8.e e403/398 8:71053626:::benchmark_data_smithi055_346788_object25494:head v 403'8510, mlcod=402'8507)", 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:39.512083+0000", 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 3.3970181410000002, 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11252994700000001, 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.512083+0000", 2023-09-23T18:11:42.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.512083+0000", 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.512082+0000", 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.512089+0000", 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9410000000000001e-06 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.512099+0000", 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9750000000000002e-06 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.512108+0000", 2023-09-23T18:11:42.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6989999999999992e-06 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.519261+0000", 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0071528390000000002 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.519282+0000", 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1659999999999999e-05 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.624587+0000", 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10530468699999999 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.624613+0000", 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5799000000000002e-05 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:136095 8.e e403/398 8:70352387:::benchmark_data_smithi055_346788_object25503:head v 403'8511, mlcod=402'8507)", 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:39.526474+0000", 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 3.382627066, 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.098169373000000004, 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.526474+0000", 2023-09-23T18:11:42.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.526474+0000", 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.526473+0000", 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.526482+0000", 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0000000000000002e-06 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.526501+0000", 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9412e-05 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.526509+0000", 2023-09-23T18:11:42.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.96e-06 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.526555+0000", 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5595e-05 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.526584+0000", 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9272e-05 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.624616+0000", 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.098031775000000002 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.624643+0000", 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7402e-05 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:136096 8.e e403/398 8:7149357d:::benchmark_data_smithi055_346788_object25504:head v 403'8512, mlcod=402'8507)", 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:39.532519+0000", 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 3.3765822280000002, 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.092151184999999997, 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.532519+0000", 2023-09-23T18:11:42.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.532519+0000", 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.532517+0000", 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.532528+0000", 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.061e-05 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.532548+0000", 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9911e-05 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.532559+0000", 2023-09-23T18:11:42.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1024e-05 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.544630+0000", 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012071442999999999 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.544653+0000", 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.285e-05 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.624646+0000", 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.079992630999999995 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.624670+0000", 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3913999999999998e-05 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:42.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:136102 8.e e403/398 8:7bf222a9:::benchmark_data_smithi055_346788_object25510:head v 403'8513, mlcod=402'8507)", 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:39.541269+0000", 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 3.3678319590000001, 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.083427395000000001, 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.541269+0000", 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.541269+0000", 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.541269+0000", 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.541273+0000", 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0890000000000002e-06 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.541278+0000", 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3369999999999999e-06 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.541282+0000", 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1509999999999997e-06 2023-09-23T18:11:43.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.546895+0000", 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0056123029999999999 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.546917+0000", 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1987000000000001e-05 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.624673+0000", 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.077756562000000001 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.624696+0000", 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3210999999999999e-05 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.004 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:136113 8.6 8.dd467196 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:39.558220+0000", 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 3.3508812479999999, 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95001246100000003, 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 136113 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:43.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.558220+0000", 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.558220+0000", 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:43.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.558218+0000", 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.558228+0000", 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9899999999999992e-06 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.558248+0000", 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0258000000000002e-05 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.558258+0000", 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4690000000000003e-06 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.558328+0000", 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9929000000000001e-05 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:43.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.558343+0000", 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5831000000000001e-05 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:40.508217+0000", 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.94987356899999997 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:40.508232+0000", 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5148e-05 2023-09-23T18:11:43.009 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:136150 8.6 8.2df54d46 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:39.661752+0000", 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 3.2473483729999999, 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.84651484600000004, 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:43.010 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 136150 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.661752+0000", 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:43.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.661752+0000", 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.661751+0000", 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.661758+0000", 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3799999999999996e-06 2023-09-23T18:11:43.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.661776+0000", 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.751e-05 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.661785+0000", 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5419999999999994e-06 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.661851+0000", 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5071000000000001e-05 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.661863+0000", 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2711000000000001e-05 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:40.508257+0000", 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.84639399100000001 2023-09-23T18:11:43.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:40.508267+0000", 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0111e-05 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:136169 8.6 8.78be65f6 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:39.698092+0000", 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 3.2110090740000001, 2023-09-23T18:11:43.015 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.81020963000000001, 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 136169 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.698092+0000", 2023-09-23T18:11:43.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.698092+0000", 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.698090+0000", 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.698099+0000", 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5929999999999995e-06 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.698118+0000", 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8995999999999999e-05 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.698127+0000", 2023-09-23T18:11:43.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2010000000000005e-06 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.698214+0000", 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6588000000000005e-05 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.698226+0000", 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.255e-05 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:40.508291+0000", 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.81006439500000005 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:40.508301+0000", 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0586e-05 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:136229 8.6 8.c0d3bc66 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:39.824100+0000", 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 3.08500043, 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.68423271600000002, 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 136229 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.021 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.824100+0000", 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.824100+0000", 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.824099+0000", 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.824108+0000", 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8380000000000008e-06 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.824127+0000", 2023-09-23T18:11:43.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9870000000000001e-05 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.824136+0000", 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2530000000000004e-06 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.824209+0000", 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2941999999999994e-05 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.824219+0000", 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0589e-05 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:40.508323+0000", 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.68410335899999997 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:43.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:40.508333+0000", 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0540000000000001e-05 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:136253 8.6 8.b2e4ee96 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:39.894308+0000", 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 3.0147931720000001, 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.61406659399999997, 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:11:43.026 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 136253 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.894308+0000", 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.894308+0000", 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.894306+0000", 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.894315+0000", 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7079999999999992e-06 2023-09-23T18:11:43.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.894336+0000", 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1178e-05 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.894352+0000", 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5427999999999999e-05 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.894417+0000", 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5339000000000006e-05 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:39.894433+0000", 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5840000000000001e-05 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:40.508364+0000", 2023-09-23T18:11:43.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.61393093200000004 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:40.508374+0000", 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0253e-05 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:11:43.031 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:11:43.032 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:43.164 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:43.418 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:43.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:43.422 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:43.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:11:43.523 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:43.610 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:43.611 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:43.689 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:43.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:43.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:43.791 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:43.871 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:43.949 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:43.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:43.958 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:44.058 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:44.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[179668]: pgmap v997: 119 pgs: 1 peering, 6 remapped+peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 41 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 42613/253821 objects degraded (16.789%); 43774/253821 objects misplaced (17.246%); 2.9 MiB/s, 65 objects/s recovering 2023-09-23T18:11:44.134 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:44.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:44.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 43176/257310 objects degraded (16.780%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:44.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2533472025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:44.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:44.135 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[184174]: pgmap v997: 119 pgs: 1 peering, 6 remapped+peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 41 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 42613/253821 objects degraded (16.789%); 43774/253821 objects misplaced (17.246%); 2.9 MiB/s, 65 objects/s recovering 2023-09-23T18:11:44.135 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:44.135 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:44.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 43176/257310 objects degraded (16.780%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:44.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2533472025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:44.136 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:43 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:44.136 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:44.197 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:44.201 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:44.202 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:44.292 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:11:44.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:44.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:43 smithi099 ceph-mon[167076]: pgmap v997: 119 pgs: 1 peering, 6 remapped+peering, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 41 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 34 active+clean; 5.2 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 42613/253821 objects degraded (16.789%); 43774/253821 objects misplaced (17.246%); 2.9 MiB/s, 65 objects/s recovering 2023-09-23T18:11:44.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:43 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:44.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:43 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:44.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 43176/257310 objects degraded (16.780%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:44.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2533472025' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:44.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:43 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:11:44.398 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:44.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:44.467 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:44.468 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:44.554 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:11:44.569 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:44.625 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:45.032 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 3] out_osds: [2, 4] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:11:45.032 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:11:45.033 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_3 2023-09-23T18:11:45.033 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:11:45.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:44 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:45.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:44 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:11:45.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:44 smithi099 ceph-mon[167076]: osdmap e406: 8 total, 7 up, 6 in 2023-09-23T18:11:45.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:44 smithi099 ceph-mon[167076]: failed to encode map e406 with expected crc 2023-09-23T18:11:45.077 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:44 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:45.077 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:44 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:11:45.077 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:44 smithi055 ceph-mon[184174]: osdmap e406: 8 total, 7 up, 6 in 2023-09-23T18:11:45.078 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:44 smithi055 ceph-mon[184174]: failed to encode map e406 with expected crc 2023-09-23T18:11:45.078 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:44 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:45.078 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:44 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:11:45.078 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:44 smithi055 ceph-mon[179668]: osdmap e406: 8 total, 7 up, 6 in 2023-09-23T18:11:45.078 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:44 smithi055 ceph-mon[179668]: failed to encode map e406 with expected crc 2023-09-23T18:11:45.253 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:45.263 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:45.263 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:45.365 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:45.414 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:45.566 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:45.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:45.573 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:45.604 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:11:45.656 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:11:45.673 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:45.693 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:11:45.730 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:11:45.746 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:45.770 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:11:45.813 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:45.815 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:11:45.815 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:11:45.816 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:11:45.823 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:45.823 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:45.880 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:11:45.880 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:11:45.880 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:11:45.880 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:11:45.881 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:11:45.881 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:11:45.881 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:11:45.881 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:11:45.881 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:11:45.924 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:45.993 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:45.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:45 smithi055 ceph-mon[179668]: failed to encode map e406 with expected crc 2023-09-23T18:11:45.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:45 smithi055 ceph-mon[179668]: pgmap v999: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 44 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 35 active+clean; 5.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 410 op/s; 40885/252648 objects degraded (16.183%); 43466/252648 objects misplaced (17.204%); 18 MiB/s, 315 objects/s recovering 2023-09-23T18:11:45.995 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3307382679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:46.007 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:46.063 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:46.070 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:46.071 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:46.172 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:46.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:45 smithi099 ceph-mon[167076]: failed to encode map e406 with expected crc 2023-09-23T18:11:46.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:45 smithi099 ceph-mon[167076]: pgmap v999: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 44 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 35 active+clean; 5.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 410 op/s; 40885/252648 objects degraded (16.183%); 43466/252648 objects misplaced (17.204%); 18 MiB/s, 315 objects/s recovering 2023-09-23T18:11:46.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3307382679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:46.231 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:11:46.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:45 smithi055 ceph-mon[184174]: failed to encode map e406 with expected crc 2023-09-23T18:11:46.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:45 smithi055 ceph-mon[184174]: pgmap v999: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 3 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 44 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 4 active+recovery_wait+degraded+remapped, 35 active+clean; 5.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 410 op/s; 40885/252648 objects degraded (16.183%); 43466/252648 objects misplaced (17.204%); 18 MiB/s, 315 objects/s recovering 2023-09-23T18:11:46.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3307382679' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:46.255 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:46.377 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:46.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:46.384 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:46.485 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:46.496 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:11:46.496 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:11:46.565 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:46.630 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:46.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:46.645 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:46.745 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:46.800 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:47.005 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:47.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:47.015 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:47.017 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:11:47.068 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:47 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T18:11:47.068 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:47 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T18:11:47.115 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:47.214 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:47.311 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:47 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T18:11:47.349 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:11:47.467 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:47.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:47.471 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:47.572 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:47.653 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:47.838 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:47.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:47.847 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:47.947 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:48.010 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:48.092 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:48.096 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:48.096 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:48.143 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:48.190 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:48 smithi099 ceph-mon[167076]: pgmap v1000: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 5.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 45139/249279 objects degraded (18.108%); 48141/249279 objects misplaced (19.312%); 18 MiB/s, 307 objects/s recovering 2023-09-23T18:11:48.190 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:48 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:48.190 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:48 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:48.197 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:48.260 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:48.296 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:11:48.304 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:11:48.340 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:48 smithi055 ceph-mon[184174]: pgmap v1000: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 5.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 45139/249279 objects degraded (18.108%); 48141/249279 objects misplaced (19.312%); 18 MiB/s, 307 objects/s recovering 2023-09-23T18:11:48.340 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:48 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:48.340 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:48 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:48.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:48 smithi055 ceph-mon[179668]: pgmap v1000: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 5.1 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 522 op/s; 45139/249279 objects degraded (18.108%); 48141/249279 objects misplaced (19.312%); 18 MiB/s, 307 objects/s recovering 2023-09-23T18:11:48.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:48 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:48.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:48 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:48.450 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:11:48.451 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:11:48.497 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:11:48.853 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:11:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:49 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 45139/249279 objects degraded (18.108%), 19 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:49 smithi099 ceph-mon[167076]: from='client.55984 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:11:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:49 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:49 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:11:49.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:49 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:49 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 45139/249279 objects degraded (18.108%), 19 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:49 smithi055 ceph-mon[184174]: from='client.55984 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:11:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:49 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:49 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:11:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:49 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:49 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 45139/249279 objects degraded (18.108%), 19 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:49 smithi055 ceph-mon[179668]: from='client.55984 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:11:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:49 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:11:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:49 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:11:49.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:49 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:49.434 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:49.440 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:49.440 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:49.541 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:49.599 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:49.647 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:11:50.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:50 smithi099 ceph-mon[167076]: pgmap v1001: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 42577/241128 objects degraded (17.657%); 46553/241128 objects misplaced (19.306%); 26 MiB/s, 424 objects/s recovering 2023-09-23T18:11:50.359 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:50 smithi055 ceph-mon[179668]: pgmap v1001: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 42577/241128 objects degraded (17.657%); 46553/241128 objects misplaced (19.306%); 26 MiB/s, 424 objects/s recovering 2023-09-23T18:11:50.360 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:50 smithi055 ceph-mon[184174]: pgmap v1001: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 673 op/s; 42577/241128 objects degraded (17.657%); 46553/241128 objects misplaced (19.306%); 26 MiB/s, 424 objects/s recovering 2023-09-23T18:11:50.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:11:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:11:50] "GET /metrics HTTP/1.1" 200 36670 "" "Prometheus/2.43.0" 2023-09-23T18:11:51.247 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:11:51.247 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:11:51.375 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1000,"stamp":"2023-09-23T18:11:46.932817+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5448330591,"num_objects":83093,"num_object_clones":0,"num_object_copies":249279,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":45139,"num_objects_misplaced":48141,"num_objects_unfound":0,"num_objects_dirty":83093,"num_whiteouts":0,"num_read":6053,"num_read_kb":12334,"num_write":139070,"num_write_kb":7090916,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13580,"num_bytes_recovered":878351300,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74835,"ondisk_log_size":74835,"up":280,"acting":332,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":342,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":14320724,"kb_used_data":11645476,"kb_used_omap":38,"kb_used_meta":2675033,"kb_avail":548150188,"statfs":{"total":575970213888,"available":561305792512,"internally_reserved":0,"allocated":11924967424,"data_stored":11915221421,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":39915,"internal_metadata":2739233813},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,2,7,6,4],"upper_bound":32},"perf_stat":{"commit_latency_ms":5,"apply_latency_ms":5,"commit_latency_ns":5000000,"apply_latency_ns":5000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-220406513,"num_objects":-3367,"num_object_clones":0,"num_object_copies":-10101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-252,"num_objects_misplaced":-5078,"num_objects_unfound":0,"num_objects_dirty":-3367,"num_whiteouts":0,"num_read":-7,"num_read_kb":-14,"num_write":3364,"num_write_kb":-37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1978,"num_bytes_recovered":121225688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.433910"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":667,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.512784+0000","last_change":"2023-09-23T18:10:54.164872+0000","last_active":"2023-09-23T18:11:40.512784+0000","last_peered":"2023-09-23T18:11:40.512784+0000","last_clean":"2023-09-23T18:11:40.512784+0000","last_became_active":"2023-09-23T18:10:54.164329+0000","last_became_peered":"2023-09-23T18:10:54.164329+0000","last_unstale":"2023-09-23T18:11:40.512784+0000","last_undegraded":"2023-09-23T18:11:40.512784+0000","last_fullsized":"2023-09-23T18:11:40.512784+0000","mapping_epoch":385,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":386,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.712627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.712627+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":999913,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,2],"avail_no_missing":["7","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":107,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.510124+0000","last_change":"2023-09-23T18:11:30.560658+0000","last_active":"2023-09-23T18:11:40.510124+0000","last_peered":"2023-09-23T18:11:40.510124+0000","last_clean":"2023-09-23T18:11:40.510124+0000","last_became_active":"2023-09-23T18:11:30.559611+0000","last_became_peered":"2023-09-23T18:11:30.559611+0000","last_unstale":"2023-09-23T18:11:40.510124+0000","last_undegraded":"2023-09-23T18:11:40.510124+0000","last_fullsized":"2023-09-23T18:11:40.510124+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:30:09.204535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,2],"avail_no_missing":["1","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":292,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.512547+0000","last_change":"2023-09-23T18:11:30.582310+0000","last_active":"2023-09-23T18:11:40.512547+0000","last_peered":"2023-09-23T18:11:40.512547+0000","last_clean":"2023-09-23T18:11:40.512547+0000","last_became_active":"2023-09-23T18:11:30.581322+0000","last_became_peered":"2023-09-23T18:11:30.581322+0000","last_unstale":"2023-09-23T18:11:40.512547+0000","last_undegraded":"2023-09-23T18:11:40.512547+0000","last_fullsized":"2023-09-23T18:11:40.512547+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:13:50.112572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,4],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":645,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.512986+0000","last_change":"2023-09-23T18:11:32.607615+0000","last_active":"2023-09-23T18:11:40.512986+0000","last_peered":"2023-09-23T18:11:40.512986+0000","last_clean":"2023-09-23T18:11:40.512986+0000","last_became_active":"2023-09-23T18:11:32.607124+0000","last_became_peered":"2023-09-23T18:11:32.607124+0000","last_unstale":"2023-09-23T18:11:40.512986+0000","last_undegraded":"2023-09-23T18:11:40.512986+0000","last_fullsized":"2023-09-23T18:11:40.512986+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:11.990299+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,4],"avail_no_missing":["7","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":149,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.514763+0000","last_change":"2023-09-23T18:11:29.585733+0000","last_active":"2023-09-23T18:11:40.514763+0000","last_peered":"2023-09-23T18:11:40.514763+0000","last_clean":"2023-09-23T18:11:40.514763+0000","last_became_active":"2023-09-23T18:11:29.585351+0000","last_became_peered":"2023-09-23T18:11:29.585351+0000","last_unstale":"2023-09-23T18:11:40.514763+0000","last_undegraded":"2023-09-23T18:11:40.514763+0000","last_fullsized":"2023-09-23T18:11:40.514763+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:23.309026+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":421,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.515448+0000","last_change":"2023-09-23T18:09:49.264642+0000","last_active":"2023-09-23T18:11:40.515448+0000","last_peered":"2023-09-23T18:11:40.515448+0000","last_clean":"2023-09-23T18:09:40.155891+0000","last_became_active":"2023-09-23T18:09:49.264642+0000","last_became_peered":"2023-09-23T18:09:49.264642+0000","last_unstale":"2023-09-23T18:11:40.515448+0000","last_undegraded":"2023-09-23T18:11:40.515448+0000","last_fullsized":"2023-09-23T18:09:49.251516+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:29.396482+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":627,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.510300+0000","last_change":"2023-09-23T18:11:30.561163+0000","last_active":"2023-09-23T18:11:40.510300+0000","last_peered":"2023-09-23T18:11:40.510300+0000","last_clean":"2023-09-23T18:11:40.510300+0000","last_became_active":"2023-09-23T18:11:30.560679+0000","last_became_peered":"2023-09-23T18:11:30.560679+0000","last_unstale":"2023-09-23T18:11:40.510300+0000","last_undegraded":"2023-09-23T18:11:40.510300+0000","last_fullsized":"2023-09-23T18:11:40.510300+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:16:42.945138+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,4],"avail_no_missing":["1","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":295,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.509360+0000","last_change":"2023-09-23T18:10:52.851415+0000","last_active":"2023-09-23T18:11:40.509360+0000","last_peered":"2023-09-23T18:11:40.509360+0000","last_clean":"2023-09-23T18:11:40.509360+0000","last_became_active":"2023-09-23T18:10:52.851001+0000","last_became_peered":"2023-09-23T18:10:52.851001+0000","last_unstale":"2023-09-23T18:11:40.509360+0000","last_undegraded":"2023-09-23T18:11:40.509360+0000","last_fullsized":"2023-09-23T18:11:40.509360+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:40.705152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,2],"avail_no_missing":["1","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":363,"reported_epoch":405,"state":"active+undersized","last_fresh":"2023-09-23T18:11:41.522730+0000","last_change":"2023-09-23T18:09:49.249873+0000","last_active":"2023-09-23T18:11:41.522730+0000","last_peered":"2023-09-23T18:11:41.522730+0000","last_clean":"2023-09-23T18:09:40.149572+0000","last_became_active":"2023-09-23T18:09:49.249873+0000","last_became_peered":"2023-09-23T18:09:49.249873+0000","last_unstale":"2023-09-23T18:11:41.522730+0000","last_undegraded":"2023-09-23T18:11:41.522730+0000","last_fullsized":"2023-09-23T18:09:49.240762+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:52.086856+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":792,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.515377+0000","last_change":"2023-09-23T18:09:49.267065+0000","last_active":"2023-09-23T18:11:40.515377+0000","last_peered":"2023-09-23T18:11:40.515377+0000","last_clean":"2023-09-23T18:09:40.155011+0000","last_became_active":"2023-09-23T18:09:49.267065+0000","last_became_peered":"2023-09-23T18:09:49.267065+0000","last_unstale":"2023-09-23T18:11:40.515377+0000","last_undegraded":"2023-09-23T18:11:40.515377+0000","last_fullsized":"2023-09-23T18:09:49.257376+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:52.249937+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":315,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.532108+0000","last_change":"2023-09-23T18:11:41.531878+0000","last_active":"2023-09-23T18:11:41.532108+0000","last_peered":"2023-09-23T18:11:41.532108+0000","last_clean":"2023-09-23T18:11:41.532108+0000","last_became_active":"2023-09-23T18:11:41.531545+0000","last_became_peered":"2023-09-23T18:11:41.531545+0000","last_unstale":"2023-09-23T18:11:41.532108+0000","last_undegraded":"2023-09-23T18:11:41.532108+0000","last_fullsized":"2023-09-23T18:11:41.532108+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:31.697705+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,1],"avail_no_missing":["6","0","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":22,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.529492+0000","last_change":"2023-09-23T18:11:18.177272+0000","last_active":"2023-09-23T18:11:40.529492+0000","last_peered":"2023-09-23T18:11:40.529492+0000","last_clean":"2023-09-23T18:11:40.529492+0000","last_became_active":"2023-09-23T18:11:18.176918+0000","last_became_peered":"2023-09-23T18:11:18.176918+0000","last_unstale":"2023-09-23T18:11:40.529492+0000","last_undegraded":"2023-09-23T18:11:40.529492+0000","last_fullsized":"2023-09-23T18:11:40.529492+0000","mapping_epoch":393,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:41.495118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":733,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.515066+0000","last_change":"2023-09-23T18:11:32.618276+0000","last_active":"2023-09-23T18:11:40.515066+0000","last_peered":"2023-09-23T18:11:40.515066+0000","last_clean":"2023-09-23T18:11:40.515066+0000","last_became_active":"2023-09-23T18:11:32.617899+0000","last_became_peered":"2023-09-23T18:11:32.617899+0000","last_unstale":"2023-09-23T18:11:40.515066+0000","last_undegraded":"2023-09-23T18:11:40.515066+0000","last_fullsized":"2023-09-23T18:11:40.515066+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:53.104697+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":316,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.515596+0000","last_change":"2023-09-23T18:09:49.264696+0000","last_active":"2023-09-23T18:11:40.515596+0000","last_peered":"2023-09-23T18:11:40.515596+0000","last_clean":"2023-09-23T18:05:59.285835+0000","last_became_active":"2023-09-23T18:09:49.264696+0000","last_became_peered":"2023-09-23T18:09:49.264696+0000","last_unstale":"2023-09-23T18:11:40.515596+0000","last_undegraded":"2023-09-23T18:11:40.515596+0000","last_fullsized":"2023-09-23T18:09:49.251195+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:24:32.680036+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":215,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.519933+0000","last_change":"2023-09-23T18:11:19.167247+0000","last_active":"2023-09-23T18:11:41.519933+0000","last_peered":"2023-09-23T18:11:41.519933+0000","last_clean":"2023-09-23T18:11:41.519933+0000","last_became_active":"2023-09-23T18:11:19.166959+0000","last_became_peered":"2023-09-23T18:11:19.166959+0000","last_unstale":"2023-09-23T18:11:41.519933+0000","last_undegraded":"2023-09-23T18:11:41.519933+0000","last_fullsized":"2023-09-23T18:11:41.519933+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:07.487191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":461,"reported_epoch":405,"state":"active+undersized","last_fresh":"2023-09-23T18:11:41.522800+0000","last_change":"2023-09-23T18:09:49.254157+0000","last_active":"2023-09-23T18:11:41.522800+0000","last_peered":"2023-09-23T18:11:41.522800+0000","last_clean":"2023-09-23T17:49:06.430604+0000","last_became_active":"2023-09-23T18:09:49.254157+0000","last_became_peered":"2023-09-23T18:09:49.254157+0000","last_unstale":"2023-09-23T18:11:41.522800+0000","last_undegraded":"2023-09-23T18:11:41.522800+0000","last_fullsized":"2023-09-23T18:09:49.240876+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.652623+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.652623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":218,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.515263+0000","last_change":"2023-09-23T18:11:29.592002+0000","last_active":"2023-09-23T18:11:40.515263+0000","last_peered":"2023-09-23T18:11:40.515263+0000","last_clean":"2023-09-23T18:11:40.515263+0000","last_became_active":"2023-09-23T18:11:29.562699+0000","last_became_peered":"2023-09-23T18:11:29.562699+0000","last_unstale":"2023-09-23T18:11:40.515263+0000","last_undegraded":"2023-09-23T18:11:40.515263+0000","last_fullsized":"2023-09-23T18:11:40.515263+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:14:21.977674+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":540,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":142,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.512433+0000","last_change":"2023-09-23T18:10:53.000402+0000","last_active":"2023-09-23T18:11:40.512433+0000","last_peered":"2023-09-23T18:11:40.512433+0000","last_clean":"2023-09-23T18:11:40.512433+0000","last_became_active":"2023-09-23T18:10:52.999952+0000","last_became_peered":"2023-09-23T18:10:52.999952+0000","last_unstale":"2023-09-23T18:11:40.512433+0000","last_undegraded":"2023-09-23T18:11:40.512433+0000","last_fullsized":"2023-09-23T18:11:40.512433+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:31.705403+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,2],"avail_no_missing":["1","2","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":816,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.512677+0000","last_change":"2023-09-23T18:11:30.559311+0000","last_active":"2023-09-23T18:11:40.512677+0000","last_peered":"2023-09-23T18:11:40.512677+0000","last_clean":"2023-09-23T18:11:40.512677+0000","last_became_active":"2023-09-23T18:11:30.557753+0000","last_became_peered":"2023-09-23T18:11:30.557753+0000","last_unstale":"2023-09-23T18:11:40.512677+0000","last_undegraded":"2023-09-23T18:11:40.512677+0000","last_fullsized":"2023-09-23T18:11:40.512677+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:51.581474+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":119,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.515267+0000","last_change":"2023-09-23T18:11:30.573731+0000","last_active":"2023-09-23T18:11:40.515267+0000","last_peered":"2023-09-23T18:11:40.515267+0000","last_clean":"2023-09-23T18:11:40.515267+0000","last_became_active":"2023-09-23T18:11:30.573316+0000","last_became_peered":"2023-09-23T18:11:30.573316+0000","last_unstale":"2023-09-23T18:11:40.515267+0000","last_undegraded":"2023-09-23T18:11:40.515267+0000","last_fullsized":"2023-09-23T18:11:40.515267+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:09.499602+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6,4],"avail_no_missing":["7","4","6"],"object_location_counts":[{"shards":"4,6,7","objects":4}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":210,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.519497+0000","last_change":"2023-09-23T18:11:30.578737+0000","last_active":"2023-09-23T18:11:41.519497+0000","last_peered":"2023-09-23T18:11:41.519497+0000","last_clean":"2023-09-23T18:11:41.519497+0000","last_became_active":"2023-09-23T18:11:30.578179+0000","last_became_peered":"2023-09-23T18:11:30.578179+0000","last_unstale":"2023-09-23T18:11:41.519497+0000","last_undegraded":"2023-09-23T18:11:41.519497+0000","last_fullsized":"2023-09-23T18:11:41.519497+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:44.163742+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,4],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":640,"reported_epoch":404,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:11:40.510623+0000","last_change":"2023-09-23T18:09:49.245471+0000","last_active":"2023-09-23T18:11:40.510623+0000","last_peered":"2023-09-23T18:11:40.510623+0000","last_clean":"2023-09-23T18:02:45.964322+0000","last_became_active":"2023-09-23T18:09:49.245471+0000","last_became_peered":"2023-09-23T18:09:49.245471+0000","last_unstale":"2023-09-23T18:11:40.510623+0000","last_undegraded":"2023-09-23T18:09:49.238416+0000","last_fullsized":"2023-09-23T18:09:49.238297+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":313,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.869576+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":365,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.513278+0000","last_change":"2023-09-23T18:11:29.596588+0000","last_active":"2023-09-23T18:11:40.513278+0000","last_peered":"2023-09-23T18:11:40.513278+0000","last_clean":"2023-09-23T18:11:40.513278+0000","last_became_active":"2023-09-23T18:11:29.562735+0000","last_became_peered":"2023-09-23T18:11:29.562735+0000","last_unstale":"2023-09-23T18:11:40.513278+0000","last_undegraded":"2023-09-23T18:11:40.513278+0000","last_fullsized":"2023-09-23T18:11:40.513278+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:02.755997+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,0],"acting":[7,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":241,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.510512+0000","last_change":"2023-09-23T18:11:19.185574+0000","last_active":"2023-09-23T18:11:40.510512+0000","last_peered":"2023-09-23T18:11:40.510512+0000","last_clean":"2023-09-23T18:11:40.510512+0000","last_became_active":"2023-09-23T18:11:19.185301+0000","last_became_peered":"2023-09-23T18:11:19.185301+0000","last_unstale":"2023-09-23T18:11:40.510512+0000","last_undegraded":"2023-09-23T18:11:40.510512+0000","last_fullsized":"2023-09-23T18:11:40.510512+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:43.081121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":652,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.512746+0000","last_change":"2023-09-23T18:09:49.260631+0000","last_active":"2023-09-23T18:11:40.512746+0000","last_peered":"2023-09-23T18:11:40.512746+0000","last_clean":"2023-09-23T18:05:06.478813+0000","last_became_active":"2023-09-23T18:09:49.260631+0000","last_became_peered":"2023-09-23T18:09:49.260631+0000","last_unstale":"2023-09-23T18:11:40.512746+0000","last_undegraded":"2023-09-23T18:11:40.512746+0000","last_fullsized":"2023-09-23T18:09:49.251470+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":278,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:23.252461+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":424,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.513480+0000","last_change":"2023-09-23T18:10:52.844134+0000","last_active":"2023-09-23T18:11:40.513480+0000","last_peered":"2023-09-23T18:11:40.513480+0000","last_clean":"2023-09-23T18:11:40.513480+0000","last_became_active":"2023-09-23T18:10:52.843850+0000","last_became_peered":"2023-09-23T18:10:52.843850+0000","last_unstale":"2023-09-23T18:11:40.513480+0000","last_undegraded":"2023-09-23T18:11:40.513480+0000","last_fullsized":"2023-09-23T18:11:40.513480+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:35:11.095314+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,2],"avail_no_missing":["7","1","2"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":278,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.515602+0000","last_change":"2023-09-23T18:11:19.167208+0000","last_active":"2023-09-23T18:11:40.515602+0000","last_peered":"2023-09-23T18:11:40.515602+0000","last_clean":"2023-09-23T18:11:40.515602+0000","last_became_active":"2023-09-23T18:11:19.166805+0000","last_became_peered":"2023-09-23T18:11:19.166805+0000","last_unstale":"2023-09-23T18:11:40.515602+0000","last_undegraded":"2023-09-23T18:11:40.515602+0000","last_fullsized":"2023-09-23T18:11:40.515602+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:17:48.150206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,6],"avail_no_missing":["7","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":779,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.513860+0000","last_change":"2023-09-23T18:11:30.575109+0000","last_active":"2023-09-23T18:11:40.513860+0000","last_peered":"2023-09-23T18:11:40.513860+0000","last_clean":"2023-09-23T18:11:40.513860+0000","last_became_active":"2023-09-23T18:11:30.574207+0000","last_became_peered":"2023-09-23T18:11:30.574207+0000","last_unstale":"2023-09-23T18:11:40.513860+0000","last_undegraded":"2023-09-23T18:11:40.513860+0000","last_fullsized":"2023-09-23T18:11:40.513860+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:03:09.570141+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":197,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.520023+0000","last_change":"2023-09-23T18:11:29.576268+0000","last_active":"2023-09-23T18:11:41.520023+0000","last_peered":"2023-09-23T18:11:41.520023+0000","last_clean":"2023-09-23T18:11:41.520023+0000","last_became_active":"2023-09-23T18:11:29.575912+0000","last_became_peered":"2023-09-23T18:11:29.575912+0000","last_unstale":"2023-09-23T18:11:41.520023+0000","last_undegraded":"2023-09-23T18:11:41.520023+0000","last_fullsized":"2023-09-23T18:11:41.520023+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:31:57.423792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":355,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.522989+0000","last_change":"2023-09-23T18:11:19.168750+0000","last_active":"2023-09-23T18:11:41.522989+0000","last_peered":"2023-09-23T18:11:41.522989+0000","last_clean":"2023-09-23T18:11:41.522989+0000","last_became_active":"2023-09-23T18:11:19.168447+0000","last_became_peered":"2023-09-23T18:11:19.168447+0000","last_unstale":"2023-09-23T18:11:41.522989+0000","last_undegraded":"2023-09-23T18:11:41.522989+0000","last_fullsized":"2023-09-23T18:11:41.522989+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:22.907052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":653,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.510906+0000","last_change":"2023-09-23T18:09:49.248584+0000","last_active":"2023-09-23T18:11:40.510906+0000","last_peered":"2023-09-23T18:11:40.510906+0000","last_clean":"2023-09-23T18:05:59.276387+0000","last_became_active":"2023-09-23T18:09:49.248584+0000","last_became_peered":"2023-09-23T18:09:49.248584+0000","last_unstale":"2023-09-23T18:11:40.510906+0000","last_undegraded":"2023-09-23T18:11:40.510906+0000","last_fullsized":"2023-09-23T18:09:49.239167+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:29.998743+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":411,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.518834+0000","last_change":"2023-09-23T18:10:53.014529+0000","last_active":"2023-09-23T18:11:41.518834+0000","last_peered":"2023-09-23T18:11:41.518834+0000","last_clean":"2023-09-23T18:11:41.518834+0000","last_became_active":"2023-09-23T18:10:52.995019+0000","last_became_peered":"2023-09-23T18:10:52.995019+0000","last_unstale":"2023-09-23T18:11:41.518834+0000","last_undegraded":"2023-09-23T18:11:41.518834+0000","last_fullsized":"2023-09-23T18:11:41.518834+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:29:36.000082+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,2],"avail_no_missing":["6","1","2"],"object_location_counts":[{"shards":"1,2,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":269,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.512971+0000","last_change":"2023-09-23T18:10:51.816822+0000","last_active":"2023-09-23T18:11:40.512971+0000","last_peered":"2023-09-23T18:11:40.512971+0000","last_clean":"2023-09-23T18:11:40.512971+0000","last_became_active":"2023-09-23T18:10:51.816526+0000","last_became_peered":"2023-09-23T18:10:51.816526+0000","last_unstale":"2023-09-23T18:11:40.512971+0000","last_undegraded":"2023-09-23T18:11:40.512971+0000","last_fullsized":"2023-09-23T18:11:40.512971+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:31.194789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.4","version":"406'8705","reported_seq":8772,"reported_epoch":406,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:11:45.577545+0000","last_change":"2023-09-23T18:11:30.662628+0000","last_active":"2023-09-23T18:11:45.577545+0000","last_peered":"2023-09-23T18:11:45.577545+0000","last_clean":"2023-09-23T18:09:48.184212+0000","last_became_active":"2023-09-23T18:11:30.643750+0000","last_became_peered":"2023-09-23T18:11:30.643750+0000","last_unstale":"2023-09-23T18:11:45.577545+0000","last_undegraded":"2023-09-23T18:11:30.570213+0000","last_fullsized":"2023-09-23T18:11:30.545289+0000","mapping_epoch":398,"log_start":"377'4685","ondisk_log_start":"377'4685","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4020,"log_dups_size":0,"ondisk_log_size":4020,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:22.486706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":329580544,"num_objects":5029,"num_object_clones":0,"num_object_copies":15087,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5029,"num_objects_misplaced":5029,"num_objects_unfound":0,"num_objects_dirty":5029,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8705,"num_write_kb":439488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":5029}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":805,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.515140+0000","last_change":"2023-09-23T18:11:32.616644+0000","last_active":"2023-09-23T18:11:40.515140+0000","last_peered":"2023-09-23T18:11:40.515140+0000","last_clean":"2023-09-23T18:11:40.515140+0000","last_became_active":"2023-09-23T18:11:32.616328+0000","last_became_peered":"2023-09-23T18:11:32.616328+0000","last_unstale":"2023-09-23T18:11:40.515140+0000","last_undegraded":"2023-09-23T18:11:40.515140+0000","last_fullsized":"2023-09-23T18:11:40.515140+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:38.926602+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":359,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.515620+0000","last_change":"2023-09-23T18:11:30.589034+0000","last_active":"2023-09-23T18:11:40.515620+0000","last_peered":"2023-09-23T18:11:40.515620+0000","last_clean":"2023-09-23T18:11:40.515620+0000","last_became_active":"2023-09-23T18:11:30.588672+0000","last_became_peered":"2023-09-23T18:11:30.588672+0000","last_unstale":"2023-09-23T18:11:40.515620+0000","last_undegraded":"2023-09-23T18:11:40.515620+0000","last_fullsized":"2023-09-23T18:11:40.515620+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:46.926334+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,4],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":276,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.512509+0000","last_change":"2023-09-23T18:09:49.260845+0000","last_active":"2023-09-23T18:11:40.512509+0000","last_peered":"2023-09-23T18:11:40.512509+0000","last_clean":"2023-09-23T18:02:45.953891+0000","last_became_active":"2023-09-23T18:09:49.260845+0000","last_became_peered":"2023-09-23T18:09:49.260845+0000","last_unstale":"2023-09-23T18:11:40.512509+0000","last_undegraded":"2023-09-23T18:11:40.512509+0000","last_fullsized":"2023-09-23T18:09:49.252007+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":250,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:35.249760+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.5","version":"406'8734","reported_seq":8798,"reported_epoch":406,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:11:45.632533+0000","last_change":"2023-09-23T18:11:30.676293+0000","last_active":"2023-09-23T18:11:45.632533+0000","last_peered":"2023-09-23T18:11:45.632533+0000","last_clean":"2023-09-23T18:11:28.500165+0000","last_became_active":"2023-09-23T18:11:30.657908+0000","last_became_peered":"2023-09-23T18:11:30.657908+0000","last_unstale":"2023-09-23T18:11:45.632533+0000","last_undegraded":"2023-09-23T18:11:45.632533+0000","last_fullsized":"2023-09-23T18:11:45.632533+0000","mapping_epoch":398,"log_start":"378'4719","ondisk_log_start":"378'4719","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4015,"log_dups_size":0,"ondisk_log_size":4015,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:21:42.049207+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":333971456,"num_objects":5096,"num_object_clones":0,"num_object_copies":15288,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5096,"num_objects_unfound":0,"num_objects_dirty":5096,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8734,"num_write_kb":442560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,4],"avail_no_missing":["6","4","7"],"object_location_counts":[{"shards":"4,6,7","objects":5096}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":483,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.524496+0000","last_change":"2023-09-23T18:11:41.523931+0000","last_active":"2023-09-23T18:11:41.524496+0000","last_peered":"2023-09-23T18:11:41.524496+0000","last_clean":"2023-09-23T18:11:41.524496+0000","last_became_active":"2023-09-23T18:11:41.523578+0000","last_became_peered":"2023-09-23T18:11:41.523578+0000","last_unstale":"2023-09-23T18:11:41.524496+0000","last_undegraded":"2023-09-23T18:11:41.524496+0000","last_fullsized":"2023-09-23T18:11:41.524496+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:41.510146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:41.510146+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:23.713117+0000","scrub_duration":0.0078303169999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[{"shards":"0,4,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":374,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.515642+0000","last_change":"2023-09-23T18:11:30.570076+0000","last_active":"2023-09-23T18:11:40.515642+0000","last_peered":"2023-09-23T18:11:40.515642+0000","last_clean":"2023-09-23T18:11:40.515642+0000","last_became_active":"2023-09-23T18:11:30.569766+0000","last_became_peered":"2023-09-23T18:11:30.569766+0000","last_unstale":"2023-09-23T18:11:40.515642+0000","last_undegraded":"2023-09-23T18:11:40.515642+0000","last_fullsized":"2023-09-23T18:11:40.515642+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:37.093951+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,4],"avail_no_missing":["7","1","4"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":339,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.510769+0000","last_change":"2023-09-23T18:11:29.570036+0000","last_active":"2023-09-23T18:11:40.510769+0000","last_peered":"2023-09-23T18:11:40.510769+0000","last_clean":"2023-09-23T18:11:40.510769+0000","last_became_active":"2023-09-23T18:11:29.569583+0000","last_became_peered":"2023-09-23T18:11:29.569583+0000","last_unstale":"2023-09-23T18:11:40.510769+0000","last_undegraded":"2023-09-23T18:11:40.510769+0000","last_fullsized":"2023-09-23T18:11:40.510769+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:12.194566+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.6","version":"406'8823","reported_seq":10489,"reported_epoch":406,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:11:45.676151+0000","last_change":"2023-09-23T18:11:41.576860+0000","last_active":"2023-09-23T18:11:45.676151+0000","last_peered":"2023-09-23T18:11:45.676151+0000","last_clean":"2023-09-23T18:10:00.095783+0000","last_became_active":"2023-09-23T18:11:41.534114+0000","last_became_peered":"2023-09-23T18:11:41.534114+0000","last_unstale":"2023-09-23T18:11:45.676151+0000","last_undegraded":"2023-09-23T18:11:41.525847+0000","last_fullsized":"2023-09-23T18:11:45.676151+0000","mapping_epoch":404,"log_start":"378'4834","ondisk_log_start":"378'4834","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3989,"log_dups_size":0,"ondisk_log_size":3989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:23.957244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336658432,"num_objects":5137,"num_object_clones":0,"num_object_copies":15411,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2826,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5137,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8823,"num_write_kb":446720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":799,"num_bytes_recovered":52363264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"0,6,7","objects":3355},{"shards":"2,6,7","objects":1782}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":604,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.523037+0000","last_change":"2023-09-23T18:11:32.611274+0000","last_active":"2023-09-23T18:11:41.523037+0000","last_peered":"2023-09-23T18:11:41.523037+0000","last_clean":"2023-09-23T18:11:41.523037+0000","last_became_active":"2023-09-23T18:11:32.610817+0000","last_became_peered":"2023-09-23T18:11:32.610817+0000","last_unstale":"2023-09-23T18:11:41.523037+0000","last_undegraded":"2023-09-23T18:11:41.523037+0000","last_fullsized":"2023-09-23T18:11:41.523037+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:12:41.445091+0000","scrub_duration":0.0018893740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3,4],"avail_no_missing":["6","3","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":278,"reported_epoch":404,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:11:40.515543+0000","last_change":"2023-09-23T18:09:49.260821+0000","last_active":"2023-09-23T18:11:40.515543+0000","last_peered":"2023-09-23T18:11:40.515543+0000","last_clean":"2023-09-23T18:09:40.155190+0000","last_became_active":"2023-09-23T18:09:49.260821+0000","last_became_peered":"2023-09-23T18:09:49.260821+0000","last_unstale":"2023-09-23T18:11:40.515543+0000","last_undegraded":"2023-09-23T18:09:49.257202+0000","last_fullsized":"2023-09-23T18:09:49.257118+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":215,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:23:05.023383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[{"shards":"6,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":381,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.523807+0000","last_change":"2023-09-23T18:06:05.371447+0000","last_active":"2023-09-23T18:11:41.523807+0000","last_peered":"2023-09-23T18:11:41.523807+0000","last_clean":"2023-09-23T18:11:41.523807+0000","last_became_active":"2023-09-23T18:06:05.371065+0000","last_became_peered":"2023-09-23T18:06:05.371065+0000","last_unstale":"2023-09-23T18:11:41.523807+0000","last_undegraded":"2023-09-23T18:11:41.523807+0000","last_fullsized":"2023-09-23T18:11:41.523807+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.025791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"405'8538","reported_seq":8617,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:42.785013+0000","last_change":"2023-09-23T18:11:28.545256+0000","last_active":"2023-09-23T18:11:42.785013+0000","last_peered":"2023-09-23T18:11:42.785013+0000","last_clean":"2023-09-23T18:11:42.785013+0000","last_became_active":"2023-09-23T18:11:19.181733+0000","last_became_peered":"2023-09-23T18:11:19.181733+0000","last_unstale":"2023-09-23T18:11:42.785013+0000","last_undegraded":"2023-09-23T18:11:42.785013+0000","last_fullsized":"2023-09-23T18:11:42.785013+0000","mapping_epoch":394,"log_start":"377'4607","ondisk_log_start":"377'4607","created":361,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3931,"log_dups_size":0,"ondisk_log_size":3931,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:18.957805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338165760,"num_objects":5160,"num_object_clones":0,"num_object_copies":15480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8538,"num_write_kb":438336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":635,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.515496+0000","last_change":"2023-09-23T18:11:32.624843+0000","last_active":"2023-09-23T18:11:40.515496+0000","last_peered":"2023-09-23T18:11:40.515496+0000","last_clean":"2023-09-23T18:11:40.515496+0000","last_became_active":"2023-09-23T18:11:32.624496+0000","last_became_peered":"2023-09-23T18:11:32.624496+0000","last_unstale":"2023-09-23T18:11:40.515496+0000","last_undegraded":"2023-09-23T18:11:40.515496+0000","last_fullsized":"2023-09-23T18:11:40.515496+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.362652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.362652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":329,"reported_epoch":405,"state":"active+undersized","last_fresh":"2023-09-23T18:11:41.522946+0000","last_change":"2023-09-23T18:09:49.253688+0000","last_active":"2023-09-23T18:11:41.522946+0000","last_peered":"2023-09-23T18:11:41.522946+0000","last_clean":"2023-09-23T18:06:48.956525+0000","last_became_active":"2023-09-23T18:09:49.253688+0000","last_became_peered":"2023-09-23T18:09:49.253688+0000","last_unstale":"2023-09-23T18:11:41.522946+0000","last_undegraded":"2023-09-23T18:11:41.522946+0000","last_fullsized":"2023-09-23T18:09:49.240299+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:58.227257+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":330,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.526009+0000","last_change":"2023-09-23T18:11:41.525721+0000","last_active":"2023-09-23T18:11:41.526009+0000","last_peered":"2023-09-23T18:11:41.526009+0000","last_clean":"2023-09-23T18:11:41.526009+0000","last_became_active":"2023-09-23T18:11:41.525380+0000","last_became_peered":"2023-09-23T18:11:41.525380+0000","last_unstale":"2023-09-23T18:11:41.526009+0000","last_undegraded":"2023-09-23T18:11:41.526009+0000","last_fullsized":"2023-09-23T18:11:41.526009+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:34.452925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"405'8554","reported_seq":8628,"reported_epoch":405,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:11:42.733408+0000","last_change":"2023-09-23T18:11:18.164509+0000","last_active":"2023-09-23T18:11:42.733408+0000","last_peered":"2023-09-23T18:11:42.733408+0000","last_clean":"2023-09-23T18:09:48.233578+0000","last_became_active":"2023-09-23T18:11:18.164509+0000","last_became_peered":"2023-09-23T18:11:18.164509+0000","last_unstale":"2023-09-23T18:11:42.733408+0000","last_undegraded":"2023-09-23T18:11:18.145440+0000","last_fullsized":"2023-09-23T18:11:18.145265+0000","mapping_epoch":393,"log_start":"377'4620","ondisk_log_start":"377'4620","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3934,"log_dups_size":0,"ondisk_log_size":3934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:06:21.888580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":344719360,"num_objects":5260,"num_object_clones":0,"num_object_copies":15780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5260,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8554,"num_write_kb":442048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":5260}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":816,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.513272+0000","last_change":"2023-09-23T18:11:19.167242+0000","last_active":"2023-09-23T18:11:40.513272+0000","last_peered":"2023-09-23T18:11:40.513272+0000","last_clean":"2023-09-23T18:11:40.513272+0000","last_became_active":"2023-09-23T18:11:19.166818+0000","last_became_peered":"2023-09-23T18:11:19.166818+0000","last_unstale":"2023-09-23T18:11:40.513272+0000","last_undegraded":"2023-09-23T18:11:40.513272+0000","last_fullsized":"2023-09-23T18:11:40.513272+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:58.563930+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,6],"avail_no_missing":["7","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":204,"reported_epoch":404,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:11:40.513544+0000","last_change":"2023-09-23T18:11:29.574559+0000","last_active":"2023-09-23T18:11:40.513544+0000","last_peered":"2023-09-23T18:11:40.513544+0000","last_clean":"2023-09-23T18:06:37.478653+0000","last_became_active":"2023-09-23T18:11:29.573933+0000","last_became_peered":"2023-09-23T18:11:29.573933+0000","last_unstale":"2023-09-23T18:11:40.513544+0000","last_undegraded":"2023-09-23T18:11:29.557329+0000","last_fullsized":"2023-09-23T18:11:40.513544+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:18.880929+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":426,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.522854+0000","last_change":"2023-09-23T18:11:29.570410+0000","last_active":"2023-09-23T18:11:41.522854+0000","last_peered":"2023-09-23T18:11:41.522854+0000","last_clean":"2023-09-23T18:11:41.522854+0000","last_became_active":"2023-09-23T18:11:29.570111+0000","last_became_peered":"2023-09-23T18:11:29.570111+0000","last_unstale":"2023-09-23T18:11:41.522854+0000","last_undegraded":"2023-09-23T18:11:41.522854+0000","last_fullsized":"2023-09-23T18:11:41.522854+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:02:28.674208+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.1","version":"405'8541","reported_seq":13016,"reported_epoch":405,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:11:42.812671+0000","last_change":"2023-09-23T18:11:29.581658+0000","last_active":"2023-09-23T18:11:42.812671+0000","last_peered":"2023-09-23T18:11:42.812671+0000","last_clean":"2023-09-23T18:09:48.228392+0000","last_became_active":"2023-09-23T18:10:52.853765+0000","last_became_peered":"2023-09-23T18:10:52.853765+0000","last_unstale":"2023-09-23T18:11:42.812671+0000","last_undegraded":"2023-09-23T18:10:52.839450+0000","last_fullsized":"2023-09-23T18:11:42.812671+0000","mapping_epoch":383,"log_start":"377'4640","ondisk_log_start":"377'4640","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3901,"log_dups_size":0,"ondisk_log_size":3901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:36:51.803952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":339935232,"num_objects":5187,"num_object_clones":0,"num_object_copies":15561,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1381,"num_objects_misplaced":5187,"num_objects_unfound":0,"num_objects_dirty":5187,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8541,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2199,"num_bytes_recovered":144113664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"0,1,2","objects":3806},{"shards":"1,2","objects":1381}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":716,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.518640+0000","last_change":"2023-09-23T18:11:32.611572+0000","last_active":"2023-09-23T18:11:41.518640+0000","last_peered":"2023-09-23T18:11:41.518640+0000","last_clean":"2023-09-23T18:11:41.518640+0000","last_became_active":"2023-09-23T18:11:32.611070+0000","last_became_peered":"2023-09-23T18:11:32.611070+0000","last_unstale":"2023-09-23T18:11:41.518640+0000","last_undegraded":"2023-09-23T18:11:41.518640+0000","last_fullsized":"2023-09-23T18:11:41.518640+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:52:32.787974+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":385,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.524151+0000","last_change":"2023-09-23T18:11:29.571830+0000","last_active":"2023-09-23T18:11:41.524151+0000","last_peered":"2023-09-23T18:11:41.524151+0000","last_clean":"2023-09-23T18:11:41.524151+0000","last_became_active":"2023-09-23T18:11:29.571285+0000","last_became_peered":"2023-09-23T18:11:29.571285+0000","last_unstale":"2023-09-23T18:11:41.524151+0000","last_undegraded":"2023-09-23T18:11:41.524151+0000","last_fullsized":"2023-09-23T18:11:41.524151+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:22:03.921391+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":214,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.511071+0000","last_change":"2023-09-23T18:11:30.581806+0000","last_active":"2023-09-23T18:11:40.511071+0000","last_peered":"2023-09-23T18:11:40.511071+0000","last_clean":"2023-09-23T18:11:40.511071+0000","last_became_active":"2023-09-23T18:11:30.580361+0000","last_became_peered":"2023-09-23T18:11:30.580361+0000","last_unstale":"2023-09-23T18:11:40.511071+0000","last_undegraded":"2023-09-23T18:11:40.511071+0000","last_fullsized":"2023-09-23T18:11:40.511071+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:16:04.856891+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,4],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.2","version":"406'8640","reported_seq":12061,"reported_epoch":406,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:11:45.676077+0000","last_change":"2023-09-23T18:11:41.575448+0000","last_active":"2023-09-23T18:11:45.676077+0000","last_peered":"2023-09-23T18:11:45.676077+0000","last_clean":"2023-09-23T18:09:48.231362+0000","last_became_active":"2023-09-23T18:11:41.533286+0000","last_became_peered":"2023-09-23T18:11:41.533286+0000","last_unstale":"2023-09-23T18:11:45.676077+0000","last_undegraded":"2023-09-23T18:11:41.522706+0000","last_fullsized":"2023-09-23T18:11:45.676077+0000","mapping_epoch":404,"log_start":"377'4671","ondisk_log_start":"377'4671","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3969,"log_dups_size":0,"ondisk_log_size":3969,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:01.504918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335675392,"num_objects":5122,"num_object_clones":0,"num_object_copies":15366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1859,"num_objects_misplaced":5122,"num_objects_unfound":0,"num_objects_dirty":5122,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8640,"num_write_kb":440384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1667,"num_bytes_recovered":109248512,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"0,4,6","objects":3354},{"shards":"4,6","objects":1768}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":789,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.513970+0000","last_change":"2023-09-23T18:10:04.176981+0000","last_active":"2023-09-23T18:11:40.513970+0000","last_peered":"2023-09-23T18:11:40.513970+0000","last_clean":"2023-09-23T18:11:40.513970+0000","last_became_active":"2023-09-23T18:10:04.176481+0000","last_became_peered":"2023-09-23T18:10:04.176481+0000","last_unstale":"2023-09-23T18:11:40.513970+0000","last_undegraded":"2023-09-23T18:11:40.513970+0000","last_fullsized":"2023-09-23T18:11:40.513970+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:12.991004+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,2],"avail_no_missing":["1","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":196,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.513408+0000","last_change":"2023-09-23T18:11:30.567537+0000","last_active":"2023-09-23T18:11:40.513408+0000","last_peered":"2023-09-23T18:11:40.513408+0000","last_clean":"2023-09-23T18:11:40.513408+0000","last_became_active":"2023-09-23T18:11:30.567098+0000","last_became_peered":"2023-09-23T18:11:30.567098+0000","last_unstale":"2023-09-23T18:11:40.513408+0000","last_undegraded":"2023-09-23T18:11:40.513408+0000","last_fullsized":"2023-09-23T18:11:40.513408+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:54.692397+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1,4],"avail_no_missing":["7","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":318,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.526630+0000","last_change":"2023-09-23T18:11:29.570256+0000","last_active":"2023-09-23T18:11:41.526630+0000","last_peered":"2023-09-23T18:11:41.526630+0000","last_clean":"2023-09-23T18:11:41.526630+0000","last_became_active":"2023-09-23T18:11:29.569884+0000","last_became_peered":"2023-09-23T18:11:29.569884+0000","last_unstale":"2023-09-23T18:11:41.526630+0000","last_undegraded":"2023-09-23T18:11:41.526630+0000","last_fullsized":"2023-09-23T18:11:41.526630+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:08.642443+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.3","version":"405'8666","reported_seq":8730,"reported_epoch":405,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:11:42.761024+0000","last_change":"2023-09-23T18:09:49.272932+0000","last_active":"2023-09-23T18:11:42.761024+0000","last_peered":"2023-09-23T18:11:42.761024+0000","last_clean":"2023-09-23T18:09:48.219569+0000","last_became_active":"2023-09-23T18:09:49.258387+0000","last_became_peered":"2023-09-23T18:09:49.258387+0000","last_unstale":"2023-09-23T18:11:42.761024+0000","last_undegraded":"2023-09-23T18:09:49.243174+0000","last_fullsized":"2023-09-23T18:09:49.240692+0000","mapping_epoch":367,"log_start":"377'4745","ondisk_log_start":"377'4745","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3921,"log_dups_size":0,"ondisk_log_size":3921,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:55.945872+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346816512,"num_objects":5292,"num_object_clones":0,"num_object_copies":15876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5292,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8666,"num_write_kb":446656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":5292}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":501,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.523663+0000","last_change":"2023-09-23T18:11:32.611579+0000","last_active":"2023-09-23T18:11:41.523663+0000","last_peered":"2023-09-23T18:11:41.523663+0000","last_clean":"2023-09-23T18:11:41.523663+0000","last_became_active":"2023-09-23T18:11:32.611123+0000","last_became_peered":"2023-09-23T18:11:32.611123+0000","last_unstale":"2023-09-23T18:11:41.523663+0000","last_undegraded":"2023-09-23T18:11:41.523663+0000","last_fullsized":"2023-09-23T18:11:41.523663+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":262,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.509205+0000","last_change":"2023-09-23T18:11:30.560512+0000","last_active":"2023-09-23T18:11:40.509205+0000","last_peered":"2023-09-23T18:11:40.509205+0000","last_clean":"2023-09-23T18:11:40.509205+0000","last_became_active":"2023-09-23T18:11:30.559234+0000","last_became_peered":"2023-09-23T18:11:30.559234+0000","last_unstale":"2023-09-23T18:11:40.509205+0000","last_undegraded":"2023-09-23T18:11:40.509205+0000","last_fullsized":"2023-09-23T18:11:40.509205+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:38:47.244371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,4],"avail_no_missing":["1","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":159,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.512927+0000","last_change":"2023-09-23T18:11:29.570941+0000","last_active":"2023-09-23T18:11:40.512927+0000","last_peered":"2023-09-23T18:11:40.512927+0000","last_clean":"2023-09-23T18:11:40.512927+0000","last_became_active":"2023-09-23T18:11:29.570364+0000","last_became_peered":"2023-09-23T18:11:29.570364+0000","last_unstale":"2023-09-23T18:11:40.512927+0000","last_undegraded":"2023-09-23T18:11:40.512927+0000","last_fullsized":"2023-09-23T18:11:40.512927+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:38:44.480232+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"405'8600","reported_seq":17558,"reported_epoch":405,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:11:43.105587+0000","last_change":"2023-09-23T18:11:30.677408+0000","last_active":"2023-09-23T18:11:43.105587+0000","last_peered":"2023-09-23T18:11:43.105587+0000","last_clean":"2023-09-23T18:10:50.781383+0000","last_became_active":"2023-09-23T18:11:30.652420+0000","last_became_peered":"2023-09-23T18:11:30.652420+0000","last_unstale":"2023-09-23T18:11:43.105587+0000","last_undegraded":"2023-09-23T18:11:43.105587+0000","last_fullsized":"2023-09-23T18:11:43.105587+0000","mapping_epoch":398,"log_start":"375'4309","ondisk_log_start":"375'4309","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4291,"log_dups_size":0,"ondisk_log_size":4291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:53:34.858535+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343015424,"num_objects":5234,"num_object_clones":0,"num_object_copies":15702,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6721,"num_objects_unfound":0,"num_objects_dirty":5234,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8600,"num_write_kb":442688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4441,"num_bytes_recovered":291045376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,4,2],"avail_no_missing":["7","2","4"],"object_location_counts":[{"shards":"2,4,7","objects":5234}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":604,"reported_epoch":405,"state":"active+undersized","last_fresh":"2023-09-23T18:11:41.522596+0000","last_change":"2023-09-23T18:09:49.254468+0000","last_active":"2023-09-23T18:11:41.522596+0000","last_peered":"2023-09-23T18:11:41.522596+0000","last_clean":"2023-09-23T18:09:40.149522+0000","last_became_active":"2023-09-23T18:09:49.254468+0000","last_became_peered":"2023-09-23T18:09:49.254468+0000","last_unstale":"2023-09-23T18:11:41.522596+0000","last_undegraded":"2023-09-23T18:11:41.522596+0000","last_fullsized":"2023-09-23T18:09:49.240580+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:11.928170+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":808,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.512418+0000","last_change":"2023-09-23T18:10:04.168459+0000","last_active":"2023-09-23T18:11:40.512418+0000","last_peered":"2023-09-23T18:11:40.512418+0000","last_clean":"2023-09-23T18:11:40.512418+0000","last_became_active":"2023-09-23T18:10:04.167807+0000","last_became_peered":"2023-09-23T18:10:04.167807+0000","last_unstale":"2023-09-23T18:11:40.512418+0000","last_undegraded":"2023-09-23T18:11:40.512418+0000","last_fullsized":"2023-09-23T18:11:40.512418+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:40.519034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":345,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.523104+0000","last_change":"2023-09-23T18:10:53.055010+0000","last_active":"2023-09-23T18:11:41.523104+0000","last_peered":"2023-09-23T18:11:41.523104+0000","last_clean":"2023-09-23T18:11:41.523104+0000","last_became_active":"2023-09-23T18:10:53.054586+0000","last_became_peered":"2023-09-23T18:10:53.054586+0000","last_unstale":"2023-09-23T18:11:41.523104+0000","last_undegraded":"2023-09-23T18:11:41.523104+0000","last_fullsized":"2023-09-23T18:11:41.523104+0000","mapping_epoch":383,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":384,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:28:20.117569+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,2],"avail_no_missing":["6","2","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":148,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.511191+0000","last_change":"2023-09-23T18:11:30.569740+0000","last_active":"2023-09-23T18:11:40.511191+0000","last_peered":"2023-09-23T18:11:40.511191+0000","last_clean":"2023-09-23T18:11:40.511191+0000","last_became_active":"2023-09-23T18:11:30.568936+0000","last_became_peered":"2023-09-23T18:11:30.568936+0000","last_unstale":"2023-09-23T18:11:40.511191+0000","last_undegraded":"2023-09-23T18:11:40.511191+0000","last_fullsized":"2023-09-23T18:11:40.511191+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:09.343388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,4],"avail_no_missing":["1","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"405'8543","reported_seq":11052,"reported_epoch":405,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T18:11:42.185165+0000","last_change":"2023-09-23T18:11:19.360101+0000","last_active":"2023-09-23T18:11:42.185165+0000","last_peered":"2023-09-23T18:11:42.185165+0000","last_clean":"2023-09-23T18:09:48.238829+0000","last_became_active":"2023-09-23T18:11:19.328296+0000","last_became_peered":"2023-09-23T18:11:19.328296+0000","last_unstale":"2023-09-23T18:11:42.185165+0000","last_undegraded":"2023-09-23T18:11:19.316471+0000","last_fullsized":"2023-09-23T18:11:42.185165+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":8543,"log_dups_size":0,"ondisk_log_size":8543,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:48:12.157987+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345702400,"num_objects":5275,"num_object_clones":0,"num_object_copies":15825,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5275,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5275,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8543,"num_write_kb":442176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1217,"num_bytes_recovered":79626240,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7,4],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,4,7","objects":110},{"shards":"3,7","objects":5165}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":645,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.527493+0000","last_change":"2023-09-23T18:11:41.527493+0000","last_active":"2023-09-23T18:11:41.527493+0000","last_peered":"2023-09-23T18:11:41.527493+0000","last_clean":"2023-09-23T18:11:41.527493+0000","last_became_active":"2023-09-23T18:11:41.527049+0000","last_became_peered":"2023-09-23T18:11:41.527049+0000","last_unstale":"2023-09-23T18:11:41.527493+0000","last_undegraded":"2023-09-23T18:11:41.527493+0000","last_fullsized":"2023-09-23T18:11:41.527493+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:56:41.522794+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1264,"reported_epoch":404,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:11:40.510542+0000","last_change":"2023-09-23T18:09:49.245522+0000","last_active":"2023-09-23T18:11:40.510542+0000","last_peered":"2023-09-23T18:11:40.510542+0000","last_clean":"2023-09-23T18:05:59.276534+0000","last_became_active":"2023-09-23T18:09:49.245522+0000","last_became_peered":"2023-09-23T18:09:49.245522+0000","last_unstale":"2023-09-23T18:11:40.510542+0000","last_undegraded":"2023-09-23T18:09:49.238958+0000","last_fullsized":"2023-09-23T18:09:49.238829+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:46.845650+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":189,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.519522+0000","last_change":"2023-09-23T18:11:40.509907+0000","last_active":"2023-09-23T18:11:41.519522+0000","last_peered":"2023-09-23T18:11:41.519522+0000","last_clean":"2023-09-23T18:11:41.519522+0000","last_became_active":"2023-09-23T18:11:40.509579+0000","last_became_peered":"2023-09-23T18:11:40.509579+0000","last_unstale":"2023-09-23T18:11:41.519522+0000","last_undegraded":"2023-09-23T18:11:41.519522+0000","last_fullsized":"2023-09-23T18:11:41.519522+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:44:04.821865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":150,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.510909+0000","last_change":"2023-09-23T18:10:51.815823+0000","last_active":"2023-09-23T18:11:40.510909+0000","last_peered":"2023-09-23T18:11:40.510909+0000","last_clean":"2023-09-23T18:11:40.510909+0000","last_became_active":"2023-09-23T18:10:51.815528+0000","last_became_peered":"2023-09-23T18:10:51.815528+0000","last_unstale":"2023-09-23T18:11:40.510909+0000","last_undegraded":"2023-09-23T18:11:40.510909+0000","last_fullsized":"2023-09-23T18:11:40.510909+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:40.194226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"405'8500","reported_seq":8560,"reported_epoch":405,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:11:42.799638+0000","last_change":"2023-09-23T18:11:30.639107+0000","last_active":"2023-09-23T18:11:42.799638+0000","last_peered":"2023-09-23T18:11:42.799638+0000","last_clean":"2023-09-23T18:11:28.515469+0000","last_became_active":"2023-09-23T18:11:30.625971+0000","last_became_peered":"2023-09-23T18:11:30.625971+0000","last_unstale":"2023-09-23T18:11:42.799638+0000","last_undegraded":"2023-09-23T18:11:42.799638+0000","last_fullsized":"2023-09-23T18:11:42.799638+0000","mapping_epoch":398,"log_start":"377'4602","ondisk_log_start":"377'4602","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3898,"log_dups_size":0,"ondisk_log_size":3898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:45:44.342957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":338165760,"num_objects":5160,"num_object_clones":0,"num_object_copies":15480,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5160,"num_objects_unfound":0,"num_objects_dirty":5160,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8500,"num_write_kb":437120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":5160}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":619,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.528742+0000","last_change":"2023-09-23T18:11:41.528742+0000","last_active":"2023-09-23T18:11:41.528742+0000","last_peered":"2023-09-23T18:11:41.528742+0000","last_clean":"2023-09-23T18:11:41.528742+0000","last_became_active":"2023-09-23T18:11:41.528226+0000","last_became_peered":"2023-09-23T18:11:41.528226+0000","last_unstale":"2023-09-23T18:11:41.528742+0000","last_undegraded":"2023-09-23T18:11:41.528742+0000","last_fullsized":"2023-09-23T18:11:41.528742+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:25:41.568587+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":656,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.516904+0000","last_change":"2023-09-23T18:11:40.522678+0000","last_active":"2023-09-23T18:11:41.516904+0000","last_peered":"2023-09-23T18:11:41.516904+0000","last_clean":"2023-09-23T18:11:41.516904+0000","last_became_active":"2023-09-23T18:11:40.521998+0000","last_became_peered":"2023-09-23T18:11:40.521998+0000","last_unstale":"2023-09-23T18:11:41.516904+0000","last_undegraded":"2023-09-23T18:11:41.516904+0000","last_fullsized":"2023-09-23T18:11:41.516904+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:12.167477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":141,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.509560+0000","last_change":"2023-09-23T18:10:51.809930+0000","last_active":"2023-09-23T18:11:40.509560+0000","last_peered":"2023-09-23T18:11:40.509560+0000","last_clean":"2023-09-23T18:11:40.509560+0000","last_became_active":"2023-09-23T18:10:51.809574+0000","last_became_peered":"2023-09-23T18:10:51.809574+0000","last_unstale":"2023-09-23T18:11:40.509560+0000","last_undegraded":"2023-09-23T18:11:40.509560+0000","last_fullsized":"2023-09-23T18:11:40.509560+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:05.356356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":147,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.513070+0000","last_change":"2023-09-23T18:09:49.257163+0000","last_active":"2023-09-23T18:11:40.513070+0000","last_peered":"2023-09-23T18:11:40.513070+0000","last_clean":"2023-09-23T18:09:40.168372+0000","last_became_active":"2023-09-23T18:09:49.257163+0000","last_became_peered":"2023-09-23T18:09:49.257163+0000","last_unstale":"2023-09-23T18:11:40.513070+0000","last_undegraded":"2023-09-23T18:11:40.513070+0000","last_fullsized":"2023-09-23T18:09:49.240945+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:37:14.517993+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"406'8794","reported_seq":8944,"reported_epoch":406,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:11:45.688747+0000","last_change":"2023-09-23T18:11:18.168627+0000","last_active":"2023-09-23T18:11:45.688747+0000","last_peered":"2023-09-23T18:11:45.688747+0000","last_clean":"2023-09-23T18:09:48.227889+0000","last_became_active":"2023-09-23T18:11:18.168627+0000","last_became_peered":"2023-09-23T18:11:18.168627+0000","last_unstale":"2023-09-23T18:11:45.688747+0000","last_undegraded":"2023-09-23T18:11:18.149078+0000","last_fullsized":"2023-09-23T18:11:18.148964+0000","mapping_epoch":393,"log_start":"378'4805","ondisk_log_start":"378'4805","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3989,"log_dups_size":0,"ondisk_log_size":3989,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:22:42.193927+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":335282176,"num_objects":5116,"num_object_clones":0,"num_object_copies":15348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5116,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8794,"num_write_kb":445120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":5116}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1100,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.513224+0000","last_change":"2023-09-23T18:11:19.172105+0000","last_active":"2023-09-23T18:11:40.513224+0000","last_peered":"2023-09-23T18:11:40.513224+0000","last_clean":"2023-09-23T18:11:40.513224+0000","last_became_active":"2023-09-23T18:11:19.171511+0000","last_became_peered":"2023-09-23T18:11:19.171511+0000","last_unstale":"2023-09-23T18:11:40.513224+0000","last_undegraded":"2023-09-23T18:11:40.513224+0000","last_fullsized":"2023-09-23T18:11:40.513224+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:37.524546+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3,2],"avail_no_missing":["7","2","3"],"object_location_counts":[{"shards":"2,3,7","objects":1}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":760,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.512511+0000","last_change":"2023-09-23T18:06:48.957422+0000","last_active":"2023-09-23T18:11:40.512511+0000","last_peered":"2023-09-23T18:11:40.512511+0000","last_clean":"2023-09-23T18:11:40.512511+0000","last_became_active":"2023-09-23T18:06:48.957044+0000","last_became_peered":"2023-09-23T18:06:48.957044+0000","last_unstale":"2023-09-23T18:11:40.512511+0000","last_undegraded":"2023-09-23T18:11:40.512511+0000","last_fullsized":"2023-09-23T18:11:40.512511+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:58.047861+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":187,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.524023+0000","last_change":"2023-09-23T18:11:30.562348+0000","last_active":"2023-09-23T18:11:41.524023+0000","last_peered":"2023-09-23T18:11:41.524023+0000","last_clean":"2023-09-23T18:11:41.524023+0000","last_became_active":"2023-09-23T18:11:30.560808+0000","last_became_peered":"2023-09-23T18:11:30.560808+0000","last_unstale":"2023-09-23T18:11:41.524023+0000","last_undegraded":"2023-09-23T18:11:41.524023+0000","last_fullsized":"2023-09-23T18:11:41.524023+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:01:34.047235+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,4],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":349,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.523611+0000","last_change":"2023-09-23T18:11:29.571435+0000","last_active":"2023-09-23T18:11:41.523611+0000","last_peered":"2023-09-23T18:11:41.523611+0000","last_clean":"2023-09-23T18:11:41.523611+0000","last_became_active":"2023-09-23T18:11:29.571081+0000","last_became_peered":"2023-09-23T18:11:29.571081+0000","last_unstale":"2023-09-23T18:11:41.523611+0000","last_undegraded":"2023-09-23T18:11:41.523611+0000","last_fullsized":"2023-09-23T18:11:41.523611+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:47:55.564070+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,0],"acting":[6,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"405'8650","reported_seq":10411,"reported_epoch":405,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:11:42.761628+0000","last_change":"2023-09-23T18:11:29.575618+0000","last_active":"2023-09-23T18:11:42.761628+0000","last_peered":"2023-09-23T18:11:42.761628+0000","last_clean":"2023-09-23T18:09:48.219556+0000","last_became_active":"2023-09-23T18:11:18.230425+0000","last_became_peered":"2023-09-23T18:11:18.230425+0000","last_unstale":"2023-09-23T18:11:42.761628+0000","last_undegraded":"2023-09-23T18:11:18.159979+0000","last_fullsized":"2023-09-23T18:11:18.146088+0000","mapping_epoch":393,"log_start":"377'4732","ondisk_log_start":"377'4732","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3918,"log_dups_size":0,"ondisk_log_size":3918,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:54:41.969324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":346685440,"num_objects":5290,"num_object_clones":0,"num_object_copies":15870,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4561,"num_objects_misplaced":5290,"num_objects_unfound":0,"num_objects_dirty":5290,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8650,"num_write_kb":446080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":838,"num_bytes_recovered":54919168,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":5290}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":577,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.513773+0000","last_change":"2023-09-23T18:11:30.574656+0000","last_active":"2023-09-23T18:11:40.513773+0000","last_peered":"2023-09-23T18:11:40.513773+0000","last_clean":"2023-09-23T18:11:40.513773+0000","last_became_active":"2023-09-23T18:11:30.573870+0000","last_became_peered":"2023-09-23T18:11:30.573870+0000","last_unstale":"2023-09-23T18:11:40.513773+0000","last_undegraded":"2023-09-23T18:11:40.513773+0000","last_fullsized":"2023-09-23T18:11:40.513773+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:24.501389+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,4],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":668,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.512880+0000","last_change":"2023-09-23T18:10:01.230791+0000","last_active":"2023-09-23T18:11:40.512880+0000","last_peered":"2023-09-23T18:11:40.512880+0000","last_clean":"2023-09-23T18:11:40.512880+0000","last_became_active":"2023-09-23T18:10:01.170909+0000","last_became_peered":"2023-09-23T18:10:01.170909+0000","last_unstale":"2023-09-23T18:11:40.512880+0000","last_undegraded":"2023-09-23T18:11:40.512880+0000","last_fullsized":"2023-09-23T18:11:40.512880+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:08.759640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":337,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.527450+0000","last_change":"2023-09-23T18:11:41.527450+0000","last_active":"2023-09-23T18:11:41.527450+0000","last_peered":"2023-09-23T18:11:41.527450+0000","last_clean":"2023-09-23T18:11:41.527450+0000","last_became_active":"2023-09-23T18:11:41.527044+0000","last_became_peered":"2023-09-23T18:11:41.527044+0000","last_unstale":"2023-09-23T18:11:41.527450+0000","last_undegraded":"2023-09-23T18:11:41.527450+0000","last_fullsized":"2023-09-23T18:11:41.527450+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:50:56.421213+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0],"acting":[7,0,4],"avail_no_missing":["7","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":390,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.511271+0000","last_change":"2023-09-23T18:09:49.263629+0000","last_active":"2023-09-23T18:11:40.511271+0000","last_peered":"2023-09-23T18:11:40.511271+0000","last_clean":"2023-09-23T18:09:40.167801+0000","last_became_active":"2023-09-23T18:09:49.263629+0000","last_became_peered":"2023-09-23T18:09:49.263629+0000","last_unstale":"2023-09-23T18:11:40.511271+0000","last_undegraded":"2023-09-23T18:11:40.511271+0000","last_fullsized":"2023-09-23T18:09:49.239463+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:25.971337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"405'8542","reported_seq":8625,"reported_epoch":405,"state":"active+recovery_wait+remapped","last_fresh":"2023-09-23T18:11:42.807538+0000","last_change":"2023-09-23T18:11:30.658129+0000","last_active":"2023-09-23T18:11:42.807538+0000","last_peered":"2023-09-23T18:11:42.807538+0000","last_clean":"2023-09-23T18:11:28.502507+0000","last_became_active":"2023-09-23T18:11:30.628065+0000","last_became_peered":"2023-09-23T18:11:30.628065+0000","last_unstale":"2023-09-23T18:11:42.807538+0000","last_undegraded":"2023-09-23T18:11:42.807538+0000","last_fullsized":"2023-09-23T18:11:42.807538+0000","mapping_epoch":398,"log_start":"377'4636","ondisk_log_start":"377'4636","created":361,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":3906,"log_dups_size":0,"ondisk_log_size":3906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:14.680914+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":336592896,"num_objects":5136,"num_object_clones":0,"num_object_copies":15408,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5136,"num_objects_unfound":0,"num_objects_dirty":5136,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8542,"num_write_kb":437696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,4],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":5136}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":629,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.526739+0000","last_change":"2023-09-23T18:11:30.579074+0000","last_active":"2023-09-23T18:11:41.526739+0000","last_peered":"2023-09-23T18:11:41.526739+0000","last_clean":"2023-09-23T18:11:41.526739+0000","last_became_active":"2023-09-23T18:11:30.578841+0000","last_became_peered":"2023-09-23T18:11:30.578841+0000","last_unstale":"2023-09-23T18:11:41.526739+0000","last_undegraded":"2023-09-23T18:11:41.526739+0000","last_fullsized":"2023-09-23T18:11:41.526739+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:42:09.442022+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,4],"avail_no_missing":["6","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":670,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.515443+0000","last_change":"2023-09-23T18:11:39.533416+0000","last_active":"2023-09-23T18:11:40.515443+0000","last_peered":"2023-09-23T18:11:40.515443+0000","last_clean":"2023-09-23T18:11:40.515443+0000","last_became_active":"2023-09-23T18:11:29.570038+0000","last_became_peered":"2023-09-23T18:11:29.570038+0000","last_unstale":"2023-09-23T18:11:40.515443+0000","last_undegraded":"2023-09-23T18:11:40.515443+0000","last_fullsized":"2023-09-23T18:11:40.515443+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":403,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:00:43.655330+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":147,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.510780+0000","last_change":"2023-09-23T18:09:49.247536+0000","last_active":"2023-09-23T18:11:40.510780+0000","last_peered":"2023-09-23T18:11:40.510780+0000","last_clean":"2023-09-23T18:09:40.168110+0000","last_became_active":"2023-09-23T18:09:49.247536+0000","last_became_peered":"2023-09-23T18:09:49.247536+0000","last_unstale":"2023-09-23T18:11:40.510780+0000","last_undegraded":"2023-09-23T18:11:40.510780+0000","last_fullsized":"2023-09-23T18:09:49.239753+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":304,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:00:23.414385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":47,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.518995+0000","last_change":"2023-09-23T18:10:02.191332+0000","last_active":"2023-09-23T18:11:41.518995+0000","last_peered":"2023-09-23T18:11:41.518995+0000","last_clean":"2023-09-23T18:11:41.518995+0000","last_became_active":"2023-09-23T18:10:02.190947+0000","last_became_peered":"2023-09-23T18:10:02.190947+0000","last_unstale":"2023-09-23T18:11:41.518995+0000","last_undegraded":"2023-09-23T18:11:41.518995+0000","last_fullsized":"2023-09-23T18:11:41.518995+0000","mapping_epoch":371,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":372,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:03:14.070652+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7],"acting":[0,7,1],"avail_no_missing":["0","1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"8.c","version":"405'8532","reported_seq":8608,"reported_epoch":405,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:11:43.065527+0000","last_change":"2023-09-23T18:11:18.162863+0000","last_active":"2023-09-23T18:11:43.065527+0000","last_peered":"2023-09-23T18:11:43.065527+0000","last_clean":"2023-09-23T18:09:48.216272+0000","last_became_active":"2023-09-23T18:11:18.162863+0000","last_became_peered":"2023-09-23T18:11:18.162863+0000","last_unstale":"2023-09-23T18:11:43.065527+0000","last_undegraded":"2023-09-23T18:11:18.153489+0000","last_fullsized":"2023-09-23T18:11:18.153384+0000","mapping_epoch":393,"log_start":"375'4246","ondisk_log_start":"375'4246","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4286,"log_dups_size":0,"ondisk_log_size":4286,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:53:33.970899+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":340000768,"num_objects":5188,"num_object_clones":0,"num_object_copies":15564,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5188,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5188,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8532,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":5188}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":444,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.523779+0000","last_change":"2023-09-23T18:11:32.605671+0000","last_active":"2023-09-23T18:11:41.523779+0000","last_peered":"2023-09-23T18:11:41.523779+0000","last_clean":"2023-09-23T18:11:41.523779+0000","last_became_active":"2023-09-23T18:11:32.605217+0000","last_became_peered":"2023-09-23T18:11:32.605217+0000","last_unstale":"2023-09-23T18:11:41.523779+0000","last_undegraded":"2023-09-23T18:11:41.523779+0000","last_fullsized":"2023-09-23T18:11:41.523779+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":20,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,4],"avail_no_missing":["6","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":209,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.515420+0000","last_change":"2023-09-23T18:11:19.178406+0000","last_active":"2023-09-23T18:11:40.515420+0000","last_peered":"2023-09-23T18:11:40.515420+0000","last_clean":"2023-09-23T18:11:40.515420+0000","last_became_active":"2023-09-23T18:11:19.178025+0000","last_became_peered":"2023-09-23T18:11:19.178025+0000","last_unstale":"2023-09-23T18:11:40.515420+0000","last_undegraded":"2023-09-23T18:11:40.515420+0000","last_fullsized":"2023-09-23T18:11:40.515420+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:42.595992+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":154,"reported_epoch":404,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:11:40.512925+0000","last_change":"2023-09-23T18:11:29.572363+0000","last_active":"2023-09-23T18:11:40.512925+0000","last_peered":"2023-09-23T18:11:40.512925+0000","last_clean":"2023-09-23T18:11:24.446423+0000","last_became_active":"2023-09-23T18:11:29.569417+0000","last_became_peered":"2023-09-23T18:11:29.569417+0000","last_unstale":"2023-09-23T18:11:40.512925+0000","last_undegraded":"2023-09-23T18:11:29.559637+0000","last_fullsized":"2023-09-23T18:11:40.512925+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:01.538253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":5,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,3],"acting":[1,6,3],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":216,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.512621+0000","last_change":"2023-09-23T18:09:49.265848+0000","last_active":"2023-09-23T18:11:40.512621+0000","last_peered":"2023-09-23T18:11:40.512621+0000","last_clean":"2023-09-23T18:09:40.154366+0000","last_became_active":"2023-09-23T18:09:49.265848+0000","last_became_peered":"2023-09-23T18:09:49.265848+0000","last_unstale":"2023-09-23T18:11:40.512621+0000","last_undegraded":"2023-09-23T18:11:40.512621+0000","last_fullsized":"2023-09-23T18:09:49.251764+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:23.255571+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"405'8753","reported_seq":13072,"reported_epoch":405,"state":"active+undersized+degraded+remapped+backfilling","last_fresh":"2023-09-23T18:11:43.119138+0000","last_change":"2023-09-23T18:11:39.540942+0000","last_active":"2023-09-23T18:11:43.119138+0000","last_peered":"2023-09-23T18:11:43.119138+0000","last_clean":"2023-09-23T18:09:48.206765+0000","last_became_active":"2023-09-23T18:10:53.062318+0000","last_became_peered":"2023-09-23T18:10:53.062318+0000","last_unstale":"2023-09-23T18:11:43.119138+0000","last_undegraded":"2023-09-23T18:10:52.842479+0000","last_fullsized":"2023-09-23T18:10:52.826867+0000","mapping_epoch":383,"log_start":"376'4451","ondisk_log_start":"376'4451","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4302,"log_dups_size":0,"ondisk_log_size":4302,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:26:52.081459+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":352714776,"num_objects":5383,"num_object_clones":0,"num_object_copies":16149,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3339,"num_objects_misplaced":5383,"num_objects_unfound":0,"num_objects_dirty":5383,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8753,"num_write_kb":452289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2123,"num_bytes_recovered":139132928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,2],"avail_no_missing":["7","2"],"object_location_counts":[{"shards":"2,7","objects":5383}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":657,"reported_epoch":404,"state":"active+undersized","last_fresh":"2023-09-23T18:11:40.515091+0000","last_change":"2023-09-23T18:09:49.258198+0000","last_active":"2023-09-23T18:11:40.515091+0000","last_peered":"2023-09-23T18:11:40.515091+0000","last_clean":"2023-09-23T18:05:06.479280+0000","last_became_active":"2023-09-23T18:09:49.258198+0000","last_became_peered":"2023-09-23T18:09:49.258198+0000","last_unstale":"2023-09-23T18:11:40.515091+0000","last_undegraded":"2023-09-23T18:11:40.515091+0000","last_fullsized":"2023-09-23T18:09:49.249014+0000","mapping_epoch":367,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":307,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:16:05.551792+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6],"acting":[7,6],"avail_no_missing":["7","6"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":707,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.522841+0000","last_change":"2023-09-23T18:11:18.202385+0000","last_active":"2023-09-23T18:11:41.522841+0000","last_peered":"2023-09-23T18:11:41.522841+0000","last_clean":"2023-09-23T18:11:41.522841+0000","last_became_active":"2023-09-23T18:11:18.168714+0000","last_became_peered":"2023-09-23T18:11:18.168714+0000","last_unstale":"2023-09-23T18:11:41.522841+0000","last_undegraded":"2023-09-23T18:11:41.522841+0000","last_fullsized":"2023-09-23T18:11:41.522841+0000","mapping_epoch":393,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:16.704449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":446,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.512505+0000","last_change":"2023-09-23T18:11:30.575165+0000","last_active":"2023-09-23T18:11:40.512505+0000","last_peered":"2023-09-23T18:11:40.512505+0000","last_clean":"2023-09-23T18:11:40.512505+0000","last_became_active":"2023-09-23T18:11:30.574450+0000","last_became_peered":"2023-09-23T18:11:30.574450+0000","last_unstale":"2023-09-23T18:11:40.512505+0000","last_undegraded":"2023-09-23T18:11:40.512505+0000","last_fullsized":"2023-09-23T18:11:40.512505+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:28.830168+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,4],"avail_no_missing":["1","4","7"],"object_location_counts":[{"shards":"1,4,7","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":208,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.519816+0000","last_change":"2023-09-23T18:11:30.563061+0000","last_active":"2023-09-23T18:11:41.519816+0000","last_peered":"2023-09-23T18:11:41.519816+0000","last_clean":"2023-09-23T18:11:41.519816+0000","last_became_active":"2023-09-23T18:11:30.562128+0000","last_became_peered":"2023-09-23T18:11:30.562128+0000","last_unstale":"2023-09-23T18:11:41.519816+0000","last_undegraded":"2023-09-23T18:11:41.519816+0000","last_fullsized":"2023-09-23T18:11:41.519816+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:41.753247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0],"acting":[6,0,4],"avail_no_missing":["6","0","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":565,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.519682+0000","last_change":"2023-09-23T18:11:30.562718+0000","last_active":"2023-09-23T18:11:41.519682+0000","last_peered":"2023-09-23T18:11:41.519682+0000","last_clean":"2023-09-23T18:11:41.519682+0000","last_became_active":"2023-09-23T18:11:30.560950+0000","last_became_peered":"2023-09-23T18:11:30.560950+0000","last_unstale":"2023-09-23T18:11:41.519682+0000","last_undegraded":"2023-09-23T18:11:41.519682+0000","last_fullsized":"2023-09-23T18:11:41.519682+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:14.580819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:14.580819+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:05.329118+0000","scrub_duration":0.003313033,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,4],"avail_no_missing":["6","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":20,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.513371+0000","last_change":"2023-09-23T18:11:29.568835+0000","last_active":"2023-09-23T18:11:40.513371+0000","last_peered":"2023-09-23T18:11:40.513371+0000","last_clean":"2023-09-23T18:11:40.513371+0000","last_became_active":"2023-09-23T18:11:29.568535+0000","last_became_peered":"2023-09-23T18:11:29.568535+0000","last_unstale":"2023-09-23T18:11:40.513371+0000","last_undegraded":"2023-09-23T18:11:40.513371+0000","last_fullsized":"2023-09-23T18:11:40.513371+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:21:54.999127+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":459,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.510620+0000","last_change":"2023-09-23T18:11:30.560720+0000","last_active":"2023-09-23T18:11:40.510620+0000","last_peered":"2023-09-23T18:11:40.510620+0000","last_clean":"2023-09-23T18:11:40.510620+0000","last_became_active":"2023-09-23T18:11:30.559535+0000","last_became_peered":"2023-09-23T18:11:30.559535+0000","last_unstale":"2023-09-23T18:11:40.510620+0000","last_undegraded":"2023-09-23T18:11:40.510620+0000","last_fullsized":"2023-09-23T18:11:40.510620+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:02.730729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7,4],"avail_no_missing":["1","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":560,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.526585+0000","last_change":"2023-09-23T18:11:30.578984+0000","last_active":"2023-09-23T18:11:41.526585+0000","last_peered":"2023-09-23T18:11:41.526585+0000","last_clean":"2023-09-23T18:11:41.526585+0000","last_became_active":"2023-09-23T18:11:30.578663+0000","last_became_peered":"2023-09-23T18:11:30.578663+0000","last_unstale":"2023-09-23T18:11:41.526585+0000","last_undegraded":"2023-09-23T18:11:41.526585+0000","last_fullsized":"2023-09-23T18:11:41.526585+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.604423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.604423+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:59.544972+0000","scrub_duration":0.0014924669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,4],"avail_no_missing":["6","4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":160,"reported_epoch":404,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:11:40.510242+0000","last_change":"2023-09-23T18:11:30.574777+0000","last_active":"2023-09-23T18:11:40.510242+0000","last_peered":"2023-09-23T18:11:40.510242+0000","last_clean":"2023-09-23T18:05:59.284452+0000","last_became_active":"2023-09-23T18:11:30.574519+0000","last_became_peered":"2023-09-23T18:11:30.574519+0000","last_unstale":"2023-09-23T18:11:40.510242+0000","last_undegraded":"2023-09-23T18:11:30.549347+0000","last_fullsized":"2023-09-23T18:11:40.510242+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:41.327858+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,4],"avail_no_missing":["1","4"],"object_location_counts":[{"shards":"1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":359,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.513718+0000","last_change":"2023-09-23T18:11:29.574454+0000","last_active":"2023-09-23T18:11:40.513718+0000","last_peered":"2023-09-23T18:11:40.513718+0000","last_clean":"2023-09-23T18:11:40.513718+0000","last_became_active":"2023-09-23T18:11:29.572329+0000","last_became_peered":"2023-09-23T18:11:29.572329+0000","last_unstale":"2023-09-23T18:11:40.513718+0000","last_undegraded":"2023-09-23T18:11:40.513718+0000","last_fullsized":"2023-09-23T18:11:40.513718+0000","mapping_epoch":397,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":398,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:50.587062+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":49,"reported_epoch":405,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:11:41.523443+0000","last_change":"2023-09-23T18:11:30.573908+0000","last_active":"2023-09-23T18:11:41.523443+0000","last_peered":"2023-09-23T18:11:41.523443+0000","last_clean":"2023-09-23T17:59:45.651889+0000","last_became_active":"2023-09-23T18:11:30.573221+0000","last_became_peered":"2023-09-23T18:11:30.573221+0000","last_unstale":"2023-09-23T18:11:41.523443+0000","last_undegraded":"2023-09-23T18:11:30.546209+0000","last_fullsized":"2023-09-23T18:11:41.523443+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":220,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:43:19.527562+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":1}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":71,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.523510+0000","last_change":"2023-09-23T18:11:40.531290+0000","last_active":"2023-09-23T18:11:41.523510+0000","last_peered":"2023-09-23T18:11:41.523510+0000","last_clean":"2023-09-23T18:11:41.523510+0000","last_became_active":"2023-09-23T18:11:40.531027+0000","last_became_peered":"2023-09-23T18:11:40.531027+0000","last_unstale":"2023-09-23T18:11:41.523510+0000","last_undegraded":"2023-09-23T18:11:41.523510+0000","last_fullsized":"2023-09-23T18:11:41.523510+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:06:41.736360+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":791,"reported_epoch":404,"state":"active+clean","last_fresh":"2023-09-23T18:11:40.515158+0000","last_change":"2023-09-23T18:06:08.440953+0000","last_active":"2023-09-23T18:11:40.515158+0000","last_peered":"2023-09-23T18:11:40.515158+0000","last_clean":"2023-09-23T18:11:40.515158+0000","last_became_active":"2023-09-23T18:06:08.440604+0000","last_became_peered":"2023-09-23T18:06:08.440604+0000","last_unstale":"2023-09-23T18:11:40.515158+0000","last_undegraded":"2023-09-23T18:11:40.515158+0000","last_fullsized":"2023-09-23T18:11:40.515158+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:53.274049+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":218,"reported_epoch":405,"state":"active+clean","last_fresh":"2023-09-23T18:11:41.526721+0000","last_change":"2023-09-23T18:10:51.816416+0000","last_active":"2023-09-23T18:11:41.526721+0000","last_peered":"2023-09-23T18:11:41.526721+0000","last_clean":"2023-09-23T18:11:41.526721+0000","last_became_active":"2023-09-23T18:10:51.815980+0000","last_became_peered":"2023-09-23T18:10:51.815980+0000","last_unstale":"2023-09-23T18:11:41.526721+0000","last_undegraded":"2023-09-23T18:11:41.526721+0000","last_fullsized":"2023-09-23T18:11:41.526721+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:04.404422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":214,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.529948+0000","last_change":"2023-09-23T18:11:41.529948+0000","last_active":"2023-09-23T18:11:41.529948+0000","last_peered":"2023-09-23T18:11:41.529948+0000","last_clean":"2023-09-23T18:11:41.529948+0000","last_became_active":"2023-09-23T18:11:41.529434+0000","last_became_peered":"2023-09-23T18:11:41.529434+0000","last_unstale":"2023-09-23T18:11:41.529948+0000","last_undegraded":"2023-09-23T18:11:41.529948+0000","last_fullsized":"2023-09-23T18:11:41.529948+0000","mapping_epoch":404,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":405,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:47:52.076003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0,2],"avail_no_missing":["1","0","2"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":766,"reported_epoch":404,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:40.513137+0000","last_change":"2023-09-23T18:11:32.618497+0000","last_active":"2023-09-23T18:11:40.513137+0000","last_peered":"2023-09-23T18:11:40.513137+0000","last_clean":"2023-09-23T18:11:40.513137+0000","last_became_active":"2023-09-23T18:11:32.617999+0000","last_became_peered":"2023-09-23T18:11:32.617999+0000","last_unstale":"2023-09-23T18:11:40.513137+0000","last_undegraded":"2023-09-23T18:11:40.513137+0000","last_fullsized":"2023-09-23T18:11:40.513137+0000","mapping_epoch":400,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":401,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:17.635974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:17.635974+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:52:13.437308+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6,4],"avail_no_missing":["1","4","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":225,"reported_epoch":405,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:11:41.522607+0000","last_change":"2023-09-23T18:11:30.562878+0000","last_active":"2023-09-23T18:11:41.522607+0000","last_peered":"2023-09-23T18:11:41.522607+0000","last_clean":"2023-09-23T18:00:42.557373+0000","last_became_active":"2023-09-23T18:11:30.562411+0000","last_became_peered":"2023-09-23T18:11:30.562411+0000","last_unstale":"2023-09-23T18:11:41.522607+0000","last_undegraded":"2023-09-23T18:11:30.546133+0000","last_fullsized":"2023-09-23T18:11:41.522607+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":299,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:53:29.625323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":2,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7,4],"avail_no_missing":["6","4"],"object_location_counts":[{"shards":"4,6","objects":2}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":320,"reported_epoch":405,"state":"active+clean+remapped","last_fresh":"2023-09-23T18:11:41.519039+0000","last_change":"2023-09-23T18:11:30.559169+0000","last_active":"2023-09-23T18:11:41.519039+0000","last_peered":"2023-09-23T18:11:41.519039+0000","last_clean":"2023-09-23T18:11:41.519039+0000","last_became_active":"2023-09-23T18:11:30.555361+0000","last_became_peered":"2023-09-23T18:11:30.555361+0000","last_unstale":"2023-09-23T18:11:41.519039+0000","last_undegraded":"2023-09-23T18:11:41.519039+0000","last_fullsized":"2023-09-23T18:11:41.519039+0000","mapping_epoch":398,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":399,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:10.158383+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1,4],"avail_no_missing":["6","1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":78,"acting":90,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":15,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":54,"num_bytes_recovered":1296,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":45031,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":75,"acting":92,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":5443682328,"num_objects":83065,"num_object_clones":0,"num_object_copies":249195,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":45126,"num_objects_misplaced":48124,"num_objects_unfound":0,"num_objects_dirty":83065,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":138115,"num_write_kb":7077697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13326,"num_bytes_recovered":872873984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15606816768,"data_stored":15606808624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":68813,"ondisk_log_size":68813,"up":38,"acting":41,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":448436,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":100,"num_read_kb":187,"num_write":48,"num_write_kb":494,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1281678,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2375680,"data_stored":2361472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":67,"acting":86,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":1,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":195,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":22,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001469,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2651248,"kb_used_data":2369960,"kb_used_omap":13,"kb_used_meta":281202,"kb_avail":91093904,"statfs":{"total":95995035648,"available":93280157696,"internally_reserved":0,"allocated":2426839040,"data_stored":2425481900,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":287950952},"hb_peers":[0,1,2,3,4,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,1,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.88300000000000001,"5min":1.04,"15min":0.80500000000000005},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":3.4609999999999999,"5min":9.3499999999999996,"15min":9.3499999999999996},"last":3.669},{"interface":"front","average":{"1min":1.1699999999999999,"5min":0.86699999999999999,"15min":1.1160000000000001},"min":{"1min":0.25,"5min":0.25,"15min":0.23000000000000001},"max":{"1min":3.2519999999999998,"5min":3.8159999999999998,"15min":6.641},"last":1.587}]},{"osd":1,"last update":"Sat Sep 23 18:10:44 2023","interfaces":[{"interface":"back","average":{"1min":1.9019999999999999,"5min":1.4179999999999999,"15min":1.024},"min":{"1min":0.25,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":6.1900000000000004,"5min":9.4909999999999997,"15min":9.4909999999999997},"last":4.0860000000000003},{"interface":"front","average":{"1min":2.133,"5min":1.2270000000000001,"15min":1.1599999999999999},"min":{"1min":0.27400000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.6100000000000003,"5min":7.6100000000000003,"15min":7.6100000000000003},"last":7.7480000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:10:44 2023","interfaces":[{"interface":"back","average":{"1min":2.3690000000000002,"5min":1.3799999999999999,"15min":1.129},"min":{"1min":0.29899999999999999,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":7.4290000000000003,"5min":7.4290000000000003,"15min":7.4290000000000003},"last":1.9930000000000001},{"interface":"front","average":{"1min":2.8500000000000001,"5min":1.532,"15min":1.151},"min":{"1min":0.314,"5min":0.307,"15min":0.29399999999999998},"max":{"1min":8.6519999999999992,"5min":8.6519999999999992,"15min":8.6519999999999992},"last":2.0190000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:11:33 2023","interfaces":[{"interface":"back","average":{"1min":2.3420000000000001,"5min":2.1589999999999998,"15min":1.7150000000000001},"min":{"1min":0.435,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":6.3070000000000004,"5min":45.752000000000002,"15min":45.752000000000002},"last":3.5619999999999998},{"interface":"front","average":{"1min":2.028,"5min":1.2729999999999999,"15min":1.242},"min":{"1min":0.32200000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":5.6699999999999999,"5min":6.7889999999999997,"15min":15.349},"last":1.137}]},{"osd":4,"last update":"Sat Sep 23 18:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":0.84299999999999997,"15min":0.996},"min":{"1min":0.26800000000000002,"5min":0.25800000000000001,"15min":0.245},"max":{"1min":4.2869999999999999,"5min":4.4669999999999996,"15min":6.9729999999999999},"last":1.4730000000000001},{"interface":"front","average":{"1min":0.70799999999999996,"5min":0.81200000000000006,"15min":0.68400000000000005},"min":{"1min":0.26700000000000002,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":2.9039999999999999,"5min":5.1059999999999999,"15min":6.3479999999999999},"last":0.876}]},{"osd":5,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.0860000000000001,"15min":1.042},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.6280000000000001,"5min":8.4749999999999996,"15min":9.5999999999999996},"last":3.2149999999999999},{"interface":"front","average":{"1min":1.127,"5min":0.97899999999999998,"15min":1.095},"min":{"1min":0.35899999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.2839999999999998,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":3.1720000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:10:44 2023","interfaces":[{"interface":"back","average":{"1min":0.66500000000000004,"5min":0.86699999999999999,"15min":0.83899999999999997},"min":{"1min":0.31,"5min":0.308,"15min":0.308},"max":{"1min":2.8359999999999999,"5min":6.782,"15min":6.782},"last":0.84599999999999997},{"interface":"front","average":{"1min":0.68600000000000005,"5min":0.79700000000000004,"15min":1.052},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":2.7189999999999999,"5min":3.1280000000000001,"15min":9.5519999999999996},"last":1.6140000000000001}]}]},{"osd":6,"up_from":163,"seq":700079669406,"num_pgs":76,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2801524,"kb_used_data":2457452,"kb_used_omap":7,"kb_used_meta":344056,"kb_avail":90943628,"statfs":{"total":95995035648,"available":93126275072,"internally_reserved":0,"allocated":2516430848,"data_stored":2515079455,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":352314148},"hb_peers":[0,1,2,3,4,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,0,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:10:55 2023","interfaces":[{"interface":"back","average":{"1min":1.375,"5min":1.137,"15min":1.1819999999999999},"min":{"1min":0.28399999999999997,"5min":0.23400000000000001,"15min":0.215},"max":{"1min":8.9429999999999996,"5min":8.9429999999999996,"15min":15.407},"last":0.25},{"interface":"front","average":{"1min":2.0579999999999998,"5min":1.4950000000000001,"15min":1.645},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.23100000000000001},"max":{"1min":8.3689999999999998,"5min":15.147,"15min":15.147},"last":0.81100000000000005}]},{"osd":1,"last update":"Sat Sep 23 18:10:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4790000000000001,"5min":1.2689999999999999,"15min":1.474},"min":{"1min":0.26400000000000001,"5min":0.25800000000000001,"15min":0.23699999999999999},"max":{"1min":4.4370000000000003,"5min":6.5309999999999997,"15min":14.233000000000001},"last":0.30399999999999999},{"interface":"front","average":{"1min":2.7650000000000001,"5min":1.4570000000000001,"15min":1.643},"min":{"1min":0.27700000000000002,"5min":0.26000000000000001,"15min":0.248},"max":{"1min":16.041,"5min":16.041,"15min":16.041},"last":3.0339999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:10:55 2023","interfaces":[{"interface":"back","average":{"1min":2.262,"5min":1.407,"15min":1.7330000000000001},"min":{"1min":0.33700000000000002,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":9.4130000000000003,"5min":9.4130000000000003,"15min":11.210000000000001},"last":0.77800000000000002},{"interface":"front","average":{"1min":2.0289999999999999,"5min":1.3839999999999999,"15min":1.8680000000000001},"min":{"1min":0.378,"5min":0.248,"15min":0.245},"max":{"1min":5.9269999999999996,"5min":8.6300000000000008,"15min":11.223000000000001},"last":0.63300000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:11:35 2023","interfaces":[{"interface":"back","average":{"1min":2.23,"5min":1.6870000000000001,"15min":1.296},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":10.257999999999999,"5min":10.257999999999999,"15min":10.257999999999999},"last":3.4020000000000001},{"interface":"front","average":{"1min":2.3090000000000002,"5min":1.621,"15min":1.3720000000000001},"min":{"1min":0.35999999999999999,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":9.5960000000000001,"5min":9.5960000000000001,"15min":9.5960000000000001},"last":0.83299999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:10:55 2023","interfaces":[{"interface":"back","average":{"1min":0.89300000000000002,"5min":0.89700000000000002,"15min":0.93100000000000005},"min":{"1min":0.27700000000000002,"5min":0.254,"15min":0.23000000000000001},"max":{"1min":5.4119999999999999,"5min":8.0280000000000005,"15min":9.9109999999999996},"last":2.2269999999999999},{"interface":"front","average":{"1min":0.80200000000000005,"5min":1.0269999999999999,"15min":0.81200000000000006},"min":{"1min":0.28399999999999997,"5min":0.27200000000000002,"15min":0.24099999999999999},"max":{"1min":4.7590000000000003,"5min":8.1159999999999997,"15min":8.3000000000000007},"last":0.52500000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":0.99299999999999999,"15min":0.89400000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7219999999999995,"5min":9.7219999999999995,"15min":11.869},"last":1.681},{"interface":"front","average":{"1min":1.5289999999999999,"5min":0.92200000000000004,"15min":0.80100000000000005},"min":{"1min":0.30599999999999999,"5min":0.251,"15min":0.251},"max":{"1min":10.436,"5min":10.436,"15min":10.436},"last":1.746}]},{"osd":7,"last update":"Sat Sep 23 18:11:40 2023","interfaces":[{"interface":"back","average":{"1min":1.427,"5min":1.115,"15min":1.022},"min":{"1min":0.28899999999999998,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":6.5350000000000001,"5min":13.460000000000001,"15min":13.460000000000001},"last":3.448},{"interface":"front","average":{"1min":1.3560000000000001,"5min":1.0329999999999999,"15min":0.94099999999999995},"min":{"1min":0.318,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":8.2780000000000005,"5min":8.2780000000000005,"15min":10.411},"last":0.38400000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795223,"num_pgs":77,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3321324,"kb_used_data":2586660,"kb_used_omap":7,"kb_used_meta":734648,"kb_avail":90423828,"statfs":{"total":95995035648,"available":92593999872,"internally_reserved":0,"allocated":2648739840,"data_stored":2646867707,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":752280253},"hb_peers":[0,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:11:24 2023","interfaces":[{"interface":"back","average":{"1min":1.696,"5min":1.7989999999999999,"15min":1.4350000000000001},"min":{"1min":0.19400000000000001,"5min":0.182,"15min":0.17699999999999999},"max":{"1min":4.1710000000000003,"5min":14.567,"15min":15.529999999999999},"last":0.254},{"interface":"front","average":{"1min":2.8220000000000001,"5min":1.9199999999999999,"15min":1.9570000000000001},"min":{"1min":0.33900000000000002,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.2780000000000005,"5min":10.992000000000001,"15min":82.244},"last":0.20499999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:10:56 2023","interfaces":[{"interface":"back","average":{"1min":2.8879999999999999,"5min":1.804,"15min":1.776},"min":{"1min":0.26800000000000002,"5min":0.22,"15min":0.20899999999999999},"max":{"1min":12.717000000000001,"5min":12.717000000000001,"15min":16.347999999999999},"last":0.53700000000000003},{"interface":"front","average":{"1min":2.359,"5min":1.6120000000000001,"15min":1.4139999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.21299999999999999},"max":{"1min":7.1950000000000003,"5min":7.641,"15min":11.305999999999999},"last":0.39100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:11:36 2023","interfaces":[{"interface":"back","average":{"1min":2.2200000000000002,"5min":1.8879999999999999,"15min":1.1399999999999999},"min":{"1min":0.25900000000000001,"5min":0.247,"15min":0.247},"max":{"1min":11.519,"5min":11.519,"15min":11.519},"last":0.313},{"interface":"front","average":{"1min":2.2989999999999999,"5min":1.6200000000000001,"15min":1.2929999999999999},"min":{"1min":0.29999999999999999,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":8.4930000000000003,"5min":10.141,"15min":11.125999999999999},"last":0.373}]},{"osd":4,"last update":"Sat Sep 23 18:11:24 2023","interfaces":[{"interface":"back","average":{"1min":1.663,"5min":1.7969999999999999,"15min":1.5760000000000001},"min":{"1min":0.26900000000000002,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":9.1289999999999996,"5min":28.446999999999999,"15min":28.446999999999999},"last":0.64700000000000002},{"interface":"front","average":{"1min":1.3140000000000001,"5min":1.8149999999999999,"15min":1.504},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":2.948,"5min":28.434000000000001,"15min":28.434000000000001},"last":0.63300000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.7869999999999999,"15min":1.4379999999999999},"min":{"1min":0.38900000000000001,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":6.7489999999999997,"5min":11.638999999999999,"15min":11.638999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.579,"15min":1.393},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":6.5609999999999999,"5min":8.4309999999999992,"15min":9.5519999999999996},"last":1.962}]},{"osd":6,"last update":"Sat Sep 23 18:10:56 2023","interfaces":[{"interface":"back","average":{"1min":3.5470000000000002,"5min":2.714,"15min":1.98},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":27.516999999999999,"5min":89.566000000000003,"15min":89.566000000000003},"last":0.33000000000000002},{"interface":"front","average":{"1min":3.742,"5min":2.71,"15min":1.8500000000000001},"min":{"1min":0.38500000000000001,"5min":0.38,"15min":0.34699999999999998},"max":{"1min":27.501999999999999,"5min":90.608999999999995,"15min":90.608999999999995},"last":3.9289999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:10:45 2023","interfaces":[{"interface":"back","average":{"1min":3.3029999999999999,"5min":1.778,"15min":1.1000000000000001},"min":{"1min":0.39900000000000002,"5min":0.39800000000000002,"15min":0.33900000000000002},"max":{"1min":27.913,"5min":27.913,"15min":27.913},"last":0.504},{"interface":"front","average":{"1min":3.677,"5min":1.8400000000000001,"15min":1.7589999999999999},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.40999999999999998},"max":{"1min":27.620999999999999,"5min":27.620999999999999,"15min":27.620999999999999},"last":0.82099999999999995}]}]},{"osd":0,"up_from":93,"seq":399431958751,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1223180,"kb_used_data":682564,"kb_used_omap":7,"kb_used_meta":540536,"kb_avail":92521972,"statfs":{"total":95995035648,"available":94742499328,"internally_reserved":0,"allocated":698945536,"data_stored":697040062,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":553509604},"hb_peers":[1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:11:27 2023","interfaces":[{"interface":"back","average":{"1min":2.3239999999999998,"5min":1.8600000000000001,"15min":1.8480000000000001},"min":{"1min":0.23400000000000001,"5min":0.217,"15min":0.17899999999999999},"max":{"1min":13.677,"5min":15.836,"15min":18.154},"last":0.35799999999999998},{"interface":"front","average":{"1min":2.3010000000000002,"5min":1.6639999999999999,"15min":1.6419999999999999},"min":{"1min":0.252,"5min":0.246,"15min":0.17399999999999999},"max":{"1min":7.9619999999999997,"5min":14.468999999999999,"15min":14.468999999999999},"last":1.7969999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.482,"5min":1.397,"15min":1.7909999999999999},"min":{"1min":0.222,"5min":0.222,"15min":0.20499999999999999},"max":{"1min":4.5,"5min":10.007999999999999,"15min":20.882000000000001},"last":0.33600000000000002},{"interface":"front","average":{"1min":2.2919999999999998,"5min":1.5900000000000001,"15min":1.75},"min":{"1min":0.40200000000000002,"5min":0.24399999999999999,"15min":0.217},"max":{"1min":6.6589999999999998,"5min":10.84,"15min":15.064},"last":3.8239999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:11:30 2023","interfaces":[{"interface":"back","average":{"1min":2.1680000000000001,"5min":1.8959999999999999,"15min":1.663},"min":{"1min":0.313,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":11.015000000000001,"5min":12.779999999999999,"15min":12.779999999999999},"last":1.272},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.7909999999999999,"15min":1.498},"min":{"1min":0.315,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":7.6689999999999996,"5min":8.4730000000000008,"15min":15.198},"last":1.244}]},{"osd":4,"last update":"Sat Sep 23 18:10:49 2023","interfaces":[{"interface":"back","average":{"1min":1.9450000000000001,"5min":1.518,"15min":1.5229999999999999},"min":{"1min":0.38400000000000001,"5min":0.32100000000000001,"15min":0.251},"max":{"1min":6.3680000000000003,"5min":8.4109999999999996,"15min":15.279999999999999},"last":1.212},{"interface":"front","average":{"1min":1.246,"5min":1.462,"15min":1.585},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.247},"max":{"1min":4.343,"5min":15.477,"15min":18.106999999999999},"last":0.376}]},{"osd":5,"last update":"Sat Sep 23 18:08:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.7230000000000001,"15min":1.5600000000000001},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":9.2010000000000005,"5min":10.194000000000001,"15min":14.720000000000001},"last":1.478},{"interface":"front","average":{"1min":1.409,"5min":1.512,"15min":1.3140000000000001},"min":{"1min":0.33300000000000002,"5min":0.32600000000000001,"15min":0.254},"max":{"1min":5.1230000000000002,"5min":10.363,"15min":12.794},"last":2.1160000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:10:49 2023","interfaces":[{"interface":"back","average":{"1min":2.1960000000000002,"5min":2.2799999999999998,"15min":1.9770000000000001},"min":{"1min":0.442,"5min":0.35199999999999998,"15min":0.34599999999999997},"max":{"1min":6.2599999999999998,"5min":58.457999999999998,"15min":58.457999999999998},"last":1.153},{"interface":"front","average":{"1min":1.625,"5min":2.1099999999999999,"15min":1.964},"min":{"1min":0.36899999999999999,"5min":0.32900000000000001,"15min":0.311},"max":{"1min":5.149,"5min":58.354999999999997,"15min":58.354999999999997},"last":1.9019999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:11:40 2023","interfaces":[{"interface":"back","average":{"1min":0.879,"5min":1.452,"15min":1.171},"min":{"1min":0.39500000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":2.7730000000000001,"5min":10.416,"15min":10.416},"last":1.867},{"interface":"front","average":{"1min":2.0230000000000001,"5min":1.835,"15min":1.4419999999999999},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.36399999999999999},"max":{"1min":6.6529999999999996,"5min":8.923,"15min":8.923},"last":1.1859999999999999}]}]},{"osd":2,"up_from":0,"seq":442381631698,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249265,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2521440,"kb_used_data":2071000,"kb_used_omap":3,"kb_used_meta":450428,"kb_avail":91223712,"statfs":{"total":95995035648,"available":93413081088,"internally_reserved":0,"allocated":2120704000,"data_stored":2118668159,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":461239016},"hb_peers":[0,1,2,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:11:31 2023","interfaces":[{"interface":"back","average":{"1min":1.089,"5min":1.0780000000000001,"15min":0.95699999999999996},"min":{"1min":0.183,"5min":0.183,"15min":0.183},"max":{"1min":4.2320000000000002,"5min":6.6500000000000004,"15min":8.5440000000000005},"last":0.24099999999999999},{"interface":"front","average":{"1min":2.6680000000000001,"5min":1.786,"15min":1.3320000000000001},"min":{"1min":0.222,"5min":0.22,"15min":0.20699999999999999},"max":{"1min":7.6840000000000002,"5min":9.1999999999999993,"15min":14.885999999999999},"last":1.5760000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:11:31 2023","interfaces":[{"interface":"back","average":{"1min":3.3300000000000001,"5min":2.2909999999999999,"15min":1.5509999999999999},"min":{"1min":0.28599999999999998,"5min":0.215,"15min":0.215},"max":{"1min":11.555999999999999,"5min":16.056000000000001,"15min":21.931999999999999},"last":1.6850000000000001},{"interface":"front","average":{"1min":3.194,"5min":1.835,"15min":1.29},"min":{"1min":0.27400000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":15.941000000000001,"5min":16.587,"15min":16.587},"last":2.8580000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:11:31 2023","interfaces":[{"interface":"back","average":{"1min":1.899,"5min":1.2430000000000001,"15min":1.0149999999999999},"min":{"1min":0.26600000000000001,"5min":0.247,"15min":0.22600000000000001},"max":{"1min":5.9080000000000004,"5min":6.6760000000000002,"15min":8.2200000000000006},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.3879999999999999,"5min":1.8069999999999999,"15min":1.4530000000000001},"min":{"1min":0.41399999999999998,"5min":0.30499999999999999,"15min":0.25800000000000001},"max":{"1min":8.1500000000000004,"5min":8.1500000000000004,"15min":21.988},"last":1.613}]},{"osd":4,"last update":"Sat Sep 23 18:11:31 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.99,"15min":1.4359999999999999},"min":{"1min":0.30199999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":10.763,"5min":37.231000000000002,"15min":37.231000000000002},"last":1.6559999999999999},{"interface":"front","average":{"1min":3.1549999999999998,"5min":2.0430000000000001,"15min":1.4630000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.30499999999999999},"max":{"1min":15.890000000000001,"5min":37.381,"15min":37.381},"last":2.9020000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.4570000000000001,"15min":1.3069999999999999},"min":{"1min":0.34699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":9.3170000000000002,"5min":12.743,"15min":12.743},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.099,"5min":1.1279999999999999,"15min":1.147},"min":{"1min":0.376,"5min":0.315,"15min":0.315},"max":{"1min":4.601,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.435}]},{"osd":6,"last update":"Sat Sep 23 18:11:31 2023","interfaces":[{"interface":"back","average":{"1min":1.234,"5min":2.359,"15min":1.522},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":4.1639999999999997,"5min":115.54000000000001,"15min":115.54000000000001},"last":0.47799999999999998},{"interface":"front","average":{"1min":1.8140000000000001,"5min":2.649,"15min":1.8640000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":7.6600000000000001,"5min":115.691,"15min":115.691},"last":1.6319999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:11:31 2023","interfaces":[{"interface":"back","average":{"1min":3.1059999999999999,"5min":1.7290000000000001,"15min":1.603},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":15.972,"5min":15.972,"15min":15.972},"last":4.867},{"interface":"front","average":{"1min":1.3100000000000001,"5min":1.4410000000000001,"15min":1.508},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":4.1950000000000003,"5min":6.5279999999999996,"15min":8.6039999999999992},"last":0.50700000000000001}]}]},{"osd":4,"up_from":0,"seq":128849019215,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":5,"up_from":38,"seq":163208757559,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1802008,"kb_used_data":1477840,"kb_used_omap":0,"kb_used_meta":324160,"kb_avail":91943144,"statfs":{"total":95995035648,"available":94149779456,"internally_reserved":0,"allocated":1513308160,"data_stored":1512084138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":331939840},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:08:55 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":0.77900000000000003,"15min":1.069},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.217},"max":{"1min":2.5649999999999999,"5min":5.4539999999999997,"15min":21.376999999999999},"last":1.3560000000000001},{"interface":"front","average":{"1min":0.98899999999999999,"5min":0.94799999999999995,"15min":1.3},"min":{"1min":0.249,"5min":0.249,"15min":0.22900000000000001},"max":{"1min":3.5910000000000002,"5min":8.5099999999999998,"15min":21.388999999999999},"last":3.379}]},{"osd":1,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":0.78400000000000003,"5min":1.038,"15min":0.96699999999999997},"min":{"1min":0.23899999999999999,"5min":0.20799999999999999,"15min":0.20799999999999999},"max":{"1min":2.6349999999999998,"5min":10.547000000000001,"15min":10.853999999999999},"last":1.385},{"interface":"front","average":{"1min":1.0780000000000001,"5min":1.1299999999999999,"15min":1.1759999999999999},"min":{"1min":0.26600000000000001,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":3.625,"5min":8.9749999999999996,"15min":10.206},"last":3.4239999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:08:55 2023","interfaces":[{"interface":"back","average":{"1min":1.1140000000000001,"5min":1.002,"15min":1.4119999999999999},"min":{"1min":0.38600000000000001,"5min":0.27700000000000002,"15min":0.255},"max":{"1min":5.0570000000000004,"5min":5.5309999999999997,"15min":38.606999999999999},"last":1.962},{"interface":"front","average":{"1min":0.97399999999999998,"5min":1.0549999999999999,"15min":1.302},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":2.6000000000000001,"5min":10.065,"15min":39.670999999999999},"last":1.335}]},{"osd":3,"last update":"Sat Sep 23 18:09:28 2023","interfaces":[{"interface":"back","average":{"1min":0.86199999999999999,"5min":1.266,"15min":0.96099999999999997},"min":{"1min":0.32200000000000001,"5min":0.27400000000000002,"15min":0.27400000000000002},"max":{"1min":3.6499999999999999,"5min":9.2309999999999999,"15min":9.2309999999999999},"last":5.194},{"interface":"front","average":{"1min":1.6930000000000001,"5min":1.3779999999999999,"15min":0.94699999999999995},"min":{"1min":0.35599999999999998,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":10.362,"5min":13.446999999999999,"15min":13.446999999999999},"last":1.988}]},{"osd":4,"last update":"Sat Sep 23 18:08:59 2023","interfaces":[{"interface":"back","average":{"1min":1.0289999999999999,"5min":0.79800000000000004,"15min":0.83699999999999997},"min":{"1min":0.29799999999999999,"5min":0.25700000000000001,"15min":0.224},"max":{"1min":3.7149999999999999,"5min":8.6229999999999993,"15min":10.092000000000001},"last":6.8600000000000003},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.94699999999999995,"15min":0.83999999999999997},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.23999999999999999},"max":{"1min":9.1750000000000007,"5min":9.1750000000000007,"15min":9.1750000000000007},"last":2.1890000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:08:59 2023","interfaces":[{"interface":"back","average":{"1min":1.119,"5min":0.86299999999999999,"15min":1.1930000000000001},"min":{"1min":0.314,"5min":0.27900000000000003,"15min":0.27700000000000002},"max":{"1min":6.032,"5min":6.032,"15min":11.287000000000001},"last":1.9279999999999999},{"interface":"front","average":{"1min":0.93200000000000005,"5min":0.872,"15min":1.0880000000000001},"min":{"1min":0.33100000000000002,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":3.6709999999999998,"5min":6.6920000000000002,"15min":8.266},"last":1.3089999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:09:38 2023","interfaces":[{"interface":"back","average":{"1min":1.853,"5min":0.99299999999999999,"15min":1.302},"min":{"1min":0.34200000000000003,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":6.1849999999999996,"5min":8.0999999999999996,"15min":8.8510000000000009},"last":2.7679999999999998},{"interface":"front","average":{"1min":1.752,"5min":1.022,"15min":1.099},"min":{"1min":0.46000000000000002,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":6.2400000000000002,"5min":17.684000000000001,"15min":17.684000000000001},"last":6.6449999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5310,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":682164224,"data_stored":682164224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2637234176,"data_stored":2637234176,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":1725960192,"data_stored":1725956120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2111045632,"data_stored":2111045632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1987248128,"data_stored":1987248128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1551040512,"data_stored":1551040512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2505900032,"data_stored":2505900032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":2406223872,"data_stored":2406219800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:11:51.383 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pgp_num 16 2023-09-23T18:11:51.691 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:11:51.714 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:51.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:51.719 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:51.820 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:51.869 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:52.280 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:52.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:52.285 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:52.291 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:52.386 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:52.451 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:52.459 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:11:52.460 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:52.594 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:52.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:52.601 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:52.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:52.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:52 smithi099 ceph-mon[167076]: pgmap v1002: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 41601/237435 objects degraded (17.521%); 45819/237435 objects misplaced (19.297%); 25 MiB/s, 411 objects/s recovering 2023-09-23T18:11:52.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:52.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:52.758 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:52.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:52.823 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:52 smithi055 ceph-mon[184174]: pgmap v1002: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 41601/237435 objects degraded (17.521%); 45819/237435 objects misplaced (19.297%); 25 MiB/s, 411 objects/s recovering 2023-09-23T18:11:52.823 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:52.823 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:52.824 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:52 smithi055 ceph-mon[179668]: pgmap v1002: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.8 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 702 op/s; 41601/237435 objects degraded (17.521%); 45819/237435 objects misplaced (19.297%); 25 MiB/s, 411 objects/s recovering 2023-09-23T18:11:52.824 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:11:52.824 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:11:52.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:52.829 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:52.929 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:52.984 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:53.295 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:53.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:53.298 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:53.399 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:53.476 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 41601/237435 objects degraded (17.521%), 19 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:53.738 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:53.741 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 41601/237435 objects degraded (17.521%), 19 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:53.742 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:53.742 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:53.744 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 41601/237435 objects degraded (17.521%), 19 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:53.843 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:11:53.898 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:53.948 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:11:53.948 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:11:53.960 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:11:53.965 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:53.965 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:11:54.066 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:54.126 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:54.220 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:54.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:54.225 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:54.325 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:54.362 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:11:54.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:11:54.402 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:54.424 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:11:54.468 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:11:54.495 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:11:54.520 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:11:54.526 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:54.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:54.536 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:54.566 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:11:54.607 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:11:54.638 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:54.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:54 smithi055 ceph-mon[179668]: pgmap v1003: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 746 op/s; 40611/231423 objects degraded (17.548%); 44829/231423 objects misplaced (19.371%); 24 MiB/s, 390 objects/s recovering 2023-09-23T18:11:54.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3554971922' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:54.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:54 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:54.650 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/726424453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:54.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:54 smithi055 ceph-mon[184174]: pgmap v1003: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 746 op/s; 40611/231423 objects degraded (17.548%); 44829/231423 objects misplaced (19.371%); 24 MiB/s, 390 objects/s recovering 2023-09-23T18:11:54.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3554971922' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:54.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:54 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:54.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/726424453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:54.657 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:11:54.658 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:11:54.658 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:11:54.658 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:11:54.658 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:11:54.658 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:11:54.658 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:11:54.658 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:11:54.658 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:11:54.696 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:54.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:54 smithi099 ceph-mon[167076]: pgmap v1003: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 746 op/s; 40611/231423 objects degraded (17.548%); 44829/231423 objects misplaced (19.371%); 24 MiB/s, 390 objects/s recovering 2023-09-23T18:11:54.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3554971922' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:54.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:54 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:11:54.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/726424453' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:11:55.054 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:11:55.152 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:11:55.153 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:11:55.161 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:55.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:55.165 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:55.266 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:11:55.319 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:55.381 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:11:55.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:55.386 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:11:55.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:55.538 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 8 pgp_num to 16 2023-09-23T18:11:55.539 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:55.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:55 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:55.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:55 smithi099 ceph-mon[167076]: failed to encode map e407 with expected crc 2023-09-23T18:11:55.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:55 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:11:55.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:55 smithi099 ceph-mon[167076]: osdmap e407: 8 total, 7 up, 6 in 2023-09-23T18:11:55.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:55 smithi099 ceph-mon[167076]: failed to encode map e407 with expected crc 2023-09-23T18:11:55.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:55 smithi099 ceph-mon[167076]: pgmap v1005: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 39094/228984 objects degraded (17.073%); 44208/228984 objects misplaced (19.306%); 22 MiB/s, 359 objects/s recovering 2023-09-23T18:11:55.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3988090491' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:11:55.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:11:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:55.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:11:55.859 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:11:55.895 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:55.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[184174]: failed to encode map e407 with expected crc 2023-09-23T18:11:55.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:11:55.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[184174]: osdmap e407: 8 total, 7 up, 6 in 2023-09-23T18:11:55.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[184174]: failed to encode map e407 with expected crc 2023-09-23T18:11:55.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[184174]: pgmap v1005: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 39094/228984 objects degraded (17.073%); 44208/228984 objects misplaced (19.306%); 22 MiB/s, 359 objects/s recovering 2023-09-23T18:11:55.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3988090491' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:11:55.896 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:11:55.896 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[179668]: failed to encode map e407 with expected crc 2023-09-23T18:11:55.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:11:55.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[179668]: osdmap e407: 8 total, 7 up, 6 in 2023-09-23T18:11:55.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[179668]: failed to encode map e407 with expected crc 2023-09-23T18:11:55.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[179668]: pgmap v1005: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 788 op/s; 39094/228984 objects degraded (17.073%); 44208/228984 objects misplaced (19.306%); 22 MiB/s, 359 objects/s recovering 2023-09-23T18:11:55.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3988090491' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:11:55.899 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:55.958 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:11:55.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:55.962 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:11:56.063 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:56.118 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:56.179 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:56.184 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:56.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:56.285 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:56.347 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:56.419 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:56.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:56.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:56.503 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:56 smithi055 ceph-mon[179668]: failed to encode map e408 with expected crc 2023-09-23T18:11:56.503 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3988090491' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:11:56.504 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:56 smithi055 ceph-mon[184174]: failed to encode map e408 with expected crc 2023-09-23T18:11:56.504 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3988090491' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:11:56.504 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:56 smithi055 ceph-mon[184174]: osdmap e408: 8 total, 7 up, 6 in 2023-09-23T18:11:56.504 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:56 smithi055 ceph-mon[184174]: failed to encode map e408 with expected crc 2023-09-23T18:11:56.504 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:56 smithi055 ceph-mon[184174]: failed to encode map e408 with expected crc 2023-09-23T18:11:56.524 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:11:56.575 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:56.635 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:11:56.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:56.639 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:11:56.740 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:56.767 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:56 smithi099 ceph-mon[167076]: failed to encode map e408 with expected crc 2023-09-23T18:11:56.768 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3988090491' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:11:56.768 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:56 smithi099 ceph-mon[167076]: osdmap e408: 8 total, 7 up, 6 in 2023-09-23T18:11:56.768 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:56 smithi099 ceph-mon[167076]: failed to encode map e408 with expected crc 2023-09-23T18:11:56.768 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:56 smithi099 ceph-mon[167076]: failed to encode map e408 with expected crc 2023-09-23T18:11:56.772 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:56 smithi055 ceph-mon[179668]: osdmap e408: 8 total, 7 up, 6 in 2023-09-23T18:11:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:56 smithi055 ceph-mon[179668]: failed to encode map e408 with expected crc 2023-09-23T18:11:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:56 smithi055 ceph-mon[179668]: failed to encode map e408 with expected crc 2023-09-23T18:11:57.153 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:11:57.223 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:11:57.793 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:57 smithi099 ceph-mon[167076]: osdmap e409: 8 total, 7 up, 6 in 2023-09-23T18:11:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:57 smithi099 ceph-mon[167076]: failed to encode map e409 with expected crc 2023-09-23T18:11:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:57 smithi099 ceph-mon[167076]: failed to encode map e409 with expected crc 2023-09-23T18:11:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:57 smithi099 ceph-mon[167076]: pgmap v1008: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 676 op/s; 38107/225258 objects degraded (16.917%); 43463/225258 objects misplaced (19.295%); 20 MiB/s, 323 objects/s recovering 2023-09-23T18:11:57.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:57 smithi055 ceph-mon[179668]: osdmap e409: 8 total, 7 up, 6 in 2023-09-23T18:11:57.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:57 smithi055 ceph-mon[179668]: failed to encode map e409 with expected crc 2023-09-23T18:11:57.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:57 smithi055 ceph-mon[179668]: failed to encode map e409 with expected crc 2023-09-23T18:11:57.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:57 smithi055 ceph-mon[179668]: pgmap v1008: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 676 op/s; 38107/225258 objects degraded (16.917%); 43463/225258 objects misplaced (19.295%); 20 MiB/s, 323 objects/s recovering 2023-09-23T18:11:57.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:57 smithi055 ceph-mon[184174]: osdmap e409: 8 total, 7 up, 6 in 2023-09-23T18:11:57.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:57 smithi055 ceph-mon[184174]: failed to encode map e409 with expected crc 2023-09-23T18:11:57.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:57 smithi055 ceph-mon[184174]: failed to encode map e409 with expected crc 2023-09-23T18:11:57.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:57 smithi055 ceph-mon[184174]: pgmap v1008: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 46 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.6 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 676 op/s; 38107/225258 objects degraded (16.917%); 43463/225258 objects misplaced (19.295%); 20 MiB/s, 323 objects/s recovering 2023-09-23T18:11:58.207 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:58.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:58.211 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:58.311 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:58.365 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:58.533 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:58 smithi099 ceph-mon[167076]: failed to encode map e410 with expected crc 2023-09-23T18:11:58.533 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:58 smithi099 ceph-mon[167076]: osdmap e410: 8 total, 7 up, 6 in 2023-09-23T18:11:58.533 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 38107/225258 objects degraded (16.917%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:58.582 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:58.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:58.586 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:58.592 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:11:58.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:58 smithi055 ceph-mon[179668]: failed to encode map e410 with expected crc 2023-09-23T18:11:58.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:58 smithi055 ceph-mon[179668]: osdmap e410: 8 total, 7 up, 6 in 2023-09-23T18:11:58.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 38107/225258 objects degraded (16.917%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:58.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:58 smithi055 ceph-mon[184174]: failed to encode map e410 with expected crc 2023-09-23T18:11:58.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:58 smithi055 ceph-mon[184174]: osdmap e410: 8 total, 7 up, 6 in 2023-09-23T18:11:58.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 38107/225258 objects degraded (16.917%), 19 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:11:58.687 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:58.766 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:58.836 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:58.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:58.858 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:58.959 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:11:58.996 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:59.434 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:11:59.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:59.438 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:11:59.539 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:11:59.595 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:11:59.643 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:11:59 smithi055 ceph-mon[184174]: pgmap v1010: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 36241/218391 objects degraded (16.595%); 42155/218391 objects misplaced (19.303%); 22 MiB/s, 350 objects/s recovering 2023-09-23T18:11:59.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:11:59 smithi055 ceph-mon[179668]: pgmap v1010: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 36241/218391 objects degraded (16.595%); 42155/218391 objects misplaced (19.303%); 22 MiB/s, 350 objects/s recovering 2023-09-23T18:11:59.667 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:11:59.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:11:59.670 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:11:59.770 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:11:59.813 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:11:59 smithi099 ceph-mon[167076]: pgmap v1010: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 790 op/s; 36241/218391 objects degraded (16.595%); 42155/218391 objects misplaced (19.303%); 22 MiB/s, 350 objects/s recovering 2023-09-23T18:11:59.817 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:00.210 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:12:00.241 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:12:00.274 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:12:00.313 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:12:00.351 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:12:00.364 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:00.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:00.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:00.397 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:12:00.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:12:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:12:00] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-09-23T18:12:00.445 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:12:00.445 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:12:00.445 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:12:00.445 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:12:00.445 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:12:00.445 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:12:00.445 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:12:00.446 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:12:00.446 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:12:00.470 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:00.554 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1698242956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:00.629 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:00.634 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:12:00.634 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:12:00.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:00.647 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:00.748 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:00.788 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1698242956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:00.788 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1698242956' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:00.788 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:00.843 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:12:00.843 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:12:00.960 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 3] out_osds: [2, 4] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:12:00.960 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:12:00.960 INFO:tasks.thrashosds.thrasher:Adding osd 4 2023-09-23T18:12:00.961 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 4 2023-09-23T18:12:01.013 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:01.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:12:01.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:01.017 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:01.118 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:01.154 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:01.210 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:01.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:01.216 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:01.317 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:01.367 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:01.430 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:01.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:01.436 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:01.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:01.595 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:01.601 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:12:01.635 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:01 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2380660047' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:12:01.635 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:01 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:12:01.635 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:01 smithi055 ceph-mon[184174]: pgmap v1011: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 35421/215277 objects degraded (16.454%); 41552/215277 objects misplaced (19.302%); 20 MiB/s, 326 objects/s recovering 2023-09-23T18:12:01.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:01 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2380660047' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:12:01.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:01 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:12:01.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:01 smithi055 ceph-mon[179668]: pgmap v1011: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 35421/215277 objects degraded (16.454%); 41552/215277 objects misplaced (19.302%); 20 MiB/s, 326 objects/s recovering 2023-09-23T18:12:01.642 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:01.644 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:01.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:01.648 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:01.693 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:12:01.749 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:01.809 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:01.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:01 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2380660047' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:12:01.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:01 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:12:01.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:01 smithi099 ceph-mon[167076]: pgmap v1011: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.4 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 761 op/s; 35421/215277 objects degraded (16.454%); 41552/215277 objects misplaced (19.302%); 20 MiB/s, 326 objects/s recovering 2023-09-23T18:12:02.022 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:02.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:02.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:02.128 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:02.194 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:02.246 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:02.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:02.255 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:02.355 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:02.420 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:02.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:02.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:02.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:02.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:02 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:02.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:02 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:12:02.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:02 smithi099 ceph-mon[167076]: osdmap e411: 8 total, 7 up, 6 in 2023-09-23T18:12:02.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:02 smithi099 ceph-mon[167076]: failed to encode map e411 with expected crc 2023-09-23T18:12:02.569 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:02 smithi099 ceph-mon[167076]: failed to encode map e411 with expected crc 2023-09-23T18:12:02.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:02.631 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:02.735 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:02.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:02.738 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:02.839 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:02.844 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:12:02.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:02 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:02.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:02 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:12:02.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:02 smithi055 ceph-mon[179668]: osdmap e411: 8 total, 7 up, 6 in 2023-09-23T18:12:02.901 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:02 smithi055 ceph-mon[179668]: failed to encode map e411 with expected crc 2023-09-23T18:12:02.901 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:02 smithi055 ceph-mon[179668]: failed to encode map e411 with expected crc 2023-09-23T18:12:02.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:02 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:02.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:02 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:12:02.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:02 smithi055 ceph-mon[184174]: osdmap e411: 8 total, 7 up, 6 in 2023-09-23T18:12:02.902 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:02 smithi055 ceph-mon[184174]: failed to encode map e411 with expected crc 2023-09-23T18:12:02.902 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:02 smithi055 ceph-mon[184174]: failed to encode map e411 with expected crc 2023-09-23T18:12:02.905 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:03.028 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:03.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:03.041 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:03.065 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:12:03.066 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:12:03.142 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:03.187 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:03.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:03.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:03.400 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:03.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:03.505 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:12:03.545 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:03.615 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:03.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:03.620 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:03.721 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:03.766 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:03 smithi099 ceph-mon[167076]: pgmap v1013: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 34179/209160 objects degraded (16.341%); 40534/209160 objects misplaced (19.379%); 19 MiB/s, 299 objects/s recovering 2023-09-23T18:12:03.767 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 35421/215277 objects degraded (16.454%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:03.768 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:03.817 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:03.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:03.824 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:03.925 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:03.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:03 smithi055 ceph-mon[179668]: pgmap v1013: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 34179/209160 objects degraded (16.341%); 40534/209160 objects misplaced (19.379%); 19 MiB/s, 299 objects/s recovering 2023-09-23T18:12:03.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 35421/215277 objects degraded (16.454%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:03.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:03 smithi055 ceph-mon[184174]: pgmap v1013: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.3 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 835 op/s; 34179/209160 objects degraded (16.341%); 40534/209160 objects misplaced (19.379%); 19 MiB/s, 299 objects/s recovering 2023-09-23T18:12:03.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 35421/215277 objects degraded (16.454%), 18 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:04.004 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:04.063 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:12:04.074 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:04.077 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:04.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:04.080 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:04.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:04.225 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:04.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:04.275 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:04 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:12:04.269+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e411 definitely_dead 0 2023-09-23T18:12:04.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:04.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:04.300 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:12:04.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:04.408 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:04.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:04.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:04.457 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:04.558 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:04.588 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:04 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2386303297' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T18:12:04.648 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:04.655 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.4. 2023-09-23T18:12:04.695 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:04.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:04.699 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:04.799 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:04.824 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:04 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2386303297' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T18:12:04.827 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:04 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2386303297' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["4"]}]: dispatch 2023-09-23T18:12:05.109 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:05.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:05.111 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:05.202 INFO:tasks.thrashosds.thrasher:Added osd 4 2023-09-23T18:12:05.213 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:05.246 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:05.328 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:05.330 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:05.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:05.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:05.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:05.462 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:05.497 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:05.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:05.500 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:05.600 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:05.614 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[184174]: failed to encode map e412 with expected crc 2023-09-23T18:12:05.614 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2386303297' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T18:12:05.614 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[184174]: osdmap e412: 8 total, 7 up, 7 in 2023-09-23T18:12:05.614 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[184174]: failed to encode map e412 with expected crc 2023-09-23T18:12:05.614 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[184174]: failed to encode map e412 with expected crc 2023-09-23T18:12:05.615 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[184174]: pgmap v1015: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 796 op/s; 33934/206913 objects degraded (16.400%); 39975/206913 objects misplaced (19.320%); 14 MiB/s, 226 objects/s recovering 2023-09-23T18:12:05.628 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:12:05.628 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:12:05.646 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:05.698 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:12:05.720 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:12:05.720 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:05 smithi099 ceph-mon[167076]: failed to encode map e412 with expected crc 2023-09-23T18:12:05.721 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2386303297' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T18:12:05.721 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:05 smithi099 ceph-mon[167076]: osdmap e412: 8 total, 7 up, 7 in 2023-09-23T18:12:05.721 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:05 smithi099 ceph-mon[167076]: failed to encode map e412 with expected crc 2023-09-23T18:12:05.721 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:05 smithi099 ceph-mon[167076]: failed to encode map e412 with expected crc 2023-09-23T18:12:05.721 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:05 smithi099 ceph-mon[167076]: pgmap v1015: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 796 op/s; 33934/206913 objects degraded (16.400%); 39975/206913 objects misplaced (19.320%); 14 MiB/s, 226 objects/s recovering 2023-09-23T18:12:05.741 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:12:05.766 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:12:05.786 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:12:05.805 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:12:05.825 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:12:05.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:12:05.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:12:05.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:12:05.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:12:05.825 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:12:05.825 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:12:05.826 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:12:05.826 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:12:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[179668]: failed to encode map e412 with expected crc 2023-09-23T18:12:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2386303297' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["4"]}]': finished 2023-09-23T18:12:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[179668]: osdmap e412: 8 total, 7 up, 7 in 2023-09-23T18:12:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[179668]: failed to encode map e412 with expected crc 2023-09-23T18:12:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[179668]: failed to encode map e412 with expected crc 2023-09-23T18:12:05.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:05 smithi055 ceph-mon[179668]: pgmap v1015: 119 pgs: 1 active+recovery_wait+remapped, 3 active+recovery_wait+degraded, 47 active+clean+remapped, 3 active+remapped+backfill_wait, 7 active+undersized+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 5 active+recovery_wait+degraded+remapped, 35 active+clean; 4.2 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 796 op/s; 33934/206913 objects degraded (16.400%); 39975/206913 objects misplaced (19.320%); 14 MiB/s, 226 objects/s recovering 2023-09-23T18:12:05.961 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:12:06.302 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:06.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:06.305 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:06.406 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:06.438 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:06.560 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:12:06.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[179668]: failed to encode map e413 with expected crc 2023-09-23T18:12:06.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[179668]: failed to encode map e413 with expected crc 2023-09-23T18:12:06.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[179668]: osdmap e413: 8 total, 7 up, 7 in 2023-09-23T18:12:06.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[179668]: failed to encode map e413 with expected crc 2023-09-23T18:12:06.777 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:06.780 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:06.781 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:06.882 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[184174]: failed to encode map e413 with expected crc 2023-09-23T18:12:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[184174]: failed to encode map e413 with expected crc 2023-09-23T18:12:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[184174]: osdmap e413: 8 total, 7 up, 7 in 2023-09-23T18:12:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[184174]: failed to encode map e413 with expected crc 2023-09-23T18:12:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1160053978' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:06.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1160053978' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:06.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:06.964 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:06.970 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:06 smithi099 ceph-mon[167076]: failed to encode map e413 with expected crc 2023-09-23T18:12:06.970 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:06 smithi099 ceph-mon[167076]: failed to encode map e413 with expected crc 2023-09-23T18:12:06.970 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:06 smithi099 ceph-mon[167076]: osdmap e413: 8 total, 7 up, 7 in 2023-09-23T18:12:06.970 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:06 smithi099 ceph-mon[167076]: failed to encode map e413 with expected crc 2023-09-23T18:12:06.970 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:06 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1160053978' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:06.970 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:07.207 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:07.218 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:07.218 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:07.319 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:07.388 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:07.892 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:12:07.892 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:12:07.892 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:12:07.892 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:12:07.892 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:12:07.927 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:07 smithi099 ceph-mon[167076]: failed to encode map e414 with expected crc 2023-09-23T18:12:07.927 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:07 smithi099 ceph-mon[167076]: osdmap e414: 8 total, 7 up, 7 in 2023-09-23T18:12:07.927 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:07 smithi099 ceph-mon[167076]: failed to encode map e414 with expected crc 2023-09-23T18:12:07.927 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:07 smithi099 ceph-mon[167076]: 8.e continuing backfill to osd.1 from (379'5309,411'9644] 8:7b7c7e01:::benchmark_data_smithi055_346788_object57885:head to 411'9644 2023-09-23T18:12:07.927 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:07 smithi099 ceph-mon[167076]: pgmap v1018: 119 pgs: 12 peering, 14 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 26 active+clean+remapped, 7 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 39 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 27988/203556 objects degraded (13.750%); 17108/203556 objects misplaced (8.405%); 10 MiB/s, 166 objects/s recovering 2023-09-23T18:12:07.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:07 smithi055 ceph-mon[179668]: failed to encode map e414 with expected crc 2023-09-23T18:12:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:07 smithi055 ceph-mon[179668]: osdmap e414: 8 total, 7 up, 7 in 2023-09-23T18:12:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:07 smithi055 ceph-mon[179668]: failed to encode map e414 with expected crc 2023-09-23T18:12:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:07 smithi055 ceph-mon[179668]: 8.e continuing backfill to osd.1 from (379'5309,411'9644] 8:7b7c7e01:::benchmark_data_smithi055_346788_object57885:head to 411'9644 2023-09-23T18:12:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:07 smithi055 ceph-mon[179668]: pgmap v1018: 119 pgs: 12 peering, 14 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 26 active+clean+remapped, 7 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 39 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 27988/203556 objects degraded (13.750%); 17108/203556 objects misplaced (8.405%); 10 MiB/s, 166 objects/s recovering 2023-09-23T18:12:07.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:07 smithi055 ceph-mon[184174]: failed to encode map e414 with expected crc 2023-09-23T18:12:07.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:07 smithi055 ceph-mon[184174]: osdmap e414: 8 total, 7 up, 7 in 2023-09-23T18:12:07.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:07 smithi055 ceph-mon[184174]: failed to encode map e414 with expected crc 2023-09-23T18:12:07.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:07 smithi055 ceph-mon[184174]: 8.e continuing backfill to osd.1 from (379'5309,411'9644] 8:7b7c7e01:::benchmark_data_smithi055_346788_object57885:head to 411'9644 2023-09-23T18:12:07.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:07 smithi055 ceph-mon[184174]: pgmap v1018: 119 pgs: 12 peering, 14 remapped+peering, 1 active+recovery_wait+remapped, 2 active+recovery_wait+degraded, 26 active+clean+remapped, 7 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 1 active+recovery_wait+degraded+remapped, 39 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 727 op/s; 27988/203556 objects degraded (13.750%); 17108/203556 objects misplaced (8.405%); 10 MiB/s, 166 objects/s recovering 2023-09-23T18:12:07.964 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:07.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:07.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:08.073 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:08.129 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:08.367 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:12:08.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:08 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-09-23T18:12:08.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:08 smithi055 ceph-mon[179668]: failed to encode map e415 with expected crc 2023-09-23T18:12:08.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:08 smithi055 ceph-mon[179668]: osdmap e415: 8 total, 7 up, 7 in 2023-09-23T18:12:08.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:08 smithi055 ceph-mon[179668]: failed to encode map e415 with expected crc 2023-09-23T18:12:08.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 27988/203556 objects degraded (13.750%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:08.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:08 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-09-23T18:12:08.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:08 smithi055 ceph-mon[184174]: failed to encode map e415 with expected crc 2023-09-23T18:12:08.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:08 smithi055 ceph-mon[184174]: osdmap e415: 8 total, 7 up, 7 in 2023-09-23T18:12:08.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:08 smithi055 ceph-mon[184174]: failed to encode map e415 with expected crc 2023-09-23T18:12:08.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 27988/203556 objects degraded (13.750%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:09.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:08 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 5 pgs inactive, 7 pgs peering (PG_AVAILABILITY) 2023-09-23T18:12:09.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:08 smithi099 ceph-mon[167076]: failed to encode map e415 with expected crc 2023-09-23T18:12:09.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:08 smithi099 ceph-mon[167076]: osdmap e415: 8 total, 7 up, 7 in 2023-09-23T18:12:09.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:08 smithi099 ceph-mon[167076]: failed to encode map e415 with expected crc 2023-09-23T18:12:09.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 27988/203556 objects degraded (13.750%), 12 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:09.149 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:12:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[184174]: osdmap e416: 8 total, 7 up, 7 in 2023-09-23T18:12:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[184174]: failed to encode map e416 with expected crc 2023-09-23T18:12:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[184174]: failed to encode map e416 with expected crc 2023-09-23T18:12:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[184174]: failed to encode map e416 with expected crc 2023-09-23T18:12:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[184174]: failed to encode map e416 with expected crc 2023-09-23T18:12:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[184174]: pgmap v1021: 119 pgs: 14 peering, 14 remapped+peering, 4 active+recovery_wait+degraded, 17 active+clean+remapped, 7 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 45 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 27046/199185 objects degraded (13.578%); 12412/199185 objects misplaced (6.231%); 14 MiB/s, 220 objects/s recovering 2023-09-23T18:12:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[179668]: osdmap e416: 8 total, 7 up, 7 in 2023-09-23T18:12:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[179668]: failed to encode map e416 with expected crc 2023-09-23T18:12:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[179668]: failed to encode map e416 with expected crc 2023-09-23T18:12:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[179668]: failed to encode map e416 with expected crc 2023-09-23T18:12:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[179668]: failed to encode map e416 with expected crc 2023-09-23T18:12:09.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:09 smithi055 ceph-mon[179668]: pgmap v1021: 119 pgs: 14 peering, 14 remapped+peering, 4 active+recovery_wait+degraded, 17 active+clean+remapped, 7 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 45 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 27046/199185 objects degraded (13.578%); 12412/199185 objects misplaced (6.231%); 14 MiB/s, 220 objects/s recovering 2023-09-23T18:12:10.067 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:09 smithi099 ceph-mon[167076]: osdmap e416: 8 total, 7 up, 7 in 2023-09-23T18:12:10.067 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:09 smithi099 ceph-mon[167076]: failed to encode map e416 with expected crc 2023-09-23T18:12:10.067 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:09 smithi099 ceph-mon[167076]: failed to encode map e416 with expected crc 2023-09-23T18:12:10.067 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:09 smithi099 ceph-mon[167076]: failed to encode map e416 with expected crc 2023-09-23T18:12:10.068 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:09 smithi099 ceph-mon[167076]: failed to encode map e416 with expected crc 2023-09-23T18:12:10.068 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:09 smithi099 ceph-mon[167076]: pgmap v1021: 119 pgs: 14 peering, 14 remapped+peering, 4 active+recovery_wait+degraded, 17 active+clean+remapped, 7 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 45 active+clean; 4.1 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 642 op/s; 27046/199185 objects degraded (13.578%); 12412/199185 objects misplaced (6.231%); 14 MiB/s, 220 objects/s recovering 2023-09-23T18:12:10.123 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:10.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:10.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:10.203 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 3, 4] out_osds: [2] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:12:10.203 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:12:10.204 INFO:tasks.thrashosds.thrasher:inject_pause on osd.0 2023-09-23T18:12:10.204 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:12:10.204 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:12:10.204 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 config set filestore_inject_stall 3 2023-09-23T18:12:10.212 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:12:10.214 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:12:10.229 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:10.317 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:10.385 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:10.390 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:12:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:12:10] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.43.0" 2023-09-23T18:12:10.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:10.392 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:10.493 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:10.543 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:10.575 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:12:10.575 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:12:10.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:12:10.576 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:12:10.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.576 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136078 8.e 8:7d936bc7:::benchmark_data_smithi055_346788_object25486:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e402)", 2023-09-23T18:12:10.577 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:39.486326+0000", 2023-09-23T18:12:10.577 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.086767418000001, 2023-09-23T18:12:10.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.138509884, 2023-09-23T18:12:10.577 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.577 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.578 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.578 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.578 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.578 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136078 2023-09-23T18:12:10.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.579 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.486326+0000", 2023-09-23T18:12:10.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.580 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.486326+0000", 2023-09-23T18:12:10.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.486325+0000", 2023-09-23T18:12:10.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:12:10.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.582 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.582 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.582 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.486328+0000", 2023-09-23T18:12:10.582 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9380000000000001e-06 2023-09-23T18:12:10.582 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.583 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.583 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.583 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.486334+0000", 2023-09-23T18:12:10.583 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.959e-06 2023-09-23T18:12:10.583 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.486338+0000", 2023-09-23T18:12:10.584 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9999999999999998e-06 2023-09-23T18:12:10.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.490417+0000", 2023-09-23T18:12:10.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040786549999999996 2023-09-23T18:12:10.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.490574+0000", 2023-09-23T18:12:10.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015656000000000001 2023-09-23T18:12:10.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.586 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,2,3,4", 2023-09-23T18:12:10.586 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.490729+0000", 2023-09-23T18:12:10.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000155735 2023-09-23T18:12:10.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.586 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.586 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.494201+0000", 2023-09-23T18:12:10.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034715359999999999 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.495263+0000", 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010627080000000001 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.495297+0000", 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3791000000000002e-05 2023-09-23T18:12:10.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.511651+0000", 2023-09-23T18:12:10.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016353416999999999 2023-09-23T18:12:10.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.624669+0000", 2023-09-23T18:12:10.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.11301865699999999 2023-09-23T18:12:10.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.624729+0000", 2023-09-23T18:12:10.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9740000000000001e-05 2023-09-23T18:12:10.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.624836+0000", 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106587 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136086 8.e 8:71053626:::benchmark_data_smithi055_346788_object25494:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:39.508870+0000", 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.064222751999999, 2023-09-23T18:12:10.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.116290637, 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136086 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.592 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.508870+0000", 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.508870+0000", 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.508869+0000", 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:10.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.508876+0000", 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0550000000000002e-06 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.508894+0000", 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8348000000000001e-05 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.508901+0000", 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2940000000000002e-06 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.511669+0000", 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027684989999999998 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.511772+0000", 2023-09-23T18:12:10.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102608 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,2,3,4", 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.511880+0000", 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108603 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.515201+0000", 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033205750000000001 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.516446+0000", 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012449189999999999 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.516895+0000", 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00044888399999999998 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.519682+0000", 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027871269999999999 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.624895+0000", 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.105213007 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.624943+0000", 2023-09-23T18:12:10.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8260000000000002e-05 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.625161+0000", 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021796399999999999 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136096 8.e 8:7149357d:::benchmark_data_smithi055_346788_object25504:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e403)", 2023-09-23T18:12:10.599 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:39.523885+0000", 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 31.04920821, 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.10172041599999999, 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136096 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.523885+0000", 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.523885+0000", 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.523885+0000", 2023-09-23T18:12:10.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:12:10.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.602 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.602 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.523888+0000", 2023-09-23T18:12:10.602 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4630000000000001e-06 2023-09-23T18:12:10.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.602 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.523895+0000", 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6590000000000001e-06 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.523899+0000", 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3919999999999996e-06 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.526750+0000", 2023-09-23T18:12:10.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002850787 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.526885+0000", 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000135499 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,2,3,4", 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.527048+0000", 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000162619 2023-09-23T18:12:10.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.530682+0000", 2023-09-23T18:12:10.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0036340700000000001 2023-09-23T18:12:10.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.531086+0000", 2023-09-23T18:12:10.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00040381899999999998 2023-09-23T18:12:10.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.534143+0000", 2023-09-23T18:12:10.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030567820000000001 2023-09-23T18:12:10.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.534409+0000", 2023-09-23T18:12:10.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00026623100000000002 2023-09-23T18:12:10.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.625419+0000", 2023-09-23T18:12:10.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.091010488 2023-09-23T18:12:10.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.625467+0000", 2023-09-23T18:12:10.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7330999999999998e-05 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:39.625605+0000", 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013870399999999999 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.608 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148346 8.d 8:b553d8e7:::benchmark_data_smithi055_346788_object37754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T18:12:10.609 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.520341+0000", 2023-09-23T18:12:10.609 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 14.052752564, 2023-09-23T18:12:10.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1563575239999999, 2023-09-23T18:12:10.609 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.609 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:12:10.609 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.609 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.609 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.609 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148346 2023-09-23T18:12:10.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.610 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520341+0000", 2023-09-23T18:12:10.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520341+0000", 2023-09-23T18:12:10.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520340+0000", 2023-09-23T18:12:10.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:12:10.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520343+0000", 2023-09-23T18:12:10.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2260000000000001e-06 2023-09-23T18:12:10.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520350+0000", 2023-09-23T18:12:10.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2009999999999996e-06 2023-09-23T18:12:10.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520353+0000", 2023-09-23T18:12:10.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8229999999999997e-06 2023-09-23T18:12:10.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520401+0000", 2023-09-23T18:12:10.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7818999999999997e-05 2023-09-23T18:12:10.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520549+0000", 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014766099999999999 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520711+0000", 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016159099999999999 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.523302+0000", 2023-09-23T18:12:10.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002591051 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.529040+0000", 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057384949999999997 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.529049+0000", 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0629999999999995e-06 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.654884+0000", 2023-09-23T18:12:10.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1258352890000001 2023-09-23T18:12:10.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:12:10.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.654994+0000", 2023-09-23T18:12:10.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000109758 2023-09-23T18:12:10.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.617 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676645+0000", 2023-09-23T18:12:10.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021651083000000002 2023-09-23T18:12:10.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.617 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676698+0000", 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2895000000000001e-05 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148356 8.d 8:b231d41c:::benchmark_data_smithi055_346788_object37764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.534689+0000", 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 14.038404197, 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1416983039999999, 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.618 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148356 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534689+0000", 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534689+0000", 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534688+0000", 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534694+0000", 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7320000000000001e-06 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534710+0000", 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5917e-05 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534717+0000", 2023-09-23T18:12:10.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7209999999999997e-06 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534759+0000", 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1794e-05 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534769+0000", 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9150000000000008e-06 2023-09-23T18:12:10.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.655033+0000", 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1202644310000001 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.655166+0000", 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013317400000000001 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.655326+0000", 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016015199999999999 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.661897+0000", 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065707750000000001 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676191+0000", 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014294118999999999 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676291+0000", 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9346999999999996e-05 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676328+0000", 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7681000000000001e-05 2023-09-23T18:12:10.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676387+0000", 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8913999999999999e-05 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148375 8.d 8:b69cd647:::benchmark_data_smithi055_346788_object37783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.552544+0000", 2023-09-23T18:12:10.626 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 14.020549690999999, 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1243204470000001, 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148375 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552544+0000", 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552544+0000", 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552543+0000", 2023-09-23T18:12:10.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552546+0000", 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8770000000000002e-06 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552580+0000", 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4294000000000001e-05 2023-09-23T18:12:10.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552591+0000", 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0567999999999999e-05 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.558426+0000", 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058347260000000001 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.558435+0000", 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2580000000000001e-06 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.657112+0000", 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.098676784 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.657237+0000", 2023-09-23T18:12:10.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012515399999999999 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.657353+0000", 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011598 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.661903+0000", 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045504259999999998 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676235+0000", 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014331468999999999 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676770+0000", 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00053503199999999996 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676803+0000", 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3618000000000002e-05 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676864+0000", 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0745000000000002e-05 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148377 8.d 8:b286308b:::benchmark_data_smithi055_346788_object37785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.553953+0000", 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 14.019140317, 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1230239500000001, 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148377 2023-09-23T18:12:10.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553953+0000", 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553953+0000", 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553952+0000", 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553957+0000", 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6260000000000003e-06 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553968+0000", 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1654999999999999e-05 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553975+0000", 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3450000000000002e-06 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.558455+0000", 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044808180000000001 2023-09-23T18:12:10.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.558460+0000", 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.42e-06 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.658566+0000", 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.100105881 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.658686+0000", 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119815 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.658819+0000", 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000133802 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.664276+0000", 2023-09-23T18:12:10.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005456982 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676258+0000", 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011981656 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676896+0000", 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063824599999999997 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676923+0000", 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6713999999999999e-05 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676977+0000", 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3909000000000002e-05 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.642 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148424 8.d 8:b27653e5:::benchmark_data_smithi055_346788_object37832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.634454+0000", 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.938639543000001, 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0427430849999999, 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.643 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148424 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634454+0000", 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634454+0000", 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634453+0000", 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634458+0000", 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3870000000000003e-06 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634469+0000", 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0871e-05 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634473+0000", 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5480000000000003e-06 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634508+0000", 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4168000000000002e-05 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634514+0000", 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0859999999999998e-06 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.660281+0000", 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0257674050000001 2023-09-23T18:12:10.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.660408+0000", 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127327 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.660494+0000", 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5332000000000002e-05 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.666754+0000", 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062598719999999997 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676422+0000", 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096688 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677081+0000", 2023-09-23T18:12:10.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00065843900000000003 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677134+0000", 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2908000000000002e-05 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677197+0000", 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2958000000000003e-05 2023-09-23T18:12:10.650 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148435 8.d 8:b5632819:::benchmark_data_smithi055_346788_object37843:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.644607+0000", 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.928485846999999, 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.032802521, 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.651 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148435 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644607+0000", 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644607+0000", 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644606+0000", 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644611+0000", 2023-09-23T18:12:10.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9599999999999999e-06 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644622+0000", 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0441e-05 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644626+0000", 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.622e-06 2023-09-23T18:12:10.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644665+0000", 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.837e-05 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644671+0000", 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8650000000000003e-06 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.661941+0000", 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.017269996 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.662026+0000", 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4456000000000006e-05 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.662106+0000", 2023-09-23T18:12:10.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0147000000000004e-05 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.666763+0000", 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046564930000000003 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676452+0000", 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096896880000000001 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677241+0000", 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00078881700000000003 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677297+0000", 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6048999999999998e-05 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677410+0000", 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112773 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148439 8.d 8:be04b9c4:::benchmark_data_smithi055_346788_object37847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.648385+0000", 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.924708581999999, 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0292518509999999, 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.659 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148439 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.648385+0000", 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.648385+0000", 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.648384+0000", 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.648388+0000", 2023-09-23T18:12:10.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1829999999999996e-06 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.648399+0000", 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0861e-05 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.648403+0000", 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3359999999999997e-06 2023-09-23T18:12:10.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.648446+0000", 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2477e-05 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.648451+0000", 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2020000000000003e-06 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.663122+0000", 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.014671248 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.663193+0000", 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0971999999999997e-05 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.663290+0000", 2023-09-23T18:12:10.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6323999999999995e-05 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.669180+0000", 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005890813 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676718+0000", 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075378119999999996 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677470+0000", 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00075180899999999998 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677522+0000", 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2369e-05 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677636+0000", 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000114023 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148444 8.d 8:b850c954:::benchmark_data_smithi055_346788_object37852:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.654979+0000", 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.918114035, 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.022879444, 2023-09-23T18:12:10.667 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148444 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.654979+0000", 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.654979+0000", 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.654978+0000", 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.654984+0000", 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3969999999999999e-06 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.654998+0000", 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4214e-05 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.670 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.655004+0000", 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5759999999999996e-06 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.655045+0000", 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1207e-05 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.655052+0000", 2023-09-23T18:12:10.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7100000000000001e-06 2023-09-23T18:12:10.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.664319+0000", 2023-09-23T18:12:10.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0092673000000001 2023-09-23T18:12:10.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.664448+0000", 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012875699999999999 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.664548+0000", 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100097 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.669191+0000", 2023-09-23T18:12:10.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046429569999999996 2023-09-23T18:12:10.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.674 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.674 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676746+0000", 2023-09-23T18:12:10.674 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075545229999999996 2023-09-23T18:12:10.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.674 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.674 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677694+0000", 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00094813099999999995 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677746+0000", 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2142999999999999e-05 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677859+0000", 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112732 2023-09-23T18:12:10.675 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.676 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.676 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.676 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.676 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.676 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148476 8.d 8:b2863299:::benchmark_data_smithi055_346788_object37884:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.676 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.707033+0000", 2023-09-23T18:12:10.676 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.866060327, 2023-09-23T18:12:10.676 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97116398000000004, 2023-09-23T18:12:10.676 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148476 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.707033+0000", 2023-09-23T18:12:10.677 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.678 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.678 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.678 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.707033+0000", 2023-09-23T18:12:10.678 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.678 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.678 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.678 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.707032+0000", 2023-09-23T18:12:10.678 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:10.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.707038+0000", 2023-09-23T18:12:10.679 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2009999999999996e-06 2023-09-23T18:12:10.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.707050+0000", 2023-09-23T18:12:10.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2602999999999999e-05 2023-09-23T18:12:10.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.680 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.707055+0000", 2023-09-23T18:12:10.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9439999999999996e-06 2023-09-23T18:12:10.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.680 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.707097+0000", 2023-09-23T18:12:10.681 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1564999999999999e-05 2023-09-23T18:12:10.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.681 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.681 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.707103+0000", 2023-09-23T18:12:10.681 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3289999999999999e-06 2023-09-23T18:12:10.681 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.681 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.681 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.665641+0000", 2023-09-23T18:12:10.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95853753799999997 2023-09-23T18:12:10.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.682 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.665718+0000", 2023-09-23T18:12:10.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7058000000000007e-05 2023-09-23T18:12:10.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.683 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.665799+0000", 2023-09-23T18:12:10.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0801000000000001e-05 2023-09-23T18:12:10.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.683 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.683 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.673523+0000", 2023-09-23T18:12:10.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007723911 2023-09-23T18:12:10.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.684 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.684 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677016+0000", 2023-09-23T18:12:10.684 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034939239999999998 2023-09-23T18:12:10.684 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.684 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.684 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.684 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678031+0000", 2023-09-23T18:12:10.684 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010149989999999999 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678087+0000", 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5343999999999997e-05 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678197+0000", 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110067 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.685 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148488 8.d 8:b27a7263:::benchmark_data_smithi055_346788_object37896:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.721362+0000", 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.851731269, 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95704133599999996, 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.686 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148488 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.721362+0000", 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.687 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.721362+0000", 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.721361+0000", 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.721371+0000", 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3389999999999999e-06 2023-09-23T18:12:10.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.721378+0000", 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8679999999999999e-06 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.721383+0000", 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9640000000000002e-06 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.689 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.721422+0000", 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8139999999999997e-05 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.721428+0000", 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9630000000000003e-06 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.666788+0000", 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94536060099999997 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.666870+0000", 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2106000000000003e-05 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.666978+0000", 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000107836 2023-09-23T18:12:10.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.673528+0000", 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065496080000000002 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677918+0000", 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004390026 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.692 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678267+0000", 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00034922600000000001 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678311+0000", 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3600000000000003e-05 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.693 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678403+0000", 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2715999999999998e-05 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148495 8.d 8:b93f6d9f:::benchmark_data_smithi055_346788_object37903:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.728680+0000", 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.844413037000001, 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94992516699999996, 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.694 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148495 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.728680+0000", 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.728680+0000", 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.728679+0000", 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.696 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.728686+0000", 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3899999999999998e-06 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.728699+0000", 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3152999999999999e-05 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.728704+0000", 2023-09-23T18:12:10.697 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9660000000000004e-06 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.728748+0000", 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3943999999999998e-05 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.728753+0000", 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4609999999999999e-06 2023-09-23T18:12:10.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.667972+0000", 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.93921920800000003 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.668046+0000", 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3739999999999995e-05 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.699 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.668140+0000", 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4190000000000005e-05 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.675933+0000", 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0077932100000000001 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677964+0000", 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002030291 2023-09-23T18:12:10.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678455+0000", 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00049118000000000002 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678503+0000", 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8634000000000002e-05 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.701 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678605+0000", 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000101854 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148511 8.d 8:b3f930af:::benchmark_data_smithi055_346788_object37919:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.745830+0000", 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.827263483999999, 2023-09-23T18:12:10.702 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.93305965000000002, 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148511 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.703 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.745830+0000", 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.745830+0000", 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.745829+0000", 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:12:10.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.745832+0000", 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3189999999999999e-06 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.745839+0000", 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6819999999999997e-06 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.745843+0000", 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8099999999999999e-06 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.745898+0000", 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5155000000000003e-05 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.745908+0000", 2023-09-23T18:12:10.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0225e-05 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.669238+0000", 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.92332963400000001 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.669354+0000", 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116394 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.707 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.669479+0000", 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012446700000000001 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.675939+0000", 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064598149999999998 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.708 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.709 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678239+0000", 2023-09-23T18:12:10.709 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023003170000000001 2023-09-23T18:12:10.709 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.709 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.709 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.709 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678746+0000", 2023-09-23T18:12:10.709 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00050742300000000001 2023-09-23T18:12:10.709 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.709 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678793+0000", 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6533000000000003e-05 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678889+0000", 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6477000000000005e-05 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.710 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.711 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.711 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.711 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148537 8.d 8:b6ea1f80:::benchmark_data_smithi055_346788_object37945:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.711 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.788641+0000", 2023-09-23T18:12:10.711 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.784452246000001, 2023-09-23T18:12:10.711 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.89045098899999997, 2023-09-23T18:12:10.711 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.711 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.711 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.711 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.712 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.712 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148537 2023-09-23T18:12:10.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.712 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.712 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.712 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.712 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.788641+0000", 2023-09-23T18:12:10.712 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.712 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.713 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.713 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.788641+0000", 2023-09-23T18:12:10.713 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.713 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.713 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.713 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.713 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.788639+0000", 2023-09-23T18:12:10.713 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:10.713 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.713 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.714 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.714 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.788648+0000", 2023-09-23T18:12:10.714 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1349999999999992e-06 2023-09-23T18:12:10.714 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.714 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.714 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.714 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.788665+0000", 2023-09-23T18:12:10.714 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7280999999999999e-05 2023-09-23T18:12:10.714 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.714 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.715 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.715 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.788672+0000", 2023-09-23T18:12:10.715 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2189999999999998e-06 2023-09-23T18:12:10.715 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.715 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.715 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.715 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.788730+0000", 2023-09-23T18:12:10.715 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8332999999999999e-05 2023-09-23T18:12:10.715 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.715 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.788747+0000", 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6847e-05 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.671005+0000", 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.88225769700000001 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.716 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.717 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.671124+0000", 2023-09-23T18:12:10.717 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119382 2023-09-23T18:12:10.717 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.717 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.717 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.717 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.671262+0000", 2023-09-23T18:12:10.717 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013775500000000001 2023-09-23T18:12:10.717 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.717 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.718 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.718 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.675944+0000", 2023-09-23T18:12:10.718 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004681546 2023-09-23T18:12:10.718 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.718 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.718 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.718 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678673+0000", 2023-09-23T18:12:10.718 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027291160000000002 2023-09-23T18:12:10.718 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.718 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.719 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.719 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678940+0000", 2023-09-23T18:12:10.719 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000267135 2023-09-23T18:12:10.719 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.719 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.719 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.719 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.678990+0000", 2023-09-23T18:12:10.719 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9719000000000003e-05 2023-09-23T18:12:10.719 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.719 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.720 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.721 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679092+0000", 2023-09-23T18:12:10.721 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010241199999999999 2023-09-23T18:12:10.721 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.721 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.721 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.722 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.722 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.722 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148542 8.d 8:bb6d9be0:::benchmark_data_smithi055_346788_object37950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.722 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.792075+0000", 2023-09-23T18:12:10.722 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.781017888000001, 2023-09-23T18:12:10.722 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.88720220299999997, 2023-09-23T18:12:10.722 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.722 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.723 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.723 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.723 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.723 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148542 2023-09-23T18:12:10.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.723 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.723 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.792075+0000", 2023-09-23T18:12:10.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.724 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.724 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.724 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.792075+0000", 2023-09-23T18:12:10.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.724 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.724 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.724 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.792075+0000", 2023-09-23T18:12:10.725 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:12:10.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.725 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.725 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.725 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.792079+0000", 2023-09-23T18:12:10.725 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1609999999999996e-06 2023-09-23T18:12:10.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.725 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.725 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.725 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.792089+0000", 2023-09-23T18:12:10.726 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9539999999999999e-06 2023-09-23T18:12:10.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.726 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.726 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.792093+0000", 2023-09-23T18:12:10.726 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2799999999999997e-06 2023-09-23T18:12:10.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.726 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.726 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.792144+0000", 2023-09-23T18:12:10.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1048000000000003e-05 2023-09-23T18:12:10.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.727 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.792157+0000", 2023-09-23T18:12:10.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.311e-05 2023-09-23T18:12:10.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.727 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.672339+0000", 2023-09-23T18:12:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.88018168600000002 2023-09-23T18:12:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.672412+0000", 2023-09-23T18:12:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2701999999999996e-05 2023-09-23T18:12:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.672490+0000", 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8109999999999996e-05 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676031+0000", 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0035411319999999998 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679145+0000", 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031142330000000001 2023-09-23T18:12:10.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679194+0000", 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8538000000000002e-05 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679225+0000", 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1078000000000003e-05 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679278+0000", 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2827000000000001e-05 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148551 8.d 8:b049ab89:::benchmark_data_smithi055_346788_object37959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.825401+0000", 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.747692399, 2023-09-23T18:12:10.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85419349700000002, 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148551 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.825401+0000", 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.825401+0000", 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.825399+0000", 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:10.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.825412+0000", 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2432000000000001e-05 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.825427+0000", 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5512e-05 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.825434+0000", 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6769999999999998e-06 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.825478+0000", 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4041000000000001e-05 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.825486+0000", 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3690000000000007e-06 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.673567+0000", 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.84808050999999995 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.673650+0000", 2023-09-23T18:12:10.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3263999999999994e-05 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.673763+0000", 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011270899999999999 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676036+0000", 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022728219999999999 2023-09-23T18:12:10.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679359+0000", 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033229330000000001 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679498+0000", 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000139617 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679528+0000", 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9499999999999999e-05 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679594+0000", 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6654000000000003e-05 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148559 8.d 8:b7ad953e:::benchmark_data_smithi055_346788_object37967:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.864454+0000", 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 13.708638819000001, 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.81529974400000005, 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.740 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148559 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.864454+0000", 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.864454+0000", 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.864453+0000", 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.864459+0000", 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7520000000000002e-06 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.864473+0000", 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4154999999999999e-05 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.864478+0000", 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1699999999999996e-06 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.864516+0000", 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8102000000000002e-05 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.864523+0000", 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2370000000000003e-06 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.674721+0000", 2023-09-23T18:12:10.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.81019781700000004 2023-09-23T18:12:10.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.674799+0000", 2023-09-23T18:12:10.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8035999999999996e-05 2023-09-23T18:12:10.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.674888+0000", 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8551e-05 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676264+0000", 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013762399999999999 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679408+0000", 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031437850000000001 2023-09-23T18:12:10.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679631+0000", 2023-09-23T18:12:10.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000223568 2023-09-23T18:12:10.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679660+0000", 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8849000000000001e-05 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.679754+0000", 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3833999999999996e-05 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.749 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:150468 8.d 8:b88dad68:::benchmark_data_smithi055_346788_object39876:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e410)", 2023-09-23T18:12:10.749 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:59.845586+0000", 2023-09-23T18:12:10.749 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 10.727507061000001, 2023-09-23T18:12:10.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.10538246599999999, 2023-09-23T18:12:10.749 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:12:10.749 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:10.749 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:12:10.749 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:12:10.749 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 150468 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.845586+0000", 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:12:10.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.845586+0000", 2023-09-23T18:12:10.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:12:10.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:12:10.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.845585+0000", 2023-09-23T18:12:10.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:12:10.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:12:10.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.845591+0000", 2023-09-23T18:12:10.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7479999999999999e-06 2023-09-23T18:12:10.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:12:10.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.845603+0000", 2023-09-23T18:12:10.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2542e-05 2023-09-23T18:12:10.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:12:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.845610+0000", 2023-09-23T18:12:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7190000000000004e-06 2023-09-23T18:12:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:12:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.845652+0000", 2023-09-23T18:12:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2208e-05 2023-09-23T18:12:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:12:10.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.845738+0000", 2023-09-23T18:12:10.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5501999999999996e-05 2023-09-23T18:12:10.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:12:10.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.845818+0000", 2023-09-23T18:12:10.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0068999999999995e-05 2023-09-23T18:12:10.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:12:10.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.848251+0000", 2023-09-23T18:12:10.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024331309999999998 2023-09-23T18:12:10.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.848673+0000", 2023-09-23T18:12:10.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00042175699999999998 2023-09-23T18:12:10.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.950815+0000", 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.10214266700000001 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.950868+0000", 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.236e-05 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:12:10.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:12:10.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:12:10.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:59.950969+0000", 2023-09-23T18:12:10.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100862 2023-09-23T18:12:10.757 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.757 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.757 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.757 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:12:10.757 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:12:10.757 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:12:10.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:10.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:10.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:10.859 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:10.906 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:10.967 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:10.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:10.976 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:10.998 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:12:11.054 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:11 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 7 pgs peering) 2023-09-23T18:12:11.077 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:11.110 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:11.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:11.157 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:11 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 7 pgs peering) 2023-09-23T18:12:11.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:11.158 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:11.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:11.280 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:11.294 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:11.335 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:11.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:11.338 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:11.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:11 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 7 pgs peering) 2023-09-23T18:12:11.439 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:11.521 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:11.604 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:11.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:11.609 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:11.710 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:11.756 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:11.803 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:11.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:11.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:11.908 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:11.949 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:12.059 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:12 smithi055 ceph-mon[179668]: pgmap v1022: 119 pgs: 2 peering, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 820 op/s; 31172/193770 objects degraded (16.087%); 17519/193770 objects misplaced (9.041%); 16 MiB/s, 263 objects/s recovering 2023-09-23T18:12:12.060 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:12 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1892793252' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:12.060 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:12 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:12.103 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:12:12.214 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:12:12.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:12 smithi099 ceph-mon[167076]: pgmap v1022: 119 pgs: 2 peering, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 820 op/s; 31172/193770 objects degraded (16.087%); 17519/193770 objects misplaced (9.041%); 16 MiB/s, 263 objects/s recovering 2023-09-23T18:12:12.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:12 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1892793252' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:12.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:12 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:12.371 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:12.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:12 smithi055 ceph-mon[184174]: pgmap v1022: 119 pgs: 2 peering, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 3 active+recovery_wait+degraded+remapped, 1 active+recovering+degraded+remapped, 56 active+clean; 3.9 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 820 op/s; 31172/193770 objects degraded (16.087%); 17519/193770 objects misplaced (9.041%); 16 MiB/s, 263 objects/s recovering 2023-09-23T18:12:12.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:12 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1892793252' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:12.379 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:12 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:12.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:12.385 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:12.485 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:12.545 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:12.611 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:12.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:12.616 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:12.717 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:12.780 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:12.796 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:13.164 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:13.169 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:13.170 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:13.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:13 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:12:13.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:13 smithi055 ceph-mon[184174]: failed to encode map e417 with expected crc 2023-09-23T18:12:13.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:13 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:12:13.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:13 smithi055 ceph-mon[184174]: osdmap e417: 8 total, 7 up, 7 in 2023-09-23T18:12:13.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:13 smithi055 ceph-mon[184174]: failed to encode map e417 with expected crc 2023-09-23T18:12:13.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:13 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:12:13.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:13 smithi055 ceph-mon[179668]: failed to encode map e417 with expected crc 2023-09-23T18:12:13.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:13 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:12:13.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:13 smithi055 ceph-mon[179668]: osdmap e417: 8 total, 7 up, 7 in 2023-09-23T18:12:13.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:13 smithi055 ceph-mon[179668]: failed to encode map e417 with expected crc 2023-09-23T18:12:13.231 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:12:13.231 INFO:teuthology.orchestra.run.smithi055.stdout: "success": "" 2023-09-23T18:12:13.231 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:12:13.271 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:13 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:12:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:13 smithi099 ceph-mon[167076]: failed to encode map e417 with expected crc 2023-09-23T18:12:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:13 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:12:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:13 smithi099 ceph-mon[167076]: osdmap e417: 8 total, 7 up, 7 in 2023-09-23T18:12:13.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:13 smithi099 ceph-mon[167076]: failed to encode map e417 with expected crc 2023-09-23T18:12:13.333 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:13.368 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:13.448 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:13.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:13.460 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:13.561 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:13.616 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:13.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:13.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:13.676 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:13.776 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:13.823 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:13.868 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:13.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:13.874 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:13.974 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:14.015 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:14.076 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:14.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:14.080 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:14.181 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:14.286 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:14.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:14 smithi099 ceph-mon[167076]: pgmap v1024: 119 pgs: 2 peering, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 56 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 29700/187398 objects degraded (15.849%); 16908/187398 objects misplaced (9.023%); 15 MiB/s, 240 objects/s recovering 2023-09-23T18:12:14.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:14 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 29700/187398 objects degraded (15.849%), 17 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:14.404 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:14 smithi055 ceph-mon[184174]: pgmap v1024: 119 pgs: 2 peering, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 56 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 29700/187398 objects degraded (15.849%); 16908/187398 objects misplaced (9.023%); 15 MiB/s, 240 objects/s recovering 2023-09-23T18:12:14.404 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:14 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 29700/187398 objects degraded (15.849%), 17 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:14.404 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:14 smithi055 ceph-mon[179668]: pgmap v1024: 119 pgs: 2 peering, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 56 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 897 op/s; 29700/187398 objects degraded (15.849%); 16908/187398 objects misplaced (9.023%); 15 MiB/s, 240 objects/s recovering 2023-09-23T18:12:14.404 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:14 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 29700/187398 objects degraded (15.849%), 17 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:14.743 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:14.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:14.747 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:14.778 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:12:14.848 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:14.907 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:12:14.907 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:12:14.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.907 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:161769 8.8 8:1da57420:::benchmark_data_smithi055_346788_object51177:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:14.877504+0000", 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.026965807000000001, 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027010032999999999, 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161769 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:14.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.877504+0000", 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.877504+0000", 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:14.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.877502+0000", 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.877512+0000", 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0244999999999999e-05 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.877533+0000", 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1013e-05 2023-09-23T18:12:14.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.877543+0000", 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0327e-05 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.880494+0000", 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029506350000000001 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.880559+0000", 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5203999999999994e-05 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.886858+0000", 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0062987959999999997 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.887005+0000", 2023-09-23T18:12:14.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014741799999999999 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.887164+0000", 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015844899999999999 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.895038+0000", 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0078745929999999992 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.902825+0000", 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0077869230000000003 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.902868+0000", 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2892000000000002e-05 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.902925+0000", 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7034999999999999e-05 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:161775 8.8 8.3dcaf148 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:14.895307+0000", 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0091622289999999992, 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093393450000000006, 2023-09-23T18:12:14.915 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161775 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.895307+0000", 2023-09-23T18:12:14.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.895307+0000", 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.895306+0000", 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.895315+0000", 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4560000000000006e-06 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.895330+0000", 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4644000000000001e-05 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:14.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.895340+0000", 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0195e-05 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:161776 8.8 8.fa1a3d28 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e417)", 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:14.896480+0000", 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0079901629999999998, 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082287060000000006, 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:14.919 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 161776 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.896480+0000", 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.896480+0000", 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.896478+0000", 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.896485+0000", 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9059999999999996e-06 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.896498+0000", 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2632999999999999e-05 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.896505+0000", 2023-09-23T18:12:14.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7060000000000004e-06 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.45923.0:161769 8.8 e417/394 ondisk, result = 0)", 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:14.900650+0000", 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.003819392, 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041166609999999998, 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:12:14.923 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.900650+0000", 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.900650+0000", 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.900649+0000", 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.900660+0000", 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0091999999999999e-05 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:14.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.900672+0000", 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2544000000000001e-05 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.900680+0000", 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9780000000000005e-06 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.902855+0000", 2023-09-23T18:12:14.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021744780000000001 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:14.902862+0000", 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4619999999999999e-06 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 4 2023-09-23T18:12:14.927 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:12:14.928 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:15.251 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:15.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:15.257 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:15.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:15.396 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:15.404 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:12:15.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:15.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:15.442 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:15.543 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:15.609 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:15.674 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:15.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:15.680 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:15.781 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:15.828 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:15.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:15.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:15.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:15.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:15.998 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:12:15.998 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:12:15.998 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:12:15.998 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:12:16.016 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:12:16.033 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:16.055 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:12:16.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:16.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:16.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:16.098 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:12:16.098 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:12:16.098 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:12:16.098 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:12:16.098 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:12:16.098 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:12:16.099 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:12:16.099 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:12:16.099 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:12:16.159 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:16.182 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:16.205 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:16.229 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:16.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:16 smithi055 ceph-mon[179668]: pgmap v1025: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 29415/185703 objects degraded (15.840%); 16628/185703 objects misplaced (8.954%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T18:12:16.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3788506678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:16.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:16 smithi055 ceph-mon[184174]: pgmap v1025: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 29415/185703 objects degraded (15.840%); 16628/185703 objects misplaced (8.954%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T18:12:16.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3788506678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:16 smithi099 ceph-mon[167076]: pgmap v1025: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.8 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 0 B/s wr, 796 op/s; 29415/185703 objects degraded (15.840%); 16628/185703 objects misplaced (8.954%); 9.9 MiB/s, 158 objects/s recovering 2023-09-23T18:12:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3788506678' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:17.190 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:17.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:17.197 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:17.298 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:17.357 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:18.032 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:18.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:18.036 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:18.138 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:18.202 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:18.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:18 smithi055 ceph-mon[184174]: pgmap v1026: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 764 op/s; 28365/180186 objects degraded (15.742%); 16302/180186 objects misplaced (9.047%); 11 MiB/s, 182 objects/s recovering 2023-09-23T18:12:18.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:18 smithi055 ceph-mon[179668]: pgmap v1026: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 764 op/s; 28365/180186 objects degraded (15.742%); 16302/180186 objects misplaced (9.047%); 11 MiB/s, 182 objects/s recovering 2023-09-23T18:12:18.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:18 smithi099 ceph-mon[167076]: pgmap v1026: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.7 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 764 op/s; 28365/180186 objects degraded (15.742%); 16302/180186 objects misplaced (9.047%); 11 MiB/s, 182 objects/s recovering 2023-09-23T18:12:18.886 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 3, 4] out_osds: [2] dead_osds: [5] live_osds: [0, 1, 2, 4, 6, 7, 3] 2023-09-23T18:12:18.887 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:12:18.887 INFO:tasks.thrashosds.thrasher:Reviving osd 5 2023-09-23T18:12:18.887 INFO:tasks.cephadm.osd.5:Restarting osd.5 (starting--it wasn't running)... 2023-09-23T18:12:18.887 DEBUG:teuthology.orchestra.run.smithi099:osd.5> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.5.service 2023-09-23T18:12:18.889 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl start ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T18:12:18.899 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:18.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:18.908 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:18.925 INFO:journalctl@ceph.osd.5.smithi099.stdout:-- Logs begin at Sat 2023-09-23 17:24:23 UTC. -- 2023-09-23T18:12:18.986 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:12:18.987 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:12:19.009 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:19.069 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:19.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:19.133 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:19.133 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:19.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:19 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 28365/180186 objects degraded (15.742%), 17 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:19 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 28365/180186 objects degraded (15.742%), 17 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:19.234 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:19.295 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:18 smithi099 systemd[1]: Starting Ceph osd.5 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:12:19.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:19 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 28365/180186 objects degraded (15.742%), 17 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:19.300 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:19.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:19.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:19.362 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:19.463 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:19.522 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:19.682 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:19.685 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:19.786 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:19.847 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:19.934 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:19.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:19.937 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:20.038 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:20.244 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:20 smithi055 ceph-mon[179668]: pgmap v1027: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 877 op/s; 26780/172860 objects degraded (15.492%); 15467/172860 objects misplaced (8.948%); 13 MiB/s, 203 objects/s recovering 2023-09-23T18:12:20.333 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:20.411 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:20.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:20.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:20.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:12:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:12:20] "GET /metrics HTTP/1.1" 200 36631 "" "Prometheus/2.43.0" 2023-09-23T18:12:20.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:20 smithi055 ceph-mon[184174]: pgmap v1027: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 877 op/s; 26780/172860 objects degraded (15.492%); 15467/172860 objects misplaced (8.948%); 13 MiB/s, 203 objects/s recovering 2023-09-23T18:12:20.518 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:20.535 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:12:20.535 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:12:20.578 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:20 smithi099 bash[251073]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T18:12:20.579 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:20 smithi099 bash[251073]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-23T18:12:20.579 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:20 smithi099 bash[251073]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-5 --no-mon-config 2023-09-23T18:12:20.579 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:20 smithi099 bash[251073]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-5/block 2023-09-23T18:12:20.579 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:20 smithi099 bash[251073]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-5/block 2023-09-23T18:12:20.579 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:20 smithi099 bash[251073]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T18:12:20.579 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:20 smithi099 bash[251073]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-5 2023-09-23T18:12:20.579 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:20 smithi099 bash[251073]: --> ceph-volume lvm activate successful for osd ID: 5 2023-09-23T18:12:20.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:20 smithi099 ceph-mon[167076]: pgmap v1027: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 877 op/s; 26780/172860 objects degraded (15.492%); 15467/172860 objects misplaced (8.948%); 13 MiB/s, 203 objects/s recovering 2023-09-23T18:12:20.598 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:20.894 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:20.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:20.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:20.986 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:12:20.988 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:12:21.006 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:21.130 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:21.223 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:12:21.225 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:21.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:21.246 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:21.247 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1395332462' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:12:21.247 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:21 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:12:21.247 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:20 smithi099 podman[251341]: 2023-09-23T18:12:21.247 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:21 smithi099 bash[251341]: d7247e58758106c2321fb80b346f57625a3fd4f9107de2852224dc2ed3bfa582 2023-09-23T18:12:21.250 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:12:21.347 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:21.412 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:21.577 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:21 smithi099 systemd[1]: Started Ceph osd.5 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:12:21.578 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:21.581 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1395332462' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:12:21.582 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:21 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:12:21.582 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1395332462' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:12:21.582 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:21 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:12:21.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:21.583 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:21.684 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:21.742 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:21.798 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:21.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:21.803 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:21.867 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:21.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:21.909 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:21.932 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:21.981 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:21.988 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:21 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:12:21.693+0000 7fecb91c1540 -1 Falling back to public interface 2023-09-23T18:12:21.989 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:21.989 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:21.995 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:12:22.090 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:22.127 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:22 smithi099 ceph-mon[167076]: pgmap v1028: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 892 op/s; 25609/166974 objects degraded (15.337%); 15063/166974 objects misplaced (9.021%); 13 MiB/s, 211 objects/s recovering 2023-09-23T18:12:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:22 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:22 smithi099 ceph-mon[167076]: failed to encode map e418 with expected crc 2023-09-23T18:12:22.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:22 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:12:22.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:22 smithi099 ceph-mon[167076]: osdmap e418: 8 total, 7 up, 7 in 2023-09-23T18:12:22.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:22.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[179668]: pgmap v1028: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 892 op/s; 25609/166974 objects degraded (15.337%); 15063/166974 objects misplaced (9.021%); 13 MiB/s, 211 objects/s recovering 2023-09-23T18:12:22.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:22.551 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[179668]: failed to encode map e418 with expected crc 2023-09-23T18:12:22.551 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:12:22.551 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[179668]: osdmap e418: 8 total, 7 up, 7 in 2023-09-23T18:12:22.551 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:22.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[184174]: pgmap v1028: 119 pgs: 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 3 active+recovery_wait+degraded+remapped, 58 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 892 op/s; 25609/166974 objects degraded (15.337%); 15063/166974 objects misplaced (9.021%); 13 MiB/s, 211 objects/s recovering 2023-09-23T18:12:22.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:22.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[184174]: failed to encode map e418 with expected crc 2023-09-23T18:12:22.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:12:22.552 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[184174]: osdmap e418: 8 total, 7 up, 7 in 2023-09-23T18:12:22.552 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:23.026 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:23.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:23.033 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:23.133 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:23.192 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:23.262 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:23.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:23.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:23.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 25609/166974 objects degraded (15.337%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:23.373 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:23.441 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 25609/166974 objects degraded (15.337%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:23.446 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:23.448 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 25609/166974 objects degraded (15.337%), 17 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:23.649 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:23.658 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:23.658 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:23.687 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:12:23.715 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:12:23.715 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:12:23.759 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:23.833 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:23.880 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:23.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:23.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:23.985 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:24.045 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:24.182 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:24.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:24.187 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:24.214 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:24 smithi099 ceph-mon[167076]: pgmap v1030: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 876 op/s; 24297/161097 objects degraded (15.082%); 14500/161097 objects misplaced (9.001%); 13 MiB/s, 215 objects/s recovering 2023-09-23T18:12:24.288 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:24.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:24 smithi055 ceph-mon[179668]: pgmap v1030: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 876 op/s; 24297/161097 objects degraded (15.082%); 14500/161097 objects misplaced (9.001%); 13 MiB/s, 215 objects/s recovering 2023-09-23T18:12:24.343 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:24 smithi055 ceph-mon[184174]: pgmap v1030: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.3 GiB data, 12 GiB used, 613 GiB / 626 GiB avail; 0 B/s wr, 876 op/s; 24297/161097 objects degraded (15.082%); 14500/161097 objects misplaced (9.001%); 13 MiB/s, 215 objects/s recovering 2023-09-23T18:12:24.347 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:26 smithi099 ceph-mon[167076]: pgmap v1031: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 24032/159441 objects degraded (15.073%); 14213/159441 objects misplaced (8.914%); 13 MiB/s, 215 objects/s recovering 2023-09-23T18:12:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1249306586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:26.609 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:12:26.609 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:12:26.609 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:12:26.609 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:12:26.610 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:12:26.610 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:12:26.610 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:12:26.610 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:12:26.610 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:12:26.610 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:12:26.610 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:12:26.610 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:12:26.610 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.5 2023-09-23T18:12:26.610 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:12:26.611 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:12:26.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:26 smithi055 ceph-mon[184174]: pgmap v1031: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 24032/159441 objects degraded (15.073%); 14213/159441 objects misplaced (8.914%); 13 MiB/s, 215 objects/s recovering 2023-09-23T18:12:26.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1249306586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:26 smithi055 ceph-mon[179668]: pgmap v1031: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.2 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 874 op/s; 24032/159441 objects degraded (15.073%); 14213/159441 objects misplaced (8.914%); 13 MiB/s, 215 objects/s recovering 2023-09-23T18:12:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1249306586' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:27.559 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:27.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:27.564 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:27.588 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:12:27.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:27.725 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:28.070 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:28.080 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:28.080 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:28.094 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:12:28.095 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:12:28.181 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:28.220 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:28.243 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:28.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:28 smithi055 ceph-mon[184174]: pgmap v1032: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 895 op/s; 23324/153315 objects degraded (15.213%); 13871/153315 objects misplaced (9.047%); 11 MiB/s, 172 objects/s recovering 2023-09-23T18:12:28.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 23324/153315 objects degraded (15.213%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:28.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:28 smithi055 ceph-mon[179668]: pgmap v1032: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 895 op/s; 23324/153315 objects degraded (15.213%); 13871/153315 objects misplaced (9.047%); 11 MiB/s, 172 objects/s recovering 2023-09-23T18:12:28.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 23324/153315 objects degraded (15.213%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:28.510 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:28.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:28 smithi099 ceph-mon[167076]: pgmap v1032: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 895 op/s; 23324/153315 objects degraded (15.213%); 13871/153315 objects misplaced (9.047%); 11 MiB/s, 172 objects/s recovering 2023-09-23T18:12:28.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 23324/153315 objects degraded (15.213%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:28.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:28.995 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:28.995 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:29.096 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:29.163 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:29.476 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:29.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:29.480 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:29.581 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:29.654 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:29.720 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:29.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:29.727 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:29.827 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:29.899 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:29.981 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:29.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:29.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:30.087 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:30.096 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:12:30.155 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:30.189 INFO:teuthology.orchestra.run.smithi099.stderr:no valid command found; 10 closest matches: 2023-09-23T18:12:30.189 INFO:teuthology.orchestra.run.smithi099.stderr:0 2023-09-23T18:12:30.189 INFO:teuthology.orchestra.run.smithi099.stderr:1 2023-09-23T18:12:30.189 INFO:teuthology.orchestra.run.smithi099.stderr:2 2023-09-23T18:12:30.189 INFO:teuthology.orchestra.run.smithi099.stderr:abort 2023-09-23T18:12:30.189 INFO:teuthology.orchestra.run.smithi099.stderr:assert 2023-09-23T18:12:30.189 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs debug_inject_read_zeros 2023-09-23T18:12:30.189 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs files list 2023-09-23T18:12:30.189 INFO:teuthology.orchestra.run.smithi099.stderr:bluefs stats 2023-09-23T18:12:30.190 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator dump block 2023-09-23T18:12:30.190 INFO:teuthology.orchestra.run.smithi099.stderr:bluestore allocator fragmentation block 2023-09-23T18:12:30.192 INFO:teuthology.orchestra.run.smithi099.stderr:admin_socket: invalid command 2023-09-23T18:12:30.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:30 smithi055 ceph-mon[179668]: pgmap v1033: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 21787/144879 objects degraded (15.038%); 12987/144879 objects misplaced (8.964%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T18:12:30.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:30 smithi055 ceph-mon[179668]: from='osd.5 [v2:172.21.15.99:6808/4029064001,v1:172.21.15.99:6809/4029064001]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:12:30.237 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:30 smithi055 ceph-mon[179668]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:12:30.237 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:30 smithi055 ceph-mon[184174]: pgmap v1033: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 21787/144879 objects degraded (15.038%); 12987/144879 objects misplaced (8.964%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T18:12:30.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:30 smithi055 ceph-mon[184174]: from='osd.5 [v2:172.21.15.99:6808/4029064001,v1:172.21.15.99:6809/4029064001]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:12:30.238 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:30 smithi055 ceph-mon[184174]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:12:30.238 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:30.249 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:30.249 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:30.283 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:12:30.284 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:12:30.284 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:12:30.284 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:12:30.351 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:30.414 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:30.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:30 smithi099 ceph-mon[167076]: pgmap v1033: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 3.0 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 21787/144879 objects degraded (15.038%); 12987/144879 objects misplaced (8.964%); 9.4 MiB/s, 150 objects/s recovering 2023-09-23T18:12:30.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:30 smithi099 ceph-mon[167076]: from='osd.5 [v2:172.21.15.99:6808/4029064001,v1:172.21.15.99:6809/4029064001]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:12:30.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:30 smithi099 ceph-mon[167076]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-09-23T18:12:30.578 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:12:30.145+0000 7fecb91c1540 -1 osd.5 366 log_to_monitors true 2023-09-23T18:12:30.578 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:12:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:12:30.371+0000 7fecac7a6700 -1 osd.5 419 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:12:30.667 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:30.669 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T18:12:30.669 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-5, ['dump_ops_in_flight'] 2023-09-23T18:12:30.670 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:12:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:12:30] "GET /metrics HTTP/1.1" 200 36635 "" "Prometheus/2.43.0" 2023-09-23T18:12:30.671 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:30.671 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:30.772 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:30.783 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:12:30.825 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:30.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:30.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:30.883 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:30.984 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:31.036 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:31.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:31.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:31.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:31.144 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:31.185 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:31.245 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:31.274 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:12:31.404 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:31.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:31.408 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:31.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[184174]: failed to encode map e419 with expected crc 2023-09-23T18:12:31.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[184174]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T18:12:31.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[184174]: osdmap e419: 8 total, 7 up, 7 in 2023-09-23T18:12:31.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[184174]: from='osd.5 [v2:172.21.15.99:6808/4029064001,v1:172.21.15.99:6809/4029064001]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:12:31.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[184174]: failed to encode map e419 with expected crc 2023-09-23T18:12:31.495 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[184174]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:12:31.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/596461254' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:31.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:31.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[179668]: failed to encode map e419 with expected crc 2023-09-23T18:12:31.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[179668]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T18:12:31.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[179668]: osdmap e419: 8 total, 7 up, 7 in 2023-09-23T18:12:31.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[179668]: from='osd.5 [v2:172.21.15.99:6808/4029064001,v1:172.21.15.99:6809/4029064001]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:12:31.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[179668]: failed to encode map e419 with expected crc 2023-09-23T18:12:31.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[179668]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:12:31.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/596461254' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:31.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:31 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:31.509 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:31.534 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:31 smithi099 ceph-mon[167076]: failed to encode map e419 with expected crc 2023-09-23T18:12:31.534 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:31 smithi099 ceph-mon[167076]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-09-23T18:12:31.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:31 smithi099 ceph-mon[167076]: osdmap e419: 8 total, 7 up, 7 in 2023-09-23T18:12:31.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:31 smithi099 ceph-mon[167076]: from='osd.5 [v2:172.21.15.99:6808/4029064001,v1:172.21.15.99:6809/4029064001]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:12:31.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:31 smithi099 ceph-mon[167076]: failed to encode map e419 with expected crc 2023-09-23T18:12:31.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:31 smithi099 ceph-mon[167076]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi099", "root=default"]}]: dispatch 2023-09-23T18:12:31.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/596461254' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:31.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:31 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:31.537 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:31.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:31.576 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:31.576 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:31.677 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:31.710 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:31.901 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:31.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:31.905 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:32.007 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:32.038 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:32.124 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:32.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:32.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:32.132 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:32.230 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:32.259 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:32.313 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:32.333 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:32.334 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:32.434 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:32.461 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:32.496 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:32.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:32.500 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:32.554 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[184174]: pgmap v1035: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 968 op/s; 21218/138591 objects degraded (15.310%); 12582/138591 objects misplaced (9.079%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T18:12:32.554 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:32.555 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[184174]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:12:32.555 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[184174]: osd.5 [v2:172.21.15.99:6808/4029064001,v1:172.21.15.99:6809/4029064001] boot 2023-09-23T18:12:32.555 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:12:32.555 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[184174]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T18:12:32.555 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[184174]: failed to encode map e420 with expected crc 2023-09-23T18:12:32.555 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:12:32.555 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[179668]: pgmap v1035: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 968 op/s; 21218/138591 objects degraded (15.310%); 12582/138591 objects misplaced (9.079%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T18:12:32.555 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:32.556 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[179668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:12:32.556 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[179668]: osd.5 [v2:172.21.15.99:6808/4029064001,v1:172.21.15.99:6809/4029064001] boot 2023-09-23T18:12:32.556 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:12:32.556 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[179668]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T18:12:32.556 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[179668]: failed to encode map e420 with expected crc 2023-09-23T18:12:32.556 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:32 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:12:32.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:32 smithi099 ceph-mon[167076]: pgmap v1035: 119 pgs: 1 active+recovery_wait+remapped, 4 active+recovery_wait+degraded, 28 active+clean+remapped, 1 active+remapped+backfill_wait, 8 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 1 active+recovering+degraded+remapped, 2 active+recovery_wait+degraded+remapped, 58 active+clean; 2.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 968 op/s; 21218/138591 objects degraded (15.310%); 12582/138591 objects misplaced (9.079%); 5.8 MiB/s, 93 objects/s recovering 2023-09-23T18:12:32.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:32 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:32.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:32 smithi099 ceph-mon[167076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:12:32.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:32 smithi099 ceph-mon[167076]: osd.5 [v2:172.21.15.99:6808/4029064001,v1:172.21.15.99:6809/4029064001] boot 2023-09-23T18:12:32.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:32 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:12:32.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:32 smithi099 ceph-mon[167076]: osdmap e420: 8 total, 8 up, 7 in 2023-09-23T18:12:32.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:32 smithi099 ceph-mon[167076]: failed to encode map e420 with expected crc 2023-09-23T18:12:32.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:32 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-09-23T18:12:32.601 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:32.643 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:33.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: failed to encode map e367 with expected crc 2023-09-23T18:12:33.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: failed to encode map e407 with expected crc 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: failed to encode map e420 with expected crc 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: failed to encode map e420 with expected crc 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: 8.1 starting backfill to osd.5 from (0'0,366'2392] MAX to 419'11002 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: 8.a continuing backfill to osd.3 from (389'7232,419'11164] 8:52229eaa:::benchmark_data_smithi055_346788_object56927:head to 419'11164 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: 8.a starting backfill to osd.5 from (0'0,366'2428] MAX to 419'11164 2023-09-23T18:12:33.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: 8.2 starting backfill to osd.5 from (0'0,366'2322] MAX to 419'10980 2023-09-23T18:12:33.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: 8.d starting backfill to osd.5 from (0'0,366'2496] MAX to 419'11289 2023-09-23T18:12:33.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 21218/138591 objects degraded (15.310%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:33.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[184174]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T18:12:33.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: failed to encode map e367 with expected crc 2023-09-23T18:12:33.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: failed to encode map e407 with expected crc 2023-09-23T18:12:33.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: failed to encode map e420 with expected crc 2023-09-23T18:12:33.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: failed to encode map e420 with expected crc 2023-09-23T18:12:33.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T18:12:33.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: 8.1 starting backfill to osd.5 from (0'0,366'2392] MAX to 419'11002 2023-09-23T18:12:33.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: 8.a continuing backfill to osd.3 from (389'7232,419'11164] 8:52229eaa:::benchmark_data_smithi055_346788_object56927:head to 419'11164 2023-09-23T18:12:33.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: 8.a starting backfill to osd.5 from (0'0,366'2428] MAX to 419'11164 2023-09-23T18:12:33.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: 8.2 starting backfill to osd.5 from (0'0,366'2322] MAX to 419'10980 2023-09-23T18:12:33.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: 8.d starting backfill to osd.5 from (0'0,366'2496] MAX to 419'11289 2023-09-23T18:12:33.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 21218/138591 objects degraded (15.310%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:33.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:33 smithi055 ceph-mon[179668]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T18:12:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: failed to encode map e367 with expected crc 2023-09-23T18:12:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: failed to encode map e407 with expected crc 2023-09-23T18:12:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: failed to encode map e420 with expected crc 2023-09-23T18:12:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: failed to encode map e420 with expected crc 2023-09-23T18:12:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: osdmap e421: 8 total, 8 up, 7 in 2023-09-23T18:12:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: failed to encode map e421 with expected crc 2023-09-23T18:12:33.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: 8.1 starting backfill to osd.5 from (0'0,366'2392] MAX to 419'11002 2023-09-23T18:12:33.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: 8.a continuing backfill to osd.3 from (389'7232,419'11164] 8:52229eaa:::benchmark_data_smithi055_346788_object56927:head to 419'11164 2023-09-23T18:12:33.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: 8.a starting backfill to osd.5 from (0'0,366'2428] MAX to 419'11164 2023-09-23T18:12:33.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: 8.2 starting backfill to osd.5 from (0'0,366'2322] MAX to 419'10980 2023-09-23T18:12:33.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: 8.d starting backfill to osd.5 from (0'0,366'2496] MAX to 419'11289 2023-09-23T18:12:33.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 21218/138591 objects degraded (15.310%), 16 pgs degraded, 23 pgs undersized (PG_DEGRADED) 2023-09-23T18:12:33.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:33 smithi099 ceph-mon[167076]: osdmap e422: 8 total, 8 up, 7 in 2023-09-23T18:12:33.612 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:12:33.612 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:12:33.612 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:12:33.612 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:12:33.635 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:12:33.642 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:33.655 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:33.655 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:33.662 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:12:33.692 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:12:33.730 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:12:33.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:12:33.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:12:33.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:12:33.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:12:33.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:12:33.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:12:33.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:12:33.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:12:33.756 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:33.843 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:33.942 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:33.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:33.956 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:33.994 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:12:34.057 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:34.140 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:34.330 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[179668]: pgmap v1038: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 4 remapped+peering, 5 peering, 4 active+recovery_wait+degraded, 22 active+clean+remapped, 2 active+remapped+backfill_wait, 6 active+undersized+degraded, 12 active+undersized, 2 active+recovery_wait+degraded+remapped, 61 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 12071/134286 objects degraded (8.989%); 12040/134286 objects misplaced (8.966%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T18:12:34.330 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[179668]: failed to encode map e422 with expected crc 2023-09-23T18:12:34.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[179668]: failed to encode map e422 with expected crc 2023-09-23T18:12:34.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[179668]: failed to encode map e422 with expected crc 2023-09-23T18:12:34.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[179668]: 8.0 starting backfill to osd.5 from (0'0,366'2425] MAX to 419'11043 2023-09-23T18:12:34.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[179668]: 8.3 starting backfill to osd.5 from (0'0,366'2408] MAX to 419'11201 2023-09-23T18:12:34.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[179668]: 8.c starting backfill to osd.5 from (0'0,366'2427] MAX to 419'10987 2023-09-23T18:12:34.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[179668]: 8.9 starting backfill to osd.5 from (0'0,366'2432] MAX to 419'11121 2023-09-23T18:12:34.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[179668]: 8.4 starting backfill to osd.5 from (0'0,366'2442] MAX to 419'10991 2023-09-23T18:12:34.331 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1346379043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:34.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:34 smithi099 ceph-mon[167076]: pgmap v1038: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 4 remapped+peering, 5 peering, 4 active+recovery_wait+degraded, 22 active+clean+remapped, 2 active+remapped+backfill_wait, 6 active+undersized+degraded, 12 active+undersized, 2 active+recovery_wait+degraded+remapped, 61 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 12071/134286 objects degraded (8.989%); 12040/134286 objects misplaced (8.966%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T18:12:34.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:34 smithi099 ceph-mon[167076]: failed to encode map e422 with expected crc 2023-09-23T18:12:34.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:34 smithi099 ceph-mon[167076]: failed to encode map e422 with expected crc 2023-09-23T18:12:34.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:34 smithi099 ceph-mon[167076]: failed to encode map e422 with expected crc 2023-09-23T18:12:34.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:34 smithi099 ceph-mon[167076]: 8.0 starting backfill to osd.5 from (0'0,366'2425] MAX to 419'11043 2023-09-23T18:12:34.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:34 smithi099 ceph-mon[167076]: 8.3 starting backfill to osd.5 from (0'0,366'2408] MAX to 419'11201 2023-09-23T18:12:34.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:34 smithi099 ceph-mon[167076]: 8.c starting backfill to osd.5 from (0'0,366'2427] MAX to 419'10987 2023-09-23T18:12:34.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:34 smithi099 ceph-mon[167076]: 8.9 starting backfill to osd.5 from (0'0,366'2432] MAX to 419'11121 2023-09-23T18:12:34.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:34 smithi099 ceph-mon[167076]: 8.4 starting backfill to osd.5 from (0'0,366'2442] MAX to 419'10991 2023-09-23T18:12:34.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1346379043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:34.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[184174]: pgmap v1038: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 4 remapped+peering, 5 peering, 4 active+recovery_wait+degraded, 22 active+clean+remapped, 2 active+remapped+backfill_wait, 6 active+undersized+degraded, 12 active+undersized, 2 active+recovery_wait+degraded+remapped, 61 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 1.05k op/s; 12071/134286 objects degraded (8.989%); 12040/134286 objects misplaced (8.966%); 7.0 MiB/s, 112 objects/s recovering 2023-09-23T18:12:34.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[184174]: failed to encode map e422 with expected crc 2023-09-23T18:12:34.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[184174]: failed to encode map e422 with expected crc 2023-09-23T18:12:34.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[184174]: failed to encode map e422 with expected crc 2023-09-23T18:12:34.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[184174]: 8.0 starting backfill to osd.5 from (0'0,366'2425] MAX to 419'11043 2023-09-23T18:12:34.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[184174]: 8.3 starting backfill to osd.5 from (0'0,366'2408] MAX to 419'11201 2023-09-23T18:12:34.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[184174]: 8.c starting backfill to osd.5 from (0'0,366'2427] MAX to 419'10987 2023-09-23T18:12:34.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[184174]: 8.9 starting backfill to osd.5 from (0'0,366'2432] MAX to 419'11121 2023-09-23T18:12:34.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[184174]: 8.4 starting backfill to osd.5 from (0'0,366'2442] MAX to 419'10991 2023-09-23T18:12:34.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1346379043' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:34.693 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:34.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:34.697 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:34.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:34.826 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:34.896 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:34.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:34.904 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:35.005 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:35.065 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:35.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:35 smithi055 ceph-mon[179668]: failed to encode map e423 with expected crc 2023-09-23T18:12:35.352 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:35 smithi055 ceph-mon[179668]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T18:12:35.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:35 smithi055 ceph-mon[184174]: failed to encode map e423 with expected crc 2023-09-23T18:12:35.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:35 smithi055 ceph-mon[184174]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T18:12:35.671 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:12:35.730 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:35 smithi099 ceph-mon[167076]: failed to encode map e423 with expected crc 2023-09-23T18:12:35.731 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:35 smithi099 ceph-mon[167076]: osdmap e423: 8 total, 8 up, 7 in 2023-09-23T18:12:36.355 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:36.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:36.360 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:36.419 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:12:36.419 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:12:36.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:36.506 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:36.528 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:12:36.568 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:36.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:36.581 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:36.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[184174]: pgmap v1041: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 5 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 19 active+clean+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+degraded, 9 active+undersized, 2 active+recovery_wait+degraded+remapped, 68 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 9135/133110 objects degraded (6.863%); 11839/133110 objects misplaced (8.894%); 5.8 MiB/s, 2 keys/s, 95 objects/s recovering 2023-09-23T18:12:36.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[184174]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T18:12:36.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[184174]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[184174]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[184174]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[184174]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[184174]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[184174]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[184174]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[179668]: pgmap v1041: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 5 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 19 active+clean+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+degraded, 9 active+undersized, 2 active+recovery_wait+degraded+remapped, 68 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 9135/133110 objects degraded (6.863%); 11839/133110 objects misplaced (8.894%); 5.8 MiB/s, 2 keys/s, 95 objects/s recovering 2023-09-23T18:12:36.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[179668]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T18:12:36.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[179668]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[179668]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[179668]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[179668]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[179668]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[179668]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:36 smithi055 ceph-mon[179668]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.682 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:36.733 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:36 smithi099 ceph-mon[167076]: pgmap v1041: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 5 remapped+peering, 7 peering, 1 active+recovery_wait+degraded, 19 active+clean+remapped, 3 active+remapped+backfill_wait, 4 active+undersized+degraded, 9 active+undersized, 2 active+recovery_wait+degraded+remapped, 68 active+clean; 2.7 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 0 B/s wr, 456 op/s; 9135/133110 objects degraded (6.863%); 11839/133110 objects misplaced (8.894%); 5.8 MiB/s, 2 keys/s, 95 objects/s recovering 2023-09-23T18:12:36.733 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:36 smithi099 ceph-mon[167076]: osdmap e424: 8 total, 8 up, 7 in 2023-09-23T18:12:36.734 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:36 smithi099 ceph-mon[167076]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.734 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:36 smithi099 ceph-mon[167076]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.734 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:36 smithi099 ceph-mon[167076]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.734 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:36 smithi099 ceph-mon[167076]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.734 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:36 smithi099 ceph-mon[167076]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.734 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:36 smithi099 ceph-mon[167076]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.734 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:36 smithi099 ceph-mon[167076]: failed to encode map e424 with expected crc 2023-09-23T18:12:36.750 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:36.819 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:36.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:36.831 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:36.862 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:36.883 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:12:36.932 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:36.987 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:37.117 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:37.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:37.124 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:37.225 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:37.276 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:37.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:37.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:37.723 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:37.723 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:37.724 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:37.724 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:37.927 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:38.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:38.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:38.032 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:38.133 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:38.214 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:38.305 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:38.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:38.315 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:38.359 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:12:38.416 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:38.491 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:38.568 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:38 smithi055 ceph-mon[184174]: pgmap v1043: 119 pgs: 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 5 remapped+peering, 7 peering, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 8.8 KiB/s rd, 24 KiB/s wr, 640 op/s; 8021/129708 objects degraded (6.184%); 11073/129708 objects misplaced (8.537%); 13 MiB/s, 2 keys/s, 371 objects/s recovering 2023-09-23T18:12:38.569 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 8021/129708 objects degraded (6.184%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:38.569 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:38 smithi055 ceph-mon[179668]: pgmap v1043: 119 pgs: 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 5 remapped+peering, 7 peering, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 8.8 KiB/s rd, 24 KiB/s wr, 640 op/s; 8021/129708 objects degraded (6.184%); 11073/129708 objects misplaced (8.537%); 13 MiB/s, 2 keys/s, 371 objects/s recovering 2023-09-23T18:12:38.569 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 8021/129708 objects degraded (6.184%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:38.621 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:38.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:38.629 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:38.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:38.771 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:38 smithi099 ceph-mon[167076]: pgmap v1043: 119 pgs: 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 5 remapped+peering, 7 peering, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 99 active+clean; 2.6 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 8.8 KiB/s rd, 24 KiB/s wr, 640 op/s; 8021/129708 objects degraded (6.184%); 11073/129708 objects misplaced (8.537%); 13 MiB/s, 2 keys/s, 371 objects/s recovering 2023-09-23T18:12:38.771 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 8021/129708 objects degraded (6.184%), 5 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:38.776 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:39.240 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:39.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:39.246 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:39.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:39.384 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:39.463 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:39.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:39.470 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:39.504 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:12:39.539 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:39.571 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:39.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3911468807' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:12:39.615 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:12:39.615 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:12:39.648 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3911468807' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:12:39.649 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3911468807' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:12:39.675 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:39.750 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:39.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:39.761 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:39.861 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:39.919 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:40.359 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:40.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:40.366 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:40.467 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:40.488 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:12:40] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.43.0" 2023-09-23T18:12:40.518 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:40.530 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:40 smithi099 ceph-mon[167076]: pgmap v1044: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 641 op/s; 17891/122832 objects degraded (14.565%); 10175/122832 objects misplaced (8.284%); 9.6 MiB/s, 1 keys/s, 285 objects/s recovering 2023-09-23T18:12:40.530 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:40 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:40.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:40 smithi099 ceph-mon[167076]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:40 smithi099 ceph-mon[167076]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3911468807' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:12:40.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:40 smithi099 ceph-mon[167076]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T18:12:40.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:40 smithi099 ceph-mon[167076]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:40 smithi099 ceph-mon[167076]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.607 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:12:40.833 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:40.837 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[179668]: pgmap v1044: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 641 op/s; 17891/122832 objects degraded (14.565%); 10175/122832 objects misplaced (8.284%); 9.6 MiB/s, 1 keys/s, 285 objects/s recovering 2023-09-23T18:12:40.837 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:40.837 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[179668]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.837 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[179668]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.837 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3911468807' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:12:40.838 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[179668]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T18:12:40.838 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[179668]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.838 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[179668]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[184174]: pgmap v1044: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.5 GiB data, 10 GiB used, 615 GiB / 626 GiB avail; 6.8 KiB/s rd, 18 KiB/s wr, 641 op/s; 17891/122832 objects degraded (14.565%); 10175/122832 objects misplaced (8.284%); 9.6 MiB/s, 1 keys/s, 285 objects/s recovering 2023-09-23T18:12:40.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:12:40.839 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[184174]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.839 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[184174]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.839 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3911468807' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:12:40.839 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[184174]: osdmap e425: 8 total, 8 up, 7 in 2023-09-23T18:12:40.839 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[184174]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.839 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:40 smithi055 ceph-mon[184174]: failed to encode map e425 with expected crc 2023-09-23T18:12:40.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:40.840 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:40.941 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:41.000 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:41.130 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:41.134 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:41.134 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:41.148 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:12:41.148 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:12:41.235 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:41.286 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:41.498 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:41.502 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:41.503 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:41.603 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:41.644 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:42.108 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:42.115 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:42.115 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:42.154 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:12:42.217 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:42.274 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:42.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:42 smithi099 ceph-mon[167076]: pgmap v1046: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 6.2 KiB/s rd, 17 KiB/s wr, 832 op/s; 17289/116676 objects degraded (14.818%); 9879/116676 objects misplaced (8.467%); 8.7 MiB/s, 257 objects/s recovering 2023-09-23T18:12:42.823 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:42 smithi055 ceph-mon[184174]: pgmap v1046: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 6.2 KiB/s rd, 17 KiB/s wr, 832 op/s; 17289/116676 objects degraded (14.818%); 9879/116676 objects misplaced (8.467%); 8.7 MiB/s, 257 objects/s recovering 2023-09-23T18:12:42.824 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:42 smithi055 ceph-mon[179668]: pgmap v1046: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.4 GiB data, 10 GiB used, 616 GiB / 626 GiB avail; 6.2 KiB/s rd, 17 KiB/s wr, 832 op/s; 17289/116676 objects degraded (14.818%); 9879/116676 objects misplaced (8.467%); 8.7 MiB/s, 257 objects/s recovering 2023-09-23T18:12:43.172 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:43.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:43.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:43.230 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:43.249 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:12:43.289 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:12:43.331 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:43.338 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:12:43.384 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:12:43.404 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:43.433 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:12:43.470 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:43.476 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:12:43.499 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:43.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:43.509 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:43.512 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:12:43.519 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:43 smithi099 ceph-mon[167076]: pgmap v1047: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 935 op/s; 16346/110757 objects degraded (14.758%); 9350/110757 objects misplaced (8.442%); 8.0 MiB/s, 225 objects/s recovering 2023-09-23T18:12:43.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 17289/116676 objects degraded (14.818%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:43.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2530909126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:43.563 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:12:43.571 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:12:43.572 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:12:43.572 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:12:43.572 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:12:43.572 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:12:43.572 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:12:43.572 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:12:43.572 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:12:43.610 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:43.674 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:43 smithi055 ceph-mon[179668]: pgmap v1047: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 935 op/s; 16346/110757 objects degraded (14.758%); 9350/110757 objects misplaced (8.442%); 8.0 MiB/s, 225 objects/s recovering 2023-09-23T18:12:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 17289/116676 objects degraded (14.818%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2530909126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:43 smithi055 ceph-mon[184174]: pgmap v1047: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.3 GiB data, 9.8 GiB used, 616 GiB / 626 GiB avail; 5.1 KiB/s rd, 14 KiB/s wr, 935 op/s; 16346/110757 objects degraded (14.758%); 9350/110757 objects misplaced (8.442%); 8.0 MiB/s, 225 objects/s recovering 2023-09-23T18:12:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 17289/116676 objects degraded (14.818%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:43.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2530909126' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:43.745 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:43.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:43.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:43.854 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:43.917 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:43.998 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:44.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:44.004 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:44.105 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:44.167 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:44.539 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:44.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:44.543 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:44.644 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:44.697 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:44.796 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:44.799 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:44.800 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:44.901 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:44.967 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:45.225 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:45.235 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:45.235 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:45.247 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:12:45.247 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:12:45.247 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.248 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:187931 8.0 8:01b7a24a:::benchmark_data_smithi055_346788_object77339:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T18:12:45.248 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:45.221819+0000", 2023-09-23T18:12:45.248 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.024551175000000001, 2023-09-23T18:12:45.248 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024585052999999999, 2023-09-23T18:12:45.248 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:45.249 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:12:45.249 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:12:45.249 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:12:45.249 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:45.249 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 187931 2023-09-23T18:12:45.250 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.250 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:45.250 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:45.250 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.221819+0000", 2023-09-23T18:12:45.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.251 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.251 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.251 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:45.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.221819+0000", 2023-09-23T18:12:45.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:45.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.221818+0000", 2023-09-23T18:12:45.253 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:45.253 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.253 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.253 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:45.253 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.221824+0000", 2023-09-23T18:12:45.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6640000000000001e-06 2023-09-23T18:12:45.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.255 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:45.255 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.221840+0000", 2023-09-23T18:12:45.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5673999999999999e-05 2023-09-23T18:12:45.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.255 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:45.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.221846+0000", 2023-09-23T18:12:45.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2639999999999997e-06 2023-09-23T18:12:45.256 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.256 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:45.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.221896+0000", 2023-09-23T18:12:45.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9295999999999997e-05 2023-09-23T18:12:45.256 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.257 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:45.257 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.222055+0000", 2023-09-23T18:12:45.257 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015902599999999999 2023-09-23T18:12:45.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.257 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:12:45.257 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.222219+0000", 2023-09-23T18:12:45.257 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016406799999999999 2023-09-23T18:12:45.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.224976+0000", 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027576699999999998 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.224990+0000", 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3591000000000001e-05 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:187941 8.f 8:f2172285:::benchmark_data_smithi055_346788_object77349:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:45.233964+0000", 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.01240636, 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012541211999999999, 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 187941 2023-09-23T18:12:45.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.233964+0000", 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.233964+0000", 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.233963+0000", 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.233969+0000", 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0140000000000004e-06 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.233978+0000", 2023-09-23T18:12:45.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8859999999999999e-06 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.233984+0000", 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5799999999999999e-06 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.235876+0000", 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001891975 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.236032+0000", 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015616299999999999 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.236180+0000", 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000147942 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:45.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.243471+0000", 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072907179999999999 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:187947 8.f 8:fb15e012:::benchmark_data_smithi055_346788_object77355:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:45.235825+0000", 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.010545527000000001, 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010768902, 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:45.264 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 187947 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.235825+0000", 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.235825+0000", 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.235824+0000", 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.235829+0000", 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8279999999999999e-06 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.235842+0000", 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2184000000000001e-05 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.235849+0000", 2023-09-23T18:12:45.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5449999999999998e-06 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240731+0000", 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004882127 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240867+0000", 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013557700000000001 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240981+0000", 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114283 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:187943 8.b e425/413 8:db14b48d:::benchmark_data_smithi055_346788_object77351:head v 425'11599, mlcod=425'11597)", 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:45.238865+0000", 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0075051340000000001, 2023-09-23T18:12:45.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0078042709999999998, 2023-09-23T18:12:45.270 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:45.270 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:12:45.270 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:45.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:45.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238865+0000", 2023-09-23T18:12:45.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238865+0000", 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238864+0000", 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238873+0000", 2023-09-23T18:12:45.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8179999999999993e-06 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238885+0000", 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1770000000000001e-05 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238894+0000", 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7050000000000005e-06 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.239024+0000", 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000130023 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.239047+0000", 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3170999999999999e-05 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:187950 8.1 8:8a46f49a:::benchmark_data_smithi055_346788_object77358:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:45.238919+0000", 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0074515520000000002, 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0078145980000000007, 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 187950 2023-09-23T18:12:45.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238919+0000", 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238919+0000", 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238917+0000", 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238926+0000", 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9190000000000005e-06 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238939+0000", 2023-09-23T18:12:45.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.394e-05 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.238949+0000", 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1640000000000007e-06 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.239006+0000", 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7358999999999997e-05 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.239178+0000", 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017201 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.239346+0000", 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000167524 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.246610+0000", 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072646550000000001 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:187951 8.a 8:5047f3dc:::benchmark_data_smithi055_346788_object77359:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:45.240338+0000", 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0060323879999999996, 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0064748699999999998, 2023-09-23T18:12:45.279 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 187951 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240338+0000", 2023-09-23T18:12:45.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240338+0000", 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240336+0000", 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:12:45.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:45.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240344+0000", 2023-09-23T18:12:45.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4340000000000003e-06 2023-09-23T18:12:45.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:45.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240357+0000", 2023-09-23T18:12:45.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3376e-05 2023-09-23T18:12:45.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:45.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240374+0000", 2023-09-23T18:12:45.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6650000000000002e-05 2023-09-23T18:12:45.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:45.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240424+0000", 2023-09-23T18:12:45.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0640999999999998e-05 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240573+0000", 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014902700000000001 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,3,5", 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.240733+0000", 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015958500000000001 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.246525+0000", 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0057918079999999999 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:187953 8.0 8:091f8389:::benchmark_data_smithi055_346788_object77361:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:45.242431+0000", 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.003939499, 2023-09-23T18:12:45.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044647519999999998, 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 187953 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:45.286 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242431+0000", 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242431+0000", 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242429+0000", 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:12:45.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242436+0000", 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.08e-06 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242450+0000", 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3305000000000001e-05 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:45.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242459+0000", 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5780000000000005e-06 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242507+0000", 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7976000000000003e-05 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242637+0000", 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000130044 2023-09-23T18:12:45.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242767+0000", 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000129667 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:187954 8.a 8:59ed5593:::benchmark_data_smithi055_346788_object77362:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e425)", 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:45.242477+0000", 2023-09-23T18:12:45.290 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0038936169999999998, 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044944980000000004, 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 187954 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:45.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242477+0000", 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242477+0000", 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242476+0000", 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:12:45.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242479+0000", 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4999999999999999e-06 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242488+0000", 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3590000000000008e-06 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:45.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.242494+0000", 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5400000000000001e-06 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.243217+0000", 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00072279899999999999 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.243341+0000", 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012391799999999999 2023-09-23T18:12:45.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,3,5", 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.243476+0000", 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000134616 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.246559+0000", 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003083218 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:45.295 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.45923.0:187947 8.f e425/412)", 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:45.246888+0000", 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 4294967295.9994822, 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000160793, 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.246888+0000", 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.246888+0000", 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.246888+0000", 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:12:45.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:12:45.298 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.246890+0000", 2023-09-23T18:12:45.298 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3740000000000001e-06 2023-09-23T18:12:45.298 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:12:45.298 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.246896+0000", 2023-09-23T18:12:45.298 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1600000000000003e-06 2023-09-23T18:12:45.298 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.246901+0000", 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8810000000000004e-06 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.247042+0000", 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014089500000000001 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:12:45.299 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:12:45.300 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:45.247049+0000", 2023-09-23T18:12:45.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4320000000000002e-06 2023-09-23T18:12:45.300 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.300 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:12:45.300 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.300 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:12:45.300 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:12:45.300 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 9 2023-09-23T18:12:45.300 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:12:45.336 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:45.411 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:46.039 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:12:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:46 smithi099 ceph-mon[167076]: pgmap v1048: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 16064/109077 objects degraded (14.727%); 9078/109077 objects misplaced (8.323%); 6.6 MiB/s, 189 objects/s recovering 2023-09-23T18:12:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/141446082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:46.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:46 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:46 smithi055 ceph-mon[179668]: pgmap v1048: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 16064/109077 objects degraded (14.727%); 9078/109077 objects misplaced (8.323%); 6.6 MiB/s, 189 objects/s recovering 2023-09-23T18:12:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/141446082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:46 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:46.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:46 smithi055 ceph-mon[184174]: pgmap v1048: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.2 GiB data, 9.7 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 833 op/s; 16064/109077 objects degraded (14.727%); 9078/109077 objects misplaced (8.323%); 6.6 MiB/s, 189 objects/s recovering 2023-09-23T18:12:46.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/141446082' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:46.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:46 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:12:46.448 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:12:47.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:47 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:12:47.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:47 smithi099 ceph-mon[167076]: failed to encode map e426 with expected crc 2023-09-23T18:12:47.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:47 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:12:47.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:47 smithi099 ceph-mon[167076]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T18:12:47.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:47 smithi099 ceph-mon[167076]: failed to encode map e426 with expected crc 2023-09-23T18:12:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:47 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:12:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:47 smithi055 ceph-mon[179668]: failed to encode map e426 with expected crc 2023-09-23T18:12:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:47 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:12:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:47 smithi055 ceph-mon[179668]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T18:12:47.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:47 smithi055 ceph-mon[179668]: failed to encode map e426 with expected crc 2023-09-23T18:12:47.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:47 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:12:47.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:47 smithi055 ceph-mon[184174]: failed to encode map e426 with expected crc 2023-09-23T18:12:47.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:47 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:12:47.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:47 smithi055 ceph-mon[184174]: osdmap e426: 8 total, 8 up, 7 in 2023-09-23T18:12:47.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:47 smithi055 ceph-mon[184174]: failed to encode map e426 with expected crc 2023-09-23T18:12:47.636 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:47.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:47.644 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:47.733 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:12:47.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:12:47.811 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:47.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:12:47.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:47.895 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:12:47.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:12:48.062 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:48 smithi099 ceph-mon[167076]: pgmap v1050: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 829 op/s; 15509/102915 objects degraded (15.070%); 8742/102915 objects misplaced (8.494%); 776 KiB/s, 12 objects/s recovering 2023-09-23T18:12:48.063 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:48.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:12:48.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:48.137 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:12:48.238 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:12:48.290 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:48.293 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 3, 4] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 4, 6, 7, 3, 5] 2023-09-23T18:12:48.293 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:12:48.294 INFO:tasks.thrashosds.thrasher:Growing pool unique_pool_3 2023-09-23T18:12:48.294 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:12:48.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:48 smithi055 ceph-mon[184174]: pgmap v1050: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 829 op/s; 15509/102915 objects degraded (15.070%); 8742/102915 objects misplaced (8.494%); 776 KiB/s, 12 objects/s recovering 2023-09-23T18:12:48.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:48 smithi055 ceph-mon[179668]: pgmap v1050: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 2.1 GiB data, 9.4 GiB used, 616 GiB / 626 GiB avail; 0 B/s wr, 829 op/s; 15509/102915 objects degraded (15.070%); 8742/102915 objects misplaced (8.494%); 776 KiB/s, 12 objects/s recovering 2023-09-23T18:12:49.289 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:12:49.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:49.294 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:12:49.395 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:49.461 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:49 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15509/102915 objects degraded (15.070%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:49.462 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:49 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:49.462 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:49 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15509/102915 objects degraded (15.070%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:49.462 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:49 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:49.465 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:49.529 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:49 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15509/102915 objects degraded (15.070%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:49.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:49 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:12:49.634 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:49.635 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:49.643 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:49.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:49.744 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:12:49.802 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:49.865 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:12:49.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:49.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:12:49.879 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:49.973 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:50.034 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:50.233 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:50.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:50.239 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:50.340 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:50.412 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:50.471 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:50.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:50.487 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:50.491 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:12:50] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.43.0" 2023-09-23T18:12:50.492 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-mon[184174]: pgmap v1051: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.9 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 14259/94722 objects degraded (15.054%); 7849/94722 objects misplaced (8.286%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:12:50.492 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.492 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.492 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.492 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-mon[179668]: pgmap v1051: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.9 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 14259/94722 objects degraded (15.054%); 7849/94722 objects misplaced (8.286%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:12:50.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:50 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.588 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:50.660 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:50 smithi099 ceph-mon[167076]: pgmap v1051: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.9 GiB data, 9.1 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 988 op/s; 14259/94722 objects degraded (15.054%); 7849/94722 objects misplaced (8.286%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:12:50.660 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:50 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.660 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:50 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.660 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:50 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.660 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:50 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:50.677 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:50.749 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:50.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:50.759 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:50.860 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:12:50.951 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:51.095 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:12:51.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:51.107 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:12:51.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:51.293 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:51.320 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:12:51.379 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:51.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:51.386 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:51.487 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:51.569 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:51.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:51.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:51.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:51.730 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:51.778 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:51.850 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:51.856 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:51.857 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:51.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:52.042 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:52.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:52.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:52.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:52.180 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:12:52.180 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:12:52.181 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:12:52.181 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:12:52.181 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:12:52.218 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:52.335 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:52.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:52 smithi055 ceph-mon[184174]: pgmap v1052: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.8 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 935 op/s; 13711/88593 objects degraded (15.476%); 7511/88593 objects misplaced (8.478%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:12:52.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:52.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:52.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:52 smithi055 ceph-mon[179668]: pgmap v1052: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.8 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 935 op/s; 13711/88593 objects degraded (15.476%); 7511/88593 objects misplaced (8.478%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:12:52.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:52.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:52.702 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:52.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:52.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:52.730 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:12:52.741 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:12:52.805 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:12:52.808 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:52.816 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:12:52.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:52 smithi099 ceph-mon[167076]: pgmap v1052: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.8 GiB data, 8.8 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 935 op/s; 13711/88593 objects degraded (15.476%); 7511/88593 objects misplaced (8.478%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:12:52.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:52.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:12:52.934 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:52.983 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:12:53.043 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:12:53.073 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:12:53.133 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:12:53.176 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:12:53.233 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:12:53.281 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:12:53.318 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:12:53.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:12:53.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:12:53.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:12:53.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:12:53.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:12:53.325 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:12:53.325 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:12:53.325 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:12:53.464 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/89325515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:53.464 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 13711/88593 objects degraded (15.476%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:53.465 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/89325515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:53.465 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 13711/88593 objects degraded (15.476%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/89325515' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:12:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 13711/88593 objects degraded (15.476%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:54.207 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:12:54.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:54.213 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:12:54.313 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:54.396 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:54.491 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:54.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:54.510 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:54.611 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:54.668 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:54.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:54 smithi099 ceph-mon[167076]: from='client.36432 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:12:54.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:54 smithi099 ceph-mon[167076]: pgmap v1053: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.7 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 12807/82761 objects degraded (15.475%); 6998/82761 objects misplaced (8.456%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:12:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:54 smithi055 ceph-mon[179668]: from='client.36432 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:12:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:54 smithi055 ceph-mon[179668]: pgmap v1053: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.7 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 12807/82761 objects degraded (15.475%); 6998/82761 objects misplaced (8.456%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:12:54.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:54 smithi055 ceph-mon[184174]: from='client.36432 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:12:54.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:54 smithi055 ceph-mon[184174]: pgmap v1053: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.7 GiB data, 8.5 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 932 op/s; 12807/82761 objects degraded (15.475%); 6998/82761 objects misplaced (8.456%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:12:56.333 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:56.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:56.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:56.442 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:56.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:56 smithi055 ceph-mon[179668]: pgmap v1054: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 12562/81108 objects degraded (15.488%); 6701/81108 objects misplaced (8.262%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:12:56.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:56 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:56.514 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:56.594 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:56.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:56.596 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:56.697 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:12:56.745 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:56 smithi055 ceph-mon[184174]: pgmap v1054: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 12562/81108 objects degraded (15.488%); 6701/81108 objects misplaced (8.262%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:12:56.746 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:56 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:56.782 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:56 smithi099 ceph-mon[167076]: pgmap v1054: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.7 GiB data, 8.4 GiB used, 617 GiB / 626 GiB avail; 0 B/s wr, 931 op/s; 12562/81108 objects degraded (15.488%); 6701/81108 objects misplaced (8.262%); 1.3 MiB/s, 21 objects/s recovering 2023-09-23T18:12:56.782 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:56 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:56.783 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:56.871 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:12:56.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:56.879 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:12:56.980 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:57.055 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:57.057 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1052,"stamp":"2023-09-23T18:12:50.966068+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1938104777,"num_objects":29531,"num_object_clones":0,"num_object_copies":88593,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13711,"num_objects_misplaced":7511,"num_objects_unfound":0,"num_objects_dirty":29531,"num_whiteouts":0,"num_read":6064,"num_read_kb":12355,"num_write":192638,"num_write_kb":7090971,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":26073,"num_bytes_recovered":1644577267,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":84197,"ondisk_log_size":84197,"up":357,"acting":350,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":366,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":9190800,"kb_used_data":6187544,"kb_used_omap":48,"kb_used_meta":3002895,"kb_avail":647025264,"statfs":{"total":671965249536,"available":662553870336,"internally_reserved":0,"allocated":6336045056,"data_stored":6324267009,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":49710,"internal_metadata":3074964946},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,3,5,6,2,6,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-613482496,"num_objects":-9361,"num_object_clones":0,"num_object_copies":-28083,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3578,"num_objects_misplaced":-2368,"num_objects_unfound":0,"num_objects_dirty":-9361,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9361,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":208,"num_bytes_recovered":13631488,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004217"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":707,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498939+0000","last_change":"2023-09-23T18:12:33.314688+0000","last_active":"2023-09-23T18:12:39.498939+0000","last_peered":"2023-09-23T18:12:39.498939+0000","last_clean":"2023-09-23T18:12:39.498939+0000","last_became_active":"2023-09-23T18:12:33.314335+0000","last_became_peered":"2023-09-23T18:12:33.314335+0000","last_unstale":"2023-09-23T18:12:39.498939+0000","last_undegraded":"2023-09-23T18:12:39.498939+0000","last_fullsized":"2023-09-23T18:12:39.498939+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.712627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.712627+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":737225,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":22,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.496177+0000","last_change":"2023-09-23T18:12:06.687066+0000","last_active":"2023-09-23T18:12:39.496177+0000","last_peered":"2023-09-23T18:12:39.496177+0000","last_clean":"2023-09-23T18:12:39.496177+0000","last_became_active":"2023-09-23T18:12:06.667541+0000","last_became_peered":"2023-09-23T18:12:06.667541+0000","last_unstale":"2023-09-23T18:12:39.496177+0000","last_undegraded":"2023-09-23T18:12:39.496177+0000","last_fullsized":"2023-09-23T18:12:39.496177+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:04:48.597217+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":324,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.488989+0000","last_change":"2023-09-23T18:12:06.645901+0000","last_active":"2023-09-23T18:12:39.488989+0000","last_peered":"2023-09-23T18:12:39.488989+0000","last_clean":"2023-09-23T18:12:39.488989+0000","last_became_active":"2023-09-23T18:12:06.645637+0000","last_became_peered":"2023-09-23T18:12:06.645637+0000","last_unstale":"2023-09-23T18:12:39.488989+0000","last_undegraded":"2023-09-23T18:12:39.488989+0000","last_fullsized":"2023-09-23T18:12:39.488989+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:40.588377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":711,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490848+0000","last_change":"2023-09-23T18:12:33.306657+0000","last_active":"2023-09-23T18:12:39.490848+0000","last_peered":"2023-09-23T18:12:39.490848+0000","last_clean":"2023-09-23T18:12:39.490848+0000","last_became_active":"2023-09-23T18:12:33.306262+0000","last_became_peered":"2023-09-23T18:12:33.306262+0000","last_unstale":"2023-09-23T18:12:39.490848+0000","last_undegraded":"2023-09-23T18:12:39.490848+0000","last_fullsized":"2023-09-23T18:12:39.490848+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:51:36.546205+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,7],"acting":[5,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":179,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.489002+0000","last_change":"2023-09-23T18:12:05.655415+0000","last_active":"2023-09-23T18:12:39.489002+0000","last_peered":"2023-09-23T18:12:39.489002+0000","last_clean":"2023-09-23T18:12:39.489002+0000","last_became_active":"2023-09-23T18:12:05.652991+0000","last_became_peered":"2023-09-23T18:12:05.652991+0000","last_unstale":"2023-09-23T18:12:39.489002+0000","last_undegraded":"2023-09-23T18:12:39.489002+0000","last_fullsized":"2023-09-23T18:12:39.489002+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:43.814491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":452,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.504922+0000","last_change":"2023-09-23T18:12:32.314383+0000","last_active":"2023-09-23T18:12:39.504922+0000","last_peered":"2023-09-23T18:12:39.504922+0000","last_clean":"2023-09-23T18:12:39.504922+0000","last_became_active":"2023-09-23T18:12:32.314092+0000","last_became_peered":"2023-09-23T18:12:32.314092+0000","last_unstale":"2023-09-23T18:12:39.504922+0000","last_undegraded":"2023-09-23T18:12:39.504922+0000","last_fullsized":"2023-09-23T18:12:39.504922+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:31.573002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":757,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.499104+0000","last_change":"2023-09-23T18:12:06.649780+0000","last_active":"2023-09-23T18:12:39.499104+0000","last_peered":"2023-09-23T18:12:39.499104+0000","last_clean":"2023-09-23T18:12:39.499104+0000","last_became_active":"2023-09-23T18:12:06.649464+0000","last_became_peered":"2023-09-23T18:12:06.649464+0000","last_unstale":"2023-09-23T18:12:39.499104+0000","last_undegraded":"2023-09-23T18:12:39.499104+0000","last_fullsized":"2023-09-23T18:12:39.499104+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:51.143336+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":327,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.488064+0000","last_change":"2023-09-23T18:12:33.378464+0000","last_active":"2023-09-23T18:12:39.488064+0000","last_peered":"2023-09-23T18:12:39.488064+0000","last_clean":"2023-09-23T18:12:39.488064+0000","last_became_active":"2023-09-23T18:12:33.378128+0000","last_became_peered":"2023-09-23T18:12:33.378128+0000","last_unstale":"2023-09-23T18:12:39.488064+0000","last_undegraded":"2023-09-23T18:12:39.488064+0000","last_fullsized":"2023-09-23T18:12:39.488064+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:02:37.376121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":393,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497240+0000","last_change":"2023-09-23T18:12:32.316801+0000","last_active":"2023-09-23T18:12:39.497240+0000","last_peered":"2023-09-23T18:12:39.497240+0000","last_clean":"2023-09-23T18:12:39.497240+0000","last_became_active":"2023-09-23T18:12:32.316076+0000","last_became_peered":"2023-09-23T18:12:32.316076+0000","last_unstale":"2023-09-23T18:12:39.497240+0000","last_undegraded":"2023-09-23T18:12:39.497240+0000","last_fullsized":"2023-09-23T18:12:39.497240+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:34.054541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":823,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.504795+0000","last_change":"2023-09-23T18:12:32.337404+0000","last_active":"2023-09-23T18:12:39.504795+0000","last_peered":"2023-09-23T18:12:39.504795+0000","last_clean":"2023-09-23T18:12:39.504795+0000","last_became_active":"2023-09-23T18:12:32.337014+0000","last_became_peered":"2023-09-23T18:12:32.337014+0000","last_unstale":"2023-09-23T18:12:39.504795+0000","last_undegraded":"2023-09-23T18:12:39.504795+0000","last_fullsized":"2023-09-23T18:12:39.504795+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:09.079475+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":317,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.491634+0000","last_change":"2023-09-23T18:12:33.309772+0000","last_active":"2023-09-23T18:12:39.491634+0000","last_peered":"2023-09-23T18:12:39.491634+0000","last_clean":"2023-09-23T18:12:39.491634+0000","last_became_active":"2023-09-23T18:12:33.308746+0000","last_became_peered":"2023-09-23T18:12:33.308746+0000","last_unstale":"2023-09-23T18:12:39.491634+0000","last_undegraded":"2023-09-23T18:12:39.491634+0000","last_fullsized":"2023-09-23T18:12:39.491634+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:08.023480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":43,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.484834+0000","last_change":"2023-09-23T18:11:18.177272+0000","last_active":"2023-09-23T18:12:39.484834+0000","last_peered":"2023-09-23T18:12:39.484834+0000","last_clean":"2023-09-23T18:12:39.484834+0000","last_became_active":"2023-09-23T18:11:18.176918+0000","last_became_peered":"2023-09-23T18:11:18.176918+0000","last_unstale":"2023-09-23T18:12:39.484834+0000","last_undegraded":"2023-09-23T18:12:39.484834+0000","last_fullsized":"2023-09-23T18:12:39.484834+0000","mapping_epoch":393,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:41.495118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":774,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.504670+0000","last_change":"2023-09-23T18:12:08.690652+0000","last_active":"2023-09-23T18:12:39.504670+0000","last_peered":"2023-09-23T18:12:39.504670+0000","last_clean":"2023-09-23T18:12:39.504670+0000","last_became_active":"2023-09-23T18:12:08.690415+0000","last_became_peered":"2023-09-23T18:12:08.690415+0000","last_unstale":"2023-09-23T18:12:39.504670+0000","last_undegraded":"2023-09-23T18:12:39.504670+0000","last_fullsized":"2023-09-23T18:12:39.504670+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:02:44.521262+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":339,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.491255+0000","last_change":"2023-09-23T18:12:32.320477+0000","last_active":"2023-09-23T18:12:39.491255+0000","last_peered":"2023-09-23T18:12:39.491255+0000","last_clean":"2023-09-23T18:12:39.491255+0000","last_became_active":"2023-09-23T18:12:32.320176+0000","last_became_peered":"2023-09-23T18:12:32.320176+0000","last_unstale":"2023-09-23T18:12:39.491255+0000","last_undegraded":"2023-09-23T18:12:39.491255+0000","last_fullsized":"2023-09-23T18:12:39.491255+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:19.404451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":235,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498066+0000","last_change":"2023-09-23T18:11:19.167247+0000","last_active":"2023-09-23T18:12:39.498066+0000","last_peered":"2023-09-23T18:12:39.498066+0000","last_clean":"2023-09-23T18:12:39.498066+0000","last_became_active":"2023-09-23T18:11:19.166959+0000","last_became_peered":"2023-09-23T18:11:19.166959+0000","last_unstale":"2023-09-23T18:12:39.498066+0000","last_undegraded":"2023-09-23T18:12:39.498066+0000","last_fullsized":"2023-09-23T18:12:39.498066+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:07.487191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":712,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.491444+0000","last_change":"2023-09-23T18:12:32.317402+0000","last_active":"2023-09-23T18:12:39.491444+0000","last_peered":"2023-09-23T18:12:39.491444+0000","last_clean":"2023-09-23T18:12:39.491444+0000","last_became_active":"2023-09-23T18:12:32.316944+0000","last_became_peered":"2023-09-23T18:12:32.316944+0000","last_unstale":"2023-09-23T18:12:39.491444+0000","last_undegraded":"2023-09-23T18:12:39.491444+0000","last_fullsized":"2023-09-23T18:12:39.491444+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.652623+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.652623+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":255,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.504632+0000","last_change":"2023-09-23T18:12:31.325394+0000","last_active":"2023-09-23T18:12:39.504632+0000","last_peered":"2023-09-23T18:12:39.504632+0000","last_clean":"2023-09-23T18:12:39.504632+0000","last_became_active":"2023-09-23T18:12:05.654979+0000","last_became_peered":"2023-09-23T18:12:05.654979+0000","last_unstale":"2023-09-23T18:12:39.504632+0000","last_undegraded":"2023-09-23T18:12:39.504632+0000","last_fullsized":"2023-09-23T18:12:39.504632+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:50.669737+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":630,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":173,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.488576+0000","last_change":"2023-09-23T18:12:33.306077+0000","last_active":"2023-09-23T18:12:39.488576+0000","last_peered":"2023-09-23T18:12:39.488576+0000","last_clean":"2023-09-23T18:12:39.488576+0000","last_became_active":"2023-09-23T18:12:33.305802+0000","last_became_peered":"2023-09-23T18:12:33.305802+0000","last_unstale":"2023-09-23T18:12:39.488576+0000","last_undegraded":"2023-09-23T18:12:39.488576+0000","last_fullsized":"2023-09-23T18:12:39.488576+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:53:54.619757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,5],"acting":[1,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":846,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.489714+0000","last_change":"2023-09-23T18:12:05.654051+0000","last_active":"2023-09-23T18:12:39.489714+0000","last_peered":"2023-09-23T18:12:39.489714+0000","last_clean":"2023-09-23T18:12:39.489714+0000","last_became_active":"2023-09-23T18:12:05.653587+0000","last_became_peered":"2023-09-23T18:12:05.653587+0000","last_unstale":"2023-09-23T18:12:39.489714+0000","last_undegraded":"2023-09-23T18:12:39.489714+0000","last_fullsized":"2023-09-23T18:12:39.489714+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:10.417643+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":212,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.502324+0000","last_change":"2023-09-23T18:12:06.679404+0000","last_active":"2023-09-23T18:12:39.502324+0000","last_peered":"2023-09-23T18:12:39.502324+0000","last_clean":"2023-09-23T18:12:39.502324+0000","last_became_active":"2023-09-23T18:12:06.679106+0000","last_became_peered":"2023-09-23T18:12:06.679106+0000","last_unstale":"2023-09-23T18:12:39.502324+0000","last_undegraded":"2023-09-23T18:12:39.502324+0000","last_fullsized":"2023-09-23T18:12:39.502324+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:48.320006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":230,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498240+0000","last_change":"2023-09-23T18:12:06.645467+0000","last_active":"2023-09-23T18:12:39.498240+0000","last_peered":"2023-09-23T18:12:39.498240+0000","last_clean":"2023-09-23T18:12:39.498240+0000","last_became_active":"2023-09-23T18:12:06.642537+0000","last_became_peered":"2023-09-23T18:12:06.642537+0000","last_unstale":"2023-09-23T18:12:39.498240+0000","last_undegraded":"2023-09-23T18:12:39.498240+0000","last_fullsized":"2023-09-23T18:12:39.498240+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:37.741322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":690,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490641+0000","last_change":"2023-09-23T18:12:32.338144+0000","last_active":"2023-09-23T18:12:39.490641+0000","last_peered":"2023-09-23T18:12:39.490641+0000","last_clean":"2023-09-23T18:12:39.490641+0000","last_became_active":"2023-09-23T18:12:32.337238+0000","last_became_peered":"2023-09-23T18:12:32.337238+0000","last_unstale":"2023-09-23T18:12:39.490641+0000","last_undegraded":"2023-09-23T18:12:39.490641+0000","last_fullsized":"2023-09-23T18:12:39.490641+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.869576+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":402,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.502854+0000","last_change":"2023-09-23T18:12:31.312539+0000","last_active":"2023-09-23T18:12:39.502854+0000","last_peered":"2023-09-23T18:12:39.502854+0000","last_clean":"2023-09-23T18:12:39.502854+0000","last_became_active":"2023-09-23T18:12:05.650685+0000","last_became_peered":"2023-09-23T18:12:05.650685+0000","last_unstale":"2023-09-23T18:12:39.502854+0000","last_undegraded":"2023-09-23T18:12:39.502854+0000","last_fullsized":"2023-09-23T18:12:39.502854+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:54.312641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":263,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.488294+0000","last_change":"2023-09-23T18:11:19.185574+0000","last_active":"2023-09-23T18:12:39.488294+0000","last_peered":"2023-09-23T18:12:39.488294+0000","last_clean":"2023-09-23T18:12:39.488294+0000","last_became_active":"2023-09-23T18:11:19.185301+0000","last_became_peered":"2023-09-23T18:11:19.185301+0000","last_unstale":"2023-09-23T18:12:39.488294+0000","last_undegraded":"2023-09-23T18:12:39.488294+0000","last_fullsized":"2023-09-23T18:12:39.488294+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:21:43.081121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":697,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498580+0000","last_change":"2023-09-23T18:12:32.307660+0000","last_active":"2023-09-23T18:12:39.498580+0000","last_peered":"2023-09-23T18:12:39.498580+0000","last_clean":"2023-09-23T18:12:39.498580+0000","last_became_active":"2023-09-23T18:12:32.307314+0000","last_became_peered":"2023-09-23T18:12:32.307314+0000","last_unstale":"2023-09-23T18:12:39.498580+0000","last_undegraded":"2023-09-23T18:12:39.498580+0000","last_fullsized":"2023-09-23T18:12:39.498580+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:38.292879+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:38.292879+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":457,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.502985+0000","last_change":"2023-09-23T18:12:33.337295+0000","last_active":"2023-09-23T18:12:39.502985+0000","last_peered":"2023-09-23T18:12:39.502985+0000","last_clean":"2023-09-23T18:12:39.502985+0000","last_became_active":"2023-09-23T18:12:33.336974+0000","last_became_peered":"2023-09-23T18:12:33.336974+0000","last_unstale":"2023-09-23T18:12:39.502985+0000","last_undegraded":"2023-09-23T18:12:39.502985+0000","last_fullsized":"2023-09-23T18:12:39.502985+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:51.011158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":357,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.492006+0000","last_change":"2023-09-23T18:12:33.305642+0000","last_active":"2023-09-23T18:12:39.492006+0000","last_peered":"2023-09-23T18:12:39.492006+0000","last_clean":"2023-09-23T18:12:39.492006+0000","last_became_active":"2023-09-23T18:12:33.305291+0000","last_became_peered":"2023-09-23T18:12:33.305291+0000","last_unstale":"2023-09-23T18:12:39.492006+0000","last_undegraded":"2023-09-23T18:12:39.492006+0000","last_fullsized":"2023-09-23T18:12:39.492006+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:17:54.709365+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":810,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490316+0000","last_change":"2023-09-23T18:12:06.645815+0000","last_active":"2023-09-23T18:12:39.490316+0000","last_peered":"2023-09-23T18:12:39.490316+0000","last_clean":"2023-09-23T18:12:39.490316+0000","last_became_active":"2023-09-23T18:12:06.645452+0000","last_became_peered":"2023-09-23T18:12:06.645452+0000","last_unstale":"2023-09-23T18:12:39.490316+0000","last_undegraded":"2023-09-23T18:12:39.490316+0000","last_fullsized":"2023-09-23T18:12:39.490316+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:24.550609+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":23,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.500209+0000","last_change":"2023-09-23T18:12:05.646034+0000","last_active":"2023-09-23T18:12:39.500209+0000","last_peered":"2023-09-23T18:12:39.500209+0000","last_clean":"2023-09-23T18:12:39.500209+0000","last_became_active":"2023-09-23T18:12:05.645701+0000","last_became_peered":"2023-09-23T18:12:05.645701+0000","last_unstale":"2023-09-23T18:12:39.500209+0000","last_undegraded":"2023-09-23T18:12:39.500209+0000","last_fullsized":"2023-09-23T18:12:39.500209+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:22.277691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":376,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.484724+0000","last_change":"2023-09-23T18:11:19.168750+0000","last_active":"2023-09-23T18:12:39.484724+0000","last_peered":"2023-09-23T18:12:39.484724+0000","last_clean":"2023-09-23T18:12:39.484724+0000","last_became_active":"2023-09-23T18:11:19.168447+0000","last_became_peered":"2023-09-23T18:11:19.168447+0000","last_unstale":"2023-09-23T18:12:39.484724+0000","last_undegraded":"2023-09-23T18:12:39.484724+0000","last_fullsized":"2023-09-23T18:12:39.484724+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:22.907052+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":687,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490801+0000","last_change":"2023-09-23T18:12:32.335469+0000","last_active":"2023-09-23T18:12:39.490801+0000","last_peered":"2023-09-23T18:12:39.490801+0000","last_clean":"2023-09-23T18:12:39.490801+0000","last_became_active":"2023-09-23T18:12:32.334685+0000","last_became_peered":"2023-09-23T18:12:32.334685+0000","last_unstale":"2023-09-23T18:12:39.490801+0000","last_undegraded":"2023-09-23T18:12:39.490801+0000","last_fullsized":"2023-09-23T18:12:39.490801+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:41.036373+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":443,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.499347+0000","last_change":"2023-09-23T18:12:33.309551+0000","last_active":"2023-09-23T18:12:39.499347+0000","last_peered":"2023-09-23T18:12:39.499347+0000","last_clean":"2023-09-23T18:12:39.499347+0000","last_became_active":"2023-09-23T18:12:33.309236+0000","last_became_peered":"2023-09-23T18:12:33.309236+0000","last_unstale":"2023-09-23T18:12:39.499347+0000","last_undegraded":"2023-09-23T18:12:39.499347+0000","last_fullsized":"2023-09-23T18:12:39.499347+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:34:07.006152+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":290,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.489468+0000","last_change":"2023-09-23T18:10:51.816822+0000","last_active":"2023-09-23T18:12:39.489468+0000","last_peered":"2023-09-23T18:12:39.489468+0000","last_clean":"2023-09-23T18:12:39.489468+0000","last_became_active":"2023-09-23T18:10:51.816526+0000","last_became_peered":"2023-09-23T18:10:51.816526+0000","last_unstale":"2023-09-23T18:12:39.489468+0000","last_undegraded":"2023-09-23T18:12:39.489468+0000","last_fullsized":"2023-09-23T18:12:39.489468+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:31.194789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.4","version":"426'11948","reported_seq":12047,"reported_epoch":426,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:12:50.916188+0000","last_change":"2023-09-23T18:12:33.428642+0000","last_active":"2023-09-23T18:12:50.916188+0000","last_peered":"2023-09-23T18:12:50.916188+0000","last_clean":"2023-09-23T18:09:48.184212+0000","last_became_active":"2023-09-23T18:12:33.397325+0000","last_became_peered":"2023-09-23T18:12:33.397325+0000","last_unstale":"2023-09-23T18:12:50.916188+0000","last_undegraded":"2023-09-23T18:12:33.314766+0000","last_fullsized":"2023-09-23T18:12:33.294901+0000","mapping_epoch":421,"log_start":"391'7385","ondisk_log_start":"391'7385","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4563,"log_dups_size":0,"ondisk_log_size":4563,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:36.787038+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117047296,"num_objects":1786,"num_object_clones":0,"num_object_copies":5358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1786,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11948,"num_write_kb":439488,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[4,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"4,6","objects":1786}],"blocked_by":[],"up_primary":5,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":846,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.504534+0000","last_change":"2023-09-23T18:12:08.689111+0000","last_active":"2023-09-23T18:12:39.504534+0000","last_peered":"2023-09-23T18:12:39.504534+0000","last_clean":"2023-09-23T18:12:39.504534+0000","last_became_active":"2023-09-23T18:12:08.688825+0000","last_became_peered":"2023-09-23T18:12:08.688825+0000","last_unstale":"2023-09-23T18:12:39.504534+0000","last_undegraded":"2023-09-23T18:12:39.504534+0000","last_fullsized":"2023-09-23T18:12:39.504534+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:29.016205+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":392,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.505171+0000","last_change":"2023-09-23T18:12:06.652515+0000","last_active":"2023-09-23T18:12:39.505171+0000","last_peered":"2023-09-23T18:12:39.505171+0000","last_clean":"2023-09-23T18:12:39.505171+0000","last_became_active":"2023-09-23T18:12:06.652196+0000","last_became_peered":"2023-09-23T18:12:06.652196+0000","last_unstale":"2023-09-23T18:12:39.505171+0000","last_undegraded":"2023-09-23T18:12:39.505171+0000","last_fullsized":"2023-09-23T18:12:39.505171+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:27.975688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":340,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498726+0000","last_change":"2023-09-23T18:12:32.311619+0000","last_active":"2023-09-23T18:12:39.498726+0000","last_peered":"2023-09-23T18:12:39.498726+0000","last_clean":"2023-09-23T18:12:39.498726+0000","last_became_active":"2023-09-23T18:12:32.311311+0000","last_became_peered":"2023-09-23T18:12:32.311311+0000","last_unstale":"2023-09-23T18:12:39.498726+0000","last_undegraded":"2023-09-23T18:12:39.498726+0000","last_fullsized":"2023-09-23T18:12:39.498726+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:30.980099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.5","version":"426'12028","reported_seq":12124,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T18:12:50.712933+0000","last_change":"2023-09-23T18:12:06.670738+0000","last_active":"2023-09-23T18:12:50.712933+0000","last_peered":"2023-09-23T18:12:50.712933+0000","last_clean":"2023-09-23T18:12:50.712933+0000","last_became_active":"2023-09-23T18:12:06.646229+0000","last_became_peered":"2023-09-23T18:12:06.646229+0000","last_unstale":"2023-09-23T18:12:50.712933+0000","last_undegraded":"2023-09-23T18:12:50.712933+0000","last_fullsized":"2023-09-23T18:12:50.712933+0000","mapping_epoch":413,"log_start":"391'7319","ondisk_log_start":"391'7319","created":361,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4709,"log_dups_size":0,"ondisk_log_size":4709,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:31.975713+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":118095872,"num_objects":1802,"num_object_clones":0,"num_object_copies":5406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12028,"num_write_kb":442560,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":513,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498861+0000","last_change":"2023-09-23T18:12:05.644364+0000","last_active":"2023-09-23T18:12:39.498861+0000","last_peered":"2023-09-23T18:12:39.498861+0000","last_clean":"2023-09-23T18:12:39.498861+0000","last_became_active":"2023-09-23T18:12:05.644014+0000","last_became_peered":"2023-09-23T18:12:05.644014+0000","last_unstale":"2023-09-23T18:12:39.498861+0000","last_undegraded":"2023-09-23T18:12:39.498861+0000","last_fullsized":"2023-09-23T18:12:39.498861+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:41.510146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:41.510146+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:03.919547+0000","scrub_duration":0.0078303169999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":407,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.505108+0000","last_change":"2023-09-23T18:12:06.653146+0000","last_active":"2023-09-23T18:12:39.505108+0000","last_peered":"2023-09-23T18:12:39.505108+0000","last_clean":"2023-09-23T18:12:39.505108+0000","last_became_active":"2023-09-23T18:12:06.652832+0000","last_became_peered":"2023-09-23T18:12:06.652832+0000","last_unstale":"2023-09-23T18:12:39.505108+0000","last_undegraded":"2023-09-23T18:12:39.505108+0000","last_fullsized":"2023-09-23T18:12:39.505108+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:34.147471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":23,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.499977+0000","last_change":"2023-09-23T18:12:05.655221+0000","last_active":"2023-09-23T18:12:39.499977+0000","last_peered":"2023-09-23T18:12:39.499977+0000","last_clean":"2023-09-23T18:12:39.499977+0000","last_became_active":"2023-09-23T18:12:05.654901+0000","last_became_peered":"2023-09-23T18:12:05.654901+0000","last_unstale":"2023-09-23T18:12:39.499977+0000","last_undegraded":"2023-09-23T18:12:39.499977+0000","last_fullsized":"2023-09-23T18:12:39.499977+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:56.947639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.6","version":"426'12197","reported_seq":19036,"reported_epoch":426,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:12:50.719254+0000","last_change":"2023-09-23T18:12:33.669607+0000","last_active":"2023-09-23T18:12:50.719254+0000","last_peered":"2023-09-23T18:12:50.719254+0000","last_clean":"2023-09-23T18:10:00.095783+0000","last_became_active":"2023-09-23T18:11:41.534114+0000","last_became_peered":"2023-09-23T18:11:41.534114+0000","last_unstale":"2023-09-23T18:12:50.719254+0000","last_undegraded":"2023-09-23T18:12:33.669450+0000","last_fullsized":"2023-09-23T18:12:50.719254+0000","mapping_epoch":404,"log_start":"391'7434","ondisk_log_start":"391'7434","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4763,"log_dups_size":0,"ondisk_log_size":4763,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:23.957244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":115539968,"num_objects":1763,"num_object_clones":0,"num_object_copies":5289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":251,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12197,"num_write_kb":446720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3374,"num_bytes_recovered":169148416,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,7],"acting":[6,0,7],"avail_no_missing":["6","2","7"],"object_location_counts":[{"shards":"0,6,7","objects":1763}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":648,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497436+0000","last_change":"2023-09-23T18:12:33.317652+0000","last_active":"2023-09-23T18:12:39.497436+0000","last_peered":"2023-09-23T18:12:39.497436+0000","last_clean":"2023-09-23T18:12:39.497436+0000","last_became_active":"2023-09-23T18:12:33.317203+0000","last_became_peered":"2023-09-23T18:12:33.317203+0000","last_unstale":"2023-09-23T18:12:39.497436+0000","last_undegraded":"2023-09-23T18:12:39.497436+0000","last_fullsized":"2023-09-23T18:12:39.497436+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:56.401433+0000","scrub_duration":0.0018893740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":309,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.504863+0000","last_change":"2023-09-23T18:12:32.336820+0000","last_active":"2023-09-23T18:12:39.504863+0000","last_peered":"2023-09-23T18:12:39.504863+0000","last_clean":"2023-09-23T18:12:39.504863+0000","last_became_active":"2023-09-23T18:12:32.336503+0000","last_became_peered":"2023-09-23T18:12:32.336503+0000","last_unstale":"2023-09-23T18:12:39.504863+0000","last_undegraded":"2023-09-23T18:12:39.504863+0000","last_fullsized":"2023-09-23T18:12:39.504863+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:18.676067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":401,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497727+0000","last_change":"2023-09-23T18:06:05.371447+0000","last_active":"2023-09-23T18:12:39.497727+0000","last_peered":"2023-09-23T18:12:39.497727+0000","last_clean":"2023-09-23T18:12:39.497727+0000","last_became_active":"2023-09-23T18:06:05.371065+0000","last_became_peered":"2023-09-23T18:06:05.371065+0000","last_unstale":"2023-09-23T18:12:39.497727+0000","last_undegraded":"2023-09-23T18:12:39.497727+0000","last_fullsized":"2023-09-23T18:12:39.497727+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.025791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.7","version":"426'11821","reported_seq":11921,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T18:12:47.786452+0000","last_change":"2023-09-23T18:11:28.545256+0000","last_active":"2023-09-23T18:12:47.786452+0000","last_peered":"2023-09-23T18:12:47.786452+0000","last_clean":"2023-09-23T18:12:47.786452+0000","last_became_active":"2023-09-23T18:11:19.181733+0000","last_became_peered":"2023-09-23T18:11:19.181733+0000","last_unstale":"2023-09-23T18:12:47.786452+0000","last_undegraded":"2023-09-23T18:12:47.786452+0000","last_fullsized":"2023-09-23T18:12:47.786452+0000","mapping_epoch":394,"log_start":"392'7507","ondisk_log_start":"392'7507","created":361,"last_epoch_clean":397,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4314,"log_dups_size":0,"ondisk_log_size":4314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:18.957805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":123011072,"num_objects":1877,"num_object_clones":0,"num_object_copies":5631,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1877,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11821,"num_write_kb":438336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,6],"acting":[1,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":638,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.499632+0000","last_change":"2023-09-23T18:12:08.703409+0000","last_active":"2023-09-23T18:12:39.499632+0000","last_peered":"2023-09-23T18:12:39.499632+0000","last_clean":"2023-09-23T18:12:39.499632+0000","last_became_active":"2023-09-23T18:12:08.703079+0000","last_became_peered":"2023-09-23T18:12:08.703079+0000","last_unstale":"2023-09-23T18:12:39.499632+0000","last_undegraded":"2023-09-23T18:12:39.499632+0000","last_fullsized":"2023-09-23T18:12:39.499632+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.362652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.362652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":351,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.491342+0000","last_change":"2023-09-23T18:12:32.320529+0000","last_active":"2023-09-23T18:12:39.491342+0000","last_peered":"2023-09-23T18:12:39.491342+0000","last_clean":"2023-09-23T18:12:39.491342+0000","last_became_active":"2023-09-23T18:12:32.320241+0000","last_became_peered":"2023-09-23T18:12:32.320241+0000","last_unstale":"2023-09-23T18:12:39.491342+0000","last_undegraded":"2023-09-23T18:12:39.491342+0000","last_fullsized":"2023-09-23T18:12:39.491342+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:31.121529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":360,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490231+0000","last_change":"2023-09-23T18:12:05.655791+0000","last_active":"2023-09-23T18:12:39.490231+0000","last_peered":"2023-09-23T18:12:39.490231+0000","last_clean":"2023-09-23T18:12:39.490231+0000","last_became_active":"2023-09-23T18:12:05.655227+0000","last_became_peered":"2023-09-23T18:12:05.655227+0000","last_unstale":"2023-09-23T18:12:39.490231+0000","last_undegraded":"2023-09-23T18:12:39.490231+0000","last_fullsized":"2023-09-23T18:12:39.490231+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:37.413645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.0","version":"426'11956","reported_seq":12055,"reported_epoch":426,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:12:47.793759+0000","last_change":"2023-09-23T18:12:33.543861+0000","last_active":"2023-09-23T18:12:47.793759+0000","last_peered":"2023-09-23T18:12:47.793759+0000","last_clean":"2023-09-23T18:09:48.233578+0000","last_became_active":"2023-09-23T18:12:33.511751+0000","last_became_peered":"2023-09-23T18:12:33.511751+0000","last_unstale":"2023-09-23T18:12:47.793759+0000","last_undegraded":"2023-09-23T18:12:33.306811+0000","last_fullsized":"2023-09-23T18:12:33.289160+0000","mapping_epoch":421,"log_start":"392'7620","ondisk_log_start":"392'7620","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4336,"log_dups_size":0,"ondisk_log_size":4336,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:18.266191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121765888,"num_objects":1858,"num_object_clones":0,"num_object_copies":5574,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1858,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1858,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11956,"num_write_kb":442048,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[1,3],"avail_no_missing":["1","3"],"object_location_counts":[{"shards":"1,3","objects":1858}],"blocked_by":[],"up_primary":5,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":849,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.502923+0000","last_change":"2023-09-23T18:12:33.375834+0000","last_active":"2023-09-23T18:12:39.502923+0000","last_peered":"2023-09-23T18:12:39.502923+0000","last_clean":"2023-09-23T18:12:39.502923+0000","last_became_active":"2023-09-23T18:12:33.375454+0000","last_became_peered":"2023-09-23T18:12:33.375454+0000","last_unstale":"2023-09-23T18:12:39.502923+0000","last_undegraded":"2023-09-23T18:12:39.502923+0000","last_fullsized":"2023-09-23T18:12:39.502923+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:10:51.059655+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":237,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498372+0000","last_change":"2023-09-23T18:12:05.656901+0000","last_active":"2023-09-23T18:12:39.498372+0000","last_peered":"2023-09-23T18:12:39.498372+0000","last_clean":"2023-09-23T18:12:39.498372+0000","last_became_active":"2023-09-23T18:12:05.656162+0000","last_became_peered":"2023-09-23T18:12:05.656162+0000","last_unstale":"2023-09-23T18:12:39.498372+0000","last_undegraded":"2023-09-23T18:12:39.498372+0000","last_fullsized":"2023-09-23T18:12:39.498372+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:54:58.044693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":23,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.499650+0000","last_change":"2023-09-23T18:12:05.654316+0000","last_active":"2023-09-23T18:12:39.499650+0000","last_peered":"2023-09-23T18:12:39.499650+0000","last_clean":"2023-09-23T18:12:39.499650+0000","last_became_active":"2023-09-23T18:12:05.653952+0000","last_became_peered":"2023-09-23T18:12:05.653952+0000","last_unstale":"2023-09-23T18:12:39.499650+0000","last_undegraded":"2023-09-23T18:12:39.499650+0000","last_fullsized":"2023-09-23T18:12:39.499650+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:35:40.264045+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.1","version":"426'11861","reported_seq":19140,"reported_epoch":426,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:12:47.857914+0000","last_change":"2023-09-23T18:12:32.427105+0000","last_active":"2023-09-23T18:12:47.857914+0000","last_peered":"2023-09-23T18:12:47.857914+0000","last_clean":"2023-09-23T18:09:48.228392+0000","last_became_active":"2023-09-23T18:12:32.373840+0000","last_became_peered":"2023-09-23T18:12:32.373840+0000","last_unstale":"2023-09-23T18:12:47.857914+0000","last_undegraded":"2023-09-23T18:12:47.857914+0000","last_fullsized":"2023-09-23T18:12:47.857914+0000","mapping_epoch":420,"log_start":"392'7540","ondisk_log_start":"392'7540","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4321,"log_dups_size":0,"ondisk_log_size":4321,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:11.153881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":122355712,"num_objects":1867,"num_object_clones":0,"num_object_copies":5601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1865,"num_objects_unfound":0,"num_objects_dirty":1867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11861,"num_write_kb":439296,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3584,"num_bytes_recovered":234881024,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,2],"avail_no_missing":["1","0","2"],"object_location_counts":[{"shards":"0,1,2","objects":1867}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":706,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.499033+0000","last_change":"2023-09-23T18:12:33.305592+0000","last_active":"2023-09-23T18:12:39.499033+0000","last_peered":"2023-09-23T18:12:39.499033+0000","last_clean":"2023-09-23T18:12:39.499033+0000","last_became_active":"2023-09-23T18:12:33.305207+0000","last_became_peered":"2023-09-23T18:12:33.305207+0000","last_unstale":"2023-09-23T18:12:39.499033+0000","last_undegraded":"2023-09-23T18:12:39.499033+0000","last_fullsized":"2023-09-23T18:12:39.499033+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:37.013028+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":415,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497983+0000","last_change":"2023-09-23T18:12:05.655148+0000","last_active":"2023-09-23T18:12:39.497983+0000","last_peered":"2023-09-23T18:12:39.497983+0000","last_clean":"2023-09-23T18:12:39.497983+0000","last_became_active":"2023-09-23T18:12:05.654825+0000","last_became_peered":"2023-09-23T18:12:05.654825+0000","last_unstale":"2023-09-23T18:12:39.497983+0000","last_undegraded":"2023-09-23T18:12:39.497983+0000","last_fullsized":"2023-09-23T18:12:39.497983+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:09.300269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":240,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497537+0000","last_change":"2023-09-23T18:12:06.646164+0000","last_active":"2023-09-23T18:12:39.497537+0000","last_peered":"2023-09-23T18:12:39.497537+0000","last_clean":"2023-09-23T18:12:39.497537+0000","last_became_active":"2023-09-23T18:12:06.645663+0000","last_became_peered":"2023-09-23T18:12:06.645663+0000","last_unstale":"2023-09-23T18:12:39.497537+0000","last_undegraded":"2023-09-23T18:12:39.497537+0000","last_fullsized":"2023-09-23T18:12:39.497537+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:09:25.444493+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.2","version":"426'12009","reported_seq":19000,"reported_epoch":426,"state":"active+recovery_wait+degraded+remapped","last_fresh":"2023-09-23T18:12:50.951267+0000","last_change":"2023-09-23T18:12:32.432775+0000","last_active":"2023-09-23T18:12:50.951267+0000","last_peered":"2023-09-23T18:12:50.951267+0000","last_clean":"2023-09-23T18:09:48.231362+0000","last_became_active":"2023-09-23T18:12:32.399543+0000","last_became_peered":"2023-09-23T18:12:32.399543+0000","last_unstale":"2023-09-23T18:12:50.951267+0000","last_undegraded":"2023-09-23T18:12:32.339100+0000","last_fullsized":"2023-09-23T18:12:50.951267+0000","mapping_epoch":420,"log_start":"391'7471","ondisk_log_start":"391'7471","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4538,"log_dups_size":0,"ondisk_log_size":4538,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:10:08.963749+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":114884608,"num_objects":1753,"num_object_clones":0,"num_object_copies":5259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":91,"num_objects_misplaced":1753,"num_objects_unfound":0,"num_objects_dirty":1753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12009,"num_write_kb":440384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3434,"num_bytes_recovered":225050624,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,6],"avail_no_missing":["4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1753}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":820,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490372+0000","last_change":"2023-09-23T18:12:33.325300+0000","last_active":"2023-09-23T18:12:39.490372+0000","last_peered":"2023-09-23T18:12:39.490372+0000","last_clean":"2023-09-23T18:12:39.490372+0000","last_became_active":"2023-09-23T18:12:33.324847+0000","last_became_peered":"2023-09-23T18:12:33.324847+0000","last_unstale":"2023-09-23T18:12:39.490372+0000","last_undegraded":"2023-09-23T18:12:39.490372+0000","last_fullsized":"2023-09-23T18:12:39.490372+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:23.403910+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":220,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.500338+0000","last_change":"2023-09-23T18:12:06.653158+0000","last_active":"2023-09-23T18:12:39.500338+0000","last_peered":"2023-09-23T18:12:39.500338+0000","last_clean":"2023-09-23T18:12:39.500338+0000","last_became_active":"2023-09-23T18:12:06.651673+0000","last_became_peered":"2023-09-23T18:12:06.651673+0000","last_unstale":"2023-09-23T18:12:39.500338+0000","last_undegraded":"2023-09-23T18:12:39.500338+0000","last_fullsized":"2023-09-23T18:12:39.500338+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:54:09.741505+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":23,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.499722+0000","last_change":"2023-09-23T18:12:05.654018+0000","last_active":"2023-09-23T18:12:39.499722+0000","last_peered":"2023-09-23T18:12:39.499722+0000","last_clean":"2023-09-23T18:12:39.499722+0000","last_became_active":"2023-09-23T18:12:05.653113+0000","last_became_peered":"2023-09-23T18:12:05.653113+0000","last_unstale":"2023-09-23T18:12:39.499722+0000","last_undegraded":"2023-09-23T18:12:39.499722+0000","last_fullsized":"2023-09-23T18:12:39.499722+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:24.208896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"8.3","version":"426'12029","reported_seq":12127,"reported_epoch":426,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:12:47.838680+0000","last_change":"2023-09-23T18:12:33.435370+0000","last_active":"2023-09-23T18:12:47.838680+0000","last_peered":"2023-09-23T18:12:47.838680+0000","last_clean":"2023-09-23T18:09:48.219569+0000","last_became_active":"2023-09-23T18:12:33.385164+0000","last_became_peered":"2023-09-23T18:12:33.385164+0000","last_unstale":"2023-09-23T18:12:47.838680+0000","last_undegraded":"2023-09-23T18:12:33.435226+0000","last_fullsized":"2023-09-23T18:12:33.289570+0000","mapping_epoch":421,"log_start":"392'7645","ondisk_log_start":"392'7645","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4384,"log_dups_size":0,"ondisk_log_size":4384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:17.435206+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":126418944,"num_objects":1929,"num_object_clones":0,"num_object_copies":5787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1929,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12029,"num_write_kb":446656,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1929}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":699,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490515+0000","last_change":"2023-09-23T18:12:33.311093+0000","last_active":"2023-09-23T18:12:39.490515+0000","last_peered":"2023-09-23T18:12:39.490515+0000","last_clean":"2023-09-23T18:12:39.490515+0000","last_became_active":"2023-09-23T18:12:33.310618+0000","last_became_peered":"2023-09-23T18:12:33.310618+0000","last_unstale":"2023-09-23T18:12:39.490515+0000","last_undegraded":"2023-09-23T18:12:39.490515+0000","last_fullsized":"2023-09-23T18:12:39.490515+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":292,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.487974+0000","last_change":"2023-09-23T18:12:05.652577+0000","last_active":"2023-09-23T18:12:39.487974+0000","last_peered":"2023-09-23T18:12:39.487974+0000","last_clean":"2023-09-23T18:12:39.487974+0000","last_became_active":"2023-09-23T18:12:05.652290+0000","last_became_peered":"2023-09-23T18:12:05.652290+0000","last_unstale":"2023-09-23T18:12:39.487974+0000","last_undegraded":"2023-09-23T18:12:39.487974+0000","last_fullsized":"2023-09-23T18:12:39.487974+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:32.320757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":189,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.489397+0000","last_change":"2023-09-23T18:12:05.679491+0000","last_active":"2023-09-23T18:12:39.489397+0000","last_peered":"2023-09-23T18:12:39.489397+0000","last_clean":"2023-09-23T18:12:39.489397+0000","last_became_active":"2023-09-23T18:12:05.657868+0000","last_became_peered":"2023-09-23T18:12:05.657868+0000","last_unstale":"2023-09-23T18:12:39.489397+0000","last_undegraded":"2023-09-23T18:12:39.489397+0000","last_fullsized":"2023-09-23T18:12:39.489397+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:30.473658+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.e","version":"426'12143","reported_seq":22292,"reported_epoch":426,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:12:50.883790+0000","last_change":"2023-09-23T18:12:32.432869+0000","last_active":"2023-09-23T18:12:50.883790+0000","last_peered":"2023-09-23T18:12:50.883790+0000","last_clean":"2023-09-23T18:10:50.781383+0000","last_became_active":"2023-09-23T18:12:06.651363+0000","last_became_peered":"2023-09-23T18:12:06.651363+0000","last_unstale":"2023-09-23T18:12:50.883790+0000","last_undegraded":"2023-09-23T18:12:50.883790+0000","last_fullsized":"2023-09-23T18:12:50.883790+0000","mapping_epoch":413,"log_start":"392'7609","ondisk_log_start":"392'7609","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4534,"log_dups_size":0,"ondisk_log_size":4534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:45:39.565633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":110821376,"num_objects":1691,"num_object_clones":0,"num_object_copies":5073,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":146,"num_objects_unfound":0,"num_objects_dirty":1691,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12143,"num_write_kb":442688,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5019,"num_bytes_recovered":328925184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,2],"avail_no_missing":["4","2","7"],"object_location_counts":[{"shards":"2,4,7","objects":1691}],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":634,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497173+0000","last_change":"2023-09-23T18:12:32.316992+0000","last_active":"2023-09-23T18:12:39.497173+0000","last_peered":"2023-09-23T18:12:39.497173+0000","last_clean":"2023-09-23T18:12:39.497173+0000","last_became_active":"2023-09-23T18:12:32.316484+0000","last_became_peered":"2023-09-23T18:12:32.316484+0000","last_unstale":"2023-09-23T18:12:39.497173+0000","last_undegraded":"2023-09-23T18:12:39.497173+0000","last_fullsized":"2023-09-23T18:12:39.497173+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:21.933684+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":830,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T18:12:46.038920+0000","last_change":"2023-09-23T18:10:04.168459+0000","last_active":"2023-09-23T18:12:46.038920+0000","last_peered":"2023-09-23T18:12:46.038920+0000","last_clean":"2023-09-23T18:12:46.038920+0000","last_became_active":"2023-09-23T18:10:04.167807+0000","last_became_peered":"2023-09-23T18:10:04.167807+0000","last_unstale":"2023-09-23T18:12:46.038920+0000","last_undegraded":"2023-09-23T18:12:46.038920+0000","last_fullsized":"2023-09-23T18:12:46.038920+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:40.519034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":353,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.491938+0000","last_change":"2023-09-23T18:12:33.306430+0000","last_active":"2023-09-23T18:12:39.491938+0000","last_peered":"2023-09-23T18:12:39.491938+0000","last_clean":"2023-09-23T18:12:39.491938+0000","last_became_active":"2023-09-23T18:12:33.305956+0000","last_became_peered":"2023-09-23T18:12:33.305956+0000","last_unstale":"2023-09-23T18:12:39.491938+0000","last_undegraded":"2023-09-23T18:12:39.491938+0000","last_fullsized":"2023-09-23T18:12:39.491938+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:02.643791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":190,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.488500+0000","last_change":"2023-09-23T18:12:33.302096+0000","last_active":"2023-09-23T18:12:39.488500+0000","last_peered":"2023-09-23T18:12:39.488500+0000","last_clean":"2023-09-23T18:12:39.488500+0000","last_became_active":"2023-09-23T18:12:33.301770+0000","last_became_peered":"2023-09-23T18:12:33.301770+0000","last_unstale":"2023-09-23T18:12:39.488500+0000","last_undegraded":"2023-09-23T18:12:39.488500+0000","last_fullsized":"2023-09-23T18:12:39.488500+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:00.314270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.8","version":"426'11808","reported_seq":19613,"reported_epoch":426,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:12:47.373695+0000","last_change":"2023-09-23T18:12:33.671349+0000","last_active":"2023-09-23T18:12:47.373695+0000","last_peered":"2023-09-23T18:12:47.373695+0000","last_clean":"2023-09-23T18:09:48.238829+0000","last_became_active":"2023-09-23T18:12:33.540152+0000","last_became_peered":"2023-09-23T18:12:33.540152+0000","last_unstale":"2023-09-23T18:12:47.373695+0000","last_undegraded":"2023-09-23T18:12:33.507360+0000","last_fullsized":"2023-09-23T18:12:33.288867+0000","mapping_epoch":421,"log_start":"366'1800","ondisk_log_start":"366'1800","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":10008,"log_dups_size":0,"ondisk_log_size":10008,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:33.636709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":131727360,"num_objects":2010,"num_object_clones":0,"num_object_copies":6030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2535,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2010,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11808,"num_write_kb":442176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3849,"num_bytes_recovered":251985920,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[{"shards":"3,7","objects":-525}],"blocked_by":[],"up_primary":5,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":713,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490920+0000","last_change":"2023-09-23T18:12:33.306235+0000","last_active":"2023-09-23T18:12:39.490920+0000","last_peered":"2023-09-23T18:12:39.490920+0000","last_clean":"2023-09-23T18:12:39.490920+0000","last_became_active":"2023-09-23T18:12:33.305801+0000","last_became_peered":"2023-09-23T18:12:33.305801+0000","last_unstale":"2023-09-23T18:12:39.490920+0000","last_undegraded":"2023-09-23T18:12:39.490920+0000","last_fullsized":"2023-09-23T18:12:39.490920+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:44.704978+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1322,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490924+0000","last_change":"2023-09-23T18:12:32.335626+0000","last_active":"2023-09-23T18:12:39.490924+0000","last_peered":"2023-09-23T18:12:39.490924+0000","last_clean":"2023-09-23T18:12:39.490924+0000","last_became_active":"2023-09-23T18:12:32.335067+0000","last_became_peered":"2023-09-23T18:12:32.335067+0000","last_unstale":"2023-09-23T18:12:39.490924+0000","last_undegraded":"2023-09-23T18:12:39.490924+0000","last_fullsized":"2023-09-23T18:12:39.490924+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:53.656722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":23,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497838+0000","last_change":"2023-09-23T18:12:05.649154+0000","last_active":"2023-09-23T18:12:39.497838+0000","last_peered":"2023-09-23T18:12:39.497838+0000","last_clean":"2023-09-23T18:12:39.497838+0000","last_became_active":"2023-09-23T18:12:05.648905+0000","last_became_peered":"2023-09-23T18:12:05.648905+0000","last_unstale":"2023-09-23T18:12:39.497838+0000","last_undegraded":"2023-09-23T18:12:39.497838+0000","last_fullsized":"2023-09-23T18:12:39.497838+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:05.562327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":172,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.488562+0000","last_change":"2023-09-23T18:10:51.815823+0000","last_active":"2023-09-23T18:12:39.488562+0000","last_peered":"2023-09-23T18:12:39.488562+0000","last_clean":"2023-09-23T18:12:39.488562+0000","last_became_active":"2023-09-23T18:10:51.815528+0000","last_became_peered":"2023-09-23T18:10:51.815528+0000","last_unstale":"2023-09-23T18:12:39.488562+0000","last_undegraded":"2023-09-23T18:12:39.488562+0000","last_fullsized":"2023-09-23T18:12:39.488562+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:16:40.194226+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.b","version":"426'11935","reported_seq":12023,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T18:12:50.919090+0000","last_change":"2023-09-23T18:12:06.680276+0000","last_active":"2023-09-23T18:12:50.919090+0000","last_peered":"2023-09-23T18:12:50.919090+0000","last_clean":"2023-09-23T18:12:50.919090+0000","last_became_active":"2023-09-23T18:12:06.648544+0000","last_became_peered":"2023-09-23T18:12:06.648544+0000","last_unstale":"2023-09-23T18:12:50.919090+0000","last_undegraded":"2023-09-23T18:12:50.919090+0000","last_fullsized":"2023-09-23T18:12:50.919090+0000","mapping_epoch":413,"log_start":"391'7402","ondisk_log_start":"391'7402","created":361,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4533,"log_dups_size":0,"ondisk_log_size":4533,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:54:01.997622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113049600,"num_objects":1725,"num_object_clones":0,"num_object_copies":5175,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1725,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11935,"num_write_kb":437120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,7],"acting":[4,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":661,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.499159+0000","last_change":"2023-09-23T18:12:06.667895+0000","last_active":"2023-09-23T18:12:39.499159+0000","last_peered":"2023-09-23T18:12:39.499159+0000","last_clean":"2023-09-23T18:12:39.499159+0000","last_became_active":"2023-09-23T18:12:06.657611+0000","last_became_peered":"2023-09-23T18:12:06.657611+0000","last_unstale":"2023-09-23T18:12:39.499159+0000","last_undegraded":"2023-09-23T18:12:39.499159+0000","last_fullsized":"2023-09-23T18:12:39.499159+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:49.208587+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":677,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.488492+0000","last_change":"2023-09-23T18:11:40.522678+0000","last_active":"2023-09-23T18:12:39.488492+0000","last_peered":"2023-09-23T18:12:39.488492+0000","last_clean":"2023-09-23T18:12:39.488492+0000","last_became_active":"2023-09-23T18:11:40.521998+0000","last_became_peered":"2023-09-23T18:11:40.521998+0000","last_unstale":"2023-09-23T18:12:39.488492+0000","last_undegraded":"2023-09-23T18:12:39.488492+0000","last_fullsized":"2023-09-23T18:12:39.488492+0000","mapping_epoch":403,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":404,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:12.167477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,6],"acting":[1,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":163,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.488722+0000","last_change":"2023-09-23T18:10:51.809930+0000","last_active":"2023-09-23T18:12:39.488722+0000","last_peered":"2023-09-23T18:12:39.488722+0000","last_clean":"2023-09-23T18:12:39.488722+0000","last_became_active":"2023-09-23T18:10:51.809574+0000","last_became_peered":"2023-09-23T18:10:51.809574+0000","last_unstale":"2023-09-23T18:12:39.488722+0000","last_undegraded":"2023-09-23T18:12:39.488722+0000","last_fullsized":"2023-09-23T18:12:39.488722+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:31:05.356356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":178,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.489538+0000","last_change":"2023-09-23T18:12:32.334537+0000","last_active":"2023-09-23T18:12:39.489538+0000","last_peered":"2023-09-23T18:12:39.489538+0000","last_clean":"2023-09-23T18:12:39.489538+0000","last_became_active":"2023-09-23T18:12:32.334248+0000","last_became_peered":"2023-09-23T18:12:32.334248+0000","last_unstale":"2023-09-23T18:12:39.489538+0000","last_undegraded":"2023-09-23T18:12:39.489538+0000","last_fullsized":"2023-09-23T18:12:39.489538+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:07.708083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.9","version":"426'12121","reported_seq":12304,"reported_epoch":426,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:12:50.704208+0000","last_change":"2023-09-23T18:12:33.435618+0000","last_active":"2023-09-23T18:12:50.704208+0000","last_peered":"2023-09-23T18:12:50.704208+0000","last_clean":"2023-09-23T18:09:48.227889+0000","last_became_active":"2023-09-23T18:12:33.375526+0000","last_became_peered":"2023-09-23T18:12:33.375526+0000","last_unstale":"2023-09-23T18:12:50.704208+0000","last_undegraded":"2023-09-23T18:12:33.300384+0000","last_fullsized":"2023-09-23T18:12:33.291754+0000","mapping_epoch":421,"log_start":"391'7405","ondisk_log_start":"391'7405","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4716,"log_dups_size":0,"ondisk_log_size":4716,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:47:58.161406+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":117243904,"num_objects":1789,"num_object_clones":0,"num_object_copies":5367,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1789,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1789,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12121,"num_write_kb":445120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":2359296,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,3],"acting":[6,3],"avail_no_missing":["6","3"],"object_location_counts":[{"shards":"3,6","objects":1789}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1268,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490776+0000","last_change":"2023-09-23T18:12:33.299474+0000","last_active":"2023-09-23T18:12:39.490776+0000","last_peered":"2023-09-23T18:12:39.490776+0000","last_clean":"2023-09-23T18:12:39.490776+0000","last_became_active":"2023-09-23T18:12:33.298954+0000","last_became_peered":"2023-09-23T18:12:33.298954+0000","last_unstale":"2023-09-23T18:12:39.490776+0000","last_undegraded":"2023-09-23T18:12:39.490776+0000","last_fullsized":"2023-09-23T18:12:39.490776+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:18:00.572633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":781,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.502496+0000","last_change":"2023-09-23T18:06:48.957422+0000","last_active":"2023-09-23T18:12:39.502496+0000","last_peered":"2023-09-23T18:12:39.502496+0000","last_clean":"2023-09-23T18:12:39.502496+0000","last_became_active":"2023-09-23T18:06:48.957044+0000","last_became_peered":"2023-09-23T18:06:48.957044+0000","last_unstale":"2023-09-23T18:12:39.502496+0000","last_undegraded":"2023-09-23T18:12:39.502496+0000","last_fullsized":"2023-09-23T18:12:39.502496+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:58.047861+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":195,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490614+0000","last_change":"2023-09-23T18:12:33.310915+0000","last_active":"2023-09-23T18:12:39.490614+0000","last_peered":"2023-09-23T18:12:39.490614+0000","last_clean":"2023-09-23T18:12:39.490614+0000","last_became_active":"2023-09-23T18:12:33.310201+0000","last_became_peered":"2023-09-23T18:12:33.310201+0000","last_unstale":"2023-09-23T18:12:39.490614+0000","last_undegraded":"2023-09-23T18:12:39.490614+0000","last_fullsized":"2023-09-23T18:12:39.490614+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:37.485005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":379,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498541+0000","last_change":"2023-09-23T18:12:05.656003+0000","last_active":"2023-09-23T18:12:39.498541+0000","last_peered":"2023-09-23T18:12:39.498541+0000","last_clean":"2023-09-23T18:12:39.498541+0000","last_became_active":"2023-09-23T18:12:05.655726+0000","last_became_peered":"2023-09-23T18:12:05.655726+0000","last_unstale":"2023-09-23T18:12:39.498541+0000","last_undegraded":"2023-09-23T18:12:39.498541+0000","last_fullsized":"2023-09-23T18:12:39.498541+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:56:47.283544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"8.a","version":"426'12001","reported_seq":14468,"reported_epoch":426,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:12:47.824274+0000","last_change":"2023-09-23T18:12:33.425699+0000","last_active":"2023-09-23T18:12:47.824274+0000","last_peered":"2023-09-23T18:12:47.824274+0000","last_clean":"2023-09-23T18:09:48.219556+0000","last_became_active":"2023-09-23T18:12:32.399069+0000","last_became_peered":"2023-09-23T18:12:32.399069+0000","last_unstale":"2023-09-23T18:12:47.824274+0000","last_undegraded":"2023-09-23T18:12:32.321220+0000","last_fullsized":"2023-09-23T18:12:32.299107+0000","mapping_epoch":420,"log_start":"392'7632","ondisk_log_start":"392'7632","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4369,"log_dups_size":0,"ondisk_log_size":4369,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:46:46.919031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":127074304,"num_objects":1939,"num_object_clones":0,"num_object_copies":5817,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1692,"num_objects_misplaced":1720,"num_objects_unfound":0,"num_objects_dirty":1939,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12001,"num_write_kb":446080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1172,"num_bytes_recovered":76808192,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,3],"acting":[1,2],"avail_no_missing":["1","2"],"object_location_counts":[{"shards":"1,2","objects":1939}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":608,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490282+0000","last_change":"2023-09-23T18:12:05.656585+0000","last_active":"2023-09-23T18:12:39.490282+0000","last_peered":"2023-09-23T18:12:39.490282+0000","last_clean":"2023-09-23T18:12:39.490282+0000","last_became_active":"2023-09-23T18:12:05.656067+0000","last_became_peered":"2023-09-23T18:12:05.656067+0000","last_unstale":"2023-09-23T18:12:39.490282+0000","last_undegraded":"2023-09-23T18:12:39.490282+0000","last_fullsized":"2023-09-23T18:12:39.490282+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:18.616905+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":689,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.502692+0000","last_change":"2023-09-23T18:10:01.230791+0000","last_active":"2023-09-23T18:12:39.502692+0000","last_peered":"2023-09-23T18:12:39.502692+0000","last_clean":"2023-09-23T18:12:39.502692+0000","last_became_active":"2023-09-23T18:10:01.170909+0000","last_became_peered":"2023-09-23T18:10:01.170909+0000","last_unstale":"2023-09-23T18:12:39.502692+0000","last_undegraded":"2023-09-23T18:12:39.502692+0000","last_fullsized":"2023-09-23T18:12:39.502692+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:08.759640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":367,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.500382+0000","last_change":"2023-09-23T18:12:33.379732+0000","last_active":"2023-09-23T18:12:39.500382+0000","last_peered":"2023-09-23T18:12:39.500382+0000","last_clean":"2023-09-23T18:12:39.500382+0000","last_became_active":"2023-09-23T18:12:33.378977+0000","last_became_peered":"2023-09-23T18:12:33.378977+0000","last_unstale":"2023-09-23T18:12:39.500382+0000","last_undegraded":"2023-09-23T18:12:39.500382+0000","last_fullsized":"2023-09-23T18:12:39.500382+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:34.334366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":420,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.489333+0000","last_change":"2023-09-23T18:12:32.324474+0000","last_active":"2023-09-23T18:12:39.489333+0000","last_peered":"2023-09-23T18:12:39.489333+0000","last_clean":"2023-09-23T18:12:39.489333+0000","last_became_active":"2023-09-23T18:12:32.324209+0000","last_became_peered":"2023-09-23T18:12:32.324209+0000","last_unstale":"2023-09-23T18:12:39.489333+0000","last_undegraded":"2023-09-23T18:12:39.489333+0000","last_fullsized":"2023-09-23T18:12:39.489333+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:17.579958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"8.f","version":"426'11828","reported_seq":11941,"reported_epoch":426,"state":"active+clean","last_fresh":"2023-09-23T18:12:47.827023+0000","last_change":"2023-09-23T18:12:05.679857+0000","last_active":"2023-09-23T18:12:47.827023+0000","last_peered":"2023-09-23T18:12:47.827023+0000","last_clean":"2023-09-23T18:12:47.827023+0000","last_became_active":"2023-09-23T18:12:05.656425+0000","last_became_peered":"2023-09-23T18:12:05.656425+0000","last_unstale":"2023-09-23T18:12:47.827023+0000","last_undegraded":"2023-09-23T18:12:47.827023+0000","last_fullsized":"2023-09-23T18:12:47.827023+0000","mapping_epoch":412,"log_start":"392'7536","ondisk_log_start":"392'7536","created":361,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4292,"log_dups_size":0,"ondisk_log_size":4292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:12:59.264671+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":121241600,"num_objects":1850,"num_object_clones":0,"num_object_copies":5550,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1850,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11828,"num_write_kb":437696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":661,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498969+0000","last_change":"2023-09-23T18:12:06.646074+0000","last_active":"2023-09-23T18:12:39.498969+0000","last_peered":"2023-09-23T18:12:39.498969+0000","last_clean":"2023-09-23T18:12:39.498969+0000","last_became_active":"2023-09-23T18:12:06.645836+0000","last_became_peered":"2023-09-23T18:12:06.645836+0000","last_unstale":"2023-09-23T18:12:39.498969+0000","last_undegraded":"2023-09-23T18:12:39.498969+0000","last_fullsized":"2023-09-23T18:12:39.498969+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:34.582299+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":717,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.504767+0000","last_change":"2023-09-23T18:12:31.342399+0000","last_active":"2023-09-23T18:12:39.504767+0000","last_peered":"2023-09-23T18:12:39.504767+0000","last_clean":"2023-09-23T18:12:39.504767+0000","last_became_active":"2023-09-23T18:12:05.655437+0000","last_became_peered":"2023-09-23T18:12:05.655437+0000","last_unstale":"2023-09-23T18:12:39.504767+0000","last_undegraded":"2023-09-23T18:12:39.504767+0000","last_fullsized":"2023-09-23T18:12:39.504767+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:20.926785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":178,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.488480+0000","last_change":"2023-09-23T18:12:32.317796+0000","last_active":"2023-09-23T18:12:39.488480+0000","last_peered":"2023-09-23T18:12:39.488480+0000","last_clean":"2023-09-23T18:12:39.488480+0000","last_became_active":"2023-09-23T18:12:32.317479+0000","last_became_peered":"2023-09-23T18:12:32.317479+0000","last_unstale":"2023-09-23T18:12:39.488480+0000","last_undegraded":"2023-09-23T18:12:39.488480+0000","last_fullsized":"2023-09-23T18:12:39.488480+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:52.596216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":202,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490636+0000","last_change":"2023-09-23T18:12:33.303060+0000","last_active":"2023-09-23T18:12:39.490636+0000","last_peered":"2023-09-23T18:12:39.490636+0000","last_clean":"2023-09-23T18:12:39.490636+0000","last_became_active":"2023-09-23T18:12:33.302734+0000","last_became_peered":"2023-09-23T18:12:33.302734+0000","last_unstale":"2023-09-23T18:12:39.490636+0000","last_undegraded":"2023-09-23T18:12:39.490636+0000","last_fullsized":"2023-09-23T18:12:39.490636+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:48.422153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"8.c","version":"426'11883","reported_seq":12146,"reported_epoch":426,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:12:48.125718+0000","last_change":"2023-09-23T18:12:33.666809+0000","last_active":"2023-09-23T18:12:48.125718+0000","last_peered":"2023-09-23T18:12:48.125718+0000","last_clean":"2023-09-23T18:09:48.216272+0000","last_became_active":"2023-09-23T18:12:33.390850+0000","last_became_peered":"2023-09-23T18:12:33.390850+0000","last_unstale":"2023-09-23T18:12:48.125718+0000","last_undegraded":"2023-09-23T18:12:33.302814+0000","last_fullsized":"2023-09-23T18:12:33.291414+0000","mapping_epoch":421,"log_start":"386'6946","ondisk_log_start":"386'6946","created":361,"last_epoch_clean":362,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4937,"log_dups_size":0,"ondisk_log_size":4937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:12:00.304877+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":120389632,"num_objects":1837,"num_object_clones":0,"num_object_copies":5511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1780,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11883,"num_write_kb":439040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":77,"num_bytes_recovered":5046272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3],"avail_no_missing":["7","3"],"object_location_counts":[{"shards":"3,7","objects":1837}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":784,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497349+0000","last_change":"2023-09-23T18:12:33.330181+0000","last_active":"2023-09-23T18:12:39.497349+0000","last_peered":"2023-09-23T18:12:39.497349+0000","last_clean":"2023-09-23T18:12:39.497349+0000","last_became_active":"2023-09-23T18:12:33.329678+0000","last_became_peered":"2023-09-23T18:12:33.329678+0000","last_unstale":"2023-09-23T18:12:39.497349+0000","last_undegraded":"2023-09-23T18:12:39.497349+0000","last_fullsized":"2023-09-23T18:12:39.497349+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:58.659729+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":230,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.505077+0000","last_change":"2023-09-23T18:11:19.178406+0000","last_active":"2023-09-23T18:12:39.505077+0000","last_peered":"2023-09-23T18:12:39.505077+0000","last_clean":"2023-09-23T18:12:39.505077+0000","last_became_active":"2023-09-23T18:11:19.178025+0000","last_became_peered":"2023-09-23T18:11:19.178025+0000","last_unstale":"2023-09-23T18:12:39.505077+0000","last_undegraded":"2023-09-23T18:12:39.505077+0000","last_fullsized":"2023-09-23T18:12:39.505077+0000","mapping_epoch":394,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":395,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:18.913342+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":185,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.489828+0000","last_change":"2023-09-23T18:12:05.654233+0000","last_active":"2023-09-23T18:12:39.489828+0000","last_peered":"2023-09-23T18:12:39.489828+0000","last_clean":"2023-09-23T18:12:39.489828+0000","last_became_active":"2023-09-23T18:12:05.653748+0000","last_became_peered":"2023-09-23T18:12:05.653748+0000","last_unstale":"2023-09-23T18:12:39.489828+0000","last_undegraded":"2023-09-23T18:12:39.489828+0000","last_fullsized":"2023-09-23T18:12:39.489828+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:48.819688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":247,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.502528+0000","last_change":"2023-09-23T18:12:32.328282+0000","last_active":"2023-09-23T18:12:39.502528+0000","last_peered":"2023-09-23T18:12:39.502528+0000","last_clean":"2023-09-23T18:12:39.502528+0000","last_became_active":"2023-09-23T18:12:32.327937+0000","last_became_peered":"2023-09-23T18:12:32.327937+0000","last_unstale":"2023-09-23T18:12:39.502528+0000","last_undegraded":"2023-09-23T18:12:39.502528+0000","last_fullsized":"2023-09-23T18:12:39.502528+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:24.813277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"8.d","version":"426'12109","reported_seq":22767,"reported_epoch":426,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:12:48.115719+0000","last_change":"2023-09-23T18:12:33.350921+0000","last_active":"2023-09-23T18:12:48.115719+0000","last_peered":"2023-09-23T18:12:48.115719+0000","last_clean":"2023-09-23T18:12:31.244290+0000","last_became_active":"2023-09-23T18:12:32.393483+0000","last_became_peered":"2023-09-23T18:12:32.393483+0000","last_unstale":"2023-09-23T18:12:48.115719+0000","last_undegraded":"2023-09-23T18:12:48.115719+0000","last_fullsized":"2023-09-23T18:12:48.115719+0000","mapping_epoch":420,"log_start":"386'7251","ondisk_log_start":"386'7251","created":361,"last_epoch_clean":410,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:09:24.998551+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.998551+0000","objects_scrubbed":0,"log_size":4858,"log_dups_size":0,"ondisk_log_size":4858,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:40.199666+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":132775960,"num_objects":2027,"num_object_clones":0,"num_object_copies":6081,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":2027,"num_objects_unfound":0,"num_objects_dirty":2027,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12109,"num_write_kb":452289,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5267,"num_bytes_recovered":345047064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[7,6,2],"avail_no_missing":["7","2","6"],"object_location_counts":[{"shards":"2,6,7","objects":2027}],"blocked_by":[],"up_primary":5,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":673,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.491176+0000","last_change":"2023-09-23T18:12:32.327790+0000","last_active":"2023-09-23T18:12:39.491176+0000","last_peered":"2023-09-23T18:12:39.491176+0000","last_clean":"2023-09-23T18:12:39.491176+0000","last_became_active":"2023-09-23T18:12:32.327436+0000","last_became_peered":"2023-09-23T18:12:32.327436+0000","last_unstale":"2023-09-23T18:12:39.491176+0000","last_undegraded":"2023-09-23T18:12:39.491176+0000","last_fullsized":"2023-09-23T18:12:39.491176+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:31.945694+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":727,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497365+0000","last_change":"2023-09-23T18:11:18.202385+0000","last_active":"2023-09-23T18:12:39.497365+0000","last_peered":"2023-09-23T18:12:39.497365+0000","last_clean":"2023-09-23T18:12:39.497365+0000","last_became_active":"2023-09-23T18:11:18.168714+0000","last_became_peered":"2023-09-23T18:11:18.168714+0000","last_unstale":"2023-09-23T18:12:39.497365+0000","last_undegraded":"2023-09-23T18:12:39.497365+0000","last_fullsized":"2023-09-23T18:12:39.497365+0000","mapping_epoch":393,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":394,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:32:16.704449+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":30,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":477,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.489425+0000","last_change":"2023-09-23T18:12:06.646490+0000","last_active":"2023-09-23T18:12:39.489425+0000","last_peered":"2023-09-23T18:12:39.489425+0000","last_clean":"2023-09-23T18:12:39.489425+0000","last_became_active":"2023-09-23T18:12:06.645565+0000","last_became_peered":"2023-09-23T18:12:06.645565+0000","last_unstale":"2023-09-23T18:12:39.489425+0000","last_undegraded":"2023-09-23T18:12:39.489425+0000","last_fullsized":"2023-09-23T18:12:39.489425+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:54.061911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":229,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.500162+0000","last_change":"2023-09-23T18:12:33.379923+0000","last_active":"2023-09-23T18:12:39.500162+0000","last_peered":"2023-09-23T18:12:39.500162+0000","last_clean":"2023-09-23T18:12:39.500162+0000","last_became_active":"2023-09-23T18:12:33.379412+0000","last_became_peered":"2023-09-23T18:12:33.379412+0000","last_unstale":"2023-09-23T18:12:39.500162+0000","last_undegraded":"2023-09-23T18:12:39.500162+0000","last_fullsized":"2023-09-23T18:12:39.500162+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:53.669871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":595,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497960+0000","last_change":"2023-09-23T18:12:05.637738+0000","last_active":"2023-09-23T18:12:39.497960+0000","last_peered":"2023-09-23T18:12:39.497960+0000","last_clean":"2023-09-23T18:12:39.497960+0000","last_became_active":"2023-09-23T18:12:05.637397+0000","last_became_peered":"2023-09-23T18:12:05.637397+0000","last_unstale":"2023-09-23T18:12:39.497960+0000","last_undegraded":"2023-09-23T18:12:39.497960+0000","last_fullsized":"2023-09-23T18:12:39.497960+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:14.580819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:14.580819+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:39.955222+0000","scrub_duration":0.003313033,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":23,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497760+0000","last_change":"2023-09-23T18:12:05.654572+0000","last_active":"2023-09-23T18:12:39.497760+0000","last_peered":"2023-09-23T18:12:39.497760+0000","last_clean":"2023-09-23T18:12:39.497760+0000","last_became_active":"2023-09-23T18:12:05.654026+0000","last_became_peered":"2023-09-23T18:12:05.654026+0000","last_unstale":"2023-09-23T18:12:39.497760+0000","last_undegraded":"2023-09-23T18:12:39.497760+0000","last_fullsized":"2023-09-23T18:12:39.497760+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:55.511372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":500,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.488383+0000","last_change":"2023-09-23T18:12:33.321764+0000","last_active":"2023-09-23T18:12:39.488383+0000","last_peered":"2023-09-23T18:12:39.488383+0000","last_clean":"2023-09-23T18:12:39.488383+0000","last_became_active":"2023-09-23T18:12:33.321413+0000","last_became_peered":"2023-09-23T18:12:33.321413+0000","last_unstale":"2023-09-23T18:12:39.488383+0000","last_undegraded":"2023-09-23T18:12:39.488383+0000","last_fullsized":"2023-09-23T18:12:39.488383+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:19.453718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":590,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498676+0000","last_change":"2023-09-23T18:12:05.644542+0000","last_active":"2023-09-23T18:12:39.498676+0000","last_peered":"2023-09-23T18:12:39.498676+0000","last_clean":"2023-09-23T18:12:39.498676+0000","last_became_active":"2023-09-23T18:12:05.643810+0000","last_became_peered":"2023-09-23T18:12:05.643810+0000","last_unstale":"2023-09-23T18:12:39.498676+0000","last_undegraded":"2023-09-23T18:12:39.498676+0000","last_fullsized":"2023-09-23T18:12:39.498676+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.604423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.604423+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:15.854751+0000","scrub_duration":0.0014924669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":214,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490983+0000","last_change":"2023-09-23T18:12:33.386729+0000","last_active":"2023-09-23T18:12:39.490983+0000","last_peered":"2023-09-23T18:12:39.490983+0000","last_clean":"2023-09-23T18:12:39.490983+0000","last_became_active":"2023-09-23T18:12:33.386238+0000","last_became_peered":"2023-09-23T18:12:33.386238+0000","last_unstale":"2023-09-23T18:12:39.490983+0000","last_undegraded":"2023-09-23T18:12:39.490983+0000","last_fullsized":"2023-09-23T18:12:39.490983+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:15.285523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":23,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.497911+0000","last_change":"2023-09-23T18:12:05.656643+0000","last_active":"2023-09-23T18:12:39.497911+0000","last_peered":"2023-09-23T18:12:39.497911+0000","last_clean":"2023-09-23T18:12:39.497911+0000","last_became_active":"2023-09-23T18:12:05.655835+0000","last_became_peered":"2023-09-23T18:12:05.655835+0000","last_unstale":"2023-09-23T18:12:39.497911+0000","last_undegraded":"2023-09-23T18:12:39.497911+0000","last_fullsized":"2023-09-23T18:12:39.497911+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:28:03.784579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":202,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.492290+0000","last_change":"2023-09-23T18:12:33.305790+0000","last_active":"2023-09-23T18:12:39.492290+0000","last_peered":"2023-09-23T18:12:39.492290+0000","last_clean":"2023-09-23T18:12:39.492290+0000","last_became_active":"2023-09-23T18:12:33.305535+0000","last_became_peered":"2023-09-23T18:12:33.305535+0000","last_unstale":"2023-09-23T18:12:39.492290+0000","last_undegraded":"2023-09-23T18:12:39.492290+0000","last_fullsized":"2023-09-23T18:12:39.492290+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:15.270191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":23,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498596+0000","last_change":"2023-09-23T18:12:05.645960+0000","last_active":"2023-09-23T18:12:39.498596+0000","last_peered":"2023-09-23T18:12:39.498596+0000","last_clean":"2023-09-23T18:12:39.498596+0000","last_became_active":"2023-09-23T18:12:05.645643+0000","last_became_peered":"2023-09-23T18:12:05.645643+0000","last_unstale":"2023-09-23T18:12:39.498596+0000","last_undegraded":"2023-09-23T18:12:39.498596+0000","last_fullsized":"2023-09-23T18:12:39.498596+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:40:39.540121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":812,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.504775+0000","last_change":"2023-09-23T18:06:08.440953+0000","last_active":"2023-09-23T18:12:39.504775+0000","last_peered":"2023-09-23T18:12:39.504775+0000","last_clean":"2023-09-23T18:12:39.504775+0000","last_became_active":"2023-09-23T18:06:08.440604+0000","last_became_peered":"2023-09-23T18:06:08.440604+0000","last_unstale":"2023-09-23T18:12:39.504775+0000","last_undegraded":"2023-09-23T18:12:39.504775+0000","last_fullsized":"2023-09-23T18:12:39.504775+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:53.274049+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":238,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.498882+0000","last_change":"2023-09-23T18:10:51.816416+0000","last_active":"2023-09-23T18:12:39.498882+0000","last_peered":"2023-09-23T18:12:39.498882+0000","last_clean":"2023-09-23T18:12:39.498882+0000","last_became_active":"2023-09-23T18:10:51.815980+0000","last_became_peered":"2023-09-23T18:10:51.815980+0000","last_unstale":"2023-09-23T18:12:39.498882+0000","last_undegraded":"2023-09-23T18:12:39.498882+0000","last_fullsized":"2023-09-23T18:12:39.498882+0000","mapping_epoch":382,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":383,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:55:04.404422+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":220,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.490467+0000","last_change":"2023-09-23T18:12:33.303210+0000","last_active":"2023-09-23T18:12:39.490467+0000","last_peered":"2023-09-23T18:12:39.490467+0000","last_clean":"2023-09-23T18:12:39.490467+0000","last_became_active":"2023-09-23T18:12:33.302901+0000","last_became_peered":"2023-09-23T18:12:33.302901+0000","last_unstale":"2023-09-23T18:12:39.490467+0000","last_undegraded":"2023-09-23T18:12:39.490467+0000","last_fullsized":"2023-09-23T18:12:39.490467+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:21:40.195799+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":814,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.489080+0000","last_change":"2023-09-23T18:12:35.373704+0000","last_active":"2023-09-23T18:12:39.489080+0000","last_peered":"2023-09-23T18:12:39.489080+0000","last_clean":"2023-09-23T18:12:39.489080+0000","last_became_active":"2023-09-23T18:12:35.373267+0000","last_became_peered":"2023-09-23T18:12:35.373267+0000","last_unstale":"2023-09-23T18:12:39.489080+0000","last_undegraded":"2023-09-23T18:12:39.489080+0000","last_fullsized":"2023-09-23T18:12:39.489080+0000","mapping_epoch":423,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":424,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:17.635974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:17.635974+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:08:36.446459+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":243,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.491820+0000","last_change":"2023-09-23T18:12:33.308688+0000","last_active":"2023-09-23T18:12:39.491820+0000","last_peered":"2023-09-23T18:12:39.491820+0000","last_clean":"2023-09-23T18:12:39.491820+0000","last_became_active":"2023-09-23T18:12:33.308396+0000","last_became_peered":"2023-09-23T18:12:33.308396+0000","last_unstale":"2023-09-23T18:12:39.491820+0000","last_undegraded":"2023-09-23T18:12:39.491820+0000","last_fullsized":"2023-09-23T18:12:39.491820+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:11.406407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":440,"reported_epoch":425,"state":"active+clean","last_fresh":"2023-09-23T18:12:39.496485+0000","last_change":"2023-09-23T18:12:06.656925+0000","last_active":"2023-09-23T18:12:39.496485+0000","last_peered":"2023-09-23T18:12:39.496485+0000","last_clean":"2023-09-23T18:12:39.496485+0000","last_became_active":"2023-09-23T18:12:06.656605+0000","last_became_peered":"2023-09-23T18:12:06.656605+0000","last_unstale":"2023-09-23T18:12:39.496485+0000","last_undegraded":"2023-09-23T18:12:39.496485+0000","last_fullsized":"2023-09-23T18:12:39.496485+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:29.874912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":6154,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54826,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":8,"num_pg":16,"stat_sum":{"num_bytes":1933443096,"num_objects":29503,"num_object_clones":0,"num_object_copies":88509,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13711,"num_objects_misplaced":7511,"num_objects_unfound":0,"num_objects_dirty":29503,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":191677,"num_write_kb":7077697,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":25819,"num_bytes_recovered":1639317528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":7024881664,"data_stored":7024869448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":78175,"ondisk_log_size":78175,"up":48,"acting":41,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":461854,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":111,"num_read_kb":208,"num_write":54,"num_write_kb":549,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1059243,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":188,"num_bytes_recovered":4194342,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001482,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1184748,"kb_used_data":888484,"kb_used_omap":13,"kb_used_meta":296242,"kb_avail":92560404,"statfs":{"total":95995035648,"available":94781853696,"internally_reserved":0,"allocated":909807616,"data_stored":908385234,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":303351912},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.744,"5min":1.099,"15min":0.88100000000000001},"min":{"1min":0.24399999999999999,"5min":0.214,"15min":0.214},"max":{"1min":10.061,"5min":10.061,"15min":10.061},"last":1.514},{"interface":"front","average":{"1min":1.4239999999999999,"5min":1.0269999999999999,"15min":1.125},"min":{"1min":0.249,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":8.7469999999999999,"5min":8.7469999999999999,"15min":8.7469999999999999},"last":0.45300000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.712,"5min":1.5580000000000001,"15min":1.093},"min":{"1min":0.25,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":4.0860000000000003,"5min":9.4909999999999997,"15min":9.4909999999999997},"last":2.923},{"interface":"front","average":{"1min":2.0139999999999998,"5min":1.3779999999999999,"15min":1.224},"min":{"1min":0.29299999999999998,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":7.7480000000000002,"5min":7.7480000000000002,"15min":7.7480000000000002},"last":2.0110000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.6080000000000001,"5min":1.496,"15min":1.1739999999999999},"min":{"1min":0.34799999999999998,"5min":0.28199999999999997,"15min":0.23300000000000001},"max":{"1min":4.7679999999999998,"5min":7.4290000000000003,"15min":7.4290000000000003},"last":0.84799999999999998},{"interface":"front","average":{"1min":1.4399999999999999,"5min":1.6240000000000001,"15min":1.1930000000000001},"min":{"1min":0.32800000000000001,"5min":0.314,"15min":0.29399999999999998},"max":{"1min":3.8460000000000001,"5min":8.6519999999999992,"15min":8.6519999999999992},"last":2.5840000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:12:37 2023","interfaces":[{"interface":"back","average":{"1min":2.9710000000000001,"5min":2.5219999999999998,"15min":1.8100000000000001},"min":{"1min":0.41199999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":10.747999999999999,"5min":45.752000000000002,"15min":45.752000000000002},"last":3.246},{"interface":"front","average":{"1min":1.835,"5min":1.409,"15min":1.3},"min":{"1min":0.34699999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":4.8899999999999997,"5min":6.7889999999999997,"15min":15.349},"last":3.1099999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:11:47 2023","interfaces":[{"interface":"back","average":{"1min":0.68200000000000005,"5min":0.83099999999999996,"15min":0.96999999999999997},"min":{"1min":0.26800000000000002,"5min":0.25800000000000001,"15min":0.245},"max":{"1min":2.8969999999999998,"5min":4.4669999999999996,"15min":6.9729999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.069,"5min":0.89700000000000002,"15min":0.71999999999999997},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.4249999999999998,"5min":6.4249999999999998,"15min":6.4249999999999998},"last":1.617}]},{"osd":5,"last update":"Sat Sep 23 18:09:43 2023","interfaces":[{"interface":"back","average":{"1min":1.1899999999999999,"5min":1.0860000000000001,"15min":1.042},"min":{"1min":0.307,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.6280000000000001,"5min":8.4749999999999996,"15min":9.5999999999999996},"last":1.5509999999999999},{"interface":"front","average":{"1min":1.127,"5min":0.97899999999999998,"15min":1.095},"min":{"1min":0.35899999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.2839999999999998,"5min":7.3739999999999997,"15min":7.3739999999999997},"last":6.0679999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:11:47 2023","interfaces":[{"interface":"back","average":{"1min":1.5089999999999999,"5min":1.0009999999999999,"15min":0.89300000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":8.0069999999999997,"5min":8.0069999999999997,"15min":8.0069999999999997},"last":2.3300000000000001},{"interface":"front","average":{"1min":0.98999999999999999,"5min":0.83599999999999997,"15min":1.042},"min":{"1min":0.29899999999999999,"5min":0.253,"15min":0.253},"max":{"1min":7.7919999999999998,"5min":7.7919999999999998,"15min":9.5519999999999996},"last":0.877}]}]},{"osd":6,"up_from":163,"seq":700079669419,"num_pgs":64,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1329256,"kb_used_data":937248,"kb_used_omap":7,"kb_used_meta":391928,"kb_avail":92415896,"statfs":{"total":95995035648,"available":94633877504,"internally_reserved":0,"allocated":959741952,"data_stored":958323583,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":401335076},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:11:58 2023","interfaces":[{"interface":"back","average":{"1min":1.8109999999999999,"5min":1.3200000000000001,"15min":1.2130000000000001},"min":{"1min":0.22500000000000001,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":10.801,"5min":10.801,"15min":15.407},"last":0.46600000000000003},{"interface":"front","average":{"1min":1.6990000000000001,"5min":1.403,"15min":1.5780000000000001},"min":{"1min":0.25800000000000001,"5min":0.25600000000000001,"15min":0.23100000000000001},"max":{"1min":9.6329999999999991,"5min":9.6329999999999991,"15min":15.147},"last":0.41799999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:11:58 2023","interfaces":[{"interface":"back","average":{"1min":2.3780000000000001,"5min":1.4790000000000001,"15min":1.49},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":21.390000000000001,"5min":21.390000000000001,"15min":21.390000000000001},"last":0.58899999999999997},{"interface":"front","average":{"1min":3.5310000000000001,"5min":1.9199999999999999,"15min":1.7170000000000001},"min":{"1min":0.308,"5min":0.26300000000000001,"15min":0.248},"max":{"1min":21.559999999999999,"5min":21.559999999999999,"15min":21.559999999999999},"last":0.51500000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:11:58 2023","interfaces":[{"interface":"back","average":{"1min":1.831,"5min":1.5329999999999999,"15min":1.6879999999999999},"min":{"1min":0.34300000000000003,"5min":0.27600000000000002,"15min":0.27400000000000002},"max":{"1min":7.6529999999999996,"5min":9.4130000000000003,"15min":11.210000000000001},"last":0.44800000000000001},{"interface":"front","average":{"1min":1.49,"5min":1.4399999999999999,"15min":1.7649999999999999},"min":{"1min":0.35099999999999998,"5min":0.248,"15min":0.245},"max":{"1min":4.133,"5min":5.9269999999999996,"15min":11.223000000000001},"last":0.55500000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:12:37 2023","interfaces":[{"interface":"back","average":{"1min":1.78,"5min":1.768,"15min":1.341},"min":{"1min":0.309,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":4.9349999999999996,"5min":10.257999999999999,"15min":10.257999999999999},"last":2.7109999999999999},{"interface":"front","average":{"1min":2.988,"5min":2.0329999999999999,"15min":1.494},"min":{"1min":0.57599999999999996,"5min":0.32400000000000001,"15min":0.31900000000000001},"max":{"1min":7.6859999999999999,"5min":9.5960000000000001,"15min":9.5960000000000001},"last":0.54800000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:11:58 2023","interfaces":[{"interface":"back","average":{"1min":1.0529999999999999,"5min":1.0189999999999999,"15min":0.94399999999999995},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":3.6000000000000001,"5min":8.0280000000000005,"15min":9.9109999999999996},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.81100000000000005,"5min":1.054,"15min":0.83999999999999997},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.24099999999999999},"max":{"1min":3.3050000000000002,"5min":8.1159999999999997,"15min":8.3000000000000007},"last":0.504}]},{"osd":5,"last update":"Sat Sep 23 18:08:49 2023","interfaces":[{"interface":"back","average":{"1min":1.377,"5min":0.99299999999999999,"15min":0.89400000000000002},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.7219999999999995,"5min":9.7219999999999995,"15min":11.869},"last":0.57799999999999996},{"interface":"front","average":{"1min":1.5289999999999999,"5min":0.92200000000000004,"15min":0.80100000000000005},"min":{"1min":0.30599999999999999,"5min":0.251,"15min":0.251},"max":{"1min":10.436,"5min":10.436,"15min":10.436},"last":0.44}]},{"osd":7,"last update":"Sat Sep 23 18:12:42 2023","interfaces":[{"interface":"back","average":{"1min":1.673,"5min":1.331,"15min":1.069},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.26800000000000002},"max":{"1min":5.9450000000000003,"5min":13.460000000000001,"15min":13.460000000000001},"last":5.532},{"interface":"front","average":{"1min":1.9630000000000001,"5min":1.294,"15min":1.02},"min":{"1min":0.33300000000000002,"5min":0.318,"15min":0.28499999999999998},"max":{"1min":12.404,"5min":12.404,"15min":12.404},"last":0.63300000000000001}]}]},{"osd":1,"up_from":98,"seq":420906795236,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1757860,"kb_used_data":991068,"kb_used_omap":7,"kb_used_meta":766648,"kb_avail":91987292,"statfs":{"total":95995035648,"available":94194987008,"internally_reserved":0,"allocated":1014853632,"data_stored":1012860473,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":785048253},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,2,0,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:12:25 2023","interfaces":[{"interface":"back","average":{"1min":1.702,"5min":1.859,"15min":1.3700000000000001},"min":{"1min":0.20399999999999999,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":7.8570000000000002,"5min":14.567,"15min":15.529999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.129,"5min":2.0059999999999998,"15min":1.9019999999999999},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20100000000000001},"max":{"1min":6.6369999999999996,"5min":10.992000000000001,"15min":82.244},"last":1.5549999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:11:56 2023","interfaces":[{"interface":"back","average":{"1min":1.669,"5min":1.7849999999999999,"15min":1.7949999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22},"max":{"1min":4.5670000000000002,"5min":12.717000000000001,"15min":16.347999999999999},"last":1.6200000000000001},{"interface":"front","average":{"1min":1.9890000000000001,"5min":1.7050000000000001,"15min":1.4350000000000001},"min":{"1min":0.22,"5min":0.22,"15min":0.21299999999999999},"max":{"1min":8.4960000000000004,"5min":8.4960000000000004,"15min":9.8379999999999992},"last":0.44600000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:12:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1760000000000002,"5min":2.008,"15min":1.2470000000000001},"min":{"1min":0.29999999999999999,"5min":0.247,"15min":0.247},"max":{"1min":8.6660000000000004,"5min":11.519,"15min":11.519},"last":0.47399999999999998},{"interface":"front","average":{"1min":3.1890000000000001,"5min":2.0329999999999999,"15min":1.4379999999999999},"min":{"1min":0.373,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":15.446999999999999,"5min":15.446999999999999,"15min":15.446999999999999},"last":1.5920000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:12:25 2023","interfaces":[{"interface":"back","average":{"1min":2.0259999999999998,"5min":1.9830000000000001,"15min":1.597},"min":{"1min":0.29899999999999999,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":11.154,"5min":28.446999999999999,"15min":28.446999999999999},"last":5.6299999999999999},{"interface":"front","average":{"1min":1.3560000000000001,"5min":1.8280000000000001,"15min":1.474},"min":{"1min":0.31,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":5.2169999999999996,"5min":28.434000000000001,"15min":28.434000000000001},"last":5.6920000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:09:18 2023","interfaces":[{"interface":"back","average":{"1min":1.607,"5min":1.7869999999999999,"15min":1.4379999999999999},"min":{"1min":0.38900000000000001,"5min":0.35299999999999998,"15min":0.317},"max":{"1min":6.7489999999999997,"5min":11.638999999999999,"15min":11.638999999999999},"last":3.3599999999999999},{"interface":"front","average":{"1min":1.7849999999999999,"5min":1.579,"15min":1.393},"min":{"1min":0.374,"5min":0.33700000000000002,"15min":0.29599999999999999},"max":{"1min":6.5609999999999999,"5min":8.4309999999999992,"15min":9.5519999999999996},"last":2.161}]},{"osd":6,"last update":"Sat Sep 23 18:11:56 2023","interfaces":[{"interface":"back","average":{"1min":1.5900000000000001,"5min":1.9450000000000001,"15min":1.9350000000000001},"min":{"1min":0.33000000000000002,"5min":0.307,"15min":0.307},"max":{"1min":9.7439999999999998,"5min":27.516999999999999,"15min":89.566000000000003},"last":2.5409999999999999},{"interface":"front","average":{"1min":1.534,"5min":1.851,"15min":1.8360000000000001},"min":{"1min":0.373,"5min":0.373,"15min":0.34699999999999998},"max":{"1min":4.5049999999999999,"5min":27.501999999999999,"15min":90.608999999999995},"last":2.363}]},{"osd":7,"last update":"Sat Sep 23 18:11:48 2023","interfaces":[{"interface":"back","average":{"1min":1.26,"5min":1.7829999999999999,"15min":1.133},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.984,"5min":27.913,"15min":27.913},"last":0.67900000000000005},{"interface":"front","average":{"1min":2.3300000000000001,"5min":2.008,"15min":1.792},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":9.6180000000000003,"5min":27.620999999999999,"15min":27.620999999999999},"last":3.6040000000000001}]}]},{"osd":0,"up_from":93,"seq":399431958764,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":933020,"kb_used_data":353364,"kb_used_omap":7,"kb_used_meta":579576,"kb_avail":92812132,"statfs":{"total":95995035648,"available":95039623168,"internally_reserved":0,"allocated":361844736,"data_stored":359856037,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":593486564},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:12:29 2023","interfaces":[{"interface":"back","average":{"1min":2.2400000000000002,"5min":1.9019999999999999,"15min":1.8720000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.18099999999999999},"max":{"1min":12.869,"5min":13.677,"15min":18.154},"last":1.266},{"interface":"front","average":{"1min":2.806,"5min":1.8460000000000001,"15min":1.7230000000000001},"min":{"1min":0.30399999999999999,"5min":0.246,"15min":0.17399999999999999},"max":{"1min":14.207000000000001,"5min":14.207000000000001,"15min":14.468999999999999},"last":2.117}]},{"osd":2,"last update":"Sat Sep 23 18:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.167,"5min":1.274,"15min":1.8100000000000001},"min":{"1min":0.246,"5min":0.222,"15min":0.20499999999999999},"max":{"1min":3.7839999999999998,"5min":9.2859999999999996,"15min":20.882000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":1.4239999999999999,"5min":1.5629999999999999,"15min":1.7929999999999999},"min":{"1min":0.26600000000000001,"5min":0.25900000000000001,"15min":0.217},"max":{"1min":6.085,"5min":10.84,"15min":15.064},"last":2.8119999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:12:35 2023","interfaces":[{"interface":"back","average":{"1min":2.355,"5min":1.9379999999999999,"15min":1.716},"min":{"1min":0.311,"5min":0.25600000000000001,"15min":0.20499999999999999},"max":{"1min":9.1750000000000007,"5min":11.015000000000001,"15min":12.779999999999999},"last":0.81599999999999995},{"interface":"front","average":{"1min":2.4359999999999999,"5min":1.9770000000000001,"15min":1.5800000000000001},"min":{"1min":0.34899999999999998,"5min":0.29599999999999999,"15min":0.28599999999999998},"max":{"1min":9.2110000000000003,"5min":9.2110000000000003,"15min":15.198},"last":0.745}]},{"osd":4,"last update":"Sat Sep 23 18:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.46,"5min":1.474,"15min":1.548},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.251},"max":{"1min":3.9910000000000001,"5min":8.4109999999999996,"15min":15.279999999999999},"last":3.6160000000000001},{"interface":"front","average":{"1min":1.165,"5min":1.2470000000000001,"15min":1.5960000000000001},"min":{"1min":0.31,"5min":0.31,"15min":0.247},"max":{"1min":6.7199999999999998,"5min":9.2520000000000007,"15min":18.106999999999999},"last":1.3080000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:08:46 2023","interfaces":[{"interface":"back","average":{"1min":1.7729999999999999,"5min":1.7230000000000001,"15min":1.5600000000000001},"min":{"1min":0.317,"5min":0.29799999999999999,"15min":0.28399999999999997},"max":{"1min":9.2010000000000005,"5min":10.194000000000001,"15min":14.720000000000001},"last":1.03},{"interface":"front","average":{"1min":1.409,"5min":1.512,"15min":1.3140000000000001},"min":{"1min":0.33300000000000002,"5min":0.32600000000000001,"15min":0.254},"max":{"1min":5.1230000000000002,"5min":10.363,"15min":12.794},"last":0.58699999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:11:51 2023","interfaces":[{"interface":"back","average":{"1min":1.5229999999999999,"5min":2.246,"15min":1.9099999999999999},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34599999999999997},"max":{"1min":4.0229999999999997,"5min":58.457999999999998,"15min":58.457999999999998},"last":3.512},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.9470000000000001,"15min":1.857},"min":{"1min":0.315,"5min":0.315,"15min":0.311},"max":{"1min":2.8490000000000002,"5min":58.354999999999997,"15min":58.354999999999997},"last":2.1800000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:12:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6619999999999999,"5min":1.508,"15min":1.22},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.34499999999999997},"max":{"1min":6.3300000000000001,"5min":7.4790000000000001,"15min":10.416},"last":3.6920000000000002},{"interface":"front","average":{"1min":2.5230000000000001,"5min":1.968,"15min":1.538},"min":{"1min":0.45900000000000002,"5min":0.40500000000000003,"15min":0.36399999999999999},"max":{"1min":9.1489999999999991,"5min":9.1489999999999991,"15min":9.1489999999999991},"last":0.84799999999999998}]}]},{"osd":2,"up_from":0,"seq":442381631711,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249278,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1259388,"kb_used_data":781044,"kb_used_omap":3,"kb_used_meta":478332,"kb_avail":92485764,"statfs":{"total":95995035648,"available":94705422336,"internally_reserved":0,"allocated":799789056,"data_stored":797657898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":489812712},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,2,1,3,1,2,1],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.655,"5min":1.268,"15min":1.008},"min":{"1min":0.19800000000000001,"5min":0.183,"15min":0.183},"max":{"1min":9.7089999999999996,"5min":9.7089999999999996,"15min":9.7089999999999996},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.5780000000000001,"5min":1.716,"15min":1.365},"min":{"1min":0.25,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":4.0149999999999997,"5min":9.1999999999999993,"15min":14.885999999999999},"last":1.5900000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.758,"5min":2.2450000000000001,"15min":1.597},"min":{"1min":0.27800000000000002,"5min":0.215,"15min":0.215},"max":{"1min":12.005000000000001,"5min":16.056000000000001,"15min":21.931999999999999},"last":1.6299999999999999},{"interface":"front","average":{"1min":2.629,"5min":2.1760000000000002,"15min":1.4019999999999999},"min":{"1min":0.39400000000000002,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":14.457000000000001,"5min":16.587,"15min":16.587},"last":2.843}]},{"osd":2,"last update":"Sat Sep 23 18:12:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0499999999999998,"5min":1.5309999999999999,"15min":1.095},"min":{"1min":0.30599999999999999,"5min":0.247,"15min":0.22600000000000001},"max":{"1min":10.318,"5min":10.318,"15min":10.318},"last":2.0760000000000001},{"interface":"front","average":{"1min":1.7110000000000001,"5min":1.7809999999999999,"15min":1.484},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25800000000000001},"max":{"1min":11.846,"5min":11.846,"15min":21.988},"last":2.1440000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:12:34 2023","interfaces":[{"interface":"back","average":{"1min":0.96699999999999997,"5min":1.8300000000000001,"15min":1.427},"min":{"1min":0.39100000000000001,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":2.5369999999999999,"5min":37.231000000000002,"15min":37.231000000000002},"last":1.665},{"interface":"front","average":{"1min":1.589,"5min":2.169,"15min":1.498},"min":{"1min":0.34000000000000002,"5min":0.32100000000000001,"15min":0.30499999999999999},"max":{"1min":7.3860000000000001,"5min":37.381,"15min":37.381},"last":0.42799999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:09:30 2023","interfaces":[{"interface":"back","average":{"1min":1.6000000000000001,"5min":1.4570000000000001,"15min":1.3069999999999999},"min":{"1min":0.34699999999999998,"5min":0.307,"15min":0.307},"max":{"1min":9.3170000000000002,"5min":12.743,"15min":12.743},"last":0.47399999999999998},{"interface":"front","average":{"1min":1.099,"5min":1.1279999999999999,"15min":1.147},"min":{"1min":0.376,"5min":0.315,"15min":0.315},"max":{"1min":4.601,"5min":8.6479999999999997,"15min":8.6479999999999997},"last":0.753}]},{"osd":6,"last update":"Sat Sep 23 18:12:34 2023","interfaces":[{"interface":"back","average":{"1min":1.774,"5min":2.5600000000000001,"15min":1.5680000000000001},"min":{"1min":0.32700000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":9.8149999999999995,"5min":115.54000000000001,"15min":115.54000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.97399999999999998,"5min":2.5009999999999999,"15min":1.8300000000000001},"min":{"1min":0.32900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":2.774,"5min":115.691,"15min":115.691},"last":1.6990000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:12:34 2023","interfaces":[{"interface":"back","average":{"1min":2.0030000000000001,"5min":1.9259999999999999,"15min":1.631},"min":{"1min":0.373,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":7.3200000000000003,"5min":15.972,"15min":15.972},"last":0.39600000000000002},{"interface":"front","average":{"1min":1.7649999999999999,"5min":1.625,"15min":1.518},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.7840000000000007,"5min":9.7840000000000007,"15min":9.7840000000000007},"last":0.502}]}]},{"osd":4,"up_from":30,"seq":128849019228,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1116520,"kb_used_data":692000,"kb_used_omap":0,"kb_used_meta":424512,"kb_avail":92628632,"statfs":{"total":95995035648,"available":94851719168,"internally_reserved":0,"allocated":708608000,"data_stored":707189953,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":434700288},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:12:50 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":1.0669999999999999,"15min":1.145},"min":{"1min":0.247,"5min":0.22500000000000001,"15min":0.20100000000000001},"max":{"1min":1.944,"5min":9.2759999999999998,"15min":14.164},"last":0.39900000000000002},{"interface":"front","average":{"1min":1.655,"5min":1.4079999999999999,"15min":1.4199999999999999},"min":{"1min":0.372,"5min":0.23000000000000001,"15min":0.22600000000000001},"max":{"1min":5.7809999999999997,"5min":8.3490000000000002,"15min":11.372},"last":4.6189999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:12:18 2023","interfaces":[{"interface":"back","average":{"1min":1.3,"5min":2.0419999999999998,"15min":1.645},"min":{"1min":0.26200000000000001,"5min":0.249,"15min":0.249},"max":{"1min":5.3899999999999997,"5min":42.929000000000002,"15min":42.929000000000002},"last":5.0330000000000004},{"interface":"front","average":{"1min":1.298,"5min":1.8740000000000001,"15min":1.5840000000000001},"min":{"1min":0.26200000000000001,"5min":0.255,"15min":0.23100000000000001},"max":{"1min":6.9809999999999999,"5min":43.494999999999997,"15min":43.494999999999997},"last":1.6919999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:11:55 2023","interfaces":[{"interface":"back","average":{"1min":1.8140000000000001,"5min":2.0609999999999999,"15min":1.6459999999999999},"min":{"1min":0.30299999999999999,"5min":0.29199999999999998,"15min":0.26600000000000001},"max":{"1min":8.8019999999999996,"5min":32.887999999999998,"15min":32.887999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":1.4099999999999999,"5min":1.79,"15min":1.5800000000000001},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.27100000000000002},"max":{"1min":6.9009999999999998,"5min":29.103999999999999,"15min":29.103999999999999},"last":0.56100000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:12:32 2023","interfaces":[{"interface":"back","average":{"1min":1.909,"5min":2.2749999999999999,"15min":1.79},"min":{"1min":0.318,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":10.288,"5min":28.754999999999999,"15min":29.154},"last":7.1200000000000001},{"interface":"front","average":{"1min":1.385,"5min":2.0470000000000002,"15min":1.7090000000000001},"min":{"1min":0.38600000000000001,"5min":0.309,"15min":0.309},"max":{"1min":5.7549999999999999,"5min":29.213000000000001,"15min":29.216000000000001},"last":4.6580000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:08:51 2023","interfaces":[{"interface":"back","average":{"1min":1.349,"5min":1.036,"15min":0.81799999999999995},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":5.5279999999999996,"5min":9.2959999999999994,"15min":9.9819999999999993},"last":0.499},{"interface":"front","average":{"1min":1.2649999999999999,"5min":1.1299999999999999,"15min":0.98199999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":9.1609999999999996,"5min":9.1999999999999993,"15min":11.170999999999999},"last":4.0670000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:11:55 2023","interfaces":[{"interface":"back","average":{"1min":1,"5min":0.85099999999999998,"15min":0.92700000000000005},"min":{"1min":0.25,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":3.1230000000000002,"5min":5.8310000000000004,"15min":9.0990000000000002},"last":4.1100000000000003},{"interface":"front","average":{"1min":1.5029999999999999,"5min":1.04,"15min":1.0309999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":10.413,"5min":10.413,"15min":10.725},"last":4.1360000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:12:50 2023","interfaces":[{"interface":"back","average":{"1min":1.3089999999999999,"5min":1.1100000000000001,"15min":1.075},"min":{"1min":0.35699999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":5.726,"5min":9.2509999999999994,"15min":13.298999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.768,"5min":1.173,"15min":1.214},"min":{"1min":0.35499999999999998,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":10.535,"5min":10.535,"15min":10.535},"last":0.54100000000000004}]}]},{"osd":5,"up_from":420,"seq":1803886264325,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1610008,"kb_used_data":1544336,"kb_used_omap":9,"kb_used_meta":65654,"kb_avail":92135144,"statfs":{"total":95995035648,"available":94346387456,"internally_reserved":0,"allocated":1581400064,"data_stored":1579993831,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":67230141},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,1,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53200000000000003}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.5579999999999998}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.8940000000000001}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40100000000000002}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":6.3410000000000002}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":2.2320000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":4.407}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":350224384,"data_stored":350224384,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":980090880,"data_stored":980090880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":845484032,"data_stored":845479960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":771227648,"data_stored":771227648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":685506560,"data_stored":685506560,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1576140800,"data_stored":1576140800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":935071744,"data_stored":935067672,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":8,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":881135616,"data_stored":881131544,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:12:57.065 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:12:57.066 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_3 pg_num 26 2023-09-23T18:12:57.118 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:57.121 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:57.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:57.222 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:57.303 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:57.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:57.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:57.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:57.393 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:57.469 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:57.511 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:57.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:57.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:57.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:57.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:57.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:12:57.706 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:57.706 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:12:57.743 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:57.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:12:57.802 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:57.802 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:12:57.903 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:57.961 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:58.093 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:58.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:58.102 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:58.203 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:58.264 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:58.333 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:58.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:58.341 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:58.443 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:12:58.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:58 smithi055 ceph-mon[179668]: pgmap v1055: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.5 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 12009/75237 objects degraded (15.962%); 6393/75237 objects misplaced (8.497%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T18:12:58.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 12009/75237 objects degraded (15.962%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:58.537 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:12:58.574 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:58.654 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:12:58.665 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:58.665 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:12:58.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:12:58.818 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:58 smithi099 ceph-mon[167076]: pgmap v1055: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.5 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 12009/75237 objects degraded (15.962%); 6393/75237 objects misplaced (8.497%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T18:12:58.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 12009/75237 objects degraded (15.962%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:58.826 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:58.835 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:58 smithi055 ceph-mon[184174]: pgmap v1055: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.5 GiB data, 8.2 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 842 op/s; 12009/75237 objects degraded (15.962%); 6393/75237 objects misplaced (8.497%); 1.2 MiB/s, 19 objects/s recovering 2023-09-23T18:12:58.835 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 12009/75237 objects degraded (15.962%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:12:58.910 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:12:58.914 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:58.915 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:12:59.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:12:59.068 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:59.098 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:12:59.127 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:12:59.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:59.135 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:12:59.236 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:59.341 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:59.538 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:59 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1174997446' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:12:59.538 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:12:59 smithi099 ceph-mon[167076]: pgmap v1056: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 10822/67626 objects degraded (16.003%); 5549/67626 objects misplaced (8.205%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T18:12:59.546 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:12:59.552 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:12:59.552 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:12:59.564 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:12:59.653 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:12:59.693 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:12:59.693 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:12:59.745 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:12:59.835 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:59 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1174997446' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:12:59.835 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:12:59 smithi055 ceph-mon[184174]: pgmap v1056: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 10822/67626 objects degraded (16.003%); 5549/67626 objects misplaced (8.205%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T18:12:59.836 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1174997446' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:12:59.836 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:12:59 smithi055 ceph-mon[179668]: pgmap v1056: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.4 GiB data, 7.8 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 979 op/s; 10822/67626 objects degraded (16.003%); 5549/67626 objects misplaced (8.205%); 1.7 MiB/s, 27 objects/s recovering 2023-09-23T18:13:00.353 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:13:00.506 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:00.512 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:13:00] "GET /metrics HTTP/1.1" 200 36622 "" "Prometheus/2.43.0" 2023-09-23T18:13:00.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:00.513 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:00.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:00.669 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:00.726 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:00.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:00.729 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:00.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:00 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:00.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:00 smithi099 ceph-mon[167076]: failed to encode map e427 with expected crc 2023-09-23T18:13:00.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:00 smithi099 ceph-mon[167076]: failed to encode map e427 with expected crc 2023-09-23T18:13:00.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1174997446' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:13:00.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:00 smithi099 ceph-mon[167076]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T18:13:00.830 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:00.858 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:00.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:00.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-mon[179668]: failed to encode map e427 with expected crc 2023-09-23T18:13:00.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-mon[179668]: failed to encode map e427 with expected crc 2023-09-23T18:13:00.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1174997446' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:13:00.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-mon[179668]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T18:13:00.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:00.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-mon[184174]: failed to encode map e427 with expected crc 2023-09-23T18:13:00.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-mon[184174]: failed to encode map e427 with expected crc 2023-09-23T18:13:00.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1174997446' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:13:00.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:00 smithi055 ceph-mon[184174]: osdmap e427: 8 total, 8 up, 7 in 2023-09-23T18:13:00.899 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:01.044 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:01.052 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:01.052 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:01.153 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:01.221 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:01.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:01.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:01.291 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:01.392 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:01.452 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:01.520 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:01.534 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:01.534 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:01.635 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:01.660 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:01 smithi055 ceph-mon[184174]: pgmap v1058: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.3 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 887 op/s; 10287/61935 objects degraded (16.609%); 5256/61935 objects misplaced (8.486%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:13:01.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:01 smithi055 ceph-mon[179668]: pgmap v1058: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.3 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 887 op/s; 10287/61935 objects degraded (16.609%); 5256/61935 objects misplaced (8.486%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:13:01.675 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:01.698 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:01.723 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:01.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:01.729 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:01.830 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:01.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:01 smithi099 ceph-mon[167076]: pgmap v1058: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.3 GiB data, 7.5 GiB used, 618 GiB / 626 GiB avail; 0 B/s wr, 887 op/s; 10287/61935 objects degraded (16.609%); 5256/61935 objects misplaced (8.486%); 1.4 MiB/s, 21 objects/s recovering 2023-09-23T18:13:01.892 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:02.240 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:13:02.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:02.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:02.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3553452214' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:13:02.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:02 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:13:02.670 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 8 pg_num to 26 2023-09-23T18:13:02.689 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:02.695 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:02.695 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:02.796 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:02.860 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3553452214' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:13:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:02 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:13:02.941 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:02.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:02.951 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:02.953 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:02.953 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:02.953 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3553452214' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:13:02.954 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:02 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]: dispatch 2023-09-23T18:13:03.052 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:03.097 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:03.156 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:03.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:03.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:03.267 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:03.433 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:03.538 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:03.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:03.545 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:03.646 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-09-23T18:13:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[179668]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T18:13:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[179668]: failed to encode map e428 with expected crc 2023-09-23T18:13:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[179668]: failed to encode map e428 with expected crc 2023-09-23T18:13:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[179668]: pgmap v1060: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 9434/56316 objects degraded (16.752%); 4749/56316 objects misplaced (8.433%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:13:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T18:13:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 10287/61935 objects degraded (16.609%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:13:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:13:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-09-23T18:13:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[184174]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T18:13:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[184174]: failed to encode map e428 with expected crc 2023-09-23T18:13:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[184174]: failed to encode map e428 with expected crc 2023-09-23T18:13:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[184174]: pgmap v1060: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 9434/56316 objects degraded (16.752%); 4749/56316 objects misplaced (8.433%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:13:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T18:13:03.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 10287/61935 objects degraded (16.609%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:03.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:13:03.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:13:03.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:03 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:03.785 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:03 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num", "val": "26"}]': finished 2023-09-23T18:13:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:03 smithi099 ceph-mon[167076]: osdmap e428: 8 total, 8 up, 7 in 2023-09-23T18:13:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:03 smithi099 ceph-mon[167076]: failed to encode map e428 with expected crc 2023-09-23T18:13:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:03 smithi099 ceph-mon[167076]: failed to encode map e428 with expected crc 2023-09-23T18:13:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:03 smithi099 ceph-mon[167076]: pgmap v1060: 119 pgs: 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.1 GiB data, 7.3 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 1.03k op/s; 9434/56316 objects degraded (16.752%); 4749/56316 objects misplaced (8.433%); 1.8 MiB/s, 28 objects/s recovering 2023-09-23T18:13:04.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:03 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]: dispatch 2023-09-23T18:13:04.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 10287/61935 objects degraded (16.609%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:04.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:03 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:13:04.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:03 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:13:04.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:03 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:04 smithi055 ceph-mon[179668]: failed to encode map e429 with expected crc 2023-09-23T18:13:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:04 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T18:13:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:04 smithi055 ceph-mon[179668]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T18:13:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:04 smithi055 ceph-mon[179668]: failed to encode map e429 with expected crc 2023-09-23T18:13:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:04 smithi055 ceph-mon[179668]: failed to encode map e429 with expected crc 2023-09-23T18:13:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:04 smithi055 ceph-mon[184174]: failed to encode map e429 with expected crc 2023-09-23T18:13:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:04 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T18:13:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:04 smithi055 ceph-mon[184174]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T18:13:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:04 smithi055 ceph-mon[184174]: failed to encode map e429 with expected crc 2023-09-23T18:13:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:04 smithi055 ceph-mon[184174]: failed to encode map e429 with expected crc 2023-09-23T18:13:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:04 smithi099 ceph-mon[167076]: failed to encode map e429 with expected crc 2023-09-23T18:13:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:04 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_3", "var": "pg_num_actual", "val": "26"}]': finished 2023-09-23T18:13:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:04 smithi099 ceph-mon[167076]: osdmap e429: 8 total, 8 up, 7 in 2023-09-23T18:13:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:04 smithi099 ceph-mon[167076]: failed to encode map e429 with expected crc 2023-09-23T18:13:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:04 smithi099 ceph-mon[167076]: failed to encode map e429 with expected crc 2023-09-23T18:13:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:05 smithi099 ceph-mon[167076]: failed to encode map e430 with expected crc 2023-09-23T18:13:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:05 smithi099 ceph-mon[167076]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T18:13:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:05 smithi099 ceph-mon[167076]: failed to encode map e430 with expected crc 2023-09-23T18:13:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:05 smithi099 ceph-mon[167076]: 8.9 continuing backfill to osd.5 from (396'8105,428'12829] MIN to 428'12829 2023-09-23T18:13:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:05 smithi099 ceph-mon[167076]: 8.1 continuing backfill to osd.5 from (402'8440,428'12742] MIN to 428'12742 2023-09-23T18:13:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:05 smithi099 ceph-mon[167076]: 8.3 continuing backfill to osd.5 from (402'8545,428'12922] MIN to 428'12922 2023-09-23T18:13:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:05 smithi099 ceph-mon[167076]: 8.4 continuing backfill to osd.5 from (401'8185,428'12677] MIN to 428'12677 2023-09-23T18:13:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:05 smithi099 ceph-mon[167076]: 8.0 starting backfill to osd.5 from (402'8420,428'12792] MIN to 428'12793 2023-09-23T18:13:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:05 smithi099 ceph-mon[167076]: 8.2 continuing backfill to osd.5 from (402'8171,428'12708] MIN to 428'12708 2023-09-23T18:13:06.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:05 smithi099 ceph-mon[167076]: pgmap v1063: 129 pgs: 10 unknown, 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.1 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 9155/54492 objects degraded (16.801%); 4429/54492 objects misplaced (8.128%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:13:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[184174]: failed to encode map e430 with expected crc 2023-09-23T18:13:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[184174]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T18:13:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[184174]: failed to encode map e430 with expected crc 2023-09-23T18:13:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[184174]: 8.9 continuing backfill to osd.5 from (396'8105,428'12829] MIN to 428'12829 2023-09-23T18:13:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[184174]: 8.1 continuing backfill to osd.5 from (402'8440,428'12742] MIN to 428'12742 2023-09-23T18:13:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[184174]: 8.3 continuing backfill to osd.5 from (402'8545,428'12922] MIN to 428'12922 2023-09-23T18:13:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[184174]: 8.4 continuing backfill to osd.5 from (401'8185,428'12677] MIN to 428'12677 2023-09-23T18:13:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[184174]: 8.0 starting backfill to osd.5 from (402'8420,428'12792] MIN to 428'12793 2023-09-23T18:13:06.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[184174]: 8.2 continuing backfill to osd.5 from (402'8171,428'12708] MIN to 428'12708 2023-09-23T18:13:06.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[184174]: pgmap v1063: 129 pgs: 10 unknown, 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.1 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 9155/54492 objects degraded (16.801%); 4429/54492 objects misplaced (8.128%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:13:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[179668]: failed to encode map e430 with expected crc 2023-09-23T18:13:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[179668]: osdmap e430: 8 total, 8 up, 7 in 2023-09-23T18:13:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[179668]: failed to encode map e430 with expected crc 2023-09-23T18:13:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[179668]: 8.9 continuing backfill to osd.5 from (396'8105,428'12829] MIN to 428'12829 2023-09-23T18:13:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[179668]: 8.1 continuing backfill to osd.5 from (402'8440,428'12742] MIN to 428'12742 2023-09-23T18:13:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[179668]: 8.3 continuing backfill to osd.5 from (402'8545,428'12922] MIN to 428'12922 2023-09-23T18:13:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[179668]: 8.4 continuing backfill to osd.5 from (401'8185,428'12677] MIN to 428'12677 2023-09-23T18:13:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[179668]: 8.0 starting backfill to osd.5 from (402'8420,428'12792] MIN to 428'12793 2023-09-23T18:13:06.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[179668]: 8.2 continuing backfill to osd.5 from (402'8171,428'12708] MIN to 428'12708 2023-09-23T18:13:06.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:05 smithi055 ceph-mon[179668]: pgmap v1063: 129 pgs: 10 unknown, 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 3 active+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 1.1 GiB data, 7.1 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 805 op/s; 9155/54492 objects degraded (16.801%); 4429/54492 objects misplaced (8.128%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:13:06.474 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:06.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:06.478 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:06.579 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:06.605 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:06.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:06.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:06.646 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:06.747 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:06.782 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[179668]: failed to encode map e431 with expected crc 2023-09-23T18:13:06.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[179668]: failed to encode map e431 with expected crc 2023-09-23T18:13:06.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[179668]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T18:13:06.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[179668]: failed to encode map e431 with expected crc 2023-09-23T18:13:06.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[179668]: failed to encode map e431 with expected crc 2023-09-23T18:13:06.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[179668]: failed to encode map e431 with expected crc 2023-09-23T18:13:06.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:06.783 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:06.783 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[184174]: failed to encode map e431 with expected crc 2023-09-23T18:13:06.784 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[184174]: failed to encode map e431 with expected crc 2023-09-23T18:13:06.784 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[184174]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T18:13:06.784 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[184174]: failed to encode map e431 with expected crc 2023-09-23T18:13:06.784 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[184174]: failed to encode map e431 with expected crc 2023-09-23T18:13:06.784 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[184174]: failed to encode map e431 with expected crc 2023-09-23T18:13:06.784 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:06.784 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:06.786 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:06.878 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:06.880 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:06.881 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:06.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:07.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:06 smithi099 ceph-mon[167076]: failed to encode map e431 with expected crc 2023-09-23T18:13:07.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:06 smithi099 ceph-mon[167076]: failed to encode map e431 with expected crc 2023-09-23T18:13:07.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:06 smithi099 ceph-mon[167076]: osdmap e431: 8 total, 8 up, 7 in 2023-09-23T18:13:07.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:06 smithi099 ceph-mon[167076]: failed to encode map e431 with expected crc 2023-09-23T18:13:07.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:06 smithi099 ceph-mon[167076]: failed to encode map e431 with expected crc 2023-09-23T18:13:07.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:06 smithi099 ceph-mon[167076]: failed to encode map e431 with expected crc 2023-09-23T18:13:07.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:07.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:07.032 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:07.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:07.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:07.080 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:07.181 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:07.221 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:07.230 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:07.281 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:07.285 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:07.285 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:07.386 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:07.424 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:07.436 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:13:07.459 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:13:07.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:07.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:07.471 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:07.482 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:13:07.504 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:13:07.527 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:13:07.547 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:13:07.568 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:13:07.572 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:07.590 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:13:07.594 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:13:07.594 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:13:07.595 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:13:07.595 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:13:07.595 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:13:07.595 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:13:07.595 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:13:07.595 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:13:07.607 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:07.723 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.723 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T18:13:07.723 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: 8.11 continuing backfill to osd.5 from (402'8440,428'12742] MIN to 428'12742 2023-09-23T18:13:07.723 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.723 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.723 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: 8.13 continuing backfill to osd.5 from (402'8545,428'12922] MIN to 428'12922 2023-09-23T18:13:07.723 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.723 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: 8.14 continuing backfill to osd.5 from (401'8185,428'12677] MIN to 428'12677 2023-09-23T18:13:07.723 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: 8.12 continuing backfill to osd.5 from (402'8171,428'12708] MIN to 428'12708 2023-09-23T18:13:07.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T18:13:07.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: 8.11 continuing backfill to osd.5 from (402'8440,428'12742] MIN to 428'12742 2023-09-23T18:13:07.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: 8.13 continuing backfill to osd.5 from (402'8545,428'12922] MIN to 428'12922 2023-09-23T18:13:07.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: 8.14 continuing backfill to osd.5 from (401'8185,428'12677] MIN to 428'12677 2023-09-23T18:13:07.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: 8.12 continuing backfill to osd.5 from (402'8171,428'12708] MIN to 428'12708 2023-09-23T18:13:07.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: 8.10 starting backfill to osd.5 from (402'8420,428'12792] MIN to 428'12793 2023-09-23T18:13:07.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: 8.19 continuing backfill to osd.5 from (396'8105,428'12829] MIN to 428'12829 2023-09-23T18:13:07.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: pgmap v1066: 129 pgs: 3 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 3 peering, 3 unknown, 5 active+undersized+degraded+remapped+backfill_wait, 3 active+remapped+backfill_wait, 111 active+clean; 1.1 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 4870/50718 objects degraded (9.602%); 3657/50718 objects misplaced (7.210%); 6.8 MiB/s, 181 objects/s recovering 2023-09-23T18:13:07.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1789101400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:07.812 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.812 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: osdmap e432: 8 total, 8 up, 7 in 2023-09-23T18:13:07.812 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: 8.11 continuing backfill to osd.5 from (402'8440,428'12742] MIN to 428'12742 2023-09-23T18:13:07.812 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.812 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.812 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: 8.13 continuing backfill to osd.5 from (402'8545,428'12922] MIN to 428'12922 2023-09-23T18:13:07.812 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: failed to encode map e432 with expected crc 2023-09-23T18:13:07.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: 8.14 continuing backfill to osd.5 from (401'8185,428'12677] MIN to 428'12677 2023-09-23T18:13:07.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: 8.12 continuing backfill to osd.5 from (402'8171,428'12708] MIN to 428'12708 2023-09-23T18:13:07.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: 8.10 starting backfill to osd.5 from (402'8420,428'12792] MIN to 428'12793 2023-09-23T18:13:07.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: 8.19 continuing backfill to osd.5 from (396'8105,428'12829] MIN to 428'12829 2023-09-23T18:13:07.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: pgmap v1066: 129 pgs: 3 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 3 peering, 3 unknown, 5 active+undersized+degraded+remapped+backfill_wait, 3 active+remapped+backfill_wait, 111 active+clean; 1.1 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 4870/50718 objects degraded (9.602%); 3657/50718 objects misplaced (7.210%); 6.8 MiB/s, 181 objects/s recovering 2023-09-23T18:13:07.813 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:07 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1789101400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:08.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: 8.10 starting backfill to osd.5 from (402'8420,428'12792] MIN to 428'12793 2023-09-23T18:13:08.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: 8.19 continuing backfill to osd.5 from (396'8105,428'12829] MIN to 428'12829 2023-09-23T18:13:08.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: pgmap v1066: 129 pgs: 3 remapped+peering, 1 active+undersized+degraded+remapped+backfilling, 3 peering, 3 unknown, 5 active+undersized+degraded+remapped+backfill_wait, 3 active+remapped+backfill_wait, 111 active+clean; 1.1 GiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 466 op/s; 4870/50718 objects degraded (9.602%); 3657/50718 objects misplaced (7.210%); 6.8 MiB/s, 181 objects/s recovering 2023-09-23T18:13:08.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:07 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1789101400' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:08.260 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:08.267 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:08.267 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:08.368 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:08.388 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:13:08.388 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:13:08.388 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:13:08.389 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:13:08.405 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:08.499 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:08.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:08.504 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:08.605 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:08.653 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:08.716 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:08.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:08.721 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:08.822 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:08.874 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:13:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[179668]: failed to encode map e433 with expected crc 2023-09-23T18:13:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[179668]: failed to encode map e433 with expected crc 2023-09-23T18:13:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[179668]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T18:13:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[179668]: failed to encode map e433 with expected crc 2023-09-23T18:13:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 4870/50718 objects degraded (9.602%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:08.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:13:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[184174]: failed to encode map e433 with expected crc 2023-09-23T18:13:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[184174]: failed to encode map e433 with expected crc 2023-09-23T18:13:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[184174]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T18:13:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[184174]: failed to encode map e433 with expected crc 2023-09-23T18:13:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 4870/50718 objects degraded (9.602%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:09.013 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:13:09.016 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:09.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:09.019 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:09.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:08 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:13:09.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:08 smithi099 ceph-mon[167076]: failed to encode map e433 with expected crc 2023-09-23T18:13:09.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:08 smithi099 ceph-mon[167076]: failed to encode map e433 with expected crc 2023-09-23T18:13:09.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:08 smithi099 ceph-mon[167076]: osdmap e433: 8 total, 8 up, 7 in 2023-09-23T18:13:09.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:08 smithi099 ceph-mon[167076]: failed to encode map e433 with expected crc 2023-09-23T18:13:09.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 4870/50718 objects degraded (9.602%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:09.120 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:09.165 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:09.209 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:09.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:09.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:09.314 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:09.356 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:09.402 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:09.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:09.405 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:09.506 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:09.563 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:09.614 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:09.617 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:09.618 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:09.710 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:09.718 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:09.756 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:13:09.767 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:09 smithi099 ceph-mon[167076]: pgmap v1068: 129 pgs: 1 active+undersized+degraded+remapped, 3 remapped+peering, 2 active+undersized+degraded+remapped+backfilling, 2 peering, 4 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 112 active+clean; 1016 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 3889/48639 objects degraded (7.996%); 3240/48639 objects misplaced (6.661%); 16 MiB/s, 323 objects/s recovering 2023-09-23T18:13:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2797949418' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:09 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:09.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:09.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:09.838 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:09.939 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:09.982 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:10.002 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:09 smithi055 ceph-mon[184174]: pgmap v1068: 129 pgs: 1 active+undersized+degraded+remapped, 3 remapped+peering, 2 active+undersized+degraded+remapped+backfilling, 2 peering, 4 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 112 active+clean; 1016 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 3889/48639 objects degraded (7.996%); 3240/48639 objects misplaced (6.661%); 16 MiB/s, 323 objects/s recovering 2023-09-23T18:13:10.002 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2797949418' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:10.002 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:09 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:10.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:09 smithi055 ceph-mon[179668]: pgmap v1068: 129 pgs: 1 active+undersized+degraded+remapped, 3 remapped+peering, 2 active+undersized+degraded+remapped+backfilling, 2 peering, 4 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 4 active+remapped+backfill_wait, 112 active+clean; 1016 MiB data, 6.9 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 454 op/s; 3889/48639 objects degraded (7.996%); 3240/48639 objects misplaced (6.661%); 16 MiB/s, 323 objects/s recovering 2023-09-23T18:13:10.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2797949418' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:10.003 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:09 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:10.058 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:13:10.204 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:10.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:10.209 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:10.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:10.348 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:10.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:10.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:10.410 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:10.511 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:10.557 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:10.605 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:10.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:10.608 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:10.667 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:13:10] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-09-23T18:13:10.709 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:10.757 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:10.805 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:10.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:10.812 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:10.894 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:10.913 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:13:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-mon[184174]: failed to encode map e434 with expected crc 2023-09-23T18:13:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-mon[184174]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T18:13:10.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:13:10.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:10.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-mon[179668]: failed to encode map e434 with expected crc 2023-09-23T18:13:10.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:10.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:10 smithi055 ceph-mon[179668]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T18:13:10.958 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:10 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:13:10.958 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:10 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:10.958 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:10 smithi099 ceph-mon[167076]: failed to encode map e434 with expected crc 2023-09-23T18:13:10.958 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:10 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:10.958 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:10 smithi099 ceph-mon[167076]: osdmap e434: 8 total, 8 up, 7 in 2023-09-23T18:13:10.959 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:11.035 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:11.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:11.041 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:11.142 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:11.188 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:11.248 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:13:11.248 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:13:11.249 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:13:11.249 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:13:11.249 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:13:11.252 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:11.257 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:11.257 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:11.358 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:11.404 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:11.459 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:11.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:11.465 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:11.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:11.601 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 3, 4] out_osds: [2] dead_osds: [] live_osds: [0, 1, 2, 4, 6, 7, 3, 5] 2023-09-23T18:13:11.602 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:13:11.602 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-09-23T18:13:11.602 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 2 2023-09-23T18:13:11.618 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:11.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:11.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:11.682 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:11.783 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:11.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:11 smithi055 ceph-mon[184174]: pgmap v1070: 129 pgs: 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 113 active+clean; 918 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 7907/43920 objects degraded (18.003%); 3580/43920 objects misplaced (8.151%); 17 MiB/s, 340 objects/s recovering 2023-09-23T18:13:11.818 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:11 smithi055 ceph-mon[179668]: pgmap v1070: 129 pgs: 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 113 active+clean; 918 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 7907/43920 objects degraded (18.003%); 3580/43920 objects misplaced (8.151%); 17 MiB/s, 340 objects/s recovering 2023-09-23T18:13:11.830 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:11 smithi099 ceph-mon[167076]: pgmap v1070: 129 pgs: 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 113 active+clean; 918 MiB data, 6.5 GiB used, 619 GiB / 626 GiB avail; 0 B/s wr, 653 op/s; 7907/43920 objects degraded (18.003%); 3580/43920 objects misplaced (8.151%); 17 MiB/s, 340 objects/s recovering 2023-09-23T18:13:12.252 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:13:12.658 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:12.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:12.661 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:12.731 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:13:12.762 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:12.803 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[184174]: failed to encode map e435 with expected crc 2023-09-23T18:13:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[184174]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T18:13:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[184174]: failed to encode map e435 with expected crc 2023-09-23T18:13:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[184174]: failed to encode map e435 with expected crc 2023-09-23T18:13:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[184174]: failed to encode map e435 with expected crc 2023-09-23T18:13:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[184174]: failed to encode map e435 with expected crc 2023-09-23T18:13:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[179668]: failed to encode map e435 with expected crc 2023-09-23T18:13:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[179668]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T18:13:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[179668]: failed to encode map e435 with expected crc 2023-09-23T18:13:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[179668]: failed to encode map e435 with expected crc 2023-09-23T18:13:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[179668]: failed to encode map e435 with expected crc 2023-09-23T18:13:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:12 smithi055 ceph-mon[179668]: failed to encode map e435 with expected crc 2023-09-23T18:13:13.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:12 smithi099 ceph-mon[167076]: failed to encode map e435 with expected crc 2023-09-23T18:13:13.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:12 smithi099 ceph-mon[167076]: osdmap e435: 8 total, 8 up, 7 in 2023-09-23T18:13:13.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:12 smithi099 ceph-mon[167076]: failed to encode map e435 with expected crc 2023-09-23T18:13:13.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:12 smithi099 ceph-mon[167076]: failed to encode map e435 with expected crc 2023-09-23T18:13:13.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:12 smithi099 ceph-mon[167076]: failed to encode map e435 with expected crc 2023-09-23T18:13:13.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:12 smithi099 ceph-mon[167076]: failed to encode map e435 with expected crc 2023-09-23T18:13:13.691 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:13.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:13.696 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:13.797 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:13.829 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:13.875 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:13.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:13.882 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:13.941 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:13.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:14.015 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:14.055 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:14.056 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:13 smithi099 ceph-mon[167076]: failed to encode map e436 with expected crc 2023-09-23T18:13:14.056 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:13 smithi099 ceph-mon[167076]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T18:13:14.056 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:13 smithi099 ceph-mon[167076]: failed to encode map e436 with expected crc 2023-09-23T18:13:14.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:13 smithi099 ceph-mon[167076]: pgmap v1073: 129 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 113 active+clean; 830 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 7326/39696 objects degraded (18.455%); 3186/39696 objects misplaced (8.026%); 12 MiB/s, 209 objects/s recovering 2023-09-23T18:13:14.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7907/43920 objects degraded (18.003%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:14.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:13 smithi099 ceph-mon[167076]: failed to encode map e437 with expected crc 2023-09-23T18:13:14.057 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:13 smithi099 ceph-mon[167076]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T18:13:14.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:14.059 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:14.160 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:14.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[184174]: failed to encode map e436 with expected crc 2023-09-23T18:13:14.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[184174]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T18:13:14.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[184174]: failed to encode map e436 with expected crc 2023-09-23T18:13:14.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[184174]: pgmap v1073: 129 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 113 active+clean; 830 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 7326/39696 objects degraded (18.455%); 3186/39696 objects misplaced (8.026%); 12 MiB/s, 209 objects/s recovering 2023-09-23T18:13:14.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7907/43920 objects degraded (18.003%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:14.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[184174]: failed to encode map e437 with expected crc 2023-09-23T18:13:14.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[184174]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T18:13:14.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[179668]: failed to encode map e436 with expected crc 2023-09-23T18:13:14.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[179668]: osdmap e436: 8 total, 8 up, 7 in 2023-09-23T18:13:14.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[179668]: failed to encode map e436 with expected crc 2023-09-23T18:13:14.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[179668]: pgmap v1073: 129 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped+backfilling, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 113 active+clean; 830 MiB data, 6.3 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 698 op/s; 7326/39696 objects degraded (18.455%); 3186/39696 objects misplaced (8.026%); 12 MiB/s, 209 objects/s recovering 2023-09-23T18:13:14.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7907/43920 objects degraded (18.003%), 11 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[179668]: failed to encode map e437 with expected crc 2023-09-23T18:13:14.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:13 smithi055 ceph-mon[179668]: osdmap e437: 8 total, 8 up, 7 in 2023-09-23T18:13:14.190 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:14.527 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:14.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:14.532 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:14.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:14.691 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:14.766 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:14.773 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:14.774 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:14.874 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:14.934 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:14 smithi099 ceph-mon[167076]: failed to encode map e437 with expected crc 2023-09-23T18:13:14.934 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:14 smithi099 ceph-mon[167076]: failed to encode map e437 with expected crc 2023-09-23T18:13:14.934 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:14 smithi099 ceph-mon[167076]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T18:13:14.936 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:14.936 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:13:14.942 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:14.965 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:13:14.989 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:14.996 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:13:14.999 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:14.999 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:15.029 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:13:15.063 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:13:15.096 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:13:15.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:14 smithi055 ceph-mon[184174]: failed to encode map e437 with expected crc 2023-09-23T18:13:15.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:14 smithi055 ceph-mon[184174]: failed to encode map e437 with expected crc 2023-09-23T18:13:15.097 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:14 smithi055 ceph-mon[184174]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T18:13:15.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:14 smithi055 ceph-mon[179668]: failed to encode map e437 with expected crc 2023-09-23T18:13:15.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:14 smithi055 ceph-mon[179668]: failed to encode map e437 with expected crc 2023-09-23T18:13:15.098 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:14 smithi055 ceph-mon[179668]: osdmap e438: 8 total, 8 up, 7 in 2023-09-23T18:13:15.100 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:15.127 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:13:15.142 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:15.159 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:13:15.162 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:13:15.162 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:13:15.162 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:13:15.162 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:13:15.162 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:13:15.162 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:13:15.163 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:13:15.163 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:13:15.253 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:15.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:15.261 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:15.361 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:15.407 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:15.459 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:15.463 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:15.464 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:15.564 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:15.614 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:15.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:15.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:15.674 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:15.774 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:15.832 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:15.852 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:15.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:15.891 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:15 smithi099 ceph-mon[167076]: failed to encode map e438 with expected crc 2023-09-23T18:13:15.891 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1182629636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:15.892 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:15 smithi099 ceph-mon[167076]: pgmap v1076: 129 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 114 active+clean; 802 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 6708/38370 objects degraded (17.482%); 2944/38370 objects misplaced (7.673%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T18:13:15.892 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:15 smithi099 ceph-mon[167076]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T18:13:15.892 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:15 smithi099 ceph-mon[167076]: failed to encode map e439 with expected crc 2023-09-23T18:13:15.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:15.893 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:15.994 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:16.031 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:16.038 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:15 smithi055 ceph-mon[184174]: failed to encode map e438 with expected crc 2023-09-23T18:13:16.038 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1182629636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:16.038 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:15 smithi055 ceph-mon[184174]: pgmap v1076: 129 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 114 active+clean; 802 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 6708/38370 objects degraded (17.482%); 2944/38370 objects misplaced (7.673%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T18:13:16.038 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:15 smithi055 ceph-mon[184174]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T18:13:16.039 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:15 smithi055 ceph-mon[184174]: failed to encode map e439 with expected crc 2023-09-23T18:13:16.039 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:15 smithi055 ceph-mon[179668]: failed to encode map e438 with expected crc 2023-09-23T18:13:16.039 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1182629636' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:16.039 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:15 smithi055 ceph-mon[179668]: pgmap v1076: 129 pgs: 1 active+undersized+degraded+remapped, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 6 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 114 active+clean; 802 MiB data, 6.2 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 462 op/s; 6708/38370 objects degraded (17.482%); 2944/38370 objects misplaced (7.673%); 8.9 MiB/s, 143 objects/s recovering 2023-09-23T18:13:16.039 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:15 smithi055 ceph-mon[179668]: osdmap e439: 8 total, 8 up, 7 in 2023-09-23T18:13:16.039 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:15 smithi055 ceph-mon[179668]: failed to encode map e439 with expected crc 2023-09-23T18:13:16.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:16.084 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:16.085 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:16.185 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:16.219 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:16.269 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:16.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:16.278 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:16.379 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:16.421 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:16.481 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:16.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:16.486 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:16.587 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:16.644 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:16.697 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:16.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:16.701 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:16.802 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:16.853 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:16 smithi055 ceph-mon[184174]: failed to encode map e439 with expected crc 2023-09-23T18:13:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:16 smithi055 ceph-mon[184174]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T18:13:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:16 smithi055 ceph-mon[184174]: failed to encode map e440 with expected crc 2023-09-23T18:13:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:16 smithi055 ceph-mon[179668]: failed to encode map e439 with expected crc 2023-09-23T18:13:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:16 smithi055 ceph-mon[179668]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T18:13:16.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:16 smithi055 ceph-mon[179668]: failed to encode map e440 with expected crc 2023-09-23T18:13:17.060 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:17.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:17.065 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:16 smithi099 ceph-mon[167076]: failed to encode map e439 with expected crc 2023-09-23T18:13:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:16 smithi099 ceph-mon[167076]: osdmap e440: 8 total, 8 up, 7 in 2023-09-23T18:13:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:16 smithi099 ceph-mon[167076]: failed to encode map e440 with expected crc 2023-09-23T18:13:17.166 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:17.212 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:17.258 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.2. 2023-09-23T18:13:17.279 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:17.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:17.282 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:17.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:17.409 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:17.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:13:17.174+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e440 definitely_dead 0 2023-09-23T18:13:17.449 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:17.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:17.452 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:17.553 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:17.589 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:17.628 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:17.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:17.631 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:17.649 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-09-23T18:13:17.732 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:17.751 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:13:17.762 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:17.857 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:17.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:17.860 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:17.919 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:13:17.920 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:13:17.944 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[184174]: failed to encode map e440 with expected crc 2023-09-23T18:13:17.944 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[184174]: pgmap v1079: 129 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 114 active+clean; 682 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 5204/32598 objects degraded (15.964%); 2695/32598 objects misplaced (8.267%); 29 MiB/s, 462 objects/s recovering 2023-09-23T18:13:17.944 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3817139352' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T18:13:17.945 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3817139352' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T18:13:17.945 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[184174]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T18:13:17.945 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3535131116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:17.945 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:17.946 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[179668]: failed to encode map e440 with expected crc 2023-09-23T18:13:17.946 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[179668]: pgmap v1079: 129 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 114 active+clean; 682 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 5204/32598 objects degraded (15.964%); 2695/32598 objects misplaced (8.267%); 29 MiB/s, 462 objects/s recovering 2023-09-23T18:13:17.946 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3817139352' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T18:13:17.946 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3817139352' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T18:13:17.946 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[179668]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T18:13:17.947 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3535131116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:17.947 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:17 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:17.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:17.989 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:18.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:18.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:18.028 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:17 smithi099 ceph-mon[167076]: failed to encode map e440 with expected crc 2023-09-23T18:13:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:17 smithi099 ceph-mon[167076]: pgmap v1079: 129 pgs: 1 peering, 1 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 1 active+undersized+degraded+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 114 active+clean; 682 MiB data, 5.8 GiB used, 620 GiB / 626 GiB avail; 0 B/s wr, 591 op/s; 5204/32598 objects degraded (15.964%); 2695/32598 objects misplaced (8.267%); 29 MiB/s, 462 objects/s recovering 2023-09-23T18:13:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3817139352' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T18:13:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3817139352' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T18:13:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:17 smithi099 ceph-mon[167076]: osdmap e441: 8 total, 8 up, 8 in 2023-09-23T18:13:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3535131116' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:17 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:18.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:18.155 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:18.191 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:18.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:18.201 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:18.201 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:13:18.302 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:18.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:13:18.340 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:18.521 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:18.525 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:18.525 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:18.626 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:18.654 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:18.675 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:18.688 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:18.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:18.691 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:18.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:18.824 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:18.868 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:18.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:18.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:18.973 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:19.001 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:19.016 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:19.043 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:19.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:19.047 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:18 smithi099 ceph-mon[167076]: failed to encode map e441 with expected crc 2023-09-23T18:13:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 5204/32598 objects degraded (15.964%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:18 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:18 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:13:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:18 smithi099 ceph-mon[167076]: osdmap e442: 8 total, 8 up, 8 in 2023-09-23T18:13:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:18 smithi099 ceph-mon[167076]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:18 smithi099 ceph-mon[167076]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:18 smithi099 ceph-mon[167076]: 8.a continuing backfill to osd.5 from (407'9246,440'13373] 8:54d9879c:::benchmark_data_smithi055_346788_object98394:head to 440'13373 2023-09-23T18:13:19.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:18 smithi099 ceph-mon[167076]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.147 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:19.151 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[179668]: failed to encode map e441 with expected crc 2023-09-23T18:13:19.151 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 5204/32598 objects degraded (15.964%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:19.151 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:19.151 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:13:19.151 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[179668]: osdmap e442: 8 total, 8 up, 8 in 2023-09-23T18:13:19.151 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[179668]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.151 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[179668]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.151 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[179668]: 8.a continuing backfill to osd.5 from (407'9246,440'13373] 8:54d9879c:::benchmark_data_smithi055_346788_object98394:head to 440'13373 2023-09-23T18:13:19.151 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[179668]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[184174]: failed to encode map e441 with expected crc 2023-09-23T18:13:19.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 5204/32598 objects degraded (15.964%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:19.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:19.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:13:19.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[184174]: osdmap e442: 8 total, 8 up, 8 in 2023-09-23T18:13:19.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[184174]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[184174]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[184174]: 8.a continuing backfill to osd.5 from (407'9246,440'13373] 8:54d9879c:::benchmark_data_smithi055_346788_object98394:head to 440'13373 2023-09-23T18:13:19.152 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:18 smithi055 ceph-mon[184174]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.178 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:19.392 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:19.397 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:19.398 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:19.498 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:19.538 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:19.618 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:19.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:19.624 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:19.724 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:19.761 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:19.854 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:19.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:19.857 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:19.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:19 smithi099 ceph-mon[167076]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:19 smithi099 ceph-mon[167076]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.868 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:19 smithi099 ceph-mon[167076]: failed to encode map e442 with expected crc 2023-09-23T18:13:19.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:19 smithi099 ceph-mon[167076]: pgmap v1082: 129 pgs: 2 remapped+peering, 14 peering, 2 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 103 active+clean; 569 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 933 op/s; 4054/27162 objects degraded (14.925%); 617/27162 objects misplaced (2.272%); 22 MiB/s, 359 objects/s recovering 2023-09-23T18:13:19.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:19 smithi099 ceph-mon[167076]: osdmap e443: 8 total, 8 up, 8 in 2023-09-23T18:13:19.869 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:19 smithi099 ceph-mon[167076]: 8.e starting backfill to osd.2 from (402'8309,428'12862] MIN to 440'13340 2023-09-23T18:13:19.930 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:13:19.949 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:13:19.958 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:19.971 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:13:19.989 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:19.993 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:13:20.013 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:13:20.023 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:20.025 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[179668]: failed to encode map e442 with expected crc 2023-09-23T18:13:20.025 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[179668]: failed to encode map e442 with expected crc 2023-09-23T18:13:20.025 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[179668]: failed to encode map e442 with expected crc 2023-09-23T18:13:20.026 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[179668]: pgmap v1082: 129 pgs: 2 remapped+peering, 14 peering, 2 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 103 active+clean; 569 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 933 op/s; 4054/27162 objects degraded (14.925%); 617/27162 objects misplaced (2.272%); 22 MiB/s, 359 objects/s recovering 2023-09-23T18:13:20.026 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[179668]: osdmap e443: 8 total, 8 up, 8 in 2023-09-23T18:13:20.026 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[179668]: 8.e starting backfill to osd.2 from (402'8309,428'12862] MIN to 440'13340 2023-09-23T18:13:20.026 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[184174]: failed to encode map e442 with expected crc 2023-09-23T18:13:20.026 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[184174]: failed to encode map e442 with expected crc 2023-09-23T18:13:20.026 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[184174]: failed to encode map e442 with expected crc 2023-09-23T18:13:20.026 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[184174]: pgmap v1082: 129 pgs: 2 remapped+peering, 14 peering, 2 active+undersized+degraded+remapped+backfilling, 1 active+recovering+degraded, 1 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovery_wait, 2 active+remapped+backfill_wait, 103 active+clean; 569 MiB data, 6.6 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 933 op/s; 4054/27162 objects degraded (14.925%); 617/27162 objects misplaced (2.272%); 22 MiB/s, 359 objects/s recovering 2023-09-23T18:13:20.026 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[184174]: osdmap e443: 8 total, 8 up, 8 in 2023-09-23T18:13:20.027 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:19 smithi055 ceph-mon[184174]: 8.e starting backfill to osd.2 from (402'8309,428'12862] MIN to 440'13340 2023-09-23T18:13:20.027 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:20.027 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:20.033 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:13:20.052 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:13:20.071 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:13:20.072 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:13:20.073 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:13:20.073 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:13:20.073 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:13:20.073 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:13:20.073 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:13:20.073 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:13:20.073 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:13:20.129 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:20.156 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:20.193 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:20.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:20.196 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:20.297 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:20.331 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:20.383 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:20.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:20.386 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:20.487 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:20.518 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:20.546 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:13:20.556 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:20.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:20.559 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:20.605 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:13:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:13:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213448 8.e 8.f5f7327e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.277918+0000", 2023-09-23T18:13:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.32582823700000002, 2023-09-23T18:13:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.32585863999999998, 2023-09-23T18:13:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.605 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213448 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.277918+0000", 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.277918+0000", 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.277917+0000", 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.277923+0000", 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5699999999999998e-06 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.277937+0000", 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3551e-05 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.277944+0000", 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9530000000000002e-06 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.277999+0000", 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5426000000000001e-05 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.278008+0000", 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1780000000000003e-06 2023-09-23T18:13:20.609 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213669 8.e 8.2e591a0e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.517278+0000", 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.086469110000000002, 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.086575535999999995, 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.610 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213669 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.517278+0000", 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.517278+0000", 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.517276+0000", 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.517284+0000", 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3039999999999999e-06 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.517300+0000", 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6748999999999998e-05 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.517306+0000", 2023-09-23T18:13:20.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0090000000000004e-06 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.517365+0000", 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8477999999999998e-05 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.517371+0000", 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5289999999999997e-06 2023-09-23T18:13:20.614 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213525 8.e 8.31157dce (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.343299+0000", 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.26044742799999998, 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.26062036700000002, 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.615 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213525 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.343299+0000", 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.343299+0000", 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.343298+0000", 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.343303+0000", 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5260000000000001e-06 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.343318+0000", 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4924e-05 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.343325+0000", 2023-09-23T18:13:20.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3060000000000003e-06 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.343381+0000", 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6165000000000002e-05 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.343390+0000", 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3000000000000007e-06 2023-09-23T18:13:20.619 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213679 8.e 8.ce4d180e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.530032+0000", 2023-09-23T18:13:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.073714452999999999, 2023-09-23T18:13:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.073953517999999996, 2023-09-23T18:13:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.620 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213679 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.530032+0000", 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.530032+0000", 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.530031+0000", 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.530036+0000", 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5770000000000004e-06 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.530044+0000", 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6149999999999997e-06 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.530048+0000", 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0130000000000004e-06 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.530092+0000", 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4024999999999999e-05 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.530099+0000", 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.37e-06 2023-09-23T18:13:20.624 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213455 8.e 8.177cdcae (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.282696+0000", 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.32105104099999998, 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.32135610799999997, 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.625 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213455 2023-09-23T18:13:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.282696+0000", 2023-09-23T18:13:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.282696+0000", 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.282695+0000", 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.282700+0000", 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1749999999999996e-06 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.282710+0000", 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5759999999999995e-06 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.282715+0000", 2023-09-23T18:13:20.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.767e-06 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.282769+0000", 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3591999999999998e-05 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.282779+0000", 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0475000000000001e-05 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.629 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213497 8.e 8.e5a0f49e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.315723+0000", 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.28802341999999997, 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.28839281900000002, 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.630 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213497 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.315723+0000", 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.315723+0000", 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.315722+0000", 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.315728+0000", 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0229999999999998e-06 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.315744+0000", 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6229e-05 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.315750+0000", 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4350000000000002e-06 2023-09-23T18:13:20.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.315827+0000", 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6649000000000004e-05 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.315835+0000", 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1450000000000008e-06 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.634 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213533 8.e 8.c0e19a2e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.348521+0000", 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.25522526000000001, 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.255658524, 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.635 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213533 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.348521+0000", 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.348521+0000", 2023-09-23T18:13:20.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.348520+0000", 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.348525+0000", 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0030000000000002e-06 2023-09-23T18:13:20.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.348535+0000", 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8789999999999998e-06 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.348541+0000", 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6740000000000002e-06 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.348613+0000", 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2167000000000002e-05 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.348621+0000", 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5590000000000006e-06 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213634 8.e 8.d27d801e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.465116+0000", 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.138630789, 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.13912993600000001, 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.640 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213634 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.465116+0000", 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.465116+0000", 2023-09-23T18:13:20.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.465114+0000", 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.465120+0000", 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3149999999999999e-06 2023-09-23T18:13:20.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.465129+0000", 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8769999999999999e-06 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.465133+0000", 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3680000000000004e-06 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.465183+0000", 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0000999999999998e-05 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.465190+0000", 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6669999999999999e-06 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213583 8.e 8.9a53207e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.397914+0000", 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.20583315399999999, 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.20639603000000001, 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213583 2023-09-23T18:13:20.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.397914+0000", 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.397914+0000", 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.397912+0000", 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.397919+0000", 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0809999999999996e-06 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.397935+0000", 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6299000000000001e-05 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.397943+0000", 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4839999999999998e-06 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.397996+0000", 2023-09-23T18:13:20.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3362000000000002e-05 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.398008+0000", 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2294e-05 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213652 8.e 8.2fe648de (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.649 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.486919+0000", 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.11682783300000001, 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.11745487, 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213652 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.486919+0000", 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.486919+0000", 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.486918+0000", 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.486926+0000", 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9670000000000001e-06 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.486943+0000", 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7093999999999999e-05 2023-09-23T18:13:20.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.486950+0000", 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5050000000000001e-06 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.487042+0000", 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1633999999999997e-05 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.653 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.487050+0000", 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7360000000000007e-06 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213417 8.e 8.8875552e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.256100+0000", 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.34764663600000001, 2023-09-23T18:13:20.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.34833629900000002, 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213417 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256100+0000", 2023-09-23T18:13:20.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256100+0000", 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256099+0000", 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256105+0000", 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7309999999999998e-06 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256115+0000", 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9690000000000006e-06 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256120+0000", 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4079999999999997e-06 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256168+0000", 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7728000000000001e-05 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256175+0000", 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7729999999999999e-06 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213466 8.e 8.273c1d3e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.291072+0000", 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.31267457500000001, 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.31343150600000003, 2023-09-23T18:13:20.659 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213466 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.291072+0000", 2023-09-23T18:13:20.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.291072+0000", 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.291071+0000", 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:20.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.291076+0000", 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8289999999999998e-06 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.291085+0000", 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1989999999999995e-06 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.662 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.291091+0000", 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6699999999999999e-06 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.291244+0000", 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015392800000000001 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.291252+0000", 2023-09-23T18:13:20.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5850000000000002e-06 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213425 8.e 8.dd434ede (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.262931+0000", 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.34081537699999997, 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.34164881000000002, 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.664 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213425 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262931+0000", 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262931+0000", 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262930+0000", 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262937+0000", 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8140000000000003e-06 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262953+0000", 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6552e-05 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262960+0000", 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5919999999999997e-06 2023-09-23T18:13:20.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.263035+0000", 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5650999999999998e-05 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.263048+0000", 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2648e-05 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.668 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213401 8.e 8.1aacb37e (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.36012863099999998, 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.361027983, 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "delayed", 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:20.669 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213401 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:20.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243620+0000", 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9649999999999998e-06 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243623+0000", 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6179999999999999e-06 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243626+0000", 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.294e-06 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:20.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243677+0000", 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1668000000000001e-05 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243686+0000", 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7730000000000005e-06 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 14 2023-09-23T18:13:20.673 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:13:20.674 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:20.675 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:13:20] "GET /metrics HTTP/1.1" 200 36621 "" "Prometheus/2.43.0" 2023-09-23T18:13:20.704 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:20.830 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:20.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:20.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:20.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[184174]: failed to encode map e443 with expected crc 2023-09-23T18:13:20.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[184174]: failed to encode map e443 with expected crc 2023-09-23T18:13:20.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[184174]: failed to encode map e443 with expected crc 2023-09-23T18:13:20.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[184174]: 8.d continuing backfill to osd.5 from (406'8927,440'13578] MIN to 440'13578 2023-09-23T18:13:20.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/70545165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:20.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[184174]: osdmap e444: 8 total, 8 up, 8 in 2023-09-23T18:13:20.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[184174]: 8.6 scrub starts 2023-09-23T18:13:20.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[179668]: failed to encode map e443 with expected crc 2023-09-23T18:13:20.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[179668]: failed to encode map e443 with expected crc 2023-09-23T18:13:20.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[179668]: failed to encode map e443 with expected crc 2023-09-23T18:13:20.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[179668]: 8.d continuing backfill to osd.5 from (406'8927,440'13578] MIN to 440'13578 2023-09-23T18:13:20.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/70545165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:20.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[179668]: osdmap e444: 8 total, 8 up, 8 in 2023-09-23T18:13:20.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:20 smithi055 ceph-mon[179668]: 8.6 scrub starts 2023-09-23T18:13:20.935 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:20.954 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:20 smithi099 ceph-mon[167076]: failed to encode map e443 with expected crc 2023-09-23T18:13:20.954 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:20 smithi099 ceph-mon[167076]: failed to encode map e443 with expected crc 2023-09-23T18:13:20.955 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:20 smithi099 ceph-mon[167076]: failed to encode map e443 with expected crc 2023-09-23T18:13:20.955 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:20 smithi099 ceph-mon[167076]: 8.d continuing backfill to osd.5 from (406'8927,440'13578] MIN to 440'13578 2023-09-23T18:13:20.955 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:20 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/70545165' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:20.955 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:20 smithi099 ceph-mon[167076]: osdmap e444: 8 total, 8 up, 8 in 2023-09-23T18:13:20.955 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:20 smithi099 ceph-mon[167076]: 8.6 scrub starts 2023-09-23T18:13:20.963 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:20.996 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:20.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:20.999 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:21.100 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:21.126 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:21.158 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:21.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:21.161 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:21.186 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:21.202 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:13:21.262 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:21.295 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:21.338 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:21.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:21.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:21.445 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:21.519 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:21.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:21.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:21.588 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:21.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:21.739 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:21.794 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:21.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:21.804 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:21.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:21 smithi099 ceph-mon[167076]: failed to encode map e444 with expected crc 2023-09-23T18:13:21.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:21 smithi099 ceph-mon[167076]: failed to encode map e444 with expected crc 2023-09-23T18:13:21.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:21 smithi099 ceph-mon[167076]: failed to encode map e444 with expected crc 2023-09-23T18:13:21.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:21 smithi099 ceph-mon[167076]: pgmap v1085: 129 pgs: 1 active+clean+remapped, 2 remapped+peering, 18 peering, 1 active+undersized+degraded+remapped+backfilling, 2 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 501 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 3645/23922 objects degraded (15.237%); 20 MiB/s, 414 objects/s recovering 2023-09-23T18:13:21.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:21 smithi099 ceph-mon[167076]: 1.e scrub starts 2023-09-23T18:13:21.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:21 smithi099 ceph-mon[167076]: failed to encode map e445 with expected crc 2023-09-23T18:13:21.837 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:21 smithi099 ceph-mon[167076]: osdmap e445: 8 total, 8 up, 8 in 2023-09-23T18:13:21.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[184174]: failed to encode map e444 with expected crc 2023-09-23T18:13:21.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[184174]: failed to encode map e444 with expected crc 2023-09-23T18:13:21.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[184174]: failed to encode map e444 with expected crc 2023-09-23T18:13:21.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[184174]: pgmap v1085: 129 pgs: 1 active+clean+remapped, 2 remapped+peering, 18 peering, 1 active+undersized+degraded+remapped+backfilling, 2 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 501 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 3645/23922 objects degraded (15.237%); 20 MiB/s, 414 objects/s recovering 2023-09-23T18:13:21.884 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[184174]: 1.e scrub starts 2023-09-23T18:13:21.884 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[184174]: failed to encode map e445 with expected crc 2023-09-23T18:13:21.884 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[184174]: osdmap e445: 8 total, 8 up, 8 in 2023-09-23T18:13:21.884 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[184174]: failed to encode map e445 with expected crc 2023-09-23T18:13:21.884 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[184174]: failed to encode map e445 with expected crc 2023-09-23T18:13:21.884 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:21.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[179668]: failed to encode map e444 with expected crc 2023-09-23T18:13:21.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[179668]: failed to encode map e444 with expected crc 2023-09-23T18:13:21.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[179668]: failed to encode map e444 with expected crc 2023-09-23T18:13:21.885 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[179668]: pgmap v1085: 129 pgs: 1 active+clean+remapped, 2 remapped+peering, 18 peering, 1 active+undersized+degraded+remapped+backfilling, 2 active+recovery_wait+degraded, 2 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 98 active+clean; 501 MiB data, 6.2 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 722 op/s; 3645/23922 objects degraded (15.237%); 20 MiB/s, 414 objects/s recovering 2023-09-23T18:13:21.885 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[179668]: 1.e scrub starts 2023-09-23T18:13:21.885 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[179668]: failed to encode map e445 with expected crc 2023-09-23T18:13:21.885 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[179668]: osdmap e445: 8 total, 8 up, 8 in 2023-09-23T18:13:21.885 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[179668]: failed to encode map e445 with expected crc 2023-09-23T18:13:21.885 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[179668]: failed to encode map e445 with expected crc 2023-09-23T18:13:21.885 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:21.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:21.943 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:21.989 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:21.993 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:21.994 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:22.028 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:22.095 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:22.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:21 smithi099 ceph-mon[167076]: failed to encode map e445 with expected crc 2023-09-23T18:13:22.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:21 smithi099 ceph-mon[167076]: failed to encode map e445 with expected crc 2023-09-23T18:13:22.122 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:22.124 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:22.165 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:22.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:22.168 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:22.269 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:22.317 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:22.368 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:22.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:22.373 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:22.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:22.540 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:22.624 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:22.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:22.631 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:22.650 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 3, 4, 2] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 6, 7, 3, 5] 2023-09-23T18:13:22.650 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:13:22.650 INFO:tasks.thrashosds.thrasher:Removing osd 3, in_osds are: [6, 7, 5, 1, 0, 3, 4, 2] 2023-09-23T18:13:22.650 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 3 2023-09-23T18:13:22.732 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:22.782 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:22.835 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:22.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:22.841 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:22.854 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:13:22.941 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:23.021 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:23.140 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:23.142 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:22 smithi099 ceph-mon[167076]: failed to encode map e445 with expected crc 2023-09-23T18:13:23.142 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:22 smithi099 ceph-mon[167076]: failed to encode map e445 with expected crc 2023-09-23T18:13:23.142 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:22 smithi099 ceph-mon[167076]: 8.2 scrub starts 2023-09-23T18:13:23.142 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:22 smithi099 ceph-mon[167076]: failed to encode map e446 with expected crc 2023-09-23T18:13:23.142 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:22 smithi099 ceph-mon[167076]: osdmap e446: 8 total, 8 up, 8 in 2023-09-23T18:13:23.142 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:22 smithi099 ceph-mon[167076]: failed to encode map e446 with expected crc 2023-09-23T18:13:23.142 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:22 smithi099 ceph-mon[167076]: failed to encode map e446 with expected crc 2023-09-23T18:13:23.142 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3999944999' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:23.142 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:22 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:23.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:23.143 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[184174]: failed to encode map e445 with expected crc 2023-09-23T18:13:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[184174]: failed to encode map e445 with expected crc 2023-09-23T18:13:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[184174]: 8.2 scrub starts 2023-09-23T18:13:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[184174]: failed to encode map e446 with expected crc 2023-09-23T18:13:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[184174]: osdmap e446: 8 total, 8 up, 8 in 2023-09-23T18:13:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[184174]: failed to encode map e446 with expected crc 2023-09-23T18:13:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[184174]: failed to encode map e446 with expected crc 2023-09-23T18:13:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3999944999' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:23.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[179668]: failed to encode map e445 with expected crc 2023-09-23T18:13:23.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[179668]: failed to encode map e445 with expected crc 2023-09-23T18:13:23.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[179668]: 8.2 scrub starts 2023-09-23T18:13:23.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[179668]: failed to encode map e446 with expected crc 2023-09-23T18:13:23.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[179668]: osdmap e446: 8 total, 8 up, 8 in 2023-09-23T18:13:23.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[179668]: failed to encode map e446 with expected crc 2023-09-23T18:13:23.184 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[179668]: failed to encode map e446 with expected crc 2023-09-23T18:13:23.185 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3999944999' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:23.185 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:22 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:23.215 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:13:23.245 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:23.295 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:23.549 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:13:23.549 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:13:23.549 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:13:23.549 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:13:23.549 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:13:23.590 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:23.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:23.595 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:23.658 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:23.696 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:23.750 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:23.863 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[184174]: failed to encode map e446 with expected crc 2023-09-23T18:13:23.863 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[184174]: pgmap v1088: 129 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 remapped+peering, 8 peering, 3 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 450 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 3692/21471 objects degraded (17.195%); 20 MiB/s, 400 objects/s recovering 2023-09-23T18:13:23.863 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 3645/23922 objects degraded (15.237%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:23.863 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:13:23.863 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:23.863 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[184174]: osdmap e447: 8 total, 8 up, 8 in 2023-09-23T18:13:23.863 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[184174]: failed to encode map e447 with expected crc 2023-09-23T18:13:23.863 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[184174]: failed to encode map e447 with expected crc 2023-09-23T18:13:23.864 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[184174]: failed to encode map e447 with expected crc 2023-09-23T18:13:23.896 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:23.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:23.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:23.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:24.003 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:13:24.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:24.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:23 smithi099 ceph-mon[167076]: failed to encode map e446 with expected crc 2023-09-23T18:13:24.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:23 smithi099 ceph-mon[167076]: pgmap v1088: 129 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 remapped+peering, 8 peering, 3 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 450 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 3692/21471 objects degraded (17.195%); 20 MiB/s, 400 objects/s recovering 2023-09-23T18:13:24.114 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 3645/23922 objects degraded (15.237%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:24.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:23 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:13:24.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:23 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:24.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:23 smithi099 ceph-mon[167076]: osdmap e447: 8 total, 8 up, 8 in 2023-09-23T18:13:24.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:23 smithi099 ceph-mon[167076]: failed to encode map e447 with expected crc 2023-09-23T18:13:24.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:23 smithi099 ceph-mon[167076]: failed to encode map e447 with expected crc 2023-09-23T18:13:24.115 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:23 smithi099 ceph-mon[167076]: failed to encode map e447 with expected crc 2023-09-23T18:13:24.117 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:24.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[179668]: failed to encode map e446 with expected crc 2023-09-23T18:13:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[179668]: pgmap v1088: 129 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 1 remapped+peering, 8 peering, 3 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 107 active+clean; 450 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 474 op/s; 3692/21471 objects degraded (17.195%); 20 MiB/s, 400 objects/s recovering 2023-09-23T18:13:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 3645/23922 objects degraded (15.237%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:13:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[179668]: osdmap e447: 8 total, 8 up, 8 in 2023-09-23T18:13:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[179668]: failed to encode map e447 with expected crc 2023-09-23T18:13:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[179668]: failed to encode map e447 with expected crc 2023-09-23T18:13:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:23 smithi055 ceph-mon[179668]: failed to encode map e447 with expected crc 2023-09-23T18:13:24.189 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:24.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:24.196 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:24.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:24.354 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:24.412 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:24.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:24.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:24.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:24.546 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:24.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:24.588 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:24.588 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:24.689 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:24.719 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:24.750 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:24.852 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:24.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:24.855 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:24.956 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:24.986 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:25.135 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:25.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:24 smithi055 ceph-mon[179668]: failed to encode map e447 with expected crc 2023-09-23T18:13:25.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:24 smithi055 ceph-mon[179668]: failed to encode map e447 with expected crc 2023-09-23T18:13:25.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:24 smithi055 ceph-mon[179668]: osdmap e448: 8 total, 8 up, 8 in 2023-09-23T18:13:25.138 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:24 smithi055 ceph-mon[184174]: failed to encode map e447 with expected crc 2023-09-23T18:13:25.138 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:24 smithi055 ceph-mon[184174]: failed to encode map e447 with expected crc 2023-09-23T18:13:25.138 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:24 smithi055 ceph-mon[184174]: osdmap e448: 8 total, 8 up, 8 in 2023-09-23T18:13:25.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:25.139 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:25.165 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:24 smithi099 ceph-mon[167076]: failed to encode map e447 with expected crc 2023-09-23T18:13:25.165 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:24 smithi099 ceph-mon[167076]: failed to encode map e447 with expected crc 2023-09-23T18:13:25.165 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:24 smithi099 ceph-mon[167076]: osdmap e448: 8 total, 8 up, 8 in 2023-09-23T18:13:25.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:25.272 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:25.495 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:25.506 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:13:25.510 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:25.511 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:25.532 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:13:25.561 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:13:25.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:13:25.611 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:25.622 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:13:25.651 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:25.652 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:13:25.671 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:13:25.692 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:13:25.720 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:25.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:25.725 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:25.730 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:13:25.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:13:25.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:13:25.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:13:25.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:13:25.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:13:25.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:13:25.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:13:25.735 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:13:25.826 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:25.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:13:25.582+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e449 definitely_dead 0 2023-09-23T18:13:25.872 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:25.909 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:25.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:25.913 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:26.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:26.081 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:26.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[184174]: failed to encode map e448 with expected crc 2023-09-23T18:13:26.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[184174]: pgmap v1091: 129 pgs: 1 active+remapped+backfilling, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 5 peering, 3 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 429 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 289 op/s; 3692/20448 objects degraded (18.056%); 57/20448 objects misplaced (0.279%); 7.7 MiB/s, 2 keys/s, 124 objects/s recovering 2023-09-23T18:13:26.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[184174]: osdmap e449: 8 total, 8 up, 8 in 2023-09-23T18:13:26.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[184174]: failed to encode map e449 with expected crc 2023-09-23T18:13:26.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[184174]: failed to encode map e449 with expected crc 2023-09-23T18:13:26.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3210817965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:26.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2575723496' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:13:26.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:13:26.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[184174]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T18:13:26.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[179668]: failed to encode map e448 with expected crc 2023-09-23T18:13:26.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[179668]: pgmap v1091: 129 pgs: 1 active+remapped+backfilling, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 5 peering, 3 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 429 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 289 op/s; 3692/20448 objects degraded (18.056%); 57/20448 objects misplaced (0.279%); 7.7 MiB/s, 2 keys/s, 124 objects/s recovering 2023-09-23T18:13:26.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[179668]: osdmap e449: 8 total, 8 up, 8 in 2023-09-23T18:13:26.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[179668]: failed to encode map e449 with expected crc 2023-09-23T18:13:26.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[179668]: failed to encode map e449 with expected crc 2023-09-23T18:13:26.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3210817965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:26.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2575723496' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:13:26.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:13:26.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:25 smithi055 ceph-mon[179668]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T18:13:26.247 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.3. 2023-09-23T18:13:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:25 smithi099 ceph-mon[167076]: failed to encode map e448 with expected crc 2023-09-23T18:13:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:25 smithi099 ceph-mon[167076]: pgmap v1091: 129 pgs: 1 active+remapped+backfilling, 1 active+undersized+remapped+backfill_wait, 1 active+clean+remapped, 5 peering, 3 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 429 MiB data, 6.1 GiB used, 709 GiB / 715 GiB avail; 0 B/s wr, 289 op/s; 3692/20448 objects degraded (18.056%); 57/20448 objects misplaced (0.279%); 7.7 MiB/s, 2 keys/s, 124 objects/s recovering 2023-09-23T18:13:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:25 smithi099 ceph-mon[167076]: osdmap e449: 8 total, 8 up, 8 in 2023-09-23T18:13:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:25 smithi099 ceph-mon[167076]: failed to encode map e449 with expected crc 2023-09-23T18:13:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:25 smithi099 ceph-mon[167076]: failed to encode map e449 with expected crc 2023-09-23T18:13:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:25 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3210817965' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:25 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2575723496' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:13:26.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:25 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["3"]}]: dispatch 2023-09-23T18:13:26.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:25 smithi099 ceph-mon[167076]: Client client.admin marked osd.3 out, while it was still marked up 2023-09-23T18:13:26.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:26.586 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:26.587 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:26.688 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:26.713 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:26.747 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:26.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:26.750 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:26.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:26.877 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:27.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[179668]: failed to encode map e449 with expected crc 2023-09-23T18:13:27.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[179668]: 8.12 deep-scrub starts 2023-09-23T18:13:27.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[179668]: 8.5 scrub starts 2023-09-23T18:13:27.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T18:13:27.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[179668]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T18:13:27.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[179668]: failed to encode map e450 with expected crc 2023-09-23T18:13:27.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[184174]: failed to encode map e449 with expected crc 2023-09-23T18:13:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[184174]: 8.12 deep-scrub starts 2023-09-23T18:13:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[184174]: 8.5 scrub starts 2023-09-23T18:13:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T18:13:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[184174]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T18:13:27.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:26 smithi055 ceph-mon[184174]: failed to encode map e450 with expected crc 2023-09-23T18:13:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:26 smithi099 ceph-mon[167076]: failed to encode map e449 with expected crc 2023-09-23T18:13:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:26 smithi099 ceph-mon[167076]: 8.12 deep-scrub starts 2023-09-23T18:13:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:26 smithi099 ceph-mon[167076]: 8.5 scrub starts 2023-09-23T18:13:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:26 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["3"]}]': finished 2023-09-23T18:13:27.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:26 smithi099 ceph-mon[167076]: osdmap e450: 8 total, 8 up, 7 in 2023-09-23T18:13:27.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:26 smithi099 ceph-mon[167076]: failed to encode map e450 with expected crc 2023-09-23T18:13:27.516 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:27.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:27.518 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:27.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:27.651 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:27.687 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:27.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:27.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:27.791 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:27.828 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:28.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:28.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:28.016 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:27 smithi099 ceph-mon[167076]: failed to encode map e450 with expected crc 2023-09-23T18:13:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:27 smithi099 ceph-mon[167076]: failed to encode map e450 with expected crc 2023-09-23T18:13:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:27 smithi099 ceph-mon[167076]: pgmap v1094: 129 pgs: 1 active+undersized+remapped+backfill_wait, 3 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 307 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3581/14598 objects degraded (24.531%); 8.7 MiB/s, 2 keys/s, 323 objects/s recovering 2023-09-23T18:13:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:27 smithi099 ceph-mon[167076]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T18:13:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:27 smithi099 ceph-mon[167076]: failed to encode map e451 with expected crc 2023-09-23T18:13:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:27 smithi099 ceph-mon[167076]: 8.0 starting backfill to osd.4 from (369'3530,392'7621] MIN to 449'13227 2023-09-23T18:13:28.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:27 smithi099 ceph-mon[167076]: 8.10 starting backfill to osd.4 from (369'3530,392'7621] MIN to 449'13196 2023-09-23T18:13:28.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:27 smithi099 ceph-mon[167076]: 8.0 continuing backfill to osd.5 from (402'8420,449'13227] 8:07754a4b:::benchmark_data_smithi055_346788_object18666:head to 449'13227 2023-09-23T18:13:28.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:27 smithi099 ceph-mon[167076]: 8.10 continuing backfill to osd.5 from (402'8420,449'13196] MIN to 449'13196 2023-09-23T18:13:28.093 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:13:28.093 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:13:28.117 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:28.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[179668]: failed to encode map e450 with expected crc 2023-09-23T18:13:28.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[179668]: failed to encode map e450 with expected crc 2023-09-23T18:13:28.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[179668]: pgmap v1094: 129 pgs: 1 active+undersized+remapped+backfill_wait, 3 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 307 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3581/14598 objects degraded (24.531%); 8.7 MiB/s, 2 keys/s, 323 objects/s recovering 2023-09-23T18:13:28.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[179668]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T18:13:28.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[179668]: failed to encode map e451 with expected crc 2023-09-23T18:13:28.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[179668]: 8.0 starting backfill to osd.4 from (369'3530,392'7621] MIN to 449'13227 2023-09-23T18:13:28.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[179668]: 8.10 starting backfill to osd.4 from (369'3530,392'7621] MIN to 449'13196 2023-09-23T18:13:28.146 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[179668]: 8.0 continuing backfill to osd.5 from (402'8420,449'13227] 8:07754a4b:::benchmark_data_smithi055_346788_object18666:head to 449'13227 2023-09-23T18:13:28.147 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[179668]: 8.10 continuing backfill to osd.5 from (402'8420,449'13196] MIN to 449'13196 2023-09-23T18:13:28.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[184174]: failed to encode map e450 with expected crc 2023-09-23T18:13:28.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[184174]: failed to encode map e450 with expected crc 2023-09-23T18:13:28.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[184174]: pgmap v1094: 129 pgs: 1 active+undersized+remapped+backfill_wait, 3 active+recovery_wait+degraded, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering, 1 active+recovery_wait, 3 active+recovery_wait+undersized+degraded+remapped, 117 active+clean; 307 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 572 op/s; 3581/14598 objects degraded (24.531%); 8.7 MiB/s, 2 keys/s, 323 objects/s recovering 2023-09-23T18:13:28.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[184174]: osdmap e451: 8 total, 8 up, 7 in 2023-09-23T18:13:28.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[184174]: failed to encode map e451 with expected crc 2023-09-23T18:13:28.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[184174]: 8.0 starting backfill to osd.4 from (369'3530,392'7621] MIN to 449'13227 2023-09-23T18:13:28.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[184174]: 8.10 starting backfill to osd.4 from (369'3530,392'7621] MIN to 449'13196 2023-09-23T18:13:28.148 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[184174]: 8.0 continuing backfill to osd.5 from (402'8420,449'13227] 8:07754a4b:::benchmark_data_smithi055_346788_object18666:head to 449'13227 2023-09-23T18:13:28.148 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:27 smithi055 ceph-mon[184174]: 8.10 continuing backfill to osd.5 from (402'8420,449'13196] MIN to 449'13196 2023-09-23T18:13:28.150 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: failed to encode map e451 with expected crc 2023-09-23T18:13:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: failed to encode map e451 with expected crc 2023-09-23T18:13:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 3581/14598 objects degraded (24.531%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T18:13:29.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: 8.17 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13119 2023-09-23T18:13:29.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: 8.f starting backfill to osd.6 from (0'0,0'0] MAX to 449'13473 2023-09-23T18:13:29.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: 8.9 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13265 2023-09-23T18:13:29.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: 8.7 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13099 2023-09-23T18:13:29.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[179668]: 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13291 2023-09-23T18:13:29.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: failed to encode map e451 with expected crc 2023-09-23T18:13:29.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: failed to encode map e451 with expected crc 2023-09-23T18:13:29.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 3581/14598 objects degraded (24.531%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:29.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T18:13:29.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: 8.17 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13119 2023-09-23T18:13:29.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: 8.f starting backfill to osd.6 from (0'0,0'0] MAX to 449'13473 2023-09-23T18:13:29.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.184 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: 8.9 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13265 2023-09-23T18:13:29.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: 8.7 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13099 2023-09-23T18:13:29.185 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:28 smithi055 ceph-mon[184174]: 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13291 2023-09-23T18:13:29.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: failed to encode map e451 with expected crc 2023-09-23T18:13:29.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: failed to encode map e451 with expected crc 2023-09-23T18:13:29.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 3581/14598 objects degraded (24.531%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:29.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: osdmap e452: 8 total, 8 up, 7 in 2023-09-23T18:13:29.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: 8.17 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13119 2023-09-23T18:13:29.333 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.334 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: 8.f starting backfill to osd.6 from (0'0,0'0] MAX to 449'13473 2023-09-23T18:13:29.334 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:29.334 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: 8.9 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13265 2023-09-23T18:13:29.334 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: 8.7 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13099 2023-09-23T18:13:29.334 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:28 smithi099 ceph-mon[167076]: 8.19 starting backfill to osd.4 from (0'0,0'0] MAX to 449'13291 2023-09-23T18:13:30.116 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:30.119 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:30.119 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:30.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:13:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[179668]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[179668]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 449'13520 2023-09-23T18:13:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[179668]: pgmap v1097: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 peering, 7 remapped+peering, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 226 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 122/10710 objects degraded (1.139%); 97/10710 objects misplaced (0.906%); 11 MiB/s, 357 objects/s recovering 2023-09-23T18:13:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[179668]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T18:13:30.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[184174]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[184174]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 449'13520 2023-09-23T18:13:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[184174]: pgmap v1097: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 peering, 7 remapped+peering, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 226 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 122/10710 objects degraded (1.139%); 97/10710 objects misplaced (0.906%); 11 MiB/s, 357 objects/s recovering 2023-09-23T18:13:30.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:29 smithi055 ceph-mon[184174]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T18:13:30.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:29 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:29 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:29 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:29 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:29 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:29 smithi099 ceph-mon[167076]: failed to encode map e452 with expected crc 2023-09-23T18:13:30.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:29 smithi099 ceph-mon[167076]: 8.c starting backfill to osd.4 from (0'0,0'0] MAX to 449'13520 2023-09-23T18:13:30.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:29 smithi099 ceph-mon[167076]: pgmap v1097: 129 pgs: 1 active+undersized+degraded+remapped+backfilling, 2 peering, 7 remapped+peering, 1 active+recovery_wait+degraded, 2 active+undersized+remapped, 1 active+recovery_wait, 2 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 226 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 0 B/s wr, 811 op/s; 122/10710 objects degraded (1.139%); 97/10710 objects misplaced (0.906%); 11 MiB/s, 357 objects/s recovering 2023-09-23T18:13:30.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:29 smithi099 ceph-mon[167076]: osdmap e453: 8 total, 8 up, 7 in 2023-09-23T18:13:30.192 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:30.220 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:30.248 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:30.328 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:30.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:30.335 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:30.435 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:30.469 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:13:30] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.43.0" 2023-09-23T18:13:30.471 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:30.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-mon[184174]: failed to encode map e453 with expected crc 2023-09-23T18:13:30.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-mon[184174]: failed to encode map e453 with expected crc 2023-09-23T18:13:30.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-mon[184174]: failed to encode map e454 with expected crc 2023-09-23T18:13:30.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-mon[184174]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T18:13:30.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-mon[184174]: failed to encode map e454 with expected crc 2023-09-23T18:13:30.973 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:30.976 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:30.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:30.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:31.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:31.162 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:31.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-mon[179668]: failed to encode map e453 with expected crc 2023-09-23T18:13:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-mon[179668]: failed to encode map e453 with expected crc 2023-09-23T18:13:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-mon[179668]: failed to encode map e454 with expected crc 2023-09-23T18:13:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-mon[179668]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T18:13:31.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:30 smithi055 ceph-mon[179668]: failed to encode map e454 with expected crc 2023-09-23T18:13:31.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:30 smithi099 ceph-mon[167076]: failed to encode map e453 with expected crc 2023-09-23T18:13:31.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:30 smithi099 ceph-mon[167076]: failed to encode map e453 with expected crc 2023-09-23T18:13:31.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:30 smithi099 ceph-mon[167076]: failed to encode map e454 with expected crc 2023-09-23T18:13:31.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:30 smithi099 ceph-mon[167076]: osdmap e454: 8 total, 8 up, 7 in 2023-09-23T18:13:31.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:30 smithi099 ceph-mon[167076]: failed to encode map e454 with expected crc 2023-09-23T18:13:31.946 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:13:31.972 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:31.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:31.978 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:32.078 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:32.138 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:31 smithi055 ceph-mon[179668]: failed to encode map e454 with expected crc 2023-09-23T18:13:32.139 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:31 smithi055 ceph-mon[179668]: failed to encode map e454 with expected crc 2023-09-23T18:13:32.139 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:31 smithi055 ceph-mon[179668]: pgmap v1100: 129 pgs: 2 peering, 7 remapped+peering, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 117 active+clean; 162 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 100/7626 objects misplaced (1.311%); 9.5 MiB/s, 159 objects/s recovering 2023-09-23T18:13:32.139 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:31 smithi055 ceph-mon[184174]: failed to encode map e454 with expected crc 2023-09-23T18:13:32.139 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:31 smithi055 ceph-mon[184174]: failed to encode map e454 with expected crc 2023-09-23T18:13:32.139 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:31 smithi055 ceph-mon[184174]: pgmap v1100: 129 pgs: 2 peering, 7 remapped+peering, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 117 active+clean; 162 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 100/7626 objects misplaced (1.311%); 9.5 MiB/s, 159 objects/s recovering 2023-09-23T18:13:32.143 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:32.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:31 smithi099 ceph-mon[167076]: failed to encode map e454 with expected crc 2023-09-23T18:13:32.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:31 smithi099 ceph-mon[167076]: failed to encode map e454 with expected crc 2023-09-23T18:13:32.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:31 smithi099 ceph-mon[167076]: pgmap v1100: 129 pgs: 2 peering, 7 remapped+peering, 2 active+remapped+backfill_wait, 1 active+undersized+remapped, 117 active+clean; 162 MiB data, 4.2 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 580 op/s; 100/7626 objects misplaced (1.311%); 9.5 MiB/s, 159 objects/s recovering 2023-09-23T18:13:32.420 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Removed 110589 objects 2023-09-23T18:13:32.420 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Clean up completed and total clean up time :151.346 2023-09-23T18:13:32.537 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:32.541 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:32.541 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:32.642 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:32.672 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:32.730 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:32.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:32.733 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:32.834 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:32.863 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:32.899 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:32.902 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:32.902 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:32.975 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:33.003 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:33.037 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:33.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:32 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 122/10710 objects degraded (1.139%), 4 pgs degraded) 2023-09-23T18:13:33.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:32 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:13:33.108 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:33.110 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:32 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 122/10710 objects degraded (1.139%), 4 pgs degraded) 2023-09-23T18:13:33.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:32 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:13:33.111 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:32 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 122/10710 objects degraded (1.139%), 4 pgs degraded) 2023-09-23T18:13:33.111 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:32 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:13:33.111 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:33.112 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:33.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:33.242 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:33.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:33.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:33.555 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:33.656 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:33.688 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:33.727 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:33.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:33.731 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:33.832 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:33.862 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:33.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:34.002 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:34.003 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:34.084 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:13:34.084 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:13:34.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:34.133 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:34.218 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:13:34.223 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:34.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:34.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:34.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:34 smithi055 ceph-mon[184174]: pgmap v1101: 129 pgs: 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 120 active+clean; 88 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 1701/4113 objects degraded (41.357%); 2809/4113 objects misplaced (68.296%); 7.6 MiB/s, 203 objects/s recovering 2023-09-23T18:13:34.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/94927297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:13:34.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:34 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:13:34.249 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:34 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 1701/4113 objects degraded (41.357%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:34.250 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:34 smithi055 ceph-mon[179668]: pgmap v1101: 129 pgs: 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 120 active+clean; 88 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 1701/4113 objects degraded (41.357%); 2809/4113 objects misplaced (68.296%); 7.6 MiB/s, 203 objects/s recovering 2023-09-23T18:13:34.250 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/94927297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:13:34.250 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:34 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:13:34.250 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:34 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 1701/4113 objects degraded (41.357%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:34.250 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:13:34.279 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:13:34.313 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:13:34.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:34.345 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:13:34.355 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:34.368 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:13:34.388 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:13:34.412 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:13:34.428 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:34 smithi099 ceph-mon[167076]: pgmap v1101: 129 pgs: 2 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 5 active+remapped+backfill_wait, 120 active+clean; 88 MiB data, 4.1 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 618 op/s; 1701/4113 objects degraded (41.357%); 2809/4113 objects misplaced (68.296%); 7.6 MiB/s, 203 objects/s recovering 2023-09-23T18:13:34.428 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/94927297' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:13:34.429 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:34 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:13:34.429 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:34 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 1701/4113 objects degraded (41.357%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:34.433 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:13:34.435 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:13:34.435 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:13:34.435 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:13:34.435 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:13:34.435 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:13:34.435 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:13:34.435 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:13:34.435 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:13:34.440 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:34.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:34.445 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:34.475 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:13:34.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:34.584 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:34.622 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:34.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:34.626 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:34.727 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:34.762 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:34.800 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:34.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:34.803 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:34.904 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:34.935 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:34.969 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:34.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:34.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:34.980 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_3 2023-09-23T18:13:34.981 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_3 unique_pool_3 --yes-i-really-really-mean-it 2023-09-23T18:13:35.075 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:35.103 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:35.137 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:35.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:35.141 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:35.166 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:35.195 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 4, 2] out_osds: [3] dead_osds: [] live_osds: [0, 1, 2, 4, 6, 7, 3, 5] 2023-09-23T18:13:35.195 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:13:35.195 INFO:tasks.thrashosds.thrasher:Adding osd 3 2023-09-23T18:13:35.195 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 3 2023-09-23T18:13:35.242 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:35.276 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:35.312 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:35.314 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:35.315 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:35.415 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:35.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:35.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[184174]: failed to encode map e455 with expected crc 2023-09-23T18:13:35.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:13:35.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[184174]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T18:13:35.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4269513748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:35.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[184174]: failed to encode map e455 with expected crc 2023-09-23T18:13:35.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[184174]: failed to encode map e455 with expected crc 2023-09-23T18:13:35.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:35.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[179668]: failed to encode map e455 with expected crc 2023-09-23T18:13:35.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:13:35.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[179668]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T18:13:35.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4269513748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:35.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[179668]: failed to encode map e455 with expected crc 2023-09-23T18:13:35.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:35 smithi055 ceph-mon[179668]: failed to encode map e455 with expected crc 2023-09-23T18:13:35.443 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:35.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:35.482 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:35 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:35.482 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:35 smithi099 ceph-mon[167076]: failed to encode map e455 with expected crc 2023-09-23T18:13:35.482 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:35 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:13:35.482 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:35 smithi099 ceph-mon[167076]: osdmap e455: 8 total, 8 up, 7 in 2023-09-23T18:13:35.483 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:35 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4269513748' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:35.483 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:35 smithi099 ceph-mon[167076]: failed to encode map e455 with expected crc 2023-09-23T18:13:35.483 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:35 smithi099 ceph-mon[167076]: failed to encode map e455 with expected crc 2023-09-23T18:13:35.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:35.484 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:35.584 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:35.618 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:35.648 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:35.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:35.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:35.896 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:35.910 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:35.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:36.026 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:36.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:36.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:36.081 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:36.182 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:36.216 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:36.256 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:36.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:36.261 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:36.361 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:36.368 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:13:36.368 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:13:36.369 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:13:36.369 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:13:36.369 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:13:36.392 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:36.428 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:36.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:36 smithi055 ceph-mon[179668]: failed to encode map e455 with expected crc 2023-09-23T18:13:36.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:36 smithi055 ceph-mon[179668]: pgmap v1103: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 121 active+clean; 76 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 1701/3513 objects degraded (48.420%); 2809/3513 objects misplaced (79.960%); 1.8 MiB/s, 105 objects/s recovering 2023-09-23T18:13:36.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:36 smithi055 ceph-mon[184174]: failed to encode map e455 with expected crc 2023-09-23T18:13:36.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:36 smithi055 ceph-mon[184174]: pgmap v1103: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 121 active+clean; 76 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 1701/3513 objects degraded (48.420%); 2809/3513 objects misplaced (79.960%); 1.8 MiB/s, 105 objects/s recovering 2023-09-23T18:13:36.430 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:36 smithi099 ceph-mon[167076]: failed to encode map e455 with expected crc 2023-09-23T18:13:36.431 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:36 smithi099 ceph-mon[167076]: pgmap v1103: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 121 active+clean; 76 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 410 op/s; 1701/3513 objects degraded (48.420%); 2809/3513 objects misplaced (79.960%); 1.8 MiB/s, 105 objects/s recovering 2023-09-23T18:13:36.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:36.431 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:36.532 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:36.559 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:36.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:36.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:36.758 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:36.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:13:36.859 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:36.876 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:13:36.892 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:36.939 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:13:36.981 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:36.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:36.986 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:37.086 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:37.139 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:37.279 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:37.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:37.283 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:37.383 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:37.384 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:37.385 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:37.418 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:37.419 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:37.558 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:37.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:37.562 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:37.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:37.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:37.692 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:37.723 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:37.728 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:37.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:37.731 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:37.833 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:37.851 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:37.863 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:37.909 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:37.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:37.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:38.013 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:38.045 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:38.135 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:38.137 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:37 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:13:37.948+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e455 definitely_dead 0 2023-09-23T18:13:38.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:38.142 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:38.245 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:38.258 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:38 smithi099 ceph-mon[167076]: pgmap v1104: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 121 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 1701/519 objects degraded (327.746%); 2719/519 objects misplaced (523.892%); 1.4 MiB/s, 81 objects/s recovering 2023-09-23T18:13:38.258 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/496444315' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:13:38.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:13:38.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4281711255' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:13:38.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:13:38.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:13:38.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T18:13:38.259 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:38 smithi099 ceph-mon[167076]: osdmap e456: 8 total, 8 up, 8 in 2023-09-23T18:13:38.265 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_3' removed 2023-09-23T18:13:38.276 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.3. 2023-09-23T18:13:38.376 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[184174]: pgmap v1104: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 121 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 1701/519 objects degraded (327.746%); 2719/519 objects misplaced (523.892%); 1.4 MiB/s, 81 objects/s recovering 2023-09-23T18:13:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/496444315' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:13:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:13:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4281711255' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:13:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:13:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:13:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T18:13:38.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[184174]: osdmap e456: 8 total, 8 up, 8 in 2023-09-23T18:13:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[179668]: pgmap v1104: 129 pgs: 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 6 active+remapped+backfill_wait, 121 active+clean; 14 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail; 0 B/s wr, 455 op/s; 1701/519 objects degraded (327.746%); 2719/519 objects misplaced (523.892%); 1.4 MiB/s, 81 objects/s recovering 2023-09-23T18:13:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/496444315' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:13:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:13:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4281711255' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:13:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["3"]}]: dispatch 2023-09-23T18:13:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_3", "pool2": "unique_pool_3", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:13:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["3"]}]': finished 2023-09-23T18:13:38.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:38 smithi055 ceph-mon[179668]: osdmap e456: 8 total, 8 up, 8 in 2023-09-23T18:13:38.522 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:38.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:38.525 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:38.626 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:38.654 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:38.689 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:38.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:38.692 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:38.793 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:38.806 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:13:38.806 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:13:38.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:13:38.806 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:13:38.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.806 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136101 8.2 8:4a1fd025:::benchmark_data_smithi055_346788_object25509:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509243+0000", 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.294351758, 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0685515059999999, 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136101 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509243+0000", 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509243+0000", 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509242+0000", 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509248+0000", 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4110000000000001e-06 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509259+0000", 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0757000000000001e-05 2023-09-23T18:13:38.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509265+0000", 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.874e-06 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509361+0000", 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5877000000000004e-05 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509370+0000", 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4720000000000001e-06 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.547253+0000", 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.037882701 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.548809+0000", 2023-09-23T18:13:38.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015562060000000001 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.548982+0000", 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017251899999999999 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.561547+0000", 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012565092999999999 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.576752+0000", 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015205023999999999 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.576807+0000", 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5451999999999997e-05 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.576848+0000", 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1063999999999999e-05 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.577795+0000", 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00094642999999999995 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.814 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136113 8.6 8:698e62bb:::benchmark_data_smithi055_346788_object25521:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509277+0000", 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.29431766899999, 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.138360233, 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136113 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.815 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509277+0000", 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509277+0000", 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509277+0000", 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509280+0000", 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8490000000000002e-06 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509287+0000", 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1720000000000001e-06 2023-09-23T18:13:38.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509291+0000", 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8519999999999997e-06 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509362+0000", 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0857000000000003e-05 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509370+0000", 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.755e-06 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.553199+0000", 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0438288469999999 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.553903+0000", 2023-09-23T18:13:38.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00070390000000000003 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.580852+0000", 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.026948666 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.580930+0000", 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8088000000000002e-05 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.581491+0000", 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00056076800000000001 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.621100+0000", 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.039608947999999998 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.625239+0000", 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041396810000000001 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.646068+0000", 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020828435999999999 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.646108+0000", 2023-09-23T18:13:38.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9777999999999998e-05 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.647638+0000", 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015301290000000001 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136115 8.2 8:4e13f92c:::benchmark_data_smithi055_346788_object25523:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.823 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509302+0000", 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.29429356, 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0748453979999999, 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136115 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509302+0000", 2023-09-23T18:13:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509302+0000", 2023-09-23T18:13:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509301+0000", 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509304+0000", 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5000000000000002e-06 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509310+0000", 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6409999999999996e-06 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509314+0000", 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.065e-06 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509393+0000", 2023-09-23T18:13:38.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8400999999999997e-05 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509399+0000", 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8590000000000001e-06 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.555775+0000", 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.046376787 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.555913+0000", 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000137124 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.556072+0000", 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015936100000000001 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.561561+0000", 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054889159999999999 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.576781+0000", 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015219939 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.577859+0000", 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010779780000000001 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.577914+0000", 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4926999999999998e-05 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584147+0000", 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00623326 2023-09-23T18:13:38.831 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136138 8.2 8:4ee3c91a:::benchmark_data_smithi055_346788_object25546:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509319+0000", 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.2942761, 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.076255135, 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.832 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136138 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509319+0000", 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509319+0000", 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509319+0000", 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509321+0000", 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2460000000000002e-06 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509327+0000", 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0290000000000002e-06 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509331+0000", 2023-09-23T18:13:38.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7850000000000002e-06 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509441+0000", 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011047100000000001 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509447+0000", 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9259999999999997e-06 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.557856+0000", 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0484084789999999 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.557995+0000", 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013896899999999999 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.558139+0000", 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014439800000000001 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.563720+0000", 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055811599999999999 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584195+0000", 2023-09-23T18:13:38.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020474598 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584414+0000", 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021974000000000001 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584502+0000", 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7262999999999994e-05 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.585574+0000", 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010724269999999999 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136150 8.6 8:62b2afb4:::benchmark_data_smithi055_346788_object25558:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509414+0000", 2023-09-23T18:13:38.840 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.294181165, 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1723984599999999, 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136150 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509414+0000", 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509414+0000", 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509413+0000", 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509428+0000", 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4487999999999999e-05 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509437+0000", 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0219999999999993e-06 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509443+0000", 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8320000000000002e-06 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511001+0000", 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001558116 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511008+0000", 2023-09-23T18:13:38.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7409999999999992e-06 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.617716+0000", 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1067071230000001 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.620358+0000", 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026421690000000002 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.650403+0000", 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.030045479 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.650475+0000", 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1815000000000003e-05 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:13:38.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.650585+0000", 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000109768 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.673499+0000", 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022914443 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680194+0000", 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066950300000000003 2023-09-23T18:13:38.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680306+0000", 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112019 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680358+0000", 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1526000000000001e-05 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.681812+0000", 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001454601 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136153 8.2 8:4ed9a1c7:::benchmark_data_smithi055_346788_object25561:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509454+0000", 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.29414111299999, 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.076902005, 2023-09-23T18:13:38.849 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136153 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509454+0000", 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509454+0000", 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509454+0000", 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509456+0000", 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6120000000000001e-06 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509462+0000", 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3029999999999996e-06 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509467+0000", 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4329999999999997e-06 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510885+0000", 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014182279999999999 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510893+0000", 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4749999999999996e-06 2023-09-23T18:13:38.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.559782+0000", 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0488890849999999 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.559879+0000", 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7108000000000006e-05 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.560003+0000", 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012399500000000001 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.566016+0000", 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006012982 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584256+0000", 2023-09-23T18:13:38.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018239986999999999 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.585620+0000", 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001363815 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.585666+0000", 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6289999999999999e-05 2023-09-23T18:13:38.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586356+0000", 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00069014300000000005 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136169 8.6 8:6fa67d1e:::benchmark_data_smithi055_346788_object25577:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509476+0000", 2023-09-23T18:13:38.857 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.294119529, 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1734557620000001, 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136169 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509476+0000", 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509476+0000", 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509475+0000", 2023-09-23T18:13:38.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509478+0000", 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4719999999999998e-06 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509484+0000", 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4590000000000003e-06 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509488+0000", 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1189999999999999e-06 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511033+0000", 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001544595 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511040+0000", 2023-09-23T18:13:38.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6660000000000002e-06 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.621116+0000", 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1100768670000001 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.622276+0000", 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001159579 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.655074+0000", 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032797699 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.655139+0000", 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5399000000000002e-05 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:13:38.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.655225+0000", 2023-09-23T18:13:38.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5869000000000001e-05 2023-09-23T18:13:38.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680207+0000", 2023-09-23T18:13:38.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.024982244000000001 2023-09-23T18:13:38.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680275+0000", 2023-09-23T18:13:38.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7720999999999998e-05 2023-09-23T18:13:38.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.681878+0000", 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016034949999999999 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.681930+0000", 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.151e-05 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.682931+0000", 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001001486 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136190 8.2 8:4369bb6d:::benchmark_data_smithi055_346788_object25598:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509493+0000", 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.294102575, 2023-09-23T18:13:38.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.077617552, 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136190 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509493+0000", 2023-09-23T18:13:38.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509493+0000", 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509492+0000", 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509494+0000", 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1490000000000001e-06 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509504+0000", 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0132000000000001e-05 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509509+0000", 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1239999999999998e-06 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510918+0000", 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014091360000000001 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510924+0000", 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.767e-06 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.561593+0000", 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0506690839999999 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.561703+0000", 2023-09-23T18:13:38.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110209 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.561829+0000", 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000126112 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.569780+0000", 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079513919999999998 2023-09-23T18:13:38.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584300+0000", 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014519776 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586489+0000", 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021893910000000002 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586532+0000", 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2703000000000001e-05 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.587110+0000", 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00057794699999999999 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.875 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136192 8.2 8:4a109c9f:::benchmark_data_smithi055_346788_object25600:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509514+0000", 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.294081331, 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0789546750000001, 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.876 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136192 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509514+0000", 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509514+0000", 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509513+0000", 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509516+0000", 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2869999999999999e-06 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509522+0000", 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2530000000000001e-06 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509526+0000", 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9360000000000001e-06 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510954+0000", 2023-09-23T18:13:38.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001427928 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510961+0000", 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8460000000000002e-06 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.563751+0000", 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0527905870000001 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.563908+0000", 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015658499999999999 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.564053+0000", 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014559999999999999 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.573474+0000", 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0094202969999999994 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584337+0000", 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010863462000000001 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.587213+0000", 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028762839999999998 2023-09-23T18:13:38.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.587261+0000", 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7006999999999999e-05 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.588468+0000", 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012079829999999999 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.883 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136204 8.2 8:4e83b308:::benchmark_data_smithi055_346788_object25612:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509534+0000", 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.29406070100001, 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.079762712, 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.884 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136204 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509534+0000", 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509534+0000", 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509534+0000", 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509536+0000", 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.131e-06 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509543+0000", 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8290000000000002e-06 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509547+0000", 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2899999999999996e-06 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510987+0000", 2023-09-23T18:13:38.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00143958 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510992+0000", 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0849999999999996e-06 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.566056+0000", 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0550638720000001 2023-09-23T18:13:38.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.566147+0000", 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1142000000000003e-05 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.566279+0000", 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000131536 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.573479+0000", 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0072009359999999998 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584378+0000", 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010898410000000001 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.588561+0000", 2023-09-23T18:13:38.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041834999999999997 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.588615+0000", 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3668999999999997e-05 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.589297+0000", 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00068208699999999995 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.891 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136213 8.2 8:4a22a6f0:::benchmark_data_smithi055_346788_object25621:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509553+0000", 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.29404226, 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0912685440000001, 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.892 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136213 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509553+0000", 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509553+0000", 2023-09-23T18:13:38.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509553+0000", 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509561+0000", 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4090000000000003e-06 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509567+0000", 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.455e-06 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509571+0000", 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8539999999999999e-06 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511006+0000", 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001435232 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511009+0000", 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.942e-06 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.568065+0000", 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0570560849999999 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.568177+0000", 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111777 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.568259+0000", 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1468e-05 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.575309+0000", 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007049792 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586397+0000", 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011088694 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.599400+0000", 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013002575000000001 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.599456+0000", 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6444999999999999e-05 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.600821+0000", 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013651760000000001 2023-09-23T18:13:38.899 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136219 8.2 8:4dc26cdb:::benchmark_data_smithi055_346788_object25627:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509576+0000", 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.294018962, 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0921537569999999, 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.900 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136219 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509576+0000", 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509576+0000", 2023-09-23T18:13:38.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509576+0000", 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509578+0000", 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2709999999999999e-06 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509584+0000", 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3219999999999998e-06 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509588+0000", 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5590000000000001e-06 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511027+0000", 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001439081 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511032+0000", 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2669999999999997e-06 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.569826+0000", 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.058793485 2023-09-23T18:13:38.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.569971+0000", 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014493399999999999 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.570110+0000", 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013977299999999999 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.575317+0000", 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052065820000000004 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586429+0000", 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011111792000000001 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.600885+0000", 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014456115 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.600938+0000", 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2673999999999997e-05 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.601730+0000", 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00079229799999999998 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.907 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136223 8.2 8:4007e85f:::benchmark_data_smithi055_346788_object25631:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509592+0000", 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.294003139, 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.092864678, 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.908 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136223 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509592+0000", 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509592+0000", 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509592+0000", 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509594+0000", 2023-09-23T18:13:38.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2239999999999998e-06 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509600+0000", 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0599999999999996e-06 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509606+0000", 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.614e-06 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511054+0000", 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001448335 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511059+0000", 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2320000000000001e-06 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.571978+0000", 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0609187710000001 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.572101+0000", 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012351800000000001 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.572190+0000", 2023-09-23T18:13:38.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8537999999999998e-05 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.575368+0000", 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031780010000000002 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586458+0000", 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011089887999999999 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.601786+0000", 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015328664000000001 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.601836+0000", 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9604999999999997e-05 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.602457+0000", 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00062059400000000003 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136229 8.6 8:663dcb03:::benchmark_data_smithi055_346788_object25637:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509610+0000", 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.293985042, 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.174026851, 2023-09-23T18:13:38.916 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136229 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509610+0000", 2023-09-23T18:13:38.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509610+0000", 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509610+0000", 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509612+0000", 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0449999999999999e-06 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509618+0000", 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5010000000000001e-06 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509623+0000", 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0300000000000001e-06 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511063+0000", 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014401259999999999 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511070+0000", 2023-09-23T18:13:38.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5390000000000004e-06 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.622337+0000", 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1112675249999999 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.623186+0000", 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00084855100000000001 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.659609+0000", 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.036422818000000003 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.659698+0000", 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9207000000000002e-05 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.659828+0000", 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012993100000000001 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680217+0000", 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020389131000000001 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.681851+0000", 2023-09-23T18:13:38.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001633497 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.683010+0000", 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001158912 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.683040+0000", 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0759000000000001e-05 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.683637+0000", 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00059668899999999997 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136253 8.6 8:6977274d:::benchmark_data_smithi055_346788_object25661:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509628+0000", 2023-09-23T18:13:38.925 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.293966803, 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1746242060000001, 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136253 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509628+0000", 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509628+0000", 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509628+0000", 2023-09-23T18:13:38.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509630+0000", 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1789999999999998e-06 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509636+0000", 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9769999999999997e-06 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509640+0000", 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1099999999999996e-06 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511101+0000", 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014610129999999999 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511108+0000", 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8689999999999998e-06 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.623221+0000", 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1121126079999999 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.623931+0000", 2023-09-23T18:13:38.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00071072500000000005 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.664256+0000", 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.040324454000000003 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.664348+0000", 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2176999999999995e-05 2023-09-23T18:13:38.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.664491+0000", 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000142483 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680225+0000", 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015734387999999998 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.682971+0000", 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027464070000000002 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.683697+0000", 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00072510999999999995 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.683723+0000", 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6183999999999999e-05 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.684253+0000", 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00052985599999999999 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136258 8.2 8:4d91b10c:::benchmark_data_smithi055_346788_object25666:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509644+0000", 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 118.29395115, 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.093560117, 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136258 2023-09-23T18:13:38.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509644+0000", 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509644+0000", 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509644+0000", 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509646+0000", 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9889999999999999e-06 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509651+0000", 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8250000000000001e-06 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509655+0000", 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5489999999999998e-06 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511128+0000", 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014725890000000001 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511133+0000", 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4870000000000002e-06 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.573520+0000", 2023-09-23T18:13:38.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.062386692 2023-09-23T18:13:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.573661+0000", 2023-09-23T18:13:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014144099999999999 2023-09-23T18:13:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:13:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.573810+0000", 2023-09-23T18:13:38.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014895699999999999 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.577812+0000", 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040020100000000003 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.587171+0000", 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0093583950000000003 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.602503+0000", 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015332070999999999 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.602563+0000", 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0265e-05 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.603204+0000", 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00064115899999999998 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177019 8.9 8:935233eb:::benchmark_data_smithi055_346788_object66427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.253339+0000", 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 67.550256273000002, 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1561469240000002, 2023-09-23T18:13:38.943 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177019 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253339+0000", 2023-09-23T18:13:38.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253339+0000", 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253338+0000", 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253344+0000", 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2029999999999997e-06 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253359+0000", 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4806e-05 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253366+0000", 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9249999999999998e-06 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253432+0000", 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6327000000000004e-05 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253574+0000", 2023-09-23T18:13:38.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014209300000000001 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3", 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253702+0000", 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012784100000000001 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.255719+0000", 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002016795 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.288188+0000", 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032469117999999998 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.288201+0000", 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2873e-05 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.298979+0000", 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0107786649999999 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.298990+0000", 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0479000000000001e-05 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.409403+0000", 2023-09-23T18:13:38.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.110413163 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.409486+0000", 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2878000000000002e-05 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.951 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177046 8.9 8:97ec04da:::benchmark_data_smithi055_346788_object66454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.334212+0000", 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 67.469383024999999, 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0928899140000001, 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177046 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334212+0000", 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334212+0000", 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334211+0000", 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334221+0000", 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0329999999999999e-05 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334236+0000", 2023-09-23T18:13:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5441e-05 2023-09-23T18:13:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334243+0000", 2023-09-23T18:13:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5590000000000002e-06 2023-09-23T18:13:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334297+0000", 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3662000000000002e-05 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334307+0000", 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0207000000000001e-05 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.299021+0000", 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.964714356 2023-09-23T18:13:38.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.299028+0000", 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9670000000000004e-06 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.409514+0000", 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.110486262 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.409646+0000", 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013124400000000001 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.409782+0000", 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013615099999999999 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.419151+0000", 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0093694659999999999 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.422507+0000", 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033560059999999999 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.426918+0000", 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004411206 2023-09-23T18:13:38.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.426976+0000", 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7358000000000002e-05 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.427102+0000", 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012621999999999999 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.960 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177048 8.9 8:993077e3:::benchmark_data_smithi055_346788_object66456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.345194+0000", 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 67.458401433000006, 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0826640959999998, 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.961 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177048 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345194+0000", 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345194+0000", 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345192+0000", 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345200+0000", 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8699999999999992e-06 2023-09-23T18:13:38.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345214+0000", 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4086e-05 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345221+0000", 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2429999999999999e-06 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345290+0000", 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8515000000000004e-05 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345303+0000", 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3321e-05 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.299049+0000", 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95374580099999995 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.299055+0000", 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3249999999999996e-06 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.411995+0000", 2023-09-23T18:13:38.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1129396570000001 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.412116+0000", 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012100299999999999 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.412239+0000", 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012302099999999999 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.419161+0000", 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069218630000000003 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.422531+0000", 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003370587 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.427711+0000", 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051796910000000002 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.427769+0000", 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8127000000000001e-05 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.427858+0000", 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8553000000000004e-05 2023-09-23T18:13:38.969 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177051 8.9 8:9801ac86:::benchmark_data_smithi055_346788_object66459:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:32.401316+0000", 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 66.402279151000002, 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0341772149999999, 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:13:38.970 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177051 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.401316+0000", 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.401316+0000", 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.401315+0000", 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.401322+0000", 2023-09-23T18:13:38.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7070000000000003e-06 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.401335+0000", 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2948999999999999e-05 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.401341+0000", 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7499999999999997e-06 2023-09-23T18:13:38.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.401404+0000", 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3077999999999995e-05 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.401418+0000", 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3203999999999999e-05 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.414181+0000", 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0127638059999999 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.414309+0000", 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012722300000000001 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:13:38.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.414441+0000", 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013229800000000001 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.419168+0000", 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004727495 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.424002+0000", 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048334379999999998 2023-09-23T18:13:38.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.435322+0000", 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011320227 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.435376+0000", 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4083999999999999e-05 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:13:38.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:13:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.435493+0000", 2023-09-23T18:13:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011710299999999999 2023-09-23T18:13:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:13:38.978 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:13:38.978 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:13:38.979 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:39.014 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T18:13:39.015 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T18:13:39.015 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_4 2023-09-23T18:13:39.015 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_4 16 2023-09-23T18:13:39.122 INFO:tasks.thrashosds.thrasher:Added osd 3 2023-09-23T18:13:39.132 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:39.135 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:39.136 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:39.216 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:13:39.236 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:39.286 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:39.321 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:39.324 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:39.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:39.381 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:39 smithi055 ceph-mon[179668]: failed to encode map e456 with expected crc 2023-09-23T18:13:39.381 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:39 smithi055 ceph-mon[179668]: failed to encode map e456 with expected crc 2023-09-23T18:13:39.382 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:39 smithi055 ceph-mon[179668]: osdmap e457: 8 total, 8 up, 8 in 2023-09-23T18:13:39.382 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:39 smithi055 ceph-mon[184174]: failed to encode map e456 with expected crc 2023-09-23T18:13:39.382 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:39 smithi055 ceph-mon[184174]: failed to encode map e456 with expected crc 2023-09-23T18:13:39.382 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:39 smithi055 ceph-mon[184174]: osdmap e457: 8 total, 8 up, 8 in 2023-09-23T18:13:39.425 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:39.457 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:39.494 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:39.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:39.500 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:39.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:39 smithi099 ceph-mon[167076]: failed to encode map e456 with expected crc 2023-09-23T18:13:39.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:39 smithi099 ceph-mon[167076]: failed to encode map e456 with expected crc 2023-09-23T18:13:39.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:39 smithi099 ceph-mon[167076]: osdmap e457: 8 total, 8 up, 8 in 2023-09-23T18:13:39.601 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:39.630 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:39.833 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:39.848 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:13:39.865 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:39.868 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:13:39.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:39.870 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:39.889 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:13:39.907 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:13:39.940 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:13:39.959 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:13:39.971 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:39.980 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:13:40.009 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:40.009 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:13:40.009 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:13:40.009 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:13:40.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:13:40.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:13:40.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:13:40.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:13:40.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:13:40.010 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:13:40.048 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:40.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:40.051 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:40.117 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:40.152 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:40.181 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:40.286 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:13:40.310 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[179668]: pgmap v1106: 103 pgs: 103 active+clean; 4.4 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:13:40.310 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[179668]: failed to encode map e457 with expected crc 2023-09-23T18:13:40.310 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[179668]: failed to encode map e457 with expected crc 2023-09-23T18:13:40.310 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[179668]: failed to encode map e457 with expected crc 2023-09-23T18:13:40.310 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1701/519 objects degraded (327.746%), 2 pgs degraded) 2023-09-23T18:13:40.311 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3799244012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:40.311 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1271420461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:40.311 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:40.311 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[184174]: pgmap v1106: 103 pgs: 103 active+clean; 4.4 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:13:40.311 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[184174]: failed to encode map e457 with expected crc 2023-09-23T18:13:40.311 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[184174]: failed to encode map e457 with expected crc 2023-09-23T18:13:40.311 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[184174]: failed to encode map e457 with expected crc 2023-09-23T18:13:40.311 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1701/519 objects degraded (327.746%), 2 pgs degraded) 2023-09-23T18:13:40.312 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3799244012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:40.312 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1271420461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:40.312 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:40.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:40 smithi099 ceph-mon[167076]: pgmap v1106: 103 pgs: 103 active+clean; 4.4 MiB data, 3.7 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:13:40.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:40 smithi099 ceph-mon[167076]: failed to encode map e457 with expected crc 2023-09-23T18:13:40.317 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:40 smithi099 ceph-mon[167076]: failed to encode map e457 with expected crc 2023-09-23T18:13:40.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:40 smithi099 ceph-mon[167076]: failed to encode map e457 with expected crc 2023-09-23T18:13:40.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:40 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 1701/519 objects degraded (327.746%), 2 pgs degraded) 2023-09-23T18:13:40.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3799244012' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:40.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1271420461' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:40.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:40 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:40.336 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:40.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:40.340 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:40.440 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:40.468 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:40.552 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:40.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:40.555 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:40.566 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:13:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:13:40] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-09-23T18:13:40.656 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:40.686 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:40.735 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:40.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:40.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:40.839 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:40.867 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:40.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:40.956 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:40.956 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:41.039 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:13:41.040 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:13:41.057 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:41.085 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:41.121 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:41.124 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:41.125 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:41.226 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:41.257 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:41.300 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:41.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:41.304 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:41.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:41 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:41.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:41 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:41.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:41 smithi099 ceph-mon[167076]: osdmap e458: 8 total, 8 up, 8 in 2023-09-23T18:13:41.375 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:41 smithi099 ceph-mon[167076]: failed to encode map e458 with expected crc 2023-09-23T18:13:41.376 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:13:41.405 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:41.434 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:41.473 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:41.476 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:41.476 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:41.522 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:41 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:41.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:41 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:41.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:41 smithi055 ceph-mon[184174]: osdmap e458: 8 total, 8 up, 8 in 2023-09-23T18:13:41.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:41 smithi055 ceph-mon[184174]: failed to encode map e458 with expected crc 2023-09-23T18:13:41.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:41 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:41.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:41 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:41.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:41 smithi055 ceph-mon[179668]: osdmap e458: 8 total, 8 up, 8 in 2023-09-23T18:13:41.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:41 smithi055 ceph-mon[179668]: failed to encode map e458 with expected crc 2023-09-23T18:13:41.577 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:41.603 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:41.637 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:41.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:41.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:41.741 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:41.769 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:41.803 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:41.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:41.807 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:41.908 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:41.941 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:41.965 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:41.975 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:41.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:41.979 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:42.080 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:42.107 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:42.192 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:42.195 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:42.196 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:42.296 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:42.324 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:42.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:42 smithi099 ceph-mon[167076]: failed to encode map e458 with expected crc 2023-09-23T18:13:42.365 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:42 smithi099 ceph-mon[167076]: pgmap v1109: 103 pgs: 103 active+clean; 4.4 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:13:42.367 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:13:42.371 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_4' created 2023-09-23T18:13:42.409 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:42.413 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:42.413 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:42.514 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:42.542 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:42.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:42 smithi055 ceph-mon[179668]: failed to encode map e458 with expected crc 2023-09-23T18:13:42.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:42 smithi055 ceph-mon[179668]: pgmap v1109: 103 pgs: 103 active+clean; 4.4 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:13:42.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:42 smithi055 ceph-mon[184174]: failed to encode map e458 with expected crc 2023-09-23T18:13:42.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:42 smithi055 ceph-mon[184174]: pgmap v1109: 103 pgs: 103 active+clean; 4.4 MiB data, 3.6 GiB used, 622 GiB / 626 GiB avail 2023-09-23T18:13:42.898 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:13:43.145 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:43.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:43.148 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:43.196 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:13:43.196 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:13:43.238 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_4 rados --yes-i-really-mean-it || true 2023-09-23T18:13:43.248 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:43.309 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:43.343 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:43.345 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:43.346 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:43.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:43.475 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/397988015' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T18:13:43.475 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:43 smithi099 ceph-mon[167076]: failed to encode map e459 with expected crc 2023-09-23T18:13:43.475 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/397988015' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T18:13:43.475 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:43 smithi099 ceph-mon[167076]: osdmap e459: 8 total, 8 up, 8 in 2023-09-23T18:13:43.478 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:43.492 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:43.518 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:43.612 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:43.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:43.616 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/397988015' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T18:13:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:43 smithi055 ceph-mon[184174]: failed to encode map e459 with expected crc 2023-09-23T18:13:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/397988015' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T18:13:43.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:43 smithi055 ceph-mon[184174]: osdmap e459: 8 total, 8 up, 8 in 2023-09-23T18:13:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/397988015' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]: dispatch 2023-09-23T18:13:43.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:43 smithi055 ceph-mon[179668]: failed to encode map e459 with expected crc 2023-09-23T18:13:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/397988015' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_4", "pg_num": 16}]': finished 2023-09-23T18:13:43.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:43 smithi055 ceph-mon[179668]: osdmap e459: 8 total, 8 up, 8 in 2023-09-23T18:13:43.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:13:43.717 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:43.777 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:43.818 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:43.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:43.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:43.923 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:43.950 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:43.985 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:43.987 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:43.988 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:44.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:13:44.118 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:44.123 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 4, 2, 3] out_osds: [] dead_osds: [] live_osds: [0, 1, 2, 4, 6, 7, 3, 5] 2023-09-23T18:13:44.123 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:13:44.123 INFO:tasks.thrashosds.thrasher:Killing osd 2, live_osds are [0, 1, 2, 4, 6, 7, 3, 5] 2023-09-23T18:13:44.123 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T18:13:44.123 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T18:13:44.259 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:44.313 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:44.316 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:13:44 smithi055 systemd[1]: Stopping Ceph osd.2 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:13:44.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[184174]: pgmap v1111: 119 pgs: 3 creating+peering, 13 unknown, 103 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T18:13:44.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[184174]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[184174]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[184174]: osdmap e460: 8 total, 8 up, 8 in 2023-09-23T18:13:44.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[184174]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[184174]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[179668]: pgmap v1111: 119 pgs: 3 creating+peering, 13 unknown, 103 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T18:13:44.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[179668]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[179668]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[179668]: osdmap e460: 8 total, 8 up, 8 in 2023-09-23T18:13:44.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[179668]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:44 smithi055 ceph-mon[179668]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.711 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:13:44.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:44.715 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:13:44.728 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:44 smithi099 ceph-mon[167076]: pgmap v1111: 119 pgs: 3 creating+peering, 13 unknown, 103 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T18:13:44.728 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:44 smithi099 ceph-mon[167076]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.729 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:44 smithi099 ceph-mon[167076]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.729 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:44 smithi099 ceph-mon[167076]: osdmap e460: 8 total, 8 up, 8 in 2023-09-23T18:13:44.729 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:44 smithi099 ceph-mon[167076]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.729 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:44 smithi099 ceph-mon[167076]: failed to encode map e460 with expected crc 2023-09-23T18:13:44.815 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:44.844 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:45.024 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:45.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:45.028 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:45.129 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:45.158 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:45.225 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:45.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:45.229 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:45.329 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:45.330 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[208602]: 2023-09-23T18:13:45.301+0000 7fc9d29cd700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:13:45.330 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[208602]: 2023-09-23T18:13:45.301+0000 7fc9d29cd700 -1 osd.2 461 *** Got signal Terminated *** 2023-09-23T18:13:45.330 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[208602]: 2023-09-23T18:13:45.301+0000 7fc9d29cd700 -1 osd.2 461 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:13:45.358 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:45.393 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:45.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:45.397 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:45.498 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:45.524 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:45.526 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:13:45.526 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:13:45.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:45.561 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:45 smithi099 ceph-mon[167076]: failed to encode map e461 with expected crc 2023-09-23T18:13:45.561 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:45 smithi099 ceph-mon[167076]: failed to encode map e461 with expected crc 2023-09-23T18:13:45.561 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:45 smithi099 ceph-mon[167076]: osdmap e461: 8 total, 8 up, 8 in 2023-09-23T18:13:45.561 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:45 smithi099 ceph-mon[167076]: failed to encode map e461 with expected crc 2023-09-23T18:13:45.561 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:45 smithi099 ceph-mon[167076]: osd.2 marked itself down and dead 2023-09-23T18:13:45.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:45.562 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:45.593 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:13:45.612 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:13:45.632 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:13:45.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-mon[179668]: failed to encode map e461 with expected crc 2023-09-23T18:13:45.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-mon[179668]: failed to encode map e461 with expected crc 2023-09-23T18:13:45.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-mon[179668]: osdmap e461: 8 total, 8 up, 8 in 2023-09-23T18:13:45.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-mon[179668]: failed to encode map e461 with expected crc 2023-09-23T18:13:45.639 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-mon[179668]: osd.2 marked itself down and dead 2023-09-23T18:13:45.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-mon[184174]: failed to encode map e461 with expected crc 2023-09-23T18:13:45.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-mon[184174]: failed to encode map e461 with expected crc 2023-09-23T18:13:45.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-mon[184174]: osdmap e461: 8 total, 8 up, 8 in 2023-09-23T18:13:45.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-mon[184174]: failed to encode map e461 with expected crc 2023-09-23T18:13:45.639 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:45 smithi055 ceph-mon[184174]: osd.2 marked itself down and dead 2023-09-23T18:13:45.654 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:13:45.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:45.674 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:13:45.690 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:45.694 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:13:45.714 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:13:45.714 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:13:45.714 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:13:45.714 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T18:13:45.714 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:13:45.714 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:13:45.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:13:45.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:13:45.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:13:45.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:13:45.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:45.890 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:45.891 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:45.992 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:46.026 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:46.327 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:13:46.328 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:13:46.408 INFO:teuthology.orchestra.run.smithi055.stderr:enabled application 'rados' on pool 'unique_pool_4' 2023-09-23T18:13:46.408 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:13:46.418 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:46.421 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:46.421 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:46.477 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:46.522 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:46.554 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:46.679 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:13:46 smithi055 bash[385817]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T18:13:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[179668]: pgmap v1114: 119 pgs: 7 creating+peering, 9 unknown, 103 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 19 B/s, 0 objects/s recovering 2023-09-23T18:13:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[179668]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:13:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[179668]: failed to encode map e462 with expected crc 2023-09-23T18:13:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[179668]: failed to encode map e462 with expected crc 2023-09-23T18:13:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[179668]: osdmap e462: 8 total, 7 up, 8 in 2023-09-23T18:13:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1336751624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1977971229' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:46.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:46.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1387669404' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:13:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[184174]: pgmap v1114: 119 pgs: 7 creating+peering, 9 unknown, 103 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 19 B/s, 0 objects/s recovering 2023-09-23T18:13:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[184174]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:13:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[184174]: failed to encode map e462 with expected crc 2023-09-23T18:13:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[184174]: failed to encode map e462 with expected crc 2023-09-23T18:13:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[184174]: osdmap e462: 8 total, 7 up, 8 in 2023-09-23T18:13:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1336751624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1977971229' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:46.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:46.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1387669404' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:13:46.698 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:46 smithi099 ceph-mon[167076]: pgmap v1114: 119 pgs: 7 creating+peering, 9 unknown, 103 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 19 B/s, 0 objects/s recovering 2023-09-23T18:13:46.698 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:46 smithi099 ceph-mon[167076]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:13:46.698 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:46 smithi099 ceph-mon[167076]: failed to encode map e462 with expected crc 2023-09-23T18:13:46.698 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:46 smithi099 ceph-mon[167076]: failed to encode map e462 with expected crc 2023-09-23T18:13:46.698 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:46 smithi099 ceph-mon[167076]: osdmap e462: 8 total, 7 up, 8 in 2023-09-23T18:13:46.698 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1336751624' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:46.698 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1977971229' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:46.698 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:46 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:13:46.698 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1387669404' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:13:46.794 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:46.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:46.797 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:46.898 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:46.928 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:46.964 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:46.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:46.967 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:47.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:47.096 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:47.240 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:47.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:47.243 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:47.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:47.372 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:47.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:47.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:47.411 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:47.512 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:47.541 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:47.627 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:47.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:47.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[184174]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[184174]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:13:47.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1387669404' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:13:47.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[184174]: osdmap e463: 8 total, 7 up, 8 in 2023-09-23T18:13:47.628 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[184174]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[184174]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[184174]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.629 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:47.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:47.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[179668]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[179668]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:13:47.629 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1387669404' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:13:47.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[179668]: osdmap e463: 8 total, 7 up, 8 in 2023-09-23T18:13:47.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[179668]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[179668]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[179668]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:47.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:47.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:47.675 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:13:47.675 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:13:47.676 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:13:47.676 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:13:47.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:47 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:13:47.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:47 smithi099 ceph-mon[167076]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:47 smithi099 ceph-mon[167076]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:47 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:13:47.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1387669404' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_4", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:13:47.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:47 smithi099 ceph-mon[167076]: osdmap e463: 8 total, 7 up, 8 in 2023-09-23T18:13:47.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:47 smithi099 ceph-mon[167076]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:47 smithi099 ceph-mon[167076]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:47 smithi099 ceph-mon[167076]: failed to encode map e463 with expected crc 2023-09-23T18:13:47.701 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:47.732 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:47.761 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:48.020 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:48.023 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:48.023 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:48.030 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:13:48.125 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:48.160 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:48.199 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:48.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:48.207 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:48.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:48.329 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:13:48.335 DEBUG:teuthology.orchestra.run.smithi055:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_4 bench 90 write' 2023-09-23T18:13:48.338 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:48.347 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T18:13:48.347 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T18:13:48.370 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:48.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:48.373 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:48 smithi055 ceph-mon[179668]: pgmap v1117: 119 pgs: 14 peering, 7 creating+peering, 98 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 22 B/s, 0 objects/s recovering 2023-09-23T18:13:48.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:48 smithi055 ceph-mon[184174]: pgmap v1117: 119 pgs: 14 peering, 7 creating+peering, 98 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 22 B/s, 0 objects/s recovering 2023-09-23T18:13:48.474 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:48.504 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:48.539 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:48.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:48.544 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:48.593 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:hints = 1 2023-09-23T18:13:48.594 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T18:13:48.594 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object prefix: benchmark_data_smithi055_386467 2023-09-23T18:13:48.594 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:13:48.594 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 0 0 0 0 0 0 - 0 2023-09-23T18:13:48.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:48.671 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:48.696 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:48 smithi099 ceph-mon[167076]: pgmap v1117: 119 pgs: 14 peering, 7 creating+peering, 98 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 22 B/s, 0 objects/s recovering 2023-09-23T18:13:48.704 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:48.824 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:48.829 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:48.830 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:48.835 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:13:48.930 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:48.977 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:49.037 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:49.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:49.041 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:49.142 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:49.214 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:49.510 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:49.514 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:49 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:49.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:49.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:49.594 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 1 16 3442 3426 214.114 214.125 0.00447388 0.0046523 2023-09-23T18:13:49.616 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:49.658 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:49 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:49.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:49 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 3/84 objects degraded (3.571%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:50.200 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:13:50.200 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:13:50.200 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:13:50.200 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:13:50.200 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:13:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:50 smithi099 ceph-mon[167076]: pgmap v1118: 119 pgs: 2 active+undersized+degraded, 11 active+undersized, 14 peering, 92 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T18:13:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:50 smithi099 ceph-mon[167076]: 1.17 scrub starts 2023-09-23T18:13:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:50 smithi099 ceph-mon[167076]: 1.17 scrub ok 2023-09-23T18:13:50.594 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 2 16 6733 6717 209.893 205.688 0.00316577 0.00475388 2023-09-23T18:13:50.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:13:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:13:50] "GET /metrics HTTP/1.1" 200 36630 "" "Prometheus/2.43.0" 2023-09-23T18:13:50.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:50 smithi055 ceph-mon[184174]: pgmap v1118: 119 pgs: 2 active+undersized+degraded, 11 active+undersized, 14 peering, 92 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T18:13:50.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:50 smithi055 ceph-mon[184174]: 1.17 scrub starts 2023-09-23T18:13:50.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:50 smithi055 ceph-mon[184174]: 1.17 scrub ok 2023-09-23T18:13:50.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:50 smithi055 ceph-mon[179668]: pgmap v1118: 119 pgs: 2 active+undersized+degraded, 11 active+undersized, 14 peering, 92 active+clean; 4.4 MiB data, 4.1 GiB used, 711 GiB / 715 GiB avail; 3/84 objects degraded (3.571%) 2023-09-23T18:13:50.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:50 smithi055 ceph-mon[179668]: 1.17 scrub starts 2023-09-23T18:13:50.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:50 smithi055 ceph-mon[179668]: 1.17 scrub ok 2023-09-23T18:13:51.331 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:51.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:51.335 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:51.435 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:51.450 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T18:13:51.493 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:51.529 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:13:51.529 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T18:13:51.553 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:51.594 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 3 16 9945 9929 206.791 200.75 0.0040661 0.00482517 2023-09-23T18:13:51.615 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:52.595 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 4 16 12548 12532 195.763 162.688 0.00189541 0.00509812 2023-09-23T18:13:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:52 smithi099 ceph-mon[167076]: pgmap v1119: 119 pgs: 4 active+undersized+degraded, 23 active+undersized, 92 active+clean; 235 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 35 MiB/s wr, 558 op/s; 8/11169 objects degraded (0.072%) 2023-09-23T18:13:52.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:52.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:52.842 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:52.848 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:52.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:52 smithi055 ceph-mon[179668]: pgmap v1119: 119 pgs: 4 active+undersized+degraded, 23 active+undersized, 92 active+clean; 235 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 35 MiB/s wr, 558 op/s; 8/11169 objects degraded (0.072%) 2023-09-23T18:13:52.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:52.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:52.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:52 smithi055 ceph-mon[184174]: pgmap v1119: 119 pgs: 4 active+undersized+degraded, 23 active+undersized, 92 active+clean; 235 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail; 35 MiB/s wr, 558 op/s; 8/11169 objects degraded (0.072%) 2023-09-23T18:13:52.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:52.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:13:52.949 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:52.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:13:53.000 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:53.069 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:53.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:53.077 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:53.177 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:53.225 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:53.332 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:53.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:53.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:53.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:53.495 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:53.541 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:53.543 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:53.544 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:53.595 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 5 16 14995 14979 187.195 152.938 0.00163443 0.00531205 2023-09-23T18:13:53.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:53.717 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:53.776 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:53.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:53.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:53.831 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:13:53.886 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:53.929 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:54.178 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:54.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:54.184 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:54.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:13:54.341 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:54.442 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:54 smithi055 ceph-mon[179668]: pgmap v1120: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 397 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 1756/18939 objects degraded (9.272%) 2023-09-23T18:13:54.442 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:54 smithi055 ceph-mon[184174]: pgmap v1120: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 397 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 1756/18939 objects degraded (9.272%) 2023-09-23T18:13:54.595 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 6 16 17350 17334 180.525 147.188 0.00453481 0.00553299 2023-09-23T18:13:54.680 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:13:54.714 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:13:54.752 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:13:54.788 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:13:54.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:54 smithi099 ceph-mon[167076]: pgmap v1120: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 397 MiB data, 5.3 GiB used, 710 GiB / 715 GiB avail; 49 MiB/s wr, 784 op/s; 1756/18939 objects degraded (9.272%) 2023-09-23T18:13:54.829 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:13:54.860 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:13:54.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:13:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:54.872 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:13:54.906 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:13:54.907 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:13:54.907 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:13:54.907 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T18:13:54.907 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:13:54.907 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:13:54.907 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:13:54.907 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:13:54.907 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:13:54.973 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:55.004 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:55.062 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:55.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:55.068 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:55.169 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:13:55.220 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:55.276 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:13:55.282 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:55.283 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:13:55.371 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:13:55.372 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:13:55.384 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:13:55.438 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:55.469 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:13:55.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:55 smithi055 ceph-mon[179668]: 1.10 scrub starts 2023-09-23T18:13:55.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:55 smithi055 ceph-mon[179668]: 1.10 scrub ok 2023-09-23T18:13:55.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/806299940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:55.492 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2803852310' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:55.492 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:55 smithi055 ceph-mon[184174]: 1.10 scrub starts 2023-09-23T18:13:55.493 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:55 smithi055 ceph-mon[184174]: 1.10 scrub ok 2023-09-23T18:13:55.493 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/806299940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:55.493 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2803852310' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:55.595 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 7 16 19672 19656 175.456 145.125 0.00495342 0.00569324 2023-09-23T18:13:55.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:13:55.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:55.614 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:13:55.715 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:55.726 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:55 smithi099 ceph-mon[167076]: 1.10 scrub starts 2023-09-23T18:13:55.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:55 smithi099 ceph-mon[167076]: 1.10 scrub ok 2023-09-23T18:13:55.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/806299940' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:13:55.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2803852310' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:13:55.758 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:55.810 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:55.815 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:55.816 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:55.916 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:55.955 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:56.006 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:13:56.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:56.010 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:13:56.111 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:56.161 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:56.530 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 4, 2, 3] out_osds: [] dead_osds: [2] live_osds: [0, 1, 4, 6, 7, 3, 5] 2023-09-23T18:13:56.530 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:13:56.530 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-09-23T18:13:56.531 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:13:56.597 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 8 16 22069 22053 172.249 149.812 0.00467802 0.00580089 2023-09-23T18:13:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:56 smithi099 ceph-mon[167076]: pgmap v1121: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 620 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 1756/29613 objects degraded (5.930%) 2023-09-23T18:13:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:56 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 1756/29613 objects degraded (5.930%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:56 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:13:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:56 smithi099 ceph-mon[167076]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2803852310' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:56 smithi099 ceph-mon[167076]: osdmap e464: 8 total, 7 up, 8 in 2023-09-23T18:13:56.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:56 smithi099 ceph-mon[167076]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:56 smithi099 ceph-mon[167076]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:56 smithi099 ceph-mon[167076]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[179668]: pgmap v1121: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 620 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 1756/29613 objects degraded (5.930%) 2023-09-23T18:13:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 1756/29613 objects degraded (5.930%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:13:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[179668]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2803852310' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[179668]: osdmap e464: 8 total, 7 up, 8 in 2023-09-23T18:13:56.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[179668]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[179668]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[179668]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[184174]: pgmap v1121: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 620 MiB data, 5.8 GiB used, 709 GiB / 715 GiB avail; 64 MiB/s wr, 1.02k op/s; 1756/29613 objects degraded (5.930%) 2023-09-23T18:13:56.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 1756/29613 objects degraded (5.930%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:13:56.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:13:56.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[184174]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2803852310' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:13:56.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[184174]: osdmap e464: 8 total, 7 up, 8 in 2023-09-23T18:13:56.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[184174]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[184174]: failed to encode map e464 with expected crc 2023-09-23T18:13:56.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:56 smithi055 ceph-mon[184174]: failed to encode map e464 with expected crc 2023-09-23T18:13:57.303 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:57.373 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:13:57.596 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 9 16 24294 24278 168.561 139.062 0.00185193 0.00588268 2023-09-23T18:13:57.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:57 smithi055 ceph-mon[179668]: failed to encode map e464 with expected crc 2023-09-23T18:13:57.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:57.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:57 smithi055 ceph-mon[184174]: failed to encode map e464 with expected crc 2023-09-23T18:13:57.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:57 smithi099 ceph-mon[167076]: failed to encode map e464 with expected crc 2023-09-23T18:13:57.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:13:57.895 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:13:58.272 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:58.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:58.279 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:58.317 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:58.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:13:58.431 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:58.494 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:13:58.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:58.502 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:13:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:58 smithi099 ceph-mon[167076]: pgmap v1123: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 1756/51213 objects degraded (3.429%) 2023-09-23T18:13:58.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:58 smithi099 ceph-mon[167076]: 1.11 scrub starts 2023-09-23T18:13:58.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:58 smithi099 ceph-mon[167076]: 1.11 scrub ok 2023-09-23T18:13:58.596 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 10 16 26306 26290 164.28 125.75 0.00756237 0.00608038 2023-09-23T18:13:58.603 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:13:58.651 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:58.749 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:58 smithi055 ceph-mon[179668]: pgmap v1123: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 1756/51213 objects degraded (3.429%) 2023-09-23T18:13:58.749 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:58 smithi055 ceph-mon[179668]: 1.11 scrub starts 2023-09-23T18:13:58.749 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:58 smithi055 ceph-mon[179668]: 1.11 scrub ok 2023-09-23T18:13:58.749 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:58 smithi055 ceph-mon[184174]: pgmap v1123: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.0 GiB data, 6.8 GiB used, 708 GiB / 715 GiB avail; 106 MiB/s wr, 1.70k op/s; 1756/51213 objects degraded (3.429%) 2023-09-23T18:13:58.749 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:58 smithi055 ceph-mon[184174]: 1.11 scrub starts 2023-09-23T18:13:58.749 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:58 smithi055 ceph-mon[184174]: 1.11 scrub ok 2023-09-23T18:13:59.005 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:13:59.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:59.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:13:59.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:59.156 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:59.282 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:13:59.449 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:13:59.457 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:13:59.476 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:59 smithi099 ceph-mon[167076]: 1.e scrub starts 2023-09-23T18:13:59.476 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:13:59 smithi099 ceph-mon[167076]: 1.e scrub ok 2023-09-23T18:13:59.490 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:59.495 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:59.496 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:59.597 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:13:59.598 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 11 16 28270 28254 160.49 122.75 0.00253845 0.00622542 2023-09-23T18:13:59.626 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:13:59.678 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:13:59.687 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:13:59.688 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:13:59.788 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:13:59.826 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:59 smithi055 ceph-mon[179668]: 1.e scrub starts 2023-09-23T18:13:59.826 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:13:59 smithi055 ceph-mon[179668]: 1.e scrub ok 2023-09-23T18:13:59.826 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:59 smithi055 ceph-mon[184174]: 1.e scrub starts 2023-09-23T18:13:59.827 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:13:59 smithi055 ceph-mon[184174]: 1.e scrub ok 2023-09-23T18:13:59.840 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:00.027 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:00.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:00.036 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:00.060 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1124,"stamp":"2023-09-23T18:13:58.993858+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1452573812,"num_objects":22122,"num_object_clones":0,"num_object_copies":66366,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3253,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22122,"num_whiteouts":0,"num_read":6030,"num_read_kb":12292,"num_write":23038,"num_write_kb":1427125,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":305,"num_bytes_recovered":9596188,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":28116,"ondisk_log_size":28116,"up":330,"acting":330,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":360,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":8318888,"kb_used_data":4296744,"kb_used_omap":60,"kb_used_meta":4022019,"kb_avail":741642328,"statfs":{"total":767960285184,"available":759441743872,"internally_reserved":0,"allocated":4399865856,"data_stored":4384952061,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":61623,"internal_metadata":4118548297},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,1,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":1447912131,"num_objects":22094,"num_object_clones":0,"num_object_copies":66282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3250,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22094,"num_whiteouts":0,"num_read":-33,"num_read_kb":-62,"num_write":22077,"num_write_kb":1413851,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":-120757,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003457"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":775,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.409403+0000","last_change":"2023-09-23T18:13:46.409304+0000","last_active":"2023-09-23T18:13:46.409403+0000","last_peered":"2023-09-23T18:13:46.409403+0000","last_clean":"2023-09-23T18:13:44.384482+0000","last_became_active":"2023-09-23T18:13:46.409304+0000","last_became_peered":"2023-09-23T18:13:46.409304+0000","last_unstale":"2023-09-23T18:13:46.409403+0000","last_undegraded":"2023-09-23T18:13:46.409403+0000","last_fullsized":"2023-09-23T18:13:46.401529+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.712627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.712627+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":999913,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":77,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.412907+0000","last_change":"2023-09-23T18:13:46.412649+0000","last_active":"2023-09-23T18:13:46.412907+0000","last_peered":"2023-09-23T18:13:46.412907+0000","last_clean":"2023-09-23T18:13:44.385598+0000","last_became_active":"2023-09-23T18:13:46.412649+0000","last_became_peered":"2023-09-23T18:13:46.412649+0000","last_unstale":"2023-09-23T18:13:46.412907+0000","last_undegraded":"2023-09-23T18:13:46.412907+0000","last_fullsized":"2023-09-23T18:13:46.406993+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:43.117460+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":362,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.403088+0000","last_change":"2023-09-23T18:12:06.645901+0000","last_active":"2023-09-23T18:13:46.403088+0000","last_peered":"2023-09-23T18:13:46.403088+0000","last_clean":"2023-09-23T18:13:46.403088+0000","last_became_active":"2023-09-23T18:12:06.645637+0000","last_became_peered":"2023-09-23T18:12:06.645637+0000","last_unstale":"2023-09-23T18:13:46.403088+0000","last_undegraded":"2023-09-23T18:13:46.403088+0000","last_fullsized":"2023-09-23T18:13:46.403088+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:12:40.588377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":775,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.413026+0000","last_change":"2023-09-23T18:13:46.412986+0000","last_active":"2023-09-23T18:13:46.413026+0000","last_peered":"2023-09-23T18:13:46.413026+0000","last_clean":"2023-09-23T18:13:44.384861+0000","last_became_active":"2023-09-23T18:13:46.412986+0000","last_became_peered":"2023-09-23T18:13:46.412986+0000","last_unstale":"2023-09-23T18:13:46.413026+0000","last_undegraded":"2023-09-23T18:13:46.413026+0000","last_fullsized":"2023-09-23T18:13:46.403235+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:05:32.839102+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4],"acting":[5,4],"avail_no_missing":["5","4"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":236,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.403406+0000","last_change":"2023-09-23T18:13:39.230590+0000","last_active":"2023-09-23T18:13:46.403406+0000","last_peered":"2023-09-23T18:13:46.403406+0000","last_clean":"2023-09-23T18:13:46.403406+0000","last_became_active":"2023-09-23T18:13:39.230265+0000","last_became_peered":"2023-09-23T18:13:39.230265+0000","last_unstale":"2023-09-23T18:13:46.403406+0000","last_undegraded":"2023-09-23T18:13:46.403406+0000","last_fullsized":"2023-09-23T18:13:46.403406+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:29.038853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":490,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406252+0000","last_change":"2023-09-23T18:12:32.314383+0000","last_active":"2023-09-23T18:13:46.406252+0000","last_peered":"2023-09-23T18:13:46.406252+0000","last_clean":"2023-09-23T18:13:46.406252+0000","last_became_active":"2023-09-23T18:12:32.314092+0000","last_became_peered":"2023-09-23T18:12:32.314092+0000","last_unstale":"2023-09-23T18:13:46.406252+0000","last_undegraded":"2023-09-23T18:13:46.406252+0000","last_fullsized":"2023-09-23T18:13:46.406252+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:33:31.573002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":795,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.407365+0000","last_change":"2023-09-23T18:12:06.649780+0000","last_active":"2023-09-23T18:13:46.407365+0000","last_peered":"2023-09-23T18:13:46.407365+0000","last_clean":"2023-09-23T18:13:46.407365+0000","last_became_active":"2023-09-23T18:12:06.649464+0000","last_became_peered":"2023-09-23T18:12:06.649464+0000","last_unstale":"2023-09-23T18:13:46.407365+0000","last_undegraded":"2023-09-23T18:13:46.407365+0000","last_fullsized":"2023-09-23T18:13:46.407365+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:51.143336+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":380,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.410416+0000","last_change":"2023-09-23T18:13:46.410242+0000","last_active":"2023-09-23T18:13:46.410416+0000","last_peered":"2023-09-23T18:13:46.410416+0000","last_clean":"2023-09-23T18:13:44.647216+0000","last_became_active":"2023-09-23T18:13:46.410242+0000","last_became_peered":"2023-09-23T18:13:46.410242+0000","last_unstale":"2023-09-23T18:13:46.410416+0000","last_undegraded":"2023-09-23T18:13:46.410416+0000","last_fullsized":"2023-09-23T18:13:46.403896+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:51:18.336504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":431,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.405198+0000","last_change":"2023-09-23T18:12:32.316801+0000","last_active":"2023-09-23T18:13:46.405198+0000","last_peered":"2023-09-23T18:13:46.405198+0000","last_clean":"2023-09-23T18:13:46.405198+0000","last_became_active":"2023-09-23T18:12:32.316076+0000","last_became_peered":"2023-09-23T18:12:32.316076+0000","last_unstale":"2023-09-23T18:13:46.405198+0000","last_undegraded":"2023-09-23T18:13:46.405198+0000","last_fullsized":"2023-09-23T18:13:46.405198+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:34.054541+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":861,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406813+0000","last_change":"2023-09-23T18:12:32.337404+0000","last_active":"2023-09-23T18:13:46.406813+0000","last_peered":"2023-09-23T18:13:46.406813+0000","last_clean":"2023-09-23T18:13:46.406813+0000","last_became_active":"2023-09-23T18:12:32.337014+0000","last_became_peered":"2023-09-23T18:12:32.337014+0000","last_unstale":"2023-09-23T18:13:46.406813+0000","last_undegraded":"2023-09-23T18:13:46.406813+0000","last_fullsized":"2023-09-23T18:13:46.406813+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:15:09.079475+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":354,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.403087+0000","last_change":"2023-09-23T18:12:33.309772+0000","last_active":"2023-09-23T18:13:45.403087+0000","last_peered":"2023-09-23T18:13:45.403087+0000","last_clean":"2023-09-23T18:13:45.403087+0000","last_became_active":"2023-09-23T18:12:33.308746+0000","last_became_peered":"2023-09-23T18:12:33.308746+0000","last_unstale":"2023-09-23T18:13:45.403087+0000","last_undegraded":"2023-09-23T18:13:45.403087+0000","last_fullsized":"2023-09-23T18:13:45.403087+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:27:08.023480+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":18,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.402041+0000","last_change":"2023-09-23T18:13:39.223092+0000","last_active":"2023-09-23T18:13:46.402041+0000","last_peered":"2023-09-23T18:13:46.402041+0000","last_clean":"2023-09-23T18:13:46.402041+0000","last_became_active":"2023-09-23T18:13:39.222773+0000","last_became_peered":"2023-09-23T18:13:39.222773+0000","last_unstale":"2023-09-23T18:13:46.402041+0000","last_undegraded":"2023-09-23T18:13:46.402041+0000","last_fullsized":"2023-09-23T18:13:46.402041+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:14:31.585063+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":839,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.414112+0000","last_change":"2023-09-23T18:13:46.413874+0000","last_active":"2023-09-23T18:13:46.414112+0000","last_peered":"2023-09-23T18:13:46.414112+0000","last_clean":"2023-09-23T18:13:45.034137+0000","last_became_active":"2023-09-23T18:13:46.413874+0000","last_became_peered":"2023-09-23T18:13:46.413874+0000","last_unstale":"2023-09-23T18:13:46.414112+0000","last_undegraded":"2023-09-23T18:13:46.414112+0000","last_fullsized":"2023-09-23T18:13:46.407576+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:24:25.700316+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":376,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.402080+0000","last_change":"2023-09-23T18:12:32.320477+0000","last_active":"2023-09-23T18:13:45.402080+0000","last_peered":"2023-09-23T18:13:45.402080+0000","last_clean":"2023-09-23T18:13:45.402080+0000","last_became_active":"2023-09-23T18:12:32.320176+0000","last_became_peered":"2023-09-23T18:12:32.320176+0000","last_unstale":"2023-09-23T18:13:45.402080+0000","last_undegraded":"2023-09-23T18:13:45.402080+0000","last_fullsized":"2023-09-23T18:13:45.402080+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:19.404451+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":293,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.407641+0000","last_change":"2023-09-23T18:13:39.225138+0000","last_active":"2023-09-23T18:13:46.407641+0000","last_peered":"2023-09-23T18:13:46.407641+0000","last_clean":"2023-09-23T18:13:46.407641+0000","last_became_active":"2023-09-23T18:13:39.224810+0000","last_became_peered":"2023-09-23T18:13:39.224810+0000","last_unstale":"2023-09-23T18:13:46.407641+0000","last_undegraded":"2023-09-23T18:13:46.407641+0000","last_fullsized":"2023-09-23T18:13:46.407641+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:51:46.902930+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":759,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:49.188721+0000","last_change":"2023-09-23T18:13:49.188664+0000","last_active":"2023-09-23T18:13:49.188721+0000","last_peered":"2023-09-23T18:13:49.188721+0000","last_clean":"2023-09-23T18:13:49.188721+0000","last_became_active":"2023-09-23T18:12:32.316944+0000","last_became_peered":"2023-09-23T18:12:32.316944+0000","last_unstale":"2023-09-23T18:13:49.188721+0000","last_undegraded":"2023-09-23T18:13:49.188721+0000","last_fullsized":"2023-09-23T18:13:49.188721+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:58:02.870292+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,7],"acting":[5,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":325,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406730+0000","last_change":"2023-09-23T18:13:39.235689+0000","last_active":"2023-09-23T18:13:46.406730+0000","last_peered":"2023-09-23T18:13:46.406730+0000","last_clean":"2023-09-23T18:13:46.406730+0000","last_became_active":"2023-09-23T18:13:39.230285+0000","last_became_peered":"2023-09-23T18:13:39.230285+0000","last_unstale":"2023-09-23T18:13:46.406730+0000","last_undegraded":"2023-09-23T18:13:46.406730+0000","last_fullsized":"2023-09-23T18:13:46.406730+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:52.139628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":227,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.410075+0000","last_change":"2023-09-23T18:13:46.409853+0000","last_active":"2023-09-23T18:13:46.410075+0000","last_peered":"2023-09-23T18:13:46.410075+0000","last_clean":"2023-09-23T18:13:44.647749+0000","last_became_active":"2023-09-23T18:13:46.409853+0000","last_became_peered":"2023-09-23T18:13:46.409853+0000","last_unstale":"2023-09-23T18:13:46.410075+0000","last_undegraded":"2023-09-23T18:13:46.410075+0000","last_fullsized":"2023-09-23T18:13:46.404840+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:06:21.557540+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":885,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404985+0000","last_change":"2023-09-23T18:12:05.654051+0000","last_active":"2023-09-23T18:13:46.404985+0000","last_peered":"2023-09-23T18:13:46.404985+0000","last_clean":"2023-09-23T18:13:46.404985+0000","last_became_active":"2023-09-23T18:12:05.653587+0000","last_became_peered":"2023-09-23T18:12:05.653587+0000","last_unstale":"2023-09-23T18:13:46.404985+0000","last_undegraded":"2023-09-23T18:13:46.404985+0000","last_fullsized":"2023-09-23T18:13:46.404985+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:10.417643+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":250,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.412030+0000","last_change":"2023-09-23T18:12:06.679404+0000","last_active":"2023-09-23T18:13:46.412030+0000","last_peered":"2023-09-23T18:13:46.412030+0000","last_clean":"2023-09-23T18:13:46.412030+0000","last_became_active":"2023-09-23T18:12:06.679106+0000","last_became_peered":"2023-09-23T18:12:06.679106+0000","last_unstale":"2023-09-23T18:13:46.412030+0000","last_undegraded":"2023-09-23T18:13:46.412030+0000","last_fullsized":"2023-09-23T18:13:46.412030+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:39:48.320006+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":268,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.410087+0000","last_change":"2023-09-23T18:12:06.645467+0000","last_active":"2023-09-23T18:13:46.410087+0000","last_peered":"2023-09-23T18:13:46.410087+0000","last_clean":"2023-09-23T18:13:46.410087+0000","last_became_active":"2023-09-23T18:12:06.642537+0000","last_became_peered":"2023-09-23T18:12:06.642537+0000","last_unstale":"2023-09-23T18:13:46.410087+0000","last_undegraded":"2023-09-23T18:13:46.410087+0000","last_fullsized":"2023-09-23T18:13:46.410087+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:24:37.741322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":727,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.403032+0000","last_change":"2023-09-23T18:12:32.338144+0000","last_active":"2023-09-23T18:13:45.403032+0000","last_peered":"2023-09-23T18:13:45.403032+0000","last_clean":"2023-09-23T18:13:45.403032+0000","last_became_active":"2023-09-23T18:12:32.337238+0000","last_became_peered":"2023-09-23T18:12:32.337238+0000","last_unstale":"2023-09-23T18:13:45.403032+0000","last_undegraded":"2023-09-23T18:13:45.403032+0000","last_fullsized":"2023-09-23T18:13:45.403032+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:09:24.869576+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":463,"reported_epoch":463,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:13:46.413424+0000","last_change":"2023-09-23T18:13:46.413169+0000","last_active":"2023-09-23T18:13:46.413424+0000","last_peered":"2023-09-23T18:13:46.413424+0000","last_clean":"2023-09-23T18:13:45.034260+0000","last_became_active":"2023-09-23T18:13:46.413169+0000","last_became_peered":"2023-09-23T18:13:46.413169+0000","last_unstale":"2023-09-23T18:13:46.413424+0000","last_undegraded":"2023-09-23T18:13:46.406068+0000","last_fullsized":"2023-09-23T18:13:46.405961+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.024567+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4],"acting":[7,4],"avail_no_missing":["7","4"],"object_location_counts":[{"shards":"4,7","objects":2}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":319,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.403345+0000","last_change":"2023-09-23T18:13:39.226565+0000","last_active":"2023-09-23T18:13:46.403345+0000","last_peered":"2023-09-23T18:13:46.403345+0000","last_clean":"2023-09-23T18:13:46.403345+0000","last_became_active":"2023-09-23T18:13:39.226211+0000","last_became_peered":"2023-09-23T18:13:39.226211+0000","last_unstale":"2023-09-23T18:13:46.403345+0000","last_undegraded":"2023-09-23T18:13:46.403345+0000","last_fullsized":"2023-09-23T18:13:46.403345+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:53.510129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":744,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:54.269599+0000","last_change":"2023-09-23T18:13:54.269536+0000","last_active":"2023-09-23T18:13:54.269599+0000","last_peered":"2023-09-23T18:13:54.269599+0000","last_clean":"2023-09-23T18:13:54.269599+0000","last_became_active":"2023-09-23T18:12:32.307314+0000","last_became_peered":"2023-09-23T18:12:32.307314+0000","last_unstale":"2023-09-23T18:13:54.269599+0000","last_undegraded":"2023-09-23T18:13:54.269599+0000","last_fullsized":"2023-09-23T18:13:54.269599+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:32:49.398606+0000","scrub_duration":0.0059717060000000002,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":512,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.412611+0000","last_change":"2023-09-23T18:13:46.412071+0000","last_active":"2023-09-23T18:13:46.412611+0000","last_peered":"2023-09-23T18:13:46.412611+0000","last_clean":"2023-09-23T18:13:45.034086+0000","last_became_active":"2023-09-23T18:13:46.412071+0000","last_became_peered":"2023-09-23T18:13:46.412071+0000","last_unstale":"2023-09-23T18:13:46.412611+0000","last_undegraded":"2023-09-23T18:13:46.412611+0000","last_fullsized":"2023-09-23T18:13:46.405682+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:16.008042+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1],"acting":[7,1],"avail_no_missing":["7","1"],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":414,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.403584+0000","last_change":"2023-09-23T18:13:39.226718+0000","last_active":"2023-09-23T18:13:45.403584+0000","last_peered":"2023-09-23T18:13:45.403584+0000","last_clean":"2023-09-23T18:13:45.403584+0000","last_became_active":"2023-09-23T18:13:39.226382+0000","last_became_peered":"2023-09-23T18:13:39.226382+0000","last_unstale":"2023-09-23T18:13:45.403584+0000","last_undegraded":"2023-09-23T18:13:45.403584+0000","last_fullsized":"2023-09-23T18:13:45.403584+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:17:34.008867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":849,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404295+0000","last_change":"2023-09-23T18:12:06.645815+0000","last_active":"2023-09-23T18:13:46.404295+0000","last_peered":"2023-09-23T18:13:46.404295+0000","last_clean":"2023-09-23T18:13:46.404295+0000","last_became_active":"2023-09-23T18:12:06.645452+0000","last_became_peered":"2023-09-23T18:12:06.645452+0000","last_unstale":"2023-09-23T18:13:46.404295+0000","last_undegraded":"2023-09-23T18:13:46.404295+0000","last_fullsized":"2023-09-23T18:13:46.404295+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:24.550609+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":61,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.410576+0000","last_change":"2023-09-23T18:12:05.646034+0000","last_active":"2023-09-23T18:13:46.410576+0000","last_peered":"2023-09-23T18:13:46.410576+0000","last_clean":"2023-09-23T18:13:46.410576+0000","last_became_active":"2023-09-23T18:12:05.645701+0000","last_became_peered":"2023-09-23T18:12:05.645701+0000","last_unstale":"2023-09-23T18:13:46.410576+0000","last_undegraded":"2023-09-23T18:13:46.410576+0000","last_fullsized":"2023-09-23T18:13:46.410576+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:22.277691+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":24,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.408628+0000","last_change":"2023-09-23T18:13:46.408429+0000","last_active":"2023-09-23T18:13:46.408628+0000","last_peered":"2023-09-23T18:13:46.408628+0000","last_clean":"2023-09-23T18:13:45.069569+0000","last_became_active":"2023-09-23T18:13:46.408429+0000","last_became_peered":"2023-09-23T18:13:46.408429+0000","last_unstale":"2023-09-23T18:13:46.408628+0000","last_undegraded":"2023-09-23T18:13:46.408628+0000","last_fullsized":"2023-09-23T18:13:46.402431+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:37:33.121491+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7],"acting":[3,7],"avail_no_missing":["3","7"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":724,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.401811+0000","last_change":"2023-09-23T18:12:32.335469+0000","last_active":"2023-09-23T18:13:45.401811+0000","last_peered":"2023-09-23T18:13:45.401811+0000","last_clean":"2023-09-23T18:13:45.401811+0000","last_became_active":"2023-09-23T18:12:32.334685+0000","last_became_peered":"2023-09-23T18:12:32.334685+0000","last_unstale":"2023-09-23T18:13:45.401811+0000","last_undegraded":"2023-09-23T18:13:45.401811+0000","last_fullsized":"2023-09-23T18:13:45.401811+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:51:41.036373+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":508,"reported_epoch":463,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:13:46.411524+0000","last_change":"2023-09-23T18:13:46.411261+0000","last_active":"2023-09-23T18:13:46.411524+0000","last_peered":"2023-09-23T18:13:46.411524+0000","last_clean":"2023-09-23T18:13:44.969333+0000","last_became_active":"2023-09-23T18:13:46.411261+0000","last_became_peered":"2023-09-23T18:13:46.411261+0000","last_unstale":"2023-09-23T18:13:46.411524+0000","last_undegraded":"2023-09-23T18:13:46.404020+0000","last_fullsized":"2023-09-23T18:13:46.403909+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:37.203507+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5],"acting":[6,5],"avail_no_missing":["6","5"],"object_location_counts":[{"shards":"5,6","objects":4}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":343,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.410636+0000","last_change":"2023-09-23T18:13:46.410487+0000","last_active":"2023-09-23T18:13:46.410636+0000","last_peered":"2023-09-23T18:13:46.410636+0000","last_clean":"2023-09-23T18:13:44.647912+0000","last_became_active":"2023-09-23T18:13:46.410487+0000","last_became_peered":"2023-09-23T18:13:46.410487+0000","last_unstale":"2023-09-23T18:13:46.410636+0000","last_undegraded":"2023-09-23T18:13:46.410636+0000","last_fullsized":"2023-09-23T18:13:46.404858+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:04:55.935710+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"463'1176","reported_seq":1187,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.394678+0000","last_change":"2023-09-23T18:13:43.955681+0000","last_active":"2023-09-23T18:13:55.394678+0000","last_peered":"2023-09-23T18:13:55.394678+0000","last_clean":"2023-09-23T18:13:55.394678+0000","last_became_active":"2023-09-23T18:13:43.955335+0000","last_became_peered":"2023-09-23T18:13:43.955335+0000","last_unstale":"2023-09-23T18:13:55.394678+0000","last_undegraded":"2023-09-23T18:13:55.394678+0000","last_fullsized":"2023-09-23T18:13:55.394678+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1176,"log_dups_size":0,"ondisk_log_size":1176,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:21:58.913204+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77070336,"num_objects":1176,"num_object_clones":0,"num_object_copies":3528,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1176,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1176,"num_write_kb":75264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":884,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406643+0000","last_change":"2023-09-23T18:12:08.689111+0000","last_active":"2023-09-23T18:13:46.406643+0000","last_peered":"2023-09-23T18:13:46.406643+0000","last_clean":"2023-09-23T18:13:46.406643+0000","last_became_active":"2023-09-23T18:12:08.688825+0000","last_became_peered":"2023-09-23T18:12:08.688825+0000","last_unstale":"2023-09-23T18:13:46.406643+0000","last_undegraded":"2023-09-23T18:13:46.406643+0000","last_fullsized":"2023-09-23T18:13:46.406643+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:29.016205+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":430,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.405484+0000","last_change":"2023-09-23T18:12:06.652515+0000","last_active":"2023-09-23T18:13:46.405484+0000","last_peered":"2023-09-23T18:13:46.405484+0000","last_clean":"2023-09-23T18:13:46.405484+0000","last_became_active":"2023-09-23T18:12:06.652196+0000","last_became_peered":"2023-09-23T18:12:06.652196+0000","last_unstale":"2023-09-23T18:13:46.405484+0000","last_undegraded":"2023-09-23T18:13:46.405484+0000","last_fullsized":"2023-09-23T18:13:46.405484+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:27.975688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":377,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.401606+0000","last_change":"2023-09-23T18:12:32.311619+0000","last_active":"2023-09-23T18:13:45.401606+0000","last_peered":"2023-09-23T18:13:45.401606+0000","last_clean":"2023-09-23T18:13:45.401606+0000","last_became_active":"2023-09-23T18:12:32.311311+0000","last_became_peered":"2023-09-23T18:12:32.311311+0000","last_unstale":"2023-09-23T18:13:45.401606+0000","last_undegraded":"2023-09-23T18:13:45.401606+0000","last_fullsized":"2023-09-23T18:13:45.401606+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:30.980099+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.4","version":"464'1550","reported_seq":1564,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T18:13:57.886604+0000","last_change":"2023-09-23T18:13:44.089940+0000","last_active":"2023-09-23T18:13:57.886604+0000","last_peered":"2023-09-23T18:13:57.886604+0000","last_clean":"2023-09-23T18:13:57.886604+0000","last_became_active":"2023-09-23T18:13:44.089583+0000","last_became_peered":"2023-09-23T18:13:44.089583+0000","last_unstale":"2023-09-23T18:13:57.886604+0000","last_undegraded":"2023-09-23T18:13:57.886604+0000","last_fullsized":"2023-09-23T18:13:57.886604+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1550,"log_dups_size":0,"ondisk_log_size":1550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:52.948719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101580800,"num_objects":1550,"num_object_clones":0,"num_object_copies":4650,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1550,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1550,"num_write_kb":99200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":551,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406935+0000","last_change":"2023-09-23T18:12:05.644364+0000","last_active":"2023-09-23T18:13:46.406935+0000","last_peered":"2023-09-23T18:13:46.406935+0000","last_clean":"2023-09-23T18:13:46.406935+0000","last_became_active":"2023-09-23T18:12:05.644014+0000","last_became_peered":"2023-09-23T18:12:05.644014+0000","last_unstale":"2023-09-23T18:13:46.406935+0000","last_undegraded":"2023-09-23T18:13:46.406935+0000","last_fullsized":"2023-09-23T18:13:46.406935+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:41.510146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:41.510146+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:03.919547+0000","scrub_duration":0.0078303169999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":445,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406531+0000","last_change":"2023-09-23T18:12:06.653146+0000","last_active":"2023-09-23T18:13:46.406531+0000","last_peered":"2023-09-23T18:13:46.406531+0000","last_clean":"2023-09-23T18:13:46.406531+0000","last_became_active":"2023-09-23T18:12:06.652832+0000","last_became_peered":"2023-09-23T18:12:06.652832+0000","last_unstale":"2023-09-23T18:13:46.406531+0000","last_undegraded":"2023-09-23T18:13:46.406531+0000","last_fullsized":"2023-09-23T18:13:46.406531+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:34.147471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":61,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.410371+0000","last_change":"2023-09-23T18:12:05.655221+0000","last_active":"2023-09-23T18:13:46.410371+0000","last_peered":"2023-09-23T18:13:46.410371+0000","last_clean":"2023-09-23T18:13:46.410371+0000","last_became_active":"2023-09-23T18:12:05.654901+0000","last_became_peered":"2023-09-23T18:12:05.654901+0000","last_unstale":"2023-09-23T18:13:46.410371+0000","last_undegraded":"2023-09-23T18:13:46.410371+0000","last_fullsized":"2023-09-23T18:13:46.410371+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:56.947639+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"463'1210","reported_seq":1222,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.398790+0000","last_change":"2023-09-23T18:13:43.380121+0000","last_active":"2023-09-23T18:13:55.398790+0000","last_peered":"2023-09-23T18:13:55.398790+0000","last_clean":"2023-09-23T18:13:55.398790+0000","last_became_active":"2023-09-23T18:13:43.379842+0000","last_became_peered":"2023-09-23T18:13:43.379842+0000","last_unstale":"2023-09-23T18:13:55.398790+0000","last_undegraded":"2023-09-23T18:13:55.398790+0000","last_fullsized":"2023-09-23T18:13:55.398790+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1210,"log_dups_size":0,"ondisk_log_size":1210,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:58:14.025467+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79298560,"num_objects":1210,"num_object_clones":0,"num_object_copies":3630,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1210,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1210,"num_write_kb":77440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":686,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.405700+0000","last_change":"2023-09-23T18:12:33.317652+0000","last_active":"2023-09-23T18:13:46.405700+0000","last_peered":"2023-09-23T18:13:46.405700+0000","last_clean":"2023-09-23T18:13:46.405700+0000","last_became_active":"2023-09-23T18:12:33.317203+0000","last_became_peered":"2023-09-23T18:12:33.317203+0000","last_unstale":"2023-09-23T18:13:46.405700+0000","last_undegraded":"2023-09-23T18:13:46.405700+0000","last_fullsized":"2023-09-23T18:13:46.405700+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:49:56.401433+0000","scrub_duration":0.0018893740000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":347,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406979+0000","last_change":"2023-09-23T18:12:32.336820+0000","last_active":"2023-09-23T18:13:46.406979+0000","last_peered":"2023-09-23T18:13:46.406979+0000","last_clean":"2023-09-23T18:13:46.406979+0000","last_became_active":"2023-09-23T18:12:32.336503+0000","last_became_peered":"2023-09-23T18:12:32.336503+0000","last_unstale":"2023-09-23T18:13:46.406979+0000","last_undegraded":"2023-09-23T18:13:46.406979+0000","last_fullsized":"2023-09-23T18:13:46.406979+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:18.676067+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":439,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404897+0000","last_change":"2023-09-23T18:06:05.371447+0000","last_active":"2023-09-23T18:13:46.404897+0000","last_peered":"2023-09-23T18:13:46.404897+0000","last_clean":"2023-09-23T18:13:46.404897+0000","last_became_active":"2023-09-23T18:06:05.371065+0000","last_became_peered":"2023-09-23T18:06:05.371065+0000","last_unstale":"2023-09-23T18:13:46.404897+0000","last_undegraded":"2023-09-23T18:13:46.404897+0000","last_fullsized":"2023-09-23T18:13:46.404897+0000","mapping_epoch":285,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":286,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:52:49.025791+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"463'1228","reported_seq":1240,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.395312+0000","last_change":"2023-09-23T18:13:43.378269+0000","last_active":"2023-09-23T18:13:55.395312+0000","last_peered":"2023-09-23T18:13:55.395312+0000","last_clean":"2023-09-23T18:13:55.395312+0000","last_became_active":"2023-09-23T18:13:43.377991+0000","last_became_peered":"2023-09-23T18:13:43.377991+0000","last_unstale":"2023-09-23T18:13:55.395312+0000","last_undegraded":"2023-09-23T18:13:55.395312+0000","last_fullsized":"2023-09-23T18:13:55.395312+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1228,"log_dups_size":0,"ondisk_log_size":1228,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:35.684985+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80478208,"num_objects":1228,"num_object_clones":0,"num_object_copies":3684,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1228,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1228,"num_write_kb":78592,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":709,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406545+0000","last_change":"2023-09-23T18:13:39.229226+0000","last_active":"2023-09-23T18:13:46.406545+0000","last_peered":"2023-09-23T18:13:46.406545+0000","last_clean":"2023-09-23T18:13:46.406545+0000","last_became_active":"2023-09-23T18:13:39.228861+0000","last_became_peered":"2023-09-23T18:13:39.228861+0000","last_unstale":"2023-09-23T18:13:46.406545+0000","last_undegraded":"2023-09-23T18:13:46.406545+0000","last_fullsized":"2023-09-23T18:13:46.406545+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.362652+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.362652+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13417,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":5,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":40252,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":388,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.404157+0000","last_change":"2023-09-23T18:12:32.320529+0000","last_active":"2023-09-23T18:13:45.404157+0000","last_peered":"2023-09-23T18:13:45.404157+0000","last_clean":"2023-09-23T18:13:45.404157+0000","last_became_active":"2023-09-23T18:12:32.320241+0000","last_became_peered":"2023-09-23T18:12:32.320241+0000","last_unstale":"2023-09-23T18:13:45.404157+0000","last_undegraded":"2023-09-23T18:13:45.404157+0000","last_fullsized":"2023-09-23T18:13:45.404157+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:31.121529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":399,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.465722+0000","last_change":"2023-09-23T18:12:05.655791+0000","last_active":"2023-09-23T18:13:55.465722+0000","last_peered":"2023-09-23T18:13:55.465722+0000","last_clean":"2023-09-23T18:13:55.465722+0000","last_became_active":"2023-09-23T18:12:05.655227+0000","last_became_peered":"2023-09-23T18:12:05.655227+0000","last_unstale":"2023-09-23T18:13:55.465722+0000","last_undegraded":"2023-09-23T18:13:55.465722+0000","last_fullsized":"2023-09-23T18:13:55.465722+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:37.413645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"464'1191","reported_seq":1204,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.732889+0000","last_change":"2023-09-23T18:13:44.089793+0000","last_active":"2023-09-23T18:13:55.732889+0000","last_peered":"2023-09-23T18:13:55.732889+0000","last_clean":"2023-09-23T18:13:55.732889+0000","last_became_active":"2023-09-23T18:13:44.089400+0000","last_became_peered":"2023-09-23T18:13:44.089400+0000","last_unstale":"2023-09-23T18:13:55.732889+0000","last_undegraded":"2023-09-23T18:13:55.732889+0000","last_fullsized":"2023-09-23T18:13:55.732889+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1191,"log_dups_size":0,"ondisk_log_size":1191,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:18:32.514230+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78053376,"num_objects":1191,"num_object_clones":0,"num_object_copies":3573,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1191,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1191,"num_write_kb":76224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":917,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.405255+0000","last_change":"2023-09-23T18:13:39.227697+0000","last_active":"2023-09-23T18:13:46.405255+0000","last_peered":"2023-09-23T18:13:46.405255+0000","last_clean":"2023-09-23T18:13:46.405255+0000","last_became_active":"2023-09-23T18:13:39.227302+0000","last_became_peered":"2023-09-23T18:13:39.227302+0000","last_unstale":"2023-09-23T18:13:46.405255+0000","last_undegraded":"2023-09-23T18:13:46.405255+0000","last_fullsized":"2023-09-23T18:13:46.405255+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:47.233519+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,5],"acting":[7,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":303,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.410222+0000","last_change":"2023-09-23T18:13:39.237303+0000","last_active":"2023-09-23T18:13:46.410222+0000","last_peered":"2023-09-23T18:13:46.410222+0000","last_clean":"2023-09-23T18:13:46.410222+0000","last_became_active":"2023-09-23T18:13:39.230309+0000","last_became_peered":"2023-09-23T18:13:39.230309+0000","last_unstale":"2023-09-23T18:13:46.410222+0000","last_undegraded":"2023-09-23T18:13:46.410222+0000","last_fullsized":"2023-09-23T18:13:46.410222+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:37.392528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":81,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.411863+0000","last_change":"2023-09-23T18:13:39.231339+0000","last_active":"2023-09-23T18:13:46.411863+0000","last_peered":"2023-09-23T18:13:46.411863+0000","last_clean":"2023-09-23T18:13:46.411863+0000","last_became_active":"2023-09-23T18:13:39.231018+0000","last_became_peered":"2023-09-23T18:13:39.231018+0000","last_unstale":"2023-09-23T18:13:46.411863+0000","last_undegraded":"2023-09-23T18:13:46.411863+0000","last_fullsized":"2023-09-23T18:13:46.411863+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:07:04.831474+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"464'1648","reported_seq":1668,"reported_epoch":464,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:13:57.884972+0000","last_change":"2023-09-23T18:13:48.601510+0000","last_active":"2023-09-23T18:13:57.884972+0000","last_peered":"2023-09-23T18:13:57.884972+0000","last_clean":"2023-09-23T18:13:44.646980+0000","last_became_active":"2023-09-23T18:13:46.408222+0000","last_became_peered":"2023-09-23T18:13:46.408222+0000","last_unstale":"2023-09-23T18:13:57.884972+0000","last_undegraded":"2023-09-23T18:13:48.599118+0000","last_fullsized":"2023-09-23T18:13:46.403288+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1648,"log_dups_size":0,"ondisk_log_size":1648,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:28:09.050093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":108003328,"num_objects":1648,"num_object_clones":0,"num_object_copies":4944,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1648,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1648,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1648,"num_write_kb":105472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"1,6","objects":1648}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":770,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.407147+0000","last_change":"2023-09-23T18:13:46.407065+0000","last_active":"2023-09-23T18:13:46.407147+0000","last_peered":"2023-09-23T18:13:46.407147+0000","last_clean":"2023-09-23T18:13:44.384586+0000","last_became_active":"2023-09-23T18:13:46.407065+0000","last_became_peered":"2023-09-23T18:13:46.407065+0000","last_unstale":"2023-09-23T18:13:46.407147+0000","last_undegraded":"2023-09-23T18:13:46.407147+0000","last_fullsized":"2023-09-23T18:13:46.401651+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:32.254440+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":453,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.405242+0000","last_change":"2023-09-23T18:12:05.655148+0000","last_active":"2023-09-23T18:13:46.405242+0000","last_peered":"2023-09-23T18:13:46.405242+0000","last_clean":"2023-09-23T18:13:46.405242+0000","last_became_active":"2023-09-23T18:12:05.654825+0000","last_became_peered":"2023-09-23T18:12:05.654825+0000","last_unstale":"2023-09-23T18:13:46.405242+0000","last_undegraded":"2023-09-23T18:13:46.405242+0000","last_fullsized":"2023-09-23T18:13:46.405242+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:09.300269+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":295,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.411604+0000","last_change":"2023-09-23T18:13:46.411337+0000","last_active":"2023-09-23T18:13:46.411604+0000","last_peered":"2023-09-23T18:13:46.411604+0000","last_clean":"2023-09-23T18:13:44.386136+0000","last_became_active":"2023-09-23T18:13:46.411337+0000","last_became_peered":"2023-09-23T18:13:46.411337+0000","last_unstale":"2023-09-23T18:13:46.411604+0000","last_undegraded":"2023-09-23T18:13:46.411604+0000","last_fullsized":"2023-09-23T18:13:46.406549+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:59:41.285378+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1],"acting":[4,1],"avail_no_missing":["4","1"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"464'1666","reported_seq":1676,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T18:13:58.174026+0000","last_change":"2023-09-23T18:13:44.089720+0000","last_active":"2023-09-23T18:13:58.174026+0000","last_peered":"2023-09-23T18:13:58.174026+0000","last_clean":"2023-09-23T18:13:58.174026+0000","last_became_active":"2023-09-23T18:13:44.089423+0000","last_became_peered":"2023-09-23T18:13:44.089423+0000","last_unstale":"2023-09-23T18:13:58.174026+0000","last_undegraded":"2023-09-23T18:13:58.174026+0000","last_fullsized":"2023-09-23T18:13:58.174026+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1666,"log_dups_size":0,"ondisk_log_size":1666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:59.848952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":109182976,"num_objects":1666,"num_object_clones":0,"num_object_copies":4998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1666,"num_write_kb":106624,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":859,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404393+0000","last_change":"2023-09-23T18:12:33.325300+0000","last_active":"2023-09-23T18:13:46.404393+0000","last_peered":"2023-09-23T18:13:46.404393+0000","last_clean":"2023-09-23T18:13:46.404393+0000","last_became_active":"2023-09-23T18:12:33.324847+0000","last_became_peered":"2023-09-23T18:12:33.324847+0000","last_unstale":"2023-09-23T18:13:46.404393+0000","last_undegraded":"2023-09-23T18:13:46.404393+0000","last_fullsized":"2023-09-23T18:13:46.404393+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:29:23.403910+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":275,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.414041+0000","last_change":"2023-09-23T18:13:46.413677+0000","last_active":"2023-09-23T18:13:46.414041+0000","last_peered":"2023-09-23T18:13:46.414041+0000","last_clean":"2023-09-23T18:13:44.385445+0000","last_became_active":"2023-09-23T18:13:46.413677+0000","last_became_peered":"2023-09-23T18:13:46.413677+0000","last_unstale":"2023-09-23T18:13:46.414041+0000","last_undegraded":"2023-09-23T18:13:46.414041+0000","last_fullsized":"2023-09-23T18:13:46.410611+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:19:31.213704+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7],"acting":[4,7],"avail_no_missing":["4","7"],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":61,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.407415+0000","last_change":"2023-09-23T18:12:05.654018+0000","last_active":"2023-09-23T18:13:46.407415+0000","last_peered":"2023-09-23T18:13:46.407415+0000","last_clean":"2023-09-23T18:13:46.407415+0000","last_became_active":"2023-09-23T18:12:05.653113+0000","last_became_peered":"2023-09-23T18:12:05.653113+0000","last_unstale":"2023-09-23T18:13:46.407415+0000","last_undegraded":"2023-09-23T18:13:46.407415+0000","last_fullsized":"2023-09-23T18:13:46.407415+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:24.208896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"464'1597","reported_seq":1615,"reported_epoch":464,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:13:57.892958+0000","last_change":"2023-09-23T18:13:48.607106+0000","last_active":"2023-09-23T18:13:57.892958+0000","last_peered":"2023-09-23T18:13:57.892958+0000","last_clean":"2023-09-23T18:13:44.647171+0000","last_became_active":"2023-09-23T18:13:46.409684+0000","last_became_peered":"2023-09-23T18:13:46.409684+0000","last_unstale":"2023-09-23T18:13:57.892958+0000","last_undegraded":"2023-09-23T18:13:48.599267+0000","last_fullsized":"2023-09-23T18:13:46.403216+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1597,"log_dups_size":0,"ondisk_log_size":1597,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:57.503924+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":104660992,"num_objects":1597,"num_object_clones":0,"num_object_copies":4791,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1597,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1597,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1597,"num_write_kb":102208,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":1597}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":763,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.408295+0000","last_change":"2023-09-23T18:13:46.408211+0000","last_active":"2023-09-23T18:13:46.408295+0000","last_peered":"2023-09-23T18:13:46.408295+0000","last_clean":"2023-09-23T18:13:44.384462+0000","last_became_active":"2023-09-23T18:13:46.408211+0000","last_became_peered":"2023-09-23T18:13:46.408211+0000","last_unstale":"2023-09-23T18:13:46.408295+0000","last_undegraded":"2023-09-23T18:13:46.408295+0000","last_fullsized":"2023-09-23T18:13:46.403083+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":331,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.403871+0000","last_change":"2023-09-23T18:12:05.652577+0000","last_active":"2023-09-23T18:13:46.403871+0000","last_peered":"2023-09-23T18:13:46.403871+0000","last_clean":"2023-09-23T18:13:46.403871+0000","last_became_active":"2023-09-23T18:12:05.652290+0000","last_became_peered":"2023-09-23T18:12:05.652290+0000","last_unstale":"2023-09-23T18:13:46.403871+0000","last_undegraded":"2023-09-23T18:13:46.403871+0000","last_fullsized":"2023-09-23T18:13:46.403871+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:32.320757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":246,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404761+0000","last_change":"2023-09-23T18:13:39.229808+0000","last_active":"2023-09-23T18:13:46.404761+0000","last_peered":"2023-09-23T18:13:46.404761+0000","last_clean":"2023-09-23T18:13:46.404761+0000","last_became_active":"2023-09-23T18:13:39.229469+0000","last_became_peered":"2023-09-23T18:13:39.229469+0000","last_unstale":"2023-09-23T18:13:46.404761+0000","last_undegraded":"2023-09-23T18:13:46.404761+0000","last_fullsized":"2023-09-23T18:13:46.404761+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:33.892578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"463'1107","reported_seq":1118,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.416850+0000","last_change":"2023-09-23T18:13:43.378777+0000","last_active":"2023-09-23T18:13:55.416850+0000","last_peered":"2023-09-23T18:13:55.416850+0000","last_clean":"2023-09-23T18:13:55.416850+0000","last_became_active":"2023-09-23T18:13:43.378487+0000","last_became_peered":"2023-09-23T18:13:43.378487+0000","last_unstale":"2023-09-23T18:13:55.416850+0000","last_undegraded":"2023-09-23T18:13:55.416850+0000","last_fullsized":"2023-09-23T18:13:55.416850+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1107,"log_dups_size":0,"ondisk_log_size":1107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:45:22.300740+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72548352,"num_objects":1107,"num_object_clones":0,"num_object_copies":3321,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1107,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1107,"num_write_kb":70848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":672,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.405118+0000","last_change":"2023-09-23T18:12:32.316992+0000","last_active":"2023-09-23T18:13:46.405118+0000","last_peered":"2023-09-23T18:13:46.405118+0000","last_clean":"2023-09-23T18:13:46.405118+0000","last_became_active":"2023-09-23T18:12:32.316484+0000","last_became_peered":"2023-09-23T18:12:32.316484+0000","last_unstale":"2023-09-23T18:13:46.405118+0000","last_undegraded":"2023-09-23T18:13:46.405118+0000","last_fullsized":"2023-09-23T18:13:46.405118+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:46:21.933684+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,5],"acting":[6,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":868,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.403598+0000","last_change":"2023-09-23T18:10:04.168459+0000","last_active":"2023-09-23T18:13:46.403598+0000","last_peered":"2023-09-23T18:13:46.403598+0000","last_clean":"2023-09-23T18:13:46.403598+0000","last_became_active":"2023-09-23T18:10:04.167807+0000","last_became_peered":"2023-09-23T18:10:04.167807+0000","last_unstale":"2023-09-23T18:13:46.403598+0000","last_undegraded":"2023-09-23T18:13:46.403598+0000","last_fullsized":"2023-09-23T18:13:46.403598+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:40.519034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":407,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.408324+0000","last_change":"2023-09-23T18:13:46.408253+0000","last_active":"2023-09-23T18:13:46.408324+0000","last_peered":"2023-09-23T18:13:46.408324+0000","last_clean":"2023-09-23T18:13:44.384592+0000","last_became_active":"2023-09-23T18:13:46.408253+0000","last_became_peered":"2023-09-23T18:13:46.408253+0000","last_unstale":"2023-09-23T18:13:46.408324+0000","last_undegraded":"2023-09-23T18:13:46.408324+0000","last_fullsized":"2023-09-23T18:13:46.403179+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:52.558866+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6],"acting":[5,6],"avail_no_missing":["5","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":228,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404751+0000","last_change":"2023-09-23T18:12:33.302096+0000","last_active":"2023-09-23T18:13:46.404751+0000","last_peered":"2023-09-23T18:13:46.404751+0000","last_clean":"2023-09-23T18:13:46.404751+0000","last_became_active":"2023-09-23T18:12:33.301770+0000","last_became_peered":"2023-09-23T18:12:33.301770+0000","last_unstale":"2023-09-23T18:13:46.404751+0000","last_undegraded":"2023-09-23T18:13:46.404751+0000","last_fullsized":"2023-09-23T18:13:46.404751+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:01:00.314270+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"464'1561","reported_seq":1574,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T18:13:58.152331+0000","last_change":"2023-09-23T18:13:43.956148+0000","last_active":"2023-09-23T18:13:58.152331+0000","last_peered":"2023-09-23T18:13:58.152331+0000","last_clean":"2023-09-23T18:13:58.152331+0000","last_became_active":"2023-09-23T18:13:43.955820+0000","last_became_peered":"2023-09-23T18:13:43.955820+0000","last_unstale":"2023-09-23T18:13:58.152331+0000","last_undegraded":"2023-09-23T18:13:58.152331+0000","last_fullsized":"2023-09-23T18:13:58.152331+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1561,"log_dups_size":0,"ondisk_log_size":1561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:21:25.835727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102301696,"num_objects":1561,"num_object_clones":0,"num_object_copies":4683,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1561,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1561,"num_write_kb":99904,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,5],"acting":[7,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":777,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.408592+0000","last_change":"2023-09-23T18:13:46.408507+0000","last_active":"2023-09-23T18:13:46.408592+0000","last_peered":"2023-09-23T18:13:46.408592+0000","last_clean":"2023-09-23T18:13:44.384683+0000","last_became_active":"2023-09-23T18:13:46.408507+0000","last_became_peered":"2023-09-23T18:13:46.408507+0000","last_unstale":"2023-09-23T18:13:46.408592+0000","last_undegraded":"2023-09-23T18:13:46.408592+0000","last_fullsized":"2023-09-23T18:13:46.402329+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:26.067367+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1359,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.401933+0000","last_change":"2023-09-23T18:12:32.335626+0000","last_active":"2023-09-23T18:13:45.401933+0000","last_peered":"2023-09-23T18:13:45.401933+0000","last_clean":"2023-09-23T18:13:45.401933+0000","last_became_active":"2023-09-23T18:12:32.335067+0000","last_became_peered":"2023-09-23T18:12:32.335067+0000","last_unstale":"2023-09-23T18:13:45.401933+0000","last_undegraded":"2023-09-23T18:13:45.401933+0000","last_fullsized":"2023-09-23T18:13:45.401933+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:02:53.656722+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":38,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":61,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.409794+0000","last_change":"2023-09-23T18:12:05.649154+0000","last_active":"2023-09-23T18:13:46.409794+0000","last_peered":"2023-09-23T18:13:46.409794+0000","last_clean":"2023-09-23T18:13:46.409794+0000","last_became_active":"2023-09-23T18:12:05.648905+0000","last_became_peered":"2023-09-23T18:12:05.648905+0000","last_unstale":"2023-09-23T18:13:46.409794+0000","last_undegraded":"2023-09-23T18:13:46.409794+0000","last_fullsized":"2023-09-23T18:13:46.409794+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:54:05.562327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":225,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.409676+0000","last_change":"2023-09-23T18:13:46.409234+0000","last_active":"2023-09-23T18:13:46.409676+0000","last_peered":"2023-09-23T18:13:46.409676+0000","last_clean":"2023-09-23T18:13:44.648101+0000","last_became_active":"2023-09-23T18:13:46.409234+0000","last_became_peered":"2023-09-23T18:13:46.409234+0000","last_unstale":"2023-09-23T18:13:46.409676+0000","last_undegraded":"2023-09-23T18:13:46.409676+0000","last_fullsized":"2023-09-23T18:13:46.403546+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:11.542905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"463'1214","reported_seq":1226,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.417177+0000","last_change":"2023-09-23T18:13:43.379192+0000","last_active":"2023-09-23T18:13:55.417177+0000","last_peered":"2023-09-23T18:13:55.417177+0000","last_clean":"2023-09-23T18:13:55.417177+0000","last_became_active":"2023-09-23T18:13:43.378923+0000","last_became_peered":"2023-09-23T18:13:43.378923+0000","last_unstale":"2023-09-23T18:13:55.417177+0000","last_undegraded":"2023-09-23T18:13:55.417177+0000","last_fullsized":"2023-09-23T18:13:55.417177+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1214,"log_dups_size":0,"ondisk_log_size":1214,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:08:01.044388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79560704,"num_objects":1214,"num_object_clones":0,"num_object_copies":3642,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1214,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1214,"num_write_kb":77696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,7],"acting":[5,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":699,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.407369+0000","last_change":"2023-09-23T18:12:06.667895+0000","last_active":"2023-09-23T18:13:46.407369+0000","last_peered":"2023-09-23T18:13:46.407369+0000","last_clean":"2023-09-23T18:13:46.407369+0000","last_became_active":"2023-09-23T18:12:06.657611+0000","last_became_peered":"2023-09-23T18:12:06.657611+0000","last_unstale":"2023-09-23T18:13:46.407369+0000","last_undegraded":"2023-09-23T18:13:46.407369+0000","last_fullsized":"2023-09-23T18:13:46.407369+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:49.208587+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":757,"reported_epoch":463,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:13:47.896170+0000","last_change":"2023-09-23T18:13:46.406918+0000","last_active":"2023-09-23T18:13:47.896170+0000","last_peered":"2023-09-23T18:13:47.896170+0000","last_clean":"2023-09-23T18:13:44.647012+0000","last_became_active":"2023-09-23T18:13:46.406918+0000","last_became_peered":"2023-09-23T18:13:46.406918+0000","last_unstale":"2023-09-23T18:13:47.896170+0000","last_undegraded":"2023-09-23T18:13:46.402983+0000","last_fullsized":"2023-09-23T18:13:46.402864+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:09:24.588021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0],"acting":[1,0],"avail_no_missing":["1","0"],"object_location_counts":[{"shards":"0,1","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":216,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.407964+0000","last_change":"2023-09-23T18:13:46.407744+0000","last_active":"2023-09-23T18:13:46.407964+0000","last_peered":"2023-09-23T18:13:46.407964+0000","last_clean":"2023-09-23T18:13:44.647324+0000","last_became_active":"2023-09-23T18:13:46.407744+0000","last_became_peered":"2023-09-23T18:13:46.407744+0000","last_unstale":"2023-09-23T18:13:46.407964+0000","last_undegraded":"2023-09-23T18:13:46.407964+0000","last_fullsized":"2023-09-23T18:13:46.404048+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:36.358104+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6],"acting":[1,6],"avail_no_missing":["1","6"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":216,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.405125+0000","last_change":"2023-09-23T18:12:32.334537+0000","last_active":"2023-09-23T18:13:46.405125+0000","last_peered":"2023-09-23T18:13:46.405125+0000","last_clean":"2023-09-23T18:13:46.405125+0000","last_became_active":"2023-09-23T18:12:32.334248+0000","last_became_peered":"2023-09-23T18:12:32.334248+0000","last_unstale":"2023-09-23T18:13:46.405125+0000","last_undegraded":"2023-09-23T18:13:46.405125+0000","last_fullsized":"2023-09-23T18:13:46.405125+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:24:07.708083+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,7],"acting":[1,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"463'1169","reported_seq":1181,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.393783+0000","last_change":"2023-09-23T18:13:43.955654+0000","last_active":"2023-09-23T18:13:55.393783+0000","last_peered":"2023-09-23T18:13:55.393783+0000","last_clean":"2023-09-23T18:13:55.393783+0000","last_became_active":"2023-09-23T18:13:43.955354+0000","last_became_peered":"2023-09-23T18:13:43.955354+0000","last_unstale":"2023-09-23T18:13:55.393783+0000","last_undegraded":"2023-09-23T18:13:55.393783+0000","last_fullsized":"2023-09-23T18:13:55.393783+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1169,"log_dups_size":0,"ondisk_log_size":1169,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:18:51.727274+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76611584,"num_objects":1169,"num_object_clones":0,"num_object_copies":3507,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1169,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1169,"num_write_kb":74816,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1362,"reported_epoch":463,"state":"active+undersized+degraded","last_fresh":"2023-09-23T18:13:46.408827+0000","last_change":"2023-09-23T18:13:46.408573+0000","last_active":"2023-09-23T18:13:46.408827+0000","last_peered":"2023-09-23T18:13:46.408827+0000","last_clean":"2023-09-23T18:13:44.384833+0000","last_became_active":"2023-09-23T18:13:46.408573+0000","last_became_peered":"2023-09-23T18:13:46.408573+0000","last_unstale":"2023-09-23T18:13:46.408827+0000","last_undegraded":"2023-09-23T18:13:46.403033+0000","last_fullsized":"2023-09-23T18:13:46.402926+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":452,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:44.471165+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":14,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7],"acting":[5,7],"avail_no_missing":["5","7"],"object_location_counts":[{"shards":"5,7","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":819,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.405931+0000","last_change":"2023-09-23T18:06:48.957422+0000","last_active":"2023-09-23T18:13:46.405931+0000","last_peered":"2023-09-23T18:13:46.405931+0000","last_clean":"2023-09-23T18:13:46.405931+0000","last_became_active":"2023-09-23T18:06:48.957044+0000","last_became_peered":"2023-09-23T18:06:48.957044+0000","last_unstale":"2023-09-23T18:13:46.405931+0000","last_undegraded":"2023-09-23T18:13:46.405931+0000","last_fullsized":"2023-09-23T18:13:46.405931+0000","mapping_epoch":301,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":302,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:58.047861+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":232,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.401670+0000","last_change":"2023-09-23T18:12:33.310915+0000","last_active":"2023-09-23T18:13:45.401670+0000","last_peered":"2023-09-23T18:13:45.401670+0000","last_clean":"2023-09-23T18:13:45.401670+0000","last_became_active":"2023-09-23T18:12:33.310201+0000","last_became_peered":"2023-09-23T18:12:33.310201+0000","last_unstale":"2023-09-23T18:13:45.401670+0000","last_undegraded":"2023-09-23T18:13:45.401670+0000","last_fullsized":"2023-09-23T18:13:45.401670+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:42:37.485005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":417,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404646+0000","last_change":"2023-09-23T18:12:05.656003+0000","last_active":"2023-09-23T18:13:46.404646+0000","last_peered":"2023-09-23T18:13:46.404646+0000","last_clean":"2023-09-23T18:13:46.404646+0000","last_became_active":"2023-09-23T18:12:05.655726+0000","last_became_peered":"2023-09-23T18:12:05.655726+0000","last_unstale":"2023-09-23T18:13:46.404646+0000","last_undegraded":"2023-09-23T18:13:46.404646+0000","last_fullsized":"2023-09-23T18:13:46.404646+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:56:47.283544+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"464'1303","reported_seq":1314,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.713073+0000","last_change":"2023-09-23T18:13:43.381156+0000","last_active":"2023-09-23T18:13:55.713073+0000","last_peered":"2023-09-23T18:13:55.713073+0000","last_clean":"2023-09-23T18:13:55.713073+0000","last_became_active":"2023-09-23T18:13:43.380809+0000","last_became_peered":"2023-09-23T18:13:43.380809+0000","last_unstale":"2023-09-23T18:13:55.713073+0000","last_undegraded":"2023-09-23T18:13:55.713073+0000","last_fullsized":"2023-09-23T18:13:55.713073+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1303,"log_dups_size":0,"ondisk_log_size":1303,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:39:00.676180+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85393408,"num_objects":1303,"num_object_clones":0,"num_object_copies":3909,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1303,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1303,"num_write_kb":83392,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":646,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404254+0000","last_change":"2023-09-23T18:12:05.656585+0000","last_active":"2023-09-23T18:13:46.404254+0000","last_peered":"2023-09-23T18:13:46.404254+0000","last_clean":"2023-09-23T18:13:46.404254+0000","last_became_active":"2023-09-23T18:12:05.656067+0000","last_became_peered":"2023-09-23T18:12:05.656067+0000","last_unstale":"2023-09-23T18:13:46.404254+0000","last_undegraded":"2023-09-23T18:13:46.404254+0000","last_fullsized":"2023-09-23T18:13:46.404254+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:18.616905+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":727,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406176+0000","last_change":"2023-09-23T18:10:01.230791+0000","last_active":"2023-09-23T18:13:46.406176+0000","last_peered":"2023-09-23T18:13:46.406176+0000","last_clean":"2023-09-23T18:13:46.406176+0000","last_became_active":"2023-09-23T18:10:01.170909+0000","last_became_peered":"2023-09-23T18:10:01.170909+0000","last_unstale":"2023-09-23T18:13:46.406176+0000","last_undegraded":"2023-09-23T18:13:46.406176+0000","last_fullsized":"2023-09-23T18:13:46.406176+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:08.759640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":405,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.410661+0000","last_change":"2023-09-23T18:12:33.379732+0000","last_active":"2023-09-23T18:13:46.410661+0000","last_peered":"2023-09-23T18:13:46.410661+0000","last_clean":"2023-09-23T18:13:46.410661+0000","last_became_active":"2023-09-23T18:12:33.378977+0000","last_became_peered":"2023-09-23T18:12:33.378977+0000","last_unstale":"2023-09-23T18:13:46.410661+0000","last_undegraded":"2023-09-23T18:13:46.410661+0000","last_fullsized":"2023-09-23T18:13:46.410661+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:49:34.334366+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,5],"acting":[4,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":459,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404449+0000","last_change":"2023-09-23T18:12:32.324474+0000","last_active":"2023-09-23T18:13:46.404449+0000","last_peered":"2023-09-23T18:13:46.404449+0000","last_clean":"2023-09-23T18:13:46.404449+0000","last_became_active":"2023-09-23T18:12:32.324209+0000","last_became_peered":"2023-09-23T18:12:32.324209+0000","last_unstale":"2023-09-23T18:13:46.404449+0000","last_undegraded":"2023-09-23T18:13:46.404449+0000","last_fullsized":"2023-09-23T18:13:46.404449+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:17.579958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,5],"acting":[1,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"464'1555","reported_seq":1568,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T18:13:58.180592+0000","last_change":"2023-09-23T18:13:43.384807+0000","last_active":"2023-09-23T18:13:58.180592+0000","last_peered":"2023-09-23T18:13:58.180592+0000","last_clean":"2023-09-23T18:13:58.180592+0000","last_became_active":"2023-09-23T18:13:43.384531+0000","last_became_peered":"2023-09-23T18:13:43.384531+0000","last_unstale":"2023-09-23T18:13:58.180592+0000","last_undegraded":"2023-09-23T18:13:58.180592+0000","last_fullsized":"2023-09-23T18:13:58.180592+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1555,"log_dups_size":0,"ondisk_log_size":1555,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:12.269073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101908480,"num_objects":1555,"num_object_clones":0,"num_object_copies":4665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1555,"num_write_kb":99520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":699,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406419+0000","last_change":"2023-09-23T18:12:06.646074+0000","last_active":"2023-09-23T18:13:46.406419+0000","last_peered":"2023-09-23T18:13:46.406419+0000","last_clean":"2023-09-23T18:13:46.406419+0000","last_became_active":"2023-09-23T18:12:06.645836+0000","last_became_peered":"2023-09-23T18:12:06.645836+0000","last_unstale":"2023-09-23T18:13:46.406419+0000","last_undegraded":"2023-09-23T18:13:46.406419+0000","last_fullsized":"2023-09-23T18:13:46.406419+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:22:34.582299+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":755,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406893+0000","last_change":"2023-09-23T18:12:31.342399+0000","last_active":"2023-09-23T18:13:46.406893+0000","last_peered":"2023-09-23T18:13:46.406893+0000","last_clean":"2023-09-23T18:13:46.406893+0000","last_became_active":"2023-09-23T18:12:05.655437+0000","last_became_peered":"2023-09-23T18:12:05.655437+0000","last_unstale":"2023-09-23T18:13:46.406893+0000","last_undegraded":"2023-09-23T18:13:46.406893+0000","last_fullsized":"2023-09-23T18:13:46.406893+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:20.926785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":216,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.403514+0000","last_change":"2023-09-23T18:12:32.317796+0000","last_active":"2023-09-23T18:13:46.403514+0000","last_peered":"2023-09-23T18:13:46.403514+0000","last_clean":"2023-09-23T18:13:46.403514+0000","last_became_active":"2023-09-23T18:12:32.317479+0000","last_became_peered":"2023-09-23T18:12:32.317479+0000","last_unstale":"2023-09-23T18:13:46.403514+0000","last_undegraded":"2023-09-23T18:13:46.403514+0000","last_fullsized":"2023-09-23T18:13:46.403514+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:52.596216+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,6],"acting":[1,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":239,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.401580+0000","last_change":"2023-09-23T18:12:33.303060+0000","last_active":"2023-09-23T18:13:45.401580+0000","last_peered":"2023-09-23T18:13:45.401580+0000","last_clean":"2023-09-23T18:13:45.401580+0000","last_became_active":"2023-09-23T18:12:33.302734+0000","last_became_peered":"2023-09-23T18:12:33.302734+0000","last_unstale":"2023-09-23T18:13:45.401580+0000","last_undegraded":"2023-09-23T18:13:45.401580+0000","last_fullsized":"2023-09-23T18:13:45.401580+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:19:48.422153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,7],"acting":[5,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"9.d","version":"464'1622","reported_seq":1635,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T18:13:58.174147+0000","last_change":"2023-09-23T18:13:43.381584+0000","last_active":"2023-09-23T18:13:58.174147+0000","last_peered":"2023-09-23T18:13:58.174147+0000","last_clean":"2023-09-23T18:13:58.174147+0000","last_became_active":"2023-09-23T18:13:43.381228+0000","last_became_peered":"2023-09-23T18:13:43.381228+0000","last_unstale":"2023-09-23T18:13:58.174147+0000","last_undegraded":"2023-09-23T18:13:58.174147+0000","last_fullsized":"2023-09-23T18:13:58.174147+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1622,"log_dups_size":0,"ondisk_log_size":1622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:57.391056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":106299392,"num_objects":1622,"num_object_clones":0,"num_object_copies":4866,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1622,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1622,"num_write_kb":103808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":822,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.407197+0000","last_change":"2023-09-23T18:12:33.330181+0000","last_active":"2023-09-23T18:13:46.407197+0000","last_peered":"2023-09-23T18:13:46.407197+0000","last_clean":"2023-09-23T18:13:46.407197+0000","last_became_active":"2023-09-23T18:12:33.329678+0000","last_became_peered":"2023-09-23T18:12:33.329678+0000","last_unstale":"2023-09-23T18:13:46.407197+0000","last_undegraded":"2023-09-23T18:13:46.407197+0000","last_fullsized":"2023-09-23T18:13:46.407197+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:15:58.659729+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,5],"acting":[4,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":296,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.405179+0000","last_change":"2023-09-23T18:13:39.230634+0000","last_active":"2023-09-23T18:13:46.405179+0000","last_peered":"2023-09-23T18:13:46.405179+0000","last_clean":"2023-09-23T18:13:46.405179+0000","last_became_active":"2023-09-23T18:13:39.224509+0000","last_became_peered":"2023-09-23T18:13:39.224509+0000","last_unstale":"2023-09-23T18:13:46.405179+0000","last_undegraded":"2023-09-23T18:13:46.405179+0000","last_fullsized":"2023-09-23T18:13:46.405179+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.715343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":223,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404816+0000","last_change":"2023-09-23T18:12:05.654233+0000","last_active":"2023-09-23T18:13:46.404816+0000","last_peered":"2023-09-23T18:13:46.404816+0000","last_clean":"2023-09-23T18:13:46.404816+0000","last_became_active":"2023-09-23T18:12:05.653748+0000","last_became_peered":"2023-09-23T18:12:05.653748+0000","last_unstale":"2023-09-23T18:13:46.404816+0000","last_undegraded":"2023-09-23T18:13:46.404816+0000","last_fullsized":"2023-09-23T18:13:46.404816+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:47:48.819688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":285,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406021+0000","last_change":"2023-09-23T18:12:32.328282+0000","last_active":"2023-09-23T18:13:46.406021+0000","last_peered":"2023-09-23T18:13:46.406021+0000","last_clean":"2023-09-23T18:13:46.406021+0000","last_became_active":"2023-09-23T18:12:32.327937+0000","last_became_peered":"2023-09-23T18:12:32.327937+0000","last_unstale":"2023-09-23T18:13:46.406021+0000","last_undegraded":"2023-09-23T18:13:46.406021+0000","last_fullsized":"2023-09-23T18:13:46.406021+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:24.813277+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,5],"acting":[7,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"464'1297","reported_seq":1309,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.953455+0000","last_change":"2023-09-23T18:13:44.089836+0000","last_active":"2023-09-23T18:13:55.953455+0000","last_peered":"2023-09-23T18:13:55.953455+0000","last_clean":"2023-09-23T18:13:55.953455+0000","last_became_active":"2023-09-23T18:13:44.089506+0000","last_became_peered":"2023-09-23T18:13:44.089506+0000","last_unstale":"2023-09-23T18:13:55.953455+0000","last_undegraded":"2023-09-23T18:13:55.953455+0000","last_fullsized":"2023-09-23T18:13:55.953455+0000","mapping_epoch":459,"log_start":"0'0","ondisk_log_start":"0'0","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":1297,"log_dups_size":0,"ondisk_log_size":1297,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:41.847812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85000192,"num_objects":1297,"num_object_clones":0,"num_object_copies":3891,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1297,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1297,"num_write_kb":83008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":710,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.402515+0000","last_change":"2023-09-23T18:12:32.327790+0000","last_active":"2023-09-23T18:13:45.402515+0000","last_peered":"2023-09-23T18:13:45.402515+0000","last_clean":"2023-09-23T18:13:45.402515+0000","last_became_active":"2023-09-23T18:12:32.327436+0000","last_became_peered":"2023-09-23T18:12:32.327436+0000","last_unstale":"2023-09-23T18:13:45.402515+0000","last_undegraded":"2023-09-23T18:13:45.402515+0000","last_fullsized":"2023-09-23T18:13:45.402515+0000","mapping_epoch":420,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":421,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:31.945694+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":804,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.411069+0000","last_change":"2023-09-23T18:13:46.410696+0000","last_active":"2023-09-23T18:13:46.411069+0000","last_peered":"2023-09-23T18:13:46.411069+0000","last_clean":"2023-09-23T18:13:44.970750+0000","last_became_active":"2023-09-23T18:13:46.410696+0000","last_became_peered":"2023-09-23T18:13:46.410696+0000","last_unstale":"2023-09-23T18:13:46.411069+0000","last_undegraded":"2023-09-23T18:13:46.411069+0000","last_fullsized":"2023-09-23T18:13:46.405339+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":450,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:23:32.008897+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7],"acting":[6,7],"avail_no_missing":["6","7"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":516,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.404572+0000","last_change":"2023-09-23T18:12:06.646490+0000","last_active":"2023-09-23T18:13:46.404572+0000","last_peered":"2023-09-23T18:13:46.404572+0000","last_clean":"2023-09-23T18:13:46.404572+0000","last_became_active":"2023-09-23T18:12:06.645565+0000","last_became_peered":"2023-09-23T18:12:06.645565+0000","last_unstale":"2023-09-23T18:13:46.404572+0000","last_undegraded":"2023-09-23T18:13:46.404572+0000","last_fullsized":"2023-09-23T18:13:46.404572+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:54.061911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":267,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.410492+0000","last_change":"2023-09-23T18:12:33.379923+0000","last_active":"2023-09-23T18:13:46.410492+0000","last_peered":"2023-09-23T18:13:46.410492+0000","last_clean":"2023-09-23T18:13:46.410492+0000","last_became_active":"2023-09-23T18:12:33.379412+0000","last_became_peered":"2023-09-23T18:12:33.379412+0000","last_unstale":"2023-09-23T18:13:46.410492+0000","last_undegraded":"2023-09-23T18:13:46.410492+0000","last_fullsized":"2023-09-23T18:13:46.410492+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:37:53.669871+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":633,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.407472+0000","last_change":"2023-09-23T18:12:05.637738+0000","last_active":"2023-09-23T18:13:46.407472+0000","last_peered":"2023-09-23T18:13:46.407472+0000","last_clean":"2023-09-23T18:13:46.407472+0000","last_became_active":"2023-09-23T18:12:05.637397+0000","last_became_peered":"2023-09-23T18:12:05.637397+0000","last_unstale":"2023-09-23T18:13:46.407472+0000","last_undegraded":"2023-09-23T18:13:46.407472+0000","last_fullsized":"2023-09-23T18:13:46.407472+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:14.580819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:14.580819+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:39.955222+0000","scrub_duration":0.003313033,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":61,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.409705+0000","last_change":"2023-09-23T18:12:05.654572+0000","last_active":"2023-09-23T18:13:46.409705+0000","last_peered":"2023-09-23T18:13:46.409705+0000","last_clean":"2023-09-23T18:13:46.409705+0000","last_became_active":"2023-09-23T18:12:05.654026+0000","last_became_peered":"2023-09-23T18:12:05.654026+0000","last_unstale":"2023-09-23T18:13:46.409705+0000","last_undegraded":"2023-09-23T18:13:46.409705+0000","last_fullsized":"2023-09-23T18:13:46.409705+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:55.511372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":539,"reported_epoch":464,"state":"active+clean","last_fresh":"2023-09-23T18:13:55.466387+0000","last_change":"2023-09-23T18:12:33.321764+0000","last_active":"2023-09-23T18:13:55.466387+0000","last_peered":"2023-09-23T18:13:55.466387+0000","last_clean":"2023-09-23T18:13:55.466387+0000","last_became_active":"2023-09-23T18:12:33.321413+0000","last_became_peered":"2023-09-23T18:12:33.321413+0000","last_unstale":"2023-09-23T18:13:55.466387+0000","last_undegraded":"2023-09-23T18:13:55.466387+0000","last_fullsized":"2023-09-23T18:13:55.466387+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:19.453718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,5],"acting":[1,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":628,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.405899+0000","last_change":"2023-09-23T18:12:05.644542+0000","last_active":"2023-09-23T18:13:46.405899+0000","last_peered":"2023-09-23T18:13:46.405899+0000","last_clean":"2023-09-23T18:13:46.405899+0000","last_became_active":"2023-09-23T18:12:05.643810+0000","last_became_peered":"2023-09-23T18:12:05.643810+0000","last_unstale":"2023-09-23T18:13:46.405899+0000","last_undegraded":"2023-09-23T18:13:46.405899+0000","last_fullsized":"2023-09-23T18:13:46.405899+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.604423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.604423+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:15.854751+0000","scrub_duration":0.0014924669999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":251,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.402420+0000","last_change":"2023-09-23T18:12:33.386729+0000","last_active":"2023-09-23T18:13:45.402420+0000","last_peered":"2023-09-23T18:13:45.402420+0000","last_clean":"2023-09-23T18:13:45.402420+0000","last_became_active":"2023-09-23T18:12:33.386238+0000","last_became_peered":"2023-09-23T18:12:33.386238+0000","last_unstale":"2023-09-23T18:13:45.402420+0000","last_undegraded":"2023-09-23T18:13:45.402420+0000","last_fullsized":"2023-09-23T18:13:45.402420+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:18:15.285523+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":81,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.409872+0000","last_change":"2023-09-23T18:13:39.231091+0000","last_active":"2023-09-23T18:13:46.409872+0000","last_peered":"2023-09-23T18:13:46.409872+0000","last_clean":"2023-09-23T18:13:46.409872+0000","last_became_active":"2023-09-23T18:13:39.230787+0000","last_became_peered":"2023-09-23T18:13:39.230787+0000","last_unstale":"2023-09-23T18:13:46.409872+0000","last_undegraded":"2023-09-23T18:13:46.409872+0000","last_fullsized":"2023-09-23T18:13:46.409872+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:40.269418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":239,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.403966+0000","last_change":"2023-09-23T18:12:33.305790+0000","last_active":"2023-09-23T18:13:45.403966+0000","last_peered":"2023-09-23T18:13:45.403966+0000","last_clean":"2023-09-23T18:13:45.403966+0000","last_became_active":"2023-09-23T18:12:33.305535+0000","last_became_peered":"2023-09-23T18:12:33.305535+0000","last_unstale":"2023-09-23T18:13:45.403966+0000","last_undegraded":"2023-09-23T18:13:45.403966+0000","last_fullsized":"2023-09-23T18:13:45.403966+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:08:15.270191+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":81,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.407166+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:13:46.407166+0000","last_peered":"2023-09-23T18:13:46.407166+0000","last_clean":"2023-09-23T18:13:46.407166+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:13:46.407166+0000","last_undegraded":"2023-09-23T18:13:46.407166+0000","last_fullsized":"2023-09-23T18:13:46.407166+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":850,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.407952+0000","last_change":"2023-09-23T18:06:08.440953+0000","last_active":"2023-09-23T18:13:46.407952+0000","last_peered":"2023-09-23T18:13:46.407952+0000","last_clean":"2023-09-23T18:13:46.407952+0000","last_became_active":"2023-09-23T18:06:08.440604+0000","last_became_peered":"2023-09-23T18:06:08.440604+0000","last_unstale":"2023-09-23T18:13:46.407952+0000","last_undegraded":"2023-09-23T18:13:46.407952+0000","last_fullsized":"2023-09-23T18:13:46.407952+0000","mapping_epoch":288,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":289,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:53.274049+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":293,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.412250+0000","last_change":"2023-09-23T18:13:46.412010+0000","last_active":"2023-09-23T18:13:46.412250+0000","last_peered":"2023-09-23T18:13:46.412250+0000","last_clean":"2023-09-23T18:13:44.970756+0000","last_became_active":"2023-09-23T18:13:46.412010+0000","last_became_peered":"2023-09-23T18:13:46.412010+0000","last_unstale":"2023-09-23T18:13:46.412250+0000","last_undegraded":"2023-09-23T18:13:46.412250+0000","last_fullsized":"2023-09-23T18:13:46.406115+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:15:29.076032+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1],"acting":[6,1],"avail_no_missing":["6","1"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":274,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.408917+0000","last_change":"2023-09-23T18:13:46.408844+0000","last_active":"2023-09-23T18:13:46.408917+0000","last_peered":"2023-09-23T18:13:46.408917+0000","last_clean":"2023-09-23T18:13:44.384330+0000","last_became_active":"2023-09-23T18:13:46.408844+0000","last_became_peered":"2023-09-23T18:13:46.408844+0000","last_unstale":"2023-09-23T18:13:46.408917+0000","last_undegraded":"2023-09-23T18:13:46.408917+0000","last_fullsized":"2023-09-23T18:13:46.401528+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":442,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:02.937198+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0],"acting":[5,0],"avail_no_missing":["5","0"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":876,"reported_epoch":463,"state":"active+undersized","last_fresh":"2023-09-23T18:13:46.411776+0000","last_change":"2023-09-23T18:13:46.411504+0000","last_active":"2023-09-23T18:13:46.411776+0000","last_peered":"2023-09-23T18:13:46.411776+0000","last_clean":"2023-09-23T18:13:44.647900+0000","last_became_active":"2023-09-23T18:13:46.411504+0000","last_became_peered":"2023-09-23T18:13:46.411504+0000","last_unstale":"2023-09-23T18:13:46.411776+0000","last_undegraded":"2023-09-23T18:13:46.411776+0000","last_fullsized":"2023-09-23T18:13:46.403180+0000","mapping_epoch":462,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":445,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:17.635974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:17.635974+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:36.201594+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4],"acting":[1,4],"avail_no_missing":["1","4"],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":280,"reported_epoch":462,"state":"active+clean","last_fresh":"2023-09-23T18:13:45.403423+0000","last_change":"2023-09-23T18:12:33.308688+0000","last_active":"2023-09-23T18:13:45.403423+0000","last_peered":"2023-09-23T18:13:45.403423+0000","last_clean":"2023-09-23T18:13:45.403423+0000","last_became_active":"2023-09-23T18:12:33.308396+0000","last_became_peered":"2023-09-23T18:12:33.308396+0000","last_unstale":"2023-09-23T18:13:45.403423+0000","last_undegraded":"2023-09-23T18:13:45.403423+0000","last_fullsized":"2023-09-23T18:13:45.403423+0000","mapping_epoch":421,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":422,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:11.406407+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":554,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":478,"reported_epoch":463,"state":"active+clean","last_fresh":"2023-09-23T18:13:46.406433+0000","last_change":"2023-09-23T18:12:06.656925+0000","last_active":"2023-09-23T18:13:46.406433+0000","last_peered":"2023-09-23T18:13:46.406433+0000","last_clean":"2023-09-23T18:13:46.406433+0000","last_became_active":"2023-09-23T18:12:06.656605+0000","last_became_peered":"2023-09-23T18:12:06.656605+0000","last_unstale":"2023-09-23T18:13:46.406433+0000","last_undegraded":"2023-09-23T18:13:46.406433+0000","last_fullsized":"2023-09-23T18:13:46.406433+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:29.874912+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":90,"acting":90,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":76,"num_bytes_recovered":6368,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":54826,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":87,"acting":87,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":421601,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":78,"num_read_kb":146,"num_write":37,"num_write_kb":384,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1201174,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":86,"acting":86,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":1447952384,"num_objects":22094,"num_object_clones":0,"num_object_copies":66282,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3245,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22094,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22094,"num_write_kb":1414016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":4262002688,"data_stored":4262002688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":22094,"ondisk_log_size":22094,"up":46,"acting":46,"num_store_stats":7},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":224,"num_bytes_recovered":8388646,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":21,"acting":21,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001496,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1466384,"kb_used_data":1118600,"kb_used_omap":13,"kb_used_meta":347762,"kb_avail":92278768,"statfs":{"total":95995035648,"available":94493458432,"internally_reserved":0,"allocated":1145446400,"data_stored":1143907276,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":356108392},"hb_peers":[0,1,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":1.286,"15min":0.99299999999999999},"min":{"1min":0.23000000000000001,"5min":0.214,"15min":0.214},"max":{"1min":15.869999999999999,"5min":15.869999999999999,"15min":15.869999999999999},"last":2.7829999999999999},{"interface":"front","average":{"1min":2.0649999999999999,"5min":1.4670000000000001,"15min":1.1899999999999999},"min":{"1min":0.25800000000000001,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":18.995999999999999,"5min":18.995999999999999,"15min":18.995999999999999},"last":1.7569999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":1.3260000000000001,"5min":1.7150000000000001,"15min":1.2410000000000001},"min":{"1min":0.28699999999999998,"5min":0.25,"15min":0.24399999999999999},"max":{"1min":8.4930000000000003,"5min":9.5389999999999997,"15min":9.5389999999999997},"last":0.66200000000000003},{"interface":"front","average":{"1min":2.1549999999999998,"5min":1.732,"15min":1.3140000000000001},"min":{"1min":0.29599999999999999,"5min":0.25700000000000001,"15min":0.23400000000000001},"max":{"1min":15.805999999999999,"5min":15.805999999999999,"15min":15.805999999999999},"last":3.2549999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":1.633,"15min":1.2290000000000001},"min":{"1min":0.29899999999999999,"5min":0.28199999999999997,"15min":0.23300000000000001},"max":{"1min":8.4939999999999998,"5min":8.4939999999999998,"15min":8.4939999999999998},"last":0.38100000000000001},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.8560000000000001,"15min":1.294},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.4789999999999992,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":0.39700000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:13:38 2023","interfaces":[{"interface":"back","average":{"1min":2.6400000000000001,"5min":2.8290000000000002,"15min":1.8819999999999999},"min":{"1min":0.32300000000000001,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":19.125,"5min":45.752000000000002,"15min":45.752000000000002},"last":2.5579999999999998},{"interface":"front","average":{"1min":4.0119999999999996,"5min":2.0630000000000002,"15min":1.5029999999999999},"min":{"1min":0.33100000000000002,"5min":0.311,"15min":0.29399999999999998},"max":{"1min":30.832999999999998,"5min":30.832999999999998,"15min":30.832999999999998},"last":3.2109999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.157,"15min":1.0209999999999999},"min":{"1min":0.27400000000000002,"5min":0.251,"15min":0.245},"max":{"1min":5.1109999999999998,"5min":9.516,"15min":9.516},"last":2.7440000000000002},{"interface":"front","average":{"1min":1.3400000000000001,"5min":1.1839999999999999,"15min":0.83099999999999996},"min":{"1min":0.32500000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.4139999999999997,"5min":6.4249999999999998,"15min":6.4249999999999998},"last":3.169}]},{"osd":5,"last update":"Sat Sep 23 18:13:33 2023","interfaces":[{"interface":"back","average":{"1min":1.3080000000000001,"5min":1.3080000000000001,"15min":1.3080000000000001},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.484,"5min":4.484,"15min":4.484},"last":3.3380000000000001},{"interface":"front","average":{"1min":1.863,"5min":1.863,"15min":1.863},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":6.0970000000000004,"5min":6.0970000000000004,"15min":6.0970000000000004},"last":2.6459999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:13:49 2023","interfaces":[{"interface":"back","average":{"1min":2.258,"5min":1.492,"15min":1.0580000000000001},"min":{"1min":0.34100000000000003,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":15.847,"5min":15.847,"15min":15.847},"last":3.3039999999999998},{"interface":"front","average":{"1min":2.4750000000000001,"5min":1.3069999999999999,"15min":1.1419999999999999},"min":{"1min":0.34300000000000003,"5min":0.253,"15min":0.253},"max":{"1min":16.975999999999999,"5min":16.975999999999999,"15min":16.975999999999999},"last":2.6160000000000001}]}]},{"osd":6,"up_from":163,"seq":700079669432,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":723328,"kb_used_data":303096,"kb_used_omap":7,"kb_used_meta":420216,"kb_avail":93021824,"statfs":{"total":95995035648,"available":95254347776,"internally_reserved":0,"allocated":310370304,"data_stored":308839432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":430301988},"hb_peers":[0,1,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:12:59 2023","interfaces":[{"interface":"back","average":{"1min":1.456,"5min":1.534,"15min":1.22},"min":{"1min":0.22700000000000001,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":4.0670000000000002,"5min":10.801,"15min":15.407},"last":0.91500000000000004},{"interface":"front","average":{"1min":1.605,"5min":1.611,"15min":1.5049999999999999},"min":{"1min":0.32200000000000001,"5min":0.25600000000000001,"15min":0.23100000000000001},"max":{"1min":6.0460000000000003,"5min":9.6329999999999991,"15min":15.147},"last":3.7000000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:12:59 2023","interfaces":[{"interface":"back","average":{"1min":1.722,"5min":1.6739999999999999,"15min":1.462},"min":{"1min":0.248,"5min":0.218,"15min":0.218},"max":{"1min":5.3099999999999996,"5min":21.390000000000001,"15min":21.390000000000001},"last":0.94699999999999995},{"interface":"front","average":{"1min":1.915,"5min":2.1230000000000002,"15min":1.6830000000000001},"min":{"1min":0.32500000000000001,"5min":0.26300000000000001,"15min":0.248},"max":{"1min":5.9219999999999997,"5min":21.559999999999999,"15min":21.559999999999999},"last":3.7759999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:12:59 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.6910000000000001,"15min":1.6259999999999999},"min":{"1min":0.42699999999999999,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":5.9880000000000004,"5min":9.4130000000000003,"15min":11.210000000000001},"last":0.46800000000000003},{"interface":"front","average":{"1min":1.625,"5min":1.6180000000000001,"15min":1.6699999999999999},"min":{"1min":0.41599999999999998,"5min":0.248,"15min":0.245},"max":{"1min":8.548,"5min":8.548,"15min":11.223000000000001},"last":0.57599999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:13:40 2023","interfaces":[{"interface":"back","average":{"1min":1.552,"5min":1.827,"15min":1.371},"min":{"1min":0.36899999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":6.1189999999999998,"5min":10.257999999999999,"15min":10.257999999999999},"last":0.97399999999999998},{"interface":"front","average":{"1min":1.198,"5min":1.915,"15min":1.496},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.31900000000000001},"max":{"1min":7.5140000000000002,"5min":9.5960000000000001,"15min":9.5960000000000001},"last":3.746}]},{"osd":4,"last update":"Sat Sep 23 18:12:59 2023","interfaces":[{"interface":"back","average":{"1min":1.6419999999999999,"5min":1.1699999999999999,"15min":0.996},"min":{"1min":0.254,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":5.8929999999999998,"5min":5.8929999999999998,"15min":9.9109999999999996},"last":1.1100000000000001},{"interface":"front","average":{"1min":1.2190000000000001,"5min":1.0860000000000001,"15min":0.89500000000000002},"min":{"1min":0.28199999999999997,"5min":0.27000000000000002,"15min":0.24099999999999999},"max":{"1min":4.0060000000000002,"5min":6.2949999999999999,"15min":8.3000000000000007},"last":0.996}]},{"osd":5,"last update":"Sat Sep 23 18:13:32 2023","interfaces":[{"interface":"back","average":{"1min":1.9550000000000001,"5min":1.9550000000000001,"15min":1.9550000000000001},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.7199999999999998,"5min":7.7199999999999998,"15min":7.7199999999999998},"last":1.1850000000000001},{"interface":"front","average":{"1min":2.5150000000000001,"5min":2.5150000000000001,"15min":2.5150000000000001},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":8.8030000000000008,"5min":8.8030000000000008,"15min":8.8030000000000008},"last":1.2230000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.4970000000000001,"15min":1.1060000000000001},"min":{"1min":0.35999999999999999,"5min":0.28000000000000003,"15min":0.26800000000000002},"max":{"1min":5.532,"5min":13.460000000000001,"15min":13.460000000000001},"last":4.1059999999999999},{"interface":"front","average":{"1min":1.423,"5min":1.4379999999999999,"15min":1.0629999999999999},"min":{"1min":0.36799999999999999,"5min":0.318,"15min":0.28499999999999998},"max":{"1min":6.1630000000000003,"5min":12.404,"15min":12.404},"last":1.0109999999999999}]}]},{"osd":1,"up_from":98,"seq":420906795250,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1892172,"kb_used_data":1025476,"kb_used_omap":7,"kb_used_meta":866680,"kb_avail":91852980,"statfs":{"total":95995035648,"available":94057451520,"internally_reserved":0,"allocated":1050087424,"data_stored":1047913695,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":887481021},"hb_peers":[0,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:25 2023","interfaces":[{"interface":"back","average":{"1min":1.042,"5min":1.6899999999999999,"15min":1.3220000000000001},"min":{"1min":0.22500000000000001,"5min":0.184,"15min":0.17699999999999999},"max":{"1min":8.0749999999999993,"5min":14.567,"15min":15.529999999999999},"last":0.23599999999999999},{"interface":"front","average":{"1min":2.335,"5min":2.2480000000000002,"15min":1.909},"min":{"1min":0.26200000000000001,"5min":0.20499999999999999,"15min":0.20100000000000001},"max":{"1min":20.257000000000001,"5min":20.257000000000001,"15min":82.244},"last":0.39600000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:13:00 2023","interfaces":[{"interface":"back","average":{"1min":3.165,"5min":2.2410000000000001,"15min":1.8180000000000001},"min":{"1min":0.29699999999999999,"5min":0.22600000000000001,"15min":0.22},"max":{"1min":23.992999999999999,"5min":23.992999999999999,"15min":23.992999999999999},"last":0.30199999999999999},{"interface":"front","average":{"1min":2.4119999999999999,"5min":1.954,"15min":1.4710000000000001},"min":{"1min":0.28899999999999998,"5min":0.22,"15min":0.21299999999999999},"max":{"1min":10.526999999999999,"5min":10.526999999999999,"15min":10.526999999999999},"last":0.38600000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:13:41 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":1.792,"15min":1.298},"min":{"1min":0.32000000000000001,"5min":0.247,"15min":0.247},"max":{"1min":9.1859999999999999,"5min":11.519,"15min":11.519},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.4099999999999999,"5min":2.0390000000000001,"15min":1.4650000000000001},"min":{"1min":0.33000000000000002,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":6.7889999999999997,"5min":15.446999999999999,"15min":15.446999999999999},"last":0.443}]},{"osd":4,"last update":"Sat Sep 23 18:13:25 2023","interfaces":[{"interface":"back","average":{"1min":2.4449999999999998,"5min":2.2040000000000002,"15min":1.6379999999999999},"min":{"1min":0.35199999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":16.395,"5min":28.446999999999999,"15min":28.446999999999999},"last":1.8160000000000001},{"interface":"front","average":{"1min":2.3780000000000001,"5min":2.0699999999999998,"15min":1.5089999999999999},"min":{"1min":0.318,"5min":0.29999999999999999,"15min":0.27000000000000002},"max":{"1min":9.6530000000000005,"5min":28.434000000000001,"15min":28.434000000000001},"last":1.5880000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:13:31 2023","interfaces":[{"interface":"back","average":{"1min":2.6480000000000001,"5min":2.6480000000000001,"15min":2.6480000000000001},"min":{"1min":0.46899999999999997,"5min":0.46899999999999997,"15min":0.46899999999999997},"max":{"1min":16.085000000000001,"5min":16.085000000000001,"15min":16.085000000000001},"last":0.65400000000000003},{"interface":"front","average":{"1min":2.4169999999999998,"5min":2.4169999999999998,"15min":2.4169999999999998},"min":{"1min":0.35899999999999999,"5min":0.35899999999999999,"15min":0.35899999999999999},"max":{"1min":21.812000000000001,"5min":21.812000000000001,"15min":21.812000000000001},"last":1.8939999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:13:00 2023","interfaces":[{"interface":"back","average":{"1min":3.202,"5min":2.3250000000000002,"15min":1.9970000000000001},"min":{"1min":0.46500000000000002,"5min":0.307,"15min":0.307},"max":{"1min":16.428000000000001,"5min":27.516999999999999,"15min":89.566000000000003},"last":1.708},{"interface":"front","average":{"1min":3.1160000000000001,"5min":2.27,"15min":1.9279999999999999},"min":{"1min":0.504,"5min":0.373,"15min":0.34699999999999998},"max":{"1min":15.151,"5min":27.501999999999999,"15min":90.608999999999995},"last":0.70199999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:13:55 2023","interfaces":[{"interface":"back","average":{"1min":1.054,"5min":1.796,"15min":1.2390000000000001},"min":{"1min":0.40799999999999997,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":3.6930000000000001,"5min":27.913,"15min":27.913},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.97799999999999998,"5min":2.3300000000000001,"15min":1.845},"min":{"1min":0.47199999999999998,"5min":0.371,"15min":0.371},"max":{"1min":2.6930000000000001,"5min":27.620999999999999,"15min":27.620999999999999},"last":1.198}]}]},{"osd":0,"up_from":93,"seq":399431958777,"num_pgs":20,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":777020,"kb_used_data":197428,"kb_used_omap":7,"kb_used_meta":579576,"kb_avail":92968132,"statfs":{"total":95995035648,"available":95199367168,"internally_reserved":0,"allocated":202166272,"data_stored":199997992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":593486564},"hb_peers":[1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:13:33 2023","interfaces":[{"interface":"back","average":{"1min":0.69899999999999995,"5min":1.6679999999999999,"15min":1.774},"min":{"1min":0.20699999999999999,"5min":0.192,"15min":0.18099999999999999},"max":{"1min":2.585,"5min":13.677,"15min":18.154},"last":0.82899999999999996},{"interface":"front","average":{"1min":1.095,"5min":1.792,"15min":1.6419999999999999},"min":{"1min":0.28699999999999998,"5min":0.246,"15min":0.17399999999999999},"max":{"1min":6.2329999999999997,"5min":14.207000000000001,"15min":14.468999999999999},"last":0.91500000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.2909999999999999,"15min":1.7090000000000001},"min":{"1min":0.24299999999999999,"5min":0.222,"15min":0.20499999999999999},"max":{"1min":3.903,"5min":9.2859999999999996,"15min":20.882000000000001},"last":0.42499999999999999},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.579,"15min":1.6890000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":6.2750000000000004,"5min":10.84,"15min":15.064},"last":0.47399999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:13:39 2023","interfaces":[{"interface":"back","average":{"1min":0.93100000000000005,"5min":1.8360000000000001,"15min":1.675},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.20499999999999999},"max":{"1min":4.8970000000000002,"5min":11.015000000000001,"15min":12.779999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.1739999999999999,"5min":1.9059999999999999,"15min":1.579},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":7.976,"5min":9.2110000000000003,"15min":15.198},"last":0.35399999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:12:51 2023","interfaces":[{"interface":"back","average":{"1min":2.5739999999999998,"5min":1.74,"15min":1.611},"min":{"1min":0.35199999999999998,"5min":0.32800000000000001,"15min":0.251},"max":{"1min":9.0709999999999997,"5min":9.0709999999999997,"15min":15.279999999999999},"last":1.0129999999999999},{"interface":"front","average":{"1min":1.7989999999999999,"5min":1.3859999999999999,"15min":1.6100000000000001},"min":{"1min":0.41899999999999998,"5min":0.31,"15min":0.247},"max":{"1min":5.0229999999999997,"5min":9.2520000000000007,"15min":18.106999999999999},"last":0.873}]},{"osd":5,"last update":"Sat Sep 23 18:13:33 2023","interfaces":[{"interface":"back","average":{"1min":0.98099999999999998,"5min":0.98099999999999998,"15min":0.98099999999999998},"min":{"1min":0.35099999999999998,"5min":0.35099999999999998,"15min":0.35099999999999998},"max":{"1min":3.3239999999999998,"5min":3.3239999999999998,"15min":3.3239999999999998},"last":1.3819999999999999},{"interface":"front","average":{"1min":1.028,"5min":1.028,"15min":1.028},"min":{"1min":0.35499999999999998,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":2.6739999999999999,"5min":2.6739999999999999,"15min":2.6739999999999999},"last":1.044}]},{"osd":6,"last update":"Sat Sep 23 18:12:51 2023","interfaces":[{"interface":"back","average":{"1min":2.5939999999999999,"5min":1.8839999999999999,"15min":1.9159999999999999},"min":{"1min":0.41299999999999998,"5min":0.34899999999999998,"15min":0.34599999999999997},"max":{"1min":9.1170000000000009,"5min":9.1170000000000009,"15min":58.457999999999998},"last":0.70099999999999996},{"interface":"front","average":{"1min":1.579,"5min":1.4059999999999999,"15min":1.778},"min":{"1min":0.40600000000000003,"5min":0.315,"15min":0.311},"max":{"1min":6.3049999999999997,"5min":7.0599999999999996,"15min":58.354999999999997},"last":1.3240000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:13:44 2023","interfaces":[{"interface":"back","average":{"1min":1.252,"5min":1.4490000000000001,"15min":1.242},"min":{"1min":0.38800000000000001,"5min":0.36599999999999999,"15min":0.34499999999999997},"max":{"1min":5.1710000000000003,"5min":7.4790000000000001,"15min":10.416},"last":1.2170000000000001},{"interface":"front","average":{"1min":1.3109999999999999,"5min":1.909,"15min":1.5529999999999999},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.36399999999999999},"max":{"1min":7.1109999999999998,"5min":9.1489999999999991,"15min":9.1489999999999991},"last":1.1599999999999999}]}]},{"osd":2,"up_from":103,"seq":442381631722,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":908604,"kb_used_data":210548,"kb_used_omap":11,"kb_used_meta":698036,"kb_avail":92836548,"statfs":{"total":95995035648,"available":95064625152,"internally_reserved":0,"allocated":215601152,"data_stored":213461087,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11913,"internal_metadata":714789239},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.1120000000000001,"5min":1.585,"15min":1.9930000000000001},"min":{"1min":0.32900000000000001,"5min":0.19900000000000001,"15min":0.17199999999999999},"max":{"1min":3.7759999999999998,"5min":11.946,"15min":36.656999999999996},"last":0.29799999999999999},{"interface":"front","average":{"1min":1.246,"5min":1.6639999999999999,"15min":1.8759999999999999},"min":{"1min":0.27600000000000002,"5min":0.24099999999999999,"15min":0.215},"max":{"1min":3.4889999999999999,"5min":11.978,"15min":36.697000000000003},"last":0.48199999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.123,"5min":1.6799999999999999,"15min":1.863},"min":{"1min":0.32600000000000001,"5min":0.23599999999999999,"15min":0.215},"max":{"1min":3.2200000000000002,"5min":12.106,"15min":20.995000000000001},"last":0.373},{"interface":"front","average":{"1min":1.1950000000000001,"5min":1.7949999999999999,"15min":1.6559999999999999},"min":{"1min":0.252,"5min":0.23899999999999999,"15min":0.193},"max":{"1min":4.2039999999999997,"5min":8.8409999999999993,"15min":15.945},"last":0.44700000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:13:42 2023","interfaces":[{"interface":"back","average":{"1min":1.075,"5min":1.9079999999999999,"15min":1.5169999999999999},"min":{"1min":0.26600000000000001,"5min":0.24199999999999999,"15min":0.24199999999999999},"max":{"1min":9.1170000000000009,"5min":9.1170000000000009,"15min":37.500999999999998},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.997,"5min":1.665,"15min":1.4359999999999999},"min":{"1min":0.318,"5min":0.28999999999999998,"15min":0.24199999999999999},"max":{"1min":3.5329999999999999,"5min":9.1579999999999995,"15min":34.915999999999997},"last":0.53600000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.9319999999999999,"5min":1.625,"15min":1.5860000000000001},"min":{"1min":0.46600000000000003,"5min":0.31900000000000001,"15min":0.26500000000000001},"max":{"1min":4.9779999999999998,"5min":7.3540000000000001,"15min":13.016999999999999},"last":0.501},{"interface":"front","average":{"1min":1.3680000000000001,"5min":1.47,"15min":1.522},"min":{"1min":0.33500000000000002,"5min":0.28699999999999998,"15min":0.26700000000000002},"max":{"1min":4.2009999999999996,"5min":12.084,"15min":20.920000000000002},"last":0.59699999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:13:32 2023","interfaces":[{"interface":"back","average":{"1min":1.038,"5min":1.038,"15min":1.038},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":6.8120000000000003,"5min":6.8120000000000003,"15min":6.8120000000000003},"last":0.51300000000000001},{"interface":"front","average":{"1min":1.7310000000000001,"5min":1.7310000000000001,"15min":1.7310000000000001},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":9.1760000000000002,"5min":9.1760000000000002,"15min":9.1760000000000002},"last":0.61299999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:12:56 2023","interfaces":[{"interface":"back","average":{"1min":1.6659999999999999,"5min":1.552,"15min":1.659},"min":{"1min":0.34599999999999997,"5min":0.30499999999999999,"15min":0.29699999999999999},"max":{"1min":4.4960000000000004,"5min":7.391,"15min":13.065},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.869,"15min":1.8360000000000001},"min":{"1min":0.58099999999999996,"5min":0.36899999999999999,"15min":0.33900000000000002},"max":{"1min":4.2990000000000004,"5min":12.032999999999999,"15min":20.893000000000001},"last":0.55200000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:13:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.915,"15min":1.556},"min":{"1min":0.40999999999999998,"5min":0.379,"15min":0.379},"max":{"1min":9.1999999999999993,"5min":9.1999999999999993,"15min":12.051},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.7230000000000001,"5min":1.8620000000000001,"15min":1.3640000000000001},"min":{"1min":0.41199999999999998,"5min":0.32800000000000001,"15min":0.309},"max":{"1min":6.4489999999999998,"5min":12.058,"15min":12.058},"last":0.63900000000000001}]}]},{"osd":3,"up_from":237,"seq":1017907249292,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":880852,"kb_used_data":376268,"kb_used_omap":3,"kb_used_meta":504572,"kb_avail":92864300,"statfs":{"total":95995035648,"available":95093043200,"internally_reserved":0,"allocated":385298432,"data_stored":383002261,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":516682472},"hb_peers":[0,1,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1],"upper_bound":2},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:37 2023","interfaces":[{"interface":"back","average":{"1min":0.755,"5min":1.169,"15min":1},"min":{"1min":0.26800000000000002,"5min":0.183,"15min":0.183},"max":{"1min":5.5270000000000001,"5min":9.7089999999999996,"15min":9.7089999999999996},"last":1.1830000000000001},{"interface":"front","average":{"1min":0.85699999999999998,"5min":1.5620000000000001,"15min":1.3500000000000001},"min":{"1min":0.253,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":3.508,"5min":9.1999999999999993,"15min":14.885999999999999},"last":0.96399999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:13:37 2023","interfaces":[{"interface":"back","average":{"1min":0.91200000000000003,"5min":2.1360000000000001,"15min":1.587},"min":{"1min":0.23999999999999999,"5min":0.215,"15min":0.215},"max":{"1min":3.5379999999999998,"5min":16.056000000000001,"15min":21.931999999999999},"last":0.27000000000000002},{"interface":"front","average":{"1min":1.5389999999999999,"5min":2.226,"15min":1.4419999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":7.3630000000000004,"5min":16.587,"15min":16.587},"last":1.167}]},{"osd":2,"last update":"Sat Sep 23 18:13:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":1.8,"15min":1.214},"min":{"1min":0.372,"5min":0.26600000000000001,"15min":0.22600000000000001},"max":{"1min":33.976999999999997,"5min":33.976999999999997,"15min":33.976999999999997},"last":0.30299999999999999},{"interface":"front","average":{"1min":2.6779999999999999,"5min":1.9750000000000001,"15min":1.5800000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.25800000000000001},"max":{"1min":34.006999999999998,"5min":34.006999999999998,"15min":34.006999999999998},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:13:37 2023","interfaces":[{"interface":"back","average":{"1min":1.399,"5min":1.484,"15min":1.4470000000000001},"min":{"1min":0.29999999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.923,"5min":10.763,"15min":37.231000000000002},"last":0.33200000000000002},{"interface":"front","average":{"1min":1.627,"5min":1.827,"15min":1.5349999999999999},"min":{"1min":0.379,"5min":0.32100000000000001,"15min":0.30499999999999999},"max":{"1min":5.165,"5min":15.890000000000001,"15min":37.381},"last":1.018}]},{"osd":5,"last update":"Sat Sep 23 18:13:32 2023","interfaces":[{"interface":"back","average":{"1min":1.2669999999999999,"5min":1.2669999999999999,"15min":1.2669999999999999},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":4.2450000000000001,"5min":4.2450000000000001,"15min":4.2450000000000001},"last":1.119},{"interface":"front","average":{"1min":1.7070000000000001,"5min":1.7070000000000001,"15min":1.7070000000000001},"min":{"1min":0.39400000000000002,"5min":0.39400000000000002,"15min":0.39400000000000002},"max":{"1min":8.9969999999999999,"5min":8.9969999999999999,"15min":8.9969999999999999},"last":0.98399999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:13:37 2023","interfaces":[{"interface":"back","average":{"1min":1.494,"5min":1.4359999999999999,"15min":1.5960000000000001},"min":{"1min":0.41199999999999998,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":9.0280000000000005,"5min":9.8149999999999995,"15min":115.54000000000001},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.663,"5min":1.4279999999999999,"15min":1.843},"min":{"1min":0.32900000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.109,"5min":7.6600000000000001,"15min":115.691},"last":0.45500000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:13:37 2023","interfaces":[{"interface":"back","average":{"1min":2.3250000000000002,"5min":2.1080000000000001,"15min":1.6819999999999999},"min":{"1min":0.39600000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":9.9760000000000009,"5min":15.972,"15min":15.972},"last":1.0609999999999999},{"interface":"front","average":{"1min":2.1480000000000001,"5min":1.77,"15min":1.5529999999999999},"min":{"1min":0.42899999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":9.9109999999999996,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":1.577}]}]},{"osd":4,"up_from":30,"seq":128849019241,"num_pgs":57,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":761224,"kb_used_data":294528,"kb_used_omap":0,"kb_used_meta":466688,"kb_avail":92983928,"statfs":{"total":95995035648,"available":95215542272,"internally_reserved":0,"allocated":301596672,"data_stored":300057988,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":477888512},"hb_peers":[0,1,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:52 2023","interfaces":[{"interface":"back","average":{"1min":1.319,"5min":1.1040000000000001,"15min":1.1919999999999999},"min":{"1min":0.249,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":7.3239999999999998,"5min":9.2759999999999998,"15min":14.164},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.393,"5min":1.48,"15min":1.383},"min":{"1min":0.26900000000000002,"5min":0.23000000000000001,"15min":0.22900000000000001},"max":{"1min":7.9569999999999999,"5min":8.3490000000000002,"15min":9.3789999999999996},"last":0.33700000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:13:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2130000000000001,"5min":2.2330000000000001,"15min":1.639},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.7119999999999997,"5min":42.929000000000002,"15min":42.929000000000002},"last":1.0660000000000001},{"interface":"front","average":{"1min":1.8360000000000001,"5min":1.9870000000000001,"15min":1.514},"min":{"1min":0.307,"5min":0.255,"15min":0.23100000000000001},"max":{"1min":7.3520000000000003,"5min":43.494999999999997,"15min":43.494999999999997},"last":0.54900000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:12:55 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":2.1150000000000002,"15min":1.6200000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":8.5530000000000008,"5min":32.887999999999998,"15min":32.887999999999998},"last":1.99},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.7849999999999999,"15min":1.536},"min":{"1min":0.34200000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.9199999999999999,"5min":29.103999999999999,"15min":29.103999999999999},"last":2.0609999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:13:33 2023","interfaces":[{"interface":"back","average":{"1min":1.95,"5min":2.327,"15min":1.8300000000000001},"min":{"1min":0.39400000000000002,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":7.3840000000000003,"5min":28.754999999999999,"15min":29.154},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.1230000000000002,"5min":2.1059999999999999,"15min":1.7589999999999999},"min":{"1min":0.35699999999999998,"5min":0.309,"15min":0.309},"max":{"1min":7.8460000000000001,"5min":29.213000000000001,"15min":29.216000000000001},"last":0.371}]},{"osd":5,"last update":"Sat Sep 23 18:13:33 2023","interfaces":[{"interface":"back","average":{"1min":3.1709999999999998,"5min":3.1709999999999998,"15min":3.1709999999999998},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":25.704999999999998,"5min":25.704999999999998,"15min":25.704999999999998},"last":1.7909999999999999},{"interface":"front","average":{"1min":3.347,"5min":3.347,"15min":3.347},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":32.640000000000001,"5min":32.640000000000001,"15min":32.640000000000001},"last":2.0179999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:12:55 2023","interfaces":[{"interface":"back","average":{"1min":1.3320000000000001,"5min":1.006,"15min":0.96399999999999997},"min":{"1min":0.30499999999999999,"5min":0.25,"15min":0.23599999999999999},"max":{"1min":5.2839999999999998,"5min":5.8310000000000004,"15min":9.0990000000000002},"last":1.016},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.302,"15min":1.0880000000000001},"min":{"1min":0.34799999999999998,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":6.6669999999999998,"5min":10.413,"15min":10.725},"last":1.0980000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:13:52 2023","interfaces":[{"interface":"back","average":{"1min":1.956,"5min":1.3400000000000001,"15min":1.145},"min":{"1min":0.39900000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.0090000000000003,"5min":9.2509999999999994,"15min":13.298999999999999},"last":1.6100000000000001},{"interface":"front","average":{"1min":1.885,"5min":1.409,"15min":1.2589999999999999},"min":{"1min":0.42199999999999999,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":7.9850000000000003,"5min":10.535,"15min":10.535},"last":1.6459999999999999}]}]},{"osd":5,"up_from":420,"seq":1803886264338,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":909304,"kb_used_data":770800,"kb_used_omap":9,"kb_used_meta":138486,"kb_avail":92835848,"statfs":{"total":95995035648,"available":95063908352,"internally_reserved":0,"allocated":789299200,"data_stored":787772330,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":141810109},"hb_peers":[0,1,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.9369999999999998,"5min":2.9369999999999998,"15min":2.9369999999999998},"last":0.22800000000000001},{"interface":"front","average":{"1min":1.5669999999999999,"5min":1.5669999999999999,"15min":1.5669999999999999},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":4.9219999999999997,"5min":4.9219999999999997,"15min":4.9219999999999997},"last":0.40000000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:13:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7150000000000001,"5min":1.7150000000000001,"15min":1.7150000000000001},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":5.0469999999999997,"5min":5.0469999999999997,"15min":5.0469999999999997},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.98799999999999999,"5min":0.98799999999999999,"15min":0.98799999999999999},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":3.7429999999999999,"5min":3.7429999999999999,"15min":3.7429999999999999},"last":3.198}]},{"osd":2,"last update":"Sat Sep 23 18:13:36 2023","interfaces":[{"interface":"back","average":{"1min":1.7250000000000001,"5min":1.7250000000000001,"15min":1.7250000000000001},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.4720000000000004,"5min":5.4720000000000004,"15min":5.4720000000000004},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.026,"5min":1.026,"15min":1.026},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.9790000000000001,"5min":3.9790000000000001,"15min":3.9790000000000001},"last":0.55000000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.95199999999999996,"5min":0.95199999999999996,"15min":0.95199999999999996},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":3.1459999999999999,"5min":3.1459999999999999,"15min":3.1459999999999999},"last":1.0589999999999999},{"interface":"front","average":{"1min":1.5089999999999999,"5min":1.5089999999999999,"15min":1.5089999999999999},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":4.8479999999999999,"5min":4.8479999999999999,"15min":4.8479999999999999},"last":1.0069999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:13:36 2023","interfaces":[{"interface":"back","average":{"1min":1.9850000000000001,"5min":1.9850000000000001,"15min":1.9850000000000001},"min":{"1min":0.251,"5min":0.251,"15min":0.251},"max":{"1min":6.3410000000000002,"5min":6.3410000000000002,"15min":6.3410000000000002},"last":1.304},{"interface":"front","average":{"1min":1.3819999999999999,"5min":1.3819999999999999,"15min":1.3819999999999999},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":8.9320000000000004,"5min":8.9320000000000004,"15min":8.9320000000000004},"last":1.03}]},{"osd":6,"last update":"Sat Sep 23 18:13:36 2023","interfaces":[{"interface":"back","average":{"1min":0.85199999999999998,"5min":0.85199999999999998,"15min":0.85199999999999998},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":2.2530000000000001,"5min":2.2530000000000001,"15min":2.2530000000000001},"last":1.085},{"interface":"front","average":{"1min":1.431,"5min":1.431,"15min":1.431},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":5.0979999999999999,"5min":5.0979999999999999,"15min":5.0979999999999999},"last":0.36199999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:13:36 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.3640000000000001,"15min":1.3640000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.307},"max":{"1min":4.407,"5min":4.407,"15min":4.407},"last":1.27},{"interface":"front","average":{"1min":1.288,"5min":1.288,"15min":1.288},"min":{"1min":0.34599999999999997,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":5.9119999999999999,"5min":5.9119999999999999,"15min":5.9119999999999999},"last":1.107}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7415,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":219742208,"data_stored":219742208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1129512960,"data_stored":1129512960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":396754944,"data_stored":396754944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":329908224,"data_stored":329908224,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":703594496,"data_stored":703594496,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":334888960,"data_stored":334888960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":1147600896,"data_stored":1147600896,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:14:00.070 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-09-23T18:14:00.138 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:00.237 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:00.357 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:00.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:00.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:00.469 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:00.535 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:00 smithi099 ceph-mon[167076]: pgmap v1124: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 3253/66366 objects degraded (4.902%) 2023-09-23T18:14:00.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:00 smithi099 ceph-mon[167076]: from='client.36570 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:14:00.598 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 12 16 30100 30084 156.631 114.375 0.00432005 0.00637858 2023-09-23T18:14:00.628 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:14:00.661 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:14:00.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:14:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:14:00] "GET /metrics HTTP/1.1" 200 36618 "" "Prometheus/2.43.0" 2023-09-23T18:14:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:00 smithi055 ceph-mon[184174]: pgmap v1124: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 3253/66366 objects degraded (4.902%) 2023-09-23T18:14:00.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:00 smithi055 ceph-mon[184174]: from='client.36570 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:14:00.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:00 smithi055 ceph-mon[179668]: pgmap v1124: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.4 GiB data, 7.9 GiB used, 707 GiB / 715 GiB avail; 138 MiB/s wr, 2.21k op/s; 3253/66366 objects degraded (4.902%) 2023-09-23T18:14:00.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:00 smithi055 ceph-mon[179668]: from='client.36570 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:14:00.687 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:00.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:00.693 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:00.697 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:14:00.735 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:14:00.768 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:14:00.793 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:00.802 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:14:00.843 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:14:00.843 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:14:00.843 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:14:00.843 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T18:14:00.844 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:14:00.844 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:14:00.844 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:14:00.844 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:14:00.844 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:14:00.853 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:00.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:00.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:00.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:01.021 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:01.070 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:01.174 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:01.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:01.180 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:01.281 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:01.347 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:01.399 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:14:01.399 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:14:01.399 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177009 8.0 8.46508c20 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.295203+0000", 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 90.102015734000005, 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99775324600000004, 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.400 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177009 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295203+0000", 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295203+0000", 2023-09-23T18:14:01.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295201+0000", 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295216+0000", 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4637999999999999e-05 2023-09-23T18:14:01.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295229+0000", 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2396e-05 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295237+0000", 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0390000000000005e-06 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.300237+0000", 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0050006570000000004 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.300247+0000", 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6339999999999996e-06 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.292935+0000", 2023-09-23T18:14:01.404 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99268783800000004 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.292956+0000", 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1806000000000001e-05 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177013 8.0 8.c1908a30 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T18:14:01.405 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.295252+0000", 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 90.101966863000001, 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99778062999999995, 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177013 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.406 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295252+0000", 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295252+0000", 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295251+0000", 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295255+0000", 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5540000000000002e-06 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295262+0000", 2023-09-23T18:14:01.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1770000000000001e-06 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295273+0000", 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0400999999999999e-05 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.300266+0000", 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049932980000000002 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.300271+0000", 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2190000000000003e-06 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.293023+0000", 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.992751936 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.410 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.293033+0000", 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.679e-06 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177039 8.0 8.50a3b3f0 (undecoded) ondisk+retry+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.295281+0000", 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 90.101937875000004, 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99778350900000001, 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.411 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177039 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295281+0000", 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.412 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295281+0000", 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295281+0000", 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295283+0000", 2023-09-23T18:14:01.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7240000000000001e-06 2023-09-23T18:14:01.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.414 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295290+0000", 2023-09-23T18:14:01.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6139999999999997e-06 2023-09-23T18:14:01.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.414 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295294+0000", 2023-09-23T18:14:01.415 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9979999999999997e-06 2023-09-23T18:14:01.415 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.415 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.415 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.415 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.300289+0000", 2023-09-23T18:14:01.415 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049950239999999998 2023-09-23T18:14:01.415 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.415 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.415 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.415 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.300293+0000", 2023-09-23T18:14:01.416 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3259999999999997e-06 2023-09-23T18:14:01.416 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.416 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.416 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.416 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.293055+0000", 2023-09-23T18:14:01.416 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99276222999999997 2023-09-23T18:14:01.416 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.416 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.416 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.417 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.293065+0000", 2023-09-23T18:14:01.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1810000000000007e-06 2023-09-23T18:14:01.418 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.418 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.418 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.418 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.418 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.418 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177043 8.0 8.afa1c210 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T18:14:01.418 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.334446+0000", 2023-09-23T18:14:01.418 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 90.062772988999996, 2023-09-23T18:14:01.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95864787600000001, 2023-09-23T18:14:01.419 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.419 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:01.419 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.419 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.419 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.419 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177043 2023-09-23T18:14:01.419 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.419 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.419 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.420 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.420 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334446+0000", 2023-09-23T18:14:01.420 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.420 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.420 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.420 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.420 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334446+0000", 2023-09-23T18:14:01.420 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.420 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.420 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.421 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.421 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334444+0000", 2023-09-23T18:14:01.421 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999976 2023-09-23T18:14:01.421 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.421 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.421 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.421 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334451+0000", 2023-09-23T18:14:01.421 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9399999999999996e-06 2023-09-23T18:14:01.421 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.421 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.422 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.422 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334462+0000", 2023-09-23T18:14:01.422 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1078e-05 2023-09-23T18:14:01.422 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.422 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.422 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.422 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334469+0000", 2023-09-23T18:14:01.422 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0690000000000004e-06 2023-09-23T18:14:01.422 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334520+0000", 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1165999999999998e-05 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334530+0000", 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8020000000000003e-06 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.423 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.424 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.424 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.293084+0000", 2023-09-23T18:14:01.424 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95855419600000003 2023-09-23T18:14:01.424 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.424 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.424 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.424 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.293094+0000", 2023-09-23T18:14:01.424 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8740000000000007e-06 2023-09-23T18:14:01.424 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.425 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.425 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.425 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.425 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.425 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177044 8.0 8.2351b7a0 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T18:14:01.425 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.334488+0000", 2023-09-23T18:14:01.425 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 90.062730852000001, 2023-09-23T18:14:01.425 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95863536199999999, 2023-09-23T18:14:01.425 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.426 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:01.426 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.426 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.426 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.426 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177044 2023-09-23T18:14:01.426 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.426 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.426 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.426 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.427 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334488+0000", 2023-09-23T18:14:01.427 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.427 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.427 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.427 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.427 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334488+0000", 2023-09-23T18:14:01.427 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.427 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.427 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334488+0000", 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334491+0000", 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2389999999999998e-06 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.428 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334497+0000", 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8499999999999999e-06 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334501+0000", 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1620000000000001e-06 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334551+0000", 2023-09-23T18:14:01.429 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9985999999999999e-05 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334569+0000", 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7873e-05 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.293115+0000", 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95854569000000001 2023-09-23T18:14:01.430 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.293123+0000", 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9819999999999997e-06 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:209416 8.4 8:207eee37:::benchmark_data_smithi055_346788_object98824:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:13.199983+0000", 2023-09-23T18:14:01.431 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 48.197236085999997, 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0412275559999999, 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 209416 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.432 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.199983+0000", 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.199983+0000", 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.199981+0000", 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:01.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.199989+0000", 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4030000000000001e-06 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.200006+0000", 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7176999999999998e-05 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.434 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.200013+0000", 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1380000000000001e-06 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.200089+0000", 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5573000000000003e-05 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.200097+0000", 2023-09-23T18:14:01.435 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9060000000000002e-06 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.205272+0000", 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051752270000000001 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.205280+0000", 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4460000000000001e-06 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.236584+0000", 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0313038000000001 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.236684+0000", 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9417e-05 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.437 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241114+0000", 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044305880000000001 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241210+0000", 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6355999999999997e-05 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.438 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:209472 8.4 8:24fd059e:::benchmark_data_smithi055_346788_object98880:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:13.312086+0000", 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 48.085132907999999, 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.92936662800000003, 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 209472 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.439 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.312086+0000", 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.312086+0000", 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.440 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.312085+0000", 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.312090+0000", 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.604e-06 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.312100+0000", 2023-09-23T18:14:01.441 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7040000000000006e-06 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.312105+0000", 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5949999999999998e-06 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.312160+0000", 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4629000000000002e-05 2023-09-23T18:14:01.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.312171+0000", 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1100999999999999e-05 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.236905+0000", 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.924734104 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.443 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.236987+0000", 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2161000000000002e-05 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.237110+0000", 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012214800000000001 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241198+0000", 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040879059999999997 2023-09-23T18:14:01.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241242+0000", 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4196000000000002e-05 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241283+0000", 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1139e-05 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.446 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241335+0000", 2023-09-23T18:14:01.446 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2623000000000001e-05 2023-09-23T18:14:01.446 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.446 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.446 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.446 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241453+0000", 2023-09-23T18:14:01.446 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117238 2023-09-23T18:14:01.446 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.446 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:209497 8.4 8:22230312:::benchmark_data_smithi055_346788_object98905:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e437)", 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:13.331663+0000", 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 48.065555818, 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91005130099999998, 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.447 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.448 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.448 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 209497 2023-09-23T18:14:01.448 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.448 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.448 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.448 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.448 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.331663+0000", 2023-09-23T18:14:01.448 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.448 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.331663+0000", 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.331661+0000", 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.449 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.450 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.331668+0000", 2023-09-23T18:14:01.450 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.029e-06 2023-09-23T18:14:01.450 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.450 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.450 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.450 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.331681+0000", 2023-09-23T18:14:01.450 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2632999999999999e-05 2023-09-23T18:14:01.450 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.450 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.450 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.331687+0000", 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0499999999999997e-06 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.331746+0000", 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9243999999999997e-05 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:13.331755+0000", 2023-09-23T18:14:01.451 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5329999999999994e-06 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.238370+0000", 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.90661559300000005 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.238487+0000", 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117112 2023-09-23T18:14:01.452 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.238607+0000", 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011997099999999999 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241208+0000", 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002600415 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.453 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241517+0000", 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00030899699999999998 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241578+0000", 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1084e-05 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.454 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241620+0000", 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2163999999999998e-05 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:14.241714+0000", 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4514999999999999e-05 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.455 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216502 8.d 8:b23d2703:::benchmark_data_smithi055_346788_object105910:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.223737+0000", 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.173481514000002, 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041598958, 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216502 2023-09-23T18:14:01.456 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223737+0000", 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223737+0000", 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.457 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223736+0000", 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223741+0000", 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4569999999999998e-06 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.458 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223750+0000", 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4809999999999995e-06 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223755+0000", 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9910000000000002e-06 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.459 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.227046+0000", 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003290598 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.227055+0000", 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8629999999999997e-06 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.265248+0000", 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.038193436 2023-09-23T18:14:01.460 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.265336+0000", 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8357999999999997e-05 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216511 8.d 8:b504248b:::benchmark_data_smithi055_346788_object105919:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.461 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.223766+0000", 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.173452863999998, 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0420443770000001, 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216511 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.462 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.463 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.463 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223766+0000", 2023-09-23T18:14:01.463 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.463 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.463 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.463 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.464 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223766+0000", 2023-09-23T18:14:01.464 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223766+0000", 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223768+0000", 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6350000000000002e-06 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.465 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223775+0000", 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1750000000000002e-06 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.223778+0000", 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9380000000000002e-06 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.466 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.227074+0000", 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032951069999999998 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.227080+0000", 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9599999999999997e-06 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.265730+0000", 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0386503600000001 2023-09-23T18:14:01.467 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.265810+0000", 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0607999999999998e-05 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216519 8.d 8:b0c0099c:::benchmark_data_smithi055_346788_object105927:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.230372+0000", 2023-09-23T18:14:01.468 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.166847441999998, 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0503545540000001, 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216519 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.469 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.230372+0000", 2023-09-23T18:14:01.470 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.470 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.470 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.470 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.470 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.230372+0000", 2023-09-23T18:14:01.470 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.470 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.470 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.470 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.470 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.230371+0000", 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.230376+0000", 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8829999999999997e-06 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.230385+0000", 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0759999999999992e-06 2023-09-23T18:14:01.471 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.230392+0000", 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0079999999999996e-06 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.230473+0000", 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0908999999999994e-05 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.472 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.230481+0000", 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5809999999999997e-06 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.265846+0000", 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.035364803 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.265961+0000", 2023-09-23T18:14:01.473 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011540399999999999 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.266351+0000", 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00038968800000000001 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.275993+0000", 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096419709999999992 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.474 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280614+0000", 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046207230000000002 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280652+0000", 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8417000000000002e-05 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.475 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280700+0000", 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7604e-05 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280726+0000", 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6376e-05 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.476 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216549 8.d 8:bab9cbb0:::benchmark_data_smithi055_346788_object105957:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.257230+0000", 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.139988928000001, 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.023662367, 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216549 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.477 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.257230+0000", 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.257230+0000", 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.478 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.479 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.479 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.257229+0000", 2023-09-23T18:14:01.479 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:14:01.479 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.479 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.479 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.479 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.257235+0000", 2023-09-23T18:14:01.479 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0360000000000003e-06 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.257247+0000", 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1827999999999999e-05 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.257253+0000", 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3389999999999998e-06 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.480 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.257302+0000", 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.88e-05 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.257311+0000", 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1779999999999992e-06 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.481 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.269173+0000", 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0118617640000001 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.269278+0000", 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104646 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:01.482 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.269394+0000", 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011622599999999999 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.275997+0000", 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066031099999999997 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280816+0000", 2023-09-23T18:14:01.483 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048189849999999996 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280847+0000", 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0886000000000002e-05 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280873+0000", 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6091000000000001e-05 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.484 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280892+0000", 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9609000000000001e-05 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216597 8.d 8:b2eb184e:::benchmark_data_smithi055_346788_object106005:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.303851+0000", 2023-09-23T18:14:01.485 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.093368052000002, 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98201729999999998, 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216597 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.486 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.303851+0000", 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.303851+0000", 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.487 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.303850+0000", 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.303857+0000", 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.985e-06 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.303869+0000", 2023-09-23T18:14:01.488 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2401e-05 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.303876+0000", 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0389999999999999e-06 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.304627+0000", 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00075032899999999999 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.304638+0000", 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1574e-05 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.271352+0000", 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.96671352499999996 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.490 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.271488+0000", 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013662 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.271620+0000", 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132022 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280620+0000", 2023-09-23T18:14:01.491 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0090001370000000001 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280939+0000", 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00031812499999999999 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.285777+0000", 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048386319999999998 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.492 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.285828+0000", 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0714000000000003e-05 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.285868+0000", 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0380999999999998e-05 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.493 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216626 8.d 8:b5d39d27:::benchmark_data_smithi055_346788_object106034:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.331076+0000", 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.066142769999999, 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95492708199999998, 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216626 2023-09-23T18:14:01.494 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.331076+0000", 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.331076+0000", 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.495 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.331075+0000", 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.331085+0000", 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0678e-05 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.496 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.331098+0000", 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2519e-05 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.331104+0000", 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5309999999999998e-06 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.331154+0000", 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9879000000000001e-05 2023-09-23T18:14:01.497 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.331164+0000", 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5179999999999993e-06 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.273820+0000", 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.94265602599999998 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.498 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.273928+0000", 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108299 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.274038+0000", 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000109926 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.499 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280626+0000", 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065881309999999997 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.285906+0000", 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052801410000000003 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.285937+0000", 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0852999999999998e-05 2023-09-23T18:14:01.500 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.285973+0000", 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6371000000000001e-05 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.286003+0000", 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9865e-05 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.501 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216671 8.d 8:b14ed93b:::benchmark_data_smithi055_346788_object106079:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.377248+0000", 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.019970745999998, 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.91558229999999996, 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.502 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216671 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.377248+0000", 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.377248+0000", 2023-09-23T18:14:01.503 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.377247+0000", 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.377253+0000", 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.784e-06 2023-09-23T18:14:01.504 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.377264+0000", 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1426999999999999e-05 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.377270+0000", 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.117e-06 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.505 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.377330+0000", 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9920000000000002e-05 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.377339+0000", 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9309999999999997e-06 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.506 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.276018+0000", 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.89867946399999998 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.276140+0000", 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121537 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:01.507 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.276292+0000", 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015174599999999999 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280826+0000", 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045341870000000003 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.286041+0000", 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005215028 2023-09-23T18:14:01.508 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.292741+0000", 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0067003640000000003 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.292792+0000", 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0949999999999998e-05 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.509 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.292831+0000", 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8232999999999998e-05 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216677 8.d 8:b8cca103:::benchmark_data_smithi055_346788_object106085:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.383225+0000", 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 37.013993485, 2023-09-23T18:14:01.510 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.90971724399999998, 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216677 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.511 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.383225+0000", 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.383225+0000", 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.383224+0000", 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:01.512 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.383230+0000", 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3090000000000001e-06 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.383241+0000", 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0198e-05 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.513 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.383246+0000", 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6459999999999998e-06 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.383299+0000", 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2510999999999999e-05 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.514 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.383308+0000", 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6970000000000005e-06 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.278429+0000", 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.89512088199999995 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.515 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.278598+0000", 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000169099 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.278761+0000", 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016292800000000001 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280950+0000", 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021882799999999999 2023-09-23T18:14:01.516 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.286069+0000", 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051194730000000003 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.292870+0000", 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0068013029999999999 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.517 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.292910+0000", 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9875e-05 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.292943+0000", 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2471000000000001e-05 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.518 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216725 8.d 8:b0495c94:::benchmark_data_smithi055_346788_object106133:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.430935+0000", 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 36.966284055000003, 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.86213059999999997, 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.519 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216725 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.430935+0000", 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.430935+0000", 2023-09-23T18:14:01.520 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.430933+0000", 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.430940+0000", 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2489999999999998e-06 2023-09-23T18:14:01.521 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.430955+0000", 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5398000000000001e-05 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.430962+0000", 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.934e-06 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.431027+0000", 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4918000000000004e-05 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.431036+0000", 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1209999999999996e-06 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.523 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.280989+0000", 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.84995282400000005 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.281122+0000", 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013348799999999999 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:01.524 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.281236+0000", 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000114094 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.285912+0000", 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046757470000000001 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.292974+0000", 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070620730000000003 2023-09-23T18:14:01.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.293001+0000", 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6271e-05 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.293033+0000", 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2267999999999997e-05 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.293066+0000", 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2706000000000002e-05 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216726 8.d 8:b67347b3:::benchmark_data_smithi055_346788_object106134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.431377+0000", 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 36.965841992000001, 2023-09-23T18:14:01.527 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.86180865900000003, 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216726 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.528 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.431377+0000", 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.431377+0000", 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.431376+0000", 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:01.529 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.431380+0000", 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0500000000000002e-06 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.431389+0000", 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4880000000000007e-06 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.530 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.431393+0000", 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2039999999999999e-06 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.431435+0000", 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1693999999999998e-05 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.531 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.431450+0000", 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4973e-05 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.283248+0000", 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85179844400000004 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.283380+0000", 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013146100000000001 2023-09-23T18:14:01.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.283509+0000", 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012892700000000001 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.286010+0000", 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025011059999999999 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.533 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.293096+0000", 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070860460000000004 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.293122+0000", 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.635e-05 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.534 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.293154+0000", 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2014000000000002e-05 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.293186+0000", 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1504000000000002e-05 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.535 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216741 8.d 8:b845f7ec:::benchmark_data_smithi055_346788_object106149:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.447276+0000", 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 36.949943234000003, 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85512940100000001, 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216741 2023-09-23T18:14:01.536 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.447276+0000", 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.447276+0000", 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.447275+0000", 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.447278+0000", 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7310000000000002e-06 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.538 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.447285+0000", 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5100000000000004e-06 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.447287+0000", 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8830000000000002e-06 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.539 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.447341+0000", 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4014999999999997e-05 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.447351+0000", 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5710000000000004e-06 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.286089+0000", 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.83873766000000005 2023-09-23T18:14:01.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.286186+0000", 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7785e-05 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.286291+0000", 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104145 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.541 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.292661+0000", 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0063703509999999998 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.293218+0000", 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00055713900000000001 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.302329+0000", 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0091112329999999998 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.302365+0000", 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6056999999999997e-05 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.302405+0000", 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9740999999999998e-05 2023-09-23T18:14:01.543 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:216744 8.d 8:b96e3143:::benchmark_data_smithi055_346788_object106152:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e448)", 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:24.453365+0000", 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 36.943854459999997, 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.85447187999999996, 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:01.544 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 216744 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.453365+0000", 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:01.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.453365+0000", 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.453363+0000", 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.453368+0000", 2023-09-23T18:14:01.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3959999999999999e-06 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.453380+0000", 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1902000000000001e-05 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.453384+0000", 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6770000000000003e-06 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.453427+0000", 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2231000000000003e-05 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:24.453435+0000", 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2050000000000002e-06 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:01.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.288366+0000", 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.83493111499999995 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.288488+0000", 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000122364 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.288625+0000", 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013655199999999999 2023-09-23T18:14:01.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.293249+0000", 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046239990000000002 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.302441+0000", 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0091920140000000001 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.307756+0000", 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0053151880000000002 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.307800+0000", 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4119999999999998e-05 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:01.551 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:25.307836+0000", 2023-09-23T18:14:01.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6155000000000002e-05 2023-09-23T18:14:01.552 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.552 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.552 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.552 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:01.552 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:01.552 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:14:01.553 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:01.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:01.554 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:01.554 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:01.565 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:14:01.565 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:14:01.598 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 13 16 31987 31971 153.655 117.938 0.00425994 0.00650276 2023-09-23T18:14:01.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:01.709 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:01 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2213501024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:01.716 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:01.754 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:01 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2213501024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:01.754 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:01 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2213501024' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:01.778 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:01.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:01.787 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:01.888 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:01.940 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:01.992 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:01.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:01.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:02.098 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:02.142 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:02.265 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:02.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:02.270 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:02.371 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:02.428 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:02.507 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:14:02.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:02 smithi099 ceph-mon[167076]: pgmap v1125: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.7 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 3253/83691 objects degraded (3.887%) 2023-09-23T18:14:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:02 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 3253/83691 objects degraded (3.887%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:02.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3499133745' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:14:02.598 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 14 16 33859 33843 151.036 117 0.00747679 0.00661456 2023-09-23T18:14:02.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:02 smithi055 ceph-mon[179668]: pgmap v1125: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.7 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 3253/83691 objects degraded (3.887%) 2023-09-23T18:14:02.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:02 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 3253/83691 objects degraded (3.887%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:02.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3499133745' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:14:02.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:02 smithi055 ceph-mon[184174]: pgmap v1125: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.7 GiB data, 8.7 GiB used, 706 GiB / 715 GiB avail; 151 MiB/s wr, 2.41k op/s; 3253/83691 objects degraded (3.887%) 2023-09-23T18:14:02.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:02 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 3253/83691 objects degraded (3.887%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:02.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3499133745' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:14:03.567 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:14:03.598 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 15 16 35790 35774 149.013 120.688 0.00524343 0.0067064 2023-09-23T18:14:03.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:03 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:03.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3499133745' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:14:03.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:03 smithi099 ceph-mon[167076]: osdmap e465: 8 total, 7 up, 8 in 2023-09-23T18:14:03.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:03 smithi099 ceph-mon[167076]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:03 smithi099 ceph-mon[167076]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:03 smithi099 ceph-mon[167076]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:03 smithi099 ceph-mon[167076]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1672094755' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:14:03.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:03 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:14:03.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:03.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3499133745' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:14:03.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[184174]: osdmap e465: 8 total, 7 up, 8 in 2023-09-23T18:14:03.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[184174]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[184174]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[184174]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[184174]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1672094755' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:14:03.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:14:03.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:03.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3499133745' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:14:03.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[179668]: osdmap e465: 8 total, 7 up, 8 in 2023-09-23T18:14:03.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[179668]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[179668]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[179668]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[179668]: failed to encode map e465 with expected crc 2023-09-23T18:14:03.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1672094755' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:14:03.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:03 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:14:04.139 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:04.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:04.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:04.217 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 9 pgp_num to 16 2023-09-23T18:14:04.235 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:14:04.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:04.252 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:04.308 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:04.376 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:04.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:04.385 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:04.486 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:04.512 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:04.545 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:04.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:04.548 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:04.598 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 16 16 37160 37144 145.051 85.625 0.00203337 0.00682237 2023-09-23T18:14:04.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:04.675 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:04.709 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:04.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:04.712 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:04.812 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:04.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:04 smithi099 ceph-mon[167076]: pgmap v1127: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 155 MiB/s wr, 2.47k op/s; 4461/89103 objects degraded (5.007%) 2023-09-23T18:14:04.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:04 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:14:04.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:04 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:14:04.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:04 smithi099 ceph-mon[167076]: osdmap e466: 8 total, 7 up, 8 in 2023-09-23T18:14:04.879 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:04 smithi055 ceph-mon[184174]: pgmap v1127: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 155 MiB/s wr, 2.47k op/s; 4461/89103 objects degraded (5.007%) 2023-09-23T18:14:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:04 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:14:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:04 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:14:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:04 smithi055 ceph-mon[184174]: osdmap e466: 8 total, 7 up, 8 in 2023-09-23T18:14:04.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:04 smithi055 ceph-mon[179668]: pgmap v1127: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 1.8 GiB data, 9.3 GiB used, 706 GiB / 715 GiB avail; 155 MiB/s wr, 2.47k op/s; 4461/89103 objects degraded (5.007%) 2023-09-23T18:14:04.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:04 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:14:04.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:04 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:14:04.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:04 smithi055 ceph-mon[179668]: osdmap e466: 8 total, 7 up, 8 in 2023-09-23T18:14:05.000 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:14:05.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:05.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:05.252 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:05.352 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:05.423 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:05.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:05.485 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:05.486 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:05.587 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:05.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 17 16 38340 38324 140.857 73.75 0.0114496 0.00709413 2023-09-23T18:14:05.666 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:05 smithi055 ceph-mon[184174]: failed to encode map e466 with expected crc 2023-09-23T18:14:05.666 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:05 smithi055 ceph-mon[184174]: failed to encode map e466 with expected crc 2023-09-23T18:14:05.667 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:05 smithi055 ceph-mon[179668]: failed to encode map e466 with expected crc 2023-09-23T18:14:05.667 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:05 smithi055 ceph-mon[179668]: failed to encode map e466 with expected crc 2023-09-23T18:14:05.667 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:05.753 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:05.757 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:05.757 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:05 smithi099 ceph-mon[167076]: failed to encode map e466 with expected crc 2023-09-23T18:14:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:05 smithi099 ceph-mon[167076]: failed to encode map e466 with expected crc 2023-09-23T18:14:05.858 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:05.905 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:06.070 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:06.117 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:06.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:06.121 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:06.219 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:06.222 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:06.299 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:06.438 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:06.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:06.446 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:06.546 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:06.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 18 16 39891 39875 138.417 96.9375 0.0093015 0.00721936 2023-09-23T18:14:06.636 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:06 smithi099 ceph-mon[167076]: pgmap v1129: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s; 4461/96348 objects degraded (4.630%) 2023-09-23T18:14:06.636 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:06.640 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:06.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:06 smithi055 ceph-mon[179668]: pgmap v1129: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s; 4461/96348 objects degraded (4.630%) 2023-09-23T18:14:06.712 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:06.712 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:06 smithi055 ceph-mon[184174]: pgmap v1129: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.0 GiB data, 9.7 GiB used, 706 GiB / 715 GiB avail; 117 MiB/s wr, 1.88k op/s; 4461/96348 objects degraded (4.630%) 2023-09-23T18:14:06.712 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:06.750 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:06.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:06.763 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:06.864 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:06.913 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:07.065 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:07.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:07.069 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:07.170 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:07.223 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:07.275 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:07.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:07.283 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:07.370 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:14:07.384 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:07.408 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:14:07.450 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:07.459 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:14:07.504 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:14:07.531 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:07.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:07.539 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:07.552 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:14:07.584 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:14:07.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 19 16 41385 41369 136.047 93.375 0.00742984 0.00734527 2023-09-23T18:14:07.621 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:14:07.621 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:14:07.621 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:14:07.622 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T18:14:07.622 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:14:07.622 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:14:07.622 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:14:07.622 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:14:07.622 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:14:07.640 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:07.651 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:07 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1023458563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:07.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:07 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1023458563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:07.725 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:07.782 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:07.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:07.787 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:07.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:07 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1023458563' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:07.887 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:07.924 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:08.168 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:08.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:08.174 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:08.259 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:14:08.259 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:14:08.276 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:08.331 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:08.389 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:08.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:08.394 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:08.473 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:14:08.473 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:14:08.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:08.561 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:08.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:08 smithi099 ceph-mon[167076]: pgmap v1130: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s; 4461/110262 objects degraded (4.046%) 2023-09-23T18:14:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 4461/110262 objects degraded (4.046%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:08.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:14:08.599914+0000 min lat: 0.00130179 max lat: 0.340435 avg lat: 0.00745532 2023-09-23T18:14:08.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:14:08.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 20 16 42921 42905 134.044 96 0.00978216 0.00745532 2023-09-23T18:14:08.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:08.641 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:08.642 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:08.679 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:14:08.717 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:08 smithi055 ceph-mon[184174]: pgmap v1130: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s; 4461/110262 objects degraded (4.046%) 2023-09-23T18:14:08.717 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 4461/110262 objects degraded (4.046%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:08.718 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:08 smithi055 ceph-mon[179668]: pgmap v1130: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.2 GiB data, 10 GiB used, 705 GiB / 715 GiB avail; 114 MiB/s wr, 1.83k op/s; 4461/110262 objects degraded (4.046%) 2023-09-23T18:14:08.718 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 4461/110262 objects degraded (4.046%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:08.742 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:08.830 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:08.906 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:08.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:08.911 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:09.011 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:09.061 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:09.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:09.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:09.137 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:09.238 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:09.295 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:09.365 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:09.372 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:09.372 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:09.422 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:09.473 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:09.532 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:09.571 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:14:09.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 21 16 44316 44300 131.813 87.1875 0.00794219 0.00758138 2023-09-23T18:14:09.747 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:09.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:09.752 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:09.820 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:09 smithi055 ceph-mon[184174]: pgmap v1131: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 5341/119847 objects degraded (4.457%) 2023-09-23T18:14:09.820 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2565049734' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:09.822 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:09 smithi055 ceph-mon[179668]: pgmap v1131: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 5341/119847 objects degraded (4.457%) 2023-09-23T18:14:09.823 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2565049734' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:09.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:09 smithi099 ceph-mon[167076]: pgmap v1131: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.4 GiB data, 11 GiB used, 704 GiB / 715 GiB avail; 94 MiB/s wr, 1.50k op/s; 5341/119847 objects degraded (4.457%) 2023-09-23T18:14:09.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2565049734' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:09.853 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:09.958 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:10.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:10.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:10.043 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:10.144 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:10.199 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:10.358 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:10.359 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:14:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:14:10] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.43.0" 2023-09-23T18:14:10.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:10.366 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:10.467 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:10.474 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:14:10.536 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:10.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 22 16 45773 45757 129.96 91.0625 0.0182378 0.00768925 2023-09-23T18:14:10.668 INFO:tasks.thrashosds.thrasher:in_osds: [6, 7, 5, 1, 0, 4, 2, 3] out_osds: [] dead_osds: [2] live_osds: [0, 1, 4, 6, 7, 3, 5] 2023-09-23T18:14:10.668 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:14:10.668 INFO:tasks.thrashosds.thrasher:Removing osd 6, in_osds are: [6, 7, 5, 1, 0, 4, 2, 3] 2023-09-23T18:14:10.668 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 6 2023-09-23T18:14:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:10 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:10 smithi055 ceph-mon[179668]: failed to encode map e467 with expected crc 2023-09-23T18:14:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2565049734' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:14:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:10 smithi055 ceph-mon[179668]: osdmap e467: 8 total, 7 up, 8 in 2023-09-23T18:14:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:10 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:10 smithi055 ceph-mon[184174]: failed to encode map e467 with expected crc 2023-09-23T18:14:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2565049734' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:14:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:10 smithi055 ceph-mon[184174]: osdmap e467: 8 total, 7 up, 8 in 2023-09-23T18:14:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:10 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:10 smithi099 ceph-mon[167076]: failed to encode map e467 with expected crc 2023-09-23T18:14:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2565049734' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:14:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:10 smithi099 ceph-mon[167076]: osdmap e467: 8 total, 7 up, 8 in 2023-09-23T18:14:11.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 23 16 47425 47409 128.798 103.25 0.00702788 0.00775893 2023-09-23T18:14:11.713 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:11.719 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:11.719 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:11.821 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:11.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:11 smithi099 ceph-mon[167076]: pgmap v1133: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 5341/132516 objects degraded (4.030%) 2023-09-23T18:14:11.832 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:11 smithi055 ceph-mon[179668]: pgmap v1133: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 5341/132516 objects degraded (4.030%) 2023-09-23T18:14:11.832 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:11 smithi055 ceph-mon[184174]: pgmap v1133: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.7 GiB data, 12 GiB used, 704 GiB / 715 GiB avail; 113 MiB/s wr, 1.81k op/s; 5341/132516 objects degraded (4.030%) 2023-09-23T18:14:11.882 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:12.048 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:12.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:12.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:12.134 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:14:12.153 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:12.189 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:14:12.189 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:14:12.220 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:12.221 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:12.291 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:12.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:12.296 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:12.397 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:12.470 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:12.536 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:12.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:12.546 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:12.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 24 16 48740 48724 126.856 82.1875 0.00643767 0.00787807 2023-09-23T18:14:12.647 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:12.694 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:12.810 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:12.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:12.814 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:12.849 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:14:12.916 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:12.941 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:12.941 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:12 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:12.979 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:13.037 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:13.041 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:13.042 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:13.142 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:13.189 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:12 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:13.190 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:12 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:13.193 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:13.257 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:12 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:13.257 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:12 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:13.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:13.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:13.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:13.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:13.443 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:13.502 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:13.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:13.507 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:13.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 25 16 50171 50155 125.359 89.4375 0.00854816 0.00797263 2023-09-23T18:14:13.608 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:13.627 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:13.657 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:13.770 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:13.775 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:13.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:13.784 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:13.885 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:13.935 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:13.999 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:14.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:14.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:14.105 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:14.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:13 smithi055 ceph-mon[179668]: pgmap v1134: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s; 6271/136650 objects degraded (4.589%) 2023-09-23T18:14:14.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 5341/132516 objects degraded (4.030%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:14.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:13 smithi055 ceph-mon[184174]: pgmap v1134: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s; 6271/136650 objects degraded (4.589%) 2023-09-23T18:14:14.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 5341/132516 objects degraded (4.030%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:14.164 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:14.167 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:14.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:13 smithi099 ceph-mon[167076]: pgmap v1134: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.8 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.53k op/s; 6271/136650 objects degraded (4.589%) 2023-09-23T18:14:14.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 5341/132516 objects degraded (4.030%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:14.379 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:14.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:14.384 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:14.485 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:14.557 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:14.599 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 26 16 51592 51576 123.954 88.8125 0.00887953 0.00806204 2023-09-23T18:14:14.976 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:14.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:14.983 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:15.084 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:15.165 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:15.318 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:15.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:15.326 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:15.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:15.488 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:15.552 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:15.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:15.558 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:15.600 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 27 16 52963 52947 122.536 85.6875 0.00716281 0.00815694 2023-09-23T18:14:15.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:15.728 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:15.787 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:15.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:15.797 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:15.859 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:15 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:14:15.433+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e467 definitely_dead 0 2023-09-23T18:14:15.898 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:15.983 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:16.185 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:16.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:16.191 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:16.236 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.6. 2023-09-23T18:14:16.293 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:16.307 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:14:16.335 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:14:16.338 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:16.358 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:14:16.382 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:14:16.403 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:14:16.428 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:14:16.447 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:14:16.448 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:14:16.448 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:14:16.448 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T18:14:16.448 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:14:16.448 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:14:16.448 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:14:16.448 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:14:16.448 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:14:16.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[184174]: pgmap v1135: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 6271/142038 objects degraded (4.415%) 2023-09-23T18:14:16.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:16.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:16.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/268496782' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T18:14:16.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T18:14:16.451 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[184174]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T18:14:16.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[179668]: pgmap v1135: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 6271/142038 objects degraded (4.415%) 2023-09-23T18:14:16.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:16.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:16.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/268496782' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T18:14:16.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T18:14:16.452 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:16 smithi055 ceph-mon[179668]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T18:14:16.533 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:16.537 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:16.537 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:16 smithi099 ceph-mon[167076]: pgmap v1135: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 2.9 GiB data, 12 GiB used, 703 GiB / 715 GiB avail; 95 MiB/s wr, 1.52k op/s; 6271/142038 objects degraded (4.415%) 2023-09-23T18:14:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:16 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:16 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/268496782' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T18:14:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:16 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["6"]}]: dispatch 2023-09-23T18:14:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:16 smithi099 ceph-mon[167076]: Client client.admin marked osd.6 out, while it was still marked up 2023-09-23T18:14:16.600 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 28 16 53803 53787 120.035 52.5 0.00412476 0.0082216 2023-09-23T18:14:16.638 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:16.667 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:16.871 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:14:16.871 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:14:16.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:14:16.872 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:14:16.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.872 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29520 7.4 7:2e3eff57:::benchmark_data_smithi055_311528_object29519:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T18:14:16.872 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.342145+0000", 2023-09-23T18:14:16.872 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 492.52717607400001, 2023-09-23T18:14:16.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.279446246, 2023-09-23T18:14:16.873 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.873 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:14:16.873 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:16.873 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:16.873 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:16.873 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29520 2023-09-23T18:14:16.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342145+0000", 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342145+0000", 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342144+0000", 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342167+0000", 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2671e-05 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.875 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342179+0000", 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2537999999999999e-05 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342199+0000", 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9791000000000001e-05 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342244+0000", 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5379e-05 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342363+0000", 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119085 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,6", 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.342477+0000", 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011398 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.344360+0000", 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018828829999999999 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.351598+0000", 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072381310000000001 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.351613+0000", 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5082999999999999e-05 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361553+0000", 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0099392469999999 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361566+0000", 2023-09-23T18:14:16.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3742e-05 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.621482+0000", 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.259916043 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.621592+0000", 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109143 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.880 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29523 7.4 7:218b7f51:::benchmark_data_smithi055_311528_object29522:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e284)", 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.362687+0000", 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 492.50663464199999, 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2720197689999999, 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:16.881 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29523 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362687+0000", 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362687+0000", 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362685+0000", 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362714+0000", 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9023999999999999e-05 2023-09-23T18:14:16.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362730+0000", 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6053999999999999e-05 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.362738+0000", 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7570000000000004e-06 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.363432+0000", 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00069362000000000004 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.363443+0000", 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1031e-05 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361597+0000", 2023-09-23T18:14:16.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99815435299999999 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361605+0000", 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7270000000000007e-06 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.623050+0000", 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2614453999999999 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.623261+0000", 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021100599999999999 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.623463+0000", 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020170799999999999 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:16.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626400+0000", 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00293728 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.628125+0000", 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001725266 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.633086+0000", 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049610030000000003 2023-09-23T18:14:16.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.633146+0000", 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0000000000000002e-05 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.634707+0000", 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001560077 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.889 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29540 7.9 7:9dcdb66e:::benchmark_data_smithi055_311528_object29539:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.375098+0000", 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 492.49422382, 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2623359889999999, 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:16.890 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29540 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375098+0000", 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375098+0000", 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375096+0000", 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375120+0000", 2023-09-23T18:14:16.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3737e-05 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375132+0000", 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2211999999999999e-05 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375140+0000", 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6350000000000006e-06 2023-09-23T18:14:16.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375200+0000", 2023-09-23T18:14:16.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.067e-05 2023-09-23T18:14:16.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.375211+0000", 2023-09-23T18:14:16.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0248000000000001e-05 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356527+0000", 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98131692100000001 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356539+0000", 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1392e-05 2023-09-23T18:14:16.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625642+0000", 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2691035230000001 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625784+0000", 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014192400000000001 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625926+0000", 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000141607 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629334+0000", 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034082079999999998 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.630984+0000", 2023-09-23T18:14:16.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00164963 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635790+0000", 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048063300000000002 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635850+0000", 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9979000000000003e-05 2023-09-23T18:14:16.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637434+0000", 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001583552 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29542 7.9 7:9a322fbf:::benchmark_data_smithi055_311528_object29541:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.376211+0000", 2023-09-23T18:14:16.899 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 492.493110043, 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2627142660000001, 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29542 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376211+0000", 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376211+0000", 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376210+0000", 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:16.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376231+0000", 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1126000000000001e-05 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376248+0000", 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6461e-05 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.902 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.376256+0000", 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5590000000000006e-06 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.378137+0000", 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001880928 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.378147+0000", 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9010000000000006e-06 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356782+0000", 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97863468399999998 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.356790+0000", 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5560000000000008e-06 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629412+0000", 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.272622081 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629587+0000", 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000174432 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629697+0000", 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000109821 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.631159+0000", 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014629059999999999 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.631648+0000", 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00048873199999999997 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637552+0000", 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0059042740000000002 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637614+0000", 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1674000000000006e-05 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.638926+0000", 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013115170000000001 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29546 7.4 7:2ffa6bd1:::benchmark_data_smithi055_311528_object29545:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.379888+0000", 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 492.48943384, 2023-09-23T18:14:16.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2559961080000002, 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29546 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379888+0000", 2023-09-23T18:14:16.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379888+0000", 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379886+0000", 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379920+0000", 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4292999999999998e-05 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379934+0000", 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4307999999999999e-05 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379943+0000", 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7010000000000002e-06 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.379996+0000", 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2849000000000002e-05 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.380004+0000", 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5900000000000002e-06 2023-09-23T18:14:16.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.361629+0000", 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.981625623 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.364509+0000", 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028797689999999999 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.624800+0000", 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.260291504 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.624960+0000", 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000159739 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:14:16.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.625123+0000", 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016252700000000001 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.628149+0000", 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00302673 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629738+0000", 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001588922 2023-09-23T18:14:16.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.634757+0000", 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050189459999999998 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.634816+0000", 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8294000000000001e-05 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635884+0000", 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001068102 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:29552 7.4 7:2f4e28dc:::benchmark_data_smithi055_311528_object29551:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e285)", 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:04.385790+0000", 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 492.48353106399998, 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2515060980000001, 2023-09-23T18:14:16.917 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 29552 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385790+0000", 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385790+0000", 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385789+0000", 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385809+0000", 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0041e-05 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385821+0000", 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1803e-05 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385829+0000", 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4889999999999995e-06 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385881+0000", 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1408e-05 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:04.385890+0000", 2023-09-23T18:14:16.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2089999999999994e-06 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.364654+0000", 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97876412000000002 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:05.364667+0000", 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2456999999999999e-05 2023-09-23T18:14:16.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626449+0000", 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2617827670000001 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626582+0000", 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013226399999999999 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 1,4", 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.626686+0000", 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104701 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629538+0000", 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028518670000000001 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.629790+0000", 2023-09-23T18:14:16.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00025145700000000002 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635940+0000", 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0061504029999999996 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.635986+0000", 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6304000000000003e-05 2023-09-23T18:14:16.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:06.637296+0000", 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013099990000000001 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:108098 7.6 7:63f0b014:::benchmark_data_smithi055_311528_object37135:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:14:16.926 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:40.505253+0000", 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 336.36406810699998, 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13445985899999999, 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108098 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.927 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505253+0000", 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505253+0000", 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505251+0000", 2023-09-23T18:14:16.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505261+0000", 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4450000000000002e-06 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505282+0000", 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1251999999999999e-05 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505290+0000", 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2879999999999993e-06 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505343+0000", 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2034999999999999e-05 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.506478+0000", 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011350469999999999 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.624282+0000", 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.117804585 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.624374+0000", 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1632000000000006e-05 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.624475+0000", 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010101400000000001 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.626282+0000", 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018073939999999999 2023-09-23T18:14:16.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.628754+0000", 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002471829 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.639295+0000", 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010541404000000001 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.639338+0000", 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2814e-05 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.639713+0000", 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000374948 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:88522 8.2 e379/371 8:4b2b0f84:::benchmark_data_smithi055_346788_object88521:head v 379'5492, mlcod=0'0)", 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:35.572982+0000", 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 221.29633967699999, 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.050153703000000001, 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.572982+0000", 2023-09-23T18:14:16.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.572982+0000", 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.572981+0000", 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:16.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.573023+0000", 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.244e-05 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.573042+0000", 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8419000000000001e-05 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.573051+0000", 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4639999999999995e-06 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575760+0000", 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027086300000000001 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575785+0000", 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5405000000000002e-05 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623088+0000", 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047302960999999998 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623135+0000", 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7342000000000002e-05 2023-09-23T18:14:16.939 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:88524 8.2 e379/371 8:46f9590d:::benchmark_data_smithi055_346788_object88523:head v 379'5493, mlcod=0'0)", 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:35.575910+0000", 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 221.29341146300001, 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047258545999999998, 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575910+0000", 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575910+0000", 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575909+0000", 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575956+0000", 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6783000000000003e-05 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575968+0000", 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2165e-05 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575975+0000", 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5469999999999999e-06 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.581529+0000", 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055535189999999998 2023-09-23T18:14:16.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.581561+0000", 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1767000000000003e-05 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623141+0000", 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.041580060000000002 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623169+0000", 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7665e-05 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPush(8.6 386/371 [PushOp(8:6c728b0a:::benchmark_data_smithi055_346788_object2082:head, version: 365'134, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:6c728b0a:::benchmark_data_smithi055_346788_object2082:head@365'134, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 202.51768894200001, 2023-09-23T18:14:16.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.060643765000000002, 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351669+0000", 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6786999999999999e-05 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351689+0000", 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9692000000000001e-05 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351702+0000", 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3709e-05 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.408798+0000", 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.057095414999999997 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.408806+0000", 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5909999999999996e-06 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.412276+0000", 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034699190000000001 2023-09-23T18:14:16.949 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:104012 8.6 e386/371 8:6f916fc8:::benchmark_data_smithi055_346788_object104011:head v 386'6585, mlcod=365'133)", 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:54.387490+0000", 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 202.48183123699999, 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047785543, 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.950 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387490+0000", 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387490+0000", 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387489+0000", 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387519+0000", 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9485e-05 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387539+0000", 2023-09-23T18:14:16.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0528999999999999e-05 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387548+0000", 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2240000000000001e-06 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.396833+0000", 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092844009999999994 2023-09-23T18:14:16.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.396858+0000", 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5072e-05 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.435227+0000", 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038369551000000002 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.435276+0000", 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8396000000000002e-05 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:105389 8.6 e386/371 8:617cba55:::benchmark_data_smithi055_346788_object105388:head v 386'6660, mlcod=365'174)", 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:55.764711+0000", 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 201.10461009700001, 2023-09-23T18:14:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.049308314999999998, 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764711+0000", 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764711+0000", 2023-09-23T18:14:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764710+0000", 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764741+0000", 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0073000000000002e-05 2023-09-23T18:14:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764755+0000", 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4827999999999999e-05 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764764+0000", 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.738e-06 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.769484+0000", 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047194649999999999 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.769511+0000", 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7542000000000002e-05 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.813963+0000", 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.044451549 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.814020+0000", 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7012999999999998e-05 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:109643 8.6 e386/371 8:6b4a303b:::benchmark_data_smithi055_346788_object109642:head v 386'6916, mlcod=365'308)", 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 196.85004595800001, 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.046387045000000002, 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019295+0000", 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9360000000000001e-05 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019301+0000", 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7490000000000001e-06 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019305+0000", 2023-09-23T18:14:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8700000000000002e-06 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.027156+0000", 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0078509070000000007 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.027176+0000", 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.037e-05 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.065607+0000", 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038430756000000003 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.065663+0000", 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5334000000000002e-05 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:136113 8.6 e405/404 8:698e62bb:::benchmark_data_smithi055_346788_object25521:head v 405'8582, mlcod=0'0)", 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:41.581664+0000", 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 155.28765776, 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.064351331999999997, 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581664+0000", 2023-09-23T18:14:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581664+0000", 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581662+0000", 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581670+0000", 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8990000000000001e-06 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581683+0000", 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2346e-05 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581692+0000", 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2900000000000008e-06 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581741+0000", 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8770000000000002e-05 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581757+0000", 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5994999999999998e-05 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.645972+0000", 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.064215229999999998 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.646015+0000", 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2985000000000003e-05 2023-09-23T18:14:16.970 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:156532 8.1 e416/383 8:8578332c:::benchmark_data_smithi055_346788_object45940:head v 416'9728, mlcod=366'2079)", 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:09.038016+0000", 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 127.831305207, 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.053601875, 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038016+0000", 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038016+0000", 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038015+0000", 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038022+0000", 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7939999999999997e-06 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085419+0000", 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047397008999999997 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085439+0000", 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9429999999999999e-05 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085493+0000", 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4848000000000001e-05 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085526+0000", 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2218999999999998e-05 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.091569+0000", 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060432519999999998 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.091618+0000", 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9156000000000002e-05 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:156516 8.2 e416/413 8:40fce4c3:::benchmark_data_smithi055_346788_object45924:head v 416'9669, mlcod=0'0)", 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:09.038816+0000", 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 127.83050554, 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.053023129000000002, 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.976 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038816+0000", 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038816+0000", 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038815+0000", 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038821+0000", 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.852e-06 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085422+0000", 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.046601112 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085438+0000", 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5879000000000002e-05 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085493+0000", 2023-09-23T18:14:16.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4710999999999998e-05 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085526+0000", 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3525999999999997e-05 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.091788+0000", 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0062620949999999996 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.091839+0000", 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0606999999999998e-05 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPush(8.6 416/404 [PushOp(8:6b0a9119:::benchmark_data_smithi055_346788_object45949:head, version: 368'2919, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:6b0a9119:::benchmark_data_smithi055_346788_object45949:head@368'2919, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:09.042065+0000", 2023-09-23T18:14:16.981 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 127.82725662999999, 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.045964787, 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.042065+0000", 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.042065+0000", 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.042064+0000", 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.042119+0000", 2023-09-23T18:14:16.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5179000000000001e-05 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085425+0000", 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043305758 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085450+0000", 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5037000000000001e-05 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085809+0000", 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00035868500000000003 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085822+0000", 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3665e-05 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.088030+0000", 2023-09-23T18:14:16.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022073050000000001 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:37066 9.8 e466/459 9:1853f8f7:::benchmark_data_smithi055_386467_object37065:head v 466'2211, mlcod=466'2208)", 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:04.407507+0000", 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 12.461814638, 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.33454793700000002, 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.986 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407507+0000", 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407507+0000", 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407506+0000", 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407569+0000", 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3313999999999997e-05 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407585+0000", 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6155999999999999e-05 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407594+0000", 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7029999999999995e-06 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737582+0000", 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32998835999999998 2023-09-23T18:14:16.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737627+0000", 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4586000000000003e-05 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.742016+0000", 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0043891629999999997 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.742055+0000", 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8816999999999998e-05 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:37075 9.9 e466/459 9:997ed14a:::benchmark_data_smithi055_386467_object37074:head v 466'2323, mlcod=466'2321)", 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:04.410872+0000", 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 12.458449869000001, 2023-09-23T18:14:16.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.33109135099999998, 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410872+0000", 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410872+0000", 2023-09-23T18:14:16.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410870+0000", 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410944+0000", 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4023999999999995e-05 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737578+0000", 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.326633379 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737600+0000", 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1971e-05 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:16.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737664+0000", 2023-09-23T18:14:16.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3850999999999996e-05 2023-09-23T18:14:16.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:16.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737684+0000", 2023-09-23T18:14:16.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0202000000000001e-05 2023-09-23T18:14:16.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:16.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.741922+0000", 2023-09-23T18:14:16.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0042384019999999996 2023-09-23T18:14:16.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.741963+0000", 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0674999999999999e-05 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:37084 9.8 e466/459 9:1c8e8c79:::benchmark_data_smithi055_386467_object37083:head v 466'2212, mlcod=466'2209)", 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:04.415320+0000", 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 12.45400092, 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32965889700000001, 2023-09-23T18:14:16.997 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415320+0000", 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415320+0000", 2023-09-23T18:14:16.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415319+0000", 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415387+0000", 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7977000000000003e-05 2023-09-23T18:14:16.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737586+0000", 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32219906100000001 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737603+0000", 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7031e-05 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:17.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.739129+0000", 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001526077 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.739145+0000", 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5176e-05 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.744934+0000", 2023-09-23T18:14:17.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0057898649999999999 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.744979+0000", 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4999000000000001e-05 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:17.002 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:14:17.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:17.004 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:17.004 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:17.088 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:14:17.089 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:14:17.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:17.137 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:17.172 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:17.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:17.175 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:17.243 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:14:17.275 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:17.304 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:17.338 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:17.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[179668]: failed to encode map e468 with expected crc 2023-09-23T18:14:17.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T18:14:17.340 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[179668]: osdmap e468: 8 total, 7 up, 7 in 2023-09-23T18:14:17.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:14:17.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:14:17.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:17.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/327427915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:17.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/283375008' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:17.341 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:17.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[184174]: failed to encode map e468 with expected crc 2023-09-23T18:14:17.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T18:14:17.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[184174]: osdmap e468: 8 total, 7 up, 7 in 2023-09-23T18:14:17.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:14:17.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:14:17.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:17.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/327427915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:17.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/283375008' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:17.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:17 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:17.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:17.343 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:17.405 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:14:17.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:17.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:17 smithi099 ceph-mon[167076]: failed to encode map e468 with expected crc 2023-09-23T18:14:17.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:17 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["6"]}]': finished 2023-09-23T18:14:17.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:17 smithi099 ceph-mon[167076]: osdmap e468: 8 total, 7 up, 7 in 2023-09-23T18:14:17.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:14:17.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:14:17.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:17.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/327427915' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:17.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/283375008' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:17.468 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:17 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:17.470 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:17.502 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:17.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:17.505 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:17.600 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 29 16 53803 53787 115.896 0 - 0.0082216 2023-09-23T18:14:17.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:17.633 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:17.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:17.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:17.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:17.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:17.794 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:17.828 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:17.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:17.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:17.932 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:17.960 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:18.211 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: pgmap v1137: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s; 6271/154158 objects degraded (4.068%) 2023-09-23T18:14:18.211 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:18.212 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: pgmap v1137: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s; 6271/154158 objects degraded (4.068%) 2023-09-23T18:14:18.212 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:18.212 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:14:18.212 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: osdmap e469: 8 total, 7 up, 7 in 2023-09-23T18:14:18.212 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.212 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.212 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.212 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.212 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.213 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.213 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.213 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 6271/154158 objects degraded (4.068%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:18.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: pgmap v1137: 119 pgs: 6 active+undersized+degraded, 21 active+undersized, 92 active+clean; 3.1 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 89 MiB/s wr, 1.43k op/s; 6271/154158 objects degraded (4.068%) 2023-09-23T18:14:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:14:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: osdmap e469: 8 total, 7 up, 7 in 2023-09-23T18:14:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 6271/154158 objects degraded (4.068%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:18.600 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 30 16 54080 54064 112.61 8.65625 0.0189584 0.00887513 2023-09-23T18:14:18.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:14:18.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: osdmap e469: 8 total, 7 up, 7 in 2023-09-23T18:14:18.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: failed to encode map e469 with expected crc 2023-09-23T18:14:18.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 6271/154158 objects degraded (4.068%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:19.090 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:14:19.259 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:19.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:19.265 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:19.367 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:19.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[184174]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[184174]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[184174]: osdmap e470: 8 total, 7 up, 7 in 2023-09-23T18:14:19.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[184174]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[184174]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[184174]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[184174]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[179668]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[179668]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[179668]: osdmap e470: 8 total, 7 up, 7 in 2023-09-23T18:14:19.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[179668]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[179668]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[179668]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:19 smithi055 ceph-mon[179668]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.448 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:19.511 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:19.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:19.521 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:19.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:19 smithi099 ceph-mon[167076]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:19 smithi099 ceph-mon[167076]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:19 smithi099 ceph-mon[167076]: osdmap e470: 8 total, 7 up, 7 in 2023-09-23T18:14:19.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:19 smithi099 ceph-mon[167076]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:19 smithi099 ceph-mon[167076]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:19 smithi099 ceph-mon[167076]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:19 smithi099 ceph-mon[167076]: failed to encode map e470 with expected crc 2023-09-23T18:14:19.600 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 31 16 55200 55184 111.235 70 0.0242731 0.0089853 2023-09-23T18:14:19.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:19.667 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:19.691 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:19.761 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:19.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:19.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:19.868 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:19.943 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:19.999 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:20.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:20.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:20.109 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:20.168 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:20.201 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:20.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:20.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:20.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:20.257 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[179668]: pgmap v1140: 119 pgs: 12 remapped+peering, 5 active+undersized+degraded, 16 active+undersized, 86 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 10 KiB/s rd, 81 MiB/s wr, 1.31k op/s; 3410/160092 objects degraded (2.130%); 20 KiB/s, 1 objects/s recovering 2023-09-23T18:14:20.257 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T18:14:20.258 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[179668]: failed to encode map e471 with expected crc 2023-09-23T18:14:20.258 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[179668]: failed to encode map e471 with expected crc 2023-09-23T18:14:20.258 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[179668]: osdmap e471: 8 total, 7 up, 7 in 2023-09-23T18:14:20.258 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[179668]: failed to encode map e471 with expected crc 2023-09-23T18:14:20.340 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:20.405 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:20.531 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:20.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:20.536 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:20.540 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[184174]: pgmap v1140: 119 pgs: 12 remapped+peering, 5 active+undersized+degraded, 16 active+undersized, 86 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 10 KiB/s rd, 81 MiB/s wr, 1.31k op/s; 3410/160092 objects degraded (2.130%); 20 KiB/s, 1 objects/s recovering 2023-09-23T18:14:20.540 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T18:14:20.540 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[184174]: failed to encode map e471 with expected crc 2023-09-23T18:14:20.540 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[184174]: failed to encode map e471 with expected crc 2023-09-23T18:14:20.540 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[184174]: osdmap e471: 8 total, 7 up, 7 in 2023-09-23T18:14:20.540 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-mon[184174]: failed to encode map e471 with expected crc 2023-09-23T18:14:20.541 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:14:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:14:20] "GET /metrics HTTP/1.1" 200 36625 "" "Prometheus/2.43.0" 2023-09-23T18:14:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:20 smithi099 ceph-mon[167076]: pgmap v1140: 119 pgs: 12 remapped+peering, 5 active+undersized+degraded, 16 active+undersized, 86 active+clean; 3.3 GiB data, 12 GiB used, 614 GiB / 626 GiB avail; 10 KiB/s rd, 81 MiB/s wr, 1.31k op/s; 3410/160092 objects degraded (2.130%); 20 KiB/s, 1 objects/s recovering 2023-09-23T18:14:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:20 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 2 pgs inactive, 3 pgs peering (PG_AVAILABILITY) 2023-09-23T18:14:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:20 smithi099 ceph-mon[167076]: failed to encode map e471 with expected crc 2023-09-23T18:14:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:20 smithi099 ceph-mon[167076]: failed to encode map e471 with expected crc 2023-09-23T18:14:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:20 smithi099 ceph-mon[167076]: osdmap e471: 8 total, 7 up, 7 in 2023-09-23T18:14:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:20 smithi099 ceph-mon[167076]: failed to encode map e471 with expected crc 2023-09-23T18:14:20.601 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 32 16 56280 56264 109.865 67.5 0.00859672 0.00909696 2023-09-23T18:14:20.638 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:20.686 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:20.746 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:20.751 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:20.751 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:20.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:20.928 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:20.984 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:20.992 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:20.992 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:21.093 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:21.172 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:21.251 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:14:21.251 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:14:21.251 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.251 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:56914 9.f e472/459 9:fb4dc4dc:::benchmark_data_smithi055_386467_object56913:head v 472'3422, mlcod=472'3420)", 2023-09-23T18:14:21.251 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:21.232299+0000", 2023-09-23T18:14:21.251 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.015942008000000001, 2023-09-23T18:14:21.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015946457000000001, 2023-09-23T18:14:21.251 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:21.251 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:21.251 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.232299+0000", 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.232299+0000", 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.232299+0000", 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.232331+0000", 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2070999999999998e-05 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:21.253 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.232339+0000", 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4590000000000001e-06 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.232344+0000", 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3070000000000002e-06 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.236768+0000", 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044238699999999999 2023-09-23T18:14:21.254 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.236787+0000", 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9213000000000001e-05 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.248216+0000", 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011429445 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.255 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.248246+0000", 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9519999999999999e-05 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:56915 9.7 e472/459 9:e3eafcfc:::benchmark_data_smithi055_386467_object56914:head v 472'3572, mlcod=472'3569)", 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:21.237037+0000", 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.011204123, 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011336888, 2023-09-23T18:14:21.256 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.237037+0000", 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.237037+0000", 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.237037+0000", 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.237077+0000", 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9561999999999998e-05 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.237083+0000", 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0190000000000003e-06 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.237087+0000", 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3100000000000002e-06 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.238781+0000", 2023-09-23T18:14:21.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016944530000000001 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.238802+0000", 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.022e-05 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:56922 9.4 9:251d2408:::benchmark_data_smithi055_386467_object56921:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T18:14:21.260 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:21.240474+0000", 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0077671709999999998, 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0079824160000000009, 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 56922 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240474+0000", 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240474+0000", 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240473+0000", 2023-09-23T18:14:21.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240500+0000", 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6560999999999999e-05 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240515+0000", 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5007999999999999e-05 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240522+0000", 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5619999999999998e-06 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240800+0000", 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000278256 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:21.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240979+0000", 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017906199999999999 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.241142+0000", 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000162344 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.247952+0000", 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006810333 2023-09-23T18:14:21.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.247993+0000", 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0729000000000003e-05 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:56924 9.4 9:28715ee5:::benchmark_data_smithi055_386467_object56923:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:21.240539+0000", 2023-09-23T18:14:21.266 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0077022200000000001, 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0080223640000000006, 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 56924 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:21.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240539+0000", 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240539+0000", 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240539+0000", 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:21.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240563+0000", 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4598999999999999e-05 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240575+0000", 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1352000000000001e-05 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240581+0000", 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8030000000000002e-06 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.243173+0000", 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025920510000000002 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.243326+0000", 2023-09-23T18:14:21.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000153249 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.243477+0000", 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015141699999999999 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.248000+0000", 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045226559999999999 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.248036+0000", 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6402000000000001e-05 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:56925 9.4 9:2ff01b76:::benchmark_data_smithi055_386467_object56924:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:21.240586+0000", 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0076555260000000002, 2023-09-23T18:14:21.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0080721850000000008, 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 56925 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240586+0000", 2023-09-23T18:14:21.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240586+0000", 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240586+0000", 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:21.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240609+0000", 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3677999999999999e-05 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240616+0000", 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7900000000000002e-06 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.240621+0000", 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7740000000000004e-06 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.245537+0000", 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049166879999999998 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:21.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.245685+0000", 2023-09-23T18:14:21.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014783900000000001 2023-09-23T18:14:21.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:14:21.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.245831+0000", 2023-09-23T18:14:21.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014571399999999999 2023-09-23T18:14:21.277 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.277 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:21.277 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.278 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:56926 9.4 9:2cbb8491:::benchmark_data_smithi055_386467_object56925:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e472)", 2023-09-23T18:14:21.278 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:21.241079+0000", 2023-09-23T18:14:21.278 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.007162351, 2023-09-23T18:14:21.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0076624149999999997, 2023-09-23T18:14:21.278 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 56926 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.241079+0000", 2023-09-23T18:14:21.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.241079+0000", 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.241078+0000", 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:21.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.241100+0000", 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2373000000000001e-05 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.241113+0000", 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.237e-05 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.241120+0000", 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8159999999999994e-06 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.248179+0000", 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0070585939999999996 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:21.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.248352+0000", 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017299700000000001 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:21.248511+0000", 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015909799999999999 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 6 2023-09-23T18:14:21.283 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:14:21.294 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:21 smithi099 ceph-mon[167076]: osdmap e472: 8 total, 7 up, 7 in 2023-09-23T18:14:21.295 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:21 smithi099 ceph-mon[167076]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.341 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:14:21.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[179668]: osdmap e472: 8 total, 7 up, 7 in 2023-09-23T18:14:21.407 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[179668]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.408 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[179668]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.408 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[179668]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.408 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[179668]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.408 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[179668]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.408 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[179668]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[184174]: osdmap e472: 8 total, 7 up, 7 in 2023-09-23T18:14:21.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[184174]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[184174]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[184174]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.408 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[184174]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[184174]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:21 smithi055 ceph-mon[184174]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:21 smithi099 ceph-mon[167076]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:21 smithi099 ceph-mon[167076]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:21 smithi099 ceph-mon[167076]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:21 smithi099 ceph-mon[167076]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:21 smithi099 ceph-mon[167076]: failed to encode map e472 with expected crc 2023-09-23T18:14:21.601 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 33 16 57285 57269 108.439 62.8125 0.018486 0.00921701 2023-09-23T18:14:21.657 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:21.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:21.667 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:21.768 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:21.789 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:14:21.834 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:21.842 INFO:tasks.thrashosds.thrasher:in_osds: [7, 5, 1, 0, 4, 2, 3] out_osds: [6] dead_osds: [2] live_osds: [0, 1, 4, 6, 7, 3, 5] 2023-09-23T18:14:21.842 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:14:21.842 INFO:tasks.thrashosds.thrasher:Adding osd 6 2023-09-23T18:14:21.842 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 6 2023-09-23T18:14:21.979 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:22.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:22.001 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:22.102 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:22.223 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:22.300 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:22.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:22.308 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:22.376 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:14:22.408 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:22.428 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:14:22.470 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:22 smithi055 ceph-mon[179668]: pgmap v1143: 119 pgs: 7 active+clean+remapped, 12 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 81 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 30 KiB/s rd, 56 MiB/s wr, 925 op/s; 10214/164979 objects degraded (6.191%); 8/164979 objects misplaced (0.005%); 469 KiB/s, 3 objects/s recovering 2023-09-23T18:14:22.470 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:22.470 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:22.471 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4061556720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:22.471 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:22 smithi055 ceph-mon[184174]: pgmap v1143: 119 pgs: 7 active+clean+remapped, 12 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 81 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 30 KiB/s rd, 56 MiB/s wr, 925 op/s; 10214/164979 objects degraded (6.191%); 8/164979 objects misplaced (0.005%); 469 KiB/s, 3 objects/s recovering 2023-09-23T18:14:22.471 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:22.471 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:22.471 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4061556720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:22.474 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:22.488 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:14:22.519 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:22.527 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:14:22.539 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:22.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:22.546 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:22.580 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:14:22.603 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 34 16 58182 58166 106.893 56.0625 0.0170435 0.00934902 2023-09-23T18:14:22.617 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:14:22.647 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:22.659 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:14:22.659 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:14:22.660 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:14:22.660 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T18:14:22.660 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:14:22.660 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:14:22.660 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:14:22.660 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:14:22.660 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:14:22.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:22 smithi099 ceph-mon[167076]: pgmap v1143: 119 pgs: 7 active+clean+remapped, 12 remapped+peering, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 81 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 30 KiB/s rd, 56 MiB/s wr, 925 op/s; 10214/164979 objects degraded (6.191%); 8/164979 objects misplaced (0.005%); 469 KiB/s, 3 objects/s recovering 2023-09-23T18:14:22.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:22.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:22.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4061556720' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:22.703 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:22.773 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:22.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:22.778 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:22.784 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:22.878 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:22.931 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:23.048 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:23.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:23.062 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:23.083 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:14:23.083 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:14:23.163 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:23.218 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:23.229 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:23.288 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:23.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:23.299 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:23.400 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:23.462 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:23.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 10214/164979 objects degraded (6.191%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:23.603 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 35 16 59231 59215 105.713 65.5625 0.0065377 0.00945415 2023-09-23T18:14:23.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 10214/164979 objects degraded (6.191%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:23.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 10214/164979 objects degraded (6.191%), 6 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:23.762 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:23.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:23.769 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:23.870 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:23.973 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:24.441 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:24.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:24.449 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:24.550 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:24.603 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 36 16 60320 60304 104.667 68.0625 0.0104511 0.00954808 2023-09-23T18:14:24.607 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:24.668 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:24.677 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:24.677 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:24.778 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:24 smithi099 ceph-mon[167076]: pgmap v1144: 119 pgs: 12 active+clean+remapped, 5 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 82 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 21 KiB/s rd, 49 MiB/s wr, 800 op/s; 14271/167742 objects degraded (8.508%); 3265/167742 objects misplaced (1.946%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:14:24.848 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:24 smithi055 ceph-mon[184174]: pgmap v1144: 119 pgs: 12 active+clean+remapped, 5 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 82 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 21 KiB/s rd, 49 MiB/s wr, 800 op/s; 14271/167742 objects degraded (8.508%); 3265/167742 objects misplaced (1.946%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:14:24.849 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:24 smithi055 ceph-mon[179668]: pgmap v1144: 119 pgs: 12 active+clean+remapped, 5 remapped+peering, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 82 active+clean; 3.4 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 21 KiB/s rd, 49 MiB/s wr, 800 op/s; 14271/167742 objects degraded (8.508%); 3265/167742 objects misplaced (1.946%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:14:24.858 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:25.084 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:14:25.490 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:14:25.525 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:25 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1115149822' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:25.525 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:25 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1115149822' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:25.603 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 37 16 61379 61363 103.627 66.1875 0.00295796 0.00964071 2023-09-23T18:14:25.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:25 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1115149822' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:26.149 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:26.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:26.159 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:26.259 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:26.303 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:26.369 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:26.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:26.379 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:26.480 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:26.549 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:26.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:26 smithi099 ceph-mon[167076]: pgmap v1145: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 8.3 KiB/s rd, 38 MiB/s wr, 620 op/s; 14271/172572 objects degraded (8.270%); 6305/172572 objects misplaced (3.654%); 4.6 MiB/s, 71 objects/s recovering 2023-09-23T18:14:26.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:26 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T18:14:26.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:26 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:14:26.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:26 smithi099 ceph-mon[167076]: failed to encode map e473 with expected crc 2023-09-23T18:14:26.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1115149822' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:14:26.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:26 smithi099 ceph-mon[167076]: osdmap e473: 8 total, 7 up, 7 in 2023-09-23T18:14:26.607 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 38 15 62443 62428 102.641 66.5625 0.00958523 0.00973625 2023-09-23T18:14:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[184174]: pgmap v1145: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 8.3 KiB/s rd, 38 MiB/s wr, 620 op/s; 14271/172572 objects degraded (8.270%); 6305/172572 objects misplaced (3.654%); 4.6 MiB/s, 71 objects/s recovering 2023-09-23T18:14:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T18:14:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:14:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[184174]: failed to encode map e473 with expected crc 2023-09-23T18:14:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1115149822' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:14:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[184174]: osdmap e473: 8 total, 7 up, 7 in 2023-09-23T18:14:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[179668]: pgmap v1145: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 3.5 GiB data, 13 GiB used, 613 GiB / 626 GiB avail; 8.3 KiB/s rd, 38 MiB/s wr, 620 op/s; 14271/172572 objects degraded (8.270%); 6305/172572 objects misplaced (3.654%); 4.6 MiB/s, 71 objects/s recovering 2023-09-23T18:14:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 2 pgs peering) 2023-09-23T18:14:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:14:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[179668]: failed to encode map e473 with expected crc 2023-09-23T18:14:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1115149822' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:14:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:26 smithi055 ceph-mon[179668]: osdmap e473: 8 total, 7 up, 7 in 2023-09-23T18:14:27.607 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 39 16 63590 63574 101.846 71.625 0.0108267 0.00981354 2023-09-23T18:14:28.146 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:28.153 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:28.154 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:28.254 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:28.322 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:28.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:28.403 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:28.404 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:28.504 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:28.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:28 smithi099 ceph-mon[167076]: pgmap v1147: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 7.2 KiB/s rd, 54 MiB/s wr, 867 op/s; 14271/180120 objects degraded (7.923%); 6305/180120 objects misplaced (3.500%); 4.0 MiB/s, 62 objects/s recovering 2023-09-23T18:14:28.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 14271/180120 objects degraded (7.923%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:28.581 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:28.607 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:14:28.608462+0000 min lat: 0.00130179 max lat: 2.18576 avg lat: 0.00990437 2023-09-23T18:14:28.607 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:14:28.608 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 40 16 64625 64609 100.917 64.6875 0.0257067 0.00990437 2023-09-23T18:14:28.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:28.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:28.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:28.747 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:28.802 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:28.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:28.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:28.850 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:28.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:28 smithi055 ceph-mon[184174]: pgmap v1147: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 7.2 KiB/s rd, 54 MiB/s wr, 867 op/s; 14271/180120 objects degraded (7.923%); 6305/180120 objects misplaced (3.500%); 4.0 MiB/s, 62 objects/s recovering 2023-09-23T18:14:28.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 14271/180120 objects degraded (7.923%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:28.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:28 smithi055 ceph-mon[179668]: pgmap v1147: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 3.7 GiB data, 13 GiB used, 612 GiB / 626 GiB avail; 7.2 KiB/s rd, 54 MiB/s wr, 867 op/s; 14271/180120 objects degraded (7.923%); 6305/180120 objects misplaced (3.500%); 4.0 MiB/s, 62 objects/s recovering 2023-09-23T18:14:28.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 14271/180120 objects degraded (7.923%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:28.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:29.002 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:29.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:29.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:29.083 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:29.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:29.246 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:29.293 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:29.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:29.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:29.310 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:29.411 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:29.469 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:29.539 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:29.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:29.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:29.607 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 41 16 65602 65586 99.9453 61.0625 0.0101827 0.0100009 2023-09-23T18:14:29.650 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:29.701 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:30.187 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:14:30.261 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:30.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:30.270 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:30.370 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:30.450 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:30.499 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:30 smithi055 ceph-mon[179668]: pgmap v1148: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 6.4 KiB/s rd, 67 MiB/s wr, 1.08k op/s; 14929/188247 objects degraded (7.931%); 5812/188247 objects misplaced (3.087%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T18:14:30.499 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:14:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:14:30] "GET /metrics HTTP/1.1" 200 36643 "" "Prometheus/2.43.0" 2023-09-23T18:14:30.500 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:30 smithi055 ceph-mon[184174]: pgmap v1148: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 6.4 KiB/s rd, 67 MiB/s wr, 1.08k op/s; 14929/188247 objects degraded (7.931%); 5812/188247 objects misplaced (3.087%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T18:14:30.607 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 42 16 66486 66470 98.8812 55.25 0.0266848 0.0101079 2023-09-23T18:14:30.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:30 smithi099 ceph-mon[167076]: pgmap v1148: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 3.8 GiB data, 14 GiB used, 612 GiB / 626 GiB avail; 6.4 KiB/s rd, 67 MiB/s wr, 1.08k op/s; 14929/188247 objects degraded (7.931%); 5812/188247 objects misplaced (3.087%); 7.1 MiB/s, 112 objects/s recovering 2023-09-23T18:14:31.049 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:31.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:31.058 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:31.159 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:31.186 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:14:31.186 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:14:31.259 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:31.336 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:31.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:31.347 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:31.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:31.526 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:31.538 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.6. 2023-09-23T18:14:31.606 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:31.607 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 43 16 67375 67359 97.874 55.5625 0.00320163 0.0102073 2023-09-23T18:14:31.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:31.611 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:31.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:31 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:14:31.360+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e473 definitely_dead 0 2023-09-23T18:14:31.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/682501228' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T18:14:31.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:31 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T18:14:31.625 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/682501228' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T18:14:31.626 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:31 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T18:14:31.655 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:14:31.712 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:31.739 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:31.814 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/682501228' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T18:14:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:31 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["6"]}]: dispatch 2023-09-23T18:14:31.945 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:31.948 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:31.948 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:31.997 INFO:tasks.thrashosds.thrasher:Added osd 6 2023-09-23T18:14:32.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:32.079 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:32.110 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:32.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:32.113 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:32.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:32.241 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:32.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:32.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:32.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:32.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:32.404 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:32.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:32.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:32.441 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:32.542 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:32.582 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:32.608 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 44 16 67416 67400 95.7083 2.5625 0.00357225 0.0102046 2023-09-23T18:14:32.707 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:32.707 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:32.710 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:32.710 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:32.752 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:14:32.780 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:14:32.807 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:14:32.811 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:32.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:32 smithi099 ceph-mon[167076]: pgmap v1149: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 14929/195771 objects degraded (7.626%); 5812/195771 objects misplaced (2.969%); 6.0 MiB/s, 97 objects/s recovering 2023-09-23T18:14:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:32 smithi099 ceph-mon[167076]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:32 smithi099 ceph-mon[167076]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:32 smithi099 ceph-mon[167076]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:32 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T18:14:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:32 smithi099 ceph-mon[167076]: osdmap e474: 8 total, 7 up, 8 in 2023-09-23T18:14:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:32 smithi099 ceph-mon[167076]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:32 smithi099 ceph-mon[167076]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:32 smithi099 ceph-mon[167076]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.830 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:14:32.839 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[179668]: pgmap v1149: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 14929/195771 objects degraded (7.626%); 5812/195771 objects misplaced (2.969%); 6.0 MiB/s, 97 objects/s recovering 2023-09-23T18:14:32.839 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[179668]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.839 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[179668]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[179668]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T18:14:32.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[179668]: osdmap e474: 8 total, 7 up, 8 in 2023-09-23T18:14:32.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[179668]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[179668]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[179668]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.841 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[184174]: pgmap v1149: 119 pgs: 13 active+clean+remapped, 1 active+recovering+undersized+degraded+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 85 active+clean; 4.0 GiB data, 14 GiB used, 611 GiB / 626 GiB avail; 64 MiB/s wr, 1.02k op/s; 14929/195771 objects degraded (7.626%); 5812/195771 objects misplaced (2.969%); 6.0 MiB/s, 97 objects/s recovering 2023-09-23T18:14:32.841 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[184174]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.841 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[184174]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.841 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[184174]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.841 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["6"]}]': finished 2023-09-23T18:14:32.841 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[184174]: osdmap e474: 8 total, 7 up, 8 in 2023-09-23T18:14:32.841 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[184174]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.841 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[184174]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.841 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:32 smithi055 ceph-mon[184174]: failed to encode map e474 with expected crc 2023-09-23T18:14:32.842 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:32.856 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:14:32.885 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:14:32.908 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:14:32.908 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:14:32.908 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:14:32.909 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T18:14:32.909 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:14:32.909 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:14:32.909 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:14:32.909 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:14:32.909 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:14:33.046 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:33.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:33.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:33.150 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:33.179 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:33.373 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:33.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:33.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:33.394 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:14:33.395 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:14:33.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:33.506 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:33.539 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:33.542 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:33.543 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:33.608 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 45 16 67572 67556 93.7985 9.75 0.00538937 0.0105102 2023-09-23T18:14:33.644 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:33.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: osdmap e475: 8 total, 7 up, 8 in 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.684 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.684 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1147094186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:33.684 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 14929/195771 objects degraded (7.626%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:33.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: osdmap e475: 8 total, 7 up, 8 in 2023-09-23T18:14:33.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1147094186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:33.685 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 14929/195771 objects degraded (7.626%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:33.707 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:33.780 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:33.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:33.788 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:33.788 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: osdmap e475: 8 total, 7 up, 8 in 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.789 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: failed to encode map e475 with expected crc 2023-09-23T18:14:33.790 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1147094186' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:33.790 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 14929/195771 objects degraded (7.626%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:33.889 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:33.952 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:34.026 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:34.035 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:34.036 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:34.137 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:34.206 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:34.276 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:34.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:34.290 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:34.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:34.454 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:34.518 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:34.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:34.523 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:34.594 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:14:34.608 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 46 16 68636 68620 93.2051 66.5 0.0285624 0.0107217 2023-09-23T18:14:34.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:34.686 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:34.732 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:14:34.732 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:14:34.732 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:14:34.732 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:108086 7.22 7:46c8def9:::benchmark_data_smithi055_311528_object37123:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:39.639458+0000", 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 355.08993453900001, 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.86476655800000002, 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108086 2023-09-23T18:14:34.733 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639458+0000", 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639458+0000", 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.734 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639455+0000", 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999971 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639469+0000", 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4528999999999999e-05 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.735 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639496+0000", 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6537e-05 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.639510+0000", 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3509999999999999e-05 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.780408+0000", 2023-09-23T18:14:34.736 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14089864299999999 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:39.780419+0000", 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0786e-05 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.500214+0000", 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.719795454 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.737 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.500334+0000", 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119627 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 2,3", 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.500459+0000", 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012500400000000001 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.738 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.501601+0000", 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011421140000000001 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.502716+0000", 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001114959 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.504110+0000", 2023-09-23T18:14:34.739 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013935799999999999 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.504161+0000", 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1282999999999997e-05 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.504224+0000", 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3471000000000003e-05 2023-09-23T18:14:34.740 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.741 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.741 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.741 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.741 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.741 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:108123 7.b 7:d79fe6bb:::benchmark_data_smithi055_311528_object37160:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:14:34.741 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:40.575125+0000", 2023-09-23T18:14:34.741 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 354.15426770699997, 2023-09-23T18:14:34.741 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95218377899999995, 2023-09-23T18:14:34.742 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.742 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.742 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.742 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.742 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.742 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108123 2023-09-23T18:14:34.742 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.742 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.743 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.743 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575125+0000", 2023-09-23T18:14:34.743 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.743 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.743 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.743 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.743 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575125+0000", 2023-09-23T18:14:34.744 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.744 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.744 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.744 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.744 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575123+0000", 2023-09-23T18:14:34.744 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:34.744 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.744 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.744 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575129+0000", 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0599999999999996e-06 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575144+0000", 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4541e-05 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575152+0000", 2023-09-23T18:14:34.745 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1429999999999998e-06 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575229+0000", 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7212999999999994e-05 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.575241+0000", 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2505e-05 2023-09-23T18:14:34.746 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.747 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.747 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.747 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.520155+0000", 2023-09-23T18:14:34.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.94491320499999998 2023-09-23T18:14:34.747 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.747 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.747 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.747 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.520539+0000", 2023-09-23T18:14:34.747 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000384736 2023-09-23T18:14:34.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:14:34.748 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.521133+0000", 2023-09-23T18:14:34.748 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00059405799999999996 2023-09-23T18:14:34.748 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.748 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.748 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.524812+0000", 2023-09-23T18:14:34.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036780530000000001 2023-09-23T18:14:34.749 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.749 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.749 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.749 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.526787+0000", 2023-09-23T18:14:34.749 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019753050000000001 2023-09-23T18:14:34.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.750 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.750 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.526873+0000", 2023-09-23T18:14:34.750 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6248000000000005e-05 2023-09-23T18:14:34.750 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.750 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.526994+0000", 2023-09-23T18:14:34.751 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120562 2023-09-23T18:14:34.751 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.751 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.751 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.751 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:41.527308+0000", 2023-09-23T18:14:34.752 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00031482999999999999 2023-09-23T18:14:34.752 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.752 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.752 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.752 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.752 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.752 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130809 7.19 7:9fc2251d:::benchmark_data_smithi055_311528_object59846:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:14:34.753 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.096744+0000", 2023-09-23T18:14:34.753 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.63264820000001, 2023-09-23T18:14:34.753 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.038223272, 2023-09-23T18:14:34.753 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.753 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.753 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.753 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.753 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.753 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130809 2023-09-23T18:14:34.753 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096744+0000", 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096744+0000", 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.754 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096743+0000", 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096750+0000", 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5590000000000002e-06 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.755 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096759+0000", 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6590000000000002e-06 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096766+0000", 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6499999999999999e-06 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.756 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.097072+0000", 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00030662300000000001 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.097082+0000", 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5710000000000004e-06 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.127052+0000", 2023-09-23T18:14:34.757 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.029969822 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.127174+0000", 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000122481 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.127480+0000", 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00030583699999999999 2023-09-23T18:14:34.758 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134732+0000", 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072513999999999999 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134740+0000", 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3990000000000004e-06 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.759 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134861+0000", 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000121248 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134922+0000", 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1042000000000003e-05 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.760 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134967+0000", 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5229999999999999e-05 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130810 7.19 7:9eaf2462:::benchmark_data_smithi055_311528_object59847:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.096843+0000", 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.63254903500001, 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.038332319, 2023-09-23T18:14:34.761 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130810 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096843+0000", 2023-09-23T18:14:34.762 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096843+0000", 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096843+0000", 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:34.763 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096847+0000", 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4139999999999996e-06 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096857+0000", 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4029999999999996e-06 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.764 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.096863+0000", 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7070000000000003e-06 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.097102+0000", 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000238958 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.097110+0000", 2023-09-23T18:14:34.765 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3579999999999997e-06 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.129315+0000", 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.032205713 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.129414+0000", 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8196999999999995e-05 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.766 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.129683+0000", 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026979100000000002 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134985+0000", 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0053020360000000004 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.767 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135052+0000", 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6750999999999998e-05 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135093+0000", 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1121999999999997e-05 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135145+0000", 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1456000000000003e-05 2023-09-23T18:14:34.768 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135176+0000", 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1028000000000002e-05 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130818 7.19 7:99afb0dd:::benchmark_data_smithi055_311528_object59855:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:14:34.769 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.108760+0000", 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.62063230899997, 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0266103639999999, 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130818 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.770 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108760+0000", 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108760+0000", 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108758+0000", 2023-09-23T18:14:34.771 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108766+0000", 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5859999999999999e-06 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108780+0000", 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4324e-05 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.772 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108789+0000", 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7849999999999997e-06 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108958+0000", 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000168937 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.773 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.108970+0000", 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2383e-05 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.131100+0000", 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0221301270000001 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.131236+0000", 2023-09-23T18:14:34.774 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013559800000000001 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.131498+0000", 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026210699999999998 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.134993+0000", 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034951940000000001 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.775 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135222+0000", 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00022855 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135265+0000", 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3387999999999999e-05 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135323+0000", 2023-09-23T18:14:34.776 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8149999999999997e-05 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135370+0000", 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7070000000000002e-05 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130851 7.19 7:99593c89:::benchmark_data_smithi055_311528_object59888:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:14:34.777 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.149542+0000", 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.57985019799997, 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98596146699999998, 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130851 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.778 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149542+0000", 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149542+0000", 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149541+0000", 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:34.779 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149547+0000", 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8769999999999998e-06 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149555+0000", 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4030000000000007e-06 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.780 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.149563+0000", 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5510000000000002e-06 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.151045+0000", 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001481818 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.151052+0000", 2023-09-23T18:14:34.781 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1679999999999998e-06 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.132864+0000", 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.981811824 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.132970+0000", 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000106037 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.782 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.133254+0000", 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00028440900000000001 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135198+0000", 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00194392 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.783 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135423+0000", 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000224711 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135456+0000", 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3707000000000001e-05 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135482+0000", 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5802000000000001e-05 2023-09-23T18:14:34.784 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135504+0000", 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1605000000000001e-05 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130859 7.19 7:9cd3c3cf:::benchmark_data_smithi055_311528_object59896:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.160780+0000", 2023-09-23T18:14:34.785 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.56861226299998, 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98387569699999999, 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130859 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.786 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160780+0000", 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160780+0000", 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160779+0000", 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:34.787 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160786+0000", 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8090000000000004e-06 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160797+0000", 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1953999999999999e-05 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.788 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160806+0000", 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2989999999999997e-06 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160869+0000", 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3701999999999994e-05 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160896+0000", 2023-09-23T18:14:34.789 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6448000000000002e-05 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135538+0000", 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97464208399999996 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135632+0000", 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3869000000000005e-05 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.790 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.135828+0000", 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000196205 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.140042+0000", 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0042135619999999997 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144498+0000", 2023-09-23T18:14:34.791 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044563279999999999 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144556+0000", 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8196000000000003e-05 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144616+0000", 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9828999999999999e-05 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.792 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144656+0000", 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9838e-05 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130860 7.19 7:9f617653:::benchmark_data_smithi055_311528_object59897:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.160819+0000", 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.56857349400002, 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98406092999999994, 2023-09-23T18:14:34.793 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130860 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160819+0000", 2023-09-23T18:14:34.794 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160819+0000", 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160818+0000", 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.795 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160822+0000", 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3390000000000001e-06 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160829+0000", 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7130000000000005e-06 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.796 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160835+0000", 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9710000000000001e-06 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160932+0000", 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6945000000000001e-05 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.160942+0000", 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2029999999999998e-06 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.797 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.137518+0000", 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97657627899999999 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.137637+0000", 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119502 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.798 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.137906+0000", 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026830400000000003 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.142645+0000", 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047394070000000002 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144709+0000", 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020640020000000001 2023-09-23T18:14:34.799 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144784+0000", 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5236999999999998e-05 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144836+0000", 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1625999999999997e-05 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.800 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144880+0000", 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3964999999999997e-05 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130895 7.19 7:9b882bdd:::benchmark_data_smithi055_311528_object59932:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.192324+0000", 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.53706826199999, 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.95269991499999995, 2023-09-23T18:14:34.801 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130895 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192324+0000", 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.802 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192324+0000", 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192323+0000", 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.803 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192329+0000", 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8669999999999999e-06 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192340+0000", 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1095e-05 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.804 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192345+0000", 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5330000000000002e-06 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192385+0000", 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9743999999999998e-05 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.192395+0000", 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0068000000000001e-05 2023-09-23T18:14:34.805 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.138986+0000", 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.94659121700000004 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.139056+0000", 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9802999999999996e-05 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.806 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.139230+0000", 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017401100000000001 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144510+0000", 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0052801319999999999 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144750+0000", 2023-09-23T18:14:34.807 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024012999999999999 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144963+0000", 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000212189 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144990+0000", 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7562000000000001e-05 2023-09-23T18:14:34.808 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145024+0000", 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3930000000000002e-05 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130917 7.19 7:99576fc8:::benchmark_data_smithi055_311528_object59954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.210519+0000", 2023-09-23T18:14:34.809 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.518873681, 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.93460085299999995, 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130917 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.810 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210519+0000", 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210519+0000", 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210518+0000", 2023-09-23T18:14:34.811 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210524+0000", 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6829999999999996e-06 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210544+0000", 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0341999999999999e-05 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.812 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210552+0000", 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5799999999999992e-06 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210613+0000", 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0959999999999999e-05 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.813 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.210628+0000", 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4586999999999999e-05 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.140077+0000", 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.929449148 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.140177+0000", 2023-09-23T18:14:34.814 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000100169 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.140402+0000", 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000224243 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144518+0000", 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041164879999999997 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.815 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144927+0000", 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000408565 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145047+0000", 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012084200000000001 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.816 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145082+0000", 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5000999999999999e-05 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145120+0000", 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7154999999999999e-05 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.817 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130975 7.19 7:9b428e91:::benchmark_data_smithi055_311528_object60012:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.264723+0000", 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.46466951600001, 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.88050636100000002, 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130975 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.818 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264723+0000", 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264723+0000", 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264721+0000", 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264728+0000", 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8340000000000001e-06 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264741+0000", 2023-09-23T18:14:34.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2666999999999999e-05 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264748+0000", 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.311e-06 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264805+0000", 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6986e-05 2023-09-23T18:14:34.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.264816+0000", 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1268e-05 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.141359+0000", 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.87654296300000001 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.141441+0000", 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1876e-05 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.141611+0000", 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000169727 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144666+0000", 2023-09-23T18:14:34.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030556799999999999 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145144+0000", 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00047729100000000002 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145160+0000", 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6844999999999999e-05 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145200+0000", 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9512999999999999e-05 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145229+0000", 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9258e-05 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:130994 7.19 7:9a8395d6:::benchmark_data_smithi055_311528_object60031:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e348)", 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:09:03.285435+0000", 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.44395718999999, 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.86362094700000003, 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 130994 2023-09-23T18:14:34.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285435+0000", 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285435+0000", 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285434+0000", 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285440+0000", 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8250000000000001e-06 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285452+0000", 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.188e-05 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285459+0000", 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5379999999999997e-06 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285513+0000", 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4119000000000001e-05 2023-09-23T18:14:34.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.830 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:03.285521+0000", 2023-09-23T18:14:34.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.072e-06 2023-09-23T18:14:34.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.830 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.142673+0000", 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.85715205299999997 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.142795+0000", 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012132499999999999 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:14:34.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.143009+0000", 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000214572 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.144717+0000", 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001707339 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.145278+0000", 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00056135099999999997 2023-09-23T18:14:34.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.148998+0000", 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037199619999999998 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.149030+0000", 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2348000000000003e-05 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:09:04.149056+0000", 2023-09-23T18:14:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5958000000000001e-05 2023-09-23T18:14:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:177036 8.3 8:cf21179b:::benchmark_data_smithi055_346788_object66444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T18:14:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:31.272982+0000", 2023-09-23T18:14:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 123.456410424, 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1620943650000002, 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 177036 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.272982+0000", 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.272982+0000", 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.272981+0000", 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.272989+0000", 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5390000000000007e-06 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.273002+0000", 2023-09-23T18:14:34.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2403e-05 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.273010+0000", 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4500000000000004e-06 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.273073+0000", 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2885999999999994e-05 2023-09-23T18:14:34.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.273250+0000", 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017687599999999999 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 6", 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.273415+0000", 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016536799999999999 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.317430+0000", 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.044014765999999997 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.323066+0000", 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0056359849999999996 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.323083+0000", 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6756999999999999e-05 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.306699+0000", 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98361607299999998 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.841 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.306714+0000", 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5603e-05 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.424828+0000", 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1181138900000001 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for missing object", 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.424961+0000", 2023-09-23T18:14:34.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000132566 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.435003+0000", 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010042697999999999 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.435076+0000", 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2904000000000005e-05 2023-09-23T18:14:34.843 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:177040 8.3 8:c9accaf3:::benchmark_data_smithi055_346788_object66448:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:31.280054+0000", 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 123.44933877, 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1541554619999999, 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.844 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 177040 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.280054+0000", 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.280054+0000", 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.280052+0000", 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.846 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.280074+0000", 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2240999999999999e-05 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.280585+0000", 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000510879 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.280600+0000", 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4771e-05 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.323115+0000", 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.042515237999999997 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.323125+0000", 2023-09-23T18:14:34.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0141e-05 2023-09-23T18:14:34.849 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.849 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.849 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.849 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.306748+0000", 2023-09-23T18:14:34.849 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98362218700000004 2023-09-23T18:14:34.849 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.849 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.849 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.849 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.309181+0000", 2023-09-23T18:14:34.849 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00243297 2023-09-23T18:14:34.850 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.425228+0000", 2023-09-23T18:14:34.850 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1160473820000001 2023-09-23T18:14:34.850 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.425356+0000", 2023-09-23T18:14:34.850 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012844199999999999 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.425484+0000", 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012712800000000001 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.427851+0000", 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023671410000000001 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.428018+0000", 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016713700000000001 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.434065+0000", 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060475090000000004 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.434107+0000", 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1257000000000002e-05 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.434209+0000", 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000102536 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:177050 8.1 8:8d757129:::benchmark_data_smithi055_346788_object66458:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:31.355597+0000", 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 123.373795636, 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0456194539999999, 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:34.854 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 177050 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.355597+0000", 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.355597+0000", 2023-09-23T18:14:34.855 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.355595+0000", 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.355603+0000", 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2120000000000003e-06 2023-09-23T18:14:34.856 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.355617+0000", 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3706999999999999e-05 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.355626+0000", 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8869999999999998e-06 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.857 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.355696+0000", 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0282000000000002e-05 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:31.355712+0000", 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5993000000000001e-05 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.858 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.397761+0000", 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0420482289999999 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.397924+0000", 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000163406 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,2,5", 2023-09-23T18:14:34.859 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.398046+0000", 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000121717 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.400804+0000", 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002758216 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.400886+0000", 2023-09-23T18:14:34.860 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2440000000000004e-05 2023-09-23T18:14:34.861 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.861 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.861 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.861 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.401046+0000", 2023-09-23T18:14:34.861 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000160128 2023-09-23T18:14:34.861 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.861 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.861 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.861 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.401053+0000", 2023-09-23T18:14:34.861 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5300000000000002e-06 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.401088+0000", 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5043000000000003e-05 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.401216+0000", 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000128231 2023-09-23T18:14:34.862 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:177009 8.0 8:04310a62:::benchmark_data_smithi055_346788_object66417:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:32.305233+0000", 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 122.424159135, 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.238562095, 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:14:34.863 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 177009 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.305233+0000", 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.864 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.305233+0000", 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.305231+0000", 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.865 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.305240+0000", 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5609999999999999e-06 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.306363+0000", 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011231640000000001 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.306376+0000", 2023-09-23T18:14:34.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2525e-05 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.313365+0000", 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0069892189999999996 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.313377+0000", 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1887e-05 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.867 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.543685+0000", 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2303080689999999 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.543795+0000", 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011063 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.868 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:177013 8.0 8:0c510983:::benchmark_data_smithi055_346788_object66421:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:32.311784+0000", 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 122.41760853300001, 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2321761200000001, 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:14:34.869 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 177013 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311784+0000", 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.870 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311784+0000", 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311783+0000", 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311790+0000", 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8189999999999992e-06 2023-09-23T18:14:34.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311804+0000", 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3047999999999999e-05 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311817+0000", 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3047000000000001e-05 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.313510+0000", 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016935190000000001 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.313521+0000", 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0601999999999999e-05 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.543921+0000", 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2304001099999999 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.543960+0000", 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9224000000000003e-05 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:177039 8.0 8:0fcdc50a:::benchmark_data_smithi055_346788_object66447:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e421)", 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:32.311825+0000", 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 122.417567224, 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2321812240000001, 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 177039 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.875 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311825+0000", 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311825+0000", 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311825+0000", 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311828+0000", 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.96e-06 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311834+0000", 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6839999999999999e-06 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.311840+0000", 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.541e-06 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.313551+0000", 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0017114280000000001 2023-09-23T18:14:34.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:32.313560+0000", 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2949999999999994e-06 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.543973+0000", 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.230413169 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:33.544006+0000", 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3615000000000003e-05 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:67505 9.0 9:0ae62737:::benchmark_data_smithi055_386467_object67504:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:32.787394+0000", 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.941998229, 2023-09-23T18:14:34.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.859944983, 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67505 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.787394+0000", 2023-09-23T18:14:34.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.787394+0000", 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.787394+0000", 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:34.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.787413+0000", 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.944e-05 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.787422+0000", 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3009999999999995e-06 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.787431+0000", 2023-09-23T18:14:34.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6349999999999995e-06 2023-09-23T18:14:34.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.787473+0000", 2023-09-23T18:14:34.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1692e-05 2023-09-23T18:14:34.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.787479+0000", 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.652e-06 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.625205+0000", 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.83772599400000003 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.625384+0000", 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017902900000000001 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.625575+0000", 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019031899999999999 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.632130+0000", 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0065553520000000004 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.632290+0000", 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000159965 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.887 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.645572+0000", 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013281864000000001 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.645634+0000", 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.177e-05 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.647339+0000", 2023-09-23T18:14:34.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0017055340000000001 2023-09-23T18:14:34.889 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.889 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.889 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.889 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:67514 9.0 9:0e87030b:::benchmark_data_smithi055_386467_object67513:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e475)", 2023-09-23T18:14:34.889 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:32.799899+0000", 2023-09-23T18:14:34.889 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 1.9294929330000001, 2023-09-23T18:14:34.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.85695507400000004, 2023-09-23T18:14:34.890 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:14:34.890 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:34.890 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:14:34.890 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:14:34.890 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:14:34.890 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67514 2023-09-23T18:14:34.890 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.890 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:14:34.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:14:34.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.799899+0000", 2023-09-23T18:14:34.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:14:34.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.799899+0000", 2023-09-23T18:14:34.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:14:34.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:14:34.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.799898+0000", 2023-09-23T18:14:34.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:34.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:14:34.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.799918+0000", 2023-09-23T18:14:34.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9782000000000001e-05 2023-09-23T18:14:34.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.893 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:14:34.893 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.799932+0000", 2023-09-23T18:14:34.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4049999999999999e-05 2023-09-23T18:14:34.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:14:34.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.799938+0000", 2023-09-23T18:14:34.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2419999999999997e-06 2023-09-23T18:14:34.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.799975+0000", 2023-09-23T18:14:34.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6878999999999997e-05 2023-09-23T18:14:34.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:14:34.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:32.799982+0000", 2023-09-23T18:14:34.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2269999999999996e-06 2023-09-23T18:14:34.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:14:34.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.627414+0000", 2023-09-23T18:14:34.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.82743134299999999 2023-09-23T18:14:34.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:14:34.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.627529+0000", 2023-09-23T18:14:34.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000115806 2023-09-23T18:14:34.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:14:34.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.627710+0000", 2023-09-23T18:14:34.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018100899999999999 2023-09-23T18:14:34.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:14:34.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.637077+0000", 2023-09-23T18:14:34.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093669500000000006 2023-09-23T18:14:34.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.898 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.639678+0000", 2023-09-23T18:14:34.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026000810000000002 2023-09-23T18:14:34.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:34.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.655199+0000", 2023-09-23T18:14:34.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015521445 2023-09-23T18:14:34.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:14:34.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.655269+0000", 2023-09-23T18:14:34.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0123999999999995e-05 2023-09-23T18:14:34.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:14:34.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:14:34.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:14:34.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:33.656855+0000", 2023-09-23T18:14:34.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001585468 2023-09-23T18:14:34.901 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.902 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.902 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.902 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:14:34.902 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:14:34.902 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:14:34.903 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[184174]: pgmap v1152: 119 pgs: 1 peering, 7 remapped+peering, 9 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 82 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s; 11092/197871 objects degraded (5.606%); 2780/197871 objects misplaced (1.405%); 5.5 MiB/s, 89 objects/s recovering 2023-09-23T18:14:34.903 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:14:34.903 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[184174]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.903 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[184174]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.903 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[184174]: osdmap e476: 8 total, 7 up, 8 in 2023-09-23T18:14:34.904 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[184174]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.904 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[184174]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.904 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[184174]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.904 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2565592614' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:14:34.904 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[179668]: pgmap v1152: 119 pgs: 1 peering, 7 remapped+peering, 9 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 82 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s; 11092/197871 objects degraded (5.606%); 2780/197871 objects misplaced (1.405%); 5.5 MiB/s, 89 objects/s recovering 2023-09-23T18:14:34.905 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:14:34.905 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[179668]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.905 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[179668]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.905 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[179668]: osdmap e476: 8 total, 7 up, 8 in 2023-09-23T18:14:34.905 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[179668]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.905 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[179668]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.905 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[179668]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.905 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2565592614' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:14:34.906 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:34.906 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:34.906 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:34.907 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:34 smithi099 ceph-mon[167076]: pgmap v1152: 119 pgs: 1 peering, 7 remapped+peering, 9 active+clean+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 82 active+clean; 4.0 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 52 MiB/s wr, 827 op/s; 11092/197871 objects degraded (5.606%); 2780/197871 objects misplaced (1.405%); 5.5 MiB/s, 89 objects/s recovering 2023-09-23T18:14:34.907 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:34 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:14:34.907 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:34 smithi099 ceph-mon[167076]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.907 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:34 smithi099 ceph-mon[167076]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.907 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:34 smithi099 ceph-mon[167076]: osdmap e476: 8 total, 7 up, 8 in 2023-09-23T18:14:34.907 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:34 smithi099 ceph-mon[167076]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.907 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:34 smithi099 ceph-mon[167076]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.907 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:34 smithi099 ceph-mon[167076]: failed to encode map e476 with expected crc 2023-09-23T18:14:34.907 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2565592614' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:14:35.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:35.074 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:35.137 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:35.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:35.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:35.247 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:35.302 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:35.340 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:14:35.355 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:35.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:35.360 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:35.396 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:14:35.461 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:35.507 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:35.565 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:35.565 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[179668]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.565 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2565592614' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:14:35.565 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[179668]: osdmap e477: 8 total, 7 up, 8 in 2023-09-23T18:14:35.566 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[179668]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.566 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[179668]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.569 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:35.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:35.580 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:35.609 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 47 16 69679 69663 92.6074 65.1875 0.00803644 0.0107929 2023-09-23T18:14:35.681 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:35.728 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:35.781 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:35.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:35.785 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:35 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:35 smithi099 ceph-mon[167076]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:35 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2565592614' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:14:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:35 smithi099 ceph-mon[167076]: osdmap e477: 8 total, 7 up, 8 in 2023-09-23T18:14:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:35 smithi099 ceph-mon[167076]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:35 smithi099 ceph-mon[167076]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:35 smithi099 ceph-mon[167076]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:35 smithi099 ceph-mon[167076]: pgmap v1155: 119 pgs: 2 peering, 8 remapped+peering, 8 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 81 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 27 MiB/s wr, 433 op/s; 11972/200973 objects degraded (5.957%); 8/200973 objects misplaced (0.004%); 6.0 MiB/s, 97 objects/s recovering 2023-09-23T18:14:35.886 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[184174]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2565592614' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:14:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[184174]: osdmap e477: 8 total, 7 up, 8 in 2023-09-23T18:14:35.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[184174]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[184174]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[184174]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[184174]: pgmap v1155: 119 pgs: 2 peering, 8 remapped+peering, 8 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 81 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 27 MiB/s wr, 433 op/s; 11972/200973 objects degraded (5.957%); 8/200973 objects misplaced (0.004%); 6.0 MiB/s, 97 objects/s recovering 2023-09-23T18:14:35.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[179668]: failed to encode map e477 with expected crc 2023-09-23T18:14:35.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:35 smithi055 ceph-mon[179668]: pgmap v1155: 119 pgs: 2 peering, 8 remapped+peering, 8 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 81 active+clean; 4.1 GiB data, 15 GiB used, 611 GiB / 626 GiB avail; 27 MiB/s wr, 433 op/s; 11972/200973 objects degraded (5.957%); 8/200973 objects misplaced (0.004%); 6.0 MiB/s, 97 objects/s recovering 2023-09-23T18:14:35.932 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:36.003 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:36.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:36.008 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:36.108 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:36.124 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:14:36.126 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:36.194 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:36.601 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[179668]: osdmap e478: 8 total, 7 up, 8 in 2023-09-23T18:14:36.602 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[179668]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.602 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[179668]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.602 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[179668]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.602 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[179668]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.602 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[179668]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.603 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[179668]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.603 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:36.610 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 48 16 70673 70657 91.9701 62.125 0.0177493 0.0108675 2023-09-23T18:14:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[184174]: osdmap e478: 8 total, 7 up, 8 in 2023-09-23T18:14:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[184174]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[184174]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[184174]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[184174]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[184174]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[184174]: failed to encode map e478 with expected crc 2023-09-23T18:14:36.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:36.995 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:14:36.998 INFO:tasks.thrashosds.thrasher:in_osds: [7, 5, 1, 0, 4, 2, 3, 6] out_osds: [] dead_osds: [2] live_osds: [0, 1, 4, 6, 7, 3, 5] 2023-09-23T18:14:36.998 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:14:36.998 INFO:tasks.thrashosds.thrasher:Reviving osd 2 2023-09-23T18:14:36.998 INFO:tasks.cephadm.osd.2:Restarting osd.2 (starting--it wasn't running)... 2023-09-23T18:14:36.998 DEBUG:teuthology.orchestra.run.smithi055:osd.2> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.2.service 2023-09-23T18:14:37.000 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl start ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T18:14:37.055 INFO:journalctl@ceph.osd.2.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T18:14:37.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:36 smithi099 ceph-mon[167076]: osdmap e478: 8 total, 7 up, 8 in 2023-09-23T18:14:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:36 smithi099 ceph-mon[167076]: failed to encode map e478 with expected crc 2023-09-23T18:14:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:36 smithi099 ceph-mon[167076]: failed to encode map e478 with expected crc 2023-09-23T18:14:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:36 smithi099 ceph-mon[167076]: failed to encode map e478 with expected crc 2023-09-23T18:14:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:36 smithi099 ceph-mon[167076]: failed to encode map e478 with expected crc 2023-09-23T18:14:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:36 smithi099 ceph-mon[167076]: failed to encode map e478 with expected crc 2023-09-23T18:14:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:36 smithi099 ceph-mon[167076]: failed to encode map e478 with expected crc 2023-09-23T18:14:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:37.430 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:37 smithi055 systemd[1]: Starting Ceph osd.2 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:14:37.436 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:14:37.464 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:14:37.510 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:14:37.554 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:14:37.582 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:37.590 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:14:37.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:37.591 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:37.611 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 49 16 71705 71689 91.4069 64.5 0.0189203 0.0109348 2023-09-23T18:14:37.636 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:14:37.664 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:14:37.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:14:37.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:14:37.664 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T18:14:37.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:14:37.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:14:37.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:14:37.664 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:14:37.665 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:14:37.692 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:37.725 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:37.786 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:37.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:37.796 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:37.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:37.917 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:37 smithi055 ceph-mon[184174]: pgmap v1157: 119 pgs: 7 peering, 6 remapped+peering, 6 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 78 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 5973/205770 objects degraded (2.903%); 448/205770 objects misplaced (0.218%); 11 MiB/s, 179 objects/s recovering 2023-09-23T18:14:37.917 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4179278365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:37.917 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:37 smithi055 ceph-mon[179668]: pgmap v1157: 119 pgs: 7 peering, 6 remapped+peering, 6 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 78 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 5973/205770 objects degraded (2.903%); 448/205770 objects misplaced (0.218%); 11 MiB/s, 179 objects/s recovering 2023-09-23T18:14:37.917 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4179278365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:37.953 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:37 smithi099 ceph-mon[167076]: pgmap v1157: 119 pgs: 7 peering, 6 remapped+peering, 6 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 78 active+clean; 4.2 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 47 MiB/s wr, 744 op/s; 5973/205770 objects degraded (2.903%); 448/205770 objects misplaced (0.218%); 11 MiB/s, 179 objects/s recovering 2023-09-23T18:14:37.953 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4179278365' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:37.958 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:38.040 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:38.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:38.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:38.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:38.223 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:38.261 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:14:38.262 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:14:38.327 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:38.613 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 50 16 72724 72708 90.8497 63.6875 0.0130523 0.0110025 2023-09-23T18:14:38.805 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:38.812 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:38.812 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:38.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:38 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:14:38.877 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 5973/205770 objects degraded (2.903%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:38.913 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:38.924 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:38 smithi055 bash[393886]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T18:14:38.924 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:38 smithi055 bash[393886]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T18:14:38.924 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:38 smithi055 bash[393886]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_2 --path /var/lib/ceph/osd/ceph-2 --no-mon-config 2023-09-23T18:14:38.925 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:38 smithi055 bash[393886]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_2 /var/lib/ceph/osd/ceph-2/block 2023-09-23T18:14:38.925 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:38 smithi055 bash[393886]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-2/block 2023-09-23T18:14:38.925 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:38 smithi055 bash[393886]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-1 2023-09-23T18:14:38.925 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:38 smithi055 bash[393886]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-2 2023-09-23T18:14:38.925 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:38 smithi055 bash[393886]: --> ceph-volume lvm activate successful for osd ID: 2 2023-09-23T18:14:38.925 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:38 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:14:38.925 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 5973/205770 objects degraded (2.903%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:38.925 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:38 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:14:38.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 5973/205770 objects degraded (2.903%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:38.990 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:39.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:39.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:39.093 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:39.194 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:39.243 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:39.357 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:39.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:39.369 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:39.470 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:39.528 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:39.584 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:39.591 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:39.592 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:39.613 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 51 16 73749 73733 90.3245 64.0625 0.0143475 0.0110664 2023-09-23T18:14:39.693 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:39.765 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:39.924 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:39.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:39 smithi055 ceph-mon[184174]: pgmap v1158: 119 pgs: 5 peering, 9 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s; 9107/212748 objects degraded (4.281%); 2190/212748 objects misplaced (1.029%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T18:14:39.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:39 smithi055 ceph-mon[179668]: pgmap v1158: 119 pgs: 5 peering, 9 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s; 9107/212748 objects degraded (4.281%); 2190/212748 objects misplaced (1.029%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T18:14:39.933 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:39.933 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:39.936 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:39 smithi099 ceph-mon[167076]: pgmap v1158: 119 pgs: 5 peering, 9 active+clean+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 82 active+clean; 4.3 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 52 MiB/s wr, 826 op/s; 9107/212748 objects degraded (4.281%); 2190/212748 objects misplaced (1.029%); 6.4 MiB/s, 103 objects/s recovering 2023-09-23T18:14:40.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:40.109 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:40.215 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:39 smithi055 podman[394355]: 2023-09-23T18:14:40.215 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:40 smithi055 bash[394355]: fac494a3493ed4889a6ee4b0c675852a52bcbcc1f0c01e1cf4a5407a0a7bb383 2023-09-23T18:14:40.219 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:14:40.246 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:40.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:40.258 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:40.263 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:14:40.359 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:40.411 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:14:40.412 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:14:40.446 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:40.614 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 52 16 74682 74666 89.709 58.3125 0.0109124 0.0111414 2023-09-23T18:14:40.651 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:40 smithi055 systemd[1]: Started Ceph osd.2 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:14:40.651 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:14:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:14:40] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.43.0" 2023-09-23T18:14:40.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2236222300' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:40.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:40 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:40.663 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:14:40.775 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:40.785 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:40.785 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:40.795 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:14:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2236222300' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:40 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:40.850 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:40.886 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:40.926 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2236222300' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:40.926 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:40 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:14:40.926 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[394395]: 2023-09-23T18:14:40.675+0000 7f18174f5640 -1 Falling back to public interface 2023-09-23T18:14:40.930 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:41.494 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:14:41.613 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 53 16 74995 74979 88.3858 19.5625 0.00450409 0.0111406 2023-09-23T18:14:41.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:41.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[184174]: failed to encode map e479 with expected crc 2023-09-23T18:14:41.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:14:41.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[184174]: osdmap e479: 8 total, 7 up, 8 in 2023-09-23T18:14:41.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[184174]: failed to encode map e479 with expected crc 2023-09-23T18:14:41.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[184174]: pgmap v1160: 119 pgs: 9 active+clean+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 9107/220299 objects degraded (4.134%); 1746/220299 objects misplaced (0.793%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T18:14:41.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:41.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[179668]: failed to encode map e479 with expected crc 2023-09-23T18:14:41.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:14:41.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[179668]: osdmap e479: 8 total, 7 up, 8 in 2023-09-23T18:14:41.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[179668]: failed to encode map e479 with expected crc 2023-09-23T18:14:41.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:41 smithi055 ceph-mon[179668]: pgmap v1160: 119 pgs: 9 active+clean+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 9107/220299 objects degraded (4.134%); 1746/220299 objects misplaced (0.793%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T18:14:41.953 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:41 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:41.953 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:41 smithi099 ceph-mon[167076]: failed to encode map e479 with expected crc 2023-09-23T18:14:41.953 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:41 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:14:41.953 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:41 smithi099 ceph-mon[167076]: osdmap e479: 8 total, 7 up, 8 in 2023-09-23T18:14:41.954 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:41 smithi099 ceph-mon[167076]: failed to encode map e479 with expected crc 2023-09-23T18:14:41.954 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:41 smithi099 ceph-mon[167076]: pgmap v1160: 119 pgs: 9 active+clean+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 63 MiB/s wr, 1.00k op/s; 9107/220299 objects degraded (4.134%); 1746/220299 objects misplaced (0.793%); 8.3 MiB/s, 132 objects/s recovering 2023-09-23T18:14:42.099 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:42.106 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:42.107 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:42.208 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:42.281 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:42.614 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 54 16 75982 75966 87.8914 61.6875 0.00667329 0.0113735 2023-09-23T18:14:42.914 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:42 smithi055 ceph-mon[184174]: failed to encode map e480 with expected crc 2023-09-23T18:14:42.914 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:42 smithi055 ceph-mon[184174]: osdmap e480: 8 total, 7 up, 8 in 2023-09-23T18:14:42.914 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:42 smithi055 ceph-mon[184174]: failed to encode map e480 with expected crc 2023-09-23T18:14:42.915 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:42 smithi055 ceph-mon[184174]: failed to encode map e480 with expected crc 2023-09-23T18:14:42.915 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:42 smithi055 ceph-mon[184174]: failed to encode map e480 with expected crc 2023-09-23T18:14:42.915 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:42 smithi055 ceph-mon[179668]: failed to encode map e480 with expected crc 2023-09-23T18:14:42.915 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:42 smithi055 ceph-mon[179668]: osdmap e480: 8 total, 7 up, 8 in 2023-09-23T18:14:42.915 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:42 smithi055 ceph-mon[179668]: failed to encode map e480 with expected crc 2023-09-23T18:14:42.915 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:42 smithi055 ceph-mon[179668]: failed to encode map e480 with expected crc 2023-09-23T18:14:42.915 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:42 smithi055 ceph-mon[179668]: failed to encode map e480 with expected crc 2023-09-23T18:14:43.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:42 smithi099 ceph-mon[167076]: failed to encode map e480 with expected crc 2023-09-23T18:14:43.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:42 smithi099 ceph-mon[167076]: osdmap e480: 8 total, 7 up, 8 in 2023-09-23T18:14:43.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:42 smithi099 ceph-mon[167076]: failed to encode map e480 with expected crc 2023-09-23T18:14:43.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:42 smithi099 ceph-mon[167076]: failed to encode map e480 with expected crc 2023-09-23T18:14:43.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:42 smithi099 ceph-mon[167076]: failed to encode map e480 with expected crc 2023-09-23T18:14:43.463 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:43.614 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 55 16 77073 77057 87.5331 68.1875 0.0227774 0.0114188 2023-09-23T18:14:43.770 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:43 smithi055 ceph-mon[179668]: pgmap v1162: 119 pgs: 9 active+clean+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s; 9351/222696 objects degraded (4.199%); 2038/222696 objects misplaced (0.915%); 3.7 MiB/s, 60 objects/s recovering 2023-09-23T18:14:43.770 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 9107/220299 objects degraded (4.134%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:43.773 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:43 smithi055 ceph-mon[184174]: pgmap v1162: 119 pgs: 9 active+clean+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s; 9351/222696 objects degraded (4.199%); 2038/222696 objects misplaced (0.915%); 3.7 MiB/s, 60 objects/s recovering 2023-09-23T18:14:43.773 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 9107/220299 objects degraded (4.134%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:44.050 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:44.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:44.057 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:44.058 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:43 smithi099 ceph-mon[167076]: pgmap v1162: 119 pgs: 9 active+clean+remapped, 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.5 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 49 MiB/s wr, 786 op/s; 9351/222696 objects degraded (4.199%); 2038/222696 objects misplaced (0.915%); 3.7 MiB/s, 60 objects/s recovering 2023-09-23T18:14:44.058 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 9107/220299 objects degraded (4.134%), 9 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:44.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:44.243 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:44.614 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 56 16 78074 78058 87.0872 62.5625 0.0217065 0.0114774 2023-09-23T18:14:44.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:44.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:44.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:45.043 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:45.079 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:45.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:45.143 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:45.144 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:45.245 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:45.298 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:45.515 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:14:45.617 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 57 16 79063 79047 86.6398 61.8125 0.020043 0.011537 2023-09-23T18:14:46.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:46 smithi099 ceph-mon[167076]: pgmap v1163: 119 pgs: 9 active+clean+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 8471/226731 objects degraded (3.736%); 2559/226731 objects misplaced (1.129%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T18:14:46.350 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:14:46.350 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:14:46.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:46 smithi055 ceph-mon[179668]: pgmap v1163: 119 pgs: 9 active+clean+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 8471/226731 objects degraded (3.736%); 2559/226731 objects misplaced (1.129%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T18:14:46.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:46 smithi055 ceph-mon[184174]: pgmap v1163: 119 pgs: 9 active+clean+remapped, 1 active+undersized+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 87 active+clean; 4.6 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 55 MiB/s wr, 873 op/s; 8471/226731 objects degraded (3.736%); 2559/226731 objects misplaced (1.129%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T18:14:46.617 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 58 16 80075 80059 86.2366 63.25 0.0283843 0.0115916 2023-09-23T18:14:46.712 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:46.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:46.716 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:46.817 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:46.878 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:47.176 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:47.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:47.181 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:47.282 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:47.354 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:47.617 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 59 16 81007 80991 85.7623 58.25 0.0161848 0.0116556 2023-09-23T18:14:47.680 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[394395]: 2023-09-23T18:14:47.423+0000 7f18174f5640 -1 osd.2 461 log_to_monitors true 2023-09-23T18:14:47.685 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:47.691 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:14:47.692 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:47.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:47.794 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:14:47.877 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:47.971 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:14:47.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:47.974 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:14:48.075 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:48.135 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:48 smithi055 ceph-mon[184174]: pgmap v1164: 119 pgs: 9 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 88 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 8471/233247 objects degraded (3.632%); 2559/233247 objects misplaced (1.097%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T18:14:48.135 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:48 smithi055 ceph-mon[184174]: from='osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:14:48.135 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:48 smithi055 ceph-mon[179668]: pgmap v1164: 119 pgs: 9 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 88 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 8471/233247 objects degraded (3.632%); 2559/233247 objects misplaced (1.097%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T18:14:48.136 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:48 smithi055 ceph-mon[179668]: from='osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:14:48.141 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:48.180 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:48.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:48.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:48.284 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:48.300 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:48.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:48 smithi099 ceph-mon[167076]: pgmap v1164: 119 pgs: 9 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 88 active+clean; 4.7 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 853 op/s; 8471/233247 objects degraded (3.632%); 2559/233247 objects misplaced (1.097%); 6.3 MiB/s, 100 objects/s recovering 2023-09-23T18:14:48.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:48 smithi099 ceph-mon[167076]: from='osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-09-23T18:14:48.359 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:48.400 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:14:48 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[394395]: 2023-09-23T18:14:48.130+0000 7f180715f700 -1 osd.2 461 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:14:48.498 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:48.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:48.504 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:48.563 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:14:48.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:48.617 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:14:48.617817+0000 min lat: 0.00130179 max lat: 2.18576 avg lat: 0.0117271 2023-09-23T18:14:48.617 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:14:48.617 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 60 16 81875 81859 85.2372 54.25 0.0106402 0.0117271 2023-09-23T18:14:48.664 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:48.749 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:14:48.751 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:14:48.752 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:14:48.752 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:14:48.752 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:14:48.798 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:14:48.833 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:14:48.865 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:14:48.909 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:14:48.946 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:14:48.975 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:14:48.976 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:14:48.976 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:14:48.976 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.2 2023-09-23T18:14:48.976 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:14:48.976 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:14:48.976 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:14:48.976 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:14:48.976 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:14:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[184174]: from='osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:14:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[184174]: osdmap e481: 8 total, 7 up, 8 in 2023-09-23T18:14:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[184174]: from='osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:14:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[184174]: failed to encode map e481 with expected crc 2023-09-23T18:14:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[184174]: failed to encode map e481 with expected crc 2023-09-23T18:14:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 8471/233247 objects degraded (3.632%), 8 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T18:14:49.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/92984958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:49.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[179668]: from='osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:14:49.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[179668]: osdmap e481: 8 total, 7 up, 8 in 2023-09-23T18:14:49.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[179668]: from='osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:14:49.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[179668]: failed to encode map e481 with expected crc 2023-09-23T18:14:49.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[179668]: failed to encode map e481 with expected crc 2023-09-23T18:14:49.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 8471/233247 objects degraded (3.632%), 8 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T18:14:49.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/92984958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:49.451 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:49.453 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:49 smithi099 ceph-mon[167076]: from='osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207]' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-09-23T18:14:49.453 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:49 smithi099 ceph-mon[167076]: osdmap e481: 8 total, 7 up, 8 in 2023-09-23T18:14:49.453 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:49 smithi099 ceph-mon[167076]: from='osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:14:49.453 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:49 smithi099 ceph-mon[167076]: failed to encode map e481 with expected crc 2023-09-23T18:14:49.453 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:49 smithi099 ceph-mon[167076]: failed to encode map e481 with expected crc 2023-09-23T18:14:49.453 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:49 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 8471/233247 objects degraded (3.632%), 8 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T18:14:49.453 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/92984958' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:49.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:49.454 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:49.555 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:49.587 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:49.617 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 61 16 82479 82463 84.4588 37.75 0.00411746 0.0117499 2023-09-23T18:14:50.320 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:50 smithi099 ceph-mon[167076]: pgmap v1166: 119 pgs: 9 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 88 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 8774/240642 objects degraded (3.646%); 2480/240642 objects misplaced (1.031%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:14:50.320 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:50 smithi099 ceph-mon[167076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:14:50.320 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:50 smithi099 ceph-mon[167076]: failed to encode map e482 with expected crc 2023-09-23T18:14:50.320 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:50 smithi099 ceph-mon[167076]: osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207] boot 2023-09-23T18:14:50.321 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:50 smithi099 ceph-mon[167076]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T18:14:50.321 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:50 smithi099 ceph-mon[167076]: failed to encode map e482 with expected crc 2023-09-23T18:14:50.321 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:50 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:14:50.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:14:50] "GET /metrics HTTP/1.1" 200 36677 "" "Prometheus/2.43.0" 2023-09-23T18:14:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[184174]: pgmap v1166: 119 pgs: 9 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 88 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 8774/240642 objects degraded (3.646%); 2480/240642 objects misplaced (1.031%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:14:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[184174]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:14:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[184174]: failed to encode map e482 with expected crc 2023-09-23T18:14:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[184174]: osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207] boot 2023-09-23T18:14:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[184174]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T18:14:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[184174]: failed to encode map e482 with expected crc 2023-09-23T18:14:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:14:50.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[179668]: pgmap v1166: 119 pgs: 9 active+clean+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 4 active+undersized+degraded, 1 active+recovering+undersized+remapped, 13 active+undersized, 1 active+recovery_wait+degraded+remapped, 88 active+clean; 4.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 53 MiB/s wr, 847 op/s; 8774/240642 objects degraded (3.646%); 2480/240642 objects misplaced (1.031%); 5.9 MiB/s, 94 objects/s recovering 2023-09-23T18:14:50.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[179668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:14:50.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[179668]: failed to encode map e482 with expected crc 2023-09-23T18:14:50.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[179668]: osd.2 [v2:172.21.15.55:6818/1022522207,v1:172.21.15.55:6819/1022522207] boot 2023-09-23T18:14:50.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[179668]: osdmap e482: 8 total, 8 up, 8 in 2023-09-23T18:14:50.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[179668]: failed to encode map e482 with expected crc 2023-09-23T18:14:50.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:50 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-09-23T18:14:50.617 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 62 16 82564 82548 83.1826 5.3125 0.00283125 0.0118478 2023-09-23T18:14:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T18:14:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[184174]: 9.0 starting backfill to osd.2 from (0'0,0'0] MAX to 481'5262 2023-09-23T18:14:51.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T18:14:51.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:51 smithi055 ceph-mon[179668]: 9.0 starting backfill to osd.2 from (0'0,0'0] MAX to 481'5262 2023-09-23T18:14:51.508 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.508 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.508 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.508 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.508 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: osdmap e483: 8 total, 8 up, 8 in 2023-09-23T18:14:51.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: failed to encode map e483 with expected crc 2023-09-23T18:14:51.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:51 smithi099 ceph-mon[167076]: 9.0 starting backfill to osd.2 from (0'0,0'0] MAX to 481'5262 2023-09-23T18:14:51.618 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 63 16 82935 82919 82.2286 23.1875 0.0126525 0.0121561 2023-09-23T18:14:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:52 smithi099 ceph-mon[167076]: pgmap v1169: 119 pgs: 8 peering, 3 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 1 active+recovery_wait+degraded+remapped, 96 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 7975/246177 objects degraded (3.240%); 2838/246177 objects misplaced (1.153%); 8.3 MiB/s, 133 objects/s recovering 2023-09-23T18:14:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:52 smithi099 ceph-mon[167076]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:52 smithi099 ceph-mon[167076]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:52 smithi099 ceph-mon[167076]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:52 smithi099 ceph-mon[167076]: 9.2 starting backfill to osd.2 from (0'0,0'0] MAX to 481'5177 2023-09-23T18:14:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:52 smithi099 ceph-mon[167076]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:52 smithi099 ceph-mon[167076]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T18:14:52.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:52.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[184174]: pgmap v1169: 119 pgs: 8 peering, 3 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 1 active+recovery_wait+degraded+remapped, 96 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 7975/246177 objects degraded (3.240%); 2838/246177 objects misplaced (1.153%); 8.3 MiB/s, 133 objects/s recovering 2023-09-23T18:14:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[184174]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[184174]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[184174]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[184174]: 9.2 starting backfill to osd.2 from (0'0,0'0] MAX to 481'5177 2023-09-23T18:14:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[184174]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[184174]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T18:14:52.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:52.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[179668]: pgmap v1169: 119 pgs: 8 peering, 3 active+clean+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 1 active+recovering+undersized+remapped, 6 active+undersized, 1 active+recovery_wait+degraded+remapped, 96 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 67 MiB/s wr, 1.08k op/s; 7975/246177 objects degraded (3.240%); 2838/246177 objects misplaced (1.153%); 8.3 MiB/s, 133 objects/s recovering 2023-09-23T18:14:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[179668]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[179668]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[179668]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[179668]: 9.2 starting backfill to osd.2 from (0'0,0'0] MAX to 481'5177 2023-09-23T18:14:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[179668]: failed to encode map e484 with expected crc 2023-09-23T18:14:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[179668]: osdmap e484: 8 total, 8 up, 8 in 2023-09-23T18:14:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:14:52.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:14:52.621 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 64 16 83999 83983 81.9792 66.5 0.00704848 0.0121938 2023-09-23T18:14:52.743 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:14:52.743 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65460 7.4 7:2d53859f:::benchmark_data_smithi055_311528_object65459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 486.84154586099999, 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9613000379999992, 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.744 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65460 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896404+0000", 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896620+0000", 2023-09-23T18:14:52.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021651099999999999 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896638+0000", 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8179999999999999e-05 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896646+0000", 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2899999999999997e-06 2023-09-23T18:14:52.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896718+0000", 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2587000000000001e-05 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896729+0000", 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1223000000000001e-05 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.853134+0000", 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9564047319999993 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.857705+0000", 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045713259999999997 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65469 7.4 7:2eb46e41:::benchmark_data_smithi055_311528_object65468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 486.84128421000003, 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 10.043735116000001, 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65469 2023-09-23T18:14:52.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896666+0000", 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896742+0000", 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5432000000000006e-05 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896755+0000", 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3275000000000001e-05 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896762+0000", 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4760000000000003e-06 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896860+0000", 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8120000000000002e-05 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896869+0000", 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1409999999999994e-06 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.858317+0000", 2023-09-23T18:14:52.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9614477719999996 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.863986+0000", 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0056689319999999998 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940290+0000", 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.076304636999999995 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940402+0000", 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111987 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65477 7.4 7:23aaada8:::benchmark_data_smithi055_311528_object65476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 486.84117628899997, 2023-09-23T18:14:52.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 10.046314513, 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65477 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896774+0000", 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:52.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896805+0000", 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0916e-05 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896816+0000", 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0519999999999999e-05 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896820+0000", 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0849999999999999e-06 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896888+0000", 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8637e-05 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896897+0000", 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6279999999999994e-06 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.864614+0000", 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9677164650000005 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.871623+0000", 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070090059999999999 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.882195+0000", 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010572560999999999 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.928966+0000", 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.046770763999999999 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.762 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940462+0000", 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011496312 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940512+0000", 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0118999999999999e-05 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940552+0000", 2023-09-23T18:14:52.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9535e-05 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943089+0000", 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025376470000000001 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65485 7.4 7:246beb72:::benchmark_data_smithi055_311528_object65484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:14:52.764 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 485.64178508200001, 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8502435909999999, 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65485 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096165+0000", 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096281+0000", 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116042 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096295+0000", 2023-09-23T18:14:52.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4524e-05 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096301+0000", 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7490000000000004e-06 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.889363+0000", 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7930615480000007 2023-09-23T18:14:52.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.896004+0000", 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066411439999999999 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.906662+0000", 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010657645 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940400+0000", 2023-09-23T18:14:52.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.033738490000000003 2023-09-23T18:14:52.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943179+0000", 2023-09-23T18:14:52.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002778675 2023-09-23T18:14:52.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943222+0000", 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2966999999999997e-05 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943292+0000", 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0043999999999995e-05 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946410+0000", 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031180190000000001 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65486 7.4 7:28b4ee06:::benchmark_data_smithi055_311528_object65485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 485.61653460299999, 2023-09-23T18:14:52.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8402319929999997, 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65486 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:14:52.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121415+0000", 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:52.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121466+0000", 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1007000000000001e-05 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121481+0000", 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5287000000000001e-05 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121489+0000", 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5010000000000004e-06 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946571+0000", 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8250813830000006 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946747+0000", 2023-09-23T18:14:52.776 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017653100000000001 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946947+0000", 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019961800000000001 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.954051+0000", 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0071042989999999997 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958525+0000", 2023-09-23T18:14:52.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044738959999999998 2023-09-23T18:14:52.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958638+0000", 2023-09-23T18:14:52.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113078 2023-09-23T18:14:52.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958698+0000", 2023-09-23T18:14:52.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0207000000000001e-05 2023-09-23T18:14:52.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961649+0000", 2023-09-23T18:14:52.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002950214 2023-09-23T18:14:52.779 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.780 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.780 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.780 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65487 7.4 7:2b3c8c2f:::benchmark_data_smithi055_311528_object65486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:14:52.780 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:14:52.780 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 485.61645053699999, 2023-09-23T18:14:52.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8433001999999998, 2023-09-23T18:14:52.780 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.781 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.781 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.781 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.781 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.781 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65487 2023-09-23T18:14:52.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.781 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:14:52.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:14:52.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121500+0000", 2023-09-23T18:14:52.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:14:52.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121563+0000", 2023-09-23T18:14:52.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3126000000000005e-05 2023-09-23T18:14:52.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121579+0000", 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6418000000000001e-05 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121585+0000", 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.575e-06 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949470+0000", 2023-09-23T18:14:52.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8278846659999992 2023-09-23T18:14:52.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949632+0000", 2023-09-23T18:14:52.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016232099999999999 2023-09-23T18:14:52.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949768+0000", 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013578700000000001 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.956292+0000", 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065241910000000004 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.786 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958569+0000", 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022767289999999999 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961776+0000", 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032076660000000001 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961847+0000", 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1137999999999996e-05 2023-09-23T18:14:52.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964801+0000", 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029533990000000002 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65498 7.4 7:27d4ee95:::benchmark_data_smithi055_311528_object65497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:14:52.788 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 485.58194102300001, 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8116268140000003, 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65498 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156009+0000", 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:52.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156082+0000", 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3093e-05 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156096+0000", 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4465e-05 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156104+0000", 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4120000000000004e-06 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.951850+0000", 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7957462389999996 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.951998+0000", 2023-09-23T18:14:52.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014826300000000001 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.952148+0000", 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000149524 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961673+0000", 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095253839999999996 2023-09-23T18:14:52.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961735+0000", 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2036e-05 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964887+0000", 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003152092 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964960+0000", 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2601999999999994e-05 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.967637+0000", 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026772739999999999 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108071 7.21 7:85ca81b0:::benchmark_data_smithi055_311528_object37108:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 373.193326442, 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9624768260000001, 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108071 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.796 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544624+0000", 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544628+0000", 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.698e-06 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.798 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544634+0000", 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3849999999999999e-06 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544638+0000", 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7560000000000001e-06 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581968+0000", 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037330020999999998 2023-09-23T18:14:52.799 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581975+0000", 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1470000000000004e-06 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487605+0000", 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9056295090000002 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.800 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487615+0000", 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0036e-05 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502365+0000", 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0147500899999999 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.801 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502491+0000", 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125716 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502677+0000", 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018593700000000001 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504929+0000", 2023-09-23T18:14:52.802 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022522219999999999 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504942+0000", 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2636999999999999e-05 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506803+0000", 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018613709999999999 2023-09-23T18:14:52.803 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506851+0000", 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8578000000000001e-05 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507102+0000", 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025014900000000002 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.804 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108080 7.21 7:85d5fb8e:::benchmark_data_smithi055_311528_object37117:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 373.19329497500001, 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.962840377, 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.805 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108080 2023-09-23T18:14:52.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.806 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:14:52.806 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.806 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.806 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.806 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.806 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544658+0000", 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9589999999999998e-06 2023-09-23T18:14:52.807 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544663+0000", 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8069999999999999e-06 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544667+0000", 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1790000000000001e-06 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.808 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581992+0000", 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037325061999999999 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581997+0000", 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5480000000000003e-06 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.809 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487637+0000", 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9056401429999998 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487642+0000", 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.503810+0000", 2023-09-23T18:14:52.810 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.01616774 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.503955+0000", 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000145291 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504156+0000", 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020058200000000001 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.811 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507170+0000", 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003013511 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507182+0000", 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.218e-05 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.812 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507234+0000", 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2349000000000003e-05 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507288+0000", 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4307e-05 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507497+0000", 2023-09-23T18:14:52.813 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000208233 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108094 7.1 7:819d9c99:::benchmark_data_smithi055_311528_object37131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 372.96993549299998, 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7316361810000001, 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.814 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108094 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.815 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768015+0000", 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.816 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768019+0000", 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3989999999999997e-06 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768032+0000", 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2707e-05 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768037+0000", 2023-09-23T18:14:52.817 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2490000000000001e-06 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768081+0000", 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3421000000000003e-05 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768093+0000", 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1989e-05 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.818 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.479295+0000", 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71120236800000003 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.479313+0000", 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8434e-05 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.819 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492409+0000", 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.013095748 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492585+0000", 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000176007 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492874+0000", 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00028859600000000001 2023-09-23T18:14:52.820 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.497854+0000", 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049804380000000002 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499314+0000", 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014593449999999999 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.821 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499349+0000", 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5802e-05 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499422+0000", 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2278000000000002e-05 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.822 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499652+0000", 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00023028400000000001 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108097 7.21 7:87bce1cb:::benchmark_data_smithi055_311528_object37134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 372.240694502, 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0105963070000001, 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.823 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108097 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.824 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497255+0000", 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.825 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497262+0000", 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.029e-06 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497275+0000", 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3349e-05 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497284+0000", 2023-09-23T18:14:52.826 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3639999999999999e-06 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497329+0000", 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5105000000000003e-05 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497335+0000", 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8989999999999998e-06 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.827 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487661+0000", 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99032663499999996 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487667+0000", 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.446e-06 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.828 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504977+0000", 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.017310079 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505119+0000", 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000142707 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:14:52.829 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505315+0000", 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019584499999999999 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507193+0000", 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001877213 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507527+0000", 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00033465300000000002 2023-09-23T18:14:52.830 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507614+0000", 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7287000000000006e-05 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507647+0000", 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2481999999999998e-05 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.831 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507853+0000", 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000206092 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108108 7.21 7:860e36b6:::benchmark_data_smithi055_311528_object37145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 372.18970192299997, 2023-09-23T18:14:52.832 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.963000675, 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108108 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:14:52.833 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548248+0000", 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.834 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548257+0000", 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1549999999999996e-06 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548278+0000", 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1229e-05 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.835 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548286+0000", 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0150000000000003e-06 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548337+0000", 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1131999999999998e-05 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548346+0000", 2023-09-23T18:14:52.836 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8070000000000006e-06 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487685+0000", 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.939339378 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487691+0000", 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7100000000000004e-06 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.837 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505973+0000", 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018282141 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506088+0000", 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011443299999999999 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.838 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506199+0000", 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011132 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507536+0000", 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013369 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507578+0000", 2023-09-23T18:14:52.839 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1971000000000002e-05 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.510980+0000", 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034023870000000002 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.511029+0000", 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9064999999999999e-05 2023-09-23T18:14:52.840 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.511250+0000", 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022076600000000001 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:129162 8.2 8:454f52aa:::benchmark_data_smithi055_346788_object18570:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:28.516171+0000", 2023-09-23T18:14:52.841 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 204.22178038800001, 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.068282459, 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129162 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.842 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516171+0000", 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516171+0000", 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.843 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516170+0000", 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516174+0000", 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3680000000000004e-06 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516183+0000", 2023-09-23T18:14:52.844 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9120000000000004e-06 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516187+0000", 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2209999999999999e-06 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516232+0000", 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4530000000000002e-05 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.845 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516388+0000", 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000156572 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516568+0000", 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000179927 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.846 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.523118+0000", 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006549933 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.528581+0000", 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054625460000000004 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.847 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.546371+0000", 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017789800000000001 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.546381+0000", 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0311e-05 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.584362+0000", 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.037981313 2023-09-23T18:14:52.848 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.584453+0000", 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1094999999999994e-05 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:129175 8.2 8:4646f8c9:::benchmark_data_smithi055_346788_object18583:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T18:14:52.849 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:28.554311+0000", 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 204.18364069699999, 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041236976, 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129175 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554311+0000", 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554311+0000", 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554309+0000", 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554315+0000", 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1929999999999998e-06 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554328+0000", 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2975e-05 2023-09-23T18:14:52.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554334+0000", 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3580000000000002e-06 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554380+0000", 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6189000000000001e-05 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554387+0000", 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8369999999999999e-06 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.585150+0000", 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0307628820000001 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.585231+0000", 2023-09-23T18:14:52.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1168000000000006e-05 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.585373+0000", 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014207099999999999 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.589993+0000", 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046203149999999998 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.590043+0000", 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0148000000000002e-05 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.594330+0000", 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0042865430000000003 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.594380+0000", 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9521000000000002e-05 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.595548+0000", 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001168012 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.857 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:129177 8.2 8:4d02a5a8:::benchmark_data_smithi055_346788_object18585:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:28.562962+0000", 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 204.17498934700001, 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.033361969, 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129177 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562962+0000", 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562962+0000", 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562960+0000", 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562967+0000", 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8240000000000002e-06 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562981+0000", 2023-09-23T18:14:52.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4100999999999999e-05 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562986+0000", 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2780000000000001e-06 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.563035+0000", 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8353000000000003e-05 2023-09-23T18:14:52.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.563046+0000", 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1547e-05 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.587649+0000", 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0246025240000001 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.587767+0000", 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011790000000000001 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.587856+0000", 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9696000000000003e-05 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.590055+0000", 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021986890000000002 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.590123+0000", 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8114000000000005e-05 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.595596+0000", 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054732649999999997 2023-09-23T18:14:52.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.595635+0000", 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8549e-05 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.596324+0000", 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000688907 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.865 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177038 8.2 8:4ced54d7:::benchmark_data_smithi055_346788_object66446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 141.46282300799999, 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1583805069999999, 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:52.866 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177038 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275127+0000", 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275133+0000", 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5910000000000001e-06 2023-09-23T18:14:52.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275146+0000", 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2302e-05 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275163+0000", 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7689e-05 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.282482+0000", 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0073186709999999997 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.282498+0000", 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5841999999999998e-05 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430240+0000", 2023-09-23T18:14:52.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1477423680000001 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430343+0000", 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102289 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5,6", 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430425+0000", 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2150000000000005e-05 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.432859+0000", 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024338519999999998 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.432958+0000", 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9440999999999999e-05 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433088+0000", 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013031100000000001 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433396+0000", 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00030761699999999999 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433437+0000", 2023-09-23T18:14:52.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1242999999999998e-05 2023-09-23T18:14:52.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433509+0000", 2023-09-23T18:14:52.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1559999999999999e-05 2023-09-23T18:14:52.874 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.874 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.874 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213401 8.e 8:7ecd3558:::benchmark_data_smithi055_346788_object102809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 92.494333163999997, 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0290741050000001, 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213401 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.875 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:14:52.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:52.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243620+0000", 2023-09-23T18:14:52.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9649999999999998e-06 2023-09-23T18:14:52.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243623+0000", 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6179999999999999e-06 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243626+0000", 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.294e-06 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243677+0000", 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1668000000000001e-05 2023-09-23T18:14:52.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243686+0000", 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7730000000000005e-06 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.235935+0000", 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99224838100000001 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.236043+0000", 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108855 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.236169+0000", 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012573700000000001 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.253075+0000", 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016906304 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272511+0000", 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019435859 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272626+0000", 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115215 2023-09-23T18:14:52.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272667+0000", 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0590000000000003e-05 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272692+0000", 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5111999999999999e-05 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.883 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213417 8.e 8:74aaae11:::benchmark_data_smithi055_346788_object102825:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.256100+0000", 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 92.481851168999995, 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0167222810000001, 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:52.884 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213417 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256100+0000", 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256100+0000", 2023-09-23T18:14:52.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256099+0000", 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256105+0000", 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7309999999999998e-06 2023-09-23T18:14:52.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256115+0000", 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9690000000000006e-06 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256120+0000", 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4079999999999997e-06 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256168+0000", 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7728000000000001e-05 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.256175+0000", 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7729999999999999e-06 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.238168+0000", 2023-09-23T18:14:52.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98199206699999997 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.238291+0000", 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012327700000000001 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.238419+0000", 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012834700000000001 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.253082+0000", 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014662616 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272593+0000", 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019511170000000001 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272741+0000", 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000147937 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272786+0000", 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5126000000000001e-05 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272822+0000", 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6334000000000001e-05 2023-09-23T18:14:52.891 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213425 8.e 8:7b72c2bb:::benchmark_data_smithi055_346788_object102833:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.262931+0000", 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 92.47501991, 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0101257320000001, 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.892 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213425 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262931+0000", 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262931+0000", 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262930+0000", 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262937+0000", 2023-09-23T18:14:52.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8140000000000003e-06 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262953+0000", 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6552e-05 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.262960+0000", 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5919999999999997e-06 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.263035+0000", 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5650999999999998e-05 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.263048+0000", 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2648e-05 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.240478+0000", 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97742947300000005 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.240607+0000", 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012959000000000001 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.240742+0000", 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013444499999999999 2023-09-23T18:14:52.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.255239+0000", 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014497701 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272871+0000", 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017631862000000002 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272978+0000", 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106356 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.273017+0000", 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9708e-05 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.273057+0000", 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9841e-05 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:69908 9.c 9:3e409780:::benchmark_data_smithi055_386467_object69907:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e478)", 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:14:35.853346+0000", 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 16.884604898999999, 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.093677790999999996, 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:52.900 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 69908 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.853346+0000", 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.853346+0000", 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.853345+0000", 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.853485+0000", 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013987900000000001 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.913546+0000", 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.06006119 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.913577+0000", 2023-09-23T18:14:52.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1479000000000001e-05 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.924286+0000", 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010708353 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.924402+0000", 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116032 2023-09-23T18:14:52.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.924531+0000", 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012898400000000001 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.930057+0000", 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055267509999999999 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.930902+0000", 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00084501599999999997 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.946926+0000", 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016023167000000001 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:52.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.946982+0000", 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6688999999999999e-05 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:35.947024+0000", 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1933e-05 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:52.907 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:52.908 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:14:53.626 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 65 16 84873 84857 81.5527 54.625 0.0227935 0.0122569 2023-09-23T18:14:53.724 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:14:53.997 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:54.001 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:54.001 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:54.102 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:54.171 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:54.265 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:54.271 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:54 smithi055 ceph-mon[184174]: pgmap v1171: 119 pgs: 8 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+undersized, 1 active+recovery_wait+degraded+remapped, 104 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 8170/247878 objects degraded (3.296%); 3038/247878 objects misplaced (1.226%); 10 MiB/s, 162 objects/s recovering 2023-09-23T18:14:54.271 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:54 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 8170/247878 objects degraded (3.296%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T18:14:54.272 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:54 smithi055 ceph-mon[179668]: pgmap v1171: 119 pgs: 8 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+undersized, 1 active+recovery_wait+degraded+remapped, 104 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 8170/247878 objects degraded (3.296%); 3038/247878 objects misplaced (1.226%); 10 MiB/s, 162 objects/s recovering 2023-09-23T18:14:54.272 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:54 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 8170/247878 objects degraded (3.296%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T18:14:54.272 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:14:54.273 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:14:54.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:54.275 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:54.375 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:54.403 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:54.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:54.440 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:54 smithi099 ceph-mon[167076]: pgmap v1171: 119 pgs: 8 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 2 active+undersized, 1 active+recovery_wait+degraded+remapped, 104 active+clean; 5.0 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 50 MiB/s wr, 803 op/s; 8170/247878 objects degraded (3.296%); 3038/247878 objects misplaced (1.226%); 10 MiB/s, 162 objects/s recovering 2023-09-23T18:14:54.440 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:54 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 8170/247878 objects degraded (3.296%), 4 pgs degraded, 3 pgs undersized (PG_DEGRADED) 2023-09-23T18:14:54.441 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:54.441 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:54.520 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:14:54.543 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:54.575 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:54.616 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:54.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:54.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:54.626 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 66 16 85643 85627 81.0463 48.125 0.00369469 0.012288 2023-09-23T18:14:54.721 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:54.751 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:55.374 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:55.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:55.378 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:55.479 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:55.530 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:55 smithi099 ceph-mon[167076]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:55 smithi099 ceph-mon[167076]: osdmap e485: 8 total, 8 up, 8 in 2023-09-23T18:14:55.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:55 smithi099 ceph-mon[167076]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:55 smithi099 ceph-mon[167076]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:55 smithi099 ceph-mon[167076]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.531 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:55 smithi099 ceph-mon[167076]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.548 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:55.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:55.623 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:55.624 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:55.626 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 67 16 85968 85952 80.1402 20.3125 0.0219849 0.0124734 2023-09-23T18:14:55.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[179668]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[179668]: osdmap e485: 8 total, 8 up, 8 in 2023-09-23T18:14:55.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[179668]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[179668]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[179668]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[179668]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[184174]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[184174]: osdmap e485: 8 total, 8 up, 8 in 2023-09-23T18:14:55.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[184174]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[184174]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[184174]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:55 smithi055 ceph-mon[184174]: failed to encode map e485 with expected crc 2023-09-23T18:14:55.726 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:55.812 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:55.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:55.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:55.884 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:55.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:56.065 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:56.119 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:14:56.119 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:14:56.119 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.119 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:86436 9.1 9:870ea6b6:::benchmark_data_smithi055_386467_object86435:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2023-09-23T18:14:56.119 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:14:56.111134+0000", 2023-09-23T18:14:56.119 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0039835089999999997, 2023-09-23T18:14:56.120 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040210070000000001, 2023-09-23T18:14:56.120 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:56.120 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:14:56.120 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:56.120 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:14:56.120 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:14:56.120 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 86436 2023-09-23T18:14:56.120 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.121 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:56.121 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.121 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:56.121 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.111134+0000", 2023-09-23T18:14:56.121 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:56.121 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.121 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.122 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:56.122 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.111134+0000", 2023-09-23T18:14:56.122 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:56.122 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.122 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.122 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:56.122 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.111132+0000", 2023-09-23T18:14:56.123 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:14:56.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.123 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.123 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:56.123 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.111201+0000", 2023-09-23T18:14:56.123 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8873000000000003e-05 2023-09-23T18:14:56.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.124 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.124 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:56.124 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.111220+0000", 2023-09-23T18:14:56.124 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8700999999999999e-05 2023-09-23T18:14:56.124 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.124 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.124 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:56.124 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.111227+0000", 2023-09-23T18:14:56.125 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6690000000000006e-06 2023-09-23T18:14:56.125 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.125 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.125 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:56.125 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.111283+0000", 2023-09-23T18:14:56.125 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5478e-05 2023-09-23T18:14:56.125 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.125 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.126 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:56.126 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.111426+0000", 2023-09-23T18:14:56.126 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000143277 2023-09-23T18:14:56.126 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.126 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.126 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:14:56.126 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.111590+0000", 2023-09-23T18:14:56.127 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016402800000000001 2023-09-23T18:14:56.127 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.127 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:56.127 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.127 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.127 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.127 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.36552.0:86427 9.1 e486/483 ondisk, result = 0)", 2023-09-23T18:14:56.127 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:14:56.114424+0000", 2023-09-23T18:14:56.128 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.00069293499999999999, 2023-09-23T18:14:56.128 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00081733100000000003, 2023-09-23T18:14:56.128 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:56.128 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:14:56.128 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:56.128 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.128 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:56.128 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114424+0000", 2023-09-23T18:14:56.128 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:56.128 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114424+0000", 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114424+0000", 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.129 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114428+0000", 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8359999999999999e-06 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114446+0000", 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7238999999999999e-05 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:56.130 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114452+0000", 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8119999999999996e-06 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.115189+0000", 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00073780299999999998 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.115196+0000", 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5320000000000003e-06 2023-09-23T18:14:56.131 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.36552.0:86430 9.1 e486/483)", 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:14:56.114547+0000", 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.00057041100000000005, 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000755363, 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:14:56.132 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:56.133 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.133 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:56.133 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114547+0000", 2023-09-23T18:14:56.133 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:56.133 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.133 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.133 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:56.133 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114547+0000", 2023-09-23T18:14:56.134 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:56.134 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.134 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.134 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:56.134 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114546+0000", 2023-09-23T18:14:56.134 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:56.134 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.135 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.135 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:56.135 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114549+0000", 2023-09-23T18:14:56.135 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9440000000000001e-06 2023-09-23T18:14:56.135 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.135 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.135 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:56.135 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114571+0000", 2023-09-23T18:14:56.136 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2019e-05 2023-09-23T18:14:56.136 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.136 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.136 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:56.136 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114576+0000", 2023-09-23T18:14:56.136 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.544e-06 2023-09-23T18:14:56.136 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.136 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:56.137 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.137 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.137 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.137 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:86430 9.1 9:8a2cb20a:::benchmark_data_smithi055_386467_object86429:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2023-09-23T18:14:56.137 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:14:56.105286+0000", 2023-09-23T18:14:56.137 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0098307189999999999, 2023-09-23T18:14:56.137 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010147679999999999, 2023-09-23T18:14:56.138 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:56.138 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:14:56.138 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:56.138 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:14:56.138 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:14:56.138 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 86430 2023-09-23T18:14:56.138 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.139 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:56.139 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.139 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:56.139 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105286+0000", 2023-09-23T18:14:56.139 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:56.139 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.139 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.139 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:56.140 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105286+0000", 2023-09-23T18:14:56.140 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:56.140 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.140 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.140 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:56.140 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105284+0000", 2023-09-23T18:14:56.140 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:14:56.141 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.141 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.141 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:56.141 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105353+0000", 2023-09-23T18:14:56.141 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8916000000000002e-05 2023-09-23T18:14:56.141 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.141 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.141 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:56.142 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105372+0000", 2023-09-23T18:14:56.142 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8861000000000001e-05 2023-09-23T18:14:56.142 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.142 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.142 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:56.142 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105379+0000", 2023-09-23T18:14:56.142 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4259999999999997e-06 2023-09-23T18:14:56.143 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.143 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.143 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:56.143 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105431+0000", 2023-09-23T18:14:56.143 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1532000000000001e-05 2023-09-23T18:14:56.143 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.143 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.144 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:56.144 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105599+0000", 2023-09-23T18:14:56.144 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000167722 2023-09-23T18:14:56.144 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.144 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.144 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:14:56.144 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105751+0000", 2023-09-23T18:14:56.144 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000152733 2023-09-23T18:14:56.145 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.145 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.145 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:56.145 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.108519+0000", 2023-09-23T18:14:56.145 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027672199999999999 2023-09-23T18:14:56.145 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.145 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.145 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:56.146 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.114500+0000", 2023-09-23T18:14:56.146 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0059813649999999998 2023-09-23T18:14:56.146 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.146 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.146 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:56.146 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.115372+0000", 2023-09-23T18:14:56.146 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00087245900000000002 2023-09-23T18:14:56.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.147 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.147 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:14:56.147 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.115424+0000", 2023-09-23T18:14:56.147 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1860000000000002e-05 2023-09-23T18:14:56.147 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.147 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:56.147 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:86432 9.b 9:ddbb62a9:::benchmark_data_smithi055_386467_object86431:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e486)", 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:14:56.105398+0000", 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0097191580000000003, 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01014486, 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 86432 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.148 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105398+0000", 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105398+0000", 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:14:56.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105397+0000", 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105455+0000", 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7818000000000001e-05 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105463+0000", 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9310000000000008e-06 2023-09-23T18:14:56.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105468+0000", 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.848e-06 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105596+0000", 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012843899999999999 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.151 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105719+0000", 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012250600000000001 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5", 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.105853+0000", 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000134272 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:14:56.152 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.108433+0000", 2023-09-23T18:14:56.153 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025797110000000002 2023-09-23T18:14:56.153 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:14:56.153 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:14:56.153 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:14:56.153 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:14:56.115093+0000", 2023-09-23T18:14:56.153 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066602609999999998 2023-09-23T18:14:56.153 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.153 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:14:56.153 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.153 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:14:56.154 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:14:56.154 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 5 2023-09-23T18:14:56.154 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:14:56.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:56.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:56.155 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:56.256 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:56.273 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:14:56.304 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:56 smithi099 ceph-mon[167076]: pgmap v1173: 119 pgs: 8 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 34 MiB/s wr, 538 op/s; 8168/250179 objects degraded (3.265%); 3038/250179 objects misplaced (1.214%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:14:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:56 smithi099 ceph-mon[167076]: failed to encode map e486 with expected crc 2023-09-23T18:14:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:56 smithi099 ceph-mon[167076]: osdmap e486: 8 total, 8 up, 8 in 2023-09-23T18:14:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:56 smithi099 ceph-mon[167076]: failed to encode map e486 with expected crc 2023-09-23T18:14:56.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:56 smithi099 ceph-mon[167076]: failed to encode map e486 with expected crc 2023-09-23T18:14:56.379 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:56.386 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:56.387 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:56.487 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:56.566 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:56 smithi055 ceph-mon[179668]: pgmap v1173: 119 pgs: 8 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 34 MiB/s wr, 538 op/s; 8168/250179 objects degraded (3.265%); 3038/250179 objects misplaced (1.214%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:14:56.567 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:56 smithi055 ceph-mon[179668]: failed to encode map e486 with expected crc 2023-09-23T18:14:56.567 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:56 smithi055 ceph-mon[179668]: osdmap e486: 8 total, 8 up, 8 in 2023-09-23T18:14:56.567 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:56 smithi055 ceph-mon[179668]: failed to encode map e486 with expected crc 2023-09-23T18:14:56.567 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:56 smithi055 ceph-mon[179668]: failed to encode map e486 with expected crc 2023-09-23T18:14:56.567 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:56 smithi055 ceph-mon[184174]: pgmap v1173: 119 pgs: 8 peering, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+undersized+remapped, 1 active+recovery_wait+degraded+remapped, 107 active+clean; 5.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 34 MiB/s wr, 538 op/s; 8168/250179 objects degraded (3.265%); 3038/250179 objects misplaced (1.214%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:14:56.567 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:56 smithi055 ceph-mon[184174]: failed to encode map e486 with expected crc 2023-09-23T18:14:56.567 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:56 smithi055 ceph-mon[184174]: osdmap e486: 8 total, 8 up, 8 in 2023-09-23T18:14:56.567 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:56 smithi055 ceph-mon[184174]: failed to encode map e486 with expected crc 2023-09-23T18:14:56.568 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:56 smithi055 ceph-mon[184174]: failed to encode map e486 with expected crc 2023-09-23T18:14:56.580 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:56.626 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 68 16 86906 86890 79.8239 58.625 0.0259895 0.012523 2023-09-23T18:14:56.657 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:14:56.658 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:56.663 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:56.664 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:56.765 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:14:56.855 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:56.926 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:14:56.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:56.932 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:14:57.033 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:14:57.095 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:57.144 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:57.172 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:14:57.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:57.177 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:14:57.273 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:57 smithi099 ceph-mon[167076]: failed to encode map e486 with expected crc 2023-09-23T18:14:57.274 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3323396125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:57.274 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:57 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:57.278 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:14:57.293 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:14:57.334 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:57.398 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:14:57.398 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:57 smithi055 ceph-mon[179668]: failed to encode map e486 with expected crc 2023-09-23T18:14:57.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3323396125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:57.399 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:57 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:57.399 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:57 smithi055 ceph-mon[184174]: failed to encode map e486 with expected crc 2023-09-23T18:14:57.399 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3323396125' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:57.399 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:57 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:14:57.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:57.402 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:14:57.503 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:57.515 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:14:57.563 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:57.626 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 69 16 87823 87807 79.4977 57.3125 0.0160041 0.0125738 2023-09-23T18:14:57.756 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:57.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:57.764 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:57.864 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:57.908 INFO:tasks.thrashosds.thrasher:in_osds: [7, 5, 1, 0, 4, 2, 3, 6] out_osds: [] dead_osds: [] live_osds: [0, 1, 4, 6, 7, 3, 5, 2] 2023-09-23T18:14:57.909 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:14:57.909 INFO:tasks.thrashosds.thrasher:Removing osd 5, in_osds are: [7, 5, 1, 0, 4, 2, 3, 6] 2023-09-23T18:14:57.909 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 5 2023-09-23T18:14:57.935 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:58.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[179668]: pgmap v1175: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 116 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 544 op/s; 8168/255975 objects degraded (3.191%); 2551/255975 objects misplaced (0.997%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T18:14:58.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:58.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[179668]: failed to encode map e487 with expected crc 2023-09-23T18:14:58.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:14:58.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[179668]: osdmap e487: 8 total, 8 up, 8 in 2023-09-23T18:14:58.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[179668]: failed to encode map e487 with expected crc 2023-09-23T18:14:58.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 8168/255975 objects degraded (3.191%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:58.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[184174]: pgmap v1175: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 116 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 544 op/s; 8168/255975 objects degraded (3.191%); 2551/255975 objects misplaced (0.997%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T18:14:58.292 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:58.292 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[184174]: failed to encode map e487 with expected crc 2023-09-23T18:14:58.292 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:14:58.292 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[184174]: osdmap e487: 8 total, 8 up, 8 in 2023-09-23T18:14:58.292 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[184174]: failed to encode map e487 with expected crc 2023-09-23T18:14:58.292 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 8168/255975 objects degraded (3.191%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:58.458 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:58.492 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:58 smithi099 ceph-mon[167076]: pgmap v1175: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+degraded+remapped, 116 active+clean; 5.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 34 MiB/s wr, 544 op/s; 8168/255975 objects degraded (3.191%); 2551/255975 objects misplaced (0.997%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T18:14:58.492 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:58 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:14:58.493 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:58 smithi099 ceph-mon[167076]: failed to encode map e487 with expected crc 2023-09-23T18:14:58.493 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:58 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:14:58.493 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:58 smithi099 ceph-mon[167076]: osdmap e487: 8 total, 8 up, 8 in 2023-09-23T18:14:58.493 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:58 smithi099 ceph-mon[167076]: failed to encode map e487 with expected crc 2023-09-23T18:14:58.493 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 8168/255975 objects degraded (3.191%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:14:58.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:58.495 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:58.595 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:14:58.627 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 70 16 88826 88810 79.2575 62.6875 0.0259909 0.012613 2023-09-23T18:14:58.667 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:58.760 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:14:58.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:58.763 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:14:58.811 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:14:58.853 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:14:58.864 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:58.866 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:14:58.885 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:14:58.925 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:14:58.944 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:58.971 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:14:59.018 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:14:59.041 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:14:59.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:59.049 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:14:59.068 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:14:59.110 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:14:59.120 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:14:59.120 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:14:59.121 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:14:59.121 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:14:59.121 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:14:59.121 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:14:59.121 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:14:59.121 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:14:59.150 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:14:59.206 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:59.338 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:14:59.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:59.343 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:14:59.397 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:14:59.397 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:14:59.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:14:59 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2774000155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:59.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:14:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2774000155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:59.444 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:14:59.497 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:14:59.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:14:59 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2774000155' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:14:59.627 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 71 16 89752 89736 78.9564 57.875 0.0228178 0.012661 2023-09-23T18:14:59.781 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:14:59.788 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:14:59.789 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:14:59.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:14:59.889 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:14:59.919 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:14:59.968 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:00.039 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:00.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:00.043 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:00.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:00.217 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:00.286 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:00.291 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:00.291 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:00.306 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:00 smithi055 ceph-mon[179668]: pgmap v1177: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 116 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s; 8134/261150 objects degraded (3.115%); 8.1 MiB/s, 128 objects/s recovering 2023-09-23T18:15:00.306 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:00 smithi055 ceph-mon[184174]: pgmap v1177: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 116 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s; 8134/261150 objects degraded (3.115%); 8.1 MiB/s, 128 objects/s recovering 2023-09-23T18:15:00.392 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:00.456 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:00.511 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:00.519 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:00.520 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:00.523 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:00 smithi099 ceph-mon[167076]: pgmap v1177: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+recovering+degraded+remapped, 116 active+clean; 5.3 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 46 MiB/s wr, 737 op/s; 8134/261150 objects degraded (3.115%); 8.1 MiB/s, 128 objects/s recovering 2023-09-23T18:15:00.621 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:00.627 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 72 15 90696 90681 78.6801 59.0625 0.0125425 0.0127043 2023-09-23T18:15:00.668 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:00.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:15:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:15:00] "GET /metrics HTTP/1.1" 200 36483 "" "Prometheus/2.43.0" 2023-09-23T18:15:00.741 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:00.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:00.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:00.768 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:00.855 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:00.881 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:15:00.929 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:00.998 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:01.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:01.008 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:01.108 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:01.155 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:15:00.857+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e487 definitely_dead 0 2023-09-23T18:15:01.166 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:01.353 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.5. 2023-09-23T18:15:01.416 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:01 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3527743124' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:15:01.416 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:01 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:15:01.416 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:01 smithi055 ceph-mon[179668]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T18:15:01.416 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:01 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3527743124' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:15:01.416 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:01 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:15:01.416 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:01 smithi055 ceph-mon[184174]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T18:15:01.418 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:01 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3527743124' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:15:01.418 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:01 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["5"]}]: dispatch 2023-09-23T18:15:01.418 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:01 smithi099 ceph-mon[167076]: Client client.admin marked osd.5 out, while it was still marked up 2023-09-23T18:15:01.627 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 73 16 91381 91365 78.188 42.75 0.0181754 0.0127374 2023-09-23T18:15:01.783 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:15:01.813 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:01.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:01.817 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:01.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:01.951 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:02.035 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:02.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:02.038 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:02.139 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:02.172 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:02.316 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:02.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:02.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:02.424 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:15:02.424 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:15:02.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[184174]: pgmap v1178: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 116 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 8134/268965 objects degraded (3.024%); 7.2 MiB/s, 113 objects/s recovering 2023-09-23T18:15:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[184174]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[184174]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T18:15:02.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[184174]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T18:15:02.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[184174]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.431 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[184174]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[184174]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[184174]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.432 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[179668]: pgmap v1178: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 116 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 8134/268965 objects degraded (3.024%); 7.2 MiB/s, 113 objects/s recovering 2023-09-23T18:15:02.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[179668]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[179668]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T18:15:02.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[179668]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T18:15:02.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[179668]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.433 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[179668]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[179668]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:02 smithi055 ceph-mon[179668]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.459 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:02.479 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:02.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:02.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:02.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:02.603 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:02.628 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 74 16 91381 91365 77.1318 0 - 0.0127374 2023-09-23T18:15:02.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:02 smithi099 ceph-mon[167076]: pgmap v1178: 119 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 1 active+recovering+degraded+remapped, 116 active+clean; 5.5 GiB data, 20 GiB used, 696 GiB / 715 GiB avail; 58 MiB/s wr, 922 op/s; 8134/268965 objects degraded (3.024%); 7.2 MiB/s, 113 objects/s recovering 2023-09-23T18:15:02.629 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:02 smithi099 ceph-mon[167076]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:02 smithi099 ceph-mon[167076]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:02 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["5"]}]': finished 2023-09-23T18:15:02.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:02 smithi099 ceph-mon[167076]: osdmap e488: 8 total, 8 up, 7 in 2023-09-23T18:15:02.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:02 smithi099 ceph-mon[167076]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:02 smithi099 ceph-mon[167076]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:02 smithi099 ceph-mon[167076]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.630 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:02 smithi099 ceph-mon[167076]: failed to encode map e488 with expected crc 2023-09-23T18:15:02.632 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:02.666 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:02.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:02.669 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:02.770 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:02.801 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:02.958 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:15:03.094 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:03.098 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:03.098 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:03.199 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:03.236 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:03.286 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:03.290 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:03.290 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:03.369 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:15:03.391 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:03.458 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:03.513 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.513 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.513 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.513 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T18:15:03.513 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.513 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.513 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.514 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.514 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3248493606' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:03.514 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:03.514 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 8134/268965 objects degraded (3.024%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:03.514 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.514 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.515 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.515 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T18:15:03.515 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.515 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.515 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.515 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.516 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3248493606' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:03.516 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:03.516 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 8134/268965 objects degraded (3.024%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:03.553 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:03.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:03.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:03.627 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 75 16 91403 91387 76.1221 0.6875 0.00761624 0.01298 2023-09-23T18:15:03.643 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:15:03.667 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:03.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: osdmap e489: 8 total, 8 up, 7 in 2023-09-23T18:15:03.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: failed to encode map e489 with expected crc 2023-09-23T18:15:03.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3248493606' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:03.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:03.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 8134/268965 objects degraded (3.024%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:03.731 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:03.737 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:15:03.767 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:15:03.797 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:15:03.831 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:15:03.865 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:15:03.890 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:15:03.921 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:15:04.143 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:15:04.145 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:15:04.145 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:15:04.146 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:15:04.146 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:15:04.146 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:15:04.146 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:15:04.146 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:15:04.146 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:15:04.188 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:04.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:04.193 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:04.294 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:04.349 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:04.543 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:04.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:04.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: pgmap v1181: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 6 remapped+peering, 1 active+recovering+degraded+remapped, 110 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.0 KiB/s rd, 53 MiB/s wr, 856 op/s; 7951/271305 objects degraded (2.931%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: failed to encode map e490 with expected crc 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: failed to encode map e490 with expected crc 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: failed to encode map e490 with expected crc 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: 9.b starting backfill to osd.3 from (0'0,473'4207] MIN to 487'5783 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 487'5618 2023-09-23T18:15:04.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 487'5714 2023-09-23T18:15:04.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: 9.6 starting backfill to osd.0 from (0'0,473'4248] MIN to 487'5718 2023-09-23T18:15:04.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 487'5662 2023-09-23T18:15:04.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: 9.8 starting backfill to osd.1 from (0'0,0'0] MAX to 487'5502 2023-09-23T18:15:04.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: 9.f starting backfill to osd.7 from (0'0,0'0] MAX to 487'5613 2023-09-23T18:15:04.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: 9.7 starting backfill to osd.6 from (0'0,0'0] MAX to 487'5712 2023-09-23T18:15:04.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: 9.9 starting backfill to osd.6 from (0'0,0'0] MAX to 487'5645 2023-09-23T18:15:04.550 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/978299266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:04.550 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: pgmap v1181: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 6 remapped+peering, 1 active+recovering+degraded+remapped, 110 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.0 KiB/s rd, 53 MiB/s wr, 856 op/s; 7951/271305 objects degraded (2.931%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T18:15:04.550 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T18:15:04.550 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: failed to encode map e490 with expected crc 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: failed to encode map e490 with expected crc 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: failed to encode map e490 with expected crc 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: 9.b starting backfill to osd.3 from (0'0,473'4207] MIN to 487'5783 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 487'5618 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 487'5714 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: 9.6 starting backfill to osd.0 from (0'0,473'4248] MIN to 487'5718 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 487'5662 2023-09-23T18:15:04.551 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: 9.8 starting backfill to osd.1 from (0'0,0'0] MAX to 487'5502 2023-09-23T18:15:04.552 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: 9.f starting backfill to osd.7 from (0'0,0'0] MAX to 487'5613 2023-09-23T18:15:04.552 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: 9.7 starting backfill to osd.6 from (0'0,0'0] MAX to 487'5712 2023-09-23T18:15:04.552 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: 9.9 starting backfill to osd.6 from (0'0,0'0] MAX to 487'5645 2023-09-23T18:15:04.552 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:04 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/978299266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:04.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: pgmap v1181: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 6 remapped+peering, 1 active+recovering+degraded+remapped, 110 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 7.0 KiB/s rd, 53 MiB/s wr, 856 op/s; 7951/271305 objects degraded (2.931%); 6.9 MiB/s, 109 objects/s recovering 2023-09-23T18:15:04.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 3 pgs peering (PG_AVAILABILITY) 2023-09-23T18:15:04.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:15:04.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: failed to encode map e490 with expected crc 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: failed to encode map e490 with expected crc 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: failed to encode map e490 with expected crc 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: osdmap e490: 8 total, 8 up, 7 in 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: 9.b starting backfill to osd.3 from (0'0,473'4207] MIN to 487'5783 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: 9.d starting backfill to osd.4 from (0'0,0'0] MAX to 487'5618 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: 9.a starting backfill to osd.6 from (0'0,0'0] MAX to 487'5714 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: 9.6 starting backfill to osd.0 from (0'0,473'4248] MIN to 487'5718 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: 9.5 starting backfill to osd.1 from (0'0,0'0] MAX to 487'5662 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: 9.8 starting backfill to osd.1 from (0'0,0'0] MAX to 487'5502 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: 9.f starting backfill to osd.7 from (0'0,0'0] MAX to 487'5613 2023-09-23T18:15:04.553 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: 9.7 starting backfill to osd.6 from (0'0,0'0] MAX to 487'5712 2023-09-23T18:15:04.554 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: 9.9 starting backfill to osd.6 from (0'0,0'0] MAX to 487'5645 2023-09-23T18:15:04.554 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:04 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/978299266' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:04.627 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 76 16 92097 92081 75.6913 43.375 0.0143055 0.0132072 2023-09-23T18:15:04.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:04.718 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:04.980 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:04.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:04.989 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:05.089 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:05.184 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:05.254 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:05.265 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:05.265 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:05.366 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:05.465 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:05.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:05 smithi099 ceph-mon[167076]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T18:15:05.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:05 smithi099 ceph-mon[167076]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:05 smithi099 ceph-mon[167076]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:05 smithi099 ceph-mon[167076]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:05 smithi099 ceph-mon[167076]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:05 smithi099 ceph-mon[167076]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:05 smithi099 ceph-mon[167076]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:05 smithi099 ceph-mon[167076]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:05 smithi099 ceph-mon[167076]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.584 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:05.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:05.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:05.627 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 77 16 92939 92923 75.3918 52.625 0.0517034 0.0132572 2023-09-23T18:15:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[184174]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T18:15:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[184174]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[184174]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[184174]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[184174]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[184174]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[184174]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[184174]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[184174]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[179668]: osdmap e491: 8 total, 8 up, 7 in 2023-09-23T18:15:05.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[179668]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[179668]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[179668]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[179668]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[179668]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[179668]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[179668]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:05 smithi055 ceph-mon[179668]: failed to encode map e491 with expected crc 2023-09-23T18:15:05.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:05.732 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:15:05.732 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:15:05.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.732 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:93004 9.d 9:bcf1adf4:::benchmark_data_smithi055_386467_object93003:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.732 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.711312+0000", 2023-09-23T18:15:05.732 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.015231109, 2023-09-23T18:15:05.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015278194, 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 93004 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711312+0000", 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711312+0000", 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711311+0000", 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:15:05.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711372+0000", 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1455e-05 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711385+0000", 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.271e-05 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.735 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711393+0000", 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8129999999999996e-06 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711446+0000", 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2967000000000003e-05 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711588+0000", 2023-09-23T18:15:05.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014192999999999999 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,5", 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711789+0000", 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000200998 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:93005 9.7 9:e52cc579:::benchmark_data_smithi055_386467_object93004:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.737 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.714332+0000", 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.012210477000000001, 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012344981, 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 93005 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.714332+0000", 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.714332+0000", 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.714331+0000", 2023-09-23T18:15:05.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.714369+0000", 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7793999999999998e-05 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.714379+0000", 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0304e-05 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.714385+0000", 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8220000000000003e-06 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.718307+0000", 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0039218739999999997 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.718437+0000", 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012969500000000001 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.718602+0000", 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016556700000000001 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:05.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.725779+0000", 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0071764380000000003 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:93011 9.3 9:ca97be02:::benchmark_data_smithi055_386467_object93010:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.716610+0000", 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0099323599999999995, 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010149981000000001, 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.743 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 93011 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716610+0000", 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716610+0000", 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716609+0000", 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716654+0000", 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4749999999999997e-05 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716670+0000", 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6147e-05 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716677+0000", 2023-09-23T18:15:05.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1629999999999999e-06 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.717071+0000", 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00039395100000000002 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.717191+0000", 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000120019 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.717324+0000", 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013282800000000001 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.722836+0000", 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055127489999999999 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:05.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.723337+0000", 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00050004200000000002 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:93012 9.9 9:99c3954c:::benchmark_data_smithi055_386467_object93011:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.716695+0000", 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0098483059999999994, 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01015257, 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.749 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 93012 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716695+0000", 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716695+0000", 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716694+0000", 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716747+0000", 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3168999999999999e-05 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716754+0000", 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1790000000000002e-06 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716759+0000", 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3789999999999999e-06 2023-09-23T18:15:05.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.718318+0000", 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001559572 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.718483+0000", 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000164215 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5,6", 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.718644+0000", 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000161204 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.724695+0000", 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0060510670000000003 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:15:05.754 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.724750+0000", 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5507000000000002e-05 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.726823+0000", 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020727150000000001 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.755 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:93013 9.9 9:95a5d1fd:::benchmark_data_smithi055_386467_object93012:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.721847+0000", 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0046960680000000003, 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0050902170000000002, 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 93013 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.721847+0000", 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.721847+0000", 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.721845+0000", 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.721925+0000", 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9399000000000003e-05 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.721938+0000", 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3426e-05 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.721945+0000", 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1339999999999998e-06 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.721996+0000", 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0893000000000002e-05 2023-09-23T18:15:05.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.722121+0000", 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012510399999999999 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5,6", 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.722296+0000", 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017514000000000001 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.725589+0000", 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032922020000000001 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.726745+0000", 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001155887 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.761 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:92994 9.7 9:e90aa224:::benchmark_data_smithi055_386467_object92993:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.698658+0000", 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.027885294000000001, 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028363824999999999, 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 92994 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.762 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.698658+0000", 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.698658+0000", 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.763 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.698656+0000", 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.698685+0000", 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9371999999999999e-05 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.698697+0000", 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1078e-05 2023-09-23T18:15:05.764 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.698703+0000", 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1489999999999999e-06 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.708178+0000", 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0094754139999999997 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.765 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.708328+0000", 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014986700000000001 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.708506+0000", 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017840299999999999 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.715743+0000", 2023-09-23T18:15:05.766 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0072366399999999999 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:92997 9.5 9:a478c535:::benchmark_data_smithi055_386467_object92996:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.700573+0000", 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.025969664, 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.026528541999999999, 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.767 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 92997 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.700573+0000", 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.768 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.700573+0000", 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.700572+0000", 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.700629+0000", 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7398999999999997e-05 2023-09-23T18:15:05.769 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.700641+0000", 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2292e-05 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.700648+0000", 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7850000000000003e-06 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.770 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.706987+0000", 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0063385009999999999 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.707151+0000", 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016444699999999999 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.707354+0000", 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020246500000000001 2023-09-23T18:15:05.771 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710193+0000", 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002839776 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711930+0000", 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001736544 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.772 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.718280+0000", 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0063504750000000004 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:92998 9.7 9:e66628f9:::benchmark_data_smithi055_386467_object92997:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.702733+0000", 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.023809466000000001, 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.024459109999999999, 2023-09-23T18:15:05.773 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 92998 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.702733+0000", 2023-09-23T18:15:05.774 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.775 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.702733+0000", 2023-09-23T18:15:05.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.775 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.775 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.775 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.775 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.702732+0000", 2023-09-23T18:15:05.775 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:15:05.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.777 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.777 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.777 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.704615+0000", 2023-09-23T18:15:05.777 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018825669999999999 2023-09-23T18:15:05.777 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.704627+0000", 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2713e-05 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.704634+0000", 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5209999999999999e-06 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.778 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711035+0000", 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064010930000000001 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711128+0000", 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2757000000000007e-05 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711226+0000", 2023-09-23T18:15:05.779 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8727000000000005e-05 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.725756+0000", 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014529698000000001 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:92999 9.7 9:e348e308:::benchmark_data_smithi055_386467_object92998:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.780 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.705734+0000", 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.020809290000000001, 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021543070000000001, 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 92999 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.781 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.705734+0000", 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.705734+0000", 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.705732+0000", 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:15:05.782 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.706322+0000", 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00058999699999999998 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.706334+0000", 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1885000000000001e-05 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.783 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.706341+0000", 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4139999999999999e-06 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.713575+0000", 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0072342159999999999 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.713702+0000", 2023-09-23T18:15:05.784 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012744000000000001 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.713854+0000", 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015126899999999999 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.725766+0000", 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011911978 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.785 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:93001 9.7 9:e9b99a99:::benchmark_data_smithi055_386467_object93000:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.710832+0000", 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.015710898000000001, 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016524898, 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.786 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 93001 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710832+0000", 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.787 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710832+0000", 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710831+0000", 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710871+0000", 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0620000000000001e-05 2023-09-23T18:15:05.788 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710882+0000", 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.081e-05 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710888+0000", 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8100000000000003e-06 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.789 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.715775+0000", 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048868469999999997 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.715907+0000", 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013265999999999999 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.716068+0000", 2023-09-23T18:15:05.790 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016108 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.725772+0000", 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097037890000000009 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:93003 9.9 9:9abf81d2:::benchmark_data_smithi055_386467_object93002:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e492)", 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:05.710895+0000", 2023-09-23T18:15:05.791 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.015647357000000001, 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016543219000000001, 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 93003 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:05.792 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710895+0000", 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710895+0000", 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710895+0000", 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.793 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710928+0000", 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2867999999999998e-05 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710936+0000", 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9959999999999993e-06 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.794 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.710941+0000", 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9300000000000002e-06 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711402+0000", 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000461552 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711529+0000", 2023-09-23T18:15:05.795 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012670100000000001 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5,6", 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.711701+0000", 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017167300000000001 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.720981+0000", 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0092807080000000004 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.796 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.720991+0000", 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5580000000000006e-06 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:05.726780+0000", 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057889619999999999 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:05.797 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:15:05.798 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 11 2023-09-23T18:15:05.798 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:15:05.798 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:05.895 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:15:05.939 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:05.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:05.946 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:06.046 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:06.112 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:06.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:06.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:06.200 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:06.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:06.320 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:15:06.401 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:06.488 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:06.498 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:06.498 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:06.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:06 smithi099 ceph-mon[167076]: pgmap v1184: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 17 remapped+peering, 1 active+recovering+degraded+remapped, 99 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10 KiB/s rd, 26 MiB/s wr, 422 op/s; 7951/273936 objects degraded (2.903%); 6.0 MiB/s, 92 objects/s recovering 2023-09-23T18:15:06.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:06 smithi099 ceph-mon[167076]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:06 smithi099 ceph-mon[167076]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:06 smithi099 ceph-mon[167076]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:06 smithi099 ceph-mon[167076]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:06 smithi099 ceph-mon[167076]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T18:15:06.501 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:06 smithi099 ceph-mon[167076]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.502 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:06 smithi099 ceph-mon[167076]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.502 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:06 smithi099 ceph-mon[167076]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.502 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:06 smithi099 ceph-mon[167076]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.564 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:15:06.599 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:06.627 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 78 16 93821 93805 75.132 55.125 0.0124941 0.0133029 2023-09-23T18:15:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[184174]: pgmap v1184: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 17 remapped+peering, 1 active+recovering+degraded+remapped, 99 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10 KiB/s rd, 26 MiB/s wr, 422 op/s; 7951/273936 objects degraded (2.903%); 6.0 MiB/s, 92 objects/s recovering 2023-09-23T18:15:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[184174]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[184174]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[184174]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[184174]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[184174]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T18:15:06.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[184174]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[184174]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[184174]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[184174]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[179668]: pgmap v1184: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 17 remapped+peering, 1 active+recovering+degraded+remapped, 99 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 10 KiB/s rd, 26 MiB/s wr, 422 op/s; 7951/273936 objects degraded (2.903%); 6.0 MiB/s, 92 objects/s recovering 2023-09-23T18:15:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[179668]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[179668]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[179668]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[179668]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[179668]: osdmap e492: 8 total, 8 up, 7 in 2023-09-23T18:15:06.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[179668]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[179668]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[179668]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:06 smithi055 ceph-mon[179668]: failed to encode map e492 with expected crc 2023-09-23T18:15:06.684 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:06.714 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:06.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:06.754 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:06.754 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:06.848 INFO:tasks.thrashosds.thrasher:in_osds: [7, 1, 0, 4, 2, 3, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 4, 6, 7, 3, 5, 2] 2023-09-23T18:15:06.848 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:15:06.848 INFO:tasks.thrashosds.thrasher:inject_pause on osd.1 2023-09-23T18:15:06.848 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:15:06.848 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:15:06.849 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 config set filestore_inject_stall 3 2023-09-23T18:15:06.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:06.923 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:07.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:07.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:07.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:07.107 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:07.160 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:07.228 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:07.235 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:15:07.236 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:07.236 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:07.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:07.384 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:07.496 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:07.501 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:07.502 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:07.595 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:07.602 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:07.638 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 79 16 94689 94673 74.8571 54.25 0.014014 0.0133529 2023-09-23T18:15:07.639 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:07.639 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:07.647 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:07.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:07.818 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:07.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:07.827 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:07.928 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:07.992 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:08.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:08.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:08.147 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:08.154 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:08.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:08.309 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:08.383 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:08.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:08.389 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:08.489 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:08.568 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:08.639 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:15:08.639717+0000 min lat: 0.00130179 max lat: 2.30896 avg lat: 0.0133935 2023-09-23T18:15:08.639 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:15:08.639 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 80 16 95594 95578 74.6285 56.5625 0.00787208 0.0133935 2023-09-23T18:15:08.652 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:08.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:08.679 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:08.708 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:08 smithi055 ceph-mon[179668]: pgmap v1186: 119 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 17 remapped+peering, 1 active+recovering+degraded+remapped, 98 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 9.0 KiB/s rd, 27 MiB/s wr, 446 op/s; 7951/275139 objects degraded (2.890%); 5547/275139 objects misplaced (2.016%); 9.7 MiB/s, 153 objects/s recovering 2023-09-23T18:15:08.708 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7951/275139 objects degraded (2.890%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:08.709 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:08 smithi055 ceph-mon[184174]: pgmap v1186: 119 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 17 remapped+peering, 1 active+recovering+degraded+remapped, 98 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 9.0 KiB/s rd, 27 MiB/s wr, 446 op/s; 7951/275139 objects degraded (2.890%); 5547/275139 objects misplaced (2.016%); 9.7 MiB/s, 153 objects/s recovering 2023-09-23T18:15:08.709 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7951/275139 objects degraded (2.890%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:08.780 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:08.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:08 smithi099 ceph-mon[167076]: pgmap v1186: 119 pgs: 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfill_wait, 1 peering, 17 remapped+peering, 1 active+recovering+degraded+remapped, 98 active+clean; 5.6 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 9.0 KiB/s rd, 27 MiB/s wr, 446 op/s; 7951/275139 objects degraded (2.890%); 5547/275139 objects misplaced (2.016%); 9.7 MiB/s, 153 objects/s recovering 2023-09-23T18:15:08.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7951/275139 objects degraded (2.890%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:08.849 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:08.872 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:15:08.872 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:15:08.872 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:15:08.872 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:15:08.872 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:15:08.924 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:08.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:08.937 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:09.037 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:09.083 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:09.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:09.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:09.240 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:09.330 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:15:09.337 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:15:09.341 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:09.367 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:15:09.402 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:15:09.412 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:09.439 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:15:09.476 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2941898234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:09.476 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:09 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:09.476 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2842552736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:09.485 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:15:09.505 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:09.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:09.514 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:09.516 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:15:09.517 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2941898234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:09.517 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:09 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:09.517 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2842552736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:09.538 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:15:09.592 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:15:09.615 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:09.630 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:15:09.639 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 81 16 96431 96415 74.3529 52.3125 0.0138743 0.0134441 2023-09-23T18:15:09.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:15:09.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:15:09.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:15:09.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:15:09.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:15:09.662 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:15:09.663 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:15:09.663 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:15:09.692 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:09.926 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2941898234' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:09.926 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:09 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:09.927 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2842552736' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:09.968 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:09.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:09.974 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:10.074 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:10.143 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:10.188 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:15:10.230 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:10.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:10.235 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:10.336 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:10.397 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:10.458 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:10.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:10.465 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:10.484 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:15:10] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.43.0" 2023-09-23T18:15:10.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[184174]: pgmap v1187: 119 pgs: 7 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 721 op/s; 7782/284295 objects degraded (2.737%); 52268/284295 objects misplaced (18.385%); 12 MiB/s, 191 objects/s recovering 2023-09-23T18:15:10.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 7 pgs peering) 2023-09-23T18:15:10.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:10.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[184174]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:15:10.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[184174]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T18:15:10.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[184174]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[184174]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.485 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[184174]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[179668]: pgmap v1187: 119 pgs: 7 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 721 op/s; 7782/284295 objects degraded (2.737%); 52268/284295 objects misplaced (18.385%); 12 MiB/s, 191 objects/s recovering 2023-09-23T18:15:10.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 7 pgs peering) 2023-09-23T18:15:10.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:10.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[179668]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:15:10.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[179668]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T18:15:10.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[179668]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[179668]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.486 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:10 smithi055 ceph-mon[179668]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.566 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:10.639 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 82 16 97378 97362 74.168 59.1875 0.0207036 0.0134776 2023-09-23T18:15:10.721 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:15:10.721 INFO:teuthology.orchestra.run.smithi055.stdout: "success": "" 2023-09-23T18:15:10.721 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:15:10.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:10 smithi099 ceph-mon[167076]: pgmap v1187: 119 pgs: 7 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 45 MiB/s wr, 721 op/s; 7782/284295 objects degraded (2.737%); 52268/284295 objects misplaced (18.385%); 12 MiB/s, 191 objects/s recovering 2023-09-23T18:15:10.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:10 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 7 pgs peering) 2023-09-23T18:15:10.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:10 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:10.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:10 smithi099 ceph-mon[167076]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:10 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:15:10.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:10 smithi099 ceph-mon[167076]: osdmap e493: 8 total, 8 up, 7 in 2023-09-23T18:15:10.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:10 smithi099 ceph-mon[167076]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:10 smithi099 ceph-mon[167076]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:10 smithi099 ceph-mon[167076]: failed to encode map e493 with expected crc 2023-09-23T18:15:10.792 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:10.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:10.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:10.871 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:10.971 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:11.078 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:11.157 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:11.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:11.167 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:11.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:11.337 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:11.540 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:11.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:11.549 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:11.639 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 83 16 98349 98333 74.0056 60.6875 0.0145138 0.0135078 2023-09-23T18:15:11.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:11.741 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:11.892 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:11.897 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:11.898 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:11.952 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:15:11.999 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:12.056 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:12.137 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:12.139 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:12.139 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:12.240 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:12.271 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:15:12.290 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:12.356 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:12.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:12.362 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:12.462 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:12.491 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:12 smithi099 ceph-mon[167076]: pgmap v1189: 119 pgs: 7 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 652 op/s; 7782/286932 objects degraded (2.712%); 52022/286932 objects misplaced (18.130%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:15:12.538 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:12 smithi055 ceph-mon[184174]: pgmap v1189: 119 pgs: 7 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 652 op/s; 7782/286932 objects degraded (2.712%); 52022/286932 objects misplaced (18.130%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:15:12.538 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:12 smithi055 ceph-mon[179668]: pgmap v1189: 119 pgs: 7 active+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 41 MiB/s wr, 652 op/s; 7782/286932 objects degraded (2.712%); 52022/286932 objects misplaced (18.130%); 16 MiB/s, 252 objects/s recovering 2023-09-23T18:15:12.543 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:12.614 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:12.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:12.619 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:12.639 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 84 16 99260 99244 73.8025 56.9375 0.0111866 0.0135441 2023-09-23T18:15:12.720 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:12.783 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:12.846 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:12.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:12.875 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:12.875 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:12.976 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:12.998 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:15:12.998 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:15:13.075 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:13.102 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:13.334 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:13.344 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:13.344 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:13.381 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:15:13.446 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:13.537 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:13.641 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 85 16 100150 100134 73.5869 55.625 0.015215 0.0135846 2023-09-23T18:15:13.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7782/286932 objects degraded (2.712%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:13.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7782/286932 objects degraded (2.712%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:13.778 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:13.787 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:13.787 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:13.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7782/286932 objects degraded (2.712%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:13.889 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:13.983 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:14.107 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:14.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:14.113 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:14.214 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:14.268 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:14.354 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:14.363 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:14.364 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:14.465 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:14.515 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:14 smithi055 ceph-mon[179668]: pgmap v1190: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 756 op/s; 7737/292104 objects degraded (2.649%); 52856/292104 objects misplaced (18.095%); 15 MiB/s, 250 objects/s recovering 2023-09-23T18:15:14.515 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:14 smithi055 ceph-mon[184174]: pgmap v1190: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 756 op/s; 7737/292104 objects degraded (2.649%); 52856/292104 objects misplaced (18.095%); 15 MiB/s, 250 objects/s recovering 2023-09-23T18:15:14.574 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:14.645 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 86 16 101055 101039 73.3862 56.5625 0.0164909 0.0136211 2023-09-23T18:15:14.664 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:14.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:14.674 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:14.774 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:14.785 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:14.824 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:14 smithi099 ceph-mon[167076]: pgmap v1190: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 47 MiB/s wr, 756 op/s; 7737/292104 objects degraded (2.649%); 52856/292104 objects misplaced (18.095%); 15 MiB/s, 250 objects/s recovering 2023-09-23T18:15:14.830 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:14.895 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:14.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:14.900 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:15.001 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:15.068 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:15.194 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:15.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:15.204 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:15.252 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:15:15.300 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:15:15.304 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:15.335 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:15:15.370 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:15.385 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:15:15.422 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:15:15.464 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:15:15.481 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:15.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:15.494 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:15.503 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:15:15.549 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:15:15.561 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:15:15.561 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:15:15.561 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:15:15.561 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:15:15.561 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:15:15.561 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:15:15.562 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:15:15.562 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:15:15.562 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:15:15.595 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:15.645 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 87 16 101964 101948 73.1958 56.8125 0.0206935 0.0136573 2023-09-23T18:15:15.695 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:15.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3786178018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:15.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1588726231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:15.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3786178018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:15.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1588726231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:15.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3786178018' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:15.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1588726231' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:15.990 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:15.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:15.997 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:16.098 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:16.134 INFO:tasks.thrashosds.thrasher:in_osds: [7, 1, 0, 4, 2, 3, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 4, 6, 7, 3, 5, 2] 2023-09-23T18:15:16.134 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:15:16.134 INFO:tasks.thrashosds.thrasher:fixing pg num pool unique_pool_4 2023-09-23T18:15:16.134 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:15:16.197 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:16.268 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:16.274 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:16.274 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:16.375 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:16.433 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:16.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:16.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:16.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:16.548 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[179668]: pgmap v1191: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 832 op/s; 7737/297987 objects degraded (2.596%); 53809/297987 objects misplaced (18.057%); 16 MiB/s, 250 objects/s recovering 2023-09-23T18:15:16.548 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:16.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[179668]: failed to encode map e494 with expected crc 2023-09-23T18:15:16.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[179668]: failed to encode map e494 with expected crc 2023-09-23T18:15:16.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3786178018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:16.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[179668]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T18:15:16.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[179668]: failed to encode map e494 with expected crc 2023-09-23T18:15:16.549 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:16.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:16.645 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 88 16 102861 102845 73.0011 56.0625 0.0187125 0.0136925 2023-09-23T18:15:16.688 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:16.752 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:16.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:16.755 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:16.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:16 smithi099 ceph-mon[167076]: pgmap v1191: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 832 op/s; 7737/297987 objects degraded (2.596%); 53809/297987 objects misplaced (18.057%); 16 MiB/s, 250 objects/s recovering 2023-09-23T18:15:16.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:16 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:16.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:16 smithi099 ceph-mon[167076]: failed to encode map e494 with expected crc 2023-09-23T18:15:16.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:16 smithi099 ceph-mon[167076]: failed to encode map e494 with expected crc 2023-09-23T18:15:16.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3786178018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:16.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:16 smithi099 ceph-mon[167076]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T18:15:16.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:16 smithi099 ceph-mon[167076]: failed to encode map e494 with expected crc 2023-09-23T18:15:16.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:16 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:16.857 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:16.911 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[184174]: pgmap v1191: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 52 MiB/s wr, 832 op/s; 7737/297987 objects degraded (2.596%); 53809/297987 objects misplaced (18.057%); 16 MiB/s, 250 objects/s recovering 2023-09-23T18:15:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[184174]: failed to encode map e494 with expected crc 2023-09-23T18:15:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[184174]: failed to encode map e494 with expected crc 2023-09-23T18:15:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3786178018' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[184174]: osdmap e494: 8 total, 8 up, 7 in 2023-09-23T18:15:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[184174]: failed to encode map e494 with expected crc 2023-09-23T18:15:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:16 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:15:16.953 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:15:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:15:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:103114 9.1 e494/483 9:85dca482:::benchmark_data_smithi055_386467_object103113:head v 494'6423, mlcod=494'6421)", 2023-09-23T18:15:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:16.939606+0000", 2023-09-23T18:15:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0089910619999999993, 2023-09-23T18:15:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0090253399999999998, 2023-09-23T18:15:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:15:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:15:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:15:16.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.939606+0000", 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.939606+0000", 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.939605+0000", 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.939672+0000", 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6611000000000003e-05 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.939681+0000", 2023-09-23T18:15:16.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.098e-06 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.939687+0000", 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0689999999999998e-06 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.939748+0000", 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0460000000000001e-05 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.939766+0000", 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8196999999999999e-05 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:103118 9.4 e494/459 9:2e8f7238:::benchmark_data_smithi055_386467_object103117:head v 494'6534, mlcod=494'6532)", 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:16.943733+0000", 2023-09-23T18:15:16.958 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0048643169999999999, 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049660199999999998, 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.943733+0000", 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.943733+0000", 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.943732+0000", 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.943752+0000", 2023-09-23T18:15:16.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0242999999999999e-05 2023-09-23T18:15:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:15:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.943765+0000", 2023-09-23T18:15:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2858e-05 2023-09-23T18:15:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:15:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.943773+0000", 2023-09-23T18:15:16.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4009999999999997e-06 2023-09-23T18:15:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:15:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.943837+0000", 2023-09-23T18:15:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3542999999999998e-05 2023-09-23T18:15:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:15:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.943852+0000", 2023-09-23T18:15:16.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4955e-05 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:103115 9.c e494/459 9:3a959a98:::benchmark_data_smithi055_386467_object103114:head v 494'6469, mlcod=494'6467)", 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:16.947220+0000", 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0013770410000000001, 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015411190000000001, 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:15:16.963 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:15:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:15:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:15:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.947220+0000", 2023-09-23T18:15:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:15:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:15:16.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.947220+0000", 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.947219+0000", 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.947263+0000", 2023-09-23T18:15:16.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4496999999999998e-05 2023-09-23T18:15:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:15:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.947283+0000", 2023-09-23T18:15:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.965e-05 2023-09-23T18:15:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:15:16.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.947291+0000", 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5960000000000004e-06 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.947335+0000", 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3822999999999997e-05 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:16.947357+0000", 2023-09-23T18:15:16.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2096e-05 2023-09-23T18:15:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:15:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:15:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:15:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:15:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:15:16.968 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 3 2023-09-23T18:15:16.968 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:15:16.979 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:16.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:16.986 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:17.087 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:17.138 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:17.168 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:17.355 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:17.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:17.360 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:17.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:17.507 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:15:17.535 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:17.607 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:17.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:17.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:17.649 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 89 16 103744 103728 72.7974 55.1875 0.0147166 0.0137316 2023-09-23T18:15:17.716 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:17.806 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:18.359 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:15:18.368 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:15:18.575 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:18 smithi055 ceph-mon[179668]: pgmap v1193: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42 MiB/s wr, 679 op/s; 7737/300603 objects degraded (2.574%); 53474/300603 objects misplaced (17.789%); 14 MiB/s, 228 objects/s recovering 2023-09-23T18:15:18.576 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7737/300603 objects degraded (2.574%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:18.649 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total time run: 90.0173 2023-09-23T18:15:18.649 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total writes made: 104657 2023-09-23T18:15:18.649 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Write size: 65536 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object size: 65536 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Bandwidth (MB/sec): 72.6645 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Bandwidth: 42.6965 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max bandwidth (MB/sec): 214.125 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min bandwidth (MB/sec): 0 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average IOPS: 1162 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev IOPS: 683.152 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max IOPS: 3426 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min IOPS: 0 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average Latency(s): 0.013758 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Latency(s): 0.0509589 2023-09-23T18:15:18.650 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max latency(s): 2.30896 2023-09-23T18:15:18.651 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min latency(s): 0.00130179 2023-09-23T18:15:18.653 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T18:15:18.668 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:18.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:18.670 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:18.771 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:18.824 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:18 smithi055 ceph-mon[184174]: pgmap v1193: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42 MiB/s wr, 679 op/s; 7737/300603 objects degraded (2.574%); 53474/300603 objects misplaced (17.789%); 14 MiB/s, 228 objects/s recovering 2023-09-23T18:15:18.824 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7737/300603 objects degraded (2.574%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:18.825 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:18.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:18 smithi099 ceph-mon[167076]: pgmap v1193: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 42 MiB/s wr, 679 op/s; 7737/300603 objects degraded (2.574%); 53474/300603 objects misplaced (17.789%); 14 MiB/s, 228 objects/s recovering 2023-09-23T18:15:18.831 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7737/300603 objects degraded (2.574%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:19.287 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:19.293 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:19.294 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:19.395 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:19.457 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:19.597 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:19 smithi055 ceph-mon[179668]: pgmap v1194: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 949 op/s; 7701/311517 objects degraded (2.472%); 55268/311517 objects misplaced (17.742%); 17 MiB/s, 266 objects/s recovering 2023-09-23T18:15:19.638 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:19.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:19.652 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:19.753 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:19.807 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:19.866 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:19.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:19.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:19.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:19 smithi099 ceph-mon[167076]: pgmap v1194: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 949 op/s; 7701/311517 objects degraded (2.472%); 55268/311517 objects misplaced (17.742%); 17 MiB/s, 266 objects/s recovering 2023-09-23T18:15:19.890 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:19.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:19 smithi055 ceph-mon[184174]: pgmap v1194: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 59 MiB/s wr, 949 op/s; 7701/311517 objects degraded (2.472%); 55268/311517 objects misplaced (17.742%); 17 MiB/s, 266 objects/s recovering 2023-09-23T18:15:19.945 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:19.979 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:20.022 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:20.053 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:20.088 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:20.095 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:20.096 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:20.197 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:20.281 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:20.360 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:20.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:20.366 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:20.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:15:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:15:20] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.43.0" 2023-09-23T18:15:20.467 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:20.532 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:20.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:20.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:20.716 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:20.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:20.857 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:20.911 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:20.923 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:20.923 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:20.974 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:15:21.000 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:15:21.024 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:21.070 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:21.118 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:21.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:21.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:21.224 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:21.277 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:21.395 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:21.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:21.402 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:21.503 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:21.565 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:21.874 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:21.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:21.884 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:21.972 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1194,"stamp":"2023-09-23T18:15:19.032263+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6807779596,"num_objects":103839,"num_object_clones":0,"num_object_copies":311517,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7701,"num_objects_misplaced":55268,"num_objects_unfound":0,"num_objects_dirty":103839,"num_whiteouts":0,"num_read":6081,"num_read_kb":12390,"num_write":104783,"num_write_kb":6657334,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8530,"num_bytes_recovered":547563408,"num_keys_recovered":60,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":74153,"ondisk_log_size":74153,"up":357,"acting":356,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":361,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":19659992,"kb_used_data":16031456,"kb_used_omap":61,"kb_used_meta":3628418,"kb_avail":636556072,"statfs":{"total":671965249536,"available":651833417728,"internally_reserved":0,"allocated":16416210944,"data_stored":16401964128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":62520,"internal_metadata":3715501000},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,6,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":594673664,"num_objects":9074,"num_object_clones":0,"num_object_copies":27222,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-81,"num_objects_misplaced":3000,"num_objects_unfound":0,"num_objects_dirty":9074,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9074,"num_write_kb":580736,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2545,"num_bytes_recovered":166789120,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.558012"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":715,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501265+0000","last_change":"2023-09-23T18:15:05.438191+0000","last_active":"2023-09-23T18:15:09.501265+0000","last_peered":"2023-09-23T18:15:09.501265+0000","last_clean":"2023-09-23T18:15:09.501265+0000","last_became_active":"2023-09-23T18:15:05.431985+0000","last_became_peered":"2023-09-23T18:15:05.431985+0000","last_unstale":"2023-09-23T18:15:09.501265+0000","last_undegraded":"2023-09-23T18:15:09.501265+0000","last_fullsized":"2023-09-23T18:15:09.501265+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.712627+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.712627+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1262601,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,1],"acting":[7,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":117,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508412+0000","last_change":"2023-09-23T18:14:50.152058+0000","last_active":"2023-09-23T18:15:09.508412+0000","last_peered":"2023-09-23T18:15:09.508412+0000","last_clean":"2023-09-23T18:15:09.508412+0000","last_became_active":"2023-09-23T18:14:50.151763+0000","last_became_peered":"2023-09-23T18:14:50.151763+0000","last_unstale":"2023-09-23T18:15:09.508412+0000","last_undegraded":"2023-09-23T18:15:09.508412+0000","last_fullsized":"2023-09-23T18:15:09.508412+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:10:08.770497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":411,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500246+0000","last_change":"2023-09-23T18:14:32.559438+0000","last_active":"2023-09-23T18:15:09.500246+0000","last_peered":"2023-09-23T18:15:09.500246+0000","last_clean":"2023-09-23T18:15:09.500246+0000","last_became_active":"2023-09-23T18:14:32.559118+0000","last_became_peered":"2023-09-23T18:14:32.559118+0000","last_unstale":"2023-09-23T18:15:09.500246+0000","last_undegraded":"2023-09-23T18:15:09.500246+0000","last_fullsized":"2023-09-23T18:15:09.500246+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:57:33.128176+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,6],"acting":[1,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":697,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509177+0000","last_change":"2023-09-23T18:15:05.432894+0000","last_active":"2023-09-23T18:15:09.509177+0000","last_peered":"2023-09-23T18:15:09.509177+0000","last_clean":"2023-09-23T18:15:09.509177+0000","last_became_active":"2023-09-23T18:15:05.432531+0000","last_became_peered":"2023-09-23T18:15:05.432531+0000","last_unstale":"2023-09-23T18:15:09.509177+0000","last_undegraded":"2023-09-23T18:15:09.509177+0000","last_fullsized":"2023-09-23T18:15:09.509177+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:18.093123+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":266,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.499832+0000","last_change":"2023-09-23T18:13:39.230590+0000","last_active":"2023-09-23T18:15:09.499832+0000","last_peered":"2023-09-23T18:15:09.499832+0000","last_clean":"2023-09-23T18:15:09.499832+0000","last_became_active":"2023-09-23T18:13:39.230265+0000","last_became_peered":"2023-09-23T18:13:39.230265+0000","last_unstale":"2023-09-23T18:15:09.499832+0000","last_undegraded":"2023-09-23T18:15:09.499832+0000","last_fullsized":"2023-09-23T18:15:09.499832+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:29.038853+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":530,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504191+0000","last_change":"2023-09-23T18:15:02.358653+0000","last_active":"2023-09-23T18:15:09.504191+0000","last_peered":"2023-09-23T18:15:09.504191+0000","last_clean":"2023-09-23T18:15:09.504191+0000","last_became_active":"2023-09-23T18:15:02.358253+0000","last_became_peered":"2023-09-23T18:15:02.358253+0000","last_unstale":"2023-09-23T18:15:09.504191+0000","last_undegraded":"2023-09-23T18:15:09.504191+0000","last_fullsized":"2023-09-23T18:15:09.504191+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:50.842301+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":825,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508671+0000","last_change":"2023-09-23T18:12:06.649780+0000","last_active":"2023-09-23T18:15:09.508671+0000","last_peered":"2023-09-23T18:15:09.508671+0000","last_clean":"2023-09-23T18:15:09.508671+0000","last_became_active":"2023-09-23T18:12:06.649464+0000","last_became_peered":"2023-09-23T18:12:06.649464+0000","last_unstale":"2023-09-23T18:15:09.508671+0000","last_undegraded":"2023-09-23T18:15:09.508671+0000","last_fullsized":"2023-09-23T18:15:09.508671+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:01:51.143336+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":420,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501988+0000","last_change":"2023-09-23T18:14:50.228351+0000","last_active":"2023-09-23T18:15:09.501988+0000","last_peered":"2023-09-23T18:15:09.501988+0000","last_clean":"2023-09-23T18:15:09.501988+0000","last_became_active":"2023-09-23T18:14:50.227024+0000","last_became_peered":"2023-09-23T18:14:50.227024+0000","last_unstale":"2023-09-23T18:15:09.501988+0000","last_undegraded":"2023-09-23T18:15:09.501988+0000","last_fullsized":"2023-09-23T18:15:09.501988+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:44.088044+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":39,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504609+0000","last_change":"2023-09-23T18:15:02.394722+0000","last_active":"2023-09-23T18:15:09.504609+0000","last_peered":"2023-09-23T18:15:09.504609+0000","last_clean":"2023-09-23T18:15:09.504609+0000","last_became_active":"2023-09-23T18:15:02.393823+0000","last_became_peered":"2023-09-23T18:15:02.393823+0000","last_unstale":"2023-09-23T18:15:09.504609+0000","last_undegraded":"2023-09-23T18:15:09.504609+0000","last_fullsized":"2023-09-23T18:15:09.504609+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:09.827921+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":911,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.502257+0000","last_change":"2023-09-23T18:15:05.426415+0000","last_active":"2023-09-23T18:15:09.502257+0000","last_peered":"2023-09-23T18:15:09.502257+0000","last_clean":"2023-09-23T18:15:09.502257+0000","last_became_active":"2023-09-23T18:15:05.426143+0000","last_became_peered":"2023-09-23T18:15:05.426143+0000","last_unstale":"2023-09-23T18:15:09.502257+0000","last_undegraded":"2023-09-23T18:15:09.502257+0000","last_fullsized":"2023-09-23T18:15:09.502257+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:17:04.301805+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,3],"acting":[7,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":395,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504331+0000","last_change":"2023-09-23T18:15:02.366915+0000","last_active":"2023-09-23T18:15:09.504331+0000","last_peered":"2023-09-23T18:15:09.504331+0000","last_clean":"2023-09-23T18:15:09.504331+0000","last_became_active":"2023-09-23T18:15:02.366532+0000","last_became_peered":"2023-09-23T18:15:02.366532+0000","last_unstale":"2023-09-23T18:15:09.504331+0000","last_undegraded":"2023-09-23T18:15:09.504331+0000","last_fullsized":"2023-09-23T18:15:09.504331+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:59:31.299084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":66,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.481925+0000","last_change":"2023-09-23T18:14:32.574436+0000","last_active":"2023-09-23T18:15:09.481925+0000","last_peered":"2023-09-23T18:15:09.481925+0000","last_clean":"2023-09-23T18:15:09.481925+0000","last_became_active":"2023-09-23T18:14:32.574125+0000","last_became_peered":"2023-09-23T18:14:32.574125+0000","last_unstale":"2023-09-23T18:15:09.481925+0000","last_undegraded":"2023-09-23T18:15:09.481925+0000","last_fullsized":"2023-09-23T18:15:09.481925+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:27:42.906327+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":879,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500038+0000","last_change":"2023-09-23T18:14:50.154908+0000","last_active":"2023-09-23T18:15:09.500038+0000","last_peered":"2023-09-23T18:15:09.500038+0000","last_clean":"2023-09-23T18:15:09.500038+0000","last_became_active":"2023-09-23T18:14:50.154588+0000","last_became_peered":"2023-09-23T18:14:50.154588+0000","last_unstale":"2023-09-23T18:15:09.500038+0000","last_undegraded":"2023-09-23T18:15:09.500038+0000","last_fullsized":"2023-09-23T18:15:09.500038+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:18.452003+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,4],"acting":[7,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":347,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504399+0000","last_change":"2023-09-23T18:15:02.358503+0000","last_active":"2023-09-23T18:15:09.504399+0000","last_peered":"2023-09-23T18:15:09.504399+0000","last_clean":"2023-09-23T18:15:09.504399+0000","last_became_active":"2023-09-23T18:15:02.357886+0000","last_became_peered":"2023-09-23T18:15:02.357886+0000","last_unstale":"2023-09-23T18:15:09.504399+0000","last_undegraded":"2023-09-23T18:15:09.504399+0000","last_fullsized":"2023-09-23T18:15:09.504399+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:19:39.795641+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":29,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504964+0000","last_change":"2023-09-23T18:14:32.584265+0000","last_active":"2023-09-23T18:15:09.504964+0000","last_peered":"2023-09-23T18:15:09.504964+0000","last_clean":"2023-09-23T18:15:09.504964+0000","last_became_active":"2023-09-23T18:14:32.583958+0000","last_became_peered":"2023-09-23T18:14:32.583958+0000","last_unstale":"2023-09-23T18:15:09.504964+0000","last_undegraded":"2023-09-23T18:15:09.504964+0000","last_fullsized":"2023-09-23T18:15:09.504964+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:39:06.513412+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":823,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504786+0000","last_change":"2023-09-23T18:15:05.454712+0000","last_active":"2023-09-23T18:15:09.504786+0000","last_peered":"2023-09-23T18:15:09.504786+0000","last_clean":"2023-09-23T18:15:09.504786+0000","last_became_active":"2023-09-23T18:15:05.453953+0000","last_became_peered":"2023-09-23T18:15:05.453953+0000","last_unstale":"2023-09-23T18:15:09.504786+0000","last_undegraded":"2023-09-23T18:15:09.504786+0000","last_fullsized":"2023-09-23T18:15:09.504786+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:01:24.699949+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,3],"acting":[6,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":355,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.502212+0000","last_change":"2023-09-23T18:13:39.235689+0000","last_active":"2023-09-23T18:15:09.502212+0000","last_peered":"2023-09-23T18:15:09.502212+0000","last_clean":"2023-09-23T18:15:09.502212+0000","last_became_active":"2023-09-23T18:13:39.230285+0000","last_became_peered":"2023-09-23T18:13:39.230285+0000","last_unstale":"2023-09-23T18:15:09.502212+0000","last_undegraded":"2023-09-23T18:15:09.502212+0000","last_fullsized":"2023-09-23T18:15:09.502212+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:50:52.139628+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":287,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501368+0000","last_change":"2023-09-23T18:14:51.173999+0000","last_active":"2023-09-23T18:15:09.501368+0000","last_peered":"2023-09-23T18:15:09.501368+0000","last_clean":"2023-09-23T18:15:09.501368+0000","last_became_active":"2023-09-23T18:14:51.173652+0000","last_became_peered":"2023-09-23T18:14:51.173652+0000","last_unstale":"2023-09-23T18:15:09.501368+0000","last_undegraded":"2023-09-23T18:15:09.501368+0000","last_fullsized":"2023-09-23T18:15:09.501368+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:50:57.800809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":915,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501142+0000","last_change":"2023-09-23T18:12:05.654051+0000","last_active":"2023-09-23T18:15:09.501142+0000","last_peered":"2023-09-23T18:15:09.501142+0000","last_clean":"2023-09-23T18:15:09.501142+0000","last_became_active":"2023-09-23T18:12:05.653587+0000","last_became_peered":"2023-09-23T18:12:05.653587+0000","last_unstale":"2023-09-23T18:15:09.501142+0000","last_undegraded":"2023-09-23T18:15:09.501142+0000","last_fullsized":"2023-09-23T18:15:09.501142+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:36:10.417643+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":302,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509522+0000","last_change":"2023-09-23T18:14:32.578146+0000","last_active":"2023-09-23T18:15:09.509522+0000","last_peered":"2023-09-23T18:15:09.509522+0000","last_clean":"2023-09-23T18:15:09.509522+0000","last_became_active":"2023-09-23T18:14:32.577862+0000","last_became_peered":"2023-09-23T18:14:32.577862+0000","last_unstale":"2023-09-23T18:15:09.509522+0000","last_undegraded":"2023-09-23T18:15:09.509522+0000","last_fullsized":"2023-09-23T18:15:09.509522+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:42.199606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":318,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509289+0000","last_change":"2023-09-23T18:14:32.566438+0000","last_active":"2023-09-23T18:15:09.509289+0000","last_peered":"2023-09-23T18:15:09.509289+0000","last_clean":"2023-09-23T18:15:09.509289+0000","last_became_active":"2023-09-23T18:14:32.566145+0000","last_became_peered":"2023-09-23T18:14:32.566145+0000","last_unstale":"2023-09-23T18:15:09.509289+0000","last_undegraded":"2023-09-23T18:15:09.509289+0000","last_fullsized":"2023-09-23T18:15:09.509289+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:28.627770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":679,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.499696+0000","last_change":"2023-09-23T18:15:05.434757+0000","last_active":"2023-09-23T18:15:09.499696+0000","last_peered":"2023-09-23T18:15:09.499696+0000","last_clean":"2023-09-23T18:15:09.499696+0000","last_became_active":"2023-09-23T18:15:05.434441+0000","last_became_peered":"2023-09-23T18:15:05.434441+0000","last_unstale":"2023-09-23T18:15:09.499696+0000","last_undegraded":"2023-09-23T18:15:09.499696+0000","last_fullsized":"2023-09-23T18:15:09.499696+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:57.330924+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:13:57.330924+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":23,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":503,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.505804+0000","last_change":"2023-09-23T18:14:50.162031+0000","last_active":"2023-09-23T18:15:09.505804+0000","last_peered":"2023-09-23T18:15:09.505804+0000","last_clean":"2023-09-23T18:15:09.505804+0000","last_became_active":"2023-09-23T18:14:50.161669+0000","last_became_peered":"2023-09-23T18:14:50.161669+0000","last_unstale":"2023-09-23T18:15:09.505804+0000","last_undegraded":"2023-09-23T18:15:09.505804+0000","last_fullsized":"2023-09-23T18:15:09.505804+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:47:23.934550+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":204,"num_keys_recovered":60,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":350,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.503373+0000","last_change":"2023-09-23T18:13:39.226565+0000","last_active":"2023-09-23T18:15:09.503373+0000","last_peered":"2023-09-23T18:15:09.503373+0000","last_clean":"2023-09-23T18:15:09.503373+0000","last_became_active":"2023-09-23T18:13:39.226211+0000","last_became_peered":"2023-09-23T18:13:39.226211+0000","last_unstale":"2023-09-23T18:15:09.503373+0000","last_undegraded":"2023-09-23T18:15:09.503373+0000","last_fullsized":"2023-09-23T18:15:09.503373+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:53.510129+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,7],"acting":[1,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":693,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501185+0000","last_change":"2023-09-23T18:15:05.436650+0000","last_active":"2023-09-23T18:15:09.501185+0000","last_peered":"2023-09-23T18:15:09.501185+0000","last_clean":"2023-09-23T18:15:09.501185+0000","last_became_active":"2023-09-23T18:15:05.436366+0000","last_became_peered":"2023-09-23T18:15:05.436366+0000","last_unstale":"2023-09-23T18:15:09.501185+0000","last_undegraded":"2023-09-23T18:15:09.501185+0000","last_fullsized":"2023-09-23T18:15:09.501185+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:54:42.613260+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":552,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500854+0000","last_change":"2023-09-23T18:14:50.155421+0000","last_active":"2023-09-23T18:15:09.500854+0000","last_peered":"2023-09-23T18:15:09.500854+0000","last_clean":"2023-09-23T18:15:09.500854+0000","last_became_active":"2023-09-23T18:14:50.155154+0000","last_became_peered":"2023-09-23T18:14:50.155154+0000","last_unstale":"2023-09-23T18:15:09.500854+0000","last_undegraded":"2023-09-23T18:15:09.500854+0000","last_fullsized":"2023-09-23T18:15:09.500854+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:48:28.679998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,2],"acting":[7,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":417,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.502442+0000","last_change":"2023-09-23T18:15:02.363582+0000","last_active":"2023-09-23T18:15:09.502442+0000","last_peered":"2023-09-23T18:15:09.502442+0000","last_clean":"2023-09-23T18:15:09.502442+0000","last_became_active":"2023-09-23T18:15:02.363253+0000","last_became_peered":"2023-09-23T18:15:02.363253+0000","last_unstale":"2023-09-23T18:15:09.502442+0000","last_undegraded":"2023-09-23T18:15:09.502442+0000","last_fullsized":"2023-09-23T18:15:09.502442+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:43:58.672620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":879,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.516143+0000","last_change":"2023-09-23T18:12:06.645815+0000","last_active":"2023-09-23T18:15:09.516143+0000","last_peered":"2023-09-23T18:15:09.516143+0000","last_clean":"2023-09-23T18:15:09.516143+0000","last_became_active":"2023-09-23T18:12:06.645452+0000","last_became_peered":"2023-09-23T18:12:06.645452+0000","last_unstale":"2023-09-23T18:15:09.516143+0000","last_undegraded":"2023-09-23T18:15:09.516143+0000","last_fullsized":"2023-09-23T18:15:09.516143+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:51:24.550609+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":111,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508710+0000","last_change":"2023-09-23T18:14:32.583669+0000","last_active":"2023-09-23T18:15:09.508710+0000","last_peered":"2023-09-23T18:15:09.508710+0000","last_clean":"2023-09-23T18:15:09.508710+0000","last_became_active":"2023-09-23T18:14:32.583317+0000","last_became_peered":"2023-09-23T18:14:32.583317+0000","last_unstale":"2023-09-23T18:15:09.508710+0000","last_undegraded":"2023-09-23T18:15:09.508710+0000","last_fullsized":"2023-09-23T18:15:09.508710+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:08.876039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":63,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.482320+0000","last_change":"2023-09-23T18:14:50.154939+0000","last_active":"2023-09-23T18:15:09.482320+0000","last_peered":"2023-09-23T18:15:09.482320+0000","last_clean":"2023-09-23T18:15:09.482320+0000","last_became_active":"2023-09-23T18:14:50.154528+0000","last_became_peered":"2023-09-23T18:14:50.154528+0000","last_unstale":"2023-09-23T18:15:09.482320+0000","last_undegraded":"2023-09-23T18:15:09.482320+0000","last_fullsized":"2023-09-23T18:15:09.482320+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:44:53.381349+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,2],"acting":[3,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":692,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501089+0000","last_change":"2023-09-23T18:15:05.438379+0000","last_active":"2023-09-23T18:15:09.501089+0000","last_peered":"2023-09-23T18:15:09.501089+0000","last_clean":"2023-09-23T18:15:09.501089+0000","last_became_active":"2023-09-23T18:15:05.438027+0000","last_became_peered":"2023-09-23T18:15:05.438027+0000","last_unstale":"2023-09-23T18:15:09.501089+0000","last_undegraded":"2023-09-23T18:15:09.501089+0000","last_fullsized":"2023-09-23T18:15:09.501089+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:24:18.418168+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,7],"acting":[1,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":563,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504914+0000","last_change":"2023-09-23T18:15:02.367636+0000","last_active":"2023-09-23T18:15:09.504914+0000","last_peered":"2023-09-23T18:15:09.504914+0000","last_clean":"2023-09-23T18:15:09.504914+0000","last_became_active":"2023-09-23T18:15:02.367231+0000","last_became_peered":"2023-09-23T18:15:02.367231+0000","last_unstale":"2023-09-23T18:15:09.504914+0000","last_undegraded":"2023-09-23T18:15:09.504914+0000","last_fullsized":"2023-09-23T18:15:09.504914+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:34:52.466896+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":16,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":383,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501958+0000","last_change":"2023-09-23T18:14:50.155076+0000","last_active":"2023-09-23T18:15:09.501958+0000","last_peered":"2023-09-23T18:15:09.501958+0000","last_clean":"2023-09-23T18:15:09.501958+0000","last_became_active":"2023-09-23T18:14:50.154736+0000","last_became_peered":"2023-09-23T18:14:50.154736+0000","last_unstale":"2023-09-23T18:15:09.501958+0000","last_undegraded":"2023-09-23T18:15:09.501958+0000","last_fullsized":"2023-09-23T18:15:09.501958+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:24.760586+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,2],"acting":[1,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.5","version":"494'6455","reported_seq":6507,"reported_epoch":494,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:15:18.245464+0000","last_change":"2023-09-23T18:15:03.526449+0000","last_active":"2023-09-23T18:15:18.245464+0000","last_peered":"2023-09-23T18:15:18.245464+0000","last_clean":"2023-09-23T18:15:01.294843+0000","last_became_active":"2023-09-23T18:15:03.521460+0000","last_became_peered":"2023-09-23T18:15:03.521460+0000","last_unstale":"2023-09-23T18:15:18.245464+0000","last_undegraded":"2023-09-23T18:15:18.245464+0000","last_fullsized":"2023-09-23T18:15:18.245464+0000","mapping_epoch":489,"log_start":"464'2077","ondisk_log_start":"464'2077","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":4378,"log_dups_size":0,"ondisk_log_size":4378,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:09:18.054344+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423034880,"num_objects":6455,"num_object_clones":0,"num_object_copies":19365,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6455,"num_objects_unfound":0,"num_objects_dirty":6455,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6455,"num_write_kb":413120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,5],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":6455}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":914,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.502162+0000","last_change":"2023-09-23T18:12:08.689111+0000","last_active":"2023-09-23T18:15:09.502162+0000","last_peered":"2023-09-23T18:15:09.502162+0000","last_clean":"2023-09-23T18:15:09.502162+0000","last_became_active":"2023-09-23T18:12:08.688825+0000","last_became_peered":"2023-09-23T18:12:08.688825+0000","last_unstale":"2023-09-23T18:15:09.502162+0000","last_undegraded":"2023-09-23T18:15:09.502162+0000","last_fullsized":"2023-09-23T18:15:09.502162+0000","mapping_epoch":415,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":416,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:29.016205+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":460,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500748+0000","last_change":"2023-09-23T18:12:06.652515+0000","last_active":"2023-09-23T18:15:09.500748+0000","last_peered":"2023-09-23T18:15:09.500748+0000","last_clean":"2023-09-23T18:15:09.500748+0000","last_became_active":"2023-09-23T18:12:06.652196+0000","last_became_peered":"2023-09-23T18:12:06.652196+0000","last_unstale":"2023-09-23T18:15:09.500748+0000","last_undegraded":"2023-09-23T18:15:09.500748+0000","last_fullsized":"2023-09-23T18:15:09.500748+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:07:27.975688+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":418,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500913+0000","last_change":"2023-09-23T18:15:02.384395+0000","last_active":"2023-09-23T18:15:09.500913+0000","last_peered":"2023-09-23T18:15:09.500913+0000","last_clean":"2023-09-23T18:15:09.500913+0000","last_became_active":"2023-09-23T18:15:02.384192+0000","last_became_peered":"2023-09-23T18:15:02.384192+0000","last_unstale":"2023-09-23T18:15:09.500913+0000","last_undegraded":"2023-09-23T18:15:09.500913+0000","last_fullsized":"2023-09-23T18:15:09.500913+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:47:29.382855+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.4","version":"494'6596","reported_seq":6640,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T18:15:17.943513+0000","last_change":"2023-09-23T18:13:44.089940+0000","last_active":"2023-09-23T18:15:17.943513+0000","last_peered":"2023-09-23T18:15:17.943513+0000","last_clean":"2023-09-23T18:15:17.943513+0000","last_became_active":"2023-09-23T18:13:44.089583+0000","last_became_peered":"2023-09-23T18:13:44.089583+0000","last_unstale":"2023-09-23T18:15:17.943513+0000","last_undegraded":"2023-09-23T18:15:17.943513+0000","last_fullsized":"2023-09-23T18:15:17.943513+0000","mapping_epoch":459,"log_start":"466'2694","ondisk_log_start":"466'2694","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":3902,"log_dups_size":0,"ondisk_log_size":3902,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:54:52.948719+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":432275456,"num_objects":6596,"num_object_clones":0,"num_object_copies":19788,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6596,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6596,"num_write_kb":422144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,3],"acting":[1,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":718,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504481+0000","last_change":"2023-09-23T18:14:51.175340+0000","last_active":"2023-09-23T18:15:09.504481+0000","last_peered":"2023-09-23T18:15:09.504481+0000","last_clean":"2023-09-23T18:15:09.504481+0000","last_became_active":"2023-09-23T18:14:51.174964+0000","last_became_peered":"2023-09-23T18:14:51.174964+0000","last_unstale":"2023-09-23T18:15:09.504481+0000","last_undegraded":"2023-09-23T18:15:09.504481+0000","last_fullsized":"2023-09-23T18:15:09.504481+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:41.510146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:41.510146+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":2047528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":475,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504536+0000","last_change":"2023-09-23T18:12:06.653146+0000","last_active":"2023-09-23T18:15:09.504536+0000","last_peered":"2023-09-23T18:15:09.504536+0000","last_clean":"2023-09-23T18:15:09.504536+0000","last_became_active":"2023-09-23T18:12:06.652832+0000","last_became_peered":"2023-09-23T18:12:06.652832+0000","last_unstale":"2023-09-23T18:15:09.504536+0000","last_undegraded":"2023-09-23T18:15:09.504536+0000","last_fullsized":"2023-09-23T18:15:09.504536+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:49:34.147471+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":111,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508525+0000","last_change":"2023-09-23T18:14:32.572356+0000","last_active":"2023-09-23T18:15:09.508525+0000","last_peered":"2023-09-23T18:15:09.508525+0000","last_clean":"2023-09-23T18:15:09.508525+0000","last_became_active":"2023-09-23T18:14:32.571962+0000","last_became_peered":"2023-09-23T18:14:32.571962+0000","last_unstale":"2023-09-23T18:15:09.508525+0000","last_undegraded":"2023-09-23T18:15:09.508525+0000","last_fullsized":"2023-09-23T18:15:09.508525+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:43:47.328734+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.7","version":"494'6491","reported_seq":6543,"reported_epoch":494,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:15:18.225025+0000","last_change":"2023-09-23T18:15:03.575458+0000","last_active":"2023-09-23T18:15:18.225025+0000","last_peered":"2023-09-23T18:15:18.225025+0000","last_clean":"2023-09-23T18:15:01.299832+0000","last_became_active":"2023-09-23T18:15:03.554908+0000","last_became_peered":"2023-09-23T18:15:03.554908+0000","last_unstale":"2023-09-23T18:15:18.225025+0000","last_undegraded":"2023-09-23T18:15:18.225025+0000","last_fullsized":"2023-09-23T18:15:18.225025+0000","mapping_epoch":489,"log_start":"465'2127","ondisk_log_start":"465'2127","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":4364,"log_dups_size":0,"ondisk_log_size":4364,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:51:00.169463+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425394176,"num_objects":6491,"num_object_clones":0,"num_object_copies":19473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6491,"num_objects_unfound":0,"num_objects_dirty":6491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6491,"num_write_kb":415424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":6491}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":479,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504527+0000","last_change":"2023-09-23T18:15:05.460702+0000","last_active":"2023-09-23T18:15:09.504527+0000","last_peered":"2023-09-23T18:15:09.504527+0000","last_clean":"2023-09-23T18:15:09.504527+0000","last_became_active":"2023-09-23T18:15:05.458403+0000","last_became_peered":"2023-09-23T18:15:05.458403+0000","last_unstale":"2023-09-23T18:15:09.504527+0000","last_undegraded":"2023-09-23T18:15:09.504527+0000","last_fullsized":"2023-09-23T18:15:09.504527+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":419,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.502348+0000","last_change":"2023-09-23T18:15:02.404221+0000","last_active":"2023-09-23T18:15:09.502348+0000","last_peered":"2023-09-23T18:15:09.502348+0000","last_clean":"2023-09-23T18:15:09.502348+0000","last_became_active":"2023-09-23T18:15:02.363973+0000","last_became_peered":"2023-09-23T18:15:02.363973+0000","last_unstale":"2023-09-23T18:15:09.502348+0000","last_undegraded":"2023-09-23T18:15:09.502348+0000","last_fullsized":"2023-09-23T18:15:09.502348+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:16:06.946021+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":210,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,3],"acting":[7,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":29,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.506175+0000","last_change":"2023-09-23T18:14:32.574101+0000","last_active":"2023-09-23T18:15:09.506175+0000","last_peered":"2023-09-23T18:15:09.506175+0000","last_clean":"2023-09-23T18:15:09.506175+0000","last_became_active":"2023-09-23T18:14:32.573776+0000","last_became_peered":"2023-09-23T18:14:32.573776+0000","last_unstale":"2023-09-23T18:15:09.506175+0000","last_undegraded":"2023-09-23T18:15:09.506175+0000","last_fullsized":"2023-09-23T18:15:09.506175+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:32.745211+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.6","version":"494'6482","reported_seq":8257,"reported_epoch":494,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:15:17.979321+0000","last_change":"2023-09-23T18:15:03.781719+0000","last_active":"2023-09-23T18:15:17.979321+0000","last_peered":"2023-09-23T18:15:17.979321+0000","last_clean":"2023-09-23T18:15:01.330528+0000","last_became_active":"2023-09-23T18:15:03.503493+0000","last_became_peered":"2023-09-23T18:15:03.503493+0000","last_unstale":"2023-09-23T18:15:17.979321+0000","last_undegraded":"2023-09-23T18:15:17.979321+0000","last_fullsized":"2023-09-23T18:15:17.979321+0000","mapping_epoch":489,"log_start":"466'2572","ondisk_log_start":"466'2572","created":459,"last_epoch_clean":480,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":3910,"log_dups_size":0,"ondisk_log_size":3910,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:19:43.579175+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424804352,"num_objects":6482,"num_object_clones":0,"num_object_copies":19446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6482,"num_objects_unfound":0,"num_objects_dirty":6482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6482,"num_write_kb":414848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":849,"num_bytes_recovered":55640064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,5],"avail_no_missing":["1","5","6"],"object_location_counts":[{"shards":"1,5,6","objects":6482}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":749,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509545+0000","last_change":"2023-09-23T18:13:58.012684+0000","last_active":"2023-09-23T18:15:09.509545+0000","last_peered":"2023-09-23T18:15:09.509545+0000","last_clean":"2023-09-23T18:15:09.509545+0000","last_became_active":"2023-09-23T18:13:39.228861+0000","last_became_peered":"2023-09-23T18:13:39.228861+0000","last_unstale":"2023-09-23T18:15:09.509545+0000","last_undegraded":"2023-09-23T18:15:09.509545+0000","last_fullsized":"2023-09-23T18:15:09.509545+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:58.012609+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:58.012609+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:46:41.805062+0000","scrub_duration":0.007530826,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":429,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504349+0000","last_change":"2023-09-23T18:15:02.377448+0000","last_active":"2023-09-23T18:15:09.504349+0000","last_peered":"2023-09-23T18:15:09.504349+0000","last_clean":"2023-09-23T18:15:09.504349+0000","last_became_active":"2023-09-23T18:15:02.377093+0000","last_became_peered":"2023-09-23T18:15:02.377093+0000","last_unstale":"2023-09-23T18:15:09.504349+0000","last_undegraded":"2023-09-23T18:15:09.504349+0000","last_fullsized":"2023-09-23T18:15:09.504349+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:57:49.531920+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":429,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.515875+0000","last_change":"2023-09-23T18:12:05.655791+0000","last_active":"2023-09-23T18:15:09.515875+0000","last_peered":"2023-09-23T18:15:09.515875+0000","last_clean":"2023-09-23T18:15:09.515875+0000","last_became_active":"2023-09-23T18:12:05.655227+0000","last_became_peered":"2023-09-23T18:12:05.655227+0000","last_unstale":"2023-09-23T18:15:09.515875+0000","last_undegraded":"2023-09-23T18:15:09.515875+0000","last_fullsized":"2023-09-23T18:15:09.515875+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:37.413645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.1","version":"494'6365","reported_seq":9648,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T18:15:15.749116+0000","last_change":"2023-09-23T18:14:51.186372+0000","last_active":"2023-09-23T18:15:15.749116+0000","last_peered":"2023-09-23T18:15:15.749116+0000","last_clean":"2023-09-23T18:15:15.749116+0000","last_became_active":"2023-09-23T18:14:51.175011+0000","last_became_peered":"2023-09-23T18:14:51.175011+0000","last_unstale":"2023-09-23T18:15:15.749116+0000","last_undegraded":"2023-09-23T18:15:15.749116+0000","last_fullsized":"2023-09-23T18:15:15.749116+0000","mapping_epoch":483,"log_start":"464'1391","ondisk_log_start":"464'1391","created":459,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":4974,"log_dups_size":0,"ondisk_log_size":4974,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:09:45.500512+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":417136640,"num_objects":6365,"num_object_clones":0,"num_object_copies":19095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6365,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6365,"num_write_kb":407360,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1603,"num_bytes_recovered":105054208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":967,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500085+0000","last_change":"2023-09-23T18:15:05.441816+0000","last_active":"2023-09-23T18:15:09.500085+0000","last_peered":"2023-09-23T18:15:09.500085+0000","last_clean":"2023-09-23T18:15:09.500085+0000","last_became_active":"2023-09-23T18:15:05.438382+0000","last_became_peered":"2023-09-23T18:15:05.438382+0000","last_unstale":"2023-09-23T18:15:09.500085+0000","last_undegraded":"2023-09-23T18:15:09.500085+0000","last_fullsized":"2023-09-23T18:15:09.500085+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:52:41.392305+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,6],"acting":[7,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":333,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509436+0000","last_change":"2023-09-23T18:13:39.237303+0000","last_active":"2023-09-23T18:15:09.509436+0000","last_peered":"2023-09-23T18:15:09.509436+0000","last_clean":"2023-09-23T18:15:09.509436+0000","last_became_active":"2023-09-23T18:13:39.230309+0000","last_became_peered":"2023-09-23T18:13:39.230309+0000","last_unstale":"2023-09-23T18:15:09.509436+0000","last_undegraded":"2023-09-23T18:15:09.509436+0000","last_fullsized":"2023-09-23T18:15:09.509436+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:37.392528+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":131,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509312+0000","last_change":"2023-09-23T18:14:32.576635+0000","last_active":"2023-09-23T18:15:09.509312+0000","last_peered":"2023-09-23T18:15:09.509312+0000","last_clean":"2023-09-23T18:15:09.509312+0000","last_became_active":"2023-09-23T18:14:32.576218+0000","last_became_peered":"2023-09-23T18:14:32.576218+0000","last_unstale":"2023-09-23T18:15:09.509312+0000","last_undegraded":"2023-09-23T18:15:09.509312+0000","last_fullsized":"2023-09-23T18:15:09.509312+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:21:10.810798+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.0","version":"494'6659","reported_seq":11341,"reported_epoch":494,"state":"active+recovering+degraded+remapped","last_fresh":"2023-09-23T18:15:17.998561+0000","last_change":"2023-09-23T18:15:03.793849+0000","last_active":"2023-09-23T18:15:17.998561+0000","last_peered":"2023-09-23T18:15:17.998561+0000","last_clean":"2023-09-23T18:13:44.646980+0000","last_became_active":"2023-09-23T18:14:50.230022+0000","last_became_peered":"2023-09-23T18:14:50.230022+0000","last_unstale":"2023-09-23T18:15:17.998561+0000","last_undegraded":"2023-09-23T18:14:50.224901+0000","last_fullsized":"2023-09-23T18:15:17.998561+0000","mapping_epoch":482,"log_start":"466'2725","ondisk_log_start":"466'2725","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":3934,"log_dups_size":0,"ondisk_log_size":3934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:54.618501+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":436404224,"num_objects":6659,"num_object_clones":0,"num_object_copies":19977,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1150,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6659,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6659,"num_write_kb":426176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2298,"num_bytes_recovered":150601728,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,0],"avail_no_missing":["1","6"],"object_location_counts":[{"shards":"0,1,6","objects":5509},{"shards":"1,6","objects":1150}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":771,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504825+0000","last_change":"2023-09-23T18:15:05.453647+0000","last_active":"2023-09-23T18:15:09.504825+0000","last_peered":"2023-09-23T18:15:09.504825+0000","last_clean":"2023-09-23T18:15:09.504825+0000","last_became_active":"2023-09-23T18:15:05.453337+0000","last_became_peered":"2023-09-23T18:15:05.453337+0000","last_unstale":"2023-09-23T18:15:09.504825+0000","last_undegraded":"2023-09-23T18:15:09.504825+0000","last_fullsized":"2023-09-23T18:15:09.504825+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:48.645713+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":29,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.506103+0000","last_change":"2023-09-23T18:14:32.566145+0000","last_active":"2023-09-23T18:15:09.506103+0000","last_peered":"2023-09-23T18:15:09.506103+0000","last_clean":"2023-09-23T18:15:09.506103+0000","last_became_active":"2023-09-23T18:14:32.565837+0000","last_became_peered":"2023-09-23T18:14:32.565837+0000","last_unstale":"2023-09-23T18:15:09.506103+0000","last_undegraded":"2023-09-23T18:15:09.506103+0000","last_fullsized":"2023-09-23T18:15:09.506103+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:14:20.416197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":335,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.511301+0000","last_change":"2023-09-23T18:14:50.152289+0000","last_active":"2023-09-23T18:15:09.511301+0000","last_peered":"2023-09-23T18:15:09.511301+0000","last_clean":"2023-09-23T18:15:09.511301+0000","last_became_active":"2023-09-23T18:14:50.151167+0000","last_became_peered":"2023-09-23T18:14:50.151167+0000","last_unstale":"2023-09-23T18:15:09.511301+0000","last_undegraded":"2023-09-23T18:15:09.511301+0000","last_fullsized":"2023-09-23T18:15:09.511301+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:24:45.752762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.3","version":"494'6555","reported_seq":6598,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T18:15:18.257496+0000","last_change":"2023-09-23T18:13:44.089720+0000","last_active":"2023-09-23T18:15:18.257496+0000","last_peered":"2023-09-23T18:15:18.257496+0000","last_clean":"2023-09-23T18:15:18.257496+0000","last_became_active":"2023-09-23T18:13:44.089423+0000","last_became_peered":"2023-09-23T18:13:44.089423+0000","last_unstale":"2023-09-23T18:15:18.257496+0000","last_undegraded":"2023-09-23T18:15:18.257496+0000","last_fullsized":"2023-09-23T18:15:18.257496+0000","mapping_epoch":459,"log_start":"465'2264","ondisk_log_start":"465'2264","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":4291,"log_dups_size":0,"ondisk_log_size":4291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:11:59.848952+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429588480,"num_objects":6555,"num_object_clones":0,"num_object_copies":19665,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6555,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6555,"num_write_kb":419520,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,1],"acting":[7,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":907,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.516318+0000","last_change":"2023-09-23T18:15:05.415849+0000","last_active":"2023-09-23T18:15:09.516318+0000","last_peered":"2023-09-23T18:15:09.516318+0000","last_clean":"2023-09-23T18:15:09.516318+0000","last_became_active":"2023-09-23T18:15:05.415448+0000","last_became_peered":"2023-09-23T18:15:05.415448+0000","last_unstale":"2023-09-23T18:15:09.516318+0000","last_undegraded":"2023-09-23T18:15:09.516318+0000","last_fullsized":"2023-09-23T18:15:09.516318+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:29:09.968103+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":315,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508759+0000","last_change":"2023-09-23T18:14:50.153729+0000","last_active":"2023-09-23T18:15:09.508759+0000","last_peered":"2023-09-23T18:15:09.508759+0000","last_clean":"2023-09-23T18:15:09.508759+0000","last_became_active":"2023-09-23T18:14:50.153432+0000","last_became_peered":"2023-09-23T18:14:50.153432+0000","last_unstale":"2023-09-23T18:15:09.508759+0000","last_undegraded":"2023-09-23T18:15:09.508759+0000","last_fullsized":"2023-09-23T18:15:09.508759+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:07:14.635756+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":111,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509632+0000","last_change":"2023-09-23T18:14:32.571811+0000","last_active":"2023-09-23T18:15:09.509632+0000","last_peered":"2023-09-23T18:15:09.509632+0000","last_clean":"2023-09-23T18:15:09.509632+0000","last_became_active":"2023-09-23T18:14:32.571501+0000","last_became_peered":"2023-09-23T18:14:32.571501+0000","last_unstale":"2023-09-23T18:15:09.509632+0000","last_undegraded":"2023-09-23T18:15:09.509632+0000","last_fullsized":"2023-09-23T18:15:09.509632+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:59.974796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"9.2","version":"494'6551","reported_seq":6619,"reported_epoch":494,"state":"active+undersized+degraded+remapped+backfill_wait","last_fresh":"2023-09-23T18:15:17.974109+0000","last_change":"2023-09-23T18:15:02.396417+0000","last_active":"2023-09-23T18:15:17.974109+0000","last_peered":"2023-09-23T18:15:17.974109+0000","last_clean":"2023-09-23T18:13:44.647171+0000","last_became_active":"2023-09-23T18:14:51.206257+0000","last_became_peered":"2023-09-23T18:14:51.206257+0000","last_unstale":"2023-09-23T18:15:17.974109+0000","last_undegraded":"2023-09-23T18:14:51.172703+0000","last_fullsized":"2023-09-23T18:14:51.163308+0000","mapping_epoch":483,"log_start":"466'2614","ondisk_log_start":"466'2614","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":3937,"log_dups_size":0,"ondisk_log_size":3937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:41.826488+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":429326336,"num_objects":6551,"num_object_clones":0,"num_object_copies":19653,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":6551,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6551,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6551,"num_write_kb":419264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,7],"acting":[1,7],"avail_no_missing":["1","7"],"object_location_counts":[{"shards":"1,7","objects":6551}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":556,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.506253+0000","last_change":"2023-09-23T18:15:05.446257+0000","last_active":"2023-09-23T18:15:09.506253+0000","last_peered":"2023-09-23T18:15:09.506253+0000","last_clean":"2023-09-23T18:15:09.506253+0000","last_became_active":"2023-09-23T18:15:05.445912+0000","last_became_peered":"2023-09-23T18:15:05.445912+0000","last_unstale":"2023-09-23T18:15:09.506253+0000","last_undegraded":"2023-09-23T18:15:09.506253+0000","last_fullsized":"2023-09-23T18:15:09.506253+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.509865+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.509865+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,4],"acting":[6,2,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":361,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501904+0000","last_change":"2023-09-23T18:12:05.652577+0000","last_active":"2023-09-23T18:15:09.501904+0000","last_peered":"2023-09-23T18:15:09.501904+0000","last_clean":"2023-09-23T18:15:09.501904+0000","last_became_active":"2023-09-23T18:12:05.652290+0000","last_became_peered":"2023-09-23T18:12:05.652290+0000","last_unstale":"2023-09-23T18:15:09.501904+0000","last_undegraded":"2023-09-23T18:15:09.501904+0000","last_fullsized":"2023-09-23T18:15:09.501904+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:18:32.320757+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":277,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501879+0000","last_change":"2023-09-23T18:13:39.229808+0000","last_active":"2023-09-23T18:15:09.501879+0000","last_peered":"2023-09-23T18:15:09.501879+0000","last_clean":"2023-09-23T18:15:09.501879+0000","last_became_active":"2023-09-23T18:13:39.229469+0000","last_became_peered":"2023-09-23T18:13:39.229469+0000","last_unstale":"2023-09-23T18:15:09.501879+0000","last_undegraded":"2023-09-23T18:15:09.501879+0000","last_fullsized":"2023-09-23T18:15:09.501879+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:33.892578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.f","version":"494'6411","reported_seq":6462,"reported_epoch":494,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:15:18.006507+0000","last_change":"2023-09-23T18:15:03.550039+0000","last_active":"2023-09-23T18:15:18.006507+0000","last_peered":"2023-09-23T18:15:18.006507+0000","last_clean":"2023-09-23T18:15:01.262016+0000","last_became_active":"2023-09-23T18:15:03.532987+0000","last_became_peered":"2023-09-23T18:15:03.532987+0000","last_unstale":"2023-09-23T18:15:18.006507+0000","last_undegraded":"2023-09-23T18:15:18.006507+0000","last_fullsized":"2023-09-23T18:15:18.006507+0000","mapping_epoch":489,"log_start":"466'2467","ondisk_log_start":"466'2467","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":3944,"log_dups_size":0,"ondisk_log_size":3944,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:38.090354+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":420151296,"num_objects":6411,"num_object_clones":0,"num_object_copies":19233,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6411,"num_objects_unfound":0,"num_objects_dirty":6411,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6411,"num_write_kb":410304,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,5],"avail_no_missing":["1","4","5"],"object_location_counts":[{"shards":"1,4,5","objects":6411}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":534,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504697+0000","last_change":"2023-09-23T18:15:05.454861+0000","last_active":"2023-09-23T18:15:09.504697+0000","last_peered":"2023-09-23T18:15:09.504697+0000","last_clean":"2023-09-23T18:15:09.504697+0000","last_became_active":"2023-09-23T18:15:05.454380+0000","last_became_peered":"2023-09-23T18:15:05.454380+0000","last_unstale":"2023-09-23T18:15:09.504697+0000","last_undegraded":"2023-09-23T18:15:09.504697+0000","last_fullsized":"2023-09-23T18:15:09.504697+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:52:48.343670+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,1],"acting":[6,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":899,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.503556+0000","last_change":"2023-09-23T18:10:04.168459+0000","last_active":"2023-09-23T18:15:09.503556+0000","last_peered":"2023-09-23T18:15:09.503556+0000","last_clean":"2023-09-23T18:15:09.503556+0000","last_became_active":"2023-09-23T18:10:04.167807+0000","last_became_peered":"2023-09-23T18:10:04.167807+0000","last_unstale":"2023-09-23T18:15:09.503556+0000","last_undegraded":"2023-09-23T18:15:09.503556+0000","last_fullsized":"2023-09-23T18:15:09.503556+0000","mapping_epoch":373,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":374,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:40.519034+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":473,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.505373+0000","last_change":"2023-09-23T18:15:02.362081+0000","last_active":"2023-09-23T18:15:09.505373+0000","last_peered":"2023-09-23T18:15:09.505373+0000","last_clean":"2023-09-23T18:15:09.505373+0000","last_became_active":"2023-09-23T18:15:02.361525+0000","last_became_peered":"2023-09-23T18:15:02.361525+0000","last_unstale":"2023-09-23T18:15:09.505373+0000","last_undegraded":"2023-09-23T18:15:09.505373+0000","last_fullsized":"2023-09-23T18:15:09.505373+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:17:15.393004+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,7],"acting":[6,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":268,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501291+0000","last_change":"2023-09-23T18:15:02.371406+0000","last_active":"2023-09-23T18:15:09.501291+0000","last_peered":"2023-09-23T18:15:09.501291+0000","last_clean":"2023-09-23T18:15:09.501291+0000","last_became_active":"2023-09-23T18:15:02.369507+0000","last_became_peered":"2023-09-23T18:15:02.369507+0000","last_unstale":"2023-09-23T18:15:09.501291+0000","last_undegraded":"2023-09-23T18:15:09.501291+0000","last_fullsized":"2023-09-23T18:15:09.501291+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:16:27.238434+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.9","version":"494'6482","reported_seq":6542,"reported_epoch":494,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:15:18.222931+0000","last_change":"2023-09-23T18:15:03.661971+0000","last_active":"2023-09-23T18:15:18.222931+0000","last_peered":"2023-09-23T18:15:18.222931+0000","last_clean":"2023-09-23T18:15:01.322651+0000","last_became_active":"2023-09-23T18:15:03.578016+0000","last_became_peered":"2023-09-23T18:15:03.578016+0000","last_unstale":"2023-09-23T18:15:18.222931+0000","last_undegraded":"2023-09-23T18:15:18.222931+0000","last_fullsized":"2023-09-23T18:15:18.222931+0000","mapping_epoch":489,"log_start":"465'2160","ondisk_log_start":"465'2160","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":4322,"log_dups_size":0,"ondisk_log_size":4322,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:44.027116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":424804352,"num_objects":6482,"num_object_clones":0,"num_object_copies":19446,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6482,"num_objects_unfound":0,"num_objects_dirty":6482,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6482,"num_write_kb":414848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,5],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":6482}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":670,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508303+0000","last_change":"2023-09-23T18:15:05.421996+0000","last_active":"2023-09-23T18:15:09.508303+0000","last_peered":"2023-09-23T18:15:09.508303+0000","last_clean":"2023-09-23T18:15:09.508303+0000","last_became_active":"2023-09-23T18:15:05.420974+0000","last_became_peered":"2023-09-23T18:15:05.420974+0000","last_unstale":"2023-09-23T18:15:09.508303+0000","last_undegraded":"2023-09-23T18:15:09.508303+0000","last_fullsized":"2023-09-23T18:15:09.508303+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:56:51.310201+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,2],"acting":[4,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1319,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500997+0000","last_change":"2023-09-23T18:15:03.770780+0000","last_active":"2023-09-23T18:15:09.500997+0000","last_peered":"2023-09-23T18:15:09.500997+0000","last_clean":"2023-09-23T18:15:09.500997+0000","last_became_active":"2023-09-23T18:15:03.372803+0000","last_became_peered":"2023-09-23T18:15:03.372803+0000","last_unstale":"2023-09-23T18:15:09.500997+0000","last_undegraded":"2023-09-23T18:15:09.500997+0000","last_fullsized":"2023-09-23T18:15:09.500997+0000","mapping_epoch":489,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":490,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:53.860579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,0],"acting":[1,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":111,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509143+0000","last_change":"2023-09-23T18:14:32.571585+0000","last_active":"2023-09-23T18:15:09.509143+0000","last_peered":"2023-09-23T18:15:09.509143+0000","last_clean":"2023-09-23T18:15:09.509143+0000","last_became_active":"2023-09-23T18:14:32.571225+0000","last_became_peered":"2023-09-23T18:14:32.571225+0000","last_unstale":"2023-09-23T18:15:09.509143+0000","last_undegraded":"2023-09-23T18:15:09.509143+0000","last_fullsized":"2023-09-23T18:15:09.509143+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:14.338611+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":286,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.503514+0000","last_change":"2023-09-23T18:14:51.171996+0000","last_active":"2023-09-23T18:15:09.503514+0000","last_peered":"2023-09-23T18:15:09.503514+0000","last_clean":"2023-09-23T18:15:09.503514+0000","last_became_active":"2023-09-23T18:14:51.171690+0000","last_became_peered":"2023-09-23T18:14:51.171690+0000","last_unstale":"2023-09-23T18:15:09.503514+0000","last_undegraded":"2023-09-23T18:15:09.503514+0000","last_fullsized":"2023-09-23T18:15:09.503514+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:24:45.002355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.a","version":"494'6457","reported_seq":6508,"reported_epoch":494,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:15:17.987056+0000","last_change":"2023-09-23T18:15:03.523003+0000","last_active":"2023-09-23T18:15:17.987056+0000","last_peered":"2023-09-23T18:15:17.987056+0000","last_clean":"2023-09-23T18:15:01.307200+0000","last_became_active":"2023-09-23T18:15:03.516197+0000","last_became_peered":"2023-09-23T18:15:03.516197+0000","last_unstale":"2023-09-23T18:15:17.987056+0000","last_undegraded":"2023-09-23T18:15:17.987056+0000","last_fullsized":"2023-09-23T18:15:17.987056+0000","mapping_epoch":489,"log_start":"466'2568","ondisk_log_start":"466'2568","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":3889,"log_dups_size":0,"ondisk_log_size":3889,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:40:24.617656+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":423165952,"num_objects":6457,"num_object_clones":0,"num_object_copies":19371,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6457,"num_objects_unfound":0,"num_objects_dirty":6457,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6457,"num_write_kb":413248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,6],"acting":[1,7,5],"avail_no_missing":["1","5","7"],"object_location_counts":[{"shards":"1,5,7","objects":6457}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":729,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508883+0000","last_change":"2023-09-23T18:12:06.667895+0000","last_active":"2023-09-23T18:15:09.508883+0000","last_peered":"2023-09-23T18:15:09.508883+0000","last_clean":"2023-09-23T18:15:09.508883+0000","last_became_active":"2023-09-23T18:12:06.657611+0000","last_became_peered":"2023-09-23T18:12:06.657611+0000","last_unstale":"2023-09-23T18:15:09.508883+0000","last_undegraded":"2023-09-23T18:15:09.508883+0000","last_fullsized":"2023-09-23T18:15:09.508883+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:42:49.208587+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,0],"acting":[4,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":796,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.499597+0000","last_change":"2023-09-23T18:14:50.149104+0000","last_active":"2023-09-23T18:15:09.499597+0000","last_peered":"2023-09-23T18:15:09.499597+0000","last_clean":"2023-09-23T18:15:09.499597+0000","last_became_active":"2023-09-23T18:14:50.148534+0000","last_became_peered":"2023-09-23T18:14:50.148534+0000","last_unstale":"2023-09-23T18:15:09.499597+0000","last_undegraded":"2023-09-23T18:15:09.499597+0000","last_fullsized":"2023-09-23T18:15:09.499597+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:07:49.593322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":277,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.502121+0000","last_change":"2023-09-23T18:14:51.170785+0000","last_active":"2023-09-23T18:15:09.502121+0000","last_peered":"2023-09-23T18:15:09.502121+0000","last_clean":"2023-09-23T18:15:09.502121+0000","last_became_active":"2023-09-23T18:14:51.170406+0000","last_became_peered":"2023-09-23T18:14:51.170406+0000","last_unstale":"2023-09-23T18:15:09.502121+0000","last_undegraded":"2023-09-23T18:15:09.502121+0000","last_fullsized":"2023-09-23T18:15:09.502121+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:44.012355+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":256,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.502100+0000","last_change":"2023-09-23T18:15:02.360389+0000","last_active":"2023-09-23T18:15:09.502100+0000","last_peered":"2023-09-23T18:15:09.502100+0000","last_clean":"2023-09-23T18:15:09.502100+0000","last_became_active":"2023-09-23T18:15:02.360126+0000","last_became_peered":"2023-09-23T18:15:02.360126+0000","last_unstale":"2023-09-23T18:15:09.502100+0000","last_undegraded":"2023-09-23T18:15:09.502100+0000","last_fullsized":"2023-09-23T18:15:09.502100+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:07.251881+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.8","version":"494'6335","reported_seq":8735,"reported_epoch":494,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T18:15:18.275803+0000","last_change":"2023-09-23T18:15:03.739771+0000","last_active":"2023-09-23T18:15:18.275803+0000","last_peered":"2023-09-23T18:15:18.275803+0000","last_clean":"2023-09-23T18:15:01.305551+0000","last_became_active":"2023-09-23T18:15:03.509202+0000","last_became_peered":"2023-09-23T18:15:03.509202+0000","last_unstale":"2023-09-23T18:15:18.275803+0000","last_undegraded":"2023-09-23T18:15:18.275803+0000","last_fullsized":"2023-09-23T18:15:18.275803+0000","mapping_epoch":489,"log_start":"465'2017","ondisk_log_start":"465'2017","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":4318,"log_dups_size":0,"ondisk_log_size":4318,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:27:31.134237+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415170560,"num_objects":6335,"num_object_clones":0,"num_object_copies":19005,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":5099,"num_objects_unfound":0,"num_objects_dirty":6335,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6335,"num_write_kb":405440,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1174,"num_bytes_recovered":76939264,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,1],"acting":[7,0,5],"avail_no_missing":["7","0","5"],"object_location_counts":[{"shards":"0,5,7","objects":6335}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1174,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.505709+0000","last_change":"2023-09-23T18:15:05.425595+0000","last_active":"2023-09-23T18:15:09.505709+0000","last_peered":"2023-09-23T18:15:09.505709+0000","last_clean":"2023-09-23T18:15:09.505709+0000","last_became_active":"2023-09-23T18:15:05.425001+0000","last_became_peered":"2023-09-23T18:15:05.425001+0000","last_unstale":"2023-09-23T18:15:09.505709+0000","last_undegraded":"2023-09-23T18:15:09.505709+0000","last_fullsized":"2023-09-23T18:15:09.505709+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:42:14.947699+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,3],"acting":[7,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":873,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501014+0000","last_change":"2023-09-23T18:14:33.574868+0000","last_active":"2023-09-23T18:15:09.501014+0000","last_peered":"2023-09-23T18:15:09.501014+0000","last_clean":"2023-09-23T18:15:09.501014+0000","last_became_active":"2023-09-23T18:14:33.574528+0000","last_became_peered":"2023-09-23T18:14:33.574528+0000","last_unstale":"2023-09-23T18:15:09.501014+0000","last_undegraded":"2023-09-23T18:15:09.501014+0000","last_fullsized":"2023-09-23T18:15:09.501014+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:21.272232+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":273,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.506027+0000","last_change":"2023-09-23T18:15:02.376729+0000","last_active":"2023-09-23T18:15:09.506027+0000","last_peered":"2023-09-23T18:15:09.506027+0000","last_clean":"2023-09-23T18:15:09.506027+0000","last_became_active":"2023-09-23T18:15:02.375966+0000","last_became_peered":"2023-09-23T18:15:02.375966+0000","last_unstale":"2023-09-23T18:15:09.506027+0000","last_undegraded":"2023-09-23T18:15:09.506027+0000","last_fullsized":"2023-09-23T18:15:09.506027+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:22.659570+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":29,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.505288+0000","last_change":"2023-09-23T18:14:32.576439+0000","last_active":"2023-09-23T18:15:09.505288+0000","last_peered":"2023-09-23T18:15:09.505288+0000","last_clean":"2023-09-23T18:15:09.505288+0000","last_became_active":"2023-09-23T18:14:32.576130+0000","last_became_peered":"2023-09-23T18:14:32.576130+0000","last_unstale":"2023-09-23T18:15:09.505288+0000","last_undegraded":"2023-09-23T18:15:09.505288+0000","last_fullsized":"2023-09-23T18:15:09.505288+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:29.142118+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"9.b","version":"494'6501","reported_seq":11078,"reported_epoch":494,"state":"active+remapped+backfilling","last_fresh":"2023-09-23T18:15:15.782490+0000","last_change":"2023-09-23T18:15:03.627630+0000","last_active":"2023-09-23T18:15:15.782490+0000","last_peered":"2023-09-23T18:15:15.782490+0000","last_clean":"2023-09-23T18:15:01.314493+0000","last_became_active":"2023-09-23T18:15:03.501230+0000","last_became_peered":"2023-09-23T18:15:03.501230+0000","last_unstale":"2023-09-23T18:15:15.782490+0000","last_undegraded":"2023-09-23T18:15:15.782490+0000","last_fullsized":"2023-09-23T18:15:15.782490+0000","mapping_epoch":489,"log_start":"464'1565","ondisk_log_start":"464'1565","created":459,"last_epoch_clean":486,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":4936,"log_dups_size":0,"ondisk_log_size":4936,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:45.936312+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":426049536,"num_objects":6501,"num_object_clones":0,"num_object_copies":19503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":4966,"num_objects_unfound":0,"num_objects_dirty":6501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6501,"num_write_kb":416064,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2244,"num_bytes_recovered":147062784,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":6501}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":711,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.516047+0000","last_change":"2023-09-23T18:14:35.624253+0000","last_active":"2023-09-23T18:15:09.516047+0000","last_peered":"2023-09-23T18:15:09.516047+0000","last_clean":"2023-09-23T18:15:09.516047+0000","last_became_active":"2023-09-23T18:14:35.623844+0000","last_became_peered":"2023-09-23T18:14:35.623844+0000","last_unstale":"2023-09-23T18:15:09.516047+0000","last_undegraded":"2023-09-23T18:15:09.516047+0000","last_fullsized":"2023-09-23T18:15:09.516047+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:18.363849+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":757,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501363+0000","last_change":"2023-09-23T18:10:01.230791+0000","last_active":"2023-09-23T18:15:09.501363+0000","last_peered":"2023-09-23T18:15:09.501363+0000","last_clean":"2023-09-23T18:15:09.501363+0000","last_became_active":"2023-09-23T18:10:01.170909+0000","last_became_peered":"2023-09-23T18:10:01.170909+0000","last_unstale":"2023-09-23T18:15:09.501363+0000","last_undegraded":"2023-09-23T18:15:09.501363+0000","last_fullsized":"2023-09-23T18:15:09.501363+0000","mapping_epoch":370,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":371,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:23:08.759640+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":445,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508754+0000","last_change":"2023-09-23T18:15:02.354373+0000","last_active":"2023-09-23T18:15:09.508754+0000","last_peered":"2023-09-23T18:15:09.508754+0000","last_clean":"2023-09-23T18:15:09.508754+0000","last_became_active":"2023-09-23T18:15:02.354067+0000","last_became_peered":"2023-09-23T18:15:02.354067+0000","last_unstale":"2023-09-23T18:15:09.508754+0000","last_undegraded":"2023-09-23T18:15:09.508754+0000","last_fullsized":"2023-09-23T18:15:09.508754+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:44.651421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,7],"acting":[4,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":498,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500869+0000","last_change":"2023-09-23T18:15:02.362891+0000","last_active":"2023-09-23T18:15:09.500869+0000","last_peered":"2023-09-23T18:15:09.500869+0000","last_clean":"2023-09-23T18:15:09.500869+0000","last_became_active":"2023-09-23T18:15:02.362533+0000","last_became_peered":"2023-09-23T18:15:02.362533+0000","last_unstale":"2023-09-23T18:15:09.500869+0000","last_undegraded":"2023-09-23T18:15:09.500869+0000","last_fullsized":"2023-09-23T18:15:09.500869+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:14:14.390983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,7,4],"acting":[1,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.e","version":"494'6639","reported_seq":6682,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T18:15:18.200807+0000","last_change":"2023-09-23T18:13:43.384807+0000","last_active":"2023-09-23T18:15:18.200807+0000","last_peered":"2023-09-23T18:15:18.200807+0000","last_clean":"2023-09-23T18:15:18.200807+0000","last_became_active":"2023-09-23T18:13:43.384531+0000","last_became_peered":"2023-09-23T18:13:43.384531+0000","last_unstale":"2023-09-23T18:15:18.200807+0000","last_undegraded":"2023-09-23T18:15:18.200807+0000","last_fullsized":"2023-09-23T18:15:18.200807+0000","mapping_epoch":459,"log_start":"465'2274","ondisk_log_start":"465'2274","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":4365,"log_dups_size":0,"ondisk_log_size":4365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:24:12.269073+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":435093504,"num_objects":6639,"num_object_clones":0,"num_object_copies":19917,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6639,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6639,"num_write_kb":424896,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":732,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.506931+0000","last_change":"2023-09-23T18:14:33.580637+0000","last_active":"2023-09-23T18:15:09.506931+0000","last_peered":"2023-09-23T18:15:09.506931+0000","last_clean":"2023-09-23T18:15:09.506931+0000","last_became_active":"2023-09-23T18:14:33.580395+0000","last_became_peered":"2023-09-23T18:14:33.580395+0000","last_unstale":"2023-09-23T18:15:09.506931+0000","last_undegraded":"2023-09-23T18:15:09.506931+0000","last_fullsized":"2023-09-23T18:15:09.506931+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:36:25.187301+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":785,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.502301+0000","last_change":"2023-09-23T18:12:31.342399+0000","last_active":"2023-09-23T18:15:09.502301+0000","last_peered":"2023-09-23T18:15:09.502301+0000","last_clean":"2023-09-23T18:15:09.502301+0000","last_became_active":"2023-09-23T18:12:05.655437+0000","last_became_peered":"2023-09-23T18:12:05.655437+0000","last_unstale":"2023-09-23T18:15:09.502301+0000","last_undegraded":"2023-09-23T18:15:09.502301+0000","last_fullsized":"2023-09-23T18:15:09.502301+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":420,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:37:20.926785+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":35,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":274,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.503474+0000","last_change":"2023-09-23T18:15:02.372918+0000","last_active":"2023-09-23T18:15:09.503474+0000","last_peered":"2023-09-23T18:15:09.503474+0000","last_clean":"2023-09-23T18:15:09.503474+0000","last_became_active":"2023-09-23T18:15:02.368706+0000","last_became_peered":"2023-09-23T18:15:02.368706+0000","last_unstale":"2023-09-23T18:15:09.503474+0000","last_undegraded":"2023-09-23T18:15:09.503474+0000","last_fullsized":"2023-09-23T18:15:09.503474+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:28:32.141074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":16,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.515599+0000","last_change":"2023-09-23T18:15:02.370461+0000","last_active":"2023-09-23T18:15:09.515599+0000","last_peered":"2023-09-23T18:15:09.515599+0000","last_clean":"2023-09-23T18:15:09.515599+0000","last_became_active":"2023-09-23T18:15:02.368131+0000","last_became_peered":"2023-09-23T18:15:02.368131+0000","last_unstale":"2023-09-23T18:15:09.515599+0000","last_undegraded":"2023-09-23T18:15:09.515599+0000","last_fullsized":"2023-09-23T18:15:09.515599+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:52:18.962417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"9.d","version":"494'6425","reported_seq":6483,"reported_epoch":494,"state":"active+remapped+backfill_wait","last_fresh":"2023-09-23T18:15:18.275343+0000","last_change":"2023-09-23T18:15:03.730815+0000","last_active":"2023-09-23T18:15:18.275343+0000","last_peered":"2023-09-23T18:15:18.275343+0000","last_clean":"2023-09-23T18:15:01.315479+0000","last_became_active":"2023-09-23T18:15:03.604080+0000","last_became_peered":"2023-09-23T18:15:03.604080+0000","last_unstale":"2023-09-23T18:15:18.275343+0000","last_undegraded":"2023-09-23T18:15:18.275343+0000","last_fullsized":"2023-09-23T18:15:18.275343+0000","mapping_epoch":489,"log_start":"465'2133","ondisk_log_start":"465'2133","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":4292,"log_dups_size":0,"ondisk_log_size":4292,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:45:00.832483+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421068800,"num_objects":6425,"num_object_clones":0,"num_object_copies":19275,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":6425,"num_objects_unfound":0,"num_objects_dirty":6425,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6425,"num_write_kb":411200,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":65536,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,4],"acting":[7,1,5],"avail_no_missing":["7","1","5"],"object_location_counts":[{"shards":"1,5,7","objects":6425}],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":912,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.511124+0000","last_change":"2023-09-23T18:15:05.443342+0000","last_active":"2023-09-23T18:15:09.511124+0000","last_peered":"2023-09-23T18:15:09.511124+0000","last_clean":"2023-09-23T18:15:09.511124+0000","last_became_active":"2023-09-23T18:15:05.442985+0000","last_became_peered":"2023-09-23T18:15:05.442985+0000","last_unstale":"2023-09-23T18:15:09.511124+0000","last_undegraded":"2023-09-23T18:15:09.511124+0000","last_fullsized":"2023-09-23T18:15:09.511124+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:20:58.369920+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,7],"acting":[4,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":326,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500436+0000","last_change":"2023-09-23T18:13:39.230634+0000","last_active":"2023-09-23T18:15:09.500436+0000","last_peered":"2023-09-23T18:15:09.500436+0000","last_clean":"2023-09-23T18:15:09.500436+0000","last_became_active":"2023-09-23T18:13:39.224509+0000","last_became_peered":"2023-09-23T18:13:39.224509+0000","last_unstale":"2023-09-23T18:15:09.500436+0000","last_undegraded":"2023-09-23T18:15:09.500436+0000","last_fullsized":"2023-09-23T18:15:09.500436+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:11:19.715343+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,0],"acting":[7,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":302,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501292+0000","last_change":"2023-09-23T18:14:32.648863+0000","last_active":"2023-09-23T18:15:09.501292+0000","last_peered":"2023-09-23T18:15:09.501292+0000","last_clean":"2023-09-23T18:15:09.501292+0000","last_became_active":"2023-09-23T18:14:32.565996+0000","last_became_peered":"2023-09-23T18:14:32.565996+0000","last_unstale":"2023-09-23T18:15:09.501292+0000","last_undegraded":"2023-09-23T18:15:09.501292+0000","last_fullsized":"2023-09-23T18:15:09.501292+0000","mapping_epoch":474,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":475,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:07.925417+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":15,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":325,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.501096+0000","last_change":"2023-09-23T18:15:02.384553+0000","last_active":"2023-09-23T18:15:09.501096+0000","last_peered":"2023-09-23T18:15:09.501096+0000","last_clean":"2023-09-23T18:15:09.501096+0000","last_became_active":"2023-09-23T18:15:02.384247+0000","last_became_peered":"2023-09-23T18:15:02.384247+0000","last_unstale":"2023-09-23T18:15:09.501096+0000","last_undegraded":"2023-09-23T18:15:09.501096+0000","last_fullsized":"2023-09-23T18:15:09.501096+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:52.496925+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,6],"acting":[7,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"9.c","version":"494'6408","reported_seq":6451,"reported_epoch":494,"state":"active+clean","last_fresh":"2023-09-23T18:15:15.911225+0000","last_change":"2023-09-23T18:13:44.089836+0000","last_active":"2023-09-23T18:15:15.911225+0000","last_peered":"2023-09-23T18:15:15.911225+0000","last_clean":"2023-09-23T18:15:15.911225+0000","last_became_active":"2023-09-23T18:13:44.089506+0000","last_became_peered":"2023-09-23T18:13:44.089506+0000","last_unstale":"2023-09-23T18:15:15.911225+0000","last_undegraded":"2023-09-23T18:15:15.911225+0000","last_fullsized":"2023-09-23T18:15:15.911225+0000","mapping_epoch":459,"log_start":"464'2033","ondisk_log_start":"464'2033","created":459,"last_epoch_clean":460,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:13:42.360929+0000","last_clean_scrub_stamp":"2023-09-23T18:13:42.360929+0000","objects_scrubbed":0,"log_size":4375,"log_dups_size":0,"ondisk_log_size":4375,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:13:41.847812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419954688,"num_objects":6408,"num_object_clones":0,"num_object_copies":19224,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6408,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6408,"num_write_kb":410112,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":698,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500174+0000","last_change":"2023-09-23T18:15:05.438785+0000","last_active":"2023-09-23T18:15:09.500174+0000","last_peered":"2023-09-23T18:15:09.500174+0000","last_clean":"2023-09-23T18:15:09.500174+0000","last_became_active":"2023-09-23T18:15:05.438415+0000","last_became_peered":"2023-09-23T18:15:05.438415+0000","last_unstale":"2023-09-23T18:15:09.500174+0000","last_undegraded":"2023-09-23T18:15:09.500174+0000","last_fullsized":"2023-09-23T18:15:09.500174+0000","mapping_epoch":491,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":492,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:33:23.131198+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":849,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504880+0000","last_change":"2023-09-23T18:14:51.176099+0000","last_active":"2023-09-23T18:15:09.504880+0000","last_peered":"2023-09-23T18:15:09.504880+0000","last_clean":"2023-09-23T18:15:09.504880+0000","last_became_active":"2023-09-23T18:14:51.175692+0000","last_became_peered":"2023-09-23T18:14:51.175692+0000","last_unstale":"2023-09-23T18:15:09.504880+0000","last_undegraded":"2023-09-23T18:15:09.504880+0000","last_fullsized":"2023-09-23T18:15:09.504880+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:37:18.968685+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":40,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":546,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500943+0000","last_change":"2023-09-23T18:12:06.646490+0000","last_active":"2023-09-23T18:15:09.500943+0000","last_peered":"2023-09-23T18:15:09.500943+0000","last_clean":"2023-09-23T18:15:09.500943+0000","last_became_active":"2023-09-23T18:12:06.645565+0000","last_became_peered":"2023-09-23T18:12:06.645565+0000","last_unstale":"2023-09-23T18:15:09.500943+0000","last_undegraded":"2023-09-23T18:15:09.500943+0000","last_fullsized":"2023-09-23T18:15:09.500943+0000","mapping_epoch":413,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":414,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:30:54.061911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":327,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508660+0000","last_change":"2023-09-23T18:15:02.364177+0000","last_active":"2023-09-23T18:15:09.508660+0000","last_peered":"2023-09-23T18:15:09.508660+0000","last_clean":"2023-09-23T18:15:09.508660+0000","last_became_active":"2023-09-23T18:15:02.363922+0000","last_became_peered":"2023-09-23T18:15:02.363922+0000","last_unstale":"2023-09-23T18:15:09.508660+0000","last_undegraded":"2023-09-23T18:15:09.508660+0000","last_fullsized":"2023-09-23T18:15:09.508660+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:43:35.761315+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,0],"acting":[4,6,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":454,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.505048+0000","last_change":"2023-09-23T18:14:35.637811+0000","last_active":"2023-09-23T18:15:09.505048+0000","last_peered":"2023-09-23T18:15:09.505048+0000","last_clean":"2023-09-23T18:15:09.505048+0000","last_became_active":"2023-09-23T18:14:35.637468+0000","last_became_peered":"2023-09-23T18:14:35.637468+0000","last_unstale":"2023-09-23T18:15:09.505048+0000","last_undegraded":"2023-09-23T18:15:09.505048+0000","last_fullsized":"2023-09-23T18:15:09.505048+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:14.580819+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:14.580819+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":91,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509061+0000","last_change":"2023-09-23T18:12:05.654572+0000","last_active":"2023-09-23T18:15:09.509061+0000","last_peered":"2023-09-23T18:15:09.509061+0000","last_clean":"2023-09-23T18:15:09.509061+0000","last_became_active":"2023-09-23T18:12:05.654026+0000","last_became_peered":"2023-09-23T18:12:05.654026+0000","last_unstale":"2023-09-23T18:15:09.509061+0000","last_undegraded":"2023-09-23T18:15:09.509061+0000","last_fullsized":"2023-09-23T18:15:09.509061+0000","mapping_epoch":412,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":413,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:55:55.511372+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":578,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.503424+0000","last_change":"2023-09-23T18:15:02.371806+0000","last_active":"2023-09-23T18:15:09.503424+0000","last_peered":"2023-09-23T18:15:09.503424+0000","last_clean":"2023-09-23T18:15:09.503424+0000","last_became_active":"2023-09-23T18:15:02.370617+0000","last_became_peered":"2023-09-23T18:15:02.370617+0000","last_unstale":"2023-09-23T18:15:09.503424+0000","last_undegraded":"2023-09-23T18:15:09.503424+0000","last_fullsized":"2023-09-23T18:15:09.503424+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:32:53.070031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,7],"acting":[1,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":454,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.506679+0000","last_change":"2023-09-23T18:14:35.637811+0000","last_active":"2023-09-23T18:15:09.506679+0000","last_peered":"2023-09-23T18:15:09.506679+0000","last_clean":"2023-09-23T18:15:09.506679+0000","last_became_active":"2023-09-23T18:14:35.637487+0000","last_became_peered":"2023-09-23T18:14:35.637487+0000","last_unstale":"2023-09-23T18:15:09.506679+0000","last_undegraded":"2023-09-23T18:15:09.506679+0000","last_fullsized":"2023-09-23T18:15:09.506679+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.604423+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.604423+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":13418,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":11,"num_read_kb":21,"num_write":6,"num_write_kb":55,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":40253,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,4],"acting":[6,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":132,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509618+0000","last_change":"2023-09-23T18:15:02.390443+0000","last_active":"2023-09-23T18:15:09.509618+0000","last_peered":"2023-09-23T18:15:09.509618+0000","last_clean":"2023-09-23T18:15:09.509618+0000","last_became_active":"2023-09-23T18:15:02.372462+0000","last_became_peered":"2023-09-23T18:15:02.372462+0000","last_unstale":"2023-09-23T18:15:09.509618+0000","last_undegraded":"2023-09-23T18:15:09.509618+0000","last_fullsized":"2023-09-23T18:15:09.509618+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:31:52.660745+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,6],"acting":[4,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":111,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.509128+0000","last_change":"2023-09-23T18:13:39.231091+0000","last_active":"2023-09-23T18:15:09.509128+0000","last_peered":"2023-09-23T18:15:09.509128+0000","last_clean":"2023-09-23T18:15:09.509128+0000","last_became_active":"2023-09-23T18:13:39.230787+0000","last_became_peered":"2023-09-23T18:13:39.230787+0000","last_unstale":"2023-09-23T18:15:09.509128+0000","last_undegraded":"2023-09-23T18:15:09.509128+0000","last_fullsized":"2023-09-23T18:15:09.509128+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:33:40.269418+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":132,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.511222+0000","last_change":"2023-09-23T18:15:02.383257+0000","last_active":"2023-09-23T18:15:09.511222+0000","last_peered":"2023-09-23T18:15:09.511222+0000","last_clean":"2023-09-23T18:15:09.511222+0000","last_became_active":"2023-09-23T18:15:02.367965+0000","last_became_peered":"2023-09-23T18:15:02.367965+0000","last_unstale":"2023-09-23T18:15:09.511222+0000","last_undegraded":"2023-09-23T18:15:09.511222+0000","last_fullsized":"2023-09-23T18:15:09.511222+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:11:39.571675+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":111,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508484+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:15:09.508484+0000","last_peered":"2023-09-23T18:15:09.508484+0000","last_clean":"2023-09-23T18:15:09.508484+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:15:09.508484+0000","last_undegraded":"2023-09-23T18:15:09.508484+0000","last_fullsized":"2023-09-23T18:15:09.508484+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":920,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500251+0000","last_change":"2023-09-23T18:14:35.639769+0000","last_active":"2023-09-23T18:15:09.500251+0000","last_peered":"2023-09-23T18:15:09.500251+0000","last_clean":"2023-09-23T18:15:09.500251+0000","last_became_active":"2023-09-23T18:14:35.639420+0000","last_became_peered":"2023-09-23T18:14:35.639420+0000","last_unstale":"2023-09-23T18:15:09.500251+0000","last_undegraded":"2023-09-23T18:15:09.500251+0000","last_fullsized":"2023-09-23T18:15:09.500251+0000","mapping_epoch":477,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":478,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:06.524828+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":338,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.506855+0000","last_change":"2023-09-23T18:14:51.174477+0000","last_active":"2023-09-23T18:15:09.506855+0000","last_peered":"2023-09-23T18:15:09.506855+0000","last_clean":"2023-09-23T18:15:09.506855+0000","last_became_active":"2023-09-23T18:14:51.174131+0000","last_became_peered":"2023-09-23T18:14:51.174131+0000","last_unstale":"2023-09-23T18:15:09.506855+0000","last_undegraded":"2023-09-23T18:15:09.506855+0000","last_fullsized":"2023-09-23T18:15:09.506855+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:08:28.525961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,1],"acting":[6,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":16,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.499511+0000","last_change":"2023-09-23T18:15:02.373168+0000","last_active":"2023-09-23T18:15:09.499511+0000","last_peered":"2023-09-23T18:15:09.499511+0000","last_clean":"2023-09-23T18:15:09.499511+0000","last_became_active":"2023-09-23T18:15:02.372566+0000","last_became_peered":"2023-09-23T18:15:02.372566+0000","last_unstale":"2023-09-23T18:15:09.499511+0000","last_undegraded":"2023-09-23T18:15:09.499511+0000","last_fullsized":"2023-09-23T18:15:09.499511+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:19.416529+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,0],"acting":[1,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":915,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.500324+0000","last_change":"2023-09-23T18:14:50.155989+0000","last_active":"2023-09-23T18:15:09.500324+0000","last_peered":"2023-09-23T18:15:09.500324+0000","last_clean":"2023-09-23T18:15:09.500324+0000","last_became_active":"2023-09-23T18:14:50.154682+0000","last_became_peered":"2023-09-23T18:14:50.154682+0000","last_unstale":"2023-09-23T18:15:09.500324+0000","last_undegraded":"2023-09-23T18:15:09.500324+0000","last_fullsized":"2023-09-23T18:15:09.500324+0000","mapping_epoch":482,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":483,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:17.635974+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:17.635974+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:25.872380+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,2],"acting":[1,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":333,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.504249+0000","last_change":"2023-09-23T18:15:02.408597+0000","last_active":"2023-09-23T18:15:09.504249+0000","last_peered":"2023-09-23T18:15:09.504249+0000","last_clean":"2023-09-23T18:15:09.504249+0000","last_became_active":"2023-09-23T18:15:02.394251+0000","last_became_peered":"2023-09-23T18:15:02.394251+0000","last_unstale":"2023-09-23T18:15:09.504249+0000","last_undegraded":"2023-09-23T18:15:09.504249+0000","last_fullsized":"2023-09-23T18:15:09.504249+0000","mapping_epoch":488,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":489,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:44.564462+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":532,"reported_epoch":493,"state":"active+clean","last_fresh":"2023-09-23T18:15:09.508307+0000","last_change":"2023-09-23T18:14:33.578457+0000","last_active":"2023-09-23T18:15:09.508307+0000","last_peered":"2023-09-23T18:15:09.508307+0000","last_clean":"2023-09-23T18:15:09.508307+0000","last_became_active":"2023-09-23T18:14:33.578186+0000","last_became_peered":"2023-09-23T18:14:33.578186+0000","last_unstale":"2023-09-23T18:15:09.508307+0000","last_undegraded":"2023-09-23T18:15:09.508307+0000","last_fullsized":"2023-09-23T18:15:09.508307+0000","mapping_epoch":475,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":476,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:41:02.084377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":7686,"num_keys_recovered":60,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":106496,"data_stored":17136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":65518,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":156537,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":129,"num_read_kb":244,"num_write":64,"num_write_kb":641,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":13,"num_bytes_recovered":3672401,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":9,"num_pg":16,"stat_sum":{"num_bytes":6803423232,"num_objects":103812,"num_object_clones":0,"num_object_copies":311436,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":7701,"num_objects_misplaced":55268,"num_objects_unfound":0,"num_objects_dirty":103812,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":103812,"num_write_kb":6643968,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8171,"num_bytes_recovered":535494656,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20085932032,"data_stored":20085932032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":68131,"ondisk_log_size":68131,"up":48,"acting":47,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":259,"num_bytes_recovered":8388665,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":12595200,"data_stored":12582969,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001512,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4971924,"kb_used_data":4573900,"kb_used_omap":13,"kb_used_meta":398002,"kb_avail":88773228,"statfs":{"total":95995035648,"available":90903785472,"internally_reserved":0,"allocated":4683673600,"data_stored":4682055848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":407554152},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,6,2,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1759999999999999,"5min":1.3420000000000001,"15min":1.03},"min":{"1min":0.20200000000000001,"5min":0.20200000000000001,"15min":0.20200000000000001},"max":{"1min":3.883,"5min":15.869999999999999,"15min":15.869999999999999},"last":1.819},{"interface":"front","average":{"1min":2.2240000000000002,"5min":1.673,"15min":1.2529999999999999},"min":{"1min":0.36699999999999999,"5min":0.249,"15min":0.23000000000000001},"max":{"1min":7.0030000000000001,"5min":18.995999999999999,"15min":18.995999999999999},"last":2.7280000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:14:51 2023","interfaces":[{"interface":"back","average":{"1min":1.3560000000000001,"5min":1.7090000000000001,"15min":1.286},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":4.7569999999999997,"5min":9.5389999999999997,"15min":9.5389999999999997},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.4199999999999999,"5min":1.8069999999999999,"15min":1.339},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.23400000000000001},"max":{"1min":4.6989999999999998,"5min":15.805999999999999,"15min":15.805999999999999},"last":1.9890000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:12:49 2023","interfaces":[{"interface":"back","average":{"1min":1.764,"5min":1.633,"15min":1.2290000000000001},"min":{"1min":0.29899999999999999,"5min":0.28199999999999997,"15min":0.23300000000000001},"max":{"1min":8.4939999999999998,"5min":8.4939999999999998,"15min":8.4939999999999998},"last":1.8999999999999999},{"interface":"front","average":{"1min":2.3290000000000002,"5min":1.8560000000000001,"15min":1.294},"min":{"1min":0.28299999999999997,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":9.4789999999999992,"5min":9.4789999999999992,"15min":9.4789999999999992},"last":2.7639999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:14:41 2023","interfaces":[{"interface":"back","average":{"1min":1.7010000000000001,"5min":2.8069999999999999,"15min":1.8919999999999999},"min":{"1min":0.27500000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.3129999999999997,"5min":45.752000000000002,"15min":45.752000000000002},"last":0.376},{"interface":"front","average":{"1min":2.0619999999999998,"5min":2.1760000000000002,"15min":1.575},"min":{"1min":0.32500000000000001,"5min":0.311,"15min":0.29399999999999998},"max":{"1min":7.0510000000000002,"5min":30.832999999999998,"15min":30.832999999999998},"last":1.8440000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:14:51 2023","interfaces":[{"interface":"back","average":{"1min":3.1349999999999998,"5min":1.5429999999999999,"15min":1.1579999999999999},"min":{"1min":0.246,"5min":0.246,"15min":0.245},"max":{"1min":29.782,"5min":29.782,"15min":29.782},"last":2.0990000000000002},{"interface":"front","average":{"1min":3.9900000000000002,"5min":1.696,"15min":1.0620000000000001},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":40.277000000000001,"5min":40.277000000000001,"15min":40.277000000000001},"last":2.6200000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:14:35 2023","interfaces":[{"interface":"back","average":{"1min":1.538,"5min":1.3540000000000001,"15min":1.323},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":5.141,"5min":5.141,"15min":5.141},"last":5.0780000000000003},{"interface":"front","average":{"1min":2.8759999999999999,"5min":2.0649999999999999,"15min":1.9299999999999999},"min":{"1min":0.42899999999999999,"5min":0.253,"15min":0.253},"max":{"1min":16.221,"5min":16.221,"15min":16.221},"last":2.9740000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:14:51 2023","interfaces":[{"interface":"back","average":{"1min":1.99,"5min":1.609,"15min":1.143},"min":{"1min":0.36899999999999999,"5min":0.28299999999999997,"15min":0.28299999999999997},"max":{"1min":4.5199999999999996,"5min":15.847,"15min":15.847},"last":1.7490000000000001},{"interface":"front","average":{"1min":2.9329999999999998,"5min":1.671,"15min":1.2629999999999999},"min":{"1min":0.35399999999999998,"5min":0.253,"15min":0.253},"max":{"1min":6.9379999999999997,"5min":16.975999999999999,"15min":16.975999999999999},"last":2.7890000000000001}]}]},{"osd":6,"up_from":163,"seq":700079669448,"num_pgs":61,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2107068,"kb_used_data":1649780,"kb_used_omap":7,"kb_used_meta":457272,"kb_avail":91638084,"statfs":{"total":95995035648,"available":93837398016,"internally_reserved":0,"allocated":1689374720,"data_stored":1687772499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":468247332},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:15:05 2023","interfaces":[{"interface":"back","average":{"1min":1.6679999999999999,"5min":1.4319999999999999,"15min":1.208},"min":{"1min":0.248,"5min":0.22500000000000001,"15min":0.215},"max":{"1min":4.8710000000000004,"5min":10.801,"15min":15.407},"last":0.90200000000000002},{"interface":"front","average":{"1min":1.962,"5min":1.7450000000000001,"15min":1.3700000000000001},"min":{"1min":0.26100000000000001,"5min":0.25600000000000001,"15min":0.23100000000000001},"max":{"1min":6.3090000000000002,"5min":9.6329999999999991,"15min":15.147},"last":0.54200000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:15:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4710000000000001,"5min":1.8009999999999999,"15min":1.405},"min":{"1min":0.245,"5min":0.218,"15min":0.218},"max":{"1min":11.083,"5min":21.390000000000001,"15min":21.390000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":2.2320000000000002,"5min":2.3540000000000001,"15min":1.5980000000000001},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.248},"max":{"1min":6.375,"5min":21.559999999999999,"15min":21.559999999999999},"last":0.47199999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:12:59 2023","interfaces":[{"interface":"back","average":{"1min":1.593,"5min":1.6910000000000001,"15min":1.6259999999999999},"min":{"1min":0.42699999999999999,"5min":0.29799999999999999,"15min":0.27400000000000002},"max":{"1min":5.9880000000000004,"5min":9.4130000000000003,"15min":11.210000000000001},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.625,"5min":1.6180000000000001,"15min":1.6699999999999999},"min":{"1min":0.41599999999999998,"5min":0.248,"15min":0.245},"max":{"1min":8.548,"5min":8.548,"15min":11.223000000000001},"last":0.72299999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:14:42 2023","interfaces":[{"interface":"back","average":{"1min":1.6559999999999999,"5min":1.7769999999999999,"15min":1.407},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.7249999999999996,"5min":10.257999999999999,"15min":10.257999999999999},"last":2.0499999999999998},{"interface":"front","average":{"1min":2.0910000000000002,"5min":2.0859999999999999,"15min":1.5569999999999999},"min":{"1min":0.378,"5min":0.32600000000000001,"15min":0.31900000000000001},"max":{"1min":6.3470000000000004,"5min":9.5960000000000001,"15min":9.5960000000000001},"last":8.1140000000000008}]},{"osd":4,"last update":"Sat Sep 23 18:15:05 2023","interfaces":[{"interface":"back","average":{"1min":1.423,"5min":1.2909999999999999,"15min":1.073},"min":{"1min":0.26800000000000002,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":6.2809999999999997,"5min":8.6059999999999999,"15min":9.9109999999999996},"last":0.45500000000000002},{"interface":"front","average":{"1min":2.3410000000000002,"5min":1.2350000000000001,"15min":1.0660000000000001},"min":{"1min":0.27900000000000003,"5min":0.27000000000000002,"15min":0.24099999999999999},"max":{"1min":8.8040000000000003,"5min":8.8040000000000003,"15min":8.8040000000000003},"last":0.35699999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:14:36 2023","interfaces":[{"interface":"back","average":{"1min":1.429,"5min":1.849,"15min":1.919},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":4.3010000000000002,"5min":7.7199999999999998,"15min":7.7199999999999998},"last":0.622},{"interface":"front","average":{"1min":1.6339999999999999,"5min":2.3380000000000001,"15min":2.456},"min":{"1min":0.438,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":7.5179999999999998,"5min":8.8030000000000008,"15min":8.8030000000000008},"last":0.49399999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:14:49 2023","interfaces":[{"interface":"back","average":{"1min":1.728,"5min":1.425,"15min":1.157},"min":{"1min":0.36299999999999999,"5min":0.28000000000000003,"15min":0.26800000000000002},"max":{"1min":6.0629999999999997,"5min":6.5350000000000001,"15min":13.460000000000001},"last":0.95499999999999996},{"interface":"front","average":{"1min":2.286,"5min":1.5109999999999999,"15min":1.163},"min":{"1min":0.32900000000000001,"5min":0.318,"15min":0.28499999999999998},"max":{"1min":8.5869999999999997,"5min":12.404,"15min":12.404},"last":0.871}]}]},{"osd":1,"up_from":98,"seq":420906795266,"num_pgs":78,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5583204,"kb_used_data":4652188,"kb_used_omap":7,"kb_used_meta":931000,"kb_avail":88161948,"statfs":{"total":95995035648,"available":90277834752,"internally_reserved":0,"allocated":4763840512,"data_stored":4761513601,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":953344701},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:30 2023","interfaces":[{"interface":"back","average":{"1min":1.2230000000000001,"5min":1.532,"15min":1.3180000000000001},"min":{"1min":0.23499999999999999,"5min":0.19400000000000001,"15min":0.17699999999999999},"max":{"1min":5.3710000000000004,"5min":12.669,"15min":15.529999999999999},"last":7.6070000000000002},{"interface":"front","average":{"1min":1.2769999999999999,"5min":2.222,"15min":1.915},"min":{"1min":0.23400000000000001,"5min":0.20499999999999999,"15min":0.20100000000000001},"max":{"1min":8.1600000000000001,"5min":20.257000000000001,"15min":82.244},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:13:00 2023","interfaces":[{"interface":"back","average":{"1min":3.165,"5min":2.2410000000000001,"15min":1.8180000000000001},"min":{"1min":0.29699999999999999,"5min":0.22600000000000001,"15min":0.22},"max":{"1min":23.992999999999999,"5min":23.992999999999999,"15min":23.992999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":2.4119999999999999,"5min":1.954,"15min":1.4710000000000001},"min":{"1min":0.28899999999999998,"5min":0.22,"15min":0.21299999999999999},"max":{"1min":10.526999999999999,"5min":10.526999999999999,"15min":10.526999999999999},"last":2.2919999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:14:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.853,"15min":1.357},"min":{"1min":0.215,"5min":0.215,"15min":0.215},"max":{"1min":11.225,"5min":11.519,"15min":11.519},"last":7.7290000000000001},{"interface":"front","average":{"1min":0.80500000000000005,"5min":1.968,"15min":1.4510000000000001},"min":{"1min":0.26100000000000001,"5min":0.248,"15min":0.23599999999999999},"max":{"1min":3.427,"5min":15.446999999999999,"15min":15.446999999999999},"last":0.438}]},{"osd":4,"last update":"Sat Sep 23 18:14:30 2023","interfaces":[{"interface":"back","average":{"1min":1.4099999999999999,"5min":2.1589999999999998,"15min":1.6839999999999999},"min":{"1min":0.28999999999999998,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":12.778,"5min":28.446999999999999,"15min":28.446999999999999},"last":2.3239999999999998},{"interface":"front","average":{"1min":1.1439999999999999,"5min":1.8700000000000001,"15min":1.5029999999999999},"min":{"1min":0.34300000000000003,"5min":0.31,"15min":0.27000000000000002},"max":{"1min":7.5170000000000003,"5min":28.434000000000001,"15min":28.434000000000001},"last":7.6559999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:14:32 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":2.3740000000000001,"15min":2.556},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":5.9249999999999998,"5min":16.085000000000001,"15min":16.085000000000001},"last":0.63800000000000001},{"interface":"front","average":{"1min":1.948,"5min":2.323,"15min":2.3849999999999998},"min":{"1min":0.31,"5min":0.31,"15min":0.31},"max":{"1min":16.268000000000001,"5min":21.812000000000001,"15min":21.812000000000001},"last":2.3580000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:15:04 2023","interfaces":[{"interface":"back","average":{"1min":1.9470000000000001,"5min":2.4420000000000002,"15min":1.9530000000000001},"min":{"1min":0.34799999999999998,"5min":0.307,"15min":0.307},"max":{"1min":4.6630000000000003,"5min":27.516999999999999,"15min":89.566000000000003},"last":2.3410000000000002},{"interface":"front","average":{"1min":1.8500000000000001,"5min":2.2210000000000001,"15min":1.8759999999999999},"min":{"1min":0.38700000000000001,"5min":0.373,"15min":0.34699999999999998},"max":{"1min":6.3840000000000003,"5min":27.501999999999999,"15min":90.608999999999995},"last":0.67200000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:14:57 2023","interfaces":[{"interface":"back","average":{"1min":1.4059999999999999,"5min":1.8120000000000001,"15min":1.282},"min":{"1min":0.34699999999999998,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":5.3019999999999996,"5min":27.913,"15min":27.913},"last":7.6929999999999996},{"interface":"front","average":{"1min":2.1469999999999998,"5min":2.5209999999999999,"15min":1.867},"min":{"1min":0.40200000000000002,"5min":0.371,"15min":0.371},"max":{"1min":5.875,"5min":27.620999999999999,"15min":27.620999999999999},"last":2.0369999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958793,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2262728,"kb_used_data":1556800,"kb_used_omap":7,"kb_used_meta":705912,"kb_avail":91482424,"statfs":{"total":95995035648,"available":93678002176,"internally_reserved":0,"allocated":1594163200,"data_stored":1591853065,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":722854628},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:14:34 2023","interfaces":[{"interface":"back","average":{"1min":0.72799999999999998,"5min":1.544,"15min":1.6539999999999999},"min":{"1min":0.19800000000000001,"5min":0.192,"15min":0.186},"max":{"1min":3.9260000000000002,"5min":13.677,"15min":15.836},"last":1.1180000000000001},{"interface":"front","average":{"1min":1.375,"5min":1.8500000000000001,"15min":1.5760000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.17399999999999999},"max":{"1min":6.4989999999999997,"5min":14.207000000000001,"15min":14.468999999999999},"last":3.8010000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:12:51 2023","interfaces":[{"interface":"back","average":{"1min":1.0860000000000001,"5min":1.2909999999999999,"15min":1.7090000000000001},"min":{"1min":0.24299999999999999,"5min":0.222,"15min":0.20499999999999999},"max":{"1min":3.903,"5min":9.2859999999999996,"15min":20.882000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":1.5189999999999999,"5min":1.579,"15min":1.6890000000000001},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.217},"max":{"1min":6.2750000000000004,"5min":10.84,"15min":15.064},"last":0.47399999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:14:40 2023","interfaces":[{"interface":"back","average":{"1min":1.111,"5min":1.74,"15min":1.6459999999999999},"min":{"1min":0.215,"5min":0.215,"15min":0.20499999999999999},"max":{"1min":5.5979999999999999,"5min":11.015000000000001,"15min":12.779999999999999},"last":0.36699999999999999},{"interface":"front","average":{"1min":1.073,"5min":1.8049999999999999,"15min":1.5700000000000001},"min":{"1min":0.29699999999999999,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":4.6289999999999996,"5min":9.2110000000000003,"15min":15.198},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:15:01 2023","interfaces":[{"interface":"back","average":{"1min":4.3419999999999996,"5min":2.2469999999999999,"15min":1.667},"min":{"1min":0.34200000000000003,"5min":0.32800000000000001,"15min":0.29299999999999998},"max":{"1min":57.746000000000002,"5min":57.746000000000002,"15min":57.746000000000002},"last":0.48799999999999999},{"interface":"front","average":{"1min":4.0949999999999998,"5min":1.853,"15min":1.655},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.247},"max":{"1min":57.816000000000003,"5min":57.816000000000003,"15min":57.816000000000003},"last":4.7469999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:14:34 2023","interfaces":[{"interface":"back","average":{"1min":1.8839999999999999,"5min":1.161,"15min":1.0409999999999999},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":6.4660000000000002,"5min":6.4660000000000002,"15min":6.4660000000000002},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.1479999999999999,"5min":1.052,"15min":1.036},"min":{"1min":0.49199999999999999,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":3.2040000000000002,"5min":3.2040000000000002,"15min":3.2040000000000002},"last":0.59399999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:15:01 2023","interfaces":[{"interface":"back","average":{"1min":1.913,"5min":1.8620000000000001,"15min":1.78},"min":{"1min":0.374,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":5.5389999999999997,"5min":9.1170000000000009,"15min":58.457999999999998},"last":0.625},{"interface":"front","average":{"1min":2.0720000000000001,"5min":1.504,"15min":1.621},"min":{"1min":0.374,"5min":0.315,"15min":0.311},"max":{"1min":5.343,"5min":6.3049999999999997,"15min":58.354999999999997},"last":3.8679999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:14:44 2023","interfaces":[{"interface":"back","average":{"1min":2.1789999999999998,"5min":1.5580000000000001,"15min":1.3260000000000001},"min":{"1min":0.47699999999999998,"5min":0.36599999999999999,"15min":0.34499999999999997},"max":{"1min":5.0410000000000004,"5min":6.3300000000000001,"15min":10.416},"last":0.56200000000000006},{"interface":"front","average":{"1min":2.2200000000000002,"5min":2.044,"15min":1.629},"min":{"1min":0.39800000000000002,"5min":0.39800000000000002,"15min":0.36399999999999999},"max":{"1min":6.0129999999999999,"5min":9.1489999999999991,"15min":9.1489999999999991},"last":4.0979999999999999}]}]},{"osd":2,"up_from":482,"seq":2070174236679,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":182572,"kb_used_data":88484,"kb_used_omap":22,"kb_used_meta":94057,"kb_avail":93562580,"statfs":{"total":95995035648,"available":95808081920,"internally_reserved":0,"allocated":90607616,"data_stored":88271256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22605,"internal_metadata":96315315},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51100000000000001}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34000000000000002}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.45200000000000001}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58699999999999997}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":1.9910000000000001}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47499999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":3.7839999999999998}]}]},{"osd":3,"up_from":237,"seq":1017907249308,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2352992,"kb_used_data":1806808,"kb_used_omap":3,"kb_used_meta":546172,"kb_avail":91392160,"statfs":{"total":95995035648,"available":93585571840,"internally_reserved":0,"allocated":1850171392,"data_stored":1847731902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":559280872},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:39 2023","interfaces":[{"interface":"back","average":{"1min":9.1750000000000007,"5min":2.8580000000000001,"15min":1.5529999999999999},"min":{"1min":0.19800000000000001,"5min":0.183,"15min":0.183},"max":{"1min":195.69800000000001,"5min":195.69800000000001,"15min":195.69800000000001},"last":0.73199999999999998},{"interface":"front","average":{"1min":9.1280000000000001,"5min":3.1869999999999998,"15min":1.8859999999999999},"min":{"1min":0.23899999999999999,"5min":0.222,"15min":0.20699999999999999},"max":{"1min":194.661,"5min":194.661,"15min":194.661},"last":0.23499999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:14:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.9470000000000001,"15min":1.5860000000000001},"min":{"1min":0.23799999999999999,"5min":0.215,"15min":0.215},"max":{"1min":7.016,"5min":16.056000000000001,"15min":21.931999999999999},"last":0.36899999999999999},{"interface":"front","average":{"1min":1.4650000000000001,"5min":2.2400000000000002,"15min":1.476},"min":{"1min":0.27300000000000002,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":8.5700000000000003,"5min":16.587,"15min":16.587},"last":2.157}]},{"osd":2,"last update":"Sat Sep 23 18:13:37 2023","interfaces":[{"interface":"back","average":{"1min":2.6419999999999999,"5min":1.8,"15min":1.214},"min":{"1min":0.372,"5min":0.26600000000000001,"15min":0.22600000000000001},"max":{"1min":33.976999999999997,"5min":33.976999999999997,"15min":33.976999999999997},"last":0.60999999999999999},{"interface":"front","average":{"1min":2.6779999999999999,"5min":1.9750000000000001,"15min":1.5800000000000001},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.25800000000000001},"max":{"1min":34.006999999999998,"5min":34.006999999999998,"15min":34.006999999999998},"last":0.78800000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:14:39 2023","interfaces":[{"interface":"back","average":{"1min":1.3009999999999999,"5min":1.4790000000000001,"15min":1.46},"min":{"1min":0.33200000000000002,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":6.234,"5min":10.763,"15min":37.231000000000002},"last":0.318},{"interface":"front","average":{"1min":1.514,"5min":1.8799999999999999,"15min":1.5649999999999999},"min":{"1min":0.38800000000000001,"5min":0.32100000000000001,"15min":0.30499999999999999},"max":{"1min":4.4139999999999997,"5min":15.890000000000001,"15min":37.381},"last":0.53800000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:14:32 2023","interfaces":[{"interface":"back","average":{"1min":1.222,"5min":1.258,"15min":1.264},"min":{"1min":0.39000000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":3.8519999999999999,"5min":4.2450000000000001,"15min":4.2450000000000001},"last":1.272},{"interface":"front","average":{"1min":1.1379999999999999,"5min":1.593,"15min":1.669},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.4020000000000001,"5min":8.9969999999999999,"15min":8.9969999999999999},"last":0.75600000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:14:39 2023","interfaces":[{"interface":"back","average":{"1min":1.2869999999999999,"5min":1.4319999999999999,"15min":1.609},"min":{"1min":0.40999999999999998,"5min":0.32700000000000001,"15min":0.32200000000000001},"max":{"1min":5.915,"5min":9.8149999999999995,"15min":115.54000000000001},"last":0.77100000000000002},{"interface":"front","average":{"1min":1.278,"5min":1.4630000000000001,"15min":1.829},"min":{"1min":0.41699999999999998,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.2000000000000002,"5min":7.6600000000000001,"15min":115.691},"last":0.40500000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:14:39 2023","interfaces":[{"interface":"back","average":{"1min":1.825,"5min":2.1309999999999998,"15min":1.6990000000000001},"min":{"1min":0.36799999999999999,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":5.2809999999999997,"5min":15.972,"15min":15.972},"last":0.58399999999999996},{"interface":"front","average":{"1min":1.413,"5min":1.7,"15min":1.5389999999999999},"min":{"1min":0.40699999999999997,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":5.5129999999999999,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":1.2110000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019257,"num_pgs":70,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2199504,"kb_used_data":1703496,"kb_used_omap":0,"kb_used_meta":496000,"kb_avail":91545648,"statfs":{"total":95995035648,"available":93742743552,"internally_reserved":0,"allocated":1744379904,"data_stored":1742765957,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":507904000},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:14:54 2023","interfaces":[{"interface":"back","average":{"1min":1.2649999999999999,"5min":1.165,"15min":1.131},"min":{"1min":0.22,"5min":0.22,"15min":0.215},"max":{"1min":5.3460000000000001,"5min":9.2759999999999998,"15min":14.164},"last":7.2199999999999998},{"interface":"front","average":{"1min":2.6400000000000001,"5min":1.734,"15min":1.4430000000000001},"min":{"1min":0.34200000000000003,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":8.3800000000000008,"5min":8.3800000000000008,"15min":9.3789999999999996},"last":0.47299999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:14:20 2023","interfaces":[{"interface":"back","average":{"1min":0.85299999999999998,"5min":2.1459999999999999,"15min":1.601},"min":{"1min":0.27300000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":2.6320000000000001,"5min":42.929000000000002,"15min":42.929000000000002},"last":9.1880000000000006},{"interface":"front","average":{"1min":0.76300000000000001,"5min":1.7969999999999999,"15min":1.4530000000000001},"min":{"1min":0.29099999999999998,"5min":0.255,"15min":0.23100000000000001},"max":{"1min":2.6819999999999999,"5min":43.494999999999997,"15min":43.494999999999997},"last":9.7409999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:12:55 2023","interfaces":[{"interface":"back","average":{"1min":1.278,"5min":2.1150000000000002,"15min":1.6200000000000001},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.26600000000000001},"max":{"1min":8.5530000000000008,"5min":32.887999999999998,"15min":32.887999999999998},"last":7.3179999999999996},{"interface":"front","average":{"1min":1.0449999999999999,"5min":1.7849999999999999,"15min":1.536},"min":{"1min":0.34200000000000003,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":2.9199999999999999,"5min":29.103999999999999,"15min":29.103999999999999},"last":2.9100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:14:35 2023","interfaces":[{"interface":"back","average":{"1min":1.274,"5min":1.954,"15min":1.825},"min":{"1min":0.36499999999999999,"5min":0.318,"15min":0.30199999999999999},"max":{"1min":7.6619999999999999,"5min":11.960000000000001,"15min":29.154},"last":3.0219999999999998},{"interface":"front","average":{"1min":1.5009999999999999,"5min":1.877,"15min":1.768},"min":{"1min":0.371,"5min":0.309,"15min":0.309},"max":{"1min":5.8579999999999997,"5min":9.343,"15min":29.216000000000001},"last":0.98399999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:14:35 2023","interfaces":[{"interface":"back","average":{"1min":2.1589999999999998,"5min":2.968,"15min":3.1030000000000002},"min":{"1min":0.34999999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9489999999999998,"5min":25.704999999999998,"15min":25.704999999999998},"last":0.502},{"interface":"front","average":{"1min":1.417,"5min":2.9609999999999999,"15min":3.218},"min":{"1min":0.36899999999999999,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":4.9690000000000003,"5min":32.640000000000001,"15min":32.640000000000001},"last":0.377}]},{"osd":6,"last update":"Sat Sep 23 18:14:59 2023","interfaces":[{"interface":"back","average":{"1min":2.3919999999999999,"5min":1.389,"15min":1.1240000000000001},"min":{"1min":0.28199999999999997,"5min":0.25,"15min":0.25},"max":{"1min":7.2210000000000001,"5min":9.7129999999999992,"15min":9.7129999999999992},"last":1.208},{"interface":"front","average":{"1min":2.2639999999999998,"5min":1.7549999999999999,"15min":1.2769999999999999},"min":{"1min":0.40100000000000002,"5min":0.27500000000000002,"15min":0.27500000000000002},"max":{"1min":7.2519999999999998,"5min":10.413,"15min":10.725},"last":0.33500000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:14:54 2023","interfaces":[{"interface":"back","average":{"1min":3.4140000000000001,"5min":1.6779999999999999,"15min":1.3109999999999999},"min":{"1min":0.52400000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.4629999999999992,"5min":9.0500000000000007,"15min":13.298999999999999},"last":0.66900000000000004},{"interface":"front","average":{"1min":3.044,"5min":1.748,"15min":1.381},"min":{"1min":0.438,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":11.779,"5min":11.779,"15min":11.779},"last":0.69799999999999995}]}]},{"osd":5,"up_from":0,"seq":1803886264354,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7491,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":34,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5358,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":32768,"data_stored":5504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1593835520,"data_stored":1593835520,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4786552832,"data_stored":4786552832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":65536,"data_stored":65536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1859387392,"data_stored":1859387392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1741881344,"data_stored":1741881344,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3729522688,"data_stored":3729522688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1689124864,"data_stored":1689124864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":9,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4685561856,"data_stored":4685561856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:15:21.986 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set unique_pool_4 pgp_num 16 2023-09-23T18:15:21.986 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:22.061 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:22.174 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:22.178 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:22.179 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:22.280 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:22.351 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:22.428 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:22.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:22.433 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:22.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:22.596 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:22 smithi099 ceph-mon[167076]: from='client.46481 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:15:22.597 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:22 smithi099 ceph-mon[167076]: pgmap v1195: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 906 op/s; 7701/311796 objects degraded (2.470%); 54748/311796 objects misplaced (17.559%); 16 MiB/s, 253 objects/s recovering 2023-09-23T18:15:22.597 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:22.597 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:22.605 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:22.618 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:15:22.619 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:15:22.619 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:15:22.619 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:15:22.619 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:15:22.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:22 smithi055 ceph-mon[179668]: from='client.46481 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:15:22.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:22 smithi055 ceph-mon[179668]: pgmap v1195: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 906 op/s; 7701/311796 objects degraded (2.470%); 54748/311796 objects misplaced (17.559%); 16 MiB/s, 253 objects/s recovering 2023-09-23T18:15:22.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:22.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:22.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:22 smithi055 ceph-mon[184174]: from='client.46481 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:15:22.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:22 smithi055 ceph-mon[184174]: pgmap v1195: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 54 MiB/s wr, 906 op/s; 7701/311796 objects degraded (2.470%); 54748/311796 objects misplaced (17.559%); 16 MiB/s, 253 objects/s recovering 2023-09-23T18:15:22.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:22.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:22.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:22.717 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:22.717 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:22.818 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:22.901 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:22.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:22.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:22.982 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:23.083 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:23.166 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:23.257 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:23.266 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:15:23.327 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:23.334 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:23.334 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:23.435 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:23.594 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:23.730 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7701/311796 objects degraded (2.470%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:23.730 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7701/311796 objects degraded (2.470%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:23.737 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:15:23.787 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:15:23.823 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:15:23.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7701/311796 objects degraded (2.470%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:23.830 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:23.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:23.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:23.859 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:15:23.898 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:15:23.940 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:23.948 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:15:23.991 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:15:24.035 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:15:24.050 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:15:24.050 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:15:24.050 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:15:24.050 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:15:24.050 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:15:24.050 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:15:24.050 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:15:24.050 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:15:24.063 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:24.521 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:15:24.825 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:24.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:24 smithi099 ceph-mon[167076]: pgmap v1196: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 909 op/s; 7148/307908 objects degraded (2.321%); 54142/307908 objects misplaced (17.584%); 16 MiB/s, 256 objects/s recovering 2023-09-23T18:15:24.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3309860828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:24.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1806830791' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:15:24.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:24.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:24 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:24.831 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:24.832 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:24.832 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:24 smithi055 ceph-mon[179668]: pgmap v1196: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 909 op/s; 7148/307908 objects degraded (2.321%); 54142/307908 objects misplaced (17.584%); 16 MiB/s, 256 objects/s recovering 2023-09-23T18:15:24.832 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3309860828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:24.832 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1806830791' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:15:24.832 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:24.833 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:24 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:24.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:24 smithi055 ceph-mon[184174]: pgmap v1196: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.3 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 45 MiB/s wr, 909 op/s; 7148/307908 objects degraded (2.321%); 54142/307908 objects misplaced (17.584%); 16 MiB/s, 256 objects/s recovering 2023-09-23T18:15:24.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3309860828' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:24.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1806830791' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:15:24.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:24.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:24 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:24.933 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:24.997 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:25.049 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:25.062 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:25.063 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:25.164 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:25.234 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:25.478 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:25.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:25.487 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:25.588 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:25.650 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:25.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:25 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:25.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:25 smithi099 ceph-mon[167076]: failed to encode map e495 with expected crc 2023-09-23T18:15:25.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:25 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1806830791' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:15:25.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:25 smithi099 ceph-mon[167076]: osdmap e495: 8 total, 8 up, 7 in 2023-09-23T18:15:25.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:25 smithi099 ceph-mon[167076]: failed to encode map e495 with expected crc 2023-09-23T18:15:25.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:25 smithi099 ceph-mon[167076]: failed to encode map e495 with expected crc 2023-09-23T18:15:25.891 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:25.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:25.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:25.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[179668]: failed to encode map e495 with expected crc 2023-09-23T18:15:25.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1806830791' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:15:25.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[179668]: osdmap e495: 8 total, 8 up, 7 in 2023-09-23T18:15:25.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[179668]: failed to encode map e495 with expected crc 2023-09-23T18:15:25.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[179668]: failed to encode map e495 with expected crc 2023-09-23T18:15:25.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:25.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[184174]: failed to encode map e495 with expected crc 2023-09-23T18:15:25.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1806830791' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:15:25.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[184174]: osdmap e495: 8 total, 8 up, 7 in 2023-09-23T18:15:25.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[184174]: failed to encode map e495 with expected crc 2023-09-23T18:15:25.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:25 smithi055 ceph-mon[184174]: failed to encode map e495 with expected crc 2023-09-23T18:15:25.902 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:26.003 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:26.058 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:26.114 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:26.125 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:26.128 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:26.129 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:26.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:26.308 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:26.387 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:26.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:26.393 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:26.493 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:26 smithi099 ceph-mon[167076]: failed to encode map e495 with expected crc 2023-09-23T18:15:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:26 smithi099 ceph-mon[167076]: pgmap v1198: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 107 B/s rd, 35 MiB/s wr, 957 op/s; 7148/302871 objects degraded (2.360%); 52592/302871 objects misplaced (17.364%); 17 MiB/s, 270 objects/s recovering 2023-09-23T18:15:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2107787651' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:15:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:26 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:15:26.589 INFO:teuthology.orchestra.run.smithi055.stderr:set pool 9 pgp_num to 16 2023-09-23T18:15:26.590 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:26 smithi055 ceph-mon[184174]: failed to encode map e495 with expected crc 2023-09-23T18:15:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:26 smithi055 ceph-mon[184174]: pgmap v1198: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 107 B/s rd, 35 MiB/s wr, 957 op/s; 7148/302871 objects degraded (2.360%); 52592/302871 objects misplaced (17.364%); 17 MiB/s, 270 objects/s recovering 2023-09-23T18:15:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2107787651' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:15:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:26 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:15:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:26 smithi055 ceph-mon[179668]: failed to encode map e495 with expected crc 2023-09-23T18:15:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:26 smithi055 ceph-mon[179668]: pgmap v1198: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.2 GiB data, 19 GiB used, 607 GiB / 626 GiB avail; 107 B/s rd, 35 MiB/s wr, 957 op/s; 7148/302871 objects degraded (2.360%); 52592/302871 objects misplaced (17.364%); 17 MiB/s, 270 objects/s recovering 2023-09-23T18:15:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2107787651' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:15:26.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:26 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]: dispatch 2023-09-23T18:15:27.009 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:27.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:27.013 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:27.114 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:27.181 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:27.295 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:27.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:27.308 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:27.409 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:27.458 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:15:27.487 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:27.612 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:27.620 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:27.620 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:27.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[179668]: failed to encode map e496 with expected crc 2023-09-23T18:15:27.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:15:27.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[179668]: osdmap e496: 8 total, 8 up, 7 in 2023-09-23T18:15:27.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[179668]: failed to encode map e496 with expected crc 2023-09-23T18:15:27.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:27.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:27.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[184174]: failed to encode map e496 with expected crc 2023-09-23T18:15:27.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:15:27.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[184174]: osdmap e496: 8 total, 8 up, 7 in 2023-09-23T18:15:27.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[184174]: failed to encode map e496 with expected crc 2023-09-23T18:15:27.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:27.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:27.722 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:27.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:27 smithi099 ceph-mon[167076]: failed to encode map e496 with expected crc 2023-09-23T18:15:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:27 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_4", "var": "pgp_num", "val": "16"}]': finished 2023-09-23T18:15:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:27 smithi099 ceph-mon[167076]: osdmap e496: 8 total, 8 up, 7 in 2023-09-23T18:15:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:27 smithi099 ceph-mon[167076]: failed to encode map e496 with expected crc 2023-09-23T18:15:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:27.847 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:27.926 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:27.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:27.939 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:28.000 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:15:28.040 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:28.128 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:15:28.128 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:15:28.176 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:28.558 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:28.564 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:28.564 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:28.613 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:28.618 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:15:28.618 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:28 smithi055 ceph-mon[179668]: pgmap v1200: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 6.6 MiB/s wr, 676 op/s; 7148/300348 objects degraded (2.380%); 51824/300348 objects misplaced (17.255%); 14 MiB/s, 225 objects/s recovering 2023-09-23T18:15:28.618 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:28 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:28.618 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:28 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:28.618 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:28 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:28.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7148/300348 objects degraded (2.380%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:28.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:28 smithi055 ceph-mon[184174]: pgmap v1200: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 6.6 MiB/s wr, 676 op/s; 7148/300348 objects degraded (2.380%); 51824/300348 objects misplaced (17.255%); 14 MiB/s, 225 objects/s recovering 2023-09-23T18:15:28.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:28 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:28.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:28 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:28.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:28 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:28.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7148/300348 objects degraded (2.380%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:28.665 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:28.711 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:28.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:28.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:28.807 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:28.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:28 smithi099 ceph-mon[167076]: pgmap v1200: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 6.1 GiB data, 18 GiB used, 607 GiB / 626 GiB avail; 127 B/s rd, 6.6 MiB/s wr, 676 op/s; 7148/300348 objects degraded (2.380%); 51824/300348 objects misplaced (17.255%); 14 MiB/s, 225 objects/s recovering 2023-09-23T18:15:28.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:28 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:15:28.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:28 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:15:28.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:28 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:28.810 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7148/300348 objects degraded (2.380%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:28.909 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:28.969 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:29.089 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:29.178 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:29.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:29.187 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:29.288 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:29.356 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:29.365 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:15:29.438 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:29.450 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:29.451 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:29.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:29.647 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:29.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:29.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:29.741 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:29.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:29.929 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:30.003 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:30.014 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:30.015 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:30.116 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:30.230 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:30.327 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:30.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:30.331 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:30.346 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:15:30.379 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:15:30.410 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:15:30.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:30.444 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:15:30.491 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:15:30.496 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:15:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:15:30] "GET /metrics HTTP/1.1" 200 36657 "" "Prometheus/2.43.0" 2023-09-23T18:15:30.517 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:30.532 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:15:30.573 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:15:30.610 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:15:30.616 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:15:30.616 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:15:30.616 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:15:30.617 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:15:30.617 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:15:30.617 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:15:30.617 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:15:30.617 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:15:30.659 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:30.670 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:30.671 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:30 smithi099 ceph-mon[167076]: pgmap v1201: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 2.8 MiB/s wr, 1.02k op/s; 6469/289458 objects degraded (2.235%); 49216/289458 objects misplaced (17.003%); 17 MiB/s, 265 objects/s recovering 2023-09-23T18:15:30.671 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3675397740' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:30.672 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:30.772 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:30.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:30 smithi055 ceph-mon[184174]: pgmap v1201: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 2.8 MiB/s wr, 1.02k op/s; 6469/289458 objects degraded (2.235%); 49216/289458 objects misplaced (17.003%); 17 MiB/s, 265 objects/s recovering 2023-09-23T18:15:30.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3675397740' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:30.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:30 smithi055 ceph-mon[179668]: pgmap v1201: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.9 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 2.8 MiB/s wr, 1.02k op/s; 6469/289458 objects degraded (2.235%); 49216/289458 objects misplaced (17.003%); 17 MiB/s, 265 objects/s recovering 2023-09-23T18:15:30.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3675397740' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:30.840 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:30.894 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:30.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:30.899 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:31.000 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:31.072 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:31.177 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:15:31.178 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:15:31.178 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.178 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.36552.0:116204 9.7 e496/489 9:e0a1bd8c:::benchmark_data_smithi055_386467_object11544:head v 496'7196, mlcod=496'7194)", 2023-09-23T18:15:31.178 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:31.164924+0000", 2023-09-23T18:15:31.178 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0034015690000000001, 2023-09-23T18:15:31.178 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034316860000000002, 2023-09-23T18:15:31.178 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:31.178 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:15:31.178 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.164924+0000", 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.164924+0000", 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.179 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.164923+0000", 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.164930+0000", 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1099999999999997e-06 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:31.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.164946+0000", 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5358000000000001e-05 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.164952+0000", 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5289999999999997e-06 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.167849+0000", 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002897204 2023-09-23T18:15:31.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.167873+0000", 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3465e-05 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.36552.0:116205 9.6 e496/489 9:651d6c09:::benchmark_data_smithi055_386467_object11545:head v 496'7242, mlcod=496'7240)", 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:31.166298+0000", 2023-09-23T18:15:31.182 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.002027867, 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021236750000000002, 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166298+0000", 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:31.184 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166298+0000", 2023-09-23T18:15:31.185 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:31.185 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166297+0000", 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166305+0000", 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6939999999999995e-06 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166319+0000", 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3886000000000001e-05 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166325+0000", 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0329999999999997e-06 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166406+0000", 2023-09-23T18:15:31.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1104000000000001e-05 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166427+0000", 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1396999999999999e-05 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.36552.0:116209 9.a e496/489 9:53e13928:::benchmark_data_smithi055_386467_object11549:head v 496'7223, mlcod=496'7220)", 2023-09-23T18:15:31.188 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:31.166339+0000", 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.00198701, 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021431639999999999, 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166339+0000", 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166339+0000", 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166339+0000", 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967296 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166343+0000", 2023-09-23T18:15:31.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2560000000000004e-06 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166350+0000", 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2599999999999999e-06 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166353+0000", 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2969999999999999e-06 2023-09-23T18:15:31.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166404+0000", 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0256999999999997e-05 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:31.166426+0000", 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2154000000000002e-05 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:31.192 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:31.193 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:31.193 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:15:31.193 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 3 2023-09-23T18:15:31.193 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:15:31.404 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:31.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:31.409 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:31.509 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:15:31.577 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:31.586 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:15:31.596 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:15:31.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/941970849' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:31.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:31 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:31.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/941970849' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:31.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:31 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:31.743 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:15:31.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:31.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:15:31.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/941970849' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:31.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:31 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:31.851 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:31.917 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:32.439 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:15:32.707 INFO:tasks.thrashosds.thrasher:in_osds: [7, 1, 0, 4, 2, 3, 6] out_osds: [5] dead_osds: [] live_osds: [0, 1, 4, 6, 7, 3, 5, 2] 2023-09-23T18:15:32.707 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:15:32.707 INFO:tasks.thrashosds.thrasher:Killing osd 1, live_osds are [0, 1, 4, 6, 7, 3, 5, 2] 2023-09-23T18:15:32.707 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T18:15:32.708 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T18:15:32.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:32 smithi099 ceph-mon[167076]: pgmap v1202: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 1.0 MiB/s wr, 915 op/s; 6469/286725 objects degraded (2.256%); 48406/286725 objects misplaced (16.882%); 18 MiB/s, 288 objects/s recovering 2023-09-23T18:15:32.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:32 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:15:32.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:32 smithi099 ceph-mon[167076]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:32 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:32.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:32 smithi099 ceph-mon[167076]: osdmap e497: 8 total, 8 up, 7 in 2023-09-23T18:15:32.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:32 smithi099 ceph-mon[167076]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:32 smithi099 ceph-mon[167076]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:32 smithi099 ceph-mon[167076]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.819 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:32 smithi099 ceph-mon[167076]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[184174]: pgmap v1202: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 1.0 MiB/s wr, 915 op/s; 6469/286725 objects degraded (2.256%); 48406/286725 objects misplaced (16.882%); 18 MiB/s, 288 objects/s recovering 2023-09-23T18:15:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:15:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[184174]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[184174]: osdmap e497: 8 total, 8 up, 7 in 2023-09-23T18:15:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[184174]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[184174]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[184174]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[184174]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.931 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:15:32 smithi055 systemd[1]: Stopping Ceph osd.1 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:15:32.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[179668]: pgmap v1202: 119 pgs: 1 active+undersized+degraded+remapped+backfill_wait, 7 active+remapped+backfill_wait, 2 active+remapped+backfilling, 1 active+recovering+degraded+remapped, 108 active+clean; 5.8 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 127 B/s rd, 1.0 MiB/s wr, 915 op/s; 6469/286725 objects degraded (2.256%); 48406/286725 objects misplaced (16.882%); 18 MiB/s, 288 objects/s recovering 2023-09-23T18:15:32.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:15:32.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[179668]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:32.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[179668]: osdmap e497: 8 total, 8 up, 7 in 2023-09-23T18:15:32.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[179668]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[179668]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[179668]: failed to encode map e497 with expected crc 2023-09-23T18:15:32.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:32 smithi055 ceph-mon[179668]: failed to encode map e497 with expected crc 2023-09-23T18:15:33.690 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 6469/286725 objects degraded (2.256%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:33.874 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 6469/286725 objects degraded (2.256%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:33.874 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 6469/286725 objects degraded (2.256%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:34.058 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:15:34.059 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:15:34.059 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:15:34.059 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:15:34.059 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:15:34.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:34 smithi055 ceph-mon[184174]: pgmap v1204: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+remapped+backfill_wait, 2 active+remapped+backfilling, 108 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 900 op/s; 5429/281256 objects degraded (1.930%); 53323/281256 objects misplaced (18.959%); 20 MiB/s, 327 objects/s recovering 2023-09-23T18:15:34.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:34 smithi055 ceph-mon[179668]: pgmap v1204: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+remapped+backfill_wait, 2 active+remapped+backfilling, 108 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 900 op/s; 5429/281256 objects degraded (1.930%); 53323/281256 objects misplaced (18.959%); 20 MiB/s, 327 objects/s recovering 2023-09-23T18:15:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:34 smithi099 ceph-mon[167076]: pgmap v1204: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+remapped+backfill_wait, 2 active+remapped+backfilling, 108 active+clean; 5.7 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 900 op/s; 5429/281256 objects degraded (1.930%); 53323/281256 objects misplaced (18.959%); 20 MiB/s, 327 objects/s recovering 2023-09-23T18:15:35.781 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:35.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:35.790 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:35.889 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:35 smithi055 ceph-mon[184174]: pgmap v1205: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+remapped+backfill_wait, 2 active+remapped+backfilling, 108 active+clean; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 995 op/s; 5429/274950 objects degraded (1.975%); 51410/274950 objects misplaced (18.698%); 19 MiB/s, 311 objects/s recovering 2023-09-23T18:15:35.891 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:35 smithi055 ceph-mon[179668]: pgmap v1205: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+remapped+backfill_wait, 2 active+remapped+backfilling, 108 active+clean; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 995 op/s; 5429/274950 objects degraded (1.975%); 51410/274950 objects misplaced (18.698%); 19 MiB/s, 311 objects/s recovering 2023-09-23T18:15:35.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:35.930 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:15:36.004 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:35 smithi099 ceph-mon[167076]: pgmap v1205: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 8 active+remapped+backfill_wait, 2 active+remapped+backfilling, 108 active+clean; 5.6 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 995 op/s; 5429/274950 objects degraded (1.975%); 51410/274950 objects misplaced (18.698%); 19 MiB/s, 311 objects/s recovering 2023-09-23T18:15:36.008 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:36.312 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:36.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:36.328 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:36.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:15:36.430 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:36.509 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:36.603 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:36.616 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:36.617 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:36.642 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:15:36 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[201753]: 2023-09-23T18:15:36.273+0000 7fc99e91b700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:15:36.642 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:15:36 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[201753]: 2023-09-23T18:15:36.273+0000 7fc99e91b700 -1 osd.1 497 *** Got signal Terminated *** 2023-09-23T18:15:36.642 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:15:36 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[201753]: 2023-09-23T18:15:36.273+0000 7fc99e91b700 -1 osd.1 497 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:15:36.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:36 smithi055 ceph-mon[179668]: osd.1 marked itself down and dead 2023-09-23T18:15:36.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:36.643 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:36.718 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:36.747 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:36.765 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:36.780 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:36.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:36.783 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:36.884 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:36.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:36 smithi099 ceph-mon[167076]: osd.1 marked itself down and dead 2023-09-23T18:15:36.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:36.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:36.912 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:36 smithi055 ceph-mon[184174]: osd.1 marked itself down and dead 2023-09-23T18:15:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:36.947 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:36.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:36.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:37.051 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:37.082 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:37.134 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:15:37.861 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:37.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:37.865 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:37.891 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:15:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[184174]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:15:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[184174]: osdmap e498: 8 total, 7 up, 7 in 2023-09-23T18:15:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[184174]: failed to encode map e498 with expected crc 2023-09-23T18:15:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[184174]: failed to encode map e498 with expected crc 2023-09-23T18:15:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[184174]: failed to encode map e498 with expected crc 2023-09-23T18:15:37.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[184174]: failed to encode map e498 with expected crc 2023-09-23T18:15:37.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[184174]: failed to encode map e498 with expected crc 2023-09-23T18:15:37.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[184174]: pgmap v1207: 119 pgs: 1 stale+active+undersized+degraded+remapped+backfilling, 4 stale+active+remapped+backfill_wait, 31 stale+active+clean, 4 active+remapped+backfill_wait, 2 active+remapped+backfilling, 77 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 5429/272247 objects degraded (1.994%); 50593/272247 objects misplaced (18.583%); 19 MiB/s, 297 objects/s recovering 2023-09-23T18:15:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[179668]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:15:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[179668]: osdmap e498: 8 total, 7 up, 7 in 2023-09-23T18:15:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[179668]: failed to encode map e498 with expected crc 2023-09-23T18:15:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[179668]: failed to encode map e498 with expected crc 2023-09-23T18:15:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[179668]: failed to encode map e498 with expected crc 2023-09-23T18:15:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[179668]: failed to encode map e498 with expected crc 2023-09-23T18:15:37.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[179668]: failed to encode map e498 with expected crc 2023-09-23T18:15:37.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:37 smithi055 ceph-mon[179668]: pgmap v1207: 119 pgs: 1 stale+active+undersized+degraded+remapped+backfilling, 4 stale+active+remapped+backfill_wait, 31 stale+active+clean, 4 active+remapped+backfill_wait, 2 active+remapped+backfilling, 77 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 5429/272247 objects degraded (1.994%); 50593/272247 objects misplaced (18.583%); 19 MiB/s, 297 objects/s recovering 2023-09-23T18:15:37.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:38.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:37 smithi099 ceph-mon[167076]: Health check failed: 1 osds down (OSD_DOWN) 2023-09-23T18:15:38.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:37 smithi099 ceph-mon[167076]: osdmap e498: 8 total, 7 up, 7 in 2023-09-23T18:15:38.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:37 smithi099 ceph-mon[167076]: failed to encode map e498 with expected crc 2023-09-23T18:15:38.005 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:37 smithi099 ceph-mon[167076]: failed to encode map e498 with expected crc 2023-09-23T18:15:38.006 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:37 smithi099 ceph-mon[167076]: failed to encode map e498 with expected crc 2023-09-23T18:15:38.006 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:37 smithi099 ceph-mon[167076]: failed to encode map e498 with expected crc 2023-09-23T18:15:38.006 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:37 smithi099 ceph-mon[167076]: failed to encode map e498 with expected crc 2023-09-23T18:15:38.006 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:37 smithi099 ceph-mon[167076]: pgmap v1207: 119 pgs: 1 stale+active+undersized+degraded+remapped+backfilling, 4 stale+active+remapped+backfill_wait, 31 stale+active+clean, 4 active+remapped+backfill_wait, 2 active+remapped+backfilling, 77 active+clean; 5.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 716 op/s; 5429/272247 objects degraded (1.994%); 50593/272247 objects misplaced (18.583%); 19 MiB/s, 297 objects/s recovering 2023-09-23T18:15:38.008 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:38.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:38.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:38.091 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:38.192 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:38.222 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:38.664 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:38.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:38.668 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:38.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:38.801 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:38.930 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:15:38 smithi055 bash[401669]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T18:15:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: osdmap e499: 8 total, 7 up, 7 in 2023-09-23T18:15:38.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: 9.7 continuing backfill to osd.6 from (467'3227,497'7557] MIN to 497'7557 2023-09-23T18:15:38.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: 9.a continuing backfill to osd.6 from (472'3668,497'7579] MIN to 497'7579 2023-09-23T18:15:38.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: 9.2 continuing backfill to osd.2 from (472'3714,497'7660] 9:43113f20:::benchmark_data_smithi055_386467_object94548:head to 497'7660 2023-09-23T18:15:38.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: 9.d continuing backfill to osd.4 from (467'3133,497'7515] MIN to 497'7515 2023-09-23T18:15:38.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: 9.0 continuing backfill to osd.2 from (473'3925,497'7821] MIN to 497'7821 2023-09-23T18:15:38.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: 9.b continuing backfill to osd.3 from (466'2765,497'7769] 9:da9efc73:::benchmark_data_smithi055_386467_object73335:head to 497'7769 2023-09-23T18:15:38.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: 9.6 continuing backfill to osd.0 from (472'3672,497'7586] MIN to 497'7586 2023-09-23T18:15:38.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: 9.f continuing backfill to osd.7 from (472'3567,497'7415] MIN to 497'7415 2023-09-23T18:15:38.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 5429/272247 objects degraded (1.994%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:15:38.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: osdmap e499: 8 total, 7 up, 7 in 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: 9.7 continuing backfill to osd.6 from (467'3227,497'7557] MIN to 497'7557 2023-09-23T18:15:38.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: 9.a continuing backfill to osd.6 from (472'3668,497'7579] MIN to 497'7579 2023-09-23T18:15:38.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: 9.2 continuing backfill to osd.2 from (472'3714,497'7660] 9:43113f20:::benchmark_data_smithi055_386467_object94548:head to 497'7660 2023-09-23T18:15:38.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: 9.d continuing backfill to osd.4 from (467'3133,497'7515] MIN to 497'7515 2023-09-23T18:15:38.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: 9.0 continuing backfill to osd.2 from (473'3925,497'7821] MIN to 497'7821 2023-09-23T18:15:38.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: 9.b continuing backfill to osd.3 from (466'2765,497'7769] 9:da9efc73:::benchmark_data_smithi055_386467_object73335:head to 497'7769 2023-09-23T18:15:38.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: 9.6 continuing backfill to osd.0 from (472'3672,497'7586] MIN to 497'7586 2023-09-23T18:15:38.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: 9.f continuing backfill to osd.7 from (472'3567,497'7415] MIN to 497'7415 2023-09-23T18:15:38.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 5429/272247 objects degraded (1.994%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:15:38.967 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.967 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.967 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.967 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: osdmap e499: 8 total, 7 up, 7 in 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: failed to encode map e499 with expected crc 2023-09-23T18:15:38.968 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: 9.7 continuing backfill to osd.6 from (467'3227,497'7557] MIN to 497'7557 2023-09-23T18:15:38.969 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: 9.a continuing backfill to osd.6 from (472'3668,497'7579] MIN to 497'7579 2023-09-23T18:15:38.969 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: 9.2 continuing backfill to osd.2 from (472'3714,497'7660] 9:43113f20:::benchmark_data_smithi055_386467_object94548:head to 497'7660 2023-09-23T18:15:38.969 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: 9.d continuing backfill to osd.4 from (467'3133,497'7515] MIN to 497'7515 2023-09-23T18:15:38.969 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: 9.0 continuing backfill to osd.2 from (473'3925,497'7821] MIN to 497'7821 2023-09-23T18:15:38.969 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: 9.b continuing backfill to osd.3 from (466'2765,497'7769] 9:da9efc73:::benchmark_data_smithi055_386467_object73335:head to 497'7769 2023-09-23T18:15:38.969 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: 9.6 continuing backfill to osd.0 from (472'3672,497'7586] MIN to 497'7586 2023-09-23T18:15:38.969 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: 9.f continuing backfill to osd.7 from (472'3567,497'7415] MIN to 497'7415 2023-09-23T18:15:38.969 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 5429/272247 objects degraded (1.994%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:15:38.985 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:38.988 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:38.988 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:39.089 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:39.124 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:39.219 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:39.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:39.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:39.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:39.357 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:39.701 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:39.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:39 smithi055 ceph-mon[184174]: pgmap v1209: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 18 active+undersized, 2 peering, 3 stale+active+remapped+backfill_wait, 21 stale+active+clean, 5 active+undersized+degraded, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 61 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 924 op/s; 37064/266001 objects degraded (13.934%); 51082/266001 objects misplaced (19.204%); 30 MiB/s, 486 objects/s recovering 2023-09-23T18:15:39.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:39 smithi055 ceph-mon[179668]: pgmap v1209: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 18 active+undersized, 2 peering, 3 stale+active+remapped+backfill_wait, 21 stale+active+clean, 5 active+undersized+degraded, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 61 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 924 op/s; 37064/266001 objects degraded (13.934%); 51082/266001 objects misplaced (19.204%); 30 MiB/s, 486 objects/s recovering 2023-09-23T18:15:40.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:39 smithi099 ceph-mon[167076]: pgmap v1209: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 1 undersized+degraded+remapped+backfilling+peered, 3 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 18 active+undersized, 2 peering, 3 stale+active+remapped+backfill_wait, 21 stale+active+clean, 5 active+undersized+degraded, 1 active+remapped+backfill_wait, 1 active+remapped+backfilling, 61 active+clean; 5.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 924 op/s; 37064/266001 objects degraded (13.934%); 51082/266001 objects misplaced (19.204%); 30 MiB/s, 486 objects/s recovering 2023-09-23T18:15:40.138 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:15:40.161 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:15:40.190 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:15:40.218 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:15:40.241 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:15:40.264 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:15:40.290 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:15:40.290 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:15:40.290 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:15:40.290 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:15:40.290 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:15:40.291 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:15:40.291 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:15:40.291 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:15:40.291 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:15:40.547 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:15:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:15:40] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.43.0" 2023-09-23T18:15:40.888 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:40 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:15:40.888 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2394760014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:40.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:40 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:15:40.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2394760014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:41.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:40 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:15:41.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2394760014' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:41.259 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:15:41.260 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:15:41.337 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T18:15:41.392 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:41.395 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:15:41.395 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T18:15:41.396 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:41.396 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:41.497 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:41.545 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:41 smithi099 ceph-mon[167076]: pgmap v1210: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 5 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 53 active+undersized, 2 peering, 12 active+undersized+degraded, 1 active+remapped+backfill_wait, 40 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.5 KiB/s rd, 9.1 KiB/s wr, 786 op/s; 57644/262434 objects degraded (21.965%); 53856/262434 objects misplaced (20.522%); 48 MiB/s, 761 objects/s recovering 2023-09-23T18:15:42.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:41 smithi055 ceph-mon[179668]: pgmap v1210: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 5 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 53 active+undersized, 2 peering, 12 active+undersized+degraded, 1 active+remapped+backfill_wait, 40 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.5 KiB/s rd, 9.1 KiB/s wr, 786 op/s; 57644/262434 objects degraded (21.965%); 53856/262434 objects misplaced (20.522%); 48 MiB/s, 761 objects/s recovering 2023-09-23T18:15:42.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:41 smithi055 ceph-mon[184174]: pgmap v1210: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 5 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 53 active+undersized, 2 peering, 12 active+undersized+degraded, 1 active+remapped+backfill_wait, 40 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 3.5 KiB/s rd, 9.1 KiB/s wr, 786 op/s; 57644/262434 objects degraded (21.965%); 53856/262434 objects misplaced (20.522%); 48 MiB/s, 761 objects/s recovering 2023-09-23T18:15:42.292 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:42.295 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:42.295 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:42.396 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:42.431 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:42.472 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:42.475 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:42.476 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:42.577 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:42.615 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:42.701 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:15:42.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[184174]: failed to encode map e500 with expected crc 2023-09-23T18:15:42.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[184174]: failed to encode map e500 with expected crc 2023-09-23T18:15:42.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[184174]: osdmap e500: 8 total, 7 up, 7 in 2023-09-23T18:15:42.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[184174]: failed to encode map e500 with expected crc 2023-09-23T18:15:42.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1661085442' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:42.990 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:42.990 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[179668]: failed to encode map e500 with expected crc 2023-09-23T18:15:42.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[179668]: failed to encode map e500 with expected crc 2023-09-23T18:15:42.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[179668]: osdmap e500: 8 total, 7 up, 7 in 2023-09-23T18:15:42.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[179668]: failed to encode map e500 with expected crc 2023-09-23T18:15:42.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1661085442' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:42.991 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:42 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:43.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:42 smithi099 ceph-mon[167076]: failed to encode map e500 with expected crc 2023-09-23T18:15:43.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:42 smithi099 ceph-mon[167076]: failed to encode map e500 with expected crc 2023-09-23T18:15:43.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:42 smithi099 ceph-mon[167076]: osdmap e500: 8 total, 7 up, 7 in 2023-09-23T18:15:43.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:42 smithi099 ceph-mon[167076]: failed to encode map e500 with expected crc 2023-09-23T18:15:43.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:42 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1661085442' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:43.032 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:42 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:15:43.038 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:43.042 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:43.042 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:43.143 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:43.174 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:43.213 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:43.216 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:43.216 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:43.261 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:15:43.317 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:43.357 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:43.407 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:43.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:43.416 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:43.465 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:15:43.465 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:15:43.517 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:43.547 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:43.585 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:43.587 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:43.588 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:43.689 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:43.719 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:43.756 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:43.759 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:43.760 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:43.856 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:15:43.861 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:43.882 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:43.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[184174]: failed to encode map e501 with expected crc 2023-09-23T18:15:43.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:15:43.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[184174]: osdmap e501: 8 total, 7 up, 7 in 2023-09-23T18:15:43.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[184174]: failed to encode map e501 with expected crc 2023-09-23T18:15:43.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[184174]: failed to encode map e501 with expected crc 2023-09-23T18:15:43.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[184174]: pgmap v1213: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 5 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 55 active+undersized, 12 active+undersized+degraded, 1 active+remapped+backfill_wait, 40 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 548 op/s; 57644/262434 objects degraded (21.965%); 53856/262434 objects misplaced (20.522%); 52 MiB/s, 829 objects/s recovering 2023-09-23T18:15:43.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 57644/262434 objects degraded (21.965%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:43.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:43.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[179668]: failed to encode map e501 with expected crc 2023-09-23T18:15:43.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:15:43.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[179668]: osdmap e501: 8 total, 7 up, 7 in 2023-09-23T18:15:43.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[179668]: failed to encode map e501 with expected crc 2023-09-23T18:15:43.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[179668]: failed to encode map e501 with expected crc 2023-09-23T18:15:43.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[179668]: pgmap v1213: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 5 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 55 active+undersized, 12 active+undersized+degraded, 1 active+remapped+backfill_wait, 40 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 548 op/s; 57644/262434 objects degraded (21.965%); 53856/262434 objects misplaced (20.522%); 52 MiB/s, 829 objects/s recovering 2023-09-23T18:15:43.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 57644/262434 objects degraded (21.965%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:43.899 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:43.914 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:44.023 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:44.026 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:44.027 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:44.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:43 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:44.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:43 smithi099 ceph-mon[167076]: failed to encode map e501 with expected crc 2023-09-23T18:15:44.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:43 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:15:44.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:43 smithi099 ceph-mon[167076]: osdmap e501: 8 total, 7 up, 7 in 2023-09-23T18:15:44.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:43 smithi099 ceph-mon[167076]: failed to encode map e501 with expected crc 2023-09-23T18:15:44.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:43 smithi099 ceph-mon[167076]: failed to encode map e501 with expected crc 2023-09-23T18:15:44.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:43 smithi099 ceph-mon[167076]: pgmap v1213: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 1 undersized+degraded+remapped+backfilling+peered, 5 active+undersized+degraded+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded, 55 active+undersized, 12 active+undersized+degraded, 1 active+remapped+backfill_wait, 40 active+clean; 5.3 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 4.7 KiB/s rd, 12 KiB/s wr, 548 op/s; 57644/262434 objects degraded (21.965%); 53856/262434 objects misplaced (20.522%); 52 MiB/s, 829 objects/s recovering 2023-09-23T18:15:44.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 57644/262434 objects degraded (21.965%), 21 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:44.128 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:44.161 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:44.199 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:44.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:44.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:44.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:44.339 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:44.382 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:44.385 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:44.385 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:44.486 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:44.517 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:44.558 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:44.561 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:44.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:44.662 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:44.699 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:44.712 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:44 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:15:44.741 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:44.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:44.745 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:44.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:44.878 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:44.894 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:44.920 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:44.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:44.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:44.984 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:44 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:15:44.989 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:44 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:15:45.026 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:15:45.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:45.057 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:45.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:45.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:45.101 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:45.202 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:45.233 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:15:45.233 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:45.256 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:15:45.274 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:45.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:45.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:45.278 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:15:45.304 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:15:45.326 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:15:45.349 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:15:45.371 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:15:45.371 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:15:45.371 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:15:45.372 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:15:45.372 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:15:45.372 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:15:45.372 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:15:45.372 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:15:45.372 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:15:45.378 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:45.409 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:45.451 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:45.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:45.455 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:45.556 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:45.588 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:45.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:45.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:45.633 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:45.731 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:45 smithi099 ceph-mon[167076]: pgmap v1214: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 1 undersized+degraded+remapped+backfilling+peered, 2 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 3.8 KiB/s rd, 9.9 KiB/s wr, 446 op/s; 55018/262434 objects degraded (20.965%); 53856/262434 objects misplaced (20.522%); 64 MiB/s, 1.03k objects/s recovering 2023-09-23T18:15:45.731 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2500940147' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:45.734 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:45.770 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:45 smithi055 ceph-mon[179668]: pgmap v1214: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 1 undersized+degraded+remapped+backfilling+peered, 2 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 3.8 KiB/s rd, 9.9 KiB/s wr, 446 op/s; 55018/262434 objects degraded (20.965%); 53856/262434 objects misplaced (20.522%); 64 MiB/s, 1.03k objects/s recovering 2023-09-23T18:15:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2500940147' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:45 smithi055 ceph-mon[184174]: pgmap v1214: 119 pgs: 1 active+recovery_wait+remapped, 1 active+clean+remapped, 1 undersized+degraded+remapped+backfilling+peered, 2 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded, 5 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 3.8 KiB/s rd, 9.9 KiB/s wr, 446 op/s; 55018/262434 objects degraded (20.965%); 53856/262434 objects misplaced (20.522%); 64 MiB/s, 1.03k objects/s recovering 2023-09-23T18:15:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2500940147' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:45.934 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:45.937 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:45.937 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:46.038 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:46.067 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:15:46.067 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:15:46.076 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:46.119 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:46.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:46.124 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:46.225 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:46.231 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:46.262 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:46.345 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:15:46.345 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:15:46.345 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:15:46.346 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:15:46.363 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:46.366 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:46.367 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:46.396 INFO:tasks.thrashosds.thrasher:in_osds: [7, 1, 0, 4, 2, 3, 6] out_osds: [5] dead_osds: [1] live_osds: [0, 4, 6, 7, 3, 5, 2] 2023-09-23T18:15:46.396 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:15:46.396 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T18:15:46.396 INFO:tasks.thrashosds.thrasher:Setting osd 1 primary_affinity to 0.000000 2023-09-23T18:15:46.397 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 0 2023-09-23T18:15:46.468 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:46.499 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:46.540 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:46.544 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:46.545 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:46.645 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:46.677 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:46.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:46.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:46.726 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:46.827 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:46.861 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:47.014 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:15:47.052 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:47.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:47.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:47.156 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:47.188 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:47.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:47.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:47.235 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:47.336 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:47.372 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:47.373 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:47.465 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:47.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:47.469 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:47.570 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:47.603 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:47.641 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:47.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:47.645 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:47.745 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:47.782 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:47.842 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:47.847 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:47.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:47.854 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[179668]: failed to encode map e502 with expected crc 2023-09-23T18:15:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[179668]: osdmap e502: 8 total, 7 up, 7 in 2023-09-23T18:15:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[179668]: failed to encode map e502 with expected crc 2023-09-23T18:15:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[179668]: failed to encode map e502 with expected crc 2023-09-23T18:15:47.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[179668]: failed to encode map e502 with expected crc 2023-09-23T18:15:47.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[179668]: pgmap v1216: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+remapped, 1 undersized+degraded+remapped+backfilling+peered, 2 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49912/262434 objects degraded (19.019%); 53856/262434 objects misplaced (20.522%); 80 MiB/s, 1.28k objects/s recovering 2023-09-23T18:15:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[184174]: failed to encode map e502 with expected crc 2023-09-23T18:15:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[184174]: osdmap e502: 8 total, 7 up, 7 in 2023-09-23T18:15:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[184174]: failed to encode map e502 with expected crc 2023-09-23T18:15:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[184174]: failed to encode map e502 with expected crc 2023-09-23T18:15:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[184174]: failed to encode map e502 with expected crc 2023-09-23T18:15:47.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:47 smithi055 ceph-mon[184174]: pgmap v1216: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+remapped, 1 undersized+degraded+remapped+backfilling+peered, 2 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49912/262434 objects degraded (19.019%); 53856/262434 objects misplaced (20.522%); 80 MiB/s, 1.28k objects/s recovering 2023-09-23T18:15:47.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:48.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:47 smithi099 ceph-mon[167076]: failed to encode map e502 with expected crc 2023-09-23T18:15:48.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:47 smithi099 ceph-mon[167076]: osdmap e502: 8 total, 7 up, 7 in 2023-09-23T18:15:48.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:47 smithi099 ceph-mon[167076]: failed to encode map e502 with expected crc 2023-09-23T18:15:48.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:47 smithi099 ceph-mon[167076]: failed to encode map e502 with expected crc 2023-09-23T18:15:48.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:47 smithi099 ceph-mon[167076]: failed to encode map e502 with expected crc 2023-09-23T18:15:48.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:47 smithi099 ceph-mon[167076]: pgmap v1216: 119 pgs: 1 active+recovery_wait+remapped, 2 active+clean+remapped, 1 undersized+degraded+remapped+backfilling+peered, 2 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+undersized+degraded, 4 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 12 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 49912/262434 objects degraded (19.019%); 53856/262434 objects misplaced (20.522%); 80 MiB/s, 1.28k objects/s recovering 2023-09-23T18:15:48.031 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:48.069 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:15:48.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:48.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:48.114 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:48.215 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:48.264 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:48.274 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:15:48.375 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:48.383 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:48.384 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:48.485 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:48.518 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:48.558 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:48.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:48.563 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:48.664 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:48.692 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:48.730 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:48.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:48.734 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:48.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: failed to encode map e503 with expected crc 2023-09-23T18:15:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: failed to encode map e503 with expected crc 2023-09-23T18:15:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: failed to encode map e503 with expected crc 2023-09-23T18:15:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: osdmap e503: 8 total, 7 up, 7 in 2023-09-23T18:15:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1178663028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 49912/262434 objects degraded (19.019%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: failed to encode map e504 with expected crc 2023-09-23T18:15:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:48.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:48 smithi099 ceph-mon[167076]: osdmap e504: 8 total, 7 up, 7 in 2023-09-23T18:15:48.835 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:48.864 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:48.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: failed to encode map e503 with expected crc 2023-09-23T18:15:48.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: failed to encode map e503 with expected crc 2023-09-23T18:15:48.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: failed to encode map e503 with expected crc 2023-09-23T18:15:48.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: osdmap e503: 8 total, 7 up, 7 in 2023-09-23T18:15:48.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1178663028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:48.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:48.886 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 49912/262434 objects degraded (19.019%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:48.887 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:48.887 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: failed to encode map e504 with expected crc 2023-09-23T18:15:48.887 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:48.887 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[179668]: osdmap e504: 8 total, 7 up, 7 in 2023-09-23T18:15:48.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: failed to encode map e503 with expected crc 2023-09-23T18:15:48.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: failed to encode map e503 with expected crc 2023-09-23T18:15:48.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: failed to encode map e503 with expected crc 2023-09-23T18:15:48.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: osdmap e503: 8 total, 7 up, 7 in 2023-09-23T18:15:48.887 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1178663028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:48.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:15:48.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 49912/262434 objects degraded (19.019%), 20 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:48.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:48.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: failed to encode map e504 with expected crc 2023-09-23T18:15:48.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:15:48.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:48 smithi055 ceph-mon[184174]: osdmap e504: 8 total, 7 up, 7 in 2023-09-23T18:15:48.902 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:48.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:48.906 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:49.006 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:49.036 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:49.074 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:49.076 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:49.077 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:49.177 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:49.212 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:49.333 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:49.350 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:49.365 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:49.365 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:49.466 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:49.549 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:49.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:49.624 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:15:49.624 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:15:49.624 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:15:49.624 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:15:49.624 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:15:49.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:49.630 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:49.731 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:49.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:49 smithi099 ceph-mon[167076]: failed to encode map e504 with expected crc 2023-09-23T18:15:49.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:49 smithi099 ceph-mon[167076]: failed to encode map e504 with expected crc 2023-09-23T18:15:49.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:49 smithi099 ceph-mon[167076]: failed to encode map e504 with expected crc 2023-09-23T18:15:49.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:49 smithi099 ceph-mon[167076]: pgmap v1219: 119 pgs: 3 active+clean+remapped, 3 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 48 op/s; 47320/261567 objects degraded (18.091%); 53696/261567 objects misplaced (20.529%); 105 MiB/s, 1.68k objects/s recovering 2023-09-23T18:15:49.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:49 smithi099 ceph-mon[167076]: osdmap e505: 8 total, 7 up, 7 in 2023-09-23T18:15:49.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:49 smithi099 ceph-mon[167076]: failed to encode map e505 with expected crc 2023-09-23T18:15:49.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2914878945' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T18:15:49.795 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:49 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T18:15:49.798 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:49.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:49.873 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:49.873 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[179668]: failed to encode map e504 with expected crc 2023-09-23T18:15:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[179668]: failed to encode map e504 with expected crc 2023-09-23T18:15:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[179668]: failed to encode map e504 with expected crc 2023-09-23T18:15:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[179668]: pgmap v1219: 119 pgs: 3 active+clean+remapped, 3 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 48 op/s; 47320/261567 objects degraded (18.091%); 53696/261567 objects misplaced (20.529%); 105 MiB/s, 1.68k objects/s recovering 2023-09-23T18:15:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[179668]: osdmap e505: 8 total, 7 up, 7 in 2023-09-23T18:15:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[179668]: failed to encode map e505 with expected crc 2023-09-23T18:15:49.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2914878945' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T18:15:49.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T18:15:49.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[184174]: failed to encode map e504 with expected crc 2023-09-23T18:15:49.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[184174]: failed to encode map e504 with expected crc 2023-09-23T18:15:49.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[184174]: failed to encode map e504 with expected crc 2023-09-23T18:15:49.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[184174]: pgmap v1219: 119 pgs: 3 active+clean+remapped, 3 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 48 op/s; 47320/261567 objects degraded (18.091%); 53696/261567 objects misplaced (20.529%); 105 MiB/s, 1.68k objects/s recovering 2023-09-23T18:15:49.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[184174]: osdmap e505: 8 total, 7 up, 7 in 2023-09-23T18:15:49.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[184174]: failed to encode map e505 with expected crc 2023-09-23T18:15:49.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2914878945' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T18:15:49.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:49 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]: dispatch 2023-09-23T18:15:49.974 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:50.034 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:50.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:50.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:50.112 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:50.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:50.276 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:15:50.284 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.1 primary-affinity to 0 (802) 2023-09-23T18:15:50.296 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:50.389 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:50.398 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:50.399 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:50.500 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:50.533 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:15:50] "GET /metrics HTTP/1.1" 200 36660 "" "Prometheus/2.43.0" 2023-09-23T18:15:50.538 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:50.684 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:50.688 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:50.688 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:50.788 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[184174]: failed to encode map e505 with expected crc 2023-09-23T18:15:50.788 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[184174]: failed to encode map e505 with expected crc 2023-09-23T18:15:50.788 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-23T18:15:50.788 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[184174]: osdmap e506: 8 total, 7 up, 7 in 2023-09-23T18:15:50.789 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[184174]: failed to encode map e506 with expected crc 2023-09-23T18:15:50.789 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[184174]: failed to encode map e506 with expected crc 2023-09-23T18:15:50.789 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[179668]: failed to encode map e505 with expected crc 2023-09-23T18:15:50.789 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[179668]: failed to encode map e505 with expected crc 2023-09-23T18:15:50.789 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-23T18:15:50.789 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[179668]: osdmap e506: 8 total, 7 up, 7 in 2023-09-23T18:15:50.789 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[179668]: failed to encode map e506 with expected crc 2023-09-23T18:15:50.789 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:50 smithi055 ceph-mon[179668]: failed to encode map e506 with expected crc 2023-09-23T18:15:50.790 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:50.811 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:15:50.840 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:50.937 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:50.943 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:50.943 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:51.044 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:51.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:50 smithi099 ceph-mon[167076]: failed to encode map e505 with expected crc 2023-09-23T18:15:51.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:50 smithi099 ceph-mon[167076]: failed to encode map e505 with expected crc 2023-09-23T18:15:51.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:50 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 0.0}]': finished 2023-09-23T18:15:51.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:50 smithi099 ceph-mon[167076]: osdmap e506: 8 total, 7 up, 7 in 2023-09-23T18:15:51.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:50 smithi099 ceph-mon[167076]: failed to encode map e506 with expected crc 2023-09-23T18:15:51.081 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:50 smithi099 ceph-mon[167076]: failed to encode map e506 with expected crc 2023-09-23T18:15:51.084 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:51.150 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:51.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:51.161 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:51.173 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:15:51.202 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:15:51.231 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:15:51.257 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:15:51.259 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:51.262 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:51.289 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:15:51.317 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:15:51.355 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:15:51.355 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:15:51.355 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:15:51.355 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:15:51.355 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:15:51.355 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:15:51.355 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:15:51.355 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:15:51.356 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:15:51.363 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:51.447 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:51.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:51.457 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:51.558 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:51.623 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:51.707 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:51.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:51.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:51.815 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:51.856 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:51.892 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[184174]: failed to encode map e506 with expected crc 2023-09-23T18:15:51.893 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[184174]: failed to encode map e506 with expected crc 2023-09-23T18:15:51.893 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[184174]: pgmap v1222: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 339 op/s; 44303/258357 objects degraded (17.148%); 53060/258357 objects misplaced (20.537%); 79 MiB/s, 1.27k objects/s recovering 2023-09-23T18:15:51.893 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1061144290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:51.893 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:51.893 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:51.893 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[179668]: failed to encode map e506 with expected crc 2023-09-23T18:15:51.893 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[179668]: failed to encode map e506 with expected crc 2023-09-23T18:15:51.893 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[179668]: pgmap v1222: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 339 op/s; 44303/258357 objects degraded (17.148%); 53060/258357 objects misplaced (20.537%); 79 MiB/s, 1.27k objects/s recovering 2023-09-23T18:15:51.893 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1061144290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:51.894 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:51.894 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:51.917 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:15:51.918 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:15:52.001 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:52.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:52.005 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:51 smithi099 ceph-mon[167076]: failed to encode map e506 with expected crc 2023-09-23T18:15:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:51 smithi099 ceph-mon[167076]: failed to encode map e506 with expected crc 2023-09-23T18:15:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:51 smithi099 ceph-mon[167076]: pgmap v1222: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 339 op/s; 44303/258357 objects degraded (17.148%); 53060/258357 objects misplaced (20.537%); 79 MiB/s, 1.27k objects/s recovering 2023-09-23T18:15:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1061144290' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:15:52.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:15:52.106 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:52.137 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:52.179 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:52.183 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:52.184 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:52.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:52.310 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:52.366 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:52.446 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:52.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:52.453 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:52.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:52.633 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:52.689 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:52.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:52.697 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:52.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:52.808 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:15:52.809 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:15:52.867 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:52.953 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:52.963 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:52.963 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:53.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:53.152 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:53.236 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:53.246 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:53.247 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:53.348 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:53.388 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:53.433 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:53.437 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:53.438 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:53.539 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:53.573 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:15:53.619 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:53.723 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:53.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:53.739 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:53.840 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:53.919 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:15:53.928 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:54.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:54.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:54.029 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:54.130 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:54.168 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:54.215 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:54.218 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:54 smithi055 ceph-mon[184174]: pgmap v1223: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 44303/258357 objects degraded (17.148%); 53060/258357 objects misplaced (20.537%); 60 MiB/s, 957 objects/s recovering 2023-09-23T18:15:54.218 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:54 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 44303/258357 objects degraded (17.148%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:54.218 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:54 smithi055 ceph-mon[179668]: pgmap v1223: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 44303/258357 objects degraded (17.148%); 53060/258357 objects misplaced (20.537%); 60 MiB/s, 957 objects/s recovering 2023-09-23T18:15:54.218 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:54 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 44303/258357 objects degraded (17.148%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:54.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:54.220 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:54.320 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:54.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:54 smithi099 ceph-mon[167076]: pgmap v1223: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.3 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 256 op/s; 44303/258357 objects degraded (17.148%); 53060/258357 objects misplaced (20.537%); 60 MiB/s, 957 objects/s recovering 2023-09-23T18:15:54.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:54 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 44303/258357 objects degraded (17.148%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:54.352 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:54.400 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:54.405 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:54.406 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:54.507 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:54.523 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:15:54.542 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:54.587 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:15:54.590 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:54.590 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:15:54.672 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:54.692 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:54.765 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:54.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:54.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:54.866 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:54.967 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:55.003 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:55.142 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:15:55.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2451011592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:15:55.171 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:55 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:15:55.172 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2451011592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:15:55.172 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:55 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:15:55.249 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:55.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:55.253 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:55.353 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:55.452 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2451011592' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:15:55.452 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:55 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:15:55.472 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:55.542 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:55.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:55.555 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:55.656 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:55.694 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:55.739 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:55.743 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:55.744 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:55.821 INFO:tasks.thrashosds.thrasher:in_osds: [7, 1, 0, 4, 2, 3, 6] out_osds: [5] dead_osds: [1] live_osds: [0, 4, 6, 7, 3, 5, 2] 2023-09-23T18:15:55.822 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:15:55.822 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T18:15:55.822 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:15:55.822 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:15:55.822 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T18:15:55.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:56.005 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:56.105 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:56.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:56.114 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:56.160 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:15:56.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[179668]: pgmap v1224: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 41860/250869 objects degraded (16.686%); 51388/250869 objects misplaced (20.484%); 39 MiB/s, 626 objects/s recovering 2023-09-23T18:15:56.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:56.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[179668]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:15:56.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[179668]: osdmap e507: 8 total, 7 up, 7 in 2023-09-23T18:15:56.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[179668]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[179668]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[179668]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[184174]: pgmap v1224: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 41860/250869 objects degraded (16.686%); 51388/250869 objects misplaced (20.484%); 39 MiB/s, 626 objects/s recovering 2023-09-23T18:15:56.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:56.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[184174]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:15:56.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[184174]: osdmap e507: 8 total, 7 up, 7 in 2023-09-23T18:15:56.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[184174]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[184174]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:56 smithi055 ceph-mon[184174]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.184 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:15:56.206 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:15:56.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:15:56.238 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:15:56.270 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:15:56.296 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:15:56.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:56 smithi099 ceph-mon[167076]: pgmap v1224: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.1 GiB data, 18 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 559 op/s; 41860/250869 objects degraded (16.686%); 51388/250869 objects misplaced (20.484%); 39 MiB/s, 626 objects/s recovering 2023-09-23T18:15:56.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:56 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:15:56.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:56 smithi099 ceph-mon[167076]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:56 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:15:56.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:56 smithi099 ceph-mon[167076]: osdmap e507: 8 total, 7 up, 7 in 2023-09-23T18:15:56.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:56 smithi099 ceph-mon[167076]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:56 smithi099 ceph-mon[167076]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.304 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:56 smithi099 ceph-mon[167076]: failed to encode map e507 with expected crc 2023-09-23T18:15:56.304 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:56.323 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:15:56.323 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:15:56.324 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:15:56.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:15:56.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:15:56.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:15:56.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:15:56.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:15:56.324 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:15:56.330 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:15:56.331 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:15:56.331 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.331 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:129710 9.c 9:38ba9611:::benchmark_data_smithi055_386467_object25050:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e507)", 2023-09-23T18:15:56.331 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:56.320620+0000", 2023-09-23T18:15:56.331 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0085404490000000003, 2023-09-23T18:15:56.331 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085645689999999993, 2023-09-23T18:15:56.331 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:56.331 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:56.332 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:56.332 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:56.332 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:56.332 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129710 2023-09-23T18:15:56.332 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.332 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:56.332 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.332 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.320620+0000", 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.320620+0000", 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:56.333 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.320619+0000", 2023-09-23T18:15:56.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:15:56.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:56.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.320629+0000", 2023-09-23T18:15:56.334 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0188e-05 2023-09-23T18:15:56.334 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.334 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.334 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:56.334 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.320653+0000", 2023-09-23T18:15:56.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3981999999999998e-05 2023-09-23T18:15:56.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.335 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.335 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:56.335 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.320662+0000", 2023-09-23T18:15:56.335 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1800000000000002e-06 2023-09-23T18:15:56.335 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.321152+0000", 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00049052700000000004 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.321289+0000", 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000136196 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.336 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.321412+0000", 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012308999999999999 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.324231+0000", 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028192590000000002 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.324508+0000", 2023-09-23T18:15:56.337 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00027714699999999999 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:129712 9.f 9:fc7713fa:::benchmark_data_smithi055_386467_object25052:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e507)", 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:56.321810+0000", 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0073511760000000001, 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0074335110000000003, 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:15:56.338 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129712 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.321810+0000", 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.339 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.321810+0000", 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.321809+0000", 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.321815+0000", 2023-09-23T18:15:56.340 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9730000000000002e-06 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.321827+0000", 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2879e-05 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.321835+0000", 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1149999999999996e-06 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.341 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.321885+0000", 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0058000000000001e-05 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.322006+0000", 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121292 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:15:56.342 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.322119+0000", 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011333299999999999 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop(client.36552.0:129711 9.d e507/498 9:ba41da16:::benchmark_data_smithi055_386467_object25051:head v 507'8026, mlcod=507'8024)", 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:56.328098+0000", 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0010629789999999999, 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001188337, 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:15:56.343 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.328098+0000", 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.328098+0000", 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:15:56.344 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.328097+0000", 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.328107+0000", 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.047e-05 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.345 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.328127+0000", 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9239999999999999e-05 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.328135+0000", 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2870000000000004e-06 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:15:56.346 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.328201+0000", 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6357000000000002e-05 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:56.328222+0000", 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1022e-05 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:15:56.347 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 3 2023-09-23T18:15:56.348 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:15:56.365 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:15:56.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:56.372 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:15:56.473 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:56.508 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:56.631 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:56.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:56.635 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:56.669 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:15:56.670 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:15:56.725 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:15:56.736 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:15:56.747 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:15:56.835 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:56.916 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:15:56.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:56.926 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:15:57.027 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:15:57.143 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:57.213 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:15:57.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:57.215 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:15:57.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:57.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3482716751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:57.391 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:57.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3482716751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:57.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3482716751' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:15:57.475 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:57.481 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:57.481 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:57.582 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:15:57.615 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:57.646 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:15:57.658 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:15:57.662 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:57.662 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:15:57.763 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:57.794 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:57.836 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:57.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:57.841 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:57.942 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:15:57.972 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:58.013 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:15:58.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:58.016 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:15:58.021 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:15:58.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:58.210 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:58.316 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:15:58.399 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:58 smithi099 ceph-mon[167076]: pgmap v1226: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 40319/244425 objects degraded (16.495%); 50156/244425 objects misplaced (20.520%); 35 MiB/s, 559 objects/s recovering 2023-09-23T18:15:58.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:58 smithi055 ceph-mon[184174]: pgmap v1226: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 40319/244425 objects degraded (16.495%); 50156/244425 objects misplaced (20.520%); 35 MiB/s, 559 objects/s recovering 2023-09-23T18:15:58.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:58 smithi055 ceph-mon[179668]: pgmap v1226: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 5.0 GiB data, 17 GiB used, 608 GiB / 626 GiB avail; 0 B/s wr, 734 op/s; 40319/244425 objects degraded (16.495%); 50156/244425 objects misplaced (20.520%); 35 MiB/s, 559 objects/s recovering 2023-09-23T18:15:58.440 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:58.448 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:58.449 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:58.549 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:15:58.570 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:15:58.571 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T18:15:58.571 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:15:58.645 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:58.671 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:15:59.054 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:15:59.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:15:59.066 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:15:59.142 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:15:59 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 40319/244425 objects degraded (16.495%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:59.166 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:15:59.205 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:15:59.332 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:15:59 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 40319/244425 objects degraded (16.495%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:15:59.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:15:59 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 40319/244425 objects degraded (16.495%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:00.036 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:00.044 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:00.044 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:00.145 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:00.176 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:00.280 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:00.286 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:00.286 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:00.287 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:00.357 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:00 smithi055 ceph-mon[184174]: pgmap v1227: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 954 op/s; 38248/236451 objects degraded (16.176%); 48437/236451 objects misplaced (20.485%); 34 MiB/s, 537 objects/s recovering 2023-09-23T18:16:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:00 smithi055 ceph-mon[179668]: pgmap v1227: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 954 op/s; 38248/236451 objects degraded (16.176%); 48437/236451 objects misplaced (20.485%); 34 MiB/s, 537 objects/s recovering 2023-09-23T18:16:00.388 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:00.416 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:00.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:00 smithi099 ceph-mon[167076]: pgmap v1227: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.8 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 954 op/s; 38248/236451 objects degraded (16.176%); 48437/236451 objects misplaced (20.485%); 34 MiB/s, 537 objects/s recovering 2023-09-23T18:16:00.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:16:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:16:00] "GET /metrics HTTP/1.1" 200 36662 "" "Prometheus/2.43.0" 2023-09-23T18:16:00.704 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:00.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:00.710 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:00.811 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:00.844 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:01.147 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:01.151 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:01.151 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:01.252 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:01.297 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:01.339 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:01.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:01.344 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:01.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:01.521 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:01.922 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:01.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:01.927 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:02.028 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:02.067 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:02.160 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:02.164 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:02.164 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:02.236 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:02.261 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:02.265 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:02.276 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:02 smithi055 ceph-mon[179668]: pgmap v1228: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 929 op/s; 36826/230460 objects degraded (15.979%); 47274/230460 objects misplaced (20.513%); 18 MiB/s, 289 objects/s recovering 2023-09-23T18:16:02.276 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/736111128' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:02.276 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/774891710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:02.281 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:02 smithi099 ceph-mon[167076]: pgmap v1228: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 929 op/s; 36826/230460 objects degraded (15.979%); 47274/230460 objects misplaced (20.513%); 18 MiB/s, 289 objects/s recovering 2023-09-23T18:16:02.281 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/736111128' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:02.282 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/774891710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:02.288 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:02.301 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:02.314 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:16:02.322 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:02.344 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:02.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:02.350 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:02.362 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:02.398 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:02.405 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:16:02.405 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:16:02.429 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:02.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:16:02.430 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:02.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:16:02.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:16:02.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:16:02.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:16:02.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:16:02.430 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:16:02.452 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:02.486 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:02 smithi055 ceph-mon[184174]: pgmap v1228: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 929 op/s; 36826/230460 objects degraded (15.979%); 47274/230460 objects misplaced (20.513%); 18 MiB/s, 289 objects/s recovering 2023-09-23T18:16:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/736111128' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/774891710' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:02.931 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:02.935 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:02.935 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:02.972 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:02.972 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:03.036 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:03.068 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:03.109 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:03.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:03.113 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:03.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:03.277 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:16:03.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[179668]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/736111128' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:03.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[179668]: osdmap e508: 8 total, 7 up, 7 in 2023-09-23T18:16:03.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[179668]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[179668]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[179668]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[179668]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 36826/230460 objects degraded (15.979%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:03.316 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:03.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:03 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:16:03.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:03 smithi099 ceph-mon[167076]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/736111128' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:03.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:03 smithi099 ceph-mon[167076]: osdmap e508: 8 total, 7 up, 7 in 2023-09-23T18:16:03.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:03 smithi099 ceph-mon[167076]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:03 smithi099 ceph-mon[167076]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:03 smithi099 ceph-mon[167076]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:03 smithi099 ceph-mon[167076]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 36826/230460 objects degraded (15.979%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:16:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[184174]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/736111128' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[184174]: osdmap e508: 8 total, 7 up, 7 in 2023-09-23T18:16:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[184174]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[184174]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[184174]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[184174]: failed to encode map e508 with expected crc 2023-09-23T18:16:03.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 36826/230460 objects degraded (15.979%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:04.170 INFO:tasks.thrashosds.thrasher:in_osds: [7, 1, 0, 4, 2, 3, 6] out_osds: [5] dead_osds: [1] live_osds: [0, 4, 6, 7, 3, 5, 2] 2023-09-23T18:16:04.171 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:16:04.171 INFO:tasks.thrashosds.thrasher:Adding osd 5 2023-09-23T18:16:04.171 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 5 2023-09-23T18:16:04.581 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:04 smithi099 ceph-mon[167076]: pgmap v1230: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 36826/230460 objects degraded (15.979%); 47274/230460 objects misplaced (20.513%); 16 MiB/s, 256 objects/s recovering 2023-09-23T18:16:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:04 smithi055 ceph-mon[179668]: pgmap v1230: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 36826/230460 objects degraded (15.979%); 47274/230460 objects misplaced (20.513%); 16 MiB/s, 256 objects/s recovering 2023-09-23T18:16:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:04 smithi055 ceph-mon[184174]: pgmap v1230: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.7 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 850 op/s; 36826/230460 objects degraded (15.979%); 47274/230460 objects misplaced (20.513%); 16 MiB/s, 256 objects/s recovering 2023-09-23T18:16:04.838 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:04.923 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:16:04.973 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:16:05.047 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:05.053 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:05.054 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:05.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:16:05.155 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:05.208 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:05.273 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:05.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:05.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:05.377 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:05.443 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:05.511 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:05.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:05.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:05.616 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:05.664 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:05.788 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:05.793 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:05.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:05.799 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:05.893 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:05.900 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:05.985 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:06.055 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:16:06.070 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:06.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:06.082 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:06.182 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:06.251 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:06.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:06 smithi055 ceph-mon[179668]: pgmap v1231: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 34649/222297 objects degraded (15.587%); 45514/222297 objects misplaced (20.474%); 16 MiB/s, 256 objects/s recovering 2023-09-23T18:16:06.523 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:06 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:16:06.410+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e508 definitely_dead 0 2023-09-23T18:16:06.523 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:06 smithi055 ceph-mon[184174]: pgmap v1231: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 34649/222297 objects degraded (15.587%); 45514/222297 objects misplaced (20.474%); 16 MiB/s, 256 objects/s recovering 2023-09-23T18:16:06.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:06 smithi099 ceph-mon[167076]: pgmap v1231: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.5 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 34649/222297 objects degraded (15.587%); 45514/222297 objects misplaced (20.474%); 16 MiB/s, 256 objects/s recovering 2023-09-23T18:16:07.005 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:07.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:07.016 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:07.117 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:07.151 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:07.186 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:07.189 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:07.190 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:07.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:07.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:07 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2073178166' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:16:07.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:07 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:16:07.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:07.316 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:07.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:07 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2073178166' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:16:07.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:07 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:16:07.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:07.318 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:07.339 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.5. 2023-09-23T18:16:07.377 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:07.572 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:07.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:07.576 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:07.676 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:07 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2073178166' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:16:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:07 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["5"]}]: dispatch 2023-09-23T18:16:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:07.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:07.711 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:07.752 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:07.774 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:07.796 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:07.817 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:07.836 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:07.838 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:07.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:07.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:07.861 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:07.862 INFO:tasks.thrashosds.thrasher:Added osd 5 2023-09-23T18:16:07.881 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:07.881 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:16:07.881 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:07.882 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:16:07.882 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:16:07.882 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:16:07.882 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:16:07.882 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:16:07.882 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:16:07.941 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:07.977 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:08.016 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:08.019 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:08.020 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:08.120 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:08.149 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:08.239 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:08.243 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:08.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:08.278 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:08.278 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:08.344 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:08.375 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: pgmap v1232: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 33210/215601 objects degraded (15.403%); 44252/215601 objects misplaced (20.525%); 16 MiB/s, 253 objects/s recovering 2023-09-23T18:16:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:16:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: osdmap e509: 8 total, 7 up, 8 in 2023-09-23T18:16:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1190854592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:08.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 33210/215601 objects degraded (15.403%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:08.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: osdmap e510: 8 total, 7 up, 8 in 2023-09-23T18:16:08.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:08 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: pgmap v1232: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 33210/215601 objects degraded (15.403%); 44252/215601 objects misplaced (20.525%); 16 MiB/s, 253 objects/s recovering 2023-09-23T18:16:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:16:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: osdmap e509: 8 total, 7 up, 8 in 2023-09-23T18:16:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1190854592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 33210/215601 objects degraded (15.403%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: osdmap e510: 8 total, 7 up, 8 in 2023-09-23T18:16:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: pgmap v1232: 119 pgs: 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 3 active+undersized+degraded+remapped+backfill_wait, 55 active+undersized, 1 active+remapped+backfill_wait, 14 active+undersized+degraded, 40 active+clean; 4.4 GiB data, 17 GiB used, 609 GiB / 626 GiB avail; 0 B/s wr, 960 op/s; 33210/215601 objects degraded (15.403%); 44252/215601 objects misplaced (20.525%); 16 MiB/s, 253 objects/s recovering 2023-09-23T18:16:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["5"]}]': finished 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: osdmap e509: 8 total, 7 up, 8 in 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: failed to encode map e509 with expected crc 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1190854592' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 33210/215601 objects degraded (15.403%), 19 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: osdmap e510: 8 total, 7 up, 8 in 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:08.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:08 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.269 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:16:09.413 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:09.418 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:09.418 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:09.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:09.568 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:09.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.589 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/249611374' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: failed to encode map e511 with expected crc 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: failed to encode map e511 with expected crc 2023-09-23T18:16:09.590 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:16:09.591 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[184174]: osdmap e511: 8 total, 7 up, 8 in 2023-09-23T18:16:09.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/249611374' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:09.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:09.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:09.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: failed to encode map e511 with expected crc 2023-09-23T18:16:09.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: failed to encode map e511 with expected crc 2023-09-23T18:16:09.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:16:09.592 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:09 smithi055 ceph-mon[179668]: osdmap e511: 8 total, 7 up, 8 in 2023-09-23T18:16:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: failed to encode map e510 with expected crc 2023-09-23T18:16:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/249611374' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:09.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: failed to encode map e511 with expected crc 2023-09-23T18:16:09.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: failed to encode map e511 with expected crc 2023-09-23T18:16:09.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:16:09.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:09 smithi099 ceph-mon[167076]: osdmap e511: 8 total, 7 up, 8 in 2023-09-23T18:16:10.279 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:16:10.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:10.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:10.513 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:10.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:10.679 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:10 smithi099 ceph-mon[167076]: pgmap v1235: 119 pgs: 5 remapped+peering, 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 51 active+undersized, 15 active+undersized+degraded, 40 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 922 op/s; 32204/211797 objects degraded (15.205%); 35727/211797 objects misplaced (16.869%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:16:10.679 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:10 smithi099 ceph-mon[167076]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.679 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:10 smithi099 ceph-mon[167076]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.679 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:10 smithi099 ceph-mon[167076]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:10 smithi099 ceph-mon[167076]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:10 smithi099 ceph-mon[167076]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:10 smithi099 ceph-mon[167076]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:10 smithi099 ceph-mon[167076]: failed to encode map e512 with expected crc 2023-09-23T18:16:10.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:10 smithi099 ceph-mon[167076]: osdmap e512: 8 total, 7 up, 8 in 2023-09-23T18:16:10.680 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:10 smithi099 ceph-mon[167076]: failed to encode map e512 with expected crc 2023-09-23T18:16:10.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:16:10] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.43.0" 2023-09-23T18:16:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[184174]: pgmap v1235: 119 pgs: 5 remapped+peering, 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 51 active+undersized, 15 active+undersized+degraded, 40 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 922 op/s; 32204/211797 objects degraded (15.205%); 35727/211797 objects misplaced (16.869%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:16:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[184174]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[184174]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[184174]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[184174]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[184174]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[184174]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[184174]: failed to encode map e512 with expected crc 2023-09-23T18:16:10.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[184174]: osdmap e512: 8 total, 7 up, 8 in 2023-09-23T18:16:10.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[184174]: failed to encode map e512 with expected crc 2023-09-23T18:16:10.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[179668]: pgmap v1235: 119 pgs: 5 remapped+peering, 4 active+clean+remapped, 2 active+undersized+degraded+remapped+backfilling, 2 active+undersized+degraded+remapped+backfill_wait, 51 active+undersized, 15 active+undersized+degraded, 40 active+clean; 4.3 GiB data, 16 GiB used, 610 GiB / 626 GiB avail; 0 B/s wr, 922 op/s; 32204/211797 objects degraded (15.205%); 35727/211797 objects misplaced (16.869%); 12 MiB/s, 195 objects/s recovering 2023-09-23T18:16:10.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[179668]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[179668]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[179668]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[179668]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[179668]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[179668]: failed to encode map e511 with expected crc 2023-09-23T18:16:10.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[179668]: failed to encode map e512 with expected crc 2023-09-23T18:16:10.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[179668]: osdmap e512: 8 total, 7 up, 8 in 2023-09-23T18:16:10.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:10 smithi055 ceph-mon[179668]: failed to encode map e512 with expected crc 2023-09-23T18:16:10.684 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:10.958 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:11.331 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:11.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:11.344 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:11.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:11.517 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:11.581 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:11 smithi099 ceph-mon[167076]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:11 smithi099 ceph-mon[167076]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:11 smithi099 ceph-mon[167076]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:11 smithi099 ceph-mon[167076]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:11 smithi099 ceph-mon[167076]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:11 smithi099 ceph-mon[167076]: osdmap e513: 8 total, 7 up, 8 in 2023-09-23T18:16:11.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:11 smithi099 ceph-mon[167076]: failed to encode map e513 with expected crc 2023-09-23T18:16:11.582 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:11 smithi099 ceph-mon[167076]: failed to encode map e513 with expected crc 2023-09-23T18:16:11.612 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:16:11.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[179668]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[179668]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[179668]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[179668]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.646 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[179668]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[179668]: osdmap e513: 8 total, 7 up, 8 in 2023-09-23T18:16:11.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[179668]: failed to encode map e513 with expected crc 2023-09-23T18:16:11.647 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[179668]: failed to encode map e513 with expected crc 2023-09-23T18:16:11.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[184174]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[184174]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[184174]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.647 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[184174]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.723 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[184174]: failed to encode map e512 with expected crc 2023-09-23T18:16:11.724 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[184174]: osdmap e513: 8 total, 7 up, 8 in 2023-09-23T18:16:11.724 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[184174]: failed to encode map e513 with expected crc 2023-09-23T18:16:11.724 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:11 smithi055 ceph-mon[184174]: failed to encode map e513 with expected crc 2023-09-23T18:16:11.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:11.725 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:11.726 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:11.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:11.853 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:16:11.853 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:16:11.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:16:11.853 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:16:11.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.853 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65460 7.4 7:2d53859f:::benchmark_data_smithi055_311528_object65459:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 565.95425199399995, 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9613000379999992, 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65460 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.854 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896405+0000", 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896404+0000", 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896620+0000", 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021651099999999999 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896638+0000", 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8179999999999999e-05 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896646+0000", 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2899999999999997e-06 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896718+0000", 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2587000000000001e-05 2023-09-23T18:16:11.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896729+0000", 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1223000000000001e-05 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.853134+0000", 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9564047319999993 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.857705+0000", 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045713259999999997 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65469 7.4 7:2eb46e41:::benchmark_data_smithi055_311528_object65468:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 565.95399034299999, 2023-09-23T18:16:11.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 10.043735116000001, 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65469 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:16:11.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896667+0000", 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896666+0000", 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896742+0000", 2023-09-23T18:16:11.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5432000000000006e-05 2023-09-23T18:16:11.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896755+0000", 2023-09-23T18:16:11.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3275000000000001e-05 2023-09-23T18:16:11.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896762+0000", 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4760000000000003e-06 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896860+0000", 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8120000000000002e-05 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896869+0000", 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1409999999999994e-06 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.858317+0000", 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9614477719999996 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:16:11.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.863986+0000", 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0056689319999999998 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940290+0000", 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.076304636999999995 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940402+0000", 2023-09-23T18:16:11.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111987 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65477 7.4 7:23aaada8:::benchmark_data_smithi055_311528_object65476:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e299)", 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 565.95388242199999, 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 10.046314513, 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.866 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65477 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896775+0000", 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896774+0000", 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896805+0000", 2023-09-23T18:16:11.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0916e-05 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896816+0000", 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0519999999999999e-05 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896820+0000", 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0849999999999999e-06 2023-09-23T18:16:11.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896888+0000", 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8637e-05 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:45.896897+0000", 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6279999999999994e-06 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.864614+0000", 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9677164650000005 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.871623+0000", 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070090059999999999 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:16:11.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.882195+0000", 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010572560999999999 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.928966+0000", 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.046770763999999999 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940462+0000", 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011496312 2023-09-23T18:16:11.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940512+0000", 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0118999999999999e-05 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940552+0000", 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9535e-05 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943089+0000", 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025376470000000001 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65485 7.4 7:246beb72:::benchmark_data_smithi055_311528_object65484:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 564.75449121500003, 2023-09-23T18:16:11.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8502435909999999, 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65485 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:16:11.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096166+0000", 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096165+0000", 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096281+0000", 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116042 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096295+0000", 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4524e-05 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.096301+0000", 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7490000000000004e-06 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.889363+0000", 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7930615480000007 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.896004+0000", 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066411439999999999 2023-09-23T18:16:11.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.906662+0000", 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010657645 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.940400+0000", 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.033738490000000003 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943179+0000", 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002778675 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943222+0000", 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2966999999999997e-05 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.943292+0000", 2023-09-23T18:16:11.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0043999999999995e-05 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946410+0000", 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031180190000000001 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65486 7.4 7:28b4ee06:::benchmark_data_smithi055_311528_object65485:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:16:11.881 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 564.72924073599995, 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8402319929999997, 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65486 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121417+0000", 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121415+0000", 2023-09-23T18:16:11.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121466+0000", 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1007000000000001e-05 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121481+0000", 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5287000000000001e-05 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121489+0000", 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5010000000000004e-06 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946571+0000", 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8250813830000006 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946747+0000", 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017653100000000001 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.946947+0000", 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019961800000000001 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.954051+0000", 2023-09-23T18:16:11.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0071042989999999997 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958525+0000", 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044738959999999998 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958638+0000", 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113078 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958698+0000", 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0207000000000001e-05 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961649+0000", 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002950214 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65487 7.4 7:2b3c8c2f:::benchmark_data_smithi055_311528_object65486:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 564.72915666999995, 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8433001999999998, 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65487 2023-09-23T18:16:11.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121501+0000", 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121500+0000", 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121563+0000", 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3126000000000005e-05 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121579+0000", 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6418000000000001e-05 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.121585+0000", 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.575e-06 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949470+0000", 2023-09-23T18:16:11.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8278846659999992 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949632+0000", 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016232099999999999 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.949768+0000", 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013578700000000001 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.956292+0000", 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065241910000000004 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.958569+0000", 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022767289999999999 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961776+0000", 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032076660000000001 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961847+0000", 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1137999999999996e-05 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964801+0000", 2023-09-23T18:16:11.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029533990000000002 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:65498 7.4 7:27d4ee95:::benchmark_data_smithi055_311528_object65497:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e300)", 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 564.69464715599997, 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8116268140000003, 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.896 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 65498 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156010+0000", 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156009+0000", 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156082+0000", 2023-09-23T18:16:11.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3093e-05 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156096+0000", 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4465e-05 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:47.156104+0000", 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4120000000000004e-06 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.951850+0000", 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7957462389999996 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.951998+0000", 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014826300000000001 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:16:11.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.952148+0000", 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000149524 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961673+0000", 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095253839999999996 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.961735+0000", 2023-09-23T18:16:11.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2036e-05 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964887+0000", 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003152092 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.964960+0000", 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2601999999999994e-05 2023-09-23T18:16:11.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:06:55.967637+0000", 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026772739999999999 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108071 7.21 7:85ca81b0:::benchmark_data_smithi055_311528_object37108:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:16:11.903 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 452.30603257500002, 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9624768260000001, 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108071 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544624+0000", 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:11.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544628+0000", 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.698e-06 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544634+0000", 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3849999999999999e-06 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544638+0000", 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7560000000000001e-06 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581968+0000", 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037330020999999998 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581975+0000", 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1470000000000004e-06 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487605+0000", 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9056295090000002 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487615+0000", 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0036e-05 2023-09-23T18:16:11.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502365+0000", 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0147500899999999 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502491+0000", 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125716 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502677+0000", 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018593700000000001 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504929+0000", 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022522219999999999 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504942+0000", 2023-09-23T18:16:11.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2636999999999999e-05 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506803+0000", 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018613709999999999 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506851+0000", 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8578000000000001e-05 2023-09-23T18:16:11.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507102+0000", 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025014900000000002 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108080 7.21 7:85d5fb8e:::benchmark_data_smithi055_311528_object37117:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:16:11.912 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 452.30600110799998, 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.962840377, 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108080 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:11.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544658+0000", 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9589999999999998e-06 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544663+0000", 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8069999999999999e-06 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544667+0000", 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1790000000000001e-06 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581992+0000", 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037325061999999999 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581997+0000", 2023-09-23T18:16:11.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5480000000000003e-06 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487637+0000", 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9056401429999998 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487642+0000", 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:16:11.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.503810+0000", 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.01616774 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.503955+0000", 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000145291 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504156+0000", 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020058200000000001 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507170+0000", 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003013511 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507182+0000", 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.218e-05 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507234+0000", 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2349000000000003e-05 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507288+0000", 2023-09-23T18:16:11.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4307e-05 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507497+0000", 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000208233 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108094 7.1 7:819d9c99:::benchmark_data_smithi055_311528_object37131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:16:11.921 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 452.082641626, 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7316361810000001, 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108094 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768015+0000", 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768019+0000", 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3989999999999997e-06 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768032+0000", 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2707e-05 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768037+0000", 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2490000000000001e-06 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768081+0000", 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3421000000000003e-05 2023-09-23T18:16:11.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768093+0000", 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1989e-05 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.479295+0000", 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71120236800000003 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.479313+0000", 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8434e-05 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492409+0000", 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.013095748 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.927 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492585+0000", 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000176007 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492874+0000", 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00028859600000000001 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.497854+0000", 2023-09-23T18:16:11.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049804380000000002 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499314+0000", 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014593449999999999 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499349+0000", 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5802e-05 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499422+0000", 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2278000000000002e-05 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499652+0000", 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00023028400000000001 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108097 7.21 7:87bce1cb:::benchmark_data_smithi055_311528_object37134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 451.35340063500001, 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0105963070000001, 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108097 2023-09-23T18:16:11.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497255+0000", 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497262+0000", 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.029e-06 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497275+0000", 2023-09-23T18:16:11.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3349e-05 2023-09-23T18:16:11.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497284+0000", 2023-09-23T18:16:11.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3639999999999999e-06 2023-09-23T18:16:11.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497329+0000", 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5105000000000003e-05 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497335+0000", 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8989999999999998e-06 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487661+0000", 2023-09-23T18:16:11.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99032663499999996 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487667+0000", 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.446e-06 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504977+0000", 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.017310079 2023-09-23T18:16:11.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505119+0000", 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000142707 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505315+0000", 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019584499999999999 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507193+0000", 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001877213 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507527+0000", 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00033465300000000002 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507614+0000", 2023-09-23T18:16:11.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7287000000000006e-05 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507647+0000", 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2481999999999998e-05 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507853+0000", 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000206092 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.939 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108108 7.21 7:860e36b6:::benchmark_data_smithi055_311528_object37145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 451.30240805599999, 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.963000675, 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.940 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108108 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548248+0000", 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548257+0000", 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1549999999999996e-06 2023-09-23T18:16:11.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548278+0000", 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1229e-05 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548286+0000", 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0150000000000003e-06 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548337+0000", 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1131999999999998e-05 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548346+0000", 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8070000000000006e-06 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487685+0000", 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.939339378 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487691+0000", 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7100000000000004e-06 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505973+0000", 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018282141 2023-09-23T18:16:11.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506088+0000", 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011443299999999999 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506199+0000", 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011132 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507536+0000", 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013369 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507578+0000", 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1971000000000002e-05 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.510980+0000", 2023-09-23T18:16:11.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034023870000000002 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.511029+0000", 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9064999999999999e-05 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.511250+0000", 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022076600000000001 2023-09-23T18:16:11.948 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:129162 8.2 8:454f52aa:::benchmark_data_smithi055_346788_object18570:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:28.516171+0000", 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 283.33448652099997, 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.068282459, 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.949 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129162 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516171+0000", 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516171+0000", 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516170+0000", 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516174+0000", 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3680000000000004e-06 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516183+0000", 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9120000000000004e-06 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516187+0000", 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2209999999999999e-06 2023-09-23T18:16:11.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516232+0000", 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4530000000000002e-05 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516388+0000", 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000156572 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516568+0000", 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000179927 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.523118+0000", 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006549933 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.528581+0000", 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054625460000000004 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.546371+0000", 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017789800000000001 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.546381+0000", 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0311e-05 2023-09-23T18:16:11.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.584362+0000", 2023-09-23T18:16:11.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.037981313 2023-09-23T18:16:11.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.584453+0000", 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1094999999999994e-05 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:129175 8.2 8:4646f8c9:::benchmark_data_smithi055_346788_object18583:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:28.554311+0000", 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 283.29634683, 2023-09-23T18:16:11.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041236976, 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129175 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554311+0000", 2023-09-23T18:16:11.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554311+0000", 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554309+0000", 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554315+0000", 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1929999999999998e-06 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554328+0000", 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2975e-05 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554334+0000", 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3580000000000002e-06 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554380+0000", 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6189000000000001e-05 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554387+0000", 2023-09-23T18:16:11.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8369999999999999e-06 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.585150+0000", 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0307628820000001 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.585231+0000", 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1168000000000006e-05 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.585373+0000", 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014207099999999999 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.589993+0000", 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046203149999999998 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.590043+0000", 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0148000000000002e-05 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.594330+0000", 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0042865430000000003 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.594380+0000", 2023-09-23T18:16:11.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9521000000000002e-05 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.595548+0000", 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001168012 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:129177 8.2 8:4d02a5a8:::benchmark_data_smithi055_346788_object18585:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T18:16:11.966 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:28.562962+0000", 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 283.28769548000002, 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.033361969, 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129177 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562962+0000", 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562962+0000", 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562960+0000", 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562967+0000", 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8240000000000002e-06 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562981+0000", 2023-09-23T18:16:11.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4100999999999999e-05 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562986+0000", 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2780000000000001e-06 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.563035+0000", 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8353000000000003e-05 2023-09-23T18:16:11.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.563046+0000", 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1547e-05 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.587649+0000", 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0246025240000001 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.587767+0000", 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011790000000000001 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.587856+0000", 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9696000000000003e-05 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.590055+0000", 2023-09-23T18:16:11.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021986890000000002 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.590123+0000", 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8114000000000005e-05 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.595596+0000", 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054732649999999997 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.595635+0000", 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8549e-05 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.596324+0000", 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000688907 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177038 8.2 8:4ced54d7:::benchmark_data_smithi055_346788_object66446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 220.575529141, 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1583805069999999, 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:11.975 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177038 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275127+0000", 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275133+0000", 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5910000000000001e-06 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275146+0000", 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2302e-05 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275163+0000", 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7689e-05 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.282482+0000", 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0073186709999999997 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.282498+0000", 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5841999999999998e-05 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430240+0000", 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1477423680000001 2023-09-23T18:16:11.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430343+0000", 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102289 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5,6", 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430425+0000", 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2150000000000005e-05 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.432859+0000", 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024338519999999998 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.432958+0000", 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9440999999999999e-05 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433088+0000", 2023-09-23T18:16:11.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013031100000000001 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433396+0000", 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00030761699999999999 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433437+0000", 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1242999999999998e-05 2023-09-23T18:16:11.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433509+0000", 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1559999999999999e-05 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213401 8.e 8:7ecd3558:::benchmark_data_smithi055_346788_object102809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:16:11.983 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 171.607039297, 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0290741050000001, 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213401 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243620+0000", 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9649999999999998e-06 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243623+0000", 2023-09-23T18:16:11.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6179999999999999e-06 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243626+0000", 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.294e-06 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243677+0000", 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1668000000000001e-05 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243686+0000", 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7730000000000005e-06 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.235935+0000", 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99224838100000001 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.236043+0000", 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108855 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.236169+0000", 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012573700000000001 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.253075+0000", 2023-09-23T18:16:11.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016906304 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272511+0000", 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019435859 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272626+0000", 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115215 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272667+0000", 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0590000000000003e-05 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272692+0000", 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5111999999999999e-05 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121636 9.f 9:f1d044dd:::benchmark_data_smithi055_386467_object16976:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.695760+0000", 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 35.154897626999997, 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0409455569999999, 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:11.992 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121636 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695760+0000", 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695760+0000", 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:11.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695759+0000", 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695762+0000", 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4040000000000002e-06 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695768+0000", 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1099999999999999e-06 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695772+0000", 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0579999999999999e-06 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.698805+0000", 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030330489999999999 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.698810+0000", 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5439999999999998e-06 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.717639+0000", 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018828426 2023-09-23T18:16:11.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.718020+0000", 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00038104099999999999 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.718155+0000", 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000134618 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.725686+0000", 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075315429999999999 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.735765+0000", 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010079210999999999 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.736527+0000", 2023-09-23T18:16:11.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00076177799999999996 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.736593+0000", 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6124000000000006e-05 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.736705+0000", 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011207 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:11.999 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:122388 9.f 9:fdc2bcfb:::benchmark_data_smithi055_386467_object17728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 23.589479090000001, 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0337659729999999, 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:12.000 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122388 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:12.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261180+0000", 2023-09-23T18:16:12.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.057e-06 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261187+0000", 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2559999999999996e-06 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261191+0000", 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4089999999999999e-06 2023-09-23T18:16:12.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261250+0000", 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9678e-05 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261258+0000", 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3209999999999999e-06 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.278415+0000", 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01715761 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.278423+0000", 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8089999999999993e-06 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.287981+0000", 2023-09-23T18:16:12.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0095577870000001 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.288127+0000", 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014562099999999999 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.288257+0000", 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000129982 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294755+0000", 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064986159999999996 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294764+0000", 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4689999999999997e-06 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:12.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294789+0000", 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5245e-05 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294838+0000", 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9027999999999999e-05 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294944+0000", 2023-09-23T18:16:12.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106378 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:122391 9.f 9:fd5978e6:::benchmark_data_smithi055_386467_object17731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:48.272754+0000", 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 23.577903112000001, 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0225196940000001, 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:12.009 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122391 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272754+0000", 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272754+0000", 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272753+0000", 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272758+0000", 2023-09-23T18:16:12.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9169999999999997e-06 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.275901+0000", 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031430659999999999 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.275910+0000", 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0569999999999999e-06 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.279182+0000", 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032713920000000001 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.279188+0000", 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.302e-06 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:12.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.290722+0000", 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.011533781 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.290834+0000", 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011197299999999999 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.290966+0000", 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132531 2023-09-23T18:16:12.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294997+0000", 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040308899999999996 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295070+0000", 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3144000000000003e-05 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295128+0000", 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8236000000000003e-05 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295169+0000", 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0176000000000003e-05 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295274+0000", 2023-09-23T18:16:12.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010541 2023-09-23T18:16:12.017 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:12.017 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:12.017 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:12.017 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:12.017 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:12.017 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:16:12.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:12.018 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:12.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:12.019 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:12.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:12.195 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:12.305 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:12.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:12.316 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:12.350 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:16:12.395 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:12.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:12.512 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:12.519 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:12.533 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:12.555 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:12.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:12 smithi099 ceph-mon[167076]: pgmap v1238: 119 pgs: 12 peering, 9 remapped+peering, 11 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 38 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 850 op/s; 34259/205500 objects degraded (16.671%); 12943/205500 objects misplaced (6.298%); 20 MiB/s, 310 objects/s recovering 2023-09-23T18:16:12.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:12 smithi099 ceph-mon[167076]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:12 smithi099 ceph-mon[167076]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:12 smithi099 ceph-mon[167076]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:12 smithi099 ceph-mon[167076]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:12 smithi099 ceph-mon[167076]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:12 smithi099 ceph-mon[167076]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:12 smithi099 ceph-mon[167076]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:12 smithi099 ceph-mon[167076]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:12 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 10 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-09-23T18:16:12.582 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:12.598 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:12.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:12.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:12.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[184174]: pgmap v1238: 119 pgs: 12 peering, 9 remapped+peering, 11 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 38 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 850 op/s; 34259/205500 objects degraded (16.671%); 12943/205500 objects misplaced (6.298%); 20 MiB/s, 310 objects/s recovering 2023-09-23T18:16:12.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[184174]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[184174]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[184174]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[184174]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[184174]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[184174]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.610 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[184174]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[184174]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.611 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 10 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-09-23T18:16:12.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[179668]: pgmap v1238: 119 pgs: 12 peering, 9 remapped+peering, 11 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 38 active+clean; 4.2 GiB data, 19 GiB used, 696 GiB / 715 GiB avail; 10 KiB/s rd, 27 KiB/s wr, 850 op/s; 34259/205500 objects degraded (16.671%); 12943/205500 objects misplaced (6.298%); 20 MiB/s, 310 objects/s recovering 2023-09-23T18:16:12.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[179668]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[179668]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[179668]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[179668]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[179668]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[179668]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.611 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[179668]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.612 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[179668]: failed to encode map e513 with expected crc 2023-09-23T18:16:12.612 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:12 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 10 pgs inactive, 11 pgs peering (PG_AVAILABILITY) 2023-09-23T18:16:12.612 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:12.643 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:12.672 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:12.672 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:16:12.672 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:12.672 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:16:12.672 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:16:12.672 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:16:12.673 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:16:12.673 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:16:12.673 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:16:12.704 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:12.781 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:12.856 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:12.862 INFO:tasks.thrashosds.thrasher:in_osds: [7, 1, 0, 4, 2, 3, 6, 5] out_osds: [] dead_osds: [1] live_osds: [0, 4, 6, 7, 3, 5, 2] 2023-09-23T18:16:12.862 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:16:12.863 INFO:tasks.thrashosds.thrasher:Reweighting osd 2 to 0.24321184706946283 2023-09-23T18:16:12.863 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 0.24321184706946283 2023-09-23T18:16:12.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:12.865 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:12.965 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:12.999 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:13.054 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:13.059 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:13.059 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:13.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:13.060 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:13.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:13.237 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:13.298 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:16:13.311 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:13.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:13.315 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:13.416 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:13.487 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:13 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/56877134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:13.487 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 34259/205500 objects degraded (16.671%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:13.491 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:13.585 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:13.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:13.596 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:13.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:13 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/56877134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:13.654 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 34259/205500 objects degraded (16.671%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:13.655 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:13 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/56877134' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:13.655 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 34259/205500 objects degraded (16.671%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:13.676 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:13.697 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:13.737 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:13.782 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:13.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:13.786 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:13.887 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:13.974 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:14.180 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:14.185 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:14.185 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:14.286 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:14.319 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:14.417 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:16:14.469 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:14.471 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:14 smithi055 ceph-mon[184174]: pgmap v1240: 119 pgs: 12 peering, 9 remapped+peering, 11 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 38 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 8.8 KiB/s rd, 23 KiB/s wr, 654 op/s; 34259/205500 objects degraded (16.671%); 12943/205500 objects misplaced (6.298%); 17 MiB/s, 259 objects/s recovering 2023-09-23T18:16:14.471 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2999533005' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:14.471 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:14 smithi055 ceph-mon[179668]: pgmap v1240: 119 pgs: 12 peering, 9 remapped+peering, 11 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 38 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 8.8 KiB/s rd, 23 KiB/s wr, 654 op/s; 34259/205500 objects degraded (16.671%); 12943/205500 objects misplaced (6.298%); 17 MiB/s, 259 objects/s recovering 2023-09-23T18:16:14.471 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2999533005' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:14.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:14.472 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:14.534 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:14 smithi099 ceph-mon[167076]: pgmap v1240: 119 pgs: 12 peering, 9 remapped+peering, 11 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 38 active+clean; 4.2 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 8.8 KiB/s rd, 23 KiB/s wr, 654 op/s; 34259/205500 objects degraded (16.671%); 12943/205500 objects misplaced (6.298%); 17 MiB/s, 259 objects/s recovering 2023-09-23T18:16:14.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2999533005' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:14.573 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:14.645 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:14.743 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:14.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:14.749 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:14.850 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:14.883 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:14.885 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:16:14.885 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:16:14.932 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:14.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:14.937 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:15.038 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:15.060 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:16:15.110 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:15.259 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:15.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:15.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:15.369 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:16:15.370 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:15.384 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:15.385 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:15.385 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[184174]: failed to encode map e514 with expected crc 2023-09-23T18:16:15.385 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[184174]: failed to encode map e514 with expected crc 2023-09-23T18:16:15.402 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:15.443 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:15.446 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:15.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:15.547 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:15.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:15 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:15.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:15 smithi099 ceph-mon[167076]: failed to encode map e514 with expected crc 2023-09-23T18:16:15.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:15 smithi099 ceph-mon[167076]: failed to encode map e514 with expected crc 2023-09-23T18:16:15.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2999533005' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:15.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:15 smithi099 ceph-mon[167076]: osdmap e514: 8 total, 7 up, 8 in 2023-09-23T18:16:15.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:15 smithi099 ceph-mon[167076]: failed to encode map e514 with expected crc 2023-09-23T18:16:15.654 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:15.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2999533005' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:15.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[184174]: osdmap e514: 8 total, 7 up, 8 in 2023-09-23T18:16:15.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[184174]: failed to encode map e514 with expected crc 2023-09-23T18:16:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[179668]: failed to encode map e514 with expected crc 2023-09-23T18:16:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[179668]: failed to encode map e514 with expected crc 2023-09-23T18:16:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2999533005' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:15.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[179668]: osdmap e514: 8 total, 7 up, 8 in 2023-09-23T18:16:15.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:15 smithi055 ceph-mon[179668]: failed to encode map e514 with expected crc 2023-09-23T18:16:15.738 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:15.742 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:15.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:15.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:15.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:15.943 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:16.062 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:16.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:16.071 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:16.119 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:16:16.172 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:16.207 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:16.434 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.2 to 0.243212 (3e43) 2023-09-23T18:16:16.460 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:16 smithi055 ceph-mon[179668]: pgmap v1242: 119 pgs: 12 peering, 4 remapped+peering, 15 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 39 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 7.2 KiB/s rd, 19 KiB/s wr, 563 op/s; 33369/202098 objects degraded (16.511%); 12943/202098 objects misplaced (6.404%); 14 MiB/s, 214 objects/s recovering 2023-09-23T18:16:16.460 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3508443400' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.24321184706946283}]: dispatch 2023-09-23T18:16:16.461 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:16 smithi055 ceph-mon[184174]: pgmap v1242: 119 pgs: 12 peering, 4 remapped+peering, 15 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 39 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 7.2 KiB/s rd, 19 KiB/s wr, 563 op/s; 33369/202098 objects degraded (16.511%); 12943/202098 objects misplaced (6.404%); 14 MiB/s, 214 objects/s recovering 2023-09-23T18:16:16.461 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3508443400' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.24321184706946283}]: dispatch 2023-09-23T18:16:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:16 smithi099 ceph-mon[167076]: pgmap v1242: 119 pgs: 12 peering, 4 remapped+peering, 15 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 39 active+clean; 4.1 GiB data, 19 GiB used, 697 GiB / 715 GiB avail; 7.2 KiB/s rd, 19 KiB/s wr, 563 op/s; 33369/202098 objects degraded (16.511%); 12943/202098 objects misplaced (6.404%); 14 MiB/s, 214 objects/s recovering 2023-09-23T18:16:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3508443400' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 0.24321184706946283}]: dispatch 2023-09-23T18:16:16.839 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:16:17.101 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:17.103 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:17.104 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:17.204 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:17.232 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:17.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[179668]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3508443400' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.24321184706946283}]': finished 2023-09-23T18:16:17.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[179668]: osdmap e515: 8 total, 7 up, 8 in 2023-09-23T18:16:17.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[179668]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[179668]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[179668]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[179668]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[184174]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3508443400' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.24321184706946283}]': finished 2023-09-23T18:16:17.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[184174]: osdmap e515: 8 total, 7 up, 8 in 2023-09-23T18:16:17.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[184174]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[184174]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[184174]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:17 smithi055 ceph-mon[184174]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:17 smithi099 ceph-mon[167076]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3508443400' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 0.24321184706946283}]': finished 2023-09-23T18:16:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:17 smithi099 ceph-mon[167076]: osdmap e515: 8 total, 7 up, 8 in 2023-09-23T18:16:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:17 smithi099 ceph-mon[167076]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:17 smithi099 ceph-mon[167076]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:17 smithi099 ceph-mon[167076]: failed to encode map e515 with expected crc 2023-09-23T18:16:17.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:17 smithi099 ceph-mon[167076]: failed to encode map e515 with expected crc 2023-09-23T18:16:18.694 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:18 smithi099 ceph-mon[167076]: pgmap v1244: 119 pgs: 19 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 51 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 32011/190938 objects degraded (16.765%); 11956/190938 objects misplaced (6.262%); 43 KiB/s, 4 objects/s recovering 2023-09-23T18:16:18.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:18 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 11 pgs peering) 2023-09-23T18:16:18.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:18 smithi099 ceph-mon[167076]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:18 smithi099 ceph-mon[167076]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:18 smithi099 ceph-mon[167076]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:18 smithi099 ceph-mon[167076]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:18 smithi099 ceph-mon[167076]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:18 smithi099 ceph-mon[167076]: osdmap e516: 8 total, 7 up, 8 in 2023-09-23T18:16:18.695 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 32011/190938 objects degraded (16.765%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:18.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[179668]: pgmap v1244: 119 pgs: 19 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 51 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 32011/190938 objects degraded (16.765%); 11956/190938 objects misplaced (6.262%); 43 KiB/s, 4 objects/s recovering 2023-09-23T18:16:18.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 11 pgs peering) 2023-09-23T18:16:18.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[179668]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[179668]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[179668]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[179668]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[179668]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[179668]: osdmap e516: 8 total, 7 up, 8 in 2023-09-23T18:16:18.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 32011/190938 objects degraded (16.765%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:18.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[184174]: pgmap v1244: 119 pgs: 19 active+clean+remapped, 35 active+undersized, 14 active+undersized+degraded, 51 active+clean; 3.9 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 808 op/s; 32011/190938 objects degraded (16.765%); 11956/190938 objects misplaced (6.262%); 43 KiB/s, 4 objects/s recovering 2023-09-23T18:16:18.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 10 pgs inactive, 11 pgs peering) 2023-09-23T18:16:18.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[184174]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[184174]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[184174]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[184174]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[184174]: failed to encode map e516 with expected crc 2023-09-23T18:16:18.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[184174]: osdmap e516: 8 total, 7 up, 8 in 2023-09-23T18:16:18.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 32011/190938 objects degraded (16.765%), 14 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:19.445 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:16:19.445 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:16:19.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: osdmap e517: 8 total, 7 up, 8 in 2023-09-23T18:16:19.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: 9.2 starting backfill to osd.6 from (0'0,0'0] MAX to 514'9275 2023-09-23T18:16:19.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[184174]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: osdmap e517: 8 total, 7 up, 8 in 2023-09-23T18:16:19.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: 9.2 starting backfill to osd.6 from (0'0,0'0] MAX to 514'9275 2023-09-23T18:16:19.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.753 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:19 smithi055 ceph-mon[179668]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.780 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:19.783 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:19.784 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: osdmap e517: 8 total, 7 up, 8 in 2023-09-23T18:16:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: 9.2 starting backfill to osd.6 from (0'0,0'0] MAX to 514'9275 2023-09-23T18:16:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:19 smithi099 ceph-mon[167076]: failed to encode map e517 with expected crc 2023-09-23T18:16:19.884 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:19.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:16:19.918 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:20.317 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:20.323 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:20.324 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:20.424 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:20.501 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:20.601 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:20.609 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:20.610 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:20.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:16:20] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.43.0" 2023-09-23T18:16:20.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[179668]: pgmap v1247: 119 pgs: 2 remapped+peering, 19 active+clean+remapped, 34 active+undersized, 13 active+undersized+degraded, 51 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 27262/187713 objects degraded (14.523%); 11956/187713 objects misplaced (6.369%); 55 KiB/s, 5 objects/s recovering 2023-09-23T18:16:20.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[179668]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[179668]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[179668]: osdmap e518: 8 total, 7 up, 8 in 2023-09-23T18:16:20.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[179668]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[179668]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[179668]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[179668]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[179668]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[179668]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[184174]: pgmap v1247: 119 pgs: 2 remapped+peering, 19 active+clean+remapped, 34 active+undersized, 13 active+undersized+degraded, 51 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 27262/187713 objects degraded (14.523%); 11956/187713 objects misplaced (6.369%); 55 KiB/s, 5 objects/s recovering 2023-09-23T18:16:20.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[184174]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[184174]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[184174]: osdmap e518: 8 total, 7 up, 8 in 2023-09-23T18:16:20.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[184174]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[184174]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[184174]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[184174]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[184174]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:20 smithi055 ceph-mon[184174]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.711 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:20.760 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:20 smithi099 ceph-mon[167076]: pgmap v1247: 119 pgs: 2 remapped+peering, 19 active+clean+remapped, 34 active+undersized, 13 active+undersized+degraded, 51 active+clean; 3.8 GiB data, 18 GiB used, 697 GiB / 715 GiB avail; 0 B/s wr, 1.03k op/s; 27262/187713 objects degraded (14.523%); 11956/187713 objects misplaced (6.369%); 55 KiB/s, 5 objects/s recovering 2023-09-23T18:16:20.760 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:20 smithi099 ceph-mon[167076]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.760 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:20 smithi099 ceph-mon[167076]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:20 smithi099 ceph-mon[167076]: osdmap e518: 8 total, 7 up, 8 in 2023-09-23T18:16:20.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:20 smithi099 ceph-mon[167076]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:20 smithi099 ceph-mon[167076]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:20 smithi099 ceph-mon[167076]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:20 smithi099 ceph-mon[167076]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:20 smithi099 ceph-mon[167076]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:20 smithi099 ceph-mon[167076]: failed to encode map e518 with expected crc 2023-09-23T18:16:20.769 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:20.791 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:16:20.838 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:20.846 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:20.847 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:20.948 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:21.007 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:21.324 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:21.766 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[179668]: failed to encode map e519 with expected crc 2023-09-23T18:16:21.766 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[179668]: osdmap e519: 8 total, 7 up, 8 in 2023-09-23T18:16:21.766 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[179668]: failed to encode map e519 with expected crc 2023-09-23T18:16:21.766 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[179668]: failed to encode map e519 with expected crc 2023-09-23T18:16:21.767 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:21.767 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:21.767 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[184174]: failed to encode map e519 with expected crc 2023-09-23T18:16:21.767 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[184174]: osdmap e519: 8 total, 7 up, 8 in 2023-09-23T18:16:21.767 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[184174]: failed to encode map e519 with expected crc 2023-09-23T18:16:21.767 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[184174]: failed to encode map e519 with expected crc 2023-09-23T18:16:21.767 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:21.768 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:21 smithi099 ceph-mon[167076]: failed to encode map e519 with expected crc 2023-09-23T18:16:21.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:21 smithi099 ceph-mon[167076]: osdmap e519: 8 total, 7 up, 8 in 2023-09-23T18:16:21.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:21 smithi099 ceph-mon[167076]: failed to encode map e519 with expected crc 2023-09-23T18:16:21.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:21 smithi099 ceph-mon[167076]: failed to encode map e519 with expected crc 2023-09-23T18:16:21.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:21.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:22.097 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:22.104 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:22.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:22.200 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:22.206 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:22.229 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:22.253 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:22.260 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:22.304 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:22.340 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:22.354 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:22.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:22.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:22.366 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:22.402 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:22.403 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:16:22.403 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:22.403 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:16:22.403 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:16:22.403 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:16:22.403 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:16:22.403 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:16:22.403 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:16:22.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:22.525 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:22 smithi099 ceph-mon[167076]: pgmap v1250: 119 pgs: 2 remapped+peering, 28 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 26482/181191 objects degraded (14.616%); 11368/181191 objects misplaced (6.274%); 3.0 MiB/s, 12 objects/s recovering 2023-09-23T18:16:22.525 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/645746902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:22.541 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:22.827 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:22 smithi055 ceph-mon[179668]: pgmap v1250: 119 pgs: 2 remapped+peering, 28 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 26482/181191 objects degraded (14.616%); 11368/181191 objects misplaced (6.274%); 3.0 MiB/s, 12 objects/s recovering 2023-09-23T18:16:22.827 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/645746902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:22.827 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:22 smithi055 ceph-mon[184174]: pgmap v1250: 119 pgs: 2 remapped+peering, 28 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.7 GiB data, 18 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 812 op/s; 26482/181191 objects degraded (14.616%); 11368/181191 objects misplaced (6.274%); 3.0 MiB/s, 12 objects/s recovering 2023-09-23T18:16:22.827 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/645746902' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:22.904 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:22.905 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:22.958 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:22.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:22.972 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:23.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:23.154 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:23.232 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:23.239 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:23.240 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:23.340 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:23.410 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:23.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:23.733 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:23.734 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:23.815 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 26482/181191 objects degraded (14.616%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:23.835 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:23.868 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 26482/181191 objects degraded (14.616%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:23.869 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 26482/181191 objects degraded (14.616%), 12 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:23.874 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:23.917 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:23.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:23.921 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:24.021 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:24.093 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:24.172 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:24.180 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:24.180 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:24.281 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:24.357 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:24.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:24.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:24.439 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:24.540 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:24.574 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:16:24.574 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:16:24.574 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:16:24.574 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:16:24.574 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.574 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136101 8.2 8:4a1fd025:::benchmark_data_smithi055_346788_object25509:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.574 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509243+0000", 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06056515900002, 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0685515059999999, 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136101 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.575 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509243+0000", 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509243+0000", 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509242+0000", 2023-09-23T18:16:24.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:24.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.577 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.577 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.577 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509248+0000", 2023-09-23T18:16:24.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4110000000000001e-06 2023-09-23T18:16:24.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.577 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.577 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.577 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509259+0000", 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0757000000000001e-05 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509265+0000", 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.874e-06 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509361+0000", 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5877000000000004e-05 2023-09-23T18:16:24.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509370+0000", 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4720000000000001e-06 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.547253+0000", 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.037882701 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.580 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.548809+0000", 2023-09-23T18:16:24.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015562060000000001 2023-09-23T18:16:24.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.580 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.548982+0000", 2023-09-23T18:16:24.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017251899999999999 2023-09-23T18:16:24.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.561547+0000", 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012565092999999999 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.576752+0000", 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015205023999999999 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.576807+0000", 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5451999999999997e-05 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.576848+0000", 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1063999999999999e-05 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.577795+0000", 2023-09-23T18:16:24.582 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00094642999999999995 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136113 8.6 8:698e62bb:::benchmark_data_smithi055_346788_object25521:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509277+0000", 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06053107000002, 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.138360233, 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.583 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136113 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509277+0000", 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509277+0000", 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509277+0000", 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509280+0000", 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8490000000000002e-06 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509287+0000", 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1720000000000001e-06 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509291+0000", 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8519999999999997e-06 2023-09-23T18:16:24.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509362+0000", 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0857000000000003e-05 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509370+0000", 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.755e-06 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.553199+0000", 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0438288469999999 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.553903+0000", 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00070390000000000003 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.580852+0000", 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.026948666 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.580930+0000", 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8088000000000002e-05 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.581491+0000", 2023-09-23T18:16:24.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00056076800000000001 2023-09-23T18:16:24.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.621100+0000", 2023-09-23T18:16:24.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.039608947999999998 2023-09-23T18:16:24.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.625239+0000", 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041396810000000001 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.646068+0000", 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020828435999999999 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.646108+0000", 2023-09-23T18:16:24.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9777999999999998e-05 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.647638+0000", 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0015301290000000001 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136115 8.2 8:4e13f92c:::benchmark_data_smithi055_346788_object25523:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.592 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509302+0000", 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06050696099999, 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0748453979999999, 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136115 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509302+0000", 2023-09-23T18:16:24.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509302+0000", 2023-09-23T18:16:24.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509301+0000", 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509304+0000", 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5000000000000002e-06 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509310+0000", 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6409999999999996e-06 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509314+0000", 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.065e-06 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509393+0000", 2023-09-23T18:16:24.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8400999999999997e-05 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509399+0000", 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8590000000000001e-06 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.555775+0000", 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.046376787 2023-09-23T18:16:24.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.555913+0000", 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000137124 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.556072+0000", 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015936100000000001 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.599 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.561561+0000", 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054889159999999999 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.576781+0000", 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015219939 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.577859+0000", 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010779780000000001 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.577914+0000", 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4926999999999998e-05 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584147+0000", 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00623326 2023-09-23T18:16:24.601 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136138 8.2 8:4ee3c91a:::benchmark_data_smithi055_346788_object25546:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509319+0000", 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06048950100001, 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.076255135, 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.602 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136138 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509319+0000", 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509319+0000", 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509319+0000", 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509321+0000", 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2460000000000002e-06 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509327+0000", 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0290000000000002e-06 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509331+0000", 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7850000000000002e-06 2023-09-23T18:16:24.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509441+0000", 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011047100000000001 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509447+0000", 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9259999999999997e-06 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.557856+0000", 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0484084789999999 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.557995+0000", 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013896899999999999 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.558139+0000", 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014439800000000001 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.563720+0000", 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0055811599999999999 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584195+0000", 2023-09-23T18:16:24.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020474598 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584414+0000", 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00021974000000000001 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584502+0000", 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7262999999999994e-05 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.585574+0000", 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010724269999999999 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136150 8.6 8:62b2afb4:::benchmark_data_smithi055_346788_object25558:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509414+0000", 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06039456600001, 2023-09-23T18:16:24.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1723984599999999, 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136150 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509414+0000", 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509414+0000", 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509413+0000", 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509428+0000", 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4487999999999999e-05 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509437+0000", 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0219999999999993e-06 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509443+0000", 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8320000000000002e-06 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511001+0000", 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001558116 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511008+0000", 2023-09-23T18:16:24.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7409999999999992e-06 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.617716+0000", 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1067071230000001 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.620358+0000", 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026421690000000002 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.650403+0000", 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.030045479 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.650475+0000", 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1815000000000003e-05 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.650585+0000", 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000109768 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.673499+0000", 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.022914443 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680194+0000", 2023-09-23T18:16:24.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0066950300000000003 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680306+0000", 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112019 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680358+0000", 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1526000000000001e-05 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.681812+0000", 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001454601 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136153 8.2 8:4ed9a1c7:::benchmark_data_smithi055_346788_object25561:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509454+0000", 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06035451399998, 2023-09-23T18:16:24.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.076902005, 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136153 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509454+0000", 2023-09-23T18:16:24.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509454+0000", 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509454+0000", 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509456+0000", 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6120000000000001e-06 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509462+0000", 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3029999999999996e-06 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509467+0000", 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4329999999999997e-06 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510885+0000", 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014182279999999999 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510893+0000", 2023-09-23T18:16:24.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4749999999999996e-06 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.559782+0000", 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0488890849999999 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.559879+0000", 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7108000000000006e-05 2023-09-23T18:16:24.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.560003+0000", 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012399500000000001 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.566016+0000", 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006012982 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584256+0000", 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018239986999999999 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.585620+0000", 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001363815 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.626 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.585666+0000", 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6289999999999999e-05 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586356+0000", 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00069014300000000005 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.627 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136169 8.6 8:6fa67d1e:::benchmark_data_smithi055_346788_object25577:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509476+0000", 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06033293000002, 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1734557620000001, 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136169 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.628 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509476+0000", 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509476+0000", 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509475+0000", 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509478+0000", 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4719999999999998e-06 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509484+0000", 2023-09-23T18:16:24.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4590000000000003e-06 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509488+0000", 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1189999999999999e-06 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511033+0000", 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001544595 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511040+0000", 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6660000000000002e-06 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.621116+0000", 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1100768670000001 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:24.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.622276+0000", 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001159579 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.655074+0000", 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032797699 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.655139+0000", 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5399000000000002e-05 2023-09-23T18:16:24.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.655225+0000", 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5869000000000001e-05 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680207+0000", 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.024982244000000001 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680275+0000", 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7720999999999998e-05 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.681878+0000", 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0016034949999999999 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.635 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.681930+0000", 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.151e-05 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.682931+0000", 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001001486 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.636 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136190 8.2 8:4369bb6d:::benchmark_data_smithi055_346788_object25598:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509493+0000", 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06031597600003, 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.077617552, 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136190 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.637 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509493+0000", 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509493+0000", 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509492+0000", 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509494+0000", 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1490000000000001e-06 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509504+0000", 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0132000000000001e-05 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509509+0000", 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1239999999999998e-06 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510918+0000", 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014091360000000001 2023-09-23T18:16:24.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510924+0000", 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.767e-06 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.561593+0000", 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0506690839999999 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.561703+0000", 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110209 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.561829+0000", 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000126112 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.569780+0000", 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0079513919999999998 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584300+0000", 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014519776 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586489+0000", 2023-09-23T18:16:24.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021893910000000002 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586532+0000", 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2703000000000001e-05 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.587110+0000", 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00057794699999999999 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.644 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136192 8.2 8:4a109c9f:::benchmark_data_smithi055_346788_object25600:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509514+0000", 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06029473199999, 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0789546750000001, 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.645 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136192 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509514+0000", 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509514+0000", 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509513+0000", 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509516+0000", 2023-09-23T18:16:24.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2869999999999999e-06 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509522+0000", 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2530000000000001e-06 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509526+0000", 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9360000000000001e-06 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510954+0000", 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001427928 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510961+0000", 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8460000000000002e-06 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.563751+0000", 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0527905870000001 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.563908+0000", 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015658499999999999 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.564053+0000", 2023-09-23T18:16:24.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014559999999999999 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.573474+0000", 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0094202969999999994 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584337+0000", 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010863462000000001 2023-09-23T18:16:24.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.587213+0000", 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028762839999999998 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.587261+0000", 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7006999999999999e-05 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.652 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.588468+0000", 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012079829999999999 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136204 8.2 8:4e83b308:::benchmark_data_smithi055_346788_object25612:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509534+0000", 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06027410199999, 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.079762712, 2023-09-23T18:16:24.653 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136204 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509534+0000", 2023-09-23T18:16:24.654 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509534+0000", 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509534+0000", 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.655 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509536+0000", 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.131e-06 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509543+0000", 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8290000000000002e-06 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.656 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509547+0000", 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2899999999999996e-06 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510987+0000", 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00143958 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.510992+0000", 2023-09-23T18:16:24.657 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0849999999999996e-06 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.566056+0000", 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0550638720000001 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.566147+0000", 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1142000000000003e-05 2023-09-23T18:16:24.658 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.566279+0000", 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000131536 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.573479+0000", 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0072009359999999998 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.659 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.584378+0000", 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010898410000000001 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.588561+0000", 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041834999999999997 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.660 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.588615+0000", 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3668999999999997e-05 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.589297+0000", 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00068208699999999995 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.661 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136213 8.2 8:4a22a6f0:::benchmark_data_smithi055_346788_object25621:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509553+0000", 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06025566099999, 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0912685440000001, 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136213 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.662 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509553+0000", 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509553+0000", 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.663 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509553+0000", 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509561+0000", 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4090000000000003e-06 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.664 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509567+0000", 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.455e-06 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509571+0000", 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8539999999999999e-06 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511006+0000", 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001435232 2023-09-23T18:16:24.665 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511009+0000", 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.942e-06 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.568065+0000", 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0570560849999999 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.666 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.568177+0000", 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111777 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.568259+0000", 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1468e-05 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.667 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.575309+0000", 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.007049792 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586397+0000", 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011088694 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.599400+0000", 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013002575000000001 2023-09-23T18:16:24.668 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.599456+0000", 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6444999999999999e-05 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.600821+0000", 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013651760000000001 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.669 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136219 8.2 8:4dc26cdb:::benchmark_data_smithi055_346788_object25627:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509576+0000", 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06023236300001, 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0921537569999999, 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.670 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136219 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509576+0000", 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.671 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509576+0000", 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509576+0000", 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509578+0000", 2023-09-23T18:16:24.672 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2709999999999999e-06 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509584+0000", 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3219999999999998e-06 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509588+0000", 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5590000000000001e-06 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.673 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511027+0000", 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001439081 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511032+0000", 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2669999999999997e-06 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.674 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.675 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.569826+0000", 2023-09-23T18:16:24.675 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.058793485 2023-09-23T18:16:24.677 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.677 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.677 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.677 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.569971+0000", 2023-09-23T18:16:24.677 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014493399999999999 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.570110+0000", 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013977299999999999 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.575317+0000", 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052065820000000004 2023-09-23T18:16:24.678 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586429+0000", 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011111792000000001 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.600885+0000", 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014456115 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.679 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.600938+0000", 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2673999999999997e-05 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.601730+0000", 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00079229799999999998 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.680 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136223 8.2 8:4007e85f:::benchmark_data_smithi055_346788_object25631:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509592+0000", 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06021654, 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.092864678, 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.681 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136223 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509592+0000", 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509592+0000", 2023-09-23T18:16:24.682 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509592+0000", 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509594+0000", 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2239999999999998e-06 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.683 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509600+0000", 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0599999999999996e-06 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509606+0000", 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.614e-06 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.684 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511054+0000", 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001448335 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511059+0000", 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2320000000000001e-06 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.685 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.571978+0000", 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0609187710000001 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.572101+0000", 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012351800000000001 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.572190+0000", 2023-09-23T18:16:24.686 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8537999999999998e-05 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.575368+0000", 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031780010000000002 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.586458+0000", 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011089887999999999 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.687 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.601786+0000", 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015328664000000001 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.601836+0000", 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9604999999999997e-05 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.688 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.602457+0000", 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00062059400000000003 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136229 8.6 8:663dcb03:::benchmark_data_smithi055_346788_object25637:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509610+0000", 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06019844299999, 2023-09-23T18:16:24.689 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.174026851, 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136229 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509610+0000", 2023-09-23T18:16:24.690 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509610+0000", 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509610+0000", 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.691 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509612+0000", 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0449999999999999e-06 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509618+0000", 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5010000000000001e-06 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.692 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509623+0000", 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0300000000000001e-06 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511063+0000", 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014401259999999999 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511070+0000", 2023-09-23T18:16:24.693 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5390000000000004e-06 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.622337+0000", 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1112675249999999 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.623186+0000", 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00084855100000000001 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.694 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.659609+0000", 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.036422818000000003 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.659698+0000", 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9207000000000002e-05 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.695 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.659828+0000", 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012993100000000001 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680217+0000", 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020389131000000001 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.681851+0000", 2023-09-23T18:16:24.696 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001633497 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.683010+0000", 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001158912 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.683040+0000", 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0759000000000001e-05 2023-09-23T18:16:24.697 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.683637+0000", 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00059668899999999997 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136253 8.6 8:6977274d:::benchmark_data_smithi055_346788_object25661:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509628+0000", 2023-09-23T18:16:24.698 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06018020400001, 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1746242060000001, 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136253 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.699 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509628+0000", 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509628+0000", 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509628+0000", 2023-09-23T18:16:24.700 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509630+0000", 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1789999999999998e-06 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509636+0000", 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9769999999999997e-06 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.701 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509640+0000", 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1099999999999996e-06 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511101+0000", 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014610129999999999 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.702 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511108+0000", 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8689999999999998e-06 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.623221+0000", 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1121126079999999 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.623931+0000", 2023-09-23T18:16:24.703 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00071072500000000005 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.664256+0000", 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.040324454000000003 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.664348+0000", 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2176999999999995e-05 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.704 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.664491+0000", 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000142483 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.680225+0000", 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015734387999999998 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.705 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.682971+0000", 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027464070000000002 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.683697+0000", 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00072510999999999995 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.683723+0000", 2023-09-23T18:16:24.706 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6183999999999999e-05 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.684253+0000", 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00052985599999999999 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:136258 8.2 8:4d91b10c:::benchmark_data_smithi055_346788_object25666:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e404)", 2023-09-23T18:16:24.707 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:40.509644+0000", 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 284.06016455100001, 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.093560117, 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 136258 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.708 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.709 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509644+0000", 2023-09-23T18:16:24.709 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.709 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.709 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.710 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:24.710 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:16:24.710 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.710 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509644+0000", 2023-09-23T18:16:24.710 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509644+0000", 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509646+0000", 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9889999999999999e-06 2023-09-23T18:16:24.711 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509651+0000", 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8250000000000001e-06 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.509655+0000", 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5489999999999998e-06 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.712 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511128+0000", 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014725890000000001 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:40.511133+0000", 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4870000000000002e-06 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.713 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.573520+0000", 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.062386692 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.573661+0000", 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014144099999999999 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.573810+0000", 2023-09-23T18:16:24.714 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014895699999999999 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.577812+0000", 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040020100000000003 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.587171+0000", 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0093583950000000003 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.715 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.602503+0000", 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015332070999999999 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.602563+0000", 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0265e-05 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.716 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:41.603204+0000", 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00064115899999999998 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177019 8.9 8:935233eb:::benchmark_data_smithi055_346788_object66427:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.253339+0000", 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 233.31646967399999, 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1561469240000002, 2023-09-23T18:16:24.717 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177019 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253339+0000", 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.718 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253339+0000", 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253338+0000", 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.719 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253344+0000", 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2029999999999997e-06 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253359+0000", 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4806e-05 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.720 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253366+0000", 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9249999999999998e-06 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253432+0000", 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6327000000000004e-05 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253574+0000", 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014209300000000001 2023-09-23T18:16:24.721 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3", 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.253702+0000", 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012784100000000001 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.255719+0000", 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002016795 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.722 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.288188+0000", 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032469117999999998 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.288201+0000", 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2873e-05 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.298979+0000", 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0107786649999999 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.298990+0000", 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0479000000000001e-05 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.409403+0000", 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.110413163 2023-09-23T18:16:24.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.409486+0000", 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2878000000000002e-05 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177046 8.9 8:97ec04da:::benchmark_data_smithi055_346788_object66454:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.334212+0000", 2023-09-23T18:16:24.725 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 233.235596426, 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0928899140000001, 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177046 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334212+0000", 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334212+0000", 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334211+0000", 2023-09-23T18:16:24.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334221+0000", 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0329999999999999e-05 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334236+0000", 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5441e-05 2023-09-23T18:16:24.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334243+0000", 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5590000000000002e-06 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334297+0000", 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3662000000000002e-05 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.729 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.334307+0000", 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0207000000000001e-05 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.299021+0000", 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.964714356 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.299028+0000", 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9670000000000004e-06 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.409514+0000", 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.110486262 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.409646+0000", 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013124400000000001 2023-09-23T18:16:24.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.409782+0000", 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013615099999999999 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.419151+0000", 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0093694659999999999 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.422507+0000", 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033560059999999999 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.426918+0000", 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004411206 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.426976+0000", 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7358000000000002e-05 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.427102+0000", 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012621999999999999 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177048 8.9 8:993077e3:::benchmark_data_smithi055_346788_object66456:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e420)", 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.345194+0000", 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 233.22461483399999, 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0826640959999998, 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177048 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.735 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345194+0000", 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345194+0000", 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345192+0000", 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345200+0000", 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8699999999999992e-06 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345214+0000", 2023-09-23T18:16:24.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4086e-05 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345221+0000", 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2429999999999999e-06 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345290+0000", 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8515000000000004e-05 2023-09-23T18:16:24.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.345303+0000", 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3321e-05 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.299049+0000", 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95374580099999995 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.299055+0000", 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3249999999999996e-06 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.411995+0000", 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1129396570000001 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.412116+0000", 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012100299999999999 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.412239+0000", 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012302099999999999 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.419161+0000", 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069218630000000003 2023-09-23T18:16:24.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.422531+0000", 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003370587 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.427711+0000", 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051796910000000002 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.427769+0000", 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8127000000000001e-05 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.427858+0000", 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8553000000000004e-05 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.743 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:140341 9.0 9:0283fca9:::benchmark_data_smithi055_386467_object35681:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e511)", 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:16:09.323225+0000", 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.246583888, 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0462216630000001, 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:24.744 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 140341 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.323225+0000", 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:24.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.323225+0000", 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.323223+0000", 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.323231+0000", 2023-09-23T18:16:24.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.439e-06 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.323247+0000", 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5007999999999999e-05 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.323254+0000", 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0430000000000002e-06 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.323314+0000", 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0741e-05 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.323323+0000", 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7250000000000003e-06 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.748 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:10.301925+0000", 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.97860187899999995 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:10.302018+0000", 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3039000000000001e-05 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,2", 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:10.302102+0000", 2023-09-23T18:16:24.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3821999999999998e-05 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:10.306916+0000", 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048143439999999999 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:10.369238+0000", 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.062321860999999999 2023-09-23T18:16:24.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:10.369293+0000", 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4793999999999997e-05 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:10.369345+0000", 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2651999999999997e-05 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:24.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:24.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:10.369446+0000", 2023-09-23T18:16:24.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100914 2023-09-23T18:16:24.752 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.752 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.752 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.752 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:24.752 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:24.752 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:16:24.752 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:24 smithi099 ceph-mon[167076]: pgmap v1251: 119 pgs: 1 remapped+peering, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 25580/177693 objects degraded (14.396%); 11368/177693 objects misplaced (6.398%); 2.1 MiB/s, 8 objects/s recovering 2023-09-23T18:16:24.753 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/245740707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:16:24.753 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:24 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:16:24.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:24.754 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:24.754 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:24.855 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:24.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:24 smithi055 ceph-mon[179668]: pgmap v1251: 119 pgs: 1 remapped+peering, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 25580/177693 objects degraded (14.396%); 11368/177693 objects misplaced (6.398%); 2.1 MiB/s, 8 objects/s recovering 2023-09-23T18:16:24.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/245740707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:16:24.884 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:24 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:16:24.884 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:24 smithi055 ceph-mon[184174]: pgmap v1251: 119 pgs: 1 remapped+peering, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 784 op/s; 25580/177693 objects degraded (14.396%); 11368/177693 objects misplaced (6.398%); 2.1 MiB/s, 8 objects/s recovering 2023-09-23T18:16:24.884 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/245740707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:16:24.884 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:24 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:16:24.889 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:24.906 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:16:24.936 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:24.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:24.940 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:25.030 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:16:25.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:25.166 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:25.263 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:25.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:25.274 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:25.336 INFO:tasks.thrashosds.thrasher:in_osds: [7, 1, 0, 4, 2, 3, 6, 5] out_osds: [] dead_osds: [1] live_osds: [0, 4, 6, 7, 3, 5, 2] 2023-09-23T18:16:25.336 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:16:25.336 INFO:tasks.thrashosds.thrasher:inject_pause on osd.5 2023-09-23T18:16:25.336 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:16:25.336 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:16:25.336 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 config set filestore_inject_stall 3 2023-09-23T18:16:25.375 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:25.525 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:25.542 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:25 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:25.542 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:25 smithi099 ceph-mon[167076]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.543 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:25 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:16:25.543 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:25 smithi099 ceph-mon[167076]: osdmap e520: 8 total, 7 up, 8 in 2023-09-23T18:16:25.543 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:25 smithi099 ceph-mon[167076]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.543 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:25 smithi099 ceph-mon[167076]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.543 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:25 smithi099 ceph-mon[167076]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.587 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:25.620 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:25.633 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:25.634 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:25.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:25.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:25.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[184174]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:16:25.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[184174]: osdmap e520: 8 total, 7 up, 8 in 2023-09-23T18:16:25.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[184174]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[184174]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[184174]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:25.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[179668]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:16:25.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[179668]: osdmap e520: 8 total, 7 up, 8 in 2023-09-23T18:16:25.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[179668]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[179668]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:25 smithi055 ceph-mon[179668]: failed to encode map e520 with expected crc 2023-09-23T18:16:25.838 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:25.913 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:16:25.927 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:25.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:25.938 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:26.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:26.115 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:26.552 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:26 smithi055 ceph-mon[179668]: failed to encode map e520 with expected crc 2023-09-23T18:16:26.552 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:26 smithi055 ceph-mon[179668]: pgmap v1253: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 28611/176727 objects degraded (16.189%); 14949/176727 objects misplaced (8.459%); 8.0 MiB/s, 104 objects/s recovering 2023-09-23T18:16:26.560 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:26 smithi099 ceph-mon[167076]: failed to encode map e520 with expected crc 2023-09-23T18:16:26.560 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:26 smithi099 ceph-mon[167076]: pgmap v1253: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 28611/176727 objects degraded (16.189%); 14949/176727 objects misplaced (8.459%); 8.0 MiB/s, 104 objects/s recovering 2023-09-23T18:16:26.834 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:26.855 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:26.876 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:26.898 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:26 smithi055 ceph-mon[184174]: failed to encode map e520 with expected crc 2023-09-23T18:16:26.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:26 smithi055 ceph-mon[184174]: pgmap v1253: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.6 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 610 op/s; 28611/176727 objects degraded (16.189%); 14949/176727 objects misplaced (8.459%); 8.0 MiB/s, 104 objects/s recovering 2023-09-23T18:16:26.960 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:26.985 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:27.019 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:27.020 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:16:27.020 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:27.020 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:16:27.020 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:16:27.020 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:16:27.020 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:16:27.020 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:16:27.020 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:16:27.033 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:16:27.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:27.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:27.078 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:27.179 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:27.224 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:27.346 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:16:27.367 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:27.368 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:27.572 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/440784608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:27.573 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/440784608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:27.716 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:27.799 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:27.808 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:27.808 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:27.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/440784608' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:27.909 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:27.989 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:28.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:28 smithi055 ceph-mon[179668]: pgmap v1254: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 27343/166140 objects degraded (16.458%); 13999/166140 objects misplaced (8.426%); 6.3 MiB/s, 82 objects/s recovering 2023-09-23T18:16:28.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 27343/166140 objects degraded (16.458%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:28.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:28 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:16:28.628 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:28.639 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:28 smithi099 ceph-mon[167076]: pgmap v1254: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 27343/166140 objects degraded (16.458%); 13999/166140 objects misplaced (8.426%); 6.3 MiB/s, 82 objects/s recovering 2023-09-23T18:16:28.640 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 27343/166140 objects degraded (16.458%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:28.640 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:28 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:16:28.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:28.641 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:28.741 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:28.777 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:28.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:28 smithi055 ceph-mon[184174]: pgmap v1254: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.4 GiB data, 17 GiB used, 698 GiB / 715 GiB avail; 0 B/s wr, 945 op/s; 27343/166140 objects degraded (16.458%); 13999/166140 objects misplaced (8.426%); 6.3 MiB/s, 82 objects/s recovering 2023-09-23T18:16:28.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 27343/166140 objects degraded (16.458%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:28.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:28 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:16:28.872 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:28.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:28.877 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:28.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:29.074 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:29.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:29.167 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:29.167 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:29.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:29.341 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:29.368 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:16:29.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:29.474 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:29.475 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:29.575 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:29.614 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:29.632 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:16:29.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3329950849' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:29.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:29 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:29.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3329950849' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:29.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:29 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:29.672 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:29.676 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:29.677 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:29.777 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:29.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3329950849' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:29.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:29 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:29.887 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:30.027 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:30.044 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:16:30.044 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:16:30.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:30.052 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:30.153 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:30.183 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:30.221 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:30.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:30.225 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:30.236 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:30.291 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:16:30.292 INFO:teuthology.orchestra.run.smithi099.stdout: "success": "" 2023-09-23T18:16:30.292 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:16:30.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:30.398 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:30.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:16:30] "GET /metrics HTTP/1.1" 200 36656 "" "Prometheus/2.43.0" 2023-09-23T18:16:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[184174]: pgmap v1255: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 25543/161625 objects degraded (15.804%); 13702/161625 objects misplaced (8.478%); 11 MiB/s, 152 objects/s recovering 2023-09-23T18:16:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:16:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[184174]: failed to encode map e521 with expected crc 2023-09-23T18:16:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[184174]: failed to encode map e521 with expected crc 2023-09-23T18:16:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[184174]: osdmap e521: 8 total, 7 up, 8 in 2023-09-23T18:16:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[184174]: failed to encode map e521 with expected crc 2023-09-23T18:16:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[184174]: failed to encode map e521 with expected crc 2023-09-23T18:16:30.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[184174]: failed to encode map e521 with expected crc 2023-09-23T18:16:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[179668]: pgmap v1255: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 25543/161625 objects degraded (15.804%); 13702/161625 objects misplaced (8.478%); 11 MiB/s, 152 objects/s recovering 2023-09-23T18:16:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:16:30.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[179668]: failed to encode map e521 with expected crc 2023-09-23T18:16:30.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:30.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[179668]: failed to encode map e521 with expected crc 2023-09-23T18:16:30.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[179668]: osdmap e521: 8 total, 7 up, 8 in 2023-09-23T18:16:30.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[179668]: failed to encode map e521 with expected crc 2023-09-23T18:16:30.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[179668]: failed to encode map e521 with expected crc 2023-09-23T18:16:30.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:30 smithi055 ceph-mon[179668]: failed to encode map e521 with expected crc 2023-09-23T18:16:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:30 smithi099 ceph-mon[167076]: pgmap v1255: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.3 GiB data, 17 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 25543/161625 objects degraded (15.804%); 13702/161625 objects misplaced (8.478%); 11 MiB/s, 152 objects/s recovering 2023-09-23T18:16:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:30 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:16:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:30 smithi099 ceph-mon[167076]: failed to encode map e521 with expected crc 2023-09-23T18:16:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:30 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:30 smithi099 ceph-mon[167076]: failed to encode map e521 with expected crc 2023-09-23T18:16:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:30 smithi099 ceph-mon[167076]: osdmap e521: 8 total, 7 up, 8 in 2023-09-23T18:16:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:30 smithi099 ceph-mon[167076]: failed to encode map e521 with expected crc 2023-09-23T18:16:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:30 smithi099 ceph-mon[167076]: failed to encode map e521 with expected crc 2023-09-23T18:16:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:30 smithi099 ceph-mon[167076]: failed to encode map e521 with expected crc 2023-09-23T18:16:31.556 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:31.584 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:31.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:31 smithi055 ceph-mon[179668]: pgmap v1257: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 24351/151968 objects degraded (16.024%); 12843/151968 objects misplaced (8.451%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T18:16:31.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3844842015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:31.610 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:31.630 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:31.683 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:31.722 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:31.750 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:31.750 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:16:31.750 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:31.750 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:16:31.750 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:16:31.750 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:16:31.750 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:16:31.751 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:16:31.751 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:16:31.900 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:31 smithi099 ceph-mon[167076]: pgmap v1257: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 24351/151968 objects degraded (16.024%); 12843/151968 objects misplaced (8.451%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T18:16:31.901 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3844842015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:31 smithi055 ceph-mon[184174]: pgmap v1257: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.1 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.07k op/s; 24351/151968 objects degraded (16.024%); 12843/151968 objects misplaced (8.451%); 9.8 MiB/s, 157 objects/s recovering 2023-09-23T18:16:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3844842015' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:32.056 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:32.056 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:32.107 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:16:32.802 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:32.942 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:32.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:32.954 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:33.055 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:33.132 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:34.057 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:16:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:34 smithi099 ceph-mon[167076]: pgmap v1258: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 23448/148386 objects degraded (15.802%); 12843/148386 objects misplaced (8.655%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T18:16:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:34 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 23448/148386 objects degraded (15.802%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:34.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:34 smithi055 ceph-mon[184174]: pgmap v1258: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 23448/148386 objects degraded (15.802%); 12843/148386 objects misplaced (8.655%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T18:16:34.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:34 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 23448/148386 objects degraded (15.802%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:34 smithi055 ceph-mon[179668]: pgmap v1258: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 1.11k op/s; 23448/148386 objects degraded (15.802%); 12843/148386 objects misplaced (8.655%); 5.0 MiB/s, 80 objects/s recovering 2023-09-23T18:16:34.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:34 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 23448/148386 objects degraded (15.802%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:34.777 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:35.162 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:35.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:35.168 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:35.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:16:35.269 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:35.330 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:16:35.386 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:35.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:35 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:35.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:35 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:35.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:35 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/623141839' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:35.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:35 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:35.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:35 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:35.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:35 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/623141839' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:35.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:35 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:35.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:35 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:35.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:35 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/623141839' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:36.005 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:36.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:36.009 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:36.110 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:36.188 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:36.199 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:36.226 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:36.250 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:36.272 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:36.280 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:36.289 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:36.289 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:36.298 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:36.339 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:36.372 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:36.372 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:16:36.372 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:36.373 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:16:36.373 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:16:36.373 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:16:36.373 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:16:36.373 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:16:36.373 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:16:36.390 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:36.463 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:36 smithi099 ceph-mon[167076]: pgmap v1259: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 22654/147462 objects degraded (15.363%); 12535/147462 objects misplaced (8.500%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T18:16:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:36 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/623141839' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:16:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:36 smithi099 ceph-mon[167076]: osdmap e522: 8 total, 7 up, 8 in 2023-09-23T18:16:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:36 smithi099 ceph-mon[167076]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:36 smithi099 ceph-mon[167076]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:36 smithi099 ceph-mon[167076]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:36 smithi099 ceph-mon[167076]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/681187643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:36.588 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:36.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:36.594 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[179668]: pgmap v1259: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 22654/147462 objects degraded (15.363%); 12535/147462 objects misplaced (8.500%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T18:16:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/623141839' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:16:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[179668]: osdmap e522: 8 total, 7 up, 8 in 2023-09-23T18:16:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[179668]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[179668]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[179668]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[179668]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/681187643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[184174]: pgmap v1259: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 3.0 GiB data, 16 GiB used, 699 GiB / 715 GiB avail; 0 B/s wr, 975 op/s; 22654/147462 objects degraded (15.363%); 12535/147462 objects misplaced (8.500%); 8.2 MiB/s, 131 objects/s recovering 2023-09-23T18:16:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/623141839' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:16:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[184174]: osdmap e522: 8 total, 7 up, 8 in 2023-09-23T18:16:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[184174]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[184174]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[184174]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[184174]: failed to encode map e522 with expected crc 2023-09-23T18:16:36.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/681187643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:36.695 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:36.711 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:36.711 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:36.762 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:36.846 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:36.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:36.855 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:36.913 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:16:36.955 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:37.050 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:37.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:37 smithi099 ceph-mon[167076]: failed to encode map e522 with expected crc 2023-09-23T18:16:37.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:37 smithi099 ceph-mon[167076]: failed to encode map e522 with expected crc 2023-09-23T18:16:37.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:37 smithi055 ceph-mon[179668]: failed to encode map e522 with expected crc 2023-09-23T18:16:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:37 smithi055 ceph-mon[179668]: failed to encode map e522 with expected crc 2023-09-23T18:16:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:37 smithi055 ceph-mon[184174]: failed to encode map e522 with expected crc 2023-09-23T18:16:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:37 smithi055 ceph-mon[184174]: failed to encode map e522 with expected crc 2023-09-23T18:16:37.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:37.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:37.919 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:37.919 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:37.925 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:16:38.021 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:38.057 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:38.107 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:38.112 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:38.112 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:38.213 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:38.308 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:38.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:38 smithi099 ceph-mon[167076]: pgmap v1261: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 21426/137337 objects degraded (15.601%); 11638/137337 objects misplaced (8.474%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:16:38.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 21426/137337 objects degraded (15.601%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:38.599 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:38.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:38 smithi055 ceph-mon[184174]: pgmap v1261: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 21426/137337 objects degraded (15.601%); 11638/137337 objects misplaced (8.474%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:16:38.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 21426/137337 objects degraded (15.601%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:38.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:38 smithi055 ceph-mon[179668]: pgmap v1261: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.8 GiB data, 16 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 21426/137337 objects degraded (15.601%); 11638/137337 objects misplaced (8.474%); 4.9 MiB/s, 78 objects/s recovering 2023-09-23T18:16:38.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 21426/137337 objects degraded (15.601%), 13 pgs degraded (PG_DEGRADED) 2023-09-23T18:16:38.712 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:16:38.903 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:38.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:38.912 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:39.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:39.063 INFO:tasks.thrashosds.thrasher:in_osds: [7, 1, 0, 4, 2, 3, 6, 5] out_osds: [] dead_osds: [1] live_osds: [0, 4, 6, 7, 3, 5, 2] 2023-09-23T18:16:39.063 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:16:39.063 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T18:16:39.063 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:16:39.063 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:16:39.064 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T18:16:39.089 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:39.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:39.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:39.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:39.261 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:39.396 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:39.572 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:39.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:39.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:39.797 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:39.897 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:39.989 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:40.066 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:40.330 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:16:40.585 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:40 smithi055 ceph-mon[179668]: pgmap v1262: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 19839/132909 objects degraded (14.927%); 11350/132909 objects misplaced (8.540%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T18:16:40.585 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2895862182' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:40.585 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:40 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:40.586 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:16:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:16:40] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.43.0" 2023-09-23T18:16:40.586 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:40 smithi055 ceph-mon[184174]: pgmap v1262: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 19839/132909 objects degraded (14.927%); 11350/132909 objects misplaced (8.540%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T18:16:40.586 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2895862182' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:40.586 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:40 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:40 smithi099 ceph-mon[167076]: pgmap v1262: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.7 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.01k op/s; 19839/132909 objects degraded (14.927%); 11350/132909 objects misplaced (8.540%); 8.0 MiB/s, 128 objects/s recovering 2023-09-23T18:16:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2895862182' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:40 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:41.148 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:41.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:41.167 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:41.268 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:41.312 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:41.345 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:41.368 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:41.387 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:41.416 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:41.447 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:41.482 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:41.515 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:41.527 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:41.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:16:41.527 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:41.527 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:16:41.528 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:16:41.528 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:16:41.528 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:16:41.528 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:16:41.528 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:16:41.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:41.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:41.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:41.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:41.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: osdmap e523: 8 total, 7 up, 8 in 2023-09-23T18:16:41.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.535 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.536 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.536 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.536 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.536 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.536 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.536 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:41 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2813110961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:41.542 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:41.542 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:41.542 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: osdmap e523: 8 total, 7 up, 8 in 2023-09-23T18:16:41.543 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.543 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.543 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.543 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.543 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.543 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.543 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.543 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.543 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2813110961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:41.543 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: osdmap e523: 8 total, 7 up, 8 in 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: failed to encode map e523 with expected crc 2023-09-23T18:16:41.544 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:41 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2813110961' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:41.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:41.638 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:16:41.638 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:16:41.638 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:16:41.638 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:16:41.639 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:16:41.742 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:41.806 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:16:41.807 INFO:teuthology.orchestra.run.smithi055.stdout: "success": "filestore_inject_stall = '3' (not observed, change may require restart) " 2023-09-23T18:16:41.807 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:16:42.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:16:42.081 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:42.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:42.086 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:42.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:42.264 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:42.303 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:42.303 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:42.356 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:42.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:42.370 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:42.471 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:42.503 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:42 smithi055 ceph-mon[179668]: pgmap v1264: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 18635/123240 objects degraded (15.121%); 10479/123240 objects misplaced (8.503%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T18:16:42.505 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:42 smithi055 ceph-mon[184174]: pgmap v1264: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 18635/123240 objects degraded (15.121%); 10479/123240 objects misplaced (8.503%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T18:16:42.507 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:42.547 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:42.551 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:42.551 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:42.652 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:42.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:42 smithi099 ceph-mon[167076]: pgmap v1264: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.5 GiB data, 15 GiB used, 700 GiB / 715 GiB avail; 0 B/s wr, 1.05k op/s; 18635/123240 objects degraded (15.121%); 10479/123240 objects misplaced (8.503%); 9.5 MiB/s, 151 objects/s recovering 2023-09-23T18:16:42.731 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:42.798 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:42.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:42.807 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:42.908 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:43.056 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:16:43.057 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:16:43.132 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:43.333 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:43.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:43.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:43.366 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 18635/123240 objects degraded (15.121%), 13 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:43.366 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:43 smithi055 ceph-mon[184174]: osdmap e524: 8 total, 7 up, 8 in 2023-09-23T18:16:43.367 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:43 smithi055 ceph-mon[184174]: failed to encode map e524 with expected crc 2023-09-23T18:16:43.367 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:43 smithi055 ceph-mon[184174]: failed to encode map e524 with expected crc 2023-09-23T18:16:43.367 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:43 smithi055 ceph-mon[184174]: failed to encode map e524 with expected crc 2023-09-23T18:16:43.367 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 18635/123240 objects degraded (15.121%), 13 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:43.367 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:43 smithi055 ceph-mon[179668]: osdmap e524: 8 total, 7 up, 8 in 2023-09-23T18:16:43.367 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:43 smithi055 ceph-mon[179668]: failed to encode map e524 with expected crc 2023-09-23T18:16:43.367 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:43 smithi055 ceph-mon[179668]: failed to encode map e524 with expected crc 2023-09-23T18:16:43.367 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:43 smithi055 ceph-mon[179668]: failed to encode map e524 with expected crc 2023-09-23T18:16:43.391 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 18635/123240 objects degraded (15.121%), 13 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:43.392 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:43 smithi099 ceph-mon[167076]: osdmap e524: 8 total, 7 up, 8 in 2023-09-23T18:16:43.392 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:43 smithi099 ceph-mon[167076]: failed to encode map e524 with expected crc 2023-09-23T18:16:43.392 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:43 smithi099 ceph-mon[167076]: failed to encode map e524 with expected crc 2023-09-23T18:16:43.392 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:43 smithi099 ceph-mon[167076]: failed to encode map e524 with expected crc 2023-09-23T18:16:43.449 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:43.502 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:43.695 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:43.790 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:43.794 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:43.794 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:43.895 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:43.922 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:44.199 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:44.202 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:44.202 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:44.303 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:44.304 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:16:44.335 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:44.373 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:44.379 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:44.379 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:44.478 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:44 smithi099 ceph-mon[167076]: pgmap v1265: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s; 17802/119934 objects degraded (14.843%); 10479/119934 objects misplaced (8.737%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:16:44.479 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:44 smithi099 ceph-mon[167076]: failed to encode map e524 with expected crc 2023-09-23T18:16:44.479 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:44 smithi099 ceph-mon[167076]: failed to encode map e524 with expected crc 2023-09-23T18:16:44.479 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:44 smithi099 ceph-mon[167076]: failed to encode map e524 with expected crc 2023-09-23T18:16:44.479 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:44 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:44.479 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:44 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:44.479 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:44 smithi099 ceph-mon[167076]: osdmap e525: 8 total, 7 up, 8 in 2023-09-23T18:16:44.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:44.571 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:44.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[184174]: pgmap v1265: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s; 17802/119934 objects degraded (14.843%); 10479/119934 objects misplaced (8.737%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:16:44.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[184174]: failed to encode map e524 with expected crc 2023-09-23T18:16:44.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[184174]: failed to encode map e524 with expected crc 2023-09-23T18:16:44.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[184174]: failed to encode map e524 with expected crc 2023-09-23T18:16:44.624 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:44.624 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:44.624 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[184174]: osdmap e525: 8 total, 7 up, 8 in 2023-09-23T18:16:44.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[179668]: pgmap v1265: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 29 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 1.15k op/s; 17802/119934 objects degraded (14.843%); 10479/119934 objects misplaced (8.737%); 4.6 MiB/s, 73 objects/s recovering 2023-09-23T18:16:44.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[179668]: failed to encode map e524 with expected crc 2023-09-23T18:16:44.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[179668]: failed to encode map e524 with expected crc 2023-09-23T18:16:44.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[179668]: failed to encode map e524 with expected crc 2023-09-23T18:16:44.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:44.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:44.625 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:44 smithi055 ceph-mon[179668]: osdmap e525: 8 total, 7 up, 8 in 2023-09-23T18:16:44.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:44.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:44.767 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:44.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:44.871 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:16:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:16:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:16:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:16:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148346 8.d 8:b553d8e7:::benchmark_data_smithi055_346788_object37754:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e408)", 2023-09-23T18:16:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.520341+0000", 2023-09-23T18:16:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 288.344760425, 2023-09-23T18:16:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1563575239999999, 2023-09-23T18:16:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148346 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520341+0000", 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520341+0000", 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520340+0000", 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520343+0000", 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2260000000000001e-06 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520350+0000", 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2009999999999996e-06 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520353+0000", 2023-09-23T18:16:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8229999999999997e-06 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520401+0000", 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7818999999999997e-05 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520549+0000", 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014766099999999999 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.520711+0000", 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016159099999999999 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.523302+0000", 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002591051 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.529040+0000", 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057384949999999997 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.529049+0000", 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0629999999999995e-06 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.654884+0000", 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1258352890000001 2023-09-23T18:16:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.654994+0000", 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000109758 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676645+0000", 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.021651083000000002 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676698+0000", 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2895000000000001e-05 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148356 8.d 8:b231d41c:::benchmark_data_smithi055_346788_object37764:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.534689+0000", 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 288.33041205799998, 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1416983039999999, 2023-09-23T18:16:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148356 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534689+0000", 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534689+0000", 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534688+0000", 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534694+0000", 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7320000000000001e-06 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534710+0000", 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5917e-05 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534717+0000", 2023-09-23T18:16:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7209999999999997e-06 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534759+0000", 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1794e-05 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.534769+0000", 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9150000000000008e-06 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.655033+0000", 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1202644310000001 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.655166+0000", 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013317400000000001 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:16:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.655326+0000", 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016015199999999999 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.661897+0000", 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065707750000000001 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676191+0000", 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014294118999999999 2023-09-23T18:16:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676291+0000", 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9346999999999996e-05 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676328+0000", 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7681000000000001e-05 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676387+0000", 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8913999999999999e-05 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148375 8.d 8:b69cd647:::benchmark_data_smithi055_346788_object37783:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.552544+0000", 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 288.31255755199999, 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1243204470000001, 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148375 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552544+0000", 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552544+0000", 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552543+0000", 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552546+0000", 2023-09-23T18:16:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8770000000000002e-06 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552580+0000", 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4294000000000001e-05 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.552591+0000", 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0567999999999999e-05 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.558426+0000", 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0058347260000000001 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.558435+0000", 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2580000000000001e-06 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.657112+0000", 2023-09-23T18:16:44.891 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.098676784 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.657237+0000", 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012515399999999999 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.657353+0000", 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011598 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.892 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.661903+0000", 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045504259999999998 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676235+0000", 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014331468999999999 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676770+0000", 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00053503199999999996 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676803+0000", 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3618000000000002e-05 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676864+0000", 2023-09-23T18:16:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0745000000000002e-05 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148377 8.d 8:b286308b:::benchmark_data_smithi055_346788_object37785:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.553953+0000", 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 288.311148178, 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1230239500000001, 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148377 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553953+0000", 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553953+0000", 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553952+0000", 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553957+0000", 2023-09-23T18:16:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6260000000000003e-06 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553968+0000", 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1654999999999999e-05 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.553975+0000", 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3450000000000002e-06 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.558455+0000", 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044808180000000001 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.558460+0000", 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.42e-06 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.658566+0000", 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.100105881 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.658686+0000", 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000119815 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.658819+0000", 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000133802 2023-09-23T18:16:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.664276+0000", 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005456982 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676258+0000", 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011981656 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676896+0000", 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063824599999999997 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676923+0000", 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6713999999999999e-05 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676977+0000", 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3909000000000002e-05 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148424 8.d 8:b27653e5:::benchmark_data_smithi055_346788_object37832:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.634454+0000", 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 288.23064740400002, 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0427430849999999, 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148424 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634454+0000", 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634454+0000", 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634453+0000", 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634458+0000", 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3870000000000003e-06 2023-09-23T18:16:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634469+0000", 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0871e-05 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634473+0000", 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5480000000000003e-06 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634508+0000", 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4168000000000002e-05 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.634514+0000", 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0859999999999998e-06 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.660281+0000", 2023-09-23T18:16:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0257674050000001 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.660408+0000", 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127327 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.660494+0000", 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5332000000000002e-05 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.666754+0000", 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0062598719999999997 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676422+0000", 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096688 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677081+0000", 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00065843900000000003 2023-09-23T18:16:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677134+0000", 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2908000000000002e-05 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677197+0000", 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2958000000000003e-05 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:148435 8.d 8:b5632819:::benchmark_data_smithi055_346788_object37843:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e409)", 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:56.644607+0000", 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 288.22049370799999, 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.032802521, 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 148435 2023-09-23T18:16:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644607+0000", 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644607+0000", 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644606+0000", 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644611+0000", 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9599999999999999e-06 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644622+0000", 2023-09-23T18:16:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0441e-05 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644626+0000", 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.622e-06 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644665+0000", 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.837e-05 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:56.644671+0000", 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8650000000000003e-06 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.661941+0000", 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.017269996 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.662026+0000", 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4456000000000006e-05 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,6", 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.662106+0000", 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0147000000000004e-05 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.666763+0000", 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046564930000000003 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.676452+0000", 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096896880000000001 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677241+0000", 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00078881700000000003 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677297+0000", 2023-09-23T18:16:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6048999999999998e-05 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:57.677410+0000", 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112773 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177035 8.c 8:3702eed5:::benchmark_data_smithi055_346788_object66443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T18:16:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.274617+0000", 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 253.59048388400001, 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1493567709999999, 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177035 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274617+0000", 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274617+0000", 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274616+0000", 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274624+0000", 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7940000000000003e-06 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274644+0000", 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0441999999999998e-05 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274652+0000", 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5569999999999998e-06 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.279258+0000", 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046061360000000003 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.279386+0000", 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127995 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3", 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.279497+0000", 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011157100000000001 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.291211+0000", 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011713889 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295830+0000", 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046190240000000002 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295843+0000", 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3239e-05 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.307974+0000", 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0121305490000001 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.307989+0000", 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5268e-05 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.423885+0000", 2023-09-23T18:16:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1158961300000001 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.423974+0000", 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8610999999999996e-05 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:208693 8.c 8:3f12a3a6:::benchmark_data_smithi055_346788_object98101:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T18:16:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:11.759582+0000", 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 213.10551938099999, 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0444234290000001, 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 208693 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759582+0000", 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759582+0000", 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759580+0000", 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759590+0000", 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8640000000000008e-06 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759602+0000", 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1640999999999999e-05 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759608+0000", 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.64e-06 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759679+0000", 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1308000000000002e-05 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759809+0000", 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000129696 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759940+0000", 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013043700000000001 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765188+0000", 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005248862 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.768203+0000", 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003014652 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776965+0000", 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.008761385 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776972+0000", 2023-09-23T18:16:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0049999999999998e-06 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.803922+0000", 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0269501240000001 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804005+0000", 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3491000000000004e-05 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:208696 8.c 8:3a4f2983:::benchmark_data_smithi055_346788_object98104:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:11.759625+0000", 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 213.10547620700001, 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0444810689999999, 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 208696 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759625+0000", 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759625+0000", 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759624+0000", 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759628+0000", 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3349999999999998e-06 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759635+0000", 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9949999999999999e-06 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759639+0000", 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5170000000000001e-06 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.762500+0000", 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028612569999999999 2023-09-23T18:16:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.762647+0000", 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014658100000000001 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.762784+0000", 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013712300000000001 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.769536+0000", 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0067522110000000002 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.773643+0000", 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041063560000000002 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776999+0000", 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033565219999999998 2023-09-23T18:16:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.777005+0000", 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6810000000000003e-06 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804039+0000", 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0270342690000001 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804106+0000", 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6701999999999999e-05 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:208697 8.c 8:3dd33f7b:::benchmark_data_smithi055_346788_object98105:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:11.765262+0000", 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 213.09983883500001, 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.03891529, 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 208697 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765262+0000", 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765262+0000", 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765261+0000", 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765268+0000", 2023-09-23T18:16:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2810000000000003e-06 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765281+0000", 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2964e-05 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765288+0000", 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.392e-06 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765340+0000", 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2036999999999997e-05 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765485+0000", 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014538800000000001 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:16:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765650+0000", 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016466500000000001 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.773678+0000", 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0080284060000000001 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.773706+0000", 2023-09-23T18:16:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7759e-05 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.777028+0000", 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033222540000000002 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.777033+0000", 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.074e-06 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804136+0000", 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0271029860000001 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804178+0000", 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1449999999999998e-05 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:208709 8.c 8:3b5457b5:::benchmark_data_smithi055_346788_object98117:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e435)", 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:11.776398+0000", 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 213.08870336000001, 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0613724870000001, 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 208709 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776398+0000", 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776398+0000", 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776396+0000", 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776405+0000", 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2940000000000006e-06 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776415+0000", 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0898e-05 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776423+0000", 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9240000000000007e-06 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.777529+0000", 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001105214 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.777533+0000", 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4610000000000001e-06 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804203+0000", 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0266699610000001 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804332+0000", 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012890199999999999 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804416+0000", 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3702999999999995e-05 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.830547+0000", 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.026131879 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837552+0000", 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070046869999999999 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837676+0000", 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012416900000000001 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837730+0000", 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3810999999999997e-05 2023-09-23T18:16:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837770+0000", 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0074999999999998e-05 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:208711 8.c 8:3f16d1b8:::benchmark_data_smithi055_346788_object98119:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e435)", 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:11.778108+0000", 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 213.08699272499999, 2023-09-23T18:16:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.059774424, 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 208711 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778108+0000", 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778108+0000", 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778107+0000", 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778113+0000", 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0769999999999998e-06 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778123+0000", 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0713000000000001e-05 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778129+0000", 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8549999999999998e-06 2023-09-23T18:16:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778241+0000", 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112157 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778248+0000", 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8850000000000001e-06 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.806861+0000", 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0286126259999999 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.807010+0000", 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014955099999999999 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.807149+0000", 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013890800000000001 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.830553+0000", 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.023403943999999999 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837614+0000", 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070608600000000004 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837807+0000", 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019263 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837851+0000", 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4180000000000001e-05 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837883+0000", 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1934000000000003e-05 2023-09-23T18:16:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121618 9.7 9:e91f6bd2:::benchmark_data_smithi055_386467_object16958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T18:16:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.648257+0000", 2023-09-23T18:16:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.216843730999997, 2023-09-23T18:16:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0535740570000001, 2023-09-23T18:16:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121618 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648257+0000", 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648257+0000", 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648257+0000", 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648260+0000", 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5859999999999999e-06 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648266+0000", 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2899999999999999e-06 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648270+0000", 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3370000000000001e-06 2023-09-23T18:16:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648396+0000", 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012564 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648520+0000", 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012453099999999999 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T18:16:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648655+0000", 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013492199999999999 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.653810+0000", 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051547549999999996 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.658227+0000", 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044168380000000002 2023-09-23T18:16:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.669168+0000", 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010940755999999999 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.700842+0000", 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.031674536000000003 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.700851+0000", 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0480000000000004e-06 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701731+0000", 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.000879589 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701831+0000", 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010064 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121620 9.7 9:e0b3f972:::benchmark_data_smithi055_386467_object16960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.648304+0000", 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.216796771999995, 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.053641834, 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121620 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648304+0000", 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648304+0000", 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648304+0000", 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648314+0000", 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5780000000000005e-06 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648323+0000", 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5899999999999997e-06 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648328+0000", 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7620000000000003e-06 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.650968+0000", 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026406110000000002 2023-09-23T18:16:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.651091+0000", 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012274000000000001 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.651209+0000", 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117797 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.653817+0000", 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026075460000000001 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.662443+0000", 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.008626791 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.669204+0000", 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006760856 2023-09-23T18:16:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.700923+0000", 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.031719134000000003 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.700931+0000", 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7749999999999993e-06 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701873+0000", 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.000942032 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701946+0000", 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2977000000000003e-05 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121631 9.8 9:16d41ba8:::benchmark_data_smithi055_386467_object16971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.670993+0000", 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.194107838999997, 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0587921170000001, 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121631 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.670993+0000", 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.670993+0000", 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.670993+0000", 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.670995+0000", 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3199999999999998e-06 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.671000+0000", 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3129999999999998e-06 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.671004+0000", 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7419999999999999e-06 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.680659+0000", 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096546359999999994 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.680813+0000", 2023-09-23T18:16:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015390400000000001 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.680960+0000", 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014744199999999999 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.701143+0000", 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020182319000000001 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.701148+0000", 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7579999999999998e-06 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701792+0000", 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.000643347 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701893+0000", 2023-09-23T18:16:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010093400000000001 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.729712+0000", 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.027819884999999999 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.729785+0000", 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2902000000000001e-05 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121635 9.3 9:c6494341:::benchmark_data_smithi055_386467_object16975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.672050+0000", 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.193051393000005, 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0872817100000001, 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121635 2023-09-23T18:16:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672050+0000", 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672050+0000", 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672049+0000", 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672053+0000", 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5929999999999999e-06 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672063+0000", 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0599999999999997e-06 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672067+0000", 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5820000000000003e-06 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672104+0000", 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6903000000000002e-05 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672187+0000", 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2817000000000003e-05 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672255+0000", 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7957e-05 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.686140+0000", 2023-09-23T18:16:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013884728000000001 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.686149+0000", 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5000000000000005e-06 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746343+0000", 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0601943570000001 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746452+0000", 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108996 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.759262+0000", 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012809123 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.759331+0000", 2023-09-23T18:16:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9881000000000004e-05 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121632 9.a 9:5963b4ad:::benchmark_data_smithi055_386467_object16972:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.687284+0000", 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.177817168999994, 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0798843389999999, 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:44.996 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121632 2023-09-23T18:16:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687284+0000", 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687284+0000", 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687282+0000", 2023-09-23T18:16:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687288+0000", 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9460000000000003e-06 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687299+0000", 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0729e-05 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687305+0000", 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1149999999999999e-06 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.688304+0000", 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00099935900000000001 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.688312+0000", 2023-09-23T18:16:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7109999999999995e-06 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746824+0000", 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0585122039999999 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.748540+0000", 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001715507 2023-09-23T18:16:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.748766+0000", 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022613200000000001 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.754863+0000", 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0060968660000000003 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.763930+0000", 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.009067238 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767002+0000", 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030723460000000001 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767053+0000", 2023-09-23T18:16:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0414000000000002e-05 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767168+0000", 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115481 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121637 9.a 9:5e2e6388:::benchmark_data_smithi055_386467_object16977:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:16:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.687315+0000", 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.177785974000003, 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0801113259999999, 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121637 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687315+0000", 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687315+0000", 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687315+0000", 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687318+0000", 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0319999999999999e-06 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687324+0000", 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1299999999999998e-06 2023-09-23T18:16:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687328+0000", 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2169999999999996e-06 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.688333+0000", 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010054479999999999 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.688339+0000", 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6810000000000003e-06 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.750580+0000", 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0622411700000001 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.750704+0000", 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012343400000000001 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.750826+0000", 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000122033 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.759396+0000", 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085699620000000004 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.763962+0000", 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045667279999999999 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767271+0000", 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033081130000000001 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767328+0000", 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7719000000000001e-05 2023-09-23T18:16:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767426+0000", 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8108000000000003e-05 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121638 9.3 9:c51846a3:::benchmark_data_smithi055_386467_object16978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.695896+0000", 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.169205140000003, 2023-09-23T18:16:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0712138659999999, 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121638 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695896+0000", 2023-09-23T18:16:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695896+0000", 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695894+0000", 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695902+0000", 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6270000000000006e-06 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695916+0000", 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3502999999999999e-05 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695923+0000", 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0049999999999998e-06 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695982+0000", 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8718999999999998e-05 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695991+0000", 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1819999999999995e-06 2023-09-23T18:16:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746475+0000", 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0504841819999999 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746584+0000", 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010859 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3", 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746783+0000", 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000199481 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.759330+0000", 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012547045999999999 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.766963+0000", 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0076329800000000001 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767016+0000", 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2680999999999999e-05 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767110+0000", 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3968000000000006e-05 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121643 9.a 9:511d9150:::benchmark_data_smithi055_386467_object16983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.723162+0000", 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 68.141938636999996, 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0495704269999999, 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:16:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121643 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723162+0000", 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723162+0000", 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:16:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723162+0000", 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723167+0000", 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1429999999999997e-06 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723177+0000", 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0135e-05 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723182+0000", 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9010000000000002e-06 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723231+0000", 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9101000000000003e-05 2023-09-23T18:16:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723239+0000", 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3499999999999997e-06 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.752774+0000", 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0295345709999999 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.752875+0000", 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010171 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.752981+0000", 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010507599999999999 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.759403+0000", 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064220290000000001 2023-09-23T18:16:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.763992+0000", 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045892930000000004 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.772571+0000", 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085792590000000005 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.772627+0000", 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5909999999999998e-05 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.772733+0000", 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010581000000000001 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:16:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:16:45.028 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:16:45.028 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:45.243 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:45.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:45.247 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:45.322 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:16:45.348 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:45.429 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:45.488 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:45.490 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[184174]: failed to encode map e525 with expected crc 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[184174]: failed to encode map e525 with expected crc 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2884447332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[179668]: failed to encode map e525 with expected crc 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[179668]: failed to encode map e525 with expected crc 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:45.491 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2884447332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:16:45.494 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:45.495 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:45.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:45 smithi099 ceph-mon[167076]: failed to encode map e525 with expected crc 2023-09-23T18:16:45.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:45 smithi099 ceph-mon[167076]: failed to encode map e525 with expected crc 2023-09-23T18:16:45.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:45 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:16:45.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:45 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:16:45.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:45 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:16:45.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2884447332' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:16:45.596 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:45.629 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:45.687 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:16:45.723 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:45.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:45.728 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:45.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:45.892 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:45.970 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:45.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:45.977 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:46.078 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:46.113 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:46.254 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:46.258 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:46.258 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:46.359 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:46.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[179668]: pgmap v1268: 119 pgs: 29 active+clean+remapped, 29 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded, 48 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 17138/119046 objects degraded (14.396%); 10183/119046 objects misplaced (8.554%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:16:46.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:46.379 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2884447332' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:16:46.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[179668]: osdmap e526: 8 total, 7 up, 8 in 2023-09-23T18:16:46.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[179668]: failed to encode map e526 with expected crc 2023-09-23T18:16:46.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[179668]: failed to encode map e526 with expected crc 2023-09-23T18:16:46.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[179668]: 1.a scrub starts 2023-09-23T18:16:46.380 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[179668]: 1.a scrub ok 2023-09-23T18:16:46.405 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:46.449 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:46.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:46.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:46.457 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:46.480 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:46.497 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:46.504 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:46.537 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:46.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:46.580 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:46.617 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:46.621 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:46.643 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:46.643 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:16:46.644 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:46.644 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:16:46.644 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:16:46.644 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:16:46.644 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:16:46.644 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:16:46.644 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:16:46.657 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[184174]: pgmap v1268: 119 pgs: 29 active+clean+remapped, 29 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded, 48 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 17138/119046 objects degraded (14.396%); 10183/119046 objects misplaced (8.554%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:16:46.657 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:46.657 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2884447332' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:16:46.658 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[184174]: osdmap e526: 8 total, 7 up, 8 in 2023-09-23T18:16:46.658 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[184174]: failed to encode map e526 with expected crc 2023-09-23T18:16:46.658 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[184174]: failed to encode map e526 with expected crc 2023-09-23T18:16:46.658 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[184174]: 1.a scrub starts 2023-09-23T18:16:46.658 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:46 smithi055 ceph-mon[184174]: 1.a scrub ok 2023-09-23T18:16:46.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:46.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:46.701 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:46.704 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:46 smithi099 ceph-mon[167076]: pgmap v1268: 119 pgs: 29 active+clean+remapped, 29 active+undersized, 1 active+undersized+remapped, 12 active+undersized+degraded, 48 active+clean; 2.4 GiB data, 15 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 769 op/s; 17138/119046 objects degraded (14.396%); 10183/119046 objects misplaced (8.554%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:16:46.704 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:46 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:16:46.705 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2884447332' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:16:46.705 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:46 smithi099 ceph-mon[167076]: osdmap e526: 8 total, 7 up, 8 in 2023-09-23T18:16:46.705 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:46 smithi099 ceph-mon[167076]: failed to encode map e526 with expected crc 2023-09-23T18:16:46.705 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:46 smithi099 ceph-mon[167076]: failed to encode map e526 with expected crc 2023-09-23T18:16:46.705 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:46 smithi099 ceph-mon[167076]: 1.a scrub starts 2023-09-23T18:16:46.705 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:46 smithi099 ceph-mon[167076]: 1.a scrub ok 2023-09-23T18:16:46.802 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:46.838 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:47.097 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:47.102 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:47.103 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:47.184 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:47.185 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:47.204 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:47.280 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:47.310 INFO:tasks.thrashosds.thrasher:in_osds: [7, 1, 0, 4, 2, 3, 6, 5] out_osds: [] dead_osds: [1] live_osds: [0, 4, 6, 7, 3, 5, 2] 2023-09-23T18:16:47.310 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:16:47.310 INFO:tasks.thrashosds.thrasher:Removing osd 7, in_osds are: [7, 1, 0, 4, 2, 3, 6, 5] 2023-09-23T18:16:47.311 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 7 2023-09-23T18:16:47.343 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:47.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:47.351 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:47.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:47.517 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:47.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:47 smithi099 ceph-mon[167076]: failed to encode map e526 with expected crc 2023-09-23T18:16:47.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:47 smithi099 ceph-mon[167076]: failed to encode map e526 with expected crc 2023-09-23T18:16:47.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2113493943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:47.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:47 smithi099 ceph-mon[167076]: 1.11 scrub starts 2023-09-23T18:16:47.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:47 smithi099 ceph-mon[167076]: 1.11 scrub ok 2023-09-23T18:16:47.602 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:47.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:47.612 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:47.622 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:47 smithi055 ceph-mon[184174]: failed to encode map e526 with expected crc 2023-09-23T18:16:47.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:47 smithi055 ceph-mon[184174]: failed to encode map e526 with expected crc 2023-09-23T18:16:47.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2113493943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:47.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:47 smithi055 ceph-mon[184174]: 1.11 scrub starts 2023-09-23T18:16:47.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:47 smithi055 ceph-mon[184174]: 1.11 scrub ok 2023-09-23T18:16:47.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:47 smithi055 ceph-mon[179668]: failed to encode map e526 with expected crc 2023-09-23T18:16:47.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:47 smithi055 ceph-mon[179668]: failed to encode map e526 with expected crc 2023-09-23T18:16:47.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2113493943' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:47.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:47 smithi055 ceph-mon[179668]: 1.11 scrub starts 2023-09-23T18:16:47.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:47 smithi055 ceph-mon[179668]: 1.11 scrub ok 2023-09-23T18:16:47.712 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:47.749 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:47.791 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:47.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:47.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:47.862 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:16:47.863 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:16:47.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:47.928 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:47.984 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:48.066 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:48.072 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:48.072 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:48.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:48.251 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:48.297 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:16:48.352 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:48.360 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:48.361 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:48.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:48.514 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:48 smithi099 ceph-mon[167076]: pgmap v1270: 119 pgs: 30 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 16255/111336 objects degraded (14.600%); 9390/111336 objects misplaced (8.434%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:16:48.514 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16255/111336 objects degraded (14.600%), 12 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:48.520 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:48.579 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:48.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:48.585 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:48.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:48 smithi055 ceph-mon[184174]: pgmap v1270: 119 pgs: 30 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 16255/111336 objects degraded (14.600%); 9390/111336 objects misplaced (8.434%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:16:48.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16255/111336 objects degraded (14.600%), 12 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:48.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:48 smithi055 ceph-mon[179668]: pgmap v1270: 119 pgs: 30 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.3 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 661 op/s; 16255/111336 objects degraded (14.600%); 9390/111336 objects misplaced (8.434%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:16:48.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16255/111336 objects degraded (14.600%), 12 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:48.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:48.762 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:48.850 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:48.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:48.858 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:48.959 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:49.034 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:49.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:49.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:49.117 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:49.186 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:16:49.218 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:49.278 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:49.352 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:49.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:49.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:49.461 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:49.556 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:49.621 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:49.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:49.627 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:49.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:49 smithi055 ceph-mon[184174]: 1.19 scrub starts 2023-09-23T18:16:49.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:49 smithi055 ceph-mon[184174]: 1.19 scrub ok 2023-09-23T18:16:49.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:49 smithi055 ceph-mon[179668]: 1.19 scrub starts 2023-09-23T18:16:49.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:49 smithi055 ceph-mon[179668]: 1.19 scrub ok 2023-09-23T18:16:49.728 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:49.779 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:49 smithi099 ceph-mon[167076]: 1.19 scrub starts 2023-09-23T18:16:49.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:49 smithi099 ceph-mon[167076]: 1.19 scrub ok 2023-09-23T18:16:49.784 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:49.850 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:49.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:49.856 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:49.956 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:49.992 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:50.305 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:16:50.337 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:50.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:50.340 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:50.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:50.475 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:50.539 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:50.547 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:50.547 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:50.569 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:50 smithi055 ceph-mon[184174]: pgmap v1271: 119 pgs: 30 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 15533/108309 objects degraded (14.341%); 9390/108309 objects misplaced (8.670%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:16:50.569 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:50 smithi055 ceph-mon[184174]: 1.18 scrub starts 2023-09-23T18:16:50.569 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:50 smithi055 ceph-mon[184174]: 1.18 scrub ok 2023-09-23T18:16:50.570 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:16:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:16:50] "GET /metrics HTTP/1.1" 200 36663 "" "Prometheus/2.43.0" 2023-09-23T18:16:50.570 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:50 smithi055 ceph-mon[179668]: pgmap v1271: 119 pgs: 30 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 15533/108309 objects degraded (14.341%); 9390/108309 objects misplaced (8.670%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:16:50.570 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:50 smithi055 ceph-mon[179668]: 1.18 scrub starts 2023-09-23T18:16:50.570 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:50 smithi055 ceph-mon[179668]: 1.18 scrub ok 2023-09-23T18:16:50.648 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:50.701 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:50.704 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:50.782 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:50.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:50 smithi099 ceph-mon[167076]: pgmap v1271: 119 pgs: 30 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.2 GiB data, 14 GiB used, 701 GiB / 715 GiB avail; 0 B/s wr, 645 op/s; 15533/108309 objects degraded (14.341%); 9390/108309 objects misplaced (8.670%); 6.5 MiB/s, 104 objects/s recovering 2023-09-23T18:16:50.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:50 smithi099 ceph-mon[167076]: 1.18 scrub starts 2023-09-23T18:16:50.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:50 smithi099 ceph-mon[167076]: 1.18 scrub ok 2023-09-23T18:16:50.878 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:50.882 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:50.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:50.983 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:51.040 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:51.100 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:51.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:51.109 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:51.152 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:51.209 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:51.262 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:51.311 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:51.316 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:51.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:51.417 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:51.498 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:51.592 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:51.602 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:51.603 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:51 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:16:51.408+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e526 definitely_dead 0 2023-09-23T18:16:51.703 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:51.799 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:51.855 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:51.863 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:51.863 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:51.964 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:51.997 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:52.089 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:52.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:52.094 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:52.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:52.262 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:52.325 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:52.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:52.336 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:52.437 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:52.445 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.7. 2023-09-23T18:16:52.503 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:52.535 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:52.537 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:52 smithi099 ceph-mon[167076]: pgmap v1272: 119 pgs: 30 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 14237/96858 objects degraded (14.699%); 8139/96858 objects misplaced (8.403%) 2023-09-23T18:16:52.537 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1334141677' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T18:16:52.537 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:52 smithi099 ceph-mon[167076]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T18:16:52.537 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:52.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:52.538 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:52.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:52 smithi055 ceph-mon[179668]: pgmap v1272: 119 pgs: 30 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 14237/96858 objects degraded (14.699%); 8139/96858 objects misplaced (8.403%) 2023-09-23T18:16:52.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1334141677' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T18:16:52.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:52 smithi055 ceph-mon[179668]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T18:16:52.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:52.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:52 smithi055 ceph-mon[184174]: pgmap v1272: 119 pgs: 30 active+clean+remapped, 29 active+undersized, 12 active+undersized+degraded, 48 active+clean; 2.0 GiB data, 14 GiB used, 702 GiB / 715 GiB avail; 0 B/s wr, 944 op/s; 14237/96858 objects degraded (14.699%); 8139/96858 objects misplaced (8.403%) 2023-09-23T18:16:52.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1334141677' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["7"]}]: dispatch 2023-09-23T18:16:52.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:52 smithi055 ceph-mon[184174]: Client client.admin marked osd.7 out, while it was still marked up 2023-09-23T18:16:52.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:16:52.639 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:52.668 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:52.913 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:52.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:52.921 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:53.022 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:53.049 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:53.082 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:53.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:53.086 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:53.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:53.213 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:53.245 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:53.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:53.248 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:53.250 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:16:53.250 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:16:53.285 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:16:53.348 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:53.395 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:53.414 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:53.441 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:53.461 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:53.481 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:53.501 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:53.522 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:53.543 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:53.543 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:16:53.543 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:53.543 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:16:53.543 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:16:53.543 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:16:53.544 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:16:53.544 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:16:53.544 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:16:53.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: 1.1e scrub starts 2023-09-23T18:16:53.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: 1.1e scrub ok 2023-09-23T18:16:53.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1334141677' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T18:16:53.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: osdmap e527: 8 total, 7 up, 7 in 2023-09-23T18:16:53.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.698 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1525788357' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 14237/96858 objects degraded (14.699%), 12 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: osdmap e528: 8 total, 7 up, 7 in 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.699 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2340540826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: 1.1e scrub starts 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: 1.1e scrub ok 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1334141677' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: osdmap e527: 8 total, 7 up, 7 in 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1525788357' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:53.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:53.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 14237/96858 objects degraded (14.699%), 12 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:53.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:16:53.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:53.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: osdmap e528: 8 total, 7 up, 7 in 2023-09-23T18:16:53.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2340540826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: 1.1e scrub starts 2023-09-23T18:16:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: 1.1e scrub ok 2023-09-23T18:16:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1334141677' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["7"]}]': finished 2023-09-23T18:16:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: osdmap e527: 8 total, 7 up, 7 in 2023-09-23T18:16:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: failed to encode map e527 with expected crc 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1525788357' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 14237/96858 objects degraded (14.699%), 12 pgs degraded, 37 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: osdmap e528: 8 total, 7 up, 7 in 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: failed to encode map e528 with expected crc 2023-09-23T18:16:53.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2340540826' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:54.089 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:16:54.126 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:54.126 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:54.170 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:54.173 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:54.174 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:54.274 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:16:54.329 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:54.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:16:54.518 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:54.519 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:16:54.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: pgmap v1274: 119 pgs: 2 peering, 2 unknown, 5 remapped+peering, 25 active+clean+remapped, 27 active+undersized, 12 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 907 op/s; 14237/96858 objects degraded (14.699%); 8139/96858 objects misplaced (8.403%) 2023-09-23T18:16:54.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: 9.2 starting backfill to osd.0 from (0'0,0'0] MAX to 526'11296 2023-09-23T18:16:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:16:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: osdmap e529: 8 total, 7 up, 7 in 2023-09-23T18:16:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: 9.9 starting backfill to osd.6 from (0'0,0'0] MAX to 526'11149 2023-09-23T18:16:54.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:54 smithi099 ceph-mon[167076]: 9.a starting backfill to osd.6 from (478'4468,508'8761] MIN to 526'11171 2023-09-23T18:16:54.620 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:54.660 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:54.700 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:54.703 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:54.704 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:54.743 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:54.805 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:16:54.897 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:54.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: pgmap v1274: 119 pgs: 2 peering, 2 unknown, 5 remapped+peering, 25 active+clean+remapped, 27 active+undersized, 12 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 907 op/s; 14237/96858 objects degraded (14.699%); 8139/96858 objects misplaced (8.403%) 2023-09-23T18:16:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: 9.2 starting backfill to osd.0 from (0'0,0'0] MAX to 526'11296 2023-09-23T18:16:54.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:16:54.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: osdmap e529: 8 total, 7 up, 7 in 2023-09-23T18:16:54.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: 9.9 starting backfill to osd.6 from (0'0,0'0] MAX to 526'11149 2023-09-23T18:16:54.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[184174]: 9.a starting backfill to osd.6 from (478'4468,508'8761] MIN to 526'11171 2023-09-23T18:16:54.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: pgmap v1274: 119 pgs: 2 peering, 2 unknown, 5 remapped+peering, 25 active+clean+remapped, 27 active+undersized, 12 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 907 op/s; 14237/96858 objects degraded (14.699%); 8139/96858 objects misplaced (8.403%) 2023-09-23T18:16:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: failed to encode map e528 with expected crc 2023-09-23T18:16:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: 9.2 starting backfill to osd.0 from (0'0,0'0] MAX to 526'11296 2023-09-23T18:16:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 2 pgs inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:16:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: osdmap e529: 8 total, 7 up, 7 in 2023-09-23T18:16:54.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: failed to encode map e529 with expected crc 2023-09-23T18:16:54.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: 9.9 starting backfill to osd.6 from (0'0,0'0] MAX to 526'11149 2023-09-23T18:16:54.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:54 smithi055 ceph-mon[179668]: 9.a starting backfill to osd.6 from (478'4468,508'8761] MIN to 526'11171 2023-09-23T18:16:54.963 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:16:54.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:54.973 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:16:55.074 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:55.110 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:55.418 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:55.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:55.422 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:55.473 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.473 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 526'11318 2023-09-23T18:16:55.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: 9.c starting backfill to osd.6 from (0'0,0'0] MAX to 526'11248 2023-09-23T18:16:55.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 526'11182 2023-09-23T18:16:55.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 526'11421 2023-09-23T18:16:55.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: 9.d starting backfill to osd.4 from (478'4333,508'8687] MIN to 526'11041 2023-09-23T18:16:55.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: osdmap e530: 8 total, 7 up, 7 in 2023-09-23T18:16:55.474 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.475 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.475 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.475 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.475 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.475 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.523 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:55.559 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:55.599 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:55.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:55.602 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:55.703 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:55.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 526'11318 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: 9.c starting backfill to osd.6 from (0'0,0'0] MAX to 526'11248 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 526'11182 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 526'11421 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: 9.d starting backfill to osd.4 from (478'4333,508'8687] MIN to 526'11041 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: osdmap e530: 8 total, 7 up, 7 in 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.736 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.737 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:55 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.739 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:55.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.833 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: 9.3 starting backfill to osd.5 from (0'0,0'0] MAX to 526'11318 2023-09-23T18:16:55.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e529 with expected crc 2023-09-23T18:16:55.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: 9.c starting backfill to osd.6 from (0'0,0'0] MAX to 526'11248 2023-09-23T18:16:55.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: 9.1 starting backfill to osd.4 from (0'0,0'0] MAX to 526'11182 2023-09-23T18:16:55.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: 9.e starting backfill to osd.0 from (0'0,0'0] MAX to 526'11421 2023-09-23T18:16:55.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: 9.d starting backfill to osd.4 from (478'4333,508'8687] MIN to 526'11041 2023-09-23T18:16:55.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: osdmap e530: 8 total, 7 up, 7 in 2023-09-23T18:16:55.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.834 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.835 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:55 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:55.837 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:55.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:55.841 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:55.916 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:16:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:16:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:16:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:16:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65522 7.f 7:f4ce2e57:::benchmark_data_smithi055_311528_object65521:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T18:16:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:55.973972+0000", 2023-09-23T18:16:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 599.93851366700005, 2023-09-23T18:16:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2270217859999999, 2023-09-23T18:16:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:16:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65522 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.973972+0000", 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.973972+0000", 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.973969+0000", 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.974001+0000", 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1111000000000001e-05 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.974022+0000", 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1293000000000001e-05 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.974037+0000", 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4935e-05 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.169722+0000", 2023-09-23T18:16:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1956848210000004 2023-09-23T18:16:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.169899+0000", 2023-09-23T18:16:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000177849 2023-09-23T18:16:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:16:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.170109+0000", 2023-09-23T18:16:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000209667 2023-09-23T18:16:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177841+0000", 2023-09-23T18:16:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0077319750000000003 2023-09-23T18:16:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177878+0000", 2023-09-23T18:16:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6709999999999999e-05 2023-09-23T18:16:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198539+0000", 2023-09-23T18:16:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020661536000000001 2023-09-23T18:16:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198675+0000", 2023-09-23T18:16:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013579400000000001 2023-09-23T18:16:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198720+0000", 2023-09-23T18:16:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5111999999999998e-05 2023-09-23T18:16:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.200994+0000", 2023-09-23T18:16:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022732540000000002 2023-09-23T18:16:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:65531 7.f 7:fb49bebd:::benchmark_data_smithi055_311528_object65530:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e308)", 2023-09-23T18:16:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:06:55.993528+0000", 2023-09-23T18:16:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 599.91895712300004, 2023-09-23T18:16:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2096306049999992, 2023-09-23T18:16:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:16:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:16:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 65531 2023-09-23T18:16:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993528+0000", 2023-09-23T18:16:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993528+0000", 2023-09-23T18:16:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993526+0000", 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993552+0000", 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5392e-05 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993568+0000", 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6087e-05 2023-09-23T18:16:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:06:55.993577+0000", 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8240000000000006e-06 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.172136+0000", 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1785597140000004 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.172295+0000", 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015823599999999999 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,5,7", 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.172472+0000", 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000176918 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177885+0000", 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0054133879999999999 2023-09-23T18:16:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.177931+0000", 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5782000000000003e-05 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.198584+0000", 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020652818999999999 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.201065+0000", 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024819600000000001 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.201131+0000", 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5105999999999996e-05 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:07:04.203159+0000", 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020283129999999999 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:126230 8.8 8:1237b1d1:::benchmark_data_smithi055_346788_object15638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:24.368296+0000", 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.54418928400003, 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.418939742, 2023-09-23T18:16:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 126230 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368296+0000", 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368296+0000", 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368294+0000", 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368300+0000", 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9560000000000002e-06 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368318+0000", 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7795000000000002e-05 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368328+0000", 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4790000000000002e-06 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.457382+0000", 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.089054340999999995 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.457459+0000", 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6725999999999997e-05 2023-09-23T18:16:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.692919+0000", 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.235460265 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.693016+0000", 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6869999999999999e-05 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:16:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.693104+0000", 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7700000000000004e-05 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.718266+0000", 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.025162766 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.783487+0000", 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.065220847999999998 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.783537+0000", 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0043000000000002e-05 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.783574+0000", 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6875000000000002e-05 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.787236+0000", 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003661756 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:126237 8.8 8:1cdbad3c:::benchmark_data_smithi055_346788_object15645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:24.377458+0000", 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.53502788499998, 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.48202789600000001, 2023-09-23T18:16:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 126237 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377458+0000", 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377458+0000", 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377456+0000", 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377463+0000", 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7309999999999996e-06 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377476+0000", 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2894999999999999e-05 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377483+0000", 2023-09-23T18:16:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.374e-06 2023-09-23T18:16:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.517115+0000", 2023-09-23T18:16:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13963230200000001 2023-09-23T18:16:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.517177+0000", 2023-09-23T18:16:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1897999999999996e-05 2023-09-23T18:16:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.825406+0000", 2023-09-23T18:16:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.30822874099999997 2023-09-23T18:16:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.825506+0000", 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010000099999999999 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.825640+0000", 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013374700000000001 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.830736+0000", 2023-09-23T18:16:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.005096125 2023-09-23T18:16:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.855124+0000", 2023-09-23T18:16:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024388184 2023-09-23T18:16:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.855433+0000", 2023-09-23T18:16:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00030889699999999998 2023-09-23T18:16:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.855497+0000", 2023-09-23T18:16:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4228999999999998e-05 2023-09-23T18:16:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.859485+0000", 2023-09-23T18:16:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039881949999999999 2023-09-23T18:16:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:126259 8.8 8:15b6e0af:::benchmark_data_smithi055_346788_object15667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2023-09-23T18:16:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:24.416937+0000", 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.49554830699998, 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.48747808799999998, 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 126259 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416937+0000", 2023-09-23T18:16:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416937+0000", 2023-09-23T18:16:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:55.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416936+0000", 2023-09-23T18:16:55.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:55.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:55.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416943+0000", 2023-09-23T18:16:55.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4170000000000003e-06 2023-09-23T18:16:55.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:55.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416971+0000", 2023-09-23T18:16:55.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7552e-05 2023-09-23T18:16:55.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:55.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416979+0000", 2023-09-23T18:16:55.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9589999999999995e-06 2023-09-23T18:16:55.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.585139+0000", 2023-09-23T18:16:55.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.16816076399999999 2023-09-23T18:16:55.956 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.956 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:55.956 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.585243+0000", 2023-09-23T18:16:55.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000103185 2023-09-23T18:16:55.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.840971+0000", 2023-09-23T18:16:55.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25572825999999999 2023-09-23T18:16:55.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:55.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.841051+0000", 2023-09-23T18:16:55.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9952999999999998e-05 2023-09-23T18:16:55.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:16:55.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.841149+0000", 2023-09-23T18:16:55.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8142999999999999e-05 2023-09-23T18:16:55.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.855376+0000", 2023-09-23T18:16:55.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014226664999999999 2023-09-23T18:16:55.959 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.959 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:55.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.855389+0000", 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3842999999999999e-05 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.900533+0000", 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.045143100999999998 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.900571+0000", 2023-09-23T18:16:55.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8325999999999999e-05 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.904415+0000", 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038443079999999998 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPushReply(8.8 396/394 [PushReplyOp(8:1cdbad3c:::benchmark_data_smithi055_346788_object15645:head)])", 2023-09-23T18:16:55.961 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:24.520324+0000", 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.39216128800001, 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.30352263200000001, 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520324+0000", 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520324+0000", 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520323+0000", 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520335+0000", 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1651000000000001e-05 2023-09-23T18:16:55.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520354+0000", 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9352e-05 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520364+0000", 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5079999999999994e-06 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.815627+0000", 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.29526276000000001 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.823847+0000", 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082201010000000005 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.965 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:126427 8.8 8:11ddd72a:::benchmark_data_smithi055_346788_object15835:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:24.716103+0000", 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 331.196382503, 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.28519597000000002, 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 126427 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.966 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716103+0000", 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716103+0000", 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716101+0000", 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716110+0000", 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1800000000000002e-06 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:55.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716133+0000", 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2909999999999999e-05 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716143+0000", 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9180000000000006e-06 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.815517+0000", 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.099374219999999999 2023-09-23T18:16:55.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.815593+0000", 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5104999999999993e-05 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.961822+0000", 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14622923400000001 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.961937+0000", 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114757 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.962090+0000", 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000153661 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.997809+0000", 2023-09-23T18:16:55.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.035718911999999998 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.997828+0000", 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9052e-05 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.997869+0000", 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0373999999999997e-05 2023-09-23T18:16:55.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.997928+0000", 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9286999999999997e-05 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.001299+0000", 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033710820000000001 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:126952 8.8 8:18980ef4:::benchmark_data_smithi055_346788_object16360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:25.392938+0000", 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 330.51954706599997, 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.28906703900000003, 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:55.974 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 126952 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392938+0000", 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392938+0000", 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392937+0000", 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392942+0000", 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2089999999999996e-06 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392954+0000", 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2072999999999999e-05 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392962+0000", 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8159999999999994e-06 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.449589+0000", 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.056626870000000003 2023-09-23T18:16:55.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:55.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.449675+0000", 2023-09-23T18:16:55.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6088999999999995e-05 2023-09-23T18:16:55.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.608823+0000", 2023-09-23T18:16:55.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.159147594 2023-09-23T18:16:55.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:55.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.608928+0000", 2023-09-23T18:16:55.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000105678 2023-09-23T18:16:55.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:16:55.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.609073+0000", 2023-09-23T18:16:55.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014435500000000001 2023-09-23T18:16:55.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.627538+0000", 2023-09-23T18:16:55.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018464960999999998 2023-09-23T18:16:55.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:55.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.627548+0000", 2023-09-23T18:16:55.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0257e-05 2023-09-23T18:16:55.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.981 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.678648+0000", 2023-09-23T18:16:55.981 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.051100356 2023-09-23T18:16:55.981 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.981 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.981 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:55.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.678689+0000", 2023-09-23T18:16:55.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1037e-05 2023-09-23T18:16:55.982 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.982 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.982 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:55.982 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.682005+0000", 2023-09-23T18:16:55.982 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003316041 2023-09-23T18:16:55.982 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.982 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:55.983 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.983 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.983 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.983 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:127014 8.8 8:1461082a:::benchmark_data_smithi055_346788_object16422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:16:55.983 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:25.470801+0000", 2023-09-23T18:16:55.983 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 330.44168424999998, 2023-09-23T18:16:55.983 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32388431200000001, 2023-09-23T18:16:55.983 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.983 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:55.984 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:55.984 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:16:55.984 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:55.984 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 127014 2023-09-23T18:16:55.984 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.984 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.984 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.984 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.984 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470801+0000", 2023-09-23T18:16:55.984 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470801+0000", 2023-09-23T18:16:55.985 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.985 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.985 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.985 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:55.985 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470799+0000", 2023-09-23T18:16:55.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:16:55.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:55.986 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470809+0000", 2023-09-23T18:16:55.986 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1560000000000001e-06 2023-09-23T18:16:55.986 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.986 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.986 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:55.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470824+0000", 2023-09-23T18:16:55.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5006e-05 2023-09-23T18:16:55.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.987 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:55.987 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470838+0000", 2023-09-23T18:16:55.987 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.446e-05 2023-09-23T18:16:55.987 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.987 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.533922+0000", 2023-09-23T18:16:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.063083976 2023-09-23T18:16:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.533990+0000", 2023-09-23T18:16:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8493999999999998e-05 2023-09-23T18:16:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.988 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.752999+0000", 2023-09-23T18:16:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.21900887999999999 2023-09-23T18:16:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.753123+0000", 2023-09-23T18:16:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012370099999999999 2023-09-23T18:16:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.989 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:16:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.753310+0000", 2023-09-23T18:16:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000186629 2023-09-23T18:16:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.779553+0000", 2023-09-23T18:16:55.990 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026243391000000001 2023-09-23T18:16:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.791116+0000", 2023-09-23T18:16:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011562862 2023-09-23T18:16:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.991 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.791169+0000", 2023-09-23T18:16:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2855999999999997e-05 2023-09-23T18:16:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.791227+0000", 2023-09-23T18:16:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7828000000000003e-05 2023-09-23T18:16:55.992 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.993 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:55.993 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.794685+0000", 2023-09-23T18:16:55.993 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034588449999999999 2023-09-23T18:16:55.993 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.993 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:55.993 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.993 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.993 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.993 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPushReply(8.8 396/394 [PushReplyOp(8:1461082a:::benchmark_data_smithi055_346788_object16422:head)])", 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:25.477212+0000", 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 330.435273446, 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.27574391999999998, 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477212+0000", 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.994 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477212+0000", 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477211+0000", 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477219+0000", 2023-09-23T18:16:55.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7479999999999993e-06 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477230+0000", 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1585e-05 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477239+0000", 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3669999999999997e-06 2023-09-23T18:16:55.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.745071+0000", 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.26783221600000001 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.752956+0000", 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0078849369999999998 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:55.997 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:127070 8.8 8:14141a6a:::benchmark_data_smithi055_346788_object16478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:25.535150+0000", 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 330.37733524599997, 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.30308834499999998, 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:16:55.998 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 127070 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535150+0000", 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535150+0000", 2023-09-23T18:16:55.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535149+0000", 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535157+0000", 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7619999999999995e-06 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535173+0000", 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.683e-05 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535199+0000", 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5420999999999999e-05 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.549349+0000", 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014150107 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.549399+0000", 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0049000000000001e-05 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.810675+0000", 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.26127647399999998 2023-09-23T18:16:56.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.810746+0000", 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0378999999999998e-05 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.810830+0000", 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4566000000000003e-05 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.003 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.820368+0000", 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0095380090000000001 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.820382+0000", 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.343e-05 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.834163+0000", 2023-09-23T18:16:56.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013781411 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.834233+0000", 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9759999999999996e-05 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.838239+0000", 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040055519999999999 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.005 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:128395 8.8 8:10f4d9ca:::benchmark_data_smithi055_346788_object17803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:27.281591+0000", 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 328.63089479899998, 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.29826881199999999, 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:56.006 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 128395 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281591+0000", 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:56.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281591+0000", 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281590+0000", 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281595+0000", 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5439999999999998e-06 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281608+0000", 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3247999999999999e-05 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281616+0000", 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7559999999999999e-06 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.482164+0000", 2023-09-23T18:16:56.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.20054759799999999 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.482252+0000", 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8307999999999995e-05 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.508276+0000", 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026023748999999999 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.011 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:56.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.508349+0000", 2023-09-23T18:16:56.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3506999999999999e-05 2023-09-23T18:16:56.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.011 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:16:56.011 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.508453+0000", 2023-09-23T18:16:56.011 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000103609 2023-09-23T18:16:56.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.527324+0000", 2023-09-23T18:16:56.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018870735 2023-09-23T18:16:56.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.012 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.012 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:56.012 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.540632+0000", 2023-09-23T18:16:56.012 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013307941 2023-09-23T18:16:56.012 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.576091+0000", 2023-09-23T18:16:56.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.035459813 2023-09-23T18:16:56.013 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.013 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.013 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:56.013 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.576138+0000", 2023-09-23T18:16:56.013 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6461999999999997e-05 2023-09-23T18:16:56.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.014 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.014 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:56.014 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.579859+0000", 2023-09-23T18:16:56.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037215429999999999 2023-09-23T18:16:56.014 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.014 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.014 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.014 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.015 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:173525 8.8 8:11c377cb:::benchmark_data_smithi055_346788_object62933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-09-23T18:16:56.015 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:27.592880+0000", 2023-09-23T18:16:56.015 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 268.31960501999998, 2023-09-23T18:16:56.015 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.27587297500000002, 2023-09-23T18:16:56.015 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:56.015 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:56.015 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:56.015 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:16:56.016 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:16:56.016 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 173525 2023-09-23T18:16:56.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.016 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:56.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:56.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592880+0000", 2023-09-23T18:16:56.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:56.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592880+0000", 2023-09-23T18:16:56.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:56.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592879+0000", 2023-09-23T18:16:56.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:56.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:56.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592884+0000", 2023-09-23T18:16:56.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4500000000000003e-06 2023-09-23T18:16:56.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:56.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592895+0000", 2023-09-23T18:16:56.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0366e-05 2023-09-23T18:16:56.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:56.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592901+0000", 2023-09-23T18:16:56.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1600000000000003e-06 2023-09-23T18:16:56.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.653327+0000", 2023-09-23T18:16:56.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.060425660999999999 2023-09-23T18:16:56.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:16:56.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.653398+0000", 2023-09-23T18:16:56.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1755000000000007e-05 2023-09-23T18:16:56.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.758477+0000", 2023-09-23T18:16:56.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10507913000000001 2023-09-23T18:16:56.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:56.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.758540+0000", 2023-09-23T18:16:56.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2232999999999999e-05 2023-09-23T18:16:56.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:16:56.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.758626+0000", 2023-09-23T18:16:56.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6231999999999997e-05 2023-09-23T18:16:56.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.022 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.022 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.764380+0000", 2023-09-23T18:16:56.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0057542340000000004 2023-09-23T18:16:56.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:56.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.765155+0000", 2023-09-23T18:16:56.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00077486000000000005 2023-09-23T18:16:56.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.866705+0000", 2023-09-23T18:16:56.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10154997 2023-09-23T18:16:56.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:56.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.866758+0000", 2023-09-23T18:16:56.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3186000000000002e-05 2023-09-23T18:16:56.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.868753+0000", 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019952580000000002 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:179817 9.3 9:c914105c:::benchmark_data_smithi055_386467_object75157:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e529)", 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:54.952303+0000", 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.960182812, 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024556786000000001, 2023-09-23T18:16:56.025 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 179817 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952303+0000", 2023-09-23T18:16:56.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952303+0000", 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952302+0000", 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952307+0000", 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2780000000000001e-06 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952322+0000", 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5062000000000001e-05 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:56.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952329+0000", 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1369999999999996e-06 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.953474+0000", 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011444669999999999 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.953631+0000", 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015719 2023-09-23T18:16:56.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.953786+0000", 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000154888 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.958834+0000", 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050480129999999996 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.030 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.966926+0000", 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0080918559999999997 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.976676+0000", 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0097507099999999992 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.976730+0000", 2023-09-23T18:16:56.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4012999999999999e-05 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.976859+0000", 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012904299999999999 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:179819 9.3 9:caa5baf6:::benchmark_data_smithi055_386467_object75159:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e529)", 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:54.952336+0000", 2023-09-23T18:16:56.032 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.96014982699999996, 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024783891999999998, 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 179819 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.033 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952336+0000", 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952336+0000", 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:56.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952335+0000", 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952338+0000", 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.225e-06 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952345+0000", 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8600000000000004e-06 2023-09-23T18:16:56.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.037 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:56.037 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.952350+0000", 2023-09-23T18:16:56.037 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3710000000000002e-06 2023-09-23T18:16:56.037 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.037 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.955773+0000", 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034234090000000001 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.955886+0000", 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000113199 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:16:56.038 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.955961+0000", 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4878999999999999e-05 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.962654+0000", 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0066928659999999996 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.966952+0000", 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0042976200000000003 2023-09-23T18:16:56.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.976929+0000", 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0099775230000000003 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.976992+0000", 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2969e-05 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.977119+0000", 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000127417 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:179822 9.3 9:c9747c4e:::benchmark_data_smithi055_386467_object75162:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e529)", 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:54.955823+0000", 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.95666244, 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021489642999999999, 2023-09-23T18:16:56.041 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 179822 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.955823+0000", 2023-09-23T18:16:56.042 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.043 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:56.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.955823+0000", 2023-09-23T18:16:56.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.043 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:56.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.955822+0000", 2023-09-23T18:16:56.044 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:16:56.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.044 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:56.044 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.955827+0000", 2023-09-23T18:16:56.044 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3989999999999997e-06 2023-09-23T18:16:56.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.044 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:56.044 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.955836+0000", 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1030000000000008e-06 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.955841+0000", 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4140000000000002e-06 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.957688+0000", 2023-09-23T18:16:56.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018473770000000001 2023-09-23T18:16:56.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.046 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:56.046 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.957771+0000", 2023-09-23T18:16:56.046 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2526000000000002e-05 2023-09-23T18:16:56.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.046 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.957843+0000", 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1970000000000004e-05 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.965119+0000", 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072761520000000001 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.047 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.966974+0000", 2023-09-23T18:16:56.048 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018548880000000001 2023-09-23T18:16:56.048 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.048 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.048 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.048 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.977176+0000", 2023-09-23T18:16:56.048 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010202149000000001 2023-09-23T18:16:56.048 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.048 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.048 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:56.049 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.977219+0000", 2023-09-23T18:16:56.049 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2846999999999998e-05 2023-09-23T18:16:56.049 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.049 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.049 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:56.049 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.977313+0000", 2023-09-23T18:16:56.049 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3638e-05 2023-09-23T18:16:56.049 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.049 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.049 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.050 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.050 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.050 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:179823 9.3 9:cf941829:::benchmark_data_smithi055_386467_object75163:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e529)", 2023-09-23T18:16:56.050 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:54.958597+0000", 2023-09-23T18:16:56.050 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.95388832400000001, 2023-09-23T18:16:56.050 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021843812000000001, 2023-09-23T18:16:56.050 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:56.050 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:56.050 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:56.051 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:16:56.051 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:56.051 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 179823 2023-09-23T18:16:56.051 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.051 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:56.051 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.051 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:56.051 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.958597+0000", 2023-09-23T18:16:56.051 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.052 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.052 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:56.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.958597+0000", 2023-09-23T18:16:56.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.052 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.052 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:56.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.958596+0000", 2023-09-23T18:16:56.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:16:56.053 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.053 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.053 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:56.053 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.958601+0000", 2023-09-23T18:16:56.053 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4510000000000002e-06 2023-09-23T18:16:56.053 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.053 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.053 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:56.053 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.958612+0000", 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1486e-05 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.958618+0000", 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4720000000000003e-06 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.960000+0000", 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013825129999999999 2023-09-23T18:16:56.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.960121+0000", 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000121093 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.960264+0000", 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000142077 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:56.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.965129+0000", 2023-09-23T18:16:56.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048658 2023-09-23T18:16:56.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.976563+0000", 2023-09-23T18:16:56.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011433731000000001 2023-09-23T18:16:56.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.980269+0000", 2023-09-23T18:16:56.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037059150000000002 2023-09-23T18:16:56.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:56.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.980333+0000", 2023-09-23T18:16:56.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3627000000000006e-05 2023-09-23T18:16:56.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:56.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.980441+0000", 2023-09-23T18:16:56.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000108319 2023-09-23T18:16:56.058 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.058 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.058 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.059 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:179824 9.3 9:c356a603:::benchmark_data_smithi055_386467_object75164:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e529)", 2023-09-23T18:16:56.059 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:54.960060+0000", 2023-09-23T18:16:56.059 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.95242559400000004, 2023-09-23T18:16:56.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.020547798999999999, 2023-09-23T18:16:56.059 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:56.059 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:56.059 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:56.059 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:16:56.059 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:56.059 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 179824 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.960060+0000", 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.960060+0000", 2023-09-23T18:16:56.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.960059+0000", 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.960065+0000", 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7680000000000003e-06 2023-09-23T18:16:56.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.960078+0000", 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2850999999999999e-05 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.960087+0000", 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7409999999999998e-06 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.962711+0000", 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026238960000000001 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.962875+0000", 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016393199999999999 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:16:56.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.963035+0000", 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016018799999999999 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.965135+0000", 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021001959999999999 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.976631+0000", 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011496282 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.980507+0000", 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038755679999999998 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.980537+0000", 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0154999999999998e-05 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:54.980608+0000", 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0450000000000005e-05 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:180139 9.3 9:cb1db29e:::benchmark_data_smithi055_386467_object75479:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e529)", 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:55.282499+0000", 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.62998646599999997, 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024627234000000001, 2023-09-23T18:16:56.066 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 180139 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.282499+0000", 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.282499+0000", 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.282498+0000", 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.282503+0000", 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1070000000000004e-06 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.282515+0000", 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2485e-05 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.282520+0000", 2023-09-23T18:16:56.069 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0860000000000001e-06 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.282816+0000", 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00029578699999999999 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.282979+0000", 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016247399999999999 2023-09-23T18:16:56.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.283134+0000", 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015534199999999999 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.286200+0000", 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003065646 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.297278+0000", 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011078609 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.306955+0000", 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0096766260000000007 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:56.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.307048+0000", 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3084000000000005e-05 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.307126+0000", 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8220999999999995e-05 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:180661 9.3 9:c258f9d7:::benchmark_data_smithi055_386467_object76001:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e530)", 2023-09-23T18:16:56.073 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:55.868290+0000", 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.044195673999999997, 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0095924930000000005, 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 180661 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.868290+0000", 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.868290+0000", 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:16:56.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.868288+0000", 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.868296+0000", 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2819999999999999e-06 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.868311+0000", 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5271e-05 2023-09-23T18:16:56.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.868317+0000", 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0580000000000003e-06 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.868356+0000", 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9566e-05 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.868451+0000", 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4667999999999996e-05 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.868527+0000", 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5712999999999998e-05 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.870589+0000", 2023-09-23T18:16:56.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020625629999999999 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.876584+0000", 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0059949599999999997 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.877750+0000", 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001165948 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:16:56.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.877789+0000", 2023-09-23T18:16:56.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8902000000000001e-05 2023-09-23T18:16:56.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:16:56.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:16:56.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:16:56.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:55.877882+0000", 2023-09-23T18:16:56.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3067999999999997e-05 2023-09-23T18:16:56.080 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.081 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.081 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.081 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:16:56.081 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:16:56.081 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:16:56.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:56.128 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:16:56.145 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:56.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:56.222 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:16:56.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:56.226 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:56.235 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:16:56.326 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:56.360 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:56.407 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:56.412 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:56.413 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:56.481 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: pgmap v1277: 119 pgs: 2 peering, 2 unknown, 5 remapped+peering, 25 active+clean+remapped, 27 active+undersized, 12 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 14237/96858 objects degraded (14.699%); 8139/96858 objects misplaced (8.403%) 2023-09-23T18:16:56.481 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.481 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.481 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.481 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.481 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.481 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.481 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.481 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.482 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: osdmap e531: 8 total, 7 up, 7 in 2023-09-23T18:16:56.482 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.482 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.482 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.482 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:56 smithi099 ceph-mon[167076]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.513 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:56.598 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:56.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:56.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:56.676 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: pgmap v1277: 119 pgs: 2 peering, 2 unknown, 5 remapped+peering, 25 active+clean+remapped, 27 active+undersized, 12 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 14237/96858 objects degraded (14.699%); 8139/96858 objects misplaced (8.403%) 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: osdmap e531: 8 total, 7 up, 7 in 2023-09-23T18:16:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[179668]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: pgmap v1277: 119 pgs: 2 peering, 2 unknown, 5 remapped+peering, 25 active+clean+remapped, 27 active+undersized, 12 active+undersized+degraded, 46 active+clean; 2.0 GiB data, 11 GiB used, 614 GiB / 626 GiB avail; 0 B/s wr, 613 op/s; 14237/96858 objects degraded (14.699%); 8139/96858 objects misplaced (8.403%) 2023-09-23T18:16:56.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e530 with expected crc 2023-09-23T18:16:56.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: osdmap e531: 8 total, 7 up, 7 in 2023-09-23T18:16:56.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:56 smithi055 ceph-mon[184174]: failed to encode map e531 with expected crc 2023-09-23T18:16:56.777 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:56.819 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:56.822 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:56.862 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:56.866 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:56.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:56.961 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:56.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:57.034 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:16:57.079 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:57.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:57.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:57.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:57.327 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:57.400 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:57.483 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:57.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:57.490 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:57.591 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:16:57.626 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[184174]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.627 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[184174]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.627 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[184174]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.627 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[184174]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.627 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[184174]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.627 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[184174]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[179668]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[179668]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.627 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[179668]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[179668]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[179668]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.628 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:57 smithi055 ceph-mon[179668]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.629 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:57.681 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:16:57.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:57.687 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:16:57.740 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:57 smithi099 ceph-mon[167076]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.740 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:57 smithi099 ceph-mon[167076]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.740 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:57 smithi099 ceph-mon[167076]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.741 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:57 smithi099 ceph-mon[167076]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.741 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:57 smithi099 ceph-mon[167076]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.741 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:57 smithi099 ceph-mon[167076]: failed to encode map e531 with expected crc 2023-09-23T18:16:57.787 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:57.823 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:57.863 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:57.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:57.870 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:57.940 INFO:tasks.thrashosds.thrasher:in_osds: [1, 0, 4, 2, 3, 6, 5] out_osds: [7] dead_osds: [1] live_osds: [0, 4, 6, 7, 3, 5, 2] 2023-09-23T18:16:57.940 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:16:57.940 INFO:tasks.thrashosds.thrasher:Removing osd 2, in_osds are: [1, 0, 4, 2, 3, 6, 5] 2023-09-23T18:16:57.940 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 2 2023-09-23T18:16:57.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:58.086 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:58.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:58.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:58.176 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:58.276 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:16:58.323 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:58.375 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:16:58.385 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:16:58.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:58.390 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:16:58.408 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:16:58.431 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:16:58.479 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:16:58.491 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:16:58.533 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:16:58.558 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:16:58.570 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:58.588 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:16:58.589 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:16:58.589 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:16:58.589 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:16:58.589 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:16:58.589 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:16:58.589 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:16:58.589 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:16:58.589 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:16:58.604 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:58 smithi055 ceph-mon[179668]: pgmap v1280: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 11 peering, 2 unknown, 5 remapped+peering, 44 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+undersized+degraded, 26 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 5.0 KiB/s rd, 13 KiB/s wr, 809 op/s; 12773/87180 objects degraded (14.651%); 22776/87180 objects misplaced (26.125%); 11 MiB/s, 205 objects/s recovering 2023-09-23T18:16:58.605 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 12773/87180 objects degraded (14.651%), 9 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:58.605 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:58 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/520072962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:58.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:58 smithi055 ceph-mon[184174]: pgmap v1280: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 11 peering, 2 unknown, 5 remapped+peering, 44 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+undersized+degraded, 26 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 5.0 KiB/s rd, 13 KiB/s wr, 809 op/s; 12773/87180 objects degraded (14.651%); 22776/87180 objects misplaced (26.125%); 11 MiB/s, 205 objects/s recovering 2023-09-23T18:16:58.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 12773/87180 objects degraded (14.651%), 9 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:58.605 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:58 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/520072962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:58.670 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:16:58.675 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:58.675 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:16:58.746 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:16:58.776 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:16:58.815 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:58.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:58 smithi099 ceph-mon[167076]: pgmap v1280: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 11 peering, 2 unknown, 5 remapped+peering, 44 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 4 active+undersized+degraded, 26 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 5.0 KiB/s rd, 13 KiB/s wr, 809 op/s; 12773/87180 objects degraded (14.651%); 22776/87180 objects misplaced (26.125%); 11 MiB/s, 205 objects/s recovering 2023-09-23T18:16:58.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 12773/87180 objects degraded (14.651%), 9 pgs degraded, 20 pgs undersized (PG_DEGRADED) 2023-09-23T18:16:58.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:58 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/520072962' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:16:58.843 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:16:58.844 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:16:58.910 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:16:58.913 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:58.913 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:16:59.014 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:16:59.114 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:16:59.457 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:16:59.458 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:16:59.507 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/462146053' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:59.507 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:16:59 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:59.538 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:16:59.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:59 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/462146053' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:59.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:16:59 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:59.894 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:59 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/462146053' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:59.894 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:16:59 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:16:59.929 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:16:59.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:16:59.934 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:00.016 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:17:00.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:00.141 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:00.239 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:00.248 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:00.248 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:00.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:00.450 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:00.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:00 smithi099 ceph-mon[167076]: pgmap v1281: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 9 peering, 5 remapped+peering, 46 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+undersized+degraded, 28 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 111 KiB/s rd, 48 KiB/s wr, 706 op/s; 12089/85917 objects degraded (14.071%); 24437/85917 objects misplaced (28.443%); 11 MiB/s, 195 objects/s recovering 2023-09-23T18:17:00.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:00 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-09-23T18:17:00.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:00 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:00.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:00 smithi099 ceph-mon[167076]: failed to encode map e532 with expected crc 2023-09-23T18:17:00.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:00 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:17:00.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:00 smithi099 ceph-mon[167076]: osdmap e532: 8 total, 7 up, 7 in 2023-09-23T18:17:00.509 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:00 smithi099 ceph-mon[167076]: failed to encode map e532 with expected crc 2023-09-23T18:17:00.510 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:00 smithi099 ceph-mon[167076]: failed to encode map e532 with expected crc 2023-09-23T18:17:00.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:00.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:00.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:00.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:17:00] "GET /metrics HTTP/1.1" 200 36641 "" "Prometheus/2.43.0" 2023-09-23T18:17:00.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[184174]: pgmap v1281: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 9 peering, 5 remapped+peering, 46 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+undersized+degraded, 28 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 111 KiB/s rd, 48 KiB/s wr, 706 op/s; 12089/85917 objects degraded (14.071%); 24437/85917 objects misplaced (28.443%); 11 MiB/s, 195 objects/s recovering 2023-09-23T18:17:00.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-09-23T18:17:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[184174]: failed to encode map e532 with expected crc 2023-09-23T18:17:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:17:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[184174]: osdmap e532: 8 total, 7 up, 7 in 2023-09-23T18:17:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[184174]: failed to encode map e532 with expected crc 2023-09-23T18:17:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[184174]: failed to encode map e532 with expected crc 2023-09-23T18:17:00.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[179668]: pgmap v1281: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 9 peering, 5 remapped+peering, 46 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 1 active+recovery_wait+undersized+degraded+remapped, 1 active+undersized+remapped, 3 active+undersized+degraded, 28 active+clean; 1.8 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 111 KiB/s rd, 48 KiB/s wr, 706 op/s; 12089/85917 objects degraded (14.071%); 24437/85917 objects misplaced (28.443%); 11 MiB/s, 195 objects/s recovering 2023-09-23T18:17:00.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 2 pgs inactive, 1 pg peering) 2023-09-23T18:17:00.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:00.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[179668]: failed to encode map e532 with expected crc 2023-09-23T18:17:00.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:17:00.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[179668]: osdmap e532: 8 total, 7 up, 7 in 2023-09-23T18:17:00.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[179668]: failed to encode map e532 with expected crc 2023-09-23T18:17:00.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-mon[179668]: failed to encode map e532 with expected crc 2023-09-23T18:17:00.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:17:00.526+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e532 definitely_dead 0 2023-09-23T18:17:00.700 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:00.774 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:00.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:00.851 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:00.852 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:00.858 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:17:00.953 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:00.984 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:01.029 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:01.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:01.032 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:01.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:01.227 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:01.459 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:17:01.513 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:01 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3998922202' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T18:17:01.513 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:01 smithi099 ceph-mon[167076]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T18:17:01.535 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.2. 2023-09-23T18:17:01.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:01 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3998922202' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T18:17:01.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:01 smithi055 ceph-mon[184174]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T18:17:01.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:01 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3998922202' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["2"]}]: dispatch 2023-09-23T18:17:01.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:01 smithi055 ceph-mon[179668]: Client client.admin marked osd.2 out, while it was still marked up 2023-09-23T18:17:01.939 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:01.942 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:01.942 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:02.009 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:17:02.043 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:02.076 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:02.118 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:02.123 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:02.124 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:02.224 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:02.227 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:02.269 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:02.367 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:02.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:02.370 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:02.471 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:02.507 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:02.603 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:02.608 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:02.608 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:02.709 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:02.746 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: pgmap v1283: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 52 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 37 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 107 KiB/s rd, 46 KiB/s wr, 1.34k op/s; 10498/74055 objects degraded (14.176%); 20462/74055 objects misplaced (27.631%); 17 MiB/s, 347 objects/s recovering 2023-09-23T18:17:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3998922202' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T18:17:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: osdmap e533: 8 total, 7 up, 6 in 2023-09-23T18:17:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:02 smithi099 ceph-mon[167076]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: pgmap v1283: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 52 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 37 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 107 KiB/s rd, 46 KiB/s wr, 1.34k op/s; 10498/74055 objects degraded (14.176%); 20462/74055 objects misplaced (27.631%); 17 MiB/s, 347 objects/s recovering 2023-09-23T18:17:02.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3998922202' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T18:17:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: osdmap e533: 8 total, 7 up, 6 in 2023-09-23T18:17:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[179668]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: pgmap v1283: 119 pgs: 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 1 active+undersized+degraded+remapped+backfilling, 52 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 37 active+clean; 1.5 GiB data, 11 GiB used, 615 GiB / 626 GiB avail; 107 KiB/s rd, 46 KiB/s wr, 1.34k op/s; 10498/74055 objects degraded (14.176%); 20462/74055 objects misplaced (27.631%); 17 MiB/s, 347 objects/s recovering 2023-09-23T18:17:02.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3998922202' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["2"]}]': finished 2023-09-23T18:17:02.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: osdmap e533: 8 total, 7 up, 6 in 2023-09-23T18:17:02.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.935 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:02 smithi055 ceph-mon[184174]: failed to encode map e533 with expected crc 2023-09-23T18:17:02.992 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:03.135 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:03.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:03.143 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:03.243 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:03.292 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:03.344 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:03.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:03.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:03.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:03.491 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:03.558 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:17:03.583 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:17:03.587 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:03.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:03.598 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:03.603 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:17:03.629 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:17:03.655 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:17:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: osdmap e534: 8 total, 7 up, 6 in 2023-09-23T18:17:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 10498/74055 objects degraded (14.176%), 9 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:03.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1207716469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:03.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: osdmap e534: 8 total, 7 up, 6 in 2023-09-23T18:17:03.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 10498/74055 objects degraded (14.176%), 9 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:03.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1207716469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:03.682 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:17:03.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:03.708 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:17:03.708 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:17:03.708 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:17:03.708 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:17:03.708 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:17:03.708 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:17:03.708 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:17:03.708 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:17:03.709 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:17:03.765 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.765 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: osdmap e534: 8 total, 7 up, 6 in 2023-09-23T18:17:03.765 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.765 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.765 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.766 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.766 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.766 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.766 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.766 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: failed to encode map e534 with expected crc 2023-09-23T18:17:03.766 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 10498/74055 objects degraded (14.176%), 9 pgs degraded, 19 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:03.766 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1207716469' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:03.773 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:03.782 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:17:03.782 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:17:03.782 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:17:03.782 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:17:03.782 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:17:03.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:03.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:03.850 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:03.951 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:03.986 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:04.112 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:04.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:04.114 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:04.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:17:04.215 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:04.223 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:17:04.223 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:17:04.271 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:04.318 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:04.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:04.326 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:04.427 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:04.463 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:04.509 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:04.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:04.513 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:04.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:04.681 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:04.762 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:04.777 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:04.777 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:04.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:04 smithi099 ceph-mon[167076]: pgmap v1286: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 51 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 36 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 104 KiB/s rd, 37 KiB/s wr, 838 op/s; 9535/73380 objects degraded (12.994%); 20237/73380 objects misplaced (27.578%); 18 MiB/s, 343 objects/s recovering 2023-09-23T18:17:04.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:04 smithi099 ceph-mon[167076]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:04 smithi099 ceph-mon[167076]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:04 smithi099 ceph-mon[167076]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:04 smithi099 ceph-mon[167076]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:04 smithi099 ceph-mon[167076]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:04 smithi099 ceph-mon[167076]: osdmap e535: 8 total, 7 up, 6 in 2023-09-23T18:17:04.778 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:04 smithi099 ceph-mon[167076]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.878 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:04.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[179668]: pgmap v1286: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 51 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 36 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 104 KiB/s rd, 37 KiB/s wr, 838 op/s; 9535/73380 objects degraded (12.994%); 20237/73380 objects misplaced (27.578%); 18 MiB/s, 343 objects/s recovering 2023-09-23T18:17:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[179668]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[179668]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[179668]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[179668]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[179668]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[179668]: osdmap e535: 8 total, 7 up, 6 in 2023-09-23T18:17:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[179668]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[184174]: pgmap v1286: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 51 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 36 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 104 KiB/s rd, 37 KiB/s wr, 838 op/s; 9535/73380 objects degraded (12.994%); 20237/73380 objects misplaced (27.578%); 18 MiB/s, 343 objects/s recovering 2023-09-23T18:17:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[184174]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[184174]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[184174]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[184174]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[184174]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[184174]: osdmap e535: 8 total, 7 up, 6 in 2023-09-23T18:17:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:04 smithi055 ceph-mon[184174]: failed to encode map e535 with expected crc 2023-09-23T18:17:04.952 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:04.967 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:17:05.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:05.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:05.057 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:05.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:05.227 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:05.589 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:17:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[184174]: failed to encode map e536 with expected crc 2023-09-23T18:17:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[184174]: failed to encode map e536 with expected crc 2023-09-23T18:17:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[184174]: osdmap e536: 8 total, 7 up, 6 in 2023-09-23T18:17:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[184174]: failed to encode map e536 with expected crc 2023-09-23T18:17:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[184174]: failed to encode map e536 with expected crc 2023-09-23T18:17:05.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2933745486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:17:05.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[179668]: failed to encode map e536 with expected crc 2023-09-23T18:17:05.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[179668]: failed to encode map e536 with expected crc 2023-09-23T18:17:05.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[179668]: osdmap e536: 8 total, 7 up, 6 in 2023-09-23T18:17:05.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[179668]: failed to encode map e536 with expected crc 2023-09-23T18:17:05.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[179668]: failed to encode map e536 with expected crc 2023-09-23T18:17:05.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2933745486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:17:06.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:05 smithi099 ceph-mon[167076]: failed to encode map e536 with expected crc 2023-09-23T18:17:06.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:05 smithi099 ceph-mon[167076]: failed to encode map e536 with expected crc 2023-09-23T18:17:06.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:05 smithi099 ceph-mon[167076]: osdmap e536: 8 total, 7 up, 6 in 2023-09-23T18:17:06.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:05 smithi099 ceph-mon[167076]: failed to encode map e536 with expected crc 2023-09-23T18:17:06.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:05 smithi099 ceph-mon[167076]: failed to encode map e536 with expected crc 2023-09-23T18:17:06.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2933745486' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:17:06.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:06.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:06.128 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:06.224 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:17:06.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:06.298 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:06.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: pgmap v1289: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 51 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 36 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 56 op/s; 9535/73380 objects degraded (12.994%); 20237/73380 objects misplaced (27.578%); 12 MiB/s, 199 objects/s recovering 2023-09-23T18:17:06.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:06.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2933745486' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:17:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: osdmap e537: 8 total, 7 up, 6 in 2023-09-23T18:17:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.584 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:06.791 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:06.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:06.799 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:06.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: pgmap v1289: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 51 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 36 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 56 op/s; 9535/73380 objects degraded (12.994%); 20237/73380 objects misplaced (27.578%); 12 MiB/s, 199 objects/s recovering 2023-09-23T18:17:06.847 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:06.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2933745486' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:17:06.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: osdmap e537: 8 total, 7 up, 6 in 2023-09-23T18:17:06.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:06.886 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:06.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: pgmap v1289: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 remapped+peering, 1 active+recovery_wait+remapped, 1 active+remapped+backfilling, 51 active+clean+remapped, 3 active+undersized+degraded+remapped+backfill_wait, 17 active+undersized, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 36 active+clean; 1.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 56 op/s; 9535/73380 objects degraded (12.994%); 20237/73380 objects misplaced (27.578%); 12 MiB/s, 199 objects/s recovering 2023-09-23T18:17:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2933745486' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:17:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: osdmap e537: 8 total, 7 up, 6 in 2023-09-23T18:17:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: failed to encode map e537 with expected crc 2023-09-23T18:17:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:06.976 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:06.978 INFO:tasks.thrashosds.thrasher:in_osds: [1, 0, 4, 3, 6, 5] out_osds: [7, 2] dead_osds: [1] live_osds: [0, 4, 6, 7, 3, 5, 2] 2023-09-23T18:17:06.979 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:17:06.979 INFO:tasks.thrashosds.thrasher:Reviving osd 1 2023-09-23T18:17:06.979 INFO:tasks.cephadm.osd.1:Restarting osd.1 (starting--it wasn't running)... 2023-09-23T18:17:06.979 DEBUG:teuthology.orchestra.run.smithi055:osd.1> sudo journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.1.service 2023-09-23T18:17:06.982 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl start ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T18:17:07.016 INFO:journalctl@ceph.osd.1.smithi055.stdout:-- Logs begin at Sat 2023-09-23 17:23:55 UTC. -- 2023-09-23T18:17:07.068 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:07.075 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:07.076 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:07.176 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:07.223 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:07.319 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:07.321 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:07 smithi055 systemd[1]: Starting Ceph osd.1 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:17:07.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:07.322 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:07.423 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:07.518 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:07.609 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:07.615 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:07.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:07.716 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:07.787 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:07.888 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:07.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:07.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:08.000 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:17:08.004 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:08.092 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:08.218 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:08.228 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:08.229 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:08.257 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:17:08.292 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:17:08.319 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:17:08.329 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:08.351 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:17:08.369 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:08.394 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:17:08.402 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:17:08.402 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:17:08.415 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:08.423 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:08.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:08.432 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:17:08.457 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:17:08.458 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:17:08.458 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:17:08.458 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:17:08.458 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:17:08.458 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:17:08.458 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:17:08.458 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:17:08.459 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:17:08.524 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:08.555 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:08.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:08 smithi055 bash[414674]: --> Failed to activate via raw: did not find any matching OSD to activate 2023-09-23T18:17:08.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:08 smithi055 bash[414674]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T18:17:08.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:08 smithi055 bash[414674]: Running command: /usr/bin/ceph-bluestore-tool --cluster=ceph prime-osd-dir --dev /dev/vg_nvme/lv_3 --path /var/lib/ceph/osd/ceph-1 --no-mon-config 2023-09-23T18:17:08.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:08 smithi055 bash[414674]: Running command: /usr/bin/ln -snf /dev/vg_nvme/lv_3 /var/lib/ceph/osd/ceph-1/block 2023-09-23T18:17:08.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:08 smithi055 bash[414674]: Running command: /usr/bin/chown -h ceph:ceph /var/lib/ceph/osd/ceph-1/block 2023-09-23T18:17:08.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:08 smithi055 bash[414674]: Running command: /usr/bin/chown -R ceph:ceph /dev/dm-2 2023-09-23T18:17:08.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:08 smithi055 bash[414674]: Running command: /usr/bin/chown -R ceph:ceph /var/lib/ceph/osd/ceph-1 2023-09-23T18:17:08.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:08 smithi055 bash[414674]: --> ceph-volume lvm activate successful for osd ID: 1 2023-09-23T18:17:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: pgmap v1291: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfilling, 57 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 35 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 451 op/s; 7287/67236 objects degraded (10.838%); 17097/67236 objects misplaced (25.428%); 42 MiB/s, 2 keys/s, 884 objects/s recovering 2023-09-23T18:17:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: osdmap e538: 8 total, 7 up, 6 in 2023-09-23T18:17:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2561784975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7287/67236 objects degraded (10.838%), 8 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:08.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: osdmap e539: 8 total, 7 up, 6 in 2023-09-23T18:17:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: failed to encode map e539 with expected crc 2023-09-23T18:17:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: failed to encode map e539 with expected crc 2023-09-23T18:17:08.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[179668]: failed to encode map e539 with expected crc 2023-09-23T18:17:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: pgmap v1291: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfilling, 57 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 35 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 451 op/s; 7287/67236 objects degraded (10.838%); 17097/67236 objects misplaced (25.428%); 42 MiB/s, 2 keys/s, 884 objects/s recovering 2023-09-23T18:17:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: osdmap e538: 8 total, 7 up, 6 in 2023-09-23T18:17:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2561784975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7287/67236 objects degraded (10.838%), 8 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: osdmap e539: 8 total, 7 up, 6 in 2023-09-23T18:17:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: failed to encode map e539 with expected crc 2023-09-23T18:17:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: failed to encode map e539 with expected crc 2023-09-23T18:17:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:08 smithi055 ceph-mon[184174]: failed to encode map e539 with expected crc 2023-09-23T18:17:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: pgmap v1291: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 1 active+remapped+backfilling, 57 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 35 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 451 op/s; 7287/67236 objects degraded (10.838%); 17097/67236 objects misplaced (25.428%); 42 MiB/s, 2 keys/s, 884 objects/s recovering 2023-09-23T18:17:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: osdmap e538: 8 total, 7 up, 6 in 2023-09-23T18:17:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: failed to encode map e538 with expected crc 2023-09-23T18:17:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2561784975' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7287/67236 objects degraded (10.838%), 8 pgs degraded, 17 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:08.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: osdmap e539: 8 total, 7 up, 6 in 2023-09-23T18:17:08.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: failed to encode map e539 with expected crc 2023-09-23T18:17:08.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: failed to encode map e539 with expected crc 2023-09-23T18:17:08.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:08 smithi099 ceph-mon[167076]: failed to encode map e539 with expected crc 2023-09-23T18:17:08.823 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:17:08.978 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:08.994 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:17:08.994 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:17:09.024 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:09.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:09.029 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:09.129 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:09.166 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:09.200 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:09.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:09.203 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:09.304 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:09.335 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:09.430 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:09.433 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:09.434 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:09.534 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:09.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[179668]: failed to encode map e539 with expected crc 2023-09-23T18:17:09.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[179668]: failed to encode map e539 with expected crc 2023-09-23T18:17:09.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[179668]: failed to encode map e539 with expected crc 2023-09-23T18:17:09.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[179668]: failed to encode map e539 with expected crc 2023-09-23T18:17:09.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[179668]: failed to encode map e540 with expected crc 2023-09-23T18:17:09.600 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[179668]: osdmap e540: 8 total, 7 up, 6 in 2023-09-23T18:17:09.612 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:09.704 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:09.704 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:09.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:09.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:09.810 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:09.849 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:09 smithi055 podman[415250]: 2023-09-23T18:17:09.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[184174]: failed to encode map e539 with expected crc 2023-09-23T18:17:09.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[184174]: failed to encode map e539 with expected crc 2023-09-23T18:17:09.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[184174]: failed to encode map e539 with expected crc 2023-09-23T18:17:09.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[184174]: failed to encode map e539 with expected crc 2023-09-23T18:17:09.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[184174]: failed to encode map e540 with expected crc 2023-09-23T18:17:09.849 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-mon[184174]: osdmap e540: 8 total, 7 up, 6 in 2023-09-23T18:17:09.850 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:09.946 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:09.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:09.949 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:09.986 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:17:10.050 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:10.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:09 smithi099 ceph-mon[167076]: failed to encode map e539 with expected crc 2023-09-23T18:17:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:09 smithi099 ceph-mon[167076]: failed to encode map e539 with expected crc 2023-09-23T18:17:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:09 smithi099 ceph-mon[167076]: failed to encode map e539 with expected crc 2023-09-23T18:17:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:09 smithi099 ceph-mon[167076]: failed to encode map e539 with expected crc 2023-09-23T18:17:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:09 smithi099 ceph-mon[167076]: failed to encode map e540 with expected crc 2023-09-23T18:17:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:09 smithi099 ceph-mon[167076]: osdmap e540: 8 total, 7 up, 6 in 2023-09-23T18:17:10.105 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:09 smithi055 bash[415250]: cdddfe2bb8e0734662e19f730a384d919d99fea30e721d8873513b113f201a63 2023-09-23T18:17:10.105 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:09 smithi055 ceph-osd[415431]: -- 172.21.15.55:0/595654094 <== mon.0 v2:172.21.15.55:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55b8f35304e0 con 0x55b8f3500c00 2023-09-23T18:17:10.105 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:09 smithi055 systemd[1]: Started Ceph osd.1 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:17:10.124 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:10.201 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:10.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:10.204 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:10.305 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:10.356 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:10.448 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:10.459 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:10.460 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:10.560 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:10.602 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:10.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:17:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:17:10] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.43.0" 2023-09-23T18:17:10.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[415375]: 2023-09-23T18:17:10.456+0000 7f5f40b57640 -1 Falling back to public interface 2023-09-23T18:17:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:10 smithi055 ceph-mon[179668]: pgmap v1294: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 57 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 36 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 454 op/s; 7287/67236 objects degraded (10.838%); 17097/67236 objects misplaced (25.428%); 42 MiB/s, 2 keys/s, 889 objects/s recovering 2023-09-23T18:17:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:10 smithi055 ceph-mon[179668]: failed to encode map e540 with expected crc 2023-09-23T18:17:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:10 smithi055 ceph-mon[179668]: failed to encode map e540 with expected crc 2023-09-23T18:17:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:10 smithi055 ceph-mon[184174]: pgmap v1294: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 57 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 36 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 454 op/s; 7287/67236 objects degraded (10.838%); 17097/67236 objects misplaced (25.428%); 42 MiB/s, 2 keys/s, 889 objects/s recovering 2023-09-23T18:17:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:10 smithi055 ceph-mon[184174]: failed to encode map e540 with expected crc 2023-09-23T18:17:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:10 smithi055 ceph-mon[184174]: failed to encode map e540 with expected crc 2023-09-23T18:17:10.796 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:10.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:10.801 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:10.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:10 smithi099 ceph-mon[167076]: pgmap v1294: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 57 active+clean+remapped, 2 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 1 active+recovering+undersized+degraded+remapped, 1 active+recovery_wait+undersized+degraded+remapped, 3 active+undersized+degraded, 36 active+clean; 1.4 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 454 op/s; 7287/67236 objects degraded (10.838%); 17097/67236 objects misplaced (25.428%); 42 MiB/s, 2 keys/s, 889 objects/s recovering 2023-09-23T18:17:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:10 smithi099 ceph-mon[167076]: failed to encode map e540 with expected crc 2023-09-23T18:17:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:10 smithi099 ceph-mon[167076]: failed to encode map e540 with expected crc 2023-09-23T18:17:10.902 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:10.985 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:10.995 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:17:11.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:11.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:11.066 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:11.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:11.169 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:11.246 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:11.338 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:11.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:11.339 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:11.414 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:17:11.414 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:17:11.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:11.524 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:11.591 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:11.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:11.599 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:11.630 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:17:11.699 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:11.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:11 smithi055 ceph-mon[184174]: pgmap v1296: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 58 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 1.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 3812/55602 objects degraded (6.856%); 15076/55602 objects misplaced (27.114%); 51 MiB/s, 1 keys/s, 994 objects/s recovering 2023-09-23T18:17:11.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:11 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/468561256' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:17:11.700 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:11 smithi055 ceph-mon[179668]: pgmap v1296: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 58 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 1.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 3812/55602 objects degraded (6.856%); 15076/55602 objects misplaced (27.114%); 51 MiB/s, 1 keys/s, 994 objects/s recovering 2023-09-23T18:17:11.701 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:11 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/468561256' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:17:11.744 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:11.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:11 smithi099 ceph-mon[167076]: pgmap v1296: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 58 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 1.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 1.08k op/s; 3812/55602 objects degraded (6.856%); 15076/55602 objects misplaced (27.114%); 51 MiB/s, 1 keys/s, 994 objects/s recovering 2023-09-23T18:17:11.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:11 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/468561256' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:17:12.045 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:12.049 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:12.049 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:12.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:12.207 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:12.271 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:12.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:12.277 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:12.309 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:17:12.377 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:12.408 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:12.508 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:12.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:12.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:12.586 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:12.615 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:12.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:12.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/468561256' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:17:12.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[184174]: osdmap e541: 8 total, 7 up, 6 in 2023-09-23T18:17:12.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[184174]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[184174]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[184174]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[184174]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[184174]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:12.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/468561256' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:17:12.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[179668]: osdmap e541: 8 total, 7 up, 6 in 2023-09-23T18:17:12.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[179668]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[179668]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[179668]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[179668]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:12 smithi055 ceph-mon[179668]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.663 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:12.743 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:12.748 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:12.748 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:12.849 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:12.922 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:12 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:12.922 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:12 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/468561256' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:17:12.923 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:12 smithi099 ceph-mon[167076]: osdmap e541: 8 total, 7 up, 6 in 2023-09-23T18:17:12.923 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:12 smithi099 ceph-mon[167076]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.923 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:12 smithi099 ceph-mon[167076]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.923 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:12 smithi099 ceph-mon[167076]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.923 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:12 smithi099 ceph-mon[167076]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.923 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:12 smithi099 ceph-mon[167076]: failed to encode map e541 with expected crc 2023-09-23T18:17:12.933 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:13.003 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:13.012 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:13.012 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:13.113 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:13.162 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:13.218 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:13.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:13.224 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:13.265 INFO:teuthology.orchestra.run.smithi055.stderr:no valid command found; 10 closest matches: 2023-09-23T18:17:13.265 INFO:teuthology.orchestra.run.smithi055.stderr:0 2023-09-23T18:17:13.265 INFO:teuthology.orchestra.run.smithi055.stderr:1 2023-09-23T18:17:13.266 INFO:teuthology.orchestra.run.smithi055.stderr:2 2023-09-23T18:17:13.266 INFO:teuthology.orchestra.run.smithi055.stderr:abort 2023-09-23T18:17:13.266 INFO:teuthology.orchestra.run.smithi055.stderr:assert 2023-09-23T18:17:13.266 INFO:teuthology.orchestra.run.smithi055.stderr:bluefs debug_inject_read_zeros 2023-09-23T18:17:13.266 INFO:teuthology.orchestra.run.smithi055.stderr:bluefs files list 2023-09-23T18:17:13.266 INFO:teuthology.orchestra.run.smithi055.stderr:bluefs stats 2023-09-23T18:17:13.266 INFO:teuthology.orchestra.run.smithi055.stderr:bluestore bluefs device info [] 2023-09-23T18:17:13.266 INFO:teuthology.orchestra.run.smithi055.stderr:config diff 2023-09-23T18:17:13.266 INFO:teuthology.orchestra.run.smithi055.stderr:admin_socket: invalid command 2023-09-23T18:17:13.325 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:13.393 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:13.477 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:13.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:13.482 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:13.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:13.652 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:13 smithi099 ceph-mon[167076]: pgmap v1298: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 58 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 1.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 707 op/s; 3812/55602 objects degraded (6.856%); 15076/55602 objects misplaced (27.114%); 16 MiB/s, 266 objects/s recovering 2023-09-23T18:17:13.653 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:13.732 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:13.744 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:13.745 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:13.771 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:13.783 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T18:17:13.783 INFO:tasks.cephadm.ceph_manager.ceph:waiting on admin_socket for osd-1, ['dump_ops_in_flight'] 2023-09-23T18:17:13.846 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:13.882 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:13.927 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:13.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:13 smithi055 ceph-mon[184174]: pgmap v1298: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 58 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 1.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 707 op/s; 3812/55602 objects degraded (6.856%); 15076/55602 objects misplaced (27.114%); 16 MiB/s, 266 objects/s recovering 2023-09-23T18:17:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 3812/55602 objects degraded (6.856%), 5 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:13 smithi055 ceph-mon[179668]: pgmap v1298: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 58 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 1.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 707 op/s; 3812/55602 objects degraded (6.856%); 15076/55602 objects misplaced (27.114%); 16 MiB/s, 266 objects/s recovering 2023-09-23T18:17:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 3812/55602 objects degraded (6.856%), 5 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:13.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:13.932 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:14.033 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 3812/55602 objects degraded (6.856%), 5 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:14.100 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:14.178 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:14.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:14.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:14.288 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:14.325 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:14.361 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:17:14.422 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:14.427 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:14.427 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:14.528 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:14.575 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:14.877 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:14.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:14.882 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:14.982 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:15.013 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:15.053 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:15.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:15.057 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:15.147 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:17:15.159 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:15.169 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:17:15.186 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:15.190 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:17:15.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:17:15.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:17:15.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:17:15.272 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:15.273 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:17:15.273 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:17:15.273 INFO:teuthology.orchestra.run.smithi055.stderr:Error ENXIO: problem getting command descriptions from osd.1 2023-09-23T18:17:15.273 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:17:15.273 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:17:15.273 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:17:15.274 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:17:15.274 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:17:15.274 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:17:15.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:15.276 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:15.377 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:15.408 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:15.500 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:15.503 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:15.504 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:15.605 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:15.635 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:15.677 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:15.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:15.684 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:15.785 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:15.817 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[179668]: failed to encode map e542 with expected crc 2023-09-23T18:17:15.817 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[179668]: osdmap e542: 8 total, 7 up, 6 in 2023-09-23T18:17:15.817 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[179668]: failed to encode map e542 with expected crc 2023-09-23T18:17:15.817 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[179668]: failed to encode map e542 with expected crc 2023-09-23T18:17:15.817 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[179668]: failed to encode map e542 with expected crc 2023-09-23T18:17:15.817 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[179668]: pgmap v1300: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 58 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 1.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 3812/55602 objects degraded (6.856%); 15076/55602 objects misplaced (27.114%); 15 MiB/s, 243 objects/s recovering 2023-09-23T18:17:15.817 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1772002004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:15.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[184174]: failed to encode map e542 with expected crc 2023-09-23T18:17:15.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[184174]: osdmap e542: 8 total, 7 up, 6 in 2023-09-23T18:17:15.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[184174]: failed to encode map e542 with expected crc 2023-09-23T18:17:15.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[184174]: failed to encode map e542 with expected crc 2023-09-23T18:17:15.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[184174]: failed to encode map e542 with expected crc 2023-09-23T18:17:15.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[184174]: pgmap v1300: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 58 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 1.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 3812/55602 objects degraded (6.856%); 15076/55602 objects misplaced (27.114%); 15 MiB/s, 243 objects/s recovering 2023-09-23T18:17:15.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1772002004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:15.821 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:15 smithi099 ceph-mon[167076]: failed to encode map e542 with expected crc 2023-09-23T18:17:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:15 smithi099 ceph-mon[167076]: osdmap e542: 8 total, 7 up, 6 in 2023-09-23T18:17:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:15 smithi099 ceph-mon[167076]: failed to encode map e542 with expected crc 2023-09-23T18:17:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:15 smithi099 ceph-mon[167076]: failed to encode map e542 with expected crc 2023-09-23T18:17:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:15 smithi099 ceph-mon[167076]: failed to encode map e542 with expected crc 2023-09-23T18:17:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:15 smithi099 ceph-mon[167076]: pgmap v1300: 119 pgs: 1 active+undersized+remapped+backfill_wait, 1 active+undersized+degraded+remapped+backfilling, 58 active+clean+remapped, 1 active+undersized+degraded+remapped+backfill_wait, 15 active+undersized, 2 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 1.1 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 645 op/s; 3812/55602 objects degraded (6.856%); 15076/55602 objects misplaced (27.114%); 15 MiB/s, 243 objects/s recovering 2023-09-23T18:17:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1772002004' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:16 smithi055 ceph-mon[184174]: failed to encode map e543 with expected crc 2023-09-23T18:17:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:16 smithi055 ceph-mon[184174]: osdmap e543: 8 total, 7 up, 6 in 2023-09-23T18:17:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:16 smithi055 ceph-mon[184174]: failed to encode map e543 with expected crc 2023-09-23T18:17:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:16 smithi055 ceph-mon[179668]: failed to encode map e543 with expected crc 2023-09-23T18:17:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:16 smithi055 ceph-mon[179668]: osdmap e543: 8 total, 7 up, 6 in 2023-09-23T18:17:16.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:16 smithi055 ceph-mon[179668]: failed to encode map e543 with expected crc 2023-09-23T18:17:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:16 smithi099 ceph-mon[167076]: failed to encode map e543 with expected crc 2023-09-23T18:17:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:16 smithi099 ceph-mon[167076]: osdmap e543: 8 total, 7 up, 6 in 2023-09-23T18:17:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:16 smithi099 ceph-mon[167076]: failed to encode map e543 with expected crc 2023-09-23T18:17:17.321 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:17.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:17.326 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:17.427 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:17.474 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:17.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:17 smithi055 ceph-mon[179668]: pgmap v1302: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 1 active+undersized+remapped+backfill_wait, 59 active+clean+remapped, 15 active+undersized, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 888 MiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 2097/42504 objects degraded (4.934%); 10905/42504 objects misplaced (25.656%); 15 MiB/s, 247 objects/s recovering 2023-09-23T18:17:17.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:17 smithi055 ceph-mon[184174]: pgmap v1302: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 1 active+undersized+remapped+backfill_wait, 59 active+clean+remapped, 15 active+undersized, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 888 MiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 2097/42504 objects degraded (4.934%); 10905/42504 objects misplaced (25.656%); 15 MiB/s, 247 objects/s recovering 2023-09-23T18:17:18.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:17 smithi099 ceph-mon[167076]: pgmap v1302: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 1 active+undersized+remapped+backfill_wait, 59 active+clean+remapped, 15 active+undersized, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 888 MiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 2097/42504 objects degraded (4.934%); 10905/42504 objects misplaced (25.656%); 15 MiB/s, 247 objects/s recovering 2023-09-23T18:17:18.680 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:18 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[415375]: 2023-09-23T18:17:18.240+0000 7f5f40b57640 -1 osd.1 497 log_to_monitors true 2023-09-23T18:17:18.785 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:17:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:18 smithi099 ceph-mon[167076]: from='osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:17:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 2097/42504 objects degraded (4.934%), 4 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:19.105 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:19.107 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:17:18 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[415375]: 2023-09-23T18:17:18.817+0000 7f5f2ffc0700 -1 osd.1 544 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-09-23T18:17:19.107 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:18 smithi055 ceph-mon[179668]: from='osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:17:19.107 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 2097/42504 objects degraded (4.934%), 4 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:19.108 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:18 smithi055 ceph-mon[184174]: from='osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-09-23T18:17:19.108 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 2097/42504 objects degraded (4.934%), 4 pgs degraded, 18 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:19.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:19.109 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:19.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:19.289 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:19.382 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:19.389 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:19.390 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:19.491 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:19.571 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:19.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:19.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:19.647 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:19.685 DEBUG:teuthology.orchestra.run:got remote process result: 6 2023-09-23T18:17:19.685 INFO:tasks.thrashosds.thrasher:Failed to tell all osds, ignoring 2023-09-23T18:17:19.748 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:19.806 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:19.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:19 smithi099 ceph-mon[167076]: from='osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:17:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:19 smithi099 ceph-mon[167076]: osdmap e544: 8 total, 7 up, 6 in 2023-09-23T18:17:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:19 smithi099 ceph-mon[167076]: from='osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:17:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:19 smithi099 ceph-mon[167076]: failed to encode map e544 with expected crc 2023-09-23T18:17:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:19 smithi099 ceph-mon[167076]: failed to encode map e544 with expected crc 2023-09-23T18:17:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:19 smithi099 ceph-mon[167076]: pgmap v1304: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 1 active+undersized+remapped+backfill_wait, 59 active+clean+remapped, 15 active+undersized, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 888 MiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 2097/42504 objects degraded (4.934%); 10905/42504 objects misplaced (25.656%); 15 MiB/s, 247 objects/s recovering 2023-09-23T18:17:19.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:19.847 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:19.848 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:19.893 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:19.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[179668]: from='osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:17:19.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[179668]: osdmap e544: 8 total, 7 up, 6 in 2023-09-23T18:17:19.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[179668]: from='osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:17:19.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[179668]: failed to encode map e544 with expected crc 2023-09-23T18:17:19.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[179668]: failed to encode map e544 with expected crc 2023-09-23T18:17:19.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[179668]: pgmap v1304: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 1 active+undersized+remapped+backfill_wait, 59 active+clean+remapped, 15 active+undersized, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 888 MiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 2097/42504 objects degraded (4.934%); 10905/42504 objects misplaced (25.656%); 15 MiB/s, 247 objects/s recovering 2023-09-23T18:17:19.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[184174]: from='osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-09-23T18:17:19.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[184174]: osdmap e544: 8 total, 7 up, 6 in 2023-09-23T18:17:19.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[184174]: from='osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi055", "root=default"]}]: dispatch 2023-09-23T18:17:19.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[184174]: failed to encode map e544 with expected crc 2023-09-23T18:17:19.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[184174]: failed to encode map e544 with expected crc 2023-09-23T18:17:19.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:19 smithi055 ceph-mon[184174]: pgmap v1304: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 active+remapped+backfilling, 1 active+undersized+remapped+backfill_wait, 59 active+clean+remapped, 15 active+undersized, 1 active+remapped+backfill_wait, 3 active+undersized+degraded, 38 active+clean; 888 MiB data, 8.5 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 727 op/s; 2097/42504 objects degraded (4.934%); 10905/42504 objects misplaced (25.656%); 15 MiB/s, 247 objects/s recovering 2023-09-23T18:17:19.949 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:19.977 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:20.463 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:20.466 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:20.466 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:20.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:20.595 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:20.613 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:17:20] "GET /metrics HTTP/1.1" 200 36644 "" "Prometheus/2.43.0" 2023-09-23T18:17:20.632 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:20.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:20.636 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:20.736 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:20.794 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:20.849 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:20.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:20 smithi099 ceph-mon[167076]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:17:20.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:20 smithi099 ceph-mon[167076]: osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292] boot 2023-09-23T18:17:20.851 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:20 smithi099 ceph-mon[167076]: osdmap e545: 8 total, 8 up, 6 in 2023-09-23T18:17:20.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:20 smithi099 ceph-mon[167076]: failed to encode map e545 with expected crc 2023-09-23T18:17:20.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:20 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:17:20.852 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:20 smithi099 ceph-mon[167076]: failed to encode map e545 with expected crc 2023-09-23T18:17:20.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:20.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:20.953 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:20.975 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[179668]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:17:20.975 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[179668]: osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292] boot 2023-09-23T18:17:20.975 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[179668]: osdmap e545: 8 total, 8 up, 6 in 2023-09-23T18:17:20.975 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[179668]: failed to encode map e545 with expected crc 2023-09-23T18:17:20.975 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:17:20.975 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[179668]: failed to encode map e545 with expected crc 2023-09-23T18:17:20.975 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[184174]: Health check cleared: OSD_DOWN (was: 1 osds down) 2023-09-23T18:17:20.975 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[184174]: osd.1 [v2:172.21.15.55:6810/3845695292,v1:172.21.15.55:6811/3845695292] boot 2023-09-23T18:17:20.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[184174]: osdmap e545: 8 total, 8 up, 6 in 2023-09-23T18:17:20.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[184174]: failed to encode map e545 with expected crc 2023-09-23T18:17:20.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-09-23T18:17:20.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:20 smithi055 ceph-mon[184174]: failed to encode map e545 with expected crc 2023-09-23T18:17:21.021 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:21.074 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:21.077 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:21.078 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:21.142 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:17:21.142 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:17:21.142 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:17:21.142 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:17:21.142 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:17:21.179 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:21.210 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:21.247 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:21.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:21.251 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:21.285 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:21.352 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:21.381 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:21.519 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:21.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:21.522 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:21.623 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:21.649 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:21.686 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:21.686 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:17:21.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:21.690 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:21.790 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:21.824 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:21.878 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:21.883 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:21.884 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:21.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: failed to encode map e546 with expected crc 2023-09-23T18:17:21.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: failed to encode map e546 with expected crc 2023-09-23T18:17:21.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: failed to encode map e546 with expected crc 2023-09-23T18:17:21.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: osdmap e546: 8 total, 8 up, 6 in 2023-09-23T18:17:21.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.3 starting backfill to osd.1 from (467'3364,497'7673] MAX to 544'12523 2023-09-23T18:17:21.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: failed to encode map e546 with expected crc 2023-09-23T18:17:21.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.f starting backfill to osd.1 from (472'3567,497'7415] MAX to 544'12267 2023-09-23T18:17:21.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.5 starting backfill to osd.1 from (467'3177,497'7518] MIN to 544'12330 2023-09-23T18:17:21.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.b starting backfill to osd.1 from (466'2765,497'7769] MAX to 544'12683 2023-09-23T18:17:21.935 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.e starting backfill to osd.1 from (470'3374,497'7718] MAX to 544'12707 2023-09-23T18:17:21.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.8 starting backfill to osd.1 from (467'3017,497'7378] 9:1818e8e0:::benchmark_data_smithi055_386467_object90902:head to 544'12082 2023-09-23T18:17:21.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.0 starting backfill to osd.1 from (473'3925,497'7821] MAX to 544'12762 2023-09-23T18:17:21.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.a starting backfill to osd.1 from (472'3668,497'7579] MAX to 544'12411 2023-09-23T18:17:21.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.d starting backfill to osd.1 from (467'3133,497'7515] MAX to 544'12271 2023-09-23T18:17:21.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.7 starting backfill to osd.1 from (467'3227,497'7557] MAX to 544'12420 2023-09-23T18:17:21.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.2 starting backfill to osd.1 from (472'3714,497'7660] MAX to 544'12555 2023-09-23T18:17:21.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: 9.7 continuing backfill to osd.6 from (508'8412,544'12420] 9:eab1a166:::benchmark_data_smithi055_386467_object99326:head to 544'12420 2023-09-23T18:17:21.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: pgmap v1307: 119 pgs: 1 activating+undersized+remapped, 1 remapped+peering, 2 activating+degraded+remapped, 35 peering, 1 activating, 7 activating+remapped, 2 active+remapped+backfilling, 17 active+clean+remapped, 10 active+undersized, 2 active+undersized+degraded, 41 active+clean; 675 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.44k op/s; 926/32286 objects degraded (2.868%); 7397/32286 objects misplaced (22.911%); 28 MiB/s, 445 objects/s recovering 2023-09-23T18:17:21.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:21.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: failed to encode map e546 with expected crc 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: failed to encode map e546 with expected crc 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: failed to encode map e546 with expected crc 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: osdmap e546: 8 total, 8 up, 6 in 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.3 starting backfill to osd.1 from (467'3364,497'7673] MAX to 544'12523 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: failed to encode map e546 with expected crc 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.f starting backfill to osd.1 from (472'3567,497'7415] MAX to 544'12267 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.5 starting backfill to osd.1 from (467'3177,497'7518] MIN to 544'12330 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.b starting backfill to osd.1 from (466'2765,497'7769] MAX to 544'12683 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.e starting backfill to osd.1 from (470'3374,497'7718] MAX to 544'12707 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.8 starting backfill to osd.1 from (467'3017,497'7378] 9:1818e8e0:::benchmark_data_smithi055_386467_object90902:head to 544'12082 2023-09-23T18:17:21.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.0 starting backfill to osd.1 from (473'3925,497'7821] MAX to 544'12762 2023-09-23T18:17:21.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.a starting backfill to osd.1 from (472'3668,497'7579] MAX to 544'12411 2023-09-23T18:17:21.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.d starting backfill to osd.1 from (467'3133,497'7515] MAX to 544'12271 2023-09-23T18:17:21.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.7 starting backfill to osd.1 from (467'3227,497'7557] MAX to 544'12420 2023-09-23T18:17:21.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.2 starting backfill to osd.1 from (472'3714,497'7660] MAX to 544'12555 2023-09-23T18:17:21.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: 9.7 continuing backfill to osd.6 from (508'8412,544'12420] 9:eab1a166:::benchmark_data_smithi055_386467_object99326:head to 544'12420 2023-09-23T18:17:21.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: pgmap v1307: 119 pgs: 1 activating+undersized+remapped, 1 remapped+peering, 2 activating+degraded+remapped, 35 peering, 1 activating, 7 activating+remapped, 2 active+remapped+backfilling, 17 active+clean+remapped, 10 active+undersized, 2 active+undersized+degraded, 41 active+clean; 675 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.44k op/s; 926/32286 objects degraded (2.868%); 7397/32286 objects misplaced (22.911%); 28 MiB/s, 445 objects/s recovering 2023-09-23T18:17:21.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:21.938 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:21.985 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:22.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: failed to encode map e546 with expected crc 2023-09-23T18:17:22.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: failed to encode map e546 with expected crc 2023-09-23T18:17:22.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: failed to encode map e546 with expected crc 2023-09-23T18:17:22.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: osdmap e546: 8 total, 8 up, 6 in 2023-09-23T18:17:22.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.3 starting backfill to osd.1 from (467'3364,497'7673] MAX to 544'12523 2023-09-23T18:17:22.024 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: failed to encode map e546 with expected crc 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.f starting backfill to osd.1 from (472'3567,497'7415] MAX to 544'12267 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.5 starting backfill to osd.1 from (467'3177,497'7518] MIN to 544'12330 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.b starting backfill to osd.1 from (466'2765,497'7769] MAX to 544'12683 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.e starting backfill to osd.1 from (470'3374,497'7718] MAX to 544'12707 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.8 starting backfill to osd.1 from (467'3017,497'7378] 9:1818e8e0:::benchmark_data_smithi055_386467_object90902:head to 544'12082 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.0 starting backfill to osd.1 from (473'3925,497'7821] MAX to 544'12762 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.a starting backfill to osd.1 from (472'3668,497'7579] MAX to 544'12411 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.d starting backfill to osd.1 from (467'3133,497'7515] MAX to 544'12271 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.7 starting backfill to osd.1 from (467'3227,497'7557] MAX to 544'12420 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.2 starting backfill to osd.1 from (472'3714,497'7660] MAX to 544'12555 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: 9.7 continuing backfill to osd.6 from (508'8412,544'12420] 9:eab1a166:::benchmark_data_smithi055_386467_object99326:head to 544'12420 2023-09-23T18:17:22.025 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: pgmap v1307: 119 pgs: 1 activating+undersized+remapped, 1 remapped+peering, 2 activating+degraded+remapped, 35 peering, 1 activating, 7 activating+remapped, 2 active+remapped+backfilling, 17 active+clean+remapped, 10 active+undersized, 2 active+undersized+degraded, 41 active+clean; 675 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.44k op/s; 926/32286 objects degraded (2.868%); 7397/32286 objects misplaced (22.911%); 28 MiB/s, 445 objects/s recovering 2023-09-23T18:17:22.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:22.026 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:22.026 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:22.070 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:22.078 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:22.079 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:22.180 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:22.232 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:22.266 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:22.269 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:22.270 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:22.343 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:17:22.370 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:22.426 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:23.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: osdmap e547: 8 total, 8 up, 6 in 2023-09-23T18:17:23.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.083 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: 9.6 starting backfill to osd.1 from (472'3672,497'7586] MAX to 544'12432 2023-09-23T18:17:23.084 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:22 smithi099 ceph-mon[167076]: 9.4 starting backfill to osd.1 from (472'3794,497'7723] MAX to 544'12572 2023-09-23T18:17:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: osdmap e547: 8 total, 8 up, 6 in 2023-09-23T18:17:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: 9.6 starting backfill to osd.1 from (472'3672,497'7586] MAX to 544'12432 2023-09-23T18:17:23.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[184174]: 9.4 starting backfill to osd.1 from (472'3794,497'7723] MAX to 544'12572 2023-09-23T18:17:23.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: osdmap e547: 8 total, 8 up, 6 in 2023-09-23T18:17:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: failed to encode map e547 with expected crc 2023-09-23T18:17:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: 9.6 starting backfill to osd.1 from (472'3672,497'7586] MAX to 544'12432 2023-09-23T18:17:23.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:22 smithi055 ceph-mon[179668]: 9.4 starting backfill to osd.1 from (472'3794,497'7723] MAX to 544'12572 2023-09-23T18:17:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:23 smithi099 ceph-mon[167076]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:23 smithi099 ceph-mon[167076]: osdmap e548: 8 total, 8 up, 6 in 2023-09-23T18:17:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:23 smithi099 ceph-mon[167076]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:23 smithi099 ceph-mon[167076]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:23 smithi099 ceph-mon[167076]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:23 smithi099 ceph-mon[167076]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:23 smithi099 ceph-mon[167076]: pgmap v1310: 119 pgs: 1 active+remapped+backfill_wait, 1 activating+undersized+remapped, 4 remapped+peering, 2 activating+degraded+remapped, 49 peering, 1 activating, 7 activating+remapped, 1 active+remapped+backfilling, 53 active+clean; 630 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 3/30123 objects degraded (0.010%); 6813/30123 objects misplaced (22.617%); 19 MiB/s, 302 objects/s recovering 2023-09-23T18:17:24.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 926/32286 objects degraded (2.868%), 4 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:24.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[179668]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[179668]: osdmap e548: 8 total, 8 up, 6 in 2023-09-23T18:17:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[179668]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[179668]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[179668]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[179668]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[179668]: pgmap v1310: 119 pgs: 1 active+remapped+backfill_wait, 1 activating+undersized+remapped, 4 remapped+peering, 2 activating+degraded+remapped, 49 peering, 1 activating, 7 activating+remapped, 1 active+remapped+backfilling, 53 active+clean; 630 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 3/30123 objects degraded (0.010%); 6813/30123 objects misplaced (22.617%); 19 MiB/s, 302 objects/s recovering 2023-09-23T18:17:24.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 926/32286 objects degraded (2.868%), 4 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[184174]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[184174]: osdmap e548: 8 total, 8 up, 6 in 2023-09-23T18:17:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[184174]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[184174]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[184174]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[184174]: failed to encode map e548 with expected crc 2023-09-23T18:17:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[184174]: pgmap v1310: 119 pgs: 1 active+remapped+backfill_wait, 1 activating+undersized+remapped, 4 remapped+peering, 2 activating+degraded+remapped, 49 peering, 1 activating, 7 activating+remapped, 1 active+remapped+backfilling, 53 active+clean; 630 MiB data, 8.1 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 1.03k op/s; 3/30123 objects degraded (0.010%); 6813/30123 objects misplaced (22.617%); 19 MiB/s, 302 objects/s recovering 2023-09-23T18:17:24.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 926/32286 objects degraded (2.868%), 4 pgs degraded, 12 pgs undersized (PG_DEGRADED) 2023-09-23T18:17:24.536 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:24.540 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:24.540 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:24.641 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:24.675 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:24.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:24.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:24.722 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:24.823 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:24.857 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:24 smithi055 ceph-mon[184174]: failed to encode map e549 with expected crc 2023-09-23T18:17:24.857 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:24 smithi055 ceph-mon[184174]: osdmap e549: 8 total, 8 up, 6 in 2023-09-23T18:17:24.857 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:24 smithi055 ceph-mon[184174]: failed to encode map e549 with expected crc 2023-09-23T18:17:24.858 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:24 smithi055 ceph-mon[179668]: failed to encode map e549 with expected crc 2023-09-23T18:17:24.858 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:24 smithi055 ceph-mon[179668]: osdmap e549: 8 total, 8 up, 6 in 2023-09-23T18:17:24.858 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:24 smithi055 ceph-mon[179668]: failed to encode map e549 with expected crc 2023-09-23T18:17:24.861 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:25.050 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:25.055 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:25.055 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:24 smithi099 ceph-mon[167076]: failed to encode map e549 with expected crc 2023-09-23T18:17:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:24 smithi099 ceph-mon[167076]: osdmap e549: 8 total, 8 up, 6 in 2023-09-23T18:17:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:24 smithi099 ceph-mon[167076]: failed to encode map e549 with expected crc 2023-09-23T18:17:25.157 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:25.194 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:25.235 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:25.238 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:25.239 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:25.340 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:25.376 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:25.856 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:25 smithi099 ceph-mon[167076]: failed to encode map e550 with expected crc 2023-09-23T18:17:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:25 smithi099 ceph-mon[167076]: osdmap e550: 8 total, 8 up, 6 in 2023-09-23T18:17:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:25 smithi099 ceph-mon[167076]: failed to encode map e550 with expected crc 2023-09-23T18:17:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:25 smithi099 ceph-mon[167076]: failed to encode map e550 with expected crc 2023-09-23T18:17:26.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:25 smithi099 ceph-mon[167076]: pgmap v1313: 119 pgs: 1 active+remapped+backfill_wait, 1 activating+undersized+remapped, 4 remapped+peering, 2 activating+degraded+remapped, 49 peering, 1 activating, 7 activating+remapped, 1 active+remapped+backfilling, 53 active+clean; 630 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 180 op/s; 3/30123 objects degraded (0.010%); 6813/30123 objects misplaced (22.617%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T18:17:26.144 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:26.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:25 smithi055 ceph-mon[184174]: failed to encode map e550 with expected crc 2023-09-23T18:17:26.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:25 smithi055 ceph-mon[184174]: osdmap e550: 8 total, 8 up, 6 in 2023-09-23T18:17:26.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:25 smithi055 ceph-mon[184174]: failed to encode map e550 with expected crc 2023-09-23T18:17:26.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:25 smithi055 ceph-mon[184174]: failed to encode map e550 with expected crc 2023-09-23T18:17:26.147 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:25 smithi055 ceph-mon[184174]: pgmap v1313: 119 pgs: 1 active+remapped+backfill_wait, 1 activating+undersized+remapped, 4 remapped+peering, 2 activating+degraded+remapped, 49 peering, 1 activating, 7 activating+remapped, 1 active+remapped+backfilling, 53 active+clean; 630 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 180 op/s; 3/30123 objects degraded (0.010%); 6813/30123 objects misplaced (22.617%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T18:17:26.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:25 smithi055 ceph-mon[179668]: failed to encode map e550 with expected crc 2023-09-23T18:17:26.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:25 smithi055 ceph-mon[179668]: osdmap e550: 8 total, 8 up, 6 in 2023-09-23T18:17:26.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:25 smithi055 ceph-mon[179668]: failed to encode map e550 with expected crc 2023-09-23T18:17:26.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:25 smithi055 ceph-mon[179668]: failed to encode map e550 with expected crc 2023-09-23T18:17:26.148 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:25 smithi055 ceph-mon[179668]: pgmap v1313: 119 pgs: 1 active+remapped+backfill_wait, 1 activating+undersized+remapped, 4 remapped+peering, 2 activating+degraded+remapped, 49 peering, 1 activating, 7 activating+remapped, 1 active+remapped+backfilling, 53 active+clean; 630 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 180 op/s; 3/30123 objects degraded (0.010%); 6813/30123 objects misplaced (22.617%); 4.4 MiB/s, 69 objects/s recovering 2023-09-23T18:17:26.148 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:26.149 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:26.249 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:26.274 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:26.311 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:26.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:26.314 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:26.415 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:26.441 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:26.845 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:27.061 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[184174]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.061 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[184174]: osdmap e551: 8 total, 8 up, 6 in 2023-09-23T18:17:27.061 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[184174]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.061 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[184174]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.061 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[184174]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.061 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[184174]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.061 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[184174]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.061 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[184174]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.062 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[179668]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.062 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[179668]: osdmap e551: 8 total, 8 up, 6 in 2023-09-23T18:17:27.062 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[179668]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.062 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[179668]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.062 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[179668]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.062 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[179668]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.062 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[179668]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.062 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:26 smithi055 ceph-mon[179668]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:26 smithi099 ceph-mon[167076]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:26 smithi099 ceph-mon[167076]: osdmap e551: 8 total, 8 up, 6 in 2023-09-23T18:17:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:26 smithi099 ceph-mon[167076]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:26 smithi099 ceph-mon[167076]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:26 smithi099 ceph-mon[167076]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:26 smithi099 ceph-mon[167076]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:26 smithi099 ceph-mon[167076]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:26 smithi099 ceph-mon[167076]: failed to encode map e551 with expected crc 2023-09-23T18:17:27.095 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:27.097 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:27.098 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:27.199 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:27.230 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:27.573 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:27.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:27.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:27.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:27.751 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:27.806 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:27.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:27.815 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:27.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:27.973 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:28.018 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:28.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:27 smithi099 ceph-mon[167076]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.022 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:27 smithi099 ceph-mon[167076]: osdmap e552: 8 total, 8 up, 6 in 2023-09-23T18:17:28.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:27 smithi099 ceph-mon[167076]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:27 smithi099 ceph-mon[167076]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:27 smithi099 ceph-mon[167076]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:27 smithi099 ceph-mon[167076]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:27 smithi099 ceph-mon[167076]: 1.13 scrub starts 2023-09-23T18:17:28.023 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:27 smithi099 ceph-mon[167076]: pgmap v1316: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 10 active+remapped+backfill_wait, 107 active+clean; 507 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 448/24225 objects degraded (1.849%); 4946/24225 objects misplaced (20.417%); 3.6 MiB/s, 60 objects/s recovering 2023-09-23T18:17:28.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:28.024 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:28.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:28.150 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[179668]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[179668]: osdmap e552: 8 total, 8 up, 6 in 2023-09-23T18:17:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[179668]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[179668]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[179668]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[179668]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[179668]: 1.13 scrub starts 2023-09-23T18:17:28.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[179668]: pgmap v1316: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 10 active+remapped+backfill_wait, 107 active+clean; 507 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 448/24225 objects degraded (1.849%); 4946/24225 objects misplaced (20.417%); 3.6 MiB/s, 60 objects/s recovering 2023-09-23T18:17:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[184174]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[184174]: osdmap e552: 8 total, 8 up, 6 in 2023-09-23T18:17:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[184174]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[184174]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[184174]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[184174]: failed to encode map e552 with expected crc 2023-09-23T18:17:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[184174]: 1.13 scrub starts 2023-09-23T18:17:28.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:27 smithi055 ceph-mon[184174]: pgmap v1316: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 10 active+remapped+backfill_wait, 107 active+clean; 507 MiB data, 7.6 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 448/24225 objects degraded (1.849%); 4946/24225 objects misplaced (20.417%); 3.6 MiB/s, 60 objects/s recovering 2023-09-23T18:17:28.184 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:28.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:28.188 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:28.288 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:28.318 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:28.355 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:28.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:28.359 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:28.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:28.490 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:28.523 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:17:28.524 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:17:28.524 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:28.524 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:202237 9.3 e553/545 9:c2024a7f:::benchmark_data_smithi055_386467_object97577:head v 553'12687, mlcod=553'12685)", 2023-09-23T18:17:28.524 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:28.523307+0000", 2023-09-23T18:17:28.524 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.00083021799999999997, 2023-09-23T18:17:28.524 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00085324100000000003, 2023-09-23T18:17:28.524 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:28.525 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:17:28.525 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:28.525 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:28.525 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:28.525 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:28.523307+0000", 2023-09-23T18:17:28.525 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:28.525 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:28.525 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:28.525 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:28.526 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:28.523307+0000", 2023-09-23T18:17:28.526 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:28.526 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:28.526 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:28.526 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:28.526 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:28.523307+0000", 2023-09-23T18:17:28.526 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:28.526 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:28.526 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:28.523313+0000", 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8039999999999998e-06 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:28.523327+0000", 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4756999999999999e-05 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:28.527 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:28.523335+0000", 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6240000000000002e-06 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:28.523389+0000", 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4066e-05 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:28.523409+0000", 2023-09-23T18:17:28.528 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0452999999999999e-05 2023-09-23T18:17:28.529 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:28.529 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:28.529 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:28.529 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:28.529 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:17:28.529 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 1 2023-09-23T18:17:28.529 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:17:28.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:28.559 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:28.559 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:28.660 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:28.699 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:28.809 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:17:28.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[184174]: failed to encode map e553 with expected crc 2023-09-23T18:17:28.838 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[184174]: failed to encode map e553 with expected crc 2023-09-23T18:17:28.839 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[184174]: osdmap e553: 8 total, 8 up, 6 in 2023-09-23T18:17:28.839 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[184174]: failed to encode map e553 with expected crc 2023-09-23T18:17:28.839 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1180777678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:17:28.839 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 448/24225 objects degraded (1.849%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:17:28.839 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[179668]: failed to encode map e553 with expected crc 2023-09-23T18:17:28.839 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[179668]: failed to encode map e553 with expected crc 2023-09-23T18:17:28.839 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[179668]: osdmap e553: 8 total, 8 up, 6 in 2023-09-23T18:17:28.839 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[179668]: failed to encode map e553 with expected crc 2023-09-23T18:17:28.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1180777678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:17:28.840 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 448/24225 objects degraded (1.849%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:17:29.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:28 smithi099 ceph-mon[167076]: failed to encode map e553 with expected crc 2023-09-23T18:17:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:28 smithi099 ceph-mon[167076]: failed to encode map e553 with expected crc 2023-09-23T18:17:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:28 smithi099 ceph-mon[167076]: osdmap e553: 8 total, 8 up, 6 in 2023-09-23T18:17:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:28 smithi099 ceph-mon[167076]: failed to encode map e553 with expected crc 2023-09-23T18:17:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:28 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1180777678' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:17:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 448/24225 objects degraded (1.849%), 1 pg degraded (PG_DEGRADED) 2023-09-23T18:17:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:17:30.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[179668]: failed to encode map e554 with expected crc 2023-09-23T18:17:30.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1180777678' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:17:30.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[179668]: osdmap e554: 8 total, 8 up, 6 in 2023-09-23T18:17:30.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[179668]: failed to encode map e554 with expected crc 2023-09-23T18:17:30.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[179668]: pgmap v1319: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 10 active+remapped+backfill_wait, 107 active+clean; 507 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 448/24225 objects degraded (1.849%); 4946/24225 objects misplaced (20.417%); 3.6 MiB/s, 60 objects/s recovering 2023-09-23T18:17:30.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:17:30.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[184174]: failed to encode map e554 with expected crc 2023-09-23T18:17:30.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1180777678' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:17:30.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[184174]: osdmap e554: 8 total, 8 up, 6 in 2023-09-23T18:17:30.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[184174]: failed to encode map e554 with expected crc 2023-09-23T18:17:30.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:29 smithi055 ceph-mon[184174]: pgmap v1319: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 10 active+remapped+backfill_wait, 107 active+clean; 507 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 448/24225 objects degraded (1.849%); 4946/24225 objects misplaced (20.417%); 3.6 MiB/s, 60 objects/s recovering 2023-09-23T18:17:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:29 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:17:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:29 smithi099 ceph-mon[167076]: failed to encode map e554 with expected crc 2023-09-23T18:17:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1180777678' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:17:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:29 smithi099 ceph-mon[167076]: osdmap e554: 8 total, 8 up, 6 in 2023-09-23T18:17:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:29 smithi099 ceph-mon[167076]: failed to encode map e554 with expected crc 2023-09-23T18:17:30.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:29 smithi099 ceph-mon[167076]: pgmap v1319: 119 pgs: 1 active+undersized+degraded+remapped+backfilling, 1 peering, 10 active+remapped+backfill_wait, 107 active+clean; 507 MiB data, 6.8 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 491 op/s; 448/24225 objects degraded (1.849%); 4946/24225 objects misplaced (20.417%); 3.6 MiB/s, 60 objects/s recovering 2023-09-23T18:17:30.646 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:30.649 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:17:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:17:30] "GET /metrics HTTP/1.1" 200 36487 "" "Prometheus/2.43.0" 2023-09-23T18:17:30.650 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:30.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:30.751 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:30.777 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:30.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:30.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:30.816 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:30.917 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:30.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:30 smithi055 ceph-mon[179668]: failed to encode map e555 with expected crc 2023-09-23T18:17:30.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:30 smithi055 ceph-mon[179668]: failed to encode map e555 with expected crc 2023-09-23T18:17:30.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:30 smithi055 ceph-mon[179668]: osdmap e555: 8 total, 8 up, 6 in 2023-09-23T18:17:30.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:30 smithi055 ceph-mon[179668]: failed to encode map e555 with expected crc 2023-09-23T18:17:30.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:30 smithi055 ceph-mon[184174]: failed to encode map e555 with expected crc 2023-09-23T18:17:30.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:30 smithi055 ceph-mon[184174]: failed to encode map e555 with expected crc 2023-09-23T18:17:30.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:30 smithi055 ceph-mon[184174]: osdmap e555: 8 total, 8 up, 6 in 2023-09-23T18:17:30.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:30 smithi055 ceph-mon[184174]: failed to encode map e555 with expected crc 2023-09-23T18:17:30.982 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:31.028 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:31.033 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:30 smithi099 ceph-mon[167076]: failed to encode map e555 with expected crc 2023-09-23T18:17:31.034 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:30 smithi099 ceph-mon[167076]: failed to encode map e555 with expected crc 2023-09-23T18:17:31.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:30 smithi099 ceph-mon[167076]: osdmap e555: 8 total, 8 up, 6 in 2023-09-23T18:17:31.035 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:30 smithi099 ceph-mon[167076]: failed to encode map e555 with expected crc 2023-09-23T18:17:31.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:31.037 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:31.138 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:31.170 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:31.442 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:31.445 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:31.446 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:31.547 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:31.576 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:31.611 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:31.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:31.615 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:31.716 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:31.746 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:31.830 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:31.833 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:31.834 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:31.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:32.000 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:32.042 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:17:32.060 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:32.065 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:32.065 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:32.067 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:17:32.093 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:17:32.126 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:17:32.146 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:17:32.165 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:17:32.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:32.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[179668]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[179668]: osdmap e556: 8 total, 8 up, 6 in 2023-09-23T18:17:32.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[179668]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[179668]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[179668]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[179668]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[179668]: pgmap v1322: 119 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 108 active+clean; 415 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 370 op/s; 3492/19779 objects misplaced (17.655%) 2023-09-23T18:17:32.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[184174]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[184174]: osdmap e556: 8 total, 8 up, 6 in 2023-09-23T18:17:32.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[184174]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[184174]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[184174]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[184174]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:31 smithi055 ceph-mon[184174]: pgmap v1322: 119 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 108 active+clean; 415 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 370 op/s; 3492/19779 objects misplaced (17.655%) 2023-09-23T18:17:32.199 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:31 smithi099 ceph-mon[167076]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.199 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:31 smithi099 ceph-mon[167076]: osdmap e556: 8 total, 8 up, 6 in 2023-09-23T18:17:32.199 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:31 smithi099 ceph-mon[167076]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.199 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:31 smithi099 ceph-mon[167076]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.199 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:31 smithi099 ceph-mon[167076]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.199 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:31 smithi099 ceph-mon[167076]: failed to encode map e556 with expected crc 2023-09-23T18:17:32.200 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:31 smithi099 ceph-mon[167076]: pgmap v1322: 119 pgs: 1 active+remapped+backfilling, 2 peering, 8 active+remapped+backfill_wait, 108 active+clean; 415 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 370 op/s; 3492/19779 objects misplaced (17.655%) 2023-09-23T18:17:32.203 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:32.215 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:17:32.240 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:17:32.243 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:17:32.243 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:17:32.243 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:17:32.243 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:17:32.244 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:17:32.244 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:17:32.244 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:17:32.244 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:17:32.255 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:32.259 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:32.260 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:32.361 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:32.419 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:32.475 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:32.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:32.480 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:32.544 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:17:32.596 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:17:32.596 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:32.629 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:32.862 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:32.865 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:32.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:32.967 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:33.027 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:33.083 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:33.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:33.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:33.156 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 448/24225 objects degraded (1.849%), 1 pg degraded) 2023-09-23T18:17:33.156 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:17:33.157 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[179668]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.157 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[179668]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.157 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[179668]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.157 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[179668]: osdmap e557: 8 total, 8 up, 6 in 2023-09-23T18:17:33.157 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[179668]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.157 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/961646132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:33.158 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 448/24225 objects degraded (1.849%), 1 pg degraded) 2023-09-23T18:17:33.158 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:17:33.158 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[184174]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.158 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[184174]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.158 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[184174]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.159 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[184174]: osdmap e557: 8 total, 8 up, 6 in 2023-09-23T18:17:33.159 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[184174]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.159 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:32 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/961646132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:33.160 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:17:33.191 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:33.232 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:33.235 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:17:33.268 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:33.272 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:33.272 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:33.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:32 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 448/24225 objects degraded (1.849%), 1 pg degraded) 2023-09-23T18:17:33.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:32 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:17:33.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:32 smithi099 ceph-mon[167076]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:32 smithi099 ceph-mon[167076]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:32 smithi099 ceph-mon[167076]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:32 smithi099 ceph-mon[167076]: osdmap e557: 8 total, 8 up, 6 in 2023-09-23T18:17:33.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:32 smithi099 ceph-mon[167076]: failed to encode map e557 with expected crc 2023-09-23T18:17:33.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:32 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/961646132' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:33.373 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:33.411 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:33.453 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:33.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:33.457 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:33.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:33.586 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:33.622 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:33.625 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:33.626 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:33.726 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:33.753 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:33.788 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:33.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:33.792 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:33.814 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:33.893 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:33.920 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:33.962 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:33.962 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:33.967 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:33.968 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:34.068 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:34.095 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:34.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:34.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:34.132 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[184174]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[184174]: osdmap e558: 8 total, 8 up, 6 in 2023-09-23T18:17:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[184174]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[184174]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[184174]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[184174]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[184174]: pgmap v1325: 119 pgs: 1 active+remapped+backfilling, 1 peering, 7 active+remapped+backfill_wait, 110 active+clean; 391 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 3045/18663 objects misplaced (16.316%); 0 B/s, 0 objects/s recovering 2023-09-23T18:17:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[179668]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[179668]: osdmap e558: 8 total, 8 up, 6 in 2023-09-23T18:17:34.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[179668]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[179668]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[179668]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[179668]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:33 smithi055 ceph-mon[179668]: pgmap v1325: 119 pgs: 1 active+remapped+backfilling, 1 peering, 7 active+remapped+backfill_wait, 110 active+clean; 391 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 3045/18663 objects misplaced (16.316%); 0 B/s, 0 objects/s recovering 2023-09-23T18:17:34.233 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:34.264 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:33 smithi099 ceph-mon[167076]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.264 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:33 smithi099 ceph-mon[167076]: osdmap e558: 8 total, 8 up, 6 in 2023-09-23T18:17:34.264 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:33 smithi099 ceph-mon[167076]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.264 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:33 smithi099 ceph-mon[167076]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.265 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:33 smithi099 ceph-mon[167076]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.265 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:33 smithi099 ceph-mon[167076]: failed to encode map e558 with expected crc 2023-09-23T18:17:34.265 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:33 smithi099 ceph-mon[167076]: pgmap v1325: 119 pgs: 1 active+remapped+backfilling, 1 peering, 7 active+remapped+backfill_wait, 110 active+clean; 391 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 463 op/s; 3045/18663 objects misplaced (16.316%); 0 B/s, 0 objects/s recovering 2023-09-23T18:17:34.267 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:34.302 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:34.304 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:34.305 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:34.405 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:34.430 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:34.466 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:34.470 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:34.471 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:34.571 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:34.600 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:34.633 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:34.635 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:34.636 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:34.736 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:34.762 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:34.807 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:34.811 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:34.811 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:34.913 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:34.929 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:17:34.940 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:35.006 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:35.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:35.011 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:35.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:35.159 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:35.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[184174]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[184174]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[184174]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[184174]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[184174]: osdmap e559: 8 total, 8 up, 6 in 2023-09-23T18:17:35.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[184174]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[184174]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[179668]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[179668]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[179668]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[179668]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[179668]: osdmap e559: 8 total, 8 up, 6 in 2023-09-23T18:17:35.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[179668]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:34 smithi055 ceph-mon[179668]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.213 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:35.219 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:35.219 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:34 smithi099 ceph-mon[167076]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.219 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:34 smithi099 ceph-mon[167076]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.219 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:34 smithi099 ceph-mon[167076]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.219 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:34 smithi099 ceph-mon[167076]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.219 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:34 smithi099 ceph-mon[167076]: osdmap e559: 8 total, 8 up, 6 in 2023-09-23T18:17:35.219 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:34 smithi099 ceph-mon[167076]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.220 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:34 smithi099 ceph-mon[167076]: failed to encode map e559 with expected crc 2023-09-23T18:17:35.220 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:35.321 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:35.366 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:35.415 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:35.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:35.419 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:35.478 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:17:35.478 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:17:35.519 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:35.560 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:35.595 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:35.598 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:35.599 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:35.699 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:35.740 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:35.841 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:35.846 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:35.854 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:35.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:35.908 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:17:35.930 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:17:35.956 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:35.980 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:36.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:36.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:36.037 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:36.122 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[184174]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.122 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[184174]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.122 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[184174]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.122 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[184174]: osdmap e560: 8 total, 8 up, 6 in 2023-09-23T18:17:36.122 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[184174]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.122 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[184174]: pgmap v1328: 119 pgs: 1 active+remapped+backfilling, 1 peering, 7 active+remapped+backfill_wait, 110 active+clean; 391 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 3045/18663 objects misplaced (16.316%); 0 B/s, 0 objects/s recovering 2023-09-23T18:17:36.123 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/605420229' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:17:36.123 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:17:36.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[179668]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[179668]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[179668]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[179668]: osdmap e560: 8 total, 8 up, 6 in 2023-09-23T18:17:36.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[179668]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[179668]: pgmap v1328: 119 pgs: 1 active+remapped+backfilling, 1 peering, 7 active+remapped+backfill_wait, 110 active+clean; 391 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 3045/18663 objects misplaced (16.316%); 0 B/s, 0 objects/s recovering 2023-09-23T18:17:36.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/605420229' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:17:36.123 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:35 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:17:36.138 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:36.179 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:36.272 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:36.281 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:36.281 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:36.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:35 smithi099 ceph-mon[167076]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:35 smithi099 ceph-mon[167076]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:35 smithi099 ceph-mon[167076]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:35 smithi099 ceph-mon[167076]: osdmap e560: 8 total, 8 up, 6 in 2023-09-23T18:17:36.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:35 smithi099 ceph-mon[167076]: failed to encode map e560 with expected crc 2023-09-23T18:17:36.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:35 smithi099 ceph-mon[167076]: pgmap v1328: 119 pgs: 1 active+remapped+backfilling, 1 peering, 7 active+remapped+backfill_wait, 110 active+clean; 391 MiB data, 5.6 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 3045/18663 objects misplaced (16.316%); 0 B/s, 0 objects/s recovering 2023-09-23T18:17:36.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:35 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/605420229' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:17:36.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:35 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:17:36.381 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:36.411 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:36.453 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:36.456 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:36.456 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:36.507 INFO:tasks.thrashosds.thrasher:in_osds: [1, 0, 4, 3, 6, 5] out_osds: [7, 2] dead_osds: [] live_osds: [0, 4, 6, 7, 3, 5, 2, 1] 2023-09-23T18:17:36.507 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:17:36.507 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T18:17:36.507 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.906208 2023-09-23T18:17:36.507 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0.9062075093714625 2023-09-23T18:17:36.557 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:36.589 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:36.627 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:36.631 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:36.631 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:36.732 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:36.757 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:36.794 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:36.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:36.798 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:36.898 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:36.929 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:36 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:37.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:36 smithi099 ceph-mon[167076]: failed to encode map e561 with expected crc 2023-09-23T18:17:37.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:36 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:17:37.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:36 smithi099 ceph-mon[167076]: osdmap e561: 8 total, 8 up, 6 in 2023-09-23T18:17:37.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:36 smithi099 ceph-mon[167076]: failed to encode map e561 with expected crc 2023-09-23T18:17:37.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:37.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:37.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:37.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[179668]: failed to encode map e561 with expected crc 2023-09-23T18:17:37.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:17:37.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[179668]: osdmap e561: 8 total, 8 up, 6 in 2023-09-23T18:17:37.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[179668]: failed to encode map e561 with expected crc 2023-09-23T18:17:37.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:37.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:37.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:37.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[184174]: failed to encode map e561 with expected crc 2023-09-23T18:17:37.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:17:37.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[184174]: osdmap e561: 8 total, 8 up, 6 in 2023-09-23T18:17:37.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[184174]: failed to encode map e561 with expected crc 2023-09-23T18:17:37.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:37.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:37.655 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:37.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:37.659 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:37.760 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:37.787 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:37.821 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:37.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:37.825 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:37.925 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:37.958 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:38.010 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:38.013 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:38.013 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:37 smithi099 ceph-mon[167076]: failed to encode map e561 with expected crc 2023-09-23T18:17:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:37 smithi099 ceph-mon[167076]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:37 smithi099 ceph-mon[167076]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:37 smithi099 ceph-mon[167076]: osdmap e562: 8 total, 8 up, 6 in 2023-09-23T18:17:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:37 smithi099 ceph-mon[167076]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:37 smithi099 ceph-mon[167076]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:37 smithi099 ceph-mon[167076]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:37 smithi099 ceph-mon[167076]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:37 smithi099 ceph-mon[167076]: pgmap v1331: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 259 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 1159/12327 objects misplaced (9.402%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:17:38.115 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:38.142 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:38.177 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:38.179 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:38.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[184174]: failed to encode map e561 with expected crc 2023-09-23T18:17:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[184174]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[184174]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[184174]: osdmap e562: 8 total, 8 up, 6 in 2023-09-23T18:17:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[184174]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[184174]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[184174]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[184174]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[184174]: pgmap v1331: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 259 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 1159/12327 objects misplaced (9.402%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:17:38.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[179668]: failed to encode map e561 with expected crc 2023-09-23T18:17:38.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[179668]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[179668]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[179668]: osdmap e562: 8 total, 8 up, 6 in 2023-09-23T18:17:38.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[179668]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[179668]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[179668]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[179668]: failed to encode map e562 with expected crc 2023-09-23T18:17:38.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:37 smithi055 ceph-mon[179668]: pgmap v1331: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 259 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 1159/12327 objects misplaced (9.402%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:17:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:38.182 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:38.224 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:38.283 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:38.321 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:38.417 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:17:38.438 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:17:38.457 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:17:38.476 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:17:38.495 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:17:38.517 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:17:38.543 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:17:38.562 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:38.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:38.569 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:38.569 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:17:38.571 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:17:38.571 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:17:38.571 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:17:38.571 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:17:38.571 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:17:38.571 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:17:38.571 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:17:38.571 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:17:38.670 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:38.696 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:38.729 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:38.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:38.733 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:38.833 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:38.864 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:39.003 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:39.009 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:39.009 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:39.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:38 smithi099 ceph-mon[167076]: failed to encode map e563 with expected crc 2023-09-23T18:17:39.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:38 smithi099 ceph-mon[167076]: failed to encode map e563 with expected crc 2023-09-23T18:17:39.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:38 smithi099 ceph-mon[167076]: osdmap e563: 8 total, 8 up, 6 in 2023-09-23T18:17:39.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:38 smithi099 ceph-mon[167076]: failed to encode map e563 with expected crc 2023-09-23T18:17:39.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/286386179' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:39.110 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:39.153 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:38 smithi055 ceph-mon[184174]: failed to encode map e563 with expected crc 2023-09-23T18:17:39.153 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:38 smithi055 ceph-mon[184174]: failed to encode map e563 with expected crc 2023-09-23T18:17:39.153 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:38 smithi055 ceph-mon[184174]: osdmap e563: 8 total, 8 up, 6 in 2023-09-23T18:17:39.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:38 smithi055 ceph-mon[184174]: failed to encode map e563 with expected crc 2023-09-23T18:17:39.154 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/286386179' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:39.154 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:38 smithi055 ceph-mon[179668]: failed to encode map e563 with expected crc 2023-09-23T18:17:39.154 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:38 smithi055 ceph-mon[179668]: failed to encode map e563 with expected crc 2023-09-23T18:17:39.154 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:38 smithi055 ceph-mon[179668]: osdmap e563: 8 total, 8 up, 6 in 2023-09-23T18:17:39.154 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:38 smithi055 ceph-mon[179668]: failed to encode map e563 with expected crc 2023-09-23T18:17:39.154 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/286386179' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:39.157 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:39.419 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:39.424 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:39.425 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:39.525 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:39.571 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:39.909 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:17:40.037 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:40.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:40.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:40.155 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:40.191 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:40.236 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:40.240 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:39 smithi099 ceph-mon[167076]: failed to encode map e564 with expected crc 2023-09-23T18:17:40.240 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:39 smithi099 ceph-mon[167076]: osdmap e564: 8 total, 8 up, 6 in 2023-09-23T18:17:40.240 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:39 smithi099 ceph-mon[167076]: failed to encode map e564 with expected crc 2023-09-23T18:17:40.240 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:39 smithi099 ceph-mon[167076]: failed to encode map e564 with expected crc 2023-09-23T18:17:40.240 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:39 smithi099 ceph-mon[167076]: pgmap v1334: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 259 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 1159/12327 objects misplaced (9.402%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:17:40.241 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:40.241 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:40.341 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:39 smithi055 ceph-mon[184174]: failed to encode map e564 with expected crc 2023-09-23T18:17:40.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:39 smithi055 ceph-mon[184174]: osdmap e564: 8 total, 8 up, 6 in 2023-09-23T18:17:40.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:39 smithi055 ceph-mon[184174]: failed to encode map e564 with expected crc 2023-09-23T18:17:40.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:39 smithi055 ceph-mon[184174]: failed to encode map e564 with expected crc 2023-09-23T18:17:40.342 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:39 smithi055 ceph-mon[184174]: pgmap v1334: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 259 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 1159/12327 objects misplaced (9.402%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:17:40.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:39 smithi055 ceph-mon[179668]: failed to encode map e564 with expected crc 2023-09-23T18:17:40.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:39 smithi055 ceph-mon[179668]: osdmap e564: 8 total, 8 up, 6 in 2023-09-23T18:17:40.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:39 smithi055 ceph-mon[179668]: failed to encode map e564 with expected crc 2023-09-23T18:17:40.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:39 smithi055 ceph-mon[179668]: failed to encode map e564 with expected crc 2023-09-23T18:17:40.342 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:39 smithi055 ceph-mon[179668]: pgmap v1334: 119 pgs: 1 active+remapped+backfilling, 1 peering, 4 active+remapped+backfill_wait, 113 active+clean; 259 MiB data, 4.2 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 527 op/s; 1159/12327 objects misplaced (9.402%); 2.3 MiB/s, 36 objects/s recovering 2023-09-23T18:17:40.343 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:40.368 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:40.403 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:40.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:40.407 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:40.508 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:40.534 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:40.568 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:40.572 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:40.573 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:40.599 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:17:40] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.43.0" 2023-09-23T18:17:40.674 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:40.700 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:40.736 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:40.739 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:40.740 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:40.811 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:17:40.811 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:17:40.811 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:17:40.811 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:17:40.811 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:17:40.840 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:40.847 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:40.877 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:40.919 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:40.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:40.922 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:40.987 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.4 primary-affinity to 0.906208 (8593892) 2023-09-23T18:17:41.023 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:41.061 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:41.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[184174]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[184174]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[184174]: osdmap e565: 8 total, 8 up, 6 in 2023-09-23T18:17:41.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[184174]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[184174]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1077189024' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9062075093714625}]: dispatch 2023-09-23T18:17:41.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[179668]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[179668]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[179668]: osdmap e565: 8 total, 8 up, 6 in 2023-09-23T18:17:41.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[179668]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[179668]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1077189024' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9062075093714625}]: dispatch 2023-09-23T18:17:41.202 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:17:41.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:40 smithi099 ceph-mon[167076]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:40 smithi099 ceph-mon[167076]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:40 smithi099 ceph-mon[167076]: osdmap e565: 8 total, 8 up, 6 in 2023-09-23T18:17:41.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:40 smithi099 ceph-mon[167076]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:40 smithi099 ceph-mon[167076]: failed to encode map e565 with expected crc 2023-09-23T18:17:41.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1077189024' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9062075093714625}]: dispatch 2023-09-23T18:17:41.472 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:41.478 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:41.478 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:41.578 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:41.600 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:17:41.608 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:41.695 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:41.698 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:41.699 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:41.799 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:41.828 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:41.864 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:41.867 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:41.867 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:41.968 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:41.986 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[179668]: failed to encode map e566 with expected crc 2023-09-23T18:17:41.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[179668]: failed to encode map e566 with expected crc 2023-09-23T18:17:41.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[179668]: failed to encode map e566 with expected crc 2023-09-23T18:17:41.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[179668]: failed to encode map e566 with expected crc 2023-09-23T18:17:41.987 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1077189024' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9062075093714625}]': finished 2023-09-23T18:17:42.009 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:42.051 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:42.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:42.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:42.162 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:42.206 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:42.217 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:42.257 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:42.262 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:42.262 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[179668]: osdmap e566: 8 total, 8 up, 6 in 2023-09-23T18:17:42.262 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[179668]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.262 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[179668]: pgmap v1337: 119 pgs: 2 active+remapped+backfilling, 2 peering, 1 active+remapped+backfill_wait, 114 active+clean; 183 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 402/8643 objects misplaced (4.651%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:17:42.263 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[184174]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.263 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[184174]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.263 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[184174]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.263 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[184174]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.263 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1077189024' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9062075093714625}]': finished 2023-09-23T18:17:42.263 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[184174]: osdmap e566: 8 total, 8 up, 6 in 2023-09-23T18:17:42.263 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[184174]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.263 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:41 smithi055 ceph-mon[184174]: pgmap v1337: 119 pgs: 2 active+remapped+backfilling, 2 peering, 1 active+remapped+backfill_wait, 114 active+clean; 183 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 402/8643 objects misplaced (4.651%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:17:42.264 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:41 smithi099 ceph-mon[167076]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:41 smithi099 ceph-mon[167076]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:41 smithi099 ceph-mon[167076]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:41 smithi099 ceph-mon[167076]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:41 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1077189024' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.9062075093714625}]': finished 2023-09-23T18:17:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:41 smithi099 ceph-mon[167076]: osdmap e566: 8 total, 8 up, 6 in 2023-09-23T18:17:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:41 smithi099 ceph-mon[167076]: failed to encode map e566 with expected crc 2023-09-23T18:17:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:41 smithi099 ceph-mon[167076]: pgmap v1337: 119 pgs: 2 active+remapped+backfilling, 2 peering, 1 active+remapped+backfill_wait, 114 active+clean; 183 MiB data, 4.0 GiB used, 532 GiB / 536 GiB avail; 0 B/s wr, 306 op/s; 402/8643 objects misplaced (4.651%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:17:42.365 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:42.396 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:42.434 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:42.438 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:42.438 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:42.507 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:42.539 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:42.569 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:42.908 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:42.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:42.911 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:43.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:43.043 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:43.102 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:43.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:43.113 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:43.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:42 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:17:43.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:42 smithi055 ceph-mon[179668]: failed to encode map e567 with expected crc 2023-09-23T18:17:43.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:42 smithi055 ceph-mon[179668]: failed to encode map e567 with expected crc 2023-09-23T18:17:43.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:42 smithi055 ceph-mon[179668]: osdmap e567: 8 total, 8 up, 6 in 2023-09-23T18:17:43.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:42 smithi055 ceph-mon[179668]: failed to encode map e567 with expected crc 2023-09-23T18:17:43.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:42 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:17:43.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:42 smithi055 ceph-mon[184174]: failed to encode map e567 with expected crc 2023-09-23T18:17:43.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:42 smithi055 ceph-mon[184174]: failed to encode map e567 with expected crc 2023-09-23T18:17:43.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:42 smithi055 ceph-mon[184174]: osdmap e567: 8 total, 8 up, 6 in 2023-09-23T18:17:43.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:42 smithi055 ceph-mon[184174]: failed to encode map e567 with expected crc 2023-09-23T18:17:43.214 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:43.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:42 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:17:43.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:42 smithi099 ceph-mon[167076]: failed to encode map e567 with expected crc 2023-09-23T18:17:43.268 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:42 smithi099 ceph-mon[167076]: failed to encode map e567 with expected crc 2023-09-23T18:17:43.269 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:42 smithi099 ceph-mon[167076]: osdmap e567: 8 total, 8 up, 6 in 2023-09-23T18:17:43.269 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:42 smithi099 ceph-mon[167076]: failed to encode map e567 with expected crc 2023-09-23T18:17:43.271 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:43.334 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:43.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:43.340 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:43.441 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:43.498 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:43.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:43.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:43.566 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:43.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:43.725 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:43.787 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:43.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:43.793 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:43.894 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:43.940 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:43.996 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:44.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:44.006 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:44.029 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[179668]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.029 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[179668]: osdmap e568: 8 total, 8 up, 6 in 2023-09-23T18:17:44.029 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[179668]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.029 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[179668]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.029 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[179668]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.029 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3485902774' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:17:44.029 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[179668]: pgmap v1340: 119 pgs: 1 active+remapped+backfilling, 3 peering, 115 active+clean; 121 MiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 33/5691 objects misplaced (0.580%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:17:44.045 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:17:44.057 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Removed 104657 objects 2023-09-23T18:17:44.058 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Clean up completed and total clean up time :145.404 2023-09-23T18:17:44.107 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:44.132 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:44.159 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:17:44.159 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:17:44.160 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:17:44.160 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:17:44.160 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.160 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:200048 9.7 e548/545 9:eb3a04c8:::benchmark_data_smithi055_386467_object95388:head v 548'12467, mlcod=548'12464)", 2023-09-23T18:17:44.160 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:23.384208+0000", 2023-09-23T18:17:44.160 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 20.768521386, 2023-09-23T18:17:44.160 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.048986330000000002, 2023-09-23T18:17:44.160 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.161 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.161 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.161 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.161 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.161 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.384208+0000", 2023-09-23T18:17:44.161 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.161 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.162 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.162 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.162 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.384208+0000", 2023-09-23T18:17:44.162 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.162 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.162 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.163 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.163 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.384208+0000", 2023-09-23T18:17:44.163 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:44.163 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.163 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.163 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.384214+0000", 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7000000000000002e-06 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.427570+0000", 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043355574000000001 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.164 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.427581+0000", 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1289e-05 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.427626+0000", 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.532e-05 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.427653+0000", 2023-09-23T18:17:44.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6585e-05 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.433148+0000", 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0054948180000000003 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.433195+0000", 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6817000000000003e-05 2023-09-23T18:17:44.166 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.167 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.167 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.167 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "backfill_remove(9.4 e548 9:20002837:::benchmark_data_smithi055_386467_object35193:head,465'2192,9:20008f28:::benchmark_data_smithi055_386467_object26637:head,464'1649,9:20011f42:::benchmark_data_smithi055_386467_object20696:head,464'1297,9:2001cbfd:::benchmark_data_smithi055_386467_object39716:head,466'2482,9:2001f9b7:::benchmark_data_smithi055_386467_object43296:head,466'2680,9:20039dd9:::benchmark_data_smithi055_386467_object52899:head,467'3284,9:2005a399:::benchmark_data_smithi055_386467_object61413:head,473'3843,9:2005ddd3:::benchmark_data_smithi055_386467_object69817:head,478'4397,9:200656bf:::benchmark_data_smithi055_386467_object40370:head,466'2515,9:2007481c:::benchmark_data_smithi055_386467_object50157:head,467'3109,9:200758dc:::benchmark_data_smithi055_386467_object53296:head,467'3307,9:2008db12:::benchmark_data_smithi055_386467_object94542:head,492'5941,9:200a3780:::benchmark_data_smithi055_386467_object77950:head,480'4918,9:200b5de0:::benchmark_data_smithi055_386467_object25066:head,464'1562,9:200c9171:::benchmark_data_smithi055_386467_object94529:head,492'5940,9:200d1489:::benchmark_data_smithi055_386467_object67031:head,473'4208,9:200d5392:::benchmark_data_smithi055_386467_object92876:head,492'5833,9:200eb125:::benchmark_data_smithi055_386467_object19115:head,463'1214,9:200ee3f6:::benchmark_data_smithi055_386467_object91796:head,490'5769,9:2012365e:::benchmark_data_smithi055_386467_object29924:head,464'1854,9:2013883f:::benchmark_data_smithi055_386467_object37077:head,466'2308,9:2014f85e:::benchmark_data_smithi055_386467_object31738:head,464'1973,9:20153eb1:::benchmark_data_smithi055_386467_object29897:head,464'1850,9:20169e8a:::benchmark_data_smithi055_386467_object89103:head,487'5599,9:2017ceb4:::benchmark_data_smithi055_386467_object21825:head,464'1369,9:20180797:::benchmark_data_smithi055_386467_object33967:head,465'2120,9:2018fc81:::benchmark_data_smithi055_386467_object89080:head,487'5596,9:201a96d9:::benchmark_data_smithi055_386467_object86132:head,486'5416,9:201c2320:::benchmark_data_smithi055_386467_object32405:head,464'2017,9:201c4de9:::benchmark_data_smithi055_386467_object44131:head,466'2736,9:201cc24e:::benchmark_data_smithi055_386467_object66793:head,473'4191,9:201f4742:::benchmark_data_smithi055_386467_object37231:head,466'2319,9:20204e72:::benchmark_data_smithi055_386467_object61565:head,473'3855,9:20210fe2:::benchmark_data_smithi055_386467_object64168:head,473'4035,9:2021e45d:::benchmark_data_smithi055_386467_object81047:head,480'5106,9:202202a7:::benchmark_data_smithi055_386467_object50290:head,467'3121,9:2023de73:::benchmark_data_smithi055_386467_object85854:head,486'5394,9:20241dcf:::benchmark_data_smithi055_386467_object82427:head,482'5199,9:2024510a:::benchmark_data_smithi055_386467_object72669:head,478'4587,9:20253e2f:::benchmark_data_smithi055_386467_object23199:head,464'1447,9:202571b8:::benchmark_data_smithi055_386467_object51726:head,467'3204,9:20270176:::benchmark_data_smithi055_386467_object29458:head,464'1813,9:202718a0:::benchmark_data_smithi055_386467_object90931:head,487'5716,9:20273c7e:::benchmark_data_smithi055_386467_object29995:head,464'1863,9:20285c89:::benchmark_data_smithi055_386467_object64411:head,473'4061,9:20299011:::benchmark_data_smithi055_386467_object64379:head,473'4057,9:202aff9d:::benchmark_data_smithi055_386467_object48414:head,467'3010,9:202bcd43:::benchmark_data_smithi055_386467_object77367:head,480'4886,9:202c0e3d:::benchmark_data_smithi055_386467_object92172:head,491'5794,9:202cded3:::benchmark_data_smithi055_386467_object37994:head,466'2374,9:202eaae4:::benchmark_data_smithi055_386467_object80950:head,480'5100,9:202ef91f:::benchmark_data_smithi055_386467_object58272:head,472'3643,9:202f4ead:::benchmark_data_smithi055_386467_object47233:head,467'2933,9:202f7978:::benchmark_data_smithi055_386467_object85984:head,486'5404,9:202fca1e:::benchmark_data_smithi055_386467_object49509:head,467'3071,9:2030108e:::benchmark_data_smithi055_386467_object92694:head,491'5825,9:20320c1e:::benchmark_data_smithi055_386467_object34750:head,465'2166,9:203268b2:::benchmark_data_smithi055_386467_object62370:head,473'3911,9:2032f3fa:::benchmark_data_smithi055_386467_object60100:head,472'3754,9:203347f6:::benchmark_data_smithi055_386467_object88235:head,487'5552,9:20338463:::benchmark_data_smithi055_386467_object41073:head,466'2562,9:20346ff8:::benchmark_data_smithi055_386467_object29594:head,464'1822,9:203585a1:::benchmark_data_smithi055_386467_object64255:head,473'4044,9:2036442f:::benchmark_data_smithi055_386467_object81958:head,481'5166,9:2037cf76:::benchmark_data_smithi055_386467_object51568:head,467'3198,9:20384873:::benchmark_data_smithi055_386467_object51385:head,467'3185,9:203b3111:::benchmark_data_smithi055_386467_object90967:head,487'5719,9:203ba47b:::benchmark_data_smithi055_386467_object60666:head,472'3792,9:203d0cd6:::benchmark_data_smithi055_386467_object54564:head,470'3391,9:203d67a1:::benchmark_data_smithi055_386467_object54946:head,471'3419,9:203dfc6c:::benchmark_data_smithi055_386467_object70300:head,478'4433,9:203e5db6:::benchmark_data_smithi055_386467_object69368:head,477'4359,9:203ecebf:::benchmark_data_smithi055_386467_object71456:head,478'4506,9:2041701b:::benchmark_data_smithi055_386467_object25578:head,464'1585,9:2041b288:::benchmark_data_smithi055_386467_object40386:head,466'2516,9:20420e69:::benchmark_data_smithi055_386467_object27893:head,464'1726,9:20422b36:::benchmark_data_smithi055_386467_object43080:head,466'2668,9:204293e5:::benchmark_data_smithi055_386467_object43798:head,466'2715,9:204327f1:::benchmark_data_smithi055_386467_object60813:head,472'3805,9:2044a461:::benchmark_data_smithi055_386467_object67426:head,475'4238,9:20454c5c:::benchmark_data_smithi055_386467_object92052:head,491'5785,9:2046ec93:::benchmark_data_smithi055_386467_object27455:head,464'1691,9:2047160a:::benchmark_data_smithi055_386467_object28179:head,464'1745,9:2047463c:::benchmark_data_smithi055_386467_object47075:head,467'2922,9:204754db:::benchmark_data_smithi055_386467_object41161:head,466'2563,9:2047560a:::benchmark_data_smithi055_386467_object49484:head,467'3067,9:20475e09:::benchmark_data_smithi055_386467_object81891:head,481'5158,9:20477a32:::benchmark_data_smithi055_386467_object30765:head,464'1909,9:2047c211:::benchmark_data_smithi055_386467_object86059:head,486'5410,9:20482966:::benchmark_data_smithi055_386467_object57388:head,472'3578,9:2049b8d0:::benchmark_data_smithi055_386467_object45832:head,467'2838,9:204b03ce:::benchmark_data_smithi055_386467_object86084:head,486'5413,9:204b3707:::benchmark_data_smithi055_386467_object90801:head,487'5706,9:204b9694:::benchmark_data_smithi055_386467_object26946:head,464'1666,9:204bc39c:::benchmark_data_smithi055_386467_object18680:head,463'1191,9:204c3d4e:::benchmark_data_smithi055_386467_object45722:head,467'2827,9:204df576:::benchmark_data_smithi055_386467_object27479:head,464'1692,9:204dfd53:::benchmark_data_smithi055_386467_object36943:head,466'2293,9:204ef75a:::benchmark_data_smithi055_386467_object47333:head,467'2936,9:204f402d:::benchmark_data_smithi055_386467_object35468:head,465'2209,9:204f4f08:::benchmark_data_smithi055_386467_object76882:head,480'4847,9:204fa3e7:::benchmark_data_smithi055_386467_object91366:head,487'5740,9:20506291:::benchmark_data_smithi055_386467_object61235:head,472'3830,9:205175c4:::benchmark_data_smithi055_386467_object60038:head,472'3751,9:20522636:::benchmark_data_smithi055_386467_object75003:head,480'4732,9:205242c4:::benchmark_data_smithi055_386467_object54682:head,470'3399,9:2052a570:::benchmark_data_smithi055_386467_object57697:head,472'3602,9:20538ab5:::benchmark_data_smithi055_386467_object73455:head,478'4633,9:2053a225:::benchmark_data_smithi055_386467_object68452:head,476'4303,9:2053d4fa:::benchmark_data_smithi055_386467_object90488:head,487'5687,9:205609bd:::benchmark_data_smithi055_386467_object31910:head,464'1983,9:20561026:::benchmark_data_smithi055_386467_object78629:head,480'4958,9:2056534d:::benchmark_data_smithi055_386467_object56525:head,472'3515,9:20567351:::benchmark_data_smithi055_386467_object24520:head,464'1532,9:20583ae6:::benchmark_data_smithi055_386467_object20922:head,464'1318,9:205b0598:::benchmark_data_smithi055_386467_object68450:head,476'4301,9:205b8786:::benchmark_data_smithi055_386467_object68451:head,476'4302,9:205da362:::benchmark_data_smithi055_386467_object44856:head,467'2777,9:205dfa50:::benchmark_data_smithi055_386467_object72794:head,478'4596,9:205e9d55:::benchmark_data_smithi055_386467_object57312:head,472'3571,9:205edbf3:::benchmark_data_smithi055_386467_object80584:head,480'5078,9:2060283a:::benchmark_data_smithi055_386467_object81157:head,480'5110,9:2060e880:::benchmark_data_smithi055_386467_object85677:head,486'5383,9:20611cdd:::benchmark_data_smithi055_386467_object86490:head,486'5442,9:20622696:::benchmark_data_smithi055_386467_object87890:head,487'5527,9:20627fa3:::benchmark_data_smithi055_386467_object31546:head,464'1956,9:2062d087:::benchmark_data_smithi055_386467_object64202:head,473'4039,9:20631614:::benchmark_data_smithi055_386467_object81501:head,481'5136,9:206462b0:::benchmark_data_smithi055_386467_object79413:head,480'5015,9:20650a3d:::benchmark_data_smithi055_386467_object54450:head,470'3382,9:20651aa8:::benchmark_data_smithi055_386467_object71891:head,478'4538,9:2065cd49:::benchmark_data_smithi055_386467_object74562:head,478'4703,9:2066fcf4:::benchmark_data_smithi055_386467_object67052:head,473'4214,9:20673e32:::benchmark_data_smithi055_386467_object46201:head,467'2866,9:2067580f:::benchmark_data_smithi055_386467_object44414:head,467'2755,9:2067a2ba:::benchmark_data_smithi055_386467_object26766:head,464'1659,9:2069d47f:::benchmark_data_smithi055_386467_object25625:head,464'1589,9:206ac033:::benchmark_data_smithi055_386467_object21097:head,464'1328,9:206c7829:::benchmark_data_smithi055_386467_object78070:head,480'4924,9:206c8dda:::benchmark_data_smithi055_386467_object50433:head,467'3129,9:206dc471:::benchmark_data_smithi055_386467_object62843:head,473'3947,9:206e9226:::benchmark_data_smithi055_386467_object53378:head,467'3313,9:206f0d8d:::benchmark_data_smithi055_386467_object33645:head,464'2100,9:206f1d32:::benchmark_data_smithi055_386467_object25151:head,464'1565,9:206f7279:::benchmark_data_smithi055_386467_object68568:head,476'4313,9:206fd4da:::benchmark_data_smithi055_386467_object67734:head,476'4255,9:206fed9d:::benchmark_data_smithi055_386467_object62746:head,473'3940,9:2070e2fc:::benchmark_data_smithi055_386467_object40434:head,466'2518,9:2071c0d1:::benchmark_data_smithi055_386467_object39114:head,466'2446,9:2071fbc2:::benchmark_data_smithi055_386467_object71785:head,478'4529,9:2072dedf:::benchmark_data_smithi055_386467_object23866:head,464'1494,9:20765eee:::benchmark_data_smithi055_386467_object78330:head,480'4937,9:207678a3:::benchmark_data_smithi055_386467_object70194:head,478'4423,9:2076b9b1:::benchmark_data_smithi055_386467_object79069:head,480'4989,9:20777393:::benchmark_data_smithi055_386467_object74038:head,478'4663,9:2077f12f:::benchmark_data_smithi055_386467_object51013:head,467'3158,9:2078a6d1:::benchmark_data_smithi055_386467_object81827:head,481'5154,9:207a5374:::benchmark_data_smithi055_386467_object80973:head,480'5104,9:207a6372:::benchmark_data_smithi055_386467_object54497:head,470'3385,9:207b0f77:::benchmark_data_smithi055_386467_object70637:head,478'4452,9:207bd4ce:::benchmark_data_smithi055_386467_object77182:head,480'4870,9:207c7de3:::benchmark_data_smithi055_386467_object94937:head,492'5970,9:207d9cb8:::benchmark_data_smithi055_386467_object82318:head,481'5187,9:207e9643:::benchmark_data_smithi055_386467_object51787:head,467'3208,9:207ea2d9:::benchmark_data_smithi055_386467_object74586:head,478'4706,9:207f59d7:::benchmark_data_smithi055_386467_object51543:head,467'3195,9:2080dfe7:::benchmark_data_smithi055_386467_object33791:head,465'2112,9:208386f5:::benchmark_data_smithi055_386467_object73554:head,478'4640,9:2083a7c6:::benchmark_data_smithi055_386467_object56611:head,472'3517,9:2083b4ab:::benchmark_data_smithi055_386467_object21710:head,464'1359,9:20849e8e:::benchmark_data_smithi055_386467_object30247:head,464'1883,9:2085d1e0:::benchmark_data_smithi055_386467_object25597:head,464'1586,9:2085fd3d:::benchmark_data_smithi055_386467_object59394:head,472'3708,9:2086d1c6:::benchmark_data_smithi055_386467_object56693:head,472'3523,9:2086d8b4:::benchmark_data_smithi055_386467_object65828:head,473'4129,9:20876a75:::benchmark_data_smithi055_386467_object67184:head,473'4225,9:20894d1b:::benchmark_data_smithi055_386467_object86035:head,486'5407,9:2089aeb6:::benchmark_data_smithi055_386467_object72444:head,478'4573,9:208a17c8:::benchmark_data_smithi055_386467_object95060:head,492'5979,9:208b0d1b:::benchmark_data_smithi055_386467_object50623:head,467'3139,9:208c223b:::benchmark_data_smithi055_386467_object34482:head,465'2149,9:208c5d7b:::benchmark_data_smithi055_386467_object35775:head,465'2229,9:208e839c:::benchmark_data_smithi055_386467_object26184:head,464'1624,9:208ea5a3:::benchmark_data_smithi055_386467_object59897:head,472'3740,9:208eea4c:::benchmark_data_smithi055_386467_object93102:head,492'5845,9:208fc682:::benchmark_data_smithi055_386467_object60601:head,472'3789,9:209000b0:::benchmark_data_smithi055_386467_object80285:head,480'5065,9:20941342:::benchmark_data_smithi055_386467_object38758:head,466'2424,9:2094ec90:::benchmark_data_smithi055_386467_object71855:head,478'4534,9:20955a16:::benchmark_data_smithi055_386467_object20384:head,464'1280,9:2095e32a:::benchmark_data_smithi055_386467_object66174:head,473'4146,9:209756ff:::benchmark_data_smithi055_386467_object22694:head,464'1418,9:209821a6:::benchmark_data_smithi055_386467_object30548:head,464'1895,9:20987fda:::benchmark_data_smithi055_386467_object87159:head,486'5477,9:2099791d:::benchmark_data_smithi055_386467_object49031:head,467'3041,9:209a1e0c:::benchmark_data_smithi055_386467_object44048:head,466'2728,9:209a707c:::benchmark_data_smithi055_386467_object90030:head,487'5654,9:209b2f98:::benchmark_data_smithi055_386467_object63365:head,473'3982,9:209b70e3:::benchmark_data_smithi055_386467_object63395:head,473'3985,9:209c5d65:::benchmark_data_smithi055_386467_object33068:head,464'2061,9:209c68d3:::benchmark_data_smithi055_386467_object69617:head,477'4380,9:209de65c:::benchmark_data_smithi055_386467_object92121:head,491'5789,9:209dfe29:::benchmark_data_smithi055_386467_object33561:head,464'2094,9:209e00a8:::benchmark_data_smithi055_386467_object94222:head,492'5926,9:209e18b7:::benchmark_data_smithi055_386467_object91710:head,490'5763,9:20a180d3:::benchmark_data_smithi055_386467_object94758:head,492'5959,9:20a34e18:::benchmark_data_smithi055_386467_object37343:head,466'2327,9:20a3ee08:::benchmark_data_smithi055_386467_object82516:head,483'5210,9:20a484c8:::benchmark_data_smithi055_386467_object26711:head,464'1656,9:20a4f053:::benchmark_data_smithi055_386467_object47745:head,467'2961,9:20a57150:::benchmark_data_smithi055_386467_object34261:head,465'2137,9:20a6cd24:::benchmark_data_smithi055_386467_object50588:head,467'3138,9:20a6f6e0:::benchmark_data_smithi055_386467_object66756:head,473'4190,9:20a70d59:::benchmark_data_smithi055_386467_object76967:head,480'4856,9:20a77ee4:::benchmark_data_smithi055_386467_object80882:head,480'5094,9:20a7e282:::benchmark_data_smithi055_386467_object17053:head,463'1103,9:20a81a60:::benchmark_data_smithi055_386467_object27563:head,464'1703,9:20a88248:::benchmark_data_smithi055_386467_object90935:head,487'5717,9:20a8c8b9:::benchmark_data_smithi055_386467_object51233:head,467'3174,9:20a96d2e:::benchmark_data_smithi055_386467_object90240:head,487'5670,9:20abdceb:::benchmark_data_smithi055_386467_object18349:head,463'1174,9:20ad348d:::benchmark_data_smithi055_386467_object44278:head,467'2745,9:20ae193a:::benchmark_data_smithi055_386467_object51298:head,467'3178,9:20ae5e82:::benchmark_data_smithi055_386467_object27448:head,464'1690,9:20afa314:::benchmark_data_smithi055_386467_object31440:head,464'1947,9:20b061dc:::benchmark_data_smithi055_386467_object77139:head,480'4864,9:20b10845:::benchmark_data_smithi055_386467_object71587:head,478'4515,9:20b10d45:::benchmark_data_smithi055_386467_object81946:head,481'5163,9:20b29315:::benchmark_data_smithi055_386467_object89512:head,487'5625,9:20b2eb9d:::benchmark_data_smithi055_386467_object61551:head,473'3853,9:20b37eae:::benchmark_data_smithi055_386467_object57820:head,472'3607,9:20b3e4b9:::benchmark_data_smithi055_386467_object92878:head,492'5834,9:20b61a8d:::benchmark_data_smithi055_386467_object77719:head,480'4906,9:20b6f974:::benchmark_data_smithi055_386467_object84576:head,484'5310,9:20b8c909:::benchmark_data_smithi055_386467_object36263:head,465'2259,9:20bfd5dd:::benchmark_data_smithi055_386467_object24340:head,464'1517,9:20c297ee:::benchmark_data_smithi055_386467_object22392:head,464'1400,9:20c2ce83:::benchmark_data_smithi055_386467_object73511:head,478'4636,9:20c55e39:::benchmark_data_smithi055_386467_object20789:head,464'1304,9:20c5d384:::benchmark_data_smithi055_386467_object68371:head,476'4295,9:20c84f62:::benchmark_data_smithi055_386467_object34760:head,465'2168,9:20c88405:::benchmark_data_smithi055_386467_object55552:head,471'3458,9:20c8a909:::benchmark_data_smithi055_386467_object67076:head,473'4216,9:20ca9cb7:::benchmark_data_smithi055_386467_object52268:head,467'3239,9:20cabfa2:::benchmark_data_smithi055_386467_object77320:head,480'4883,9:20cc90e7:::benchmark_data_smithi055_386467_object49066:head,467'3046,9:20cd9751:::benchmark_data_smithi055_386467_object38645:head,466'2410,9:20ce32d1:::benchmark_data_smithi055_386467_object73888:head,478'4655,9:20d1bc9f:::benchmark_data_smithi055_386467_object50885:head,467'3151,9:20d1f2b5:::benchmark_data_smithi055_386467_object66452:head,473'4164,9:20d206f7:::benchmark_data_smithi055_386467_object44322:head,467'2750,9:20d43cd8:::benchmark_data_smithi055_386467_object70887:head,478'4468,9:20d581e7:::benchmark_data_smithi055_386467_object84795:head,484'5324,9:20d6383d:::benchmark_data_smithi055_386467_object23740:head,464'1480,9:20d689db:::benchmark_data_smithi055_386467_object76103:head,480'4804,9:20d68bf8:::benchmark_data_smithi055_386467_object56674:head,472'3522,9:20d84a5e:::benchmark_data_smithi055_386467_object61668:head,473'3866,9:20d87ea2:::benchmark_data_smithi055_386467_object85893:head,486'5398,9:20d9e59d:::benchmark_data_smithi055_386467_object81846:head,481'5155,9:20da32c8:::benchmark_data_smithi055_386467_object95188:head,492'5987,9:20db0760:::benchmark_data_smithi055_386467_object38717:head,466'2418,9:20db5222:::benchmark_data_smithi055_386467_object84180:head,484'5289,9:20db7600:::benchmark_data_smithi055_386467_object66693:head,473'4184,9:20dc8930:::benchmark_data_smithi055_386467_object36338:head,465'2263,9:20dc8e2e:::benchmark_data_smithi055_386467_object93267:head,492'5861,9:20dcb1a9:::benchmark_data_smithi055_386467_object88129:head,487'5543,9:20dcbe46:::benchmark_data_smithi055_386467_object78879:head,480'4973,9:20df5da7:::benchmark_data_smithi055_386467_object23908:head,464'1497,9:20dfb706:::benchmark_data_smithi055_386467_object17753:head,463'1136,9:20e0b792:::benchmark_data_smithi055_386467_object27819:head,464'1716,9:20e0e93c:::benchmark_data_smithi055_386467_object59603:head,472'3720,9:20e18ede:::benchmark_data_smithi055_386467_object59260:head,472'3698,9:20e23155:::benchmark_data_smithi055_386467_object83579:head,484'5267,9:20e27efb:::benchmark_data_smithi055_386467_object83419:head,484'5257,9:20e512bf:::benchmark_data_smithi055_386467_object33373:head,464'2080,9:20e570ea:::benchmark_data_smithi055_386467_object63110:head,473'3968,9:20e75d3f:::benchmark_data_smithi055_386467_object18935:head,463'1204,9:20e8c460:::benchmark_data_smithi055_386467_object42703:head,466'2653,9:20e93ba6:::benchmark_data_smithi055_386467_object40518:head,466'2525,9:20e95b18:::benchmark_data_smithi055_386467_object78390:head,480'4940,9:20e9a5df:::benchmark_data_smithi055_386467_object85875:head,486'5395,9:20ea420c:::benchmark_data_smithi055_386467_object94436:head,492'5934,9:20ea8506:::benchmark_data_smithi055_386467_object40454:head,466'2520,9:20ead76f:::benchmark_data_smithi055_386467_object42680:head,466'2650,9:20eae669:::benchmark_data_smithi055_386467_object17691:head,463'1131,9:20ec5c74:::benchmark_data_smithi055_386467_object94685:head,492'5951,9:20ec76dc:::benchmark_data_smithi055_386467_object91413:head,490'5741,9:20eca925:::benchmark_data_smithi055_386467_object67505:head,475'4245,9:20eccc22:::benchmark_data_smithi055_386467_object30789:head,464'1911,9:20ecf21c:::benchmark_data_smithi055_386467_object18332:head,463'1171,9:20edae04:::benchmark_data_smithi055_386467_object55765:head,471'3470,9:20eed89a:::benchmark_data_smithi055_386467_object87464:head,486'5497,9:20ef2a75:::benchmark_data_smithi055_386467_object72081:head,478'4556,9:20ef7882:::benchmark_data_smithi055_386467_object66463:head,473'4165,9:20f0d069:::benchmark_data_smithi055_386467_object37126:head,466'2313,9:20f1d2c2:::benchmark_data_smithi055_386467_object28502:head,464'1764,9:20f2c33c:::benchmark_data_smithi055_386467_object54401:head,470'3378,9:20f3027c:::benchmark_data_smithi055_386467_object81169:head,480'5112,9:20f3a0c2:::benchmark_data_smithi055_386467_object71995:head,478'4548,9:20f3ea55:::benchmark_data_smithi055_386467_object24536:head,464'1536,9:20f4ac4b:::benchmark_data_smithi055_386467_object51040:head,467'3160,9:20f4dfa1:::benchmark_data_smithi055_386467_object25179:head,464'1567,9:20f5897c:::benchmark_data_smithi055_386467_object18402:head,463'1178,9:20f6d7ba:::benchmark_data_smithi055_386467_object31139:head,464'1930,9:20f747bf:::benchmark_data_smithi055_386467_object88048:head,487'5537,9:20f76e75:::benchmark_data_smithi055_386467_object33265:head,464'2075,9:20f817b8:::benchmark_data_smithi055_386467_object24217:head,464'1511,9:20f9e236:::benchmark_data_smithi055_386467_object20141:head,464'1268,9:20f9f9f4:::benchmark_data_smithi055_386467_object93439:head,492'5875,9:20fa0ecb:::benchmark_data_smithi055_386467_object72580:head,478'4581,9:20fa3eed:::benchmark_data_smithi055_386467_object92283:head,491'5800,9:20fad3ba:::benchmark_data_smithi055_386467_object84391:head,484'5297,9:20fb844a:::benchmark_data_smithi055_386467_object47555:head,467'2946,9:20fca5e7:::benchmark_data_smithi055_386467_object22882:head,464'1429,9:20fcef53:::benchmark_data_smithi055_386467_object94320:head,492'5929,9:20fd0718:::benchmark_data_smithi055_386467_object73184:head,478'4616,9:20fd134f:::benchmark_data_smithi055_386467_object49126:head,467'3049,9:20fe20f0:::benchmark_data_smithi055_386467_object17174:head,463'1106,9:20fe7011:::benchmark_data_smithi055_386467_object50948:head,467'3156,9:20fe8b9d:::benchmark_data_smithi055_386467_object72315:head,478'4565,9:20fef28f:::benchmark_data_smithi055_386467_object50193:head,467'3112,9:21005441:::benchmark_data_smithi055_386467_object39797:head,466'2485,9:2100836a:::benchmark_data_smithi055_386467_object87488:head,486'5500,9:21017c8a:::benchmark_data_smithi055_386467_object57319:head,472'3572,9:21037c62:::benchmark_data_smithi055_386467_object57450:head,472'3582,9:2104d51a:::benchmark_data_smithi055_386467_object37598:head,466'2355,9:2105b773:::benchmark_data_smithi055_386467_object49261:head,467'3055,9:2107fea6:::benchmark_data_smithi055_386467_object54129:head,470'3364,9:210828a7:::benchmark_data_smithi055_386467_object42726:head,466'2655,9:210afec5:::benchmark_data_smithi055_386467_object53822:head,470'3341,9:210b54b1:::benchmark_data_smithi055_386467_object85028:head,484'5341,9:210d237d:::benchmark_data_smithi055_386467_object47455:head,467'2942,9:210d6e9d:::benchmark_data_smithi055_386467_object43929:head,466'2722,9:210d79e1:::benchmark_data_smithi055_386467_object55070:head,471'3427,9:210e58d0:::benchmark_data_smithi055_386467_object51353:head,467'3183,9:2110268a:::benchmark_data_smithi055_386467_object53718:head,467'3336,9:211199b1:::benchmark_data_smithi055_386467_object50374:head,467'3125,9:2113a716:::benchmark_data_smithi055_386467_object32214:head,464'2008,9:21141e7a:::benchmark_data_smithi055_386467_object64686:head,473'4075,9:211515ee:::benchmark_data_smithi055_386467_object86563:head,486'5447,9:2115fd2e:::benchmark_data_smithi055_386467_object75329:head,480'4754,9:21189076:::benchmark_data_smithi055_386467_object80919:head,480'5097,9:211988bc:::benchmark_data_smithi055_386467_object85795:head,486'5391,9:211a29fb:::benchmark_data_smithi055_386467_object37040:head,466'2304,9:211a8708:::benchmark_data_smithi055_386467_object42315:head,466'2631,9:211b0acc:::benchmark_data_smithi055_386467_object54944:head,471'3417,9:211b0ec7:::benchmark_data_smithi055_386467_object89295:head,487'5610,9:211c4fb4:::benchmark_data_smithi055_386467_object20177:head,464'1272,9:211cfeb1:::benchmark_data_smithi055_386467_object65994:head,473'4139,9:211dd172:::benchmark_data_smithi055_386467_object45826:head,467'2836,9:211ed0b8:::benchmark_data_smithi055_386467_object57987:head,472'3621,9:2120f5e9:::benchmark_data_smithi055_386467_object71314:head,478'4493,9:2123cda6:::benchmark_data_smithi055_386467_object94006:head,492'5913,9:2124f304:::benchmark_data_smithi055_386467_object26425:head,464'1635,9:2125aff7:::benchmark_data_smithi055_386467_object78538:head,480'4954,9:2125d79e:::benchmark_data_smithi055_386467_object36741:head,465'2287,9:21278569:::benchmark_data_smithi055_386467_object42602:head,466'2644,9:2127abfe:::benchmark_data_smithi055_386467_object69128:head,477'4348,9:21281e8e:::benchmark_data_smithi055_386467_object57338:head,472'3574,9:21287664:::benchmark_data_smithi055_386467_object51481:head,467'3191,9:2129eba4:::benchmark_data_smithi055_386467_object38568:head,466'2406,9:212abb3d:::benchmark_data_smithi055_386467_object45777:head,467'2831,9:212b1722:::benchmark_data_smithi055_386467_object26321:head,464'1631,9:212b6c1c:::benchmark_data_smithi055_386467_object34571:head,465'2156,9:212cad8c:::benchmark_data_smithi055_386467_object21058:head,464'1325,9:212d44fd:::benchmark_data_smithi055_386467_object74199:head,478'4672,9:212d77e8:::benchmark_data_smithi055_386467_object91839:head,490'5770,9:212e678e:::benchmark_data_smithi055_386467_object46832:head,467'2904,9:21309d3d:::benchmark_data_smithi055_386467_object65804:head,473'4128,9:2130f146:::benchmark_data_smithi055_386467_object86706:head,486'5456,9:2131ea87:::benchmark_data_smithi055_386467_object52542:head,467'3260,9:21325f82:::benchmark_data_smithi055_386467_object65719:head,473'4126,9:21330950:::benchmark_data_smithi055_386467_object26360:head,464'1634,9:2134e22b:::benchmark_data_smithi055_386467_object36042:head,465'2250,9:21351106:::benchmark_data_smithi055_386467_object61510:head,473'3851,9:2137632a:::benchmark_data_smithi055_386467_object71879:head,478'4536,9:2137653a:::benchmark_data_smithi055_386467_object32241:head,464'2010,9:2138a0bb:::benchmark_data_smithi055_386467_object46799:head,467'2901,9:2139555d:::benchmark_data_smithi055_386467_object38200:head,466'2385,9:21397721:::benchmark_data_smithi055_386467_object69610:head,477'4378,9:21398787:::benchmark_data_smithi055_386467_object59834:head,472'3734,9:213a89eb:::benchmark_data_smithi055_386467_object56128:head,472'3491,9:213a9ab4:::benchmark_data_smithi055_386467_object93407:head,492'5873,9:213b5e8c:::benchmark_data_smithi055_386467_object44802:head,467'2771,9:213c2c80:::benchmark_data_smithi055_386467_object33061:head,464'2060,9:213e1904:::benchmark_data_smithi055_386467_object60734:head,472'3797,9:2140136b:::benchmark_data_smithi055_386467_object86223:head,486'5423,9:2140925c:::benchmark_data_smithi055_386467_object88740:head,487'5581,9:2140d534:::benchmark_data_smithi055_386467_object67798:head,476'4259,9:2141680c:::benchmark_data_smithi055_386467_object37693:head,466'2361,9:2141e51d:::benchmark_data_smithi055_386467_object60530:head,472'3786,9:21427aba:::benchmark_data_smithi055_386467_object80957:head,480'5102,9:2142df8d:::benchmark_data_smithi055_386467_object81599:head,481'5143,9:214370c7:::benchmark_data_smithi055_386467_object42725:head,466'2654,9:2144e758:::benchmark_data_smithi055_386467_object64569:head,473'4068,9:21457da6:::benchmark_data_smithi055_386467_object49781:head,467'3090,9:2146c6cc:::benchmark_data_smithi055_386467_object26429:head,464'1636,9:2147c395:::benchmark_data_smithi055_386467_object42541:head,466'2642,9:214889c7:::benchmark_data_smithi055_386467_object80509:head,480'5076,9:214c92c2:::benchmark_data_smithi055_386467_object26542:head,464'1643,9:214cdbbe:::benchmark_data_smithi055_386467_object38819:head,466'2429,9:214efb91:::benchmark_data_smithi055_386467_object74742:head,479'4713,9:214feff2:::benchmark_data_smithi055_386467_object17439:head,463'1117,9:214ffbf5:::benchmark_data_smithi055_386467_object31444:head,464'1950,9:215151ab:::benchmark_data_smithi055_386467_object41515:head,466'2585,9:215177b7:::benchmark_data_smithi055_386467_object61395:head,473'3841,9:21528f13:::benchmark_data_smithi055_386467_object60250:head,472'3767,9:21569c7b:::benchmark_data_smithi055_386467_object80958:head,480'5103,9:2157a718:::benchmark_data_smithi055_386467_object53251:head,467'3303,9:21593721:::benchmark_data_smithi055_386467_object47771:head,467'2964,9:215b9102:::benchmark_data_smithi055_386467_object33072:head,464'2062,9:215dc3f6:::benchmark_data_smithi055_386467_object62841:head,473'3946,9:21606dcc:::benchmark_data_smithi055_386467_object46371:head,467'2877,9:21607684:::benchmark_data_smithi055_386467_object29691:head,464'1833,9:2160d995:::benchmark_data_smithi055_386467_object44726:head,467'2766,9:21610b04:::benchmark_data_smithi055_386467_object27199:head,464'1677,9:2161db2c:::benchmark_data_smithi055_386467_object58166:head,472'3636,9:2162d31c:::benchmark_data_smithi055_386467_object49721:head,467'3083,9:216600f8:::benchmark_data_smithi055_386467_object69129:head,477'4349,9:216675c4:::benchmark_data_smithi055_386467_object57040:head,472'3552,9:2166f436:::benchmark_data_smithi055_386467_object35437:head,465'2207,9:21678aed:::benchmark_data_smithi055_386467_object38263:head,466'2388,9:2168ba74:::benchmark_data_smithi055_386467_object30922:head,464'1918,9:21691d8a:::benchmark_data_smithi055_386467_object87976:head,487'5535,9:2169d1de:::benchmark_data_smithi055_386467_object42205:head,466'2623,9:216f9be3:::benchmark_data_smithi055_386467_object46656:head,467'2893,9:216fa721:::benchmark_data_smithi055_386467_object47571:head,467'2948,9:21702f28:::benchmark_data_smithi055_386467_object48258:head,467'2996,9:21727470:::benchmark_data_smithi055_386467_object91463:head,490'5744,9:217392d0:::benchmark_data_smithi055_386467_object34119:head,465'2130,9:21754010:::benchmark_data_smithi055_386467_object78858:head,480'4970,9:2175c0db:::benchmark_data_smithi055_386467_object52488:head,467'3257,9:2176f0a8:::benchmark_data_smithi055_386467_object51548:head,467'3196,9:21791dae:::benchmark_data_smithi055_386467_object20051:head,464'1260,9:21798bf8:::benchmark_data_smithi055_386467_object46307:head,467'2873,9:2179a8d5:::benchmark_data_smithi055_386467_object42111:head,466'2617,9:217a0dee:::benchmark_data_smithi055_386467_object60108:head,472'3756,9:217db0cb:::benchmark_data_smithi055_386467_object71893:head,478'4539,9:217e001c:::benchmark_data_smithi055_386467_object21947:head,464'1377,9:217ff720:::benchmark_data_smithi055_386467_object79135:head,480'4996,9:2181972b:::benchmark_data_smithi055_386467_object64384:head,473'4058,9:2182a793:::benchmark_data_smithi055_386467_object91552:head,490'5752,9:21841a53:::benchmark_data_smithi055_386467_object92063:head,491'5786,9:21846da9:::benchmark_data_smithi055_386467_object27324:head,464'1684,9:2185c339:::benchmark_data_smithi055_386467_object73721:head,478'4647,9:2186c780:::benchmark_data_smithi055_386467_object47851:head,467'2970,9:2186f69b:::benchmark_data_smithi055_386467_object66502:head,473'4167,9:21898ff6:::benchmark_data_smithi055_386467_object19247:head,463'1219,9:218ab440:::benchmark_data_smithi055_386467_object72027:head,478'4551,9:218bfb17:::benchmark_data_smithi055_386467_object70076:head,478'4416,9:218cd7de:::benchmark_data_smithi055_386467_object37549:head,466'2353)", 2023-09-23T18:17:44.167 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:23.725932+0000", 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 20.426797691000001, 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.053801209000000003, 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.725932+0000", 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.725932+0000", 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.725931+0000", 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.726166+0000", 2023-09-23T18:17:44.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000234675 2023-09-23T18:17:44.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.726395+0000", 2023-09-23T18:17:44.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000228755 2023-09-23T18:17:44.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.726851+0000", 2023-09-23T18:17:44.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00045608699999999999 2023-09-23T18:17:44.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.730898+0000", 2023-09-23T18:17:44.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040472260000000001 2023-09-23T18:17:44.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.731373+0000", 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00047487300000000001 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.779733+0000", 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.048360433000000001 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.173 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.173 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "pg_backfill(progress 9.4 e 548/548 lb 9:218cd7de:::benchmark_data_smithi055_386467_object37549:head)", 2023-09-23T18:17:44.173 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:23.727364+0000", 2023-09-23T18:17:44.173 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 20.425366216, 2023-09-23T18:17:44.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.060121962000000001, 2023-09-23T18:17:44.173 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.173 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:17:44.173 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.173 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.173 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.727364+0000", 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.727364+0000", 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.727363+0000", 2023-09-23T18:17:44.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.175 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.175 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.727372+0000", 2023-09-23T18:17:44.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1540000000000008e-06 2023-09-23T18:17:44.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.175 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.175 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.727380+0000", 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.418e-06 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.727384+0000", 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7210000000000002e-06 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.787448+0000", 2023-09-23T18:17:44.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.060063456000000001 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.787456+0000", 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3979999999999999e-06 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.787485+0000", 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9337000000000001e-05 2023-09-23T18:17:44.177 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:200108 9.4 e548/546 9:2a9af7b9:::benchmark_data_smithi055_386467_object95448:head v 548'12626, mlcod=548'12624)", 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:23.728109+0000", 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 20.424620568000002, 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.056641665000000001, 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.178 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.728109+0000", 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.728109+0000", 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.728109+0000", 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.728114+0000", 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8150000000000002e-06 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.180 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.728123+0000", 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6300000000000004e-06 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.728128+0000", 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0579999999999997e-06 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.779761+0000", 2023-09-23T18:17:44.182 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.051632641999999999 2023-09-23T18:17:44.182 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.182 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.182 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.182 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.779782+0000", 2023-09-23T18:17:44.183 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1429000000000001e-05 2023-09-23T18:17:44.183 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.183 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.183 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.183 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.784710+0000", 2023-09-23T18:17:44.183 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049283850000000004 2023-09-23T18:17:44.183 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.183 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.183 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.184 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.784751+0000", 2023-09-23T18:17:44.184 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0349000000000003e-05 2023-09-23T18:17:44.184 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.184 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.184 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.184 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.184 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.184 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:200110 9.4 e548/546 9:204c3d24:::benchmark_data_smithi055_386467_object95450:head v 548'12627, mlcod=548'12624)", 2023-09-23T18:17:44.184 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:23.730973+0000", 2023-09-23T18:17:44.184 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 20.421756500000001, 2023-09-23T18:17:44.185 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.053798886999999997, 2023-09-23T18:17:44.185 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.185 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.185 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.185 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.730973+0000", 2023-09-23T18:17:44.185 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.186 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.730973+0000", 2023-09-23T18:17:44.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.186 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.186 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.730973+0000", 2023-09-23T18:17:44.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.187 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.730978+0000", 2023-09-23T18:17:44.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9429999999999997e-06 2023-09-23T18:17:44.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.730986+0000", 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8199999999999997e-06 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.730991+0000", 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1490000000000002e-06 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.189 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.189 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.781152+0000", 2023-09-23T18:17:44.189 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.05016077 2023-09-23T18:17:44.189 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.190 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.190 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.190 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.781166+0000", 2023-09-23T18:17:44.190 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4198e-05 2023-09-23T18:17:44.190 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.190 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.190 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.190 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.784755+0000", 2023-09-23T18:17:44.190 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0035882539999999999 2023-09-23T18:17:44.191 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.191 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.191 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.191 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.784772+0000", 2023-09-23T18:17:44.191 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7461000000000001e-05 2023-09-23T18:17:44.192 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.192 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.192 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.192 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.192 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.192 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:200119 9.4 e548/546 9:2e0776a8:::benchmark_data_smithi055_386467_object95459:head v 548'12628, mlcod=548'12624)", 2023-09-23T18:17:44.192 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:23.732969+0000", 2023-09-23T18:17:44.192 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 20.419760393000001, 2023-09-23T18:17:44.192 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.051820008000000001, 2023-09-23T18:17:44.193 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.193 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.193 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.193 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.193 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.194 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.732969+0000", 2023-09-23T18:17:44.194 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.194 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.194 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.195 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.195 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.732969+0000", 2023-09-23T18:17:44.195 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.195 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.195 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.195 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.196 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.732969+0000", 2023-09-23T18:17:44.196 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.196 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.196 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.196 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.196 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.732974+0000", 2023-09-23T18:17:44.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4160000000000003e-06 2023-09-23T18:17:44.197 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.197 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.197 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.197 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.774134+0000", 2023-09-23T18:17:44.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.041159631000000002 2023-09-23T18:17:44.198 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.198 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.198 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.198 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.774143+0000", 2023-09-23T18:17:44.198 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3689999999999996e-06 2023-09-23T18:17:44.198 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.198 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.198 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.199 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.782348+0000", 2023-09-23T18:17:44.199 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082052029999999995 2023-09-23T18:17:44.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.199 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.782366+0000", 2023-09-23T18:17:44.199 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.73e-05 2023-09-23T18:17:44.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.784775+0000", 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024091870000000001 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.784789+0000", 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4501999999999999e-05 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201423 9.e e552/545 9:73127b0d:::benchmark_data_smithi055_386467_object96763:head v 552'12847, mlcod=552'12845)", 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 17.262143749, 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.101742522, 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:17:44.201 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890590+0000", 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0280000000000002e-06 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890596+0000", 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0299999999999999e-06 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.203 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890601+0000", 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0660000000000003e-06 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890652+0000", 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1104999999999999e-05 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890674+0000", 2023-09-23T18:17:44.204 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2274000000000001e-05 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992278+0000", 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.1016035 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992329+0000", 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0816999999999998e-05 2023-09-23T18:17:44.205 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201426 9.e e552/545 9:7aa9c16e:::benchmark_data_smithi055_386467_object96766:head v 552'12848, mlcod=552'12845)", 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 17.262114636, 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.101739732, 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.206 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.207 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890619+0000", 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0570000000000003e-06 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890625+0000", 2023-09-23T18:17:44.208 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1970000000000001e-06 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890630+0000", 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0289999999999996e-06 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892129+0000", 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014991309999999999 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.209 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892151+0000", 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2036e-05 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992332+0000", 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10018059 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.210 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992355+0000", 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2989e-05 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201428 9.3 e552/545 9:ce8adaf4:::benchmark_data_smithi055_386467_object96768:head v 552'12637, mlcod=551'12635)", 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.892727+0000", 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 17.260002774, 2023-09-23T18:17:44.211 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10402589299999999, 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892727+0000", 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892727+0000", 2023-09-23T18:17:44.212 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892726+0000", 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892732+0000", 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6280000000000002e-06 2023-09-23T18:17:44.213 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892740+0000", 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2050000000000002e-06 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892745+0000", 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8590000000000004e-06 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.214 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892789+0000", 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3699000000000003e-05 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892812+0000", 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.368e-05 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.215 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.216 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.996720+0000", 2023-09-23T18:17:44.216 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10390730099999999 2023-09-23T18:17:44.216 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.216 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.216 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.216 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.996753+0000", 2023-09-23T18:17:44.216 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3195000000000003e-05 2023-09-23T18:17:44.216 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.216 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.216 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.217 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.217 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.217 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "pg_scan(get_digest 9.0 MIN-MIN e 552/545)", 2023-09-23T18:17:44.217 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:17:44.217 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 17.259479244000001, 2023-09-23T18:17:44.217 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.126230497, 2023-09-23T18:17:44.217 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.217 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:17:44.217 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.218 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.218 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.218 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:17:44.218 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.218 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.218 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.218 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.218 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:17:44.219 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.219 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.219 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.219 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:17:44.219 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.219 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.219 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.220 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893254+0000", 2023-09-23T18:17:44.220 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3780000000000003e-06 2023-09-23T18:17:44.220 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.220 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.220 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.220 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992277+0000", 2023-09-23T18:17:44.220 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.099022751000000006 2023-09-23T18:17:44.220 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992290+0000", 2023-09-23T18:17:44.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3297e-05 2023-09-23T18:17:44.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992361+0000", 2023-09-23T18:17:44.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0791999999999996e-05 2023-09-23T18:17:44.222 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992377+0000", 2023-09-23T18:17:44.222 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6331e-05 2023-09-23T18:17:44.222 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:27.019481+0000", 2023-09-23T18:17:44.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027103558 2023-09-23T18:17:44.223 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.223 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.223 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.223 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.223 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.223 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201433 9.e e552/545 9:772bf683:::benchmark_data_smithi055_386467_object96773:head v 552'12849, mlcod=552'12845)", 2023-09-23T18:17:44.223 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.893949+0000", 2023-09-23T18:17:44.223 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 17.258781217999999, 2023-09-23T18:17:44.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.130621881, 2023-09-23T18:17:44.224 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.224 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.224 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.224 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.224 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.224 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893949+0000", 2023-09-23T18:17:44.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.225 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893949+0000", 2023-09-23T18:17:44.225 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.225 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893948+0000", 2023-09-23T18:17:44.226 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.226 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.226 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893952+0000", 2023-09-23T18:17:44.226 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3640000000000001e-06 2023-09-23T18:17:44.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.226 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992298+0000", 2023-09-23T18:17:44.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.098345455999999998 2023-09-23T18:17:44.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.227 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992310+0000", 2023-09-23T18:17:44.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1708e-05 2023-09-23T18:17:44.228 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.228 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.228 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.228 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992386+0000", 2023-09-23T18:17:44.228 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6688999999999997e-05 2023-09-23T18:17:44.228 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.228 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.228 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.228 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992407+0000", 2023-09-23T18:17:44.229 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0678000000000001e-05 2023-09-23T18:17:44.229 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.229 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.229 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.229 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:27.024537+0000", 2023-09-23T18:17:44.229 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032130134999999997 2023-09-23T18:17:44.229 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.229 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.230 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.230 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:27.024570+0000", 2023-09-23T18:17:44.230 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3327999999999997e-05 2023-09-23T18:17:44.230 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.230 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.230 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.230 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.231 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.231 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "backfill_remove(9.d e555 9:b93c3518:::benchmark_data_smithi055_386467_object77466:head,480'4754,9:b93d43b3:::benchmark_data_smithi055_386467_object18496:head,463'1169,9:b93d82da:::benchmark_data_smithi055_386467_object46117:head,467'2889,9:b93daaf5:::benchmark_data_smithi055_386467_object84751:head,484'5225,9:b93e0cf0:::benchmark_data_smithi055_386467_object30762:head,464'1950,9:b93ed3e2:::benchmark_data_smithi055_386467_object65044:head,473'4032,9:b93fda1b:::benchmark_data_smithi055_386467_object73930:head,478'4552,9:b9411fea:::benchmark_data_smithi055_386467_object97407:head,493'5988,9:b941e089:::benchmark_data_smithi055_386467_object43117:head,466'2715,9:b94429e5:::benchmark_data_smithi055_386467_object23995:head,464'1513,9:b94474e7:::benchmark_data_smithi055_386467_object43001:head,466'2706,9:b944988d:::benchmark_data_smithi055_386467_object54100:head,470'3383,9:b9460462:::benchmark_data_smithi055_386467_object29221:head,464'1861,9:b946dac7:::benchmark_data_smithi055_386467_object79987:head,480'4929,9:b94744cb:::benchmark_data_smithi055_386467_object57170:head,472'3567,9:b947ddaf:::benchmark_data_smithi055_386467_object59027:head,472'3676,9:b949993a:::benchmark_data_smithi055_386467_object85897:head,486'5287,9:b94a0873:::benchmark_data_smithi055_386467_object50311:head,467'3150,9:b94b80a4:::benchmark_data_smithi055_386467_object21038:head,464'1333,9:b94f98d9:::benchmark_data_smithi055_386467_object38264:head,466'2408,9:b94ffd36:::benchmark_data_smithi055_386467_object62794:head,473'3908,9:b950cd0b:::benchmark_data_smithi055_386467_object96360:head,493'5933,9:b9524d07:::benchmark_data_smithi055_386467_object80669:head,480'4974,9:b952d54a:::benchmark_data_smithi055_386467_object40651:head,466'2549,9:b95407dd:::benchmark_data_smithi055_386467_object68889:head,477'4247,9:b954cf84:::benchmark_data_smithi055_386467_object70702:head,478'4354,9:b954e99f:::benchmark_data_smithi055_386467_object26290:head,464'1671,9:b9554760:::benchmark_data_smithi055_386467_object21371:head,464'1355,9:b95571a7:::benchmark_data_smithi055_386467_object83411:head,484'5136,9:b956a1e8:::benchmark_data_smithi055_386467_object88597:head,487'5463,9:b959b2f8:::benchmark_data_smithi055_386467_object21599:head,464'1375,9:b959cd07:::benchmark_data_smithi055_386467_object29134:head,464'1850,9:b95a3b60:::benchmark_data_smithi055_386467_object91334:head,487'5617,9:b95a8d85:::benchmark_data_smithi055_386467_object46425:head,467'2910,9:b95a99e5:::benchmark_data_smithi055_386467_object32108:head,464'2019,9:b95abb38:::benchmark_data_smithi055_386467_object24096:head,464'1522,9:b95aee11:::benchmark_data_smithi055_386467_object21890:head,464'1387,9:b95b16d1:::benchmark_data_smithi055_386467_object63898:head,473'3974,9:b95c4f76:::benchmark_data_smithi055_386467_object59209:head,472'3696,9:b95caf41:::benchmark_data_smithi055_386467_object51483:head,467'3236,9:b95e3dfa:::benchmark_data_smithi055_386467_object60435:head,472'3758,9:b9607b42:::benchmark_data_smithi055_386467_object94172:head,492'5803,9:b960b73e:::benchmark_data_smithi055_386467_object71101:head,478'4375,9:b961b756:::benchmark_data_smithi055_386467_object35448:head,465'2229,9:b9622b37:::benchmark_data_smithi055_386467_object86412:head,486'5322,9:b9622c34:::benchmark_data_smithi055_386467_object69022:head,477'4255,9:b962f8f4:::benchmark_data_smithi055_386467_object41878:head,466'2621,9:b9631151:::benchmark_data_smithi055_386467_object41598:head,466'2606,9:b963765d:::benchmark_data_smithi055_386467_object80969:head,480'4992,9:b963dfa6:::benchmark_data_smithi055_386467_object32498:head,464'2035,9:b963e52a:::benchmark_data_smithi055_386467_object50523:head,467'3164,9:b9650013:::benchmark_data_smithi055_386467_object91758:head,490'5654,9:b967c534:::benchmark_data_smithi055_386467_object28142:head,464'1792,9:b96876bc:::benchmark_data_smithi055_386467_object88672:head,487'5471,9:b968cd2a:::benchmark_data_smithi055_386467_object85121:head,484'5248,9:b969f292:::benchmark_data_smithi055_386467_object22000:head,464'1400,9:b96a7404:::benchmark_data_smithi055_386467_object80966:head,480'4991,9:b96b2d52:::benchmark_data_smithi055_386467_object87852:head,487'5411,9:b96bc3fd:::benchmark_data_smithi055_386467_object53620:head,467'3353,9:b96bed18:::benchmark_data_smithi055_386467_object97190:head,493'5981,9:b96dbc92:::benchmark_data_smithi055_386467_object73171:head,478'4505,9:b96ea048:::benchmark_data_smithi055_386467_object46727:head,467'2923,9:b9722cdb:::benchmark_data_smithi055_386467_object34344:head,465'2162,9:b9723587:::benchmark_data_smithi055_386467_object72538:head,478'4464,9:b972dcd2:::benchmark_data_smithi055_386467_object49773:head,467'3116,9:b973a3b3:::benchmark_data_smithi055_386467_object72460:head,478'4461,9:b9763800:::benchmark_data_smithi055_386467_object66930:head,473'4141,9:b9769ed5:::benchmark_data_smithi055_386467_object38136:head,466'2399,9:b976e51e:::benchmark_data_smithi055_386467_object42493:head,466'2666,9:b97707a6:::benchmark_data_smithi055_386467_object64205:head,473'3987,9:b9777619:::benchmark_data_smithi055_386467_object34925:head,465'2198,9:b9783ba3:::benchmark_data_smithi055_386467_object57742:head,472'3596,9:b9784f22:::benchmark_data_smithi055_386467_object55459:head,471'3464,9:b97a742f:::benchmark_data_smithi055_386467_object97916:head,493'6028,9:b97bab49:::benchmark_data_smithi055_386467_object24587:head,464'1559,9:b97e6a5e:::benchmark_data_smithi055_386467_object68172:head,476'4208,9:b97ebc5d:::benchmark_data_smithi055_386467_object19979:head,464'1264,9:b97f4b2d:::benchmark_data_smithi055_386467_object22247:head,464'1419,9:b980371d:::benchmark_data_smithi055_386467_object61251:head,472'3804,9:b980bdce:::benchmark_data_smithi055_386467_object91980:head,491'5666,9:b9817240:::benchmark_data_smithi055_386467_object69229:head,477'4268,9:b98284a9:::benchmark_data_smithi055_386467_object38992:head,466'2450,9:b982ab5d:::benchmark_data_smithi055_386467_object57139:head,472'3566,9:b982ee42:::benchmark_data_smithi055_386467_object26186:head,464'1662,9:b9832eab:::benchmark_data_smithi055_386467_object29096:head,464'1849,9:b9851f5f:::benchmark_data_smithi055_386467_object41406:head,466'2593,9:b9868b0e:::benchmark_data_smithi055_386467_object42586:head,466'2677,9:b9879b81:::benchmark_data_smithi055_386467_object84747:head,484'5224,9:b9879ce7:::benchmark_data_smithi055_386467_object63345:head,473'3942,9:b987a0b7:::benchmark_data_smithi055_386467_object90194:head,487'5559,9:b988013a:::benchmark_data_smithi055_386467_object36904:head,466'2327,9:b988b719:::benchmark_data_smithi055_386467_object38776:head,466'2437,9:b98a9451:::benchmark_data_smithi055_386467_object38113:head,466'2397,9:b98ab362:::benchmark_data_smithi055_386467_object59795:head,472'3725,9:b98b8808:::benchmark_data_smithi055_386467_object89084:head,487'5492,9:b98c1187:::benchmark_data_smithi055_386467_object94966:head,492'5839,9:b98d6636:::benchmark_data_smithi055_386467_object43931:head,466'2759,9:b98d70fb:::benchmark_data_smithi055_386467_object20537:head,464'1308,9:b98e8e6a:::benchmark_data_smithi055_386467_object36032:head,465'2267,9:b98f61a0:::benchmark_data_smithi055_386467_object56312:head,472'3510,9:b98fc319:::benchmark_data_smithi055_386467_object87454:head,486'5389,9:b990acc4:::benchmark_data_smithi055_386467_object41705:head,466'2610,9:b9925990:::benchmark_data_smithi055_386467_object41577:head,466'2604,9:b9929d33:::benchmark_data_smithi055_386467_object49861:head,467'3119,9:b9936f6c:::benchmark_data_smithi055_386467_object49489:head,467'3096,9:b9937fb3:::benchmark_data_smithi055_386467_object87258:head,486'5373,9:b993fce3:::benchmark_data_smithi055_386467_object73080:head,478'4499,9:b9945b86:::benchmark_data_smithi055_386467_object46475:head,467'2912,9:b994c072:::benchmark_data_smithi055_386467_object19852:head,464'1256,9:b9950191:::benchmark_data_smithi055_386467_object71798:head,478'4429,9:b9967787:::benchmark_data_smithi055_386467_object87953:head,487'5418,9:b996ae01:::benchmark_data_smithi055_386467_object43543:head,466'2743,9:b996c55b:::benchmark_data_smithi055_386467_object36240:head,465'2276,9:b9995b40:::benchmark_data_smithi055_386467_object82163:head,481'5068,9:b999670c:::benchmark_data_smithi055_386467_object48385:head,467'3022,9:b9998f18:::benchmark_data_smithi055_386467_object74444:head,478'4577,9:b99a5a2d:::benchmark_data_smithi055_386467_object62440:head,473'3880,9:b99a8cd9:::benchmark_data_smithi055_386467_object81859:head,481'5047,9:b99ad0b8:::benchmark_data_smithi055_386467_object81715:head,481'5039,9:b99b5a23:::benchmark_data_smithi055_386467_object53260:head,467'3330,9:b99ca381:::benchmark_data_smithi055_386467_object54989:head,471'3435,9:b99f903e:::benchmark_data_smithi055_386467_object49237:head,467'3077,9:b9a21ccb:::benchmark_data_smithi055_386467_object33460:head,464'2103,9:b9a3fdb4:::benchmark_data_smithi055_386467_object43940:head,466'2761,9:b9a43328:::benchmark_data_smithi055_386467_object93041:head,492'5722,9:b9a4577d:::benchmark_data_smithi055_386467_object88003:head,487'5420,9:b9a5c20d:::benchmark_data_smithi055_386467_object28208:head,464'1800,9:b9a5c903:::benchmark_data_smithi055_386467_object96811:head,493'5960,9:b9a7c063:::benchmark_data_smithi055_386467_object26554:head,464'1691,9:b9a8efe8:::benchmark_data_smithi055_386467_object31119:head,464'1970,9:b9aa6ca0:::benchmark_data_smithi055_386467_object32756:head,464'2053,9:b9aaa4fd:::benchmark_data_smithi055_386467_object77863:head,480'4785,9:b9ab5d58:::benchmark_data_smithi055_386467_object51947:head,467'3263,9:b9ac54b1:::benchmark_data_smithi055_386467_object55516:head,471'3467,9:b9ae025a:::benchmark_data_smithi055_386467_object82059:head,481'5063,9:b9ae5e57:::benchmark_data_smithi055_386467_object62039:head,473'3852,9:b9aeba7a:::benchmark_data_smithi055_386467_object19787:head,464'1250,9:b9af1833:::benchmark_data_smithi055_386467_object88260:head,487'5433,9:b9af195a:::benchmark_data_smithi055_386467_object36358:head,465'2290,9:b9b031fc:::benchmark_data_smithi055_386467_object49578:head,467'3100,9:b9b15806:::benchmark_data_smithi055_386467_object36643:head,465'2307,9:b9b266f7:::benchmark_data_smithi055_386467_object91033:head,487'5602,9:b9b2f80f:::benchmark_data_smithi055_386467_object54562:head,470'3415,9:b9b2fed6:::benchmark_data_smithi055_386467_object45139:head,467'2834,9:b9b3773a:::benchmark_data_smithi055_386467_object63039:head,473'3921,9:b9b3aeea:::benchmark_data_smithi055_386467_object22301:head,464'1422,9:b9b421d6:::benchmark_data_smithi055_386467_object86392:head,486'5320,9:b9b4678b:::benchmark_data_smithi055_386467_object97917:head,493'6029,9:b9b48b3b:::benchmark_data_smithi055_386467_object27055:head,464'1722,9:b9b50180:::benchmark_data_smithi055_386467_object77921:head,480'4789,9:b9b67a9e:::benchmark_data_smithi055_386467_object53919:head,470'3373,9:b9b85304:::benchmark_data_smithi055_386467_object77502:head,480'4757,9:b9b8a6a6:::benchmark_data_smithi055_386467_object94068:head,492'5795,9:b9b8c0a9:::benchmark_data_smithi055_386467_object73476:head,478'4525,9:b9b979a0:::benchmark_data_smithi055_386467_object50037:head,467'3128,9:b9b9f758:::benchmark_data_smithi055_386467_object92206:head,491'5677,9:b9bafcde:::benchmark_data_smithi055_386467_object17949:head,463'1125,9:b9bafef8:::benchmark_data_smithi055_386467_object90453:head,487'5579,9:b9bb2a0d:::benchmark_data_smithi055_386467_object21919:head,464'1391,9:b9bb3ff3:::benchmark_data_smithi055_386467_object86157:head,486'5305,9:b9bc39d7:::benchmark_data_smithi055_386467_object42765:head,466'2692,9:b9bf19d3:::benchmark_data_smithi055_386467_object97748:head,493'6016,9:b9bf448d:::benchmark_data_smithi055_386467_object96704:head,493'5950,9:b9bfa62a:::benchmark_data_smithi055_386467_object94204:head,492'5805,9:b9bfae08:::benchmark_data_smithi055_386467_object67617:head,476'4175,9:b9bfef7a:::benchmark_data_smithi055_386467_object25838:head,464'1640,9:b9c006f3:::benchmark_data_smithi055_386467_object54392:head,470'3402,9:b9c01673:::benchmark_data_smithi055_386467_object65475:head,473'4061,9:b9c1da09:::benchmark_data_smithi055_386467_object83308:head,484'5133,9:b9c294e6:::benchmark_data_smithi055_386467_object27429:head,464'1748,9:b9c343ad:::benchmark_data_smithi055_386467_object32625:head,464'2045,9:b9c44dcd:::benchmark_data_smithi055_386467_object33871:head,465'2127,9:b9c45b8b:::benchmark_data_smithi055_386467_object73706:head,478'4537,9:b9c50449:::benchmark_data_smithi055_386467_object57897:head,472'3604,9:b9c54d80:::benchmark_data_smithi055_386467_object65286:head,473'4050,9:b9c5d103:::benchmark_data_smithi055_386467_object76245:head,480'4682,9:b9c5fd1c:::benchmark_data_smithi055_386467_object44439:head,467'2799,9:b9c7bc6e:::benchmark_data_smithi055_386467_object97307:head,493'5984,9:b9c87188:::benchmark_data_smithi055_386467_object47604:head,467'2972,9:b9c8a012:::benchmark_data_smithi055_386467_object32081:head,464'2015,9:b9c903e1:::benchmark_data_smithi055_386467_object42191:head,466'2641,9:b9c9cd5a:::benchmark_data_smithi055_386467_object92693:head,491'5702,9:b9cae1b9:::benchmark_data_smithi055_386467_object20368:head,464'1293,9:b9cbc91c:::benchmark_data_smithi055_386467_object58064:head,472'3615,9:b9cd579b:::benchmark_data_smithi055_386467_object34038:head,465'2134,9:b9ce258c:::benchmark_data_smithi055_386467_object28908:head,464'1838,9:b9ce2636:::benchmark_data_smithi055_386467_object69175:head,477'4265,9:b9cefaf5:::benchmark_data_smithi055_386467_object87545:head,487'5392,9:b9cf5531:::benchmark_data_smithi055_386467_object48401:head,467'3024,9:b9d0031f:::benchmark_data_smithi055_386467_object29799:head,464'1902,9:b9d01a56:::benchmark_data_smithi055_386467_object72200:head,478'4448,9:b9d0a637:::benchmark_data_smithi055_386467_object57777:head,472'3599,9:b9d177d9:::benchmark_data_smithi055_386467_object85198:head,484'5256,9:b9d1a041:::benchmark_data_smithi055_386467_object84905:head,484'5231,9:b9d20906:::benchmark_data_smithi055_386467_object59371:head,472'3703,9:b9d3d33a:::benchmark_data_smithi055_386467_object46987:head,467'2942,9:b9d41e1b:::benchmark_data_smithi055_386467_object35241:head,465'2211,9:b9d48a47:::benchmark_data_smithi055_386467_object64700:head,473'4010,9:b9d4fec9:::benchmark_data_smithi055_386467_object78165:head,480'4809,9:b9d585ab:::benchmark_data_smithi055_386467_object93049:head,492'5723,9:b9d58961:::benchmark_data_smithi055_386467_object34835:head,465'2189,9:b9d65eef:::benchmark_data_smithi055_386467_object42368:head,466'2653,9:b9d6bb3d:::benchmark_data_smithi055_386467_object53229:head,467'3327,9:b9d7535a:::benchmark_data_smithi055_386467_object65538:head,473'4065,9:b9d77ad0:::benchmark_data_smithi055_386467_object45904:head,467'2877,9:b9d92ede:::benchmark_data_smithi055_386467_object74163:head,478'4565,9:b9dcb4ab:::benchmark_data_smithi055_386467_object35987:head,465'2262,9:b9e05d5b:::benchmark_data_smithi055_386467_object18114:head,463'1139,9:b9e0613f:::benchmark_data_smithi055_386467_object47792:head,467'2986,9:b9e118ef:::benchmark_data_smithi055_386467_object19359:head,463'1220,9:b9e26825:::benchmark_data_smithi055_386467_object35861:head,465'2257,9:b9e2d3a8:::benchmark_data_smithi055_386467_object74524:head,478'4581,9:b9e2eec7:::benchmark_data_smithi055_386467_object60470:head,472'3759,9:b9e2f020:::benchmark_data_smithi055_386467_object27871:head,464'1769,9:b9e50acf:::benchmark_data_smithi055_386467_object26435:head,464'1680,9:b9e516d1:::benchmark_data_smithi055_386467_object64727:head,473'4011,9:b9e58edf:::benchmark_data_smithi055_386467_object55022:head,471'3437,9:b9e5e517:::benchmark_data_smithi055_386467_object95065:head,492'5844,9:b9eb427f:::benchmark_data_smithi055_386467_object76648:head,480'4709,9:b9eb818e:::benchmark_data_smithi055_386467_object52233:head,467'3280,9:b9eda409:::benchmark_data_smithi055_386467_object51577:head,467'3246,9:b9ee33c5:::benchmark_data_smithi055_386467_object77031:head,480'4726,9:b9efce22:::benchmark_data_smithi055_386467_object38012:head,466'2394,9:b9efdce9:::benchmark_data_smithi055_386467_object48476:head,467'3034,9:b9f00ef7:::benchmark_data_smithi055_386467_object34214:head,465'2150,9:b9f16054:::benchmark_data_smithi055_386467_object37102:head,466'2338,9:b9f287e7:::benchmark_data_smithi055_386467_object62014:head,473'3849,9:b9f443db:::benchmark_data_smithi055_386467_object44106:head,466'2769,9:b9f5d277:::benchmark_data_smithi055_386467_object89297:head,487'5508,9:b9f61f70:::benchmark_data_smithi055_386467_object60288:head,472'3749,9:b9f7856a:::benchmark_data_smithi055_386467_object26274:head,464'1670,9:b9f7e96a:::benchmark_data_smithi055_386467_object33277:head,464'2088,9:b9f8941b:::benchmark_data_smithi055_386467_object62070:head,473'3856,9:b9f9fcfe:::benchmark_data_smithi055_386467_object95451:head,492'5871,9:b9fa226b:::benchmark_data_smithi055_386467_object49064:head,467'3070,9:b9fa866a:::benchmark_data_smithi055_386467_object28672:head,464'1825,9:b9fad611:::benchmark_data_smithi055_386467_object97223:head,493'5983,9:b9fc9f83:::benchmark_data_smithi055_386467_object37120:head,466'2339,9:ba002f5c:::benchmark_data_smithi055_386467_object58841:head,472'3661,9:ba008b78:::benchmark_data_smithi055_386467_object97081:head,493'5976,9:ba00c378:::benchmark_data_smithi055_386467_object72153:head,478'4446,9:ba01934b:::benchmark_data_smithi055_386467_object62344:head,473'3873,9:ba02676e:::benchmark_data_smithi055_386467_object73744:head,478'4540,9:ba03373a:::benchmark_data_smithi055_386467_object22657:head,464'1448,9:ba04a03f:::benchmark_data_smithi055_386467_object76865:head,480'4717,9:ba05b4cd:::benchmark_data_smithi055_386467_object82574:head,484'5092,9:ba077228:::benchmark_data_smithi055_386467_object60382:head,472'3755,9:ba08da89:::benchmark_data_smithi055_386467_object39714:head,466'2499,9:ba0d012a:::benchmark_data_smithi055_386467_object97468:head,493'5995,9:ba0efdef:::benchmark_data_smithi055_386467_object52965:head,467'3315,9:ba115801:::benchmark_data_smithi055_386467_object95357:head,492'5866,9:ba12808a:::benchmark_data_smithi055_386467_object56366:head,472'3517,9:ba129466:::benchmark_data_smithi055_386467_object76938:head,480'4720,9:ba129d91:::benchmark_data_smithi055_386467_object84623:head,484'5217,9:ba14d66a:::benchmark_data_smithi055_386467_object39379:head,466'2475,9:ba15346b:::benchmark_data_smithi055_386467_object93238:head,492'5739,9:ba155ccf:::benchmark_data_smithi055_386467_object30066:head,464'1914,9:ba155e14:::benchmark_data_smithi055_386467_object64856:head,473'4023,9:ba15bd21:::benchmark_data_smithi055_386467_object50149:head,467'3138,9:ba16f399:::benchmark_data_smithi055_386467_object48553:head,467'3038,9:ba173f40:::benchmark_data_smithi055_386467_object47153:head,467'2949,9:ba1895e2:::benchmark_data_smithi055_386467_object92145:head,491'5674,9:ba1970d4:::benchmark_data_smithi055_386467_object43982:head,466'2764,9:ba19d1a5:::benchmark_data_smithi055_386467_object81349:head,480'5011,9:ba1ccb08:::benchmark_data_smithi055_386467_object44729:head,467'2818,9:ba1cfee8:::benchmark_data_smithi055_386467_object76469:head,480'4694,9:ba1de5d7:::benchmark_data_smithi055_386467_object30158:head,464'1919,9:ba1f0028:::benchmark_data_smithi055_386467_object58317:head,472'3629,9:ba1f9f0b:::benchmark_data_smithi055_386467_object81198:head,480'5004,9:ba218cdf:::benchmark_data_smithi055_386467_object34972:head,465'2201,9:ba21abd4:::benchmark_data_smithi055_386467_object19500:head,464'1231,9:ba223e8c:::benchmark_data_smithi055_386467_object46469:head,467'2911,9:ba249d95:::benchmark_data_smithi055_386467_object29229:head,464'1863,9:ba26ac5d:::benchmark_data_smithi055_386467_object76169:head,480'4677,9:ba26dc98:::benchmark_data_smithi055_386467_object49978:head,467'3123,9:ba273b56:::benchmark_data_smithi055_386467_object34466:head,465'2168,9:ba278240:::benchmark_data_smithi055_386467_object61470:head,473'3813,9:ba27d3da:::benchmark_data_smithi055_386467_object77353:head,480'4748,9:ba27f656:::benchmark_data_smithi055_386467_object95192:head,492'5854,9:ba28e021:::benchmark_data_smithi055_386467_object17556:head,463'1099,9:ba29fc7a:::benchmark_data_smithi055_386467_object17153:head,463'1081,9:ba2a3d86:::benchmark_data_smithi055_386467_object65571:head,473'4067,9:ba2a509f:::benchmark_data_smithi055_386467_object89159:head,487'5502,9:ba2b9367:::benchmark_data_smithi055_386467_object82226:head,481'5072,9:ba2c553f:::benchmark_data_smithi055_386467_object57055:head,472'3559,9:ba2c8ef9:::benchmark_data_smithi055_386467_object60840:head,472'3781,9:ba2c932d:::benchmark_data_smithi055_386467_object83891:head,484'5174,9:ba2db281:::benchmark_data_smithi055_386467_object48967:head,467'3062,9:ba2e3c30:::benchmark_data_smithi055_386467_object43137:head,466'2716,9:ba2e81b1:::benchmark_data_smithi055_386467_object71234:head,478'4389,9:ba2eac86:::benchmark_data_smithi055_386467_object59571:head,472'3712,9:ba3092a9:::benchmark_data_smithi055_386467_object36463:head,465'2296,9:ba30e71b:::benchmark_data_smithi055_386467_object39668:head,466'2496,9:ba31434d:::benchmark_data_smithi055_386467_object70947:head,478'4369,9:ba3161f9:::benchmark_data_smithi055_386467_object76764:head,480'4714,9:ba31ea96:::benchmark_data_smithi055_386467_object81986:head,481'5058,9:ba3299f7:::benchmark_data_smithi055_386467_object89302:head,487'5509,9:ba3490a4:::benchmark_data_smithi055_386467_object56334:head,472'3513,9:ba34a739:::benchmark_data_smithi055_386467_object59597:head,472'3713,9:ba35fffd:::benchmark_data_smithi055_386467_object81956:head,481'5056,9:ba3613df:::benchmark_data_smithi055_386467_object76270:head,480'4683,9:ba376d0c:::benchmark_data_smithi055_386467_object83953:head,484'5179,9:ba382735:::benchmark_data_smithi055_386467_object46118:head,467'2890,9:ba39058c:::benchmark_data_smithi055_386467_object67764:head,476'4182,9:ba3984ce:::benchmark_data_smithi055_386467_object97524:head,493'5999,9:ba39f319:::benchmark_data_smithi055_386467_object96354:head,493'5931,9:ba3a86c3:::benchmark_data_smithi055_386467_object50542:head,467'3165,9:ba3ad879:::benchmark_data_smithi055_386467_object68639:head,477'4232,9:ba3adcb3:::benchmark_data_smithi055_386467_object77741:head,480'4777,9:ba3ade93:::benchmark_data_smithi055_386467_object60204:head,472'3742,9:ba3b0913:::benchmark_data_smithi055_386467_object22175:head,464'1416,9:ba3b4fa6:::benchmark_data_smithi055_386467_object82988:head,484'5115,9:ba3b6082:::benchmark_data_smithi055_386467_object88272:head,487'5435,9:ba3bd82b:::benchmark_data_smithi055_386467_object56020:head,472'3496,9:ba3ccebb:::benchmark_data_smithi055_386467_object79073:head,480'4870,9:ba3d6092:::benchmark_data_smithi055_386467_object88944:head,487'5481,9:ba3d6c8b:::benchmark_data_smithi055_386467_object17080:head,463'1078,9:ba3efd2d:::benchmark_data_smithi055_386467_object51064:head,467'3209,9:ba3f3e25:::benchmark_data_smithi055_386467_object22495:head,464'1439,9:ba3f7810:::benchmark_data_smithi055_386467_object67373:head,474'4159,9:ba3f8628:::benchmark_data_smithi055_386467_object46615:head,467'2919,9:ba3faf0c:::benchmark_data_smithi055_386467_object33411:head,464'2099,9:ba4003ab:::benchmark_data_smithi055_386467_object66661:head,473'4128,9:ba401eed:::benchmark_data_smithi055_386467_object22248:head,464'1420,9:ba40b257:::benchmark_data_smithi055_386467_object68862:head,477'4246,9:ba411b84:::benchmark_data_smithi055_386467_object47614:head,467'2973,9:ba412f3a:::benchmark_data_smithi055_386467_object78663:head,480'4840,9:ba41da16:::benchmark_data_smithi055_386467_object25051:head,464'1582,9:ba41f96c:::benchmark_data_smithi055_386467_object95520:head,492'5876,9:ba42885c:::benchmark_data_smithi055_386467_object26598:head,464'1695,9:ba436b0b:::benchmark_data_smithi055_386467_object86303:head,486'5316,9:ba448c00:::benchmark_data_smithi055_386467_object86797:head,486'5348,9:ba45cec8:::benchmark_data_smithi055_386467_object47971:head,467'2997,9:ba460433:::benchmark_data_smithi055_386467_object65395:head,473'4056,9:ba464fd8:::benchmark_data_smithi055_386467_object76327:head,480'4685,9:ba4650c4:::benchmark_data_smithi055_386467_object57437:head,472'3582,9:ba46d06d:::benchmark_data_smithi055_386467_object50218:head,467'3144,9:ba472c3b:::benchmark_data_smithi055_386467_object90532:head,487'5585,9:ba48af6e:::benchmark_data_smithi055_386467_object68034:head,476'4201,9:ba49c085:::benchmark_data_smithi055_386467_object90729:head,487'5591,9:ba4caec3:::benchmark_data_smithi055_386467_object32656:head,464'2049,9:ba4d55d5:::benchmark_data_smithi055_386467_object69526:head,477'4282,9:ba4e630e:::benchmark_data_smithi055_386467_object24881:head,464'1576,9:ba51c521:::benchmark_data_smithi055_386467_object84189:head,484'5193,9:ba53173c:::benchmark_data_smithi055_386467_object25898:head,464'1646,9:ba53ce19:::benchmark_data_smithi055_386467_object97937:head,493'6031,9:ba54a687:::benchmark_data_smithi055_386467_object49622:head,467'3103,9:ba55aef7:::benchmark_data_smithi055_386467_object50640:head,467'3172,9:ba560cb6:::benchmark_data_smithi055_386467_object54045:head,470'3379,9:ba575b96:::benchmark_data_smithi055_386467_object69943:head,478'4309,9:ba592e03:::benchmark_data_smithi055_386467_object20974:head,464'1329,9:ba5cfb67:::benchmark_data_smithi055_386467_object38944:head,466'2449,9:ba5daa7e:::benchmark_data_smithi055_386467_object52874:head,467'3312,9:ba5dad08:::benchmark_data_smithi055_386467_object75988:head,480'4666,9:ba5e72df:::benchmark_data_smithi055_386467_object95704:head,492'5885,9:ba60baf6:::benchmark_data_smithi055_386467_object48516:head,467'3035,9:ba60be87:::benchmark_data_smithi055_386467_object95139:head,492'5850,9:ba6167de:::benchmark_data_smithi055_386467_object61852:head,473'3836,9:ba61dd4c:::benchmark_data_smithi055_386467_object93485:head,492'5757,9:ba620ed9:::benchmark_data_smithi055_386467_object42578:head,466'2675,9:ba629104:::benchmark_data_smithi055_386467_object38870:head,466'2444,9:ba640288:::benchmark_data_smithi055_386467_object53815:head,470'3363,9:ba645f08:::benchmark_data_smithi055_386467_object43163:head,466'2718,9:ba64f4f7:::benchmark_data_smithi055_386467_object82689:head,484'5097,9:ba65b9b6:::benchmark_data_smithi055_386467_object62510:head,473'3888,9:ba65bb8a:::benchmark_data_smithi055_386467_object59556:head,472'3710,9:ba68130e:::benchmark_data_smithi055_386467_object54217:head,470'3393,9:ba683c18:::benchmark_data_smithi055_386467_object32926:head,464'2063,9:ba684d9f:::benchmark_data_smithi055_386467_object85679:head,486'5275,9:ba6a71db:::benchmark_data_smithi055_386467_object76186:head,480'4679,9:ba6d4c68:::benchmark_data_smithi055_386467_object70136:head,478'4326,9:ba6f5fbe:::benchmark_data_smithi055_386467_object87298:head,486'5378,9:ba70d26c:::benchmark_data_smithi055_386467_object88483:head,487'5456,9:ba7107cc:::benchmark_data_smithi055_386467_object80841:head,480'4986,9:ba7250b0:::benchmark_data_smithi055_386467_object26491:head,464'1686,9:ba72f908:::benchmark_data_smithi055_386467_object90094:head,487'5554,9:ba73a451:::benchmark_data_smithi055_386467_object89307:head,487'5510,9:ba73b622:::benchmark_data_smithi055_386467_object34842:head,465'2190,9:ba73e398:::benchmark_data_smithi055_386467_object90762:head,487'5595,9:ba752921:::benchmark_data_smithi055_386467_object63170:head,473'3932,9:ba76526d:::benchmark_data_smithi055_386467_object33041:head,464'2071,9:ba7790d8:::benchmark_data_smithi055_386467_object50243:head,467'3146,9:ba7a00c3:::benchmark_data_smithi055_386467_object29885:head,464'1906,9:ba7a2ed7:::benchmark_data_smithi055_386467_object79453:head,480'4896,9:ba7b0aa5:::benchmark_data_smithi055_386467_object90051:head,487'5550,9:ba7b1491:::benchmark_data_smithi055_386467_object97974:head,493'6033,9:ba7bd648:::benchmark_data_smithi055_386467_object80408:head,480'4958,9:ba7bf6c5:::benchmark_data_smithi055_386467_object80894:head,480'4988,9:ba7d8cef:::benchmark_data_smithi055_386467_object40656:head,466'2550,9:ba7dd143:::benchmark_data_smithi055_386467_object45081:head,467'2829,9:ba7ecd86:::benchmark_data_smithi055_386467_object98090:head,493'6039,9:ba7ecf21:::benchmark_data_smithi055_386467_object56075:head,472'3497,9:ba7eefa1:::benchmark_data_smithi055_386467_object90224:head,487'5561,9:ba803284:::benchmark_data_smithi055_386467_object84921:head,484'5236,9:ba820e50:::benchmark_data_smithi055_386467_object50358:head,467'3155,9:ba822ee8:::benchmark_data_smithi055_386467_object90485:head,487'5583,9:ba8266a5:::benchmark_data_smithi055_386467_object46730:head,467'2924,9:ba829ba2:::benchmark_data_smithi055_386467_object84989:head,484'5240,9:ba829cc9:::benchmark_data_smithi055_386467_object23425:head,464'1485,9:ba829f9c:::benchmark_data_smithi055_386467_object27816:head,464'1766,9:ba82d24c:::benchmark_data_smithi055_386467_object94319:head,492'5810,9:ba8340c0:::benchmark_data_smithi055_386467_object33387:head,464'2096,9:ba834d66:::benchmark_data_smithi055_386467_object50959:head,467'3199,9:ba85063f:::benchmark_data_smithi055_386467_object56149:head,472'3500,9:ba8510a6:::benchmark_data_smithi055_386467_object71454:head,478'4407,9:ba856471:::benchmark_data_smithi055_386467_object50892:head,467'3194,9:ba86513b:::benchmark_data_smithi055_386467_object56316:head,472'3511,9:ba86f2ce:::benchmark_data_smithi055_386467_object27937:head,464'1776,9:ba8759fa:::benchmark_data_smithi055_386467_object91703:head,490'5647,9:ba87e2f9:::benchmark_data_smithi055_386467_object58393:head,472'3637,9:ba8800ab:::benchmark_data_smithi055_386467_object48817:head,467'3050,9:ba88a6ca:::benchmark_data_smithi055_386467_object60667:head,472'3770,9:ba8b30e0:::benchmark_data_smithi055_386467_object39561:head,466'2487,9:ba8b4ca6:::benchmark_data_smithi055_386467_object64620:head,473'4006,9:ba8bf9e7:::benchmark_data_smithi055_386467_object95824:head,492'5893,9:ba8d4d11:::benchmark_data_smithi055_386467_object89442:head,487'5520,9:ba8f2160:::benchmark_data_smithi055_386467_object47867:head,467'2993,9:ba91eacc:::benchmark_data_smithi055_386467_object29820:head,464'1904,9:ba923aed:::benchmark_data_smithi055_386467_object41495:head,466'2601,9:ba92e559:::benchmark_data_smithi055_386467_object44475:head,467'2800,9:ba941e22:::benchmark_data_smithi055_386467_object88942:head,487'5480,9:ba956a27:::benchmark_data_smithi055_386467_object28056:head,464'1783,9:ba958ebf:::benchmark_data_smithi055_386467_object28578:head,464'1819,9:ba960512:::benchmark_data_smithi055_386467_object21477:head,464'1366,9:ba96a12a:::benchmark_data_smithi055_386467_object80350:head,480'4951,9:ba971f47:::benchmark_data_smithi055_386467_object18352:head,463'1161,9:ba975e35:::benchmark_data_smithi055_386467_object62674:head,473'3901,9:ba97e879:::benchmark_data_smithi055_386467_object87787:head,487'5405,9:ba99db6b:::benchmark_data_smithi055_386467_object61704:head,473'3827,9:ba9aa21f:::benchmark_data_smithi055_386467_object75834:head,480'4655,9:ba9bb3db:::benchmark_data_smithi055_386467_object97562:head,493'6003,9:ba9c7e6f:::benchmark_data_smithi055_386467_object25089:head,464'1585,9:ba9d12a2:::benchmark_data_smithi055_386467_object82984:head,484'5113,9:ba9f11fc:::benchmark_data_smithi055_386467_object41439:head,466'2596,9:ba9f484e:::benchmark_data_smithi055_386467_object66187:head,473'4103,9:ba9f73ce:::benchmark_data_smithi055_386467_object74872:head,479'4602,9:baa18940:::benchmark_data_smithi055_386467_object86762:head,486'5343,9:baa4d404:::benchmark_data_smithi055_386467_object52479:head,467'3290,9:baa5965c:::benchmark_data_smithi055_386467_object62441:head,473'3881,9:baa6508d:::benchmark_data_smithi055_386467_object96330:head,493'5929,9:baa6b5df:::benchmark_data_smithi055_386467_object59715:head,472'3718,9:baa6beb9:::benchmark_data_smithi055_386467_object17376:head,463'1095,9:baa7ec82:::benchmark_data_smithi055_386467_object68962:head,477'4252,9:baa819f3:::benchmark_data_smithi055_386467_object74169:head,478'4566,9:baa833f3:::benchmark_data_smithi055_386467_object78810:head,480'4855,9:baa98192:::benchmark_data_smithi055_386467_object49745:head,467'3112,9:baaa3610:::benchmark_data_smithi055_386467_object59517:head,472'3708,9:baab1f56:::benchmark_data_smithi055_386467_object62053:head,473'3853,9:baac4566:::benchmark_data_smithi055_386467_object51354:head,467'3227,9:baac4928:::benchmark_data_smithi055_386467_object54638:head,470'3423,9:baad3d39:::benchmark_data_smithi055_386467_object29342:head,464'1870,9:baad552b:::benchmark_data_smithi055_386467_object36351:head,465'2288,9:bab1601a:::benchmark_data_smithi055_386467_object94842:head,492'5836,9:bab23cd4:::benchmark_data_smithi055_386467_object58713:head,472'3652,9:bab39d74:::benchmark_data_smithi055_386467_object38715:head,466'2433,9:bab47d4f:::benchmark_data_smithi055_386467_object89265:head,487'5505,9:bab4f619:::benchmark_data_smithi055_386467_object97998:head,493'6036,9:bab590f7:::benchmark_data_smithi055_386467_object25273:head,464'1605,9:bab715b7:::benchmark_data_smithi055_386467_object63563:head,473'3959,9:bab72813:::benchmark_data_smithi055_386467_object87526:head,487'5391,9:bab7d977:::benchmark_data_smithi055_386467_object97356:head,493'5985,9:bab9818a:::benchmark_data_smithi055_386467_object83435:head,484'5138,9:bab9cec3:::benchmark_data_smithi055_386467_object95888:head,492'5896,9:babaf9ff:::benchmark_data_smithi055_386467_object39698:head,466'2498,9:babb05cd:::benchmark_data_smithi055_386467_object23360:head,464'1484,9:babc4205:::benchmark_data_smithi055_386467_object59719:head,472'3719,9:babd0ee5:::benchmark_data_smithi055_386467_object35506:head,465'2237,9:babef111:::benchmark_data_smithi055_386467_object61111:head,472'3798,9:babf07e0:::benchmark_data_smithi055_386467_object51096:head,467'3212,9:babf0cd0:::benchmark_data_smithi055_386467_object57205:head,472'3570,9:babf5179:::benchmark_data_smithi055_386467_object50322:head,467'3151,9:babfa373:::benchmark_data_smithi055_386467_object19294:head,463'1218,9:babfa6b4:::benchmark_data_smithi055_386467_object44038:head,466'2767,9:bac155e5:::benchmark_data_smithi055_386467_object32874:head,464'2058,9:bac1993f:::benchmark_data_smithi055_386467_object78026:head,480'4799,9:bac1c92a:::benchmark_data_smithi055_386467_object91103:head,487'5605)", 2023-09-23T18:17:44.231 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:30.249740+0000", 2023-09-23T18:17:44.231 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.902989657999999, 2023-09-23T18:17:44.231 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.073730479000000002, 2023-09-23T18:17:44.231 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.231 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.249740+0000", 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.249740+0000", 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.249739+0000", 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.249915+0000", 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017546799999999999 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.250162+0000", 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00024715300000000001 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.250643+0000", 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00048102300000000002 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.254654+0000", 2023-09-23T18:17:44.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040111089999999997 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.255258+0000", 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00060366800000000002 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.323471+0000", 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.068212877000000005 2023-09-23T18:17:44.235 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "pg_backfill(progress 9.d e 555/555 lb 9:bac1c92a:::benchmark_data_smithi055_386467_object91103:head)", 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:30.251380+0000", 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.901349582, 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.072176268000000002, 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.236 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.251380+0000", 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.251380+0000", 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.237 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.251380+0000", 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.251392+0000", 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2126999999999999e-05 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.238 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.251401+0000", 2023-09-23T18:17:44.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9029999999999993e-06 2023-09-23T18:17:44.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.251406+0000", 2023-09-23T18:17:44.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6899999999999997e-06 2023-09-23T18:17:44.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.323515+0000", 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.072108574999999994 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.323525+0000", 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.006e-05 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.323556+0000", 2023-09-23T18:17:44.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1495999999999998e-05 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:202892 9.5 e555/545 9:aaf09590:::benchmark_data_smithi055_386467_object98232:head v 555'12556, mlcod=555'12554)", 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:30.254755+0000", 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.897974584, 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.071579972000000006, 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.241 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.254755+0000", 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.254755+0000", 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.254755+0000", 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.254758+0000", 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7659999999999999e-06 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.254763+0000", 2023-09-23T18:17:44.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9599999999999999e-06 2023-09-23T18:17:44.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.254766+0000", 2023-09-23T18:17:44.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0620000000000001e-06 2023-09-23T18:17:44.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.323503+0000", 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.068737595999999998 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.323523+0000", 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9327e-05 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.326304+0000", 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027816 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.326335+0000", 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0870000000000001e-05 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.246 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.247 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.247 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:202896 9.0 e555/545 9:0f975c26:::benchmark_data_smithi055_386467_object98236:head v 555'12990, mlcod=555'12988)", 2023-09-23T18:17:44.247 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:30.256938+0000", 2023-09-23T18:17:44.247 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.895792085, 2023-09-23T18:17:44.247 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.062071763000000002, 2023-09-23T18:17:44.247 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.247 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.247 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.247 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.247 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.256938+0000", 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.256938+0000", 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.248 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.256937+0000", 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.256944+0000", 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2989999999999999e-06 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.256954+0000", 2023-09-23T18:17:44.249 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8800000000000003e-06 2023-09-23T18:17:44.250 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.250 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.250 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.256961+0000", 2023-09-23T18:17:44.250 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0469999999999996e-06 2023-09-23T18:17:44.250 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.250 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.251 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.257139+0000", 2023-09-23T18:17:44.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017819799999999999 2023-09-23T18:17:44.251 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.251 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.251 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.251 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.257153+0000", 2023-09-23T18:17:44.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4331000000000001e-05 2023-09-23T18:17:44.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.318979+0000", 2023-09-23T18:17:44.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.061825542999999997 2023-09-23T18:17:44.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.252 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.319009+0000", 2023-09-23T18:17:44.253 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0371000000000001e-05 2023-09-23T18:17:44.253 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.253 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.253 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.253 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.253 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.253 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:202897 9.d e555/545 9:b0ac7132:::benchmark_data_smithi055_386467_object98237:head v 555'12494, mlcod=555'12492)", 2023-09-23T18:17:44.254 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:30.257009+0000", 2023-09-23T18:17:44.254 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.895720442, 2023-09-23T18:17:44.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.067885368000000001, 2023-09-23T18:17:44.254 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.254 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.254 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.254 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.254 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.255 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.257009+0000", 2023-09-23T18:17:44.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.255 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.255 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.257009+0000", 2023-09-23T18:17:44.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.256 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.256 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.257009+0000", 2023-09-23T18:17:44.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.256 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.256 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.257014+0000", 2023-09-23T18:17:44.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3319999999999999e-06 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.257023+0000", 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4109999999999996e-06 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.257029+0000", 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.212e-06 2023-09-23T18:17:44.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.323573+0000", 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.066543982000000002 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.323587+0000", 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4299e-05 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.324860+0000", 2023-09-23T18:17:44.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0012732220000000001 2023-09-23T18:17:44.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.259 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.324895+0000", 2023-09-23T18:17:44.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4394999999999998e-05 2023-09-23T18:17:44.260 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.260 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.260 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.260 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:202898 9.f e555/545 9:f804fdd7:::benchmark_data_smithi055_386467_object98238:head v 555'12481, mlcod=555'12479)", 2023-09-23T18:17:44.260 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:30.258781+0000", 2023-09-23T18:17:44.260 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.893948847000001, 2023-09-23T18:17:44.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.061446705999999997, 2023-09-23T18:17:44.261 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.261 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.261 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.258781+0000", 2023-09-23T18:17:44.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.258781+0000", 2023-09-23T18:17:44.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.258780+0000", 2023-09-23T18:17:44.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.258786+0000", 2023-09-23T18:17:44.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4909999999999996e-06 2023-09-23T18:17:44.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.317351+0000", 2023-09-23T18:17:44.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.058565101000000001 2023-09-23T18:17:44.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.317362+0000", 2023-09-23T18:17:44.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0657e-05 2023-09-23T18:17:44.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.317460+0000", 2023-09-23T18:17:44.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7961000000000006e-05 2023-09-23T18:17:44.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.317481+0000", 2023-09-23T18:17:44.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1586999999999998e-05 2023-09-23T18:17:44.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.320196+0000", 2023-09-23T18:17:44.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002714943 2023-09-23T18:17:44.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.320228+0000", 2023-09-23T18:17:44.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1554999999999999e-05 2023-09-23T18:17:44.267 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.267 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.267 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.268 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "backfill_remove(9.b e555 9:d0005871:::benchmark_data_smithi055_386467_object87343:head,486'5521,9:d000b776:::benchmark_data_smithi055_386467_object82992:head,484'5259,9:d001e402:::benchmark_data_smithi055_386467_object72212:head,478'4508,9:d001fba5:::benchmark_data_smithi055_386467_object84068:head,484'5328,9:d0027e0b:::benchmark_data_smithi055_386467_object36037:head,465'2336,9:d003491c:::benchmark_data_smithi055_386467_object48573:head,467'3089,9:d0044b49:::benchmark_data_smithi055_386467_object57579:head,472'3618,9:d0055c2e:::benchmark_data_smithi055_386467_object17534:head,463'1143,9:d0067e75:::benchmark_data_smithi055_386467_object33263:head,464'2148,9:d007d921:::benchmark_data_smithi055_386467_object40201:head,466'2606,9:d0081df3:::benchmark_data_smithi055_386467_object43927:head,466'2806,9:d0094ece:::benchmark_data_smithi055_386467_object77925:head,480'4909,9:d009a812:::benchmark_data_smithi055_386467_object17353:head,463'1127,9:d009ff16:::benchmark_data_smithi055_386467_object37911:head,466'2454,9:d00a44df:::benchmark_data_smithi055_386467_object69810:head,478'4353,9:d00a5f31:::benchmark_data_smithi055_386467_object94033:head,492'5964,9:d00a871d:::benchmark_data_smithi055_386467_object55175:head,471'3483,9:d00ab637:::benchmark_data_smithi055_386467_object78221:head,480'4922,9:d00b3744:::benchmark_data_smithi055_386467_object38815:head,466'2507,9:d00b3e9d:::benchmark_data_smithi055_386467_object57688:head,472'3624,9:d00b5ff7:::benchmark_data_smithi055_386467_object72516:head,478'4531,9:d00b9b9b:::benchmark_data_smithi055_386467_object38837:head,466'2508,9:d00c850f:::benchmark_data_smithi055_386467_object60950:head,472'3793,9:d01002a2:::benchmark_data_smithi055_386467_object77229:head,480'4863,9:d0103d8f:::benchmark_data_smithi055_386467_object59889:head,472'3727,9:d010e9e1:::benchmark_data_smithi055_386467_object89877:head,487'5685,9:d01141d5:::benchmark_data_smithi055_386467_object60061:head,472'3733,9:d0114d95:::benchmark_data_smithi055_386467_object34649:head,465'2242,9:d0116d98:::benchmark_data_smithi055_386467_object42517:head,466'2734,9:d012e33f:::benchmark_data_smithi055_386467_object89045:head,487'5627,9:d0132567:::benchmark_data_smithi055_386467_object65073:head,473'4055,9:d0133ca2:::benchmark_data_smithi055_386467_object67331:head,473'4206,9:d013a8bd:::benchmark_data_smithi055_386467_object97428:head,493'6204,9:d01445ad:::benchmark_data_smithi055_386467_object78536:head,480'4950,9:d0148107:::benchmark_data_smithi055_386467_object59137:head,472'3693,9:d016cef1:::benchmark_data_smithi055_386467_object18967:head,463'1238,9:d0176df5:::benchmark_data_smithi055_386467_object34277:head,465'2216,9:d0177078:::benchmark_data_smithi055_386467_object60757:head,472'3778,9:d01a25b7:::benchmark_data_smithi055_386467_object50929:head,467'3249,9:d01a31af:::benchmark_data_smithi055_386467_object22541:head,464'1467,9:d01efaa6:::benchmark_data_smithi055_386467_object46690:head,467'2966,9:d01fd220:::benchmark_data_smithi055_386467_object50520:head,467'3221,9:d0202799:::benchmark_data_smithi055_386467_object38252:head,466'2469,9:d020b8ba:::benchmark_data_smithi055_386467_object44085:head,466'2818,9:d020f412:::benchmark_data_smithi055_386467_object88918:head,487'5614,9:d021472f:::benchmark_data_smithi055_386467_object79347:head,480'5012,9:d021b235:::benchmark_data_smithi055_386467_object41293:head,466'2661,9:d0226b43:::benchmark_data_smithi055_386467_object78764:head,480'4960,9:d02329f9:::benchmark_data_smithi055_386467_object83028:head,484'5265,9:d024c347:::benchmark_data_smithi055_386467_object63482:head,473'3954,9:d02577df:::benchmark_data_smithi055_386467_object22829:head,464'1488,9:d027148f:::benchmark_data_smithi055_386467_object83094:head,484'5272,9:d0271ad0:::benchmark_data_smithi055_386467_object37394:head,466'2411,9:d027750a:::benchmark_data_smithi055_386467_object95973:head,492'6113,9:d027cc70:::benchmark_data_smithi055_386467_object24663:head,464'1586,9:d0283688:::benchmark_data_smithi055_386467_object46139:head,467'2937,9:d02844d4:::benchmark_data_smithi055_386467_object73629:head,478'4609,9:d02974e5:::benchmark_data_smithi055_386467_object32822:head,464'2124,9:d029e740:::benchmark_data_smithi055_386467_object64879:head,473'4039,9:d02a0b1b:::benchmark_data_smithi055_386467_object53478:head,467'3394,9:d02b0463:::benchmark_data_smithi055_386467_object46494:head,467'2950,9:d02baf06:::benchmark_data_smithi055_386467_object21437:head,464'1399,9:d02c583f:::benchmark_data_smithi055_386467_object55966:head,472'3519,9:d02c87f1:::benchmark_data_smithi055_386467_object77549:head,480'4882,9:d02ca1d1:::benchmark_data_smithi055_386467_object29639:head,464'1911,9:d02e555f:::benchmark_data_smithi055_386467_object91702:head,490'5806,9:d02eaf3a:::benchmark_data_smithi055_386467_object78863:head,480'4967,9:d030afa1:::benchmark_data_smithi055_386467_object25503:head,464'1643,9:d03100dd:::benchmark_data_smithi055_386467_object26645:head,464'1716,9:d031bdeb:::benchmark_data_smithi055_386467_object89360:head,487'5651,9:d0320f44:::benchmark_data_smithi055_386467_object37677:head,466'2435,9:d032a7ab:::benchmark_data_smithi055_386467_object39999:head,466'2591,9:d033c39a:::benchmark_data_smithi055_386467_object19553:head,464'1275,9:d03481ec:::benchmark_data_smithi055_386467_object66757:head,473'4169,9:d0355645:::benchmark_data_smithi055_386467_object43528:head,466'2783,9:d035bb66:::benchmark_data_smithi055_386467_object35819:head,465'2322,9:d03680ea:::benchmark_data_smithi055_386467_object87460:head,486'5528,9:d03722c9:::benchmark_data_smithi055_386467_object97742:head,493'6228,9:d037a7ee:::benchmark_data_smithi055_386467_object46919:head,467'2984,9:d0392f71:::benchmark_data_smithi055_386467_object49075:head,467'3122,9:d03b144b:::benchmark_data_smithi055_386467_object44477:head,467'2847,9:d03c1011:::benchmark_data_smithi055_386467_object43387:head,466'2774,9:d03ec9e2:::benchmark_data_smithi055_386467_object59058:head,472'3687,9:d03f93af:::benchmark_data_smithi055_386467_object39812:head,466'2583,9:d03fc508:::benchmark_data_smithi055_386467_object31095:head,464'2007,9:d0412b31:::benchmark_data_smithi055_386467_object70061:head,478'4374,9:d044bbf6:::benchmark_data_smithi055_386467_object48372:head,467'3078,9:d044c2b1:::benchmark_data_smithi055_386467_object97490:head,493'6210,9:d0489782:::benchmark_data_smithi055_386467_object80125:head,480'5056,9:d048ac49:::benchmark_data_smithi055_386467_object23257:head,464'1511,9:d048e8d3:::benchmark_data_smithi055_386467_object60045:head,472'3732,9:d0495ed1:::benchmark_data_smithi055_386467_object98131:head,493'6246,9:d0499037:::benchmark_data_smithi055_386467_object52422:head,467'3333,9:d049cd78:::benchmark_data_smithi055_386467_object43427:head,466'2776,9:d04a9eb5:::benchmark_data_smithi055_386467_object18979:head,463'1241,9:d04ac1ae:::benchmark_data_smithi055_386467_object95099:head,492'6049,9:d04b495f:::benchmark_data_smithi055_386467_object85043:head,484'5377,9:d04bbceb:::benchmark_data_smithi055_386467_object30433:head,464'1965,9:d04ccd59:::benchmark_data_smithi055_386467_object42014:head,466'2704,9:d04d3860:::benchmark_data_smithi055_386467_object90515:head,487'5729,9:d04da16f:::benchmark_data_smithi055_386467_object70672:head,478'4412,9:d04e6e23:::benchmark_data_smithi055_386467_object96263:head,492'6135,9:d04f25c0:::benchmark_data_smithi055_386467_object28294:head,464'1823,9:d05120c4:::benchmark_data_smithi055_386467_object34340:head,465'2220,9:d05357d9:::benchmark_data_smithi055_386467_object34999:head,465'2270,9:d0558e8b:::benchmark_data_smithi055_386467_object54766:head,470'3451,9:d055af12:::benchmark_data_smithi055_386467_object52627:head,467'3344,9:d055dffc:::benchmark_data_smithi055_386467_object34837:head,465'2254,9:d057baa9:::benchmark_data_smithi055_386467_object41158:head,466'2654,9:d05887fd:::benchmark_data_smithi055_386467_object18845:head,463'1228,9:d058b6eb:::benchmark_data_smithi055_386467_object52078:head,467'3317,9:d058bcf1:::benchmark_data_smithi055_386467_object29723:head,464'1921,9:d058ec4d:::benchmark_data_smithi055_386467_object71345:head,478'4452,9:d059558d:::benchmark_data_smithi055_386467_object33685:head,464'2171,9:d059c28f:::benchmark_data_smithi055_386467_object37651:head,466'2432,9:d05a763f:::benchmark_data_smithi055_386467_object79593:head,480'5031,9:d05acb02:::benchmark_data_smithi055_386467_object23545:head,464'1528,9:d05c7aec:::benchmark_data_smithi055_386467_object56470:head,472'3546,9:d05d4ce5:::benchmark_data_smithi055_386467_object18533:head,463'1203,9:d060a3e1:::benchmark_data_smithi055_386467_object31586:head,464'2043,9:d0619bcc:::benchmark_data_smithi055_386467_object49128:head,467'3127,9:d062bb84:::benchmark_data_smithi055_386467_object20988:head,464'1367,9:d062bf1b:::benchmark_data_smithi055_386467_object17967:head,463'1167,9:d063b602:::benchmark_data_smithi055_386467_object47140:head,467'3000,9:d0640e49:::benchmark_data_smithi055_386467_object65296:head,473'4073,9:d0649efb:::benchmark_data_smithi055_386467_object39852:head,466'2585,9:d064a2ef:::benchmark_data_smithi055_386467_object94853:head,492'6034,9:d066d8d2:::benchmark_data_smithi055_386467_object19409:head,464'1268,9:d067ae7c:::benchmark_data_smithi055_386467_object68549:head,476'4272,9:d068574b:::benchmark_data_smithi055_386467_object72035:head,478'4495,9:d069ba23:::benchmark_data_smithi055_386467_object28441:head,464'1835,9:d06a4de2:::benchmark_data_smithi055_386467_object37418:head,466'2413,9:d06a64ec:::benchmark_data_smithi055_386467_object45910:head,467'2922,9:d06bf450:::benchmark_data_smithi055_386467_object56893:head,472'3573,9:d06c10e2:::benchmark_data_smithi055_386467_object56645:head,472'3552,9:d06cd88a:::benchmark_data_smithi055_386467_object82450:head,482'5224,9:d06d3ce6:::benchmark_data_smithi055_386467_object80579:head,480'5096,9:d06d6c79:::benchmark_data_smithi055_386467_object26757:head,464'1723,9:d06e7786:::benchmark_data_smithi055_386467_object34736:head,465'2247,9:d06ea60a:::benchmark_data_smithi055_386467_object85968:head,486'5444,9:d06eb33c:::benchmark_data_smithi055_386467_object32184:head,464'2086,9:d06ee858:::benchmark_data_smithi055_386467_object35490:head,465'2302,9:d06f58ac:::benchmark_data_smithi055_386467_object45309:head,467'2887,9:d0700375:::benchmark_data_smithi055_386467_object32995:head,464'2134,9:d0700e56:::benchmark_data_smithi055_386467_object62310:head,473'3883,9:d070762d:::benchmark_data_smithi055_386467_object73033:head,478'4560,9:d0711dbd:::benchmark_data_smithi055_386467_object35288:head,465'2288,9:d0731c82:::benchmark_data_smithi055_386467_object57557:head,472'3617,9:d0738dd6:::benchmark_data_smithi055_386467_object17759:head,463'1156,9:d073c027:::benchmark_data_smithi055_386467_object98151:head,493'6247,9:d0744c5f:::benchmark_data_smithi055_386467_object34059:head,465'2198,9:d074753f:::benchmark_data_smithi055_386467_object58611:head,472'3667,9:d074a3ea:::benchmark_data_smithi055_386467_object84083:head,484'5329,9:d074f7c5:::benchmark_data_smithi055_386467_object18766:head,463'1221,9:d075b7db:::benchmark_data_smithi055_386467_object56416:head,472'3539,9:d0760f51:::benchmark_data_smithi055_386467_object38845:head,466'2509,9:d0762f7f:::benchmark_data_smithi055_386467_object78245:head,480'4930,9:d0763474:::benchmark_data_smithi055_386467_object59920:head,472'3728,9:d076e5b3:::benchmark_data_smithi055_386467_object69798:head,478'4352,9:d07763dd:::benchmark_data_smithi055_386467_object43138:head,466'2764,9:d077ad7b:::benchmark_data_smithi055_386467_object86407:head,486'5470,9:d079089e:::benchmark_data_smithi055_386467_object23397:head,464'1519,9:d07a21c7:::benchmark_data_smithi055_386467_object82968:head,484'5256,9:d07a2907:::benchmark_data_smithi055_386467_object62898:head,473'3916,9:d07a9920:::benchmark_data_smithi055_386467_object72355:head,478'4518,9:d07acca4:::benchmark_data_smithi055_386467_object21070:head,464'1372,9:d07cbffd:::benchmark_data_smithi055_386467_object43505:head,466'2780,9:d080870e:::benchmark_data_smithi055_386467_object55616:head,471'3505,9:d080e5ef:::benchmark_data_smithi055_386467_object96547:head,493'6156,9:d081734a:::benchmark_data_smithi055_386467_object81552:head,481'5156,9:d08200c9:::benchmark_data_smithi055_386467_object79246:head,480'4998,9:d082ba49:::benchmark_data_smithi055_386467_object68724:head,477'4281,9:d0835c1d:::benchmark_data_smithi055_386467_object87610:head,487'5533,9:d0864a01:::benchmark_data_smithi055_386467_object97812:head,493'6229,9:d087178b:::benchmark_data_smithi055_386467_object17426:head,463'1134,9:d0874d46:::benchmark_data_smithi055_386467_object22571:head,464'1469,9:d087e23d:::benchmark_data_smithi055_386467_object88546:head,487'5592,9:d088db1d:::benchmark_data_smithi055_386467_object26707:head,464'1718,9:d0891c83:::benchmark_data_smithi055_386467_object44994:head,467'2871,9:d089288b:::benchmark_data_smithi055_386467_object76089:head,480'4793,9:d0898c55:::benchmark_data_smithi055_386467_object88586:head,487'5595,9:d089eac6:::benchmark_data_smithi055_386467_object80849:head,480'5112,9:d08a3237:::benchmark_data_smithi055_386467_object37438:head,466'2416,9:d08b4f8e:::benchmark_data_smithi055_386467_object35699:head,465'2317,9:d08bc5c0:::benchmark_data_smithi055_386467_object39098:head,466'2528,9:d08c029a:::benchmark_data_smithi055_386467_object17434:head,463'1135,9:d08cc137:::benchmark_data_smithi055_386467_object93289:head,492'5914,9:d08e1585:::benchmark_data_smithi055_386467_object63722:head,473'3967,9:d08eac93:::benchmark_data_smithi055_386467_object47668:head,467'3032,9:d08ee80c:::benchmark_data_smithi055_386467_object34070:head,465'2199,9:d0906cbd:::benchmark_data_smithi055_386467_object85435:head,484'5403,9:d0908355:::benchmark_data_smithi055_386467_object34485:head,465'2236,9:d091288f:::benchmark_data_smithi055_386467_object28219:head,464'1819,9:d0936eb7:::benchmark_data_smithi055_386467_object76704:head,480'4825,9:d094a540:::benchmark_data_smithi055_386467_object86149:head,486'5456,9:d094fe1f:::benchmark_data_smithi055_386467_object70007:head,478'4367,9:d095c8d1:::benchmark_data_smithi055_386467_object85333:head,484'5397,9:d096ac4e:::benchmark_data_smithi055_386467_object82699:head,484'5238,9:d096d12f:::benchmark_data_smithi055_386467_object34713:head,465'2246,9:d096d883:::benchmark_data_smithi055_386467_object94881:head,492'6036,9:d0972e11:::benchmark_data_smithi055_386467_object49876:head,467'3178,9:d0991534:::benchmark_data_smithi055_386467_object57516:head,472'3615,9:d09a568f:::benchmark_data_smithi055_386467_object51074:head,467'3258,9:d09b0e8d:::benchmark_data_smithi055_386467_object37322:head,466'2406,9:d09c0f96:::benchmark_data_smithi055_386467_object27231:head,464'1751,9:d09e128a:::benchmark_data_smithi055_386467_object77217:head,480'4862,9:d09f0eaa:::benchmark_data_smithi055_386467_object60609:head,472'3770,9:d09f5eeb:::benchmark_data_smithi055_386467_object33001:head,464'2135,9:d0a14ab5:::benchmark_data_smithi055_386467_object20085:head,464'1309,9:d0a37a1a:::benchmark_data_smithi055_386467_object79269:head,480'5001,9:d0a39de8:::benchmark_data_smithi055_386467_object25029:head,464'1618,9:d0a48296:::benchmark_data_smithi055_386467_object91314:head,487'5781,9:d0a4bf87:::benchmark_data_smithi055_386467_object28369:head,464'1829,9:d0a51a0b:::benchmark_data_smithi055_386467_object73908:head,478'4625,9:d0a73bfe:::benchmark_data_smithi055_386467_object49776:head,467'3169,9:d0a7db56:::benchmark_data_smithi055_386467_object80353:head,480'5071,9:d0a807aa:::benchmark_data_smithi055_386467_object37385:head,466'2410,9:d0a93042:::benchmark_data_smithi055_386467_object29728:head,464'1922,9:d0a9c16d:::benchmark_data_smithi055_386467_object79095:head,480'4985,9:d0aa8e55:::benchmark_data_smithi055_386467_object19336:head,463'1264,9:d0ac5e94:::benchmark_data_smithi055_386467_object24647:head,464'1584,9:d0acfbdf:::benchmark_data_smithi055_386467_object19568:head,464'1277,9:d0ae0e54:::benchmark_data_smithi055_386467_object53169:head,467'3375,9:d0ae4cc4:::benchmark_data_smithi055_386467_object29687:head,464'1916,9:d0ae8676:::benchmark_data_smithi055_386467_object25363:head,464'1637,9:d0aea6d8:::benchmark_data_smithi055_386467_object94187:head,492'5976,9:d0aea75b:::benchmark_data_smithi055_386467_object76941:head,480'4842,9:d0aeddfd:::benchmark_data_smithi055_386467_object92942:head,492'5891,9:d0afe1bf:::benchmark_data_smithi055_386467_object47740:head,467'3038,9:d0b054cc:::benchmark_data_smithi055_386467_object72436:head,478'4526,9:d0b0e206:::benchmark_data_smithi055_386467_object84490:head,484'5354,9:d0b119da:::benchmark_data_smithi055_386467_object67050:head,473'4193,9:d0b1b631:::benchmark_data_smithi055_386467_object86111:head,486'5453,9:d0b341cc:::benchmark_data_smithi055_386467_object56920:head,472'3577,9:d0b374f0:::benchmark_data_smithi055_386467_object38657:head,466'2495,9:d0b41c0b:::benchmark_data_smithi055_386467_object82357:head,481'5215,9:d0b4c15c:::benchmark_data_smithi055_386467_object92321:head,491'5851,9:d0b5dd64:::benchmark_data_smithi055_386467_object44040:head,466'2811,9:d0b60277:::benchmark_data_smithi055_386467_object85803:head,486'5427,9:d0b6a645:::benchmark_data_smithi055_386467_object82376:head,481'5220,9:d0b722e9:::benchmark_data_smithi055_386467_object35692:head,465'2315,9:d0b7c7bb:::benchmark_data_smithi055_386467_object64525:head,473'4017,9:d0b94307:::benchmark_data_smithi055_386467_object86186:head,486'5459,9:d0b98120:::benchmark_data_smithi055_386467_object76726:head,480'4829,9:d0ba9777:::benchmark_data_smithi055_386467_object69227:head,477'4313,9:d0bb0c5f:::benchmark_data_smithi055_386467_object30396:head,464'1962,9:d0bbe7c2:::benchmark_data_smithi055_386467_object75421:head,480'4756,9:d0bd42c4:::benchmark_data_smithi055_386467_object37475:head,466'2419,9:d0bdcbe3:::benchmark_data_smithi055_386467_object20232:head,464'1320,9:d0bdd1f6:::benchmark_data_smithi055_386467_object41725:head,466'2688,9:d0be0d9b:::benchmark_data_smithi055_386467_object79680:head,480'5036,9:d0c0c3bc:::benchmark_data_smithi055_386467_object91047:head,487'5769,9:d0c167de:::benchmark_data_smithi055_386467_object27891:head,464'1797,9:d0c2b885:::benchmark_data_smithi055_386467_object66110:head,473'4132,9:d0c3f0be:::benchmark_data_smithi055_386467_object79333:head,480'5008,9:d0c586b1:::benchmark_data_smithi055_386467_object62696:head,473'3906,9:d0c9186c:::benchmark_data_smithi055_386467_object45613:head,467'2907,9:d0c987bb:::benchmark_data_smithi055_386467_object94409:head,492'5994,9:d0c9d304:::benchmark_data_smithi055_386467_object80998:head,480'5122,9:d0ca4be2:::benchmark_data_smithi055_386467_object40500:head,466'2621,9:d0cb3e30:::benchmark_data_smithi055_386467_object41434:head,466'2672,9:d0cc8ebd:::benchmark_data_smithi055_386467_object74297:head,478'4669,9:d0cd3629:::benchmark_data_smithi055_386467_object94828:head,492'6029,9:d0cd36d4:::benchmark_data_smithi055_386467_object90383:head,487'5711,9:d0ce6edc:::benchmark_data_smithi055_386467_object93508:head,492'5932,9:d0cf4bba:::benchmark_data_smithi055_386467_object38471:head,466'2481,9:d0cfdfc2:::benchmark_data_smithi055_386467_object40602:head,466'2625,9:d0d02157:::benchmark_data_smithi055_386467_object32140:head,464'2081,9:d0d0366f:::benchmark_data_smithi055_386467_object98082:head,493'6244,9:d0d1b813:::benchmark_data_smithi055_386467_object22775:head,464'1482,9:d0d238c7:::benchmark_data_smithi055_386467_object31450:head,464'2032,9:d0d449bc:::benchmark_data_smithi055_386467_object94627:head,492'6016,9:d0d4769d:::benchmark_data_smithi055_386467_object70210:head,478'4384,9:d0d49e15:::benchmark_data_smithi055_386467_object95506:head,492'6073,9:d0d509aa:::benchmark_data_smithi055_386467_object43621:head,466'2788,9:d0d6df34:::benchmark_data_smithi055_386467_object69845:head,478'4354,9:d0d794d3:::benchmark_data_smithi055_386467_object17918:head,463'1165,9:d0d82be9:::benchmark_data_smithi055_386467_object78432:head,480'4944,9:d0d891b9:::benchmark_data_smithi055_386467_object41926:head,466'2696,9:d0d90186:::benchmark_data_smithi055_386467_object27989:head,464'1806,9:d0d91e58:::benchmark_data_smithi055_386467_object20646:head,464'1346,9:d0dbf1fa:::benchmark_data_smithi055_386467_object97477:head,493'6209,9:d0dc2ea6:::benchmark_data_smithi055_386467_object23469:head,464'1523,9:d0dc3ae4:::benchmark_data_smithi055_386467_object72784:head,478'4545,9:d0dc47cd:::benchmark_data_smithi055_386467_object33831:head,465'2180,9:d0dde2b9:::benchmark_data_smithi055_386467_object44200:head,466'2825,9:d0df5868:::benchmark_data_smithi055_386467_object80118:head,480'5055,9:d0e0c6cb:::benchmark_data_smithi055_386467_object92914:head,492'5888,9:d0e0d3dd:::benchmark_data_smithi055_386467_object47735:head,467'3035,9:d0e11b18:::benchmark_data_smithi055_386467_object35739:head,465'2318,9:d0e15cf3:::benchmark_data_smithi055_386467_object59372:head,472'3707,9:d0e24d26:::benchmark_data_smithi055_386467_object19408:head,464'1267,9:d0e4aaea:::benchmark_data_smithi055_386467_object90267:head,487'5705,9:d0e5e06a:::benchmark_data_smithi055_386467_object18062:head,463'1175,9:d0e807c4:::benchmark_data_smithi055_386467_object23466:head,464'1522,9:d0e82d38:::benchmark_data_smithi055_386467_object33965:head,465'2191,9:d0e84d40:::benchmark_data_smithi055_386467_object77888:head,480'4907,9:d0e85d32:::benchmark_data_smithi055_386467_object94568:head,492'6008,9:d0e9a816:::benchmark_data_smithi055_386467_object38260:head,466'2470,9:d0ea6436:::benchmark_data_smithi055_386467_object50676:head,467'3231,9:d0eb5373:::benchmark_data_smithi055_386467_object78061:head,480'4914,9:d0eb7a3b:::benchmark_data_smithi055_386467_object62885:head,473'3915,9:d0ebeb6f:::benchmark_data_smithi055_386467_object42480:head,466'2731,9:d0ec5d37:::benchmark_data_smithi055_386467_object82806:head,484'5245,9:d0ecd06b:::benchmark_data_smithi055_386467_object72680:head,478'4540,9:d0ed67f9:::benchmark_data_smithi055_386467_object21290:head,464'1389,9:d0ef7ef5:::benchmark_data_smithi055_386467_object45967:head,467'2925,9:d0f03804:::benchmark_data_smithi055_386467_object94445:head,492'5996,9:d0f0e3a2:::benchmark_data_smithi055_386467_object90809:head,487'5753,9:d0f1f622:::benchmark_data_smithi055_386467_object92759:head,491'5879,9:d0f24a40:::benchmark_data_smithi055_386467_object48355:head,467'3077,9:d0f2947d:::benchmark_data_smithi055_386467_object76696:head,480'4824,9:d0f40292:::benchmark_data_smithi055_386467_object27597:head,464'1777,9:d0f4c3f4:::benchmark_data_smithi055_386467_object96246:head,492'6132,9:d0f76ae7:::benchmark_data_smithi055_386467_object17101:head,463'1110,9:d0f78f9d:::benchmark_data_smithi055_386467_object57661:head,472'3623,9:d0f7fe53:::benchmark_data_smithi055_386467_object80855:head,480'5113,9:d0f87044:::benchmark_data_smithi055_386467_object80459:head,480'5079,9:d0f89159:::benchmark_data_smithi055_386467_object20013:head,464'1303,9:d0f939b2:::benchmark_data_smithi055_386467_object79543:head,480'5029,9:d0f9af10:::benchmark_data_smithi055_386467_object30913:head,464'1997,9:d0f9ffb2:::benchmark_data_smithi055_386467_object64654:head,473'4025,9:d0fa4907:::benchmark_data_smithi055_386467_object85081:head,484'5380,9:d0fc129b:::benchmark_data_smithi055_386467_object95966:head,492'6112,9:d0fc4bdb:::benchmark_data_smithi055_386467_object79334:head,480'5009,9:d0fc5893:::benchmark_data_smithi055_386467_object20390:head,464'1329,9:d0fc5c47:::benchmark_data_smithi055_386467_object31535:head,464'2040,9:d0fcb48f:::benchmark_data_smithi055_386467_object88779:head,487'5605,9:d0fe9755:::benchmark_data_smithi055_386467_object32353:head,464'2098,9:d0ff4e8f:::benchmark_data_smithi055_386467_object53818:head,470'3411,9:d101e7cb:::benchmark_data_smithi055_386467_object88954:head,487'5618,9:d1027c92:::benchmark_data_smithi055_386467_object79804:head,480'5042,9:d103d247:::benchmark_data_smithi055_386467_object68412:head,476'4263,9:d1045447:::benchmark_data_smithi055_386467_object67012:head,473'4191,9:d1046eeb:::benchmark_data_smithi055_386467_object87781:head,487'5543,9:d105288e:::benchmark_data_smithi055_386467_object69672:head,478'4346,9:d106d39b:::benchmark_data_smithi055_386467_object73655:head,478'4610,9:d107b6ab:::benchmark_data_smithi055_386467_object75577:head,480'4766,9:d108fe64:::benchmark_data_smithi055_386467_object94269:head,492'5986,9:d1090647:::benchmark_data_smithi055_386467_object93558:head,492'5936,9:d1094e0c:::benchmark_data_smithi055_386467_object79106:head,480'4987,9:d10a15c1:::benchmark_data_smithi055_386467_object92294:head,491'5846,9:d10a4562:::benchmark_data_smithi055_386467_object69374:head,477'4325,9:d10bc8a2:::benchmark_data_smithi055_386467_object17170:head,463'1115,9:d10cd95a:::benchmark_data_smithi055_386467_object65136:head,473'4061,9:d110145f:::benchmark_data_smithi055_386467_object67244:head,473'4203,9:d110c392:::benchmark_data_smithi055_386467_object56450:head,472'3544,9:d110ea55:::benchmark_data_smithi055_386467_object53294:head,467'3378,9:d112a8fe:::benchmark_data_smithi055_386467_object26330:head,464'1692,9:d113dd9a:::benchmark_data_smithi055_386467_object57729:head,472'3625,9:d114ef60:::benchmark_data_smithi055_386467_object42942:head,466'2753,9:d1157772:::benchmark_data_smithi055_386467_object45218:head,467'2882,9:d11589e7:::benchmark_data_smithi055_386467_object36340:head,465'2353,9:d115dc2b:::benchmark_data_smithi055_386467_object55144:head,471'3480,9:d115ee76:::benchmark_data_smithi055_386467_object70984:head,478'4429,9:d116b9f6:::benchmark_data_smithi055_386467_object27970:head,464'1805,9:d1176e72:::benchmark_data_smithi055_386467_object38793:head,466'2505,9:d117a1f6:::benchmark_data_smithi055_386467_object56491:head,472'3547,9:d11955f3:::benchmark_data_smithi055_386467_object88124:head,487'5566,9:d11a7195:::benchmark_data_smithi055_386467_object64091:head,473'3986,9:d11abe02:::benchmark_data_smithi055_386467_object79654:head,480'5032,9:d11b0e4d:::benchmark_data_smithi055_386467_object73711:head,478'4613,9:d11d4627:::benchmark_data_smithi055_386467_object94470:head,492'6000,9:d11d6f7d:::benchmark_data_smithi055_386467_object26085:head,464'1677,9:d11db168:::benchmark_data_smithi055_386467_object66509:head,473'4155,9:d11db609:::benchmark_data_smithi055_386467_object85393:head,484'5399,9:d11dee8b:::benchmark_data_smithi055_386467_object66058:head,473'4127,9:d11e02fa:::benchmark_data_smithi055_386467_object78763:head,480'4959,9:d11ecb04:::benchmark_data_smithi055_386467_object75216:head,480'4741,9:d12042cb:::benchmark_data_smithi055_386467_object21066:head,464'1371,9:d1207122:::benchmark_data_smithi055_386467_object91806:head,490'5813,9:d121617a:::benchmark_data_smithi055_386467_object82999:head,484'5261,9:d1222d8c:::benchmark_data_smithi055_386467_object65004:head,473'4049,9:d123315f:::benchmark_data_smithi055_386467_object78175:head,480'4920,9:d1246229:::benchmark_data_smithi055_386467_object93501:head,492'5931,9:d124f51e:::benchmark_data_smithi055_386467_object32540:head,464'2107,9:d125a642:::benchmark_data_smithi055_386467_object39482:head,466'2558,9:d128182c:::benchmark_data_smithi055_386467_object58133:head,472'3644,9:d1296fa0:::benchmark_data_smithi055_386467_object79198:head,480'4995,9:d129b756:::benchmark_data_smithi055_386467_object44608:head,467'2854,9:d12ab70b:::benchmark_data_smithi055_386467_object18443:head,463'1198,9:d12ae178:::benchmark_data_smithi055_386467_object86071:head,486'5448,9:d12be0ac:::benchmark_data_smithi055_386467_object69135:head,477'4306,9:d12c143c:::benchmark_data_smithi055_386467_object20171:head,464'1313,9:d12d099b:::benchmark_data_smithi055_386467_object54999:head,471'3472,9:d12e483f:::benchmark_data_smithi055_386467_object68019:head,476'4243,9:d12ff956:::benchmark_data_smithi055_386467_object17159:head,463'1114,9:d130c789:::benchmark_data_smithi055_386467_object31496:head,464'2036,9:d131d23d:::benchmark_data_smithi055_386467_object81759:head,481'5169,9:d13224aa:::benchmark_data_smithi055_386467_object25523:head,464'1644,9:d1328078:::benchmark_data_smithi055_386467_object92190:head,491'5838,9:d132da9b:::benchmark_data_smithi055_386467_object54906:head,471'3465,9:d133b4f9:::benchmark_data_smithi055_386467_object19883:head,464'1296,9:d133dcf1:::benchmark_data_smithi055_386467_object18902:head,463'1232,9:d1340e22:::benchmark_data_smithi055_386467_object92466:head,491'5860,9:d135a9f1:::benchmark_data_smithi055_386467_object82713:head,484'5241,9:d13668ac:::benchmark_data_smithi055_386467_object85330:head,484'5396,9:d1392acd:::benchmark_data_smithi055_386467_object57096:head,472'3592,9:d13bca76:::benchmark_data_smithi055_386467_object93136:head,492'5907,9:d13bcf8e:::benchmark_data_smithi055_386467_object97324:head,493'6198,9:d13c6235:::benchmark_data_smithi055_386467_object84753:head,484'5367,9:d13c867d:::benchmark_data_smithi055_386467_object92772:head,491'5881,9:d13c98ce:::benchmark_data_smithi055_386467_object51387:head,467'3276,9:d142e84b:::benchmark_data_smithi055_386467_object73870:head,478'4621,9:d1441732:::benchmark_data_smithi055_386467_object69945:head,478'4362,9:d145b4c9:::benchmark_data_smithi055_386467_object83585:head,484'5301,9:d145d6f8:::benchmark_data_smithi055_386467_object80162:head,480'5058,9:d1463f43:::benchmark_data_smithi055_386467_object74161:head,478'4647,9:d1481b2f:::benchmark_data_smithi055_386467_object26042:head,464'1673,9:d148b3da:::benchmark_data_smithi055_386467_object50233:head,467'3200,9:d148b9c8:::benchmark_data_smithi055_386467_object19347:head,463'1265,9:d1495d96:::benchmark_data_smithi055_386467_object54730:head,470'3448,9:d14a57bf:::benchmark_data_smithi055_386467_object21792:head,464'1420,9:d14ab795:::benchmark_data_smithi055_386467_object86327:head,486'5468,9:d14af008:::benchmark_data_smithi055_386467_object28105:head,464'1813,9:d14bc84f:::benchmark_data_smithi055_386467_object80884:head,480'5115,9:d14be4c9:::benchmark_data_smithi055_386467_object92160:head,491'5836,9:d14be865:::benchmark_data_smithi055_386467_object60063:head,472'3734,9:d14ca2a4:::benchmark_data_smithi055_386467_object83091:head,484'5271,9:d14f9903:::benchmark_data_smithi055_386467_object47633:head,467'3029,9:d1519634:::benchmark_data_smithi055_386467_object67119:head,473'4196,9:d15230cb:::benchmark_data_smithi055_386467_object21642:head,464'1413,9:d152e605:::benchmark_data_smithi055_386467_object91830:head,490'5814,9:d1534a72:::benchmark_data_smithi055_386467_object74683:head,478'4699,9:d153f933:::benchmark_data_smithi055_386467_object32694:head,464'2113,9:d15493d8:::benchmark_data_smithi055_386467_object36465:head,465'2359,9:d157259d:::benchmark_data_smithi055_386467_object47198:head,467'3004,9:d1572d1a:::benchmark_data_smithi055_386467_object17421:head,463'1133,9:d1596a03:::benchmark_data_smithi055_386467_object38341:head,466'2471,9:d15a806b:::benchmark_data_smithi055_386467_object66197:head,473'4138,9:d15b3cc4:::benchmark_data_smithi055_386467_object70399:head,478'4396,9:d15c6dc9:::benchmark_data_smithi055_386467_object18977:head,463'1240,9:d15cad6d:::benchmark_data_smithi055_386467_object49624:head,467'3163,9:d15cbaf0:::benchmark_data_smithi055_386467_object66150:head,473'4135,9:d15d2b6a:::benchmark_data_smithi055_386467_object89123:head,487'5629,9:d15d6aec:::benchmark_data_smithi055_386467_object43804:head,466'2797,9:d15d95da:::benchmark_data_smithi055_386467_object50704:head,467'3234,9:d15dc344:::benchmark_data_smithi055_386467_object78428:head,480'4943,9:d1624326:::benchmark_data_smithi055_386467_object88432:head,487'5585,9:d1645f34:::benchmark_data_smithi055_386467_object35559:head,465'2306,9:d166332f:::benchmark_data_smithi055_386467_object74978:head,479'4722,9:d16645ca:::benchmark_data_smithi055_386467_object59062:head,472'3688,9:d16752df:::benchmark_data_smithi055_386467_object67280:head,473'4204,9:d169fe15:::benchmark_data_smithi055_386467_object20912:head,464'1363,9:d16a0fdd:::benchmark_data_smithi055_386467_object67755:head,476'4228,9:d16b751f:::benchmark_data_smithi055_386467_object32648:head,464'2109,9:d16c51a7:::benchmark_data_smithi055_386467_object89978:head,487'5691,9:d16d12ca:::benchmark_data_smithi055_386467_object75054:head,480'4726,9:d16e403e:::benchmark_data_smithi055_386467_object51926:head,467'3304,9:d16ee28f:::benchmark_data_smithi055_386467_object56857:head,472'3566,9:d16f0d8e:::benchmark_data_smithi055_386467_object48927:head,467'3113,9:d16f6d5c:::benchmark_data_smithi055_386467_object83396:head,484'5292,9:d16fae6f:::benchmark_data_smithi055_386467_object50408:head,467'3211,9:d170622b:::benchmark_data_smithi055_386467_object41101:head,466'2650,9:d1707065:::benchmark_data_smithi055_386467_object72025:head,478'4494,9:d1707c21:::benchmark_data_smithi055_386467_object69201:head,477'4312,9:d171844b:::benchmark_data_smithi055_386467_object73470:head,478'4592,9:d1741dd4:::benchmark_data_smithi055_386467_object41678:head,466'2685,9:d1765e72:::benchmark_data_smithi055_386467_object70098:head,478'4376,9:d176c5fb:::benchmark_data_smithi055_386467_object85840:head,486'5432,9:d177fcf7:::benchmark_data_smithi055_386467_object27938:head,464'1801,9:d17ad0e0:::benchmark_data_smithi055_386467_object80333:head,480'5069,9:d17b3961:::benchmark_data_smithi055_386467_object72429:head,478'4523,9:d17b6cb9:::benchmark_data_smithi055_386467_object59007:head,472'3682,9:d17d408b:::benchmark_data_smithi055_386467_object70754:head,478'4419,9:d17dbc2e:::benchmark_data_smithi055_386467_object77860:head,480'4904,9:d17f3567:::benchmark_data_smithi055_386467_object70337:head,478'4393,9:d17f56be:::benchmark_data_smithi055_386467_object21396:head,464'1394,9:d17fcba1:::benchmark_data_smithi055_386467_object61507:head,473'3832,9:d1801391:::benchmark_data_smithi055_386467_object77527:head,480'4880,9:d18054b8:::benchmark_data_smithi055_386467_object74391:head,478'4675,9:d18063d5:::benchmark_data_smithi055_386467_object55118:head,471'3479,9:d180ce9c:::benchmark_data_smithi055_386467_object89840:head,487'5683,9:d18235b1:::benchmark_data_smithi055_386467_object84086:head,484'5330,9:d18302bf:::benchmark_data_smithi055_386467_object69985:head,478'4366,9:d1833b2c:::benchmark_data_smithi055_386467_object25249:head,464'1627,9:d183a1ae:::benchmark_data_smithi055_386467_object31858:head,464'2062)", 2023-09-23T18:17:44.268 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:30.794924+0000", 2023-09-23T18:17:44.268 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.357805403, 2023-09-23T18:17:44.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.053821905000000003, 2023-09-23T18:17:44.268 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.268 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:17:44.268 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.794924+0000", 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.794924+0000", 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.794924+0000", 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.795109+0000", 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018528300000000001 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.795373+0000", 2023-09-23T18:17:44.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026422300000000001 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.795845+0000", 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00047165800000000002 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.797770+0000", 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019250490000000001 2023-09-23T18:17:44.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.798248+0000", 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00047781499999999998 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.848746+0000", 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.050498559999999998 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.272 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "pg_backfill(progress 9.b e 555/555 lb 9:d183a1ae:::benchmark_data_smithi055_386467_object31858:head)", 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:30.798423+0000", 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.354306321999999, 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.050557146999999997, 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.798423+0000", 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.798423+0000", 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.798423+0000", 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.798435+0000", 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2165e-05 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.798445+0000", 2023-09-23T18:17:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7450000000000007e-06 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.798452+0000", 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5379999999999997e-06 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.848938+0000", 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.050485765000000002 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.276 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.848950+0000", 2023-09-23T18:17:44.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2455e-05 2023-09-23T18:17:44.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.848981+0000", 2023-09-23T18:17:44.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0148e-05 2023-09-23T18:17:44.277 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.277 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:203005 9.b e555/545 9:d9ec473b:::benchmark_data_smithi055_386467_object98345:head v 555'12927, mlcod=555'12925)", 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:30.799390+0000", 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.353339649, 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.051677753999999999, 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:44.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.799390+0000", 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.799390+0000", 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.799390+0000", 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.799394+0000", 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.515e-06 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:44.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.799404+0000", 2023-09-23T18:17:44.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7000000000000003e-06 2023-09-23T18:17:44.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:44.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.799410+0000", 2023-09-23T18:17:44.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3960000000000003e-06 2023-09-23T18:17:44.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:44.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.849334+0000", 2023-09-23T18:17:44.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.049923285999999997 2023-09-23T18:17:44.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:44.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.849363+0000", 2023-09-23T18:17:44.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9189000000000001e-05 2023-09-23T18:17:44.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:44.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.851034+0000", 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016711079999999999 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:30.851068+0000", 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3954000000000001e-05 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:44.283 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:44.284 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:17:44.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:44.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:44.285 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:44.286 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:17:44.305 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:17:44.324 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:17:44.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:44 smithi099 ceph-mon[167076]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:44 smithi099 ceph-mon[167076]: osdmap e568: 8 total, 8 up, 6 in 2023-09-23T18:17:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:44 smithi099 ceph-mon[167076]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:44 smithi099 ceph-mon[167076]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:44 smithi099 ceph-mon[167076]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3485902774' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:17:44.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:44 smithi099 ceph-mon[167076]: pgmap v1340: 119 pgs: 1 active+remapped+backfilling, 3 peering, 115 active+clean; 121 MiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 33/5691 objects misplaced (0.580%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:17:44.343 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:17:44.363 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:17:44.382 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:17:44.386 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:44.401 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:17:44.404 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[184174]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.404 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[184174]: osdmap e568: 8 total, 8 up, 6 in 2023-09-23T18:17:44.405 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[184174]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.405 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[184174]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.405 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[184174]: failed to encode map e568 with expected crc 2023-09-23T18:17:44.405 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3485902774' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:17:44.405 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:44 smithi055 ceph-mon[184174]: pgmap v1340: 119 pgs: 1 active+remapped+backfilling, 3 peering, 115 active+clean; 121 MiB data, 3.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 552 op/s; 33/5691 objects misplaced (0.580%); 2.4 MiB/s, 38 objects/s recovering 2023-09-23T18:17:44.418 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:44.422 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:17:44.424 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:17:44.424 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:17:44.424 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:17:44.424 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:17:44.424 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:17:44.425 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:17:44.425 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:17:44.425 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:17:45.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:45.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.290 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3485902774' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:17:45.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: osdmap e569: 8 total, 8 up, 6 in 2023-09-23T18:17:45.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3397601225' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:45.291 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:45.291 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3485902774' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: osdmap e569: 8 total, 8 up, 6 in 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3397601225' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:45.292 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:45 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:45.307 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:17:45.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3485902774' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: osdmap e569: 8 total, 8 up, 6 in 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: failed to encode map e569 with expected crc 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3397601225' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:45.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:45 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:17:45.565 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:45.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:45.568 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:45.669 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:45.696 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:45.729 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:45.731 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:45.732 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:45.833 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:45.860 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:45.994 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:45.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:45.998 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:46.099 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:46.124 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:46 smithi055 ceph-mon[179668]: failed to encode map e569 with expected crc 2023-09-23T18:17:46.125 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:46 smithi055 ceph-mon[179668]: pgmap v1342: 119 pgs: 1 active+remapped+backfilling, 3 peering, 115 active+clean; 121 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 538 op/s; 33/5691 objects misplaced (0.580%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:17:46.125 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:46 smithi055 ceph-mon[184174]: failed to encode map e569 with expected crc 2023-09-23T18:17:46.125 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:46 smithi055 ceph-mon[184174]: pgmap v1342: 119 pgs: 1 active+remapped+backfilling, 3 peering, 115 active+clean; 121 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 538 op/s; 33/5691 objects misplaced (0.580%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:17:46.126 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:46.164 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:46.165 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:46.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:46.168 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:46.269 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:46.293 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:46 smithi099 ceph-mon[167076]: failed to encode map e569 with expected crc 2023-09-23T18:17:46.293 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:46 smithi099 ceph-mon[167076]: pgmap v1342: 119 pgs: 1 active+remapped+backfilling, 3 peering, 115 active+clean; 121 MiB data, 3.2 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 538 op/s; 33/5691 objects misplaced (0.580%); 2.4 MiB/s, 37 objects/s recovering 2023-09-23T18:17:46.294 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_4 2023-09-23T18:17:46.294 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_4 unique_pool_4 --yes-i-really-really-mean-it 2023-09-23T18:17:46.296 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:46.332 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:46.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:46.336 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:46.437 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:46.467 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:46.510 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:46.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:46.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:46.517 INFO:tasks.thrashosds.thrasher:in_osds: [1, 0, 4, 3, 6, 5] out_osds: [7, 2] dead_osds: [] live_osds: [0, 4, 6, 7, 3, 5, 2, 1] 2023-09-23T18:17:46.517 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:17:46.517 INFO:tasks.thrashosds.thrasher:primary_affinity 2023-09-23T18:17:46.517 INFO:tasks.thrashosds.thrasher:Setting osd 4 primary_affinity to 0.000000 2023-09-23T18:17:46.517 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 0 2023-09-23T18:17:46.615 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:46.650 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:46.690 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:46.694 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:46.694 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:46.795 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:46.828 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:46.915 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:46.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:46.918 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:47.019 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:47.050 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:47.089 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:47.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:47.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:47.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:47.220 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:47.358 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:47.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:47.361 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:47.435 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:47.462 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:47.487 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:47.496 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:47 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:17:47.496 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:47 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:17:47.499 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:47.567 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:47.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:47.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:47.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:47 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:17:47.588 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:17:47.672 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:47.700 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:47.735 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:47.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:47.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:47.839 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:47.871 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:47.957 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:47.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:47.961 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:48.031 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:17:48.062 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:48.105 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:48.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:48 smithi055 ceph-mon[179668]: pgmap v1343: 119 pgs: 119 active+clean; 4.4 MiB data, 2.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 475 op/s 2023-09-23T18:17:48.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:48 smithi055 ceph-mon[184174]: pgmap v1343: 119 pgs: 119 active+clean; 4.4 MiB data, 2.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 475 op/s 2023-09-23T18:17:48.576 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:48.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:48 smithi099 ceph-mon[167076]: pgmap v1343: 119 pgs: 119 active+clean; 4.4 MiB data, 2.9 GiB used, 533 GiB / 536 GiB avail; 0 B/s wr, 475 op/s 2023-09-23T18:17:48.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:48.580 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:48.681 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:48.708 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:48.742 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:48.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:48.745 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:48.811 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:17:48.812 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:17:48.846 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:48.873 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:48.959 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:48.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:48.963 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:49.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:49.090 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:49.129 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:49.132 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:49.132 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:49.233 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:49.266 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:49.504 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:17:49.508 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:49.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:49.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:49.613 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:49.644 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:49.780 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:49.817 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:49.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:49.822 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:49.923 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:49.955 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:50.007 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:50.142 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:50.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:50.146 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:50.246 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:50.273 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:50.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:50.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:50.311 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:50.312 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_4' removed 2023-09-23T18:17:50.412 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:50.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:17:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:17:50] "GET /metrics HTTP/1.1" 200 36610 "" "Prometheus/2.43.0" 2023-09-23T18:17:50.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:50 smithi055 ceph-mon[179668]: pgmap v1344: 119 pgs: 119 active+clean; 4.4 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 263 op/s 2023-09-23T18:17:50.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:50 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2861290035' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:17:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:50 smithi055 ceph-mon[184174]: pgmap v1344: 119 pgs: 119 active+clean; 4.4 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 263 op/s 2023-09-23T18:17:50.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:50 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2861290035' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:17:50.438 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:50.473 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:50.477 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:50.477 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:50 smithi099 ceph-mon[167076]: pgmap v1344: 119 pgs: 119 active+clean; 4.4 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail; 0 B/s wr, 263 op/s 2023-09-23T18:17:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:50 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2861290035' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:17:50.578 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:50.603 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:50.636 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:50.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:50.640 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:50.740 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:50.766 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:50.801 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:50.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:50.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:50.905 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:50.948 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:51.286 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:51.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:51.294 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:51.382 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.4 primary-affinity to 0 (802) 2023-09-23T18:17:51.395 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:51.428 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:51.520 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:51.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:51.523 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:51.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:51 smithi099 ceph-mon[167076]: failed to encode map e570 with expected crc 2023-09-23T18:17:51.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2861290035' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:17:51.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:51 smithi099 ceph-mon[167076]: osdmap e570: 8 total, 8 up, 6 in 2023-09-23T18:17:51.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:51.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:51.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:51 smithi099 ceph-mon[167076]: failed to encode map e570 with expected crc 2023-09-23T18:17:51.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3961844910' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T18:17:51.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:51 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T18:17:51.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:17:51.656 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[179668]: failed to encode map e570 with expected crc 2023-09-23T18:17:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2861290035' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:17:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[179668]: osdmap e570: 8 total, 8 up, 6 in 2023-09-23T18:17:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[179668]: failed to encode map e570 with expected crc 2023-09-23T18:17:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3961844910' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T18:17:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T18:17:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[184174]: failed to encode map e570 with expected crc 2023-09-23T18:17:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2861290035' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_4", "pool2": "unique_pool_4", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:17:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[184174]: osdmap e570: 8 total, 8 up, 6 in 2023-09-23T18:17:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[184174]: failed to encode map e570 with expected crc 2023-09-23T18:17:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3961844910' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T18:17:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:51 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]: dispatch 2023-09-23T18:17:51.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:17:51.696 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:51.697 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:17:51.798 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:51.827 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:52.066 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:52.067 INFO:teuthology.task.full_sequential:In full_sequential, running task radosbench... 2023-09-23T18:17:52.068 INFO:tasks.radosbench:Beginning radosbench... 2023-09-23T18:17:52.068 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_5 2023-09-23T18:17:52.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool create unique_pool_5 16 2023-09-23T18:17:52.069 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:52.070 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:52.170 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:52.199 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:52.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:52.237 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:52.238 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:52.338 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:52.376 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:52.414 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:52.415 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:52.417 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[179668]: failed to encode map e570 with expected crc 2023-09-23T18:17:52.417 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[179668]: pgmap v1346: 103 pgs: 103 active+clean; 4.4 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:52.417 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[179668]: failed to encode map e571 with expected crc 2023-09-23T18:17:52.417 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T18:17:52.417 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[179668]: osdmap e571: 8 total, 8 up, 6 in 2023-09-23T18:17:52.417 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:52.418 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[184174]: failed to encode map e570 with expected crc 2023-09-23T18:17:52.418 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[184174]: pgmap v1346: 103 pgs: 103 active+clean; 4.4 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:52.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[184174]: failed to encode map e571 with expected crc 2023-09-23T18:17:52.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T18:17:52.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[184174]: osdmap e571: 8 total, 8 up, 6 in 2023-09-23T18:17:52.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:52.419 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:52.420 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:52.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:52.545 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:52 smithi099 ceph-mon[167076]: failed to encode map e570 with expected crc 2023-09-23T18:17:52.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:52 smithi099 ceph-mon[167076]: pgmap v1346: 103 pgs: 103 active+clean; 4.4 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:52.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:52 smithi099 ceph-mon[167076]: failed to encode map e571 with expected crc 2023-09-23T18:17:52.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:52 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 0.0}]': finished 2023-09-23T18:17:52.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:52 smithi099 ceph-mon[167076]: osdmap e571: 8 total, 8 up, 6 in 2023-09-23T18:17:52.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:17:52.548 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:52.581 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:52.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:52.584 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:52.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:52.712 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:52.746 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:52.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:52.749 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:52.850 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:52.875 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:52.908 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:52.911 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:52.912 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:53.012 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:53.038 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:53.071 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:53.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:53.074 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:53.175 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:53.211 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:53.322 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:17:53.356 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:53.359 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:17:53.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:53.362 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:53.383 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:17:53.404 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:17:53.428 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:17:53.447 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:17:53.463 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:53.467 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:17:53.487 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:17:53.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:17:53.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:17:53.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:17:53.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:17:53.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:17:53.488 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:17:53.489 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:17:53.489 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:17:53.494 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:53.528 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:53.531 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:53.535 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:53.535 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:53.636 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:53.663 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: osdmap e572: 8 total, 8 up, 6 in 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2506303037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:53.664 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:53.665 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1953029821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:17:53.666 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: osdmap e572: 8 total, 8 up, 6 in 2023-09-23T18:17:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2506303037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:53.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1953029821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:17:53.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: osdmap e572: 8 total, 8 up, 6 in 2023-09-23T18:17:53.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: failed to encode map e572 with expected crc 2023-09-23T18:17:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2506303037' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:53.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1953029821' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:17:53.701 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:53.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:53.706 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:53.807 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:53.835 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:53.971 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:53.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:53.975 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:54.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:54.107 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:54.143 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:54.146 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:54.147 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:54.247 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:54.278 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:54.337 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:17:54.525 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:54.528 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:54.529 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:54.556 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:17:54.556 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:17:54.556 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:17:54.556 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:17:54.556 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:17:54.582 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[184174]: pgmap v1349: 103 pgs: 103 active+clean; 4.4 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:54.582 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:54.582 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:54.582 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:54.582 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:54.583 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1953029821' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:17:54.583 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[184174]: osdmap e573: 8 total, 8 up, 6 in 2023-09-23T18:17:54.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[179668]: pgmap v1349: 103 pgs: 103 active+clean; 4.4 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:54.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:54.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:54.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:54.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:54.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1953029821' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:17:54.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:54 smithi055 ceph-mon[179668]: osdmap e573: 8 total, 8 up, 6 in 2023-09-23T18:17:54.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:54.659 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:54 smithi099 ceph-mon[167076]: pgmap v1349: 103 pgs: 103 active+clean; 4.4 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:54.659 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:54 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:17:54.659 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:54 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:17:54.659 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:54 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:54.660 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:54 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:17:54.660 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1953029821' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:17:54.660 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:54 smithi099 ceph-mon[167076]: osdmap e573: 8 total, 8 up, 6 in 2023-09-23T18:17:54.662 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:54.699 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:54.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:54.702 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:54.803 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:54.833 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:54.869 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:54.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:54.873 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:54.973 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:55.003 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:55.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:55.040 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:55.040 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:55.141 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:55.169 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:55.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:55.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:55.207 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:55.290 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:17:55.307 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:55.364 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:55.408 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:55.410 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:55 smithi055 ceph-mon[184174]: failed to encode map e573 with expected crc 2023-09-23T18:17:55.410 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:55 smithi055 ceph-mon[184174]: 1.e scrub starts 2023-09-23T18:17:55.410 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:55 smithi055 ceph-mon[184174]: 1.e scrub ok 2023-09-23T18:17:55.411 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:55 smithi055 ceph-mon[179668]: failed to encode map e573 with expected crc 2023-09-23T18:17:55.411 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:55 smithi055 ceph-mon[179668]: 1.e scrub starts 2023-09-23T18:17:55.411 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:55 smithi055 ceph-mon[179668]: 1.e scrub ok 2023-09-23T18:17:55.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:55.412 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:55.512 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:55.543 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:55.579 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:55.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:55.582 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:55.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:17:55.713 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:55 smithi099 ceph-mon[167076]: failed to encode map e573 with expected crc 2023-09-23T18:17:55.713 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:55 smithi099 ceph-mon[167076]: 1.e scrub starts 2023-09-23T18:17:55.713 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:55 smithi099 ceph-mon[167076]: 1.e scrub ok 2023-09-23T18:17:55.716 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:55.755 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:17:55.758 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:55.758 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:17:55.859 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:55.890 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:55.923 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:55.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:55.926 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:55.974 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:56.027 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:56.055 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:56.083 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:17:56.090 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:56.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:56.094 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:56.195 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:56.226 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:56.269 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:56.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:56.273 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:56.374 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:56.410 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_5' created 2023-09-23T18:17:56.411 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:56.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:56 smithi055 ceph-mon[179668]: 1.1d scrub starts 2023-09-23T18:17:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:56 smithi055 ceph-mon[179668]: 1.1d scrub ok 2023-09-23T18:17:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:56 smithi055 ceph-mon[179668]: pgmap v1351: 103 pgs: 103 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:56.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2448523567' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T18:17:56.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:56 smithi055 ceph-mon[184174]: 1.1d scrub starts 2023-09-23T18:17:56.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:56 smithi055 ceph-mon[184174]: 1.1d scrub ok 2023-09-23T18:17:56.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:56 smithi055 ceph-mon[184174]: pgmap v1351: 103 pgs: 103 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:56.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2448523567' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T18:17:56.447 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:56.449 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:56.450 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:56.551 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:56.579 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:56.815 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:56.817 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:56.817 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:56.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:56 smithi099 ceph-mon[167076]: 1.1d scrub starts 2023-09-23T18:17:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:56 smithi099 ceph-mon[167076]: 1.1d scrub ok 2023-09-23T18:17:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:56 smithi099 ceph-mon[167076]: pgmap v1351: 103 pgs: 103 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2448523567' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]: dispatch 2023-09-23T18:17:56.918 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:17:56.947 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:57.014 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool application enable unique_pool_5 rados --yes-i-really-mean-it || true 2023-09-23T18:17:57.089 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:17:57.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:57.093 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:17:57.094 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:57.194 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:57.226 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:57.345 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:17:57.345 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:17:57.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:17:57.345 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:17:57.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.345 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45557.0:108098 7.6 7:63f0b014:::benchmark_data_smithi055_311528_object37135:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:17:57.345 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:08:40.505253+0000", 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 556.83830417299998, 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13445985899999999, 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45557", 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 108098 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505253+0000", 2023-09-23T18:17:57.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505253+0000", 2023-09-23T18:17:57.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505251+0000", 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505261+0000", 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4450000000000002e-06 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505282+0000", 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1251999999999999e-05 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505290+0000", 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2879999999999993e-06 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.505343+0000", 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2034999999999999e-05 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.506478+0000", 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011350469999999999 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.624282+0000", 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.117804585 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.624374+0000", 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1632000000000006e-05 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.624475+0000", 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010101400000000001 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.626282+0000", 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018073939999999999 2023-09-23T18:17:57.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.628754+0000", 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002471829 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.639295+0000", 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010541404000000001 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.639338+0000", 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2814e-05 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:08:40.639713+0000", 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000374948 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:88522 8.2 e379/371 8:4b2b0f84:::benchmark_data_smithi055_346788_object88521:head v 379'5492, mlcod=0'0)", 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:35.572982+0000", 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 441.77057574299999, 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.050153703000000001, 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.572982+0000", 2023-09-23T18:17:57.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.572982+0000", 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.572981+0000", 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.573023+0000", 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.244e-05 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.573042+0000", 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8419000000000001e-05 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.573051+0000", 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4639999999999995e-06 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575760+0000", 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027086300000000001 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575785+0000", 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5405000000000002e-05 2023-09-23T18:17:57.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623088+0000", 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047302960999999998 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623135+0000", 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7342000000000002e-05 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.358 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:88524 8.2 e379/371 8:46f9590d:::benchmark_data_smithi055_346788_object88523:head v 379'5493, mlcod=0'0)", 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:35.575910+0000", 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 441.76764752899999, 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047258545999999998, 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.359 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575910+0000", 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575910+0000", 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575909+0000", 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575956+0000", 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6783000000000003e-05 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575968+0000", 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2165e-05 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.575975+0000", 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5469999999999999e-06 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.581529+0000", 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055535189999999998 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.581561+0000", 2023-09-23T18:17:57.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1767000000000003e-05 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623141+0000", 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.041580060000000002 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:35.623169+0000", 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7665e-05 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.363 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPush(8.6 386/371 [PushOp(8:6c728b0a:::benchmark_data_smithi055_346788_object2082:head, version: 365'134, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:6c728b0a:::benchmark_data_smithi055_346788_object2082:head@365'134, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 422.99192500800001, 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.060643765000000002, 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351632+0000", 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:57.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351669+0000", 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6786999999999999e-05 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351689+0000", 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9692000000000001e-05 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.351702+0000", 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3709e-05 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.408798+0000", 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.057095414999999997 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.408806+0000", 2023-09-23T18:17:57.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5909999999999996e-06 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.412276+0000", 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034699190000000001 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:104012 8.6 e386/371 8:6f916fc8:::benchmark_data_smithi055_346788_object104011:head v 386'6585, mlcod=365'133)", 2023-09-23T18:17:57.368 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:54.387490+0000", 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 422.956067303, 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047785543, 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387490+0000", 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387490+0000", 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387489+0000", 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387519+0000", 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9485e-05 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387539+0000", 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0528999999999999e-05 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.387548+0000", 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2240000000000001e-06 2023-09-23T18:17:57.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.396833+0000", 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0092844009999999994 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.396858+0000", 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5072e-05 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.435227+0000", 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038369551000000002 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:54.435276+0000", 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8396000000000002e-05 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:105389 8.6 e386/371 8:617cba55:::benchmark_data_smithi055_346788_object105388:head v 386'6660, mlcod=365'174)", 2023-09-23T18:17:57.373 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:10:55.764711+0000", 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 421.57884616299998, 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.049308314999999998, 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764711+0000", 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764711+0000", 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764710+0000", 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.375 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764741+0000", 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0073000000000002e-05 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764755+0000", 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4827999999999999e-05 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.764764+0000", 2023-09-23T18:17:57.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.738e-06 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.769484+0000", 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047194649999999999 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.769511+0000", 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7542000000000002e-05 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.813963+0000", 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.044451549 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:10:55.814020+0000", 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7012999999999998e-05 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:109643 8.6 e386/371 8:6b4a303b:::benchmark_data_smithi055_346788_object109642:head v 386'6916, mlcod=365'308)", 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 417.32428202400001, 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.046387045000000002, 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019275+0000", 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019295+0000", 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9360000000000001e-05 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019301+0000", 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7490000000000001e-06 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.019305+0000", 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8700000000000002e-06 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.027156+0000", 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0078509070000000007 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.027176+0000", 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.037e-05 2023-09-23T18:17:57.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.065607+0000", 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038430756000000003 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:00.065663+0000", 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5334000000000002e-05 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.383 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:136113 8.6 e405/404 8:698e62bb:::benchmark_data_smithi055_346788_object25521:head v 405'8582, mlcod=0'0)", 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:41.581664+0000", 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 375.761893826, 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.064351331999999997, 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581664+0000", 2023-09-23T18:17:57.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581664+0000", 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581662+0000", 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581670+0000", 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8990000000000001e-06 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581683+0000", 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2346e-05 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581692+0000", 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2900000000000008e-06 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581741+0000", 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8770000000000002e-05 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.581757+0000", 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5994999999999998e-05 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.645972+0000", 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.064215229999999998 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:41.646015+0000", 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2985000000000003e-05 2023-09-23T18:17:57.388 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:156532 8.1 e416/383 8:8578332c:::benchmark_data_smithi055_346788_object45940:head v 416'9728, mlcod=366'2079)", 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:09.038016+0000", 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 348.30554127300002, 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.053601875, 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038016+0000", 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038016+0000", 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038015+0000", 2023-09-23T18:17:57.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038022+0000", 2023-09-23T18:17:57.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7939999999999997e-06 2023-09-23T18:17:57.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085419+0000", 2023-09-23T18:17:57.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.047397008999999997 2023-09-23T18:17:57.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085439+0000", 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9429999999999999e-05 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085493+0000", 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4848000000000001e-05 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085526+0000", 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2218999999999998e-05 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.091569+0000", 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060432519999999998 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.091618+0000", 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9156000000000002e-05 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.45923.0:156516 8.2 e416/413 8:40fce4c3:::benchmark_data_smithi055_346788_object45924:head v 416'9669, mlcod=0'0)", 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:09.038816+0000", 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 348.30474160599999, 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.053023129000000002, 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.396 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038816+0000", 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038816+0000", 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038815+0000", 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.038821+0000", 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.852e-06 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085422+0000", 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.046601112 2023-09-23T18:17:57.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085438+0000", 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5879000000000002e-05 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085493+0000", 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4710999999999998e-05 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085526+0000", 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3525999999999997e-05 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.091788+0000", 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0062620949999999996 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.091839+0000", 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0606999999999998e-05 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.400 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPush(8.6 416/404 [PushOp(8:6b0a9119:::benchmark_data_smithi055_346788_object45949:head, version: 368'2919, data_included: [0~65536], data_size: 65536, omap_header_size: 0, omap_entries_size: 0, attrset_size: 2, recovery_info: ObjectRecoveryInfo(8:6b0a9119:::benchmark_data_smithi055_346788_object45949:head@368'2919, size: 65536, copy_subset: [0~65536], clone_subset: {}, snapset: 0=[]:{}, object_exist: 0), after_progress: ObjectRecoveryProgress(!first, data_recovered_to:65536, data_complete:true, omap_recovered_to:, omap_complete:true, error:false), before_progress: ObjectRecoveryProgress(first, data_recovered_to:0, data_complete:false, omap_recovered_to:, omap_complete:false, error:false))])", 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:09.042065+0000", 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 348.30149269600003, 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.045964787, 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.042065+0000", 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.042065+0000", 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.042064+0000", 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.042119+0000", 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5179000000000001e-05 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085425+0000", 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043305758 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085450+0000", 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5037000000000001e-05 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085809+0000", 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00035868500000000003 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.085822+0000", 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3665e-05 2023-09-23T18:17:57.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:09.088030+0000", 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022073050000000001 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:37066 9.8 e466/459 9:1853f8f7:::benchmark_data_smithi055_386467_object37065:head v 466'2211, mlcod=466'2208)", 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:04.407507+0000", 2023-09-23T18:17:57.405 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 232.936050704, 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.33454793700000002, 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407507+0000", 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.406 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407507+0000", 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407506+0000", 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407569+0000", 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3313999999999997e-05 2023-09-23T18:17:57.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407585+0000", 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6155999999999999e-05 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407594+0000", 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7029999999999995e-06 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737582+0000", 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32998835999999998 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737627+0000", 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4586000000000003e-05 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.742016+0000", 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0043891629999999997 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.742055+0000", 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8816999999999998e-05 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.410 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:37075 9.9 e466/459 9:997ed14a:::benchmark_data_smithi055_386467_object37074:head v 466'2323, mlcod=466'2321)", 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:04.410872+0000", 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 232.93268593499999, 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.33109135099999998, 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410872+0000", 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410872+0000", 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410870+0000", 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410944+0000", 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4023999999999995e-05 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737578+0000", 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.326633379 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737600+0000", 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1971e-05 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737664+0000", 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3850999999999996e-05 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737684+0000", 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0202000000000001e-05 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.414 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.741922+0000", 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0042384019999999996 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.741963+0000", 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0674999999999999e-05 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.415 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:37084 9.8 e466/459 9:1c8e8c79:::benchmark_data_smithi055_386467_object37083:head v 466'2212, mlcod=466'2209)", 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:04.415320+0000", 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 232.928236986, 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32965889700000001, 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415320+0000", 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.416 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415320+0000", 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415319+0000", 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415387+0000", 2023-09-23T18:17:57.417 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7977000000000003e-05 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737586+0000", 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32219906100000001 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737603+0000", 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7031e-05 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.418 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.739129+0000", 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001526077 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.739145+0000", 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5176e-05 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.744934+0000", 2023-09-23T18:17:57.419 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0057898649999999999 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.744979+0000", 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4999000000000001e-05 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:140378 9.0 e512/511 9:09fb90c2:::benchmark_data_smithi055_386467_object35718:head v 512'9018, mlcod=0'0)", 2023-09-23T18:17:57.420 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:10.307202+0000", 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 107.036355973, 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015825197999999999, 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:10.307202+0000", 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.421 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:10.307202+0000", 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:10.307201+0000", 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:10.307208+0000", 2023-09-23T18:17:57.422 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7789999999999998e-06 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:10.307223+0000", 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5330000000000001e-05 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:10.307230+0000", 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4569999999999999e-06 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.423 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:10.309571+0000", 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023409809999999998 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:10.309588+0000", 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6915e-05 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:10.322980+0000", 2023-09-23T18:17:57.424 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013391839000000001 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:10.323027+0000", 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6606000000000001e-05 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.425 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:152423 9.8 e519/510 9:14fc7188:::benchmark_data_smithi055_386467_object47763:head v 519'9200, mlcod=519'9197)", 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:23.315649+0000", 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 94.027908257000007, 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015799245, 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:23.315649+0000", 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.426 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:23.315649+0000", 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:23.315649+0000", 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:23.315654+0000", 2023-09-23T18:17:57.427 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1440000000000002e-06 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:23.315661+0000", 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9439999999999999e-06 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:23.315666+0000", 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8409999999999999e-06 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.428 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:23.318158+0000", 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0024919360000000002 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:23.318187+0000", 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8963999999999999e-05 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:23.331404+0000", 2023-09-23T18:17:57.429 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013216946 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:23.331448+0000", 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4873999999999998e-05 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:199990 9.0 e548/545 9:0696b908:::benchmark_data_smithi055_386467_object95330:head v 548'12805, mlcod=548'12803)", 2023-09-23T18:17:57.430 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:23.323786+0000", 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 34.019771399, 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018607945000000001, 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.323786+0000", 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.431 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.323786+0000", 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.323785+0000", 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.323797+0000", 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1586000000000001e-05 2023-09-23T18:17:57.432 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.323811+0000", 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4209000000000001e-05 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.323820+0000", 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6090000000000001e-06 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.433 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.324802+0000", 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00098223899999999999 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.324828+0000", 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5672000000000001e-05 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.342343+0000", 2023-09-23T18:17:57.434 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017515781000000001 2023-09-23T18:17:57.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.435 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.435 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.435 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.342394+0000", 2023-09-23T18:17:57.435 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0729999999999997e-05 2023-09-23T18:17:57.435 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.435 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.435 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.435 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.435 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:199991 9.5 e548/545 9:a1e42554:::benchmark_data_smithi055_386467_object95331:head v 548'12377, mlcod=548'12374)", 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:23.325708+0000", 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 34.017849689999998, 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018568874999999999, 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.325708+0000", 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.436 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.325708+0000", 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.325707+0000", 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.437 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.325714+0000", 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3069999999999997e-06 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.325725+0000", 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1002000000000001e-05 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.325733+0000", 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4499999999999998e-06 2023-09-23T18:17:57.438 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.325789+0000", 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6128000000000002e-05 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.325812+0000", 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3271000000000001e-05 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.439 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.344208+0000", 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018395525999999999 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.344277+0000", 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9092999999999997e-05 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.440 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:199997 9.9 e548/547 9:90d7b99d:::benchmark_data_smithi055_386467_object95337:head v 548'12426, mlcod=548'12424)", 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:23.328229+0000", 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 34.015328678000003, 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016036331000000001, 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328229+0000", 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.441 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328229+0000", 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328228+0000", 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328235+0000", 2023-09-23T18:17:57.442 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9299999999999997e-06 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328247+0000", 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2292e-05 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328254+0000", 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9369999999999998e-06 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.443 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328301+0000", 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6736000000000001e-05 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328325+0000", 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3906000000000001e-05 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.444 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.344221+0000", 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015896434000000001 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.344265+0000", 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4082999999999998e-05 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.445 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:199999 9.8 e548/545 9:151d3687:::benchmark_data_smithi055_386467_object95339:head v 548'12142, mlcod=548'12140)", 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:23.328905+0000", 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 34.014652564999999, 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015718082000000001, 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328905+0000", 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.446 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328905+0000", 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328904+0000", 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:17:57.447 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328911+0000", 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6749999999999996e-06 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328924+0000", 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2670000000000001e-05 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328933+0000", 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5000000000000005e-06 2023-09-23T18:17:57.448 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.328995+0000", 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1495999999999995e-05 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.329027+0000", 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2891000000000001e-05 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.344580+0000", 2023-09-23T18:17:57.449 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015552218 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:23.344623+0000", 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3390999999999998e-05 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:17:57.450 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:17:57.451 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:17:57.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:57.452 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:57.452 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:57.553 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:57.582 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[184174]: failed to encode map e574 with expected crc 2023-09-23T18:17:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2448523567' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T18:17:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[184174]: osdmap e574: 8 total, 8 up, 6 in 2023-09-23T18:17:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[184174]: failed to encode map e574 with expected crc 2023-09-23T18:17:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[184174]: failed to encode map e574 with expected crc 2023-09-23T18:17:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:57.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[179668]: failed to encode map e574 with expected crc 2023-09-23T18:17:57.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2448523567' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T18:17:57.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[179668]: osdmap e574: 8 total, 8 up, 6 in 2023-09-23T18:17:57.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[179668]: failed to encode map e574 with expected crc 2023-09-23T18:17:57.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[179668]: failed to encode map e574 with expected crc 2023-09-23T18:17:57.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:57 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:57.746 INFO:tasks.thrashosds.thrasher:in_osds: [1, 0, 4, 3, 6, 5] out_osds: [7, 2] dead_osds: [] live_osds: [0, 4, 6, 7, 3, 5, 2, 1] 2023-09-23T18:17:57.746 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:17:57.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:57 smithi099 ceph-mon[167076]: failed to encode map e574 with expected crc 2023-09-23T18:17:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2448523567' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_5", "pg_num": 16}]': finished 2023-09-23T18:17:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:57 smithi099 ceph-mon[167076]: osdmap e574: 8 total, 8 up, 6 in 2023-09-23T18:17:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:57 smithi099 ceph-mon[167076]: failed to encode map e574 with expected crc 2023-09-23T18:17:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:57 smithi099 ceph-mon[167076]: failed to encode map e574 with expected crc 2023-09-23T18:17:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:57 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:17:57.880 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:17:57.881 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:57.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:57.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:57.985 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:17:58.015 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:58.141 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:58.189 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:17:58.192 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:58.192 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:17:58.293 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:17:58.324 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:58.413 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:17:58.416 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:58.417 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:17:58.484 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:17:58.517 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:58.544 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:58.596 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:17:58.629 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:58.632 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:58.633 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:58.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[184174]: failed to encode map e574 with expected crc 2023-09-23T18:17:58.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[184174]: pgmap v1353: 119 pgs: 16 unknown, 103 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:58.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[184174]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[184174]: osdmap e575: 8 total, 8 up, 6 in 2023-09-23T18:17:58.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[184174]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[184174]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[184174]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[184174]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.671 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[184174]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[179668]: failed to encode map e574 with expected crc 2023-09-23T18:17:58.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[179668]: pgmap v1353: 119 pgs: 16 unknown, 103 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:58.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[179668]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[179668]: osdmap e575: 8 total, 8 up, 6 in 2023-09-23T18:17:58.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[179668]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[179668]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[179668]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[179668]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.672 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:58 smithi055 ceph-mon[179668]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:58 smithi099 ceph-mon[167076]: failed to encode map e574 with expected crc 2023-09-23T18:17:58.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:58 smithi099 ceph-mon[167076]: pgmap v1353: 119 pgs: 16 unknown, 103 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:17:58.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:58 smithi099 ceph-mon[167076]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:58 smithi099 ceph-mon[167076]: osdmap e575: 8 total, 8 up, 6 in 2023-09-23T18:17:58.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:58 smithi099 ceph-mon[167076]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:58 smithi099 ceph-mon[167076]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.716 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:58 smithi099 ceph-mon[167076]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.717 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:58 smithi099 ceph-mon[167076]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.717 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:58 smithi099 ceph-mon[167076]: failed to encode map e575 with expected crc 2023-09-23T18:17:58.734 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:58.766 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:58.870 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:17:58.889 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:17:58.908 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:17:58.928 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:17:58.947 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:17:58.967 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:17:58.980 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:58.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:58.983 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:58.986 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:17:59.006 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:17:59.008 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:17:59.008 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:17:59.008 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:17:59.008 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:17:59.008 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:17:59.008 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:17:59.008 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:17:59.008 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:17:59.084 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:59.113 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:59.315 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:59.318 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:59.319 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:59.419 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:59.454 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:59.494 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:17:59.497 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:59.497 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:17:59.573 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:59 smithi099 ceph-mon[167076]: failed to encode map e576 with expected crc 2023-09-23T18:17:59.574 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:59 smithi099 ceph-mon[167076]: osdmap e576: 8 total, 8 up, 6 in 2023-09-23T18:17:59.574 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:59 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/89157555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:59.574 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:59 smithi099 ceph-mon[167076]: failed to encode map e576 with expected crc 2023-09-23T18:17:59.574 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:17:59 smithi099 ceph-mon[167076]: failed to encode map e576 with expected crc 2023-09-23T18:17:59.597 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:17:59.625 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:59.660 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:17:59.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:59.664 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:17:59.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:59 smithi055 ceph-mon[179668]: failed to encode map e576 with expected crc 2023-09-23T18:17:59.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:59 smithi055 ceph-mon[179668]: osdmap e576: 8 total, 8 up, 6 in 2023-09-23T18:17:59.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/89157555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:59.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:59 smithi055 ceph-mon[179668]: failed to encode map e576 with expected crc 2023-09-23T18:17:59.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:17:59 smithi055 ceph-mon[179668]: failed to encode map e576 with expected crc 2023-09-23T18:17:59.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:59 smithi055 ceph-mon[184174]: failed to encode map e576 with expected crc 2023-09-23T18:17:59.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:59 smithi055 ceph-mon[184174]: osdmap e576: 8 total, 8 up, 6 in 2023-09-23T18:17:59.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:59 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/89157555' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:17:59.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:59 smithi055 ceph-mon[184174]: failed to encode map e576 with expected crc 2023-09-23T18:17:59.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:17:59 smithi055 ceph-mon[184174]: failed to encode map e576 with expected crc 2023-09-23T18:17:59.765 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:17:59.785 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:17:59.785 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:17:59.794 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:17:59.829 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:17:59.832 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:17:59.832 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:17:59.933 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:17:59.964 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:00.054 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:00.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:00.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:00.159 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:00.186 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:00.222 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:00.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:00.226 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:00.230 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:18:00.327 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:00.355 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:00.398 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:00.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:00.401 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:00.456 INFO:teuthology.orchestra.run.smithi055.stderr:enabled application 'rados' on pool 'unique_pool_5' 2023-09-23T18:18:00.503 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:00.529 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:00.562 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:00.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:00.566 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:00.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:00.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:18:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:18:00] "GET /metrics HTTP/1.1" 200 34255 "" "Prometheus/2.43.0" 2023-09-23T18:18:00.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:00 smithi055 ceph-mon[179668]: failed to encode map e576 with expected crc 2023-09-23T18:18:00.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:00 smithi055 ceph-mon[179668]: pgmap v1356: 119 pgs: 15 unknown, 104 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:18:00.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/170860613' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:00.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:00 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:00 smithi055 ceph-mon[184174]: failed to encode map e576 with expected crc 2023-09-23T18:18:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:00 smithi055 ceph-mon[184174]: pgmap v1356: 119 pgs: 15 unknown, 104 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:18:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/170860613' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:00 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:00.690 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:00 smithi099 ceph-mon[167076]: failed to encode map e576 with expected crc 2023-09-23T18:18:00.691 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:00 smithi099 ceph-mon[167076]: pgmap v1356: 119 pgs: 15 unknown, 104 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:18:00.691 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/170860613' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:00.691 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:00 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:18:00.694 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:00.727 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:00.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:00.731 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:00.831 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:00.861 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:00.870 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:18:00.896 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:00.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:00.900 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:01.000 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:01.029 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:01.061 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:01.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:01.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:01.165 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:01.192 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:01.224 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:01.227 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:01.228 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:01.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:01.356 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:01.462 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:18:01.513 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:18:01.634 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:01.636 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:01 smithi099 ceph-mon[167076]: failed to encode map e577 with expected crc 2023-09-23T18:18:01.636 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:01 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:18:01.637 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:01 smithi099 ceph-mon[167076]: osdmap e577: 8 total, 8 up, 6 in 2023-09-23T18:18:01.637 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:01 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1254251337' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:01.637 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:01 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:01.637 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:01 smithi099 ceph-mon[167076]: failed to encode map e577 with expected crc 2023-09-23T18:18:01.638 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:01.638 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:01.739 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:01.766 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:01.803 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:01.806 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:01.806 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:01.845 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[184174]: failed to encode map e577 with expected crc 2023-09-23T18:18:01.845 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:18:01.845 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[184174]: osdmap e577: 8 total, 8 up, 6 in 2023-09-23T18:18:01.846 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1254251337' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:01.846 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:01.846 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[184174]: failed to encode map e577 with expected crc 2023-09-23T18:18:01.846 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[179668]: failed to encode map e577 with expected crc 2023-09-23T18:18:01.846 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_5", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-09-23T18:18:01.846 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[179668]: osdmap e577: 8 total, 8 up, 6 in 2023-09-23T18:18:01.846 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1254251337' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:01.846 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:01.846 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:01 smithi055 ceph-mon[179668]: failed to encode map e577 with expected crc 2023-09-23T18:18:01.894 DEBUG:teuthology.orchestra.run.smithi055:> /bin/sh -c 'adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage rados --no-log-to-stderr --name client.0 -b 65536 --object-size 65536 -p unique_pool_5 bench 90 write' 2023-09-23T18:18:01.894 INFO:tasks.thrashosds.thrasher:Growing pool .mgr 2023-09-23T18:18:01.894 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:18:01.897 INFO:tasks.radosbench:joining radosbench (timing out after 3000s) 2023-09-23T18:18:01.897 INFO:teuthology.orchestra.run:waiting for 3000 2023-09-23T18:18:01.907 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:01.934 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:01.946 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:hints = 1 2023-09-23T18:18:01.946 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Maintaining 16 concurrent writes of 65536 bytes to objects of size 65536 for up to 90 seconds or 0 objects 2023-09-23T18:18:01.946 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object prefix: benchmark_data_smithi055_425050 2023-09-23T18:18:01.947 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:18:01.947 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 0 0 0 0 0 0 - 0 2023-09-23T18:18:01.990 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:01.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:01.998 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:02.099 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:02.155 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:02.161 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:02.214 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:18:02.214 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:18:02.223 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:02.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:02.231 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:02.332 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:02.389 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:02.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:02.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:02.604 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:02.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:18:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: failed to encode map e577 with expected crc 2023-09-23T18:18:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: 1.c scrub starts 2023-09-23T18:18:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: 1.c scrub ok 2023-09-23T18:18:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: pgmap v1358: 119 pgs: 10 unknown, 109 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:18:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:18:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:18:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: osdmap e578: 8 total, 8 up, 6 in 2023-09-23T18:18:02.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[184174]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: failed to encode map e577 with expected crc 2023-09-23T18:18:02.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: 1.c scrub starts 2023-09-23T18:18:02.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: 1.c scrub ok 2023-09-23T18:18:02.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: pgmap v1358: 119 pgs: 10 unknown, 109 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:18:02.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:18:02.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:18:02.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:02.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: osdmap e578: 8 total, 8 up, 6 in 2023-09-23T18:18:02.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:02 smithi055 ceph-mon[179668]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.705 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:02.748 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: failed to encode map e577 with expected crc 2023-09-23T18:18:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: 1.c scrub starts 2023-09-23T18:18:02.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: 1.c scrub ok 2023-09-23T18:18:02.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: pgmap v1358: 119 pgs: 10 unknown, 109 active+clean; 4.3 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:18:02.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:18:02.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:18:02.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:02.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: osdmap e578: 8 total, 8 up, 6 in 2023-09-23T18:18:02.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:02 smithi099 ceph-mon[167076]: failed to encode map e578 with expected crc 2023-09-23T18:18:02.892 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:02.896 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:02.896 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:02.947 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 1 16 3205 3189 199.301 199.312 0.0100296 0.00499883 2023-09-23T18:18:02.965 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:02.997 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:03.052 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:03.115 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:03.120 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:03.120 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:03.221 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:03.269 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:03.336 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:03.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:03.346 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:03.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:03.479 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:18:03.505 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:03.633 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:03.639 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:03.639 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:03.709 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:18:03.740 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:18:03.740 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:03.771 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:18:03.826 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:03.827 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:18:03.838 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:18:03.846 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:03.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:03.855 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:03.860 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:18:03.876 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:18:03.915 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:18:03.947 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 2 16 6103 6087 190.205 181.125 0.00942979 0.00524606 2023-09-23T18:18:03.949 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:18:03.953 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:18:03.953 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:18:03.953 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:18:03.953 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:18:03.953 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:18:03.953 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:18:03.953 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:18:03.953 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:18:03.957 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:04.012 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:04 smithi099 ceph-mon[167076]: pgmap v1360: 119 pgs: 119 active+clean; 4.6 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:18:04.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:04 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3086822062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:04 smithi055 ceph-mon[184174]: pgmap v1360: 119 pgs: 119 active+clean; 4.6 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:18:04.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:04 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3086822062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:04.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:04 smithi055 ceph-mon[179668]: pgmap v1360: 119 pgs: 119 active+clean; 4.6 MiB data, 2.9 GiB used, 534 GiB / 536 GiB avail 2023-09-23T18:18:04.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:04 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3086822062' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:04.765 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:04.947 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 3 16 9078 9062 188.763 185.938 0.00698897 0.00528931 2023-09-23T18:18:05.004 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:18:05.010 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:18:05.436 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1360,"stamp":"2023-09-23T18:18:03.093919+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4843864,"num_objects":28,"num_object_clones":0,"num_object_copies":84,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":28,"num_whiteouts":0,"num_read":5996,"num_read_kb":12229,"num_write":928,"num_write_kb":12944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":378,"num_bytes_recovered":17203326,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":357,"acting":357,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":352,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":3024376,"kb_used_data":140104,"kb_used_omap":40,"kb_used_meta":2884183,"kb_avail":559446536,"statfs":{"total":575970213888,"available":572873252864,"internally_reserved":0,"allocated":143466496,"data_stored":129622229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":2953403706},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":318735,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":-7,"num_read_kb":-14,"num_write":-4,"num_write_kb":-37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-6,"num_bytes_recovered":-2047528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"5.688516"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":865,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434233+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:17:58.434233+0000","last_peered":"2023-09-23T18:17:58.434233+0000","last_clean":"2023-09-23T18:17:58.434233+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:17:58.434233+0000","last_undegraded":"2023-09-23T18:17:58.434233+0000","last_fullsized":"2023-09-23T18:17:58.434233+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":192,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434493+0000","last_change":"2023-09-23T18:17:52.384763+0000","last_active":"2023-09-23T18:17:58.434493+0000","last_peered":"2023-09-23T18:17:58.434493+0000","last_clean":"2023-09-23T18:17:58.434493+0000","last_became_active":"2023-09-23T18:17:52.384412+0000","last_became_peered":"2023-09-23T18:17:52.384412+0000","last_unstale":"2023-09-23T18:17:58.434493+0000","last_undegraded":"2023-09-23T18:17:58.434493+0000","last_fullsized":"2023-09-23T18:17:58.434493+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:52:55.974256+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":459,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953748+0000","last_change":"2023-09-23T18:17:52.385931+0000","last_active":"2023-09-23T18:17:58.953748+0000","last_peered":"2023-09-23T18:17:58.953748+0000","last_clean":"2023-09-23T18:17:58.953748+0000","last_became_active":"2023-09-23T18:17:52.385640+0000","last_became_peered":"2023-09-23T18:17:52.385640+0000","last_unstale":"2023-09-23T18:17:58.953748+0000","last_undegraded":"2023-09-23T18:17:58.953748+0000","last_fullsized":"2023-09-23T18:17:58.953748+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:38:45.315239+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":851,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434427+0000","last_change":"2023-09-23T18:16:56.355044+0000","last_active":"2023-09-23T18:17:58.434427+0000","last_peered":"2023-09-23T18:17:58.434427+0000","last_clean":"2023-09-23T18:17:58.434427+0000","last_became_active":"2023-09-23T18:16:56.354698+0000","last_became_peered":"2023-09-23T18:16:56.354698+0000","last_unstale":"2023-09-23T18:17:58.434427+0000","last_undegraded":"2023-09-23T18:17:58.434427+0000","last_fullsized":"2023-09-23T18:17:58.434427+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:57:01.359288+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":316,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434889+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:17:58.434889+0000","last_peered":"2023-09-23T18:17:58.434889+0000","last_clean":"2023-09-23T18:17:58.434889+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:17:58.434889+0000","last_undegraded":"2023-09-23T18:17:58.434889+0000","last_fullsized":"2023-09-23T18:17:58.434889+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":630,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434944+0000","last_change":"2023-09-23T18:17:21.859064+0000","last_active":"2023-09-23T18:17:58.434944+0000","last_peered":"2023-09-23T18:17:58.434944+0000","last_clean":"2023-09-23T18:17:58.434944+0000","last_became_active":"2023-09-23T18:17:21.858411+0000","last_became_peered":"2023-09-23T18:17:21.858411+0000","last_unstale":"2023-09-23T18:17:58.434944+0000","last_undegraded":"2023-09-23T18:17:58.434944+0000","last_fullsized":"2023-09-23T18:17:58.434944+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:34:25.052706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":888,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434102+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:17:58.434102+0000","last_peered":"2023-09-23T18:17:58.434102+0000","last_clean":"2023-09-23T18:17:58.434102+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:17:58.434102+0000","last_undegraded":"2023-09-23T18:17:58.434102+0000","last_fullsized":"2023-09-23T18:17:58.434102+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":504,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.433877+0000","last_change":"2023-09-23T18:17:21.793684+0000","last_active":"2023-09-23T18:17:58.433877+0000","last_peered":"2023-09-23T18:17:58.433877+0000","last_clean":"2023-09-23T18:17:58.433877+0000","last_became_active":"2023-09-23T18:17:21.791678+0000","last_became_peered":"2023-09-23T18:17:21.791678+0000","last_unstale":"2023-09-23T18:17:58.433877+0000","last_undegraded":"2023-09-23T18:17:58.433877+0000","last_fullsized":"2023-09-23T18:17:58.433877+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:29:36.718693+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":153,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953898+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:17:58.953898+0000","last_peered":"2023-09-23T18:17:58.953898+0000","last_clean":"2023-09-23T18:17:58.953898+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:17:58.953898+0000","last_undegraded":"2023-09-23T18:17:58.953898+0000","last_fullsized":"2023-09-23T18:17:58.953898+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1011,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434529+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:17:58.434529+0000","last_peered":"2023-09-23T18:17:58.434529+0000","last_clean":"2023-09-23T18:17:58.434529+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:17:58.434529+0000","last_undegraded":"2023-09-23T18:17:58.434529+0000","last_fullsized":"2023-09-23T18:17:58.434529+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":77,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.435011+0000","last_change":"2023-09-23T18:16:08.297760+0000","last_active":"2023-09-23T18:17:58.435011+0000","last_peered":"2023-09-23T18:17:58.435011+0000","last_clean":"2023-09-23T18:17:58.435011+0000","last_became_active":"2023-09-23T18:16:08.296846+0000","last_became_peered":"2023-09-23T18:16:08.296846+0000","last_unstale":"2023-09-23T18:17:58.435011+0000","last_undegraded":"2023-09-23T18:17:58.435011+0000","last_fullsized":"2023-09-23T18:17:58.435011+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:09:11.358278+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":165,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434389+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:17:58.434389+0000","last_peered":"2023-09-23T18:17:58.434389+0000","last_clean":"2023-09-23T18:17:58.434389+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:17:58.434389+0000","last_undegraded":"2023-09-23T18:17:58.434389+0000","last_fullsized":"2023-09-23T18:17:58.434389+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":788,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952740+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:17:58.952740+0000","last_peered":"2023-09-23T18:17:58.952740+0000","last_clean":"2023-09-23T18:17:58.952740+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:17:58.952740+0000","last_undegraded":"2023-09-23T18:17:58.952740+0000","last_fullsized":"2023-09-23T18:17:58.952740+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":103,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434896+0000","last_change":"2023-09-23T18:17:21.858712+0000","last_active":"2023-09-23T18:17:58.434896+0000","last_peered":"2023-09-23T18:17:58.434896+0000","last_clean":"2023-09-23T18:17:58.434896+0000","last_became_active":"2023-09-23T18:17:21.857968+0000","last_became_peered":"2023-09-23T18:17:21.857968+0000","last_unstale":"2023-09-23T18:17:58.434896+0000","last_undegraded":"2023-09-23T18:17:58.434896+0000","last_fullsized":"2023-09-23T18:17:58.434896+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:10:59.198935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":122,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952310+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:17:58.952310+0000","last_peered":"2023-09-23T18:17:58.952310+0000","last_clean":"2023-09-23T18:17:58.952310+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:17:58.952310+0000","last_undegraded":"2023-09-23T18:17:58.952310+0000","last_fullsized":"2023-09-23T18:17:58.952310+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":902,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.435005+0000","last_change":"2023-09-23T18:16:56.365256+0000","last_active":"2023-09-23T18:17:58.435005+0000","last_peered":"2023-09-23T18:17:58.435005+0000","last_clean":"2023-09-23T18:17:58.435005+0000","last_became_active":"2023-09-23T18:16:56.364932+0000","last_became_peered":"2023-09-23T18:16:56.364932+0000","last_unstale":"2023-09-23T18:17:58.435005+0000","last_undegraded":"2023-09-23T18:17:58.435005+0000","last_fullsized":"2023-09-23T18:17:58.435005+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:49:54.515789+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":370,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434830+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:17:58.434830+0000","last_peered":"2023-09-23T18:17:58.434830+0000","last_clean":"2023-09-23T18:17:58.434830+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:17:58.434830+0000","last_undegraded":"2023-09-23T18:17:58.434830+0000","last_fullsized":"2023-09-23T18:17:58.434830+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":380,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953847+0000","last_change":"2023-09-23T18:17:21.782039+0000","last_active":"2023-09-23T18:17:58.953847+0000","last_peered":"2023-09-23T18:17:58.953847+0000","last_clean":"2023-09-23T18:17:58.953847+0000","last_became_active":"2023-09-23T18:17:21.781726+0000","last_became_peered":"2023-09-23T18:17:21.781726+0000","last_unstale":"2023-09-23T18:17:58.953847+0000","last_undegraded":"2023-09-23T18:17:58.953847+0000","last_fullsized":"2023-09-23T18:17:58.953847+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:58:29.767815+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":932,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T18:17:57.409754+0000","last_change":"2023-09-23T18:17:52.384140+0000","last_active":"2023-09-23T18:17:57.409754+0000","last_peered":"2023-09-23T18:17:57.409754+0000","last_clean":"2023-09-23T18:17:57.409754+0000","last_became_active":"2023-09-23T18:17:52.383802+0000","last_became_peered":"2023-09-23T18:17:52.383802+0000","last_unstale":"2023-09-23T18:17:57.409754+0000","last_undegraded":"2023-09-23T18:17:57.409754+0000","last_fullsized":"2023-09-23T18:17:57.409754+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:25:39.074135+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":109,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953358+0000","last_change":"2023-09-23T18:17:52.390718+0000","last_active":"2023-09-23T18:17:58.953358+0000","last_peered":"2023-09-23T18:17:58.953358+0000","last_clean":"2023-09-23T18:17:58.953358+0000","last_became_active":"2023-09-23T18:17:52.390191+0000","last_became_peered":"2023-09-23T18:17:52.390191+0000","last_unstale":"2023-09-23T18:17:58.953358+0000","last_undegraded":"2023-09-23T18:17:58.953358+0000","last_fullsized":"2023-09-23T18:17:58.953358+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:11:34.000012+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":396,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953255+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:17:58.953255+0000","last_peered":"2023-09-23T18:17:58.953255+0000","last_clean":"2023-09-23T18:17:58.953255+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:17:58.953255+0000","last_undegraded":"2023-09-23T18:17:58.953255+0000","last_fullsized":"2023-09-23T18:17:58.953255+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":763,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434213+0000","last_change":"2023-09-23T18:17:21.789814+0000","last_active":"2023-09-23T18:17:58.434213+0000","last_peered":"2023-09-23T18:17:58.434213+0000","last_clean":"2023-09-23T18:17:58.434213+0000","last_became_active":"2023-09-23T18:17:21.789472+0000","last_became_peered":"2023-09-23T18:17:21.789472+0000","last_unstale":"2023-09-23T18:17:58.434213+0000","last_undegraded":"2023-09-23T18:17:58.434213+0000","last_fullsized":"2023-09-23T18:17:58.434213+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:38:55.247538+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":396,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953172+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:17:58.953172+0000","last_peered":"2023-09-23T18:17:58.953172+0000","last_clean":"2023-09-23T18:17:58.953172+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:17:58.953172+0000","last_undegraded":"2023-09-23T18:17:58.953172+0000","last_fullsized":"2023-09-23T18:17:58.953172+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":64,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434301+0000","last_change":"2023-09-23T18:17:21.791098+0000","last_active":"2023-09-23T18:17:58.434301+0000","last_peered":"2023-09-23T18:17:58.434301+0000","last_clean":"2023-09-23T18:17:58.434301+0000","last_became_active":"2023-09-23T18:17:21.784777+0000","last_became_peered":"2023-09-23T18:17:21.784777+0000","last_unstale":"2023-09-23T18:17:58.434301+0000","last_undegraded":"2023-09-23T18:17:58.434301+0000","last_fullsized":"2023-09-23T18:17:58.434301+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:48:44.394135+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,1],"acting":[5,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":774,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.433982+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:17:58.433982+0000","last_peered":"2023-09-23T18:17:58.433982+0000","last_clean":"2023-09-23T18:17:58.433982+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:17:58.433982+0000","last_undegraded":"2023-09-23T18:17:58.433982+0000","last_fullsized":"2023-09-23T18:17:58.433982+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":85,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434454+0000","last_change":"2023-09-23T18:17:21.788887+0000","last_active":"2023-09-23T18:17:58.434454+0000","last_peered":"2023-09-23T18:17:58.434454+0000","last_clean":"2023-09-23T18:17:58.434454+0000","last_became_active":"2023-09-23T18:17:21.785275+0000","last_became_peered":"2023-09-23T18:17:21.785275+0000","last_unstale":"2023-09-23T18:17:58.434454+0000","last_undegraded":"2023-09-23T18:17:58.434454+0000","last_fullsized":"2023-09-23T18:17:58.434454+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:37.806340+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":87,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434766+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:17:58.434766+0000","last_peered":"2023-09-23T18:17:58.434766+0000","last_clean":"2023-09-23T18:17:58.434766+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:17:58.434766+0000","last_undegraded":"2023-09-23T18:17:58.434766+0000","last_fullsized":"2023-09-23T18:17:58.434766+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":897,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T18:17:57.408868+0000","last_change":"2023-09-23T18:17:52.384062+0000","last_active":"2023-09-23T18:17:57.408868+0000","last_peered":"2023-09-23T18:17:57.408868+0000","last_clean":"2023-09-23T18:17:57.408868+0000","last_became_active":"2023-09-23T18:17:52.383641+0000","last_became_peered":"2023-09-23T18:17:52.383641+0000","last_unstale":"2023-09-23T18:17:57.408868+0000","last_undegraded":"2023-09-23T18:17:57.408868+0000","last_fullsized":"2023-09-23T18:17:57.408868+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:18:18.490810+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":164,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952177+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:17:58.952177+0000","last_peered":"2023-09-23T18:17:58.952177+0000","last_clean":"2023-09-23T18:17:58.952177+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:17:58.952177+0000","last_undegraded":"2023-09-23T18:17:58.952177+0000","last_fullsized":"2023-09-23T18:17:58.952177+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":165,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.435078+0000","last_change":"2023-09-23T18:17:02.568080+0000","last_active":"2023-09-23T18:17:58.435078+0000","last_peered":"2023-09-23T18:17:58.435078+0000","last_clean":"2023-09-23T18:17:58.435078+0000","last_became_active":"2023-09-23T18:17:02.567769+0000","last_became_peered":"2023-09-23T18:17:02.567769+0000","last_unstale":"2023-09-23T18:17:58.435078+0000","last_undegraded":"2023-09-23T18:17:58.435078+0000","last_fullsized":"2023-09-23T18:17:58.435078+0000","mapping_epoch":533,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":534,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:20:58.972439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,5],"acting":[3,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":872,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434287+0000","last_change":"2023-09-23T18:17:21.789620+0000","last_active":"2023-09-23T18:17:58.434287+0000","last_peered":"2023-09-23T18:17:58.434287+0000","last_clean":"2023-09-23T18:17:58.434287+0000","last_became_active":"2023-09-23T18:17:21.788835+0000","last_became_peered":"2023-09-23T18:17:21.788835+0000","last_unstale":"2023-09-23T18:17:58.434287+0000","last_undegraded":"2023-09-23T18:17:58.434287+0000","last_fullsized":"2023-09-23T18:17:58.434287+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:23:49.908440+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":697,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.951997+0000","last_change":"2023-09-23T18:17:21.777450+0000","last_active":"2023-09-23T18:17:58.951997+0000","last_peered":"2023-09-23T18:17:58.951997+0000","last_clean":"2023-09-23T18:17:58.951997+0000","last_became_active":"2023-09-23T18:17:21.777078+0000","last_became_peered":"2023-09-23T18:17:21.777078+0000","last_unstale":"2023-09-23T18:17:58.951997+0000","last_undegraded":"2023-09-23T18:17:58.951997+0000","last_fullsized":"2023-09-23T18:17:58.951997+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:04:33.517731+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":85,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953719+0000","last_change":"2023-09-23T18:17:21.832285+0000","last_active":"2023-09-23T18:17:58.953719+0000","last_peered":"2023-09-23T18:17:58.953719+0000","last_clean":"2023-09-23T18:17:58.953719+0000","last_became_active":"2023-09-23T18:17:21.831436+0000","last_became_peered":"2023-09-23T18:17:21.831436+0000","last_unstale":"2023-09-23T18:17:58.953719+0000","last_undegraded":"2023-09-23T18:17:58.953719+0000","last_fullsized":"2023-09-23T18:17:58.953719+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:16:13.687158+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.6","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434629+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:17:58.434629+0000","last_peered":"2023-09-23T18:17:58.434629+0000","last_clean":"2023-09-23T18:17:58.434629+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:17:58.434629+0000","last_undegraded":"2023-09-23T18:17:58.434629+0000","last_fullsized":"2023-09-23T18:17:58.434629+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":842,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953474+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:17:58.953474+0000","last_peered":"2023-09-23T18:17:58.953474+0000","last_clean":"2023-09-23T18:17:58.953474+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:17:58.953474+0000","last_undegraded":"2023-09-23T18:17:58.953474+0000","last_fullsized":"2023-09-23T18:17:58.953474+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":161,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434820+0000","last_change":"2023-09-23T18:17:52.385847+0000","last_active":"2023-09-23T18:17:58.434820+0000","last_peered":"2023-09-23T18:17:58.434820+0000","last_clean":"2023-09-23T18:17:58.434820+0000","last_became_active":"2023-09-23T18:17:52.384670+0000","last_became_peered":"2023-09-23T18:17:52.384670+0000","last_unstale":"2023-09-23T18:17:58.434820+0000","last_undegraded":"2023-09-23T18:17:58.434820+0000","last_fullsized":"2023-09-23T18:17:58.434820+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:37.507906+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":22,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":87,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434060+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:17:58.434060+0000","last_peered":"2023-09-23T18:17:58.434060+0000","last_clean":"2023-09-23T18:17:58.434060+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:17:58.434060+0000","last_undegraded":"2023-09-23T18:17:58.434060+0000","last_fullsized":"2023-09-23T18:17:58.434060+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953708+0000","last_change":"2023-09-23T18:17:57.427372+0000","last_active":"2023-09-23T18:17:58.953708+0000","last_peered":"2023-09-23T18:17:58.953708+0000","last_clean":"2023-09-23T18:17:58.953708+0000","last_became_active":"2023-09-23T18:17:57.427030+0000","last_became_peered":"2023-09-23T18:17:57.427030+0000","last_unstale":"2023-09-23T18:17:58.953708+0000","last_undegraded":"2023-09-23T18:17:58.953708+0000","last_fullsized":"2023-09-23T18:17:58.953708+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:09:53.987393+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":811,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:18:01.016154+0000","last_change":"2023-09-23T18:18:01.015990+0000","last_active":"2023-09-23T18:18:01.016154+0000","last_peered":"2023-09-23T18:18:01.016154+0000","last_clean":"2023-09-23T18:18:01.016154+0000","last_became_active":"2023-09-23T18:14:51.174964+0000","last_became_peered":"2023-09-23T18:14:51.174964+0000","last_unstale":"2023-09-23T18:18:01.016154+0000","last_undegraded":"2023-09-23T18:18:01.016154+0000","last_fullsized":"2023-09-23T18:18:01.016154+0000","mapping_epoch":483,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:53.894493+0000","scrub_duration":0.0018562780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,4],"acting":[6,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":162,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434734+0000","last_change":"2023-09-23T18:17:52.385641+0000","last_active":"2023-09-23T18:17:58.434734+0000","last_peered":"2023-09-23T18:17:58.434734+0000","last_clean":"2023-09-23T18:17:58.434734+0000","last_became_active":"2023-09-23T18:17:52.384936+0000","last_became_peered":"2023-09-23T18:17:52.384936+0000","last_unstale":"2023-09-23T18:17:58.434734+0000","last_undegraded":"2023-09-23T18:17:58.434734+0000","last_fullsized":"2023-09-23T18:17:58.434734+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:24.391572+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":194,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952040+0000","last_change":"2023-09-23T18:17:42.001740+0000","last_active":"2023-09-23T18:17:58.952040+0000","last_peered":"2023-09-23T18:17:58.952040+0000","last_clean":"2023-09-23T18:17:58.952040+0000","last_became_active":"2023-09-23T18:17:42.001442+0000","last_became_peered":"2023-09-23T18:17:42.001442+0000","last_unstale":"2023-09-23T18:17:58.952040+0000","last_undegraded":"2023-09-23T18:17:58.952040+0000","last_fullsized":"2023-09-23T18:17:58.952040+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:38:58.704712+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.4","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952505+0000","last_change":"2023-09-23T18:17:57.424994+0000","last_active":"2023-09-23T18:17:58.952505+0000","last_peered":"2023-09-23T18:17:58.952505+0000","last_clean":"2023-09-23T18:17:58.952505+0000","last_became_active":"2023-09-23T18:17:57.424661+0000","last_became_peered":"2023-09-23T18:17:57.424661+0000","last_unstale":"2023-09-23T18:17:58.952505+0000","last_undegraded":"2023-09-23T18:17:58.952505+0000","last_fullsized":"2023-09-23T18:17:58.952505+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:07:15.895653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":582,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953865+0000","last_change":"2023-09-23T18:16:11.305989+0000","last_active":"2023-09-23T18:17:58.953865+0000","last_peered":"2023-09-23T18:17:58.953865+0000","last_clean":"2023-09-23T18:17:58.953865+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:17:58.953865+0000","last_undegraded":"2023-09-23T18:17:58.953865+0000","last_fullsized":"2023-09-23T18:17:58.953865+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:02.805268+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:02.805268+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26835,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":42,"num_write":11,"num_write_kb":110,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80505,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":502,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434632+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:17:58.434632+0000","last_peered":"2023-09-23T18:17:58.434632+0000","last_clean":"2023-09-23T18:17:58.434632+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:17:58.434632+0000","last_undegraded":"2023-09-23T18:17:58.434632+0000","last_fullsized":"2023-09-23T18:17:58.434632+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":143,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953790+0000","last_change":"2023-09-23T18:17:21.782112+0000","last_active":"2023-09-23T18:17:58.953790+0000","last_peered":"2023-09-23T18:17:58.953790+0000","last_clean":"2023-09-23T18:17:58.953790+0000","last_became_active":"2023-09-23T18:17:21.781809+0000","last_became_peered":"2023-09-23T18:17:21.781809+0000","last_unstale":"2023-09-23T18:17:58.953790+0000","last_undegraded":"2023-09-23T18:17:58.953790+0000","last_fullsized":"2023-09-23T18:17:58.953790+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:49:02.164958+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"0'0","reported_seq":12,"reported_epoch":577,"state":"active+clean","last_fresh":"2023-09-23T18:18:00.453135+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:18:00.453135+0000","last_peered":"2023-09-23T18:18:00.453135+0000","last_clean":"2023-09-23T18:18:00.453135+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:18:00.453135+0000","last_undegraded":"2023-09-23T18:18:00.453135+0000","last_fullsized":"2023-09-23T18:18:00.453135+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":607,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.435118+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:17:58.435118+0000","last_peered":"2023-09-23T18:17:58.435118+0000","last_clean":"2023-09-23T18:17:58.435118+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:17:58.435118+0000","last_undegraded":"2023-09-23T18:17:58.435118+0000","last_fullsized":"2023-09-23T18:17:58.435118+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":91,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434683+0000","last_change":"2023-09-23T18:17:21.786518+0000","last_active":"2023-09-23T18:17:58.434683+0000","last_peered":"2023-09-23T18:17:58.434683+0000","last_clean":"2023-09-23T18:17:58.434683+0000","last_became_active":"2023-09-23T18:17:21.784127+0000","last_became_peered":"2023-09-23T18:17:21.784127+0000","last_unstale":"2023-09-23T18:17:58.434683+0000","last_undegraded":"2023-09-23T18:17:58.434683+0000","last_fullsized":"2023-09-23T18:17:58.434683+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:17.023991+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":412,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T18:17:57.627711+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:17:57.627711+0000","last_peered":"2023-09-23T18:17:57.627711+0000","last_clean":"2023-09-23T18:17:57.627711+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:17:57.627711+0000","last_undegraded":"2023-09-23T18:17:57.627711+0000","last_fullsized":"2023-09-23T18:17:57.627711+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.435725+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:17:58.435725+0000","last_peered":"2023-09-23T18:17:58.435725+0000","last_clean":"2023-09-23T18:17:58.435725+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:17:58.435725+0000","last_undegraded":"2023-09-23T18:17:58.435725+0000","last_fullsized":"2023-09-23T18:17:58.435725+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1059,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434104+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:17:58.434104+0000","last_peered":"2023-09-23T18:17:58.434104+0000","last_clean":"2023-09-23T18:17:58.434104+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:17:58.434104+0000","last_undegraded":"2023-09-23T18:17:58.434104+0000","last_fullsized":"2023-09-23T18:17:58.434104+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":300,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434715+0000","last_change":"2023-09-23T18:17:52.383751+0000","last_active":"2023-09-23T18:17:58.434715+0000","last_peered":"2023-09-23T18:17:58.434715+0000","last_clean":"2023-09-23T18:17:58.434715+0000","last_became_active":"2023-09-23T18:17:52.383373+0000","last_became_peered":"2023-09-23T18:17:52.383373+0000","last_unstale":"2023-09-23T18:17:58.434715+0000","last_undegraded":"2023-09-23T18:17:58.434715+0000","last_fullsized":"2023-09-23T18:17:58.434715+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:18:27.384066+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":121,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952902+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:17:58.952902+0000","last_peered":"2023-09-23T18:17:58.952902+0000","last_clean":"2023-09-23T18:17:58.952902+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:17:58.952902+0000","last_undegraded":"2023-09-23T18:17:58.952902+0000","last_fullsized":"2023-09-23T18:17:58.952902+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434332+0000","last_change":"2023-09-23T18:17:57.629287+0000","last_active":"2023-09-23T18:17:58.434332+0000","last_peered":"2023-09-23T18:17:58.434332+0000","last_clean":"2023-09-23T18:17:58.434332+0000","last_became_active":"2023-09-23T18:17:57.628964+0000","last_became_peered":"2023-09-23T18:17:57.628964+0000","last_unstale":"2023-09-23T18:17:58.434332+0000","last_undegraded":"2023-09-23T18:17:58.434332+0000","last_fullsized":"2023-09-23T18:17:58.434332+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:13.370659+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":826,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434199+0000","last_change":"2023-09-23T18:17:05.625604+0000","last_active":"2023-09-23T18:17:58.434199+0000","last_peered":"2023-09-23T18:17:58.434199+0000","last_clean":"2023-09-23T18:17:58.434199+0000","last_became_active":"2023-09-23T18:17:05.625256+0000","last_became_peered":"2023-09-23T18:17:05.625256+0000","last_unstale":"2023-09-23T18:17:58.434199+0000","last_undegraded":"2023-09-23T18:17:58.434199+0000","last_fullsized":"2023-09-23T18:17:58.434199+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:46.272850+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":129,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953700+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:17:58.953700+0000","last_peered":"2023-09-23T18:17:58.953700+0000","last_clean":"2023-09-23T18:17:58.953700+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:17:58.953700+0000","last_undegraded":"2023-09-23T18:17:58.953700+0000","last_fullsized":"2023-09-23T18:17:58.953700+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":428,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953580+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:17:58.953580+0000","last_peered":"2023-09-23T18:17:58.953580+0000","last_clean":"2023-09-23T18:17:58.953580+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:17:58.953580+0000","last_undegraded":"2023-09-23T18:17:58.953580+0000","last_fullsized":"2023-09-23T18:17:58.953580+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.951835+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:17:58.951835+0000","last_peered":"2023-09-23T18:17:58.951835+0000","last_clean":"2023-09-23T18:17:58.951835+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:17:58.951835+0000","last_undegraded":"2023-09-23T18:17:58.951835+0000","last_fullsized":"2023-09-23T18:17:58.951835+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":848,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434491+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:17:58.434491+0000","last_peered":"2023-09-23T18:17:58.434491+0000","last_clean":"2023-09-23T18:17:58.434491+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:17:58.434491+0000","last_undegraded":"2023-09-23T18:17:58.434491+0000","last_fullsized":"2023-09-23T18:17:58.434491+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":413,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.433948+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:17:58.433948+0000","last_peered":"2023-09-23T18:17:58.433948+0000","last_clean":"2023-09-23T18:17:58.433948+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:17:58.433948+0000","last_undegraded":"2023-09-23T18:17:58.433948+0000","last_fullsized":"2023-09-23T18:17:58.433948+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":189,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953540+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:17:58.953540+0000","last_peered":"2023-09-23T18:17:58.953540+0000","last_clean":"2023-09-23T18:17:58.953540+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:17:58.953540+0000","last_undegraded":"2023-09-23T18:17:58.953540+0000","last_fullsized":"2023-09-23T18:17:58.953540+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.951444+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:17:58.951444+0000","last_peered":"2023-09-23T18:17:58.951444+0000","last_clean":"2023-09-23T18:17:58.951444+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:17:58.951444+0000","last_undegraded":"2023-09-23T18:17:58.951444+0000","last_fullsized":"2023-09-23T18:17:58.951444+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":829,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434638+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:17:58.434638+0000","last_peered":"2023-09-23T18:17:58.434638+0000","last_clean":"2023-09-23T18:17:58.434638+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:17:58.434638+0000","last_undegraded":"2023-09-23T18:17:58.434638+0000","last_fullsized":"2023-09-23T18:17:58.434638+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":329,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.433883+0000","last_change":"2023-09-23T18:17:52.384882+0000","last_active":"2023-09-23T18:17:58.433883+0000","last_peered":"2023-09-23T18:17:58.433883+0000","last_clean":"2023-09-23T18:17:58.433883+0000","last_became_active":"2023-09-23T18:17:52.384538+0000","last_became_peered":"2023-09-23T18:17:52.384538+0000","last_unstale":"2023-09-23T18:17:58.433883+0000","last_undegraded":"2023-09-23T18:17:58.433883+0000","last_fullsized":"2023-09-23T18:17:58.433883+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:54:32.494297+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":303,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T18:17:57.409196+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:17:57.409196+0000","last_peered":"2023-09-23T18:17:57.409196+0000","last_clean":"2023-09-23T18:17:57.409196+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:17:57.409196+0000","last_undegraded":"2023-09-23T18:17:57.409196+0000","last_fullsized":"2023-09-23T18:17:57.409196+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953326+0000","last_change":"2023-09-23T18:17:57.629189+0000","last_active":"2023-09-23T18:17:58.953326+0000","last_peered":"2023-09-23T18:17:58.953326+0000","last_clean":"2023-09-23T18:17:58.953326+0000","last_became_active":"2023-09-23T18:17:57.628865+0000","last_became_peered":"2023-09-23T18:17:57.628865+0000","last_unstale":"2023-09-23T18:17:58.953326+0000","last_undegraded":"2023-09-23T18:17:58.953326+0000","last_fullsized":"2023-09-23T18:17:58.953326+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:33.396143+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":668,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953920+0000","last_change":"2023-09-23T18:17:21.831805+0000","last_active":"2023-09-23T18:17:58.953920+0000","last_peered":"2023-09-23T18:17:58.953920+0000","last_clean":"2023-09-23T18:17:58.953920+0000","last_became_active":"2023-09-23T18:17:21.830942+0000","last_became_peered":"2023-09-23T18:17:21.830942+0000","last_unstale":"2023-09-23T18:17:58.953920+0000","last_undegraded":"2023-09-23T18:17:58.953920+0000","last_fullsized":"2023-09-23T18:17:58.953920+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:10:12.468788+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":917,"reported_epoch":575,"state":"active+clean","last_fresh":"2023-09-23T18:17:57.409278+0000","last_change":"2023-09-23T18:17:52.385464+0000","last_active":"2023-09-23T18:17:57.409278+0000","last_peered":"2023-09-23T18:17:57.409278+0000","last_clean":"2023-09-23T18:17:57.409278+0000","last_became_active":"2023-09-23T18:17:52.384893+0000","last_became_peered":"2023-09-23T18:17:52.384893+0000","last_unstale":"2023-09-23T18:17:57.409278+0000","last_undegraded":"2023-09-23T18:17:57.409278+0000","last_fullsized":"2023-09-23T18:17:57.409278+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:39:55.538323+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,4],"acting":[1,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":101,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434807+0000","last_change":"2023-09-23T18:17:21.788601+0000","last_active":"2023-09-23T18:17:58.434807+0000","last_peered":"2023-09-23T18:17:58.434807+0000","last_clean":"2023-09-23T18:17:58.434807+0000","last_became_active":"2023-09-23T18:17:21.785178+0000","last_became_peered":"2023-09-23T18:17:21.785178+0000","last_unstale":"2023-09-23T18:17:58.434807+0000","last_undegraded":"2023-09-23T18:17:58.434807+0000","last_fullsized":"2023-09-23T18:17:58.434807+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:12:53.920425+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,1],"acting":[5,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":91,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434588+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:17:58.434588+0000","last_peered":"2023-09-23T18:17:58.434588+0000","last_clean":"2023-09-23T18:17:58.434588+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:17:58.434588+0000","last_undegraded":"2023-09-23T18:17:58.434588+0000","last_fullsized":"2023-09-23T18:17:58.434588+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953442+0000","last_change":"2023-09-23T18:17:57.427177+0000","last_active":"2023-09-23T18:17:58.953442+0000","last_peered":"2023-09-23T18:17:58.953442+0000","last_clean":"2023-09-23T18:17:58.953442+0000","last_became_active":"2023-09-23T18:17:57.426876+0000","last_became_peered":"2023-09-23T18:17:57.426876+0000","last_unstale":"2023-09-23T18:17:58.953442+0000","last_undegraded":"2023-09-23T18:17:58.953442+0000","last_fullsized":"2023-09-23T18:17:58.953442+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:19:39.604476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":760,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434300+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:17:58.434300+0000","last_peered":"2023-09-23T18:17:58.434300+0000","last_clean":"2023-09-23T18:17:58.434300+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:17:58.434300+0000","last_undegraded":"2023-09-23T18:17:58.434300+0000","last_fullsized":"2023-09-23T18:17:58.434300+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1461,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434538+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:17:58.434538+0000","last_peered":"2023-09-23T18:17:58.434538+0000","last_clean":"2023-09-23T18:17:58.434538+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:17:58.434538+0000","last_undegraded":"2023-09-23T18:17:58.434538+0000","last_fullsized":"2023-09-23T18:17:58.434538+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":101,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953215+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:17:58.953215+0000","last_peered":"2023-09-23T18:17:58.953215+0000","last_clean":"2023-09-23T18:17:58.953215+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:17:58.953215+0000","last_undegraded":"2023-09-23T18:17:58.953215+0000","last_fullsized":"2023-09-23T18:17:58.953215+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":391,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953237+0000","last_change":"2023-09-23T18:17:21.777404+0000","last_active":"2023-09-23T18:17:58.953237+0000","last_peered":"2023-09-23T18:17:58.953237+0000","last_clean":"2023-09-23T18:17:58.953237+0000","last_became_active":"2023-09-23T18:17:21.777074+0000","last_became_peered":"2023-09-23T18:17:21.777074+0000","last_unstale":"2023-09-23T18:17:58.953237+0000","last_undegraded":"2023-09-23T18:17:58.953237+0000","last_fullsized":"2023-09-23T18:17:58.953237+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:37:45.237817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.433943+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:17:58.433943+0000","last_peered":"2023-09-23T18:17:58.433943+0000","last_clean":"2023-09-23T18:17:58.433943+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:17:58.433943+0000","last_undegraded":"2023-09-23T18:17:58.433943+0000","last_fullsized":"2023-09-23T18:17:58.433943+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":785,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434698+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:17:58.434698+0000","last_peered":"2023-09-23T18:17:58.434698+0000","last_clean":"2023-09-23T18:17:58.434698+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:17:58.434698+0000","last_undegraded":"2023-09-23T18:17:58.434698+0000","last_fullsized":"2023-09-23T18:17:58.434698+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":111,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.951955+0000","last_change":"2023-09-23T18:17:21.774714+0000","last_active":"2023-09-23T18:17:58.951955+0000","last_peered":"2023-09-23T18:17:58.951955+0000","last_clean":"2023-09-23T18:17:58.951955+0000","last_became_active":"2023-09-23T18:17:21.774179+0000","last_became_peered":"2023-09-23T18:17:21.774179+0000","last_unstale":"2023-09-23T18:17:58.951955+0000","last_undegraded":"2023-09-23T18:17:58.951955+0000","last_fullsized":"2023-09-23T18:17:58.951955+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:51:50.639823+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":370,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952103+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:17:58.952103+0000","last_peered":"2023-09-23T18:17:58.952103+0000","last_clean":"2023-09-23T18:17:58.952103+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:17:58.952103+0000","last_undegraded":"2023-09-23T18:17:58.952103+0000","last_fullsized":"2023-09-23T18:17:58.952103+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":103,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434949+0000","last_change":"2023-09-23T18:17:21.828891+0000","last_active":"2023-09-23T18:17:58.434949+0000","last_peered":"2023-09-23T18:17:58.434949+0000","last_clean":"2023-09-23T18:17:58.434949+0000","last_became_active":"2023-09-23T18:17:21.828579+0000","last_became_peered":"2023-09-23T18:17:21.828579+0000","last_unstale":"2023-09-23T18:17:58.434949+0000","last_undegraded":"2023-09-23T18:17:58.434949+0000","last_fullsized":"2023-09-23T18:17:58.434949+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:00:22.904810+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953029+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:17:58.953029+0000","last_peered":"2023-09-23T18:17:58.953029+0000","last_clean":"2023-09-23T18:17:58.953029+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:17:58.953029+0000","last_undegraded":"2023-09-23T18:17:58.953029+0000","last_fullsized":"2023-09-23T18:17:58.953029+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1474,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434226+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:17:58.434226+0000","last_peered":"2023-09-23T18:17:58.434226+0000","last_clean":"2023-09-23T18:17:58.434226+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:17:58.434226+0000","last_undegraded":"2023-09-23T18:17:58.434226+0000","last_fullsized":"2023-09-23T18:17:58.434226+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":497,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952045+0000","last_change":"2023-09-23T18:17:21.780963+0000","last_active":"2023-09-23T18:17:58.952045+0000","last_peered":"2023-09-23T18:17:58.952045+0000","last_clean":"2023-09-23T18:17:58.952045+0000","last_became_active":"2023-09-23T18:17:21.780614+0000","last_became_peered":"2023-09-23T18:17:21.780614+0000","last_unstale":"2023-09-23T18:17:58.952045+0000","last_undegraded":"2023-09-23T18:17:58.952045+0000","last_fullsized":"2023-09-23T18:17:58.952045+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:53.385496+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:53.385496+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":26834,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":22,"num_read_kb":41,"num_write":10,"num_write_kb":109,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":80504,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":77,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434189+0000","last_change":"2023-09-23T18:16:08.289677+0000","last_active":"2023-09-23T18:17:58.434189+0000","last_peered":"2023-09-23T18:17:58.434189+0000","last_clean":"2023-09-23T18:17:58.434189+0000","last_became_active":"2023-09-23T18:16:08.289317+0000","last_became_peered":"2023-09-23T18:16:08.289317+0000","last_unstale":"2023-09-23T18:17:58.434189+0000","last_undegraded":"2023-09-23T18:17:58.434189+0000","last_fullsized":"2023-09-23T18:17:58.434189+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:30:33.684123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":122,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953818+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:17:58.953818+0000","last_peered":"2023-09-23T18:17:58.953818+0000","last_clean":"2023-09-23T18:17:58.953818+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:17:58.953818+0000","last_undegraded":"2023-09-23T18:17:58.953818+0000","last_fullsized":"2023-09-23T18:17:58.953818+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434094+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:17:58.434094+0000","last_peered":"2023-09-23T18:17:58.434094+0000","last_clean":"2023-09-23T18:17:58.434094+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:17:58.434094+0000","last_undegraded":"2023-09-23T18:17:58.434094+0000","last_fullsized":"2023-09-23T18:17:58.434094+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":775,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953297+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:17:58.953297+0000","last_peered":"2023-09-23T18:17:58.953297+0000","last_clean":"2023-09-23T18:17:58.953297+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:17:58.953297+0000","last_undegraded":"2023-09-23T18:17:58.953297+0000","last_fullsized":"2023-09-23T18:17:58.953297+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":89,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434310+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:17:58.434310+0000","last_peered":"2023-09-23T18:17:58.434310+0000","last_clean":"2023-09-23T18:17:58.434310+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:17:58.434310+0000","last_undegraded":"2023-09-23T18:17:58.434310+0000","last_fullsized":"2023-09-23T18:17:58.434310+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":480,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434205+0000","last_change":"2023-09-23T18:17:52.385316+0000","last_active":"2023-09-23T18:17:58.434205+0000","last_peered":"2023-09-23T18:17:58.434205+0000","last_clean":"2023-09-23T18:17:58.434205+0000","last_became_active":"2023-09-23T18:17:52.384774+0000","last_became_peered":"2023-09-23T18:17:52.384774+0000","last_unstale":"2023-09-23T18:17:58.434205+0000","last_undegraded":"2023-09-23T18:17:58.434205+0000","last_fullsized":"2023-09-23T18:17:58.434205+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:44.576338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":582,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434937+0000","last_change":"2023-09-23T18:17:21.858891+0000","last_active":"2023-09-23T18:17:58.434937+0000","last_peered":"2023-09-23T18:17:58.434937+0000","last_clean":"2023-09-23T18:17:58.434937+0000","last_became_active":"2023-09-23T18:17:21.858030+0000","last_became_peered":"2023-09-23T18:17:21.858030+0000","last_unstale":"2023-09-23T18:17:58.434937+0000","last_undegraded":"2023-09-23T18:17:58.434937+0000","last_fullsized":"2023-09-23T18:17:58.434937+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:33:10.554587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952748+0000","last_change":"2023-09-23T18:17:57.426419+0000","last_active":"2023-09-23T18:17:58.952748+0000","last_peered":"2023-09-23T18:17:58.952748+0000","last_clean":"2023-09-23T18:17:58.952748+0000","last_became_active":"2023-09-23T18:17:57.426061+0000","last_became_peered":"2023-09-23T18:17:57.426061+0000","last_unstale":"2023-09-23T18:17:58.952748+0000","last_undegraded":"2023-09-23T18:17:58.952748+0000","last_fullsized":"2023-09-23T18:17:58.952748+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:36:09.739723+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":835,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953841+0000","last_change":"2023-09-23T18:16:56.370006+0000","last_active":"2023-09-23T18:17:58.953841+0000","last_peered":"2023-09-23T18:17:58.953841+0000","last_clean":"2023-09-23T18:17:58.953841+0000","last_became_active":"2023-09-23T18:16:56.369251+0000","last_became_peered":"2023-09-23T18:16:56.369251+0000","last_unstale":"2023-09-23T18:17:58.953841+0000","last_undegraded":"2023-09-23T18:17:58.953841+0000","last_fullsized":"2023-09-23T18:17:58.953841+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:05:45.183676+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":704,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953458+0000","last_change":"2023-09-23T18:17:52.388200+0000","last_active":"2023-09-23T18:17:58.953458+0000","last_peered":"2023-09-23T18:17:58.953458+0000","last_clean":"2023-09-23T18:17:58.953458+0000","last_became_active":"2023-09-23T18:17:52.387094+0000","last_became_peered":"2023-09-23T18:17:52.387094+0000","last_unstale":"2023-09-23T18:17:58.953458+0000","last_undegraded":"2023-09-23T18:17:58.953458+0000","last_fullsized":"2023-09-23T18:17:58.953458+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:17:20.975580+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":28,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":91,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434467+0000","last_change":"2023-09-23T18:17:21.791604+0000","last_active":"2023-09-23T18:17:58.434467+0000","last_peered":"2023-09-23T18:17:58.434467+0000","last_clean":"2023-09-23T18:17:58.434467+0000","last_became_active":"2023-09-23T18:17:21.786029+0000","last_became_peered":"2023-09-23T18:17:21.786029+0000","last_unstale":"2023-09-23T18:17:58.434467+0000","last_undegraded":"2023-09-23T18:17:58.434467+0000","last_fullsized":"2023-09-23T18:17:58.434467+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:14:45.193525+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":101,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434339+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:17:58.434339+0000","last_peered":"2023-09-23T18:17:58.434339+0000","last_clean":"2023-09-23T18:17:58.434339+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:17:58.434339+0000","last_undegraded":"2023-09-23T18:17:58.434339+0000","last_fullsized":"2023-09-23T18:17:58.434339+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953654+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:17:58.953654+0000","last_peered":"2023-09-23T18:17:58.953654+0000","last_clean":"2023-09-23T18:17:58.953654+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:17:58.953654+0000","last_undegraded":"2023-09-23T18:17:58.953654+0000","last_fullsized":"2023-09-23T18:17:58.953654+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":875,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953650+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:17:58.953650+0000","last_peered":"2023-09-23T18:17:58.953650+0000","last_clean":"2023-09-23T18:17:58.953650+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:17:58.953650+0000","last_undegraded":"2023-09-23T18:17:58.953650+0000","last_fullsized":"2023-09-23T18:17:58.953650+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":63,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434851+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:17:58.434851+0000","last_peered":"2023-09-23T18:17:58.434851+0000","last_clean":"2023-09-23T18:17:58.434851+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:17:58.434851+0000","last_undegraded":"2023-09-23T18:17:58.434851+0000","last_fullsized":"2023-09-23T18:17:58.434851+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":356,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953994+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:17:58.953994+0000","last_peered":"2023-09-23T18:17:58.953994+0000","last_clean":"2023-09-23T18:17:58.953994+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:17:58.953994+0000","last_undegraded":"2023-09-23T18:17:58.953994+0000","last_fullsized":"2023-09-23T18:17:58.953994+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":425,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.433964+0000","last_change":"2023-09-23T18:17:21.786294+0000","last_active":"2023-09-23T18:17:58.433964+0000","last_peered":"2023-09-23T18:17:58.433964+0000","last_clean":"2023-09-23T18:17:58.433964+0000","last_became_active":"2023-09-23T18:17:21.784868+0000","last_became_peered":"2023-09-23T18:17:21.784868+0000","last_unstale":"2023-09-23T18:17:58.433964+0000","last_undegraded":"2023-09-23T18:17:58.433964+0000","last_fullsized":"2023-09-23T18:17:58.433964+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:39.892879+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.f","version":"0'0","reported_seq":11,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953947+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:17:58.953947+0000","last_peered":"2023-09-23T18:17:58.953947+0000","last_clean":"2023-09-23T18:17:58.953947+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:17:58.953947+0000","last_undegraded":"2023-09-23T18:17:58.953947+0000","last_fullsized":"2023-09-23T18:17:58.953947+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":779,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434546+0000","last_change":"2023-09-23T18:16:56.353747+0000","last_active":"2023-09-23T18:17:58.434546+0000","last_peered":"2023-09-23T18:17:58.434546+0000","last_clean":"2023-09-23T18:17:58.434546+0000","last_became_active":"2023-09-23T18:16:56.352208+0000","last_became_peered":"2023-09-23T18:16:56.352208+0000","last_unstale":"2023-09-23T18:17:58.434546+0000","last_undegraded":"2023-09-23T18:17:58.434546+0000","last_fullsized":"2023-09-23T18:17:58.434546+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:12:54.881986+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1013,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953996+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:17:58.953996+0000","last_peered":"2023-09-23T18:17:58.953996+0000","last_clean":"2023-09-23T18:17:58.953996+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:17:58.953996+0000","last_undegraded":"2023-09-23T18:17:58.953996+0000","last_fullsized":"2023-09-23T18:17:58.953996+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":267,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434824+0000","last_change":"2023-09-23T18:17:52.385654+0000","last_active":"2023-09-23T18:17:58.434824+0000","last_peered":"2023-09-23T18:17:58.434824+0000","last_clean":"2023-09-23T18:17:58.434824+0000","last_became_active":"2023-09-23T18:17:52.385344+0000","last_became_peered":"2023-09-23T18:17:52.385344+0000","last_unstale":"2023-09-23T18:17:58.434824+0000","last_undegraded":"2023-09-23T18:17:58.434824+0000","last_fullsized":"2023-09-23T18:17:58.434824+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:30.457398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":367,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434113+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:17:58.434113+0000","last_peered":"2023-09-23T18:17:58.434113+0000","last_clean":"2023-09-23T18:17:58.434113+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:17:58.434113+0000","last_undegraded":"2023-09-23T18:17:58.434113+0000","last_fullsized":"2023-09-23T18:17:58.434113+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":567,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952395+0000","last_change":"2023-09-23T18:16:56.371701+0000","last_active":"2023-09-23T18:17:58.952395+0000","last_peered":"2023-09-23T18:17:58.952395+0000","last_clean":"2023-09-23T18:17:58.952395+0000","last_became_active":"2023-09-23T18:16:56.371354+0000","last_became_peered":"2023-09-23T18:16:56.371354+0000","last_unstale":"2023-09-23T18:17:58.952395+0000","last_undegraded":"2023-09-23T18:17:58.952395+0000","last_fullsized":"2023-09-23T18:17:58.952395+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:00:04.624441+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":154,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953119+0000","last_change":"2023-09-23T18:17:52.388507+0000","last_active":"2023-09-23T18:17:58.953119+0000","last_peered":"2023-09-23T18:17:58.953119+0000","last_clean":"2023-09-23T18:17:58.953119+0000","last_became_active":"2023-09-23T18:17:52.388191+0000","last_became_peered":"2023-09-23T18:17:52.388191+0000","last_unstale":"2023-09-23T18:17:58.953119+0000","last_undegraded":"2023-09-23T18:17:58.953119+0000","last_fullsized":"2023-09-23T18:17:58.953119+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:41:30.207976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":655,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434440+0000","last_change":"2023-09-23T18:17:52.385165+0000","last_active":"2023-09-23T18:17:58.434440+0000","last_peered":"2023-09-23T18:17:58.434440+0000","last_clean":"2023-09-23T18:17:58.434440+0000","last_became_active":"2023-09-23T18:17:52.384867+0000","last_became_peered":"2023-09-23T18:17:52.384867+0000","last_unstale":"2023-09-23T18:17:58.434440+0000","last_undegraded":"2023-09-23T18:17:58.434440+0000","last_fullsized":"2023-09-23T18:17:58.434440+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:02.133934+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":567,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953593+0000","last_change":"2023-09-23T18:16:56.370183+0000","last_active":"2023-09-23T18:17:58.953593+0000","last_peered":"2023-09-23T18:17:58.953593+0000","last_clean":"2023-09-23T18:17:58.953593+0000","last_became_active":"2023-09-23T18:16:56.369701+0000","last_became_peered":"2023-09-23T18:16:56.369701+0000","last_unstale":"2023-09-23T18:17:58.953593+0000","last_undegraded":"2023-09-23T18:17:58.953593+0000","last_fullsized":"2023-09-23T18:17:58.953593+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:36:41.796195+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":95,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434545+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:17:58.434545+0000","last_peered":"2023-09-23T18:17:58.434545+0000","last_clean":"2023-09-23T18:17:58.434545+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:17:58.434545+0000","last_undegraded":"2023-09-23T18:17:58.434545+0000","last_fullsized":"2023-09-23T18:17:58.434545+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":211,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952005+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:17:58.952005+0000","last_peered":"2023-09-23T18:17:58.952005+0000","last_clean":"2023-09-23T18:17:58.952005+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:17:58.952005+0000","last_undegraded":"2023-09-23T18:17:58.952005+0000","last_fullsized":"2023-09-23T18:17:58.952005+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":81,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434713+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:17:58.434713+0000","last_peered":"2023-09-23T18:17:58.434713+0000","last_clean":"2023-09-23T18:17:58.434713+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:17:58.434713+0000","last_undegraded":"2023-09-23T18:17:58.434713+0000","last_fullsized":"2023-09-23T18:17:58.434713+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":194,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.951847+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:17:58.951847+0000","last_peered":"2023-09-23T18:17:58.951847+0000","last_clean":"2023-09-23T18:17:58.951847+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:17:58.951847+0000","last_undegraded":"2023-09-23T18:17:58.951847+0000","last_fullsized":"2023-09-23T18:17:58.951847+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":969,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952829+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:17:58.952829+0000","last_peered":"2023-09-23T18:17:58.952829+0000","last_clean":"2023-09-23T18:17:58.952829+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:17:58.952829+0000","last_undegraded":"2023-09-23T18:17:58.952829+0000","last_fullsized":"2023-09-23T18:17:58.952829+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":464,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.953750+0000","last_change":"2023-09-23T18:17:21.774518+0000","last_active":"2023-09-23T18:17:58.953750+0000","last_peered":"2023-09-23T18:17:58.953750+0000","last_clean":"2023-09-23T18:17:58.953750+0000","last_became_active":"2023-09-23T18:17:21.774015+0000","last_became_peered":"2023-09-23T18:17:21.774015+0000","last_unstale":"2023-09-23T18:17:58.953750+0000","last_undegraded":"2023-09-23T18:17:58.953750+0000","last_fullsized":"2023-09-23T18:17:58.953750+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:36:48.043536+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":101,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434270+0000","last_change":"2023-09-23T18:17:21.789770+0000","last_active":"2023-09-23T18:17:58.434270+0000","last_peered":"2023-09-23T18:17:58.434270+0000","last_clean":"2023-09-23T18:17:58.434270+0000","last_became_active":"2023-09-23T18:17:21.788932+0000","last_became_peered":"2023-09-23T18:17:21.788932+0000","last_unstale":"2023-09-23T18:17:58.434270+0000","last_undegraded":"2023-09-23T18:17:58.434270+0000","last_fullsized":"2023-09-23T18:17:58.434270+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:45:12.662092+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":654,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434491+0000","last_change":"2023-09-23T18:17:54.598634+0000","last_active":"2023-09-23T18:17:58.434491+0000","last_peered":"2023-09-23T18:17:58.434491+0000","last_clean":"2023-09-23T18:17:58.434491+0000","last_became_active":"2023-09-23T18:17:52.385183+0000","last_became_peered":"2023-09-23T18:17:52.385183+0000","last_unstale":"2023-09-23T18:17:58.434491+0000","last_undegraded":"2023-09-23T18:17:58.434491+0000","last_fullsized":"2023-09-23T18:17:58.434491+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:13:31.898180+0000","scrub_duration":0.0020689219999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":83,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.434932+0000","last_change":"2023-09-23T18:16:08.310119+0000","last_active":"2023-09-23T18:17:58.434932+0000","last_peered":"2023-09-23T18:17:58.434932+0000","last_clean":"2023-09-23T18:17:58.434932+0000","last_became_active":"2023-09-23T18:16:08.297391+0000","last_became_peered":"2023-09-23T18:16:08.297391+0000","last_unstale":"2023-09-23T18:17:58.434932+0000","last_undegraded":"2023-09-23T18:17:58.434932+0000","last_fullsized":"2023-09-23T18:17:58.434932+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:19:31.289701+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":1108,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":610,"reported_epoch":576,"state":"active+clean","last_fresh":"2023-09-23T18:17:58.952033+0000","last_change":"2023-09-23T18:17:52.385291+0000","last_active":"2023-09-23T18:17:58.952033+0000","last_peered":"2023-09-23T18:17:58.952033+0000","last_clean":"2023-09-23T18:17:58.952033+0000","last_became_active":"2023-09-23T18:17:52.384989+0000","last_became_peered":"2023-09-23T18:17:52.384989+0000","last_unstale":"2023-09-23T18:17:58.952033+0000","last_undegraded":"2023-09-23T18:17:58.952033+0000","last_fullsized":"2023-09-23T18:17:58.952033+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:48:54.809895+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":48,"acting":48,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":72,"num_bytes_recovered":2356,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":644037,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":44,"num_read_kb":83,"num_write":21,"num_write_kb":219,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":423697,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":305,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001545,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669481,"num_pgs":73,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":556008,"kb_used_data":10720,"kb_used_omap":7,"kb_used_meta":545272,"kb_avail":93189144,"statfs":{"total":95995035648,"available":95425683456,"internally_reserved":0,"allocated":10977280,"data_stored":9142374,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":558359332},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:17:09 2023","interfaces":[{"interface":"back","average":{"1min":1.7769999999999999,"5min":1.3480000000000001,"15min":1.2170000000000001},"min":{"1min":0.255,"5min":0.22700000000000001,"15min":0.224},"max":{"1min":7.3680000000000003,"5min":7.3680000000000003,"15min":15.407},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.2280000000000002,"5min":1.6879999999999999,"15min":1.454},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23100000000000001},"max":{"1min":7.423,"5min":7.423,"15min":15.147},"last":0.41699999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:15:05 2023","interfaces":[{"interface":"back","average":{"1min":2.4710000000000001,"5min":1.8009999999999999,"15min":1.405},"min":{"1min":0.245,"5min":0.218,"15min":0.218},"max":{"1min":11.083,"5min":21.390000000000001,"15min":21.390000000000001},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.2320000000000002,"5min":2.3540000000000001,"15min":1.5980000000000001},"min":{"1min":0.29499999999999998,"5min":0.27700000000000002,"15min":0.248},"max":{"1min":6.375,"5min":21.559999999999999,"15min":21.559999999999999},"last":0.441}]},{"osd":2,"last update":"Sat Sep 23 18:17:53 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.359,"15min":1.272},"min":{"1min":0.314,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":4.5339999999999998,"5min":7.4189999999999996,"15min":7.4189999999999996},"last":0.54700000000000004},{"interface":"front","average":{"1min":2.1909999999999998,"5min":1.623,"15min":1.413},"min":{"1min":0.29699999999999999,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":14.771000000000001,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.52700000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:17:47 2023","interfaces":[{"interface":"back","average":{"1min":1.331,"5min":1.667,"15min":1.5289999999999999},"min":{"1min":0.34799999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.5720000000000001,"5min":7.3940000000000001,"15min":10.257999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.9259999999999999,"5min":1.948,"15min":1.754},"min":{"1min":0.33800000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.5940000000000003,"5min":9.4049999999999994,"15min":9.5960000000000001},"last":0.55500000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:17:09 2023","interfaces":[{"interface":"back","average":{"1min":2.5150000000000001,"5min":1.6779999999999999,"15min":1.2010000000000001},"min":{"1min":0.29999999999999999,"5min":0.254,"15min":0.23999999999999999},"max":{"1min":8.2400000000000002,"5min":8.6059999999999999,"15min":8.6059999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.3769999999999998,"5min":1.7929999999999999,"15min":1.2210000000000001},"min":{"1min":0.374,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":7.3310000000000004,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.50900000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:17:40 2023","interfaces":[{"interface":"back","average":{"1min":2.5089999999999999,"5min":2.1040000000000001,"15min":2.004},"min":{"1min":0.378,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":13.502000000000001,"5min":13.502000000000001,"15min":13.502000000000001},"last":0.49299999999999999},{"interface":"front","average":{"1min":3.0630000000000002,"5min":2.371,"15min":2.4670000000000001},"min":{"1min":0.28599999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.081,"5min":10.081,"15min":10.081},"last":0.56399999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:17:53 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.0750000000000002,"15min":1.4390000000000001},"min":{"1min":0.435,"5min":0.35999999999999999,"15min":0.26800000000000002},"max":{"1min":10.743,"5min":10.743,"15min":13.460000000000001},"last":0.53900000000000003},{"interface":"front","average":{"1min":1.5680000000000001,"5min":2.1539999999999999,"15min":1.478},"min":{"1min":0.40100000000000002,"5min":0.32900000000000001,"15min":0.28499999999999998},"max":{"1min":4.7089999999999996,"5min":11.244,"15min":12.404},"last":0.67000000000000004}]}]},{"osd":1,"up_from":545,"seq":2340757176329,"num_pgs":67,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":165336,"kb_used_data":11728,"kb_used_omap":13,"kb_used_meta":153586,"kb_avail":93579816,"statfs":{"total":95995035648,"available":95825731584,"internally_reserved":0,"allocated":12009472,"data_stored":9202680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":157272717},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.25600000000000001}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.29899999999999999}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.34300000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55300000000000005}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.41799999999999998}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.51300000000000001}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40000000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958826,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":857080,"kb_used_data":97008,"kb_used_omap":7,"kb_used_meta":760056,"kb_avail":92888072,"statfs":{"total":95995035648,"available":95117385728,"internally_reserved":0,"allocated":99336192,"data_stored":96675430,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":778298084},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:15:35 2023","interfaces":[{"interface":"back","average":{"1min":1.54,"5min":1.506,"15min":1.4910000000000001},"min":{"1min":0.19700000000000001,"5min":0.192,"15min":0.192},"max":{"1min":8.7810000000000006,"5min":13.677,"15min":15.836},"last":0.316},{"interface":"front","average":{"1min":1.635,"5min":1.8420000000000001,"15min":1.462},"min":{"1min":0.251,"5min":0.23400000000000001,"15min":0.186},"max":{"1min":4.6989999999999998,"5min":14.207000000000001,"15min":14.468999999999999},"last":0.24199999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:17:57 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.82299999999999995,"15min":1.02},"min":{"1min":0.19800000000000001,"5min":0.185,"15min":0.185},"max":{"1min":1.2609999999999999,"5min":8.8300000000000001,"15min":8.8300000000000001},"last":0.27800000000000002},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.67500000000000004,"15min":0.73999999999999999},"min":{"1min":0.22500000000000001,"5min":0.22,"15min":0.22},"max":{"1min":2.79,"5min":3.6899999999999999,"15min":3.6899999999999999},"last":0.33300000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:17:43 2023","interfaces":[{"interface":"back","average":{"1min":0.45400000000000001,"5min":0.86499999999999999,"15min":1.4870000000000001},"min":{"1min":0.20000000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":1.599,"5min":8.5619999999999994,"15min":12.779999999999999},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.49199999999999999,"5min":0.93899999999999995,"15min":1.4930000000000001},"min":{"1min":0.315,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":1.643,"5min":7.976,"15min":15.198},"last":0.26600000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:17:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6879999999999999,"5min":2.3719999999999999,"15min":1.744},"min":{"1min":0.35099999999999998,"5min":0.31900000000000001,"15min":0.29299999999999998},"max":{"1min":6.3869999999999996,"5min":57.746000000000002,"15min":57.746000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":1.149,"5min":2,"15min":1.6200000000000001},"min":{"1min":0.27600000000000002,"5min":0.25600000000000001,"15min":0.247},"max":{"1min":6.5229999999999997,"5min":57.816000000000003,"15min":57.816000000000003},"last":0.57799999999999996}]},{"osd":5,"last update":"Sat Sep 23 18:17:40 2023","interfaces":[{"interface":"back","average":{"1min":1.47,"5min":1.7869999999999999,"15min":1.2490000000000001},"min":{"1min":0.35899999999999999,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.4480000000000004,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":0.61399999999999999},{"interface":"front","average":{"1min":1.575,"5min":1.6259999999999999,"15min":1.2270000000000001},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":5.9379999999999997,"5min":8.8879999999999999,"15min":8.8879999999999999},"last":0.49299999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:17:03 2023","interfaces":[{"interface":"back","average":{"1min":1.6919999999999999,"5min":1.907,"15min":1.829},"min":{"1min":0.38800000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":6.3140000000000001,"5min":9.1170000000000009,"15min":58.457999999999998},"last":0.56399999999999995},{"interface":"front","average":{"1min":1.3460000000000001,"5min":1.631,"15min":1.5920000000000001},"min":{"1min":0.373,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":3.8330000000000002,"5min":9.1519999999999992,"15min":58.354999999999997},"last":0.47999999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:17:50 2023","interfaces":[{"interface":"back","average":{"1min":1.127,"5min":1.7030000000000001,"15min":1.48},"min":{"1min":0.45000000000000001,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":5.532,"5min":8.5009999999999994,"15min":10.416},"last":0.59599999999999997},{"interface":"front","average":{"1min":0.872,"5min":1.647,"15min":1.726},"min":{"1min":0.371,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":1.895,"5min":10.404,"15min":10.404},"last":0.50700000000000001}]}]},{"osd":2,"up_from":0,"seq":2070174236711,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249341,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":615968,"kb_used_data":7448,"kb_used_omap":3,"kb_used_meta":608508,"kb_avail":93129184,"statfs":{"total":95995035648,"available":95364284416,"internally_reserved":0,"allocated":7626752,"data_stored":4762248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":623112936},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:17:44 2023","interfaces":[{"interface":"back","average":{"1min":0.29299999999999998,"5min":2.4169999999999998,"15min":1.52},"min":{"1min":0.222,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":0.60899999999999999,"5min":195.69800000000001,"15min":195.69800000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.36299999999999999,"5min":2.423,"15min":1.8120000000000001},"min":{"1min":0.248,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":0.877,"5min":194.661,"15min":194.661},"last":0.28999999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:14:39 2023","interfaces":[{"interface":"back","average":{"1min":1.0389999999999999,"5min":1.9470000000000001,"15min":1.5860000000000001},"min":{"1min":0.23799999999999999,"5min":0.215,"15min":0.215},"max":{"1min":7.016,"5min":16.056000000000001,"15min":21.931999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.4650000000000001,"5min":2.2400000000000002,"15min":1.476},"min":{"1min":0.27300000000000002,"5min":0.23999999999999999,"15min":0.23100000000000001},"max":{"1min":8.5700000000000003,"5min":16.587,"15min":16.587},"last":0.39400000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:17:54 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":1.0089999999999999,"15min":1.2010000000000001},"min":{"1min":0.249,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":1.9890000000000001,"5min":6.3849999999999998,"15min":6.3849999999999998},"last":0.439},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.98699999999999999,"15min":1.2290000000000001},"min":{"1min":0.27900000000000003,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":0.54200000000000004,"5min":5.931,"15min":5.931},"last":0.35199999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:17:44 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.383,"15min":1.52},"min":{"1min":0.29499999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":2.5550000000000002,"5min":21.062999999999999,"15min":37.231000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.093,"5min":1.74,"15min":1.722},"min":{"1min":0.317,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":6.6849999999999996,"5min":21.128,"15min":37.381},"last":0.54300000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:17:36 2023","interfaces":[{"interface":"back","average":{"1min":1.5609999999999999,"5min":1.5620000000000001,"15min":1.365},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":8.2959999999999994,"5min":8.2959999999999994,"15min":8.2959999999999994},"last":0.498},{"interface":"front","average":{"1min":0.879,"5min":1.5509999999999999,"15min":1.655},"min":{"1min":0.38100000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.1779999999999999,"5min":8.9969999999999999,"15min":8.9969999999999999},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:17:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4379999999999999,"5min":1.5009999999999999,"15min":1.7070000000000001},"min":{"1min":0.441,"5min":0.38500000000000001,"15min":0.32200000000000001},"max":{"1min":9.8089999999999993,"5min":9.8089999999999993,"15min":115.54000000000001},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.1850000000000001,"5min":1.663,"15min":1.891},"min":{"1min":0.39000000000000001,"5min":0.32900000000000001,"15min":0.30499999999999999},"max":{"1min":4.218,"5min":14.321,"15min":115.691},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:17:44 2023","interfaces":[{"interface":"back","average":{"1min":0.61399999999999999,"5min":1.6679999999999999,"15min":1.663},"min":{"1min":0.39300000000000002,"5min":0.36799999999999999,"15min":0.35799999999999998},"max":{"1min":1.4319999999999999,"5min":9.9760000000000009,"15min":15.972},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.97599999999999998,"5min":1.5469999999999999,"15min":1.494},"min":{"1min":0.47499999999999998,"5min":0.40699999999999997,"15min":0.34899999999999998},"max":{"1min":3.9929999999999999,"5min":9.9109999999999996,"15min":9.9109999999999996},"last":0.56499999999999995}]}]},{"osd":4,"up_from":30,"seq":128849019290,"num_pgs":79,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":551660,"kb_used_data":6628,"kb_used_omap":0,"kb_used_meta":545024,"kb_avail":93193492,"statfs":{"total":95995035648,"available":95430135808,"internally_reserved":0,"allocated":6787072,"data_stored":4948416,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":558104576},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:17:00 2023","interfaces":[{"interface":"back","average":{"1min":1.4810000000000001,"5min":1.202,"15min":1.119},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.4560000000000004,"5min":7.3239999999999998,"15min":13.375999999999999},"last":0.59599999999999997},{"interface":"front","average":{"1min":1.7909999999999999,"5min":1.7769999999999999,"15min":1.4419999999999999},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":10.890000000000001,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.65200000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:15:21 2023","interfaces":[{"interface":"back","average":{"1min":2.3620000000000001,"5min":1.821,"15min":1.5880000000000001},"min":{"1min":0.35699999999999998,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":9.1880000000000006,"5min":9.1880000000000006,"15min":42.929000000000002},"last":0.61099999999999999},{"interface":"front","average":{"1min":2.419,"5min":1.5880000000000001,"15min":1.4990000000000001},"min":{"1min":0.34200000000000003,"5min":0.255,"15min":0.255},"max":{"1min":9.7409999999999997,"5min":9.7409999999999997,"15min":43.494999999999997},"last":0.66300000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:17:54 2023","interfaces":[{"interface":"back","average":{"1min":0.77100000000000002,"5min":1.5389999999999999,"15min":1.774},"min":{"1min":0.376,"5min":0.255,"15min":0.255},"max":{"1min":1.444,"5min":9.9960000000000004,"15min":9.9960000000000004},"last":0.63700000000000001},{"interface":"front","average":{"1min":0.88700000000000001,"5min":1.5680000000000001,"15min":1.6759999999999999},"min":{"1min":0.40300000000000002,"5min":0.253,"15min":0.253},"max":{"1min":2.391,"5min":10.938000000000001,"15min":10.938000000000001},"last":0.57799999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:17:38 2023","interfaces":[{"interface":"back","average":{"1min":1.1699999999999999,"5min":1.589,"15min":1.8700000000000001},"min":{"1min":0.34499999999999997,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":4.327,"5min":7.6619999999999999,"15min":29.154},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.72299999999999998,"5min":1.7809999999999999,"15min":1.8460000000000001},"min":{"1min":0.34799999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":2.3530000000000002,"5min":10.968,"15min":29.216000000000001},"last":0.624}]},{"osd":5,"last update":"Sat Sep 23 18:17:38 2023","interfaces":[{"interface":"back","average":{"1min":3.2829999999999999,"5min":2.8450000000000002,"15min":3.0619999999999998},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":27.675000000000001,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.7599999999999998,"5min":2.2250000000000001,"15min":2.9729999999999999},"min":{"1min":0.35899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":13.938000000000001,"5min":32.640000000000001,"15min":32.640000000000001},"last":0.55800000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:17:02 2023","interfaces":[{"interface":"back","average":{"1min":2.387,"5min":1.8440000000000001,"15min":1.2470000000000001},"min":{"1min":0.30099999999999999,"5min":0.28199999999999997,"15min":0.25},"max":{"1min":18.315000000000001,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.42299999999999999},{"interface":"front","average":{"1min":2.3330000000000002,"5min":2.0960000000000001,"15min":1.381},"min":{"1min":0.311,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":18.431000000000001,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.47199999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:17:00 2023","interfaces":[{"interface":"back","average":{"1min":4.415,"5min":2.7160000000000002,"15min":1.6499999999999999},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":19.298999999999999,"5min":19.298999999999999,"15min":19.298999999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":2.5539999999999998,"5min":2.2679999999999998,"15min":1.5289999999999999},"min":{"1min":0.42499999999999999,"5min":0.35499999999999998,"15min":0.28399999999999997},"max":{"1min":12.800000000000001,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.51300000000000001}]}]},{"osd":5,"up_from":420,"seq":1803886264387,"num_pgs":75,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":278324,"kb_used_data":6572,"kb_used_omap":9,"kb_used_meta":271734,"kb_avail":93466828,"statfs":{"total":95995035648,"available":95710031872,"internally_reserved":0,"allocated":6729728,"data_stored":4891081,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":278256061},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:17:44 2023","interfaces":[{"interface":"back","average":{"1min":1.0249999999999999,"5min":1.173,"15min":0.81799999999999995},"min":{"1min":0.214,"5min":0.214,"15min":0.214},"max":{"1min":5.1740000000000004,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.52400000000000002},{"interface":"front","average":{"1min":1.0580000000000001,"5min":1.5289999999999999,"15min":1.554},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":3.871,"5min":7.5439999999999996,"15min":7.5439999999999996},"last":0.44800000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:14:38 2023","interfaces":[{"interface":"back","average":{"1min":1.2050000000000001,"5min":1.613,"15min":1.681},"min":{"1min":0.34899999999999998,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.8490000000000002,"5min":5.0469999999999997,"15min":5.0469999999999997},"last":0.35299999999999998},{"interface":"front","average":{"1min":1.546,"5min":1.099,"15min":1.0249999999999999},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":8.1609999999999996,"5min":8.1609999999999996,"15min":8.1609999999999996},"last":0.505}]},{"osd":2,"last update":"Sat Sep 23 18:17:51 2023","interfaces":[{"interface":"back","average":{"1min":1.1990000000000001,"5min":1.528,"15min":1.5820000000000001},"min":{"1min":0.26200000000000001,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.2000000000000002,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":0.48499999999999999},{"interface":"front","average":{"1min":1.1899999999999999,"5min":2.0369999999999999,"15min":2.1899999999999999},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":3.8980000000000001,"5min":7.2359999999999998,"15min":7.2359999999999998},"last":0.41199999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:17:44 2023","interfaces":[{"interface":"back","average":{"1min":1.117,"5min":1.518,"15min":1.1399999999999999},"min":{"1min":0.26200000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.2249999999999996,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":0.57299999999999995},{"interface":"front","average":{"1min":1.47,"5min":1.581,"15min":1.5329999999999999},"min":{"1min":0.33600000000000002,"5min":0.313,"15min":0.313},"max":{"1min":5.6929999999999996,"5min":7.5179999999999998,"15min":7.5179999999999998},"last":0.47499999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:17:44 2023","interfaces":[{"interface":"back","average":{"1min":1.3160000000000001,"5min":2.2109999999999999,"15min":2.0600000000000001},"min":{"1min":0.28199999999999997,"5min":0.251,"15min":0.251},"max":{"1min":7.4870000000000001,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.36499999999999999},{"interface":"front","average":{"1min":1.8460000000000001,"5min":1.883,"15min":1.5489999999999999},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":9.0690000000000008,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:17:44 2023","interfaces":[{"interface":"back","average":{"1min":1.321,"5min":1.7869999999999999,"15min":1.163},"min":{"1min":0.28000000000000003,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.0949999999999998,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.38600000000000001},{"interface":"front","average":{"1min":2.3500000000000001,"5min":2.044,"15min":1.635},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":12.504,"5min":12.504,"15min":12.504},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:17:44 2023","interfaces":[{"interface":"back","average":{"1min":1.4350000000000001,"5min":2.3650000000000002,"15min":1.6970000000000001},"min":{"1min":0.32500000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.8810000000000002,"5min":13.506,"15min":13.506},"last":0.46200000000000002},{"interface":"front","average":{"1min":1.611,"5min":2.0430000000000001,"15min":1.5389999999999999},"min":{"1min":0.35299999999999998,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":9.0210000000000008,"5min":13.138,"15min":13.138},"last":0.55200000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5380,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:18:05.443 INFO:tasks.cephadm.ceph_manager.ceph:increase pool size by 10 2023-09-23T18:18:05.444 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pg_num 51 2023-09-23T18:18:05.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:05.763 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:05.764 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:05.865 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:05.915 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:05.948 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 4 16 11917 11901 185.891 177.438 0.00598939 0.00537276 2023-09-23T18:18:05.965 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:05.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:05.971 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:06.072 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:06.135 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:06.314 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:06.453 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:18:06.739 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:06.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:06 smithi099 ceph-mon[167076]: 1.f scrub starts 2023-09-23T18:18:06.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:06 smithi099 ceph-mon[167076]: 1.f scrub ok 2023-09-23T18:18:06.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:06 smithi099 ceph-mon[167076]: from='client.37029 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:18:06.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:06 smithi099 ceph-mon[167076]: pgmap v1361: 119 pgs: 119 active+clean; 25 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 3.1 MiB/s wr, 49 op/s 2023-09-23T18:18:06.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.12", "id": [5, 3]}]: dispatch 2023-09-23T18:18:06.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 4]}]: dispatch 2023-09-23T18:18:06.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 0]}]: dispatch 2023-09-23T18:18:06.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]: dispatch 2023-09-23T18:18:06.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:06.745 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:06.746 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:06.746 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:06.847 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:06.898 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[179668]: 1.f scrub starts 2023-09-23T18:18:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[179668]: 1.f scrub ok 2023-09-23T18:18:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[179668]: from='client.37029 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:18:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[179668]: pgmap v1361: 119 pgs: 119 active+clean; 25 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 3.1 MiB/s wr, 49 op/s 2023-09-23T18:18:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.12", "id": [5, 3]}]: dispatch 2023-09-23T18:18:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 4]}]: dispatch 2023-09-23T18:18:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 0]}]: dispatch 2023-09-23T18:18:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]: dispatch 2023-09-23T18:18:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:06.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[184174]: 1.f scrub starts 2023-09-23T18:18:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[184174]: 1.f scrub ok 2023-09-23T18:18:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[184174]: from='client.37029 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:18:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[184174]: pgmap v1361: 119 pgs: 119 active+clean; 25 MiB data, 3.0 GiB used, 533 GiB / 536 GiB avail; 3.1 MiB/s wr, 49 op/s 2023-09-23T18:18:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.12", "id": [5, 3]}]: dispatch 2023-09-23T18:18:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 4]}]: dispatch 2023-09-23T18:18:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 0]}]: dispatch 2023-09-23T18:18:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]: dispatch 2023-09-23T18:18:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:06.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:06.948 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 5 16 14219 14203 177.487 143.875 0.00446986 0.00562846 2023-09-23T18:18:06.965 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:06.971 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:06.971 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:07.072 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:07.128 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:07.188 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:07.191 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:07.193 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:07.294 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:07.305 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:07.345 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:07.399 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:07.404 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:07.404 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:07.505 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:07.523 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:18:07.565 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:07.625 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:07.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:07.631 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:07.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: 1.0 scrub starts 2023-09-23T18:18:07.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: 1.0 scrub ok 2023-09-23T18:18:07.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: failed to encode map e579 with expected crc 2023-09-23T18:18:07.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.12", "id": [5, 3]}]': finished 2023-09-23T18:18:07.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 4]}]': finished 2023-09-23T18:18:07.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 0]}]': finished 2023-09-23T18:18:07.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]': finished 2023-09-23T18:18:07.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: osdmap e579: 8 total, 8 up, 6 in 2023-09-23T18:18:07.631 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: failed to encode map e579 with expected crc 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1328680028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: 1.0 scrub starts 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: 1.0 scrub ok 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: failed to encode map e579 with expected crc 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.12", "id": [5, 3]}]': finished 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 4]}]': finished 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 0]}]': finished 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]': finished 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: osdmap e579: 8 total, 8 up, 6 in 2023-09-23T18:18:07.632 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: failed to encode map e579 with expected crc 2023-09-23T18:18:07.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1328680028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:07.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:07.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:07.731 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:07.795 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:07.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: 1.0 scrub starts 2023-09-23T18:18:07.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: 1.0 scrub ok 2023-09-23T18:18:07.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: failed to encode map e579 with expected crc 2023-09-23T18:18:07.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "1.12", "id": [5, 3]}]': finished 2023-09-23T18:18:07.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.a", "id": [1, 4]}]': finished 2023-09-23T18:18:07.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.14", "id": [3, 0]}]': finished 2023-09-23T18:18:07.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.18", "id": [1, 3]}]': finished 2023-09-23T18:18:07.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: osdmap e579: 8 total, 8 up, 6 in 2023-09-23T18:18:07.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: failed to encode map e579 with expected crc 2023-09-23T18:18:07.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1328680028' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:07.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:07.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:07.949 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 6 16 16559 16543 172.28 146.25 0.00761884 0.00579774 2023-09-23T18:18:08.019 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:18:08.019 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:18:08.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:18:08.019 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108071 7.21 7:85ca81b0:::benchmark_data_smithi055_311528_object37108:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 568.46727309000005, 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9624768260000001, 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:18:08.020 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108071 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544625+0000", 2023-09-23T18:18:08.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544624+0000", 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544628+0000", 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.698e-06 2023-09-23T18:18:08.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544634+0000", 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3849999999999999e-06 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544638+0000", 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7560000000000001e-06 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581968+0000", 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037330020999999998 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581975+0000", 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1470000000000004e-06 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487605+0000", 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9056295090000002 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487615+0000", 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0036e-05 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502365+0000", 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0147500899999999 2023-09-23T18:18:08.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502491+0000", 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125716 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.502677+0000", 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018593700000000001 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504929+0000", 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022522219999999999 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504942+0000", 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2636999999999999e-05 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506803+0000", 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0018613709999999999 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506851+0000", 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8578000000000001e-05 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507102+0000", 2023-09-23T18:18:08.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00025014900000000002 2023-09-23T18:18:08.029 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.029 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.029 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.029 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108080 7.21 7:85d5fb8e:::benchmark_data_smithi055_311528_object37117:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:18:08.029 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:18:08.029 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 568.46724162299995, 2023-09-23T18:18:08.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.962840377, 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108080 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544656+0000", 2023-09-23T18:18:08.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544658+0000", 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9589999999999998e-06 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544663+0000", 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8069999999999999e-06 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.033 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.544667+0000", 2023-09-23T18:18:08.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1790000000000001e-06 2023-09-23T18:18:08.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.033 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.033 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.033 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581992+0000", 2023-09-23T18:18:08.033 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.037325061999999999 2023-09-23T18:18:08.033 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.581997+0000", 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5480000000000003e-06 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487637+0000", 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9056401429999998 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.034 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.035 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487642+0000", 2023-09-23T18:18:08.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7459999999999997e-06 2023-09-23T18:18:08.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.035 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.035 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.035 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.503810+0000", 2023-09-23T18:18:08.035 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.01616774 2023-09-23T18:18:08.035 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.035 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.035 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.503955+0000", 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000145291 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504156+0000", 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020058200000000001 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507170+0000", 2023-09-23T18:18:08.036 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003013511 2023-09-23T18:18:08.037 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.037 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.037 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.037 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507182+0000", 2023-09-23T18:18:08.037 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.218e-05 2023-09-23T18:18:08.037 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.037 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.037 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.037 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507234+0000", 2023-09-23T18:18:08.037 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2349000000000003e-05 2023-09-23T18:18:08.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.038 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.038 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507288+0000", 2023-09-23T18:18:08.038 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4307e-05 2023-09-23T18:18:08.038 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.038 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.038 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.038 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507497+0000", 2023-09-23T18:18:08.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000208233 2023-09-23T18:18:08.039 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.039 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.039 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.039 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.039 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.039 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108094 7.1 7:819d9c99:::benchmark_data_smithi055_311528_object37131:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e340)", 2023-09-23T18:18:08.039 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:18:08.039 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 568.24388214099997, 2023-09-23T18:18:08.039 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7316361810000001, 2023-09-23T18:18:08.040 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.040 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.040 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.040 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:18:08.040 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:18:08.040 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108094 2023-09-23T18:18:08.040 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.040 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.040 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768016+0000", 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.041 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.042 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768015+0000", 2023-09-23T18:18:08.042 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:18:08.042 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.042 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.042 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.042 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768019+0000", 2023-09-23T18:18:08.042 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3989999999999997e-06 2023-09-23T18:18:08.042 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.043 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.043 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.043 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768032+0000", 2023-09-23T18:18:08.043 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2707e-05 2023-09-23T18:18:08.043 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.043 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.043 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.043 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768037+0000", 2023-09-23T18:18:08.043 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2490000000000001e-06 2023-09-23T18:18:08.043 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.044 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.044 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.044 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768081+0000", 2023-09-23T18:18:08.044 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3421000000000003e-05 2023-09-23T18:18:08.044 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.044 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.044 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.044 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:39.768093+0000", 2023-09-23T18:18:08.044 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1989e-05 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.479295+0000", 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.71120236800000003 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.479313+0000", 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.8434e-05 2023-09-23T18:18:08.045 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492409+0000", 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.013095748 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492585+0000", 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000176007 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.046 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:18:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.492874+0000", 2023-09-23T18:18:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00028859600000000001 2023-09-23T18:18:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.497854+0000", 2023-09-23T18:18:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049804380000000002 2023-09-23T18:18:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.047 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.048 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499314+0000", 2023-09-23T18:18:08.048 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014593449999999999 2023-09-23T18:18:08.048 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.048 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.048 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.048 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499349+0000", 2023-09-23T18:18:08.048 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5802e-05 2023-09-23T18:18:08.048 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.049 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.049 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.049 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499422+0000", 2023-09-23T18:18:08.049 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2278000000000002e-05 2023-09-23T18:18:08.049 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.049 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.049 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.049 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.499652+0000", 2023-09-23T18:18:08.049 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00023028400000000001 2023-09-23T18:18:08.050 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.050 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.050 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.050 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.050 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.050 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108097 7.21 7:87bce1cb:::benchmark_data_smithi055_311528_object37134:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:18:08.050 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:18:08.050 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 567.51464114999999, 2023-09-23T18:18:08.051 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0105963070000001, 2023-09-23T18:18:08.051 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.051 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.051 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.051 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:18:08.051 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:18:08.051 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108097 2023-09-23T18:18:08.051 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497257+0000", 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.052 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497255+0000", 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497262+0000", 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.029e-06 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.053 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.054 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497275+0000", 2023-09-23T18:18:08.054 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3349e-05 2023-09-23T18:18:08.054 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.054 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.054 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.054 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497284+0000", 2023-09-23T18:18:08.054 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3639999999999999e-06 2023-09-23T18:18:08.054 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.054 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.054 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.055 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497329+0000", 2023-09-23T18:18:08.055 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5105000000000003e-05 2023-09-23T18:18:08.055 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.055 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.055 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.055 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.497335+0000", 2023-09-23T18:18:08.055 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8989999999999998e-06 2023-09-23T18:18:08.055 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.055 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.056 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.056 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487661+0000", 2023-09-23T18:18:08.056 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99032663499999996 2023-09-23T18:18:08.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.056 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.056 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.056 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487667+0000", 2023-09-23T18:18:08.056 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.446e-06 2023-09-23T18:18:08.056 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.056 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.504977+0000", 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.017310079 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505119+0000", 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000142707 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.057 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:18:08.058 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505315+0000", 2023-09-23T18:18:08.058 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019584499999999999 2023-09-23T18:18:08.058 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.058 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.058 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.058 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507193+0000", 2023-09-23T18:18:08.058 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001877213 2023-09-23T18:18:08.058 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.058 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.058 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507527+0000", 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00033465300000000002 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507614+0000", 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7287000000000006e-05 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507647+0000", 2023-09-23T18:18:08.059 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2481999999999998e-05 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507853+0000", 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000206092 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.060 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45557.0:108108 7.21 7:860e36b6:::benchmark_data_smithi055_311528_object37145:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e341)", 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 567.46364857100002, 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.963000675, 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45557", 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/241824703", 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 108108 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.061 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548249+0000", 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.062 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548248+0000", 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548257+0000", 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1549999999999996e-06 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.063 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548278+0000", 2023-09-23T18:18:08.064 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1229e-05 2023-09-23T18:18:08.064 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.064 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.064 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.064 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548286+0000", 2023-09-23T18:18:08.064 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.0150000000000003e-06 2023-09-23T18:18:08.064 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.064 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.064 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548337+0000", 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1131999999999998e-05 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:40.548346+0000", 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8070000000000006e-06 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.065 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487685+0000", 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.939339378 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:41.487691+0000", 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7100000000000004e-06 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.505973+0000", 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018282141 2023-09-23T18:18:08.066 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506088+0000", 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011443299999999999 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.506199+0000", 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011132 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.067 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507536+0000", 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0013369 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.507578+0000", 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1971000000000002e-05 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.068 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.510980+0000", 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034023870000000002 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.511029+0000", 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9064999999999999e-05 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:08:42.511250+0000", 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022076600000000001 2023-09-23T18:18:08.069 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:129162 8.2 8:454f52aa:::benchmark_data_smithi055_346788_object18570:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:28.516171+0000", 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 399.49572703600001, 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.068282459, 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:18:08.070 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129162 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516171+0000", 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.071 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516171+0000", 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516170+0000", 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.072 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516174+0000", 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3680000000000004e-06 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516183+0000", 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9120000000000004e-06 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516187+0000", 2023-09-23T18:18:08.073 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2209999999999999e-06 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516232+0000", 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4530000000000002e-05 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516388+0000", 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000156572 2023-09-23T18:18:08.074 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.516568+0000", 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000179927 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.523118+0000", 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006549933 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.075 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.528581+0000", 2023-09-23T18:18:08.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054625460000000004 2023-09-23T18:18:08.076 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.076 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.076 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.076 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.546371+0000", 2023-09-23T18:18:08.076 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.017789800000000001 2023-09-23T18:18:08.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.546381+0000", 2023-09-23T18:18:08.077 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0311e-05 2023-09-23T18:18:08.077 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.077 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.077 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.077 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.584362+0000", 2023-09-23T18:18:08.078 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.037981313 2023-09-23T18:18:08.078 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.078 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.078 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.078 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.584453+0000", 2023-09-23T18:18:08.078 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1094999999999994e-05 2023-09-23T18:18:08.078 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.078 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.079 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.079 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.079 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.079 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:129175 8.2 8:4646f8c9:::benchmark_data_smithi055_346788_object18583:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T18:18:08.079 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:28.554311+0000", 2023-09-23T18:18:08.079 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 399.45758734499998, 2023-09-23T18:18:08.079 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.041236976, 2023-09-23T18:18:08.079 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.079 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.080 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.080 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:18:08.080 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:18:08.080 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129175 2023-09-23T18:18:08.080 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.080 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.080 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.080 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.080 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554311+0000", 2023-09-23T18:18:08.081 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.081 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.081 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.081 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.081 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554311+0000", 2023-09-23T18:18:08.081 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.081 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.081 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.081 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.082 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554309+0000", 2023-09-23T18:18:08.082 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:18:08.082 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.082 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.082 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.082 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554315+0000", 2023-09-23T18:18:08.082 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1929999999999998e-06 2023-09-23T18:18:08.082 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.083 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.083 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.083 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554328+0000", 2023-09-23T18:18:08.083 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2975e-05 2023-09-23T18:18:08.083 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.083 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.083 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.083 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554334+0000", 2023-09-23T18:18:08.083 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3580000000000002e-06 2023-09-23T18:18:08.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.084 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.084 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.084 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554380+0000", 2023-09-23T18:18:08.084 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6189000000000001e-05 2023-09-23T18:18:08.084 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.084 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.084 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.084 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.554387+0000", 2023-09-23T18:18:08.085 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8369999999999999e-06 2023-09-23T18:18:08.085 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.085 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.085 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.085 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.585150+0000", 2023-09-23T18:18:08.085 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0307628820000001 2023-09-23T18:18:08.085 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.085 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.086 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.086 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.585231+0000", 2023-09-23T18:18:08.086 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.1168000000000006e-05 2023-09-23T18:18:08.086 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.086 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.086 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:18:08.086 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.585373+0000", 2023-09-23T18:18:08.087 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014207099999999999 2023-09-23T18:18:08.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.087 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.087 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.087 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.589993+0000", 2023-09-23T18:18:08.087 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046203149999999998 2023-09-23T18:18:08.087 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.087 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.087 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.088 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.590043+0000", 2023-09-23T18:18:08.088 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0148000000000002e-05 2023-09-23T18:18:08.088 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.088 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.088 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.088 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.594330+0000", 2023-09-23T18:18:08.088 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0042865430000000003 2023-09-23T18:18:08.088 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.088 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.088 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.594380+0000", 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9521000000000002e-05 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.595548+0000", 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001168012 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.089 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:129177 8.2 8:4d02a5a8:::benchmark_data_smithi055_346788_object18585:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e397)", 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:11:28.562962+0000", 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 399.448935995, 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.033361969, 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 129177 2023-09-23T18:18:08.090 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562962+0000", 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562962+0000", 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.091 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562960+0000", 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562967+0000", 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8240000000000002e-06 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.092 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562981+0000", 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4100999999999999e-05 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.562986+0000", 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2780000000000001e-06 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.093 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.563035+0000", 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8353000000000003e-05 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:28.563046+0000", 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1547e-05 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.587649+0000", 2023-09-23T18:18:08.094 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0246025240000001 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.587767+0000", 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011790000000000001 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,6", 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.587856+0000", 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9696000000000003e-05 2023-09-23T18:18:08.095 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.590055+0000", 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021986890000000002 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.590123+0000", 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8114000000000005e-05 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.096 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.595596+0000", 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054732649999999997 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.595635+0000", 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8549e-05 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:11:29.596324+0000", 2023-09-23T18:18:08.097 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000688907 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177038 8.2 8:4ced54d7:::benchmark_data_smithi055_346788_object66446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 336.73676965599998, 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1583805069999999, 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.098 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177038 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.099 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275127+0000", 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.100 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275133+0000", 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5910000000000001e-06 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275146+0000", 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2302e-05 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.101 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275163+0000", 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7689e-05 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.282482+0000", 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0073186709999999997 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.282498+0000", 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5841999999999998e-05 2023-09-23T18:18:08.102 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430240+0000", 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1477423680000001 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430343+0000", 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102289 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.103 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5,6", 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430425+0000", 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2150000000000005e-05 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.432859+0000", 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024338519999999998 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.432958+0000", 2023-09-23T18:18:08.104 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9440999999999999e-05 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433088+0000", 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013031100000000001 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433396+0000", 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00030761699999999999 2023-09-23T18:18:08.105 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433437+0000", 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1242999999999998e-05 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433509+0000", 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1559999999999999e-05 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.106 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213401 8.e 8:7ecd3558:::benchmark_data_smithi055_346788_object102809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 287.768279812, 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0290741050000001, 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:18:08.107 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213401 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.108 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243620+0000", 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9649999999999998e-06 2023-09-23T18:18:08.109 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.110 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.110 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.110 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243623+0000", 2023-09-23T18:18:08.110 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6179999999999999e-06 2023-09-23T18:18:08.110 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.110 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.110 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.110 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243626+0000", 2023-09-23T18:18:08.110 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.294e-06 2023-09-23T18:18:08.110 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.111 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.111 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.111 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243677+0000", 2023-09-23T18:18:08.111 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1668000000000001e-05 2023-09-23T18:18:08.111 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.111 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.111 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.111 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243686+0000", 2023-09-23T18:18:08.112 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7730000000000005e-06 2023-09-23T18:18:08.112 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.112 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.112 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.112 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.235935+0000", 2023-09-23T18:18:08.112 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99224838100000001 2023-09-23T18:18:08.112 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.112 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.112 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.112 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.236043+0000", 2023-09-23T18:18:08.113 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108855 2023-09-23T18:18:08.113 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.113 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.113 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:18:08.113 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.236169+0000", 2023-09-23T18:18:08.113 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012573700000000001 2023-09-23T18:18:08.113 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.113 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.114 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.114 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.253075+0000", 2023-09-23T18:18:08.114 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016906304 2023-09-23T18:18:08.114 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.114 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.114 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.114 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272511+0000", 2023-09-23T18:18:08.114 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019435859 2023-09-23T18:18:08.114 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.115 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.115 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.115 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272626+0000", 2023-09-23T18:18:08.115 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115215 2023-09-23T18:18:08.115 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.115 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.115 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.116 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272667+0000", 2023-09-23T18:18:08.116 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0590000000000003e-05 2023-09-23T18:18:08.116 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.116 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.116 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.116 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272692+0000", 2023-09-23T18:18:08.116 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5111999999999999e-05 2023-09-23T18:18:08.116 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.116 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.117 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.117 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.117 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.117 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121636 9.f 9:f1d044dd:::benchmark_data_smithi055_386467_object16976:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:18:08.117 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.695760+0000", 2023-09-23T18:18:08.117 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 151.316138142, 2023-09-23T18:18:08.117 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0409455569999999, 2023-09-23T18:18:08.117 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.117 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.118 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.118 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:18:08.118 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:18:08.118 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121636 2023-09-23T18:18:08.118 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.118 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.118 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.118 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.119 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695760+0000", 2023-09-23T18:18:08.119 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.119 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.119 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.119 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.119 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695760+0000", 2023-09-23T18:18:08.119 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.119 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.120 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.120 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.120 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695759+0000", 2023-09-23T18:18:08.120 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:08.120 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.120 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.120 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.120 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695762+0000", 2023-09-23T18:18:08.120 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4040000000000002e-06 2023-09-23T18:18:08.121 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.121 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.121 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.121 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695768+0000", 2023-09-23T18:18:08.121 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1099999999999999e-06 2023-09-23T18:18:08.121 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.121 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.121 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.121 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695772+0000", 2023-09-23T18:18:08.121 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0579999999999999e-06 2023-09-23T18:18:08.122 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.122 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.122 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.122 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.698805+0000", 2023-09-23T18:18:08.122 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030330489999999999 2023-09-23T18:18:08.122 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.122 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.122 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.122 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.698810+0000", 2023-09-23T18:18:08.123 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5439999999999998e-06 2023-09-23T18:18:08.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.123 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.123 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.123 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.717639+0000", 2023-09-23T18:18:08.123 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018828426 2023-09-23T18:18:08.123 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.124 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.124 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.124 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.718020+0000", 2023-09-23T18:18:08.124 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00038104099999999999 2023-09-23T18:18:08.124 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.124 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.124 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:18:08.124 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.718155+0000", 2023-09-23T18:18:08.124 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000134618 2023-09-23T18:18:08.124 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.725686+0000", 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075315429999999999 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.735765+0000", 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010079210999999999 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.125 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.736527+0000", 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00076177799999999996 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.736593+0000", 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6124000000000006e-05 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.736705+0000", 2023-09-23T18:18:08.126 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011207 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:122388 9.f 9:fdc2bcfb:::benchmark_data_smithi055_386467_object17728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 139.750719605, 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0337659729999999, 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.127 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122388 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.128 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261180+0000", 2023-09-23T18:18:08.129 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.057e-06 2023-09-23T18:18:08.130 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.130 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.130 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.130 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261187+0000", 2023-09-23T18:18:08.130 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2559999999999996e-06 2023-09-23T18:18:08.130 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.130 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.130 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.130 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261191+0000", 2023-09-23T18:18:08.130 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4089999999999999e-06 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261250+0000", 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9678e-05 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261258+0000", 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3209999999999999e-06 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.131 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.278415+0000", 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01715761 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.278423+0000", 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8089999999999993e-06 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.132 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.287981+0000", 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0095577870000001 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.288127+0000", 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014562099999999999 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.288257+0000", 2023-09-23T18:18:08.133 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000129982 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294755+0000", 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064986159999999996 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294764+0000", 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4689999999999997e-06 2023-09-23T18:18:08.134 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294789+0000", 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5245e-05 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294838+0000", 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9027999999999999e-05 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.135 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294944+0000", 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106378 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:122391 9.f 9:fd5978e6:::benchmark_data_smithi055_386467_object17731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:48.272754+0000", 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 139.739143627, 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0225196940000001, 2023-09-23T18:18:08.136 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122391 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272754+0000", 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.137 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272754+0000", 2023-09-23T18:18:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272753+0000", 2023-09-23T18:18:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:18:08.138 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272758+0000", 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9169999999999997e-06 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.275901+0000", 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031430659999999999 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.139 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.275910+0000", 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0569999999999999e-06 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.279182+0000", 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032713920000000001 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.279188+0000", 2023-09-23T18:18:08.140 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.302e-06 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.290722+0000", 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.011533781 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.290834+0000", 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011197299999999999 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.141 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.290966+0000", 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132531 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294997+0000", 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040308899999999996 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.142 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295070+0000", 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3144000000000003e-05 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295128+0000", 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8236000000000003e-05 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295169+0000", 2023-09-23T18:18:08.143 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0176000000000003e-05 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295274+0000", 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010541 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199300 9.c 9:3f680051:::benchmark_data_smithi055_386467_object94640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:18:08.144 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.729844+0000", 2023-09-23T18:18:08.145 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 48.282054299000002, 2023-09-23T18:18:08.145 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.107413183, 2023-09-23T18:18:08.145 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.145 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:18:08.145 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.145 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:18:08.145 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:18:08.145 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199300 2023-09-23T18:18:08.145 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.145 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.146 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.146 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.146 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729844+0000", 2023-09-23T18:18:08.146 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.146 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.146 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.146 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.146 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729844+0000", 2023-09-23T18:18:08.146 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.147 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.147 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.147 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729842+0000", 2023-09-23T18:18:08.147 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:18:08.147 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.147 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.147 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.148 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729848+0000", 2023-09-23T18:18:08.148 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4229999999999996e-06 2023-09-23T18:18:08.148 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.148 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.148 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.148 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729858+0000", 2023-09-23T18:18:08.148 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0064999999999999e-05 2023-09-23T18:18:08.148 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.148 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729864+0000", 2023-09-23T18:18:08.149 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2550000000000003e-06 2023-09-23T18:18:08.149 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.149 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.149 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.149 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.732111+0000", 2023-09-23T18:18:08.149 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022468330000000002 2023-09-23T18:18:08.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.150 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.150 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.150 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.732247+0000", 2023-09-23T18:18:08.150 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013641200000000001 2023-09-23T18:18:08.150 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.150 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.150 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T18:18:08.150 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.732401+0000", 2023-09-23T18:18:08.151 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015389900000000001 2023-09-23T18:18:08.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.151 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.151 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.151 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.764773+0000", 2023-09-23T18:18:08.151 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032371687000000003 2023-09-23T18:18:08.151 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.151 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.152 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.152 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.764782+0000", 2023-09-23T18:18:08.152 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7350000000000002e-06 2023-09-23T18:18:08.152 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.152 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.152 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.152 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.820636+0000", 2023-09-23T18:18:08.152 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0558545260000001 2023-09-23T18:18:08.152 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.153 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.153 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:18:08.153 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.820762+0000", 2023-09-23T18:18:08.153 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125988 2023-09-23T18:18:08.153 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.153 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.153 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.153 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.837194+0000", 2023-09-23T18:18:08.153 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016432303999999998 2023-09-23T18:18:08.154 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.154 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.154 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.154 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.837257+0000", 2023-09-23T18:18:08.154 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2335000000000005e-05 2023-09-23T18:18:08.154 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.154 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.154 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.154 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.154 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.155 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199301 9.c 9:3dcb5cd4:::benchmark_data_smithi055_386467_object94641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:18:08.155 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.729873+0000", 2023-09-23T18:18:08.155 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 48.282024440000001, 2023-09-23T18:18:08.155 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.107041538, 2023-09-23T18:18:08.155 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.155 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.155 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.155 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:18:08.155 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:18:08.156 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199301 2023-09-23T18:18:08.156 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.156 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.156 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.156 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.156 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729873+0000", 2023-09-23T18:18:08.156 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.156 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.156 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.156 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.157 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729873+0000", 2023-09-23T18:18:08.157 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.157 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.157 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.157 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.157 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729873+0000", 2023-09-23T18:18:08.157 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:08.157 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.158 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.158 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.158 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729876+0000", 2023-09-23T18:18:08.158 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.813e-06 2023-09-23T18:18:08.158 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.158 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.158 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.158 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729882+0000", 2023-09-23T18:18:08.159 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7059999999999998e-06 2023-09-23T18:18:08.159 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.159 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.159 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.159 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729887+0000", 2023-09-23T18:18:08.159 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2910000000000001e-06 2023-09-23T18:18:08.159 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.159 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.764844+0000", 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.034956813000000003 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.764847+0000", 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8670000000000004e-06 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.160 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.820787+0000", 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.055939384 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.820905+0000", 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117979 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,6", 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.821043+0000", 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013790800000000001 2023-09-23T18:18:08.161 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.835902+0000", 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014859659000000001 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.836434+0000", 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00053201699999999997 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.162 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.163 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.163 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.836466+0000", 2023-09-23T18:18:08.163 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1909999999999998e-05 2023-09-23T18:18:08.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.163 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.163 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.163 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.836520+0000", 2023-09-23T18:18:08.163 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3516000000000001e-05 2023-09-23T18:18:08.163 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.163 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.836915+0000", 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00039519400000000002 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:201222 9.4 9:27ad1e07:::benchmark_data_smithi055_386467_object96562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:25.790582+0000", 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 42.221315408999999, 2023-09-23T18:18:08.164 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0766256249999999, 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201222 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.165 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790582+0000", 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790582+0000", 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790581+0000", 2023-09-23T18:18:08.166 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790589+0000", 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6750000000000002e-06 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790601+0000", 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2081e-05 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.167 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790608+0000", 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9450000000000004e-06 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790662+0000", 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4673000000000003e-05 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.168 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790781+0000", 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000118943 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790909+0000", 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012749300000000001 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.794044+0000", 2023-09-23T18:18:08.169 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031351450000000002 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.802883+0000", 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0088395729999999999 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.802892+0000", 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5140000000000001e-06 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.170 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.848461+0000", 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.045569451 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.848575+0000", 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113906 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.171 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.867156+0000", 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018581033 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.867208+0000", 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1820999999999997e-05 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.172 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:201232 9.4 9:23829d9b:::benchmark_data_smithi055_386467_object96572:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:25.806549+0000", 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 42.205348555999997, 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.05999021, 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201232 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.173 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806549+0000", 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806549+0000", 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.174 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806549+0000", 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806552+0000", 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9620000000000002e-06 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.175 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806567+0000", 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5602000000000001e-05 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806572+0000", 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0170000000000004e-06 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806696+0000", 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123621 2023-09-23T18:18:08.176 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806703+0000", 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0559999999999999e-06 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.848619+0000", 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0419157489999999 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.177 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.848707+0000", 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7841000000000001e-05 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.848795+0000", 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8675000000000001e-05 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.178 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.859785+0000", 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010989103 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.865959+0000", 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0061744790000000001 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866346+0000", 2023-09-23T18:18:08.179 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00038720099999999999 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866412+0000", 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6030000000000003e-05 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866540+0000", 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012732299999999999 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.180 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.181 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.181 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.181 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.181 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:201265 9.4 9:295971ad:::benchmark_data_smithi055_386467_object96605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T18:18:08.181 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:25.842742+0000", 2023-09-23T18:18:08.181 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 42.169155480000001, 2023-09-23T18:18:08.181 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0239629960000001, 2023-09-23T18:18:08.181 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.181 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.182 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.182 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:18:08.182 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:18:08.182 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201265 2023-09-23T18:18:08.182 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.182 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.182 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.182 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842742+0000", 2023-09-23T18:18:08.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.183 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842742+0000", 2023-09-23T18:18:08.183 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.183 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.183 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.183 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.184 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842741+0000", 2023-09-23T18:18:08.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:18:08.184 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.184 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.184 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.184 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842748+0000", 2023-09-23T18:18:08.184 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5810000000000002e-06 2023-09-23T18:18:08.184 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842760+0000", 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1878999999999999e-05 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842767+0000", 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9820000000000002e-06 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.185 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842814+0000", 2023-09-23T18:18:08.186 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7670000000000003e-05 2023-09-23T18:18:08.186 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.186 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.186 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.186 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842821+0000", 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9979999999999997e-06 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.850540+0000", 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0077192370000001 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.850617+0000", 2023-09-23T18:18:08.187 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6699000000000005e-05 2023-09-23T18:18:08.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.188 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:18:08.188 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.850688+0000", 2023-09-23T18:18:08.188 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0770000000000002e-05 2023-09-23T18:18:08.188 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.188 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.188 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.862765+0000", 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012077044 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866147+0000", 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033823690000000001 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.189 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866574+0000", 2023-09-23T18:18:08.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00042635999999999998 2023-09-23T18:18:08.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.190 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.190 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.190 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866606+0000", 2023-09-23T18:18:08.190 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2583000000000003e-05 2023-09-23T18:18:08.190 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.191 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.191 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866705+0000", 2023-09-23T18:18:08.191 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9137999999999998e-05 2023-09-23T18:18:08.191 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.191 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.191 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.191 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.191 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.192 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:201274 9.4 9:29d51ac9:::benchmark_data_smithi055_386467_object96614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T18:18:08.192 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:25.853318+0000", 2023-09-23T18:18:08.192 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 42.158579664000001, 2023-09-23T18:18:08.192 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.014229311, 2023-09-23T18:18:08.192 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.192 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.192 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.192 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:18:08.192 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:18:08.193 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201274 2023-09-23T18:18:08.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.193 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.193 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.193 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.193 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853318+0000", 2023-09-23T18:18:08.193 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.193 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853318+0000", 2023-09-23T18:18:08.194 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.194 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.194 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.194 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853318+0000", 2023-09-23T18:18:08.194 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:08.194 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853321+0000", 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3160000000000001e-06 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853328+0000", 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9179999999999997e-06 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.195 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853333+0000", 2023-09-23T18:18:08.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6149999999999999e-06 2023-09-23T18:18:08.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.196 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.196 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853384+0000", 2023-09-23T18:18:08.196 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1100999999999997e-05 2023-09-23T18:18:08.196 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.196 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.197 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.197 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853392+0000", 2023-09-23T18:18:08.197 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6629999999999999e-06 2023-09-23T18:18:08.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.197 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.197 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.197 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.851766+0000", 2023-09-23T18:18:08.197 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99837366299999997 2023-09-23T18:18:08.197 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.851835+0000", 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9090000000000004e-05 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.851910+0000", 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4946999999999999e-05 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.198 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.862769+0000", 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010859291 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866196+0000", 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034264439999999998 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.199 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.867401+0000", 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012055989999999999 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.867456+0000", 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4262000000000003e-05 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.867548+0000", 2023-09-23T18:18:08.200 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1908000000000002e-05 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:201278 9.4 9:26c0438d:::benchmark_data_smithi055_386467_object96618:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:25.859092+0000", 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 42.152805911999998, 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0122590970000001, 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:08.201 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201278 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859092+0000", 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.202 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859092+0000", 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859091+0000", 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859097+0000", 2023-09-23T18:18:08.203 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4430000000000002e-06 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859111+0000", 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4606e-05 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859118+0000", 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8539999999999999e-06 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.204 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859164+0000", 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6180000000000002e-05 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859170+0000", 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1410000000000002e-06 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.854090+0000", 2023-09-23T18:18:08.205 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99491981600000001 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.854152+0000", 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1598000000000002e-05 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.854226+0000", 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4345999999999996e-05 2023-09-23T18:18:08.206 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.862777+0000", 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085508010000000002 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866272+0000", 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034955300000000002 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.207 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.871174+0000", 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049011030000000004 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.871234+0000", 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0041000000000003e-05 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.871351+0000", 2023-09-23T18:18:08.208 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117516 2023-09-23T18:18:08.209 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.209 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.209 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.209 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:08.209 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:08.209 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:18:08.210 INFO:teuthology.orchestra.run.smithi055.stderr:Error EINVAL: specified pg_num 51 < pg_num_max 32 2023-09-23T18:18:08.460 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:08.464 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:08.464 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:08.483 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:18:08.560 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: pgmap v1363: 119 pgs: 119 active+clean; 749 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T18:18:08.561 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:08.561 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.561 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:18:08.561 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: osdmap e580: 8 total, 8 up, 6 in 2023-09-23T18:18:08.561 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.561 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.561 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.561 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.561 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2826318215' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T18:18:08.561 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T18:18:08.562 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: pgmap v1363: 119 pgs: 119 active+clean; 749 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T18:18:08.562 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:08.562 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.562 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:18:08.562 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: osdmap e580: 8 total, 8 up, 6 in 2023-09-23T18:18:08.562 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.562 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.563 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.563 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.563 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2826318215' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T18:18:08.563 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:08 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T18:18:08.563 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T18:18:08.565 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:08.651 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:08.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:08.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: pgmap v1363: 119 pgs: 119 active+clean; 749 MiB data, 4.7 GiB used, 532 GiB / 536 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T18:18:08.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:08.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:18:08.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: osdmap e580: 8 total, 8 up, 6 in 2023-09-23T18:18:08.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.725 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.726 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.726 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: failed to encode map e580 with expected crc 2023-09-23T18:18:08.726 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2826318215' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T18:18:08.726 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:08 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pg_num", "val": "51"}]: dispatch 2023-09-23T18:18:08.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:08.727 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:08.829 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:08.872 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:08.932 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:08.936 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:08.936 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:08.948 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 7 16 18749 18733 167.221 136.875 0.00513652 0.00597332 2023-09-23T18:18:08.970 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:18:08.998 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:18:09.026 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:18:09.037 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:09.053 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:18:09.089 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:18:09.097 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:09.123 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:18:09.159 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:18:09.169 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:09.175 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:09.175 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:09.184 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:18:09.186 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:18:09.186 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:18:09.186 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:18:09.187 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:18:09.187 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:18:09.187 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:18:09.187 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:18:09.187 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:18:09.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:09.327 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:09.333 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:18:09.395 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:09.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:09.402 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:09.502 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:09.572 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:09.622 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:09.626 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:09.627 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:09.661 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:09 smithi099 ceph-mon[167076]: failed to encode map e581 with expected crc 2023-09-23T18:18:09.662 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:09 smithi099 ceph-mon[167076]: failed to encode map e581 with expected crc 2023-09-23T18:18:09.662 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:09 smithi099 ceph-mon[167076]: osdmap e581: 8 total, 8 up, 6 in 2023-09-23T18:18:09.662 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3219405957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:09.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:09.789 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:09.913 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:09.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:09.918 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:09 smithi055 ceph-mon[179668]: failed to encode map e581 with expected crc 2023-09-23T18:18:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:09 smithi055 ceph-mon[179668]: failed to encode map e581 with expected crc 2023-09-23T18:18:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:09 smithi055 ceph-mon[179668]: osdmap e581: 8 total, 8 up, 6 in 2023-09-23T18:18:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3219405957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:09 smithi055 ceph-mon[184174]: failed to encode map e581 with expected crc 2023-09-23T18:18:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:09 smithi055 ceph-mon[184174]: failed to encode map e581 with expected crc 2023-09-23T18:18:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:09 smithi055 ceph-mon[184174]: osdmap e581: 8 total, 8 up, 6 in 2023-09-23T18:18:09.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3219405957' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:09.948 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 8 16 20883 20867 162.989 133.375 0.00711548 0.00612993 2023-09-23T18:18:10.019 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:10.020 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:18:10.077 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:10.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:18:10] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.43.0" 2023-09-23T18:18:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-mon[179668]: pgmap v1366: 119 pgs: 119 active+clean; 800 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-09-23T18:18:10.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-mon[179668]: failed to encode map e582 with expected crc 2023-09-23T18:18:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-mon[179668]: failed to encode map e582 with expected crc 2023-09-23T18:18:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-mon[179668]: osdmap e582: 8 total, 8 up, 6 in 2023-09-23T18:18:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-mon[179668]: failed to encode map e582 with expected crc 2023-09-23T18:18:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-mon[184174]: pgmap v1366: 119 pgs: 119 active+clean; 800 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-09-23T18:18:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-mon[184174]: failed to encode map e582 with expected crc 2023-09-23T18:18:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-mon[184174]: failed to encode map e582 with expected crc 2023-09-23T18:18:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-mon[184174]: osdmap e582: 8 total, 8 up, 6 in 2023-09-23T18:18:10.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:10 smithi055 ceph-mon[184174]: failed to encode map e582 with expected crc 2023-09-23T18:18:10.747 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:10 smithi099 ceph-mon[167076]: pgmap v1366: 119 pgs: 119 active+clean; 800 MiB data, 5.0 GiB used, 531 GiB / 536 GiB avail; 133 MiB/s wr, 2.12k op/s 2023-09-23T18:18:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:10 smithi099 ceph-mon[167076]: failed to encode map e582 with expected crc 2023-09-23T18:18:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:10 smithi099 ceph-mon[167076]: failed to encode map e582 with expected crc 2023-09-23T18:18:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:10 smithi099 ceph-mon[167076]: osdmap e582: 8 total, 8 up, 6 in 2023-09-23T18:18:10.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:10 smithi099 ceph-mon[167076]: failed to encode map e582 with expected crc 2023-09-23T18:18:10.949 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 9 16 23142 23126 160.549 141.188 0.0073231 0.00622059 2023-09-23T18:18:11.487 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:18:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:11 smithi099 ceph-mon[167076]: osdmap e583: 8 total, 8 up, 6 in 2023-09-23T18:18:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:11 smithi099 ceph-mon[167076]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:11 smithi099 ceph-mon[167076]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:11 smithi099 ceph-mon[167076]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:11 smithi099 ceph-mon[167076]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:11 smithi099 ceph-mon[167076]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[179668]: osdmap e583: 8 total, 8 up, 6 in 2023-09-23T18:18:11.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[179668]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[179668]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[179668]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[179668]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[179668]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[184174]: osdmap e583: 8 total, 8 up, 6 in 2023-09-23T18:18:11.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[184174]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.857 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[184174]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.857 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[184174]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.857 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[184174]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.857 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:11 smithi055 ceph-mon[184174]: failed to encode map e583 with expected crc 2023-09-23T18:18:11.949 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 10 16 25339 25323 158.225 137.312 0.00727331 0.00631384 2023-09-23T18:18:12.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:12.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:12.105 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:12.206 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:12.256 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:12.311 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:12.315 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:12.316 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:12.416 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:12.459 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:12.504 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:12.514 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:12.520 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:12.520 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:12.552 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:12 smithi099 ceph-mon[167076]: pgmap v1369: 119 pgs: 119 active+clean; 1018 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T18:18:12.621 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:12.673 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:12.779 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:12.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:12.784 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:12.784 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:12 smithi055 ceph-mon[184174]: pgmap v1369: 119 pgs: 119 active+clean; 1018 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T18:18:12.785 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:12 smithi055 ceph-mon[179668]: pgmap v1369: 119 pgs: 119 active+clean; 1018 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 67 MiB/s wr, 1.08k op/s 2023-09-23T18:18:12.885 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:12.943 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:12.949 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 11 16 27377 27361 155.42 127.375 0.0132443 0.00642743 2023-09-23T18:18:13.104 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:13.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:13.107 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:13.207 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:13.272 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:13.330 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:13.337 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:13.337 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:13.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:13.495 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:13.559 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:13 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1988267654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:13.565 INFO:tasks.thrashosds.thrasher:in_osds: [1, 0, 4, 3, 6, 5] out_osds: [7, 2] dead_osds: [] live_osds: [0, 4, 6, 7, 3, 5, 2, 1] 2023-09-23T18:18:13.565 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:18:13.565 INFO:tasks.thrashosds.thrasher:inject_pause on osd.3 2023-09-23T18:18:13.565 INFO:tasks.thrashosds.thrasher:Testing filestore_inject_stall pause injection for duration 3 2023-09-23T18:18:13.565 INFO:tasks.thrashosds.thrasher:Checking after 0, should_be_down=False 2023-09-23T18:18:13.565 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 config set filestore_inject_stall 3 2023-09-23T18:18:13.586 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:18:13.827 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 18:18:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a[151613]: logger=cleanup t=2023-09-23T18:18:13.734799075Z level=info msg="Completed cleanup jobs" duration=75.849384ms 2023-09-23T18:18:13.830 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:18:13.861 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:18:13.867 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:13.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:13.871 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:13.893 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:18:13.923 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:18:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:13 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1988267654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:13 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1988267654' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:13.950 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 12 16 28906 28890 150.432 95.5625 0.0148559 0.00664214 2023-09-23T18:18:13.963 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:18:13.972 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:13.995 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:18:14.027 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:18:14.038 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:14.175 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:18:14.175 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:18:14.175 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:18:14.175 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:18:14.175 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:18:14.175 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:18:14.175 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:18:14.175 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:18:14.176 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:18:14.254 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:14.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:14.262 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:14.363 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:14.380 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:18:14.380 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:18:14.380 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.380 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:29644 10.7 10:e867ec0d:::benchmark_data_smithi055_425050_object29643:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:18:14.380 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:14.364690+0000", 2023-09-23T18:18:14.380 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0096928750000000001, 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0097266020000000009, 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29644 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.381 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.364690+0000", 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.364690+0000", 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:14.382 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.364689+0000", 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.364741+0000", 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.185e-05 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.364748+0000", 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5409999999999997e-06 2023-09-23T18:18:14.383 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.364751+0000", 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2440000000000002e-06 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.367706+0000", 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029544430000000002 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.384 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.367829+0000", 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000122996 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.367965+0000", 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013621400000000001 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:14.385 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.371344+0000", 2023-09-23T18:18:14.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033796410000000001 2023-09-23T18:18:14.386 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.386 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.386 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:14.386 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372466+0000", 2023-09-23T18:18:14.386 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001121115 2023-09-23T18:18:14.386 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.386 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:14.386 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.386 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:29648 10.7 10:e0afff55:::benchmark_data_smithi055_425050_object29647:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:14.366125+0000", 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0082576100000000003, 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0083878719999999993, 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29648 2023-09-23T18:18:14.387 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366125+0000", 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366125+0000", 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.388 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366124+0000", 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366150+0000", 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5565e-05 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:14.389 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366157+0000", 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3440000000000003e-06 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366161+0000", 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1350000000000002e-06 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.368800+0000", 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026383589999999998 2023-09-23T18:18:14.390 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.368906+0000", 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106227 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.369020+0000", 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000114407 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.391 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.371349+0000", 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023283470000000001 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372953+0000", 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001603742 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.392 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:29649 10.7 10:e26e100a:::benchmark_data_smithi055_425050_object29648:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:14.366276+0000", 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0081069560000000002, 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0083372150000000003, 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:18:14.393 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29649 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366276+0000", 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366276+0000", 2023-09-23T18:18:14.394 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366275+0000", 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366316+0000", 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0694e-05 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.395 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366329+0000", 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3699e-05 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.366336+0000", 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6109999999999999e-06 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:14.396 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.369951+0000", 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0036151999999999998 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.370057+0000", 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000105398 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.370181+0000", 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012434999999999999 2023-09-23T18:18:14.397 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.371354+0000", 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0011729749999999999 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:29657 10.e 10:75b84716:::benchmark_data_smithi055_425050_object29656:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:18:14.398 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:14.372656+0000", 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0017267649999999999, 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002040203, 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29657 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:14.399 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372656+0000", 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372656+0000", 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:14.400 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372655+0000", 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372697+0000", 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2038e-05 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372714+0000", 2023-09-23T18:18:14.401 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.719e-05 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372721+0000", 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2529999999999998e-06 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372772+0000", 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0966e-05 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.402 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372915+0000", 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014341899999999999 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.373054+0000", 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000138918 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:14.403 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.374463+0000", 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001409169 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:29659 10.c 10:35bfeb86:::benchmark_data_smithi055_425050_object29658:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:14.372736+0000", 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.001646843, 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029190610000000001, 2023-09-23T18:18:14.404 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29659 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372736+0000", 2023-09-23T18:18:14.405 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.406 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.406 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:14.406 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372736+0000", 2023-09-23T18:18:14.406 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.406 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.406 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.406 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:14.406 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372735+0000", 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372768+0000", 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2342999999999998e-05 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372776+0000", 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9189999999999999e-06 2023-09-23T18:18:14.407 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372780+0000", 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8400000000000002e-06 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372826+0000", 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5518e-05 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.408 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.372952+0000", 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012592699999999999 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.373100+0000", 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014771700000000001 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.409 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.375591+0000", 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024910739999999998 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:29661 10.d 10:b497aaf7:::benchmark_data_smithi055_425050_object29660:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536 in=65536b] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:14.374330+0000", 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 5.2845e-05, 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0014016549999999999, 2023-09-23T18:18:14.410 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 29661 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.374330+0000", 2023-09-23T18:18:14.411 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.374330+0000", 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.374329+0000", 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:18:14.412 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.374366+0000", 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7537000000000003e-05 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.374381+0000", 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4345000000000001e-05 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.413 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.374387+0000", 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.55e-06 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.374431+0000", 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4304999999999997e-05 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:18:14.414 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.374561+0000", 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012960700000000001 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.374686+0000", 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012509899999999999 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.415 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.46799.0:29659 10.c e584/574)", 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:14.375601+0000", 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 4294967295.9987817, 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00020779899999999999, 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.375601+0000", 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.416 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.375601+0000", 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.375600+0000", 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:18:14.417 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.375605+0000", 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0969999999999999e-06 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.375615+0000", 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0895e-05 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:14.375621+0000", 2023-09-23T18:18:14.418 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0079999999999999e-06 2023-09-23T18:18:14.419 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.419 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:18:14.419 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.419 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:18:14.419 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:18:14.419 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 7 2023-09-23T18:18:14.419 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:18:14.419 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:14.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:14 smithi099 ceph-mon[167076]: pgmap v1370: 119 pgs: 119 active+clean; 1.4 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T18:18:14.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:14 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:14.825 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1988267654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:14.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:14 smithi099 ceph-mon[167076]: osdmap e584: 8 total, 8 up, 6 in 2023-09-23T18:18:14.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:14 smithi099 ceph-mon[167076]: failed to encode map e584 with expected crc 2023-09-23T18:18:14.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/442476185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:14.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:18:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[184174]: pgmap v1370: 119 pgs: 119 active+clean; 1.4 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T18:18:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1988267654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[184174]: osdmap e584: 8 total, 8 up, 6 in 2023-09-23T18:18:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[184174]: failed to encode map e584 with expected crc 2023-09-23T18:18:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/442476185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[179668]: pgmap v1370: 119 pgs: 119 active+clean; 1.4 GiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 127 MiB/s wr, 2.03k op/s 2023-09-23T18:18:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1988267654' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[179668]: osdmap e584: 8 total, 8 up, 6 in 2023-09-23T18:18:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[179668]: failed to encode map e584 with expected crc 2023-09-23T18:18:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/442476185' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:14.950 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 13 16 30761 30745 147.778 115.938 0.00847391 0.00676321 2023-09-23T18:18:15.638 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:18:15.950 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 14 16 32561 32545 145.258 112.5 0.00628547 0.00687896 2023-09-23T18:18:16.699 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:16.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:16.705 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:16.805 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:16 smithi099 ceph-mon[167076]: pgmap v1372: 119 pgs: 119 active+clean; 1.5 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-09-23T18:18:16.843 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:16 smithi055 ceph-mon[179668]: pgmap v1372: 119 pgs: 119 active+clean; 1.5 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-09-23T18:18:16.843 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:16 smithi055 ceph-mon[184174]: pgmap v1372: 119 pgs: 119 active+clean; 1.5 GiB data, 7.1 GiB used, 529 GiB / 536 GiB avail; 116 MiB/s wr, 1.85k op/s 2023-09-23T18:18:16.849 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:16.950 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 15 16 34418 34402 143.311 116.062 0.00799115 0.00697195 2023-09-23T18:18:17.559 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:17.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:17.562 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:17.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:17.693 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:17.764 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:17.950 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 16 16 35727 35711 139.467 81.8125 0.00665558 0.00716455 2023-09-23T18:18:18.499 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:18.505 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:18.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:18.606 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:18.703 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:18.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:18 smithi099 ceph-mon[167076]: pgmap v1373: 119 pgs: 119 active+clean; 2.0 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 164 MiB/s wr, 2.62k op/s 2023-09-23T18:18:18.832 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:18 smithi055 ceph-mon[179668]: pgmap v1373: 119 pgs: 119 active+clean; 2.0 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 164 MiB/s wr, 2.62k op/s 2023-09-23T18:18:18.832 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:18 smithi055 ceph-mon[184174]: pgmap v1373: 119 pgs: 119 active+clean; 2.0 GiB data, 8.7 GiB used, 528 GiB / 536 GiB avail; 164 MiB/s wr, 2.62k op/s 2023-09-23T18:18:18.950 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 17 16 37249 37233 136.859 95.125 0.0105221 0.00730093 2023-09-23T18:18:19.244 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:18:19.399 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:18:19.399 INFO:teuthology.orchestra.run.smithi055.stdout: "success": "" 2023-09-23T18:18:19.399 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:18:19.483 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:19.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:19.491 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:19.522 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:18:19.592 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:19.623 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:19.725 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:19.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:19.729 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:19.829 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:19.874 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:19.918 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:19.921 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:19.922 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:19.950 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 18 16 38773 38757 134.547 95.25 0.0101128 0.00742767 2023-09-23T18:18:20.023 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:20.091 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:20.097 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:20.105 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:18:20.105 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:18:20.106 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:18:20.106 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:18:20.106 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:18:20.178 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:20.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:20.190 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:20.291 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:20.323 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:20.344 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:20.405 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:20.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:20.410 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:20.511 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:20.555 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:20.587 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:18:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:18:20] "GET /metrics HTTP/1.1" 200 36654 "" "Prometheus/2.43.0" 2023-09-23T18:18:20.587 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:20 smithi055 ceph-mon[184174]: pgmap v1374: 119 pgs: 119 active+clean; 2.0 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-09-23T18:18:20.587 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:20 smithi055 ceph-mon[179668]: pgmap v1374: 119 pgs: 119 active+clean; 2.0 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-09-23T18:18:20.633 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:18:20.707 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:20.711 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:20.711 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:20.812 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:20.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:20 smithi099 ceph-mon[167076]: pgmap v1374: 119 pgs: 119 active+clean; 2.0 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 149 MiB/s wr, 2.38k op/s 2023-09-23T18:18:20.850 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:20.946 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:20.950 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:20.950 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:20.951 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 19 16 40274 40258 132.403 93.8125 0.0103088 0.00754781 2023-09-23T18:18:21.051 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:21.094 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:21.144 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:21.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:21.148 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:21.248 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:21.304 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:21.376 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:21.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:21.392 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:21.492 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:21.520 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:18:21.547 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:21.759 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:21.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:21.767 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:21.782 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:18:21.817 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:18:21.844 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:18:21.868 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:21.873 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:18:21.909 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:21.909 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:18:21.951 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:18:21.951628+0000 min lat: 0.00128263 max lat: 0.189938 avg lat: 0.00765863 2023-09-23T18:18:21.951 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:18:21.951 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 20 16 41774 41758 130.47 93.75 0.0107232 0.00765863 2023-09-23T18:18:21.956 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:18:21.966 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:21.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:21.976 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:21.993 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:18:22.024 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:18:22.026 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:18:22.026 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:18:22.026 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:18:22.026 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:18:22.026 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:18:22.027 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:18:22.027 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:18:22.027 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:18:22.077 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:22.147 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:22.615 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:18:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:22 smithi099 ceph-mon[167076]: pgmap v1375: 119 pgs: 119 active+clean; 2.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-09-23T18:18:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/55027399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2076402141' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:18:22.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:22 smithi055 ceph-mon[184174]: pgmap v1375: 119 pgs: 119 active+clean; 2.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-09-23T18:18:22.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:22.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:22.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/55027399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:22.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2076402141' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:18:22.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:22 smithi055 ceph-mon[179668]: pgmap v1375: 119 pgs: 119 active+clean; 2.2 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 121 MiB/s wr, 1.94k op/s 2023-09-23T18:18:22.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:22.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:22.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/55027399' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:22.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2076402141' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:18:22.950 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 21 16 43388 43372 129.06 100.875 0.00771761 0.0077439 2023-09-23T18:18:23.449 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:23.453 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:23.454 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:23.554 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:23.576 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:23.576 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[184174]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.576 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2076402141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:18:23.576 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[184174]: osdmap e585: 8 total, 8 up, 6 in 2023-09-23T18:18:23.576 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[184174]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[184174]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[184174]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[184174]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[179668]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2076402141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[179668]: osdmap e585: 8 total, 8 up, 6 in 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[179668]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[179668]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[179668]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.577 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:23 smithi055 ceph-mon[179668]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.620 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:23 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:23 smithi099 ceph-mon[167076]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:23 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2076402141' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:18:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:23 smithi099 ceph-mon[167076]: osdmap e585: 8 total, 8 up, 6 in 2023-09-23T18:18:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:23 smithi099 ceph-mon[167076]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:23 smithi099 ceph-mon[167076]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:23 smithi099 ceph-mon[167076]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:23 smithi099 ceph-mon[167076]: failed to encode map e585 with expected crc 2023-09-23T18:18:23.951 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 22 16 44855 44839 127.361 91.6875 0.010349 0.00784774 2023-09-23T18:18:24.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:24.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:24.682 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:24.765 INFO:tasks.thrashosds.thrasher:in_osds: [1, 0, 4, 3, 6, 5] out_osds: [7, 2] dead_osds: [] live_osds: [0, 4, 6, 7, 3, 5, 2, 1] 2023-09-23T18:18:24.765 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T18:18:24.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:24.809 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:24.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:24 smithi055 ceph-mon[184174]: pgmap v1377: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T18:18:24.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:24 smithi055 ceph-mon[179668]: pgmap v1377: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T18:18:24.949 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:24.950 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 23 16 46124 46108 125.272 79.3125 0.00690621 0.00797754 2023-09-23T18:18:24.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:24.955 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:24.956 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:24 smithi099 ceph-mon[167076]: pgmap v1377: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 112 MiB/s wr, 1.79k op/s 2023-09-23T18:18:25.056 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:25.139 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:25.311 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:25.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:25.320 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:25.420 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:25.458 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:18:25.458 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:18:25.464 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:25.488 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:25.511 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:25.517 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:25.518 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:25.576 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:18:25.619 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:25.645 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:18:25.684 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:25.840 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:25.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:25.846 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:25.883 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:18:25.946 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:25.951 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 24 16 47498 47482 123.63 85.875 0.0148125 0.00808365 2023-09-23T18:18:25.998 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:26.056 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:26.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:26.067 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:26.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:26.238 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:26.306 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:26.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:26.312 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:26.412 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:26.448 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:26.462 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:26.679 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:26.724 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:26.728 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:26.729 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:26.818 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:18:26.829 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:26.878 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:26 smithi055 ceph-mon[184174]: pgmap v1378: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-23T18:18:26.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:26 smithi055 ceph-mon[179668]: pgmap v1378: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-23T18:18:26.882 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:26.953 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 25 16 48847 48831 122.047 84.3125 0.00951154 0.00818829 2023-09-23T18:18:26.979 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:26.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:26.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:27.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:26 smithi099 ceph-mon[167076]: pgmap v1378: 119 pgs: 119 active+clean; 2.5 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 106 MiB/s wr, 1.69k op/s 2023-09-23T18:18:27.085 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:27.141 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:27.215 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:27.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:27.222 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:27.322 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:27.388 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:27.644 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T18:18:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:27 smithi099 ceph-mon[167076]: pgmap v1379: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T18:18:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1438419630' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:18:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:27 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:18:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:27 smithi055 ceph-mon[179668]: pgmap v1379: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T18:18:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1438419630' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:18:27.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:27 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:18:27.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:27 smithi055 ceph-mon[184174]: pgmap v1379: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 93 MiB/s wr, 1.49k op/s 2023-09-23T18:18:27.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1438419630' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:18:27.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:27 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:18:27.953 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 26 16 50137 50121 120.454 80.625 0.00747112 0.00828366 2023-09-23T18:18:28.744 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:28.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:28.750 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:28.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[179668]: failed to encode map e586 with expected crc 2023-09-23T18:18:28.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:18:28.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[179668]: osdmap e586: 8 total, 8 up, 6 in 2023-09-23T18:18:28.809 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[179668]: failed to encode map e586 with expected crc 2023-09-23T18:18:28.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[179668]: failed to encode map e586 with expected crc 2023-09-23T18:18:28.810 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[179668]: failed to encode map e586 with expected crc 2023-09-23T18:18:28.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[184174]: failed to encode map e586 with expected crc 2023-09-23T18:18:28.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:18:28.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[184174]: osdmap e586: 8 total, 8 up, 6 in 2023-09-23T18:18:28.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[184174]: failed to encode map e586 with expected crc 2023-09-23T18:18:28.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[184174]: failed to encode map e586 with expected crc 2023-09-23T18:18:28.810 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:28 smithi055 ceph-mon[184174]: failed to encode map e586 with expected crc 2023-09-23T18:18:28.828 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T18:18:28.853 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:28.928 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:28.953 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 27 16 51242 51226 118.549 69.0625 0.0140225 0.00843014 2023-09-23T18:18:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:28 smithi099 ceph-mon[167076]: failed to encode map e586 with expected crc 2023-09-23T18:18:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:28 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:18:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:28 smithi099 ceph-mon[167076]: osdmap e586: 8 total, 8 up, 6 in 2023-09-23T18:18:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:28 smithi099 ceph-mon[167076]: failed to encode map e586 with expected crc 2023-09-23T18:18:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:28 smithi099 ceph-mon[167076]: failed to encode map e586 with expected crc 2023-09-23T18:18:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:28 smithi099 ceph-mon[167076]: failed to encode map e586 with expected crc 2023-09-23T18:18:29.248 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:29.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:29.255 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:29.355 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:29.429 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:29.508 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:29.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:29.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:29.613 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:29.631 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:18:29.670 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:18:29.678 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:29.699 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:18:29.714 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:29.735 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:18:29.737 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:29.741 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:29.741 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:29.767 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:18:29.797 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:18:29.804 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:18:29.805 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:18:29.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:29 smithi099 ceph-mon[167076]: pgmap v1381: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T18:18:29.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/339921711' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:29.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:18:29.842 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:29.861 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:18:29.863 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:18:29.863 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:18:29.863 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:18:29.863 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:18:29.863 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:18:29.863 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:18:29.863 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:18:29.863 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:18:29.902 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:29 smithi055 ceph-mon[184174]: pgmap v1381: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T18:18:29.902 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/339921711' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:29.902 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:29 smithi055 ceph-mon[179668]: pgmap v1381: 119 pgs: 119 active+clean; 2.9 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 96 MiB/s wr, 1.54k op/s 2023-09-23T18:18:29.903 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/339921711' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:29.904 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:29.954 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 28 16 52391 52375 116.88 71.8125 0.0122879 0.00854993 2023-09-23T18:18:30.166 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:30.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:30.171 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:30.229 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:18:30.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:30.349 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:30.417 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:30.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:30.422 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:30.494 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:18:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:18:30] "GET /metrics HTTP/1.1" 200 36656 "" "Prometheus/2.43.0" 2023-09-23T18:18:30.523 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:30.607 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:30.662 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:18:30.685 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:30.690 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:30.691 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:30.691 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4023193859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:30.691 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:30 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:30.792 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:30.794 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4023193859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:30.794 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:30 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:30.794 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4023193859' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:30.794 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:30 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:30.861 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:30.931 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:30.939 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:30.939 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:30.954 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 29 16 53618 53602 115.494 76.6875 0.0132819 0.00865253 2023-09-23T18:18:31.040 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:31.086 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:31.091 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:18:31.137 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:31.147 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:31.147 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:31.249 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:31.319 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:31.388 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:31.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:31.401 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:31.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:31.559 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:31.638 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:31.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:31.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:31.754 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:31.823 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:18:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:18:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[184174]: failed to encode map e587 with expected crc 2023-09-23T18:18:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[184174]: failed to encode map e587 with expected crc 2023-09-23T18:18:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[184174]: osdmap e587: 8 total, 8 up, 6 in 2023-09-23T18:18:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[184174]: failed to encode map e587 with expected crc 2023-09-23T18:18:31.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[184174]: pgmap v1383: 119 pgs: 119 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T18:18:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:18:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:18:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[179668]: failed to encode map e587 with expected crc 2023-09-23T18:18:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[179668]: failed to encode map e587 with expected crc 2023-09-23T18:18:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[179668]: osdmap e587: 8 total, 8 up, 6 in 2023-09-23T18:18:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[179668]: failed to encode map e587 with expected crc 2023-09-23T18:18:31.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:31 smithi055 ceph-mon[179668]: pgmap v1383: 119 pgs: 119 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T18:18:31.953 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 30 16 54882 54866 114.278 79 0.00696956 0.00874549 2023-09-23T18:18:32.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:31 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:18:32.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:31 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:18:32.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:31 smithi099 ceph-mon[167076]: failed to encode map e587 with expected crc 2023-09-23T18:18:32.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:31 smithi099 ceph-mon[167076]: failed to encode map e587 with expected crc 2023-09-23T18:18:32.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:31 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:32.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:31 smithi099 ceph-mon[167076]: osdmap e587: 8 total, 8 up, 6 in 2023-09-23T18:18:32.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:31 smithi099 ceph-mon[167076]: failed to encode map e587 with expected crc 2023-09-23T18:18:32.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:31 smithi099 ceph-mon[167076]: pgmap v1383: 119 pgs: 119 active+clean; 3.0 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 74 MiB/s wr, 1.19k op/s 2023-09-23T18:18:32.374 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:18:32.708 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T18:18:32.954 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 31 16 55990 55974 112.825 69.25 0.00353288 0.0088009 2023-09-23T18:18:33.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:32 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/34185691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:18:33.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:32 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:18:33.093 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:32 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/34185691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:18:33.093 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:32 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:18:33.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:32 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/34185691' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:18:33.093 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:32 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:18:33.094 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T18:18:33.153 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:18:33.271 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:33.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:33.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:33.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:33.442 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:33.474 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:33.592 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:33.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:33.596 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:33.696 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:33.727 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:33.743 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:18:33.743 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: osdmap e588: 8 total, 8 up, 6 in 2023-09-23T18:18:33.743 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.743 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.743 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.743 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.744 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.744 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.744 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.744 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.744 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.745 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.745 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: pgmap v1385: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T18:18:33.745 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e589 with expected crc 2023-09-23T18:18:33.745 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: osdmap e589: 8 total, 8 up, 6 in 2023-09-23T18:18:33.745 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e589 with expected crc 2023-09-23T18:18:33.745 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[179668]: failed to encode map e589 with expected crc 2023-09-23T18:18:33.746 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:18:33.746 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: osdmap e588: 8 total, 8 up, 6 in 2023-09-23T18:18:33.746 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.746 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.746 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.748 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.748 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: pgmap v1385: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T18:18:33.748 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e589 with expected crc 2023-09-23T18:18:33.748 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: osdmap e589: 8 total, 8 up, 6 in 2023-09-23T18:18:33.749 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e589 with expected crc 2023-09-23T18:18:33.749 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:33 smithi055 ceph-mon[184174]: failed to encode map e589 with expected crc 2023-09-23T18:18:33.813 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:33.816 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:33.817 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:33.890 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:18:33.891 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:18:33.891 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:18:33.891 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:18:33.891 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:18:33.910 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:18:33.910 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: osdmap e588: 8 total, 8 up, 6 in 2023-09-23T18:18:33.910 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.910 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.910 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.911 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.911 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.911 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.911 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.911 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.911 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.912 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e588 with expected crc 2023-09-23T18:18:33.912 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: pgmap v1385: 119 pgs: 119 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 64 MiB/s wr, 1.02k op/s 2023-09-23T18:18:33.912 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e589 with expected crc 2023-09-23T18:18:33.912 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: osdmap e589: 8 total, 8 up, 6 in 2023-09-23T18:18:33.912 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e589 with expected crc 2023-09-23T18:18:33.912 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:33 smithi099 ceph-mon[167076]: failed to encode map e589 with expected crc 2023-09-23T18:18:33.918 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:33.944 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:33.954 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 32 16 55990 55974 109.3 0 - 0.0088009 2023-09-23T18:18:33.999 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:34.031 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:34.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:34.034 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:34.128 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:34.135 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:34.170 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:34.205 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:34.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:34.211 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:34.285 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:18:34.311 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:34.359 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:34.399 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:34.402 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:34.403 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:34.504 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:34.534 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:34.949 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:18:34.954 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 33 16 56259 56243 106.497 8.40625 0.0112585 0.00938504 2023-09-23T18:18:35.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:34 smithi099 ceph-mon[167076]: failed to encode map e589 with expected crc 2023-09-23T18:18:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:34 smithi099 ceph-mon[167076]: failed to encode map e589 with expected crc 2023-09-23T18:18:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:34 smithi099 ceph-mon[167076]: failed to encode map e590 with expected crc 2023-09-23T18:18:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:34 smithi099 ceph-mon[167076]: osdmap e590: 8 total, 8 up, 6 in 2023-09-23T18:18:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:34 smithi099 ceph-mon[167076]: failed to encode map e590 with expected crc 2023-09-23T18:18:35.085 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:34 smithi055 ceph-mon[179668]: failed to encode map e589 with expected crc 2023-09-23T18:18:35.085 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:34 smithi055 ceph-mon[179668]: failed to encode map e589 with expected crc 2023-09-23T18:18:35.085 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:34 smithi055 ceph-mon[179668]: failed to encode map e590 with expected crc 2023-09-23T18:18:35.085 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:34 smithi055 ceph-mon[179668]: osdmap e590: 8 total, 8 up, 6 in 2023-09-23T18:18:35.085 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:34 smithi055 ceph-mon[179668]: failed to encode map e590 with expected crc 2023-09-23T18:18:35.085 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:34 smithi055 ceph-mon[184174]: failed to encode map e589 with expected crc 2023-09-23T18:18:35.085 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:34 smithi055 ceph-mon[184174]: failed to encode map e589 with expected crc 2023-09-23T18:18:35.085 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:34 smithi055 ceph-mon[184174]: failed to encode map e590 with expected crc 2023-09-23T18:18:35.086 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:34 smithi055 ceph-mon[184174]: osdmap e590: 8 total, 8 up, 6 in 2023-09-23T18:18:35.086 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:34 smithi055 ceph-mon[184174]: failed to encode map e590 with expected crc 2023-09-23T18:18:35.424 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:18:35.459 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:18:35.498 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:18:35.533 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:18:35.570 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:18:35.607 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:18:35.636 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:35.642 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:35.643 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:35.650 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:18:35.684 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:18:35.693 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:18:35.693 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:18:35.693 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:18:35.694 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:18:35.694 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:18:35.694 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:18:35.694 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:18:35.694 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:18:35.743 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:35.791 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:35.857 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:35.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:35.872 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:35.954 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 34 16 57342 57326 105.355 67.6875 0.0130335 0.00948686 2023-09-23T18:18:35.973 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:36.036 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:36.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:36.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:36.114 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:36.215 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:36.289 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:36.501 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:18:36.501 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:18:36.531 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:36.537 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:36 smithi055 ceph-mon[179668]: pgmap v1388: 119 pgs: 1 remapped+peering, 118 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T18:18:36.538 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:36 smithi055 ceph-mon[179668]: failed to encode map e591 with expected crc 2023-09-23T18:18:36.538 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:36 smithi055 ceph-mon[179668]: osdmap e591: 8 total, 8 up, 6 in 2023-09-23T18:18:36.538 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:36 smithi055 ceph-mon[179668]: failed to encode map e591 with expected crc 2023-09-23T18:18:36.538 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3114212739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:36.538 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:36 smithi055 ceph-mon[184174]: pgmap v1388: 119 pgs: 1 remapped+peering, 118 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T18:18:36.538 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:36 smithi055 ceph-mon[184174]: failed to encode map e591 with expected crc 2023-09-23T18:18:36.538 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:36 smithi055 ceph-mon[184174]: osdmap e591: 8 total, 8 up, 6 in 2023-09-23T18:18:36.539 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:36 smithi055 ceph-mon[184174]: failed to encode map e591 with expected crc 2023-09-23T18:18:36.539 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3114212739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:36.539 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:36.539 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:36.640 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:36.675 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:36 smithi099 ceph-mon[167076]: pgmap v1388: 119 pgs: 1 remapped+peering, 118 active+clean; 3.3 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 87 MiB/s wr, 1.39k op/s 2023-09-23T18:18:36.675 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:36 smithi099 ceph-mon[167076]: failed to encode map e591 with expected crc 2023-09-23T18:18:36.675 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:36 smithi099 ceph-mon[167076]: osdmap e591: 8 total, 8 up, 6 in 2023-09-23T18:18:36.676 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:36 smithi099 ceph-mon[167076]: failed to encode map e591 with expected crc 2023-09-23T18:18:36.676 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3114212739' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:36.712 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:36.787 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:36.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:36.796 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:36.897 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:36.932 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:18:36.954 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 35 16 58396 58380 104.227 65.875 0.00935222 0.00958916 2023-09-23T18:18:36.959 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:37.025 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:37.032 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:37.033 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:37.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:37.185 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:37.240 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:37.244 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:37.245 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:37.345 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:37.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: osdmap e592: 8 total, 8 up, 6 in 2023-09-23T18:18:37.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.403 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:37.461 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T18:18:37.470 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:18:37.483 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:37.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:37.492 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:37.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: osdmap e592: 8 total, 8 up, 6 in 2023-09-23T18:18:37.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.579 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:37.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:37.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/117335591' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:18:37.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4158989840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:37.580 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:37 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:37.593 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:37.646 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:37.649 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/117335591' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:18:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4158989840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:37.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: osdmap e592: 8 total, 8 up, 6 in 2023-09-23T18:18:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: failed to encode map e592 with expected crc 2023-09-23T18:18:37.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:37.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:37.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/117335591' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:18:37.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4158989840' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:37.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:37 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:18:37.720 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:37.726 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:37.727 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:37.828 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:37.893 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:37.954 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 36 16 59448 59432 103.158 65.75 0.00744813 0.009689 2023-09-23T18:18:37.965 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:37.975 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:37.975 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:38.076 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:38.122 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:38.178 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:38.182 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:38.183 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:38.284 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:38.341 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:38.354 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:18:38.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:38 smithi099 ceph-mon[167076]: pgmap v1391: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 1 remapped+peering, 109 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 908 op/s; 7012/172026 objects degraded (4.076%); 1.3 KiB/s, 3 objects/s recovering 2023-09-23T18:18:38.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:38 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:38.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:38 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 7012/172026 objects degraded (4.076%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:38.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/117335591' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:18:38.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:18:38.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:38 smithi099 ceph-mon[167076]: osdmap e593: 8 total, 8 up, 6 in 2023-09-23T18:18:38.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[179668]: pgmap v1391: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 1 remapped+peering, 109 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 908 op/s; 7012/172026 objects degraded (4.076%); 1.3 KiB/s, 3 objects/s recovering 2023-09-23T18:18:38.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:38.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 7012/172026 objects degraded (4.076%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:38.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/117335591' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:18:38.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:18:38.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[179668]: osdmap e593: 8 total, 8 up, 6 in 2023-09-23T18:18:38.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[184174]: pgmap v1391: 119 pgs: 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 1 remapped+peering, 109 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 57 MiB/s wr, 908 op/s; 7012/172026 objects degraded (4.076%); 1.3 KiB/s, 3 objects/s recovering 2023-09-23T18:18:38.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:38.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 7012/172026 objects degraded (4.076%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:38.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/117335591' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:18:38.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:18:38.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:38 smithi055 ceph-mon[184174]: osdmap e593: 8 total, 8 up, 6 in 2023-09-23T18:18:38.955 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 37 16 60559 60543 102.247 69.4375 0.010054 0.00977579 2023-09-23T18:18:39.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:39 smithi055 ceph-mon[179668]: failed to encode map e593 with expected crc 2023-09-23T18:18:39.432 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:39 smithi055 ceph-mon[184174]: failed to encode map e593 with expected crc 2023-09-23T18:18:39.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:39 smithi099 ceph-mon[167076]: failed to encode map e593 with expected crc 2023-09-23T18:18:39.955 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 38 16 61734 61718 101.489 73.4375 0.00983966 0.00984776 2023-09-23T18:18:40.551 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:40.560 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:40.560 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:40.661 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:40.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:40 smithi055 ceph-mon[184174]: pgmap v1393: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 109 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s; 7012/173796 objects degraded (4.035%); 3250/173796 objects misplaced (1.870%); 2.9 MiB/s, 48 objects/s recovering 2023-09-23T18:18:40.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:18:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:18:40] "GET /metrics HTTP/1.1" 200 36653 "" "Prometheus/2.43.0" 2023-09-23T18:18:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:40 smithi055 ceph-mon[179668]: pgmap v1393: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 109 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s; 7012/173796 objects degraded (4.035%); 3250/173796 objects misplaced (1.870%); 2.9 MiB/s, 48 objects/s recovering 2023-09-23T18:18:40.711 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:40.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:40 smithi099 ceph-mon[167076]: pgmap v1393: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 7 peering, 109 active+clean; 3.5 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 49 MiB/s wr, 786 op/s; 7012/173796 objects degraded (4.035%); 3250/173796 objects misplaced (1.870%); 2.9 MiB/s, 48 objects/s recovering 2023-09-23T18:18:40.926 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T18:18:40.956 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 39 16 62763 62747 100.532 64.3125 0.00872389 0.00994127 2023-09-23T18:18:40.973 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:40.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:40.984 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:41.085 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:41.173 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:41.256 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:41.260 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:41.260 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:41.361 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:41.425 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:41.511 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:41.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:41.517 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:41.617 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:41.684 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:41.745 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:41.748 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:41.753 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:41.754 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:41.855 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:41.923 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:41.956 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:18:41.957449+0000 min lat: 0.00128263 max lat: 2.04688 avg lat: 0.0100277 2023-09-23T18:18:41.956 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:18:41.956 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 40 16 63819 63803 99.6686 66 0.0132169 0.0100277 2023-09-23T18:18:42.065 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:42.073 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:42.073 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:42.174 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:42.255 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:42.339 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:42.348 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:42.349 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:42.450 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:42.503 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:42 smithi055 ceph-mon[179668]: pgmap v1394: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering, 112 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 915 op/s; 7012/178992 objects degraded (3.917%); 3250/178992 objects misplaced (1.816%); 2.3 MiB/s, 38 objects/s recovering 2023-09-23T18:18:42.503 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:42 smithi055 ceph-mon[184174]: pgmap v1394: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering, 112 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 915 op/s; 7012/178992 objects degraded (3.917%); 3250/178992 objects misplaced (1.816%); 2.3 MiB/s, 38 objects/s recovering 2023-09-23T18:18:42.505 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:42.567 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:18:42.567 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:18:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:42 smithi099 ceph-mon[167076]: pgmap v1394: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 4 peering, 112 active+clean; 3.6 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 57 MiB/s wr, 915 op/s; 7012/178992 objects degraded (3.917%); 3250/178992 objects misplaced (1.816%); 2.3 MiB/s, 38 objects/s recovering 2023-09-23T18:18:42.957 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 41 16 64918 64902 98.9112 68.6875 0.0217731 0.0101052 2023-09-23T18:18:43.007 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:18:43.018 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:43.045 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:43.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:43.062 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:43.071 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:18:43.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:43.208 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:43.261 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:43.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:43.269 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:43.370 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:43.498 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:43.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7012/178992 objects degraded (3.917%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:43.584 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:43.592 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:43.593 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:43.598 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7012/178992 objects degraded (3.917%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:43.598 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7012/178992 objects degraded (3.917%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:43.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:43.744 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:43.820 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:43.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:43.825 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:43.926 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:43.957 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 42 16 65852 65836 97.9461 58.375 0.0129973 0.0102039 2023-09-23T18:18:44.011 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:44.080 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:18:44.109 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:18:44.132 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:44.137 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:44.138 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:44.147 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:18:44.205 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:18:44.239 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:44.254 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:18:44.278 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:44.307 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:18:44.337 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:44.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:44.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:44.346 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:18:44.380 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:18:44.384 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:18:44.384 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:18:44.384 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:18:44.384 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:18:44.384 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:18:44.385 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:18:44.385 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:18:44.385 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:18:44.402 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:44 smithi055 ceph-mon[179668]: pgmap v1395: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 7012/188010 objects degraded (3.730%); 3250/188010 objects misplaced (1.729%); 1.8 MiB/s, 30 objects/s recovering 2023-09-23T18:18:44.402 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1128824115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:44.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:44 smithi055 ceph-mon[184174]: pgmap v1395: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 7012/188010 objects degraded (3.730%); 3250/188010 objects misplaced (1.729%); 1.8 MiB/s, 30 objects/s recovering 2023-09-23T18:18:44.402 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1128824115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:44.434 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:44.434 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:44.442 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:44.538 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:44.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:44 smithi099 ceph-mon[167076]: pgmap v1395: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.8 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 68 MiB/s wr, 1.09k op/s; 7012/188010 objects degraded (3.730%); 3250/188010 objects misplaced (1.729%); 1.8 MiB/s, 30 objects/s recovering 2023-09-23T18:18:44.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1128824115' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:44.892 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:44.901 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:44.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:44.957 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 43 16 66853 66837 97.1232 62.5625 0.00874866 0.0102911 2023-09-23T18:18:45.003 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:45.078 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:45.414 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1154539249' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:18:45.424 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T18:18:45.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1154539249' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:18:45.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1154539249' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:18:45.957 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 44 16 67218 67202 95.4345 22.8125 0.00267464 0.0103531 2023-09-23T18:18:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[179668]: pgmap v1396: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 43 MiB/s wr, 681 op/s; 7012/189963 objects degraded (3.691%); 2993/189963 objects misplaced (1.576%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T18:18:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1154539249' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:18:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[179668]: osdmap e594: 8 total, 8 up, 6 in 2023-09-23T18:18:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[179668]: failed to encode map e594 with expected crc 2023-09-23T18:18:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[179668]: failed to encode map e594 with expected crc 2023-09-23T18:18:46.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[179668]: failed to encode map e594 with expected crc 2023-09-23T18:18:46.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[184174]: pgmap v1396: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 43 MiB/s wr, 681 op/s; 7012/189963 objects degraded (3.691%); 2993/189963 objects misplaced (1.576%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T18:18:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1154539249' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:18:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[184174]: osdmap e594: 8 total, 8 up, 6 in 2023-09-23T18:18:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[184174]: failed to encode map e594 with expected crc 2023-09-23T18:18:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[184174]: failed to encode map e594 with expected crc 2023-09-23T18:18:46.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:46 smithi055 ceph-mon[184174]: failed to encode map e594 with expected crc 2023-09-23T18:18:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:46 smithi099 ceph-mon[167076]: pgmap v1396: 119 pgs: 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 116 active+clean; 3.9 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 43 MiB/s wr, 681 op/s; 7012/189963 objects degraded (3.691%); 2993/189963 objects misplaced (1.576%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T18:18:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1154539249' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:18:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:46 smithi099 ceph-mon[167076]: osdmap e594: 8 total, 8 up, 6 in 2023-09-23T18:18:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:46 smithi099 ceph-mon[167076]: failed to encode map e594 with expected crc 2023-09-23T18:18:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:46 smithi099 ceph-mon[167076]: failed to encode map e594 with expected crc 2023-09-23T18:18:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:46 smithi099 ceph-mon[167076]: failed to encode map e594 with expected crc 2023-09-23T18:18:46.957 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 45 16 67218 67202 93.314 0 - 0.0103531 2023-09-23T18:18:47.200 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:47.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:47.205 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:47.305 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:47.336 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: failed to encode map e594 with expected crc 2023-09-23T18:18:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: osdmap e595: 8 total, 8 up, 6 in 2023-09-23T18:18:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:47.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[179668]: osdmap e596: 8 total, 8 up, 6 in 2023-09-23T18:18:47.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: failed to encode map e594 with expected crc 2023-09-23T18:18:47.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: osdmap e595: 8 total, 8 up, 6 in 2023-09-23T18:18:47.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:47.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:47 smithi055 ceph-mon[184174]: osdmap e596: 8 total, 8 up, 6 in 2023-09-23T18:18:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: failed to encode map e594 with expected crc 2023-09-23T18:18:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: osdmap e595: 8 total, 8 up, 6 in 2023-09-23T18:18:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: failed to encode map e595 with expected crc 2023-09-23T18:18:47.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:47.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:47 smithi099 ceph-mon[167076]: osdmap e596: 8 total, 8 up, 6 in 2023-09-23T18:18:47.958 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 46 16 67595 67579 91.7979 11.7812 0.0101977 0.0108886 2023-09-23T18:18:48.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: pgmap v1399: 119 pgs: 24 peering, 4 unknown, 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 38 MiB/s wr, 605 op/s; 7012/188337 objects degraded (3.723%); 2993/188337 objects misplaced (1.589%); 1.5 MiB/s, 30 objects/s recovering 2023-09-23T18:18:48.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.813 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:18:48.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 7012/188337 objects degraded (3.723%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:48.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: osdmap e597: 8 total, 8 up, 6 in 2023-09-23T18:18:48.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: failed to encode map e597 with expected crc 2023-09-23T18:18:48.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: failed to encode map e597 with expected crc 2023-09-23T18:18:48.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[179668]: failed to encode map e597 with expected crc 2023-09-23T18:18:48.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: pgmap v1399: 119 pgs: 24 peering, 4 unknown, 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 38 MiB/s wr, 605 op/s; 7012/188337 objects degraded (3.723%); 2993/188337 objects misplaced (1.589%); 1.5 MiB/s, 30 objects/s recovering 2023-09-23T18:18:48.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:18:48.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 7012/188337 objects degraded (3.723%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:48.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: osdmap e597: 8 total, 8 up, 6 in 2023-09-23T18:18:48.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: failed to encode map e597 with expected crc 2023-09-23T18:18:48.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: failed to encode map e597 with expected crc 2023-09-23T18:18:48.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:48 smithi055 ceph-mon[184174]: failed to encode map e597 with expected crc 2023-09-23T18:18:48.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: pgmap v1399: 119 pgs: 24 peering, 4 unknown, 1 active+recovering+remapped, 2 active+recovery_wait+undersized+degraded+remapped, 88 active+clean; 3.8 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 38 MiB/s wr, 605 op/s; 7012/188337 objects degraded (3.723%); 2993/188337 objects misplaced (1.589%); 1.5 MiB/s, 30 objects/s recovering 2023-09-23T18:18:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: failed to encode map e596 with expected crc 2023-09-23T18:18:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 4 pgs inactive, 5 pgs peering (PG_AVAILABILITY) 2023-09-23T18:18:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 7012/188337 objects degraded (3.723%), 2 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: osdmap e597: 8 total, 8 up, 6 in 2023-09-23T18:18:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: failed to encode map e597 with expected crc 2023-09-23T18:18:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: failed to encode map e597 with expected crc 2023-09-23T18:18:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:48 smithi099 ceph-mon[167076]: failed to encode map e597 with expected crc 2023-09-23T18:18:48.900 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:48.917 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:48.917 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:48.958 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 47 16 68567 68551 91.1373 60.75 0.0344064 0.010968 2023-09-23T18:18:49.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:49.022 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T18:18:49.066 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:49.113 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:49.118 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:49.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:49.219 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:49.260 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:49.309 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:49.313 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:49.313 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:49.414 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:49.461 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:49.521 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:49.524 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:49.525 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:49.625 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:49.706 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:49.749 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[184174]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.749 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[184174]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.750 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[184174]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.750 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[184174]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.750 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[184174]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.750 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[184174]: osdmap e598: 8 total, 8 up, 6 in 2023-09-23T18:18:49.750 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[184174]: failed to encode map e598 with expected crc 2023-09-23T18:18:49.750 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[184174]: failed to encode map e598 with expected crc 2023-09-23T18:18:49.750 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[179668]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.751 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[179668]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.751 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[179668]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.751 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[179668]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.751 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[179668]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.751 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[179668]: osdmap e598: 8 total, 8 up, 6 in 2023-09-23T18:18:49.751 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[179668]: failed to encode map e598 with expected crc 2023-09-23T18:18:49.751 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:49 smithi055 ceph-mon[179668]: failed to encode map e598 with expected crc 2023-09-23T18:18:49.780 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:49.792 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:49.792 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:49.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:49 smithi099 ceph-mon[167076]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:49 smithi099 ceph-mon[167076]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:49 smithi099 ceph-mon[167076]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:49 smithi099 ceph-mon[167076]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:49 smithi099 ceph-mon[167076]: failed to encode map e597 with expected crc 2023-09-23T18:18:49.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:49 smithi099 ceph-mon[167076]: osdmap e598: 8 total, 8 up, 6 in 2023-09-23T18:18:49.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:49 smithi099 ceph-mon[167076]: failed to encode map e598 with expected crc 2023-09-23T18:18:49.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:49 smithi099 ceph-mon[167076]: failed to encode map e598 with expected crc 2023-09-23T18:18:49.893 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:49.958 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 48 16 69350 69334 90.2581 48.9375 0.00751504 0.0110726 2023-09-23T18:18:50.004 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:18:50.004 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:18:50.004 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:18:50.004 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:18:50.004 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:18:50.013 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:50.073 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:50.076 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:18:50.287 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:50.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:50.292 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:50.393 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:50.525 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:18:50.527 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:50.575 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:18:50] "GET /metrics HTTP/1.1" 200 36653 "" "Prometheus/2.43.0" 2023-09-23T18:18:50.575 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-mon[184174]: pgmap v1402: 119 pgs: 1 active+recovering+remapped, 26 peering, 4 unknown, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 7025/189321 objects degraded (3.711%); 2547/189321 objects misplaced (1.345%); 6.0 MiB/s, 108 objects/s recovering 2023-09-23T18:18:50.575 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-mon[184174]: failed to encode map e598 with expected crc 2023-09-23T18:18:50.576 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-mon[184174]: failed to encode map e598 with expected crc 2023-09-23T18:18:50.576 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1128152097' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:50.576 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:50.576 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-mon[179668]: pgmap v1402: 119 pgs: 1 active+recovering+remapped, 26 peering, 4 unknown, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 7025/189321 objects degraded (3.711%); 2547/189321 objects misplaced (1.345%); 6.0 MiB/s, 108 objects/s recovering 2023-09-23T18:18:50.576 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-mon[179668]: failed to encode map e598 with expected crc 2023-09-23T18:18:50.576 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-mon[179668]: failed to encode map e598 with expected crc 2023-09-23T18:18:50.576 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1128152097' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:50.576 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:50 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:50.645 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:18:50.656 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:50.664 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:50.665 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:50.766 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:50.821 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:50 smithi099 ceph-mon[167076]: pgmap v1402: 119 pgs: 1 active+recovering+remapped, 26 peering, 4 unknown, 3 active+recovery_wait+undersized+degraded+remapped, 85 active+clean; 3.9 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 7025/189321 objects degraded (3.711%); 2547/189321 objects misplaced (1.345%); 6.0 MiB/s, 108 objects/s recovering 2023-09-23T18:18:50.821 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:50 smithi099 ceph-mon[167076]: failed to encode map e598 with expected crc 2023-09-23T18:18:50.821 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:50 smithi099 ceph-mon[167076]: failed to encode map e598 with expected crc 2023-09-23T18:18:50.821 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:50 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1128152097' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:50.822 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:50 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:18:50.823 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:50.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:50.892 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:50.893 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:50.959 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 49 16 70111 70095 89.3836 47.5625 0.00766274 0.0111816 2023-09-23T18:18:50.994 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:51.060 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:51.131 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:51.136 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:51.137 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:51.237 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:51.281 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:51.345 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:51.351 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:51.352 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:51.430 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:51.452 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:51.494 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:51.551 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:51.555 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:51 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:51.555 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:51 smithi099 ceph-mon[167076]: failed to encode map e599 with expected crc 2023-09-23T18:18:51.555 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:51 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:51.556 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:51 smithi099 ceph-mon[167076]: osdmap e599: 8 total, 8 up, 6 in 2023-09-23T18:18:51.556 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:51 smithi099 ceph-mon[167076]: failed to encode map e599 with expected crc 2023-09-23T18:18:51.556 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:51.556 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:51.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:51.557 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:51.657 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:51.758 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:51.758 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[184174]: failed to encode map e599 with expected crc 2023-09-23T18:18:51.758 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:51.758 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[184174]: osdmap e599: 8 total, 8 up, 6 in 2023-09-23T18:18:51.758 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[184174]: failed to encode map e599 with expected crc 2023-09-23T18:18:51.758 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:51.758 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:51.759 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:18:51.759 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[179668]: failed to encode map e599 with expected crc 2023-09-23T18:18:51.759 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:18:51.759 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[179668]: osdmap e599: 8 total, 8 up, 6 in 2023-09-23T18:18:51.759 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[179668]: failed to encode map e599 with expected crc 2023-09-23T18:18:51.759 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:51.759 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:18:51.780 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:51.874 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:51.878 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:51.881 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:51.881 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:51.959 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 50 16 70934 70918 88.6247 51.4375 0.0310499 0.011278 2023-09-23T18:18:51.982 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:52.106 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:52.219 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:52.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:52.224 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:52.324 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:52.423 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:52.485 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:52.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:52.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:52.526 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T18:18:52.601 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:52.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:52 smithi055 ceph-mon[179668]: pgmap v1405: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 10 peering, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 3.5 KiB/s rd, 89 MiB/s wr, 1.43k op/s; 15549/205425 objects degraded (7.569%); 2547/205425 objects misplaced (1.240%); 7.5 MiB/s, 112 objects/s recovering 2023-09-23T18:18:52.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:52 smithi055 ceph-mon[179668]: failed to encode map e599 with expected crc 2023-09-23T18:18:52.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1755757856' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:18:52.645 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:52 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:18:52.645 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:52 smithi055 ceph-mon[184174]: pgmap v1405: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 10 peering, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 3.5 KiB/s rd, 89 MiB/s wr, 1.43k op/s; 15549/205425 objects degraded (7.569%); 2547/205425 objects misplaced (1.240%); 7.5 MiB/s, 112 objects/s recovering 2023-09-23T18:18:52.646 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:52 smithi055 ceph-mon[184174]: failed to encode map e599 with expected crc 2023-09-23T18:18:52.646 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1755757856' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:18:52.646 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:52 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:18:52.716 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:52 smithi099 ceph-mon[167076]: pgmap v1405: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 10 peering, 6 active+recovery_wait+undersized+degraded+remapped, 101 active+clean; 4.2 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 3.5 KiB/s rd, 89 MiB/s wr, 1.43k op/s; 15549/205425 objects degraded (7.569%); 2547/205425 objects misplaced (1.240%); 7.5 MiB/s, 112 objects/s recovering 2023-09-23T18:18:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:52 smithi099 ceph-mon[167076]: failed to encode map e599 with expected crc 2023-09-23T18:18:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1755757856' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:18:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:52 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:18:52.962 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 51 16 71767 71751 87.903 52.0625 0.022054 0.0113704 2023-09-23T18:18:52.990 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:53.004 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:53.004 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:53.105 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:53.174 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:53.322 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:53.329 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:53.329 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:53.392 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T18:18:53.430 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:53.449 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:18:53.477 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:53.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:53.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:53.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:53.573 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:53 smithi055 ceph-mon[184174]: failed to encode map e600 with expected crc 2023-09-23T18:18:53.573 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:53 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:18:53.573 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:53 smithi055 ceph-mon[184174]: osdmap e600: 8 total, 8 up, 6 in 2023-09-23T18:18:53.574 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:53 smithi055 ceph-mon[184174]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:18:53.574 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15549/205425 objects degraded (7.569%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:53.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:53 smithi055 ceph-mon[179668]: failed to encode map e600 with expected crc 2023-09-23T18:18:53.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:53 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:18:53.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:53 smithi055 ceph-mon[179668]: osdmap e600: 8 total, 8 up, 6 in 2023-09-23T18:18:53.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:53 smithi055 ceph-mon[179668]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:18:53.574 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15549/205425 objects degraded (7.569%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:53.632 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:53.806 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:18:53.825 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:53.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:53 smithi099 ceph-mon[167076]: failed to encode map e600 with expected crc 2023-09-23T18:18:53.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:53 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:18:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:53 smithi099 ceph-mon[167076]: osdmap e600: 8 total, 8 up, 6 in 2023-09-23T18:18:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:53 smithi099 ceph-mon[167076]: Health check update: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:18:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15549/205425 objects degraded (7.569%), 7 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:53.842 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:18:53.880 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:18:53.915 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:53.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:53.926 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:53.927 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:18:53.961 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:18:53.964 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 52 16 72517 72501 87.1118 46.875 0.0171617 0.0114743 2023-09-23T18:18:54.011 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:18:54.027 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:54.049 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:18:54.064 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:54.086 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:18:54.090 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:18:54.090 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:18:54.090 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:18:54.090 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:18:54.090 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:18:54.091 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:18:54.091 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:18:54.091 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:18:54.114 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:54.117 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:54.118 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:54.218 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:54.272 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:54.339 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:54.349 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:54.349 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:54.450 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:54.479 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:54.539 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:54.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:54.549 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:54.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:18:54.736 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:54 smithi099 ceph-mon[167076]: pgmap v1407: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 peering, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 2.9 KiB/s rd, 94 MiB/s wr, 1.51k op/s; 19697/209997 objects degraded (9.380%); 2547/209997 objects misplaced (1.213%); 6.2 MiB/s, 94 objects/s recovering 2023-09-23T18:18:54.736 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:54 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:18:54.736 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3756529356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:54.737 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:54 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:54.740 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:54.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:18:54.813 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:18:54.813 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:18:54.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:54.814 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:18:54.872 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:54 smithi055 ceph-mon[179668]: pgmap v1407: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 peering, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 2.9 KiB/s rd, 94 MiB/s wr, 1.51k op/s; 19697/209997 objects degraded (9.380%); 2547/209997 objects misplaced (1.213%); 6.2 MiB/s, 94 objects/s recovering 2023-09-23T18:18:54.872 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:54 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:18:54.872 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3756529356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:54.872 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:54 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:54.874 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:54 smithi055 ceph-mon[184174]: pgmap v1407: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 2 peering, 7 active+recovery_wait+undersized+degraded+remapped, 108 active+clean; 4.3 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 2.9 KiB/s rd, 94 MiB/s wr, 1.51k op/s; 19697/209997 objects degraded (9.380%); 2547/209997 objects misplaced (1.213%); 6.2 MiB/s, 94 objects/s recovering 2023-09-23T18:18:54.874 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:54 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:18:54.874 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3756529356' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:18:54.874 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:54 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:18:54.916 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:18:54.953 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:54.965 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 53 16 73365 73349 86.467 53 0.0260364 0.0115608 2023-09-23T18:18:54.977 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:55.009 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:18:55.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:55.018 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:18:55.119 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:55.166 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:55.234 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:55.251 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:55.251 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:55.352 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:55.427 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:55.507 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:55.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:55.514 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:55.615 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:18:55.663 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:55.965 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 54 16 74227 74211 85.8635 53.875 0.0111497 0.0116387 2023-09-23T18:18:56.911 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:18:56.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:56.919 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:18:56.919 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:56 smithi055 ceph-mon[184174]: pgmap v1408: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 2.3 KiB/s rd, 77 MiB/s wr, 1.23k op/s; 19697/211461 objects degraded (9.315%); 2334/211461 objects misplaced (1.104%); 2.5 MiB/s, 36 objects/s recovering 2023-09-23T18:18:56.919 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:56 smithi055 ceph-mon[179668]: pgmap v1408: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 2.3 KiB/s rd, 77 MiB/s wr, 1.23k op/s; 19697/211461 objects degraded (9.315%); 2334/211461 objects misplaced (1.104%); 2.5 MiB/s, 36 objects/s recovering 2023-09-23T18:18:56.968 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 55 16 75104 75088 85.295 54.8125 0.0200993 0.0117177 2023-09-23T18:18:57.020 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:18:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:56 smithi099 ceph-mon[167076]: pgmap v1408: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.3 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 2.3 KiB/s rd, 77 MiB/s wr, 1.23k op/s; 19697/211461 objects degraded (9.315%); 2334/211461 objects misplaced (1.104%); 2.5 MiB/s, 36 objects/s recovering 2023-09-23T18:18:57.088 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:57.216 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:18:57.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:57.227 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:18:57.257 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:18:57.322 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:18:57.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:57.426 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:57.532 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:57.538 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:57.539 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:57.640 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:18:57.719 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:57.797 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:18:57.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:57.804 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:18:57.905 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:18:57.968 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 56 16 75941 75925 84.7062 52.3125 0.022247 0.0117994 2023-09-23T18:18:57.983 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:58.127 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:18:58.210 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:18:58.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:58.224 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:18:58.325 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:58.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:58 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:58.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:58 smithi099 ceph-mon[167076]: pgmap v1409: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 1.8 KiB/s rd, 88 MiB/s wr, 1.41k op/s; 19697/222318 objects degraded (8.860%); 2334/222318 objects misplaced (1.050%); 2.0 MiB/s, 28 objects/s recovering 2023-09-23T18:18:58.395 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:58 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:58.396 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:58 smithi055 ceph-mon[179668]: pgmap v1409: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 1.8 KiB/s rd, 88 MiB/s wr, 1.41k op/s; 19697/222318 objects degraded (8.860%); 2334/222318 objects misplaced (1.050%); 2.0 MiB/s, 28 objects/s recovering 2023-09-23T18:18:58.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:58 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:18:58.396 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:58 smithi055 ceph-mon[184174]: pgmap v1409: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.5 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 1.8 KiB/s rd, 88 MiB/s wr, 1.41k op/s; 19697/222318 objects degraded (8.860%); 2334/222318 objects misplaced (1.050%); 2.0 MiB/s, 28 objects/s recovering 2023-09-23T18:18:58.448 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:58.516 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:58.522 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:58.523 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:58.623 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:58.685 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:58.858 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:58.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:58.872 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:58.971 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 57 16 76746 76730 84.1025 50.3125 0.0082553 0.0118857 2023-09-23T18:18:58.973 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:59.064 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:18:59 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19697/222318 objects degraded (8.860%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:59.073 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:59.115 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:18:59.196 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:59.209 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:59.210 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:59.310 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:59.412 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:18:59 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19697/222318 objects degraded (8.860%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:59.413 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:18:59 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19697/222318 objects degraded (8.860%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:18:59.416 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:59.506 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:18:59.512 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:59.512 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:18:59.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:18:59.657 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:18:59.759 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:18:59.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:18:59.768 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:18:59.868 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:18:59.948 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:18:59.948 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:18:59.968 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 58 16 77379 77363 83.3347 39.5625 0.0099687 0.0119923 2023-09-23T18:18:59.969 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:00.041 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:00.050 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:00.050 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:00.076 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:00.102 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T18:19:00.113 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:19:00.151 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:00.193 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:00.322 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:00.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:00.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:00.357 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:19:00.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:00 smithi099 ceph-mon[167076]: pgmap v1410: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 823 op/s; 19697/223893 objects degraded (8.798%); 2066/223893 objects misplaced (0.923%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:19:00.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3790745260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:00.357 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:00 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:00.358 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2364194120' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:19:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:00 smithi055 ceph-mon[179668]: pgmap v1410: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 823 op/s; 19697/223893 objects degraded (8.798%); 2066/223893 objects misplaced (0.923%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:19:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3790745260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:00 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:00.358 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2364194120' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:19:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:00 smithi055 ceph-mon[184174]: pgmap v1410: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 51 MiB/s wr, 823 op/s; 19697/223893 objects degraded (8.798%); 2066/223893 objects misplaced (0.923%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:19:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3790745260' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:00 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:00.359 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2364194120' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:19:00.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:00.502 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:00.573 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:00.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:00.581 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:00.642 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:19:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:19:00] "GET /metrics HTTP/1.1" 200 36697 "" "Prometheus/2.43.0" 2023-09-23T18:19:00.682 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:00.751 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:00.968 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 59 16 78180 78164 82.7708 50.0625 0.0295107 0.0120766 2023-09-23T18:19:00.978 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:00.983 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:00.984 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:01.085 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:01.155 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:01.265 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:19:01.327 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:01.332 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:01.333 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:01.333 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:01.333 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[184174]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.333 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:19:01.334 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2364194120' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:19:01.334 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[184174]: osdmap e601: 8 total, 8 up, 6 in 2023-09-23T18:19:01.334 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[184174]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.334 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[184174]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.334 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[184174]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.334 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:01.334 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[179668]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.334 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:19:01.335 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2364194120' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:19:01.335 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[179668]: osdmap e601: 8 total, 8 up, 6 in 2023-09-23T18:19:01.335 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[179668]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.335 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[179668]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.335 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:01 smithi055 ceph-mon[179668]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.433 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:01.503 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:01.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:01 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:01.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:01 smithi099 ceph-mon[167076]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:01 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:19:01.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:01 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2364194120' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:19:01.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:01 smithi099 ceph-mon[167076]: osdmap e601: 8 total, 8 up, 6 in 2023-09-23T18:19:01.546 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:01 smithi099 ceph-mon[167076]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.547 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:01 smithi099 ceph-mon[167076]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.547 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:01 smithi099 ceph-mon[167076]: failed to encode map e601 with expected crc 2023-09-23T18:19:01.558 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T18:19:01.558 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:19:01.583 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:01.593 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:01.593 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:01.694 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:01.762 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:01.881 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:01.888 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:01.888 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:01.968 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:19:01.969484+0000 min lat: 0.00128263 max lat: 2.2401 avg lat: 0.0121595 2023-09-23T18:19:01.968 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:19:01.969 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 60 16 78958 78942 82.2018 48.625 0.0089882 0.0121595 2023-09-23T18:19:01.989 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:02.037 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:02.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:02.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:02.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:02.137 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:02.138 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:02 smithi099 ceph-mon[167076]: pgmap v1412: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 708 op/s; 19697/228261 objects degraded (8.629%); 2066/228261 objects misplaced (0.905%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:19:02.138 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:02 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:02.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:02.250 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:02.338 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:02.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:02.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:02.377 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:19:02.414 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:19:02.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:02.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:02 smithi055 ceph-mon[179668]: pgmap v1412: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 708 op/s; 19697/228261 objects degraded (8.629%); 2066/228261 objects misplaced (0.905%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:19:02.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:02 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:02.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:02.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:02 smithi055 ceph-mon[184174]: pgmap v1412: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.6 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 44 MiB/s wr, 708 op/s; 19697/228261 objects degraded (8.629%); 2066/228261 objects misplaced (0.905%); 3.5 MiB/s, 55 objects/s recovering 2023-09-23T18:19:02.435 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:02 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:02.444 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:02.449 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:19:02.503 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:19:02.536 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:02.564 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:19:02.601 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:19:02.614 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:02.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:02.621 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:02.655 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:19:02.679 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:02.697 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:19:02.704 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:19:02.704 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:19:02.705 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:19:02.705 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:19:02.705 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:19:02.714 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:19:02.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:19:02.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:19:02.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:19:02.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:19:02.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:19:02.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:19:02.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:19:02.722 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:02.797 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:02.871 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:02.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:02.877 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:02.947 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:19:02.968 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 61 16 79756 79740 81.6719 49.875 0.00623957 0.0122386 2023-09-23T18:19:02.978 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:03.056 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:03.148 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:19:03.292 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:03.302 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2590308311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:03.304 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2590308311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:03.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:03.306 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:03.406 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:03.478 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:03.560 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:03.568 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:03.569 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:03.578 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2590308311' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:03.670 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:03.756 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:03.823 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:03.837 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:03.837 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:03.892 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:19:03.938 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:03.968 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 62 16 80521 80505 81.1258 47.8125 0.0129162 0.0123193 2023-09-23T18:19:03.983 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:04.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:04.043 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:04.043 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:04.129 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:04.144 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:04.158 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:04 smithi055 ceph-mon[179668]: pgmap v1413: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 19697/234201 objects degraded (8.410%); 2066/234201 objects misplaced (0.882%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:19:04.158 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:04 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19697/234201 objects degraded (8.410%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:04.187 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:04.244 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:04.252 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:04.253 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:04.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:04 smithi099 ceph-mon[167076]: pgmap v1413: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 19697/234201 objects degraded (8.410%); 2066/234201 objects misplaced (0.882%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:19:04.257 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:04 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19697/234201 objects degraded (8.410%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:04.354 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:04.425 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:04 smithi055 ceph-mon[184174]: pgmap v1413: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 19697/234201 objects degraded (8.410%); 2066/234201 objects misplaced (0.882%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:19:04.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:04 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19697/234201 objects degraded (8.410%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:04.697 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:04.704 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:04.705 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:04.806 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:04.875 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:04.969 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 63 16 81319 81303 80.6293 49.875 0.0291054 0.0123982 2023-09-23T18:19:05.011 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:05.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:05.019 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:05.120 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:05.207 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:05.332 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:05.342 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:19:05.346 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:05.347 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:05.419 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:19:05.419 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:19:05.419 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:19:05.419 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:19:05.419 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.419 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:140311 9.f 9:ff5e59f6:::benchmark_data_smithi055_386467_object35651:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T18:19:05.420 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:16:08.274063+0000", 2023-09-23T18:19:05.420 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 177.14086728999999, 2023-09-23T18:19:05.420 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.081975385, 2023-09-23T18:19:05.420 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.420 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.420 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.420 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.420 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.420 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 140311 2023-09-23T18:19:05.420 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274063+0000", 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274063+0000", 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.421 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.422 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.422 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.422 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274063+0000", 2023-09-23T18:19:05.422 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:05.422 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.422 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.422 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.422 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274065+0000", 2023-09-23T18:19:05.422 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9539999999999998e-06 2023-09-23T18:19:05.422 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274070+0000", 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6969999999999998e-06 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274074+0000", 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7419999999999999e-06 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.423 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.424 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.424 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.279237+0000", 2023-09-23T18:19:05.424 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051623440000000001 2023-09-23T18:19:05.424 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.424 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.424 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:19:05.424 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.279244+0000", 2023-09-23T18:19:05.424 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5429999999999996e-06 2023-09-23T18:19:05.424 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.424 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.283291+0000", 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040470569999999997 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.283299+0000", 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3560000000000004e-06 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.425 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.426 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.339507+0000", 2023-09-23T18:19:05.426 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0562083 2023-09-23T18:19:05.426 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.426 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.426 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.426 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.339618+0000", 2023-09-23T18:19:05.426 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111382 2023-09-23T18:19:05.426 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.426 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.426 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.339717+0000", 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8845999999999995e-05 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.348179+0000", 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0084614519999999995 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.353282+0000", 2023-09-23T18:19:05.427 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051035469999999999 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.355898+0000", 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026161159999999999 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.355945+0000", 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6841000000000001e-05 2023-09-23T18:19:05.428 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.356039+0000", 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3576e-05 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:140325 9.f 9:f04bcc13:::benchmark_data_smithi055_386467_object35665:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T18:19:05.429 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:16:08.274424+0000", 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 177.14050603499999, 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0817564, 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 140325 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.430 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274424+0000", 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274424+0000", 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.431 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274424+0000", 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274428+0000", 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1579999999999998e-06 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274437+0000", 2023-09-23T18:19:05.432 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7469999999999994e-06 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274442+0000", 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6319999999999996e-06 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.279295+0000", 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048526280000000003 2023-09-23T18:19:05.433 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.279302+0000", 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2400000000000001e-06 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.283320+0000", 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040175460000000003 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.434 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.283327+0000", 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2640000000000002e-06 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.342129+0000", 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0588019900000001 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.342226+0000", 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7421000000000001e-05 2023-09-23T18:19:05.435 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.436 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:19:05.436 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.342329+0000", 2023-09-23T18:19:05.436 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102264 2023-09-23T18:19:05.436 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.436 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.436 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.436 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.353252+0000", 2023-09-23T18:19:05.436 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010923042000000001 2023-09-23T18:19:05.436 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.355890+0000", 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026384709999999999 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.356046+0000", 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000155501 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.437 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.356085+0000", 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8974000000000003e-05 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.356181+0000", 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6189999999999999e-05 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.438 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199290 9.3 9:cbcdf350:::benchmark_data_smithi055_386467_object94630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.715714+0000", 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.69921621100001, 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2707276919999999, 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199290 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.439 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715714+0000", 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715714+0000", 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.440 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715714+0000", 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715717+0000", 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0599999999999999e-06 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715723+0000", 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5889999999999999e-06 2023-09-23T18:19:05.441 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715728+0000", 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5909999999999998e-06 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716971+0000", 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012426220000000001 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.442 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717081+0000", 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110571 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717196+0000", 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115203 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.443 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.722675+0000", 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054787109999999998 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.723307+0000", 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063171099999999997 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.755425+0000", 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032118345999999999 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.444 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.755435+0000", 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6560000000000004e-06 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.986367+0000", 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2309317420000001 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.445 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.986442+0000", 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5337000000000001e-05 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199294 9.8 9:12efe5c6:::benchmark_data_smithi055_386467_object94634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.716893+0000", 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.69803783099999, 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2851745619999999, 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.446 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199294 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716893+0000", 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.447 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716893+0000", 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716892+0000", 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.448 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716897+0000", 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4859999999999997e-06 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716912+0000", 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5034e-05 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716919+0000", 2023-09-23T18:19:05.449 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3840000000000002e-06 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716961+0000", 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2438999999999998e-05 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717078+0000", 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117118 2023-09-23T18:19:05.450 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717194+0000", 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011586699999999999 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.722355+0000", 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051609560000000004 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.753140+0000", 2023-09-23T18:19:05.451 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.030785605000000001 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.753151+0000", 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0198e-05 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.001993+0000", 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2488427980000001 2023-09-23T18:19:05.452 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.453 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.453 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.453 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.002067+0000", 2023-09-23T18:19:05.453 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3726999999999994e-05 2023-09-23T18:19:05.453 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.453 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.453 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.453 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.453 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.453 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199295 9.5 9:ad8338c2:::benchmark_data_smithi055_386467_object94635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.716932+0000", 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.69799855300001, 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.305777551, 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199295 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.454 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716932+0000", 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716932+0000", 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.455 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716931+0000", 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716934+0000", 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0259999999999999e-06 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716941+0000", 2023-09-23T18:19:05.456 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0599999999999996e-06 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716952+0000", 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1178e-05 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716992+0000", 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0167999999999999e-05 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.457 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717094+0000", 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000101951 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717198+0000", 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104474 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.458 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.723586+0000", 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0063876339999999997 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.724705+0000", 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001118775 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771588+0000", 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.046883263000000001 2023-09-23T18:19:05.459 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771596+0000", 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2490000000000001e-06 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022627+0000", 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2510307110000001 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.460 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022709+0000", 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2565999999999995e-05 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199297 9.6 9:6d14a21c:::benchmark_data_smithi055_386467_object94637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.742942+0000", 2023-09-23T18:19:05.461 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.67198811999999, 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1789063959999999, 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199297 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.462 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742942+0000", 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742942+0000", 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742941+0000", 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:05.463 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742948+0000", 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6979999999999998e-06 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742960+0000", 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1433999999999999e-05 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.464 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742968+0000", 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7980000000000006e-06 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.751116+0000", 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.008147886 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.751128+0000", 2023-09-23T18:19:05.465 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2094e-05 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.759543+0000", 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0084149099999999 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.759560+0000", 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7638e-05 2023-09-23T18:19:05.466 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.903201+0000", 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1436407420000001 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.903324+0000", 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123025 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:19:05.467 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.904060+0000", 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00073632600000000004 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.916453+0000", 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012393131 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921602+0000", 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051480709999999997 2023-09-23T18:19:05.468 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921664+0000", 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2304999999999994e-05 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921726+0000", 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1891999999999996e-05 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.469 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921849+0000", 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123044 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199298 9.7 9:e7115a81:::benchmark_data_smithi055_386467_object94638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.742975+0000", 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.671955649, 2023-09-23T18:19:05.470 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3049147640000001, 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199298 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742975+0000", 2023-09-23T18:19:05.471 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742975+0000", 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742974+0000", 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.472 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742977+0000", 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0570000000000001e-06 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742983+0000", 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0009999999999998e-06 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.473 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742987+0000", 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9190000000000001e-06 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.772191+0000", 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.029203289 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.772200+0000", 2023-09-23T18:19:05.474 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1079999999999999e-06 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.015514+0000", 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.243313772 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.015648+0000", 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013492800000000001 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.475 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6,7", 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.015776+0000", 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012704 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.018114+0000", 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023388879999999999 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047670+0000", 2023-09-23T18:19:05.476 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.029555352999999999 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047711+0000", 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0816999999999999e-05 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047773+0000", 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1932000000000003e-05 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.477 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047820+0000", 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7136999999999999e-05 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047890+0000", 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9951999999999997e-05 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.478 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199299 9.d 9:b372b2c2:::benchmark_data_smithi055_386467_object94639:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.742992+0000", 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.671938263, 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.280050796, 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199299 2023-09-23T18:19:05.479 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742992+0000", 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742992+0000", 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.480 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742992+0000", 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742994+0000", 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6060000000000001e-06 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.481 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.743000+0000", 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8429999999999998e-06 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.743004+0000", 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0570000000000003e-06 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771616+0000", 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028612047000000002 2023-09-23T18:19:05.482 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771622+0000", 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9630000000000003e-06 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.985340+0000", 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2137177180000001 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.483 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.985469+0000", 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012850700000000001 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.985617+0000", 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000148553 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.987979+0000", 2023-09-23T18:19:05.484 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023620759999999998 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022765+0000", 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.034785592999999997 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022835+0000", 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0127000000000001e-05 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.485 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022868+0000", 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3479999999999998e-05 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022920+0000", 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1329000000000003e-05 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.486 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023043+0000", 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012324699999999999 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199303 9.f 9:f765a439:::benchmark_data_smithi055_386467_object94643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.756527+0000", 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.658403533, 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2992920190000001, 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.487 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199303 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.756527+0000", 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.488 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.756527+0000", 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.756526+0000", 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.756531+0000", 2023-09-23T18:19:05.489 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1900000000000003e-06 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757137+0000", 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00060600400000000005 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757147+0000", 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8700000000000004e-06 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.490 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.772195+0000", 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015048598 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.772201+0000", 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7409999999999998e-06 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.491 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.045405+0000", 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2732037970000001 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.045497+0000", 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1938e-05 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.045590+0000", 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2592999999999999e-05 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.492 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047756+0000", 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021669079999999999 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.048160+0000", 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00040372800000000001 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.493 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.048189+0000", 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9183000000000002e-05 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.055687+0000", 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0074980619999999998 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.055734+0000", 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6418000000000002e-05 2023-09-23T18:19:05.494 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.055819+0000", 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5177000000000001e-05 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199304 9.d 9:bc343161:::benchmark_data_smithi055_386467_object94644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.757155+0000", 2023-09-23T18:19:05.495 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.657775683, 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.266180844, 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199304 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757155+0000", 2023-09-23T18:19:05.496 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757155+0000", 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757154+0000", 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.497 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757159+0000", 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8539999999999996e-06 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757168+0000", 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6819999999999992e-06 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.498 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757173+0000", 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9710000000000003e-06 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771640+0000", 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014467617 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771648+0000", 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7659999999999998e-06 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.499 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.988003+0000", 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.216355106 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.988116+0000", 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011312899999999999 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T18:19:05.500 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.988249+0000", 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000133106 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.992664+0000", 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004414327 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022771+0000", 2023-09-23T18:19:05.501 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.030107643999999999 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023122+0000", 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00035060100000000003 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023168+0000", 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5799e-05 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.502 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023219+0000", 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1851999999999998e-05 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023336+0000", 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116197 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.503 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199306 9.5 9:a2b69264:::benchmark_data_smithi055_386467_object94646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.765909+0000", 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.649021323, 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2612426189999999, 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199306 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.504 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765909+0000", 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765909+0000", 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.505 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765908+0000", 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765914+0000", 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3640000000000004e-06 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765926+0000", 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1460999999999999e-05 2023-09-23T18:19:05.506 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765935+0000", 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7630000000000006e-06 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771666+0000", 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057316600000000004 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.507 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771672+0000", 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8379999999999998e-06 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023404+0000", 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.251732098 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023529+0000", 2023-09-23T18:19:05.508 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012490900000000001 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023682+0000", 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015258099999999999 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.026185+0000", 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025030669999999999 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.509 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.026377+0000", 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019275800000000001 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.026900+0000", 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00052258399999999998 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.510 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.026994+0000", 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3902999999999999e-05 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.027043+0000", 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8639e-05 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.027152+0000", 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000109197 2023-09-23T18:19:05.511 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199307 9.6 9:60908162:::benchmark_data_smithi055_386467_object94647:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.765940+0000", 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.648990398, 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1561716029999998, 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.512 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199307 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765940+0000", 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765940+0000", 2023-09-23T18:19:05.513 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765940+0000", 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765943+0000", 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1099999999999999e-06 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.514 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765949+0000", 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2169999999999999e-06 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765953+0000", 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.788e-06 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.767010+0000", 2023-09-23T18:19:05.515 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001057058 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.767020+0000", 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9089999999999995e-06 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.759589+0000", 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99256918599999999 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.516 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.759597+0000", 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2090000000000005e-06 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.906374+0000", 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.146776995 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.517 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.906482+0000", 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000107633 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.906617+0000", 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000135341 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.916497+0000", 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.009879489 2023-09-23T18:19:05.518 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921610+0000", 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051130739999999996 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921949+0000", 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00033926300000000002 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.519 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.922000+0000", 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0912999999999999e-05 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.922112+0000", 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111897 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.520 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199308 9.8 9:1df43a87:::benchmark_data_smithi055_386467_object94648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.765958+0000", 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 105.64897293999999, 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.244589932, 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199308 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.521 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765958+0000", 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765958+0000", 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.522 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765957+0000", 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765960+0000", 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6749999999999998e-06 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765965+0000", 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6080000000000003e-06 2023-09-23T18:19:05.523 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765969+0000", 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8550000000000004e-06 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.770519+0000", 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045492950000000001 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.524 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.770535+0000", 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6084000000000001e-05 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.002452+0000", 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2319171680000001 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.004743+0000", 2023-09-23T18:19:05.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022912919999999999 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.004876+0000", 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013298100000000001 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.007521+0000", 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026448959999999999 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.007529+0000", 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8650000000000001e-06 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.007964+0000", 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000434976 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.527 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.010368+0000", 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002403793 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.010420+0000", 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2784999999999997e-05 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.010547+0000", 2023-09-23T18:19:05.528 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127081 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67151 10.3 10:c85af3e6:::benchmark_data_smithi055_425050_object67150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.355657+0000", 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 20.059273388000001, 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2389400689999999, 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:19:05.529 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67151 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355657+0000", 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.530 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355657+0000", 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355655+0000", 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.531 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355688+0000", 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2824999999999998e-05 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355702+0000", 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3193999999999999e-05 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355707+0000", 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.976e-06 2023-09-23T18:19:05.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355770+0000", 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2339000000000001e-05 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355958+0000", 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018780000000000001 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.533 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.356087+0000", 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000129383 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.358394+0000", 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023073450000000001 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.361318+0000", 2023-09-23T18:19:05.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029238490000000001 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.424224+0000", 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.062906161000000002 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.424237+0000", 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3063e-05 2023-09-23T18:19:05.535 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426092+0000", 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0018549569999999 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426106+0000", 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3603e-05 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.536 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.594476+0000", 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.16837009 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.594597+0000", 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121097 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67164 10.3 10:c93b5b80:::benchmark_data_smithi055_425050_object67163:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.428830+0000", 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.986100635, 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2067956679999998, 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67164 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.538 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428830+0000", 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428830+0000", 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428829+0000", 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428877+0000", 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8142000000000001e-05 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428889+0000", 2023-09-23T18:19:05.540 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.253e-05 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428895+0000", 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9349999999999999e-06 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428931+0000", 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5407999999999997e-05 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.541 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428938+0000", 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1430000000000001e-06 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426131+0000", 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99719329800000001 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426138+0000", 2023-09-23T18:19:05.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8120000000000002e-06 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.597584+0000", 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1714459260000001 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.597724+0000", 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014053199999999999 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.543 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.597929+0000", 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000204448 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604471+0000", 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065424530000000002 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604486+0000", 2023-09-23T18:19:05.544 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4440000000000001e-05 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.632642+0000", 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028155961 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.632690+0000", 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7898000000000001e-05 2023-09-23T18:19:05.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.635626+0000", 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029359920000000001 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67171 10.3 10:cede541a:::benchmark_data_smithi055_425050_object67170:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.435663+0000", 2023-09-23T18:19:05.546 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.979267838999998, 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2024671709999999, 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67171 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435663+0000", 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435663+0000", 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435662+0000", 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:05.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435759+0000", 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6586e-05 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435763+0000", 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4680000000000003e-06 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.549 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435766+0000", 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0400000000000001e-06 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435799+0000", 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2561000000000002e-05 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435808+0000", 2023-09-23T18:19:05.550 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3749999999999992e-06 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426159+0000", 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99035037599999998 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426179+0000", 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0211999999999999e-05 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.600032+0000", 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.173853203 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.600141+0000", 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0001093 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.600325+0000", 2023-09-23T18:19:05.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000183265 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604495+0000", 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041700979999999997 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604538+0000", 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2914000000000003e-05 2023-09-23T18:19:05.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.635675+0000", 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.031136938999999999 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.635729+0000", 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4243000000000002e-05 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.554 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.638130+0000", 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002400976 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67175 10.3 10:c0e0e754:::benchmark_data_smithi055_425050_object67174:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.435944+0000", 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.978986133999999, 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2040770759999999, 2023-09-23T18:19:05.555 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67175 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435944+0000", 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.556 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435944+0000", 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435944+0000", 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.557 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.436036+0000", 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2362999999999996e-05 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.436045+0000", 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4010000000000003e-06 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.436050+0000", 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1229999999999999e-06 2023-09-23T18:19:05.558 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.436098+0000", 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7287000000000003e-05 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.436111+0000", 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3691e-05 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.559 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426201+0000", 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99008958800000002 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426209+0000", 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5310000000000004e-06 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.602179+0000", 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.17597065 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.602335+0000", 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015603000000000001 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.602509+0000", 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017391300000000001 2023-09-23T18:19:05.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604574+0000", 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020652299999999999 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604583+0000", 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8980000000000002e-06 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.638159+0000", 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.033576175 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.638195+0000", 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5228999999999998e-05 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.563 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.640021+0000", 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001826725 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67180 10.3 10:cb0581d8:::benchmark_data_smithi055_425050_object67179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.445326+0000", 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.969604736000001, 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1972762810000002, 2023-09-23T18:19:05.564 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67180 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445326+0000", 2023-09-23T18:19:05.565 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445326+0000", 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445325+0000", 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.566 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445438+0000", 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113678 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445449+0000", 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0993e-05 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.567 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445454+0000", 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1089999999999997e-06 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445502+0000", 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7259999999999998e-05 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445508+0000", 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.297e-06 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426229+0000", 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98072124400000005 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426236+0000", 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9489999999999999e-06 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604614+0000", 2023-09-23T18:19:05.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1783773929999999 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604709+0000", 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5921999999999994e-05 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604853+0000", 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014312299999999999 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.570 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.611827+0000", 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069741430000000004 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.612776+0000", 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00094902099999999998 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.571 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.640063+0000", 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.027287227000000001 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.640112+0000", 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9088000000000002e-05 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.642602+0000", 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024899879999999998 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.572 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67205 10.3 10:ce268af2:::benchmark_data_smithi055_425050_object67204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.474799+0000", 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.940131222000002, 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.170461601, 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:19:05.573 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67205 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474799+0000", 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474799+0000", 2023-09-23T18:19:05.574 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474797+0000", 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999971 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474893+0000", 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6348000000000007e-05 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.575 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474902+0000", 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0440000000000001e-06 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474906+0000", 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0759999999999996e-06 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474968+0000", 2023-09-23T18:19:05.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1721000000000001e-05 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474981+0000", 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.289e-05 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426255+0000", 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95127468599999998 2023-09-23T18:19:05.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426262+0000", 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8589999999999999e-06 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.606693+0000", 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1804307679999999 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.606822+0000", 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012950400000000001 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.607021+0000", 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019809800000000001 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.611833+0000", 2023-09-23T18:19:05.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048126979999999998 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.612807+0000", 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00097327799999999995 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.642656+0000", 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.02984966 2023-09-23T18:19:05.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.642717+0000", 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0656000000000003e-05 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.645261+0000", 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025439849999999999 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67218 10.3 10:c5b8088f:::benchmark_data_smithi055_425050_object67217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.502196+0000", 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 19.912734706999998, 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1458930650000001, 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67218 2023-09-23T18:19:05.582 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502196+0000", 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502196+0000", 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.583 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502194+0000", 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502247+0000", 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2800000000000003e-05 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:19:05.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502262+0000", 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4459e-05 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502267+0000", 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2599999999999996e-06 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502317+0000", 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0407e-05 2023-09-23T18:19:05.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502324+0000", 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7889999999999997e-06 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426283+0000", 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.92395870300000005 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426290+0000", 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8480000000000003e-06 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.609467+0000", 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.183177476 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:19:05.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.609609+0000", 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000141927 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.609777+0000", 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016761200000000001 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.612002+0000", 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022256839999999999 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.612942+0000", 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00093938799999999999 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.645322+0000", 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032380474999999999 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:19:05.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.645382+0000", 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0148000000000001e-05 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.648089+0000", 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027063980000000001 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:19:05.590 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:19:05.591 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:05.708 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:05.849 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:19:05.858 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:05.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:05.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:05.919 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:19:05.947 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:19:05.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:05.975 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 64 16 82013 81997 80.0401 43.375 0.0261724 0.0124884 2023-09-23T18:19:05.996 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:06.037 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:05 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:06.037 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:05 smithi055 ceph-mon[179668]: pgmap v1414: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 19697/235653 objects degraded (8.358%); 1825/235653 objects misplaced (0.774%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T18:19:06.038 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:06.038 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:05 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:06.038 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:05 smithi055 ceph-mon[184174]: pgmap v1414: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 19697/235653 objects degraded (8.358%); 1825/235653 objects misplaced (0.774%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T18:19:06.038 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:06.038 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:05 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:06.038 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:05 smithi099 ceph-mon[167076]: pgmap v1414: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 4.8 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 19697/235653 objects degraded (8.358%); 1825/235653 objects misplaced (0.774%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T18:19:06.039 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:06.065 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:06.071 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:06.072 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:06.172 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:06.206 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:06.253 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:06.264 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:06.264 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:06.365 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:06.443 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:06.464 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1414,"stamp":"2023-09-23T18:19:05.120277+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5150620324,"num_objects":78551,"num_object_clones":0,"num_object_copies":235653,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":19697,"num_objects_misplaced":1825,"num_objects_unfound":0,"num_objects_dirty":78551,"num_whiteouts":0,"num_read":5959,"num_read_kb":12160,"num_write":79435,"num_write_kb":5038298,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2031,"num_bytes_recovered":122773129,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":79946,"ondisk_log_size":79946,"up":357,"acting":350,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":379,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18394696,"kb_used_data":15215768,"kb_used_omap":40,"kb_used_meta":3178839,"kb_avail":544076216,"statfs":{"total":575970213888,"available":557134045184,"internally_reserved":0,"allocated":15580946432,"data_stored":15566619777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3255131450},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":528482304,"num_objects":8064,"num_object_clones":0,"num_object_copies":24192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-509,"num_objects_unfound":0,"num_objects_dirty":8064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8064,"num_write_kb":516096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":509,"num_bytes_recovered":33357824,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003964"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":889,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520673+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:18:52.520673+0000","last_peered":"2023-09-23T18:18:52.520673+0000","last_clean":"2023-09-23T18:18:52.520673+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:18:52.520673+0000","last_undegraded":"2023-09-23T18:18:52.520673+0000","last_fullsized":"2023-09-23T18:18:52.520673+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":260,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.524093+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:18:52.524093+0000","last_peered":"2023-09-23T18:18:52.524093+0000","last_clean":"2023-09-23T18:18:52.524093+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:18:52.524093+0000","last_undegraded":"2023-09-23T18:18:52.524093+0000","last_fullsized":"2023-09-23T18:18:52.524093+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":17,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.541127+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:18:52.541127+0000","last_peered":"2023-09-23T18:18:52.541127+0000","last_clean":"2023-09-23T18:18:52.541127+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:18:52.541127+0000","last_undegraded":"2023-09-23T18:18:52.541127+0000","last_fullsized":"2023-09-23T18:18:52.541127+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":895,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521036+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:18:52.521036+0000","last_peered":"2023-09-23T18:18:52.521036+0000","last_clean":"2023-09-23T18:18:52.521036+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:18:52.521036+0000","last_undegraded":"2023-09-23T18:18:52.521036+0000","last_fullsized":"2023-09-23T18:18:52.521036+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":341,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.540031+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:18:52.540031+0000","last_peered":"2023-09-23T18:18:52.540031+0000","last_clean":"2023-09-23T18:18:52.540031+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:18:52.540031+0000","last_undegraded":"2023-09-23T18:18:52.540031+0000","last_fullsized":"2023-09-23T18:18:52.540031+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":661,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.533541+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:18:52.533541+0000","last_peered":"2023-09-23T18:18:52.533541+0000","last_clean":"2023-09-23T18:18:52.533541+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:18:52.533541+0000","last_undegraded":"2023-09-23T18:18:52.533541+0000","last_fullsized":"2023-09-23T18:18:52.533541+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":912,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521670+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:18:52.521670+0000","last_peered":"2023-09-23T18:18:52.521670+0000","last_clean":"2023-09-23T18:18:52.521670+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:18:52.521670+0000","last_undegraded":"2023-09-23T18:18:52.521670+0000","last_fullsized":"2023-09-23T18:18:52.521670+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":17,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.538685+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:18:52.538685+0000","last_peered":"2023-09-23T18:18:52.538685+0000","last_clean":"2023-09-23T18:18:52.538685+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:18:52.538685+0000","last_undegraded":"2023-09-23T18:18:52.538685+0000","last_fullsized":"2023-09-23T18:18:52.538685+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":177,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.538337+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:18:52.538337+0000","last_peered":"2023-09-23T18:18:52.538337+0000","last_clean":"2023-09-23T18:18:52.538337+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:18:52.538337+0000","last_undegraded":"2023-09-23T18:18:52.538337+0000","last_fullsized":"2023-09-23T18:18:52.538337+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1035,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521274+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:18:52.521274+0000","last_peered":"2023-09-23T18:18:52.521274+0000","last_clean":"2023-09-23T18:18:52.521274+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:18:52.521274+0000","last_undegraded":"2023-09-23T18:18:52.521274+0000","last_fullsized":"2023-09-23T18:18:52.521274+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":111,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521834+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:18:52.521834+0000","last_peered":"2023-09-23T18:18:52.521834+0000","last_clean":"2023-09-23T18:18:52.521834+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:18:52.521834+0000","last_undegraded":"2023-09-23T18:18:52.521834+0000","last_fullsized":"2023-09-23T18:18:52.521834+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":190,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.539092+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:18:52.539092+0000","last_peered":"2023-09-23T18:18:52.539092+0000","last_clean":"2023-09-23T18:18:52.539092+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:18:52.539092+0000","last_undegraded":"2023-09-23T18:18:52.539092+0000","last_fullsized":"2023-09-23T18:18:52.539092+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":812,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536305+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:18:52.536305+0000","last_peered":"2023-09-23T18:18:52.536305+0000","last_clean":"2023-09-23T18:18:52.536305+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:18:52.536305+0000","last_undegraded":"2023-09-23T18:18:52.536305+0000","last_fullsized":"2023-09-23T18:18:52.536305+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":147,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520844+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:18:52.520844+0000","last_peered":"2023-09-23T18:18:52.520844+0000","last_clean":"2023-09-23T18:18:52.520844+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:18:52.520844+0000","last_undegraded":"2023-09-23T18:18:52.520844+0000","last_fullsized":"2023-09-23T18:18:52.520844+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":146,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.534888+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:18:52.534888+0000","last_peered":"2023-09-23T18:18:52.534888+0000","last_clean":"2023-09-23T18:18:52.534888+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:18:52.534888+0000","last_undegraded":"2023-09-23T18:18:52.534888+0000","last_fullsized":"2023-09-23T18:18:52.534888+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":946,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521888+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:18:52.521888+0000","last_peered":"2023-09-23T18:18:52.521888+0000","last_clean":"2023-09-23T18:18:52.521888+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:18:52.521888+0000","last_undegraded":"2023-09-23T18:18:52.521888+0000","last_fullsized":"2023-09-23T18:18:52.521888+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":394,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.541491+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:18:52.541491+0000","last_peered":"2023-09-23T18:18:52.541491+0000","last_clean":"2023-09-23T18:18:52.541491+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:18:52.541491+0000","last_undegraded":"2023-09-23T18:18:52.541491+0000","last_fullsized":"2023-09-23T18:18:52.541491+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":414,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536255+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:18:52.536255+0000","last_peered":"2023-09-23T18:18:52.536255+0000","last_clean":"2023-09-23T18:18:52.536255+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:18:52.536255+0000","last_undegraded":"2023-09-23T18:18:52.536255+0000","last_fullsized":"2023-09-23T18:18:52.536255+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":974,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.550250+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:18:52.550250+0000","last_peered":"2023-09-23T18:18:52.550250+0000","last_clean":"2023-09-23T18:18:52.550250+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:18:52.550250+0000","last_undegraded":"2023-09-23T18:18:52.550250+0000","last_fullsized":"2023-09-23T18:18:52.550250+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":153,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536863+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:18:52.536863+0000","last_peered":"2023-09-23T18:18:52.536863+0000","last_clean":"2023-09-23T18:18:52.536863+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:18:52.536863+0000","last_undegraded":"2023-09-23T18:18:52.536863+0000","last_fullsized":"2023-09-23T18:18:52.536863+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":420,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.537673+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:18:52.537673+0000","last_peered":"2023-09-23T18:18:52.537673+0000","last_clean":"2023-09-23T18:18:52.537673+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:18:52.537673+0000","last_undegraded":"2023-09-23T18:18:52.537673+0000","last_fullsized":"2023-09-23T18:18:52.537673+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":827,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520777+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:18:52.520777+0000","last_peered":"2023-09-23T18:18:52.520777+0000","last_clean":"2023-09-23T18:18:52.520777+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:18:52.520777+0000","last_undegraded":"2023-09-23T18:18:52.520777+0000","last_fullsized":"2023-09-23T18:18:52.520777+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":420,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.537532+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:18:52.537532+0000","last_peered":"2023-09-23T18:18:52.537532+0000","last_clean":"2023-09-23T18:18:52.537532+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:18:52.537532+0000","last_undegraded":"2023-09-23T18:18:52.537532+0000","last_fullsized":"2023-09-23T18:18:52.537532+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":98,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520687+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:18:52.520687+0000","last_peered":"2023-09-23T18:18:52.520687+0000","last_clean":"2023-09-23T18:18:52.520687+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:18:52.520687+0000","last_undegraded":"2023-09-23T18:18:52.520687+0000","last_fullsized":"2023-09-23T18:18:52.520687+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":798,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520393+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:18:52.520393+0000","last_peered":"2023-09-23T18:18:52.520393+0000","last_clean":"2023-09-23T18:18:52.520393+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:18:52.520393+0000","last_undegraded":"2023-09-23T18:18:52.520393+0000","last_fullsized":"2023-09-23T18:18:52.520393+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":116,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.534010+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:18:52.534010+0000","last_peered":"2023-09-23T18:18:52.534010+0000","last_clean":"2023-09-23T18:18:52.534010+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:18:52.534010+0000","last_undegraded":"2023-09-23T18:18:52.534010+0000","last_fullsized":"2023-09-23T18:18:52.534010+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":111,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521420+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:18:52.521420+0000","last_peered":"2023-09-23T18:18:52.521420+0000","last_clean":"2023-09-23T18:18:52.521420+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:18:52.521420+0000","last_undegraded":"2023-09-23T18:18:52.521420+0000","last_fullsized":"2023-09-23T18:18:52.521420+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":736,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.542022+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:18:52.542022+0000","last_peered":"2023-09-23T18:18:52.542022+0000","last_clean":"2023-09-23T18:18:52.542022+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:18:52.542022+0000","last_undegraded":"2023-09-23T18:18:52.542022+0000","last_fullsized":"2023-09-23T18:18:52.542022+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":188,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.534721+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:18:52.534721+0000","last_peered":"2023-09-23T18:18:52.534721+0000","last_clean":"2023-09-23T18:18:52.534721+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:18:52.534721+0000","last_undegraded":"2023-09-23T18:18:52.534721+0000","last_fullsized":"2023-09-23T18:18:52.534721+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":199,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.541602+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:18:52.541602+0000","last_peered":"2023-09-23T18:18:52.541602+0000","last_clean":"2023-09-23T18:18:52.541602+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:18:52.541602+0000","last_undegraded":"2023-09-23T18:18:52.541602+0000","last_fullsized":"2023-09-23T18:18:52.541602+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":832,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.535484+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:18:52.535484+0000","last_peered":"2023-09-23T18:18:52.535484+0000","last_clean":"2023-09-23T18:18:52.535484+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:18:52.535484+0000","last_undegraded":"2023-09-23T18:18:52.535484+0000","last_fullsized":"2023-09-23T18:18:52.535484+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":741,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.534690+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:18:52.534690+0000","last_peered":"2023-09-23T18:18:52.534690+0000","last_clean":"2023-09-23T18:18:52.534690+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:18:52.534690+0000","last_undegraded":"2023-09-23T18:18:52.534690+0000","last_fullsized":"2023-09-23T18:18:52.534690+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":17,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.541755+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:18:52.541755+0000","last_peered":"2023-09-23T18:18:52.541755+0000","last_clean":"2023-09-23T18:18:52.541755+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:18:52.541755+0000","last_undegraded":"2023-09-23T18:18:52.541755+0000","last_fullsized":"2023-09-23T18:18:52.541755+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"601'4929","reported_seq":4964,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:01.030670+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:19:01.030670+0000","last_peered":"2023-09-23T18:19:01.030670+0000","last_clean":"2023-09-23T18:19:01.030670+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:19:01.030670+0000","last_undegraded":"2023-09-23T18:19:01.030670+0000","last_fullsized":"2023-09-23T18:19:01.030670+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4929,"log_dups_size":0,"ondisk_log_size":4929,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":323026944,"num_objects":4929,"num_object_clones":0,"num_object_copies":14787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4929,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4929,"num_write_kb":315456,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":866,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536396+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:18:52.536396+0000","last_peered":"2023-09-23T18:18:52.536396+0000","last_clean":"2023-09-23T18:18:52.536396+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:18:52.536396+0000","last_undegraded":"2023-09-23T18:18:52.536396+0000","last_fullsized":"2023-09-23T18:18:52.536396+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":237,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.524488+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:18:52.524488+0000","last_peered":"2023-09-23T18:18:52.524488+0000","last_clean":"2023-09-23T18:18:52.524488+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:18:52.524488+0000","last_undegraded":"2023-09-23T18:18:52.524488+0000","last_fullsized":"2023-09-23T18:18:52.524488+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":111,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520387+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:18:52.520387+0000","last_peered":"2023-09-23T18:18:52.520387+0000","last_clean":"2023-09-23T18:18:52.520387+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:18:52.520387+0000","last_undegraded":"2023-09-23T18:18:52.520387+0000","last_fullsized":"2023-09-23T18:18:52.520387+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"601'4817","reported_seq":4861,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:01.166671+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:19:01.166671+0000","last_peered":"2023-09-23T18:19:01.166671+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:19:01.166671+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"578'400","ondisk_log_start":"578'400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4417,"log_dups_size":0,"ondisk_log_size":4417,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315686912,"num_objects":4817,"num_object_clones":0,"num_object_copies":14451,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4817,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4817,"num_write_kb":308288,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":4148},{"shards":"5,6","objects":669}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":726,"reported_epoch":600,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:18:52.548688+0000","last_change":"2023-09-23T18:18:47.441716+0000","last_active":"2023-09-23T18:18:52.548688+0000","last_peered":"2023-09-23T18:18:52.548688+0000","last_clean":"2023-09-23T18:11:33.614788+0000","last_became_active":"2023-09-23T18:18:47.441414+0000","last_became_peered":"2023-09-23T18:18:47.441414+0000","last_unstale":"2023-09-23T18:18:52.548688+0000","last_undegraded":"2023-09-23T18:18:47.413783+0000","last_fullsized":"2023-09-23T18:18:47.413246+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":2047528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[0,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":238,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.524698+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:18:52.524698+0000","last_peered":"2023-09-23T18:18:52.524698+0000","last_clean":"2023-09-23T18:18:52.524698+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:18:52.524698+0000","last_undegraded":"2023-09-23T18:18:52.524698+0000","last_fullsized":"2023-09-23T18:18:52.524698+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":217,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.523996+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:18:52.523996+0000","last_peered":"2023-09-23T18:18:52.523996+0000","last_clean":"2023-09-23T18:18:52.523996+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:18:52.523996+0000","last_undegraded":"2023-09-23T18:18:52.523996+0000","last_fullsized":"2023-09-23T18:18:52.523996+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"601'4991","reported_seq":8325,"reported_epoch":601,"state":"active+recovering+remapped","last_fresh":"2023-09-23T18:19:03.843588+0000","last_change":"2023-09-23T18:18:34.707552+0000","last_active":"2023-09-23T18:19:03.843588+0000","last_peered":"2023-09-23T18:19:03.843588+0000","last_clean":"2023-09-23T18:18:32.618842+0000","last_became_active":"2023-09-23T18:18:34.683472+0000","last_became_peered":"2023-09-23T18:18:34.683472+0000","last_unstale":"2023-09-23T18:19:03.843588+0000","last_undegraded":"2023-09-23T18:19:03.843588+0000","last_fullsized":"2023-09-23T18:19:03.843588+0000","mapping_epoch":589,"log_start":"578'600","ondisk_log_start":"578'600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4391,"log_dups_size":0,"ondisk_log_size":4391,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:31.959133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327090176,"num_objects":4991,"num_object_clones":0,"num_object_copies":14973,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1825,"num_objects_unfound":0,"num_objects_dirty":4991,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4991,"num_write_kb":319424,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1646,"num_bytes_recovered":107872256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":3166}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":616,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.538258+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:18:52.538258+0000","last_peered":"2023-09-23T18:18:52.538258+0000","last_clean":"2023-09-23T18:18:52.538258+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:18:52.538258+0000","last_undegraded":"2023-09-23T18:18:52.538258+0000","last_fullsized":"2023-09-23T18:18:52.538258+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":526,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521322+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:18:52.521322+0000","last_peered":"2023-09-23T18:18:52.521322+0000","last_clean":"2023-09-23T18:18:52.521322+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:18:52.521322+0000","last_undegraded":"2023-09-23T18:18:52.521322+0000","last_fullsized":"2023-09-23T18:18:52.521322+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":177,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536125+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:18:52.536125+0000","last_peered":"2023-09-23T18:18:52.536125+0000","last_clean":"2023-09-23T18:18:52.536125+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:18:52.536125+0000","last_undegraded":"2023-09-23T18:18:52.536125+0000","last_fullsized":"2023-09-23T18:18:52.536125+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"601'4976","reported_seq":5012,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:01.006710+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:19:01.006710+0000","last_peered":"2023-09-23T18:19:01.006710+0000","last_clean":"2023-09-23T18:19:01.006710+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:19:01.006710+0000","last_undegraded":"2023-09-23T18:19:01.006710+0000","last_fullsized":"2023-09-23T18:19:01.006710+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4976,"log_dups_size":0,"ondisk_log_size":4976,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":326107136,"num_objects":4976,"num_object_clones":0,"num_object_copies":14928,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4976,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4976,"num_write_kb":318464,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":631,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.541688+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:18:52.541688+0000","last_peered":"2023-09-23T18:18:52.541688+0000","last_clean":"2023-09-23T18:18:52.541688+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:18:52.541688+0000","last_undegraded":"2023-09-23T18:18:52.541688+0000","last_fullsized":"2023-09-23T18:18:52.541688+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":125,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521044+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:18:52.521044+0000","last_peered":"2023-09-23T18:18:52.521044+0000","last_clean":"2023-09-23T18:18:52.521044+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:18:52.521044+0000","last_undegraded":"2023-09-23T18:18:52.521044+0000","last_fullsized":"2023-09-23T18:18:52.521044+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":437,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.533455+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:18:52.533455+0000","last_peered":"2023-09-23T18:18:52.533455+0000","last_clean":"2023-09-23T18:18:52.533455+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:18:52.533455+0000","last_undegraded":"2023-09-23T18:18:52.533455+0000","last_fullsized":"2023-09-23T18:18:52.533455+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"601'5105","reported_seq":5141,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:03.823748+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:19:03.823748+0000","last_peered":"2023-09-23T18:19:03.823748+0000","last_clean":"2023-09-23T18:19:03.823748+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:19:03.823748+0000","last_undegraded":"2023-09-23T18:19:03.823748+0000","last_fullsized":"2023-09-23T18:19:03.823748+0000","mapping_epoch":574,"log_start":"578'700","ondisk_log_start":"578'700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4405,"log_dups_size":0,"ondisk_log_size":4405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":334561280,"num_objects":5105,"num_object_clones":0,"num_object_copies":15315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5105,"num_write_kb":326720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1083,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520679+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:18:52.520679+0000","last_peered":"2023-09-23T18:18:52.520679+0000","last_clean":"2023-09-23T18:18:52.520679+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:18:52.520679+0000","last_undegraded":"2023-09-23T18:18:52.520679+0000","last_fullsized":"2023-09-23T18:18:52.520679+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":337,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.542231+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:18:52.542231+0000","last_peered":"2023-09-23T18:18:52.542231+0000","last_clean":"2023-09-23T18:18:52.542231+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:18:52.542231+0000","last_undegraded":"2023-09-23T18:18:52.542231+0000","last_fullsized":"2023-09-23T18:18:52.542231+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":145,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536463+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:18:52.536463+0000","last_peered":"2023-09-23T18:18:52.536463+0000","last_clean":"2023-09-23T18:18:52.536463+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:18:52.536463+0000","last_undegraded":"2023-09-23T18:18:52.536463+0000","last_fullsized":"2023-09-23T18:18:52.536463+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"601'4926","reported_seq":4970,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:01.096051+0000","last_change":"2023-09-23T18:18:47.594759+0000","last_active":"2023-09-23T18:19:01.096051+0000","last_peered":"2023-09-23T18:19:01.096051+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:19:01.096051+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4926,"log_dups_size":0,"ondisk_log_size":4926,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322830336,"num_objects":4926,"num_object_clones":0,"num_object_copies":14778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4926,"num_write_kb":315264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4226},{"shards":"4,5","objects":700}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":870,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520583+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:18:52.520583+0000","last_peered":"2023-09-23T18:18:52.520583+0000","last_clean":"2023-09-23T18:18:52.520583+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:18:52.520583+0000","last_undegraded":"2023-09-23T18:18:52.520583+0000","last_fullsized":"2023-09-23T18:18:52.520583+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":153,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536044+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:18:52.536044+0000","last_peered":"2023-09-23T18:18:52.536044+0000","last_clean":"2023-09-23T18:18:52.536044+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:18:52.536044+0000","last_undegraded":"2023-09-23T18:18:52.536044+0000","last_fullsized":"2023-09-23T18:18:52.536044+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":452,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536533+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:18:52.536533+0000","last_peered":"2023-09-23T18:18:52.536533+0000","last_clean":"2023-09-23T18:18:52.536533+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:18:52.536533+0000","last_undegraded":"2023-09-23T18:18:52.536533+0000","last_fullsized":"2023-09-23T18:18:52.536533+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"601'4875","reported_seq":4910,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:01.108782+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:19:01.108782+0000","last_peered":"2023-09-23T18:19:01.108782+0000","last_clean":"2023-09-23T18:19:01.108782+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:19:01.108782+0000","last_undegraded":"2023-09-23T18:19:01.108782+0000","last_fullsized":"2023-09-23T18:19:01.108782+0000","mapping_epoch":574,"log_start":"578'400","ondisk_log_start":"578'400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4475,"log_dups_size":0,"ondisk_log_size":4475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":319488000,"num_objects":4875,"num_object_clones":0,"num_object_copies":14625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4875,"num_write_kb":312000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":872,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521121+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:18:52.521121+0000","last_peered":"2023-09-23T18:18:52.521121+0000","last_clean":"2023-09-23T18:18:52.521121+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:18:52.521121+0000","last_undegraded":"2023-09-23T18:18:52.521121+0000","last_fullsized":"2023-09-23T18:18:52.521121+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":437,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520534+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:18:52.520534+0000","last_peered":"2023-09-23T18:18:52.520534+0000","last_clean":"2023-09-23T18:18:52.520534+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:18:52.520534+0000","last_undegraded":"2023-09-23T18:18:52.520534+0000","last_fullsized":"2023-09-23T18:18:52.520534+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":213,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536419+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:18:52.536419+0000","last_peered":"2023-09-23T18:18:52.536419+0000","last_clean":"2023-09-23T18:18:52.536419+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:18:52.536419+0000","last_undegraded":"2023-09-23T18:18:52.536419+0000","last_fullsized":"2023-09-23T18:18:52.536419+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"601'4910","reported_seq":4945,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:01.211130+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:19:01.211130+0000","last_peered":"2023-09-23T18:19:01.211130+0000","last_clean":"2023-09-23T18:19:01.211130+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:19:01.211130+0000","last_undegraded":"2023-09-23T18:19:01.211130+0000","last_fullsized":"2023-09-23T18:19:01.211130+0000","mapping_epoch":574,"log_start":"578'300","ondisk_log_start":"578'300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4610,"log_dups_size":0,"ondisk_log_size":4610,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321781760,"num_objects":4910,"num_object_clones":0,"num_object_copies":14730,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4910,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4910,"num_write_kb":314240,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":853,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521885+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:18:52.521885+0000","last_peered":"2023-09-23T18:18:52.521885+0000","last_clean":"2023-09-23T18:18:52.521885+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:18:52.521885+0000","last_undegraded":"2023-09-23T18:18:52.521885+0000","last_fullsized":"2023-09-23T18:18:52.521885+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":17,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.538887+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:18:52.538887+0000","last_peered":"2023-09-23T18:18:52.538887+0000","last_clean":"2023-09-23T18:18:52.538887+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:18:52.538887+0000","last_undegraded":"2023-09-23T18:18:52.538887+0000","last_fullsized":"2023-09-23T18:18:52.538887+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":328,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.550690+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:18:52.550690+0000","last_peered":"2023-09-23T18:18:52.550690+0000","last_clean":"2023-09-23T18:18:52.550690+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:18:52.550690+0000","last_undegraded":"2023-09-23T18:18:52.550690+0000","last_fullsized":"2023-09-23T18:18:52.550690+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"601'4922","reported_seq":4960,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:00.108691+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:19:00.108691+0000","last_peered":"2023-09-23T18:19:00.108691+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:19:00.108691+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4922,"log_dups_size":0,"ondisk_log_size":4922,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":322568192,"num_objects":4922,"num_object_clones":0,"num_object_copies":14766,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4922,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4922,"num_write_kb":315008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":637},{"shards":"0,4,6","objects":4285}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":712,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.538420+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:18:52.538420+0000","last_peered":"2023-09-23T18:18:52.538420+0000","last_clean":"2023-09-23T18:18:52.538420+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:18:52.538420+0000","last_undegraded":"2023-09-23T18:18:52.538420+0000","last_fullsized":"2023-09-23T18:18:52.538420+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":950,"reported_epoch":600,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:18:52.549731+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:18:52.549731+0000","last_peered":"2023-09-23T18:18:52.549731+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:18:52.549731+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":135,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521461+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:18:52.521461+0000","last_peered":"2023-09-23T18:18:52.521461+0000","last_clean":"2023-09-23T18:18:52.521461+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:18:52.521461+0000","last_undegraded":"2023-09-23T18:18:52.521461+0000","last_fullsized":"2023-09-23T18:18:52.521461+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":115,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521812+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:18:52.521812+0000","last_peered":"2023-09-23T18:18:52.521812+0000","last_clean":"2023-09-23T18:18:52.521812+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:18:52.521812+0000","last_undegraded":"2023-09-23T18:18:52.521812+0000","last_fullsized":"2023-09-23T18:18:52.521812+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"601'4973","reported_seq":5017,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:01.163131+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:19:01.163131+0000","last_peered":"2023-09-23T18:19:01.163131+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:19:01.163131+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"578'500","ondisk_log_start":"578'500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4473,"log_dups_size":0,"ondisk_log_size":4473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":325910528,"num_objects":4973,"num_object_clones":0,"num_object_copies":14919,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4973,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4973,"num_write_kb":318272,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":3558},{"shards":"5,6","objects":1414}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":784,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520866+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:18:52.520866+0000","last_peered":"2023-09-23T18:18:52.520866+0000","last_clean":"2023-09-23T18:18:52.520866+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:18:52.520866+0000","last_undegraded":"2023-09-23T18:18:52.520866+0000","last_fullsized":"2023-09-23T18:18:52.520866+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1485,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521794+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:18:52.521794+0000","last_peered":"2023-09-23T18:18:52.521794+0000","last_clean":"2023-09-23T18:18:52.521794+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:18:52.521794+0000","last_undegraded":"2023-09-23T18:18:52.521794+0000","last_fullsized":"2023-09-23T18:18:52.521794+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":125,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.537607+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:18:52.537607+0000","last_peered":"2023-09-23T18:18:52.537607+0000","last_clean":"2023-09-23T18:18:52.537607+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:18:52.537607+0000","last_undegraded":"2023-09-23T18:18:52.537607+0000","last_fullsized":"2023-09-23T18:18:52.537607+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":425,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536715+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:18:52.536715+0000","last_peered":"2023-09-23T18:18:52.536715+0000","last_clean":"2023-09-23T18:18:52.536715+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:18:52.536715+0000","last_undegraded":"2023-09-23T18:18:52.536715+0000","last_fullsized":"2023-09-23T18:18:52.536715+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"601'4799","reported_seq":4834,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:01.029877+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:19:01.029877+0000","last_peered":"2023-09-23T18:19:01.029877+0000","last_clean":"2023-09-23T18:19:01.029877+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:19:01.029877+0000","last_undegraded":"2023-09-23T18:19:01.029877+0000","last_fullsized":"2023-09-23T18:19:01.029877+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4799,"log_dups_size":0,"ondisk_log_size":4799,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":314507264,"num_objects":4799,"num_object_clones":0,"num_object_copies":14397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4799,"num_write_kb":307136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":809,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521894+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:18:52.521894+0000","last_peered":"2023-09-23T18:18:52.521894+0000","last_clean":"2023-09-23T18:18:52.521894+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:18:52.521894+0000","last_undegraded":"2023-09-23T18:18:52.521894+0000","last_fullsized":"2023-09-23T18:18:52.521894+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":15,"reported_epoch":600,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:18:52.539761+0000","last_change":"2023-09-23T18:18:46.432707+0000","last_active":"2023-09-23T18:18:52.539761+0000","last_peered":"2023-09-23T18:18:52.539761+0000","last_clean":"2023-09-23T18:11:33.614822+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:18:52.539761+0000","last_undegraded":"2023-09-23T18:18:46.418246+0000","last_fullsized":"2023-09-23T18:18:52.539761+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":394,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.534817+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:18:52.534817+0000","last_peered":"2023-09-23T18:18:52.534817+0000","last_clean":"2023-09-23T18:18:52.534817+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:18:52.534817+0000","last_undegraded":"2023-09-23T18:18:52.534817+0000","last_fullsized":"2023-09-23T18:18:52.534817+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":137,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521791+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:18:52.521791+0000","last_peered":"2023-09-23T18:18:52.521791+0000","last_clean":"2023-09-23T18:18:52.521791+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:18:52.521791+0000","last_undegraded":"2023-09-23T18:18:52.521791+0000","last_fullsized":"2023-09-23T18:18:52.521791+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"601'4913","reported_seq":4948,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:01.147436+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:19:01.147436+0000","last_peered":"2023-09-23T18:19:01.147436+0000","last_clean":"2023-09-23T18:19:01.147436+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:19:01.147436+0000","last_undegraded":"2023-09-23T18:19:01.147436+0000","last_fullsized":"2023-09-23T18:19:01.147436+0000","mapping_epoch":574,"log_start":"578'500","ondisk_log_start":"578'500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4413,"log_dups_size":0,"ondisk_log_size":4413,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":321978368,"num_objects":4913,"num_object_clones":0,"num_object_copies":14739,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4913,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4913,"num_write_kb":314432,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1498,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520956+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:18:52.520956+0000","last_peered":"2023-09-23T18:18:52.520956+0000","last_clean":"2023-09-23T18:18:52.520956+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:18:52.520956+0000","last_undegraded":"2023-09-23T18:18:52.520956+0000","last_fullsized":"2023-09-23T18:18:52.520956+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":551,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.534772+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:18:52.534772+0000","last_peered":"2023-09-23T18:18:52.534772+0000","last_clean":"2023-09-23T18:18:52.534772+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:18:52.534772+0000","last_undegraded":"2023-09-23T18:18:52.534772+0000","last_fullsized":"2023-09-23T18:18:52.534772+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":111,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521716+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:18:52.521716+0000","last_peered":"2023-09-23T18:18:52.521716+0000","last_clean":"2023-09-23T18:18:52.521716+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:18:52.521716+0000","last_undegraded":"2023-09-23T18:18:52.521716+0000","last_fullsized":"2023-09-23T18:18:52.521716+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":146,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536860+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:18:52.536860+0000","last_peered":"2023-09-23T18:18:52.536860+0000","last_clean":"2023-09-23T18:18:52.536860+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:18:52.536860+0000","last_undegraded":"2023-09-23T18:18:52.536860+0000","last_fullsized":"2023-09-23T18:18:52.536860+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"601'4834","reported_seq":4869,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:01.056116+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:19:01.056116+0000","last_peered":"2023-09-23T18:19:01.056116+0000","last_clean":"2023-09-23T18:19:01.056116+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:19:01.056116+0000","last_undegraded":"2023-09-23T18:19:01.056116+0000","last_fullsized":"2023-09-23T18:19:01.056116+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4834,"log_dups_size":0,"ondisk_log_size":4834,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316801024,"num_objects":4834,"num_object_clones":0,"num_object_copies":14502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4834,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4834,"num_write_kb":309376,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":799,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.537746+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:18:52.537746+0000","last_peered":"2023-09-23T18:18:52.537746+0000","last_clean":"2023-09-23T18:18:52.537746+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:18:52.537746+0000","last_undegraded":"2023-09-23T18:18:52.537746+0000","last_fullsized":"2023-09-23T18:18:52.537746+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":114,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:56.060732+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:18:56.060732+0000","last_peered":"2023-09-23T18:18:56.060732+0000","last_clean":"2023-09-23T18:18:56.060732+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:18:56.060732+0000","last_undegraded":"2023-09-23T18:18:56.060732+0000","last_fullsized":"2023-09-23T18:18:56.060732+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":17,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.539621+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:18:52.539621+0000","last_peered":"2023-09-23T18:18:52.539621+0000","last_clean":"2023-09-23T18:18:52.539621+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:18:52.539621+0000","last_undegraded":"2023-09-23T18:18:52.539621+0000","last_fullsized":"2023-09-23T18:18:52.539621+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":616,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520925+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:18:52.520925+0000","last_peered":"2023-09-23T18:18:52.520925+0000","last_clean":"2023-09-23T18:18:52.520925+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:18:52.520925+0000","last_undegraded":"2023-09-23T18:18:52.520925+0000","last_fullsized":"2023-09-23T18:18:52.520925+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"601'4840","reported_seq":4885,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:01.157404+0000","last_change":"2023-09-23T18:18:34.726444+0000","last_active":"2023-09-23T18:19:01.157404+0000","last_peered":"2023-09-23T18:19:01.157404+0000","last_clean":"2023-09-23T18:18:32.699126+0000","last_became_active":"2023-09-23T18:18:34.716591+0000","last_became_peered":"2023-09-23T18:18:34.716591+0000","last_unstale":"2023-09-23T18:19:01.157404+0000","last_undegraded":"2023-09-23T18:18:34.351246+0000","last_fullsized":"2023-09-23T18:18:34.327168+0000","mapping_epoch":589,"log_start":"578'400","ondisk_log_start":"578'400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4440,"log_dups_size":0,"ondisk_log_size":4440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:13.611234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":317194240,"num_objects":4840,"num_object_clones":0,"num_object_copies":14520,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4840,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4840,"num_write_kb":309760,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":3453},{"shards":"4,6","objects":1387}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":899,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536903+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:18:52.536903+0000","last_peered":"2023-09-23T18:18:52.536903+0000","last_clean":"2023-09-23T18:18:52.536903+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:18:52.536903+0000","last_undegraded":"2023-09-23T18:18:52.536903+0000","last_fullsized":"2023-09-23T18:18:52.536903+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":39,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.534127+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:18:52.534127+0000","last_peered":"2023-09-23T18:18:52.534127+0000","last_clean":"2023-09-23T18:18:52.534127+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:18:52.534127+0000","last_undegraded":"2023-09-23T18:18:52.534127+0000","last_fullsized":"2023-09-23T18:18:52.534127+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":125,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520833+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:18:52.520833+0000","last_peered":"2023-09-23T18:18:52.520833+0000","last_clean":"2023-09-23T18:18:52.520833+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:18:52.520833+0000","last_undegraded":"2023-09-23T18:18:52.520833+0000","last_fullsized":"2023-09-23T18:18:52.520833+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":125,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520584+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:18:52.520584+0000","last_peered":"2023-09-23T18:18:52.520584+0000","last_clean":"2023-09-23T18:18:52.520584+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:18:52.520584+0000","last_undegraded":"2023-09-23T18:18:52.520584+0000","last_fullsized":"2023-09-23T18:18:52.520584+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"601'4857","reported_seq":4892,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:01.175775+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:19:01.175775+0000","last_peered":"2023-09-23T18:19:01.175775+0000","last_clean":"2023-09-23T18:19:01.175775+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:19:01.175775+0000","last_undegraded":"2023-09-23T18:19:01.175775+0000","last_fullsized":"2023-09-23T18:19:01.175775+0000","mapping_epoch":574,"log_start":"578'400","ondisk_log_start":"578'400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4457,"log_dups_size":0,"ondisk_log_size":4457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318308352,"num_objects":4857,"num_object_clones":0,"num_object_copies":14571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4857,"num_write_kb":310848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":899,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536619+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:18:52.536619+0000","last_peered":"2023-09-23T18:18:52.536619+0000","last_clean":"2023-09-23T18:18:52.536619+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:18:52.536619+0000","last_undegraded":"2023-09-23T18:18:52.536619+0000","last_fullsized":"2023-09-23T18:18:52.536619+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":87,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521609+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:18:52.521609+0000","last_peered":"2023-09-23T18:18:52.521609+0000","last_clean":"2023-09-23T18:18:52.521609+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:18:52.521609+0000","last_undegraded":"2023-09-23T18:18:52.521609+0000","last_fullsized":"2023-09-23T18:18:52.521609+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":380,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536754+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:18:52.536754+0000","last_peered":"2023-09-23T18:18:52.536754+0000","last_clean":"2023-09-23T18:18:52.536754+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:18:52.536754+0000","last_undegraded":"2023-09-23T18:18:52.536754+0000","last_fullsized":"2023-09-23T18:18:52.536754+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":17,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.538791+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:18:52.538791+0000","last_peered":"2023-09-23T18:18:52.538791+0000","last_clean":"2023-09-23T18:18:52.538791+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:18:52.538791+0000","last_undegraded":"2023-09-23T18:18:52.538791+0000","last_fullsized":"2023-09-23T18:18:52.538791+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"601'4857","reported_seq":4892,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:01.111419+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:19:01.111419+0000","last_peered":"2023-09-23T18:19:01.111419+0000","last_clean":"2023-09-23T18:19:01.111419+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:19:01.111419+0000","last_undegraded":"2023-09-23T18:19:01.111419+0000","last_fullsized":"2023-09-23T18:19:01.111419+0000","mapping_epoch":574,"log_start":"578'400","ondisk_log_start":"578'400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4457,"log_dups_size":0,"ondisk_log_size":4457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":318308352,"num_objects":4857,"num_object_clones":0,"num_object_copies":14571,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4857,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4857,"num_write_kb":310848,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":733,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.542807+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:18:52.542807+0000","last_peered":"2023-09-23T18:18:52.542807+0000","last_clean":"2023-09-23T18:18:52.542807+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:18:52.542807+0000","last_undegraded":"2023-09-23T18:18:52.542807+0000","last_fullsized":"2023-09-23T18:18:52.542807+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1037,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.538658+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:18:52.538658+0000","last_peered":"2023-09-23T18:18:52.538658+0000","last_clean":"2023-09-23T18:18:52.538658+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:18:52.538658+0000","last_undegraded":"2023-09-23T18:18:52.538658+0000","last_fullsized":"2023-09-23T18:18:52.538658+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":27,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.533982+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:18:52.533982+0000","last_peered":"2023-09-23T18:18:52.533982+0000","last_clean":"2023-09-23T18:18:52.533982+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:18:52.533982+0000","last_undegraded":"2023-09-23T18:18:52.533982+0000","last_fullsized":"2023-09-23T18:18:52.533982+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":391,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520762+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:18:52.520762+0000","last_peered":"2023-09-23T18:18:52.520762+0000","last_clean":"2023-09-23T18:18:52.520762+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:18:52.520762+0000","last_undegraded":"2023-09-23T18:18:52.520762+0000","last_fullsized":"2023-09-23T18:18:52.520762+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":631,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.534901+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:18:52.534901+0000","last_peered":"2023-09-23T18:18:52.534901+0000","last_clean":"2023-09-23T18:18:52.534901+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:18:52.534901+0000","last_undegraded":"2023-09-23T18:18:52.534901+0000","last_fullsized":"2023-09-23T18:18:52.534901+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":23,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.533756+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:18:52.533756+0000","last_peered":"2023-09-23T18:18:52.533756+0000","last_clean":"2023-09-23T18:18:52.533756+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:18:52.533756+0000","last_undegraded":"2023-09-23T18:18:52.533756+0000","last_fullsized":"2023-09-23T18:18:52.533756+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":673,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.549587+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:18:52.549587+0000","last_peered":"2023-09-23T18:18:52.549587+0000","last_clean":"2023-09-23T18:18:52.549587+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:18:52.549587+0000","last_undegraded":"2023-09-23T18:18:52.549587+0000","last_fullsized":"2023-09-23T18:18:52.549587+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":631,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536604+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:18:52.536604+0000","last_peered":"2023-09-23T18:18:52.536604+0000","last_clean":"2023-09-23T18:18:52.536604+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:18:52.536604+0000","last_undegraded":"2023-09-23T18:18:52.536604+0000","last_fullsized":"2023-09-23T18:18:52.536604+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":119,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520912+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:18:52.520912+0000","last_peered":"2023-09-23T18:18:52.520912+0000","last_clean":"2023-09-23T18:18:52.520912+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:18:52.520912+0000","last_undegraded":"2023-09-23T18:18:52.520912+0000","last_fullsized":"2023-09-23T18:18:52.520912+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":235,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.524141+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:18:52.524141+0000","last_peered":"2023-09-23T18:18:52.524141+0000","last_clean":"2023-09-23T18:18:52.524141+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:18:52.524141+0000","last_undegraded":"2023-09-23T18:18:52.524141+0000","last_fullsized":"2023-09-23T18:18:52.524141+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":105,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521371+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:18:52.521371+0000","last_peered":"2023-09-23T18:18:52.521371+0000","last_clean":"2023-09-23T18:18:52.521371+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:18:52.521371+0000","last_undegraded":"2023-09-23T18:18:52.521371+0000","last_fullsized":"2023-09-23T18:18:52.521371+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":218,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.524176+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:18:52.524176+0000","last_peered":"2023-09-23T18:18:52.524176+0000","last_clean":"2023-09-23T18:18:52.524176+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:18:52.524176+0000","last_undegraded":"2023-09-23T18:18:52.524176+0000","last_fullsized":"2023-09-23T18:18:52.524176+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":993,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536389+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:18:52.536389+0000","last_peered":"2023-09-23T18:18:52.536389+0000","last_clean":"2023-09-23T18:18:52.536389+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:18:52.536389+0000","last_undegraded":"2023-09-23T18:18:52.536389+0000","last_fullsized":"2023-09-23T18:18:52.536389+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":498,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.536830+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:18:52.536830+0000","last_peered":"2023-09-23T18:18:52.536830+0000","last_clean":"2023-09-23T18:18:52.536830+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:18:52.536830+0000","last_undegraded":"2023-09-23T18:18:52.536830+0000","last_fullsized":"2023-09-23T18:18:52.536830+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.520850+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:18:52.520850+0000","last_peered":"2023-09-23T18:18:52.520850+0000","last_clean":"2023-09-23T18:18:52.520850+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:18:52.520850+0000","last_undegraded":"2023-09-23T18:18:52.520850+0000","last_fullsized":"2023-09-23T18:18:52.520850+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":929,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.549964+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:18:52.549964+0000","last_peered":"2023-09-23T18:18:52.549964+0000","last_clean":"2023-09-23T18:18:52.549964+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:18:52.549964+0000","last_undegraded":"2023-09-23T18:18:52.549964+0000","last_fullsized":"2023-09-23T18:18:52.549964+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":123,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.521743+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:18:52.521743+0000","last_peered":"2023-09-23T18:18:52.521743+0000","last_clean":"2023-09-23T18:18:52.521743+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:18:52.521743+0000","last_undegraded":"2023-09-23T18:18:52.521743+0000","last_fullsized":"2023-09-23T18:18:52.521743+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":644,"reported_epoch":600,"state":"active+clean","last_fresh":"2023-09-23T18:18:52.534690+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:18:52.534690+0000","last_peered":"2023-09-23T18:18:52.534690+0000","last_clean":"2023-09-23T18:18:52.534690+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:18:52.534690+0000","last_undegraded":"2023-09-23T18:18:52.534690+0000","last_fullsized":"2023-09-23T18:18:52.534690+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5146148864,"num_objects":78524,"num_object_clones":0,"num_object_copies":235572,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19671,"num_objects_misplaced":1825,"num_objects_unfound":0,"num_objects_dirty":78524,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":78524,"num_write_kb":5025536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1646,"num_bytes_recovered":107872256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15597174784,"data_stored":15597174784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":73924,"ondisk_log_size":73924,"up":48,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2310216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001557,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669493,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3038372,"kb_used_data":2461212,"kb_used_omap":7,"kb_used_meta":577144,"kb_avail":90706780,"statfs":{"total":95995035648,"available":92883742720,"internally_reserved":0,"allocated":2520281088,"data_stored":2518394435,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":590996260},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:13 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.2470000000000001,"15min":1.2270000000000001},"min":{"1min":0.252,"5min":0.248,"15min":0.22500000000000001},"max":{"1min":4.4669999999999996,"5min":7.3680000000000003,"15min":15.407},"last":0.90000000000000002},{"interface":"front","average":{"1min":1.347,"5min":1.637,"15min":1.478},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23100000000000001},"max":{"1min":7.5529999999999999,"5min":7.5529999999999999,"15min":15.147},"last":4.6909999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.216,"15min":1.216},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.3029999999999999,"5min":5.3029999999999999,"15min":5.3029999999999999},"last":1.0289999999999999},{"interface":"front","average":{"1min":1.208,"5min":1.208,"15min":1.208},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.476,"5min":3.476,"15min":3.476},"last":14.082000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.403,"15min":1.2869999999999999},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.2679999999999998,"5min":7.4189999999999996,"15min":7.4189999999999996},"last":0.95799999999999996},{"interface":"front","average":{"1min":1.927,"5min":1.7470000000000001,"15min":1.454},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0439999999999996,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.98599999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.6459999999999999,"15min":1.5509999999999999},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.3150000000000004,"5min":7.3940000000000001,"15min":10.257999999999999},"last":1.0009999999999999},{"interface":"front","average":{"1min":2.282,"5min":2.165,"15min":1.829},"min":{"1min":0.28999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.9539999999999997,"5min":9.4049999999999994,"15min":9.5960000000000001},"last":1.0960000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:18:13 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":1.647,"15min":1.2230000000000001},"min":{"1min":0.32800000000000001,"5min":0.26800000000000002,"15min":0.23999999999999999},"max":{"1min":5.8819999999999997,"5min":8.6059999999999999,"15min":8.6059999999999999},"last":4.4000000000000004},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.7829999999999999,"15min":1.248},"min":{"1min":0.33800000000000002,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":4.7480000000000002,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.82599999999999996}]},{"osd":5,"last update":"Sat Sep 23 18:18:42 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":2.0699999999999998,"15min":1.9930000000000001},"min":{"1min":0.36399999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.8179999999999996,"5min":13.502000000000001,"15min":13.502000000000001},"last":2.6480000000000001},{"interface":"front","average":{"1min":2.1520000000000001,"5min":2.298,"15min":2.4420000000000002},"min":{"1min":0.46400000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.9080000000000004,"5min":10.081,"15min":10.081},"last":3.6789999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.1930000000000001,"15min":1.4970000000000001},"min":{"1min":0.45700000000000002,"5min":0.36299999999999999,"15min":0.26800000000000002},"max":{"1min":5.851,"5min":10.743,"15min":13.460000000000001},"last":2.6080000000000001},{"interface":"front","average":{"1min":1.702,"5min":2.21,"15min":1.536},"min":{"1min":0.46400000000000002,"5min":0.32900000000000001,"15min":0.28499999999999998},"max":{"1min":6.2409999999999997,"5min":11.244,"15min":12.404},"last":0.875}]}]},{"osd":1,"up_from":545,"seq":2340757176342,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2043864,"kb_used_data":1850064,"kb_used_omap":13,"kb_used_meta":193778,"kb_avail":91701288,"statfs":{"total":95995035648,"available":93902118912,"internally_reserved":0,"allocated":1894465536,"data_stored":1891546035,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":198429325},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":1.6599999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.1480000000000001,"5min":2.1480000000000001,"15min":2.1480000000000001},"last":1.8759999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.6379999999999999,"5min":1.6379999999999999,"15min":1.6379999999999999},"last":0.41399999999999998},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":2.621}]},{"osd":3,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.54700000000000004},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.6080000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":0.66600000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":12.663,"5min":12.663,"15min":12.663},"last":7.6609999999999996},{"interface":"front","average":{"1min":1.514,"5min":1.514,"15min":1.514},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.222,"5min":13.222,"15min":13.222},"last":0.68999999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":2.79,"5min":2.79,"15min":2.79},"last":1.923},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":3.7629999999999999,"5min":3.7629999999999999,"15min":3.7629999999999999},"last":0.628}]},{"osd":6,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.9350000000000001,"5min":1.9350000000000001,"15min":1.9350000000000001},"last":13.134},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":2.762,"5min":2.762,"15min":2.762},"last":0.51800000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.575,"5min":1.575,"15min":1.575},"last":7.319},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":2.1819999999999999,"5min":2.1819999999999999,"15min":2.1819999999999999},"last":0.73399999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958838,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3629236,"kb_used_data":2796268,"kb_used_omap":7,"kb_used_meta":832952,"kb_avail":90115916,"statfs":{"total":95995035648,"available":92278697984,"internally_reserved":0,"allocated":2863378432,"data_stored":2860604145,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":852943588},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:18:24 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":0.46999999999999997},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.2679999999999998,"5min":2.2679999999999998,"15min":2.2679999999999998},"last":3.5190000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:17:57 2023","interfaces":[{"interface":"back","average":{"1min":0.41299999999999998,"5min":0.82299999999999995,"15min":1.02},"min":{"1min":0.19800000000000001,"5min":0.185,"15min":0.185},"max":{"1min":1.2609999999999999,"5min":8.8300000000000001,"15min":8.8300000000000001},"last":6.1239999999999997},{"interface":"front","average":{"1min":0.51100000000000001,"5min":0.67500000000000004,"15min":0.73999999999999999},"min":{"1min":0.22500000000000001,"5min":0.22,"15min":0.22},"max":{"1min":2.79,"5min":3.6899999999999999,"15min":3.6899999999999999},"last":3.3740000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:18:44 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.86299999999999999,"15min":1.4450000000000001},"min":{"1min":0.315,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.5350000000000001,"5min":8.5619999999999994,"15min":12.779999999999999},"last":0.50600000000000001},{"interface":"front","average":{"1min":0.878,"5min":0.88,"15min":1.472},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.677,"5min":6.8200000000000003,"15min":15.198},"last":2.427}]},{"osd":4,"last update":"Sat Sep 23 18:18:05 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":2.0129999999999999,"15min":1.7470000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.7519999999999998,"5min":57.746000000000002,"15min":57.746000000000002},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.61299999999999999,"5min":1.7629999999999999,"15min":1.5780000000000001},"min":{"1min":0.29199999999999998,"5min":0.25600000000000001,"15min":0.247},"max":{"1min":1.266,"5min":57.816000000000003,"15min":57.816000000000003},"last":4.0289999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:18:40 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.823,"15min":1.2609999999999999},"min":{"1min":0.35299999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.3140000000000001,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.6379999999999999,"15min":1.2310000000000001},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.2799999999999998,"5min":8.8879999999999999,"15min":8.8879999999999999},"last":6.1929999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:18:05 2023","interfaces":[{"interface":"back","average":{"1min":0.92400000000000004,"5min":1.573,"15min":1.8169999999999999},"min":{"1min":0.377,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":4.4039999999999999,"5min":6.702,"15min":58.457999999999998},"last":5.6740000000000004},{"interface":"front","average":{"1min":0.79300000000000004,"5min":1.4730000000000001,"15min":1.5880000000000001},"min":{"1min":0.375,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":2.7530000000000001,"5min":9.1519999999999992,"15min":58.354999999999997},"last":6.0499999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.6719999999999999,"15min":1.488},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":3.1459999999999999,"5min":8.5009999999999994,"15min":10.416},"last":6.173},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.5609999999999999,"15min":1.6759999999999999},"min":{"1min":0.48199999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.6139999999999999,"5min":10.404,"15min":10.404},"last":5.6260000000000003}]}]},{"osd":2,"up_from":0,"seq":2070174236724,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249353,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2048476,"kb_used_data":1385812,"kb_used_omap":3,"kb_used_meta":662652,"kb_avail":91696676,"statfs":{"total":95995035648,"available":93897396224,"internally_reserved":0,"allocated":1419071488,"data_stored":1416090724,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":678556392},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.407,"15min":1.508},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":4.0369999999999999,"5min":195.69800000000001,"15min":195.69800000000001},"last":1.552},{"interface":"front","average":{"1min":0.97999999999999998,"5min":2.4470000000000001,"15min":1.8049999999999999},"min":{"1min":0.22600000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":6.1340000000000003,"5min":194.661,"15min":194.661},"last":4.7569999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":1.8109999999999999,"5min":1.8109999999999999,"15min":1.8109999999999999},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.9060000000000001,"5min":3.9060000000000001,"15min":3.9060000000000001},"last":78.980000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.94299999999999995,"15min":1.179},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.597,"5min":6.3849999999999998,"15min":6.3849999999999998},"last":0.82099999999999995},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.93100000000000005,"15min":1.21},"min":{"1min":0.33100000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":4.1040000000000001,"5min":5.931,"15min":5.931},"last":1.516}]},{"osd":4,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.4650000000000001,"15min":1.5669999999999999},"min":{"1min":0.40000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":20.850000000000001,"5min":21.062999999999999,"15min":37.231000000000002},"last":0.48499999999999999},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.833,"15min":1.79},"min":{"1min":0.40300000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":20.786000000000001,"5min":21.128,"15min":37.381},"last":3.0489999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:18:36 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.4790000000000001,"15min":1.337},"min":{"1min":0.40200000000000002,"5min":0.316,"15min":0.316},"max":{"1min":2.8210000000000002,"5min":8.2959999999999994,"15min":8.2959999999999994},"last":11.805999999999999},{"interface":"front","average":{"1min":1.069,"5min":1.423,"15min":1.6120000000000001},"min":{"1min":0.47699999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.6349999999999998,"5min":6.4020000000000001,"15min":8.9969999999999999},"last":3.097}]},{"osd":6,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.4299999999999999,"15min":1.7110000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":3.8079999999999998,"5min":9.8089999999999993,"15min":115.54000000000001},"last":1.4630000000000001},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.53,"15min":1.859},"min":{"1min":0.44900000000000001,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":3.8460000000000001,"5min":14.321,"15min":115.691},"last":0.433}]},{"osd":7,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.4219999999999999,"15min":1.6319999999999999},"min":{"1min":0.436,"5min":0.36799999999999999,"15min":0.35799999999999998},"max":{"1min":3.5449999999999999,"5min":5.2809999999999997,"15min":15.972},"last":0.753},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.327,"15min":1.4550000000000001},"min":{"1min":0.50800000000000001,"5min":0.40699999999999997,"15min":0.34899999999999998},"max":{"1min":2.069,"5min":5.8570000000000002,"15min":9.9109999999999996},"last":3.9630000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019302,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4624420,"kb_used_data":4015068,"kb_used_omap":0,"kb_used_meta":609344,"kb_avail":89120732,"statfs":{"total":95995035648,"available":91259629568,"internally_reserved":0,"allocated":4111429632,"data_stored":4109542771,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":623968256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:01 2023","interfaces":[{"interface":"back","average":{"1min":0.63400000000000001,"5min":1.2230000000000001,"15min":1.111},"min":{"1min":0.246,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":1.3500000000000001,"5min":7.3239999999999998,"15min":13.375999999999999},"last":0.499},{"interface":"front","average":{"1min":0.751,"5min":1.597,"15min":1.4159999999999999},"min":{"1min":0.29499999999999998,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":2.3119999999999998,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.30299999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":0.80700000000000005},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2170000000000001,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.53900000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.524,"15min":1.7689999999999999},"min":{"1min":0.30299999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.173,"5min":11.173,"15min":11.173},"last":0.56299999999999994},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.6459999999999999,"15min":1.702},"min":{"1min":0.40600000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.0909999999999993,"5min":10.938000000000001,"15min":10.938000000000001},"last":0.42099999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:18:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.52,"15min":1.887},"min":{"1min":0.35599999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":11.130000000000001,"5min":11.130000000000001,"15min":29.154},"last":0.59199999999999997},{"interface":"front","average":{"1min":1.78,"5min":1.712,"15min":1.873},"min":{"1min":0.41399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.1099999999999994,"5min":10.968,"15min":29.216000000000001},"last":0.45300000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:18:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":2.6499999999999999,"15min":2.9969999999999999},"min":{"1min":0.36499999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.0660000000000007,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.373,"5min":1.8300000000000001,"15min":2.8410000000000002},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.8789999999999996,"5min":13.938000000000001,"15min":32.640000000000001},"last":0.878}]},{"osd":6,"last update":"Sat Sep 23 18:18:04 2023","interfaces":[{"interface":"back","average":{"1min":2.343,"5min":2.0459999999999998,"15min":1.331},"min":{"1min":0.32000000000000001,"5min":0.28199999999999997,"15min":0.25},"max":{"1min":13.904,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.84599999999999997},{"interface":"front","average":{"1min":2.0899999999999999,"5min":2.161,"15min":1.4530000000000001},"min":{"1min":0.40000000000000002,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":13.972,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.90900000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:18:01 2023","interfaces":[{"interface":"back","average":{"1min":1.25,"5min":2.7040000000000002,"15min":1.6719999999999999},"min":{"1min":0.39900000000000002,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":9.3149999999999995,"5min":19.298999999999999,"15min":19.298999999999999},"last":0.48099999999999998},{"interface":"front","average":{"1min":0.85199999999999998,"5min":2.085,"15min":1.5049999999999999},"min":{"1min":0.46300000000000002,"5min":0.42199999999999999,"15min":0.28399999999999997},"max":{"1min":2.4169999999999998,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.71899999999999997}]}]},{"osd":5,"up_from":420,"seq":1803886264399,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3010328,"kb_used_data":2707344,"kb_used_omap":9,"kb_used_meta":302966,"kb_avail":90734824,"statfs":{"total":95995035648,"available":92912459776,"internally_reserved":0,"allocated":2772320256,"data_stored":2770441667,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":310237629},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.2330000000000001,"15min":0.83799999999999997},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.206,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":1.4039999999999999},{"interface":"front","average":{"1min":1.038,"5min":1.423,"15min":1.5189999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.6309999999999998,"5min":7.5439999999999996,"15min":7.5439999999999996},"last":20.937999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:18:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.1619999999999999,"5min":3.1619999999999999,"15min":3.1619999999999999},"last":1.6279999999999999},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.238,"5min":2.238,"15min":2.238},"last":12.763999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.4590000000000001,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.4930000000000003,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":1.131},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.891,"15min":2.141},"min":{"1min":0.314,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.6749999999999998,"5min":7.2359999999999998,"15min":7.2359999999999998},"last":5.9749999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.556,"15min":1.153},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.2330000000000001,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":1.8120000000000001},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.534,"15min":1.5169999999999999},"min":{"1min":0.33500000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.8599999999999999,"5min":7.5179999999999998,"15min":7.5179999999999998},"last":20.035}]},{"osd":4,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":2.1440000000000001,"15min":2.0379999999999998},"min":{"1min":0.36499999999999999,"5min":0.254,"15min":0.251},"max":{"1min":5.375,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.514,"5min":1.9099999999999999,"15min":1.5580000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.069,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":1.095}]},{"osd":6,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.867,"15min":1.1899999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1920000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":5.8360000000000003},{"interface":"front","average":{"1min":1.232,"5min":2.004,"15min":1.6220000000000001},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.9649999999999999,"5min":12.504,"15min":12.504},"last":5.9080000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":2.363,"15min":1.6970000000000001},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.4240000000000004,"5min":13.506,"15min":13.506},"last":4.0209999999999999},{"interface":"front","average":{"1min":1.2310000000000001,"5min":2.0310000000000001,"15min":1.5349999999999999},"min":{"1min":0.378,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.2610000000000001,"5min":13.138,"15min":13.138},"last":1.2350000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2874540032,"data_stored":2874540032,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1886715904,"data_stored":1886715904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1411186688,"data_stored":1411186688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4117889024,"data_stored":4117889024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2788884480,"data_stored":2788884480,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2517958656,"data_stored":2517958656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:19:06.472 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:19:06.498 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:06.541 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:06.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:06.546 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:06.646 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:06.700 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:06.756 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:06.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:06.766 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:06.867 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:06.919 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:06.975 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 65 16 82749 82733 79.5165 46 0.016836 0.0125712 2023-09-23T18:19:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:06 smithi099 ceph-mon[167076]: from='client.37134 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:19:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:19:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:06 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2474163959' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:06 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[179668]: from='client.37134 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:19:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:19:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:07.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:07.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:07.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2474163959' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:07.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:07.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[184174]: from='client.37134 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:07.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:19:07.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:19:07.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:07.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:07.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:07.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2474163959' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:07.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:06 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:07.475 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:19:07.720 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:07.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:07.730 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:07.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:07.864 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:07.903 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:07.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:07.911 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:07.975 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 66 16 83655 83639 79.1697 56.625 0.0154052 0.0126264 2023-09-23T18:19:08.011 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:08.084 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:08.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:07 smithi099 ceph-mon[167076]: pgmap v1415: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 49 MiB/s wr, 779 op/s; 19697/245718 objects degraded (8.016%); 1825/245718 objects misplaced (0.743%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T18:19:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:07 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:19:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:07 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:08.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:07 smithi099 ceph-mon[167076]: osdmap e602: 8 total, 8 up, 6 in 2023-09-23T18:19:08.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:07 smithi055 ceph-mon[179668]: pgmap v1415: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 49 MiB/s wr, 779 op/s; 19697/245718 objects degraded (8.016%); 1825/245718 objects misplaced (0.743%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T18:19:08.349 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:07 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:19:08.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:07 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:08.350 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:07 smithi055 ceph-mon[179668]: osdmap e602: 8 total, 8 up, 6 in 2023-09-23T18:19:08.350 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:07 smithi055 ceph-mon[184174]: pgmap v1415: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 49 MiB/s wr, 779 op/s; 19697/245718 objects degraded (8.016%); 1825/245718 objects misplaced (0.743%); 3.2 MiB/s, 50 objects/s recovering 2023-09-23T18:19:08.350 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:07 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:19:08.350 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:07 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:08.350 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:07 smithi055 ceph-mon[184174]: osdmap e602: 8 total, 8 up, 6 in 2023-09-23T18:19:08.790 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:08.796 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:08.797 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:08.898 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:08.957 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:08.975 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 67 16 84554 84538 78.8267 56.1875 0.0321193 0.0126817 2023-09-23T18:19:09.023 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:09.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:09.030 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:09.117 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:09.129 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:08 smithi055 ceph-mon[184174]: failed to encode map e602 with expected crc 2023-09-23T18:19:09.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:08 smithi055 ceph-mon[184174]: failed to encode map e602 with expected crc 2023-09-23T18:19:09.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:08 smithi055 ceph-mon[184174]: failed to encode map e602 with expected crc 2023-09-23T18:19:09.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19697/245718 objects degraded (8.016%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:09.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:08 smithi055 ceph-mon[179668]: failed to encode map e602 with expected crc 2023-09-23T18:19:09.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:08 smithi055 ceph-mon[179668]: failed to encode map e602 with expected crc 2023-09-23T18:19:09.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:08 smithi055 ceph-mon[179668]: failed to encode map e602 with expected crc 2023-09-23T18:19:09.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19697/245718 objects degraded (8.016%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:09.130 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:09.225 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:09.293 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:09.311 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:09.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:09.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:08 smithi099 ceph-mon[167076]: failed to encode map e602 with expected crc 2023-09-23T18:19:09.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:08 smithi099 ceph-mon[167076]: failed to encode map e602 with expected crc 2023-09-23T18:19:09.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:08 smithi099 ceph-mon[167076]: failed to encode map e602 with expected crc 2023-09-23T18:19:09.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19697/245718 objects degraded (8.016%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:09.413 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:09.459 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:09.541 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:09.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:09.548 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:09.649 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:09.662 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:09.755 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:09.960 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:19:09.975 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 68 16 85367 85351 78.4148 50.8125 0.00764527 0.0127483 2023-09-23T18:19:09.999 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:19:10.022 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:10.029 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:10.029 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:10.037 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:19:10.076 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:19:10.113 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:19:10.130 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:10.155 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:19:10.192 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:19:10.192 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:10.228 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:19:10.240 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:19:10.241 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:19:10.241 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:19:10.241 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:19:10.241 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:19:10.241 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:19:10.241 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:19:10.241 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:19:10.277 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:09 smithi055 ceph-mon[184174]: pgmap v1417: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 54 MiB/s wr, 865 op/s; 19697/247359 objects degraded (7.963%); 1581/247359 objects misplaced (0.639%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T18:19:10.278 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:09 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4151616596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:10.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:09 smithi055 ceph-mon[179668]: pgmap v1417: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 54 MiB/s wr, 865 op/s; 19697/247359 objects degraded (7.963%); 1581/247359 objects misplaced (0.639%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T18:19:10.278 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:09 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4151616596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:09 smithi099 ceph-mon[167076]: pgmap v1417: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.0 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 54 MiB/s wr, 865 op/s; 19697/247359 objects degraded (7.963%); 1581/247359 objects misplaced (0.639%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T18:19:10.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:09 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4151616596' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:10.556 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:19:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:19:10] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2023-09-23T18:19:10.877 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:19:10.975 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 69 16 86125 86109 77.965 47.375 0.0136918 0.0128197 2023-09-23T18:19:11.975 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 70 16 87036 87020 77.6646 56.9375 0.0169971 0.0128711 2023-09-23T18:19:11.979 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:11.986 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:11.986 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:12.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:12.139 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:12.193 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:12.203 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:12.203 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:12.304 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:12.340 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:12.405 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:12.410 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:12 smithi099 ceph-mon[167076]: pgmap v1418: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 49 MiB/s wr, 790 op/s; 19697/251997 objects degraded (7.816%); 1581/251997 objects misplaced (0.627%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:19:12.411 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:12.411 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:12.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:12 smithi055 ceph-mon[179668]: pgmap v1418: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 49 MiB/s wr, 790 op/s; 19697/251997 objects degraded (7.816%); 1581/251997 objects misplaced (0.627%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:19:12.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:12 smithi055 ceph-mon[184174]: pgmap v1418: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.1 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 49 MiB/s wr, 790 op/s; 19697/251997 objects degraded (7.816%); 1581/251997 objects misplaced (0.627%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:19:12.442 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:19:12.442 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:19:12.472 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:12.512 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:12.612 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:12.935 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:12.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:12.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:12.976 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 71 16 87869 87853 77.3041 52.0625 0.0145189 0.0129305 2023-09-23T18:19:13.042 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:13.084 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:13.138 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:13.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:13.145 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:13.246 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:13.277 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:19:13.330 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:13.376 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:19:13.395 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:19:13.595 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:13.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:13.603 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:13.704 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:13.774 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:19:13.774 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:13.930 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:13.938 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:13.939 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:13.979 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 72 16 88643 88627 76.9023 48.375 0.027979 0.0129969 2023-09-23T18:19:13.994 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1419,"stamp":"2023-09-23T18:19:13.123220+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5644761764,"num_objects":86091,"num_object_clones":0,"num_object_copies":258273,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":19697,"num_objects_misplaced":1581,"num_objects_unfound":0,"num_objects_dirty":86091,"num_whiteouts":0,"num_read":5959,"num_read_kb":12160,"num_write":86975,"num_write_kb":5520858,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2275,"num_bytes_recovered":138763913,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82086,"ondisk_log_size":82086,"up":357,"acting":350,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":379,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":20004008,"kb_used_data":16767096,"kb_used_omap":40,"kb_used_meta":3236823,"kb_avail":542466904,"statfs":{"total":575970213888,"available":555486109696,"internally_reserved":0,"allocated":17169506304,"data_stored":17155141002,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3314507066},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,0,1,5,4,2,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":525860864,"num_objects":8024,"num_object_clones":0,"num_object_copies":24072,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-485,"num_objects_unfound":0,"num_objects_dirty":8024,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8024,"num_write_kb":513536,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":485,"num_bytes_recovered":31784960,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003596"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":890,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097819+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:19:00.097819+0000","last_peered":"2023-09-23T18:19:00.097819+0000","last_clean":"2023-09-23T18:19:00.097819+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:19:00.097819+0000","last_undegraded":"2023-09-23T18:19:00.097819+0000","last_fullsized":"2023-09-23T18:19:00.097819+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":261,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.114070+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:19:00.114070+0000","last_peered":"2023-09-23T18:19:00.114070+0000","last_clean":"2023-09-23T18:19:00.114070+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:19:00.114070+0000","last_undegraded":"2023-09-23T18:19:00.114070+0000","last_fullsized":"2023-09-23T18:19:00.114070+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":18,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.089684+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:19:00.089684+0000","last_peered":"2023-09-23T18:19:00.089684+0000","last_clean":"2023-09-23T18:19:00.089684+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:19:00.089684+0000","last_undegraded":"2023-09-23T18:19:00.089684+0000","last_fullsized":"2023-09-23T18:19:00.089684+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":896,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104196+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:19:00.104196+0000","last_peered":"2023-09-23T18:19:00.104196+0000","last_clean":"2023-09-23T18:19:00.104196+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:19:00.104196+0000","last_undegraded":"2023-09-23T18:19:00.104196+0000","last_fullsized":"2023-09-23T18:19:00.104196+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":342,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.090902+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:19:00.090902+0000","last_peered":"2023-09-23T18:19:00.090902+0000","last_clean":"2023-09-23T18:19:00.090902+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:19:00.090902+0000","last_undegraded":"2023-09-23T18:19:00.090902+0000","last_fullsized":"2023-09-23T18:19:00.090902+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":662,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.101968+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:19:00.101968+0000","last_peered":"2023-09-23T18:19:00.101968+0000","last_clean":"2023-09-23T18:19:00.101968+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:19:00.101968+0000","last_undegraded":"2023-09-23T18:19:00.101968+0000","last_fullsized":"2023-09-23T18:19:00.101968+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":913,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.098648+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:19:00.098648+0000","last_peered":"2023-09-23T18:19:00.098648+0000","last_clean":"2023-09-23T18:19:00.098648+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:00.098648+0000","last_undegraded":"2023-09-23T18:19:00.098648+0000","last_fullsized":"2023-09-23T18:19:00.098648+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":18,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.088513+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:00.088513+0000","last_peered":"2023-09-23T18:19:00.088513+0000","last_clean":"2023-09-23T18:19:00.088513+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:19:00.088513+0000","last_undegraded":"2023-09-23T18:19:00.088513+0000","last_fullsized":"2023-09-23T18:19:00.088513+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":178,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104689+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:19:00.104689+0000","last_peered":"2023-09-23T18:19:00.104689+0000","last_clean":"2023-09-23T18:19:00.104689+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:19:00.104689+0000","last_undegraded":"2023-09-23T18:19:00.104689+0000","last_fullsized":"2023-09-23T18:19:00.104689+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1036,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.098363+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:19:00.098363+0000","last_peered":"2023-09-23T18:19:00.098363+0000","last_clean":"2023-09-23T18:19:00.098363+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:19:00.098363+0000","last_undegraded":"2023-09-23T18:19:00.098363+0000","last_fullsized":"2023-09-23T18:19:00.098363+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":112,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097693+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:19:00.097693+0000","last_peered":"2023-09-23T18:19:00.097693+0000","last_clean":"2023-09-23T18:19:00.097693+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:19:00.097693+0000","last_undegraded":"2023-09-23T18:19:00.097693+0000","last_fullsized":"2023-09-23T18:19:00.097693+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":191,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.088872+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:19:00.088872+0000","last_peered":"2023-09-23T18:19:00.088872+0000","last_clean":"2023-09-23T18:19:00.088872+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:19:00.088872+0000","last_undegraded":"2023-09-23T18:19:00.088872+0000","last_fullsized":"2023-09-23T18:19:00.088872+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":813,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.103885+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:19:00.103885+0000","last_peered":"2023-09-23T18:19:00.103885+0000","last_clean":"2023-09-23T18:19:00.103885+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:19:00.103885+0000","last_undegraded":"2023-09-23T18:19:00.103885+0000","last_fullsized":"2023-09-23T18:19:00.103885+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":148,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097730+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:19:00.097730+0000","last_peered":"2023-09-23T18:19:00.097730+0000","last_clean":"2023-09-23T18:19:00.097730+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:19:00.097730+0000","last_undegraded":"2023-09-23T18:19:00.097730+0000","last_fullsized":"2023-09-23T18:19:00.097730+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":147,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.103861+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:19:00.103861+0000","last_peered":"2023-09-23T18:19:00.103861+0000","last_clean":"2023-09-23T18:19:00.103861+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:19:00.103861+0000","last_undegraded":"2023-09-23T18:19:00.103861+0000","last_fullsized":"2023-09-23T18:19:00.103861+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":947,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097785+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:19:00.097785+0000","last_peered":"2023-09-23T18:19:00.097785+0000","last_clean":"2023-09-23T18:19:00.097785+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:19:00.097785+0000","last_undegraded":"2023-09-23T18:19:00.097785+0000","last_fullsized":"2023-09-23T18:19:00.097785+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":395,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.090004+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:19:00.090004+0000","last_peered":"2023-09-23T18:19:00.090004+0000","last_clean":"2023-09-23T18:19:00.090004+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:19:00.090004+0000","last_undegraded":"2023-09-23T18:19:00.090004+0000","last_fullsized":"2023-09-23T18:19:00.090004+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":415,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.110814+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:19:00.110814+0000","last_peered":"2023-09-23T18:19:00.110814+0000","last_clean":"2023-09-23T18:19:00.110814+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:19:00.110814+0000","last_undegraded":"2023-09-23T18:19:00.110814+0000","last_fullsized":"2023-09-23T18:19:00.110814+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":975,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.170935+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:19:00.170935+0000","last_peered":"2023-09-23T18:19:00.170935+0000","last_clean":"2023-09-23T18:19:00.170935+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:19:00.170935+0000","last_undegraded":"2023-09-23T18:19:00.170935+0000","last_fullsized":"2023-09-23T18:19:00.170935+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":154,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104849+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:19:00.104849+0000","last_peered":"2023-09-23T18:19:00.104849+0000","last_clean":"2023-09-23T18:19:00.104849+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:19:00.104849+0000","last_undegraded":"2023-09-23T18:19:00.104849+0000","last_fullsized":"2023-09-23T18:19:00.104849+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":421,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104181+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:19:00.104181+0000","last_peered":"2023-09-23T18:19:00.104181+0000","last_clean":"2023-09-23T18:19:00.104181+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:19:00.104181+0000","last_undegraded":"2023-09-23T18:19:00.104181+0000","last_fullsized":"2023-09-23T18:19:00.104181+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":828,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.100735+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:19:00.100735+0000","last_peered":"2023-09-23T18:19:00.100735+0000","last_clean":"2023-09-23T18:19:00.100735+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:19:00.100735+0000","last_undegraded":"2023-09-23T18:19:00.100735+0000","last_fullsized":"2023-09-23T18:19:00.100735+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":421,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104023+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:19:00.104023+0000","last_peered":"2023-09-23T18:19:00.104023+0000","last_clean":"2023-09-23T18:19:00.104023+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:19:00.104023+0000","last_undegraded":"2023-09-23T18:19:00.104023+0000","last_fullsized":"2023-09-23T18:19:00.104023+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":99,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097206+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:19:00.097206+0000","last_peered":"2023-09-23T18:19:00.097206+0000","last_clean":"2023-09-23T18:19:00.097206+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:19:00.097206+0000","last_undegraded":"2023-09-23T18:19:00.097206+0000","last_fullsized":"2023-09-23T18:19:00.097206+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":799,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097531+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:19:00.097531+0000","last_peered":"2023-09-23T18:19:00.097531+0000","last_clean":"2023-09-23T18:19:00.097531+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:19:00.097531+0000","last_undegraded":"2023-09-23T18:19:00.097531+0000","last_fullsized":"2023-09-23T18:19:00.097531+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":117,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.101984+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:19:00.101984+0000","last_peered":"2023-09-23T18:19:00.101984+0000","last_clean":"2023-09-23T18:19:00.101984+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:19:00.101984+0000","last_undegraded":"2023-09-23T18:19:00.101984+0000","last_fullsized":"2023-09-23T18:19:00.101984+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":112,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.098546+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:19:00.098546+0000","last_peered":"2023-09-23T18:19:00.098546+0000","last_clean":"2023-09-23T18:19:00.098546+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:19:00.098546+0000","last_undegraded":"2023-09-23T18:19:00.098546+0000","last_fullsized":"2023-09-23T18:19:00.098546+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":737,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.089269+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:19:00.089269+0000","last_peered":"2023-09-23T18:19:00.089269+0000","last_clean":"2023-09-23T18:19:00.089269+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:19:00.089269+0000","last_undegraded":"2023-09-23T18:19:00.089269+0000","last_fullsized":"2023-09-23T18:19:00.089269+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":189,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.103699+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:19:00.103699+0000","last_peered":"2023-09-23T18:19:00.103699+0000","last_clean":"2023-09-23T18:19:00.103699+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:19:00.103699+0000","last_undegraded":"2023-09-23T18:19:00.103699+0000","last_fullsized":"2023-09-23T18:19:00.103699+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":200,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.090088+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:19:00.090088+0000","last_peered":"2023-09-23T18:19:00.090088+0000","last_clean":"2023-09-23T18:19:00.090088+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:19:00.090088+0000","last_undegraded":"2023-09-23T18:19:00.090088+0000","last_fullsized":"2023-09-23T18:19:00.090088+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":833,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.103702+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:19:00.103702+0000","last_peered":"2023-09-23T18:19:00.103702+0000","last_clean":"2023-09-23T18:19:00.103702+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:19:00.103702+0000","last_undegraded":"2023-09-23T18:19:00.103702+0000","last_fullsized":"2023-09-23T18:19:00.103702+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":742,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.103692+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:19:00.103692+0000","last_peered":"2023-09-23T18:19:00.103692+0000","last_clean":"2023-09-23T18:19:00.103692+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:19:00.103692+0000","last_undegraded":"2023-09-23T18:19:00.103692+0000","last_fullsized":"2023-09-23T18:19:00.103692+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":18,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.091290+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:19:00.091290+0000","last_peered":"2023-09-23T18:19:00.091290+0000","last_clean":"2023-09-23T18:19:00.091290+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:19:00.091290+0000","last_undegraded":"2023-09-23T18:19:00.091290+0000","last_fullsized":"2023-09-23T18:19:00.091290+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"602'5447","reported_seq":5483,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:11.034164+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:19:11.034164+0000","last_peered":"2023-09-23T18:19:11.034164+0000","last_clean":"2023-09-23T18:19:11.034164+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:19:11.034164+0000","last_undegraded":"2023-09-23T18:19:11.034164+0000","last_fullsized":"2023-09-23T18:19:11.034164+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5447,"log_dups_size":0,"ondisk_log_size":5447,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356974592,"num_objects":5447,"num_object_clones":0,"num_object_copies":16341,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5447,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5447,"num_write_kb":348608,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":867,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104181+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:19:00.104181+0000","last_peered":"2023-09-23T18:19:00.104181+0000","last_clean":"2023-09-23T18:19:00.104181+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:19:00.104181+0000","last_undegraded":"2023-09-23T18:19:00.104181+0000","last_fullsized":"2023-09-23T18:19:00.104181+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":238,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.115656+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:19:00.115656+0000","last_peered":"2023-09-23T18:19:00.115656+0000","last_clean":"2023-09-23T18:19:00.115656+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:19:00.115656+0000","last_undegraded":"2023-09-23T18:19:00.115656+0000","last_fullsized":"2023-09-23T18:19:00.115656+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":112,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097583+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:19:00.097583+0000","last_peered":"2023-09-23T18:19:00.097583+0000","last_clean":"2023-09-23T18:19:00.097583+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:19:00.097583+0000","last_undegraded":"2023-09-23T18:19:00.097583+0000","last_fullsized":"2023-09-23T18:19:00.097583+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"602'5326","reported_seq":5372,"reported_epoch":602,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:11.150221+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:19:11.150221+0000","last_peered":"2023-09-23T18:19:11.150221+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:19:11.150221+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"579'900","ondisk_log_start":"579'900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4426,"log_dups_size":0,"ondisk_log_size":4426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349044736,"num_objects":5326,"num_object_clones":0,"num_object_copies":15978,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5326,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5326,"num_write_kb":340864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":4148},{"shards":"5,6","objects":1178}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":727,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:00.104567+0000","last_change":"2023-09-23T18:18:47.441716+0000","last_active":"2023-09-23T18:19:00.104567+0000","last_peered":"2023-09-23T18:19:00.104567+0000","last_clean":"2023-09-23T18:11:33.614788+0000","last_became_active":"2023-09-23T18:18:47.441414+0000","last_became_peered":"2023-09-23T18:18:47.441414+0000","last_unstale":"2023-09-23T18:19:00.104567+0000","last_undegraded":"2023-09-23T18:18:47.413783+0000","last_fullsized":"2023-09-23T18:18:47.413246+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":2047528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[0,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":239,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.114464+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:19:00.114464+0000","last_peered":"2023-09-23T18:19:00.114464+0000","last_clean":"2023-09-23T18:19:00.114464+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:19:00.114464+0000","last_undegraded":"2023-09-23T18:19:00.114464+0000","last_fullsized":"2023-09-23T18:19:00.114464+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":218,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.113963+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:19:00.113963+0000","last_peered":"2023-09-23T18:19:00.113963+0000","last_clean":"2023-09-23T18:19:00.113963+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:19:00.113963+0000","last_undegraded":"2023-09-23T18:19:00.113963+0000","last_fullsized":"2023-09-23T18:19:00.113963+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"602'5236","reported_seq":9059,"reported_epoch":602,"state":"active+recovering+remapped","last_fresh":"2023-09-23T18:19:08.838286+0000","last_change":"2023-09-23T18:18:34.707552+0000","last_active":"2023-09-23T18:19:08.838286+0000","last_peered":"2023-09-23T18:19:08.838286+0000","last_clean":"2023-09-23T18:18:32.618842+0000","last_became_active":"2023-09-23T18:18:34.683472+0000","last_became_peered":"2023-09-23T18:18:34.683472+0000","last_unstale":"2023-09-23T18:19:08.838286+0000","last_undegraded":"2023-09-23T18:19:08.838286+0000","last_fullsized":"2023-09-23T18:19:08.838286+0000","mapping_epoch":589,"log_start":"578'800","ondisk_log_start":"578'800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4436,"log_dups_size":0,"ondisk_log_size":4436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:31.959133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":343146496,"num_objects":5236,"num_object_clones":0,"num_object_copies":15708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":1581,"num_objects_unfound":0,"num_objects_dirty":5236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5236,"num_write_kb":335104,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1890,"num_bytes_recovered":123863040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":3655}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":617,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104620+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:19:00.104620+0000","last_peered":"2023-09-23T18:19:00.104620+0000","last_clean":"2023-09-23T18:19:00.104620+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:19:00.104620+0000","last_undegraded":"2023-09-23T18:19:00.104620+0000","last_fullsized":"2023-09-23T18:19:00.104620+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":527,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.098467+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:19:00.098467+0000","last_peered":"2023-09-23T18:19:00.098467+0000","last_clean":"2023-09-23T18:19:00.098467+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:19:00.098467+0000","last_undegraded":"2023-09-23T18:19:00.098467+0000","last_fullsized":"2023-09-23T18:19:00.098467+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":178,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.109727+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:19:00.109727+0000","last_peered":"2023-09-23T18:19:00.109727+0000","last_clean":"2023-09-23T18:19:00.109727+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:19:00.109727+0000","last_undegraded":"2023-09-23T18:19:00.109727+0000","last_fullsized":"2023-09-23T18:19:00.109727+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"602'5441","reported_seq":5478,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:10.978716+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:19:10.978716+0000","last_peered":"2023-09-23T18:19:10.978716+0000","last_clean":"2023-09-23T18:19:10.978716+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:19:10.978716+0000","last_undegraded":"2023-09-23T18:19:10.978716+0000","last_fullsized":"2023-09-23T18:19:10.978716+0000","mapping_epoch":574,"log_start":"578'300","ondisk_log_start":"578'300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5141,"log_dups_size":0,"ondisk_log_size":5141,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356581376,"num_objects":5441,"num_object_clones":0,"num_object_copies":16323,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5441,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5441,"num_write_kb":348224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":632,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.090174+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:19:00.090174+0000","last_peered":"2023-09-23T18:19:00.090174+0000","last_clean":"2023-09-23T18:19:00.090174+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:19:00.090174+0000","last_undegraded":"2023-09-23T18:19:00.090174+0000","last_fullsized":"2023-09-23T18:19:00.090174+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":126,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097616+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:19:00.097616+0000","last_peered":"2023-09-23T18:19:00.097616+0000","last_clean":"2023-09-23T18:19:00.097616+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:19:00.097616+0000","last_undegraded":"2023-09-23T18:19:00.097616+0000","last_fullsized":"2023-09-23T18:19:00.097616+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":438,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.101210+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:19:00.101210+0000","last_peered":"2023-09-23T18:19:00.101210+0000","last_clean":"2023-09-23T18:19:00.101210+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:00.101210+0000","last_undegraded":"2023-09-23T18:19:00.101210+0000","last_fullsized":"2023-09-23T18:19:00.101210+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"602'5346","reported_seq":5383,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:08.849947+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:19:08.849947+0000","last_peered":"2023-09-23T18:19:08.849947+0000","last_clean":"2023-09-23T18:19:08.849947+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:19:08.849947+0000","last_undegraded":"2023-09-23T18:19:08.849947+0000","last_fullsized":"2023-09-23T18:19:08.849947+0000","mapping_epoch":574,"log_start":"579'900","ondisk_log_start":"579'900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4446,"log_dups_size":0,"ondisk_log_size":4446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350355456,"num_objects":5346,"num_object_clones":0,"num_object_copies":16038,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5346,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5346,"num_write_kb":342144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1084,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.100646+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:19:00.100646+0000","last_peered":"2023-09-23T18:19:00.100646+0000","last_clean":"2023-09-23T18:19:00.100646+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:19:00.100646+0000","last_undegraded":"2023-09-23T18:19:00.100646+0000","last_fullsized":"2023-09-23T18:19:00.100646+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":338,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.089474+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:19:00.089474+0000","last_peered":"2023-09-23T18:19:00.089474+0000","last_clean":"2023-09-23T18:19:00.089474+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:19:00.089474+0000","last_undegraded":"2023-09-23T18:19:00.089474+0000","last_fullsized":"2023-09-23T18:19:00.089474+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":146,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.103961+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:19:00.103961+0000","last_peered":"2023-09-23T18:19:00.103961+0000","last_clean":"2023-09-23T18:19:00.103961+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:19:00.103961+0000","last_undegraded":"2023-09-23T18:19:00.103961+0000","last_fullsized":"2023-09-23T18:19:00.103961+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"602'5451","reported_seq":5496,"reported_epoch":602,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:11.024913+0000","last_change":"2023-09-23T18:18:47.594759+0000","last_active":"2023-09-23T18:19:11.024913+0000","last_peered":"2023-09-23T18:19:11.024913+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:19:11.024913+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5451,"log_dups_size":0,"ondisk_log_size":5451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357236736,"num_objects":5451,"num_object_clones":0,"num_object_copies":16353,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5451,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5451,"num_write_kb":348864,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4226},{"shards":"4,5","objects":1225}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":871,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097734+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:19:00.097734+0000","last_peered":"2023-09-23T18:19:00.097734+0000","last_clean":"2023-09-23T18:19:00.097734+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:19:00.097734+0000","last_undegraded":"2023-09-23T18:19:00.097734+0000","last_fullsized":"2023-09-23T18:19:00.097734+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":154,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.109649+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:19:00.109649+0000","last_peered":"2023-09-23T18:19:00.109649+0000","last_clean":"2023-09-23T18:19:00.109649+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:19:00.109649+0000","last_undegraded":"2023-09-23T18:19:00.109649+0000","last_fullsized":"2023-09-23T18:19:00.109649+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":453,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104292+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:19:00.104292+0000","last_peered":"2023-09-23T18:19:00.104292+0000","last_clean":"2023-09-23T18:19:00.104292+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:19:00.104292+0000","last_undegraded":"2023-09-23T18:19:00.104292+0000","last_fullsized":"2023-09-23T18:19:00.104292+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"602'5406","reported_seq":5442,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:11.163330+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:19:11.163330+0000","last_peered":"2023-09-23T18:19:11.163330+0000","last_clean":"2023-09-23T18:19:11.163330+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:19:11.163330+0000","last_undegraded":"2023-09-23T18:19:11.163330+0000","last_fullsized":"2023-09-23T18:19:11.163330+0000","mapping_epoch":574,"log_start":"579'900","ondisk_log_start":"579'900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4506,"log_dups_size":0,"ondisk_log_size":4506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354287616,"num_objects":5406,"num_object_clones":0,"num_object_copies":16218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5406,"num_write_kb":345984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":873,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104284+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:19:00.104284+0000","last_peered":"2023-09-23T18:19:00.104284+0000","last_clean":"2023-09-23T18:19:00.104284+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:19:00.104284+0000","last_undegraded":"2023-09-23T18:19:00.104284+0000","last_fullsized":"2023-09-23T18:19:00.104284+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":438,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.100526+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:19:00.100526+0000","last_peered":"2023-09-23T18:19:00.100526+0000","last_clean":"2023-09-23T18:19:00.100526+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:19:00.100526+0000","last_undegraded":"2023-09-23T18:19:00.100526+0000","last_fullsized":"2023-09-23T18:19:00.100526+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":214,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.105373+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:19:00.105373+0000","last_peered":"2023-09-23T18:19:00.105373+0000","last_clean":"2023-09-23T18:19:00.105373+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:19:00.105373+0000","last_undegraded":"2023-09-23T18:19:00.105373+0000","last_fullsized":"2023-09-23T18:19:00.105373+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"602'5485","reported_seq":5521,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:12.285552+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:19:12.285552+0000","last_peered":"2023-09-23T18:19:12.285552+0000","last_clean":"2023-09-23T18:19:12.285552+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:19:12.285552+0000","last_undegraded":"2023-09-23T18:19:12.285552+0000","last_fullsized":"2023-09-23T18:19:12.285552+0000","mapping_epoch":574,"log_start":"579'900","ondisk_log_start":"579'900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4585,"log_dups_size":0,"ondisk_log_size":4585,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359464960,"num_objects":5485,"num_object_clones":0,"num_object_copies":16455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5485,"num_write_kb":351040,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":854,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.099872+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:19:00.099872+0000","last_peered":"2023-09-23T18:19:00.099872+0000","last_clean":"2023-09-23T18:19:00.099872+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:19:00.099872+0000","last_undegraded":"2023-09-23T18:19:00.099872+0000","last_fullsized":"2023-09-23T18:19:00.099872+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":18,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.088664+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:19:00.088664+0000","last_peered":"2023-09-23T18:19:00.088664+0000","last_clean":"2023-09-23T18:19:00.088664+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:19:00.088664+0000","last_undegraded":"2023-09-23T18:19:00.088664+0000","last_fullsized":"2023-09-23T18:19:00.088664+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":329,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.170751+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:19:00.170751+0000","last_peered":"2023-09-23T18:19:00.170751+0000","last_clean":"2023-09-23T18:19:00.170751+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:19:00.170751+0000","last_undegraded":"2023-09-23T18:19:00.170751+0000","last_fullsized":"2023-09-23T18:19:00.170751+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"602'5440","reported_seq":5478,"reported_epoch":602,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:10.154442+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:19:10.154442+0000","last_peered":"2023-09-23T18:19:10.154442+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:19:10.154442+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"578'600","ondisk_log_start":"578'600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4840,"log_dups_size":0,"ondisk_log_size":4840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356515840,"num_objects":5440,"num_object_clones":0,"num_object_copies":16320,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5440,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5440,"num_write_kb":348160,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":1155},{"shards":"0,4,6","objects":4285}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":713,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104760+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:19:00.104760+0000","last_peered":"2023-09-23T18:19:00.104760+0000","last_clean":"2023-09-23T18:19:00.104760+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:19:00.104760+0000","last_undegraded":"2023-09-23T18:19:00.104760+0000","last_fullsized":"2023-09-23T18:19:00.104760+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":951,"reported_epoch":601,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:00.173238+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:19:00.173238+0000","last_peered":"2023-09-23T18:19:00.173238+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:19:00.173238+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":136,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.098552+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:19:00.098552+0000","last_peered":"2023-09-23T18:19:00.098552+0000","last_clean":"2023-09-23T18:19:00.098552+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:19:00.098552+0000","last_undegraded":"2023-09-23T18:19:00.098552+0000","last_fullsized":"2023-09-23T18:19:00.098552+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":116,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.099825+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:19:00.099825+0000","last_peered":"2023-09-23T18:19:00.099825+0000","last_clean":"2023-09-23T18:19:00.099825+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:19:00.099825+0000","last_undegraded":"2023-09-23T18:19:00.099825+0000","last_fullsized":"2023-09-23T18:19:00.099825+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"602'5450","reported_seq":5494,"reported_epoch":602,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:11.143924+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:19:11.143924+0000","last_peered":"2023-09-23T18:19:11.143924+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:19:11.143924+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"580'1000","ondisk_log_start":"580'1000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4450,"log_dups_size":0,"ondisk_log_size":4450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357171200,"num_objects":5450,"num_object_clones":0,"num_object_copies":16350,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5450,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5450,"num_write_kb":348800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":3558},{"shards":"5,6","objects":1891}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":785,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.100825+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:19:00.100825+0000","last_peered":"2023-09-23T18:19:00.100825+0000","last_clean":"2023-09-23T18:19:00.100825+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:19:00.100825+0000","last_undegraded":"2023-09-23T18:19:00.100825+0000","last_fullsized":"2023-09-23T18:19:00.100825+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1486,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.099780+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:19:00.099780+0000","last_peered":"2023-09-23T18:19:00.099780+0000","last_clean":"2023-09-23T18:19:00.099780+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:19:00.099780+0000","last_undegraded":"2023-09-23T18:19:00.099780+0000","last_fullsized":"2023-09-23T18:19:00.099780+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":126,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104097+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:19:00.104097+0000","last_peered":"2023-09-23T18:19:00.104097+0000","last_clean":"2023-09-23T18:19:00.104097+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:19:00.104097+0000","last_undegraded":"2023-09-23T18:19:00.104097+0000","last_fullsized":"2023-09-23T18:19:00.104097+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":426,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104716+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:19:00.104716+0000","last_peered":"2023-09-23T18:19:00.104716+0000","last_clean":"2023-09-23T18:19:00.104716+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:19:00.104716+0000","last_undegraded":"2023-09-23T18:19:00.104716+0000","last_fullsized":"2023-09-23T18:19:00.104716+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"602'5265","reported_seq":5301,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:11.098041+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:19:11.098041+0000","last_peered":"2023-09-23T18:19:11.098041+0000","last_clean":"2023-09-23T18:19:11.098041+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:19:11.098041+0000","last_undegraded":"2023-09-23T18:19:11.098041+0000","last_fullsized":"2023-09-23T18:19:11.098041+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5265,"log_dups_size":0,"ondisk_log_size":5265,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345047040,"num_objects":5265,"num_object_clones":0,"num_object_copies":15795,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5265,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5265,"num_write_kb":336960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":810,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.099921+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:19:00.099921+0000","last_peered":"2023-09-23T18:19:00.099921+0000","last_clean":"2023-09-23T18:19:00.099921+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:19:00.099921+0000","last_undegraded":"2023-09-23T18:19:00.099921+0000","last_fullsized":"2023-09-23T18:19:00.099921+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":16,"reported_epoch":601,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:19:00.090661+0000","last_change":"2023-09-23T18:18:46.432707+0000","last_active":"2023-09-23T18:19:00.090661+0000","last_peered":"2023-09-23T18:19:00.090661+0000","last_clean":"2023-09-23T18:11:33.614822+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:19:00.090661+0000","last_undegraded":"2023-09-23T18:18:46.418246+0000","last_fullsized":"2023-09-23T18:19:00.090661+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":395,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.103840+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:19:00.103840+0000","last_peered":"2023-09-23T18:19:00.103840+0000","last_clean":"2023-09-23T18:19:00.103840+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:19:00.103840+0000","last_undegraded":"2023-09-23T18:19:00.103840+0000","last_fullsized":"2023-09-23T18:19:00.103840+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":138,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097593+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:19:00.097593+0000","last_peered":"2023-09-23T18:19:00.097593+0000","last_clean":"2023-09-23T18:19:00.097593+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:19:00.097593+0000","last_undegraded":"2023-09-23T18:19:00.097593+0000","last_fullsized":"2023-09-23T18:19:00.097593+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"602'5422","reported_seq":5458,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:11.169489+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:19:11.169489+0000","last_peered":"2023-09-23T18:19:11.169489+0000","last_clean":"2023-09-23T18:19:11.169489+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:19:11.169489+0000","last_undegraded":"2023-09-23T18:19:11.169489+0000","last_fullsized":"2023-09-23T18:19:11.169489+0000","mapping_epoch":574,"log_start":"580'1000","ondisk_log_start":"580'1000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4422,"log_dups_size":0,"ondisk_log_size":4422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":355336192,"num_objects":5422,"num_object_clones":0,"num_object_copies":16266,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5422,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5422,"num_write_kb":347008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1499,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.098496+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:19:00.098496+0000","last_peered":"2023-09-23T18:19:00.098496+0000","last_clean":"2023-09-23T18:19:00.098496+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:19:00.098496+0000","last_undegraded":"2023-09-23T18:19:00.098496+0000","last_fullsized":"2023-09-23T18:19:00.098496+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":552,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104342+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:19:00.104342+0000","last_peered":"2023-09-23T18:19:00.104342+0000","last_clean":"2023-09-23T18:19:00.104342+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:19:00.104342+0000","last_undegraded":"2023-09-23T18:19:00.104342+0000","last_fullsized":"2023-09-23T18:19:00.104342+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":112,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.099689+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:19:00.099689+0000","last_peered":"2023-09-23T18:19:00.099689+0000","last_clean":"2023-09-23T18:19:00.099689+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:19:00.099689+0000","last_undegraded":"2023-09-23T18:19:00.099689+0000","last_fullsized":"2023-09-23T18:19:00.099689+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":147,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104576+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:19:00.104576+0000","last_peered":"2023-09-23T18:19:00.104576+0000","last_clean":"2023-09-23T18:19:00.104576+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:19:00.104576+0000","last_undegraded":"2023-09-23T18:19:00.104576+0000","last_fullsized":"2023-09-23T18:19:00.104576+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"602'5316","reported_seq":5352,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:11.069126+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:19:11.069126+0000","last_peered":"2023-09-23T18:19:11.069126+0000","last_clean":"2023-09-23T18:19:11.069126+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:19:11.069126+0000","last_undegraded":"2023-09-23T18:19:11.069126+0000","last_fullsized":"2023-09-23T18:19:11.069126+0000","mapping_epoch":574,"log_start":"0'0","ondisk_log_start":"0'0","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5316,"log_dups_size":0,"ondisk_log_size":5316,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":348389376,"num_objects":5316,"num_object_clones":0,"num_object_copies":15948,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5316,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5316,"num_write_kb":340224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":800,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104267+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:19:00.104267+0000","last_peered":"2023-09-23T18:19:00.104267+0000","last_clean":"2023-09-23T18:19:00.104267+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:19:00.104267+0000","last_undegraded":"2023-09-23T18:19:00.104267+0000","last_fullsized":"2023-09-23T18:19:00.104267+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":115,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.088805+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:19:00.088805+0000","last_peered":"2023-09-23T18:19:00.088805+0000","last_clean":"2023-09-23T18:19:00.088805+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:19:00.088805+0000","last_undegraded":"2023-09-23T18:19:00.088805+0000","last_fullsized":"2023-09-23T18:19:00.088805+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":18,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.090538+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:19:00.090538+0000","last_peered":"2023-09-23T18:19:00.090538+0000","last_clean":"2023-09-23T18:19:00.090538+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:19:00.090538+0000","last_undegraded":"2023-09-23T18:19:00.090538+0000","last_fullsized":"2023-09-23T18:19:00.090538+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":617,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097808+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:19:00.097808+0000","last_peered":"2023-09-23T18:19:00.097808+0000","last_clean":"2023-09-23T18:19:00.097808+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:19:00.097808+0000","last_undegraded":"2023-09-23T18:19:00.097808+0000","last_fullsized":"2023-09-23T18:19:00.097808+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"602'5352","reported_seq":5398,"reported_epoch":602,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:11.163318+0000","last_change":"2023-09-23T18:18:34.726444+0000","last_active":"2023-09-23T18:19:11.163318+0000","last_peered":"2023-09-23T18:19:11.163318+0000","last_clean":"2023-09-23T18:18:32.699126+0000","last_became_active":"2023-09-23T18:18:34.716591+0000","last_became_peered":"2023-09-23T18:18:34.716591+0000","last_unstale":"2023-09-23T18:19:11.163318+0000","last_undegraded":"2023-09-23T18:18:34.351246+0000","last_fullsized":"2023-09-23T18:18:34.327168+0000","mapping_epoch":589,"log_start":"579'900","ondisk_log_start":"579'900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4452,"log_dups_size":0,"ondisk_log_size":4452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:13.611234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350748672,"num_objects":5352,"num_object_clones":0,"num_object_copies":16056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5352,"num_write_kb":342528,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":3453},{"shards":"4,6","objects":1899}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":900,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.105831+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:19:00.105831+0000","last_peered":"2023-09-23T18:19:00.105831+0000","last_clean":"2023-09-23T18:19:00.105831+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:19:00.105831+0000","last_undegraded":"2023-09-23T18:19:00.105831+0000","last_fullsized":"2023-09-23T18:19:00.105831+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":40,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.101469+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:19:00.101469+0000","last_peered":"2023-09-23T18:19:00.101469+0000","last_clean":"2023-09-23T18:19:00.101469+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:19:00.101469+0000","last_undegraded":"2023-09-23T18:19:00.101469+0000","last_fullsized":"2023-09-23T18:19:00.101469+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":126,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097373+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:19:00.097373+0000","last_peered":"2023-09-23T18:19:00.097373+0000","last_clean":"2023-09-23T18:19:00.097373+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:19:00.097373+0000","last_undegraded":"2023-09-23T18:19:00.097373+0000","last_fullsized":"2023-09-23T18:19:00.097373+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":126,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.098407+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:19:00.098407+0000","last_peered":"2023-09-23T18:19:00.098407+0000","last_clean":"2023-09-23T18:19:00.098407+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:19:00.098407+0000","last_undegraded":"2023-09-23T18:19:00.098407+0000","last_fullsized":"2023-09-23T18:19:00.098407+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"602'5332","reported_seq":5368,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:11.175661+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:19:11.175661+0000","last_peered":"2023-09-23T18:19:11.175661+0000","last_clean":"2023-09-23T18:19:11.175661+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:19:11.175661+0000","last_undegraded":"2023-09-23T18:19:11.175661+0000","last_fullsized":"2023-09-23T18:19:11.175661+0000","mapping_epoch":574,"log_start":"579'900","ondisk_log_start":"579'900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4432,"log_dups_size":0,"ondisk_log_size":4432,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":349437952,"num_objects":5332,"num_object_clones":0,"num_object_copies":15996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5332,"num_write_kb":341248,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":900,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104360+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:19:00.104360+0000","last_peered":"2023-09-23T18:19:00.104360+0000","last_clean":"2023-09-23T18:19:00.104360+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:19:00.104360+0000","last_undegraded":"2023-09-23T18:19:00.104360+0000","last_fullsized":"2023-09-23T18:19:00.104360+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":88,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097404+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:19:00.097404+0000","last_peered":"2023-09-23T18:19:00.097404+0000","last_clean":"2023-09-23T18:19:00.097404+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:19:00.097404+0000","last_undegraded":"2023-09-23T18:19:00.097404+0000","last_fullsized":"2023-09-23T18:19:00.097404+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":381,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.105649+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:19:00.105649+0000","last_peered":"2023-09-23T18:19:00.105649+0000","last_clean":"2023-09-23T18:19:00.105649+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:19:00.105649+0000","last_undegraded":"2023-09-23T18:19:00.105649+0000","last_fullsized":"2023-09-23T18:19:00.105649+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":18,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.088591+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:00.088591+0000","last_peered":"2023-09-23T18:19:00.088591+0000","last_clean":"2023-09-23T18:19:00.088591+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:19:00.088591+0000","last_undegraded":"2023-09-23T18:19:00.088591+0000","last_fullsized":"2023-09-23T18:19:00.088591+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"602'5349","reported_seq":5385,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:11.150795+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:19:11.150795+0000","last_peered":"2023-09-23T18:19:11.150795+0000","last_clean":"2023-09-23T18:19:11.150795+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:19:11.150795+0000","last_undegraded":"2023-09-23T18:19:11.150795+0000","last_fullsized":"2023-09-23T18:19:11.150795+0000","mapping_epoch":574,"log_start":"579'900","ondisk_log_start":"579'900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4449,"log_dups_size":0,"ondisk_log_size":4449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":350552064,"num_objects":5349,"num_object_clones":0,"num_object_copies":16047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5349,"num_write_kb":342336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":734,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.096308+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:19:00.096308+0000","last_peered":"2023-09-23T18:19:00.096308+0000","last_clean":"2023-09-23T18:19:00.096308+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:19:00.096308+0000","last_undegraded":"2023-09-23T18:19:00.096308+0000","last_fullsized":"2023-09-23T18:19:00.096308+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1038,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.104961+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:19:00.104961+0000","last_peered":"2023-09-23T18:19:00.104961+0000","last_clean":"2023-09-23T18:19:00.104961+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:19:00.104961+0000","last_undegraded":"2023-09-23T18:19:00.104961+0000","last_fullsized":"2023-09-23T18:19:00.104961+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":28,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.102158+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:19:00.102158+0000","last_peered":"2023-09-23T18:19:00.102158+0000","last_clean":"2023-09-23T18:19:00.102158+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:19:00.102158+0000","last_undegraded":"2023-09-23T18:19:00.102158+0000","last_fullsized":"2023-09-23T18:19:00.102158+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":392,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.100688+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:19:00.100688+0000","last_peered":"2023-09-23T18:19:00.100688+0000","last_clean":"2023-09-23T18:19:00.100688+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:19:00.100688+0000","last_undegraded":"2023-09-23T18:19:00.100688+0000","last_fullsized":"2023-09-23T18:19:00.100688+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":632,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.103802+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:19:00.103802+0000","last_peered":"2023-09-23T18:19:00.103802+0000","last_clean":"2023-09-23T18:19:00.103802+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:19:00.103802+0000","last_undegraded":"2023-09-23T18:19:00.103802+0000","last_fullsized":"2023-09-23T18:19:00.103802+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":24,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.101084+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:19:00.101084+0000","last_peered":"2023-09-23T18:19:00.101084+0000","last_clean":"2023-09-23T18:19:00.101084+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:19:00.101084+0000","last_undegraded":"2023-09-23T18:19:00.101084+0000","last_fullsized":"2023-09-23T18:19:00.101084+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":674,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.170846+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:19:00.170846+0000","last_peered":"2023-09-23T18:19:00.170846+0000","last_clean":"2023-09-23T18:19:00.170846+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:19:00.170846+0000","last_undegraded":"2023-09-23T18:19:00.170846+0000","last_fullsized":"2023-09-23T18:19:00.170846+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":632,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.105460+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:19:00.105460+0000","last_peered":"2023-09-23T18:19:00.105460+0000","last_clean":"2023-09-23T18:19:00.105460+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:19:00.105460+0000","last_undegraded":"2023-09-23T18:19:00.105460+0000","last_fullsized":"2023-09-23T18:19:00.105460+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":120,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097458+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:19:00.097458+0000","last_peered":"2023-09-23T18:19:00.097458+0000","last_clean":"2023-09-23T18:19:00.097458+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:19:00.097458+0000","last_undegraded":"2023-09-23T18:19:00.097458+0000","last_fullsized":"2023-09-23T18:19:00.097458+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":236,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.114477+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:19:00.114477+0000","last_peered":"2023-09-23T18:19:00.114477+0000","last_clean":"2023-09-23T18:19:00.114477+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:19:00.114477+0000","last_undegraded":"2023-09-23T18:19:00.114477+0000","last_fullsized":"2023-09-23T18:19:00.114477+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":106,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.098471+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:19:00.098471+0000","last_peered":"2023-09-23T18:19:00.098471+0000","last_clean":"2023-09-23T18:19:00.098471+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:19:00.098471+0000","last_undegraded":"2023-09-23T18:19:00.098471+0000","last_fullsized":"2023-09-23T18:19:00.098471+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":219,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.114125+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:19:00.114125+0000","last_peered":"2023-09-23T18:19:00.114125+0000","last_clean":"2023-09-23T18:19:00.114125+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:19:00.114125+0000","last_undegraded":"2023-09-23T18:19:00.114125+0000","last_fullsized":"2023-09-23T18:19:00.114125+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":994,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.109443+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:19:00.109443+0000","last_peered":"2023-09-23T18:19:00.109443+0000","last_clean":"2023-09-23T18:19:00.109443+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:19:00.109443+0000","last_undegraded":"2023-09-23T18:19:00.109443+0000","last_fullsized":"2023-09-23T18:19:00.109443+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":499,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.105746+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:19:00.105746+0000","last_peered":"2023-09-23T18:19:00.105746+0000","last_clean":"2023-09-23T18:19:00.105746+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:19:00.105746+0000","last_undegraded":"2023-09-23T18:19:00.105746+0000","last_fullsized":"2023-09-23T18:19:00.105746+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":136,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.100777+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:19:00.100777+0000","last_peered":"2023-09-23T18:19:00.100777+0000","last_clean":"2023-09-23T18:19:00.100777+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:19:00.100777+0000","last_undegraded":"2023-09-23T18:19:00.100777+0000","last_fullsized":"2023-09-23T18:19:00.100777+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":930,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097137+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:19:00.097137+0000","last_peered":"2023-09-23T18:19:00.097137+0000","last_clean":"2023-09-23T18:19:00.097137+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:19:00.097137+0000","last_undegraded":"2023-09-23T18:19:00.097137+0000","last_fullsized":"2023-09-23T18:19:00.097137+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":124,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.097538+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:19:00.097538+0000","last_peered":"2023-09-23T18:19:00.097538+0000","last_clean":"2023-09-23T18:19:00.097538+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:19:00.097538+0000","last_undegraded":"2023-09-23T18:19:00.097538+0000","last_fullsized":"2023-09-23T18:19:00.097538+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":645,"reported_epoch":601,"state":"active+clean","last_fresh":"2023-09-23T18:19:00.103680+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:19:00.103680+0000","last_peered":"2023-09-23T18:19:00.103680+0000","last_clean":"2023-09-23T18:19:00.103680+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:19:00.103680+0000","last_undegraded":"2023-09-23T18:19:00.103680+0000","last_fullsized":"2023-09-23T18:19:00.103680+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5640290304,"num_objects":86064,"num_object_clones":0,"num_object_copies":258192,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19671,"num_objects_misplaced":1581,"num_objects_unfound":0,"num_objects_dirty":86064,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":86064,"num_write_kb":5508096,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1890,"num_bytes_recovered":123863040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17193762816,"data_stored":17193762816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76064,"ondisk_log_size":76064,"up":48,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2310216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001559,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669495,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3276976,"kb_used_data":2687080,"kb_used_omap":7,"kb_used_meta":589880,"kb_avail":90468176,"statfs":{"total":95995035648,"available":92639412224,"internally_reserved":0,"allocated":2751569920,"data_stored":2749679346,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":604037924},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,1,3],"upper_bound":128},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:13 2023","interfaces":[{"interface":"back","average":{"1min":0.94699999999999995,"5min":1.2470000000000001,"15min":1.2270000000000001},"min":{"1min":0.252,"5min":0.248,"15min":0.22500000000000001},"max":{"1min":4.4669999999999996,"5min":7.3680000000000003,"15min":15.407},"last":24.472999999999999},{"interface":"front","average":{"1min":1.347,"5min":1.637,"15min":1.478},"min":{"1min":0.251,"5min":0.23300000000000001,"15min":0.23100000000000001},"max":{"1min":7.5529999999999999,"5min":7.5529999999999999,"15min":15.147},"last":22.832999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.216,"15min":1.216},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.3029999999999999,"5min":5.3029999999999999,"15min":5.3029999999999999},"last":22.972000000000001},{"interface":"front","average":{"1min":1.208,"5min":1.208,"15min":1.208},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.476,"5min":3.476,"15min":3.476},"last":22.664000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.403,"15min":1.2869999999999999},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.2679999999999998,"5min":7.4189999999999996,"15min":7.4189999999999996},"last":0.37},{"interface":"front","average":{"1min":1.927,"5min":1.7470000000000001,"15min":1.454},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0439999999999996,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.34000000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.6459999999999999,"15min":1.5509999999999999},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.3150000000000004,"5min":7.3940000000000001,"15min":10.257999999999999},"last":22.614999999999998},{"interface":"front","average":{"1min":2.282,"5min":2.165,"15min":1.829},"min":{"1min":0.28999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.9539999999999997,"5min":9.4049999999999994,"15min":9.5960000000000001},"last":23.692}]},{"osd":4,"last update":"Sat Sep 23 18:18:13 2023","interfaces":[{"interface":"back","average":{"1min":1.486,"5min":1.647,"15min":1.2230000000000001},"min":{"1min":0.32800000000000001,"5min":0.26800000000000002,"15min":0.23999999999999999},"max":{"1min":5.8819999999999997,"5min":8.6059999999999999,"15min":8.6059999999999999},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.1679999999999999,"5min":1.7829999999999999,"15min":1.248},"min":{"1min":0.33800000000000002,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":4.7480000000000002,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.41799999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:18:42 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":2.0699999999999998,"15min":1.9930000000000001},"min":{"1min":0.36399999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.8179999999999996,"5min":13.502000000000001,"15min":13.502000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":2.1520000000000001,"5min":2.298,"15min":2.4420000000000002},"min":{"1min":0.46400000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.9080000000000004,"5min":10.081,"15min":10.081},"last":0.40000000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.1930000000000001,"15min":1.4970000000000001},"min":{"1min":0.45700000000000002,"5min":0.36299999999999999,"15min":0.26800000000000002},"max":{"1min":5.851,"5min":10.743,"15min":13.460000000000001},"last":0.42899999999999999},{"interface":"front","average":{"1min":1.702,"5min":2.21,"15min":1.536},"min":{"1min":0.46400000000000002,"5min":0.32900000000000001,"15min":0.28499999999999998},"max":{"1min":6.2409999999999997,"5min":11.244,"15min":12.404},"last":0.51800000000000002}]}]},{"osd":1,"up_from":545,"seq":2340757176343,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2126440,"kb_used_data":1928544,"kb_used_omap":13,"kb_used_meta":197874,"kb_avail":91618712,"statfs":{"total":95995035648,"available":93817561088,"internally_reserved":0,"allocated":1974829056,"data_stored":1971901898,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":202623629},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":1.6599999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":0.78000000000000003},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.1480000000000001,"5min":2.1480000000000001,"15min":2.1480000000000001},"last":0.29799999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.6379999999999999,"5min":1.6379999999999999,"15min":1.6379999999999999},"last":0.33600000000000002},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":1.367}]},{"osd":3,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.6080000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":0.41499999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":12.663,"5min":12.663,"15min":12.663},"last":1.3260000000000001},{"interface":"front","average":{"1min":1.514,"5min":1.514,"15min":1.514},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.222,"5min":13.222,"15min":13.222},"last":0.45600000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":2.79,"5min":2.79,"15min":2.79},"last":1.413},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":3.7629999999999999,"5min":3.7629999999999999,"15min":3.7629999999999999},"last":0.83499999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.9350000000000001,"5min":1.9350000000000001,"15min":1.9350000000000001},"last":1.3899999999999999},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":2.762,"5min":2.762,"15min":2.762},"last":0.49299999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.575,"5min":1.575,"15min":1.575},"last":0.73099999999999998},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":2.1819999999999999,"5min":2.1819999999999999,"15min":2.1819999999999999},"last":9.0640000000000001}]}]},{"osd":0,"up_from":93,"seq":399431958840,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3993744,"kb_used_data":3142728,"kb_used_omap":7,"kb_used_meta":851000,"kb_avail":89751408,"statfs":{"total":95995035648,"available":91905441792,"internally_reserved":0,"allocated":3218153472,"data_stored":3215367615,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":871424740},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:18:24 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":5.4809999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.2679999999999998,"5min":2.2679999999999998,"15min":2.2679999999999998},"last":0.23400000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:19:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":1.1519999999999999,"15min":1.1299999999999999},"min":{"1min":0.27000000000000002,"5min":0.185,"15min":0.185},"max":{"1min":18.550999999999998,"5min":18.550999999999998,"15min":18.550999999999998},"last":1.288},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.84199999999999997,"15min":0.79600000000000004},"min":{"1min":0.29299999999999998,"5min":0.22,"15min":0.22},"max":{"1min":13.618,"5min":13.618,"15min":13.618},"last":1.2070000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:18:44 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.86299999999999999,"15min":1.4450000000000001},"min":{"1min":0.315,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.5350000000000001,"5min":8.5619999999999994,"15min":12.779999999999999},"last":5.2949999999999999},{"interface":"front","average":{"1min":0.878,"5min":0.88,"15min":1.472},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.677,"5min":6.8200000000000003,"15min":15.198},"last":5.3179999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.2599999999999998,"15min":1.7789999999999999},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":13.574999999999999,"5min":57.746000000000002,"15min":57.746000000000002},"last":2.3759999999999999},{"interface":"front","average":{"1min":2.032,"5min":1.9770000000000001,"15min":1.6499999999999999},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.8970000000000002,"5min":57.816000000000003,"15min":57.816000000000003},"last":0.433}]},{"osd":5,"last update":"Sat Sep 23 18:18:40 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.823,"15min":1.2609999999999999},"min":{"1min":0.35299999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.3140000000000001,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":5.2320000000000002},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.6379999999999999,"15min":1.2310000000000001},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.2799999999999998,"5min":8.8879999999999999,"15min":8.8879999999999999},"last":5.3419999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":1.8049999999999999,"15min":1.861},"min":{"1min":0.433,"5min":0.374,"15min":0.32000000000000001},"max":{"1min":7.1020000000000003,"5min":7.1020000000000003,"15min":58.457999999999998},"last":5.3650000000000002},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.798,"15min":1.7050000000000001},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":13.672000000000001,"5min":13.672000000000001,"15min":58.354999999999997},"last":5.1970000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.6719999999999999,"15min":1.488},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":3.1459999999999999,"5min":8.5009999999999994,"15min":10.416},"last":0.46899999999999997},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.5609999999999999,"15min":1.6759999999999999},"min":{"1min":0.48199999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.6139999999999999,"5min":10.404,"15min":10.404},"last":5.2610000000000001}]}]},{"osd":2,"up_from":0,"seq":2070174236725,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249355,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2258540,"kb_used_data":1586020,"kb_used_omap":3,"kb_used_meta":672508,"kb_avail":91486612,"statfs":{"total":95995035648,"available":93682290688,"internally_reserved":0,"allocated":1624084480,"data_stored":1621096059,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":688648936},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.407,"15min":1.508},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":4.0369999999999999,"5min":195.69800000000001,"15min":195.69800000000001},"last":0.36199999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":2.4470000000000001,"15min":1.8049999999999999},"min":{"1min":0.22600000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":6.1340000000000003,"5min":194.661,"15min":194.661},"last":0.34699999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":1.8109999999999999,"5min":1.8109999999999999,"15min":1.8109999999999999},"last":0.38600000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.9060000000000001,"5min":3.9060000000000001,"15min":3.9060000000000001},"last":0.46700000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.94299999999999995,"15min":1.179},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.597,"5min":6.3849999999999998,"15min":6.3849999999999998},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.93100000000000005,"15min":1.21},"min":{"1min":0.33100000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":4.1040000000000001,"5min":5.931,"15min":5.931},"last":2.105}]},{"osd":4,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.4650000000000001,"15min":1.5669999999999999},"min":{"1min":0.40000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":20.850000000000001,"5min":21.062999999999999,"15min":37.231000000000002},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.833,"15min":1.79},"min":{"1min":0.40300000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":20.786000000000001,"5min":21.128,"15min":37.381},"last":0.57699999999999996}]},{"osd":5,"last update":"Sat Sep 23 18:18:36 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.4790000000000001,"15min":1.337},"min":{"1min":0.40200000000000002,"5min":0.316,"15min":0.316},"max":{"1min":2.8210000000000002,"5min":8.2959999999999994,"15min":8.2959999999999994},"last":0.62},{"interface":"front","average":{"1min":1.069,"5min":1.423,"15min":1.6120000000000001},"min":{"1min":0.47699999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.6349999999999998,"5min":6.4020000000000001,"15min":8.9969999999999999},"last":0.71499999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.4299999999999999,"15min":1.7110000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":3.8079999999999998,"5min":9.8089999999999993,"15min":115.54000000000001},"last":1.7589999999999999},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.53,"15min":1.859},"min":{"1min":0.44900000000000001,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":3.8460000000000001,"5min":14.321,"15min":115.691},"last":0.54100000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.4219999999999999,"15min":1.6319999999999999},"min":{"1min":0.436,"5min":0.36799999999999999,"15min":0.35799999999999998},"max":{"1min":3.5449999999999999,"5min":5.2809999999999997,"15min":15.972},"last":0.77000000000000002},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.327,"15min":1.4550000000000001},"min":{"1min":0.50800000000000001,"5min":0.40699999999999997,"15min":0.34899999999999998},"max":{"1min":2.069,"5min":5.8570000000000002,"15min":9.9109999999999996},"last":2.2749999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019305,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5064176,"kb_used_data":4442664,"kb_used_omap":0,"kb_used_meta":621504,"kb_avail":88680976,"statfs":{"total":95995035648,"available":90809319424,"internally_reserved":0,"allocated":4549287936,"data_stored":4547397154,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":636420096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":1.375,"15min":1.21},"min":{"1min":0.27200000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9660000000000002,"5min":7.2199999999999998,"15min":13.375999999999999},"last":6.5510000000000002},{"interface":"front","average":{"1min":2.2709999999999999,"5min":1.772,"15min":1.4710000000000001},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":8.9429999999999996,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.29199999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":6.5730000000000004},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2170000000000001,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":0.28000000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.524,"15min":1.7689999999999999},"min":{"1min":0.30299999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.173,"5min":11.173,"15min":11.173},"last":3.2509999999999999},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.6459999999999999,"15min":1.702},"min":{"1min":0.40600000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.0909999999999993,"5min":10.938000000000001,"15min":10.938000000000001},"last":3.3100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:18:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.52,"15min":1.887},"min":{"1min":0.35599999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":11.130000000000001,"5min":11.130000000000001,"15min":29.154},"last":0.51500000000000001},{"interface":"front","average":{"1min":1.78,"5min":1.712,"15min":1.873},"min":{"1min":0.41399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.1099999999999994,"5min":10.968,"15min":29.216000000000001},"last":13.470000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:18:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":2.6499999999999999,"15min":2.9969999999999999},"min":{"1min":0.36499999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.0660000000000007,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.496},{"interface":"front","average":{"1min":1.373,"5min":1.8300000000000001,"15min":2.8410000000000002},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.8789999999999996,"5min":13.938000000000001,"15min":32.640000000000001},"last":0.378}]},{"osd":6,"last update":"Sat Sep 23 18:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":2.0739999999999998,"15min":1.391},"min":{"1min":0.379,"5min":0.28199999999999997,"15min":0.25},"max":{"1min":7.7249999999999996,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.42099999999999999},{"interface":"front","average":{"1min":1.782,"5min":2.044,"15min":1.51},"min":{"1min":0.33900000000000002,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.6310000000000002,"5min":18.431000000000001,"15min":18.431000000000001},"last":2.4039999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":2.7919999999999998,"15min":1.7490000000000001},"min":{"1min":0.42699999999999999,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":8.9849999999999994,"5min":19.298999999999999,"15min":19.298999999999999},"last":2.339},{"interface":"front","average":{"1min":2.234,"5min":2.1549999999999998,"15min":1.5569999999999999},"min":{"1min":0.50900000000000001,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.1319999999999997,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.45700000000000002}]}]},{"osd":5,"up_from":420,"seq":1803886264401,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3284132,"kb_used_data":2980060,"kb_used_omap":9,"kb_used_meta":304054,"kb_avail":90461020,"statfs":{"total":95995035648,"available":92632084480,"internally_reserved":0,"allocated":3051581440,"data_stored":3049698930,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":311351741},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,5,3,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.2330000000000001,"15min":0.83799999999999997},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.206,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.29099999999999998},{"interface":"front","average":{"1min":1.038,"5min":1.423,"15min":1.5189999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.6309999999999998,"5min":7.5439999999999996,"15min":7.5439999999999996},"last":0.48699999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:18:19 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.0960000000000001,"15min":1.0960000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":3.1619999999999999,"5min":3.1619999999999999,"15min":3.1619999999999999},"last":0.78400000000000003},{"interface":"front","average":{"1min":0.84699999999999998,"5min":0.84699999999999998,"15min":0.84699999999999998},"min":{"1min":0.27700000000000002,"5min":0.27700000000000002,"15min":0.27700000000000002},"max":{"1min":2.238,"5min":2.238,"15min":2.238},"last":3.9500000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.4590000000000001,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.4930000000000003,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":0.35699999999999998},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.891,"15min":2.141},"min":{"1min":0.314,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.6749999999999998,"5min":7.2359999999999998,"15min":7.2359999999999998},"last":0.56699999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.556,"15min":1.153},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.2330000000000001,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":5.5099999999999998},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.534,"15min":1.5169999999999999},"min":{"1min":0.33500000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.8599999999999999,"5min":7.5179999999999998,"15min":7.5179999999999998},"last":0.54100000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":2.1440000000000001,"15min":2.0379999999999998},"min":{"1min":0.36499999999999999,"5min":0.254,"15min":0.251},"max":{"1min":5.375,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.514,"5min":1.9099999999999999,"15min":1.5580000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.069,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.81100000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.867,"15min":1.1899999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1920000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.39100000000000001},{"interface":"front","average":{"1min":1.232,"5min":2.004,"15min":1.6220000000000001},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.9649999999999999,"5min":12.504,"15min":12.504},"last":0.72399999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":2.363,"15min":1.6970000000000001},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.4240000000000004,"5min":13.506,"15min":13.506},"last":0.55000000000000004},{"interface":"front","average":{"1min":1.2310000000000001,"5min":2.0310000000000001,"15min":1.5349999999999999},"min":{"1min":0.378,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.2610000000000001,"5min":13.138,"15min":13.138},"last":0.747}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3236429824,"data_stored":3236429824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1968898048,"data_stored":1968898048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1637679104,"data_stored":1637679104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4557701120,"data_stored":4557701120,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3045457920,"data_stored":3045457920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2747596800,"data_stored":2747596800,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:19:14.003 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:19:14.039 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:14.085 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:14.132 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:14.138 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:14.139 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:14.239 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:14.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:14 smithi099 ceph-mon[167076]: pgmap v1419: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 802 op/s; 19697/258273 objects degraded (7.626%); 1581/258273 objects misplaced (0.612%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:19:14.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:14 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19697/258273 objects degraded (7.626%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:14.331 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:14 smithi099 ceph-mon[167076]: from='client.46913 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:14.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:14 smithi055 ceph-mon[179668]: pgmap v1419: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 802 op/s; 19697/258273 objects degraded (7.626%); 1581/258273 objects misplaced (0.612%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:19:14.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:14 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19697/258273 objects degraded (7.626%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:14.346 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:14 smithi055 ceph-mon[179668]: from='client.46913 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:14.347 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:14 smithi055 ceph-mon[184174]: pgmap v1419: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 802 op/s; 19697/258273 objects degraded (7.626%); 1581/258273 objects misplaced (0.612%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:19:14.347 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:14 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19697/258273 objects degraded (7.626%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:14.347 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:14 smithi055 ceph-mon[184174]: from='client.46913 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:14.359 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:14.599 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:14.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:14.606 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:14.683 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:14.707 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:14.743 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:14.786 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:14.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:14.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:14.892 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:14.946 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:14.976 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 73 16 89413 89397 76.5077 48.125 0.02195 0.0130655 2023-09-23T18:19:15.008 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:15.016 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:15.016 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:15.117 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:15.176 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:15.242 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:15.244 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:19:15.254 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:15.254 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:15.328 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/600485' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:15.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:15 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:15.355 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:15.416 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/600485' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:15 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:15.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/600485' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:15.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:15 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:15.977 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 74 16 90104 90088 76.0568 43.1875 0.186222 0.0131206 2023-09-23T18:19:16.289 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:16.298 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:16.298 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:16.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[179668]: pgmap v1420: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 804 op/s; 19697/259806 objects degraded (7.581%); 1299/259806 objects misplaced (0.500%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:19:16.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:16.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[179668]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:19:16.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[179668]: osdmap e603: 8 total, 8 up, 6 in 2023-09-23T18:19:16.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[179668]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[179668]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.351 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[179668]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[184174]: pgmap v1420: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 804 op/s; 19697/259806 objects degraded (7.581%); 1299/259806 objects misplaced (0.500%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:19:16.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:16.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[184174]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:19:16.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[184174]: osdmap e603: 8 total, 8 up, 6 in 2023-09-23T18:19:16.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[184174]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[184174]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.352 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:16 smithi055 ceph-mon[184174]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.399 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:16.476 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:16.534 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:16.545 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:16.546 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:16.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:16 smithi099 ceph-mon[167076]: pgmap v1420: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.3 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 50 MiB/s wr, 804 op/s; 19697/259806 objects degraded (7.581%); 1299/259806 objects misplaced (0.500%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:19:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:16 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:16 smithi099 ceph-mon[167076]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:16 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:19:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:16 smithi099 ceph-mon[167076]: osdmap e603: 8 total, 8 up, 6 in 2023-09-23T18:19:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:16 smithi099 ceph-mon[167076]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:16 smithi099 ceph-mon[167076]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:16 smithi099 ceph-mon[167076]: failed to encode map e603 with expected crc 2023-09-23T18:19:16.647 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:16.708 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:16.744 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:16.839 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:16.852 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:16.852 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:16.878 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:16.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:16.979 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 75 16 90985 90969 75.7747 55.0625 0.00924261 0.0131923 2023-09-23T18:19:16.996 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:17.101 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:17.109 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:17.109 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:17.210 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:17.262 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:17.441 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:17.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:17.451 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:17.552 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:17.579 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:17.864 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:17.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:17.876 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:17.977 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:17.980 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 76 16 91753 91737 75.4088 48 0.0140126 0.0132553 2023-09-23T18:19:18.025 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:18.078 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:18.082 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:18.082 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:18.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:18.242 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:18.296 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:18.306 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:18 smithi099 ceph-mon[167076]: pgmap v1422: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.5 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 51 MiB/s wr, 812 op/s; 19697/269460 objects degraded (7.310%); 1299/269460 objects misplaced (0.482%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T18:19:18.307 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:18.308 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:18.348 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:19:18.408 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:18.493 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:18 smithi055 ceph-mon[184174]: pgmap v1422: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.5 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 51 MiB/s wr, 812 op/s; 19697/269460 objects degraded (7.310%); 1299/269460 objects misplaced (0.482%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T18:19:18.493 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:18 smithi055 ceph-mon[179668]: pgmap v1422: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.5 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 51 MiB/s wr, 812 op/s; 19697/269460 objects degraded (7.310%); 1299/269460 objects misplaced (0.482%); 3.4 MiB/s, 54 objects/s recovering 2023-09-23T18:19:18.528 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:18.786 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:18.797 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:18.798 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:18.899 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:18.919 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:19:18.919 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:19:18.940 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:18.980 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 77 16 92513 92497 75.0464 47.5 0.0236842 0.0133197 2023-09-23T18:19:19.001 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:19.007 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:19.007 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:19.108 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:19.182 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:19.224 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:19 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19697/269460 objects degraded (7.310%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:19.395 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:19 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19697/269460 objects degraded (7.310%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:19.395 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:19 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19697/269460 objects degraded (7.310%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:19.761 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:19.768 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:19.768 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:19.869 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:19.869 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:19.902 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:19:19.928 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:19:19.986 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 78 16 93396 93380 74.7858 55.1875 0.0201602 0.0133659 2023-09-23T18:19:19.987 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:19:19.992 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:20.032 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:19:20.094 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:19:20.132 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:19:20.162 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:19:20.204 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:19:20.272 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:19:20.289 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:19:20.289 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:19:20.289 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:19:20.289 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:19:20.290 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:19:20.290 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:19:20.290 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:19:20.290 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:19:20.315 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:20.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:20.331 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:20.431 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:20.504 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:20.569 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:20.573 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:20.574 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:20.674 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:20.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:19:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:19:20] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2023-09-23T18:19:20.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:20 smithi055 ceph-mon[184174]: pgmap v1423: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 811 op/s; 19697/271713 objects degraded (7.249%); 1009/271713 objects misplaced (0.371%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:19:20.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:20 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4131996040' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:20.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:20 smithi055 ceph-mon[179668]: pgmap v1423: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 811 op/s; 19697/271713 objects degraded (7.249%); 1009/271713 objects misplaced (0.371%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:19:20.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:20 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4131996040' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:20.743 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:20 smithi099 ceph-mon[167076]: pgmap v1423: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 51 MiB/s wr, 811 op/s; 19697/271713 objects degraded (7.249%); 1009/271713 objects misplaced (0.371%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:19:20.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:20 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4131996040' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:20.862 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:20.871 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:20.871 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:20.972 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:20.986 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 79 16 94169 94153 74.4508 48.3125 0.0105005 0.0134254 2023-09-23T18:19:21.079 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:21.149 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:21.155 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:21.156 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:21.256 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:21.307 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:21.317 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:19:21.317 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 19697/271713 objects degraded (7.249%), 8 pgs degraded","count":8},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1650,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":603,"num_osds":8,"num_up_osds":8,"osd_up_since":1695493039,"num_in_osds":6,"osd_in_since":1695493021,"num_remapped_pgs":8},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":110},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":7},{"state_name":"active+recovery_wait+degraded","count":1},{"state_name":"active+recovering+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":90571,"data_bytes":5938363044,"bytes_used":21426053120,"bytes_avail":554544160768,"bytes_total":575970213888,"degraded_objects":19697,"degraded_total":271713,"degraded_ratio":0.072491930824068046,"misplaced_objects":1009,"misplaced_total":271713,"misplaced_ratio":0.0037134770879567779,"recovering_objects_per_sec":57,"recovering_bytes_per_sec":3747186,"recovering_keys_per_sec":0,"num_objects_recovered":572,"num_bytes_recovered":37486592,"num_keys_recovered":0,"write_bytes_sec":53181228,"read_op_per_sec":0,"write_op_per_sec":811},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T18:18:19.098325+0000","services":{}},"progress_events":{"57c27436-1bc8-4f63-b41f-a80b88a05748":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T18:19:21.363 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:21.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:21.369 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:21.470 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:21.515 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:21.574 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:21.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3486660259' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:19:21.580 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:21.580 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:21.681 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:21.687 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3486660259' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:19:21.687 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3486660259' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:19:21.748 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:21.801 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:19:21.802 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:19:21.844 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:21.845 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:21.859 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:21.859 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:21.960 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:21.986 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:2023-09-23T18:19:21.987690+0000 min lat: 0.00128263 max lat: 2.2401 avg lat: 0.0134792 2023-09-23T18:19:21.987 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: sec Cur ops started finished avg MB/s cur MB/s last lat(s) avg lat(s) 2023-09-23T18:19:21.987 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 80 16 94983 94967 74.1562 50.875 0.0109565 0.0134792 2023-09-23T18:19:22.060 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:22.188 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:22.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:22.201 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:22.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:22.350 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:22.411 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:22.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:22.417 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:22.518 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:22.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:22 smithi055 ceph-mon[179668]: pgmap v1424: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 19697/276216 objects degraded (7.131%); 1009/276216 objects misplaced (0.365%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:19:22.593 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:22.593 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:22 smithi055 ceph-mon[184174]: pgmap v1424: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 19697/276216 objects degraded (7.131%); 1009/276216 objects misplaced (0.365%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:19:22.594 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:22.603 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:22.818 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:22.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:22 smithi099 ceph-mon[167076]: pgmap v1424: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.6 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 50 MiB/s wr, 806 op/s; 19697/276216 objects degraded (7.131%); 1009/276216 objects misplaced (0.365%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:19:22.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:22.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:22.830 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:22.877 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:22.931 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:22.989 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 81 16 95763 95747 73.8425 48.75 0.030918 0.0135363 2023-09-23T18:19:23.026 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:23.079 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:19:23.409 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:23 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/619720202' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:23.409 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19697/276216 objects degraded (7.131%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:23.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:23 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/619720202' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:23.409 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19697/276216 objects degraded (7.131%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:23.502 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:19:23.549 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:23.557 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:23.558 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:23.659 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:23.723 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:23 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/619720202' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:23.724 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19697/276216 objects degraded (7.131%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:23.726 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:23.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:23.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:23.790 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:23.890 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:23.967 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:23.990 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 82 16 96621 96605 73.5959 53.625 0.0168717 0.0135827 2023-09-23T18:19:24.383 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:24.390 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:24.390 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:24.491 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:24.559 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:24.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[184174]: pgmap v1425: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 50 MiB/s wr, 802 op/s; 19697/282351 objects degraded (6.976%); 1009/282351 objects misplaced (0.357%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:19:24.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:24.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[184174]: failed to encode map e604 with expected crc 2023-09-23T18:19:24.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/619720202' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:24.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[184174]: osdmap e604: 8 total, 8 up, 6 in 2023-09-23T18:19:24.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[184174]: failed to encode map e604 with expected crc 2023-09-23T18:19:24.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[179668]: pgmap v1425: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 50 MiB/s wr, 802 op/s; 19697/282351 objects degraded (6.976%); 1009/282351 objects misplaced (0.357%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:19:24.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:24.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[179668]: failed to encode map e604 with expected crc 2023-09-23T18:19:24.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/619720202' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:24.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[179668]: osdmap e604: 8 total, 8 up, 6 in 2023-09-23T18:19:24.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:24 smithi055 ceph-mon[179668]: failed to encode map e604 with expected crc 2023-09-23T18:19:24.721 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:24.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:24.731 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:24.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:24 smithi099 ceph-mon[167076]: pgmap v1425: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.7 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 50 MiB/s wr, 802 op/s; 19697/282351 objects degraded (6.976%); 1009/282351 objects misplaced (0.357%); 3.6 MiB/s, 57 objects/s recovering 2023-09-23T18:19:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:24 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:24 smithi099 ceph-mon[167076]: failed to encode map e604 with expected crc 2023-09-23T18:19:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/619720202' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:24 smithi099 ceph-mon[167076]: osdmap e604: 8 total, 8 up, 6 in 2023-09-23T18:19:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:24 smithi099 ceph-mon[167076]: failed to encode map e604 with expected crc 2023-09-23T18:19:24.831 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:24.912 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:24.988 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 83 16 97484 97468 73.3591 53.9375 0.0108759 0.0136271 2023-09-23T18:19:24.995 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:19:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:19:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:19:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:19:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121680 9.2 9:406fa8f9:::benchmark_data_smithi055_386467_object17020:head [delete] snapc 0=[] RETRY=2 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748820+0000", 2023-09-23T18:19:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24439991099999, 2023-09-23T18:19:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.036138982, 2023-09-23T18:19:24.995 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121680 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748820+0000", 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:24.996 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748820+0000", 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748819+0000", 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:24.997 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748824+0000", 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7069999999999997e-06 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748835+0000", 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0951999999999999e-05 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748843+0000", 2023-09-23T18:19:24.998 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1180000000000003e-06 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748890+0000", 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7219000000000003e-05 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748898+0000", 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7499999999999992e-06 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:24.999 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.776933+0000", 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.028034573 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.777049+0000", 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000115674 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.000 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.777167+0000", 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118578 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.781984+0000", 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048167610000000001 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.784853+0000", 2023-09-23T18:19:25.001 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028689060000000001 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.784884+0000", 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0855999999999997e-05 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.784959+0000", 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5092000000000005e-05 2023-09-23T18:19:25.002 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121712 9.2 9:44d3445a:::benchmark_data_smithi055_386467_object17052:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748853+0000", 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24436699500001, 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0381930109999999, 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.003 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121712 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748853+0000", 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.004 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748853+0000", 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748853+0000", 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748854+0000", 2023-09-23T18:19:25.005 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.765e-06 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748861+0000", 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3219999999999998e-06 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748864+0000", 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3069999999999998e-06 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.006 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748939+0000", 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4762e-05 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748948+0000", 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6029999999999994e-06 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.007 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.779026+0000", 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.030077779 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.779115+0000", 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8441000000000002e-05 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.779195+0000", 2023-09-23T18:19:25.008 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0343e-05 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.782934+0000", 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037391770000000002 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.786948+0000", 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040140990000000001 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.009 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.786975+0000", 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6910000000000002e-05 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.787046+0000", 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0801000000000002e-05 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.010 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.011 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.011 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.011 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121738 9.2 9:475767a4:::benchmark_data_smithi055_386467_object17078:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.014 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748868+0000", 2023-09-23T18:19:25.014 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24435138600001, 2023-09-23T18:19:25.014 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.040964792, 2023-09-23T18:19:25.014 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.014 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121738 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748868+0000", 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.015 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748868+0000", 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748868+0000", 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.016 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748870+0000", 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.421e-06 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748873+0000", 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0989999999999999e-06 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748876+0000", 2023-09-23T18:19:25.017 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1209999999999998e-06 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748983+0000", 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010746799999999999 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748987+0000", 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5090000000000002e-06 2023-09-23T18:19:25.018 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.780167+0000", 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0311796449999999 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.780245+0000", 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8461000000000007e-05 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.019 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.780310+0000", 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4579999999999995e-05 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.784825+0000", 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045154139999999997 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.020 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.789720+0000", 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.004894598 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.789749+0000", 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9589000000000001e-05 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.789833+0000", 2023-09-23T18:19:25.021 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4109000000000004e-05 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121766 9.2 9:4eab0f17:::benchmark_data_smithi055_386467_object17106:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748899+0000", 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24432056399999, 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.043422123, 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.022 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121766 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748899+0000", 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.023 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748899+0000", 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748899+0000", 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748901+0000", 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3410000000000001e-06 2023-09-23T18:19:25.024 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748905+0000", 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.23e-06 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748907+0000", 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6560000000000001e-06 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.025 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.026 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749027+0000", 2023-09-23T18:19:25.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011943999999999999 2023-09-23T18:19:25.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.026 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.026 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749034+0000", 2023-09-23T18:19:25.026 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4830000000000002e-06 2023-09-23T18:19:25.026 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.026 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.781057+0000", 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0320223829999999 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.781130+0000", 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2990000000000004e-05 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.027 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.781204+0000", 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4900000000000005e-05 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.784828+0000", 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036237750000000001 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.792220+0000", 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0073916429999999998 2023-09-23T18:19:25.028 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.792250+0000", 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9978999999999999e-05 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.792321+0000", 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1574000000000003e-05 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.029 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121774 9.2 9:4d3d1368:::benchmark_data_smithi055_386467_object17114:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748910+0000", 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24430991700001, 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0474983229999999, 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.030 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121774 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748910+0000", 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748910+0000", 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.031 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748910+0000", 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748911+0000", 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.195e-06 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.032 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748914+0000", 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8339999999999999e-06 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748916+0000", 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.57e-06 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.033 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749065+0000", 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014838300000000001 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749069+0000", 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3250000000000001e-06 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.782008+0000", 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.032938731 2023-09-23T18:19:25.034 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.782076+0000", 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8198e-05 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.782147+0000", 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1440999999999996e-05 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.035 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.784970+0000", 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028228239999999998 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.796264+0000", 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011293287000000001 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.036 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.796318+0000", 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4472999999999998e-05 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.796408+0000", 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0289000000000002e-05 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121779 9.2 9:45bb1eda:::benchmark_data_smithi055_386467_object17119:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.037 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748929+0000", 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24429127299999, 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.047732289, 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121779 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.038 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748929+0000", 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748929+0000", 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748928+0000", 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:25.039 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748930+0000", 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.361e-06 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748935+0000", 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9649999999999999e-06 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.040 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748937+0000", 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.802e-06 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749101+0000", 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016309699999999999 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749107+0000", 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4710000000000004e-06 2023-09-23T18:19:25.041 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.782971+0000", 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0338636130000001 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.783039+0000", 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7972999999999994e-05 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.042 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.783103+0000", 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4832999999999994e-05 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.784975+0000", 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001871663 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.043 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.796560+0000", 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011584835999999999 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.796589+0000", 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9037999999999999e-05 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.796661+0000", 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1878999999999994e-05 2023-09-23T18:19:25.044 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121785 9.2 9:4827d468:::benchmark_data_smithi055_386467_object17125:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748940+0000", 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24427994800001, 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.05971708, 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.045 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121785 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748940+0000", 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.046 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748940+0000", 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748940+0000", 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748941+0000", 2023-09-23T18:19:25.047 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.418e-06 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748944+0000", 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.835e-06 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748947+0000", 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5959999999999998e-06 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.048 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749137+0000", 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000190008 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749140+0000", 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1209999999999998e-06 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.049 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.783981+0000", 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0348408609999999 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.784059+0000", 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8855000000000003e-05 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.784123+0000", 2023-09-23T18:19:25.050 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.41e-05 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.787057+0000", 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029333110000000001 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.808564+0000", 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021506925 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.051 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.808601+0000", 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6949000000000001e-05 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.808657+0000", 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6331e-05 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.052 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121789 9.2 9:46fcc006:::benchmark_data_smithi055_386467_object17129:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748949+0000", 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.244271056, 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0598910530000001, 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121789 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.053 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748949+0000", 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748949+0000", 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.054 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748949+0000", 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748952+0000", 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2909999999999999e-06 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748955+0000", 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4039999999999999e-06 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748958+0000", 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7420000000000002e-06 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749172+0000", 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000213484 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749180+0000", 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6710000000000005e-06 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.785019+0000", 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0358390099999999 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.785091+0000", 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1605000000000003e-05 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.785153+0000", 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2456e-05 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.787061+0000", 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019082389999999999 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.808719+0000", 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021657644 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.808755+0000", 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6199999999999999e-05 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.808840+0000", 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4525999999999997e-05 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121802 9.2 9:4534d9cb:::benchmark_data_smithi055_386467_object17142:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748980+0000", 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24423971499999, 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.059985089, 2023-09-23T18:19:25.060 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121802 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748980+0000", 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748980+0000", 2023-09-23T18:19:25.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748980+0000", 2023-09-23T18:19:25.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:25.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748981+0000", 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2890000000000001e-06 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748985+0000", 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0620000000000002e-06 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748988+0000", 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8270000000000002e-06 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749218+0000", 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00022969399999999999 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749222+0000", 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2640000000000002e-06 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.786084+0000", 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0368619029999999 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.786154+0000", 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0217999999999998e-05 2023-09-23T18:19:25.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.786228+0000", 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3767e-05 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.792151+0000", 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.005922704 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.808888+0000", 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016737655000000001 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.808918+0000", 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0139999999999999e-05 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.808965+0000", 2023-09-23T18:19:25.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6808000000000003e-05 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121805 9.2 9:44f9cfd2:::benchmark_data_smithi055_386467_object17145:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748990+0000", 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.244229437, 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0601463280000001, 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.068 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121805 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748990+0000", 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.069 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748990+0000", 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748990+0000", 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.070 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748991+0000", 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2610000000000001e-06 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748994+0000", 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8940000000000002e-06 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748997+0000", 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5959999999999998e-06 2023-09-23T18:19:25.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749256+0000", 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00025863699999999998 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749264+0000", 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8270000000000004e-06 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.787100+0000", 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.037835638 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.787176+0000", 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6180000000000006e-05 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.787249+0000", 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3077999999999994e-05 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.792160+0000", 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049110940000000004 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.809023+0000", 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016862663 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.809054+0000", 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0476000000000001e-05 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.809137+0000", 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3207999999999993e-05 2023-09-23T18:19:25.075 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121806 9.2 9:4bbb21c0:::benchmark_data_smithi055_386467_object17146:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.748999+0000", 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.244220609, 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.060269353, 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.076 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121806 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748999+0000", 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748999+0000", 2023-09-23T18:19:25.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.748999+0000", 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749000+0000", 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1489999999999999e-06 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749003+0000", 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7999999999999999e-06 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749005+0000", 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4540000000000001e-06 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749293+0000", 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00028749100000000002 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749297+0000", 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0080000000000004e-06 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.788276+0000", 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0389789920000001 2023-09-23T18:19:25.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.788345+0000", 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8938999999999998e-05 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.788456+0000", 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111011 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.793465+0000", 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050092950000000004 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.809178+0000", 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015712791 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.809223+0000", 2023-09-23T18:19:25.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5475000000000001e-05 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.809269+0000", 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5167999999999999e-05 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121857 9.2 9:48b8dd07:::benchmark_data_smithi055_386467_object17197:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.749015+0000", 2023-09-23T18:19:25.083 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24420472599999, 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0604276029999999, 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121857 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749015+0000", 2023-09-23T18:19:25.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749015+0000", 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749015+0000", 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749019+0000", 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0910000000000003e-06 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749028+0000", 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3179999999999996e-06 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749032+0000", 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8260000000000003e-06 2023-09-23T18:19:25.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749329+0000", 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00029656399999999997 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749337+0000", 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4670000000000004e-06 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.087 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.789886+0000", 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0405491950000001 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.789991+0000", 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104413 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.790119+0000", 2023-09-23T18:19:25.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012835900000000001 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.793474+0000", 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033548319999999999 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.809309+0000", 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015835213000000001 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.809362+0000", 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2997000000000001e-05 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.809443+0000", 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0557999999999997e-05 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121863 9.2 9:47bf114b:::benchmark_data_smithi055_386467_object17203:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.749035+0000", 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24418497900001, 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0706029880000001, 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121863 2023-09-23T18:19:25.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749035+0000", 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749035+0000", 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749035+0000", 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749036+0000", 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8080000000000001e-06 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749040+0000", 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0699999999999998e-06 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749042+0000", 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5809999999999999e-06 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749370+0000", 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00032831499999999998 2023-09-23T18:19:25.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749378+0000", 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0790000000000001e-06 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.791046+0000", 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.041667425 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.791196+0000", 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014979699999999999 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.791324+0000", 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012786899999999999 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.796220+0000", 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048964789999999996 2023-09-23T18:19:25.096 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.819499+0000", 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.02327891 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.819549+0000", 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0399999999999999e-05 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.097 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.819638+0000", 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8500999999999999e-05 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121866 9.2 9:48f566f2:::benchmark_data_smithi055_386467_object17206:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.749063+0000", 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24415644199999, 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.070764158, 2023-09-23T18:19:25.098 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121866 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749063+0000", 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.099 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749063+0000", 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749063+0000", 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.100 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749064+0000", 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2759999999999999e-06 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749068+0000", 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7960000000000002e-06 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749071+0000", 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7080000000000002e-06 2023-09-23T18:19:25.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749418+0000", 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00034684400000000002 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749427+0000", 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9220000000000003e-06 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.102 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.792368+0000", 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0429416709999999 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.792468+0000", 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9896999999999996e-05 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.103 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.792536+0000", 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7829000000000004e-05 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.796424+0000", 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038881710000000002 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.819693+0000", 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.023269149999999999 2023-09-23T18:19:25.104 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.819740+0000", 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6152999999999997e-05 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.819828+0000", 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7967000000000007e-05 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.105 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121872 9.2 9:43475b8a:::benchmark_data_smithi055_386467_object17212:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.749082+0000", 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.244137728, 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.070904297, 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.106 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121872 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749082+0000", 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749082+0000", 2023-09-23T18:19:25.107 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749082+0000", 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749083+0000", 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.291e-06 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.108 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749088+0000", 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9320000000000004e-06 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749091+0000", 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7070000000000001e-06 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.109 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749458+0000", 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00036695599999999999 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749461+0000", 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4979999999999998e-06 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.793525+0000", 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.044063747 2023-09-23T18:19:25.110 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.793669+0000", 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014404700000000001 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.793819+0000", 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015004600000000001 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.111 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.796430+0000", 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002610897 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.819867+0000", 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.023437054999999998 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.819911+0000", 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4283999999999998e-05 2023-09-23T18:19:25.112 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.819986+0000", 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5067999999999993e-05 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:121881 9.2 9:40dbfb1a:::benchmark_data_smithi055_386467_object17221:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e502)", 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:15:46.749093+0000", 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 218.24412669899999, 2023-09-23T18:19:25.113 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.07107223, 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 121881 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.114 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749093+0000", 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749093+0000", 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749093+0000", 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:19:25.115 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749094+0000", 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1990000000000001e-06 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749097+0000", 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8279999999999999e-06 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.116 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749099+0000", 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4509999999999999e-06 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749484+0000", 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00038486899999999998 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:25.117 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:46.749493+0000", 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5269999999999998e-06 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.795201+0000", 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0457086609999999 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.795313+0000", 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011181299999999999 2023-09-23T18:19:25.118 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 7", 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.795405+0000", 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1812999999999997e-05 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.796506+0000", 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001100679 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.119 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.820033+0000", 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.023527329 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.820082+0000", 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9135999999999998e-05 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:15:47.820165+0000", 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3163000000000003e-05 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.120 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:155104 9.2 9:4aff3af9:::benchmark_data_smithi055_386467_object50444:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e520)", 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:26.118912+0000", 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 178.87430798400001, 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.048601144999999998, 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.121 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 155104 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.118912+0000", 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.122 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.118912+0000", 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.118911+0000", 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.123 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.118917+0000", 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9669999999999998e-06 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.118929+0000", 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1975e-05 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.118935+0000", 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9499999999999998e-06 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.124 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.119372+0000", 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00043765200000000002 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.119516+0000", 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000143273 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.125 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.119649+0000", 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013353900000000001 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.121938+0000", 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002289068 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.126 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.123887+0000", 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019491160000000001 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.167381+0000", 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.043493974999999997 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.167446+0000", 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4418999999999994e-05 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.127 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:26.167513+0000", 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7272000000000002e-05 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:170024 9.2 9:490709fd:::benchmark_data_smithi055_386467_object65364:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:41.973760+0000", 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 163.019460356, 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.034843671999999999, 2023-09-23T18:19:25.128 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 170024 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.973760+0000", 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.129 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.973760+0000", 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.973759+0000", 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.130 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.973763+0000", 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.464e-06 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.973774+0000", 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0747999999999999e-05 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.973779+0000", 2023-09-23T18:19:25.131 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6240000000000001e-06 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.976804+0000", 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003024971 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.976892+0000", 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8636999999999999e-05 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.132 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.976972+0000", 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9902999999999997e-05 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.978861+0000", 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001889169 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:41.995899+0000", 2023-09-23T18:19:25.133 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.017037272999999999 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.008495+0000", 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012596703000000001 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.008527+0000", 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1965999999999999e-05 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.134 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.008603+0000", 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5897999999999997e-05 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:170603 9.2 9:4a665dfa:::benchmark_data_smithi055_386467_object65943:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:42.683452+0000", 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 162.30976790299999, 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.035519716, 2023-09-23T18:19:25.135 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 170603 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.683452+0000", 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.136 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.683452+0000", 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.683451+0000", 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.137 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.683457+0000", 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3060000000000003e-06 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.683469+0000", 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1936999999999999e-05 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.683477+0000", 2023-09-23T18:19:25.138 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1510000000000004e-06 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.683558+0000", 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0327000000000005e-05 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.683669+0000", 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000111418 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.139 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.683821+0000", 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000152144 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.686864+0000", 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030425780000000002 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.696234+0000", 2023-09-23T18:19:25.140 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093702379999999995 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.718844+0000", 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022610109999999999 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.718885+0000", 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0998000000000003e-05 2023-09-23T18:19:25.141 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:42.718972+0000", 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6601999999999995e-05 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.36552.0:171057 9.2 9:4a05eadf:::benchmark_data_smithi055_386467_object66397:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e523)", 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:16:43.185938+0000", 2023-09-23T18:19:25.142 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 161.80728212700001, 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.035871552000000001, 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.36552", 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 171057 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:25.143 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.185938+0000", 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.185938+0000", 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.185936+0000", 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:25.144 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.185944+0000", 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5859999999999999e-06 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.185960+0000", 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6175e-05 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.145 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.185968+0000", 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0390000000000005e-06 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.186023+0000", 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4481999999999998e-05 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.186125+0000", 2023-09-23T18:19:25.146 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010287 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 6,7", 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.186218+0000", 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2886999999999993e-05 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.188807+0000", 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00258896 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.147 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.198978+0000", 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010170627999999999 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.221688+0000", 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022710415000000001 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.221730+0000", 2023-09-23T18:19:25.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1196000000000003e-05 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:16:43.221809+0000", 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9785999999999998e-05 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:25.149 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:19:25.482 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:25.500 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:25.501 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:25.603 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:25.672 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:25.852 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:25.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:25.861 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:25.948 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:25.962 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:25.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 84 16 98361 98345 73.1361 54.8125 0.0186519 0.0136676 2023-09-23T18:19:25.992 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:19:26.040 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:26.117 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:26.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:26.128 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:26.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:26.282 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:26.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:26.362 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:26.362 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:26.463 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:26.538 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:26 smithi055 ceph-mon[179668]: pgmap v1427: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 49 MiB/s wr, 785 op/s; 19697/283806 objects degraded (6.940%); 681/283806 objects misplaced (0.240%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:19:26.539 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:26 smithi055 ceph-mon[184174]: pgmap v1427: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 49 MiB/s wr, 785 op/s; 19697/283806 objects degraded (6.940%); 681/283806 objects misplaced (0.240%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:19:26.564 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:26 smithi099 ceph-mon[167076]: pgmap v1427: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 5.8 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 49 MiB/s wr, 785 op/s; 19697/283806 objects degraded (6.940%); 681/283806 objects misplaced (0.240%); 3.9 MiB/s, 62 objects/s recovering 2023-09-23T18:19:26.744 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:26.755 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:26.756 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:26.856 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:26.904 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:19:26.931 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:19:26.974 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:26.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:26.978 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:26.990 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 85 16 99136 99120 72.8456 48.4375 0.0197281 0.0137231 2023-09-23T18:19:27.079 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:27.128 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:27.165 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:27.342 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:27.350 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:27.350 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:27.451 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:27.498 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1427,"stamp":"2023-09-23T18:19:25.129969+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6202538660,"num_objects":94602,"num_object_clones":0,"num_object_copies":283806,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":19697,"num_objects_misplaced":681,"num_objects_unfound":0,"num_objects_dirty":94602,"num_whiteouts":0,"num_read":5959,"num_read_kb":12160,"num_write":95486,"num_write_kb":6065562,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3175,"num_bytes_recovered":197746313,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82697,"ondisk_log_size":82697,"up":357,"acting":350,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":379,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":21757048,"kb_used_data":18463304,"kb_used_omap":40,"kb_used_meta":3293655,"kb_avail":540713864,"statfs":{"total":575970213888,"available":553690996736,"internally_reserved":0,"allocated":18906423296,"data_stored":18892026996,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3372703034},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,3,0,4,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":509739008,"num_objects":7778,"num_object_clones":0,"num_object_copies":23334,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-618,"num_objects_unfound":0,"num_objects_dirty":7778,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7778,"num_write_kb":497792,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":618,"num_bytes_recovered":40501248,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.901348"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":891,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464209+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:19:07.464209+0000","last_peered":"2023-09-23T18:19:07.464209+0000","last_clean":"2023-09-23T18:19:07.464209+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:19:07.464209+0000","last_undegraded":"2023-09-23T18:19:07.464209+0000","last_fullsized":"2023-09-23T18:19:07.464209+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":262,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.476029+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:19:07.476029+0000","last_peered":"2023-09-23T18:19:07.476029+0000","last_clean":"2023-09-23T18:19:07.476029+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:19:07.476029+0000","last_undegraded":"2023-09-23T18:19:07.476029+0000","last_fullsized":"2023-09-23T18:19:07.476029+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":19,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470592+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:19:07.470592+0000","last_peered":"2023-09-23T18:19:07.470592+0000","last_clean":"2023-09-23T18:19:07.470592+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:19:07.470592+0000","last_undegraded":"2023-09-23T18:19:07.470592+0000","last_fullsized":"2023-09-23T18:19:07.470592+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":897,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464323+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:19:07.464323+0000","last_peered":"2023-09-23T18:19:07.464323+0000","last_clean":"2023-09-23T18:19:07.464323+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:19:07.464323+0000","last_undegraded":"2023-09-23T18:19:07.464323+0000","last_fullsized":"2023-09-23T18:19:07.464323+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":343,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470145+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:19:07.470145+0000","last_peered":"2023-09-23T18:19:07.470145+0000","last_clean":"2023-09-23T18:19:07.470145+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:19:07.470145+0000","last_undegraded":"2023-09-23T18:19:07.470145+0000","last_fullsized":"2023-09-23T18:19:07.470145+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":663,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.472732+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:19:07.472732+0000","last_peered":"2023-09-23T18:19:07.472732+0000","last_clean":"2023-09-23T18:19:07.472732+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:19:07.472732+0000","last_undegraded":"2023-09-23T18:19:07.472732+0000","last_fullsized":"2023-09-23T18:19:07.472732+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":914,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.463986+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:19:07.463986+0000","last_peered":"2023-09-23T18:19:07.463986+0000","last_clean":"2023-09-23T18:19:07.463986+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:07.463986+0000","last_undegraded":"2023-09-23T18:19:07.463986+0000","last_fullsized":"2023-09-23T18:19:07.463986+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":19,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.469016+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:07.469016+0000","last_peered":"2023-09-23T18:19:07.469016+0000","last_clean":"2023-09-23T18:19:07.469016+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:19:07.469016+0000","last_undegraded":"2023-09-23T18:19:07.469016+0000","last_fullsized":"2023-09-23T18:19:07.469016+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":179,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.478925+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:19:07.478925+0000","last_peered":"2023-09-23T18:19:07.478925+0000","last_clean":"2023-09-23T18:19:07.478925+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:19:07.478925+0000","last_undegraded":"2023-09-23T18:19:07.478925+0000","last_fullsized":"2023-09-23T18:19:07.478925+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1037,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464525+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:19:07.464525+0000","last_peered":"2023-09-23T18:19:07.464525+0000","last_clean":"2023-09-23T18:19:07.464525+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:19:07.464525+0000","last_undegraded":"2023-09-23T18:19:07.464525+0000","last_fullsized":"2023-09-23T18:19:07.464525+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":113,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464778+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:19:07.464778+0000","last_peered":"2023-09-23T18:19:07.464778+0000","last_clean":"2023-09-23T18:19:07.464778+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:19:07.464778+0000","last_undegraded":"2023-09-23T18:19:07.464778+0000","last_fullsized":"2023-09-23T18:19:07.464778+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":192,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.469409+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:19:07.469409+0000","last_peered":"2023-09-23T18:19:07.469409+0000","last_clean":"2023-09-23T18:19:07.469409+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:19:07.469409+0000","last_undegraded":"2023-09-23T18:19:07.469409+0000","last_fullsized":"2023-09-23T18:19:07.469409+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":814,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.473607+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:19:07.473607+0000","last_peered":"2023-09-23T18:19:07.473607+0000","last_clean":"2023-09-23T18:19:07.473607+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:19:07.473607+0000","last_undegraded":"2023-09-23T18:19:07.473607+0000","last_fullsized":"2023-09-23T18:19:07.473607+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":149,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.467021+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:19:07.467021+0000","last_peered":"2023-09-23T18:19:07.467021+0000","last_clean":"2023-09-23T18:19:07.467021+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:19:07.467021+0000","last_undegraded":"2023-09-23T18:19:07.467021+0000","last_fullsized":"2023-09-23T18:19:07.467021+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":148,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.473058+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:19:07.473058+0000","last_peered":"2023-09-23T18:19:07.473058+0000","last_clean":"2023-09-23T18:19:07.473058+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:19:07.473058+0000","last_undegraded":"2023-09-23T18:19:07.473058+0000","last_fullsized":"2023-09-23T18:19:07.473058+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":948,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464852+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:19:07.464852+0000","last_peered":"2023-09-23T18:19:07.464852+0000","last_clean":"2023-09-23T18:19:07.464852+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:19:07.464852+0000","last_undegraded":"2023-09-23T18:19:07.464852+0000","last_fullsized":"2023-09-23T18:19:07.464852+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":397,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.242443+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:19:15.242443+0000","last_peered":"2023-09-23T18:19:15.242443+0000","last_clean":"2023-09-23T18:19:15.242443+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:19:15.242443+0000","last_undegraded":"2023-09-23T18:19:15.242443+0000","last_fullsized":"2023-09-23T18:19:15.242443+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":416,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.474560+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:19:07.474560+0000","last_peered":"2023-09-23T18:19:07.474560+0000","last_clean":"2023-09-23T18:19:07.474560+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:19:07.474560+0000","last_undegraded":"2023-09-23T18:19:07.474560+0000","last_fullsized":"2023-09-23T18:19:07.474560+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":977,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.228495+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:19:15.228495+0000","last_peered":"2023-09-23T18:19:15.228495+0000","last_clean":"2023-09-23T18:19:15.228495+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:19:15.228495+0000","last_undegraded":"2023-09-23T18:19:15.228495+0000","last_fullsized":"2023-09-23T18:19:15.228495+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":155,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.472995+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:19:07.472995+0000","last_peered":"2023-09-23T18:19:07.472995+0000","last_clean":"2023-09-23T18:19:07.472995+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:19:07.472995+0000","last_undegraded":"2023-09-23T18:19:07.472995+0000","last_fullsized":"2023-09-23T18:19:07.472995+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":422,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470351+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:19:07.470351+0000","last_peered":"2023-09-23T18:19:07.470351+0000","last_clean":"2023-09-23T18:19:07.470351+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:19:07.470351+0000","last_undegraded":"2023-09-23T18:19:07.470351+0000","last_fullsized":"2023-09-23T18:19:07.470351+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":829,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.467264+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:19:07.467264+0000","last_peered":"2023-09-23T18:19:07.467264+0000","last_clean":"2023-09-23T18:19:07.467264+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:19:07.467264+0000","last_undegraded":"2023-09-23T18:19:07.467264+0000","last_fullsized":"2023-09-23T18:19:07.467264+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":422,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470208+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:19:07.470208+0000","last_peered":"2023-09-23T18:19:07.470208+0000","last_clean":"2023-09-23T18:19:07.470208+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:19:07.470208+0000","last_undegraded":"2023-09-23T18:19:07.470208+0000","last_fullsized":"2023-09-23T18:19:07.470208+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":100,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.463996+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:19:07.463996+0000","last_peered":"2023-09-23T18:19:07.463996+0000","last_clean":"2023-09-23T18:19:07.463996+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:19:07.463996+0000","last_undegraded":"2023-09-23T18:19:07.463996+0000","last_fullsized":"2023-09-23T18:19:07.463996+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":800,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.463705+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:19:07.463705+0000","last_peered":"2023-09-23T18:19:07.463705+0000","last_clean":"2023-09-23T18:19:07.463705+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:19:07.463705+0000","last_undegraded":"2023-09-23T18:19:07.463705+0000","last_fullsized":"2023-09-23T18:19:07.463705+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":118,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470850+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:19:07.470850+0000","last_peered":"2023-09-23T18:19:07.470850+0000","last_clean":"2023-09-23T18:19:07.470850+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:19:07.470850+0000","last_undegraded":"2023-09-23T18:19:07.470850+0000","last_fullsized":"2023-09-23T18:19:07.470850+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":113,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464792+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:19:07.464792+0000","last_peered":"2023-09-23T18:19:07.464792+0000","last_clean":"2023-09-23T18:19:07.464792+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:19:07.464792+0000","last_undegraded":"2023-09-23T18:19:07.464792+0000","last_fullsized":"2023-09-23T18:19:07.464792+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":738,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470673+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:19:07.470673+0000","last_peered":"2023-09-23T18:19:07.470673+0000","last_clean":"2023-09-23T18:19:07.470673+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:19:07.470673+0000","last_undegraded":"2023-09-23T18:19:07.470673+0000","last_fullsized":"2023-09-23T18:19:07.470673+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":190,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470331+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:19:07.470331+0000","last_peered":"2023-09-23T18:19:07.470331+0000","last_clean":"2023-09-23T18:19:07.470331+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:19:07.470331+0000","last_undegraded":"2023-09-23T18:19:07.470331+0000","last_fullsized":"2023-09-23T18:19:07.470331+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":201,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470938+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:19:07.470938+0000","last_peered":"2023-09-23T18:19:07.470938+0000","last_clean":"2023-09-23T18:19:07.470938+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:19:07.470938+0000","last_undegraded":"2023-09-23T18:19:07.470938+0000","last_fullsized":"2023-09-23T18:19:07.470938+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":834,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.473304+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:19:07.473304+0000","last_peered":"2023-09-23T18:19:07.473304+0000","last_clean":"2023-09-23T18:19:07.473304+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:19:07.473304+0000","last_undegraded":"2023-09-23T18:19:07.473304+0000","last_fullsized":"2023-09-23T18:19:07.473304+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":743,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470102+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:19:07.470102+0000","last_peered":"2023-09-23T18:19:07.470102+0000","last_clean":"2023-09-23T18:19:07.470102+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:19:07.470102+0000","last_undegraded":"2023-09-23T18:19:07.470102+0000","last_fullsized":"2023-09-23T18:19:07.470102+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":19,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.471297+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:19:07.471297+0000","last_peered":"2023-09-23T18:19:07.471297+0000","last_clean":"2023-09-23T18:19:07.471297+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:19:07.471297+0000","last_undegraded":"2023-09-23T18:19:07.471297+0000","last_fullsized":"2023-09-23T18:19:07.471297+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"603'5963","reported_seq":6000,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:21.049128+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:19:21.049128+0000","last_peered":"2023-09-23T18:19:21.049128+0000","last_clean":"2023-09-23T18:19:21.049128+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:19:21.049128+0000","last_undegraded":"2023-09-23T18:19:21.049128+0000","last_fullsized":"2023-09-23T18:19:21.049128+0000","mapping_epoch":574,"log_start":"578'500","ondisk_log_start":"578'500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5463,"log_dups_size":0,"ondisk_log_size":5463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390791168,"num_objects":5963,"num_object_clones":0,"num_object_copies":17889,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5963,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5963,"num_write_kb":381632,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":868,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.478760+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:19:07.478760+0000","last_peered":"2023-09-23T18:19:07.478760+0000","last_clean":"2023-09-23T18:19:07.478760+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:19:07.478760+0000","last_undegraded":"2023-09-23T18:19:07.478760+0000","last_fullsized":"2023-09-23T18:19:07.478760+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":239,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.476953+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:19:07.476953+0000","last_peered":"2023-09-23T18:19:07.476953+0000","last_clean":"2023-09-23T18:19:07.476953+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:19:07.476953+0000","last_undegraded":"2023-09-23T18:19:07.476953+0000","last_fullsized":"2023-09-23T18:19:07.476953+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":113,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.463699+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:19:07.463699+0000","last_peered":"2023-09-23T18:19:07.463699+0000","last_clean":"2023-09-23T18:19:07.463699+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:19:07.463699+0000","last_undegraded":"2023-09-23T18:19:07.463699+0000","last_fullsized":"2023-09-23T18:19:07.463699+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"603'5799","reported_seq":5846,"reported_epoch":603,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:21.170462+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:19:21.170462+0000","last_peered":"2023-09-23T18:19:21.170462+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:19:21.170462+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"582'1300","ondisk_log_start":"582'1300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4499,"log_dups_size":0,"ondisk_log_size":4499,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380043264,"num_objects":5799,"num_object_clones":0,"num_object_copies":17397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5799,"num_write_kb":371136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":4148},{"shards":"5,6","objects":1651}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":728,"reported_epoch":602,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:07.473306+0000","last_change":"2023-09-23T18:18:47.441716+0000","last_active":"2023-09-23T18:19:07.473306+0000","last_peered":"2023-09-23T18:19:07.473306+0000","last_clean":"2023-09-23T18:11:33.614788+0000","last_became_active":"2023-09-23T18:18:47.441414+0000","last_became_peered":"2023-09-23T18:18:47.441414+0000","last_unstale":"2023-09-23T18:19:07.473306+0000","last_undegraded":"2023-09-23T18:18:47.413783+0000","last_fullsized":"2023-09-23T18:18:47.413246+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":484,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":2047528,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[0,6],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4,6","objects":1},{"shards":"0,6","objects":-1}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":240,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.476218+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:19:07.476218+0000","last_peered":"2023-09-23T18:19:07.476218+0000","last_clean":"2023-09-23T18:19:07.476218+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:19:07.476218+0000","last_undegraded":"2023-09-23T18:19:07.476218+0000","last_fullsized":"2023-09-23T18:19:07.476218+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":219,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.475507+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:19:07.475507+0000","last_peered":"2023-09-23T18:19:07.475507+0000","last_clean":"2023-09-23T18:19:07.475507+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:19:07.475507+0000","last_undegraded":"2023-09-23T18:19:07.475507+0000","last_fullsized":"2023-09-23T18:19:07.475507+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"604'5966","reported_seq":11591,"reported_epoch":604,"state":"active+recovering+remapped","last_fresh":"2023-09-23T18:19:23.871649+0000","last_change":"2023-09-23T18:18:34.707552+0000","last_active":"2023-09-23T18:19:23.871649+0000","last_peered":"2023-09-23T18:19:23.871649+0000","last_clean":"2023-09-23T18:18:32.618842+0000","last_became_active":"2023-09-23T18:18:34.683472+0000","last_became_peered":"2023-09-23T18:18:34.683472+0000","last_unstale":"2023-09-23T18:19:23.871649+0000","last_undegraded":"2023-09-23T18:19:23.871649+0000","last_fullsized":"2023-09-23T18:19:23.871649+0000","mapping_epoch":589,"log_start":"583'1600","ondisk_log_start":"583'1600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4366,"log_dups_size":0,"ondisk_log_size":4366,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:33:31.959133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":390987776,"num_objects":5966,"num_object_clones":0,"num_object_copies":17898,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":681,"num_objects_unfound":0,"num_objects_dirty":5966,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5966,"num_write_kb":381824,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2790,"num_bytes_recovered":182845440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[1,4,3],"avail_no_missing":["1","3","4"],"object_location_counts":[{"shards":"1,3,4","objects":5285}],"blocked_by":[],"up_primary":0,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":618,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.478854+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:19:07.478854+0000","last_peered":"2023-09-23T18:19:07.478854+0000","last_clean":"2023-09-23T18:19:07.478854+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:19:07.478854+0000","last_undegraded":"2023-09-23T18:19:07.478854+0000","last_fullsized":"2023-09-23T18:19:07.478854+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":528,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464624+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:19:07.464624+0000","last_peered":"2023-09-23T18:19:07.464624+0000","last_clean":"2023-09-23T18:19:07.464624+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:19:07.464624+0000","last_undegraded":"2023-09-23T18:19:07.464624+0000","last_fullsized":"2023-09-23T18:19:07.464624+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":179,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.474437+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:19:07.474437+0000","last_peered":"2023-09-23T18:19:07.474437+0000","last_clean":"2023-09-23T18:19:07.474437+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:19:07.474437+0000","last_undegraded":"2023-09-23T18:19:07.474437+0000","last_fullsized":"2023-09-23T18:19:07.474437+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"603'5967","reported_seq":6005,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:20.968988+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:19:20.968988+0000","last_peered":"2023-09-23T18:19:20.968988+0000","last_clean":"2023-09-23T18:19:20.968988+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:19:20.968988+0000","last_undegraded":"2023-09-23T18:19:20.968988+0000","last_fullsized":"2023-09-23T18:19:20.968988+0000","mapping_epoch":574,"log_start":"578'800","ondisk_log_start":"578'800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5167,"log_dups_size":0,"ondisk_log_size":5167,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391053312,"num_objects":5967,"num_object_clones":0,"num_object_copies":17901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5967,"num_write_kb":381888,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":633,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.471023+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:19:07.471023+0000","last_peered":"2023-09-23T18:19:07.471023+0000","last_clean":"2023-09-23T18:19:07.471023+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:19:07.471023+0000","last_undegraded":"2023-09-23T18:19:07.471023+0000","last_fullsized":"2023-09-23T18:19:07.471023+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":127,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464349+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:19:07.464349+0000","last_peered":"2023-09-23T18:19:07.464349+0000","last_clean":"2023-09-23T18:19:07.464349+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:19:07.464349+0000","last_undegraded":"2023-09-23T18:19:07.464349+0000","last_fullsized":"2023-09-23T18:19:07.464349+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":439,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.471487+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:19:07.471487+0000","last_peered":"2023-09-23T18:19:07.471487+0000","last_clean":"2023-09-23T18:19:07.471487+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:07.471487+0000","last_undegraded":"2023-09-23T18:19:07.471487+0000","last_fullsized":"2023-09-23T18:19:07.471487+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"604'6105","reported_seq":6144,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T18:19:23.871630+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:19:23.871630+0000","last_peered":"2023-09-23T18:19:23.871630+0000","last_clean":"2023-09-23T18:19:23.871630+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:19:23.871630+0000","last_undegraded":"2023-09-23T18:19:23.871630+0000","last_fullsized":"2023-09-23T18:19:23.871630+0000","mapping_epoch":574,"log_start":"583'1700","ondisk_log_start":"583'1700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4405,"log_dups_size":0,"ondisk_log_size":4405,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400097280,"num_objects":6105,"num_object_clones":0,"num_object_copies":18315,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6105,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6105,"num_write_kb":390720,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1085,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.466938+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:19:07.466938+0000","last_peered":"2023-09-23T18:19:07.466938+0000","last_clean":"2023-09-23T18:19:07.466938+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:19:07.466938+0000","last_undegraded":"2023-09-23T18:19:07.466938+0000","last_fullsized":"2023-09-23T18:19:07.466938+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":339,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470859+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:19:07.470859+0000","last_peered":"2023-09-23T18:19:07.470859+0000","last_clean":"2023-09-23T18:19:07.470859+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:19:07.470859+0000","last_undegraded":"2023-09-23T18:19:07.470859+0000","last_fullsized":"2023-09-23T18:19:07.470859+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":147,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.472751+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:19:07.472751+0000","last_peered":"2023-09-23T18:19:07.472751+0000","last_clean":"2023-09-23T18:19:07.472751+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:19:07.472751+0000","last_undegraded":"2023-09-23T18:19:07.472751+0000","last_fullsized":"2023-09-23T18:19:07.472751+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"603'5970","reported_seq":6016,"reported_epoch":603,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:21.007851+0000","last_change":"2023-09-23T18:18:47.594759+0000","last_active":"2023-09-23T18:19:21.007851+0000","last_peered":"2023-09-23T18:19:21.007851+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:19:21.007851+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"578'500","ondisk_log_start":"578'500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5470,"log_dups_size":0,"ondisk_log_size":5470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":391249920,"num_objects":5970,"num_object_clones":0,"num_object_copies":17910,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5970,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5970,"num_write_kb":382080,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4226},{"shards":"4,5","objects":1744}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":872,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464482+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:19:07.464482+0000","last_peered":"2023-09-23T18:19:07.464482+0000","last_clean":"2023-09-23T18:19:07.464482+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:19:07.464482+0000","last_undegraded":"2023-09-23T18:19:07.464482+0000","last_fullsized":"2023-09-23T18:19:07.464482+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":155,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.474371+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:19:07.474371+0000","last_peered":"2023-09-23T18:19:07.474371+0000","last_clean":"2023-09-23T18:19:07.474371+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:19:07.474371+0000","last_undegraded":"2023-09-23T18:19:07.474371+0000","last_fullsized":"2023-09-23T18:19:07.474371+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":454,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.478889+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:19:07.478889+0000","last_peered":"2023-09-23T18:19:07.478889+0000","last_clean":"2023-09-23T18:19:07.478889+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:19:07.478889+0000","last_undegraded":"2023-09-23T18:19:07.478889+0000","last_fullsized":"2023-09-23T18:19:07.478889+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"603'5938","reported_seq":5975,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:21.193401+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:19:21.193401+0000","last_peered":"2023-09-23T18:19:21.193401+0000","last_clean":"2023-09-23T18:19:21.193401+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:19:21.193401+0000","last_undegraded":"2023-09-23T18:19:21.193401+0000","last_fullsized":"2023-09-23T18:19:21.193401+0000","mapping_epoch":574,"log_start":"583'1500","ondisk_log_start":"583'1500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4438,"log_dups_size":0,"ondisk_log_size":4438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389152768,"num_objects":5938,"num_object_clones":0,"num_object_copies":17814,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5938,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5938,"num_write_kb":380032,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":874,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464401+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:19:07.464401+0000","last_peered":"2023-09-23T18:19:07.464401+0000","last_clean":"2023-09-23T18:19:07.464401+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:19:07.464401+0000","last_undegraded":"2023-09-23T18:19:07.464401+0000","last_fullsized":"2023-09-23T18:19:07.464401+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":439,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.466812+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:19:07.466812+0000","last_peered":"2023-09-23T18:19:07.466812+0000","last_clean":"2023-09-23T18:19:07.466812+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:19:07.466812+0000","last_undegraded":"2023-09-23T18:19:07.466812+0000","last_fullsized":"2023-09-23T18:19:07.466812+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":215,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.473226+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:19:07.473226+0000","last_peered":"2023-09-23T18:19:07.473226+0000","last_clean":"2023-09-23T18:19:07.473226+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:19:07.473226+0000","last_undegraded":"2023-09-23T18:19:07.473226+0000","last_fullsized":"2023-09-23T18:19:07.473226+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"603'6020","reported_seq":6057,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:22.170692+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:19:22.170692+0000","last_peered":"2023-09-23T18:19:22.170692+0000","last_clean":"2023-09-23T18:19:22.170692+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:19:22.170692+0000","last_undegraded":"2023-09-23T18:19:22.170692+0000","last_fullsized":"2023-09-23T18:19:22.170692+0000","mapping_epoch":574,"log_start":"582'1400","ondisk_log_start":"582'1400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4620,"log_dups_size":0,"ondisk_log_size":4620,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":394526720,"num_objects":6020,"num_object_clones":0,"num_object_copies":18060,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6020,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6020,"num_write_kb":385280,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":855,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464369+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:19:07.464369+0000","last_peered":"2023-09-23T18:19:07.464369+0000","last_clean":"2023-09-23T18:19:07.464369+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:19:07.464369+0000","last_undegraded":"2023-09-23T18:19:07.464369+0000","last_fullsized":"2023-09-23T18:19:07.464369+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":19,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.469200+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:19:07.469200+0000","last_peered":"2023-09-23T18:19:07.469200+0000","last_clean":"2023-09-23T18:19:07.469200+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:19:07.469200+0000","last_undegraded":"2023-09-23T18:19:07.469200+0000","last_fullsized":"2023-09-23T18:19:07.469200+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":331,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.228598+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:19:15.228598+0000","last_peered":"2023-09-23T18:19:15.228598+0000","last_clean":"2023-09-23T18:19:15.228598+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:19:15.228598+0000","last_undegraded":"2023-09-23T18:19:15.228598+0000","last_fullsized":"2023-09-23T18:19:15.228598+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"603'5924","reported_seq":5964,"reported_epoch":603,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:20.141858+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:19:20.141858+0000","last_peered":"2023-09-23T18:19:20.141858+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:19:20.141858+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"580'1000","ondisk_log_start":"580'1000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4924,"log_dups_size":0,"ondisk_log_size":4924,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388235264,"num_objects":5924,"num_object_clones":0,"num_object_copies":17772,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5924,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5924,"num_write_kb":379136,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":1639},{"shards":"0,4,6","objects":4285}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":714,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.479001+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:19:07.479001+0000","last_peered":"2023-09-23T18:19:07.479001+0000","last_clean":"2023-09-23T18:19:07.479001+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:19:07.479001+0000","last_undegraded":"2023-09-23T18:19:07.479001+0000","last_fullsized":"2023-09-23T18:19:07.479001+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":954,"reported_epoch":604,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:23.426749+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:19:23.426749+0000","last_peered":"2023-09-23T18:19:23.426749+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:19:23.426749+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":137,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464873+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:19:07.464873+0000","last_peered":"2023-09-23T18:19:07.464873+0000","last_clean":"2023-09-23T18:19:07.464873+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:19:07.464873+0000","last_undegraded":"2023-09-23T18:19:07.464873+0000","last_fullsized":"2023-09-23T18:19:07.464873+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":117,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464294+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:19:07.464294+0000","last_peered":"2023-09-23T18:19:07.464294+0000","last_clean":"2023-09-23T18:19:07.464294+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:19:07.464294+0000","last_undegraded":"2023-09-23T18:19:07.464294+0000","last_fullsized":"2023-09-23T18:19:07.464294+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"603'5926","reported_seq":5972,"reported_epoch":603,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:21.157648+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:19:21.157648+0000","last_peered":"2023-09-23T18:19:21.157648+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:19:21.157648+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"583'1500","ondisk_log_start":"583'1500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4426,"log_dups_size":0,"ondisk_log_size":4426,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388366336,"num_objects":5926,"num_object_clones":0,"num_object_copies":17778,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5926,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5926,"num_write_kb":379264,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":3558},{"shards":"5,6","objects":2367}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":786,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.467439+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:19:07.467439+0000","last_peered":"2023-09-23T18:19:07.467439+0000","last_clean":"2023-09-23T18:19:07.467439+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:19:07.467439+0000","last_undegraded":"2023-09-23T18:19:07.467439+0000","last_fullsized":"2023-09-23T18:19:07.467439+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1487,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464220+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:19:07.464220+0000","last_peered":"2023-09-23T18:19:07.464220+0000","last_clean":"2023-09-23T18:19:07.464220+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:19:07.464220+0000","last_undegraded":"2023-09-23T18:19:07.464220+0000","last_fullsized":"2023-09-23T18:19:07.464220+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":127,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470284+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:19:07.470284+0000","last_peered":"2023-09-23T18:19:07.470284+0000","last_clean":"2023-09-23T18:19:07.470284+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:19:07.470284+0000","last_undegraded":"2023-09-23T18:19:07.470284+0000","last_fullsized":"2023-09-23T18:19:07.470284+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":427,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.472907+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:19:07.472907+0000","last_peered":"2023-09-23T18:19:07.472907+0000","last_clean":"2023-09-23T18:19:07.472907+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:19:07.472907+0000","last_undegraded":"2023-09-23T18:19:07.472907+0000","last_fullsized":"2023-09-23T18:19:07.472907+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"603'5748","reported_seq":5785,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:21.053671+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:19:21.053671+0000","last_peered":"2023-09-23T18:19:21.053671+0000","last_clean":"2023-09-23T18:19:21.053671+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:19:21.053671+0000","last_undegraded":"2023-09-23T18:19:21.053671+0000","last_fullsized":"2023-09-23T18:19:21.053671+0000","mapping_epoch":574,"log_start":"578'200","ondisk_log_start":"578'200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5548,"log_dups_size":0,"ondisk_log_size":5548,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376700928,"num_objects":5748,"num_object_clones":0,"num_object_copies":17244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5748,"num_write_kb":367872,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":811,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464455+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:19:07.464455+0000","last_peered":"2023-09-23T18:19:07.464455+0000","last_clean":"2023-09-23T18:19:07.464455+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:19:07.464455+0000","last_undegraded":"2023-09-23T18:19:07.464455+0000","last_fullsized":"2023-09-23T18:19:07.464455+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":17,"reported_epoch":602,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:19:07.469762+0000","last_change":"2023-09-23T18:18:46.432707+0000","last_active":"2023-09-23T18:19:07.469762+0000","last_peered":"2023-09-23T18:19:07.469762+0000","last_clean":"2023-09-23T18:11:33.614822+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:19:07.469762+0000","last_undegraded":"2023-09-23T18:18:46.418246+0000","last_fullsized":"2023-09-23T18:19:07.469762+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":396,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470439+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:19:07.470439+0000","last_peered":"2023-09-23T18:19:07.470439+0000","last_clean":"2023-09-23T18:19:07.470439+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:19:07.470439+0000","last_undegraded":"2023-09-23T18:19:07.470439+0000","last_fullsized":"2023-09-23T18:19:07.470439+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":139,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464710+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:19:07.464710+0000","last_peered":"2023-09-23T18:19:07.464710+0000","last_clean":"2023-09-23T18:19:07.464710+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:19:07.464710+0000","last_undegraded":"2023-09-23T18:19:07.464710+0000","last_fullsized":"2023-09-23T18:19:07.464710+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"603'5898","reported_seq":5935,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:21.074421+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:19:21.074421+0000","last_peered":"2023-09-23T18:19:21.074421+0000","last_clean":"2023-09-23T18:19:21.074421+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:19:21.074421+0000","last_undegraded":"2023-09-23T18:19:21.074421+0000","last_fullsized":"2023-09-23T18:19:21.074421+0000","mapping_epoch":574,"log_start":"583'1400","ondisk_log_start":"583'1400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4498,"log_dups_size":0,"ondisk_log_size":4498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":386531328,"num_objects":5898,"num_object_clones":0,"num_object_copies":17694,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5898,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5898,"num_write_kb":377472,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1500,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464240+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:19:07.464240+0000","last_peered":"2023-09-23T18:19:07.464240+0000","last_clean":"2023-09-23T18:19:07.464240+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:19:07.464240+0000","last_undegraded":"2023-09-23T18:19:07.464240+0000","last_fullsized":"2023-09-23T18:19:07.464240+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":553,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470256+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:19:07.470256+0000","last_peered":"2023-09-23T18:19:07.470256+0000","last_clean":"2023-09-23T18:19:07.470256+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:19:07.470256+0000","last_undegraded":"2023-09-23T18:19:07.470256+0000","last_fullsized":"2023-09-23T18:19:07.470256+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":113,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464074+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:19:07.464074+0000","last_peered":"2023-09-23T18:19:07.464074+0000","last_clean":"2023-09-23T18:19:07.464074+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:19:07.464074+0000","last_undegraded":"2023-09-23T18:19:07.464074+0000","last_fullsized":"2023-09-23T18:19:07.464074+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":148,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.479189+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:19:07.479189+0000","last_peered":"2023-09-23T18:19:07.479189+0000","last_clean":"2023-09-23T18:19:07.479189+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:19:07.479189+0000","last_undegraded":"2023-09-23T18:19:07.479189+0000","last_fullsized":"2023-09-23T18:19:07.479189+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"603'5802","reported_seq":5839,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:21.092310+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:19:21.092310+0000","last_peered":"2023-09-23T18:19:21.092310+0000","last_clean":"2023-09-23T18:19:21.092310+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:19:21.092310+0000","last_undegraded":"2023-09-23T18:19:21.092310+0000","last_fullsized":"2023-09-23T18:19:21.092310+0000","mapping_epoch":574,"log_start":"578'300","ondisk_log_start":"578'300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5502,"log_dups_size":0,"ondisk_log_size":5502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380239872,"num_objects":5802,"num_object_clones":0,"num_object_copies":17406,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5802,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5802,"num_write_kb":371328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":801,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470443+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:19:07.470443+0000","last_peered":"2023-09-23T18:19:07.470443+0000","last_clean":"2023-09-23T18:19:07.470443+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:19:07.470443+0000","last_undegraded":"2023-09-23T18:19:07.470443+0000","last_fullsized":"2023-09-23T18:19:07.470443+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":116,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.469334+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:19:07.469334+0000","last_peered":"2023-09-23T18:19:07.469334+0000","last_clean":"2023-09-23T18:19:07.469334+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:19:07.469334+0000","last_undegraded":"2023-09-23T18:19:07.469334+0000","last_fullsized":"2023-09-23T18:19:07.469334+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":19,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.469638+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:19:07.469638+0000","last_peered":"2023-09-23T18:19:07.469638+0000","last_clean":"2023-09-23T18:19:07.469638+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:19:07.469638+0000","last_undegraded":"2023-09-23T18:19:07.469638+0000","last_fullsized":"2023-09-23T18:19:07.469638+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":618,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.467106+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:19:07.467106+0000","last_peered":"2023-09-23T18:19:07.467106+0000","last_clean":"2023-09-23T18:19:07.467106+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:19:07.467106+0000","last_undegraded":"2023-09-23T18:19:07.467106+0000","last_fullsized":"2023-09-23T18:19:07.467106+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"603'5875","reported_seq":5921,"reported_epoch":603,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:21.158043+0000","last_change":"2023-09-23T18:18:34.726444+0000","last_active":"2023-09-23T18:19:21.158043+0000","last_peered":"2023-09-23T18:19:21.158043+0000","last_clean":"2023-09-23T18:18:32.699126+0000","last_became_active":"2023-09-23T18:18:34.716591+0000","last_became_peered":"2023-09-23T18:18:34.716591+0000","last_unstale":"2023-09-23T18:19:21.158043+0000","last_undegraded":"2023-09-23T18:18:34.351246+0000","last_fullsized":"2023-09-23T18:18:34.327168+0000","mapping_epoch":589,"log_start":"583'1400","ondisk_log_start":"583'1400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4475,"log_dups_size":0,"ondisk_log_size":4475,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:13.611234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":385024000,"num_objects":5875,"num_object_clones":0,"num_object_copies":17625,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3453,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5875,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5875,"num_write_kb":376000,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":3453},{"shards":"4,6","objects":2422}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":901,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.474069+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:19:07.474069+0000","last_peered":"2023-09-23T18:19:07.474069+0000","last_clean":"2023-09-23T18:19:07.474069+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:19:07.474069+0000","last_undegraded":"2023-09-23T18:19:07.474069+0000","last_fullsized":"2023-09-23T18:19:07.474069+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":41,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.476727+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:19:07.476727+0000","last_peered":"2023-09-23T18:19:07.476727+0000","last_clean":"2023-09-23T18:19:07.476727+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:19:07.476727+0000","last_undegraded":"2023-09-23T18:19:07.476727+0000","last_fullsized":"2023-09-23T18:19:07.476727+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":127,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464139+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:19:07.464139+0000","last_peered":"2023-09-23T18:19:07.464139+0000","last_clean":"2023-09-23T18:19:07.464139+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:19:07.464139+0000","last_undegraded":"2023-09-23T18:19:07.464139+0000","last_fullsized":"2023-09-23T18:19:07.464139+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":127,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.463842+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:19:07.463842+0000","last_peered":"2023-09-23T18:19:07.463842+0000","last_clean":"2023-09-23T18:19:07.463842+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:19:07.463842+0000","last_undegraded":"2023-09-23T18:19:07.463842+0000","last_fullsized":"2023-09-23T18:19:07.463842+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"603'5830","reported_seq":5867,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:21.193278+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:19:21.193278+0000","last_peered":"2023-09-23T18:19:21.193278+0000","last_clean":"2023-09-23T18:19:21.193278+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:19:21.193278+0000","last_undegraded":"2023-09-23T18:19:21.193278+0000","last_fullsized":"2023-09-23T18:19:21.193278+0000","mapping_epoch":574,"log_start":"582'1400","ondisk_log_start":"582'1400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4430,"log_dups_size":0,"ondisk_log_size":4430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382074880,"num_objects":5830,"num_object_clones":0,"num_object_copies":17490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5830,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5830,"num_write_kb":373120,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":901,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.478963+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:19:07.478963+0000","last_peered":"2023-09-23T18:19:07.478963+0000","last_clean":"2023-09-23T18:19:07.478963+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:19:07.478963+0000","last_undegraded":"2023-09-23T18:19:07.478963+0000","last_fullsized":"2023-09-23T18:19:07.478963+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":89,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464505+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:19:07.464505+0000","last_peered":"2023-09-23T18:19:07.464505+0000","last_clean":"2023-09-23T18:19:07.464505+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:19:07.464505+0000","last_undegraded":"2023-09-23T18:19:07.464505+0000","last_fullsized":"2023-09-23T18:19:07.464505+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":382,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.473893+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:19:07.473893+0000","last_peered":"2023-09-23T18:19:07.473893+0000","last_clean":"2023-09-23T18:19:07.473893+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:19:07.473893+0000","last_undegraded":"2023-09-23T18:19:07.473893+0000","last_fullsized":"2023-09-23T18:19:07.473893+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.469113+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:07.469113+0000","last_peered":"2023-09-23T18:19:07.469113+0000","last_clean":"2023-09-23T18:19:07.469113+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:19:07.469113+0000","last_undegraded":"2023-09-23T18:19:07.469113+0000","last_fullsized":"2023-09-23T18:19:07.469113+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"603'5844","reported_seq":5881,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:21.166778+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:19:21.166778+0000","last_peered":"2023-09-23T18:19:21.166778+0000","last_clean":"2023-09-23T18:19:21.166778+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:19:21.166778+0000","last_undegraded":"2023-09-23T18:19:21.166778+0000","last_fullsized":"2023-09-23T18:19:21.166778+0000","mapping_epoch":574,"log_start":"583'1400","ondisk_log_start":"583'1400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4444,"log_dups_size":0,"ondisk_log_size":4444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":382992384,"num_objects":5844,"num_object_clones":0,"num_object_copies":17532,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5844,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":5844,"num_write_kb":374016,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":735,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470395+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:19:07.470395+0000","last_peered":"2023-09-23T18:19:07.470395+0000","last_clean":"2023-09-23T18:19:07.470395+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:19:07.470395+0000","last_undegraded":"2023-09-23T18:19:07.470395+0000","last_fullsized":"2023-09-23T18:19:07.470395+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1039,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.479222+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:19:07.479222+0000","last_peered":"2023-09-23T18:19:07.479222+0000","last_clean":"2023-09-23T18:19:07.479222+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:19:07.479222+0000","last_undegraded":"2023-09-23T18:19:07.479222+0000","last_fullsized":"2023-09-23T18:19:07.479222+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":29,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.472926+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:19:07.472926+0000","last_peered":"2023-09-23T18:19:07.472926+0000","last_clean":"2023-09-23T18:19:07.472926+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:19:07.472926+0000","last_undegraded":"2023-09-23T18:19:07.472926+0000","last_fullsized":"2023-09-23T18:19:07.472926+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":393,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.467062+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:19:07.467062+0000","last_peered":"2023-09-23T18:19:07.467062+0000","last_clean":"2023-09-23T18:19:07.467062+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:19:07.467062+0000","last_undegraded":"2023-09-23T18:19:07.467062+0000","last_fullsized":"2023-09-23T18:19:07.467062+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":633,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.473137+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:19:07.473137+0000","last_peered":"2023-09-23T18:19:07.473137+0000","last_clean":"2023-09-23T18:19:07.473137+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:19:07.473137+0000","last_undegraded":"2023-09-23T18:19:07.473137+0000","last_fullsized":"2023-09-23T18:19:07.473137+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":25,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.471369+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:19:07.471369+0000","last_peered":"2023-09-23T18:19:07.471369+0000","last_clean":"2023-09-23T18:19:07.471369+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:19:07.471369+0000","last_undegraded":"2023-09-23T18:19:07.471369+0000","last_fullsized":"2023-09-23T18:19:07.471369+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":676,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.228405+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:19:15.228405+0000","last_peered":"2023-09-23T18:19:15.228405+0000","last_clean":"2023-09-23T18:19:15.228405+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:19:15.228405+0000","last_undegraded":"2023-09-23T18:19:15.228405+0000","last_fullsized":"2023-09-23T18:19:15.228405+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":633,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.473719+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:19:07.473719+0000","last_peered":"2023-09-23T18:19:07.473719+0000","last_clean":"2023-09-23T18:19:07.473719+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:19:07.473719+0000","last_undegraded":"2023-09-23T18:19:07.473719+0000","last_fullsized":"2023-09-23T18:19:07.473719+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":121,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464218+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:19:07.464218+0000","last_peered":"2023-09-23T18:19:07.464218+0000","last_clean":"2023-09-23T18:19:07.464218+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:19:07.464218+0000","last_undegraded":"2023-09-23T18:19:07.464218+0000","last_fullsized":"2023-09-23T18:19:07.464218+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":237,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.477301+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:19:07.477301+0000","last_peered":"2023-09-23T18:19:07.477301+0000","last_clean":"2023-09-23T18:19:07.477301+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:19:07.477301+0000","last_undegraded":"2023-09-23T18:19:07.477301+0000","last_fullsized":"2023-09-23T18:19:07.477301+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":107,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464712+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:19:07.464712+0000","last_peered":"2023-09-23T18:19:07.464712+0000","last_clean":"2023-09-23T18:19:07.464712+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:19:07.464712+0000","last_undegraded":"2023-09-23T18:19:07.464712+0000","last_fullsized":"2023-09-23T18:19:07.464712+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":220,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.476100+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:19:07.476100+0000","last_peered":"2023-09-23T18:19:07.476100+0000","last_clean":"2023-09-23T18:19:07.476100+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:19:07.476100+0000","last_undegraded":"2023-09-23T18:19:07.476100+0000","last_fullsized":"2023-09-23T18:19:07.476100+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":995,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.472672+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:19:07.472672+0000","last_peered":"2023-09-23T18:19:07.472672+0000","last_clean":"2023-09-23T18:19:07.472672+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:19:07.472672+0000","last_undegraded":"2023-09-23T18:19:07.472672+0000","last_fullsized":"2023-09-23T18:19:07.472672+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":500,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.473988+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:19:07.473988+0000","last_peered":"2023-09-23T18:19:07.473988+0000","last_clean":"2023-09-23T18:19:07.473988+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:19:07.473988+0000","last_undegraded":"2023-09-23T18:19:07.473988+0000","last_fullsized":"2023-09-23T18:19:07.473988+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":137,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.467350+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:19:07.467350+0000","last_peered":"2023-09-23T18:19:07.467350+0000","last_clean":"2023-09-23T18:19:07.467350+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:19:07.467350+0000","last_undegraded":"2023-09-23T18:19:07.467350+0000","last_fullsized":"2023-09-23T18:19:07.467350+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":932,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.228872+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:19:15.228872+0000","last_peered":"2023-09-23T18:19:15.228872+0000","last_clean":"2023-09-23T18:19:15.228872+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:19:15.228872+0000","last_undegraded":"2023-09-23T18:19:15.228872+0000","last_fullsized":"2023-09-23T18:19:15.228872+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":125,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.464642+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:19:07.464642+0000","last_peered":"2023-09-23T18:19:07.464642+0000","last_clean":"2023-09-23T18:19:07.464642+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:19:07.464642+0000","last_undegraded":"2023-09-23T18:19:07.464642+0000","last_fullsized":"2023-09-23T18:19:07.464642+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":646,"reported_epoch":602,"state":"active+clean","last_fresh":"2023-09-23T18:19:07.470176+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:19:07.470176+0000","last_peered":"2023-09-23T18:19:07.470176+0000","last_clean":"2023-09-23T18:19:07.470176+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:19:07.470176+0000","last_undegraded":"2023-09-23T18:19:07.470176+0000","last_fullsized":"2023-09-23T18:19:07.470176+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6198067200,"num_objects":94575,"num_object_clones":0,"num_object_copies":283725,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":19671,"num_objects_misplaced":681,"num_objects_unfound":0,"num_objects_dirty":94575,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":94575,"num_write_kb":6052800,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2790,"num_bytes_recovered":182845440,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18918408192,"data_stored":18918408192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76675,"ondisk_log_size":76675,"up":48,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2310216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":92,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001561,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669497,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3502588,"kb_used_data":2912308,"kb_used_omap":7,"kb_used_meta":590264,"kb_avail":90242564,"statfs":{"total":95995035648,"available":92408385536,"internally_reserved":0,"allocated":2982203392,"data_stored":2980308897,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":604431140},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":1.599,"15min":1.2949999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":24.472999999999999,"5min":24.472999999999999,"15min":24.472999999999999},"last":3.7839999999999998},{"interface":"front","average":{"1min":3.6230000000000002,"5min":2.081,"15min":1.663},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":22.832999999999998,"5min":22.832999999999998,"15min":22.832999999999998},"last":3.6880000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":1.216,"5min":1.216,"15min":1.216},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":5.3029999999999999,"5min":5.3029999999999999,"15min":5.3029999999999999},"last":8.7940000000000005},{"interface":"front","average":{"1min":1.208,"5min":1.208,"15min":1.208},"min":{"1min":0.315,"5min":0.315,"15min":0.315},"max":{"1min":3.476,"5min":3.476,"15min":3.476},"last":0.747}]},{"osd":2,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.403,"15min":1.2869999999999999},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.2679999999999998,"5min":7.4189999999999996,"15min":7.4189999999999996},"last":0.35499999999999998},{"interface":"front","average":{"1min":1.927,"5min":1.7470000000000001,"15min":1.454},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0439999999999996,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.39300000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.6459999999999999,"15min":1.5509999999999999},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.3150000000000004,"5min":7.3940000000000001,"15min":10.257999999999999},"last":0.61399999999999999},{"interface":"front","average":{"1min":2.282,"5min":2.165,"15min":1.829},"min":{"1min":0.28999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.9539999999999997,"5min":9.4049999999999994,"15min":9.5960000000000001},"last":0.33400000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.8140000000000001,"15min":1.3280000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.2800000000000002,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.70999999999999996},{"interface":"front","average":{"1min":1.7569999999999999,"5min":1.9330000000000001,"15min":1.3200000000000001},"min":{"1min":0.36099999999999999,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":6.2030000000000003,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":1.018}]},{"osd":5,"last update":"Sat Sep 23 18:18:42 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":2.0699999999999998,"15min":1.9930000000000001},"min":{"1min":0.36399999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.8179999999999996,"5min":13.502000000000001,"15min":13.502000000000001},"last":0.77400000000000002},{"interface":"front","average":{"1min":2.1520000000000001,"5min":2.298,"15min":2.4420000000000002},"min":{"1min":0.46400000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.9080000000000004,"5min":10.081,"15min":10.081},"last":0.96999999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.1930000000000001,"15min":1.4970000000000001},"min":{"1min":0.45700000000000002,"5min":0.36299999999999999,"15min":0.26800000000000002},"max":{"1min":5.851,"5min":10.743,"15min":13.460000000000001},"last":0.80300000000000005},{"interface":"front","average":{"1min":1.702,"5min":2.21,"15min":1.536},"min":{"1min":0.46400000000000002,"5min":0.32900000000000001,"15min":0.28499999999999998},"max":{"1min":6.2409999999999997,"5min":11.244,"15min":12.404},"last":0.41199999999999998}]}]},{"osd":1,"up_from":545,"seq":2340757176346,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2379384,"kb_used_data":2167536,"kb_used_omap":13,"kb_used_meta":211826,"kb_avail":91365768,"statfs":{"total":95995035648,"available":93558546432,"internally_reserved":0,"allocated":2219556864,"data_stored":2216622049,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":216910477},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"min":{"1min":0.189,"5min":0.189,"15min":0.189},"max":{"1min":1.6599999999999999,"5min":1.6599999999999999,"15min":1.6599999999999999},"last":1.55},{"interface":"front","average":{"1min":0.48999999999999999,"5min":0.48999999999999999,"15min":0.48999999999999999},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":2.1480000000000001,"5min":2.1480000000000001,"15min":2.1480000000000001},"last":2.2200000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":1.6379999999999999,"5min":1.6379999999999999,"15min":1.6379999999999999},"last":1.589},{"interface":"front","average":{"1min":0.505,"5min":0.505,"15min":0.505},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":1.9219999999999999,"5min":1.9219999999999999,"15min":1.9219999999999999},"last":2.1749999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":1.032},{"interface":"front","average":{"1min":0.51200000000000001,"5min":0.51200000000000001,"15min":0.51200000000000001},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":1.6080000000000001,"5min":1.6080000000000001,"15min":1.6080000000000001},"last":3.3879999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":1.3169999999999999,"5min":1.3169999999999999,"15min":1.3169999999999999},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":12.663,"5min":12.663,"15min":12.663},"last":2.081},{"interface":"front","average":{"1min":1.514,"5min":1.514,"15min":1.514},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":13.222,"5min":13.222,"15min":13.222},"last":0.53900000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":2.79,"5min":2.79,"15min":2.79},"last":2.278},{"interface":"front","average":{"1min":0.80400000000000005,"5min":0.80400000000000005,"15min":0.80400000000000005},"min":{"1min":0.42299999999999999,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":3.7629999999999999,"5min":3.7629999999999999,"15min":3.7629999999999999},"last":4.1890000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"min":{"1min":0.435,"5min":0.435,"15min":0.435},"max":{"1min":1.9350000000000001,"5min":1.9350000000000001,"15min":1.9350000000000001},"last":2.141},{"interface":"front","average":{"1min":0.77600000000000002,"5min":0.77600000000000002,"15min":0.77600000000000002},"min":{"1min":0.378,"5min":0.378,"15min":0.378},"max":{"1min":2.762,"5min":2.762,"15min":2.762},"last":0.621}]},{"osd":7,"last update":"Sat Sep 23 18:18:20 2023","interfaces":[{"interface":"back","average":{"1min":0.68100000000000005,"5min":0.68100000000000005,"15min":0.68100000000000005},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.575,"5min":1.575,"15min":1.575},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"min":{"1min":0.44400000000000001,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":2.1819999999999999,"5min":2.1819999999999999,"15min":2.1819999999999999},"last":2.2480000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958842,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4381600,"kb_used_data":3496152,"kb_used_omap":7,"kb_used_meta":885432,"kb_avail":89363552,"statfs":{"total":95995035648,"available":91508277248,"internally_reserved":0,"allocated":3580059648,"data_stored":3577266134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":906683108},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:18:24 2023","interfaces":[{"interface":"back","average":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.627,"5min":0.627,"15min":0.627},"last":3.5329999999999999},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.52300000000000002,"15min":0.52300000000000002},"min":{"1min":0.23599999999999999,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":2.2679999999999998,"5min":2.2679999999999998,"15min":2.2679999999999998},"last":3.9929999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:19:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":1.1519999999999999,"15min":1.1299999999999999},"min":{"1min":0.27000000000000002,"5min":0.185,"15min":0.185},"max":{"1min":18.550999999999998,"5min":18.550999999999998,"15min":18.550999999999998},"last":3.6429999999999998},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.84199999999999997,"15min":0.79600000000000004},"min":{"1min":0.29299999999999998,"5min":0.22,"15min":0.22},"max":{"1min":13.618,"5min":13.618,"15min":13.618},"last":5.2220000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:18:44 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.86299999999999999,"15min":1.4450000000000001},"min":{"1min":0.315,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.5350000000000001,"5min":8.5619999999999994,"15min":12.779999999999999},"last":3.4489999999999998},{"interface":"front","average":{"1min":0.878,"5min":0.88,"15min":1.472},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.677,"5min":6.8200000000000003,"15min":15.198},"last":3.4740000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.2599999999999998,"15min":1.7789999999999999},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":13.574999999999999,"5min":57.746000000000002,"15min":57.746000000000002},"last":3.3959999999999999},{"interface":"front","average":{"1min":2.032,"5min":1.9770000000000001,"15min":1.6499999999999999},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.8970000000000002,"5min":57.816000000000003,"15min":57.816000000000003},"last":3.585}]},{"osd":5,"last update":"Sat Sep 23 18:18:40 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.823,"15min":1.2609999999999999},"min":{"1min":0.35299999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.3140000000000001,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":3.4249999999999998},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.6379999999999999,"15min":1.2310000000000001},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.2799999999999998,"5min":8.8879999999999999,"15min":8.8879999999999999},"last":3.6150000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":1.8049999999999999,"15min":1.861},"min":{"1min":0.433,"5min":0.374,"15min":0.32000000000000001},"max":{"1min":7.1020000000000003,"5min":7.1020000000000003,"15min":58.457999999999998},"last":3.3300000000000001},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.798,"15min":1.7050000000000001},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":13.672000000000001,"5min":13.672000000000001,"15min":58.354999999999997},"last":3.2109999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.6719999999999999,"15min":1.488},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":3.1459999999999999,"5min":8.5009999999999994,"15min":10.416},"last":3.2450000000000001},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.5609999999999999,"15min":1.6759999999999999},"min":{"1min":0.48199999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.6139999999999999,"5min":10.404,"15min":10.404},"last":3.3690000000000002}]}]},{"osd":2,"up_from":0,"seq":2070174236728,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249357,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2495160,"kb_used_data":1826736,"kb_used_omap":3,"kb_used_meta":668412,"kb_avail":91249992,"statfs":{"total":95995035648,"available":93439991808,"internally_reserved":0,"allocated":1870577664,"data_stored":1867585322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":684454632},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,0,3],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.407,"15min":1.508},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":4.0369999999999999,"5min":195.69800000000001,"15min":195.69800000000001},"last":2.1419999999999999},{"interface":"front","average":{"1min":0.97999999999999998,"5min":2.4470000000000001,"15min":1.8049999999999999},"min":{"1min":0.22600000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":6.1340000000000003,"5min":194.661,"15min":194.661},"last":1.3200000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.247,"5min":0.247,"15min":0.247},"max":{"1min":1.8109999999999999,"5min":1.8109999999999999,"15min":1.8109999999999999},"last":4.6890000000000001},{"interface":"front","average":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":3.9060000000000001,"5min":3.9060000000000001,"15min":3.9060000000000001},"last":1.8180000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.94299999999999995,"15min":1.179},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.597,"5min":6.3849999999999998,"15min":6.3849999999999998},"last":5.3689999999999998},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.93100000000000005,"15min":1.21},"min":{"1min":0.33100000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":4.1040000000000001,"5min":5.931,"15min":5.931},"last":1.857}]},{"osd":4,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.4650000000000001,"15min":1.5669999999999999},"min":{"1min":0.40000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":20.850000000000001,"5min":21.062999999999999,"15min":37.231000000000002},"last":1.379},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.833,"15min":1.79},"min":{"1min":0.40300000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":20.786000000000001,"5min":21.128,"15min":37.381},"last":4.7389999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:18:36 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.4790000000000001,"15min":1.337},"min":{"1min":0.40200000000000002,"5min":0.316,"15min":0.316},"max":{"1min":2.8210000000000002,"5min":8.2959999999999994,"15min":8.2959999999999994},"last":6.2089999999999996},{"interface":"front","average":{"1min":1.069,"5min":1.423,"15min":1.6120000000000001},"min":{"1min":0.47699999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.6349999999999998,"5min":6.4020000000000001,"15min":8.9969999999999999},"last":1.8939999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.4299999999999999,"15min":1.7110000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":3.8079999999999998,"5min":9.8089999999999993,"15min":115.54000000000001},"last":1.925},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.53,"15min":1.859},"min":{"1min":0.44900000000000001,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":3.8460000000000001,"5min":14.321,"15min":115.691},"last":1.413}]},{"osd":7,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.4219999999999999,"15min":1.6319999999999999},"min":{"1min":0.436,"5min":0.36799999999999999,"15min":0.35799999999999998},"max":{"1min":3.5449999999999999,"5min":5.2809999999999997,"15min":15.972},"last":7.0670000000000002},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.327,"15min":1.4550000000000001},"min":{"1min":0.50800000000000001,"5min":0.40699999999999997,"15min":0.34899999999999998},"max":{"1min":2.069,"5min":5.8570000000000002,"15min":9.9109999999999996},"last":1.964}]}]},{"osd":4,"up_from":30,"seq":128849019307,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5441468,"kb_used_data":4828980,"kb_used_omap":0,"kb_used_meta":612480,"kb_avail":88303684,"statfs":{"total":95995035648,"available":90422972416,"internally_reserved":0,"allocated":4944875520,"data_stored":4942980817,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":627179520},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":1.375,"15min":1.21},"min":{"1min":0.27200000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9660000000000002,"5min":7.2199999999999998,"15min":13.375999999999999},"last":2.4129999999999998},{"interface":"front","average":{"1min":2.2709999999999999,"5min":1.772,"15min":1.4710000000000001},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":8.9429999999999996,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.59199999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:18:21 2023","interfaces":[{"interface":"back","average":{"1min":0.69499999999999995,"5min":0.69499999999999995,"15min":0.69499999999999995},"min":{"1min":0.24299999999999999,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":1.552,"5min":1.552,"15min":1.552},"last":5.6740000000000004},{"interface":"front","average":{"1min":0.65200000000000002,"5min":0.65200000000000002,"15min":0.65200000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":1.2170000000000001,"5min":1.2170000000000001,"15min":1.2170000000000001},"last":2.3069999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.524,"15min":1.7689999999999999},"min":{"1min":0.30299999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.173,"5min":11.173,"15min":11.173},"last":3.9209999999999998},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.6459999999999999,"15min":1.702},"min":{"1min":0.40600000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.0909999999999993,"5min":10.938000000000001,"15min":10.938000000000001},"last":3.2850000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:18:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.52,"15min":1.887},"min":{"1min":0.35599999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":11.130000000000001,"5min":11.130000000000001,"15min":29.154},"last":1.51},{"interface":"front","average":{"1min":1.78,"5min":1.712,"15min":1.873},"min":{"1min":0.41399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.1099999999999994,"5min":10.968,"15min":29.216000000000001},"last":4.2389999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:18:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":2.6499999999999999,"15min":2.9969999999999999},"min":{"1min":0.36499999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.0660000000000007,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.61599999999999999},{"interface":"front","average":{"1min":1.373,"5min":1.8300000000000001,"15min":2.8410000000000002},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.8789999999999996,"5min":13.938000000000001,"15min":32.640000000000001},"last":1.613}]},{"osd":6,"last update":"Sat Sep 23 18:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":2.0739999999999998,"15min":1.391},"min":{"1min":0.379,"5min":0.28199999999999997,"15min":0.25},"max":{"1min":7.7249999999999996,"5min":18.315000000000001,"15min":18.315000000000001},"last":1.6830000000000001},{"interface":"front","average":{"1min":1.782,"5min":2.044,"15min":1.51},"min":{"1min":0.33900000000000002,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.6310000000000002,"5min":18.431000000000001,"15min":18.431000000000001},"last":2.8370000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":2.7919999999999998,"15min":1.7490000000000001},"min":{"1min":0.42699999999999999,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":8.9849999999999994,"5min":19.298999999999999,"15min":19.298999999999999},"last":0.56000000000000005},{"interface":"front","average":{"1min":2.234,"5min":2.1549999999999998,"15min":1.5569999999999999},"min":{"1min":0.50900000000000001,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.1319999999999997,"5min":12.800000000000001,"15min":12.800000000000001},"last":1.4239999999999999}]}]},{"osd":5,"up_from":420,"seq":1803886264403,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3556848,"kb_used_data":3231592,"kb_used_omap":9,"kb_used_meta":325238,"kb_avail":90188304,"statfs":{"total":95995035648,"available":92352823296,"internally_reserved":0,"allocated":3309150208,"data_stored":3307263777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":333044157},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.2330000000000001,"15min":0.83799999999999997},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.206,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":2.2080000000000002},{"interface":"front","average":{"1min":1.038,"5min":1.423,"15min":1.5189999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.6309999999999998,"5min":7.5439999999999996,"15min":7.5439999999999996},"last":2.1499999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:19:20 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":1.4770000000000001,"15min":1.2230000000000001},"min":{"1min":0.32100000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.946000000000002,"5min":23.946000000000002,"15min":23.946000000000002},"last":6.1280000000000001},{"interface":"front","average":{"1min":3.927,"5min":1.4630000000000001,"15min":1.052},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":23.268999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":2.3159999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.4590000000000001,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.4930000000000003,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":2.286},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.891,"15min":2.141},"min":{"1min":0.314,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.6749999999999998,"5min":7.2359999999999998,"15min":7.2359999999999998},"last":2.1779999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.556,"15min":1.153},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.2330000000000001,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":3.375},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.534,"15min":1.5169999999999999},"min":{"1min":0.33500000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.8599999999999999,"5min":7.5179999999999998,"15min":7.5179999999999998},"last":3.1680000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":2.1440000000000001,"15min":2.0379999999999998},"min":{"1min":0.36499999999999999,"5min":0.254,"15min":0.251},"max":{"1min":5.375,"5min":12.747999999999999,"15min":12.747999999999999},"last":7.875},{"interface":"front","average":{"1min":1.514,"5min":1.9099999999999999,"15min":1.5580000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.069,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":5.0069999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.867,"15min":1.1899999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1920000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":2.673},{"interface":"front","average":{"1min":1.232,"5min":2.004,"15min":1.6220000000000001},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.9649999999999999,"5min":12.504,"15min":12.504},"last":2.645}]},{"osd":7,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":2.363,"15min":1.6970000000000001},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.4240000000000004,"5min":13.506,"15min":13.506},"last":7.9119999999999999},{"interface":"front","average":{"1min":1.2310000000000001,"5min":2.0310000000000001,"15min":1.5349999999999999},"min":{"1min":0.378,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.2610000000000001,"5min":13.138,"15min":13.138},"last":2.2530000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3602841600,"data_stored":3602841600,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2218590208,"data_stored":2218590208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1864040448,"data_stored":1864040448,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4951310336,"data_stored":4951310336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3306225664,"data_stored":3306225664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2975399936,"data_stored":2975399936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:19:27.506 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:27.532 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:19:27.574 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:27.582 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:27.583 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:27.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:27.733 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:27.785 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:27.789 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:27.790 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:27.891 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:27.944 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:27.990 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 86 16 99985 99969 72.6156 53.0625 0.014517 0.0137655 2023-09-23T18:19:28.006 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:28.011 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:28.011 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:28.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:28.163 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:28.165 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:19:28.204 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:19:28.220 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:28.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:28.223 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:28.239 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:19:28.292 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:19:28.324 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:19:28.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:28.355 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:19:28.374 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:28.410 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:19:28.437 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:28 smithi099 ceph-mon[167076]: from='client.46931 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:28.438 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:28 smithi099 ceph-mon[167076]: pgmap v1428: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.0 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s; 19697/293994 objects degraded (6.700%); 681/293994 objects misplaced (0.232%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T18:19:28.438 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:28 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1317813877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:28.438 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19697/293994 objects degraded (6.700%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:28.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:28.447 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:19:28.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:28.448 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:28.449 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:19:28.449 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:19:28.449 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:19:28.449 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:19:28.449 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:19:28.449 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:19:28.449 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:19:28.449 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:19:28.549 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:28.639 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:28.918 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:28.923 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:28 smithi055 ceph-mon[179668]: from='client.46931 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:28.923 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:28 smithi055 ceph-mon[179668]: pgmap v1428: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.0 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s; 19697/293994 objects degraded (6.700%); 681/293994 objects misplaced (0.232%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T18:19:28.923 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:28 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1317813877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:28.923 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19697/293994 objects degraded (6.700%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:28.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:28 smithi055 ceph-mon[184174]: from='client.46931 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:28.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:28 smithi055 ceph-mon[184174]: pgmap v1428: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.0 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 817 op/s; 19697/293994 objects degraded (6.700%); 681/293994 objects misplaced (0.232%); 3.9 MiB/s, 61 objects/s recovering 2023-09-23T18:19:28.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:28 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1317813877' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:28.924 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19697/293994 objects degraded (6.700%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:28.925 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:28.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:28.990 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 87 16 100854 100838 72.4052 54.3125 0.0163612 0.0138062 2023-09-23T18:19:29.025 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:29.026 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:29.105 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:29.167 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:29.171 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:29.172 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:29.273 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:29.329 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:29.412 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:29.422 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:29.423 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:29.524 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:29.539 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:19:29.539 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:19:29.589 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:29.937 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:29.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:29.944 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:29.990 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 88 16 101697 101681 72.1812 52.6875 0.0117876 0.0138492 2023-09-23T18:19:30.045 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:30.115 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:30.362 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:19:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:19:30] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.43.0" 2023-09-23T18:19:30.507 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:19:30.678 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:30 smithi055 ceph-mon[184174]: pgmap v1429: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 820 op/s; 19697/296331 objects degraded (6.647%); 321/296331 objects misplaced (0.108%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:19:30.678 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:30 smithi055 ceph-mon[179668]: pgmap v1429: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 820 op/s; 19697/296331 objects degraded (6.647%); 321/296331 objects misplaced (0.108%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:19:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:30 smithi099 ceph-mon[167076]: pgmap v1429: 119 pgs: 1 active+recovering+remapped, 1 active+recovery_wait+degraded, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 51 MiB/s wr, 820 op/s; 19697/296331 objects degraded (6.647%); 321/296331 objects misplaced (0.108%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:19:30.936 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:19:30.964 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:19:30.990 INFO:tasks.radosbench.radosbench.0.smithi055.stdout: 89 15 102569 102554 71.9832 54.5625 0.0078539 0.0138866 2023-09-23T18:19:31.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total time run: 90.0122 2023-09-23T18:19:31.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Total writes made: 103403 2023-09-23T18:19:31.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Write size: 65536 2023-09-23T18:19:31.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Object size: 65536 2023-09-23T18:19:31.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Bandwidth (MB/sec): 71.7979 2023-09-23T18:19:31.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Bandwidth: 39.0028 2023-09-23T18:19:31.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max bandwidth (MB/sec): 199.312 2023-09-23T18:19:31.991 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min bandwidth (MB/sec): 0 2023-09-23T18:19:31.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average IOPS: 1148 2023-09-23T18:19:31.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev IOPS: 624.062 2023-09-23T18:19:31.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max IOPS: 3189 2023-09-23T18:19:31.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min IOPS: 0 2023-09-23T18:19:31.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Average Latency(s): 0.0139231 2023-09-23T18:19:31.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Stddev Latency(s): 0.0377992 2023-09-23T18:19:31.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Max latency(s): 2.2401 2023-09-23T18:19:31.992 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Min latency(s): 0.00128263 2023-09-23T18:19:31.999 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Cleaning up (deleting benchmark objects) 2023-09-23T18:19:32.421 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:32.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:32.429 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:32.530 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:32.612 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:32.755 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:32.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:32.766 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:32.766 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:32 smithi055 ceph-mon[184174]: pgmap v1430: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.1 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 52 MiB/s wr, 833 op/s; 19697/301224 objects degraded (6.539%); 321/301224 objects misplaced (0.107%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:19:32.767 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:32 smithi055 ceph-mon[179668]: pgmap v1430: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.1 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 52 MiB/s wr, 833 op/s; 19697/301224 objects degraded (6.539%); 321/301224 objects misplaced (0.107%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:19:32.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:32 smithi099 ceph-mon[167076]: pgmap v1430: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.1 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 52 MiB/s wr, 833 op/s; 19697/301224 objects degraded (6.539%); 321/301224 objects misplaced (0.107%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:19:32.867 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:32.947 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:33.030 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:33.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:33.039 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:33.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:33.194 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:33.195 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:33.248 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:33.252 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:33.253 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:33.253 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:33.274 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:33.354 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:33.442 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:33.809 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:33.811 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 19697/301224 objects degraded (6.539%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:33.811 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:33 smithi055 ceph-mon[184174]: failed to encode map e605 with expected crc 2023-09-23T18:19:33.811 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:33 smithi055 ceph-mon[184174]: osdmap e605: 8 total, 8 up, 6 in 2023-09-23T18:19:33.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 19697/301224 objects degraded (6.539%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:33.811 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:33 smithi055 ceph-mon[179668]: failed to encode map e605 with expected crc 2023-09-23T18:19:33.812 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:33 smithi055 ceph-mon[179668]: osdmap e605: 8 total, 8 up, 6 in 2023-09-23T18:19:33.813 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:33.813 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:33.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 19697/301224 objects degraded (6.539%), 8 pgs degraded (PG_DEGRADED) 2023-09-23T18:19:33.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:33 smithi099 ceph-mon[167076]: failed to encode map e605 with expected crc 2023-09-23T18:19:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:33 smithi099 ceph-mon[167076]: osdmap e605: 8 total, 8 up, 6 in 2023-09-23T18:19:33.914 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:33.940 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:33.976 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:33.979 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:33.980 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:34.081 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:34.107 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:34.142 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:34.145 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:34.145 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:34.246 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:34.279 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:34.400 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:19:34.621 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:34.629 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:34.629 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[179668]: pgmap v1431: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 52 MiB/s wr, 837 op/s; 19697/307368 objects degraded (6.408%); 321/307368 objects misplaced (0.104%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:19:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1581960732' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[179668]: failed to encode map e606 with expected crc 2023-09-23T18:19:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:19:34.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[179668]: osdmap e606: 8 total, 8 up, 6 in 2023-09-23T18:19:34.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[184174]: pgmap v1431: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 52 MiB/s wr, 837 op/s; 19697/307368 objects degraded (6.408%); 321/307368 objects misplaced (0.104%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:19:34.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1581960732' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:34.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:34.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:34.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[184174]: failed to encode map e606 with expected crc 2023-09-23T18:19:34.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:19:34.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:34 smithi055 ceph-mon[184174]: osdmap e606: 8 total, 8 up, 6 in 2023-09-23T18:19:34.730 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:34.814 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:34.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:34 smithi099 ceph-mon[167076]: pgmap v1431: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 52 MiB/s wr, 837 op/s; 19697/307368 objects degraded (6.408%); 321/307368 objects misplaced (0.104%); 4.3 MiB/s, 68 objects/s recovering 2023-09-23T18:19:34.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1581960732' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:34.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:34 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:19:34.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:34 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:34.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:34 smithi099 ceph-mon[167076]: failed to encode map e606 with expected crc 2023-09-23T18:19:34.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:34 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:19:34.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:34 smithi099 ceph-mon[167076]: osdmap e606: 8 total, 8 up, 6 in 2023-09-23T18:19:34.896 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:34.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:34.901 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:35.002 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:35.063 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:35.114 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:35.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:35.122 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:35.223 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:35.238 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:19:35.238 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:19:35.285 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:35.697 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:35 smithi055 ceph-mon[179668]: failed to encode map e606 with expected crc 2023-09-23T18:19:35.697 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:35 smithi055 ceph-mon[179668]: failed to encode map e606 with expected crc 2023-09-23T18:19:35.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:35 smithi055 ceph-mon[184174]: failed to encode map e606 with expected crc 2023-09-23T18:19:35.700 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:35 smithi055 ceph-mon[184174]: failed to encode map e606 with expected crc 2023-09-23T18:19:35.752 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:35.767 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:35.768 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:35.782 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:19:35.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:35 smithi099 ceph-mon[167076]: failed to encode map e606 with expected crc 2023-09-23T18:19:35.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:35 smithi099 ceph-mon[167076]: failed to encode map e606 with expected crc 2023-09-23T18:19:35.868 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:35.977 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:36.050 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:36.058 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:36.058 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:36.140 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:19:36.159 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:36.183 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:19:36.210 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:36.228 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:19:36.261 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:19:36.267 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:36.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:36.277 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:36.295 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:19:36.334 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:19:36.376 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:19:36.377 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:36.417 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:19:36.427 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:19:36.427 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:19:36.427 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:19:36.428 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:19:36.428 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:19:36.428 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:19:36.428 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:19:36.428 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:19:36.432 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:36.485 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:36.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:36.493 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:36.594 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:36.665 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:36.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:36 smithi055 ceph-mon[184174]: pgmap v1434: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 36 MiB/s wr, 595 op/s; 19697/307587 objects degraded (6.404%); 321/307587 objects misplaced (0.104%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T18:19:36.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/401615562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:36.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:36 smithi055 ceph-mon[179668]: pgmap v1434: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 36 MiB/s wr, 595 op/s; 19697/307587 objects degraded (6.404%); 321/307587 objects misplaced (0.104%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T18:19:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/401615562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:36.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:36 smithi099 ceph-mon[167076]: pgmap v1434: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+remapped, 7 active+recovery_wait+undersized+degraded+remapped, 110 active+clean; 6.3 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 36 MiB/s wr, 595 op/s; 19697/307587 objects degraded (6.404%); 321/307587 objects misplaced (0.104%); 2.8 MiB/s, 44 objects/s recovering 2023-09-23T18:19:36.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/401615562' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:36.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:36.944 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:36.953 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:36.953 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:37.054 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:37.101 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:37.169 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:37.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:37.176 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:37.206 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:37.277 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:37.341 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:37.346 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:19:37.356 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:19:37.402 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:37.408 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:37.409 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:37.509 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:37.585 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:37.699 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:19:37.865 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:37.872 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:37.873 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:37.974 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:37.978 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1435,"stamp":"2023-09-23T18:19:37.135418+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6610500284,"num_objects":100828,"num_object_clones":0,"num_object_copies":302484,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":16314,"num_objects_misplaced":3193,"num_objects_unfound":0,"num_objects_dirty":100828,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":106918,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4046,"num_bytes_recovered":255090313,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82629,"ondisk_log_size":82629,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":379,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":23321324,"kb_used_data":20035004,"kb_used_omap":40,"kb_used_meta":3285911,"kb_avail":539149588,"statfs":{"total":575970213888,"available":552089178112,"internally_reserved":0,"allocated":20515844096,"data_stored":20501373375,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3364773178},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":134348824,"num_objects":2051,"num_object_clones":0,"num_object_copies":6153,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-3383,"num_objects_misplaced":2872,"num_objects_unfound":0,"num_objects_dirty":2051,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7257,"num_write_kb":297793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":511,"num_bytes_recovered":33751040,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.003148"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":894,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365062+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:19:33.365062+0000","last_peered":"2023-09-23T18:19:33.365062+0000","last_clean":"2023-09-23T18:19:33.365062+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:19:33.365062+0000","last_undegraded":"2023-09-23T18:19:33.365062+0000","last_fullsized":"2023-09-23T18:19:33.365062+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":263,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.246129+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:19:15.246129+0000","last_peered":"2023-09-23T18:19:15.246129+0000","last_clean":"2023-09-23T18:19:15.246129+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:19:15.246129+0000","last_undegraded":"2023-09-23T18:19:15.246129+0000","last_fullsized":"2023-09-23T18:19:15.246129+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":22,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379749+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:19:33.379749+0000","last_peered":"2023-09-23T18:19:33.379749+0000","last_clean":"2023-09-23T18:19:33.379749+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:19:33.379749+0000","last_undegraded":"2023-09-23T18:19:33.379749+0000","last_fullsized":"2023-09-23T18:19:33.379749+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":900,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.372653+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:19:33.372653+0000","last_peered":"2023-09-23T18:19:33.372653+0000","last_clean":"2023-09-23T18:19:33.372653+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:19:33.372653+0000","last_undegraded":"2023-09-23T18:19:33.372653+0000","last_fullsized":"2023-09-23T18:19:33.372653+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":346,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.386173+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:19:33.386173+0000","last_peered":"2023-09-23T18:19:33.386173+0000","last_clean":"2023-09-23T18:19:33.386173+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:19:33.386173+0000","last_undegraded":"2023-09-23T18:19:33.386173+0000","last_fullsized":"2023-09-23T18:19:33.386173+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":666,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.382430+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:19:33.382430+0000","last_peered":"2023-09-23T18:19:33.382430+0000","last_clean":"2023-09-23T18:19:33.382430+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:19:33.382430+0000","last_undegraded":"2023-09-23T18:19:33.382430+0000","last_fullsized":"2023-09-23T18:19:33.382430+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":917,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365012+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:19:33.365012+0000","last_peered":"2023-09-23T18:19:33.365012+0000","last_clean":"2023-09-23T18:19:33.365012+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:33.365012+0000","last_undegraded":"2023-09-23T18:19:33.365012+0000","last_fullsized":"2023-09-23T18:19:33.365012+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":22,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379114+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:33.379114+0000","last_peered":"2023-09-23T18:19:33.379114+0000","last_clean":"2023-09-23T18:19:33.379114+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:19:33.379114+0000","last_undegraded":"2023-09-23T18:19:33.379114+0000","last_fullsized":"2023-09-23T18:19:33.379114+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":182,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.381751+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:19:33.381751+0000","last_peered":"2023-09-23T18:19:33.381751+0000","last_clean":"2023-09-23T18:19:33.381751+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:19:33.381751+0000","last_undegraded":"2023-09-23T18:19:33.381751+0000","last_fullsized":"2023-09-23T18:19:33.381751+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1040,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.366796+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:19:33.366796+0000","last_peered":"2023-09-23T18:19:33.366796+0000","last_clean":"2023-09-23T18:19:33.366796+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:19:33.366796+0000","last_undegraded":"2023-09-23T18:19:33.366796+0000","last_fullsized":"2023-09-23T18:19:33.366796+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":116,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.367426+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:19:33.367426+0000","last_peered":"2023-09-23T18:19:33.367426+0000","last_clean":"2023-09-23T18:19:33.367426+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:19:33.367426+0000","last_undegraded":"2023-09-23T18:19:33.367426+0000","last_fullsized":"2023-09-23T18:19:33.367426+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":195,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379466+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:19:33.379466+0000","last_peered":"2023-09-23T18:19:33.379466+0000","last_clean":"2023-09-23T18:19:33.379466+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:19:33.379466+0000","last_undegraded":"2023-09-23T18:19:33.379466+0000","last_fullsized":"2023-09-23T18:19:33.379466+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":817,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.377070+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:19:33.377070+0000","last_peered":"2023-09-23T18:19:33.377070+0000","last_clean":"2023-09-23T18:19:33.377070+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:19:33.377070+0000","last_undegraded":"2023-09-23T18:19:33.377070+0000","last_fullsized":"2023-09-23T18:19:33.377070+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":152,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.367681+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:19:33.367681+0000","last_peered":"2023-09-23T18:19:33.367681+0000","last_clean":"2023-09-23T18:19:33.367681+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:19:33.367681+0000","last_undegraded":"2023-09-23T18:19:33.367681+0000","last_fullsized":"2023-09-23T18:19:33.367681+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":151,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.376849+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:19:33.376849+0000","last_peered":"2023-09-23T18:19:33.376849+0000","last_clean":"2023-09-23T18:19:33.376849+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:19:33.376849+0000","last_undegraded":"2023-09-23T18:19:33.376849+0000","last_fullsized":"2023-09-23T18:19:33.376849+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":951,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.367507+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:19:33.367507+0000","last_peered":"2023-09-23T18:19:33.367507+0000","last_clean":"2023-09-23T18:19:33.367507+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:19:33.367507+0000","last_undegraded":"2023-09-23T18:19:33.367507+0000","last_fullsized":"2023-09-23T18:19:33.367507+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":400,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.385889+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:19:33.385889+0000","last_peered":"2023-09-23T18:19:33.385889+0000","last_clean":"2023-09-23T18:19:33.385889+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:19:33.385889+0000","last_undegraded":"2023-09-23T18:19:33.385889+0000","last_fullsized":"2023-09-23T18:19:33.385889+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":419,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.376381+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:19:33.376381+0000","last_peered":"2023-09-23T18:19:33.376381+0000","last_clean":"2023-09-23T18:19:33.376381+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:19:33.376381+0000","last_undegraded":"2023-09-23T18:19:33.376381+0000","last_fullsized":"2023-09-23T18:19:33.376381+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":978,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T18:19:23.429039+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:19:23.429039+0000","last_peered":"2023-09-23T18:19:23.429039+0000","last_clean":"2023-09-23T18:19:23.429039+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:19:23.429039+0000","last_undegraded":"2023-09-23T18:19:23.429039+0000","last_fullsized":"2023-09-23T18:19:23.429039+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":158,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378174+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:19:33.378174+0000","last_peered":"2023-09-23T18:19:33.378174+0000","last_clean":"2023-09-23T18:19:33.378174+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:19:33.378174+0000","last_undegraded":"2023-09-23T18:19:33.378174+0000","last_fullsized":"2023-09-23T18:19:33.378174+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":425,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378352+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:19:33.378352+0000","last_peered":"2023-09-23T18:19:33.378352+0000","last_clean":"2023-09-23T18:19:33.378352+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:19:33.378352+0000","last_undegraded":"2023-09-23T18:19:33.378352+0000","last_fullsized":"2023-09-23T18:19:33.378352+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":832,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365528+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:19:33.365528+0000","last_peered":"2023-09-23T18:19:33.365528+0000","last_clean":"2023-09-23T18:19:33.365528+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:19:33.365528+0000","last_undegraded":"2023-09-23T18:19:33.365528+0000","last_fullsized":"2023-09-23T18:19:33.365528+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":425,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378202+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:19:33.378202+0000","last_peered":"2023-09-23T18:19:33.378202+0000","last_clean":"2023-09-23T18:19:33.378202+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:19:33.378202+0000","last_undegraded":"2023-09-23T18:19:33.378202+0000","last_fullsized":"2023-09-23T18:19:33.378202+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":103,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365682+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:19:33.365682+0000","last_peered":"2023-09-23T18:19:33.365682+0000","last_clean":"2023-09-23T18:19:33.365682+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:19:33.365682+0000","last_undegraded":"2023-09-23T18:19:33.365682+0000","last_fullsized":"2023-09-23T18:19:33.365682+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":803,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.364816+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:19:33.364816+0000","last_peered":"2023-09-23T18:19:33.364816+0000","last_clean":"2023-09-23T18:19:33.364816+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:19:33.364816+0000","last_undegraded":"2023-09-23T18:19:33.364816+0000","last_fullsized":"2023-09-23T18:19:33.364816+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":121,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.381228+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:19:33.381228+0000","last_peered":"2023-09-23T18:19:33.381228+0000","last_clean":"2023-09-23T18:19:33.381228+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:19:33.381228+0000","last_undegraded":"2023-09-23T18:19:33.381228+0000","last_fullsized":"2023-09-23T18:19:33.381228+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":116,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.367023+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:19:33.367023+0000","last_peered":"2023-09-23T18:19:33.367023+0000","last_clean":"2023-09-23T18:19:33.367023+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:19:33.367023+0000","last_undegraded":"2023-09-23T18:19:33.367023+0000","last_fullsized":"2023-09-23T18:19:33.367023+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":741,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379718+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:19:33.379718+0000","last_peered":"2023-09-23T18:19:33.379718+0000","last_clean":"2023-09-23T18:19:33.379718+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:19:33.379718+0000","last_undegraded":"2023-09-23T18:19:33.379718+0000","last_fullsized":"2023-09-23T18:19:33.379718+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":193,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.376713+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:19:33.376713+0000","last_peered":"2023-09-23T18:19:33.376713+0000","last_clean":"2023-09-23T18:19:33.376713+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:19:33.376713+0000","last_undegraded":"2023-09-23T18:19:33.376713+0000","last_fullsized":"2023-09-23T18:19:33.376713+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":204,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.387206+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:19:33.387206+0000","last_peered":"2023-09-23T18:19:33.387206+0000","last_clean":"2023-09-23T18:19:33.387206+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:19:33.387206+0000","last_undegraded":"2023-09-23T18:19:33.387206+0000","last_fullsized":"2023-09-23T18:19:33.387206+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":837,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.376038+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:19:33.376038+0000","last_peered":"2023-09-23T18:19:33.376038+0000","last_clean":"2023-09-23T18:19:33.376038+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:19:33.376038+0000","last_undegraded":"2023-09-23T18:19:33.376038+0000","last_fullsized":"2023-09-23T18:19:33.376038+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":746,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.375666+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:19:33.375666+0000","last_peered":"2023-09-23T18:19:33.375666+0000","last_clean":"2023-09-23T18:19:33.375666+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:19:33.375666+0000","last_undegraded":"2023-09-23T18:19:33.375666+0000","last_fullsized":"2023-09-23T18:19:33.375666+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":22,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379571+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:19:33.379571+0000","last_peered":"2023-09-23T18:19:33.379571+0000","last_clean":"2023-09-23T18:19:33.379571+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:19:33.379571+0000","last_undegraded":"2023-09-23T18:19:33.379571+0000","last_fullsized":"2023-09-23T18:19:33.379571+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"606'6725","reported_seq":6765,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:36.090237+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:19:36.090237+0000","last_peered":"2023-09-23T18:19:36.090237+0000","last_clean":"2023-09-23T18:19:36.090237+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:19:36.090237+0000","last_undegraded":"2023-09-23T18:19:36.090237+0000","last_fullsized":"2023-09-23T18:19:36.090237+0000","mapping_epoch":574,"log_start":"581'1200","ondisk_log_start":"581'1200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5525,"log_dups_size":0,"ondisk_log_size":5525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416088064,"num_objects":6349,"num_object_clones":0,"num_object_copies":19047,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6349,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6725,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":871,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378873+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:19:33.378873+0000","last_peered":"2023-09-23T18:19:33.378873+0000","last_clean":"2023-09-23T18:19:33.378873+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:19:33.378873+0000","last_undegraded":"2023-09-23T18:19:33.378873+0000","last_fullsized":"2023-09-23T18:19:33.378873+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":240,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.258326+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:19:15.258326+0000","last_peered":"2023-09-23T18:19:15.258326+0000","last_clean":"2023-09-23T18:19:15.258326+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:19:15.258326+0000","last_undegraded":"2023-09-23T18:19:15.258326+0000","last_fullsized":"2023-09-23T18:19:15.258326+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":116,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.364908+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:19:33.364908+0000","last_peered":"2023-09-23T18:19:33.364908+0000","last_clean":"2023-09-23T18:19:33.364908+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:19:33.364908+0000","last_undegraded":"2023-09-23T18:19:33.364908+0000","last_fullsized":"2023-09-23T18:19:33.364908+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"606'6531","reported_seq":6581,"reported_epoch":606,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:36.189866+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:19:36.189866+0000","last_peered":"2023-09-23T18:19:36.189866+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:19:36.189866+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"584'2100","ondisk_log_start":"584'2100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4431,"log_dups_size":0,"ondisk_log_size":4431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":403767296,"num_objects":6161,"num_object_clones":0,"num_object_copies":18483,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6161,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6531,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":3963},{"shards":"5,6","objects":2013}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":725,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.407955+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:19:34.407955+0000","last_peered":"2023-09-23T18:19:34.407955+0000","last_clean":"2023-09-23T18:19:34.407955+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:19:34.407955+0000","last_undegraded":"2023-09-23T18:19:34.407955+0000","last_fullsized":"2023-09-23T18:19:34.407955+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":241,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.246110+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:19:15.246110+0000","last_peered":"2023-09-23T18:19:15.246110+0000","last_clean":"2023-09-23T18:19:15.246110+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:19:15.246110+0000","last_undegraded":"2023-09-23T18:19:15.246110+0000","last_fullsized":"2023-09-23T18:19:15.246110+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":220,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.245486+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:19:15.245486+0000","last_peered":"2023-09-23T18:19:15.245486+0000","last_clean":"2023-09-23T18:19:15.245486+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:19:15.245486+0000","last_undegraded":"2023-09-23T18:19:15.245486+0000","last_fullsized":"2023-09-23T18:19:15.245486+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"606'6572","reported_seq":13569,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:35.197290+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:19:35.197290+0000","last_peered":"2023-09-23T18:19:35.197290+0000","last_clean":"2023-09-23T18:19:35.197290+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:19:35.197290+0000","last_undegraded":"2023-09-23T18:19:35.197290+0000","last_fullsized":"2023-09-23T18:19:35.197290+0000","mapping_epoch":605,"log_start":"584'2100","ondisk_log_start":"584'2100","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4472,"log_dups_size":0,"ondisk_log_size":4472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411041792,"num_objects":6272,"num_object_clones":0,"num_object_copies":18816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6572,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":621,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.381661+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:19:33.381661+0000","last_peered":"2023-09-23T18:19:33.381661+0000","last_clean":"2023-09-23T18:19:33.381661+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:19:33.381661+0000","last_undegraded":"2023-09-23T18:19:33.381661+0000","last_fullsized":"2023-09-23T18:19:33.381661+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":531,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.366882+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:19:33.366882+0000","last_peered":"2023-09-23T18:19:33.366882+0000","last_clean":"2023-09-23T18:19:33.366882+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:19:33.366882+0000","last_undegraded":"2023-09-23T18:19:33.366882+0000","last_fullsized":"2023-09-23T18:19:33.366882+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":182,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.376274+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:19:33.376274+0000","last_peered":"2023-09-23T18:19:33.376274+0000","last_clean":"2023-09-23T18:19:33.376274+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:19:33.376274+0000","last_undegraded":"2023-09-23T18:19:33.376274+0000","last_fullsized":"2023-09-23T18:19:33.376274+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"606'6740","reported_seq":6781,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:36.079790+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:19:36.079790+0000","last_peered":"2023-09-23T18:19:36.079790+0000","last_clean":"2023-09-23T18:19:36.079790+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:19:36.079790+0000","last_undegraded":"2023-09-23T18:19:36.079790+0000","last_fullsized":"2023-09-23T18:19:36.079790+0000","mapping_epoch":574,"log_start":"583'1600","ondisk_log_start":"583'1600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5140,"log_dups_size":0,"ondisk_log_size":5140,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":416284672,"num_objects":6352,"num_object_clones":0,"num_object_copies":19056,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6352,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6740,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":636,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.387305+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:19:33.387305+0000","last_peered":"2023-09-23T18:19:33.387305+0000","last_clean":"2023-09-23T18:19:33.387305+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:19:33.387305+0000","last_undegraded":"2023-09-23T18:19:33.387305+0000","last_fullsized":"2023-09-23T18:19:33.387305+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":130,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.368747+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:19:33.368747+0000","last_peered":"2023-09-23T18:19:33.368747+0000","last_clean":"2023-09-23T18:19:33.368747+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:19:33.368747+0000","last_undegraded":"2023-09-23T18:19:33.368747+0000","last_fullsized":"2023-09-23T18:19:33.368747+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":442,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.386770+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:19:33.386770+0000","last_peered":"2023-09-23T18:19:33.386770+0000","last_clean":"2023-09-23T18:19:33.386770+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:33.386770+0000","last_undegraded":"2023-09-23T18:19:33.386770+0000","last_fullsized":"2023-09-23T18:19:33.386770+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"605'6620","reported_seq":6660,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.661548+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:19:33.661548+0000","last_peered":"2023-09-23T18:19:33.661548+0000","last_clean":"2023-09-23T18:19:33.661548+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:19:33.661548+0000","last_undegraded":"2023-09-23T18:19:33.661548+0000","last_fullsized":"2023-09-23T18:19:33.661548+0000","mapping_epoch":574,"log_start":"584'2200","ondisk_log_start":"584'2200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4420,"log_dups_size":0,"ondisk_log_size":4420,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":421003264,"num_objects":6424,"num_object_clones":0,"num_object_copies":19272,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6424,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6620,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1088,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365384+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:19:33.365384+0000","last_peered":"2023-09-23T18:19:33.365384+0000","last_clean":"2023-09-23T18:19:33.365384+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:19:33.365384+0000","last_undegraded":"2023-09-23T18:19:33.365384+0000","last_fullsized":"2023-09-23T18:19:33.365384+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":342,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379920+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:19:33.379920+0000","last_peered":"2023-09-23T18:19:33.379920+0000","last_clean":"2023-09-23T18:19:33.379920+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:19:33.379920+0000","last_undegraded":"2023-09-23T18:19:33.379920+0000","last_fullsized":"2023-09-23T18:19:33.379920+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":150,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.377218+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:19:33.377218+0000","last_peered":"2023-09-23T18:19:33.377218+0000","last_clean":"2023-09-23T18:19:33.377218+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:19:33.377218+0000","last_undegraded":"2023-09-23T18:19:33.377218+0000","last_fullsized":"2023-09-23T18:19:33.377218+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"606'6688","reported_seq":6737,"reported_epoch":606,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:36.119096+0000","last_change":"2023-09-23T18:18:47.594759+0000","last_active":"2023-09-23T18:19:36.119096+0000","last_peered":"2023-09-23T18:19:36.119096+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:19:36.119096+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"581'1200","ondisk_log_start":"581'1200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5488,"log_dups_size":0,"ondisk_log_size":5488,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":415629312,"num_objects":6342,"num_object_clones":0,"num_object_copies":19026,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6342,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6688,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":4053},{"shards":"4,5","objects":2116}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":875,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365116+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:19:33.365116+0000","last_peered":"2023-09-23T18:19:33.365116+0000","last_clean":"2023-09-23T18:19:33.365116+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:19:33.365116+0000","last_undegraded":"2023-09-23T18:19:33.365116+0000","last_fullsized":"2023-09-23T18:19:33.365116+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":158,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.376212+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:19:33.376212+0000","last_peered":"2023-09-23T18:19:33.376212+0000","last_clean":"2023-09-23T18:19:33.376212+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:19:33.376212+0000","last_undegraded":"2023-09-23T18:19:33.376212+0000","last_fullsized":"2023-09-23T18:19:33.376212+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":457,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378995+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:19:33.378995+0000","last_peered":"2023-09-23T18:19:33.378995+0000","last_clean":"2023-09-23T18:19:33.378995+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:19:33.378995+0000","last_undegraded":"2023-09-23T18:19:33.378995+0000","last_fullsized":"2023-09-23T18:19:33.378995+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"606'6739","reported_seq":6779,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:36.213288+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:19:36.213288+0000","last_peered":"2023-09-23T18:19:36.213288+0000","last_clean":"2023-09-23T18:19:36.213288+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:19:36.213288+0000","last_undegraded":"2023-09-23T18:19:36.213288+0000","last_fullsized":"2023-09-23T18:19:36.213288+0000","mapping_epoch":574,"log_start":"584'2300","ondisk_log_start":"584'2300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4439,"log_dups_size":0,"ondisk_log_size":4439,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413728768,"num_objects":6313,"num_object_clones":0,"num_object_copies":18939,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6313,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6739,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":877,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.372728+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:19:33.372728+0000","last_peered":"2023-09-23T18:19:33.372728+0000","last_clean":"2023-09-23T18:19:33.372728+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:19:33.372728+0000","last_undegraded":"2023-09-23T18:19:33.372728+0000","last_fullsized":"2023-09-23T18:19:33.372728+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":442,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365257+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:19:33.365257+0000","last_peered":"2023-09-23T18:19:33.365257+0000","last_clean":"2023-09-23T18:19:33.365257+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:19:33.365257+0000","last_undegraded":"2023-09-23T18:19:33.365257+0000","last_fullsized":"2023-09-23T18:19:33.365257+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":218,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378308+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:19:33.378308+0000","last_peered":"2023-09-23T18:19:33.378308+0000","last_clean":"2023-09-23T18:19:33.378308+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:19:33.378308+0000","last_undegraded":"2023-09-23T18:19:33.378308+0000","last_fullsized":"2023-09-23T18:19:33.378308+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"604'6535","reported_seq":6573,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T18:19:32.315683+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:19:32.315683+0000","last_peered":"2023-09-23T18:19:32.315683+0000","last_clean":"2023-09-23T18:19:32.315683+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:19:32.315683+0000","last_undegraded":"2023-09-23T18:19:32.315683+0000","last_fullsized":"2023-09-23T18:19:32.315683+0000","mapping_epoch":574,"log_start":"584'1900","ondisk_log_start":"584'1900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4635,"log_dups_size":0,"ondisk_log_size":4635,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":425918464,"num_objects":6499,"num_object_clones":0,"num_object_copies":19497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6535,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":858,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365369+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:19:33.365369+0000","last_peered":"2023-09-23T18:19:33.365369+0000","last_clean":"2023-09-23T18:19:33.365369+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:19:33.365369+0000","last_undegraded":"2023-09-23T18:19:33.365369+0000","last_fullsized":"2023-09-23T18:19:33.365369+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":22,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379263+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:19:33.379263+0000","last_peered":"2023-09-23T18:19:33.379263+0000","last_clean":"2023-09-23T18:19:33.379263+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:19:33.379263+0000","last_undegraded":"2023-09-23T18:19:33.379263+0000","last_fullsized":"2023-09-23T18:19:33.379263+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":333,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.377869+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:19:33.377869+0000","last_peered":"2023-09-23T18:19:33.377869+0000","last_clean":"2023-09-23T18:19:33.377869+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:19:33.377869+0000","last_undegraded":"2023-09-23T18:19:33.377869+0000","last_fullsized":"2023-09-23T18:19:33.377869+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"606'6662","reported_seq":6705,"reported_epoch":606,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:35.143254+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:19:35.143254+0000","last_peered":"2023-09-23T18:19:35.143254+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:19:35.143254+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"584'1800","ondisk_log_start":"584'1800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4862,"log_dups_size":0,"ondisk_log_size":4862,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":419823616,"num_objects":6406,"num_object_clones":0,"num_object_copies":19218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6662,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":2121},{"shards":"0,4,6","objects":4157}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":717,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.381839+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:19:33.381839+0000","last_peered":"2023-09-23T18:19:33.381839+0000","last_clean":"2023-09-23T18:19:33.381839+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:19:33.381839+0000","last_undegraded":"2023-09-23T18:19:33.381839+0000","last_fullsized":"2023-09-23T18:19:33.381839+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":955,"reported_epoch":604,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:23.872910+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:19:23.872910+0000","last_peered":"2023-09-23T18:19:23.872910+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:19:23.872910+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":140,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.367090+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:19:33.367090+0000","last_peered":"2023-09-23T18:19:33.367090+0000","last_clean":"2023-09-23T18:19:33.367090+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:19:33.367090+0000","last_undegraded":"2023-09-23T18:19:33.367090+0000","last_fullsized":"2023-09-23T18:19:33.367090+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":120,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365296+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:19:33.365296+0000","last_peered":"2023-09-23T18:19:33.365296+0000","last_clean":"2023-09-23T18:19:33.365296+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:19:33.365296+0000","last_undegraded":"2023-09-23T18:19:33.365296+0000","last_fullsized":"2023-09-23T18:19:33.365296+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"606'6640","reported_seq":6689,"reported_epoch":606,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:36.187872+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:19:36.187872+0000","last_peered":"2023-09-23T18:19:36.187872+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:19:36.187872+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"584'2200","ondisk_log_start":"584'2200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4440,"log_dups_size":0,"ondisk_log_size":4440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":412614656,"num_objects":6296,"num_object_clones":0,"num_object_copies":18888,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6296,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6640,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":3386},{"shards":"5,6","objects":2737}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":789,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365704+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:19:33.365704+0000","last_peered":"2023-09-23T18:19:33.365704+0000","last_clean":"2023-09-23T18:19:33.365704+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:19:33.365704+0000","last_undegraded":"2023-09-23T18:19:33.365704+0000","last_fullsized":"2023-09-23T18:19:33.365704+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1490,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365226+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:19:33.365226+0000","last_peered":"2023-09-23T18:19:33.365226+0000","last_clean":"2023-09-23T18:19:33.365226+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:19:33.365226+0000","last_undegraded":"2023-09-23T18:19:33.365226+0000","last_fullsized":"2023-09-23T18:19:33.365226+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":130,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378275+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:19:33.378275+0000","last_peered":"2023-09-23T18:19:33.378275+0000","last_clean":"2023-09-23T18:19:33.378275+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:19:33.378275+0000","last_undegraded":"2023-09-23T18:19:33.378275+0000","last_fullsized":"2023-09-23T18:19:33.378275+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":430,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378034+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:19:33.378034+0000","last_peered":"2023-09-23T18:19:33.378034+0000","last_clean":"2023-09-23T18:19:33.378034+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:19:33.378034+0000","last_undegraded":"2023-09-23T18:19:33.378034+0000","last_fullsized":"2023-09-23T18:19:33.378034+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"606'6482","reported_seq":6522,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:36.087857+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:19:36.087857+0000","last_peered":"2023-09-23T18:19:36.087857+0000","last_clean":"2023-09-23T18:19:36.087857+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:19:36.087857+0000","last_undegraded":"2023-09-23T18:19:36.087857+0000","last_fullsized":"2023-09-23T18:19:36.087857+0000","mapping_epoch":574,"log_start":"580'1000","ondisk_log_start":"580'1000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5482,"log_dups_size":0,"ondisk_log_size":5482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":400031744,"num_objects":6104,"num_object_clones":0,"num_object_copies":18312,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6104,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6482,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":814,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365451+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:19:33.365451+0000","last_peered":"2023-09-23T18:19:33.365451+0000","last_clean":"2023-09-23T18:19:33.365451+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:19:33.365451+0000","last_undegraded":"2023-09-23T18:19:33.365451+0000","last_fullsized":"2023-09-23T18:19:33.365451+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":20,"reported_epoch":605,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:19:33.385975+0000","last_change":"2023-09-23T18:18:46.432707+0000","last_active":"2023-09-23T18:19:33.385975+0000","last_peered":"2023-09-23T18:19:33.385975+0000","last_clean":"2023-09-23T18:11:33.614822+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:19:33.385975+0000","last_undegraded":"2023-09-23T18:18:46.418246+0000","last_fullsized":"2023-09-23T18:19:33.385975+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":399,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.375952+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:19:33.375952+0000","last_peered":"2023-09-23T18:19:33.375952+0000","last_clean":"2023-09-23T18:19:33.375952+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:19:33.375952+0000","last_undegraded":"2023-09-23T18:19:33.375952+0000","last_fullsized":"2023-09-23T18:19:33.375952+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":142,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.367359+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:19:33.367359+0000","last_peered":"2023-09-23T18:19:33.367359+0000","last_clean":"2023-09-23T18:19:33.367359+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:19:33.367359+0000","last_undegraded":"2023-09-23T18:19:33.367359+0000","last_fullsized":"2023-09-23T18:19:33.367359+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"606'6640","reported_seq":6680,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:36.179885+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:19:36.179885+0000","last_peered":"2023-09-23T18:19:36.179885+0000","last_clean":"2023-09-23T18:19:36.179885+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:19:36.179885+0000","last_undegraded":"2023-09-23T18:19:36.179885+0000","last_fullsized":"2023-09-23T18:19:36.179885+0000","mapping_epoch":574,"log_start":"584'2200","ondisk_log_start":"584'2200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4440,"log_dups_size":0,"ondisk_log_size":4440,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":413401088,"num_objects":6308,"num_object_clones":0,"num_object_copies":18924,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6308,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6640,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1503,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.366623+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:19:33.366623+0000","last_peered":"2023-09-23T18:19:33.366623+0000","last_clean":"2023-09-23T18:19:33.366623+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:19:33.366623+0000","last_undegraded":"2023-09-23T18:19:33.366623+0000","last_fullsized":"2023-09-23T18:19:33.366623+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":556,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.375823+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:19:33.375823+0000","last_peered":"2023-09-23T18:19:33.375823+0000","last_clean":"2023-09-23T18:19:33.375823+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:19:33.375823+0000","last_undegraded":"2023-09-23T18:19:33.375823+0000","last_fullsized":"2023-09-23T18:19:33.375823+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":116,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365092+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:19:33.365092+0000","last_peered":"2023-09-23T18:19:33.365092+0000","last_clean":"2023-09-23T18:19:33.365092+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:19:33.365092+0000","last_undegraded":"2023-09-23T18:19:33.365092+0000","last_fullsized":"2023-09-23T18:19:33.365092+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":151,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379286+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:19:33.379286+0000","last_peered":"2023-09-23T18:19:33.379286+0000","last_clean":"2023-09-23T18:19:33.379286+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:19:33.379286+0000","last_undegraded":"2023-09-23T18:19:33.379286+0000","last_fullsized":"2023-09-23T18:19:33.379286+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"606'6563","reported_seq":6603,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:36.093655+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:19:36.093655+0000","last_peered":"2023-09-23T18:19:36.093655+0000","last_clean":"2023-09-23T18:19:36.093655+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:19:36.093655+0000","last_undegraded":"2023-09-23T18:19:36.093655+0000","last_fullsized":"2023-09-23T18:19:36.093655+0000","mapping_epoch":574,"log_start":"581'1100","ondisk_log_start":"581'1100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5463,"log_dups_size":0,"ondisk_log_size":5463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":407699456,"num_objects":6221,"num_object_clones":0,"num_object_copies":18663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6563,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":804,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378425+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:19:33.378425+0000","last_peered":"2023-09-23T18:19:33.378425+0000","last_clean":"2023-09-23T18:19:33.378425+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:19:33.378425+0000","last_undegraded":"2023-09-23T18:19:33.378425+0000","last_fullsized":"2023-09-23T18:19:33.378425+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":119,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379391+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:19:33.379391+0000","last_peered":"2023-09-23T18:19:33.379391+0000","last_clean":"2023-09-23T18:19:33.379391+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:19:33.379391+0000","last_undegraded":"2023-09-23T18:19:33.379391+0000","last_fullsized":"2023-09-23T18:19:33.379391+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":22,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.385865+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:19:33.385865+0000","last_peered":"2023-09-23T18:19:33.385865+0000","last_clean":"2023-09-23T18:19:33.385865+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:19:33.385865+0000","last_undegraded":"2023-09-23T18:19:33.385865+0000","last_fullsized":"2023-09-23T18:19:33.385865+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":621,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.367745+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:19:33.367745+0000","last_peered":"2023-09-23T18:19:33.367745+0000","last_clean":"2023-09-23T18:19:33.367745+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:19:33.367745+0000","last_undegraded":"2023-09-23T18:19:33.367745+0000","last_fullsized":"2023-09-23T18:19:33.367745+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"606'6672","reported_seq":7102,"reported_epoch":606,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:36.192432+0000","last_change":"2023-09-23T18:19:32.858380+0000","last_active":"2023-09-23T18:19:36.192432+0000","last_peered":"2023-09-23T18:19:36.192432+0000","last_clean":"2023-09-23T18:18:32.699126+0000","last_became_active":"2023-09-23T18:18:34.716591+0000","last_became_peered":"2023-09-23T18:18:34.716591+0000","last_unstale":"2023-09-23T18:19:36.192432+0000","last_undegraded":"2023-09-23T18:18:34.351246+0000","last_fullsized":"2023-09-23T18:18:34.327168+0000","mapping_epoch":589,"log_start":"584'2200","ondisk_log_start":"584'2200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4472,"log_dups_size":0,"ondisk_log_size":4472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:13.611234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":411893784,"num_objects":6286,"num_object_clones":0,"num_object_copies":18858,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":71,"num_objects_misplaced":3193,"num_objects_unfound":0,"num_objects_dirty":6286,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":6672,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":189,"num_bytes_recovered":12386304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":3193},{"shards":"4,6","objects":3022}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":904,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378669+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:19:33.378669+0000","last_peered":"2023-09-23T18:19:33.378669+0000","last_clean":"2023-09-23T18:19:33.378669+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:19:33.378669+0000","last_undegraded":"2023-09-23T18:19:33.378669+0000","last_fullsized":"2023-09-23T18:19:33.378669+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":44,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.384267+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:19:33.384267+0000","last_peered":"2023-09-23T18:19:33.384267+0000","last_clean":"2023-09-23T18:19:33.384267+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:19:33.384267+0000","last_undegraded":"2023-09-23T18:19:33.384267+0000","last_fullsized":"2023-09-23T18:19:33.384267+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":130,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.368806+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:19:33.368806+0000","last_peered":"2023-09-23T18:19:33.368806+0000","last_clean":"2023-09-23T18:19:33.368806+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:19:33.368806+0000","last_undegraded":"2023-09-23T18:19:33.368806+0000","last_fullsized":"2023-09-23T18:19:33.368806+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":130,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365736+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:19:33.365736+0000","last_peered":"2023-09-23T18:19:33.365736+0000","last_clean":"2023-09-23T18:19:33.365736+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:19:33.365736+0000","last_undegraded":"2023-09-23T18:19:33.365736+0000","last_fullsized":"2023-09-23T18:19:33.365736+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"606'6624","reported_seq":6664,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:36.135417+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:19:36.135417+0000","last_peered":"2023-09-23T18:19:36.135417+0000","last_clean":"2023-09-23T18:19:36.135417+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:19:36.135417+0000","last_undegraded":"2023-09-23T18:19:36.135417+0000","last_fullsized":"2023-09-23T18:19:36.135417+0000","mapping_epoch":574,"log_start":"584'2200","ondisk_log_start":"584'2200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4424,"log_dups_size":0,"ondisk_log_size":4424,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408289280,"num_objects":6230,"num_object_clones":0,"num_object_copies":18690,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6230,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6624,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":904,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379067+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:19:33.379067+0000","last_peered":"2023-09-23T18:19:33.379067+0000","last_clean":"2023-09-23T18:19:33.379067+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:19:33.379067+0000","last_undegraded":"2023-09-23T18:19:33.379067+0000","last_fullsized":"2023-09-23T18:19:33.379067+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":92,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.366800+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:19:33.366800+0000","last_peered":"2023-09-23T18:19:33.366800+0000","last_clean":"2023-09-23T18:19:33.366800+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:19:33.366800+0000","last_undegraded":"2023-09-23T18:19:33.366800+0000","last_fullsized":"2023-09-23T18:19:33.366800+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":385,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378520+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:19:33.378520+0000","last_peered":"2023-09-23T18:19:33.378520+0000","last_clean":"2023-09-23T18:19:33.378520+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:19:33.378520+0000","last_undegraded":"2023-09-23T18:19:33.378520+0000","last_fullsized":"2023-09-23T18:19:33.378520+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":22,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.379193+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:33.379193+0000","last_peered":"2023-09-23T18:19:33.379193+0000","last_clean":"2023-09-23T18:19:33.379193+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:19:33.379193+0000","last_undegraded":"2023-09-23T18:19:33.379193+0000","last_fullsized":"2023-09-23T18:19:33.379193+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"606'6574","reported_seq":6614,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:36.199408+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:19:36.199408+0000","last_peered":"2023-09-23T18:19:36.199408+0000","last_clean":"2023-09-23T18:19:36.199408+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:19:36.199408+0000","last_undegraded":"2023-09-23T18:19:36.199408+0000","last_fullsized":"2023-09-23T18:19:36.199408+0000","mapping_epoch":574,"log_start":"584'2100","ondisk_log_start":"584'2100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4474,"log_dups_size":0,"ondisk_log_size":4474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":408813568,"num_objects":6238,"num_object_clones":0,"num_object_copies":18714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":6238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6574,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":738,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.390983+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:19:33.390983+0000","last_peered":"2023-09-23T18:19:33.390983+0000","last_clean":"2023-09-23T18:19:33.390983+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:19:33.390983+0000","last_undegraded":"2023-09-23T18:19:33.390983+0000","last_fullsized":"2023-09-23T18:19:33.390983+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1042,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.382081+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:19:33.382081+0000","last_peered":"2023-09-23T18:19:33.382081+0000","last_clean":"2023-09-23T18:19:33.382081+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:19:33.382081+0000","last_undegraded":"2023-09-23T18:19:33.382081+0000","last_fullsized":"2023-09-23T18:19:33.382081+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":32,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.382600+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:19:33.382600+0000","last_peered":"2023-09-23T18:19:33.382600+0000","last_clean":"2023-09-23T18:19:33.382600+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:19:33.382600+0000","last_undegraded":"2023-09-23T18:19:33.382600+0000","last_fullsized":"2023-09-23T18:19:33.382600+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":396,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365452+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:19:33.365452+0000","last_peered":"2023-09-23T18:19:33.365452+0000","last_clean":"2023-09-23T18:19:33.365452+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:19:33.365452+0000","last_undegraded":"2023-09-23T18:19:33.365452+0000","last_fullsized":"2023-09-23T18:19:33.365452+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":636,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.376916+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:19:33.376916+0000","last_peered":"2023-09-23T18:19:33.376916+0000","last_clean":"2023-09-23T18:19:33.376916+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:19:33.376916+0000","last_undegraded":"2023-09-23T18:19:33.376916+0000","last_fullsized":"2023-09-23T18:19:33.376916+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":28,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.386576+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:19:33.386576+0000","last_peered":"2023-09-23T18:19:33.386576+0000","last_clean":"2023-09-23T18:19:33.386576+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:19:33.386576+0000","last_undegraded":"2023-09-23T18:19:33.386576+0000","last_fullsized":"2023-09-23T18:19:33.386576+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":677,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T18:19:23.426684+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:19:23.426684+0000","last_peered":"2023-09-23T18:19:23.426684+0000","last_clean":"2023-09-23T18:19:23.426684+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:19:23.426684+0000","last_undegraded":"2023-09-23T18:19:23.426684+0000","last_fullsized":"2023-09-23T18:19:23.426684+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":636,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378376+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:19:33.378376+0000","last_peered":"2023-09-23T18:19:33.378376+0000","last_clean":"2023-09-23T18:19:33.378376+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:19:33.378376+0000","last_undegraded":"2023-09-23T18:19:33.378376+0000","last_fullsized":"2023-09-23T18:19:33.378376+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":124,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.368639+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:19:33.368639+0000","last_peered":"2023-09-23T18:19:33.368639+0000","last_clean":"2023-09-23T18:19:33.368639+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:19:33.368639+0000","last_undegraded":"2023-09-23T18:19:33.368639+0000","last_fullsized":"2023-09-23T18:19:33.368639+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":238,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.246597+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:19:15.246597+0000","last_peered":"2023-09-23T18:19:15.246597+0000","last_clean":"2023-09-23T18:19:15.246597+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:19:15.246597+0000","last_undegraded":"2023-09-23T18:19:15.246597+0000","last_fullsized":"2023-09-23T18:19:15.246597+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":110,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.366955+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:19:33.366955+0000","last_peered":"2023-09-23T18:19:33.366955+0000","last_clean":"2023-09-23T18:19:33.366955+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:19:33.366955+0000","last_undegraded":"2023-09-23T18:19:33.366955+0000","last_fullsized":"2023-09-23T18:19:33.366955+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":221,"reported_epoch":603,"state":"active+clean","last_fresh":"2023-09-23T18:19:15.246200+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:19:15.246200+0000","last_peered":"2023-09-23T18:19:15.246200+0000","last_clean":"2023-09-23T18:19:15.246200+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:19:15.246200+0000","last_undegraded":"2023-09-23T18:19:15.246200+0000","last_fullsized":"2023-09-23T18:19:15.246200+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":998,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.377144+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:19:33.377144+0000","last_peered":"2023-09-23T18:19:33.377144+0000","last_clean":"2023-09-23T18:19:33.377144+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:19:33.377144+0000","last_undegraded":"2023-09-23T18:19:33.377144+0000","last_fullsized":"2023-09-23T18:19:33.377144+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":503,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.378599+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:19:33.378599+0000","last_peered":"2023-09-23T18:19:33.378599+0000","last_clean":"2023-09-23T18:19:33.378599+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:19:33.378599+0000","last_undegraded":"2023-09-23T18:19:33.378599+0000","last_fullsized":"2023-09-23T18:19:33.378599+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":140,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.365625+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:19:33.365625+0000","last_peered":"2023-09-23T18:19:33.365625+0000","last_clean":"2023-09-23T18:19:33.365625+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:19:33.365625+0000","last_undegraded":"2023-09-23T18:19:33.365625+0000","last_fullsized":"2023-09-23T18:19:33.365625+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":933,"reported_epoch":604,"state":"active+clean","last_fresh":"2023-09-23T18:19:23.430461+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:19:23.430461+0000","last_peered":"2023-09-23T18:19:23.430461+0000","last_clean":"2023-09-23T18:19:23.430461+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:19:23.430461+0000","last_undegraded":"2023-09-23T18:19:23.430461+0000","last_fullsized":"2023-09-23T18:19:23.430461+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":128,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.367292+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:19:33.367292+0000","last_peered":"2023-09-23T18:19:33.367292+0000","last_clean":"2023-09-23T18:19:33.367292+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:19:33.367292+0000","last_undegraded":"2023-09-23T18:19:33.367292+0000","last_fullsized":"2023-09-23T18:19:33.367292+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":649,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.375744+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:19:33.375744+0000","last_peered":"2023-09-23T18:19:33.375744+0000","last_clean":"2023-09-23T18:19:33.375744+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:19:33.375744+0000","last_undegraded":"2023-09-23T18:19:33.375744+0000","last_fullsized":"2023-09-23T18:19:33.375744+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6606028824,"num_objects":100801,"num_object_clones":0,"num_object_copies":302403,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16289,"num_objects_misplaced":3193,"num_objects_unfound":0,"num_objects_dirty":100801,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":106007,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3660,"num_bytes_recovered":239861760,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20411396096,"data_stored":20411383880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76607,"ondisk_log_size":76607,"up":48,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2109440,"data_stored":2098784,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001563,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669500,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3686884,"kb_used_data":3090140,"kb_used_omap":7,"kb_used_meta":596600,"kb_avail":90058268,"statfs":{"total":95995035648,"available":92219666432,"internally_reserved":0,"allocated":3164303360,"data_stored":3162392906,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":610919204},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":1.599,"15min":1.2949999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":24.472999999999999,"5min":24.472999999999999,"15min":24.472999999999999},"last":0.498},{"interface":"front","average":{"1min":3.6230000000000002,"5min":2.081,"15min":1.663},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":22.832999999999998,"5min":22.832999999999998,"15min":22.832999999999998},"last":0.32400000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:19:21 2023","interfaces":[{"interface":"back","average":{"1min":4.1740000000000004,"5min":1.8069999999999999,"15min":1.413},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":22.972000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":1.458},{"interface":"front","average":{"1min":3.448,"5min":1.6559999999999999,"15min":1.357},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":22.664000000000001,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.377}]},{"osd":2,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.403,"15min":1.2869999999999999},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.2679999999999998,"5min":7.4189999999999996,"15min":7.4189999999999996},"last":0.72999999999999998},{"interface":"front","average":{"1min":1.927,"5min":1.7470000000000001,"15min":1.454},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0439999999999996,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.60799999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.6459999999999999,"15min":1.5509999999999999},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.3150000000000004,"5min":7.3940000000000001,"15min":10.257999999999999},"last":1.2549999999999999},{"interface":"front","average":{"1min":2.282,"5min":2.165,"15min":1.829},"min":{"1min":0.28999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.9539999999999997,"5min":9.4049999999999994,"15min":9.5960000000000001},"last":0.46600000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.8140000000000001,"15min":1.3280000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.2800000000000002,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.40899999999999997},{"interface":"front","average":{"1min":1.7569999999999999,"5min":1.9330000000000001,"15min":1.3200000000000001},"min":{"1min":0.36099999999999999,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":6.2030000000000003,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.67300000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:18:42 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":2.0699999999999998,"15min":1.9930000000000001},"min":{"1min":0.36399999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.8179999999999996,"5min":13.502000000000001,"15min":13.502000000000001},"last":0.64700000000000002},{"interface":"front","average":{"1min":2.1520000000000001,"5min":2.298,"15min":2.4420000000000002},"min":{"1min":0.46400000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.9080000000000004,"5min":10.081,"15min":10.081},"last":3.1600000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.1930000000000001,"15min":1.4970000000000001},"min":{"1min":0.45700000000000002,"5min":0.36299999999999999,"15min":0.26800000000000002},"max":{"1min":5.851,"5min":10.743,"15min":13.460000000000001},"last":0.51400000000000001},{"interface":"front","average":{"1min":1.702,"5min":2.21,"15min":1.536},"min":{"1min":0.46400000000000002,"5min":0.32900000000000001,"15min":0.28499999999999998},"max":{"1min":6.2409999999999997,"5min":11.244,"15min":12.404},"last":0.76400000000000001}]}]},{"osd":1,"up_from":545,"seq":2340757176348,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2499976,"kb_used_data":2296320,"kb_used_omap":13,"kb_used_meta":203634,"kb_avail":91245176,"statfs":{"total":95995035648,"available":93435060224,"internally_reserved":0,"allocated":2351431680,"data_stored":2348489208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":208521869},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.776,"5min":0.66800000000000004,"15min":0.48299999999999998},"min":{"1min":0.221,"5min":0.189,"15min":0.189},"max":{"1min":5.8079999999999998,"5min":5.8079999999999998,"15min":5.8079999999999998},"last":3.5169999999999999},{"interface":"front","average":{"1min":1.2430000000000001,"5min":0.64000000000000001,"15min":0.54000000000000004},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6159999999999997,"5min":4.6159999999999997,"15min":4.6159999999999997},"last":0.72499999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":0.67900000000000005,"15min":0.53300000000000003},"min":{"1min":0.28299999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9290000000000003,"5min":4.9290000000000003,"15min":4.9290000000000003},"last":3.1040000000000001},{"interface":"front","average":{"1min":1.272,"5min":0.65800000000000003,"15min":0.55600000000000005},"min":{"1min":0.29899999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2759999999999998,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":0.59899999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":0.746,"15min":0.50600000000000001},"min":{"1min":0.34300000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.3219999999999992,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":2.7189999999999999},{"interface":"front","average":{"1min":2.2429999999999999,"5min":0.85799999999999998,"15min":0.627},"min":{"1min":0.38500000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":8.3659999999999997},"last":2.9740000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":1.347,"15min":1.327},"min":{"1min":0.40200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.6609999999999996,"5min":12.663,"15min":12.663},"last":2.1509999999999998},{"interface":"front","average":{"1min":1.95,"5min":1.601,"15min":1.5429999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.1769999999999996,"5min":13.222,"15min":13.222},"last":2.7610000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":0.88200000000000001,"15min":0.77600000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.0589999999999999,"15min":0.88900000000000001},"min":{"1min":0.46000000000000002,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":8.3190000000000008,"5min":8.3190000000000008,"15min":8.3190000000000008},"last":3.7589999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":0.997,"15min":0.83899999999999997},"min":{"1min":0.45400000000000001,"5min":0.435,"15min":0.435},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":2.1840000000000002},{"interface":"front","average":{"1min":2.6070000000000002,"5min":1.1419999999999999,"15min":0.89800000000000002},"min":{"1min":0.38900000000000001,"5min":0.378,"15min":0.378},"max":{"1min":9.2579999999999991,"5min":9.2579999999999991,"15min":9.2579999999999991},"last":2.8029999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":1.0109999999999999,"15min":0.79100000000000004},"min":{"1min":0.436,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":2.8290000000000002},{"interface":"front","average":{"1min":2.6419999999999999,"5min":1.173,"15min":0.92800000000000005},"min":{"1min":0.48099999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":9.0640000000000001,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":3.012}]}]},{"osd":0,"up_from":93,"seq":399431958845,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4825216,"kb_used_data":3947960,"kb_used_omap":7,"kb_used_meta":877240,"kb_avail":88919936,"statfs":{"total":95995035648,"available":91054014464,"internally_reserved":0,"allocated":4042711040,"data_stored":4039901814,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":898294500},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:19:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7229999999999999,"5min":0.80000000000000004,"15min":0.47999999999999998},"min":{"1min":0.214,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.2309999999999999,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":4.3700000000000001},{"interface":"front","average":{"1min":2.2010000000000001,"5min":0.85799999999999998,"15min":0.63400000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.714,"5min":13.714,"15min":13.714},"last":3.839}]},{"osd":2,"last update":"Sat Sep 23 18:19:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":1.1519999999999999,"15min":1.1299999999999999},"min":{"1min":0.27000000000000002,"5min":0.185,"15min":0.185},"max":{"1min":18.550999999999998,"5min":18.550999999999998,"15min":18.550999999999998},"last":13.351000000000001},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.84199999999999997,"15min":0.79600000000000004},"min":{"1min":0.29299999999999998,"5min":0.22,"15min":0.22},"max":{"1min":13.618,"5min":13.618,"15min":13.618},"last":3.9319999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:18:44 2023","interfaces":[{"interface":"back","average":{"1min":0.92200000000000004,"5min":0.86299999999999999,"15min":1.4450000000000001},"min":{"1min":0.315,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":5.5350000000000001,"5min":8.5619999999999994,"15min":12.779999999999999},"last":13.250999999999999},{"interface":"front","average":{"1min":0.878,"5min":0.88,"15min":1.472},"min":{"1min":0.26600000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.677,"5min":6.8200000000000003,"15min":15.198},"last":13.285}]},{"osd":4,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.2599999999999998,"15min":1.7789999999999999},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":13.574999999999999,"5min":57.746000000000002,"15min":57.746000000000002},"last":3.601},{"interface":"front","average":{"1min":2.032,"5min":1.9770000000000001,"15min":1.6499999999999999},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.8970000000000002,"5min":57.816000000000003,"15min":57.816000000000003},"last":13.315}]},{"osd":5,"last update":"Sat Sep 23 18:18:40 2023","interfaces":[{"interface":"back","average":{"1min":1.161,"5min":1.823,"15min":1.2609999999999999},"min":{"1min":0.35299999999999998,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":4.3140000000000001,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":3.6259999999999999},{"interface":"front","average":{"1min":1.0880000000000001,"5min":1.6379999999999999,"15min":1.2310000000000001},"min":{"1min":0.35799999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":3.2799999999999998,"5min":8.8879999999999999,"15min":8.8879999999999999},"last":13.192}]},{"osd":6,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":1.8049999999999999,"15min":1.861},"min":{"1min":0.433,"5min":0.374,"15min":0.32000000000000001},"max":{"1min":7.1020000000000003,"5min":7.1020000000000003,"15min":58.457999999999998},"last":3.5659999999999998},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.798,"15min":1.7050000000000001},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":13.672000000000001,"5min":13.672000000000001,"15min":58.354999999999997},"last":3.871}]},{"osd":7,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.6719999999999999,"15min":1.488},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":3.1459999999999999,"5min":8.5009999999999994,"15min":10.416},"last":3.907},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.5609999999999999,"15min":1.6759999999999999},"min":{"1min":0.48199999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.6139999999999999,"5min":10.404,"15min":10.404},"last":4.4459999999999997}]}]},{"osd":2,"up_from":0,"seq":2070174236730,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249360,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2702564,"kb_used_data":2033948,"kb_used_omap":3,"kb_used_meta":668412,"kb_avail":91042588,"statfs":{"total":95995035648,"available":93227610112,"internally_reserved":0,"allocated":2082762752,"data_stored":2079751125,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":684454632},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.407,"15min":1.508},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":4.0369999999999999,"5min":195.69800000000001,"15min":195.69800000000001},"last":8.8599999999999994},{"interface":"front","average":{"1min":0.97999999999999998,"5min":2.4470000000000001,"15min":1.8049999999999999},"min":{"1min":0.22600000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":6.1340000000000003,"5min":194.661,"15min":194.661},"last":1.6359999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":0.95799999999999996,"15min":0.66600000000000004},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":21.777000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":4.8959999999999999},{"interface":"front","average":{"1min":6.5110000000000001,"5min":1.8660000000000001,"15min":1.0920000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":78.980000000000004,"5min":78.980000000000004,"15min":78.980000000000004},"last":10.888999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.94299999999999995,"15min":1.179},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.597,"5min":6.3849999999999998,"15min":6.3849999999999998},"last":1.8460000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.93100000000000005,"15min":1.21},"min":{"1min":0.33100000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":4.1040000000000001,"5min":5.931,"15min":5.931},"last":11.565}]},{"osd":4,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.4650000000000001,"15min":1.5669999999999999},"min":{"1min":0.40000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":20.850000000000001,"5min":21.062999999999999,"15min":37.231000000000002},"last":0.64400000000000002},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.833,"15min":1.79},"min":{"1min":0.40300000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":20.786000000000001,"5min":21.128,"15min":37.381},"last":1.8899999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:18:36 2023","interfaces":[{"interface":"back","average":{"1min":0.85399999999999998,"5min":1.4790000000000001,"15min":1.337},"min":{"1min":0.40200000000000002,"5min":0.316,"15min":0.316},"max":{"1min":2.8210000000000002,"5min":8.2959999999999994,"15min":8.2959999999999994},"last":1.982},{"interface":"front","average":{"1min":1.069,"5min":1.423,"15min":1.6120000000000001},"min":{"1min":0.47699999999999998,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":3.6349999999999998,"5min":6.4020000000000001,"15min":8.9969999999999999},"last":8.9359999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.4299999999999999,"15min":1.7110000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":3.8079999999999998,"5min":9.8089999999999993,"15min":115.54000000000001},"last":8.9700000000000006},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.53,"15min":1.859},"min":{"1min":0.44900000000000001,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":3.8460000000000001,"5min":14.321,"15min":115.691},"last":0.37}]},{"osd":7,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.4219999999999999,"15min":1.6319999999999999},"min":{"1min":0.436,"5min":0.36799999999999999,"15min":0.35799999999999998},"max":{"1min":3.5449999999999999,"5min":5.2809999999999997,"15min":15.972},"last":1.9390000000000001},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.327,"15min":1.4550000000000001},"min":{"1min":0.50800000000000001,"5min":0.40699999999999997,"15min":0.34899999999999998},"max":{"1min":2.069,"5min":5.8570000000000002,"15min":9.9109999999999996},"last":9.8070000000000004}]}]},{"osd":4,"up_from":30,"seq":128849019309,"num_pgs":66,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5854856,"kb_used_data":5230592,"kb_used_omap":0,"kb_used_meta":624256,"kb_avail":87890296,"statfs":{"total":95995035648,"available":89999663104,"internally_reserved":0,"allocated":5356126208,"data_stored":5354227584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":639238144},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":1.375,"15min":1.21},"min":{"1min":0.27200000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9660000000000002,"5min":7.2199999999999998,"15min":13.375999999999999},"last":0.498},{"interface":"front","average":{"1min":2.2709999999999999,"5min":1.772,"15min":1.4710000000000001},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":8.9429999999999996,"5min":10.890000000000001,"15min":10.890000000000001},"last":2.149}]},{"osd":1,"last update":"Sat Sep 23 18:19:22 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":1.47,"15min":0.95299999999999996},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":44.146999999999998,"5min":44.146999999999998,"15min":44.146999999999998},"last":2.0510000000000002},{"interface":"front","average":{"1min":4.5419999999999998,"5min":1.4299999999999999,"15min":0.91100000000000003},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":43.497999999999998,"5min":43.497999999999998,"15min":43.497999999999998},"last":2.1349999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.524,"15min":1.7689999999999999},"min":{"1min":0.30299999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.173,"5min":11.173,"15min":11.173},"last":1.9350000000000001},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.6459999999999999,"15min":1.702},"min":{"1min":0.40600000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.0909999999999993,"5min":10.938000000000001,"15min":10.938000000000001},"last":0.37}]},{"osd":3,"last update":"Sat Sep 23 18:18:39 2023","interfaces":[{"interface":"back","average":{"1min":1.6060000000000001,"5min":1.52,"15min":1.887},"min":{"1min":0.35599999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":11.130000000000001,"5min":11.130000000000001,"15min":29.154},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.78,"5min":1.712,"15min":1.873},"min":{"1min":0.41399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":9.1099999999999994,"5min":10.968,"15min":29.216000000000001},"last":1.2789999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:18:39 2023","interfaces":[{"interface":"back","average":{"1min":2.1970000000000001,"5min":2.6499999999999999,"15min":2.9969999999999999},"min":{"1min":0.36499999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":9.0660000000000007,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.56599999999999995},{"interface":"front","average":{"1min":1.373,"5min":1.8300000000000001,"15min":2.8410000000000002},"min":{"1min":0.29899999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":5.8789999999999996,"5min":13.938000000000001,"15min":32.640000000000001},"last":0.51900000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":2.0739999999999998,"15min":1.391},"min":{"1min":0.379,"5min":0.28199999999999997,"15min":0.25},"max":{"1min":7.7249999999999996,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.48699999999999999},{"interface":"front","average":{"1min":1.782,"5min":2.044,"15min":1.51},"min":{"1min":0.33900000000000002,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.6310000000000002,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.54400000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":2.7919999999999998,"15min":1.7490000000000001},"min":{"1min":0.42699999999999999,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":8.9849999999999994,"5min":19.298999999999999,"15min":19.298999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.234,"5min":2.1549999999999998,"15min":1.5569999999999999},"min":{"1min":0.50900000000000001,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.1319999999999997,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.435}]}]},{"osd":5,"up_from":420,"seq":1803886264406,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3751828,"kb_used_data":3436044,"kb_used_omap":9,"kb_used_meta":315766,"kb_avail":89993324,"statfs":{"total":95995035648,"available":92153163776,"internally_reserved":0,"allocated":3518509056,"data_stored":3516610738,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":323344829},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.2330000000000001,"15min":0.83799999999999997},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.206,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.40100000000000002},{"interface":"front","average":{"1min":1.038,"5min":1.423,"15min":1.5189999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.6309999999999998,"5min":7.5439999999999996,"15min":7.5439999999999996},"last":2.9169999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:19:20 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":1.4770000000000001,"15min":1.2230000000000001},"min":{"1min":0.32100000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.946000000000002,"5min":23.946000000000002,"15min":23.946000000000002},"last":3.7320000000000002},{"interface":"front","average":{"1min":3.927,"5min":1.4630000000000001,"15min":1.052},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":23.268999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":3.6309999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.4590000000000001,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.4930000000000003,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":3.8069999999999999},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.891,"15min":2.141},"min":{"1min":0.314,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.6749999999999998,"5min":7.2359999999999998,"15min":7.2359999999999998},"last":2.9489999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.556,"15min":1.153},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.2330000000000001,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":4.1890000000000001},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.534,"15min":1.5169999999999999},"min":{"1min":0.33500000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.8599999999999999,"5min":7.5179999999999998,"15min":7.5179999999999998},"last":7.0750000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":2.1440000000000001,"15min":2.0379999999999998},"min":{"1min":0.36499999999999999,"5min":0.254,"15min":0.251},"max":{"1min":5.375,"5min":12.747999999999999,"15min":12.747999999999999},"last":7.6900000000000004},{"interface":"front","average":{"1min":1.514,"5min":1.9099999999999999,"15min":1.5580000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.069,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":4.391}]},{"osd":6,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.867,"15min":1.1899999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1920000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":3.2450000000000001},{"interface":"front","average":{"1min":1.232,"5min":2.004,"15min":1.6220000000000001},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.9649999999999999,"5min":12.504,"15min":12.504},"last":2.8809999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":2.363,"15min":1.6970000000000001},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.4240000000000004,"5min":13.506,"15min":13.506},"last":7.6509999999999998},{"interface":"front","average":{"1min":1.2310000000000001,"5min":2.0310000000000001,"15min":1.5349999999999999},"min":{"1min":0.378,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.2610000000000001,"5min":13.138,"15min":13.138},"last":3.2130000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4005171200,"data_stored":4005167128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2332295168,"data_stored":2332295168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":2073427968,"data_stored":2073427968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":5340729344,"data_stored":5340725272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3506962432,"data_stored":3506962432,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":3152809984,"data_stored":3152805912,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:19:37.986 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:19:38.036 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:38.301 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:38.312 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:38.312 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:38.413 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:38.495 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:38.556 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:38.567 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:38.567 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:38.668 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:38.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:38 smithi055 ceph-mon[184174]: pgmap v1435: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 6.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 127 B/s rd, 36 MiB/s wr, 906 op/s; 16314/302484 objects degraded (5.393%); 3193/302484 objects misplaced (1.056%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:19:38.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:38 smithi055 ceph-mon[184174]: from='client.37191 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:38.684 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16314/302484 objects degraded (5.393%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:38.684 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:38 smithi055 ceph-mon[179668]: pgmap v1435: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 6.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 127 B/s rd, 36 MiB/s wr, 906 op/s; 16314/302484 objects degraded (5.393%); 3193/302484 objects misplaced (1.056%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:19:38.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:38 smithi055 ceph-mon[179668]: from='client.37191 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:38.685 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16314/302484 objects degraded (5.393%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:38.741 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:38 smithi099 ceph-mon[167076]: pgmap v1435: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 6.2 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 127 B/s rd, 36 MiB/s wr, 906 op/s; 16314/302484 objects degraded (5.393%); 3193/302484 objects misplaced (1.056%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:19:38.741 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:38 smithi099 ceph-mon[167076]: from='client.37191 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:38.741 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16314/302484 objects degraded (5.393%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:38.742 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:38.798 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:38.804 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:38.804 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:38.905 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:38.940 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:38.960 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:38.999 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:19:39.014 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:39.020 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:39.020 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:39.049 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:19:39.049 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:19:39.049 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201423 9.e e552/545 9:73127b0d:::benchmark_data_smithi055_386467_object96763:head v 552'12847, mlcod=552'12845)", 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 132.15540253899999, 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.101742522, 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.050 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.051 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890590+0000", 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0280000000000002e-06 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890596+0000", 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0299999999999999e-06 2023-09-23T18:19:39.052 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890601+0000", 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0660000000000003e-06 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890652+0000", 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1104999999999999e-05 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.053 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890674+0000", 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2274000000000001e-05 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992278+0000", 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.1016035 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.054 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992329+0000", 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0816999999999998e-05 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201426 9.e e552/545 9:7aa9c16e:::benchmark_data_smithi055_386467_object96766:head v 552'12848, mlcod=552'12845)", 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 132.15537342600001, 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.101739732, 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.055 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890619+0000", 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0570000000000003e-06 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890625+0000", 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1970000000000001e-06 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890630+0000", 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0289999999999996e-06 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892129+0000", 2023-09-23T18:19:39.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014991309999999999 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892151+0000", 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2036e-05 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992332+0000", 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10018059 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992355+0000", 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2989e-05 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201428 9.3 e552/545 9:ce8adaf4:::benchmark_data_smithi055_386467_object96768:head v 552'12637, mlcod=551'12635)", 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.892727+0000", 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 132.15326156399999, 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10402589299999999, 2023-09-23T18:19:39.060 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892727+0000", 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892727+0000", 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892726+0000", 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892732+0000", 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6280000000000002e-06 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.062 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892740+0000", 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2050000000000002e-06 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892745+0000", 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8590000000000004e-06 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892789+0000", 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3699000000000003e-05 2023-09-23T18:19:39.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892812+0000", 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.368e-05 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.996720+0000", 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10390730099999999 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.996753+0000", 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3195000000000003e-05 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "pg_scan(get_digest 9.0 MIN-MIN e 552/545)", 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 132.15273803400001, 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.126230497, 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:19:39.065 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893254+0000", 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3780000000000003e-06 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992277+0000", 2023-09-23T18:19:39.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.099022751000000006 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992290+0000", 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3297e-05 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992361+0000", 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0791999999999996e-05 2023-09-23T18:19:39.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992377+0000", 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6331e-05 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:27.019481+0000", 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027103558 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.069 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201433 9.e e552/545 9:772bf683:::benchmark_data_smithi055_386467_object96773:head v 552'12849, mlcod=552'12845)", 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.893949+0000", 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 132.152040008, 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.130621881, 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893949+0000", 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893949+0000", 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893948+0000", 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893952+0000", 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3640000000000001e-06 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992298+0000", 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.098345455999999998 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992310+0000", 2023-09-23T18:19:39.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1708e-05 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992386+0000", 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6688999999999997e-05 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992407+0000", 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0678000000000001e-05 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:27.024537+0000", 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032130134999999997 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:27.024570+0000", 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3327999999999997e-05 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.074 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:35571 10.2 10:446a4c6c:::benchmark_data_smithi055_425050_object35570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:17.660369+0000", 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 81.385619782999996, 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.169126159, 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.075 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 35571 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660369+0000", 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.076 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660369+0000", 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660367+0000", 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660393+0000", 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5347e-05 2023-09-23T18:19:39.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660411+0000", 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7903000000000001e-05 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660417+0000", 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9630000000000003e-06 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660468+0000", 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1572e-05 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660588+0000", 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119657 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:19:39.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660772+0000", 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000184167 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.662259+0000", 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014871649999999999 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.662719+0000", 2023-09-23T18:19:39.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00046007099999999999 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.829386+0000", 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.16666637400000001 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.829449+0000", 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3450999999999999e-05 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.082 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.829495+0000", 2023-09-23T18:19:39.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5933999999999998e-05 2023-09-23T18:19:39.082 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.082 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.082 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.082 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55937 10.4 10:2db53d3f:::benchmark_data_smithi055_425050_object55936:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:19:39.082 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.319675+0000", 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 65.726313740999998, 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3796218790000001, 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55937 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319675+0000", 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319675+0000", 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319673+0000", 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:19:39.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319716+0000", 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2919e-05 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319731+0000", 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5237e-05 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319743+0000", 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.225e-05 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321777+0000", 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020343399999999999 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321791+0000", 2023-09-23T18:19:39.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3355e-05 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.686788+0000", 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.364997741 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.687744+0000", 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00095536700000000004 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.687877+0000", 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000132908 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693069+0000", 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0051921600000000004 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693082+0000", 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.314e-05 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693115+0000", 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3355999999999997e-05 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697881+0000", 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047652900000000002 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697954+0000", 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3436999999999995e-05 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.699297+0000", 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00134252 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.090 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55942 10.4 10:25b9d4ec:::benchmark_data_smithi055_425050_object55941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.321753+0000", 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 65.724235145999998, 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3789243330000001, 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.091 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.092 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55942 2023-09-23T18:19:39.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.092 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321753+0000", 2023-09-23T18:19:39.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.092 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321753+0000", 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321753+0000", 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321791+0000", 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8318000000000001e-05 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321805+0000", 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4208e-05 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321815+0000", 2023-09-23T18:19:39.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6930000000000002e-06 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321860+0000", 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5293000000000002e-05 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321869+0000", 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1400000000000006e-06 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.690763+0000", 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.368893624 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.690928+0000", 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016521400000000001 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.691049+0000", 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012102799999999999 2023-09-23T18:19:39.096 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693150+0000", 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021011099999999998 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697641+0000", 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044906629999999998 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.097 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697713+0000", 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2111000000000001e-05 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.699368+0000", 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016551000000000001 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.098 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.699427+0000", 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.859e-05 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.700678+0000", 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001251114 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55950 10.4 10:2586d6ac:::benchmark_data_smithi055_425050_object55949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:19:39.099 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.321829+0000", 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 65.724159138999994, 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3802366020000001, 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55950 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.100 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321829+0000", 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321829+0000", 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321829+0000", 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321853+0000", 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3898999999999999e-05 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321861+0000", 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9009999999999994e-06 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.102 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321867+0000", 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.387e-06 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321900+0000", 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3447e-05 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321910+0000", 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2629999999999992e-06 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.103 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693287+0000", 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3713770679999999 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693442+0000", 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015501600000000001 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.104 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693626+0000", 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018425900000000001 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697760+0000", 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041337860000000004 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697773+0000", 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.358e-05 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.105 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697826+0000", 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2553999999999999e-05 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.700869+0000", 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030429720000000001 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.700906+0000", 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6801999999999998e-05 2023-09-23T18:19:39.106 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.702066+0000", 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001160276 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55965 10.4 10:2d25bc98:::benchmark_data_smithi055_425050_object55964:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.321878+0000", 2023-09-23T18:19:39.107 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 65.724110866999993, 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3949194659999999, 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55965 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321878+0000", 2023-09-23T18:19:39.108 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321878+0000", 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321877+0000", 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.109 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321900+0000", 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2762999999999999e-05 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321909+0000", 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9379999999999998e-06 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.110 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321915+0000", 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1809999999999998e-06 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321958+0000", 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3473999999999997e-05 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:39.111 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321967+0000", 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6070000000000008e-06 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.695651+0000", 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.373683609 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.695822+0000", 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017090399999999999 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.112 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.695961+0000", 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013964700000000001 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.699322+0000", 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033606560000000001 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.113 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.700745+0000", 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001422942 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.700778+0000", 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2872e-05 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.715581+0000", 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014802996000000001 2023-09-23T18:19:39.114 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.715626+0000", 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5742999999999999e-05 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.716797+0000", 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011706889999999999 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.115 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55971 10.4 10:231c3e9f:::benchmark_data_smithi055_425050_object55970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.321925+0000", 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 65.724063908999995, 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3957034830000001, 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55971 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.116 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321925+0000", 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321925+0000", 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321924+0000", 2023-09-23T18:19:39.117 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321950+0000", 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6506999999999999e-05 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321956+0000", 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.801e-06 2023-09-23T18:19:39.118 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321962+0000", 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5999999999999997e-06 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322005+0000", 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2996e-05 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322013+0000", 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9500000000000001e-06 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.702122+0000", 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.380109322 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.702307+0000", 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018535200000000001 2023-09-23T18:19:39.120 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.702450+0000", 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014205199999999999 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.707495+0000", 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050456310000000001 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.121 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.709169+0000", 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016743249999999999 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.709180+0000", 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0181e-05 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.716863+0000", 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0076837950000000002 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.122 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.716904+0000", 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0802e-05 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.717628+0000", 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00072384800000000003 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.123 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55990 10.4 10:25823440:::benchmark_data_smithi055_425050_object55989:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.321970+0000", 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 65.724018829000002, 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3964889010000001, 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55990 2023-09-23T18:19:39.124 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321970+0000", 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321970+0000", 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.125 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321969+0000", 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322021+0000", 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2144000000000001e-05 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322027+0000", 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8889999999999999e-06 2023-09-23T18:19:39.126 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322031+0000", 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9609999999999999e-06 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322070+0000", 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9218999999999998e-05 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:39.127 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322078+0000", 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5560000000000002e-06 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.705051+0000", 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3829731540000001 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.705240+0000", 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018871599999999999 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.128 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.705435+0000", 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019560700000000001 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.709196+0000", 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037605849999999999 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.129 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.709232+0000", 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6625000000000003e-05 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.715537+0000", 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063041709999999999 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.717683+0000", 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021465159999999998 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.130 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.717729+0000", 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5457000000000002e-05 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.718459+0000", 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00072998799999999999 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.131 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67069 10.4 10:25914e1c:::benchmark_data_smithi055_425050_object67068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.197763+0000", 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 53.848225057999997, 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.130859224, 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67069 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.132 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197763+0000", 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197763+0000", 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197762+0000", 2023-09-23T18:19:39.133 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197792+0000", 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0354000000000001e-05 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197813+0000", 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0839000000000002e-05 2023-09-23T18:19:39.134 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197823+0000", 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0178e-05 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.202503+0000", 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0046796060000000002 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.135 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.202697+0000", 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000193697 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.202901+0000", 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020478099999999999 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.136 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.207446+0000", 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045442390000000003 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.211087+0000", 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036408669999999999 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.215890+0000", 2023-09-23T18:19:39.137 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048037890000000001 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.327070+0000", 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.111180026 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.327133+0000", 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2765e-05 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.138 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.328623+0000", 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014895959999999999 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67093 10.4 10:270b9556:::benchmark_data_smithi055_425050_object67092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.219838+0000", 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 53.826150396999999, 2023-09-23T18:19:39.139 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11352698899999999, 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67093 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219838+0000", 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.140 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219838+0000", 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219836+0000", 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219865+0000", 2023-09-23T18:19:39.141 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8988000000000001e-05 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219882+0000", 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7547999999999999e-05 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219895+0000", 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2412000000000001e-05 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.142 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.220160+0000", 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026516400000000002 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.220284+0000", 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000124404 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.220399+0000", 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114948 2023-09-23T18:19:39.143 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.227313+0000", 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0069139500000000003 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.227354+0000", 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0824000000000001e-05 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.144 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.227379+0000", 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4864000000000001e-05 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.331996+0000", 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.104617342 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.332057+0000", 2023-09-23T18:19:39.145 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1183e-05 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.333365+0000", 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013077360000000001 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67102 10.4 10:2528425f:::benchmark_data_smithi055_425050_object67101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:19:39.146 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.230801+0000", 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 53.815187043999998, 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.103525952, 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67102 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.147 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230801+0000", 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230801+0000", 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230800+0000", 2023-09-23T18:19:39.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230826+0000", 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5854999999999999e-05 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230839+0000", 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2743e-05 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.149 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230845+0000", 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1840000000000004e-06 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.231280+0000", 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00043508799999999998 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.231471+0000", 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019095900000000001 2023-09-23T18:19:39.150 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.231713+0000", 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000242248 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.239490+0000", 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0077772129999999998 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.151 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.239536+0000", 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6171999999999998e-05 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.239556+0000", 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9157999999999999e-05 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.333433+0000", 2023-09-23T18:19:39.152 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.093877304999999994 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.333489+0000", 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6233999999999997e-05 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.334327+0000", 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00083832600000000002 2023-09-23T18:19:39.153 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67112 10.4 10:2686efcb:::benchmark_data_smithi055_425050_object67111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.243689+0000", 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 53.802299818000002, 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.093717647000000001, 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.154 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67112 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243689+0000", 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243689+0000", 2023-09-23T18:19:39.155 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243687+0000", 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243713+0000", 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5333999999999998e-05 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.156 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243726+0000", 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3794999999999999e-05 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243735+0000", 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.32e-06 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.247533+0000", 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003798413 2023-09-23T18:19:39.157 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.247651+0000", 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118192 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.247763+0000", 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011155899999999999 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.158 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.251684+0000", 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039215109999999999 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.251753+0000", 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8285000000000001e-05 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.259065+0000", 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0073120659999999999 2023-09-23T18:19:39.159 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.336340+0000", 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.077275132999999996 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.336403+0000", 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2880999999999996e-05 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.160 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.337406+0000", 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001003557 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67113 10.4 10:2c8fad6a:::benchmark_data_smithi055_425050_object67112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.244958+0000", 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 53.801030502000003, 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.093448478000000001, 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.161 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67113 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244958+0000", 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.162 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244958+0000", 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244957+0000", 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.163 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244982+0000", 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5035e-05 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244992+0000", 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3449999999999995e-06 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244999+0000", 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6410000000000002e-06 2023-09-23T18:19:39.164 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.249567+0000", 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045676069999999996 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.249721+0000", 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015449600000000001 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.249834+0000", 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000112039 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.251740+0000", 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001906196 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.251859+0000", 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011937799999999999 2023-09-23T18:19:39.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.259121+0000", 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072623979999999998 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.337468+0000", 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.078346247999999993 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.337526+0000", 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8489000000000002e-05 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.338406+0000", 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00088027600000000004 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67132 10.4 10:23155624:::benchmark_data_smithi055_425050_object67131:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.340445+0000", 2023-09-23T18:19:39.168 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 53.705543648000003, 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.088696630999999998, 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67132 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340445+0000", 2023-09-23T18:19:39.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340445+0000", 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340443+0000", 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340472+0000", 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8585000000000002e-05 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340491+0000", 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9244000000000001e-05 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340501+0000", 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6779999999999995e-06 2023-09-23T18:19:39.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.343768+0000", 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0032674819999999999 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.343936+0000", 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016819200000000001 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.344136+0000", 2023-09-23T18:19:39.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019938200000000001 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.351346+0000", 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072103590000000004 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.351387+0000", 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1423e-05 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.173 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.351429+0000", 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1378999999999998e-05 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.427790+0000", 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.076360753000000003 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.427856+0000", 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6674999999999994e-05 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.429142+0000", 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0012852009999999999 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67135 10.4 10:256accc7:::benchmark_data_smithi055_425050_object67134:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.340516+0000", 2023-09-23T18:19:39.175 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 53.705472481999998, 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.089923407999999996, 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67135 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340516+0000", 2023-09-23T18:19:39.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340516+0000", 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340515+0000", 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340545+0000", 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9499e-05 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340558+0000", 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2914999999999999e-05 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340569+0000", 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.083e-05 2023-09-23T18:19:39.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.346165+0000", 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0055960799999999998 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.346303+0000", 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013825600000000001 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.179 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.346455+0000", 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015165 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.351396+0000", 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0049410740000000002 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.351466+0000", 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9868000000000003e-05 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.180 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.354364+0000", 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028985349999999998 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.429207+0000", 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.074843153999999995 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.429250+0000", 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.286e-05 2023-09-23T18:19:39.181 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.430439+0000", 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001189329 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67137 10.4 10:217dc21f:::benchmark_data_smithi055_425050_object67136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.340575+0000", 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 53.705413669999999, 2023-09-23T18:19:39.182 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.094444057999999997, 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67137 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340575+0000", 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.183 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340575+0000", 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340574+0000", 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340598+0000", 2023-09-23T18:19:39.184 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3149000000000001e-05 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340610+0000", 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2245e-05 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340615+0000", 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.519e-06 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:39.185 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.348604+0000", 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0079891879999999995 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.348811+0000", 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020653699999999999 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.349042+0000", 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00023097600000000001 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.186 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.353098+0000", 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040564579999999998 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.354411+0000", 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001312192 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.356938+0000", 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025268809999999999 2023-09-23T18:19:39.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.432957+0000", 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.076019171999999996 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.433021+0000", 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4101000000000002e-05 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:39.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:39.189 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.435019+0000", 2023-09-23T18:19:39.189 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019981259999999998 2023-09-23T18:19:39.189 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.189 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.189 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.189 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:39.189 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:39.189 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:19:39.189 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:39.254 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:39.321 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:39.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:39.328 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:39.429 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:39.488 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:39.539 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:39.561 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:39.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:39.570 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:39.634 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:19:39.670 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:39.741 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:39.927 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:39.934 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:39.934 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:40.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:40.081 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:40.154 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:40.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:40.159 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:40.259 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:40.325 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:40.412 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:40.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:40.417 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:40.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:40.573 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:40.623 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:40.641 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:40.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:40.650 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:40.652 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:19:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:19:40] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.43.0" 2023-09-23T18:19:40.652 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:40 smithi055 ceph-mon[184174]: pgmap v1436: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 6.1 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 127 B/s rd, 24 MiB/s wr, 760 op/s; 16314/301095 objects degraded (5.418%); 3193/301095 objects misplaced (1.060%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:19:40.653 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:40 smithi055 ceph-mon[179668]: pgmap v1436: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 6.1 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 127 B/s rd, 24 MiB/s wr, 760 op/s; 16314/301095 objects degraded (5.418%); 3193/301095 objects misplaced (1.060%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:19:40.750 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:40.796 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:40.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:40 smithi099 ceph-mon[167076]: pgmap v1436: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 6.1 GiB data, 22 GiB used, 514 GiB / 536 GiB avail; 127 B/s rd, 24 MiB/s wr, 760 op/s; 16314/301095 objects degraded (5.418%); 3193/301095 objects misplaced (1.060%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:19:40.844 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:40.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:40.851 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:40.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:41.015 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:41.081 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:41.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:41.088 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:41.189 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:41.253 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:41.264 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:41.300 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:41.308 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:41.308 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:41.409 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:41.499 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:42.279 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:42.284 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:42.284 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:42.385 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:42.416 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:42.575 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:19:42.575 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"nodeep-scrub flag(s) set","count":12},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 16314/295275 objects degraded (5.525%), 7 pgs degraded, 2 pgs undersized","count":9},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1672,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":606,"num_osds":8,"num_up_osds":8,"osd_up_since":1695493039,"num_in_osds":6,"osd_in_since":1695493021,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":112},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":98425,"data_bytes":6453017276,"bytes_used":23305641984,"bytes_avail":552664571904,"bytes_total":575970213888,"degraded_objects":16314,"degraded_total":295275,"degraded_ratio":0.055250190500381005,"misplaced_objects":3193,"misplaced_total":295275,"misplaced_ratio":0.010813648293963257,"recovering_objects_per_sec":63,"recovering_bytes_per_sec":4216856,"recovering_keys_per_sec":0,"num_objects_recovered":511,"num_bytes_recovered":33751040,"num_keys_recovered":0,"read_bytes_sec":127,"write_bytes_sec":7827923,"read_op_per_sec":0,"write_op_per_sec":742},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T18:18:19.098325+0000","services":{}},"progress_events":{"57c27436-1bc8-4f63-b41f-a80b88a05748":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T18:19:42.604 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:19:42.610 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:42.618 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:42.618 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:42.719 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:42.766 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:42.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:42 smithi099 ceph-mon[167076]: pgmap v1437: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 6.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 7.5 MiB/s wr, 742 op/s; 16314/295275 objects degraded (5.525%); 3193/295275 objects misplaced (1.081%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:19:42.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:42 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3387050884' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:42 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/752347575' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:19:42.828 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:42.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:42.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:42.842 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:42 smithi055 ceph-mon[184174]: pgmap v1437: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 6.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 7.5 MiB/s wr, 742 op/s; 16314/295275 objects degraded (5.525%); 3193/295275 objects misplaced (1.081%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:19:42.842 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:42 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3387050884' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:42.842 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:42 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/752347575' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:19:42.845 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:42 smithi055 ceph-mon[179668]: pgmap v1437: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 6.0 GiB data, 22 GiB used, 515 GiB / 536 GiB avail; 127 B/s rd, 7.5 MiB/s wr, 742 op/s; 16314/295275 objects degraded (5.525%); 3193/295275 objects misplaced (1.081%); 4.0 MiB/s, 63 objects/s recovering 2023-09-23T18:19:42.845 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:42 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3387050884' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:42.845 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:42 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/752347575' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:19:42.941 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:43.020 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:43.095 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:43.101 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:43.102 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:43.202 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:43.268 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:43.564 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:43.571 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:43.572 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:43.652 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:19:43.652 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:19:43.672 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:43.803 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:43.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:43 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:19:43.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3387050884' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:43.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:43 smithi099 ceph-mon[167076]: osdmap e607: 8 total, 8 up, 6 in 2023-09-23T18:19:43.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:43 smithi099 ceph-mon[167076]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:43 smithi099 ceph-mon[167076]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:43 smithi099 ceph-mon[167076]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:43 smithi099 ceph-mon[167076]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16314/295275 objects degraded (5.525%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:43.918 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:43.932 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:43.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:19:43.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3387050884' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:43.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[179668]: osdmap e607: 8 total, 8 up, 6 in 2023-09-23T18:19:43.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[179668]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[179668]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[179668]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[179668]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16314/295275 objects degraded (5.525%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:43.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:19:43.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3387050884' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:43.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[184174]: osdmap e607: 8 total, 8 up, 6 in 2023-09-23T18:19:43.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[184174]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[184174]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[184174]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[184174]: failed to encode map e607 with expected crc 2023-09-23T18:19:43.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16314/295275 objects degraded (5.525%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:43.935 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:44.035 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:44.078 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:44.145 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:44.151 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:19:44.152 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:44.152 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:44.183 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:19:44.226 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:19:44.253 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:44.267 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:19:44.302 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:19:44.328 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:44.355 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:19:44.389 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:19:44.400 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:44.406 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:44.406 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:44.426 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:19:44.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:19:44.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:19:44.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:19:44.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:19:44.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:19:44.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:19:44.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:19:44.437 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:19:44.507 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:44.548 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:44.604 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:44.614 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:44.615 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:44.712 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:19:44.713 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:19:44.715 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:44.786 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:44.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:44 smithi099 ceph-mon[167076]: pgmap v1439: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 117 B/s rd, 5.6 MiB/s wr, 904 op/s; 16314/288549 objects degraded (5.654%); 2991/288549 objects misplaced (1.037%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T18:19:44.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1051454506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:44.895 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:44 smithi055 ceph-mon[184174]: pgmap v1439: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 117 B/s rd, 5.6 MiB/s wr, 904 op/s; 16314/288549 objects degraded (5.654%); 2991/288549 objects misplaced (1.037%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T18:19:44.895 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1051454506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:44.895 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:44 smithi055 ceph-mon[179668]: pgmap v1439: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 117 B/s rd, 5.6 MiB/s wr, 904 op/s; 16314/288549 objects degraded (5.654%); 2991/288549 objects misplaced (1.037%); 5.1 MiB/s, 81 objects/s recovering 2023-09-23T18:19:44.895 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1051454506' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:45.109 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:45.113 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:45.114 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:45.214 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:45.255 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:19:45.261 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:45.298 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:45.366 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:19:45.411 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:45.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:45.420 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:45.520 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:45.575 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:45.632 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:45.640 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:45.641 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:45.742 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:45.813 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:45.895 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:45.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:45.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:46.004 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:46.060 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:46.118 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:46.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:46.127 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:46.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:46.262 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:46.320 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:46.328 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:46.328 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:46.402 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:46.405 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:46.429 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:46.519 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:46.668 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:46.673 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:46.673 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:46.774 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:46.843 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:46 smithi055 ceph-mon[179668]: pgmap v1440: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 4.9 MiB/s wr, 818 op/s; 16314/287748 objects degraded (5.670%); 2991/287748 objects misplaced (1.039%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:19:46.844 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:46 smithi055 ceph-mon[184174]: pgmap v1440: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 4.9 MiB/s wr, 818 op/s; 16314/287748 objects degraded (5.670%); 2991/287748 objects misplaced (1.039%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:19:46.847 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:47.011 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:19:47.020 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:47.030 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:47.030 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:47.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:46 smithi099 ceph-mon[167076]: pgmap v1440: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.9 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 102 B/s rd, 4.9 MiB/s wr, 818 op/s; 16314/287748 objects degraded (5.670%); 2991/287748 objects misplaced (1.039%); 4.5 MiB/s, 71 objects/s recovering 2023-09-23T18:19:47.131 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:47.245 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:47.354 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:19:47.364 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:19:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:48 smithi055 ceph-mon[184174]: pgmap v1441: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 869 op/s; 16372/276381 objects degraded (5.924%); 2719/276381 objects misplaced (0.984%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:19:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:48 smithi055 ceph-mon[184174]: from='client.37209 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16372/276381 objects degraded (5.924%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:48.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:48 smithi055 ceph-mon[179668]: pgmap v1441: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 869 op/s; 16372/276381 objects degraded (5.924%); 2719/276381 objects misplaced (0.984%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:19:48.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:48 smithi055 ceph-mon[179668]: from='client.37209 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:48.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16372/276381 objects degraded (5.924%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:49.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:48 smithi099 ceph-mon[167076]: pgmap v1441: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.6 GiB data, 21 GiB used, 515 GiB / 536 GiB avail; 0 B/s wr, 869 op/s; 16372/276381 objects degraded (5.924%); 2719/276381 objects misplaced (0.984%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:19:49.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:48 smithi099 ceph-mon[167076]: from='client.37209 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:49.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16372/276381 objects degraded (5.924%), 7 pgs degraded, 2 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:49.817 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:49.826 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:49.826 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:49.927 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:50.001 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1441,"stamp":"2023-09-23T18:19:47.139541+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":6040271548,"num_objects":92127,"num_object_clones":0,"num_object_copies":276381,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":16372,"num_objects_misplaced":2719,"num_objects_unfound":0,"num_objects_dirty":92127,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":115619,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4462,"num_bytes_recovered":282353289,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83330,"ondisk_log_size":83330,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":21939840,"kb_used_data":18585360,"kb_used_omap":40,"kb_used_meta":3354135,"kb_avail":540531072,"statfs":{"total":575970213888,"available":553503817728,"internally_reserved":0,"allocated":19031408640,"data_stored":19016871691,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3434634554},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,7,8,2,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-570228736,"num_objects":-8701,"num_object_clones":0,"num_object_copies":-26103,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":58,"num_objects_misplaced":-474,"num_objects_unfound":0,"num_objects_dirty":-8701,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8701,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":416,"num_bytes_recovered":27262976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004123"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":895,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391168+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:19:34.391168+0000","last_peered":"2023-09-23T18:19:34.391168+0000","last_clean":"2023-09-23T18:19:34.391168+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:19:34.391168+0000","last_undegraded":"2023-09-23T18:19:34.391168+0000","last_fullsized":"2023-09-23T18:19:34.391168+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":265,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.369208+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:19:33.369208+0000","last_peered":"2023-09-23T18:19:33.369208+0000","last_clean":"2023-09-23T18:19:33.369208+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:19:33.369208+0000","last_undegraded":"2023-09-23T18:19:33.369208+0000","last_fullsized":"2023-09-23T18:19:33.369208+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398176+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:19:34.398176+0000","last_peered":"2023-09-23T18:19:34.398176+0000","last_clean":"2023-09-23T18:19:34.398176+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:19:34.398176+0000","last_undegraded":"2023-09-23T18:19:34.398176+0000","last_fullsized":"2023-09-23T18:19:34.398176+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":901,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390616+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:19:34.390616+0000","last_peered":"2023-09-23T18:19:34.390616+0000","last_clean":"2023-09-23T18:19:34.390616+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:19:34.390616+0000","last_undegraded":"2023-09-23T18:19:34.390616+0000","last_fullsized":"2023-09-23T18:19:34.390616+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":347,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398721+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:19:34.398721+0000","last_peered":"2023-09-23T18:19:34.398721+0000","last_clean":"2023-09-23T18:19:34.398721+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:19:34.398721+0000","last_undegraded":"2023-09-23T18:19:34.398721+0000","last_fullsized":"2023-09-23T18:19:34.398721+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":667,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396592+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:19:34.396592+0000","last_peered":"2023-09-23T18:19:34.396592+0000","last_clean":"2023-09-23T18:19:34.396592+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:19:34.396592+0000","last_undegraded":"2023-09-23T18:19:34.396592+0000","last_fullsized":"2023-09-23T18:19:34.396592+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":918,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390373+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:19:34.390373+0000","last_peered":"2023-09-23T18:19:34.390373+0000","last_clean":"2023-09-23T18:19:34.390373+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:34.390373+0000","last_undegraded":"2023-09-23T18:19:34.390373+0000","last_fullsized":"2023-09-23T18:19:34.390373+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.397821+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:34.397821+0000","last_peered":"2023-09-23T18:19:34.397821+0000","last_clean":"2023-09-23T18:19:34.397821+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:19:34.397821+0000","last_undegraded":"2023-09-23T18:19:34.397821+0000","last_fullsized":"2023-09-23T18:19:34.397821+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":183,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401669+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:19:34.401669+0000","last_peered":"2023-09-23T18:19:34.401669+0000","last_clean":"2023-09-23T18:19:34.401669+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:19:34.401669+0000","last_undegraded":"2023-09-23T18:19:34.401669+0000","last_fullsized":"2023-09-23T18:19:34.401669+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1041,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390840+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:19:34.390840+0000","last_peered":"2023-09-23T18:19:34.390840+0000","last_clean":"2023-09-23T18:19:34.390840+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:19:34.390840+0000","last_undegraded":"2023-09-23T18:19:34.390840+0000","last_fullsized":"2023-09-23T18:19:34.390840+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":117,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391102+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:19:34.391102+0000","last_peered":"2023-09-23T18:19:34.391102+0000","last_clean":"2023-09-23T18:19:34.391102+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:19:34.391102+0000","last_undegraded":"2023-09-23T18:19:34.391102+0000","last_fullsized":"2023-09-23T18:19:34.391102+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":196,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398219+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:19:34.398219+0000","last_peered":"2023-09-23T18:19:34.398219+0000","last_clean":"2023-09-23T18:19:34.398219+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:19:34.398219+0000","last_undegraded":"2023-09-23T18:19:34.398219+0000","last_fullsized":"2023-09-23T18:19:34.398219+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":818,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400857+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:19:34.400857+0000","last_peered":"2023-09-23T18:19:34.400857+0000","last_clean":"2023-09-23T18:19:34.400857+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:19:34.400857+0000","last_undegraded":"2023-09-23T18:19:34.400857+0000","last_fullsized":"2023-09-23T18:19:34.400857+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":153,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390974+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:19:34.390974+0000","last_peered":"2023-09-23T18:19:34.390974+0000","last_clean":"2023-09-23T18:19:34.390974+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:19:34.390974+0000","last_undegraded":"2023-09-23T18:19:34.390974+0000","last_fullsized":"2023-09-23T18:19:34.390974+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":152,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400947+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:19:34.400947+0000","last_peered":"2023-09-23T18:19:34.400947+0000","last_clean":"2023-09-23T18:19:34.400947+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:19:34.400947+0000","last_undegraded":"2023-09-23T18:19:34.400947+0000","last_fullsized":"2023-09-23T18:19:34.400947+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":952,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391175+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:19:34.391175+0000","last_peered":"2023-09-23T18:19:34.391175+0000","last_clean":"2023-09-23T18:19:34.391175+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:19:34.391175+0000","last_undegraded":"2023-09-23T18:19:34.391175+0000","last_fullsized":"2023-09-23T18:19:34.391175+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":401,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398559+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:19:34.398559+0000","last_peered":"2023-09-23T18:19:34.398559+0000","last_clean":"2023-09-23T18:19:34.398559+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:19:34.398559+0000","last_undegraded":"2023-09-23T18:19:34.398559+0000","last_fullsized":"2023-09-23T18:19:34.398559+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":420,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.402025+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:19:34.402025+0000","last_peered":"2023-09-23T18:19:34.402025+0000","last_clean":"2023-09-23T18:19:34.402025+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:19:34.402025+0000","last_undegraded":"2023-09-23T18:19:34.402025+0000","last_fullsized":"2023-09-23T18:19:34.402025+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":981,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:42.592426+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:19:42.592426+0000","last_peered":"2023-09-23T18:19:42.592426+0000","last_clean":"2023-09-23T18:19:42.592426+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:19:42.592426+0000","last_undegraded":"2023-09-23T18:19:42.592426+0000","last_fullsized":"2023-09-23T18:19:42.592426+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":159,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401271+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:19:34.401271+0000","last_peered":"2023-09-23T18:19:34.401271+0000","last_clean":"2023-09-23T18:19:34.401271+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:19:34.401271+0000","last_undegraded":"2023-09-23T18:19:34.401271+0000","last_fullsized":"2023-09-23T18:19:34.401271+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":426,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401302+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:19:34.401302+0000","last_peered":"2023-09-23T18:19:34.401302+0000","last_clean":"2023-09-23T18:19:34.401302+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:19:34.401302+0000","last_undegraded":"2023-09-23T18:19:34.401302+0000","last_fullsized":"2023-09-23T18:19:34.401302+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":833,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390615+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:19:34.390615+0000","last_peered":"2023-09-23T18:19:34.390615+0000","last_clean":"2023-09-23T18:19:34.390615+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:19:34.390615+0000","last_undegraded":"2023-09-23T18:19:34.390615+0000","last_fullsized":"2023-09-23T18:19:34.390615+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":426,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401221+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:19:34.401221+0000","last_peered":"2023-09-23T18:19:34.401221+0000","last_clean":"2023-09-23T18:19:34.401221+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:19:34.401221+0000","last_undegraded":"2023-09-23T18:19:34.401221+0000","last_fullsized":"2023-09-23T18:19:34.401221+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":104,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390873+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:19:34.390873+0000","last_peered":"2023-09-23T18:19:34.390873+0000","last_clean":"2023-09-23T18:19:34.390873+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:19:34.390873+0000","last_undegraded":"2023-09-23T18:19:34.390873+0000","last_fullsized":"2023-09-23T18:19:34.390873+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":804,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390243+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:19:34.390243+0000","last_peered":"2023-09-23T18:19:34.390243+0000","last_clean":"2023-09-23T18:19:34.390243+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:19:34.390243+0000","last_undegraded":"2023-09-23T18:19:34.390243+0000","last_fullsized":"2023-09-23T18:19:34.390243+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":122,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396224+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:19:34.396224+0000","last_peered":"2023-09-23T18:19:34.396224+0000","last_clean":"2023-09-23T18:19:34.396224+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:19:34.396224+0000","last_undegraded":"2023-09-23T18:19:34.396224+0000","last_fullsized":"2023-09-23T18:19:34.396224+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390982+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:19:34.390982+0000","last_peered":"2023-09-23T18:19:34.390982+0000","last_clean":"2023-09-23T18:19:34.390982+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:19:34.390982+0000","last_undegraded":"2023-09-23T18:19:34.390982+0000","last_fullsized":"2023-09-23T18:19:34.390982+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":742,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398346+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:19:34.398346+0000","last_peered":"2023-09-23T18:19:34.398346+0000","last_clean":"2023-09-23T18:19:34.398346+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:19:34.398346+0000","last_undegraded":"2023-09-23T18:19:34.398346+0000","last_fullsized":"2023-09-23T18:19:34.398346+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":194,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400859+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:19:34.400859+0000","last_peered":"2023-09-23T18:19:34.400859+0000","last_clean":"2023-09-23T18:19:34.400859+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:19:34.400859+0000","last_undegraded":"2023-09-23T18:19:34.400859+0000","last_fullsized":"2023-09-23T18:19:34.400859+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":205,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400051+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:19:34.400051+0000","last_peered":"2023-09-23T18:19:34.400051+0000","last_clean":"2023-09-23T18:19:34.400051+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:19:34.400051+0000","last_undegraded":"2023-09-23T18:19:34.400051+0000","last_fullsized":"2023-09-23T18:19:34.400051+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":838,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400969+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:19:34.400969+0000","last_peered":"2023-09-23T18:19:34.400969+0000","last_clean":"2023-09-23T18:19:34.400969+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:19:34.400969+0000","last_undegraded":"2023-09-23T18:19:34.400969+0000","last_fullsized":"2023-09-23T18:19:34.400969+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":747,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400877+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:19:34.400877+0000","last_peered":"2023-09-23T18:19:34.400877+0000","last_clean":"2023-09-23T18:19:34.400877+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:19:34.400877+0000","last_undegraded":"2023-09-23T18:19:34.400877+0000","last_fullsized":"2023-09-23T18:19:34.400877+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400010+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:19:34.400010+0000","last_peered":"2023-09-23T18:19:34.400010+0000","last_clean":"2023-09-23T18:19:34.400010+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:19:34.400010+0000","last_undegraded":"2023-09-23T18:19:34.400010+0000","last_fullsized":"2023-09-23T18:19:34.400010+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"607'7274","reported_seq":7315,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:46.056301+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:19:46.056301+0000","last_peered":"2023-09-23T18:19:46.056301+0000","last_clean":"2023-09-23T18:19:46.056301+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:19:46.056301+0000","last_undegraded":"2023-09-23T18:19:46.056301+0000","last_fullsized":"2023-09-23T18:19:46.056301+0000","mapping_epoch":574,"log_start":"583'1800","ondisk_log_start":"583'1800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5474,"log_dups_size":0,"ondisk_log_size":5474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":380108800,"num_objects":5800,"num_object_clones":0,"num_object_copies":17400,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5800,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7274,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":872,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401856+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:19:34.401856+0000","last_peered":"2023-09-23T18:19:34.401856+0000","last_clean":"2023-09-23T18:19:34.401856+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:19:34.401856+0000","last_undegraded":"2023-09-23T18:19:34.401856+0000","last_fullsized":"2023-09-23T18:19:34.401856+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":242,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.369484+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:19:33.369484+0000","last_peered":"2023-09-23T18:19:33.369484+0000","last_clean":"2023-09-23T18:19:33.369484+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:19:33.369484+0000","last_undegraded":"2023-09-23T18:19:33.369484+0000","last_fullsized":"2023-09-23T18:19:33.369484+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":117,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390279+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:19:34.390279+0000","last_peered":"2023-09-23T18:19:34.390279+0000","last_clean":"2023-09-23T18:19:34.390279+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:19:34.390279+0000","last_undegraded":"2023-09-23T18:19:34.390279+0000","last_fullsized":"2023-09-23T18:19:34.390279+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"607'7063","reported_seq":7114,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:46.207683+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:19:46.207683+0000","last_peered":"2023-09-23T18:19:46.207683+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:19:46.207683+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"584'2600","ondisk_log_start":"584'2600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4463,"log_dups_size":0,"ondisk_log_size":4463,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":368902144,"num_objects":5629,"num_object_clones":0,"num_object_copies":16887,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5629,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7063,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":3431},{"shards":"5,6","objects":1481}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":726,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.408314+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:19:34.408314+0000","last_peered":"2023-09-23T18:19:34.408314+0000","last_clean":"2023-09-23T18:19:34.408314+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:19:34.408314+0000","last_undegraded":"2023-09-23T18:19:34.408314+0000","last_fullsized":"2023-09-23T18:19:34.408314+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":243,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.370649+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:19:33.370649+0000","last_peered":"2023-09-23T18:19:33.370649+0000","last_clean":"2023-09-23T18:19:33.370649+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:19:33.370649+0000","last_undegraded":"2023-09-23T18:19:33.370649+0000","last_fullsized":"2023-09-23T18:19:33.370649+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":222,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.369897+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:19:33.369897+0000","last_peered":"2023-09-23T18:19:33.369897+0000","last_clean":"2023-09-23T18:19:33.369897+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:19:33.369897+0000","last_undegraded":"2023-09-23T18:19:33.369897+0000","last_fullsized":"2023-09-23T18:19:33.369897+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"607'7123","reported_seq":14124,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:45.201270+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:19:45.201270+0000","last_peered":"2023-09-23T18:19:45.201270+0000","last_clean":"2023-09-23T18:19:45.201270+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:19:45.201270+0000","last_undegraded":"2023-09-23T18:19:45.201270+0000","last_fullsized":"2023-09-23T18:19:45.201270+0000","mapping_epoch":605,"log_start":"584'2200","ondisk_log_start":"584'2200","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4923,"log_dups_size":0,"ondisk_log_size":4923,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":374931456,"num_objects":5721,"num_object_clones":0,"num_object_copies":17163,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5721,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7123,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":622,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401597+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:19:34.401597+0000","last_peered":"2023-09-23T18:19:34.401597+0000","last_clean":"2023-09-23T18:19:34.401597+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:19:34.401597+0000","last_undegraded":"2023-09-23T18:19:34.401597+0000","last_fullsized":"2023-09-23T18:19:34.401597+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":532,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390898+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:19:34.390898+0000","last_peered":"2023-09-23T18:19:34.390898+0000","last_clean":"2023-09-23T18:19:34.390898+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:19:34.390898+0000","last_undegraded":"2023-09-23T18:19:34.390898+0000","last_fullsized":"2023-09-23T18:19:34.390898+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":183,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401899+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:19:34.401899+0000","last_peered":"2023-09-23T18:19:34.401899+0000","last_clean":"2023-09-23T18:19:34.401899+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:19:34.401899+0000","last_undegraded":"2023-09-23T18:19:34.401899+0000","last_fullsized":"2023-09-23T18:19:34.401899+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"607'7307","reported_seq":7349,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:46.094323+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:19:46.094323+0000","last_peered":"2023-09-23T18:19:46.094323+0000","last_clean":"2023-09-23T18:19:46.094323+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:19:46.094323+0000","last_undegraded":"2023-09-23T18:19:46.094323+0000","last_fullsized":"2023-09-23T18:19:46.094323+0000","mapping_epoch":574,"log_start":"584'2200","ondisk_log_start":"584'2200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5107,"log_dups_size":0,"ondisk_log_size":5107,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379125760,"num_objects":5785,"num_object_clones":0,"num_object_copies":17355,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5785,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7307,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":637,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.399888+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:19:34.399888+0000","last_peered":"2023-09-23T18:19:34.399888+0000","last_clean":"2023-09-23T18:19:34.399888+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:19:34.399888+0000","last_undegraded":"2023-09-23T18:19:34.399888+0000","last_fullsized":"2023-09-23T18:19:34.399888+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":131,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390904+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:19:34.390904+0000","last_peered":"2023-09-23T18:19:34.390904+0000","last_clean":"2023-09-23T18:19:34.390904+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:19:34.390904+0000","last_undegraded":"2023-09-23T18:19:34.390904+0000","last_fullsized":"2023-09-23T18:19:34.390904+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":443,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396600+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:19:34.396600+0000","last_peered":"2023-09-23T18:19:34.396600+0000","last_clean":"2023-09-23T18:19:34.396600+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:34.396600+0000","last_undegraded":"2023-09-23T18:19:34.396600+0000","last_fullsized":"2023-09-23T18:19:34.396600+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"607'7114","reported_seq":7156,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:43.837054+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:19:43.837054+0000","last_peered":"2023-09-23T18:19:43.837054+0000","last_clean":"2023-09-23T18:19:43.837054+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:19:43.837054+0000","last_undegraded":"2023-09-23T18:19:43.837054+0000","last_fullsized":"2023-09-23T18:19:43.837054+0000","mapping_epoch":574,"log_start":"585'2700","ondisk_log_start":"585'2700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4414,"log_dups_size":0,"ondisk_log_size":4414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":388628480,"num_objects":5930,"num_object_clones":0,"num_object_copies":17790,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5930,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7114,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1089,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390522+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:19:34.390522+0000","last_peered":"2023-09-23T18:19:34.390522+0000","last_clean":"2023-09-23T18:19:34.390522+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:19:34.390522+0000","last_undegraded":"2023-09-23T18:19:34.390522+0000","last_fullsized":"2023-09-23T18:19:34.390522+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":343,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398581+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:19:34.398581+0000","last_peered":"2023-09-23T18:19:34.398581+0000","last_clean":"2023-09-23T18:19:34.398581+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:19:34.398581+0000","last_undegraded":"2023-09-23T18:19:34.398581+0000","last_fullsized":"2023-09-23T18:19:34.398581+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":151,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401037+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:19:34.401037+0000","last_peered":"2023-09-23T18:19:34.401037+0000","last_clean":"2023-09-23T18:19:34.401037+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:19:34.401037+0000","last_undegraded":"2023-09-23T18:19:34.401037+0000","last_fullsized":"2023-09-23T18:19:34.401037+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"607'7253","reported_seq":7303,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:46.089099+0000","last_change":"2023-09-23T18:18:47.594759+0000","last_active":"2023-09-23T18:19:46.089099+0000","last_peered":"2023-09-23T18:19:46.089099+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:19:46.089099+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"583'1700","ondisk_log_start":"583'1700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5553,"log_dups_size":0,"ondisk_log_size":5553,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378601472,"num_objects":5777,"num_object_clones":0,"num_object_copies":17331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7253,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3488},{"shards":"4,5","objects":1551}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":876,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391152+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:19:34.391152+0000","last_peered":"2023-09-23T18:19:34.391152+0000","last_clean":"2023-09-23T18:19:34.391152+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:19:34.391152+0000","last_undegraded":"2023-09-23T18:19:34.391152+0000","last_fullsized":"2023-09-23T18:19:34.391152+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":159,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401810+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:19:34.401810+0000","last_peered":"2023-09-23T18:19:34.401810+0000","last_clean":"2023-09-23T18:19:34.401810+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:19:34.401810+0000","last_undegraded":"2023-09-23T18:19:34.401810+0000","last_fullsized":"2023-09-23T18:19:34.401810+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":458,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401997+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:19:34.401997+0000","last_peered":"2023-09-23T18:19:34.401997+0000","last_clean":"2023-09-23T18:19:34.401997+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:19:34.401997+0000","last_undegraded":"2023-09-23T18:19:34.401997+0000","last_fullsized":"2023-09-23T18:19:34.401997+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"607'7269","reported_seq":7310,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:46.191995+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:19:46.191995+0000","last_peered":"2023-09-23T18:19:46.191995+0000","last_clean":"2023-09-23T18:19:46.191995+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:19:46.191995+0000","last_undegraded":"2023-09-23T18:19:46.191995+0000","last_fullsized":"2023-09-23T18:19:46.191995+0000","mapping_epoch":574,"log_start":"585'2800","ondisk_log_start":"585'2800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4469,"log_dups_size":0,"ondisk_log_size":4469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378994688,"num_objects":5783,"num_object_clones":0,"num_object_copies":17349,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5783,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7269,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":878,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390700+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:19:34.390700+0000","last_peered":"2023-09-23T18:19:34.390700+0000","last_clean":"2023-09-23T18:19:34.390700+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:19:34.390700+0000","last_undegraded":"2023-09-23T18:19:34.390700+0000","last_fullsized":"2023-09-23T18:19:34.390700+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":443,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390352+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:19:34.390352+0000","last_peered":"2023-09-23T18:19:34.390352+0000","last_clean":"2023-09-23T18:19:34.390352+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:19:34.390352+0000","last_undegraded":"2023-09-23T18:19:34.390352+0000","last_fullsized":"2023-09-23T18:19:34.390352+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":219,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401292+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:19:34.401292+0000","last_peered":"2023-09-23T18:19:34.401292+0000","last_clean":"2023-09-23T18:19:34.401292+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:19:34.401292+0000","last_undegraded":"2023-09-23T18:19:34.401292+0000","last_fullsized":"2023-09-23T18:19:34.401292+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"606'7093","reported_seq":7133,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:42.327100+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:19:42.327100+0000","last_peered":"2023-09-23T18:19:42.327100+0000","last_clean":"2023-09-23T18:19:42.327100+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:19:42.327100+0000","last_undegraded":"2023-09-23T18:19:42.327100+0000","last_fullsized":"2023-09-23T18:19:42.327100+0000","mapping_epoch":574,"log_start":"584'2400","ondisk_log_start":"584'2400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4693,"log_dups_size":0,"ondisk_log_size":4693,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":389349376,"num_objects":5941,"num_object_clones":0,"num_object_copies":17823,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5941,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7093,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":859,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390649+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:19:34.390649+0000","last_peered":"2023-09-23T18:19:34.390649+0000","last_clean":"2023-09-23T18:19:34.390649+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:19:34.390649+0000","last_undegraded":"2023-09-23T18:19:34.390649+0000","last_fullsized":"2023-09-23T18:19:34.390649+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.397986+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:19:34.397986+0000","last_peered":"2023-09-23T18:19:34.397986+0000","last_clean":"2023-09-23T18:19:34.397986+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:19:34.397986+0000","last_undegraded":"2023-09-23T18:19:34.397986+0000","last_fullsized":"2023-09-23T18:19:34.397986+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":335,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.395636+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:19:34.395636+0000","last_peered":"2023-09-23T18:19:34.395636+0000","last_clean":"2023-09-23T18:19:34.395636+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:19:34.395636+0000","last_undegraded":"2023-09-23T18:19:34.395636+0000","last_fullsized":"2023-09-23T18:19:34.395636+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"607'7206","reported_seq":7250,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:45.190054+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:19:45.190054+0000","last_peered":"2023-09-23T18:19:45.190054+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:19:45.190054+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"584'2300","ondisk_log_start":"584'2300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4906,"log_dups_size":0,"ondisk_log_size":4906,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":384172032,"num_objects":5862,"num_object_clones":0,"num_object_copies":17586,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5862,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7206,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":1577},{"shards":"0,4,6","objects":3613}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":718,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401768+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:19:34.401768+0000","last_peered":"2023-09-23T18:19:34.401768+0000","last_clean":"2023-09-23T18:19:34.401768+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:19:34.401768+0000","last_undegraded":"2023-09-23T18:19:34.401768+0000","last_fullsized":"2023-09-23T18:19:34.401768+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":957,"reported_epoch":606,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:34.394940+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:19:34.394940+0000","last_peered":"2023-09-23T18:19:34.394940+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:19:34.394940+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":141,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391024+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:19:34.391024+0000","last_peered":"2023-09-23T18:19:34.391024+0000","last_clean":"2023-09-23T18:19:34.391024+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:19:34.391024+0000","last_undegraded":"2023-09-23T18:19:34.391024+0000","last_fullsized":"2023-09-23T18:19:34.391024+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":121,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390568+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:19:34.390568+0000","last_peered":"2023-09-23T18:19:34.390568+0000","last_clean":"2023-09-23T18:19:34.390568+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:19:34.390568+0000","last_undegraded":"2023-09-23T18:19:34.390568+0000","last_fullsized":"2023-09-23T18:19:34.390568+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"607'7166","reported_seq":7216,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:46.224018+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:19:46.224018+0000","last_peered":"2023-09-23T18:19:46.224018+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:19:46.224018+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"585'2700","ondisk_log_start":"585'2700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4466,"log_dups_size":0,"ondisk_log_size":4466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":378142720,"num_objects":5770,"num_object_clones":0,"num_object_copies":17310,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5770,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7166,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":2860},{"shards":"5,6","objects":2211}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":790,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390698+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:19:34.390698+0000","last_peered":"2023-09-23T18:19:34.390698+0000","last_clean":"2023-09-23T18:19:34.390698+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:19:34.390698+0000","last_undegraded":"2023-09-23T18:19:34.390698+0000","last_fullsized":"2023-09-23T18:19:34.390698+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1491,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390535+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:19:34.390535+0000","last_peered":"2023-09-23T18:19:34.390535+0000","last_clean":"2023-09-23T18:19:34.390535+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:19:34.390535+0000","last_undegraded":"2023-09-23T18:19:34.390535+0000","last_fullsized":"2023-09-23T18:19:34.390535+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":131,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401303+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:19:34.401303+0000","last_peered":"2023-09-23T18:19:34.401303+0000","last_clean":"2023-09-23T18:19:34.401303+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:19:34.401303+0000","last_undegraded":"2023-09-23T18:19:34.401303+0000","last_fullsized":"2023-09-23T18:19:34.401303+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":431,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401192+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:19:34.401192+0000","last_peered":"2023-09-23T18:19:34.401192+0000","last_clean":"2023-09-23T18:19:34.401192+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:19:34.401192+0000","last_undegraded":"2023-09-23T18:19:34.401192+0000","last_fullsized":"2023-09-23T18:19:34.401192+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"607'7046","reported_seq":7086,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:46.121572+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:19:46.121572+0000","last_peered":"2023-09-23T18:19:46.121572+0000","last_clean":"2023-09-23T18:19:46.121572+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:19:46.121572+0000","last_undegraded":"2023-09-23T18:19:46.121572+0000","last_fullsized":"2023-09-23T18:19:46.121572+0000","mapping_epoch":574,"log_start":"583'1500","ondisk_log_start":"583'1500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5546,"log_dups_size":0,"ondisk_log_size":5546,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":363069440,"num_objects":5540,"num_object_clones":0,"num_object_copies":16620,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5540,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7046,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":815,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390661+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:19:34.390661+0000","last_peered":"2023-09-23T18:19:34.390661+0000","last_clean":"2023-09-23T18:19:34.390661+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:19:34.390661+0000","last_undegraded":"2023-09-23T18:19:34.390661+0000","last_fullsized":"2023-09-23T18:19:34.390661+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":21,"reported_epoch":606,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:19:34.398494+0000","last_change":"2023-09-23T18:18:46.432707+0000","last_active":"2023-09-23T18:19:34.398494+0000","last_peered":"2023-09-23T18:19:34.398494+0000","last_clean":"2023-09-23T18:11:33.614822+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:19:34.398494+0000","last_undegraded":"2023-09-23T18:18:46.418246+0000","last_fullsized":"2023-09-23T18:19:34.398494+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":400,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401031+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:19:34.401031+0000","last_peered":"2023-09-23T18:19:34.401031+0000","last_clean":"2023-09-23T18:19:34.401031+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:19:34.401031+0000","last_undegraded":"2023-09-23T18:19:34.401031+0000","last_fullsized":"2023-09-23T18:19:34.401031+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":143,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391087+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:19:34.391087+0000","last_peered":"2023-09-23T18:19:34.391087+0000","last_clean":"2023-09-23T18:19:34.391087+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:19:34.391087+0000","last_undegraded":"2023-09-23T18:19:34.391087+0000","last_fullsized":"2023-09-23T18:19:34.391087+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"607'7164","reported_seq":7205,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:46.164606+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:19:46.164606+0000","last_peered":"2023-09-23T18:19:46.164606+0000","last_clean":"2023-09-23T18:19:46.164606+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:19:46.164606+0000","last_undegraded":"2023-09-23T18:19:46.164606+0000","last_fullsized":"2023-09-23T18:19:46.164606+0000","mapping_epoch":574,"log_start":"584'2700","ondisk_log_start":"584'2700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4464,"log_dups_size":0,"ondisk_log_size":4464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":379060224,"num_objects":5784,"num_object_clones":0,"num_object_copies":17352,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5784,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7164,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1504,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390521+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:19:34.390521+0000","last_peered":"2023-09-23T18:19:34.390521+0000","last_clean":"2023-09-23T18:19:34.390521+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:19:34.390521+0000","last_undegraded":"2023-09-23T18:19:34.390521+0000","last_fullsized":"2023-09-23T18:19:34.390521+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":557,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400941+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:19:34.400941+0000","last_peered":"2023-09-23T18:19:34.400941+0000","last_clean":"2023-09-23T18:19:34.400941+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:19:34.400941+0000","last_undegraded":"2023-09-23T18:19:34.400941+0000","last_fullsized":"2023-09-23T18:19:34.400941+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390424+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:19:34.390424+0000","last_peered":"2023-09-23T18:19:34.390424+0000","last_clean":"2023-09-23T18:19:34.390424+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:19:34.390424+0000","last_undegraded":"2023-09-23T18:19:34.390424+0000","last_fullsized":"2023-09-23T18:19:34.390424+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":152,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.402349+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:19:34.402349+0000","last_peered":"2023-09-23T18:19:34.402349+0000","last_clean":"2023-09-23T18:19:34.402349+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:19:34.402349+0000","last_undegraded":"2023-09-23T18:19:34.402349+0000","last_fullsized":"2023-09-23T18:19:34.402349+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"607'7098","reported_seq":7139,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:46.082440+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:19:46.082440+0000","last_peered":"2023-09-23T18:19:46.082440+0000","last_clean":"2023-09-23T18:19:46.082440+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:19:46.082440+0000","last_undegraded":"2023-09-23T18:19:46.082440+0000","last_fullsized":"2023-09-23T18:19:46.082440+0000","mapping_epoch":574,"log_start":"583'1600","ondisk_log_start":"583'1600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5498,"log_dups_size":0,"ondisk_log_size":5498,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":372637696,"num_objects":5686,"num_object_clones":0,"num_object_copies":17058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5686,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7098,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":805,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401376+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:19:34.401376+0000","last_peered":"2023-09-23T18:19:34.401376+0000","last_clean":"2023-09-23T18:19:34.401376+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:19:34.401376+0000","last_undegraded":"2023-09-23T18:19:34.401376+0000","last_fullsized":"2023-09-23T18:19:34.401376+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":120,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398117+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:19:34.398117+0000","last_peered":"2023-09-23T18:19:34.398117+0000","last_clean":"2023-09-23T18:19:34.398117+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:19:34.398117+0000","last_undegraded":"2023-09-23T18:19:34.398117+0000","last_fullsized":"2023-09-23T18:19:34.398117+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398366+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:19:34.398366+0000","last_peered":"2023-09-23T18:19:34.398366+0000","last_clean":"2023-09-23T18:19:34.398366+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:19:34.398366+0000","last_undegraded":"2023-09-23T18:19:34.398366+0000","last_fullsized":"2023-09-23T18:19:34.398366+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":622,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390958+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:19:34.390958+0000","last_peered":"2023-09-23T18:19:34.390958+0000","last_clean":"2023-09-23T18:19:34.390958+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:19:34.390958+0000","last_undegraded":"2023-09-23T18:19:34.390958+0000","last_fullsized":"2023-09-23T18:19:34.390958+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"607'7212","reported_seq":8475,"reported_epoch":607,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:46.224007+0000","last_change":"2023-09-23T18:19:32.858380+0000","last_active":"2023-09-23T18:19:46.224007+0000","last_peered":"2023-09-23T18:19:46.224007+0000","last_clean":"2023-09-23T18:18:32.699126+0000","last_became_active":"2023-09-23T18:18:34.716591+0000","last_became_peered":"2023-09-23T18:18:34.716591+0000","last_unstale":"2023-09-23T18:19:46.224007+0000","last_undegraded":"2023-09-23T18:18:34.351246+0000","last_fullsized":"2023-09-23T18:18:34.327168+0000","mapping_epoch":589,"log_start":"585'2800","ondisk_log_start":"585'2800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4412,"log_dups_size":0,"ondisk_log_size":4412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:13.611234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":376504344,"num_objects":5746,"num_object_clones":0,"num_object_copies":17238,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":129,"num_objects_misplaced":2719,"num_objects_unfound":0,"num_objects_dirty":5746,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7212,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":605,"num_bytes_recovered":39649280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":2719},{"shards":"4,6","objects":2898}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":905,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401452+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:19:34.401452+0000","last_peered":"2023-09-23T18:19:34.401452+0000","last_clean":"2023-09-23T18:19:34.401452+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:19:34.401452+0000","last_undegraded":"2023-09-23T18:19:34.401452+0000","last_fullsized":"2023-09-23T18:19:34.401452+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":45,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396087+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:19:34.396087+0000","last_peered":"2023-09-23T18:19:34.396087+0000","last_clean":"2023-09-23T18:19:34.396087+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:19:34.396087+0000","last_undegraded":"2023-09-23T18:19:34.396087+0000","last_fullsized":"2023-09-23T18:19:34.396087+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":131,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390610+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:19:34.390610+0000","last_peered":"2023-09-23T18:19:34.390610+0000","last_clean":"2023-09-23T18:19:34.390610+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:19:34.390610+0000","last_undegraded":"2023-09-23T18:19:34.390610+0000","last_fullsized":"2023-09-23T18:19:34.390610+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":131,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390447+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:19:34.390447+0000","last_peered":"2023-09-23T18:19:34.390447+0000","last_clean":"2023-09-23T18:19:34.390447+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:19:34.390447+0000","last_undegraded":"2023-09-23T18:19:34.390447+0000","last_fullsized":"2023-09-23T18:19:34.390447+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"607'7178","reported_seq":7219,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:46.202068+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:19:46.202068+0000","last_peered":"2023-09-23T18:19:46.202068+0000","last_clean":"2023-09-23T18:19:46.202068+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:19:46.202068+0000","last_undegraded":"2023-09-23T18:19:46.202068+0000","last_fullsized":"2023-09-23T18:19:46.202068+0000","mapping_epoch":574,"log_start":"585'2700","ondisk_log_start":"585'2700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4478,"log_dups_size":0,"ondisk_log_size":4478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371982336,"num_objects":5676,"num_object_clones":0,"num_object_copies":17028,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5676,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7178,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":905,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.402088+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:19:34.402088+0000","last_peered":"2023-09-23T18:19:34.402088+0000","last_clean":"2023-09-23T18:19:34.402088+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:19:34.402088+0000","last_undegraded":"2023-09-23T18:19:34.402088+0000","last_fullsized":"2023-09-23T18:19:34.402088+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":93,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391005+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:19:34.391005+0000","last_peered":"2023-09-23T18:19:34.391005+0000","last_clean":"2023-09-23T18:19:34.391005+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:19:34.391005+0000","last_undegraded":"2023-09-23T18:19:34.391005+0000","last_fullsized":"2023-09-23T18:19:34.391005+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":386,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401402+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:19:34.401402+0000","last_peered":"2023-09-23T18:19:34.401402+0000","last_clean":"2023-09-23T18:19:34.401402+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:19:34.401402+0000","last_undegraded":"2023-09-23T18:19:34.401402+0000","last_fullsized":"2023-09-23T18:19:34.401402+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.397898+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:34.397898+0000","last_peered":"2023-09-23T18:19:34.397898+0000","last_clean":"2023-09-23T18:19:34.397898+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:19:34.397898+0000","last_undegraded":"2023-09-23T18:19:34.397898+0000","last_fullsized":"2023-09-23T18:19:34.397898+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"607'7142","reported_seq":7183,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:46.225404+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:19:46.225404+0000","last_peered":"2023-09-23T18:19:46.225404+0000","last_clean":"2023-09-23T18:19:46.225404+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:19:46.225404+0000","last_undegraded":"2023-09-23T18:19:46.225404+0000","last_fullsized":"2023-09-23T18:19:46.225404+0000","mapping_epoch":574,"log_start":"585'2700","ondisk_log_start":"585'2700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4442,"log_dups_size":0,"ondisk_log_size":4442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":371589120,"num_objects":5670,"num_object_clones":0,"num_object_copies":17010,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5670,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7142,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":739,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398646+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:19:34.398646+0000","last_peered":"2023-09-23T18:19:34.398646+0000","last_clean":"2023-09-23T18:19:34.398646+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:19:34.398646+0000","last_undegraded":"2023-09-23T18:19:34.398646+0000","last_fullsized":"2023-09-23T18:19:34.398646+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1043,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401980+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:19:34.401980+0000","last_peered":"2023-09-23T18:19:34.401980+0000","last_clean":"2023-09-23T18:19:34.401980+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:19:34.401980+0000","last_undegraded":"2023-09-23T18:19:34.401980+0000","last_fullsized":"2023-09-23T18:19:34.401980+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":33,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396809+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:19:34.396809+0000","last_peered":"2023-09-23T18:19:34.396809+0000","last_clean":"2023-09-23T18:19:34.396809+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:19:34.396809+0000","last_undegraded":"2023-09-23T18:19:34.396809+0000","last_fullsized":"2023-09-23T18:19:34.396809+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":397,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390572+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:19:34.390572+0000","last_peered":"2023-09-23T18:19:34.390572+0000","last_clean":"2023-09-23T18:19:34.390572+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:19:34.390572+0000","last_undegraded":"2023-09-23T18:19:34.390572+0000","last_fullsized":"2023-09-23T18:19:34.390572+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":637,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400969+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:19:34.400969+0000","last_peered":"2023-09-23T18:19:34.400969+0000","last_clean":"2023-09-23T18:19:34.400969+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:19:34.400969+0000","last_undegraded":"2023-09-23T18:19:34.400969+0000","last_fullsized":"2023-09-23T18:19:34.400969+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":29,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396456+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:19:34.396456+0000","last_peered":"2023-09-23T18:19:34.396456+0000","last_clean":"2023-09-23T18:19:34.396456+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:19:34.396456+0000","last_undegraded":"2023-09-23T18:19:34.396456+0000","last_fullsized":"2023-09-23T18:19:34.396456+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":679,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.394809+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:19:34.394809+0000","last_peered":"2023-09-23T18:19:34.394809+0000","last_clean":"2023-09-23T18:19:34.394809+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:19:34.394809+0000","last_undegraded":"2023-09-23T18:19:34.394809+0000","last_fullsized":"2023-09-23T18:19:34.394809+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":637,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401338+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:19:34.401338+0000","last_peered":"2023-09-23T18:19:34.401338+0000","last_clean":"2023-09-23T18:19:34.401338+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:19:34.401338+0000","last_undegraded":"2023-09-23T18:19:34.401338+0000","last_fullsized":"2023-09-23T18:19:34.401338+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":125,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390701+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:19:34.390701+0000","last_peered":"2023-09-23T18:19:34.390701+0000","last_clean":"2023-09-23T18:19:34.390701+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:19:34.390701+0000","last_undegraded":"2023-09-23T18:19:34.390701+0000","last_fullsized":"2023-09-23T18:19:34.390701+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":240,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.369273+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:19:33.369273+0000","last_peered":"2023-09-23T18:19:33.369273+0000","last_clean":"2023-09-23T18:19:33.369273+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:19:33.369273+0000","last_undegraded":"2023-09-23T18:19:33.369273+0000","last_fullsized":"2023-09-23T18:19:33.369273+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":111,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390943+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:19:34.390943+0000","last_peered":"2023-09-23T18:19:34.390943+0000","last_clean":"2023-09-23T18:19:34.390943+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:19:34.390943+0000","last_undegraded":"2023-09-23T18:19:34.390943+0000","last_fullsized":"2023-09-23T18:19:34.390943+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":223,"reported_epoch":605,"state":"active+clean","last_fresh":"2023-09-23T18:19:33.369248+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:19:33.369248+0000","last_peered":"2023-09-23T18:19:33.369248+0000","last_clean":"2023-09-23T18:19:33.369248+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:19:33.369248+0000","last_undegraded":"2023-09-23T18:19:33.369248+0000","last_fullsized":"2023-09-23T18:19:33.369248+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":999,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400953+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:19:34.400953+0000","last_peered":"2023-09-23T18:19:34.400953+0000","last_clean":"2023-09-23T18:19:34.400953+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:19:34.400953+0000","last_undegraded":"2023-09-23T18:19:34.400953+0000","last_fullsized":"2023-09-23T18:19:34.400953+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":504,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401453+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:19:34.401453+0000","last_peered":"2023-09-23T18:19:34.401453+0000","last_clean":"2023-09-23T18:19:34.401453+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:19:34.401453+0000","last_undegraded":"2023-09-23T18:19:34.401453+0000","last_fullsized":"2023-09-23T18:19:34.401453+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":141,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390651+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:19:34.390651+0000","last_peered":"2023-09-23T18:19:34.390651+0000","last_clean":"2023-09-23T18:19:34.390651+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:19:34.390651+0000","last_undegraded":"2023-09-23T18:19:34.390651+0000","last_fullsized":"2023-09-23T18:19:34.390651+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":935,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.395092+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:19:34.395092+0000","last_peered":"2023-09-23T18:19:34.395092+0000","last_clean":"2023-09-23T18:19:34.395092+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:19:34.395092+0000","last_undegraded":"2023-09-23T18:19:34.395092+0000","last_fullsized":"2023-09-23T18:19:34.395092+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":129,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391058+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:19:34.391058+0000","last_peered":"2023-09-23T18:19:34.391058+0000","last_clean":"2023-09-23T18:19:34.391058+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:19:34.391058+0000","last_undegraded":"2023-09-23T18:19:34.391058+0000","last_fullsized":"2023-09-23T18:19:34.391058+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":650,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400629+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:19:34.400629+0000","last_peered":"2023-09-23T18:19:34.400629+0000","last_clean":"2023-09-23T18:19:34.400629+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:19:34.400629+0000","last_undegraded":"2023-09-23T18:19:34.400629+0000","last_fullsized":"2023-09-23T18:19:34.400629+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":6035800088,"num_objects":92100,"num_object_clones":0,"num_object_copies":276300,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16347,"num_objects_misplaced":2719,"num_objects_unfound":0,"num_objects_dirty":92100,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":114708,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4076,"num_bytes_recovered":267124736,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":18898104320,"data_stored":18898092104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77308,"ondisk_log_size":77308,"up":48,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001565,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669502,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3445552,"kb_used_data":2849832,"kb_used_omap":7,"kb_used_meta":595704,"kb_avail":90299600,"statfs":{"total":95995035648,"available":92466790400,"internally_reserved":0,"allocated":2918227968,"data_stored":2916313531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":610001700},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,3],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":1.599,"15min":1.2949999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":24.472999999999999,"5min":24.472999999999999,"15min":24.472999999999999},"last":7.6580000000000004},{"interface":"front","average":{"1min":3.6230000000000002,"5min":2.081,"15min":1.663},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":22.832999999999998,"5min":22.832999999999998,"15min":22.832999999999998},"last":8.2029999999999994}]},{"osd":1,"last update":"Sat Sep 23 18:19:21 2023","interfaces":[{"interface":"back","average":{"1min":4.1740000000000004,"5min":1.8069999999999999,"15min":1.413},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":22.972000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.38200000000000001},{"interface":"front","average":{"1min":3.448,"5min":1.6559999999999999,"15min":1.357},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":22.664000000000001,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.41999999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.403,"15min":1.2869999999999999},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.2679999999999998,"5min":7.4189999999999996,"15min":7.4189999999999996},"last":0.40300000000000002},{"interface":"front","average":{"1min":1.927,"5min":1.7470000000000001,"15min":1.454},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0439999999999996,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.44800000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.6459999999999999,"15min":1.5509999999999999},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.3150000000000004,"5min":7.3940000000000001,"15min":10.257999999999999},"last":7.718},{"interface":"front","average":{"1min":2.282,"5min":2.165,"15min":1.829},"min":{"1min":0.28999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.9539999999999997,"5min":9.4049999999999994,"15min":9.5960000000000001},"last":7.5199999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.8140000000000001,"15min":1.3280000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.2800000000000002,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.7569999999999999,"5min":1.9330000000000001,"15min":1.3200000000000001},"min":{"1min":0.36099999999999999,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":6.2030000000000003,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.46200000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:18:42 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":2.0699999999999998,"15min":1.9930000000000001},"min":{"1min":0.36399999999999999,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":5.8179999999999996,"5min":13.502000000000001,"15min":13.502000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.1520000000000001,"5min":2.298,"15min":2.4420000000000002},"min":{"1min":0.46400000000000002,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.9080000000000004,"5min":10.081,"15min":10.081},"last":0.629}]},{"osd":7,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.1930000000000001,"15min":1.4970000000000001},"min":{"1min":0.45700000000000002,"5min":0.36299999999999999,"15min":0.26800000000000002},"max":{"1min":5.851,"5min":10.743,"15min":13.460000000000001},"last":0.64900000000000002},{"interface":"front","average":{"1min":1.702,"5min":2.21,"15min":1.536},"min":{"1min":0.46400000000000002,"5min":0.32900000000000001,"15min":0.28499999999999998},"max":{"1min":6.2409999999999997,"5min":11.244,"15min":12.404},"last":0.55700000000000005}]}]},{"osd":1,"up_from":545,"seq":2340757176350,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2377464,"kb_used_data":2165616,"kb_used_omap":13,"kb_used_meta":211826,"kb_avail":91367688,"statfs":{"total":95995035648,"available":93560512512,"internally_reserved":0,"allocated":2217590784,"data_stored":2214625229,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":216910477},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.776,"5min":0.66800000000000004,"15min":0.48299999999999998},"min":{"1min":0.221,"5min":0.189,"15min":0.189},"max":{"1min":5.8079999999999998,"5min":5.8079999999999998,"15min":5.8079999999999998},"last":1.8720000000000001},{"interface":"front","average":{"1min":1.2430000000000001,"5min":0.64000000000000001,"15min":0.54000000000000004},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6159999999999997,"5min":4.6159999999999997,"15min":4.6159999999999997},"last":0.82599999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":0.67900000000000005,"15min":0.53300000000000003},"min":{"1min":0.28299999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9290000000000003,"5min":4.9290000000000003,"15min":4.9290000000000003},"last":1.994},{"interface":"front","average":{"1min":1.272,"5min":0.65800000000000003,"15min":0.55600000000000005},"min":{"1min":0.29899999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2759999999999998,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":0.28599999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":0.746,"15min":0.50600000000000001},"min":{"1min":0.34300000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.3219999999999992,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":3.778},{"interface":"front","average":{"1min":2.2429999999999999,"5min":0.85799999999999998,"15min":0.627},"min":{"1min":0.38500000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":8.3659999999999997},"last":2.2549999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":1.347,"15min":1.327},"min":{"1min":0.40200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.6609999999999996,"5min":12.663,"15min":12.663},"last":0.34899999999999998},{"interface":"front","average":{"1min":1.95,"5min":1.601,"15min":1.5429999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.1769999999999996,"5min":13.222,"15min":13.222},"last":3.8180000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":0.88200000000000001,"15min":0.77600000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":0.32200000000000001},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.0589999999999999,"15min":0.88900000000000001},"min":{"1min":0.46000000000000002,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":8.3190000000000008,"5min":8.3190000000000008,"15min":8.3190000000000008},"last":1.913}]},{"osd":6,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":0.997,"15min":0.83899999999999997},"min":{"1min":0.45400000000000001,"5min":0.435,"15min":0.435},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":0.372},{"interface":"front","average":{"1min":2.6070000000000002,"5min":1.1419999999999999,"15min":0.89800000000000002},"min":{"1min":0.38900000000000001,"5min":0.378,"15min":0.378},"max":{"1min":9.2579999999999991,"5min":9.2579999999999991,"15min":9.2579999999999991},"last":3.871}]},{"osd":7,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":1.0109999999999999,"15min":0.79100000000000004},"min":{"1min":0.436,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":3.843},{"interface":"front","average":{"1min":2.6419999999999999,"5min":1.173,"15min":0.92800000000000005},"min":{"1min":0.48099999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":9.0640000000000001,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":1.9419999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958847,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4556448,"kb_used_data":3661080,"kb_used_omap":7,"kb_used_meta":895288,"kb_avail":89188704,"statfs":{"total":95995035648,"available":91329232896,"internally_reserved":0,"allocated":3748945920,"data_stored":3746121272,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":916775652},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:19:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7229999999999999,"5min":0.80000000000000004,"15min":0.47999999999999998},"min":{"1min":0.214,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.2309999999999999,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":6.4779999999999998},{"interface":"front","average":{"1min":2.2010000000000001,"5min":0.85799999999999998,"15min":0.63400000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.714,"5min":13.714,"15min":13.714},"last":2.0710000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:19:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":1.1519999999999999,"15min":1.1299999999999999},"min":{"1min":0.27000000000000002,"5min":0.185,"15min":0.185},"max":{"1min":18.550999999999998,"5min":18.550999999999998,"15min":18.550999999999998},"last":0.64700000000000002},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.84199999999999997,"15min":0.79600000000000004},"min":{"1min":0.29299999999999998,"5min":0.22,"15min":0.22},"max":{"1min":13.618,"5min":13.618,"15min":13.618},"last":2.105}]},{"osd":3,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7610000000000001,"5min":1.393,"15min":1.637},"min":{"1min":0.38400000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.539999999999999,"5min":13.539999999999999,"15min":13.539999999999999},"last":0.53400000000000003},{"interface":"front","average":{"1min":4.1390000000000002,"5min":1.4930000000000001,"15min":1.637},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.285,"5min":13.285,"15min":15.198},"last":0.58699999999999997}]},{"osd":4,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.2599999999999998,"15min":1.7789999999999999},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":13.574999999999999,"5min":57.746000000000002,"15min":57.746000000000002},"last":6.532},{"interface":"front","average":{"1min":2.032,"5min":1.9770000000000001,"15min":1.6499999999999999},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.8970000000000002,"5min":57.816000000000003,"15min":57.816000000000003},"last":0.623}]},{"osd":5,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.032,"15min":1.391},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":6.6059999999999999},{"interface":"front","average":{"1min":4.2859999999999996,"5min":2.2650000000000001,"15min":1.448},"min":{"1min":0.502,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":13.425000000000001},"last":1.579}]},{"osd":6,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":1.8049999999999999,"15min":1.861},"min":{"1min":0.433,"5min":0.374,"15min":0.32000000000000001},"max":{"1min":7.1020000000000003,"5min":7.1020000000000003,"15min":58.457999999999998},"last":1.355},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.798,"15min":1.7050000000000001},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":13.672000000000001,"5min":13.672000000000001,"15min":58.354999999999997},"last":5.1299999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.6719999999999999,"15min":1.488},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":3.1459999999999999,"5min":8.5009999999999994,"15min":10.416},"last":2.8010000000000002},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.5609999999999999,"15min":1.6759999999999999},"min":{"1min":0.48199999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.6139999999999999,"5min":10.404,"15min":10.404},"last":0.56399999999999995}]}]},{"osd":2,"up_from":0,"seq":2070174236732,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249362,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2603444,"kb_used_data":1904684,"kb_used_omap":3,"kb_used_meta":698748,"kb_avail":91141708,"statfs":{"total":95995035648,"available":93329108992,"internally_reserved":0,"allocated":1950396416,"data_stored":1947377070,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":715518696},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,3,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":0.70399999999999996,"5min":2.407,"15min":1.508},"min":{"1min":0.20499999999999999,"5min":0.19800000000000001,"15min":0.183},"max":{"1min":4.0369999999999999,"5min":195.69800000000001,"15min":195.69800000000001},"last":2.6080000000000001},{"interface":"front","average":{"1min":0.97999999999999998,"5min":2.4470000000000001,"15min":1.8049999999999999},"min":{"1min":0.22600000000000001,"5min":0.20999999999999999,"15min":0.20699999999999999},"max":{"1min":6.1340000000000003,"5min":194.661,"15min":194.661},"last":4.4550000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":0.95799999999999996,"15min":0.66600000000000004},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":21.777000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.52100000000000002},{"interface":"front","average":{"1min":6.5110000000000001,"5min":1.8660000000000001,"15min":1.0920000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":78.980000000000004,"5min":78.980000000000004,"15min":78.980000000000004},"last":2.0270000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.94299999999999995,"15min":1.179},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.597,"5min":6.3849999999999998,"15min":6.3849999999999998},"last":0.45000000000000001},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.93100000000000005,"15min":1.21},"min":{"1min":0.33100000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":4.1040000000000001,"5min":5.931,"15min":5.931},"last":2.1379999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.8089999999999999,"5min":1.4650000000000001,"15min":1.5669999999999999},"min":{"1min":0.40000000000000002,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":20.850000000000001,"5min":21.062999999999999,"15min":37.231000000000002},"last":4.4870000000000001},{"interface":"front","average":{"1min":2.0899999999999999,"5min":1.833,"15min":1.79},"min":{"1min":0.40300000000000002,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":20.786000000000001,"5min":21.128,"15min":37.381},"last":0.60399999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:19:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.8009999999999999,"15min":1.4419999999999999},"min":{"1min":0.40999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":11.805999999999999,"5min":11.805999999999999,"15min":11.805999999999999},"last":0.54900000000000004},{"interface":"front","average":{"1min":2.028,"5min":1.601,"15min":1.6339999999999999},"min":{"1min":0.44500000000000001,"5min":0.33100000000000002,"15min":0.32000000000000001},"max":{"1min":8.9359999999999999,"5min":8.9359999999999999,"15min":8.9969999999999999},"last":2.0859999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.135,"5min":1.4299999999999999,"15min":1.7110000000000001},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":3.8079999999999998,"5min":9.8089999999999993,"15min":115.54000000000001},"last":2.282},{"interface":"front","average":{"1min":1.0009999999999999,"5min":1.53,"15min":1.859},"min":{"1min":0.44900000000000001,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":3.8460000000000001,"5min":14.321,"15min":115.691},"last":7.6200000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:18:47 2023","interfaces":[{"interface":"back","average":{"1min":1.0960000000000001,"5min":1.4219999999999999,"15min":1.6319999999999999},"min":{"1min":0.436,"5min":0.36799999999999999,"15min":0.35799999999999998},"max":{"1min":3.5449999999999999,"5min":5.2809999999999997,"15min":15.972},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.0469999999999999,"5min":1.327,"15min":1.4550000000000001},"min":{"1min":0.50800000000000001,"5min":0.40699999999999997,"15min":0.34899999999999998},"max":{"1min":2.069,"5min":5.8570000000000002,"15min":9.9109999999999996},"last":2.7189999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019311,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5480740,"kb_used_data":4847388,"kb_used_omap":0,"kb_used_meta":633152,"kb_avail":88264412,"statfs":{"total":95995035648,"available":90382757888,"internally_reserved":0,"allocated":4963725312,"data_stored":4961814650,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":648347648},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,4,6],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":1.375,"15min":1.21},"min":{"1min":0.27200000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9660000000000002,"5min":7.2199999999999998,"15min":13.375999999999999},"last":0.438},{"interface":"front","average":{"1min":2.2709999999999999,"5min":1.772,"15min":1.4710000000000001},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":8.9429999999999996,"5min":10.890000000000001,"15min":10.890000000000001},"last":2.9020000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:19:22 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":1.47,"15min":0.95299999999999996},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":44.146999999999998,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.65300000000000002},{"interface":"front","average":{"1min":4.5419999999999998,"5min":1.4299999999999999,"15min":0.91100000000000003},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":43.497999999999998,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.56000000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.524,"15min":1.7689999999999999},"min":{"1min":0.30299999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.173,"5min":11.173,"15min":11.173},"last":5.359},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.6459999999999999,"15min":1.702},"min":{"1min":0.40600000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.0909999999999993,"5min":10.938000000000001,"15min":10.938000000000001},"last":0.96899999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.5669999999999999,"15min":1.9350000000000001},"min":{"1min":0.33700000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.0609999999999999,"5min":11.130000000000001,"15min":29.154},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.1800000000000002,"5min":1.8480000000000001,"15min":1.9390000000000001},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":13.470000000000001,"5min":13.470000000000001,"15min":29.216000000000001},"last":0.39200000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":2.4279999999999999,"15min":2.8559999999999999},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4950000000000001,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.318},{"interface":"front","average":{"1min":1.401,"5min":1.827,"15min":2.7109999999999999},"min":{"1min":0.30499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.1040000000000001,"5min":13.938000000000001,"15min":32.640000000000001},"last":0.77900000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":2.0739999999999998,"15min":1.391},"min":{"1min":0.379,"5min":0.28199999999999997,"15min":0.25},"max":{"1min":7.7249999999999996,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.72299999999999998},{"interface":"front","average":{"1min":1.782,"5min":2.044,"15min":1.51},"min":{"1min":0.33900000000000002,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.6310000000000002,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.69699999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":2.7919999999999998,"15min":1.7490000000000001},"min":{"1min":0.42699999999999999,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":8.9849999999999994,"5min":19.298999999999999,"15min":19.298999999999999},"last":1.1519999999999999},{"interface":"front","average":{"1min":2.234,"5min":2.1549999999999998,"15min":1.5569999999999999},"min":{"1min":0.50900000000000001,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.1319999999999997,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.52000000000000002}]}]},{"osd":5,"up_from":420,"seq":1803886264408,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3476192,"kb_used_data":3156760,"kb_used_omap":9,"kb_used_meta":319414,"kb_avail":90268960,"statfs":{"total":95995035648,"available":92435415040,"internally_reserved":0,"allocated":3232522240,"data_stored":3230619939,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":327080381},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":0.93799999999999994,"5min":1.2330000000000001,"15min":0.83799999999999997},"min":{"1min":0.20899999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.206,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.68400000000000005},{"interface":"front","average":{"1min":1.038,"5min":1.423,"15min":1.5189999999999999},"min":{"1min":0.23499999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":2.6309999999999998,"5min":7.5439999999999996,"15min":7.5439999999999996},"last":0.53100000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:19:20 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":1.4770000000000001,"15min":1.2230000000000001},"min":{"1min":0.32100000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.946000000000002,"5min":23.946000000000002,"15min":23.946000000000002},"last":3.5270000000000001},{"interface":"front","average":{"1min":3.927,"5min":1.4630000000000001,"15min":1.052},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":23.268999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":0.32200000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.4590000000000001,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.4930000000000003,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":0.59999999999999998},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.891,"15min":2.141},"min":{"1min":0.314,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.6749999999999998,"5min":7.2359999999999998,"15min":7.2359999999999998},"last":0.38500000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.1419999999999999,"5min":1.556,"15min":1.153},"min":{"1min":0.31,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":3.2330000000000001,"5min":8.7449999999999992,"15min":8.7449999999999992},"last":0.71099999999999997},{"interface":"front","average":{"1min":1.2769999999999999,"5min":1.534,"15min":1.5169999999999999},"min":{"1min":0.33500000000000002,"5min":0.313,"15min":0.313},"max":{"1min":3.8599999999999999,"5min":7.5179999999999998,"15min":7.5179999999999998},"last":0.69699999999999995}]},{"osd":4,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.6539999999999999,"5min":2.1440000000000001,"15min":2.0379999999999998},"min":{"1min":0.36499999999999999,"5min":0.254,"15min":0.251},"max":{"1min":5.375,"5min":12.747999999999999,"15min":12.747999999999999},"last":1.002},{"interface":"front","average":{"1min":1.514,"5min":1.9099999999999999,"15min":1.5580000000000001},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.069,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.51000000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.2509999999999999,"5min":1.867,"15min":1.1899999999999999},"min":{"1min":0.26400000000000001,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":4.1920000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.56100000000000005},{"interface":"front","average":{"1min":1.232,"5min":2.004,"15min":1.6220000000000001},"min":{"1min":0.35699999999999998,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":3.9649999999999999,"5min":12.504,"15min":12.504},"last":0.47799999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:18:45 2023","interfaces":[{"interface":"back","average":{"1min":1.3520000000000001,"5min":2.363,"15min":1.6970000000000001},"min":{"1min":0.36699999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":5.4240000000000004,"5min":13.506,"15min":13.506},"last":0.497},{"interface":"front","average":{"1min":1.2310000000000001,"5min":2.0310000000000001,"15min":1.5349999999999999},"min":{"1min":0.378,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":3.2610000000000001,"5min":13.138,"15min":13.138},"last":0.64700000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3711700992,"data_stored":3711696920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2195128320,"data_stored":2195128320,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1928003584,"data_stored":1928003584,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4937486336,"data_stored":4937482264,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3219980288,"data_stored":3219980288,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2905804800,"data_stored":2905800728,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:19:50.010 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:50.080 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:50.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:50.092 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:50.193 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:50.242 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:50.316 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:50.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:50.321 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:50.361 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:19:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:19:50] "GET /metrics HTTP/1.1" 200 36682 "" "Prometheus/2.43.0" 2023-09-23T18:19:50.421 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:50.463 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:50.528 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:50.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:50.532 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:50.560 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:50.626 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:50 smithi055 ceph-mon[179668]: pgmap v1442: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.6 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 16372/274710 objects degraded (5.960%); 2719/274710 objects misplaced (0.990%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:19:50.633 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:50.721 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:50.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:50 smithi099 ceph-mon[167076]: pgmap v1442: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.6 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 16372/274710 objects degraded (5.960%); 2719/274710 objects misplaced (0.990%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:19:50.842 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:50.855 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:50.855 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:50.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:50 smithi055 ceph-mon[184174]: pgmap v1442: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.6 GiB data, 21 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 879 op/s; 16372/274710 objects degraded (5.960%); 2719/274710 objects misplaced (0.990%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:19:50.956 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:51.018 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:51.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:51.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:51.085 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:51.173 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:19:51.174 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:19:51.186 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:19:51.259 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:51.379 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:19:51.391 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:51.391 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:19:51.492 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:51.530 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:51.566 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:51.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:51.569 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:51.653 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:19:51.670 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:51.699 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:51.734 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:51.736 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:51.737 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2180642160' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:51.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:51 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:51.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:51.837 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:51.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2180642160' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:51.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:51 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:51.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:51.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:51.899 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2180642160' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:51.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:51 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:19:51.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:19:51.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:19:51.901 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:52.298 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:52.303 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:52.304 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:52.404 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:52.463 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:52.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:52.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:52.536 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:52.637 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:52.696 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:52.762 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:52.766 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:52.767 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:52.769 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:52 smithi099 ceph-mon[167076]: pgmap v1443: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 858 op/s; 16372/269502 objects degraded (6.075%); 2719/269502 objects misplaced (1.009%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:19:52.769 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:52 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:52.769 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:52 smithi099 ceph-mon[167076]: failed to encode map e608 with expected crc 2023-09-23T18:19:52.769 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:52 smithi099 ceph-mon[167076]: failed to encode map e608 with expected crc 2023-09-23T18:19:52.769 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:52 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:19:52.769 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:52 smithi099 ceph-mon[167076]: osdmap e608: 8 total, 8 up, 6 in 2023-09-23T18:19:52.769 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:52 smithi099 ceph-mon[167076]: failed to encode map e608 with expected crc 2023-09-23T18:19:52.776 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:19:52.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[184174]: pgmap v1443: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 858 op/s; 16372/269502 objects degraded (6.075%); 2719/269502 objects misplaced (1.009%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:19:52.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:52.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[184174]: failed to encode map e608 with expected crc 2023-09-23T18:19:52.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[184174]: failed to encode map e608 with expected crc 2023-09-23T18:19:52.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:19:52.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[184174]: osdmap e608: 8 total, 8 up, 6 in 2023-09-23T18:19:52.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[184174]: failed to encode map e608 with expected crc 2023-09-23T18:19:52.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[179668]: pgmap v1443: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.5 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 858 op/s; 16372/269502 objects degraded (6.075%); 2719/269502 objects misplaced (1.009%); 2.6 MiB/s, 41 objects/s recovering 2023-09-23T18:19:52.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:52.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[179668]: failed to encode map e608 with expected crc 2023-09-23T18:19:52.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[179668]: failed to encode map e608 with expected crc 2023-09-23T18:19:52.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:19:52.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[179668]: osdmap e608: 8 total, 8 up, 6 in 2023-09-23T18:19:52.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:52 smithi055 ceph-mon[179668]: failed to encode map e608 with expected crc 2023-09-23T18:19:52.868 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:52.931 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:52.993 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:52.997 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:53.000 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:53.011 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:19:53.101 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:53.138 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:53.196 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:53.200 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:53.201 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:53.219 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:19:53.271 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:19:53.302 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:53.317 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:19:53.367 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:19:53.386 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:53.416 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:19:53.458 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:19:53.463 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:53.483 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:53.484 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:53.512 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:19:53.546 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:19:53.555 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:19:53.555 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:19:53.555 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:19:53.555 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:19:53.555 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:19:53.555 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:19:53.555 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:19:53.556 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:19:53.584 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:53.584 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:19:53.684 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:53.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/707827038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:53.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16372/269502 objects degraded (6.075%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:53.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/707827038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:53.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16372/269502 objects degraded (6.075%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:53.963 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:19:53.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:53.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:19:54.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/707827038' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:54.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16372/269502 objects degraded (6.075%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:54.104 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:54.135 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:54.185 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:54.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:54.200 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:54.301 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:54.342 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:54.419 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:54.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:54.426 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:54.446 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:19:54.527 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:54.620 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:54.877 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:54 smithi055 ceph-mon[184174]: pgmap v1445: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 893 op/s; 16444/261717 objects degraded (6.283%); 2432/261717 objects misplaced (0.929%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:19:54.878 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:54 smithi055 ceph-mon[179668]: pgmap v1445: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 893 op/s; 16444/261717 objects degraded (6.283%); 2432/261717 objects misplaced (0.929%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:19:54.980 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:54.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:54.998 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:55.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:54 smithi099 ceph-mon[167076]: pgmap v1445: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 893 op/s; 16444/261717 objects degraded (6.283%); 2432/261717 objects misplaced (0.929%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:19:55.099 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:55.170 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:55.229 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:55.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:55.246 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:55.347 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:55.413 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:55.484 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:55.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:55.490 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:55.591 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:55.662 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:55.683 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:55.722 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:55.727 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:55.728 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:55.818 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:19:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:19:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:19:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:19:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.818 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:37066 9.8 e466/459 9:1853f8f7:::benchmark_data_smithi055_386467_object37065:head v 466'2211, mlcod=466'2208)", 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:04.407507+0000", 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 351.40306999000001, 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.33454793700000002, 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407507+0000", 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.819 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407507+0000", 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407506+0000", 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.820 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407569+0000", 2023-09-23T18:19:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3313999999999997e-05 2023-09-23T18:19:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407585+0000", 2023-09-23T18:19:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6155999999999999e-05 2023-09-23T18:19:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.821 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.407594+0000", 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7029999999999995e-06 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737582+0000", 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32998835999999998 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.822 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737627+0000", 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4586000000000003e-05 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.742016+0000", 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0043891629999999997 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.742055+0000", 2023-09-23T18:19:55.823 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8816999999999998e-05 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:37075 9.9 e466/459 9:997ed14a:::benchmark_data_smithi055_386467_object37074:head v 466'2323, mlcod=466'2321)", 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:04.410872+0000", 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 351.39970522099998, 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.33109135099999998, 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.824 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410872+0000", 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410872+0000", 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.825 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410870+0000", 2023-09-23T18:19:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:19:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.410944+0000", 2023-09-23T18:19:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4023999999999995e-05 2023-09-23T18:19:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.826 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737578+0000", 2023-09-23T18:19:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.326633379 2023-09-23T18:19:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737600+0000", 2023-09-23T18:19:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1971e-05 2023-09-23T18:19:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.827 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737664+0000", 2023-09-23T18:19:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3850999999999996e-05 2023-09-23T18:19:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737684+0000", 2023-09-23T18:19:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0202000000000001e-05 2023-09-23T18:19:55.828 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.741922+0000", 2023-09-23T18:19:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0042384019999999996 2023-09-23T18:19:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.741963+0000", 2023-09-23T18:19:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0674999999999999e-05 2023-09-23T18:19:55.829 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:37084 9.8 e466/459 9:1c8e8c79:::benchmark_data_smithi055_386467_object37083:head v 466'2212, mlcod=466'2209)", 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:14:04.415320+0000", 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 351.39525627199998, 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32965889700000001, 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.830 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415320+0000", 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415320+0000", 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.831 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415319+0000", 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.415387+0000", 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7977000000000003e-05 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.832 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737586+0000", 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32219906100000001 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.737603+0000", 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7031e-05 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.739129+0000", 2023-09-23T18:19:55.833 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001526077 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.739145+0000", 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5176e-05 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.744934+0000", 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0057898649999999999 2023-09-23T18:19:55.834 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:14:04.744979+0000", 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4999000000000001e-05 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:35548 10.1 e584/574 10:830fdfd9:::benchmark_data_smithi055_425050_object35547:head v 584'2264, mlcod=584'2262)", 2023-09-23T18:19:55.835 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:17.647921+0000", 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 98.162656088000006, 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.181947689, 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647921+0000", 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.836 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647921+0000", 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647905+0000", 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999847 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647966+0000", 2023-09-23T18:19:55.837 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0943999999999998e-05 2023-09-23T18:19:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647976+0000", 2023-09-23T18:19:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5859999999999994e-06 2023-09-23T18:19:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647990+0000", 2023-09-23T18:19:55.838 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.377e-05 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648035+0000", 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5182999999999998e-05 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648054+0000", 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9023999999999999e-05 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.839 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.829828+0000", 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.181774198 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.829868+0000", 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0373999999999997e-05 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.840 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:35553 10.5 e584/574 10:a3327171:::benchmark_data_smithi055_425050_object35552:head v 584'2234, mlcod=584'2230)", 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:17.648042+0000", 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 98.162534401000002, 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18191095099999999, 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.841 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648042+0000", 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648042+0000", 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648042+0000", 2023-09-23T18:19:55.842 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648060+0000", 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8189000000000002e-05 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648069+0000", 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5750000000000001e-06 2023-09-23T18:19:55.843 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648074+0000", 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4890000000000003e-06 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.827305+0000", 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.17923097099999999 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.844 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.827333+0000", 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7797000000000002e-05 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.829922+0000", 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025888730000000002 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.845 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.829953+0000", 2023-09-23T18:19:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1504999999999997e-05 2023-09-23T18:19:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:35554 10.1 e584/574 10:839c1a15:::benchmark_data_smithi055_425050_object35553:head v 584'2265, mlcod=584'2262)", 2023-09-23T18:19:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:17.648660+0000", 2023-09-23T18:19:55.846 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 98.161917060999997, 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.181599287, 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648660+0000", 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.847 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648660+0000", 2023-09-23T18:19:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648659+0000", 2023-09-23T18:19:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.848 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648712+0000", 2023-09-23T18:19:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2856999999999999e-05 2023-09-23T18:19:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.827278+0000", 2023-09-23T18:19:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.17856539099999999 2023-09-23T18:19:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.849 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.827293+0000", 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5021999999999999e-05 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.828734+0000", 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014413620000000001 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.850 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.828753+0000", 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9029e-05 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830226+0000", 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001472905 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830259+0000", 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3111000000000002e-05 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.851 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:35555 10.6 e584/574 10:68209d95:::benchmark_data_smithi055_425050_object35554:head v 584'2283, mlcod=584'2281)", 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:17.650932+0000", 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 98.159644521999994, 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.17979478300000001, 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.852 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.650932+0000", 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.650932+0000", 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.650931+0000", 2023-09-23T18:19:55.853 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.650983+0000", 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2447000000000001e-05 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.827278+0000", 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.17629429899999999 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.854 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.827293+0000", 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5125e-05 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.827354+0000", 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0896000000000001e-05 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.855 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.827376+0000", 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1722999999999999e-05 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830686+0000", 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033103899999999999 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830727+0000", 2023-09-23T18:19:55.856 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1111e-05 2023-09-23T18:19:55.857 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.857 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.857 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.857 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.857 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.857 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55937 10.4 10.fcbcadb4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e588)", 2023-09-23T18:19:55.857 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:32.742172+0000", 2023-09-23T18:19:55.857 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 83.068404658999995, 2023-09-23T18:19:55.857 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.57452243800000002, 2023-09-23T18:19:55.857 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55937 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.742172+0000", 2023-09-23T18:19:55.858 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.742172+0000", 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.742171+0000", 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.859 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.742206+0000", 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4728999999999999e-05 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.742217+0000", 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1775e-05 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.860 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.742225+0000", 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1359999999999997e-06 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.742288+0000", 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2979999999999997e-05 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.861 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.742297+0000", 2023-09-23T18:19:55.862 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9670000000000007e-06 2023-09-23T18:19:55.862 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.862 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.862 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.862 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.316683+0000", 2023-09-23T18:19:55.862 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.57438562900000001 2023-09-23T18:19:55.862 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.862 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.862 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.862 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.316695+0000", 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1545e-05 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55942 10.4 10.372b9da4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e588)", 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:32.744881+0000", 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 83.065696020999994, 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.57212552999999999, 2023-09-23T18:19:55.863 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55942 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.744881+0000", 2023-09-23T18:19:55.864 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.744881+0000", 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.744879+0000", 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.865 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.866 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.866 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.744904+0000", 2023-09-23T18:19:55.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4185e-05 2023-09-23T18:19:55.866 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.866 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.866 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.866 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.744916+0000", 2023-09-23T18:19:55.866 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2058999999999999e-05 2023-09-23T18:19:55.866 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.866 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.744923+0000", 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3350000000000001e-06 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.744971+0000", 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7834000000000003e-05 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.867 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.744979+0000", 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1750000000000005e-06 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.316996+0000", 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.57201725999999997 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.868 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.317006+0000", 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9860000000000006e-06 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55950 10.4 10.356b61a4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e588)", 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:32.747361+0000", 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 83.063215782, 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.56967949100000004, 2023-09-23T18:19:55.869 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.870 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:19:55.870 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.870 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.870 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.870 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55950 2023-09-23T18:19:55.870 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.870 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.870 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.870 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.870 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.747361+0000", 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.747361+0000", 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.747360+0000", 2023-09-23T18:19:55.871 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:55.872 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.872 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.747380+0000", 2023-09-23T18:19:55.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0543e-05 2023-09-23T18:19:55.872 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.872 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.872 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.872 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.747392+0000", 2023-09-23T18:19:55.872 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1732999999999999e-05 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.747400+0000", 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.1950000000000003e-06 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.747449+0000", 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8770999999999998e-05 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.873 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.747459+0000", 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5279999999999992e-06 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.317033+0000", 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.56957429100000001 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.874 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.317040+0000", 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6499999999999996e-06 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55965 10.4 10.193da4b4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e588)", 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:32.761473+0000", 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 83.049103422000002, 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.55560252499999996, 2023-09-23T18:19:55.875 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55965 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.761473+0000", 2023-09-23T18:19:55.876 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.761473+0000", 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.761472+0000", 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.877 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.761500+0000", 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8456e-05 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.761518+0000", 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7569000000000001e-05 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.878 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.761528+0000", 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0104e-05 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.761585+0000", 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7423000000000002e-05 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.761594+0000", 2023-09-23T18:19:55.879 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1419999999999999e-06 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.317066+0000", 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.55547122900000001 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.317076+0000", 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0195e-05 2023-09-23T18:19:55.880 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55971 10.4 10.f97c38c4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e588)", 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:32.768330+0000", 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 83.042246730000002, 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.54877966600000005, 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:19:55.881 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.882 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.882 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.882 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55971 2023-09-23T18:19:55.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.882 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.882 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.882 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.882 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.768330+0000", 2023-09-23T18:19:55.882 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.882 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.768330+0000", 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.768328+0000", 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.883 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.768357+0000", 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8435000000000001e-05 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.768371+0000", 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4691999999999999e-05 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.884 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.768381+0000", 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8099999999999992e-06 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.768438+0000", 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6456999999999998e-05 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.768449+0000", 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1243000000000001e-05 2023-09-23T18:19:55.885 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.317102+0000", 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.54865319700000004 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.317110+0000", 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7389999999999999e-06 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.886 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55990 10.4 10.22c41a4 (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e588)", 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:32.800909+0000", 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 83.009667378000003, 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.51623253099999999, 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.887 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55990 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.800909+0000", 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.800909+0000", 2023-09-23T18:19:55.888 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.800908+0000", 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.800933+0000", 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5046999999999999e-05 2023-09-23T18:19:55.889 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.800948+0000", 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5007999999999999e-05 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.800957+0000", 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7499999999999992e-06 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.890 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.801018+0000", 2023-09-23T18:19:55.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1087000000000007e-05 2023-09-23T18:19:55.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.891 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:32.801029+0000", 2023-09-23T18:19:55.891 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0733999999999999e-05 2023-09-23T18:19:55.891 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.891 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.891 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.317132+0000", 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.51610352400000004 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.317142+0000", 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8870000000000004e-06 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.892 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67163 10.c 10:36302d77:::benchmark_data_smithi055_425050_object67162:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e595)", 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:46.424677+0000", 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 69.385900090999996, 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.182796306, 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67163 2023-09-23T18:19:55.893 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424677+0000", 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424677+0000", 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.894 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424675+0000", 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424720+0000", 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4589000000000003e-05 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.895 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424735+0000", 2023-09-23T18:19:55.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.526e-05 2023-09-23T18:19:55.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.896 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.896 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424746+0000", 2023-09-23T18:19:55.896 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1e-05 2023-09-23T18:19:55.896 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.896 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424964+0000", 2023-09-23T18:19:55.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021825100000000001 2023-09-23T18:19:55.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.897 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.897 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424974+0000", 2023-09-23T18:19:55.897 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0185e-05 2023-09-23T18:19:55.897 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.897 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.579362+0000", 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1543879880000001 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.580681+0000", 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013183820000000001 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.898 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:55.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.580801+0000", 2023-09-23T18:19:55.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000120704 2023-09-23T18:19:55.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.899 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.899 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.585375+0000", 2023-09-23T18:19:55.899 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045738130000000004 2023-09-23T18:19:55.899 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.899 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.587649+0000", 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022740019999999998 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.604563+0000", 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.016914118999999998 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.900 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.604629+0000", 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.5468000000000004e-05 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.607473+0000", 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028441180000000001 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.901 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67177 10.c 10:30b7fa6f:::benchmark_data_smithi055_425050_object67176:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e595)", 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:46.424760+0000", 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 69.385816368999997, 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1847201359999999, 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67177 2023-09-23T18:19:55.902 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424760+0000", 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424760+0000", 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.903 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424760+0000", 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424787+0000", 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7081999999999999e-05 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.904 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424796+0000", 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3680000000000008e-06 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424803+0000", 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.28e-06 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424998+0000", 2023-09-23T18:19:55.905 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019519700000000001 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.425005+0000", 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3969999999999997e-06 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.585422+0000", 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.160416463 2023-09-23T18:19:55.906 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.585547+0000", 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000125272 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.585672+0000", 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000125554 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.907 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.591713+0000", 2023-09-23T18:19:55.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0060401700000000001 2023-09-23T18:19:55.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:55.908 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.594565+0000", 2023-09-23T18:19:55.908 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0028525909999999998 2023-09-23T18:19:55.908 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.908 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.607549+0000", 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.012983793 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.607600+0000", 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0967000000000002e-05 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.909 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.609481+0000", 2023-09-23T18:19:55.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018806070000000001 2023-09-23T18:19:55.910 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.910 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.910 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.910 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67189 10.c 10:3dc2fede:::benchmark_data_smithi055_425050_object67188:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e595)", 2023-09-23T18:19:55.910 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:46.424808+0000", 2023-09-23T18:19:55.910 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 69.385768936000005, 2023-09-23T18:19:55.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1883613310000001, 2023-09-23T18:19:55.911 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.911 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.911 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.911 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.911 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.911 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67189 2023-09-23T18:19:55.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.911 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424808+0000", 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424808+0000", 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424807+0000", 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424826+0000", 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8776000000000001e-05 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424834+0000", 2023-09-23T18:19:55.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5240000000000003e-06 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424839+0000", 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7910000000000001e-06 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.425030+0000", 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019097100000000001 2023-09-23T18:19:55.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.425038+0000", 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0779999999999998e-06 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.587685+0000", 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.162647634 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.587848+0000", 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000162461 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.587954+0000", 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000106525 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:55.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.594576+0000", 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0066221170000000003 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.594712+0000", 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013580600000000001 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.610679+0000", 2023-09-23T18:19:55.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015966569 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.610777+0000", 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8791999999999999e-05 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.613169+0000", 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0023917470000000001 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67195 10.c 10:3e185847:::benchmark_data_smithi055_425050_object67194:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e595)", 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:46.424845+0000", 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 69.385732140000002, 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.193909667, 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.920 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67195 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424845+0000", 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424845+0000", 2023-09-23T18:19:55.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424844+0000", 2023-09-23T18:19:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424863+0000", 2023-09-23T18:19:55.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8465999999999999e-05 2023-09-23T18:19:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424870+0000", 2023-09-23T18:19:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5700000000000004e-06 2023-09-23T18:19:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424877+0000", 2023-09-23T18:19:55.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1010000000000003e-06 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.425060+0000", 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018301400000000001 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.425067+0000", 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0330000000000003e-06 2023-09-23T18:19:55.924 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.589691+0000", 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1646239629999999 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.589797+0000", 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000105387 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.589899+0000", 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0001021 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.594586+0000", 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0046875989999999998 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.595009+0000", 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000422378 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.616456+0000", 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.021447246 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.616509+0000", 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.329e-05 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.618754+0000", 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022450320000000001 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.928 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67196 10.c 10:32ac17d3:::benchmark_data_smithi055_425050_object67195:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e595)", 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:46.424883+0000", 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 69.385694182999998, 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1961849309999999, 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67196 2023-09-23T18:19:55.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424883+0000", 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424883+0000", 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424882+0000", 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424914+0000", 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.1417000000000001e-05 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424922+0000", 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7490000000000004e-06 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.424929+0000", 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3149999999999997e-06 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.425113+0000", 2023-09-23T18:19:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018482500000000001 2023-09-23T18:19:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:19:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.425123+0000", 2023-09-23T18:19:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4620000000000002e-06 2023-09-23T18:19:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.591741+0000", 2023-09-23T18:19:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.16661825 2023-09-23T18:19:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.591894+0000", 2023-09-23T18:19:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015285299999999999 2023-09-23T18:19:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:55.934 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.592100+0000", 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000206219 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.594830+0000", 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027302609999999999 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.595162+0000", 2023-09-23T18:19:55.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000331811 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.618825+0000", 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.023662341999999999 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.618882+0000", 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7558e-05 2023-09-23T18:19:55.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:47.621068+0000", 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021853710000000002 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:111874 10.4 10:2b09dd55:::benchmark_data_smithi055_425050_object8468:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:19:42.366687+0000", 2023-09-23T18:19:55.937 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.443889950999999, 2023-09-23T18:19:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.17510456499999999, 2023-09-23T18:19:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 111874 2023-09-23T18:19:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.366687+0000", 2023-09-23T18:19:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.366687+0000", 2023-09-23T18:19:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.366686+0000", 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.366692+0000", 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1680000000000001e-06 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.366705+0000", 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3495e-05 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.366717+0000", 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1715e-05 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.366766+0000", 2023-09-23T18:19:55.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9357999999999997e-05 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.366915+0000", 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014854899999999999 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.367030+0000", 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000115222 2023-09-23T18:19:55.942 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.369621+0000", 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025909589999999999 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.375282+0000", 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.005661485 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.541699+0000", 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.16641688700000001 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.541753+0000", 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3968999999999998e-05 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.541791+0000", 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8059999999999998e-05 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:111882 10.4 10:292253fa:::benchmark_data_smithi055_425050_object8476:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:19:42.370941+0000", 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 13.439635643999999, 2023-09-23T18:19:55.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.17097789699999999, 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 111882 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:19:55.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.370941+0000", 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.370941+0000", 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.370940+0000", 2023-09-23T18:19:55.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.370947+0000", 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5830000000000001e-06 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.370962+0000", 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4525999999999999e-05 2023-09-23T18:19:55.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.370969+0000", 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4250000000000001e-06 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.371013+0000", 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3492999999999998e-05 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.949 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.371111+0000", 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8139000000000003e-05 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.371193+0000", 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2464000000000002e-05 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:19:55.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.373232+0000", 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020388289999999998 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.375362+0000", 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021301179999999999 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.541834+0000", 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.16647230700000001 2023-09-23T18:19:55.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.541879+0000", 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5111999999999998e-05 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.541919+0000", 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9524999999999999e-05 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.952 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:19:55.953 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:19:55.953 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:19:55.953 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:56.028 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:56.077 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:56.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:56.082 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:56.084 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:19:56.183 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:56.250 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:56.317 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:56.320 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:56.320 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:56.421 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:19:56.460 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:19:56.474 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:56.500 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:19:56.501 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:19:56.523 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:19:56.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:56.530 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:19:56.630 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:19:56.731 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:56.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:19:56.806 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:56 smithi055 ceph-mon[184174]: pgmap v1446: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 895 op/s; 16444/260868 objects degraded (6.304%); 2432/260868 objects misplaced (0.932%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:19:56.806 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:56 smithi055 ceph-mon[179668]: pgmap v1446: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 895 op/s; 16444/260868 objects degraded (6.304%); 2432/260868 objects misplaced (0.932%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:19:56.807 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:56.808 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:19:56.909 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:56.966 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1446,"stamp":"2023-09-23T18:19:55.146882+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":5701384892,"num_objects":86956,"num_object_clones":0,"num_object_copies":260868,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":16444,"num_objects_misplaced":2432,"num_objects_unfound":0,"num_objects_dirty":86956,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":120790,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4677,"num_bytes_recovered":296443529,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83101,"ondisk_log_size":83101,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":20929512,"kb_used_data":17509688,"kb_used_omap":40,"kb_used_meta":3419479,"kb_avail":541541400,"statfs":{"total":575970213888,"available":554538393600,"internally_reserved":0,"allocated":17929920512,"data_stored":17915367878,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3501546810},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-587202560,"num_objects":-8960,"num_object_clones":0,"num_object_copies":-26880,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":130,"num_objects_misplaced":-559,"num_objects_unfound":0,"num_objects_dirty":-8960,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8960,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":429,"num_bytes_recovered":28114944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007922"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":895,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391168+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:19:34.391168+0000","last_peered":"2023-09-23T18:19:34.391168+0000","last_clean":"2023-09-23T18:19:34.391168+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:19:34.391168+0000","last_undegraded":"2023-09-23T18:19:34.391168+0000","last_fullsized":"2023-09-23T18:19:34.391168+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":267,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:42.596897+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:19:42.596897+0000","last_peered":"2023-09-23T18:19:42.596897+0000","last_clean":"2023-09-23T18:19:42.596897+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:19:42.596897+0000","last_undegraded":"2023-09-23T18:19:42.596897+0000","last_fullsized":"2023-09-23T18:19:42.596897+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398176+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:19:34.398176+0000","last_peered":"2023-09-23T18:19:34.398176+0000","last_clean":"2023-09-23T18:19:34.398176+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:19:34.398176+0000","last_undegraded":"2023-09-23T18:19:34.398176+0000","last_fullsized":"2023-09-23T18:19:34.398176+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":901,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390616+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:19:34.390616+0000","last_peered":"2023-09-23T18:19:34.390616+0000","last_clean":"2023-09-23T18:19:34.390616+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:19:34.390616+0000","last_undegraded":"2023-09-23T18:19:34.390616+0000","last_fullsized":"2023-09-23T18:19:34.390616+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":347,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398721+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:19:34.398721+0000","last_peered":"2023-09-23T18:19:34.398721+0000","last_clean":"2023-09-23T18:19:34.398721+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:19:34.398721+0000","last_undegraded":"2023-09-23T18:19:34.398721+0000","last_fullsized":"2023-09-23T18:19:34.398721+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":667,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396592+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:19:34.396592+0000","last_peered":"2023-09-23T18:19:34.396592+0000","last_clean":"2023-09-23T18:19:34.396592+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:19:34.396592+0000","last_undegraded":"2023-09-23T18:19:34.396592+0000","last_fullsized":"2023-09-23T18:19:34.396592+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":918,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390373+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:19:34.390373+0000","last_peered":"2023-09-23T18:19:34.390373+0000","last_clean":"2023-09-23T18:19:34.390373+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:34.390373+0000","last_undegraded":"2023-09-23T18:19:34.390373+0000","last_fullsized":"2023-09-23T18:19:34.390373+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.397821+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:34.397821+0000","last_peered":"2023-09-23T18:19:34.397821+0000","last_clean":"2023-09-23T18:19:34.397821+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:19:34.397821+0000","last_undegraded":"2023-09-23T18:19:34.397821+0000","last_fullsized":"2023-09-23T18:19:34.397821+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":183,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401669+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:19:34.401669+0000","last_peered":"2023-09-23T18:19:34.401669+0000","last_clean":"2023-09-23T18:19:34.401669+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:19:34.401669+0000","last_undegraded":"2023-09-23T18:19:34.401669+0000","last_fullsized":"2023-09-23T18:19:34.401669+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1041,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390840+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:19:34.390840+0000","last_peered":"2023-09-23T18:19:34.390840+0000","last_clean":"2023-09-23T18:19:34.390840+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:19:34.390840+0000","last_undegraded":"2023-09-23T18:19:34.390840+0000","last_fullsized":"2023-09-23T18:19:34.390840+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":117,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391102+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:19:34.391102+0000","last_peered":"2023-09-23T18:19:34.391102+0000","last_clean":"2023-09-23T18:19:34.391102+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:19:34.391102+0000","last_undegraded":"2023-09-23T18:19:34.391102+0000","last_fullsized":"2023-09-23T18:19:34.391102+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":196,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398219+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:19:34.398219+0000","last_peered":"2023-09-23T18:19:34.398219+0000","last_clean":"2023-09-23T18:19:34.398219+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:19:34.398219+0000","last_undegraded":"2023-09-23T18:19:34.398219+0000","last_fullsized":"2023-09-23T18:19:34.398219+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":818,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400857+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:19:34.400857+0000","last_peered":"2023-09-23T18:19:34.400857+0000","last_clean":"2023-09-23T18:19:34.400857+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:19:34.400857+0000","last_undegraded":"2023-09-23T18:19:34.400857+0000","last_fullsized":"2023-09-23T18:19:34.400857+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":153,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390974+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:19:34.390974+0000","last_peered":"2023-09-23T18:19:34.390974+0000","last_clean":"2023-09-23T18:19:34.390974+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:19:34.390974+0000","last_undegraded":"2023-09-23T18:19:34.390974+0000","last_fullsized":"2023-09-23T18:19:34.390974+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":152,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400947+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:19:34.400947+0000","last_peered":"2023-09-23T18:19:34.400947+0000","last_clean":"2023-09-23T18:19:34.400947+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:19:34.400947+0000","last_undegraded":"2023-09-23T18:19:34.400947+0000","last_fullsized":"2023-09-23T18:19:34.400947+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":952,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391175+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:19:34.391175+0000","last_peered":"2023-09-23T18:19:34.391175+0000","last_clean":"2023-09-23T18:19:34.391175+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:19:34.391175+0000","last_undegraded":"2023-09-23T18:19:34.391175+0000","last_fullsized":"2023-09-23T18:19:34.391175+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":401,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398559+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:19:34.398559+0000","last_peered":"2023-09-23T18:19:34.398559+0000","last_clean":"2023-09-23T18:19:34.398559+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:19:34.398559+0000","last_undegraded":"2023-09-23T18:19:34.398559+0000","last_fullsized":"2023-09-23T18:19:34.398559+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":420,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.402025+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:19:34.402025+0000","last_peered":"2023-09-23T18:19:34.402025+0000","last_clean":"2023-09-23T18:19:34.402025+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:19:34.402025+0000","last_undegraded":"2023-09-23T18:19:34.402025+0000","last_fullsized":"2023-09-23T18:19:34.402025+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":982,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:43.886954+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:19:43.886954+0000","last_peered":"2023-09-23T18:19:43.886954+0000","last_clean":"2023-09-23T18:19:43.886954+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:19:43.886954+0000","last_undegraded":"2023-09-23T18:19:43.886954+0000","last_fullsized":"2023-09-23T18:19:43.886954+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":159,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401271+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:19:34.401271+0000","last_peered":"2023-09-23T18:19:34.401271+0000","last_clean":"2023-09-23T18:19:34.401271+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:19:34.401271+0000","last_undegraded":"2023-09-23T18:19:34.401271+0000","last_fullsized":"2023-09-23T18:19:34.401271+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":426,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401302+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:19:34.401302+0000","last_peered":"2023-09-23T18:19:34.401302+0000","last_clean":"2023-09-23T18:19:34.401302+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:19:34.401302+0000","last_undegraded":"2023-09-23T18:19:34.401302+0000","last_fullsized":"2023-09-23T18:19:34.401302+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":833,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390615+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:19:34.390615+0000","last_peered":"2023-09-23T18:19:34.390615+0000","last_clean":"2023-09-23T18:19:34.390615+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:19:34.390615+0000","last_undegraded":"2023-09-23T18:19:34.390615+0000","last_fullsized":"2023-09-23T18:19:34.390615+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":426,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401221+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:19:34.401221+0000","last_peered":"2023-09-23T18:19:34.401221+0000","last_clean":"2023-09-23T18:19:34.401221+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:19:34.401221+0000","last_undegraded":"2023-09-23T18:19:34.401221+0000","last_fullsized":"2023-09-23T18:19:34.401221+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":104,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390873+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:19:34.390873+0000","last_peered":"2023-09-23T18:19:34.390873+0000","last_clean":"2023-09-23T18:19:34.390873+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:19:34.390873+0000","last_undegraded":"2023-09-23T18:19:34.390873+0000","last_fullsized":"2023-09-23T18:19:34.390873+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":804,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390243+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:19:34.390243+0000","last_peered":"2023-09-23T18:19:34.390243+0000","last_clean":"2023-09-23T18:19:34.390243+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:19:34.390243+0000","last_undegraded":"2023-09-23T18:19:34.390243+0000","last_fullsized":"2023-09-23T18:19:34.390243+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":122,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396224+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:19:34.396224+0000","last_peered":"2023-09-23T18:19:34.396224+0000","last_clean":"2023-09-23T18:19:34.396224+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:19:34.396224+0000","last_undegraded":"2023-09-23T18:19:34.396224+0000","last_fullsized":"2023-09-23T18:19:34.396224+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":117,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390982+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:19:34.390982+0000","last_peered":"2023-09-23T18:19:34.390982+0000","last_clean":"2023-09-23T18:19:34.390982+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:19:34.390982+0000","last_undegraded":"2023-09-23T18:19:34.390982+0000","last_fullsized":"2023-09-23T18:19:34.390982+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":742,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398346+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:19:34.398346+0000","last_peered":"2023-09-23T18:19:34.398346+0000","last_clean":"2023-09-23T18:19:34.398346+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:19:34.398346+0000","last_undegraded":"2023-09-23T18:19:34.398346+0000","last_fullsized":"2023-09-23T18:19:34.398346+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":194,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400859+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:19:34.400859+0000","last_peered":"2023-09-23T18:19:34.400859+0000","last_clean":"2023-09-23T18:19:34.400859+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:19:34.400859+0000","last_undegraded":"2023-09-23T18:19:34.400859+0000","last_fullsized":"2023-09-23T18:19:34.400859+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":205,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400051+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:19:34.400051+0000","last_peered":"2023-09-23T18:19:34.400051+0000","last_clean":"2023-09-23T18:19:34.400051+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:19:34.400051+0000","last_undegraded":"2023-09-23T18:19:34.400051+0000","last_fullsized":"2023-09-23T18:19:34.400051+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":838,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400969+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:19:34.400969+0000","last_peered":"2023-09-23T18:19:34.400969+0000","last_clean":"2023-09-23T18:19:34.400969+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:19:34.400969+0000","last_undegraded":"2023-09-23T18:19:34.400969+0000","last_fullsized":"2023-09-23T18:19:34.400969+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":747,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400877+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:19:34.400877+0000","last_peered":"2023-09-23T18:19:34.400877+0000","last_clean":"2023-09-23T18:19:34.400877+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:19:34.400877+0000","last_undegraded":"2023-09-23T18:19:34.400877+0000","last_fullsized":"2023-09-23T18:19:34.400877+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400010+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:19:34.400010+0000","last_peered":"2023-09-23T18:19:34.400010+0000","last_clean":"2023-09-23T18:19:34.400010+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:19:34.400010+0000","last_undegraded":"2023-09-23T18:19:34.400010+0000","last_fullsized":"2023-09-23T18:19:34.400010+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"607'7576","reported_seq":7617,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.123667+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:19:51.123667+0000","last_peered":"2023-09-23T18:19:51.123667+0000","last_clean":"2023-09-23T18:19:51.123667+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:19:51.123667+0000","last_undegraded":"2023-09-23T18:19:51.123667+0000","last_fullsized":"2023-09-23T18:19:51.123667+0000","mapping_epoch":574,"log_start":"584'2100","ondisk_log_start":"584'2100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5476,"log_dups_size":0,"ondisk_log_size":5476,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360316928,"num_objects":5498,"num_object_clones":0,"num_object_copies":16494,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5498,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7576,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":872,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401856+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:19:34.401856+0000","last_peered":"2023-09-23T18:19:34.401856+0000","last_clean":"2023-09-23T18:19:34.401856+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:19:34.401856+0000","last_undegraded":"2023-09-23T18:19:34.401856+0000","last_fullsized":"2023-09-23T18:19:34.401856+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":244,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:42.597055+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:19:42.597055+0000","last_peered":"2023-09-23T18:19:42.597055+0000","last_clean":"2023-09-23T18:19:42.597055+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:19:42.597055+0000","last_undegraded":"2023-09-23T18:19:42.597055+0000","last_fullsized":"2023-09-23T18:19:42.597055+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":117,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390279+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:19:34.390279+0000","last_peered":"2023-09-23T18:19:34.390279+0000","last_clean":"2023-09-23T18:19:34.390279+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:19:34.390279+0000","last_undegraded":"2023-09-23T18:19:34.390279+0000","last_fullsized":"2023-09-23T18:19:34.390279+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"607'7334","reported_seq":7385,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:51.196754+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:19:51.196754+0000","last_peered":"2023-09-23T18:19:51.196754+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:19:51.196754+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"585'2900","ondisk_log_start":"585'2900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4434,"log_dups_size":0,"ondisk_log_size":4434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351141888,"num_objects":5358,"num_object_clones":0,"num_object_copies":16074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7334,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":3160},{"shards":"5,6","objects":1210}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":726,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.408314+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:19:34.408314+0000","last_peered":"2023-09-23T18:19:34.408314+0000","last_clean":"2023-09-23T18:19:34.408314+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:19:34.408314+0000","last_undegraded":"2023-09-23T18:19:34.408314+0000","last_fullsized":"2023-09-23T18:19:34.408314+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":245,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:42.602462+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:19:42.602462+0000","last_peered":"2023-09-23T18:19:42.602462+0000","last_clean":"2023-09-23T18:19:42.602462+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:19:42.602462+0000","last_undegraded":"2023-09-23T18:19:42.602462+0000","last_fullsized":"2023-09-23T18:19:42.602462+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":224,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:42.597969+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:19:42.597969+0000","last_peered":"2023-09-23T18:19:42.597969+0000","last_clean":"2023-09-23T18:19:42.597969+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:19:42.597969+0000","last_undegraded":"2023-09-23T18:19:42.597969+0000","last_fullsized":"2023-09-23T18:19:42.597969+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"607'7399","reported_seq":14400,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:50.214527+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:19:50.214527+0000","last_peered":"2023-09-23T18:19:50.214527+0000","last_clean":"2023-09-23T18:19:50.214527+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:19:50.214527+0000","last_undegraded":"2023-09-23T18:19:50.214527+0000","last_fullsized":"2023-09-23T18:19:50.214527+0000","mapping_epoch":605,"log_start":"584'2500","ondisk_log_start":"584'2500","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4899,"log_dups_size":0,"ondisk_log_size":4899,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":356843520,"num_objects":5445,"num_object_clones":0,"num_object_copies":16335,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5445,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7399,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":622,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401597+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:19:34.401597+0000","last_peered":"2023-09-23T18:19:34.401597+0000","last_clean":"2023-09-23T18:19:34.401597+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:19:34.401597+0000","last_undegraded":"2023-09-23T18:19:34.401597+0000","last_fullsized":"2023-09-23T18:19:34.401597+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":532,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390898+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:19:34.390898+0000","last_peered":"2023-09-23T18:19:34.390898+0000","last_clean":"2023-09-23T18:19:34.390898+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:19:34.390898+0000","last_undegraded":"2023-09-23T18:19:34.390898+0000","last_fullsized":"2023-09-23T18:19:34.390898+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":183,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401899+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:19:34.401899+0000","last_peered":"2023-09-23T18:19:34.401899+0000","last_clean":"2023-09-23T18:19:34.401899+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:19:34.401899+0000","last_undegraded":"2023-09-23T18:19:34.401899+0000","last_fullsized":"2023-09-23T18:19:34.401899+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"607'7598","reported_seq":7640,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.077927+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:19:51.077927+0000","last_peered":"2023-09-23T18:19:51.077927+0000","last_clean":"2023-09-23T18:19:51.077927+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:19:51.077927+0000","last_undegraded":"2023-09-23T18:19:51.077927+0000","last_fullsized":"2023-09-23T18:19:51.077927+0000","mapping_epoch":574,"log_start":"584'2500","ondisk_log_start":"584'2500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5098,"log_dups_size":0,"ondisk_log_size":5098,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360054784,"num_objects":5494,"num_object_clones":0,"num_object_copies":16482,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5494,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7598,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":637,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.399888+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:19:34.399888+0000","last_peered":"2023-09-23T18:19:34.399888+0000","last_clean":"2023-09-23T18:19:34.399888+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:19:34.399888+0000","last_undegraded":"2023-09-23T18:19:34.399888+0000","last_fullsized":"2023-09-23T18:19:34.399888+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":131,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390904+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:19:34.390904+0000","last_peered":"2023-09-23T18:19:34.390904+0000","last_clean":"2023-09-23T18:19:34.390904+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:19:34.390904+0000","last_undegraded":"2023-09-23T18:19:34.390904+0000","last_fullsized":"2023-09-23T18:19:34.390904+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":443,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396600+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:19:34.396600+0000","last_peered":"2023-09-23T18:19:34.396600+0000","last_clean":"2023-09-23T18:19:34.396600+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:34.396600+0000","last_undegraded":"2023-09-23T18:19:34.396600+0000","last_fullsized":"2023-09-23T18:19:34.396600+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"608'7686","reported_seq":7728,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:53.864227+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:19:53.864227+0000","last_peered":"2023-09-23T18:19:53.864227+0000","last_clean":"2023-09-23T18:19:53.864227+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:19:53.864227+0000","last_undegraded":"2023-09-23T18:19:53.864227+0000","last_fullsized":"2023-09-23T18:19:53.864227+0000","mapping_epoch":574,"log_start":"586'3300","ondisk_log_start":"586'3300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4386,"log_dups_size":0,"ondisk_log_size":4386,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351141888,"num_objects":5358,"num_object_clones":0,"num_object_copies":16074,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5358,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7686,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1089,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390522+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:19:34.390522+0000","last_peered":"2023-09-23T18:19:34.390522+0000","last_clean":"2023-09-23T18:19:34.390522+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:19:34.390522+0000","last_undegraded":"2023-09-23T18:19:34.390522+0000","last_fullsized":"2023-09-23T18:19:34.390522+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":343,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398581+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:19:34.398581+0000","last_peered":"2023-09-23T18:19:34.398581+0000","last_clean":"2023-09-23T18:19:34.398581+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:19:34.398581+0000","last_undegraded":"2023-09-23T18:19:34.398581+0000","last_fullsized":"2023-09-23T18:19:34.398581+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":151,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401037+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:19:34.401037+0000","last_peered":"2023-09-23T18:19:34.401037+0000","last_clean":"2023-09-23T18:19:34.401037+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:19:34.401037+0000","last_undegraded":"2023-09-23T18:19:34.401037+0000","last_fullsized":"2023-09-23T18:19:34.401037+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"607'7561","reported_seq":7609,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:51.099298+0000","last_change":"2023-09-23T18:18:47.594759+0000","last_active":"2023-09-23T18:19:51.099298+0000","last_peered":"2023-09-23T18:19:51.099298+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:19:51.099298+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"584'2100","ondisk_log_start":"584'2100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5461,"log_dups_size":0,"ondisk_log_size":5461,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358416384,"num_objects":5469,"num_object_clones":0,"num_object_copies":16407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5469,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7561,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":3180},{"shards":"4,5","objects":1243}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":876,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391152+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:19:34.391152+0000","last_peered":"2023-09-23T18:19:34.391152+0000","last_clean":"2023-09-23T18:19:34.391152+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:19:34.391152+0000","last_undegraded":"2023-09-23T18:19:34.391152+0000","last_fullsized":"2023-09-23T18:19:34.391152+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":159,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401810+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:19:34.401810+0000","last_peered":"2023-09-23T18:19:34.401810+0000","last_clean":"2023-09-23T18:19:34.401810+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:19:34.401810+0000","last_undegraded":"2023-09-23T18:19:34.401810+0000","last_fullsized":"2023-09-23T18:19:34.401810+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":458,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401997+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:19:34.401997+0000","last_peered":"2023-09-23T18:19:34.401997+0000","last_clean":"2023-09-23T18:19:34.401997+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:19:34.401997+0000","last_undegraded":"2023-09-23T18:19:34.401997+0000","last_fullsized":"2023-09-23T18:19:34.401997+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"607'7548","reported_seq":7589,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.205020+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:19:51.205020+0000","last_peered":"2023-09-23T18:19:51.205020+0000","last_clean":"2023-09-23T18:19:51.205020+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:19:51.205020+0000","last_undegraded":"2023-09-23T18:19:51.205020+0000","last_fullsized":"2023-09-23T18:19:51.205020+0000","mapping_epoch":574,"log_start":"586'3100","ondisk_log_start":"586'3100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4448,"log_dups_size":0,"ondisk_log_size":4448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":360710144,"num_objects":5504,"num_object_clones":0,"num_object_copies":16512,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5504,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7548,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":878,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390700+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:19:34.390700+0000","last_peered":"2023-09-23T18:19:34.390700+0000","last_clean":"2023-09-23T18:19:34.390700+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:19:34.390700+0000","last_undegraded":"2023-09-23T18:19:34.390700+0000","last_fullsized":"2023-09-23T18:19:34.390700+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":443,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390352+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:19:34.390352+0000","last_peered":"2023-09-23T18:19:34.390352+0000","last_clean":"2023-09-23T18:19:34.390352+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:19:34.390352+0000","last_undegraded":"2023-09-23T18:19:34.390352+0000","last_fullsized":"2023-09-23T18:19:34.390352+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":219,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401292+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:19:34.401292+0000","last_peered":"2023-09-23T18:19:34.401292+0000","last_clean":"2023-09-23T18:19:34.401292+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:19:34.401292+0000","last_undegraded":"2023-09-23T18:19:34.401292+0000","last_fullsized":"2023-09-23T18:19:34.401292+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"608'7641","reported_seq":7683,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:52.260673+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:19:52.260673+0000","last_peered":"2023-09-23T18:19:52.260673+0000","last_clean":"2023-09-23T18:19:52.260673+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:19:52.260673+0000","last_undegraded":"2023-09-23T18:19:52.260673+0000","last_fullsized":"2023-09-23T18:19:52.260673+0000","mapping_epoch":574,"log_start":"585'3000","ondisk_log_start":"585'3000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4641,"log_dups_size":0,"ondisk_log_size":4641,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353435648,"num_objects":5393,"num_object_clones":0,"num_object_copies":16179,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5393,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7641,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":859,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390649+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:19:34.390649+0000","last_peered":"2023-09-23T18:19:34.390649+0000","last_clean":"2023-09-23T18:19:34.390649+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:19:34.390649+0000","last_undegraded":"2023-09-23T18:19:34.390649+0000","last_fullsized":"2023-09-23T18:19:34.390649+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.397986+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:19:34.397986+0000","last_peered":"2023-09-23T18:19:34.397986+0000","last_clean":"2023-09-23T18:19:34.397986+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:19:34.397986+0000","last_undegraded":"2023-09-23T18:19:34.397986+0000","last_fullsized":"2023-09-23T18:19:34.397986+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":336,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:42.592750+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:19:42.592750+0000","last_peered":"2023-09-23T18:19:42.592750+0000","last_clean":"2023-09-23T18:19:42.592750+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:19:42.592750+0000","last_undegraded":"2023-09-23T18:19:42.592750+0000","last_fullsized":"2023-09-23T18:19:42.592750+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"607'7501","reported_seq":7545,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:50.193495+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:19:50.193495+0000","last_peered":"2023-09-23T18:19:50.193495+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:19:50.193495+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"584'2600","ondisk_log_start":"584'2600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4901,"log_dups_size":0,"ondisk_log_size":4901,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":364838912,"num_objects":5567,"num_object_clones":0,"num_object_copies":16701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5567,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7501,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":1282},{"shards":"0,4,6","objects":3318}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":718,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401768+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:19:34.401768+0000","last_peered":"2023-09-23T18:19:34.401768+0000","last_clean":"2023-09-23T18:19:34.401768+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:19:34.401768+0000","last_undegraded":"2023-09-23T18:19:34.401768+0000","last_fullsized":"2023-09-23T18:19:34.401768+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":958,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:42.592317+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:19:42.592317+0000","last_peered":"2023-09-23T18:19:42.592317+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:19:42.592317+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":141,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391024+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:19:34.391024+0000","last_peered":"2023-09-23T18:19:34.391024+0000","last_clean":"2023-09-23T18:19:34.391024+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:19:34.391024+0000","last_undegraded":"2023-09-23T18:19:34.391024+0000","last_fullsized":"2023-09-23T18:19:34.391024+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":121,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390568+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:19:34.390568+0000","last_peered":"2023-09-23T18:19:34.390568+0000","last_clean":"2023-09-23T18:19:34.390568+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:19:34.390568+0000","last_undegraded":"2023-09-23T18:19:34.390568+0000","last_fullsized":"2023-09-23T18:19:34.390568+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"607'7468","reported_seq":7518,"reported_epoch":607,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:51.211635+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:19:51.211635+0000","last_peered":"2023-09-23T18:19:51.211635+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:19:51.211635+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"585'3000","ondisk_log_start":"585'3000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4468,"log_dups_size":0,"ondisk_log_size":4468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":358350848,"num_objects":5468,"num_object_clones":0,"num_object_copies":16404,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5468,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7468,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":2558},{"shards":"5,6","objects":1909}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":790,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390698+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:19:34.390698+0000","last_peered":"2023-09-23T18:19:34.390698+0000","last_clean":"2023-09-23T18:19:34.390698+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:19:34.390698+0000","last_undegraded":"2023-09-23T18:19:34.390698+0000","last_fullsized":"2023-09-23T18:19:34.390698+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1491,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390535+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:19:34.390535+0000","last_peered":"2023-09-23T18:19:34.390535+0000","last_clean":"2023-09-23T18:19:34.390535+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:19:34.390535+0000","last_undegraded":"2023-09-23T18:19:34.390535+0000","last_fullsized":"2023-09-23T18:19:34.390535+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":131,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401303+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:19:34.401303+0000","last_peered":"2023-09-23T18:19:34.401303+0000","last_clean":"2023-09-23T18:19:34.401303+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:19:34.401303+0000","last_undegraded":"2023-09-23T18:19:34.401303+0000","last_fullsized":"2023-09-23T18:19:34.401303+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":431,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401192+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:19:34.401192+0000","last_peered":"2023-09-23T18:19:34.401192+0000","last_clean":"2023-09-23T18:19:34.401192+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:19:34.401192+0000","last_undegraded":"2023-09-23T18:19:34.401192+0000","last_fullsized":"2023-09-23T18:19:34.401192+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"607'7318","reported_seq":7359,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.118398+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:19:51.118398+0000","last_peered":"2023-09-23T18:19:51.118398+0000","last_clean":"2023-09-23T18:19:51.118398+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:19:51.118398+0000","last_undegraded":"2023-09-23T18:19:51.118398+0000","last_fullsized":"2023-09-23T18:19:51.118398+0000","mapping_epoch":574,"log_start":"584'1800","ondisk_log_start":"584'1800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5518,"log_dups_size":0,"ondisk_log_size":5518,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":345243648,"num_objects":5268,"num_object_clones":0,"num_object_copies":15804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7318,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":815,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390661+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:19:34.390661+0000","last_peered":"2023-09-23T18:19:34.390661+0000","last_clean":"2023-09-23T18:19:34.390661+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:19:34.390661+0000","last_undegraded":"2023-09-23T18:19:34.390661+0000","last_fullsized":"2023-09-23T18:19:34.390661+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":21,"reported_epoch":606,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:19:34.398494+0000","last_change":"2023-09-23T18:18:46.432707+0000","last_active":"2023-09-23T18:19:34.398494+0000","last_peered":"2023-09-23T18:19:34.398494+0000","last_clean":"2023-09-23T18:11:33.614822+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:19:34.398494+0000","last_undegraded":"2023-09-23T18:18:46.418246+0000","last_fullsized":"2023-09-23T18:19:34.398494+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":400,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401031+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:19:34.401031+0000","last_peered":"2023-09-23T18:19:34.401031+0000","last_clean":"2023-09-23T18:19:34.401031+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:19:34.401031+0000","last_undegraded":"2023-09-23T18:19:34.401031+0000","last_fullsized":"2023-09-23T18:19:34.401031+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":143,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391087+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:19:34.391087+0000","last_peered":"2023-09-23T18:19:34.391087+0000","last_clean":"2023-09-23T18:19:34.391087+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:19:34.391087+0000","last_undegraded":"2023-09-23T18:19:34.391087+0000","last_fullsized":"2023-09-23T18:19:34.391087+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"607'7462","reported_seq":7503,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.232324+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:19:51.232324+0000","last_peered":"2023-09-23T18:19:51.232324+0000","last_clean":"2023-09-23T18:19:51.232324+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:19:51.232324+0000","last_undegraded":"2023-09-23T18:19:51.232324+0000","last_fullsized":"2023-09-23T18:19:51.232324+0000","mapping_epoch":574,"log_start":"585'3000","ondisk_log_start":"585'3000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4462,"log_dups_size":0,"ondisk_log_size":4462,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":359530496,"num_objects":5486,"num_object_clones":0,"num_object_copies":16458,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5486,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7462,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1504,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390521+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:19:34.390521+0000","last_peered":"2023-09-23T18:19:34.390521+0000","last_clean":"2023-09-23T18:19:34.390521+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:19:34.390521+0000","last_undegraded":"2023-09-23T18:19:34.390521+0000","last_fullsized":"2023-09-23T18:19:34.390521+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":557,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400941+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:19:34.400941+0000","last_peered":"2023-09-23T18:19:34.400941+0000","last_clean":"2023-09-23T18:19:34.400941+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:19:34.400941+0000","last_undegraded":"2023-09-23T18:19:34.400941+0000","last_fullsized":"2023-09-23T18:19:34.400941+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390424+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:19:34.390424+0000","last_peered":"2023-09-23T18:19:34.390424+0000","last_clean":"2023-09-23T18:19:34.390424+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:19:34.390424+0000","last_undegraded":"2023-09-23T18:19:34.390424+0000","last_fullsized":"2023-09-23T18:19:34.390424+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":152,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.402349+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:19:34.402349+0000","last_peered":"2023-09-23T18:19:34.402349+0000","last_clean":"2023-09-23T18:19:34.402349+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:19:34.402349+0000","last_undegraded":"2023-09-23T18:19:34.402349+0000","last_fullsized":"2023-09-23T18:19:34.402349+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"607'7381","reported_seq":7422,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.101571+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:19:51.101571+0000","last_peered":"2023-09-23T18:19:51.101571+0000","last_clean":"2023-09-23T18:19:51.101571+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:19:51.101571+0000","last_undegraded":"2023-09-23T18:19:51.101571+0000","last_fullsized":"2023-09-23T18:19:51.101571+0000","mapping_epoch":574,"log_start":"584'1900","ondisk_log_start":"584'1900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5481,"log_dups_size":0,"ondisk_log_size":5481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":354091008,"num_objects":5403,"num_object_clones":0,"num_object_copies":16209,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5403,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7381,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":805,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401376+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:19:34.401376+0000","last_peered":"2023-09-23T18:19:34.401376+0000","last_clean":"2023-09-23T18:19:34.401376+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:19:34.401376+0000","last_undegraded":"2023-09-23T18:19:34.401376+0000","last_fullsized":"2023-09-23T18:19:34.401376+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":120,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398117+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:19:34.398117+0000","last_peered":"2023-09-23T18:19:34.398117+0000","last_clean":"2023-09-23T18:19:34.398117+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:19:34.398117+0000","last_undegraded":"2023-09-23T18:19:34.398117+0000","last_fullsized":"2023-09-23T18:19:34.398117+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398366+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:19:34.398366+0000","last_peered":"2023-09-23T18:19:34.398366+0000","last_clean":"2023-09-23T18:19:34.398366+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:19:34.398366+0000","last_undegraded":"2023-09-23T18:19:34.398366+0000","last_fullsized":"2023-09-23T18:19:34.398366+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":622,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390958+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:19:34.390958+0000","last_peered":"2023-09-23T18:19:34.390958+0000","last_clean":"2023-09-23T18:19:34.390958+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:19:34.390958+0000","last_undegraded":"2023-09-23T18:19:34.390958+0000","last_fullsized":"2023-09-23T18:19:34.390958+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"607'7496","reported_seq":9189,"reported_epoch":607,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:51.232351+0000","last_change":"2023-09-23T18:19:32.858380+0000","last_active":"2023-09-23T18:19:51.232351+0000","last_peered":"2023-09-23T18:19:51.232351+0000","last_clean":"2023-09-23T18:18:32.699126+0000","last_became_active":"2023-09-23T18:18:34.716591+0000","last_became_peered":"2023-09-23T18:18:34.716591+0000","last_unstale":"2023-09-23T18:19:51.232351+0000","last_undegraded":"2023-09-23T18:18:34.351246+0000","last_fullsized":"2023-09-23T18:18:34.327168+0000","mapping_epoch":589,"log_start":"585'3000","ondisk_log_start":"585'3000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4496,"log_dups_size":0,"ondisk_log_size":4496,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:13.611234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":357892120,"num_objects":5462,"num_object_clones":0,"num_object_copies":16386,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":201,"num_objects_misplaced":2432,"num_objects_unfound":0,"num_objects_dirty":5462,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":7496,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":820,"num_bytes_recovered":53739520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":2432},{"shards":"4,6","objects":2829}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":905,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401452+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:19:34.401452+0000","last_peered":"2023-09-23T18:19:34.401452+0000","last_clean":"2023-09-23T18:19:34.401452+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:19:34.401452+0000","last_undegraded":"2023-09-23T18:19:34.401452+0000","last_fullsized":"2023-09-23T18:19:34.401452+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":45,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396087+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:19:34.396087+0000","last_peered":"2023-09-23T18:19:34.396087+0000","last_clean":"2023-09-23T18:19:34.396087+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:19:34.396087+0000","last_undegraded":"2023-09-23T18:19:34.396087+0000","last_fullsized":"2023-09-23T18:19:34.396087+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":131,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390610+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:19:34.390610+0000","last_peered":"2023-09-23T18:19:34.390610+0000","last_clean":"2023-09-23T18:19:34.390610+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:19:34.390610+0000","last_undegraded":"2023-09-23T18:19:34.390610+0000","last_fullsized":"2023-09-23T18:19:34.390610+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":131,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390447+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:19:34.390447+0000","last_peered":"2023-09-23T18:19:34.390447+0000","last_clean":"2023-09-23T18:19:34.390447+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:19:34.390447+0000","last_undegraded":"2023-09-23T18:19:34.390447+0000","last_fullsized":"2023-09-23T18:19:34.390447+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"607'7487","reported_seq":7528,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.181716+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:19:51.181716+0000","last_peered":"2023-09-23T18:19:51.181716+0000","last_clean":"2023-09-23T18:19:51.181716+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:19:51.181716+0000","last_undegraded":"2023-09-23T18:19:51.181716+0000","last_fullsized":"2023-09-23T18:19:51.181716+0000","mapping_epoch":574,"log_start":"585'3000","ondisk_log_start":"585'3000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4487,"log_dups_size":0,"ondisk_log_size":4487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":351731712,"num_objects":5367,"num_object_clones":0,"num_object_copies":16101,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5367,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7487,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":905,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.402088+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:19:34.402088+0000","last_peered":"2023-09-23T18:19:34.402088+0000","last_clean":"2023-09-23T18:19:34.402088+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:19:34.402088+0000","last_undegraded":"2023-09-23T18:19:34.402088+0000","last_fullsized":"2023-09-23T18:19:34.402088+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":93,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391005+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:19:34.391005+0000","last_peered":"2023-09-23T18:19:34.391005+0000","last_clean":"2023-09-23T18:19:34.391005+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:19:34.391005+0000","last_undegraded":"2023-09-23T18:19:34.391005+0000","last_fullsized":"2023-09-23T18:19:34.391005+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":386,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401402+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:19:34.401402+0000","last_peered":"2023-09-23T18:19:34.401402+0000","last_clean":"2023-09-23T18:19:34.401402+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:19:34.401402+0000","last_undegraded":"2023-09-23T18:19:34.401402+0000","last_fullsized":"2023-09-23T18:19:34.401402+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":23,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.397898+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:34.397898+0000","last_peered":"2023-09-23T18:19:34.397898+0000","last_clean":"2023-09-23T18:19:34.397898+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:19:34.397898+0000","last_undegraded":"2023-09-23T18:19:34.397898+0000","last_fullsized":"2023-09-23T18:19:34.397898+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"607'7423","reported_seq":7464,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.228347+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:19:51.228347+0000","last_peered":"2023-09-23T18:19:51.228347+0000","last_clean":"2023-09-23T18:19:51.228347+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:19:51.228347+0000","last_undegraded":"2023-09-23T18:19:51.228347+0000","last_fullsized":"2023-09-23T18:19:51.228347+0000","mapping_epoch":574,"log_start":"585'3000","ondisk_log_start":"585'3000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4423,"log_dups_size":0,"ondisk_log_size":4423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":353173504,"num_objects":5389,"num_object_clones":0,"num_object_copies":16167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7423,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":739,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.398646+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:19:34.398646+0000","last_peered":"2023-09-23T18:19:34.398646+0000","last_clean":"2023-09-23T18:19:34.398646+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:19:34.398646+0000","last_undegraded":"2023-09-23T18:19:34.398646+0000","last_fullsized":"2023-09-23T18:19:34.398646+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1043,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401980+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:19:34.401980+0000","last_peered":"2023-09-23T18:19:34.401980+0000","last_clean":"2023-09-23T18:19:34.401980+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:19:34.401980+0000","last_undegraded":"2023-09-23T18:19:34.401980+0000","last_fullsized":"2023-09-23T18:19:34.401980+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":33,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396809+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:19:34.396809+0000","last_peered":"2023-09-23T18:19:34.396809+0000","last_clean":"2023-09-23T18:19:34.396809+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:19:34.396809+0000","last_undegraded":"2023-09-23T18:19:34.396809+0000","last_fullsized":"2023-09-23T18:19:34.396809+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":397,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390572+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:19:34.390572+0000","last_peered":"2023-09-23T18:19:34.390572+0000","last_clean":"2023-09-23T18:19:34.390572+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:19:34.390572+0000","last_undegraded":"2023-09-23T18:19:34.390572+0000","last_fullsized":"2023-09-23T18:19:34.390572+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":637,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400969+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:19:34.400969+0000","last_peered":"2023-09-23T18:19:34.400969+0000","last_clean":"2023-09-23T18:19:34.400969+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:19:34.400969+0000","last_undegraded":"2023-09-23T18:19:34.400969+0000","last_fullsized":"2023-09-23T18:19:34.400969+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":29,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.396456+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:19:34.396456+0000","last_peered":"2023-09-23T18:19:34.396456+0000","last_clean":"2023-09-23T18:19:34.396456+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:19:34.396456+0000","last_undegraded":"2023-09-23T18:19:34.396456+0000","last_fullsized":"2023-09-23T18:19:34.396456+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":681,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:48.891855+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:19:48.891855+0000","last_peered":"2023-09-23T18:19:48.891855+0000","last_clean":"2023-09-23T18:19:48.891855+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:19:48.891855+0000","last_undegraded":"2023-09-23T18:19:48.891855+0000","last_fullsized":"2023-09-23T18:19:48.891855+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":637,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401338+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:19:34.401338+0000","last_peered":"2023-09-23T18:19:34.401338+0000","last_clean":"2023-09-23T18:19:34.401338+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:19:34.401338+0000","last_undegraded":"2023-09-23T18:19:34.401338+0000","last_fullsized":"2023-09-23T18:19:34.401338+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":125,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390701+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:19:34.390701+0000","last_peered":"2023-09-23T18:19:34.390701+0000","last_clean":"2023-09-23T18:19:34.390701+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:19:34.390701+0000","last_undegraded":"2023-09-23T18:19:34.390701+0000","last_fullsized":"2023-09-23T18:19:34.390701+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":242,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:42.597529+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:19:42.597529+0000","last_peered":"2023-09-23T18:19:42.597529+0000","last_clean":"2023-09-23T18:19:42.597529+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:19:42.597529+0000","last_undegraded":"2023-09-23T18:19:42.597529+0000","last_fullsized":"2023-09-23T18:19:42.597529+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":111,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390943+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:19:34.390943+0000","last_peered":"2023-09-23T18:19:34.390943+0000","last_clean":"2023-09-23T18:19:34.390943+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:19:34.390943+0000","last_undegraded":"2023-09-23T18:19:34.390943+0000","last_fullsized":"2023-09-23T18:19:34.390943+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":225,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:42.596969+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:19:42.596969+0000","last_peered":"2023-09-23T18:19:42.596969+0000","last_clean":"2023-09-23T18:19:42.596969+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:19:42.596969+0000","last_undegraded":"2023-09-23T18:19:42.596969+0000","last_fullsized":"2023-09-23T18:19:42.596969+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":999,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400953+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:19:34.400953+0000","last_peered":"2023-09-23T18:19:34.400953+0000","last_clean":"2023-09-23T18:19:34.400953+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:19:34.400953+0000","last_undegraded":"2023-09-23T18:19:34.400953+0000","last_fullsized":"2023-09-23T18:19:34.400953+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":504,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.401453+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:19:34.401453+0000","last_peered":"2023-09-23T18:19:34.401453+0000","last_clean":"2023-09-23T18:19:34.401453+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:19:34.401453+0000","last_undegraded":"2023-09-23T18:19:34.401453+0000","last_fullsized":"2023-09-23T18:19:34.401453+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":141,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.390651+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:19:34.390651+0000","last_peered":"2023-09-23T18:19:34.390651+0000","last_clean":"2023-09-23T18:19:34.390651+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:19:34.390651+0000","last_undegraded":"2023-09-23T18:19:34.390651+0000","last_fullsized":"2023-09-23T18:19:34.390651+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":937,"reported_epoch":607,"state":"active+clean","last_fresh":"2023-09-23T18:19:48.891897+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:19:48.891897+0000","last_peered":"2023-09-23T18:19:48.891897+0000","last_clean":"2023-09-23T18:19:48.891897+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:19:48.891897+0000","last_undegraded":"2023-09-23T18:19:48.891897+0000","last_fullsized":"2023-09-23T18:19:48.891897+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":129,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.391058+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:19:34.391058+0000","last_peered":"2023-09-23T18:19:34.391058+0000","last_clean":"2023-09-23T18:19:34.391058+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:19:34.391058+0000","last_undegraded":"2023-09-23T18:19:34.391058+0000","last_fullsized":"2023-09-23T18:19:34.391058+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":650,"reported_epoch":606,"state":"active+clean","last_fresh":"2023-09-23T18:19:34.400629+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:19:34.400629+0000","last_peered":"2023-09-23T18:19:34.400629+0000","last_clean":"2023-09-23T18:19:34.400629+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:19:34.400629+0000","last_undegraded":"2023-09-23T18:19:34.400629+0000","last_fullsized":"2023-09-23T18:19:34.400629+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":5696913432,"num_objects":86929,"num_object_clones":0,"num_object_copies":260787,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16419,"num_objects_misplaced":2432,"num_objects_unfound":0,"num_objects_dirty":86929,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":119879,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4291,"num_bytes_recovered":281214976,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":17803128832,"data_stored":17803116616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77079,"ondisk_log_size":77079,"up":48,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001567,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669503,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3325744,"kb_used_data":2720040,"kb_used_omap":7,"kb_used_meta":605688,"kb_avail":90419408,"statfs":{"total":95995035648,"available":92589473792,"internally_reserved":0,"allocated":2785320960,"data_stored":2783406523,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":620225316},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":1.599,"15min":1.2949999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":24.472999999999999,"5min":24.472999999999999,"15min":24.472999999999999},"last":0.249},{"interface":"front","average":{"1min":3.6230000000000002,"5min":2.081,"15min":1.663},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":22.832999999999998,"5min":22.832999999999998,"15min":22.832999999999998},"last":5.8959999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:19:21 2023","interfaces":[{"interface":"back","average":{"1min":4.1740000000000004,"5min":1.8069999999999999,"15min":1.413},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":22.972000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.57499999999999996},{"interface":"front","average":{"1min":3.448,"5min":1.6559999999999999,"15min":1.357},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":22.664000000000001,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.55800000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.45,"5min":1.403,"15min":1.2869999999999999},"min":{"1min":0.26800000000000002,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":6.2679999999999998,"5min":7.4189999999999996,"15min":7.4189999999999996},"last":8.0559999999999992},{"interface":"front","average":{"1min":1.927,"5min":1.7470000000000001,"15min":1.454},"min":{"1min":0.311,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0439999999999996,"5min":14.771000000000001,"15min":14.771000000000001},"last":5.6760000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":1.446,"5min":1.6459999999999999,"15min":1.5509999999999999},"min":{"1min":0.28699999999999998,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":6.3150000000000004,"5min":7.3940000000000001,"15min":10.257999999999999},"last":4.3170000000000002},{"interface":"front","average":{"1min":2.282,"5min":2.165,"15min":1.829},"min":{"1min":0.28999999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":7.9539999999999997,"5min":9.4049999999999994,"15min":9.5960000000000001},"last":1.7130000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.8140000000000001,"15min":1.3280000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.2800000000000002,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.7569999999999999,"5min":1.9330000000000001,"15min":1.3200000000000001},"min":{"1min":0.36099999999999999,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":6.2030000000000003,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.97199999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:19:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":2.0609999999999999,"15min":1.9550000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":6.4710000000000001,"5min":13.502000000000001,"15min":13.502000000000001},"last":0.35899999999999999},{"interface":"front","average":{"1min":1.3720000000000001,"5min":2.246,"15min":2.3660000000000001},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.1070000000000002,"5min":10.081,"15min":10.081},"last":0.627}]},{"osd":7,"last update":"Sat Sep 23 18:18:53 2023","interfaces":[{"interface":"back","average":{"1min":2.109,"5min":2.1930000000000001,"15min":1.4970000000000001},"min":{"1min":0.45700000000000002,"5min":0.36299999999999999,"15min":0.26800000000000002},"max":{"1min":5.851,"5min":10.743,"15min":13.460000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":1.702,"5min":2.21,"15min":1.536},"min":{"1min":0.46400000000000002,"5min":0.32900000000000001,"15min":0.28499999999999998},"max":{"1min":6.2409999999999997,"5min":11.244,"15min":12.404},"last":0.59099999999999997}]}]},{"osd":1,"up_from":545,"seq":2340757176352,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2243144,"kb_used_data":2023040,"kb_used_omap":13,"kb_used_meta":220018,"kb_avail":91502008,"statfs":{"total":95995035648,"available":93698056192,"internally_reserved":0,"allocated":2071592960,"data_stored":2068619686,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":225299085},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.776,"5min":0.66800000000000004,"15min":0.48299999999999998},"min":{"1min":0.221,"5min":0.189,"15min":0.189},"max":{"1min":5.8079999999999998,"5min":5.8079999999999998,"15min":5.8079999999999998},"last":0.33400000000000002},{"interface":"front","average":{"1min":1.2430000000000001,"5min":0.64000000000000001,"15min":0.54000000000000004},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6159999999999997,"5min":4.6159999999999997,"15min":4.6159999999999997},"last":0.77500000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":0.67900000000000005,"15min":0.53300000000000003},"min":{"1min":0.28299999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9290000000000003,"5min":4.9290000000000003,"15min":4.9290000000000003},"last":0.94799999999999995},{"interface":"front","average":{"1min":1.272,"5min":0.65800000000000003,"15min":0.55600000000000005},"min":{"1min":0.29899999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2759999999999998,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":0.63400000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":0.746,"15min":0.50600000000000001},"min":{"1min":0.34300000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.3219999999999992,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":0.28999999999999998},{"interface":"front","average":{"1min":2.2429999999999999,"5min":0.85799999999999998,"15min":0.627},"min":{"1min":0.38500000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":8.3659999999999997},"last":0.65700000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":1.347,"15min":1.327},"min":{"1min":0.40200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.6609999999999996,"5min":12.663,"15min":12.663},"last":0.60999999999999999},{"interface":"front","average":{"1min":1.95,"5min":1.601,"15min":1.5429999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.1769999999999996,"5min":13.222,"15min":13.222},"last":0.34399999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":0.88200000000000001,"15min":0.77600000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":7.4870000000000001},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.0589999999999999,"15min":0.88900000000000001},"min":{"1min":0.46000000000000002,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":8.3190000000000008,"5min":8.3190000000000008,"15min":8.3190000000000008},"last":7.5460000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":0.997,"15min":0.83899999999999997},"min":{"1min":0.45400000000000001,"5min":0.435,"15min":0.435},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":1.095},{"interface":"front","average":{"1min":2.6070000000000002,"5min":1.1419999999999999,"15min":0.89800000000000002},"min":{"1min":0.38900000000000001,"5min":0.378,"15min":0.378},"max":{"1min":9.2579999999999991,"5min":9.2579999999999991,"15min":9.2579999999999991},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":1.0109999999999999,"15min":0.79100000000000004},"min":{"1min":0.436,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.6419999999999999,"5min":1.173,"15min":0.92800000000000005},"min":{"1min":0.48099999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":9.0640000000000001,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":0.67500000000000004}]}]},{"osd":0,"up_from":93,"seq":399431958848,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4421344,"kb_used_data":3512024,"kb_used_omap":7,"kb_used_meta":909240,"kb_avail":89323808,"statfs":{"total":95995035648,"available":91467579392,"internally_reserved":0,"allocated":3596312576,"data_stored":3593487928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":931062500},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:19:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7229999999999999,"5min":0.80000000000000004,"15min":0.47999999999999998},"min":{"1min":0.214,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.2309999999999999,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.28399999999999997},{"interface":"front","average":{"1min":2.2010000000000001,"5min":0.85799999999999998,"15min":0.63400000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.714,"5min":13.714,"15min":13.714},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:19:00 2023","interfaces":[{"interface":"back","average":{"1min":2.7610000000000001,"5min":1.1519999999999999,"15min":1.1299999999999999},"min":{"1min":0.27000000000000002,"5min":0.185,"15min":0.185},"max":{"1min":18.550999999999998,"5min":18.550999999999998,"15min":18.550999999999998},"last":2.3780000000000001},{"interface":"front","average":{"1min":1.6100000000000001,"5min":0.84199999999999997,"15min":0.79600000000000004},"min":{"1min":0.29299999999999998,"5min":0.22,"15min":0.22},"max":{"1min":13.618,"5min":13.618,"15min":13.618},"last":2.2400000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7610000000000001,"5min":1.393,"15min":1.637},"min":{"1min":0.38400000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.539999999999999,"5min":13.539999999999999,"15min":13.539999999999999},"last":0.375},{"interface":"front","average":{"1min":4.1390000000000002,"5min":1.4930000000000001,"15min":1.637},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.285,"5min":13.285,"15min":15.198},"last":3.6200000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.2599999999999998,"15min":1.7789999999999999},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":13.574999999999999,"5min":57.746000000000002,"15min":57.746000000000002},"last":2.8690000000000002},{"interface":"front","average":{"1min":2.032,"5min":1.9770000000000001,"15min":1.6499999999999999},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.8970000000000002,"5min":57.816000000000003,"15min":57.816000000000003},"last":2.9159999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.032,"15min":1.391},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":0.40899999999999997},{"interface":"front","average":{"1min":4.2859999999999996,"5min":2.2650000000000001,"15min":1.448},"min":{"1min":0.502,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":13.425000000000001},"last":3.1269999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":1.8049999999999999,"15min":1.861},"min":{"1min":0.433,"5min":0.374,"15min":0.32000000000000001},"max":{"1min":7.1020000000000003,"5min":7.1020000000000003,"15min":58.457999999999998},"last":0.495},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.798,"15min":1.7050000000000001},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":13.672000000000001,"5min":13.672000000000001,"15min":58.354999999999997},"last":0.71099999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.101,"5min":1.6719999999999999,"15min":1.488},"min":{"1min":0.45600000000000002,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":3.1459999999999999,"5min":8.5009999999999994,"15min":10.416},"last":0.47299999999999998},{"interface":"front","average":{"1min":0.88100000000000001,"5min":1.5609999999999999,"15min":1.6759999999999999},"min":{"1min":0.48199999999999998,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":3.6139999999999999,"5min":10.404,"15min":10.404},"last":2.8210000000000002}]}]},{"osd":2,"up_from":0,"seq":2070174236734,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249363,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2527348,"kb_used_data":1818604,"kb_used_omap":3,"kb_used_meta":708604,"kb_avail":91217804,"statfs":{"total":95995035648,"available":93407031296,"internally_reserved":0,"allocated":1862250496,"data_stored":1859231150,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":725611240},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3919999999999999,"5min":1.25,"15min":1.7050000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.183},"max":{"1min":17.036000000000001,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.42199999999999999},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.1599999999999999,"15min":1.96},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.0980000000000008,"5min":9.0980000000000008,"15min":194.661},"last":0.34599999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":0.95799999999999996,"15min":0.66600000000000004},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":21.777000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.32700000000000001},{"interface":"front","average":{"1min":6.5110000000000001,"5min":1.8660000000000001,"15min":1.0920000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":78.980000000000004,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.39000000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:18:54 2023","interfaces":[{"interface":"back","average":{"1min":0.96899999999999997,"5min":0.94299999999999995,"15min":1.179},"min":{"1min":0.308,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":2.597,"5min":6.3849999999999998,"15min":6.3849999999999998},"last":0.41199999999999998},{"interface":"front","average":{"1min":1.0700000000000001,"5min":0.93100000000000005,"15min":1.21},"min":{"1min":0.33100000000000002,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":4.1040000000000001,"5min":5.931,"15min":5.931},"last":0.44900000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.6140000000000001,"15min":1.659},"min":{"1min":0.31900000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2300000000000004,"5min":21.062999999999999,"15min":37.231000000000002},"last":0.58899999999999997},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.9810000000000001,"15min":1.8939999999999999},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.2959999999999994,"5min":21.128,"15min":37.381},"last":0.44}]},{"osd":5,"last update":"Sat Sep 23 18:19:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.8009999999999999,"15min":1.4419999999999999},"min":{"1min":0.40999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":11.805999999999999,"5min":11.805999999999999,"15min":11.805999999999999},"last":4.6390000000000002},{"interface":"front","average":{"1min":2.028,"5min":1.601,"15min":1.6339999999999999},"min":{"1min":0.44500000000000001,"5min":0.33100000000000002,"15min":0.32000000000000001},"max":{"1min":8.9359999999999999,"5min":8.9359999999999999,"15min":8.9969999999999999},"last":1.411}]},{"osd":6,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":1.7809999999999999,"15min":1.847},"min":{"1min":0.432,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":17.192,"5min":17.192,"15min":115.54000000000001},"last":0.99399999999999999},{"interface":"front","average":{"1min":2.2530000000000001,"5min":1.7250000000000001,"15min":1.948},"min":{"1min":0.37,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":8.0350000000000001,"5min":14.321,"15min":115.691},"last":0.45800000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.5069999999999999,"15min":1.7110000000000001},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":8.2070000000000007,"5min":8.2070000000000007,"15min":15.972},"last":0.70499999999999996},{"interface":"front","average":{"1min":3.5920000000000001,"5min":1.7629999999999999,"15min":1.623},"min":{"1min":0.48799999999999999,"5min":0.41999999999999998,"15min":0.34899999999999998},"max":{"1min":17.164000000000001,"5min":17.164000000000001,"15min":17.164000000000001},"last":2.0790000000000002}]}]},{"osd":4,"up_from":30,"seq":128849019313,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5072700,"kb_used_data":4426228,"kb_used_omap":0,"kb_used_meta":646464,"kb_avail":88672452,"statfs":{"total":95995035648,"available":90800590848,"internally_reserved":0,"allocated":4532457472,"data_stored":4530538844,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":661979136},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":1.375,"15min":1.21},"min":{"1min":0.27200000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9660000000000002,"5min":7.2199999999999998,"15min":13.375999999999999},"last":5.798},{"interface":"front","average":{"1min":2.2709999999999999,"5min":1.772,"15min":1.4710000000000001},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":8.9429999999999996,"5min":10.890000000000001,"15min":10.890000000000001},"last":6.5540000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:19:22 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":1.47,"15min":0.95299999999999996},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":44.146999999999998,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.86599999999999999},{"interface":"front","average":{"1min":4.5419999999999998,"5min":1.4299999999999999,"15min":0.91100000000000003},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":43.497999999999998,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.61399999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:18:56 2023","interfaces":[{"interface":"back","average":{"1min":1.8149999999999999,"5min":1.524,"15min":1.7689999999999999},"min":{"1min":0.30299999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.173,"5min":11.173,"15min":11.173},"last":7.0250000000000004},{"interface":"front","average":{"1min":2.1190000000000002,"5min":1.6459999999999999,"15min":1.702},"min":{"1min":0.40600000000000003,"5min":0.253,"15min":0.253},"max":{"1min":9.0909999999999993,"5min":10.938000000000001,"15min":10.938000000000001},"last":4.6050000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.5669999999999999,"15min":1.9350000000000001},"min":{"1min":0.33700000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.0609999999999999,"5min":11.130000000000001,"15min":29.154},"last":5.976},{"interface":"front","average":{"1min":2.1800000000000002,"5min":1.8480000000000001,"15min":1.9390000000000001},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":13.470000000000001,"5min":13.470000000000001,"15min":29.216000000000001},"last":1.7769999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":2.4279999999999999,"15min":2.8559999999999999},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4950000000000001,"5min":27.675000000000001,"15min":27.675000000000001},"last":1.6899999999999999},{"interface":"front","average":{"1min":1.401,"5min":1.827,"15min":2.7109999999999999},"min":{"1min":0.30499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.1040000000000001,"5min":13.938000000000001,"15min":32.640000000000001},"last":0.83399999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":2.0739999999999998,"15min":1.391},"min":{"1min":0.379,"5min":0.28199999999999997,"15min":0.25},"max":{"1min":7.7249999999999996,"5min":18.315000000000001,"15min":18.315000000000001},"last":3.7280000000000002},{"interface":"front","average":{"1min":1.782,"5min":2.044,"15min":1.51},"min":{"1min":0.33900000000000002,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.6310000000000002,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.90000000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":2.7919999999999998,"15min":1.7490000000000001},"min":{"1min":0.42699999999999999,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":8.9849999999999994,"5min":19.298999999999999,"15min":19.298999999999999},"last":1.7450000000000001},{"interface":"front","average":{"1min":2.234,"5min":2.1549999999999998,"15min":1.5569999999999999},"min":{"1min":0.50900000000000001,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.1319999999999997,"5min":12.800000000000001,"15min":12.800000000000001},"last":4.6340000000000003}]}]},{"osd":5,"up_from":420,"seq":1803886264409,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3339232,"kb_used_data":3009752,"kb_used_omap":9,"kb_used_meta":329462,"kb_avail":90405920,"statfs":{"total":95995035648,"available":92575662080,"internally_reserved":0,"allocated":3081986048,"data_stored":3080083747,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":337369533},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":1.2549999999999999,"15min":0.88100000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.6000000000000001,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":2.7170000000000001},{"interface":"front","average":{"1min":2.911,"5min":1.8149999999999999,"15min":1.6080000000000001},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":20.937999999999999,"5min":20.937999999999999,"15min":20.937999999999999},"last":0.252}]},{"osd":1,"last update":"Sat Sep 23 18:19:20 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":1.4770000000000001,"15min":1.2230000000000001},"min":{"1min":0.32100000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.946000000000002,"5min":23.946000000000002,"15min":23.946000000000002},"last":2.8090000000000002},{"interface":"front","average":{"1min":3.927,"5min":1.4630000000000001,"15min":1.052},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":23.268999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":4.6239999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:18:52 2023","interfaces":[{"interface":"back","average":{"1min":1.262,"5min":1.4590000000000001,"15min":1.5589999999999999},"min":{"1min":0.34300000000000003,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":4.4930000000000003,"5min":7.6020000000000003,"15min":7.6020000000000003},"last":7.6769999999999996},{"interface":"front","average":{"1min":1.5369999999999999,"5min":1.891,"15min":2.141},"min":{"1min":0.314,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.6749999999999998,"5min":7.2359999999999998,"15min":7.2359999999999998},"last":0.499}]},{"osd":3,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":1.8740000000000001,"15min":1.2789999999999999},"min":{"1min":0.443,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.8780000000000001,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":4.7830000000000004},{"interface":"front","average":{"1min":3.7320000000000002,"5min":2.0640000000000001,"15min":1.665},"min":{"1min":0.38100000000000001,"5min":0.315,"15min":0.313},"max":{"1min":20.035,"5min":20.035,"15min":20.035},"last":4.3929999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.2770000000000001,"15min":2.0640000000000001},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":7.875,"5min":12.747999999999999,"15min":12.747999999999999},"last":3.2210000000000001},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.0899999999999999,"15min":1.6160000000000001},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7610000000000001,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":2.7839999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":1.9650000000000001,"15min":1.2609999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2480000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":2.7549999999999999},{"interface":"front","average":{"1min":2.343,"5min":2.2040000000000002,"15min":1.6819999999999999},"min":{"1min":0.34599999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.8410000000000002,"5min":12.504,"15min":12.504},"last":0.52800000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.3980000000000001,"15min":1.758},"min":{"1min":0.40699999999999997,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.528,"5min":13.506,"15min":13.506},"last":3.2469999999999999},{"interface":"front","average":{"1min":1.381,"5min":1.9630000000000001,"15min":1.542},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":5.0970000000000004,"5min":13.138,"15min":13.138},"last":3.0659999999999998}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3558084608,"data_stored":3558080536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":2049245184,"data_stored":2049245184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1853489152,"data_stored":1853489152,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4503179264,"data_stored":4503175192,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":3066363904,"data_stored":3066363904,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2772766720,"data_stored":2772762648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:19:56.974 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:19:56.975 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:56 smithi099 ceph-mon[167076]: pgmap v1446: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.3 GiB data, 20 GiB used, 516 GiB / 536 GiB avail; 0 B/s wr, 895 op/s; 16444/260868 objects degraded (6.304%); 2432/260868 objects misplaced (0.932%); 2.7 MiB/s, 42 objects/s recovering 2023-09-23T18:19:57.150 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:57.157 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:57.157 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:57.258 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:19:57.339 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:57.361 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:57.369 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:19:57.460 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:19:57.469 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:57.469 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:19:57.570 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:57.597 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:57.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:57 smithi055 ceph-mon[184174]: from='client.56632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:57.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:57 smithi055 ceph-mon[184174]: pgmap v1447: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 877 op/s; 16454/250026 objects degraded (6.581%); 2179/250026 objects misplaced (0.872%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T18:19:57.775 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:57 smithi055 ceph-mon[179668]: from='client.56632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:57.775 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:57 smithi055 ceph-mon[179668]: pgmap v1447: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 877 op/s; 16454/250026 objects degraded (6.581%); 2179/250026 objects misplaced (0.872%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T18:19:58.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:57 smithi099 ceph-mon[167076]: from='client.56632 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:19:58.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:57 smithi099 ceph-mon[167076]: pgmap v1447: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.1 GiB data, 20 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 877 op/s; 16454/250026 objects degraded (6.581%); 2179/250026 objects misplaced (0.872%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T18:19:58.317 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:58.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:58.324 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:58.424 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:19:58.437 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:19:58.472 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:58.526 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:19:58.531 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:58.531 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:19:58.632 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:58.694 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:58.762 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:19:58.765 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:58.765 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:19:58.866 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:19:58.894 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:58.975 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:19:58.982 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:19:58.982 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:19:58.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16454/250026 objects degraded (6.581%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:58.986 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:58 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1209214555' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:59.083 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:19:59.124 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16454/250026 objects degraded (6.581%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:59.124 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:58 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1209214555' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:59.125 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16454/250026 objects degraded (6.581%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:19:59.125 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:58 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1209214555' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:19:59.174 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:19:59.205 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:19:59.247 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:19:59.279 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:19:59.309 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:19:59.355 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:19:59.394 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:19:59.403 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:19:59.447 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:19:59.479 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:19:59.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:19:59.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:19:59.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:19:59.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:19:59.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:19:59.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:19:59.502 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:19:59.503 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:19:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[184174]: pgmap v1448: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 875 op/s; 16454/248430 objects degraded (6.623%); 2179/248430 objects misplaced (0.877%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T18:19:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1940198912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[184174]: failed to encode map e609 with expected crc 2023-09-23T18:19:59.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1209214555' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:59.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[184174]: osdmap e609: 8 total, 8 up, 6 in 2023-09-23T18:19:59.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[179668]: pgmap v1448: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 875 op/s; 16454/248430 objects degraded (6.623%); 2179/248430 objects misplaced (0.877%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T18:19:59.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1940198912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:19:59.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:19:59.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[179668]: failed to encode map e609 with expected crc 2023-09-23T18:19:59.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1209214555' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:19:59.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:19:59 smithi055 ceph-mon[179668]: osdmap e609: 8 total, 8 up, 6 in 2023-09-23T18:20:00.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:59 smithi099 ceph-mon[167076]: pgmap v1448: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.1 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 875 op/s; 16454/248430 objects degraded (6.623%); 2179/248430 objects misplaced (0.877%); 2.9 MiB/s, 45 objects/s recovering 2023-09-23T18:20:00.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:59 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1940198912' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:00.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:59 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:00.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:59 smithi099 ceph-mon[167076]: failed to encode map e609 with expected crc 2023-09-23T18:20:00.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:59 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1209214555' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:20:00.016 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:19:59 smithi099 ceph-mon[167076]: osdmap e609: 8 total, 8 up, 6 in 2023-09-23T18:20:00.370 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:20:00.371 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:20:00.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:20:00] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.43.0" 2023-09-23T18:20:00.748 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:20:01.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: failed to encode map e609 with expected crc 2023-09-23T18:20:01.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 16454/248430 objects degraded (6.623%), 7 pgs degraded, 6 pgs undersized 2023-09-23T18:20:01.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T18:20:01.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: [WRN] PG_DEGRADED: Degraded data redundancy: 16454/248430 objects degraded (6.623%), 7 pgs degraded, 6 pgs undersized 2023-09-23T18:20:01.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: pg 2.1 is active+recovery_wait+degraded, acting [3,0,1] 2023-09-23T18:20:01.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: pg 2.4 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-09-23T18:20:01.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: pg 10.3 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,4] 2023-09-23T18:20:01.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: pg 10.7 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2023-09-23T18:20:01.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: pg 10.a is stuck undersized for 84s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2023-09-23T18:20:01.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: pg 10.c is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-09-23T18:20:01.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:00 smithi099 ceph-mon[167076]: pg 10.d is stuck undersized for 84s, current state active+recovering+undersized+degraded+remapped, last acting [6,4] 2023-09-23T18:20:01.128 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: failed to encode map e609 with expected crc 2023-09-23T18:20:01.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 16454/248430 objects degraded (6.623%), 7 pgs degraded, 6 pgs undersized 2023-09-23T18:20:01.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T18:20:01.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: [WRN] PG_DEGRADED: Degraded data redundancy: 16454/248430 objects degraded (6.623%), 7 pgs degraded, 6 pgs undersized 2023-09-23T18:20:01.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: pg 2.1 is active+recovery_wait+degraded, acting [3,0,1] 2023-09-23T18:20:01.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: pg 2.4 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-09-23T18:20:01.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: pg 10.3 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,4] 2023-09-23T18:20:01.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: pg 10.7 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2023-09-23T18:20:01.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: pg 10.a is stuck undersized for 84s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2023-09-23T18:20:01.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: pg 10.c is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-09-23T18:20:01.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[179668]: pg 10.d is stuck undersized for 84s, current state active+recovering+undersized+degraded+remapped, last acting [6,4] 2023-09-23T18:20:01.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: failed to encode map e609 with expected crc 2023-09-23T18:20:01.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: Health detail: HEALTH_WARN noscrub,nodeep-scrub flag(s) set; Degraded data redundancy: 16454/248430 objects degraded (6.623%), 7 pgs degraded, 6 pgs undersized 2023-09-23T18:20:01.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: [WRN] OSDMAP_FLAGS: noscrub,nodeep-scrub flag(s) set 2023-09-23T18:20:01.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: [WRN] PG_DEGRADED: Degraded data redundancy: 16454/248430 objects degraded (6.623%), 7 pgs degraded, 6 pgs undersized 2023-09-23T18:20:01.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: pg 2.1 is active+recovery_wait+degraded, acting [3,0,1] 2023-09-23T18:20:01.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: pg 2.4 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [1,0] 2023-09-23T18:20:01.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: pg 10.3 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [5,4] 2023-09-23T18:20:01.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: pg 10.7 is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2023-09-23T18:20:01.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: pg 10.a is stuck undersized for 84s, current state active+recovery_wait+undersized+degraded+remapped, last acting [6,5] 2023-09-23T18:20:01.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: pg 10.c is stuck undersized for 71s, current state active+recovery_wait+undersized+degraded+remapped, last acting [0,4] 2023-09-23T18:20:01.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:00 smithi055 ceph-mon[184174]: pg 10.d is stuck undersized for 84s, current state active+recovering+undersized+degraded+remapped, last acting [6,4] 2023-09-23T18:20:01.393 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:01.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:01.401 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:01.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:01.569 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:01.594 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:01.647 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:01.660 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:01.661 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:01.761 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:01.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:01 smithi099 ceph-mon[167076]: pgmap v1450: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 16454/243588 objects degraded (6.755%); 2179/243588 objects misplaced (0.895%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:20:01.836 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:01.901 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:01.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:01.906 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:01.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:01 smithi055 ceph-mon[184174]: pgmap v1450: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 16454/243588 objects degraded (6.755%); 2179/243588 objects misplaced (0.895%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:20:01.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:01 smithi055 ceph-mon[179668]: pgmap v1450: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 5.0 GiB data, 19 GiB used, 517 GiB / 536 GiB avail; 0 B/s wr, 664 op/s; 16454/243588 objects degraded (6.755%); 2179/243588 objects misplaced (0.895%); 1.6 MiB/s, 25 objects/s recovering 2023-09-23T18:20:02.007 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:02.094 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:02.177 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:02.187 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:02.187 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:02.288 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:02.363 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:02.427 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:02.432 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:02.432 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:02.533 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:02.574 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:20:02.575 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 16454/243588 objects degraded (6.755%), 7 pgs degraded, 6 pgs undersized","count":13},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1692,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":609,"num_osds":8,"num_up_osds":8,"osd_up_since":1695493039,"num_in_osds":6,"osd_in_since":1695493021,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":112},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":81196,"data_bytes":5323897532,"bytes_used":20423348224,"bytes_avail":555546865664,"bytes_total":575970213888,"degraded_objects":16454,"degraded_total":243588,"degraded_ratio":0.067548483504934559,"misplaced_objects":2179,"misplaced_total":243588,"misplaced_ratio":0.0089454324515164958,"recovering_objects_per_sec":25,"recovering_bytes_per_sec":1674651,"recovering_keys_per_sec":0,"num_objects_recovered":243,"num_bytes_recovered":15925248,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":664},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":32,"modified":"2023-09-23T18:18:19.098325+0000","services":{}},"progress_events":{"57c27436-1bc8-4f63-b41f-a80b88a05748":{"message":"Global Recovery Event (60s)\n [==========================..] (remaining: 3s)","progress":0.94117647409439087,"add_to_ceph_s":true},"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T18:20:02.628 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:02.732 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:02.738 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:02.738 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:02.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/25670308' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:20:02.839 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:02.881 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/25670308' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:20:02.882 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/25670308' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:20:02.891 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:02.959 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:02.966 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:02.966 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:03.031 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:20:03.031 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:20:03.067 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:03.131 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:03.164 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:20:03.165 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:20:03.221 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:03.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:03.234 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:03.335 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:03.411 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:03.441 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:20:03.477 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:03.484 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:03.484 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:03.584 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:20:03.585 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:03.669 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:03.698 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:20:03.741 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:03.756 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:03.757 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:03.858 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:03.868 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:03.929 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:03.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:03 smithi055 ceph-mon[184174]: pgmap v1451: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 16493/236331 objects degraded (6.979%); 1918/236331 objects misplaced (0.812%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T18:20:03.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16454/243588 objects degraded (6.755%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:03.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:03 smithi055 ceph-mon[179668]: pgmap v1451: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 16493/236331 objects degraded (6.979%); 1918/236331 objects misplaced (0.812%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T18:20:03.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16454/243588 objects degraded (6.755%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:04.030 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:04.038 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:04.039 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:04.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:03 smithi099 ceph-mon[167076]: pgmap v1451: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 16493/236331 objects degraded (6.979%); 1918/236331 objects misplaced (0.812%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T18:20:04.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16454/243588 objects degraded (6.755%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:04.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:04.179 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:04.240 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:04.245 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:04.245 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:04.346 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:04.392 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:04.427 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:04.434 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:04.481 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:04.486 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:04.487 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:04.587 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:04.660 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:04.958 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:04.983 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:04.994 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:04.995 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:05.095 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:05.169 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:05.541 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:05.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:05.548 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:05.649 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:05.771 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:05.845 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:20:05.845 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:20:06.294 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:20:06.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:06 smithi055 ceph-mon[184174]: pgmap v1452: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 16493/235485 objects degraded (7.004%); 1918/235485 objects misplaced (0.814%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T18:20:06.431 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:06 smithi055 ceph-mon[179668]: pgmap v1452: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 16493/235485 objects degraded (7.004%); 1918/235485 objects misplaced (0.814%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T18:20:06.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:06 smithi099 ceph-mon[167076]: pgmap v1452: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.8 GiB data, 19 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 845 op/s; 16493/235485 objects degraded (7.004%); 1918/235485 objects misplaced (0.814%); 2.9 MiB/s, 46 objects/s recovering 2023-09-23T18:20:06.898 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:06.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:06.904 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:07.004 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:07.062 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:07.094 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:07.212 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:07.222 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:07.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:07.323 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:07.385 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:07.464 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:07.472 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:07.472 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:07.472 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:07.473 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:07.473 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:07.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:07.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:07.501 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:07.501 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:20:07.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:07.502 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:07.558 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:20:07.574 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:07.574 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:20:07.634 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:07.708 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:07.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:07.715 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:07.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:07.873 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:20:07.873 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:07.904 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:20:07.935 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:20:07.965 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:20:08.008 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:08.017 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:08.018 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:08.029 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:20:08.065 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:20:08.095 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:20:08.119 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:08.131 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1453,"stamp":"2023-09-23T18:20:07.152035+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4894505660,"num_objects":74644,"num_object_clones":0,"num_object_copies":223932,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":16502,"num_objects_misplaced":1644,"num_objects_unfound":0,"num_objects_dirty":74644,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":133102,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5407,"num_bytes_recovered":344284809,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83613,"ondisk_log_size":83613,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":19018412,"kb_used_data":15599932,"kb_used_omap":40,"kb_used_meta":3417943,"kb_avail":543452500,"statfs":{"total":575970213888,"available":556495360000,"internally_reserved":0,"allocated":15974330368,"data_stored":15959726696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3499973946},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-570032128,"num_objects":-8698,"num_object_clones":0,"num_object_copies":-26094,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":48,"num_objects_misplaced":-535,"num_objects_unfound":0,"num_objects_dirty":-8698,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8698,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":487,"num_bytes_recovered":31916032,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.004657"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":897,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651644+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:19:51.651644+0000","last_peered":"2023-09-23T18:19:51.651644+0000","last_clean":"2023-09-23T18:19:51.651644+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:19:51.651644+0000","last_undegraded":"2023-09-23T18:19:51.651644+0000","last_fullsized":"2023-09-23T18:19:51.651644+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":268,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651592+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:19:51.651592+0000","last_peered":"2023-09-23T18:19:51.651592+0000","last_clean":"2023-09-23T18:19:51.651592+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:19:51.651592+0000","last_undegraded":"2023-09-23T18:19:51.651592+0000","last_fullsized":"2023-09-23T18:19:51.651592+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":25,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.790451+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:19:51.790451+0000","last_peered":"2023-09-23T18:19:51.790451+0000","last_clean":"2023-09-23T18:19:51.790451+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:19:51.790451+0000","last_undegraded":"2023-09-23T18:19:51.790451+0000","last_fullsized":"2023-09-23T18:19:51.790451+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":903,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652963+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:19:51.652963+0000","last_peered":"2023-09-23T18:19:51.652963+0000","last_clean":"2023-09-23T18:19:51.652963+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:19:51.652963+0000","last_undegraded":"2023-09-23T18:19:51.652963+0000","last_fullsized":"2023-09-23T18:19:51.652963+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":349,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.788559+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:19:51.788559+0000","last_peered":"2023-09-23T18:19:51.788559+0000","last_clean":"2023-09-23T18:19:51.788559+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:19:51.788559+0000","last_undegraded":"2023-09-23T18:19:51.788559+0000","last_fullsized":"2023-09-23T18:19:51.788559+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":669,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.694953+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:19:51.694953+0000","last_peered":"2023-09-23T18:19:51.694953+0000","last_clean":"2023-09-23T18:19:51.694953+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:19:51.694953+0000","last_undegraded":"2023-09-23T18:19:51.694953+0000","last_fullsized":"2023-09-23T18:19:51.694953+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":920,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652228+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:19:51.652228+0000","last_peered":"2023-09-23T18:19:51.652228+0000","last_clean":"2023-09-23T18:19:51.652228+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:51.652228+0000","last_undegraded":"2023-09-23T18:19:51.652228+0000","last_fullsized":"2023-09-23T18:19:51.652228+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":25,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.788805+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:51.788805+0000","last_peered":"2023-09-23T18:19:51.788805+0000","last_clean":"2023-09-23T18:19:51.788805+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:19:51.788805+0000","last_undegraded":"2023-09-23T18:19:51.788805+0000","last_fullsized":"2023-09-23T18:19:51.788805+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":185,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.686653+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:19:51.686653+0000","last_peered":"2023-09-23T18:19:51.686653+0000","last_clean":"2023-09-23T18:19:51.686653+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:19:51.686653+0000","last_undegraded":"2023-09-23T18:19:51.686653+0000","last_fullsized":"2023-09-23T18:19:51.686653+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1043,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651921+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:19:51.651921+0000","last_peered":"2023-09-23T18:19:51.651921+0000","last_clean":"2023-09-23T18:19:51.651921+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:19:51.651921+0000","last_undegraded":"2023-09-23T18:19:51.651921+0000","last_fullsized":"2023-09-23T18:19:51.651921+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":119,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652858+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:19:51.652858+0000","last_peered":"2023-09-23T18:19:51.652858+0000","last_clean":"2023-09-23T18:19:51.652858+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:19:51.652858+0000","last_undegraded":"2023-09-23T18:19:51.652858+0000","last_fullsized":"2023-09-23T18:19:51.652858+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":198,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.789327+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:19:51.789327+0000","last_peered":"2023-09-23T18:19:51.789327+0000","last_clean":"2023-09-23T18:19:51.789327+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:19:51.789327+0000","last_undegraded":"2023-09-23T18:19:51.789327+0000","last_fullsized":"2023-09-23T18:19:51.789327+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":820,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.684937+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:19:51.684937+0000","last_peered":"2023-09-23T18:19:51.684937+0000","last_clean":"2023-09-23T18:19:51.684937+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:19:51.684937+0000","last_undegraded":"2023-09-23T18:19:51.684937+0000","last_fullsized":"2023-09-23T18:19:51.684937+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":155,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652343+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:19:51.652343+0000","last_peered":"2023-09-23T18:19:51.652343+0000","last_clean":"2023-09-23T18:19:51.652343+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:19:51.652343+0000","last_undegraded":"2023-09-23T18:19:51.652343+0000","last_fullsized":"2023-09-23T18:19:51.652343+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":154,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.684737+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:19:51.684737+0000","last_peered":"2023-09-23T18:19:51.684737+0000","last_clean":"2023-09-23T18:19:51.684737+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:19:51.684737+0000","last_undegraded":"2023-09-23T18:19:51.684737+0000","last_fullsized":"2023-09-23T18:19:51.684737+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":954,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652939+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:19:51.652939+0000","last_peered":"2023-09-23T18:19:51.652939+0000","last_clean":"2023-09-23T18:19:51.652939+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:19:51.652939+0000","last_undegraded":"2023-09-23T18:19:51.652939+0000","last_fullsized":"2023-09-23T18:19:51.652939+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":403,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.790767+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:19:51.790767+0000","last_peered":"2023-09-23T18:19:51.790767+0000","last_clean":"2023-09-23T18:19:51.790767+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:19:51.790767+0000","last_undegraded":"2023-09-23T18:19:51.790767+0000","last_fullsized":"2023-09-23T18:19:51.790767+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":422,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.686112+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:19:51.686112+0000","last_peered":"2023-09-23T18:19:51.686112+0000","last_clean":"2023-09-23T18:19:51.686112+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:19:51.686112+0000","last_undegraded":"2023-09-23T18:19:51.686112+0000","last_fullsized":"2023-09-23T18:19:51.686112+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":983,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.646765+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:19:51.646765+0000","last_peered":"2023-09-23T18:19:51.646765+0000","last_clean":"2023-09-23T18:19:51.646765+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:19:51.646765+0000","last_undegraded":"2023-09-23T18:19:51.646765+0000","last_fullsized":"2023-09-23T18:19:51.646765+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":161,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685485+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:19:51.685485+0000","last_peered":"2023-09-23T18:19:51.685485+0000","last_clean":"2023-09-23T18:19:51.685485+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:19:51.685485+0000","last_undegraded":"2023-09-23T18:19:51.685485+0000","last_fullsized":"2023-09-23T18:19:51.685485+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":428,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685008+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:19:51.685008+0000","last_peered":"2023-09-23T18:19:51.685008+0000","last_clean":"2023-09-23T18:19:51.685008+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:19:51.685008+0000","last_undegraded":"2023-09-23T18:19:51.685008+0000","last_fullsized":"2023-09-23T18:19:51.685008+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":835,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651980+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:19:51.651980+0000","last_peered":"2023-09-23T18:19:51.651980+0000","last_clean":"2023-09-23T18:19:51.651980+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:19:51.651980+0000","last_undegraded":"2023-09-23T18:19:51.651980+0000","last_fullsized":"2023-09-23T18:19:51.651980+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":428,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.684956+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:19:51.684956+0000","last_peered":"2023-09-23T18:19:51.684956+0000","last_clean":"2023-09-23T18:19:51.684956+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:19:51.684956+0000","last_undegraded":"2023-09-23T18:19:51.684956+0000","last_fullsized":"2023-09-23T18:19:51.684956+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":106,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651865+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:19:51.651865+0000","last_peered":"2023-09-23T18:19:51.651865+0000","last_clean":"2023-09-23T18:19:51.651865+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:19:51.651865+0000","last_undegraded":"2023-09-23T18:19:51.651865+0000","last_fullsized":"2023-09-23T18:19:51.651865+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":806,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651183+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:19:51.651183+0000","last_peered":"2023-09-23T18:19:51.651183+0000","last_clean":"2023-09-23T18:19:51.651183+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:19:51.651183+0000","last_undegraded":"2023-09-23T18:19:51.651183+0000","last_fullsized":"2023-09-23T18:19:51.651183+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":124,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.694393+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:19:51.694393+0000","last_peered":"2023-09-23T18:19:51.694393+0000","last_clean":"2023-09-23T18:19:51.694393+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:19:51.694393+0000","last_undegraded":"2023-09-23T18:19:51.694393+0000","last_fullsized":"2023-09-23T18:19:51.694393+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":119,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652082+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:19:51.652082+0000","last_peered":"2023-09-23T18:19:51.652082+0000","last_clean":"2023-09-23T18:19:51.652082+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:19:51.652082+0000","last_undegraded":"2023-09-23T18:19:51.652082+0000","last_fullsized":"2023-09-23T18:19:51.652082+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":744,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.792040+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:19:51.792040+0000","last_peered":"2023-09-23T18:19:51.792040+0000","last_clean":"2023-09-23T18:19:51.792040+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:19:51.792040+0000","last_undegraded":"2023-09-23T18:19:51.792040+0000","last_fullsized":"2023-09-23T18:19:51.792040+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":196,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.684655+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:19:51.684655+0000","last_peered":"2023-09-23T18:19:51.684655+0000","last_clean":"2023-09-23T18:19:51.684655+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:19:51.684655+0000","last_undegraded":"2023-09-23T18:19:51.684655+0000","last_fullsized":"2023-09-23T18:19:51.684655+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":207,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.788586+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:19:51.788586+0000","last_peered":"2023-09-23T18:19:51.788586+0000","last_clean":"2023-09-23T18:19:51.788586+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:19:51.788586+0000","last_undegraded":"2023-09-23T18:19:51.788586+0000","last_fullsized":"2023-09-23T18:19:51.788586+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":840,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.684657+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:19:51.684657+0000","last_peered":"2023-09-23T18:19:51.684657+0000","last_clean":"2023-09-23T18:19:51.684657+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:19:51.684657+0000","last_undegraded":"2023-09-23T18:19:51.684657+0000","last_fullsized":"2023-09-23T18:19:51.684657+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":749,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.684654+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:19:51.684654+0000","last_peered":"2023-09-23T18:19:51.684654+0000","last_clean":"2023-09-23T18:19:51.684654+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:19:51.684654+0000","last_undegraded":"2023-09-23T18:19:51.684654+0000","last_fullsized":"2023-09-23T18:19:51.684654+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":25,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.831705+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:19:51.831705+0000","last_peered":"2023-09-23T18:19:51.831705+0000","last_clean":"2023-09-23T18:19:51.831705+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:19:51.831705+0000","last_undegraded":"2023-09-23T18:19:51.831705+0000","last_fullsized":"2023-09-23T18:19:51.831705+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"609'8412","reported_seq":8455,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:06.124171+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:20:06.124171+0000","last_peered":"2023-09-23T18:20:06.124171+0000","last_clean":"2023-09-23T18:20:06.124171+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:20:06.124171+0000","last_undegraded":"2023-09-23T18:20:06.124171+0000","last_fullsized":"2023-09-23T18:20:06.124171+0000","mapping_epoch":574,"log_start":"585'2900","ondisk_log_start":"585'2900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5512,"log_dups_size":0,"ondisk_log_size":5512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305528832,"num_objects":4662,"num_object_clones":0,"num_object_copies":13986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8412,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":874,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685158+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:19:51.685158+0000","last_peered":"2023-09-23T18:19:51.685158+0000","last_clean":"2023-09-23T18:19:51.685158+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:19:51.685158+0000","last_undegraded":"2023-09-23T18:19:51.685158+0000","last_fullsized":"2023-09-23T18:19:51.685158+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":245,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652301+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:19:51.652301+0000","last_peered":"2023-09-23T18:19:51.652301+0000","last_clean":"2023-09-23T18:19:51.652301+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:19:51.652301+0000","last_undegraded":"2023-09-23T18:19:51.652301+0000","last_fullsized":"2023-09-23T18:19:51.652301+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":119,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651205+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:19:51.651205+0000","last_peered":"2023-09-23T18:19:51.651205+0000","last_clean":"2023-09-23T18:19:51.651205+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:19:51.651205+0000","last_undegraded":"2023-09-23T18:19:51.651205+0000","last_fullsized":"2023-09-23T18:19:51.651205+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"609'8166","reported_seq":8219,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:06.242753+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:20:06.242753+0000","last_peered":"2023-09-23T18:20:06.242753+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:20:06.242753+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"593'3700","ondisk_log_start":"593'3700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4466,"log_dups_size":0,"ondisk_log_size":4466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":296615936,"num_objects":4526,"num_object_clones":0,"num_object_copies":13578,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4526,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8166,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":2328},{"shards":"5,6","objects":378}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":728,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.788935+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:19:51.788935+0000","last_peered":"2023-09-23T18:19:51.788935+0000","last_clean":"2023-09-23T18:19:51.788935+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:19:51.788935+0000","last_undegraded":"2023-09-23T18:19:51.788935+0000","last_fullsized":"2023-09-23T18:19:51.788935+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":246,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652745+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:19:51.652745+0000","last_peered":"2023-09-23T18:19:51.652745+0000","last_clean":"2023-09-23T18:19:51.652745+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:19:51.652745+0000","last_undegraded":"2023-09-23T18:19:51.652745+0000","last_fullsized":"2023-09-23T18:19:51.652745+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":225,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652619+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:19:51.652619+0000","last_peered":"2023-09-23T18:19:51.652619+0000","last_clean":"2023-09-23T18:19:51.652619+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:19:51.652619+0000","last_undegraded":"2023-09-23T18:19:51.652619+0000","last_fullsized":"2023-09-23T18:19:51.652619+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"609'8217","reported_seq":15220,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:05.207823+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:20:05.207823+0000","last_peered":"2023-09-23T18:20:05.207823+0000","last_clean":"2023-09-23T18:20:05.207823+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:20:05.207823+0000","last_undegraded":"2023-09-23T18:20:05.207823+0000","last_fullsized":"2023-09-23T18:20:05.207823+0000","mapping_epoch":605,"log_start":"587'3300","ondisk_log_start":"587'3300","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4917,"log_dups_size":0,"ondisk_log_size":4917,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303235072,"num_objects":4627,"num_object_clones":0,"num_object_copies":13881,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4627,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8217,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":624,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.686577+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:19:51.686577+0000","last_peered":"2023-09-23T18:19:51.686577+0000","last_clean":"2023-09-23T18:19:51.686577+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:19:51.686577+0000","last_undegraded":"2023-09-23T18:19:51.686577+0000","last_fullsized":"2023-09-23T18:19:51.686577+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":534,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651983+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:19:51.651983+0000","last_peered":"2023-09-23T18:19:51.651983+0000","last_clean":"2023-09-23T18:19:51.651983+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:19:51.651983+0000","last_undegraded":"2023-09-23T18:19:51.651983+0000","last_fullsized":"2023-09-23T18:19:51.651983+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":185,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685982+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:19:51.685982+0000","last_peered":"2023-09-23T18:19:51.685982+0000","last_clean":"2023-09-23T18:19:51.685982+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:19:51.685982+0000","last_undegraded":"2023-09-23T18:19:51.685982+0000","last_fullsized":"2023-09-23T18:19:51.685982+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"609'8360","reported_seq":8404,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:06.239918+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:20:06.239918+0000","last_peered":"2023-09-23T18:20:06.239918+0000","last_clean":"2023-09-23T18:20:06.239918+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:20:06.239918+0000","last_undegraded":"2023-09-23T18:20:06.239918+0000","last_fullsized":"2023-09-23T18:20:06.239918+0000","mapping_epoch":574,"log_start":"586'3200","ondisk_log_start":"586'3200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5160,"log_dups_size":0,"ondisk_log_size":5160,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310116352,"num_objects":4732,"num_object_clones":0,"num_object_copies":14196,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4732,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8360,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":639,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.788668+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:19:51.788668+0000","last_peered":"2023-09-23T18:19:51.788668+0000","last_clean":"2023-09-23T18:19:51.788668+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:19:51.788668+0000","last_undegraded":"2023-09-23T18:19:51.788668+0000","last_fullsized":"2023-09-23T18:19:51.788668+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":133,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652101+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:19:51.652101+0000","last_peered":"2023-09-23T18:19:51.652101+0000","last_clean":"2023-09-23T18:19:51.652101+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:19:51.652101+0000","last_undegraded":"2023-09-23T18:19:51.652101+0000","last_fullsized":"2023-09-23T18:19:51.652101+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":445,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.694829+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:19:51.694829+0000","last_peered":"2023-09-23T18:19:51.694829+0000","last_clean":"2023-09-23T18:19:51.694829+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:51.694829+0000","last_undegraded":"2023-09-23T18:19:51.694829+0000","last_fullsized":"2023-09-23T18:19:51.694829+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"609'8234","reported_seq":8278,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:03.887179+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:20:03.887179+0000","last_peered":"2023-09-23T18:20:03.887179+0000","last_clean":"2023-09-23T18:20:03.887179+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:20:03.887179+0000","last_undegraded":"2023-09-23T18:20:03.887179+0000","last_fullsized":"2023-09-23T18:20:03.887179+0000","mapping_epoch":574,"log_start":"593'3800","ondisk_log_start":"593'3800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4434,"log_dups_size":0,"ondisk_log_size":4434,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":315228160,"num_objects":4810,"num_object_clones":0,"num_object_copies":14430,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4810,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8234,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1091,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651758+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:19:51.651758+0000","last_peered":"2023-09-23T18:19:51.651758+0000","last_clean":"2023-09-23T18:19:51.651758+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:19:51.651758+0000","last_undegraded":"2023-09-23T18:19:51.651758+0000","last_fullsized":"2023-09-23T18:19:51.651758+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":345,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.792175+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:19:51.792175+0000","last_peered":"2023-09-23T18:19:51.792175+0000","last_clean":"2023-09-23T18:19:51.792175+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:19:51.792175+0000","last_undegraded":"2023-09-23T18:19:51.792175+0000","last_fullsized":"2023-09-23T18:19:51.792175+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":153,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685098+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:19:51.685098+0000","last_peered":"2023-09-23T18:19:51.685098+0000","last_clean":"2023-09-23T18:19:51.685098+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:19:51.685098+0000","last_undegraded":"2023-09-23T18:19:51.685098+0000","last_fullsized":"2023-09-23T18:19:51.685098+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"609'8390","reported_seq":8442,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:06.094998+0000","last_change":"2023-09-23T18:18:47.594759+0000","last_active":"2023-09-23T18:20:06.094998+0000","last_peered":"2023-09-23T18:20:06.094998+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:20:06.094998+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"585'2900","ondisk_log_start":"585'2900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5490,"log_dups_size":0,"ondisk_log_size":5490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304087040,"num_objects":4640,"num_object_clones":0,"num_object_copies":13920,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4640,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8390,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2351},{"shards":"4,5","objects":414}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":878,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651545+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:19:51.651545+0000","last_peered":"2023-09-23T18:19:51.651545+0000","last_clean":"2023-09-23T18:19:51.651545+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:19:51.651545+0000","last_undegraded":"2023-09-23T18:19:51.651545+0000","last_fullsized":"2023-09-23T18:19:51.651545+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":161,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685909+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:19:51.685909+0000","last_peered":"2023-09-23T18:19:51.685909+0000","last_clean":"2023-09-23T18:19:51.685909+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:19:51.685909+0000","last_undegraded":"2023-09-23T18:19:51.685909+0000","last_fullsized":"2023-09-23T18:19:51.685909+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":460,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685298+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:19:51.685298+0000","last_peered":"2023-09-23T18:19:51.685298+0000","last_clean":"2023-09-23T18:19:51.685298+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:19:51.685298+0000","last_undegraded":"2023-09-23T18:19:51.685298+0000","last_fullsized":"2023-09-23T18:19:51.685298+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"609'8304","reported_seq":8347,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:06.086081+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:20:06.086081+0000","last_peered":"2023-09-23T18:20:06.086081+0000","last_clean":"2023-09-23T18:20:06.086081+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:20:06.086081+0000","last_undegraded":"2023-09-23T18:20:06.086081+0000","last_fullsized":"2023-09-23T18:20:06.086081+0000","mapping_epoch":574,"log_start":"593'3800","ondisk_log_start":"593'3800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4504,"log_dups_size":0,"ondisk_log_size":4504,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":311164928,"num_objects":4748,"num_object_clones":0,"num_object_copies":14244,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4748,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8304,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":880,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.653059+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:19:51.653059+0000","last_peered":"2023-09-23T18:19:51.653059+0000","last_clean":"2023-09-23T18:19:51.653059+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:19:51.653059+0000","last_undegraded":"2023-09-23T18:19:51.653059+0000","last_fullsized":"2023-09-23T18:19:51.653059+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":445,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651607+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:19:51.651607+0000","last_peered":"2023-09-23T18:19:51.651607+0000","last_clean":"2023-09-23T18:19:51.651607+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:19:51.651607+0000","last_undegraded":"2023-09-23T18:19:51.651607+0000","last_fullsized":"2023-09-23T18:19:51.651607+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":221,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.684937+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:19:51.684937+0000","last_peered":"2023-09-23T18:19:51.684937+0000","last_clean":"2023-09-23T18:19:51.684937+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:19:51.684937+0000","last_undegraded":"2023-09-23T18:19:51.684937+0000","last_fullsized":"2023-09-23T18:19:51.684937+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"609'8197","reported_seq":8240,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:02.326867+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:20:02.326867+0000","last_peered":"2023-09-23T18:20:02.326867+0000","last_clean":"2023-09-23T18:20:02.326867+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:20:02.326867+0000","last_undegraded":"2023-09-23T18:20:02.326867+0000","last_fullsized":"2023-09-23T18:20:02.326867+0000","mapping_epoch":574,"log_start":"587'3500","ondisk_log_start":"587'3500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4697,"log_dups_size":0,"ondisk_log_size":4697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":316997632,"num_objects":4837,"num_object_clones":0,"num_object_copies":14511,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4837,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8197,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":861,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652480+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:19:51.652480+0000","last_peered":"2023-09-23T18:19:51.652480+0000","last_clean":"2023-09-23T18:19:51.652480+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:19:51.652480+0000","last_undegraded":"2023-09-23T18:19:51.652480+0000","last_fullsized":"2023-09-23T18:19:51.652480+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":25,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.788954+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:19:51.788954+0000","last_peered":"2023-09-23T18:19:51.788954+0000","last_clean":"2023-09-23T18:19:51.788954+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:19:51.788954+0000","last_undegraded":"2023-09-23T18:19:51.788954+0000","last_fullsized":"2023-09-23T18:19:51.788954+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":337,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.646788+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:19:51.646788+0000","last_peered":"2023-09-23T18:19:51.646788+0000","last_clean":"2023-09-23T18:19:51.646788+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:19:51.646788+0000","last_undegraded":"2023-09-23T18:19:51.646788+0000","last_fullsized":"2023-09-23T18:19:51.646788+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"609'8334","reported_seq":8378,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:05.228381+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:20:05.228381+0000","last_peered":"2023-09-23T18:20:05.228381+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:20:05.228381+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"586'3400","ondisk_log_start":"586'3400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4934,"log_dups_size":0,"ondisk_log_size":4934,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":310247424,"num_objects":4734,"num_object_clones":0,"num_object_copies":14202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8334,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":449},{"shards":"0,4,6","objects":2485}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":720,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.686731+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:19:51.686731+0000","last_peered":"2023-09-23T18:19:51.686731+0000","last_clean":"2023-09-23T18:19:51.686731+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:19:51.686731+0000","last_undegraded":"2023-09-23T18:19:51.686731+0000","last_fullsized":"2023-09-23T18:19:51.686731+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":959,"reported_epoch":608,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:51.646612+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:19:51.646612+0000","last_peered":"2023-09-23T18:19:51.646612+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:19:51.646612+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":143,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652088+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:19:51.652088+0000","last_peered":"2023-09-23T18:19:51.652088+0000","last_clean":"2023-09-23T18:19:51.652088+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:19:51.652088+0000","last_undegraded":"2023-09-23T18:19:51.652088+0000","last_fullsized":"2023-09-23T18:19:51.652088+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":123,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652437+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:19:51.652437+0000","last_peered":"2023-09-23T18:19:51.652437+0000","last_clean":"2023-09-23T18:19:51.652437+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:19:51.652437+0000","last_undegraded":"2023-09-23T18:19:51.652437+0000","last_fullsized":"2023-09-23T18:19:51.652437+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"609'8274","reported_seq":8326,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:06.101847+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:20:06.101847+0000","last_peered":"2023-09-23T18:20:06.101847+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:20:06.101847+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"593'3800","ondisk_log_start":"593'3800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4474,"log_dups_size":0,"ondisk_log_size":4474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":305528832,"num_objects":4662,"num_object_clones":0,"num_object_copies":13986,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4662,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8274,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":1752},{"shards":"5,6","objects":1103}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":792,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652139+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:19:51.652139+0000","last_peered":"2023-09-23T18:19:51.652139+0000","last_clean":"2023-09-23T18:19:51.652139+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:19:51.652139+0000","last_undegraded":"2023-09-23T18:19:51.652139+0000","last_fullsized":"2023-09-23T18:19:51.652139+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1493,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652397+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:19:51.652397+0000","last_peered":"2023-09-23T18:19:51.652397+0000","last_clean":"2023-09-23T18:19:51.652397+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:19:51.652397+0000","last_undegraded":"2023-09-23T18:19:51.652397+0000","last_fullsized":"2023-09-23T18:19:51.652397+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":133,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685040+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:19:51.685040+0000","last_peered":"2023-09-23T18:19:51.685040+0000","last_clean":"2023-09-23T18:19:51.685040+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:19:51.685040+0000","last_undegraded":"2023-09-23T18:19:51.685040+0000","last_fullsized":"2023-09-23T18:19:51.685040+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":433,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685335+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:19:51.685335+0000","last_peered":"2023-09-23T18:19:51.685335+0000","last_clean":"2023-09-23T18:19:51.685335+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:19:51.685335+0000","last_undegraded":"2023-09-23T18:19:51.685335+0000","last_fullsized":"2023-09-23T18:19:51.685335+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"609'8102","reported_seq":8145,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:06.101576+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:20:06.101576+0000","last_peered":"2023-09-23T18:20:06.101576+0000","last_clean":"2023-09-23T18:20:06.101576+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:20:06.101576+0000","last_undegraded":"2023-09-23T18:20:06.101576+0000","last_fullsized":"2023-09-23T18:20:06.101576+0000","mapping_epoch":574,"log_start":"584'2600","ondisk_log_start":"584'2600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5502,"log_dups_size":0,"ondisk_log_size":5502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":293863424,"num_objects":4484,"num_object_clones":0,"num_object_copies":13452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8102,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":817,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652532+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:19:51.652532+0000","last_peered":"2023-09-23T18:19:51.652532+0000","last_clean":"2023-09-23T18:19:51.652532+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:19:51.652532+0000","last_undegraded":"2023-09-23T18:19:51.652532+0000","last_fullsized":"2023-09-23T18:19:51.652532+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":23,"reported_epoch":608,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:19:51.788357+0000","last_change":"2023-09-23T18:18:46.432707+0000","last_active":"2023-09-23T18:19:51.788357+0000","last_peered":"2023-09-23T18:19:51.788357+0000","last_clean":"2023-09-23T18:11:33.614822+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:19:51.788357+0000","last_undegraded":"2023-09-23T18:18:46.418246+0000","last_fullsized":"2023-09-23T18:19:51.788357+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":402,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.684801+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:19:51.684801+0000","last_peered":"2023-09-23T18:19:51.684801+0000","last_clean":"2023-09-23T18:19:51.684801+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:19:51.684801+0000","last_undegraded":"2023-09-23T18:19:51.684801+0000","last_fullsized":"2023-09-23T18:19:51.684801+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":145,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652788+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:19:51.652788+0000","last_peered":"2023-09-23T18:19:51.652788+0000","last_clean":"2023-09-23T18:19:51.652788+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:19:51.652788+0000","last_undegraded":"2023-09-23T18:19:51.652788+0000","last_fullsized":"2023-09-23T18:19:51.652788+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"609'8303","reported_seq":8346,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:06.243860+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:20:06.243860+0000","last_peered":"2023-09-23T18:20:06.243860+0000","last_clean":"2023-09-23T18:20:06.243860+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:20:06.243860+0000","last_undegraded":"2023-09-23T18:20:06.243860+0000","last_fullsized":"2023-09-23T18:20:06.243860+0000","mapping_epoch":574,"log_start":"593'3800","ondisk_log_start":"593'3800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4503,"log_dups_size":0,"ondisk_log_size":4503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":304414720,"num_objects":4645,"num_object_clones":0,"num_object_copies":13935,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4645,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8303,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1506,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652879+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:19:51.652879+0000","last_peered":"2023-09-23T18:19:51.652879+0000","last_clean":"2023-09-23T18:19:51.652879+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:19:51.652879+0000","last_undegraded":"2023-09-23T18:19:51.652879+0000","last_fullsized":"2023-09-23T18:19:51.652879+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":559,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685528+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:19:51.685528+0000","last_peered":"2023-09-23T18:19:51.685528+0000","last_clean":"2023-09-23T18:19:51.685528+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:19:51.685528+0000","last_undegraded":"2023-09-23T18:19:51.685528+0000","last_fullsized":"2023-09-23T18:19:51.685528+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":119,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652298+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:19:51.652298+0000","last_peered":"2023-09-23T18:19:51.652298+0000","last_clean":"2023-09-23T18:19:51.652298+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:19:51.652298+0000","last_undegraded":"2023-09-23T18:19:51.652298+0000","last_fullsized":"2023-09-23T18:19:51.652298+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":154,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685644+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:19:51.685644+0000","last_peered":"2023-09-23T18:19:51.685644+0000","last_clean":"2023-09-23T18:19:51.685644+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:19:51.685644+0000","last_undegraded":"2023-09-23T18:19:51.685644+0000","last_fullsized":"2023-09-23T18:19:51.685644+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"609'8151","reported_seq":8194,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:06.115951+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:20:06.115951+0000","last_peered":"2023-09-23T18:20:06.115951+0000","last_clean":"2023-09-23T18:20:06.115951+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:20:06.115951+0000","last_undegraded":"2023-09-23T18:20:06.115951+0000","last_fullsized":"2023-09-23T18:20:06.115951+0000","mapping_epoch":574,"log_start":"584'2600","ondisk_log_start":"584'2600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5551,"log_dups_size":0,"ondisk_log_size":5551,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303628288,"num_objects":4633,"num_object_clones":0,"num_object_copies":13899,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4633,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8151,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":807,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685056+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:19:51.685056+0000","last_peered":"2023-09-23T18:19:51.685056+0000","last_clean":"2023-09-23T18:19:51.685056+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:19:51.685056+0000","last_undegraded":"2023-09-23T18:19:51.685056+0000","last_fullsized":"2023-09-23T18:19:51.685056+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":122,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.789080+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:19:51.789080+0000","last_peered":"2023-09-23T18:19:51.789080+0000","last_clean":"2023-09-23T18:19:51.789080+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:19:51.789080+0000","last_undegraded":"2023-09-23T18:19:51.789080+0000","last_fullsized":"2023-09-23T18:19:51.789080+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":25,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.788247+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:19:51.788247+0000","last_peered":"2023-09-23T18:19:51.788247+0000","last_clean":"2023-09-23T18:19:51.788247+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:19:51.788247+0000","last_undegraded":"2023-09-23T18:19:51.788247+0000","last_fullsized":"2023-09-23T18:19:51.788247+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":624,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652422+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:19:51.652422+0000","last_peered":"2023-09-23T18:19:51.652422+0000","last_clean":"2023-09-23T18:19:51.652422+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:19:51.652422+0000","last_undegraded":"2023-09-23T18:19:51.652422+0000","last_fullsized":"2023-09-23T18:19:51.652422+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"609'8272","reported_seq":11425,"reported_epoch":609,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:06.249011+0000","last_change":"2023-09-23T18:19:32.858380+0000","last_active":"2023-09-23T18:20:06.249011+0000","last_peered":"2023-09-23T18:20:06.249011+0000","last_clean":"2023-09-23T18:18:32.699126+0000","last_became_active":"2023-09-23T18:18:34.716591+0000","last_became_peered":"2023-09-23T18:18:34.716591+0000","last_unstale":"2023-09-23T18:20:06.249011+0000","last_undegraded":"2023-09-23T18:18:34.351246+0000","last_fullsized":"2023-09-23T18:18:34.327168+0000","mapping_epoch":589,"log_start":"593'3800","ondisk_log_start":"593'3800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4472,"log_dups_size":0,"ondisk_log_size":4472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:13.611234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":307036184,"num_objects":4686,"num_object_clones":0,"num_object_copies":14058,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":259,"num_objects_misplaced":1644,"num_objects_unfound":0,"num_objects_dirty":4686,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8272,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1550,"num_bytes_recovered":101580800,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":1644},{"shards":"4,6","objects":2783}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":907,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685340+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:19:51.685340+0000","last_peered":"2023-09-23T18:19:51.685340+0000","last_clean":"2023-09-23T18:19:51.685340+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:19:51.685340+0000","last_undegraded":"2023-09-23T18:19:51.685340+0000","last_fullsized":"2023-09-23T18:19:51.685340+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":47,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.694770+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:19:51.694770+0000","last_peered":"2023-09-23T18:19:51.694770+0000","last_clean":"2023-09-23T18:19:51.694770+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:19:51.694770+0000","last_undegraded":"2023-09-23T18:19:51.694770+0000","last_fullsized":"2023-09-23T18:19:51.694770+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":133,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651960+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:19:51.651960+0000","last_peered":"2023-09-23T18:19:51.651960+0000","last_clean":"2023-09-23T18:19:51.651960+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:19:51.651960+0000","last_undegraded":"2023-09-23T18:19:51.651960+0000","last_fullsized":"2023-09-23T18:19:51.651960+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":133,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652871+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:19:51.652871+0000","last_peered":"2023-09-23T18:19:51.652871+0000","last_clean":"2023-09-23T18:19:51.652871+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:19:51.652871+0000","last_undegraded":"2023-09-23T18:19:51.652871+0000","last_fullsized":"2023-09-23T18:19:51.652871+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"609'8295","reported_seq":8337,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:06.253707+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:20:06.253707+0000","last_peered":"2023-09-23T18:20:06.253707+0000","last_clean":"2023-09-23T18:20:06.253707+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:20:06.253707+0000","last_undegraded":"2023-09-23T18:20:06.253707+0000","last_fullsized":"2023-09-23T18:20:06.253707+0000","mapping_epoch":574,"log_start":"593'3800","ondisk_log_start":"593'3800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4495,"log_dups_size":0,"ondisk_log_size":4495,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":298778624,"num_objects":4559,"num_object_clones":0,"num_object_copies":13677,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4559,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8295,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":907,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685384+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:19:51.685384+0000","last_peered":"2023-09-23T18:19:51.685384+0000","last_clean":"2023-09-23T18:19:51.685384+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:19:51.685384+0000","last_undegraded":"2023-09-23T18:19:51.685384+0000","last_fullsized":"2023-09-23T18:19:51.685384+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":95,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652672+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:19:51.652672+0000","last_peered":"2023-09-23T18:19:51.652672+0000","last_clean":"2023-09-23T18:19:51.652672+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:19:51.652672+0000","last_undegraded":"2023-09-23T18:19:51.652672+0000","last_fullsized":"2023-09-23T18:19:51.652672+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":388,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685177+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:19:51.685177+0000","last_peered":"2023-09-23T18:19:51.685177+0000","last_clean":"2023-09-23T18:19:51.685177+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:19:51.685177+0000","last_undegraded":"2023-09-23T18:19:51.685177+0000","last_fullsized":"2023-09-23T18:19:51.685177+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":25,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.788878+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:51.788878+0000","last_peered":"2023-09-23T18:19:51.788878+0000","last_clean":"2023-09-23T18:19:51.788878+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:19:51.788878+0000","last_undegraded":"2023-09-23T18:19:51.788878+0000","last_fullsized":"2023-09-23T18:19:51.788878+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"609'8180","reported_seq":8223,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:20:06.243748+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:20:06.243748+0000","last_peered":"2023-09-23T18:20:06.243748+0000","last_clean":"2023-09-23T18:20:06.243748+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:20:06.243748+0000","last_undegraded":"2023-09-23T18:20:06.243748+0000","last_fullsized":"2023-09-23T18:20:06.243748+0000","mapping_epoch":574,"log_start":"593'3700","ondisk_log_start":"593'3700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4480,"log_dups_size":0,"ondisk_log_size":4480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":303562752,"num_objects":4632,"num_object_clones":0,"num_object_copies":13896,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4632,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8180,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":741,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.789020+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:19:51.789020+0000","last_peered":"2023-09-23T18:19:51.789020+0000","last_clean":"2023-09-23T18:19:51.789020+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:19:51.789020+0000","last_undegraded":"2023-09-23T18:19:51.789020+0000","last_fullsized":"2023-09-23T18:19:51.789020+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1045,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.686959+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:19:51.686959+0000","last_peered":"2023-09-23T18:19:51.686959+0000","last_clean":"2023-09-23T18:19:51.686959+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:19:51.686959+0000","last_undegraded":"2023-09-23T18:19:51.686959+0000","last_fullsized":"2023-09-23T18:19:51.686959+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":35,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.695046+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:19:51.695046+0000","last_peered":"2023-09-23T18:19:51.695046+0000","last_clean":"2023-09-23T18:19:51.695046+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:19:51.695046+0000","last_undegraded":"2023-09-23T18:19:51.695046+0000","last_fullsized":"2023-09-23T18:19:51.695046+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":399,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651839+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:19:51.651839+0000","last_peered":"2023-09-23T18:19:51.651839+0000","last_clean":"2023-09-23T18:19:51.651839+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:19:51.651839+0000","last_undegraded":"2023-09-23T18:19:51.651839+0000","last_fullsized":"2023-09-23T18:19:51.651839+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":639,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.684820+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:19:51.684820+0000","last_peered":"2023-09-23T18:19:51.684820+0000","last_clean":"2023-09-23T18:19:51.684820+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:19:51.684820+0000","last_undegraded":"2023-09-23T18:19:51.684820+0000","last_fullsized":"2023-09-23T18:19:51.684820+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":31,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.694739+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:19:51.694739+0000","last_peered":"2023-09-23T18:19:51.694739+0000","last_clean":"2023-09-23T18:19:51.694739+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:19:51.694739+0000","last_undegraded":"2023-09-23T18:19:51.694739+0000","last_fullsized":"2023-09-23T18:19:51.694739+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":682,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.646481+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:19:51.646481+0000","last_peered":"2023-09-23T18:19:51.646481+0000","last_clean":"2023-09-23T18:19:51.646481+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:19:51.646481+0000","last_undegraded":"2023-09-23T18:19:51.646481+0000","last_fullsized":"2023-09-23T18:19:51.646481+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":639,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685018+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:19:51.685018+0000","last_peered":"2023-09-23T18:19:51.685018+0000","last_clean":"2023-09-23T18:19:51.685018+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:19:51.685018+0000","last_undegraded":"2023-09-23T18:19:51.685018+0000","last_fullsized":"2023-09-23T18:19:51.685018+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":127,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652018+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:19:51.652018+0000","last_peered":"2023-09-23T18:19:51.652018+0000","last_clean":"2023-09-23T18:19:51.652018+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:19:51.652018+0000","last_undegraded":"2023-09-23T18:19:51.652018+0000","last_fullsized":"2023-09-23T18:19:51.652018+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":243,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651935+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:19:51.651935+0000","last_peered":"2023-09-23T18:19:51.651935+0000","last_clean":"2023-09-23T18:19:51.651935+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:19:51.651935+0000","last_undegraded":"2023-09-23T18:19:51.651935+0000","last_fullsized":"2023-09-23T18:19:51.651935+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":113,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652008+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:19:51.652008+0000","last_peered":"2023-09-23T18:19:51.652008+0000","last_clean":"2023-09-23T18:19:51.652008+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:19:51.652008+0000","last_undegraded":"2023-09-23T18:19:51.652008+0000","last_fullsized":"2023-09-23T18:19:51.652008+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":226,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.651670+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:19:51.651670+0000","last_peered":"2023-09-23T18:19:51.651670+0000","last_clean":"2023-09-23T18:19:51.651670+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:19:51.651670+0000","last_undegraded":"2023-09-23T18:19:51.651670+0000","last_fullsized":"2023-09-23T18:19:51.651670+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1001,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685023+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:19:51.685023+0000","last_peered":"2023-09-23T18:19:51.685023+0000","last_clean":"2023-09-23T18:19:51.685023+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:19:51.685023+0000","last_undegraded":"2023-09-23T18:19:51.685023+0000","last_fullsized":"2023-09-23T18:19:51.685023+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":506,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.685258+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:19:51.685258+0000","last_peered":"2023-09-23T18:19:51.685258+0000","last_clean":"2023-09-23T18:19:51.685258+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:19:51.685258+0000","last_undegraded":"2023-09-23T18:19:51.685258+0000","last_fullsized":"2023-09-23T18:19:51.685258+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":143,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652062+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:19:51.652062+0000","last_peered":"2023-09-23T18:19:51.652062+0000","last_clean":"2023-09-23T18:19:51.652062+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:19:51.652062+0000","last_undegraded":"2023-09-23T18:19:51.652062+0000","last_fullsized":"2023-09-23T18:19:51.652062+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":938,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.646680+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:19:51.646680+0000","last_peered":"2023-09-23T18:19:51.646680+0000","last_clean":"2023-09-23T18:19:51.646680+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:19:51.646680+0000","last_undegraded":"2023-09-23T18:19:51.646680+0000","last_fullsized":"2023-09-23T18:19:51.646680+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":131,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.652748+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:19:51.652748+0000","last_peered":"2023-09-23T18:19:51.652748+0000","last_clean":"2023-09-23T18:19:51.652748+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:19:51.652748+0000","last_undegraded":"2023-09-23T18:19:51.652748+0000","last_fullsized":"2023-09-23T18:19:51.652748+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":652,"reported_epoch":608,"state":"active+clean","last_fresh":"2023-09-23T18:19:51.684647+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:19:51.684647+0000","last_peered":"2023-09-23T18:19:51.684647+0000","last_clean":"2023-09-23T18:19:51.684647+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:19:51.684647+0000","last_undegraded":"2023-09-23T18:19:51.684647+0000","last_fullsized":"2023-09-23T18:19:51.684647+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4890034200,"num_objects":74617,"num_object_clones":0,"num_object_copies":223851,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16477,"num_objects_misplaced":1644,"num_objects_unfound":0,"num_objects_dirty":74617,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":132191,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5021,"num_bytes_recovered":329056256,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":15823417344,"data_stored":15823405128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77591,"ondisk_log_size":77591,"up":48,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001569,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669506,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2995976,"kb_used_data":2385728,"kb_used_omap":7,"kb_used_meta":610232,"kb_avail":90749176,"statfs":{"total":95995035648,"available":92927156224,"internally_reserved":0,"allocated":2442985472,"data_stored":2441063069,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":624878372},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":1.599,"15min":1.2949999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":24.472999999999999,"5min":24.472999999999999,"15min":24.472999999999999},"last":0.28000000000000003},{"interface":"front","average":{"1min":3.6230000000000002,"5min":2.081,"15min":1.663},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":22.832999999999998,"5min":22.832999999999998,"15min":22.832999999999998},"last":0.36399999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:19:21 2023","interfaces":[{"interface":"back","average":{"1min":4.1740000000000004,"5min":1.8069999999999999,"15min":1.413},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":22.972000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.312},{"interface":"front","average":{"1min":3.448,"5min":1.6559999999999999,"15min":1.357},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":22.664000000000001,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.379}]},{"osd":2,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.52,"15min":1.3260000000000001},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":8.0559999999999992,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":1.371},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.861,"15min":1.492},"min":{"1min":0.34000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.1230000000000002,"5min":14.771000000000001,"15min":14.771000000000001},"last":3.2509999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":9.1259999999999994,"5min":3.1400000000000001,"15min":2.1059999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":103.348,"5min":103.348,"15min":103.348},"last":0.34999999999999998},{"interface":"front","average":{"1min":9.4610000000000003,"5min":3.6389999999999998,"15min":2.3610000000000002},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":103.40300000000001,"5min":103.40300000000001,"15min":103.40300000000001},"last":0.39600000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.8140000000000001,"15min":1.3280000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.2800000000000002,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.7569999999999999,"5min":1.9330000000000001,"15min":1.3200000000000001},"min":{"1min":0.36099999999999999,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":6.2030000000000003,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.66600000000000004}]},{"osd":5,"last update":"Sat Sep 23 18:19:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":2.0609999999999999,"15min":1.9550000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":6.4710000000000001,"5min":13.502000000000001,"15min":13.502000000000001},"last":0.52200000000000002},{"interface":"front","average":{"1min":1.3720000000000001,"5min":2.246,"15min":2.3660000000000001},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.1070000000000002,"5min":10.081,"15min":10.081},"last":0.56799999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":2.1459999999999999,"15min":1.5309999999999999},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.26800000000000002},"max":{"1min":6.4320000000000004,"5min":10.743,"15min":13.460000000000001},"last":1.756},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.0710000000000002,"15min":1.5740000000000001},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.28499999999999998},"max":{"1min":6.2770000000000001,"5min":11.244,"15min":12.404},"last":0.72399999999999998}]}]},{"osd":1,"up_from":545,"seq":2340757176354,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2095832,"kb_used_data":1883984,"kb_used_omap":13,"kb_used_meta":211826,"kb_avail":91649320,"statfs":{"total":95995035648,"available":93848903680,"internally_reserved":0,"allocated":1929199616,"data_stored":1926218623,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":216910477},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.776,"5min":0.66800000000000004,"15min":0.48299999999999998},"min":{"1min":0.221,"5min":0.189,"15min":0.189},"max":{"1min":5.8079999999999998,"5min":5.8079999999999998,"15min":5.8079999999999998},"last":2.5880000000000001},{"interface":"front","average":{"1min":1.2430000000000001,"5min":0.64000000000000001,"15min":0.54000000000000004},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6159999999999997,"5min":4.6159999999999997,"15min":4.6159999999999997},"last":0.40699999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":0.67900000000000005,"15min":0.53300000000000003},"min":{"1min":0.28299999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9290000000000003,"5min":4.9290000000000003,"15min":4.9290000000000003},"last":4.9269999999999996},{"interface":"front","average":{"1min":1.272,"5min":0.65800000000000003,"15min":0.55600000000000005},"min":{"1min":0.29899999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2759999999999998,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":0.497}]},{"osd":3,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":0.746,"15min":0.50600000000000001},"min":{"1min":0.34300000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.3219999999999992,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":2.827},{"interface":"front","average":{"1min":2.2429999999999999,"5min":0.85799999999999998,"15min":0.627},"min":{"1min":0.38500000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":8.3659999999999997},"last":2.8849999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":1.347,"15min":1.327},"min":{"1min":0.40200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.6609999999999996,"5min":12.663,"15min":12.663},"last":2.4700000000000002},{"interface":"front","average":{"1min":1.95,"5min":1.601,"15min":1.5429999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.1769999999999996,"5min":13.222,"15min":13.222},"last":1.8660000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":0.88200000000000001,"15min":0.77600000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":3.0830000000000002},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.0589999999999999,"15min":0.88900000000000001},"min":{"1min":0.46000000000000002,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":8.3190000000000008,"5min":8.3190000000000008,"15min":8.3190000000000008},"last":4.8849999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":0.997,"15min":0.83899999999999997},"min":{"1min":0.45400000000000001,"5min":0.435,"15min":0.435},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":0.46600000000000003},{"interface":"front","average":{"1min":2.6070000000000002,"5min":1.1419999999999999,"15min":0.89800000000000002},"min":{"1min":0.38900000000000001,"5min":0.378,"15min":0.378},"max":{"1min":9.2579999999999991,"5min":9.2579999999999991,"15min":9.2579999999999991},"last":2.6080000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":1.0109999999999999,"15min":0.79100000000000004},"min":{"1min":0.436,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":2.3149999999999999},{"interface":"front","average":{"1min":2.6419999999999999,"5min":1.173,"15min":0.92800000000000005},"min":{"1min":0.48099999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":9.0640000000000001,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":2.843}]}]},{"osd":0,"up_from":93,"seq":399431958851,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4003580,"kb_used_data":3098420,"kb_used_omap":7,"kb_used_meta":905144,"kb_avail":89741572,"statfs":{"total":95995035648,"available":91895369728,"internally_reserved":0,"allocated":3172782080,"data_stored":3169945730,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":926868196},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:19:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7229999999999999,"5min":0.80000000000000004,"15min":0.47999999999999998},"min":{"1min":0.214,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.2309999999999999,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.33900000000000002},{"interface":"front","average":{"1min":2.2010000000000001,"5min":0.85799999999999998,"15min":0.63400000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.714,"5min":13.714,"15min":13.714},"last":1.6819999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:20:04 2023","interfaces":[{"interface":"back","average":{"1min":3.911,"5min":1.71,"15min":1.3160000000000001},"min":{"1min":0.48999999999999999,"5min":0.185,"15min":0.185},"max":{"1min":15.002000000000001,"5min":18.550999999999998,"15min":18.550999999999998},"last":1.0129999999999999},{"interface":"front","average":{"1min":2.8530000000000002,"5min":1.258,"15min":0.93400000000000005},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":11.644,"5min":13.618,"15min":13.618},"last":1.722}]},{"osd":3,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7610000000000001,"5min":1.393,"15min":1.637},"min":{"1min":0.38400000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.539999999999999,"5min":13.539999999999999,"15min":13.539999999999999},"last":0.53300000000000003},{"interface":"front","average":{"1min":4.1390000000000002,"5min":1.4930000000000001,"15min":1.637},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.285,"5min":13.285,"15min":15.198},"last":0.57799999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.2599999999999998,"15min":1.7789999999999999},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":13.574999999999999,"5min":57.746000000000002,"15min":57.746000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":2.032,"5min":1.9770000000000001,"15min":1.6499999999999999},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.8970000000000002,"5min":57.816000000000003,"15min":57.816000000000003},"last":0.72999999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.032,"15min":1.391},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":0.63},{"interface":"front","average":{"1min":4.2859999999999996,"5min":2.2650000000000001,"15min":1.448},"min":{"1min":0.502,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":13.425000000000001},"last":0.70599999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":1.8049999999999999,"15min":1.861},"min":{"1min":0.433,"5min":0.374,"15min":0.32000000000000001},"max":{"1min":7.1020000000000003,"5min":7.1020000000000003,"15min":58.457999999999998},"last":0.67700000000000005},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.798,"15min":1.7050000000000001},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":13.672000000000001,"5min":13.672000000000001,"15min":58.354999999999997},"last":1.754}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":1.923,"15min":1.639},"min":{"1min":0.46899999999999997,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":13.644,"5min":13.644,"15min":13.644},"last":1.7829999999999999},{"interface":"front","average":{"1min":3.5179999999999998,"5min":1.821,"15min":1.8},"min":{"1min":0.55900000000000005,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.7519999999999998,"5min":10.404,"15min":10.404},"last":0.65600000000000003}]}]},{"osd":2,"up_from":0,"seq":2070174236736,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249366,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2331856,"kb_used_data":1619144,"kb_used_omap":3,"kb_used_meta":712700,"kb_avail":91413296,"statfs":{"total":95995035648,"available":93607215104,"internally_reserved":0,"allocated":1658003456,"data_stored":1654972408,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":729805544},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3919999999999999,"5min":1.25,"15min":1.7050000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.183},"max":{"1min":17.036000000000001,"5min":17.036000000000001,"15min":195.69800000000001},"last":9.2560000000000002},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.1599999999999999,"15min":1.96},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.0980000000000008,"5min":9.0980000000000008,"15min":194.661},"last":3.3450000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":0.95799999999999996,"15min":0.66600000000000004},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":21.777000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":9.7609999999999992},{"interface":"front","average":{"1min":6.5110000000000001,"5min":1.8660000000000001,"15min":1.0920000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":78.980000000000004,"5min":78.980000000000004,"15min":78.980000000000004},"last":6.1299999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":1.274,"15min":1.29},"min":{"1min":0.27400000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.151999999999999,"5min":12.151999999999999,"15min":12.151999999999999},"last":6.9119999999999999},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.298,"15min":1.3320000000000001},"min":{"1min":0.318,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.565,"5min":11.565,"15min":11.565},"last":3.1070000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.6140000000000001,"15min":1.659},"min":{"1min":0.31900000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2300000000000004,"5min":21.062999999999999,"15min":37.231000000000002},"last":0.45900000000000002},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.9810000000000001,"15min":1.8939999999999999},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.2959999999999994,"5min":21.128,"15min":37.381},"last":5.1630000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:19:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.8009999999999999,"15min":1.4419999999999999},"min":{"1min":0.40999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":11.805999999999999,"5min":11.805999999999999,"15min":11.805999999999999},"last":5.2370000000000001},{"interface":"front","average":{"1min":2.028,"5min":1.601,"15min":1.6339999999999999},"min":{"1min":0.44500000000000001,"5min":0.33100000000000002,"15min":0.32000000000000001},"max":{"1min":8.9359999999999999,"5min":8.9359999999999999,"15min":8.9969999999999999},"last":3.0619999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":1.7809999999999999,"15min":1.847},"min":{"1min":0.432,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":17.192,"5min":17.192,"15min":115.54000000000001},"last":3.3279999999999998},{"interface":"front","average":{"1min":2.2530000000000001,"5min":1.7250000000000001,"15min":1.948},"min":{"1min":0.37,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":8.0350000000000001,"5min":14.321,"15min":115.691},"last":3.4199999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.5069999999999999,"15min":1.7110000000000001},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":8.2070000000000007,"5min":8.2070000000000007,"15min":15.972},"last":5.21},{"interface":"front","average":{"1min":3.5920000000000001,"5min":1.7629999999999999,"15min":1.623},"min":{"1min":0.48799999999999999,"5min":0.41999999999999998,"15min":0.34899999999999998},"max":{"1min":17.164000000000001,"5min":17.164000000000001,"15min":17.164000000000001},"last":3.2869999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019315,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4652424,"kb_used_data":4013120,"kb_used_omap":0,"kb_used_meta":639104,"kb_avail":89092728,"statfs":{"total":95995035648,"available":91230953472,"internally_reserved":0,"allocated":4109434880,"data_stored":4107512269,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":654442496},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.0750000000000002,"5min":1.375,"15min":1.21},"min":{"1min":0.27200000000000002,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.9660000000000002,"5min":7.2199999999999998,"15min":13.375999999999999},"last":0.31900000000000001},{"interface":"front","average":{"1min":2.2709999999999999,"5min":1.772,"15min":1.4710000000000001},"min":{"1min":0.29199999999999998,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":8.9429999999999996,"5min":10.890000000000001,"15min":10.890000000000001},"last":9.3710000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:19:22 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":1.47,"15min":0.95299999999999996},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":44.146999999999998,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.36299999999999999},{"interface":"front","average":{"1min":4.5419999999999998,"5min":1.4299999999999999,"15min":0.91100000000000003},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":43.497999999999998,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.27200000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:19:56 2023","interfaces":[{"interface":"back","average":{"1min":2.621,"5min":1.6699999999999999,"15min":1.8180000000000001},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.896000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.34300000000000003},{"interface":"front","average":{"1min":1.6020000000000001,"5min":1.6200000000000001,"15min":1.6930000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":6.6890000000000001,"5min":10.938000000000001,"15min":10.938000000000001},"last":4.3710000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.5669999999999999,"15min":1.9350000000000001},"min":{"1min":0.33700000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.0609999999999999,"5min":11.130000000000001,"15min":29.154},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.1800000000000002,"5min":1.8480000000000001,"15min":1.9390000000000001},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":13.470000000000001,"5min":13.470000000000001,"15min":29.216000000000001},"last":4.4279999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":2.4279999999999999,"15min":2.8559999999999999},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4950000000000001,"5min":27.675000000000001,"15min":27.675000000000001},"last":9.5009999999999994},{"interface":"front","average":{"1min":1.401,"5min":1.827,"15min":2.7109999999999999},"min":{"1min":0.30499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.1040000000000001,"5min":13.938000000000001,"15min":32.640000000000001},"last":1.2829999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:19:07 2023","interfaces":[{"interface":"back","average":{"1min":1.681,"5min":2.0739999999999998,"15min":1.391},"min":{"1min":0.379,"5min":0.28199999999999997,"15min":0.25},"max":{"1min":7.7249999999999996,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.40200000000000002},{"interface":"front","average":{"1min":1.782,"5min":2.044,"15min":1.51},"min":{"1min":0.33900000000000002,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.6310000000000002,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.57899999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:19:03 2023","interfaces":[{"interface":"back","average":{"1min":2.3980000000000001,"5min":2.7919999999999998,"15min":1.7490000000000001},"min":{"1min":0.42699999999999999,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":8.9849999999999994,"5min":19.298999999999999,"15min":19.298999999999999},"last":4.4089999999999998},{"interface":"front","average":{"1min":2.234,"5min":2.1549999999999998,"15min":1.5569999999999999},"min":{"1min":0.50900000000000001,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.1319999999999997,"5min":12.800000000000001,"15min":12.800000000000001},"last":9.4730000000000008}]}]},{"osd":5,"up_from":420,"seq":1803886264412,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2938744,"kb_used_data":2599536,"kb_used_omap":9,"kb_used_meta":338934,"kb_avail":90806408,"statfs":{"total":95995035648,"available":92985761792,"internally_reserved":0,"allocated":2661924864,"data_stored":2660014597,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":347068861},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":1.2549999999999999,"15min":0.88100000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.6000000000000001,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.26100000000000001},{"interface":"front","average":{"1min":2.911,"5min":1.8149999999999999,"15min":1.6080000000000001},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":20.937999999999999,"5min":20.937999999999999,"15min":20.937999999999999},"last":0.29299999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:19:20 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":1.4770000000000001,"15min":1.2230000000000001},"min":{"1min":0.32100000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.946000000000002,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.32600000000000001},{"interface":"front","average":{"1min":3.927,"5min":1.4630000000000001,"15min":1.052},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":23.268999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":1.0669999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:19:52 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":1.629,"15min":1.615},"min":{"1min":0.28699999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":10.705,"5min":10.705,"15min":10.705},"last":0.46300000000000002},{"interface":"front","average":{"1min":2.5409999999999999,"5min":1.946,"15min":2.1600000000000001},"min":{"1min":0.373,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.8099999999999996,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.443}]},{"osd":3,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":1.8740000000000001,"15min":1.2789999999999999},"min":{"1min":0.443,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.8780000000000001,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":0.996},{"interface":"front","average":{"1min":3.7320000000000002,"5min":2.0640000000000001,"15min":1.665},"min":{"1min":0.38100000000000001,"5min":0.315,"15min":0.313},"max":{"1min":20.035,"5min":20.035,"15min":20.035},"last":0.47699999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.2770000000000001,"15min":2.0640000000000001},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":7.875,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.36799999999999999},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.0899999999999999,"15min":1.6160000000000001},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7610000000000001,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.38}]},{"osd":6,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":1.9650000000000001,"15min":1.2609999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2480000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.42799999999999999},{"interface":"front","average":{"1min":2.343,"5min":2.2040000000000002,"15min":1.6819999999999999},"min":{"1min":0.34599999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.8410000000000002,"5min":12.504,"15min":12.504},"last":0.39700000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.3980000000000001,"15min":1.758},"min":{"1min":0.40699999999999997,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.528,"5min":13.506,"15min":13.506},"last":0.56499999999999995},{"interface":"front","average":{"1min":1.381,"5min":1.9630000000000001,"15min":1.542},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":5.0970000000000004,"5min":13.138,"15min":13.138},"last":0.49199999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":3136032768,"data_stored":3136028696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1907294208,"data_stored":1907294208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1649016832,"data_stored":1649016832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4078833664,"data_stored":4078829592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2644574208,"data_stored":2644574208,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2407665664,"data_stored":2407661592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:20:08.139 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:20:08.141 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:20:08.141 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:20:08.141 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:20:08.141 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:20:08.141 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:20:08.141 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:20:08.141 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:20:08.141 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:20:08.215 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:08.296 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:08.305 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:08.306 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:08.406 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:08.468 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:08.504 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:20:08.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[179668]: pgmap v1453: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 869 op/s; 16502/223932 objects degraded (7.369%); 1644/223932 objects misplaced (0.734%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:20:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[179668]: from='client.47009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3785817521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2736912709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:08.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16502/223932 objects degraded (7.369%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[184174]: pgmap v1453: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 869 op/s; 16502/223932 objects degraded (7.369%); 1644/223932 objects misplaced (0.734%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:20:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[184174]: from='client.47009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:08.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3785817521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2736912709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:08.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16502/223932 objects degraded (7.369%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:08.720 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:20:08.720 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:20:08.720 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.720 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.46799.0:134577 10.7 e610/595 ondisk, result = 0)", 2023-09-23T18:20:08.720 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:20:08.706646+0000", 2023-09-23T18:20:08.720 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0012795969999999999, 2023-09-23T18:20:08.720 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001306331, 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "started", 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706646+0000", 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:08.721 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706646+0000", 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706645+0000", 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706651+0000", 2023-09-23T18:20:08.722 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4010000000000002e-06 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706663+0000", 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1267000000000001e-05 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706668+0000", 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7620000000000001e-06 2023-09-23T18:20:08.723 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706845+0000", 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000176804 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706853+0000", 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8029999999999997e-06 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.724 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_repop_reply(client.46799.0:134579 10.a e610/589)", 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:20:08.707386+0000", 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.00053929200000000003, 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063735500000000004, 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "queued for pg", 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:08.725 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.707386+0000", 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.707386+0000", 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.726 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.707386+0000", 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.707390+0000", 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6999999999999999e-06 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.707400+0000", 2023-09-23T18:20:08.727 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3000000000000007e-06 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.707404+0000", 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6700000000000002e-06 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:134571 10.e 10:71cdae2a:::benchmark_data_smithi055_425050_object31165:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e610)", 2023-09-23T18:20:08.728 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:20:08.688198+0000", 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.019727865000000001, 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019883858000000001, 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 134571 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:08.729 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688198+0000", 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688198+0000", 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.730 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688197+0000", 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688202+0000", 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7699999999999998e-06 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:08.731 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688214+0000", 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2061e-05 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688220+0000", 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0350000000000001e-06 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688631+0000", 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000410999 2023-09-23T18:20:08.732 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:08.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688720+0000", 2023-09-23T18:20:08.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.9262e-05 2023-09-23T18:20:08.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.733 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.733 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:20:08.733 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688792+0000", 2023-09-23T18:20:08.733 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1717999999999993e-05 2023-09-23T18:20:08.733 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.693428+0000", 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046361629999999996 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.702485+0000", 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0090577990000000001 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.734 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:134574 10.e 10:7960089c:::benchmark_data_smithi055_425050_object31168:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e610)", 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:20:08.688365+0000", 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.019560547000000001, 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019809953000000002, 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:08.735 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 134574 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688365+0000", 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.736 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688365+0000", 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688364+0000", 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:08.737 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688368+0000", 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2160000000000002e-06 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688374+0000", 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.4230000000000002e-06 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.688377+0000", 2023-09-23T18:20:08.738 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3890000000000001e-06 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.690770+0000", 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023928880000000001 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.690881+0000", 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110503 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.739 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.690985+0000", 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104609 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.693799+0000", 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002813421 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.740 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.702509+0000", 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0087104209999999994 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:134577 10.7 10:e592a29a:::benchmark_data_smithi055_425050_object31171:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e610)", 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:20:08.694597+0000", 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.013328098, 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013670484, 2023-09-23T18:20:08.741 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 134577 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.694597+0000", 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.742 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.694597+0000", 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.694596+0000", 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.743 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.694603+0000", 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2189999999999998e-06 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.694617+0000", 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3378e-05 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.694623+0000", 2023-09-23T18:20:08.744 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5529999999999998e-06 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.694669+0000", 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6397000000000003e-05 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.694818+0000", 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014854000000000001 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.745 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.694965+0000", 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014688200000000001 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.697870+0000", 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002905024 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:08.746 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.704724+0000", 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0068544649999999997 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706860+0000", 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002135924 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706913+0000", 2023-09-23T18:20:08.747 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2605000000000002e-05 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:134578 10.e 10:7c6107e8:::benchmark_data_smithi055_425050_object31172:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e610)", 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:20:08.697350+0000", 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.010575786, 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011018231, 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:20:08.748 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 134578 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.697350+0000", 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.749 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.697350+0000", 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.697349+0000", 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.697355+0000", 2023-09-23T18:20:08.750 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9059999999999996e-06 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.697371+0000", 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5690000000000001e-05 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.697378+0000", 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.55e-06 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.751 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.697422+0000", 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4762000000000003e-05 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.697578+0000", 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000155635 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,5", 2023-09-23T18:20:08.752 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.697709+0000", 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013072499999999999 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.701932+0000", 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0042227660000000002 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706797+0000", 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004864948 2023-09-23T18:20:08.753 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:134579 10.a 10:52a41675:::benchmark_data_smithi055_425050_object31173:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e610)", 2023-09-23T18:20:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:20:08.698902+0000", 2023-09-23T18:20:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 0.0090239110000000008, 2023-09-23T18:20:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0095693029999999995, 2023-09-23T18:20:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:08.754 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 134579 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.698902+0000", 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.755 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.698902+0000", 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.698901+0000", 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.756 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.698913+0000", 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2099000000000001e-05 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.698926+0000", 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.341e-05 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.698933+0000", 2023-09-23T18:20:08.757 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9949999999999999e-06 2023-09-23T18:20:08.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:08.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.698995+0000", 2023-09-23T18:20:08.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1937e-05 2023-09-23T18:20:08.758 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.758 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.758 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:08.758 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.699157+0000", 2023-09-23T18:20:08.758 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016230899999999999 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.699325+0000", 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016758800000000001 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706185+0000", 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0068599680000000001 2023-09-23T18:20:08.759 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:08.706815+0000", 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00062970799999999998 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout: ], 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 7 2023-09-23T18:20:08.760 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:20:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:08 smithi099 ceph-mon[167076]: pgmap v1453: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.6 GiB data, 18 GiB used, 518 GiB / 536 GiB avail; 0 B/s wr, 869 op/s; 16502/223932 objects degraded (7.369%); 1644/223932 objects misplaced (0.734%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:20:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:08 smithi099 ceph-mon[167076]: from='client.47009 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3785817521' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:08 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2736912709' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:08.761 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16502/223932 objects degraded (7.369%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:08.972 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:08.978 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:08.978 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:09.079 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:09.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:20:09.162 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:09.216 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:09.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:09.225 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:09.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:09.465 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:09 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:09.465 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:09 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:20:09.465 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:09 smithi099 ceph-mon[167076]: osdmap e610: 8 total, 8 up, 6 in 2023-09-23T18:20:09.477 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:09.549 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:09.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:09.553 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:09.654 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:09.718 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:09.822 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:09.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:09.834 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:09.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:09 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:09.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:09 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:20:09.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:09 smithi055 ceph-mon[184174]: osdmap e610: 8 total, 8 up, 6 in 2023-09-23T18:20:09.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:09 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:09.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:09 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:20:09.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:09 smithi055 ceph-mon[179668]: osdmap e610: 8 total, 8 up, 6 in 2023-09-23T18:20:09.935 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:09.983 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:10.019 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:10.044 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:10.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:10.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:10.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:10.219 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:10.348 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:10.355 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:10.355 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:10.358 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:20:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:20:10] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2023-09-23T18:20:10.457 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:10.514 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:10 smithi099 ceph-mon[167076]: failed to encode map e610 with expected crc 2023-09-23T18:20:10.514 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:10 smithi099 ceph-mon[167076]: pgmap v1455: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 890 op/s; 16502/222315 objects degraded (7.423%); 1644/222315 objects misplaced (0.739%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:20:10.546 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:10.632 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:10.636 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:10.637 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:10 smithi055 ceph-mon[184174]: failed to encode map e610 with expected crc 2023-09-23T18:20:10.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:10 smithi055 ceph-mon[184174]: pgmap v1455: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 890 op/s; 16502/222315 objects degraded (7.423%); 1644/222315 objects misplaced (0.739%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:20:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:10 smithi055 ceph-mon[179668]: failed to encode map e610 with expected crc 2023-09-23T18:20:10.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:10 smithi055 ceph-mon[179668]: pgmap v1455: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.5 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 890 op/s; 16502/222315 objects degraded (7.423%); 1644/222315 objects misplaced (0.739%); 3.1 MiB/s, 49 objects/s recovering 2023-09-23T18:20:10.737 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:10.800 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:10.815 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:20:10.869 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:10.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:10.878 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:10.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:11.031 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:20:11.055 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:11.140 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:20:11.881 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:11.928 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:12.574 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:12.683 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:12 smithi055 ceph-mon[184174]: pgmap v1456: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 866 op/s; 16502/217587 objects degraded (7.584%); 1644/217587 objects misplaced (0.756%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:20:12.689 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:12 smithi055 ceph-mon[179668]: pgmap v1456: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 866 op/s; 16502/217587 objects degraded (7.584%); 1644/217587 objects misplaced (0.756%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:20:12.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:12 smithi099 ceph-mon[167076]: pgmap v1456: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.4 GiB data, 18 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 866 op/s; 16502/217587 objects degraded (7.584%); 1644/217587 objects misplaced (0.756%); 3.0 MiB/s, 48 objects/s recovering 2023-09-23T18:20:12.837 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:12.845 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:12.845 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:12.946 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:13.013 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:13.216 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:13.226 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:13.227 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:13.328 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:13.408 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:13.500 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:13.504 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:13.504 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:13 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:13.504 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16502/217587 objects degraded (7.584%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:13.515 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:13.515 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:13.616 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:13.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:13.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:13 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:13.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16502/217587 objects degraded (7.584%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:13.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:13.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:13 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:13.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16502/217587 objects degraded (7.584%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:13.692 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:13.776 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:13.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:13.784 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:13.885 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:13.965 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:14.080 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:20:14.080 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:20:14.080 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:20:14.081 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:20:14.081 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:20:14.098 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:14.110 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:14.110 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:14.211 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:14.296 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:14.523 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:14 smithi099 ceph-mon[167076]: pgmap v1457: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 849 op/s; 16502/210828 objects degraded (7.827%); 1393/210828 objects misplaced (0.661%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:20:14.524 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:14.532 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:14.532 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:14.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:20:14.633 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:14.688 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:14.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:14 smithi055 ceph-mon[184174]: pgmap v1457: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 849 op/s; 16502/210828 objects degraded (7.827%); 1393/210828 objects misplaced (0.661%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:20:14.775 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:14 smithi055 ceph-mon[179668]: pgmap v1457: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 849 op/s; 16502/210828 objects degraded (7.827%); 1393/210828 objects misplaced (0.661%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:20:14.819 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:14.824 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:14.825 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:14.925 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:15.016 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:15.080 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:15.085 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:15.085 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:15.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:15.198 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:20:15.231 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:20:15.241 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:15.267 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:20:15.302 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:20:15.332 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:15.336 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:20:15.343 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:15.343 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:15.391 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:20:15.427 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:20:15.444 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:15.466 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:20:15.476 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:20:15.476 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:20:15.476 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:20:15.476 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:20:15.476 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:20:15.476 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:20:15.476 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:20:15.477 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:20:15.489 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:15.511 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:15.532 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/426440693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:15.578 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:15.584 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:15.585 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:15.685 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:15.744 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/426440693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:15.750 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:15.815 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:15.821 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:15.822 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:15.922 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:15.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/426440693' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:15.989 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:16.107 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:16.114 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:16.115 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:16.215 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:16.281 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:16.361 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:16.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:16.369 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:16.470 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:16.559 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:16.608 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:16.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:16.611 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:16.704 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:20:16.706 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:20:16.711 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:16.759 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:16.824 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:16 smithi099 ceph-mon[167076]: pgmap v1458: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 16502/210033 objects degraded (7.857%); 1393/210033 objects misplaced (0.663%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:20:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:16 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:16 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:16.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3761510810' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:16.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:16.828 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:16.829 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:16 smithi055 ceph-mon[184174]: pgmap v1458: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 16502/210033 objects degraded (7.857%); 1393/210033 objects misplaced (0.663%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:20:16.829 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:16 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:16.829 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:16 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:16.829 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3761510810' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:16.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:16 smithi055 ceph-mon[179668]: pgmap v1458: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.3 GiB data, 17 GiB used, 519 GiB / 536 GiB avail; 0 B/s wr, 847 op/s; 16502/210033 objects degraded (7.857%); 1393/210033 objects misplaced (0.663%); 3.2 MiB/s, 51 objects/s recovering 2023-09-23T18:20:16.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:16.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:16 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:16.834 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3761510810' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:16.929 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:16.950 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:20:17.005 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:17.186 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1458,"stamp":"2023-09-23T18:20:15.157093+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4590877372,"num_objects":70011,"num_object_clones":0,"num_object_copies":210033,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":16502,"num_objects_misplaced":1393,"num_objects_unfound":0,"num_objects_dirty":70011,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":137735,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5658,"num_bytes_recovered":360734345,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82946,"ondisk_log_size":82946,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":18080052,"kb_used_data":14637252,"kb_used_omap":40,"kb_used_meta":3442391,"kb_avail":544390860,"statfs":{"total":575970213888,"available":557456240640,"internally_reserved":0,"allocated":14988546048,"data_stored":14973918478,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3525008698},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,1,2,8,6],"upper_bound":32},"perf_stat":{"commit_latency_ms":7,"apply_latency_ms":7,"commit_latency_ns":7000000,"apply_latency_ns":7000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-556007424,"num_objects":-8484,"num_object_clones":0,"num_object_copies":-25452,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":9,"num_objects_misplaced":-525,"num_objects_unfound":0,"num_objects_dirty":-8484,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8484,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":516,"num_bytes_recovered":33816576,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005666"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":898,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.387750+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:19:59.387750+0000","last_peered":"2023-09-23T18:19:59.387750+0000","last_clean":"2023-09-23T18:19:59.387750+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:19:59.387750+0000","last_undegraded":"2023-09-23T18:19:59.387750+0000","last_fullsized":"2023-09-23T18:19:59.387750+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":269,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.407350+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:19:59.407350+0000","last_peered":"2023-09-23T18:19:59.407350+0000","last_clean":"2023-09-23T18:19:59.407350+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:19:59.407350+0000","last_undegraded":"2023-09-23T18:19:59.407350+0000","last_fullsized":"2023-09-23T18:19:59.407350+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":26,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.389583+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:19:59.389583+0000","last_peered":"2023-09-23T18:19:59.389583+0000","last_clean":"2023-09-23T18:19:59.389583+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:19:59.389583+0000","last_undegraded":"2023-09-23T18:19:59.389583+0000","last_fullsized":"2023-09-23T18:19:59.389583+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":904,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.387444+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:19:59.387444+0000","last_peered":"2023-09-23T18:19:59.387444+0000","last_clean":"2023-09-23T18:19:59.387444+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:19:59.387444+0000","last_undegraded":"2023-09-23T18:19:59.387444+0000","last_fullsized":"2023-09-23T18:19:59.387444+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":350,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.390119+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:19:59.390119+0000","last_peered":"2023-09-23T18:19:59.390119+0000","last_clean":"2023-09-23T18:19:59.390119+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:19:59.390119+0000","last_undegraded":"2023-09-23T18:19:59.390119+0000","last_fullsized":"2023-09-23T18:19:59.390119+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":670,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.399164+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:19:59.399164+0000","last_peered":"2023-09-23T18:19:59.399164+0000","last_clean":"2023-09-23T18:19:59.399164+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:19:59.399164+0000","last_undegraded":"2023-09-23T18:19:59.399164+0000","last_fullsized":"2023-09-23T18:19:59.399164+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":921,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.385704+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:19:59.385704+0000","last_peered":"2023-09-23T18:19:59.385704+0000","last_clean":"2023-09-23T18:19:59.385704+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:19:59.385704+0000","last_undegraded":"2023-09-23T18:19:59.385704+0000","last_fullsized":"2023-09-23T18:19:59.385704+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":26,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.388848+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:59.388848+0000","last_peered":"2023-09-23T18:19:59.388848+0000","last_clean":"2023-09-23T18:19:59.388848+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:19:59.388848+0000","last_undegraded":"2023-09-23T18:19:59.388848+0000","last_fullsized":"2023-09-23T18:19:59.388848+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":186,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.400923+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:19:59.400923+0000","last_peered":"2023-09-23T18:19:59.400923+0000","last_clean":"2023-09-23T18:19:59.400923+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:19:59.400923+0000","last_undegraded":"2023-09-23T18:19:59.400923+0000","last_fullsized":"2023-09-23T18:19:59.400923+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1044,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.387817+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:19:59.387817+0000","last_peered":"2023-09-23T18:19:59.387817+0000","last_clean":"2023-09-23T18:19:59.387817+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:19:59.387817+0000","last_undegraded":"2023-09-23T18:19:59.387817+0000","last_fullsized":"2023-09-23T18:19:59.387817+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":120,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386499+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:19:59.386499+0000","last_peered":"2023-09-23T18:19:59.386499+0000","last_clean":"2023-09-23T18:19:59.386499+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:19:59.386499+0000","last_undegraded":"2023-09-23T18:19:59.386499+0000","last_fullsized":"2023-09-23T18:19:59.386499+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":199,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.389098+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:19:59.389098+0000","last_peered":"2023-09-23T18:19:59.389098+0000","last_clean":"2023-09-23T18:19:59.389098+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:19:59.389098+0000","last_undegraded":"2023-09-23T18:19:59.389098+0000","last_fullsized":"2023-09-23T18:19:59.389098+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":821,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.395773+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:19:59.395773+0000","last_peered":"2023-09-23T18:19:59.395773+0000","last_clean":"2023-09-23T18:19:59.395773+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:19:59.395773+0000","last_undegraded":"2023-09-23T18:19:59.395773+0000","last_fullsized":"2023-09-23T18:19:59.395773+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":156,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.387529+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:19:59.387529+0000","last_peered":"2023-09-23T18:19:59.387529+0000","last_clean":"2023-09-23T18:19:59.387529+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:19:59.387529+0000","last_undegraded":"2023-09-23T18:19:59.387529+0000","last_fullsized":"2023-09-23T18:19:59.387529+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":155,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.395599+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:19:59.395599+0000","last_peered":"2023-09-23T18:19:59.395599+0000","last_clean":"2023-09-23T18:19:59.395599+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:19:59.395599+0000","last_undegraded":"2023-09-23T18:19:59.395599+0000","last_fullsized":"2023-09-23T18:19:59.395599+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":955,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386571+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:19:59.386571+0000","last_peered":"2023-09-23T18:19:59.386571+0000","last_clean":"2023-09-23T18:19:59.386571+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:19:59.386571+0000","last_undegraded":"2023-09-23T18:19:59.386571+0000","last_fullsized":"2023-09-23T18:19:59.386571+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":404,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.389927+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:19:59.389927+0000","last_peered":"2023-09-23T18:19:59.389927+0000","last_clean":"2023-09-23T18:19:59.389927+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:19:59.389927+0000","last_undegraded":"2023-09-23T18:19:59.389927+0000","last_fullsized":"2023-09-23T18:19:59.389927+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":423,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.397998+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:19:59.397998+0000","last_peered":"2023-09-23T18:19:59.397998+0000","last_clean":"2023-09-23T18:19:59.397998+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:19:59.397998+0000","last_undegraded":"2023-09-23T18:19:59.397998+0000","last_fullsized":"2023-09-23T18:19:59.397998+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":984,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.415949+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:19:59.415949+0000","last_peered":"2023-09-23T18:19:59.415949+0000","last_clean":"2023-09-23T18:19:59.415949+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:19:59.415949+0000","last_undegraded":"2023-09-23T18:19:59.415949+0000","last_fullsized":"2023-09-23T18:19:59.415949+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":162,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396273+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:19:59.396273+0000","last_peered":"2023-09-23T18:19:59.396273+0000","last_clean":"2023-09-23T18:19:59.396273+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:19:59.396273+0000","last_undegraded":"2023-09-23T18:19:59.396273+0000","last_fullsized":"2023-09-23T18:19:59.396273+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":429,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.400001+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:19:59.400001+0000","last_peered":"2023-09-23T18:19:59.400001+0000","last_clean":"2023-09-23T18:19:59.400001+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:19:59.400001+0000","last_undegraded":"2023-09-23T18:19:59.400001+0000","last_fullsized":"2023-09-23T18:19:59.400001+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":836,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396494+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:19:59.396494+0000","last_peered":"2023-09-23T18:19:59.396494+0000","last_clean":"2023-09-23T18:19:59.396494+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:19:59.396494+0000","last_undegraded":"2023-09-23T18:19:59.396494+0000","last_fullsized":"2023-09-23T18:19:59.396494+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":429,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.399876+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:19:59.399876+0000","last_peered":"2023-09-23T18:19:59.399876+0000","last_clean":"2023-09-23T18:19:59.399876+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:19:59.399876+0000","last_undegraded":"2023-09-23T18:19:59.399876+0000","last_fullsized":"2023-09-23T18:19:59.399876+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":107,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.385835+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:19:59.385835+0000","last_peered":"2023-09-23T18:19:59.385835+0000","last_clean":"2023-09-23T18:19:59.385835+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:19:59.385835+0000","last_undegraded":"2023-09-23T18:19:59.385835+0000","last_fullsized":"2023-09-23T18:19:59.385835+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":807,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.387442+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:19:59.387442+0000","last_peered":"2023-09-23T18:19:59.387442+0000","last_clean":"2023-09-23T18:19:59.387442+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:19:59.387442+0000","last_undegraded":"2023-09-23T18:19:59.387442+0000","last_fullsized":"2023-09-23T18:19:59.387442+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":125,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.399300+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:19:59.399300+0000","last_peered":"2023-09-23T18:19:59.399300+0000","last_clean":"2023-09-23T18:19:59.399300+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:19:59.399300+0000","last_undegraded":"2023-09-23T18:19:59.399300+0000","last_fullsized":"2023-09-23T18:19:59.399300+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":120,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.388031+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:19:59.388031+0000","last_peered":"2023-09-23T18:19:59.388031+0000","last_clean":"2023-09-23T18:19:59.388031+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:19:59.388031+0000","last_undegraded":"2023-09-23T18:19:59.388031+0000","last_fullsized":"2023-09-23T18:19:59.388031+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":745,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.391679+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:19:59.391679+0000","last_peered":"2023-09-23T18:19:59.391679+0000","last_clean":"2023-09-23T18:19:59.391679+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:19:59.391679+0000","last_undegraded":"2023-09-23T18:19:59.391679+0000","last_fullsized":"2023-09-23T18:19:59.391679+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":197,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.395463+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:19:59.395463+0000","last_peered":"2023-09-23T18:19:59.395463+0000","last_clean":"2023-09-23T18:19:59.395463+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:19:59.395463+0000","last_undegraded":"2023-09-23T18:19:59.395463+0000","last_fullsized":"2023-09-23T18:19:59.395463+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":208,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.389747+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:19:59.389747+0000","last_peered":"2023-09-23T18:19:59.389747+0000","last_clean":"2023-09-23T18:19:59.389747+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:19:59.389747+0000","last_undegraded":"2023-09-23T18:19:59.389747+0000","last_fullsized":"2023-09-23T18:19:59.389747+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":841,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.395312+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:19:59.395312+0000","last_peered":"2023-09-23T18:19:59.395312+0000","last_clean":"2023-09-23T18:19:59.395312+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:19:59.395312+0000","last_undegraded":"2023-09-23T18:19:59.395312+0000","last_fullsized":"2023-09-23T18:19:59.395312+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":750,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.397913+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:19:59.397913+0000","last_peered":"2023-09-23T18:19:59.397913+0000","last_clean":"2023-09-23T18:19:59.397913+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:19:59.397913+0000","last_undegraded":"2023-09-23T18:19:59.397913+0000","last_fullsized":"2023-09-23T18:19:59.397913+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":26,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.398933+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:19:59.398933+0000","last_peered":"2023-09-23T18:19:59.398933+0000","last_clean":"2023-09-23T18:19:59.398933+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:19:59.398933+0000","last_undegraded":"2023-09-23T18:19:59.398933+0000","last_fullsized":"2023-09-23T18:19:59.398933+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"610'8667","reported_seq":8711,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:11.091663+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:20:11.091663+0000","last_peered":"2023-09-23T18:20:11.091663+0000","last_clean":"2023-09-23T18:20:11.091663+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:20:11.091663+0000","last_undegraded":"2023-09-23T18:20:11.091663+0000","last_fullsized":"2023-09-23T18:20:11.091663+0000","mapping_epoch":574,"log_start":"586'3200","ondisk_log_start":"586'3200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5467,"log_dups_size":0,"ondisk_log_size":5467,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288817152,"num_objects":4407,"num_object_clones":0,"num_object_copies":13221,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4407,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8667,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":875,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.398052+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:19:59.398052+0000","last_peered":"2023-09-23T18:19:59.398052+0000","last_clean":"2023-09-23T18:19:59.398052+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:19:59.398052+0000","last_undegraded":"2023-09-23T18:19:59.398052+0000","last_fullsized":"2023-09-23T18:19:59.398052+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":246,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.407675+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:19:59.407675+0000","last_peered":"2023-09-23T18:19:59.407675+0000","last_clean":"2023-09-23T18:19:59.407675+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:19:59.407675+0000","last_undegraded":"2023-09-23T18:19:59.407675+0000","last_fullsized":"2023-09-23T18:19:59.407675+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":120,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.387604+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:19:59.387604+0000","last_peered":"2023-09-23T18:19:59.387604+0000","last_clean":"2023-09-23T18:19:59.387604+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:19:59.387604+0000","last_undegraded":"2023-09-23T18:19:59.387604+0000","last_fullsized":"2023-09-23T18:19:59.387604+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"610'8414","reported_seq":8468,"reported_epoch":610,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:11.209391+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:20:11.209391+0000","last_peered":"2023-09-23T18:20:11.209391+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:20:11.209391+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"593'4000","ondisk_log_start":"593'4000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4414,"log_dups_size":0,"ondisk_log_size":4414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280363008,"num_objects":4278,"num_object_clones":0,"num_object_copies":12834,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4278,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8414,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":2080},{"shards":"5,6","objects":130}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":729,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.389568+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:19:59.389568+0000","last_peered":"2023-09-23T18:19:59.389568+0000","last_clean":"2023-09-23T18:19:59.389568+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:19:59.389568+0000","last_undegraded":"2023-09-23T18:19:59.389568+0000","last_fullsized":"2023-09-23T18:19:59.389568+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":247,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.408378+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:19:59.408378+0000","last_peered":"2023-09-23T18:19:59.408378+0000","last_clean":"2023-09-23T18:19:59.408378+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:19:59.408378+0000","last_undegraded":"2023-09-23T18:19:59.408378+0000","last_fullsized":"2023-09-23T18:19:59.408378+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":226,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.407549+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:19:59.407549+0000","last_peered":"2023-09-23T18:19:59.407549+0000","last_clean":"2023-09-23T18:19:59.407549+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:19:59.407549+0000","last_undegraded":"2023-09-23T18:19:59.407549+0000","last_fullsized":"2023-09-23T18:19:59.407549+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"610'8473","reported_seq":15477,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:10.222780+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:20:10.222780+0000","last_peered":"2023-09-23T18:20:10.222780+0000","last_clean":"2023-09-23T18:20:10.222780+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:20:10.222780+0000","last_undegraded":"2023-09-23T18:20:10.222780+0000","last_fullsized":"2023-09-23T18:20:10.222780+0000","mapping_epoch":605,"log_start":"591'3600","ondisk_log_start":"591'3600","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4873,"log_dups_size":0,"ondisk_log_size":4873,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286457856,"num_objects":4371,"num_object_clones":0,"num_object_copies":13113,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4371,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8473,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":625,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.400850+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:19:59.400850+0000","last_peered":"2023-09-23T18:19:59.400850+0000","last_clean":"2023-09-23T18:19:59.400850+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:19:59.400850+0000","last_undegraded":"2023-09-23T18:19:59.400850+0000","last_fullsized":"2023-09-23T18:19:59.400850+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":535,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.387896+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:19:59.387896+0000","last_peered":"2023-09-23T18:19:59.387896+0000","last_clean":"2023-09-23T18:19:59.387896+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:19:59.387896+0000","last_undegraded":"2023-09-23T18:19:59.387896+0000","last_fullsized":"2023-09-23T18:19:59.387896+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":186,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.397865+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:19:59.397865+0000","last_peered":"2023-09-23T18:19:59.397865+0000","last_clean":"2023-09-23T18:19:59.397865+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:19:59.397865+0000","last_undegraded":"2023-09-23T18:19:59.397865+0000","last_fullsized":"2023-09-23T18:19:59.397865+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"610'8627","reported_seq":8673,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:11.101862+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:20:11.101862+0000","last_peered":"2023-09-23T18:20:11.101862+0000","last_clean":"2023-09-23T18:20:11.101862+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:20:11.101862+0000","last_undegraded":"2023-09-23T18:20:11.101862+0000","last_fullsized":"2023-09-23T18:20:11.101862+0000","mapping_epoch":574,"log_start":"587'3500","ondisk_log_start":"587'3500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5127,"log_dups_size":0,"ondisk_log_size":5127,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292618240,"num_objects":4465,"num_object_clones":0,"num_object_copies":13395,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4465,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8627,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":641,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.489902+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:20:08.489902+0000","last_peered":"2023-09-23T18:20:08.489902+0000","last_clean":"2023-09-23T18:20:08.489902+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:20:08.489902+0000","last_undegraded":"2023-09-23T18:20:08.489902+0000","last_fullsized":"2023-09-23T18:20:08.489902+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":134,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386181+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:19:59.386181+0000","last_peered":"2023-09-23T18:19:59.386181+0000","last_clean":"2023-09-23T18:19:59.386181+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:19:59.386181+0000","last_undegraded":"2023-09-23T18:19:59.386181+0000","last_fullsized":"2023-09-23T18:19:59.386181+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":447,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.490674+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:20:08.490674+0000","last_peered":"2023-09-23T18:20:08.490674+0000","last_clean":"2023-09-23T18:20:08.490674+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:08.490674+0000","last_undegraded":"2023-09-23T18:20:08.490674+0000","last_fullsized":"2023-09-23T18:20:08.490674+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"610'8765","reported_seq":8811,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:13.885150+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:20:13.885150+0000","last_peered":"2023-09-23T18:20:13.885150+0000","last_clean":"2023-09-23T18:20:13.885150+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:20:13.885150+0000","last_undegraded":"2023-09-23T18:20:13.885150+0000","last_fullsized":"2023-09-23T18:20:13.885150+0000","mapping_epoch":574,"log_start":"598'4400","ondisk_log_start":"598'4400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4365,"log_dups_size":0,"ondisk_log_size":4365,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":280428544,"num_objects":4279,"num_object_clones":0,"num_object_copies":12837,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4279,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8765,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1092,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396354+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:19:59.396354+0000","last_peered":"2023-09-23T18:19:59.396354+0000","last_clean":"2023-09-23T18:19:59.396354+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:19:59.396354+0000","last_undegraded":"2023-09-23T18:19:59.396354+0000","last_fullsized":"2023-09-23T18:19:59.396354+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":347,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.488298+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:20:08.488298+0000","last_peered":"2023-09-23T18:20:08.488298+0000","last_clean":"2023-09-23T18:20:08.488298+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:20:08.488298+0000","last_undegraded":"2023-09-23T18:20:08.488298+0000","last_fullsized":"2023-09-23T18:20:08.488298+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":154,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.395913+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:19:59.395913+0000","last_peered":"2023-09-23T18:19:59.395913+0000","last_clean":"2023-09-23T18:19:59.395913+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:19:59.395913+0000","last_undegraded":"2023-09-23T18:19:59.395913+0000","last_fullsized":"2023-09-23T18:19:59.395913+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"610'8650","reported_seq":8703,"reported_epoch":610,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:11.131002+0000","last_change":"2023-09-23T18:18:47.594759+0000","last_active":"2023-09-23T18:20:11.131002+0000","last_peered":"2023-09-23T18:20:11.131002+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:20:11.131002+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"585'3100","ondisk_log_start":"585'3100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5550,"log_dups_size":0,"ondisk_log_size":5550,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287047680,"num_objects":4380,"num_object_clones":0,"num_object_copies":13140,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4380,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8650,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":2091},{"shards":"4,5","objects":154}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":879,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.387724+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:19:59.387724+0000","last_peered":"2023-09-23T18:19:59.387724+0000","last_clean":"2023-09-23T18:19:59.387724+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:19:59.387724+0000","last_undegraded":"2023-09-23T18:19:59.387724+0000","last_fullsized":"2023-09-23T18:19:59.387724+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":162,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.397795+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:19:59.397795+0000","last_peered":"2023-09-23T18:19:59.397795+0000","last_clean":"2023-09-23T18:19:59.397795+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:19:59.397795+0000","last_undegraded":"2023-09-23T18:19:59.397795+0000","last_fullsized":"2023-09-23T18:19:59.397795+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":461,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.398177+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:19:59.398177+0000","last_peered":"2023-09-23T18:19:59.398177+0000","last_clean":"2023-09-23T18:19:59.398177+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:19:59.398177+0000","last_undegraded":"2023-09-23T18:19:59.398177+0000","last_fullsized":"2023-09-23T18:19:59.398177+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"610'8551","reported_seq":8595,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:11.213598+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:20:11.213598+0000","last_peered":"2023-09-23T18:20:11.213598+0000","last_clean":"2023-09-23T18:20:11.213598+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:20:11.213598+0000","last_undegraded":"2023-09-23T18:20:11.213598+0000","last_fullsized":"2023-09-23T18:20:11.213598+0000","mapping_epoch":574,"log_start":"593'4100","ondisk_log_start":"593'4100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4451,"log_dups_size":0,"ondisk_log_size":4451,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":294977536,"num_objects":4501,"num_object_clones":0,"num_object_copies":13503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8551,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":881,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.389670+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:19:59.389670+0000","last_peered":"2023-09-23T18:19:59.389670+0000","last_clean":"2023-09-23T18:19:59.389670+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:19:59.389670+0000","last_undegraded":"2023-09-23T18:19:59.389670+0000","last_fullsized":"2023-09-23T18:19:59.389670+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":446,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396210+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:19:59.396210+0000","last_peered":"2023-09-23T18:19:59.396210+0000","last_clean":"2023-09-23T18:19:59.396210+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:19:59.396210+0000","last_undegraded":"2023-09-23T18:19:59.396210+0000","last_fullsized":"2023-09-23T18:19:59.396210+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":222,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396155+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:19:59.396155+0000","last_peered":"2023-09-23T18:19:59.396155+0000","last_clean":"2023-09-23T18:19:59.396155+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:19:59.396155+0000","last_undegraded":"2023-09-23T18:19:59.396155+0000","last_fullsized":"2023-09-23T18:19:59.396155+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"610'8722","reported_seq":8766,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:12.319786+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:20:12.319786+0000","last_peered":"2023-09-23T18:20:12.319786+0000","last_clean":"2023-09-23T18:20:12.319786+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:20:12.319786+0000","last_undegraded":"2023-09-23T18:20:12.319786+0000","last_fullsized":"2023-09-23T18:20:12.319786+0000","mapping_epoch":574,"log_start":"593'4100","ondisk_log_start":"593'4100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4622,"log_dups_size":0,"ondisk_log_size":4622,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":282591232,"num_objects":4312,"num_object_clones":0,"num_object_copies":12936,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4312,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8722,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":862,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386066+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:19:59.386066+0000","last_peered":"2023-09-23T18:19:59.386066+0000","last_clean":"2023-09-23T18:19:59.386066+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:19:59.386066+0000","last_undegraded":"2023-09-23T18:19:59.386066+0000","last_fullsized":"2023-09-23T18:19:59.386066+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":26,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.388996+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:19:59.388996+0000","last_peered":"2023-09-23T18:19:59.388996+0000","last_clean":"2023-09-23T18:19:59.388996+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:19:59.388996+0000","last_undegraded":"2023-09-23T18:19:59.388996+0000","last_fullsized":"2023-09-23T18:19:59.388996+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":338,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.416467+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:19:59.416467+0000","last_peered":"2023-09-23T18:19:59.416467+0000","last_clean":"2023-09-23T18:19:59.416467+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:19:59.416467+0000","last_undegraded":"2023-09-23T18:19:59.416467+0000","last_fullsized":"2023-09-23T18:19:59.416467+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"610'8607","reported_seq":8654,"reported_epoch":610,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:10.195847+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:20:10.195847+0000","last_peered":"2023-09-23T18:20:10.195847+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:20:10.195847+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"592'3700","ondisk_log_start":"592'3700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4907,"log_dups_size":0,"ondisk_log_size":4907,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":292356096,"num_objects":4461,"num_object_clones":0,"num_object_copies":13383,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4461,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8607,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":176},{"shards":"0,4,6","objects":2212}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":721,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.401025+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:19:59.401025+0000","last_peered":"2023-09-23T18:19:59.401025+0000","last_clean":"2023-09-23T18:19:59.401025+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:19:59.401025+0000","last_undegraded":"2023-09-23T18:19:59.401025+0000","last_fullsized":"2023-09-23T18:19:59.401025+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":960,"reported_epoch":609,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:19:59.415412+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:19:59.415412+0000","last_peered":"2023-09-23T18:19:59.415412+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:19:59.415412+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":144,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.388095+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:19:59.388095+0000","last_peered":"2023-09-23T18:19:59.388095+0000","last_clean":"2023-09-23T18:19:59.388095+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:19:59.388095+0000","last_undegraded":"2023-09-23T18:19:59.388095+0000","last_fullsized":"2023-09-23T18:19:59.388095+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":124,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.385992+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:19:59.385992+0000","last_peered":"2023-09-23T18:19:59.385992+0000","last_clean":"2023-09-23T18:19:59.385992+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:19:59.385992+0000","last_undegraded":"2023-09-23T18:19:59.385992+0000","last_fullsized":"2023-09-23T18:19:59.385992+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"610'8530","reported_seq":8582,"reported_epoch":610,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:11.242654+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:20:11.242654+0000","last_peered":"2023-09-23T18:20:11.242654+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:20:11.242654+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"593'4100","ondisk_log_start":"593'4100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4430,"log_dups_size":0,"ondisk_log_size":4430,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":288751616,"num_objects":4406,"num_object_clones":0,"num_object_copies":13218,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4406,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8530,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":1496},{"shards":"5,6","objects":847}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":793,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396646+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:19:59.396646+0000","last_peered":"2023-09-23T18:19:59.396646+0000","last_clean":"2023-09-23T18:19:59.396646+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:19:59.396646+0000","last_undegraded":"2023-09-23T18:19:59.396646+0000","last_fullsized":"2023-09-23T18:19:59.396646+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1494,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.385924+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:19:59.385924+0000","last_peered":"2023-09-23T18:19:59.385924+0000","last_clean":"2023-09-23T18:19:59.385924+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:19:59.385924+0000","last_undegraded":"2023-09-23T18:19:59.385924+0000","last_fullsized":"2023-09-23T18:19:59.385924+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":134,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.399941+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:19:59.399941+0000","last_peered":"2023-09-23T18:19:59.399941+0000","last_clean":"2023-09-23T18:19:59.399941+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:19:59.399941+0000","last_undegraded":"2023-09-23T18:19:59.399941+0000","last_fullsized":"2023-09-23T18:19:59.399941+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":434,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396130+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:19:59.396130+0000","last_peered":"2023-09-23T18:19:59.396130+0000","last_clean":"2023-09-23T18:19:59.396130+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:19:59.396130+0000","last_undegraded":"2023-09-23T18:19:59.396130+0000","last_fullsized":"2023-09-23T18:19:59.396130+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"610'8373","reported_seq":8417,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:11.127014+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:20:11.127014+0000","last_peered":"2023-09-23T18:20:11.127014+0000","last_clean":"2023-09-23T18:20:11.127014+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:20:11.127014+0000","last_undegraded":"2023-09-23T18:20:11.127014+0000","last_fullsized":"2023-09-23T18:20:11.127014+0000","mapping_epoch":574,"log_start":"585'2900","ondisk_log_start":"585'2900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5473,"log_dups_size":0,"ondisk_log_size":5473,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":276103168,"num_objects":4213,"num_object_clones":0,"num_object_copies":12639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8373,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":818,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386146+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:19:59.386146+0000","last_peered":"2023-09-23T18:19:59.386146+0000","last_clean":"2023-09-23T18:19:59.386146+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:19:59.386146+0000","last_undegraded":"2023-09-23T18:19:59.386146+0000","last_fullsized":"2023-09-23T18:19:59.386146+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":24,"reported_epoch":609,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:19:59.389944+0000","last_change":"2023-09-23T18:18:46.432707+0000","last_active":"2023-09-23T18:19:59.389944+0000","last_peered":"2023-09-23T18:19:59.389944+0000","last_clean":"2023-09-23T18:11:33.614822+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:19:59.389944+0000","last_undegraded":"2023-09-23T18:18:46.418246+0000","last_fullsized":"2023-09-23T18:19:59.389944+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":403,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.398203+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:19:59.398203+0000","last_peered":"2023-09-23T18:19:59.398203+0000","last_clean":"2023-09-23T18:19:59.398203+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:19:59.398203+0000","last_undegraded":"2023-09-23T18:19:59.398203+0000","last_fullsized":"2023-09-23T18:19:59.398203+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":146,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386438+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:19:59.386438+0000","last_peered":"2023-09-23T18:19:59.386438+0000","last_clean":"2023-09-23T18:19:59.386438+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:19:59.386438+0000","last_undegraded":"2023-09-23T18:19:59.386438+0000","last_fullsized":"2023-09-23T18:19:59.386438+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"610'8574","reported_seq":8618,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:11.185803+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:20:11.185803+0000","last_peered":"2023-09-23T18:20:11.185803+0000","last_clean":"2023-09-23T18:20:11.185803+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:20:11.185803+0000","last_undegraded":"2023-09-23T18:20:11.185803+0000","last_fullsized":"2023-09-23T18:20:11.185803+0000","mapping_epoch":574,"log_start":"593'4100","ondisk_log_start":"593'4100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4474,"log_dups_size":0,"ondisk_log_size":4474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286654464,"num_objects":4374,"num_object_clones":0,"num_object_copies":13122,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4374,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8574,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1507,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386312+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:19:59.386312+0000","last_peered":"2023-09-23T18:19:59.386312+0000","last_clean":"2023-09-23T18:19:59.386312+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:19:59.386312+0000","last_undegraded":"2023-09-23T18:19:59.386312+0000","last_fullsized":"2023-09-23T18:19:59.386312+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":560,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.398080+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:19:59.398080+0000","last_peered":"2023-09-23T18:19:59.398080+0000","last_clean":"2023-09-23T18:19:59.398080+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:19:59.398080+0000","last_undegraded":"2023-09-23T18:19:59.398080+0000","last_fullsized":"2023-09-23T18:19:59.398080+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":120,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.385789+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:19:59.385789+0000","last_peered":"2023-09-23T18:19:59.385789+0000","last_clean":"2023-09-23T18:19:59.385789+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:19:59.385789+0000","last_undegraded":"2023-09-23T18:19:59.385789+0000","last_fullsized":"2023-09-23T18:19:59.385789+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":155,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.398399+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:19:59.398399+0000","last_peered":"2023-09-23T18:19:59.398399+0000","last_clean":"2023-09-23T18:19:59.398399+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:19:59.398399+0000","last_undegraded":"2023-09-23T18:19:59.398399+0000","last_fullsized":"2023-09-23T18:19:59.398399+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"610'8412","reported_seq":8456,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:11.002605+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:20:11.002605+0000","last_peered":"2023-09-23T18:20:11.002605+0000","last_clean":"2023-09-23T18:20:11.002605+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:20:11.002605+0000","last_undegraded":"2023-09-23T18:20:11.002605+0000","last_fullsized":"2023-09-23T18:20:11.002605+0000","mapping_epoch":574,"log_start":"585'2900","ondisk_log_start":"585'2900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5512,"log_dups_size":0,"ondisk_log_size":5512,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":286523392,"num_objects":4372,"num_object_clones":0,"num_object_copies":13116,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4372,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8412,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":808,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.400065+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:19:59.400065+0000","last_peered":"2023-09-23T18:19:59.400065+0000","last_clean":"2023-09-23T18:19:59.400065+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:19:59.400065+0000","last_undegraded":"2023-09-23T18:19:59.400065+0000","last_fullsized":"2023-09-23T18:19:59.400065+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":123,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.389107+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:19:59.389107+0000","last_peered":"2023-09-23T18:19:59.389107+0000","last_clean":"2023-09-23T18:19:59.389107+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:19:59.389107+0000","last_undegraded":"2023-09-23T18:19:59.389107+0000","last_fullsized":"2023-09-23T18:19:59.389107+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":26,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.389992+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:19:59.389992+0000","last_peered":"2023-09-23T18:19:59.389992+0000","last_clean":"2023-09-23T18:19:59.389992+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:19:59.389992+0000","last_undegraded":"2023-09-23T18:19:59.389992+0000","last_fullsized":"2023-09-23T18:19:59.389992+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":625,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.387629+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:19:59.387629+0000","last_peered":"2023-09-23T18:19:59.387629+0000","last_clean":"2023-09-23T18:19:59.387629+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:19:59.387629+0000","last_undegraded":"2023-09-23T18:19:59.387629+0000","last_fullsized":"2023-09-23T18:19:59.387629+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"610'8514","reported_seq":12172,"reported_epoch":610,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:11.262562+0000","last_change":"2023-09-23T18:19:32.858380+0000","last_active":"2023-09-23T18:20:11.262562+0000","last_peered":"2023-09-23T18:20:11.262562+0000","last_clean":"2023-09-23T18:18:32.699126+0000","last_became_active":"2023-09-23T18:18:34.716591+0000","last_became_peered":"2023-09-23T18:18:34.716591+0000","last_unstale":"2023-09-23T18:20:11.262562+0000","last_undegraded":"2023-09-23T18:18:34.351246+0000","last_fullsized":"2023-09-23T18:18:34.327168+0000","mapping_epoch":589,"log_start":"593'4100","ondisk_log_start":"593'4100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4414,"log_dups_size":0,"ondisk_log_size":4414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:13.611234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":291176472,"num_objects":4444,"num_object_clones":0,"num_object_copies":13332,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":259,"num_objects_misplaced":1393,"num_objects_unfound":0,"num_objects_dirty":4444,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":8514,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1801,"num_bytes_recovered":118030336,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":1393},{"shards":"4,6","objects":2792}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":908,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396502+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:19:59.396502+0000","last_peered":"2023-09-23T18:19:59.396502+0000","last_clean":"2023-09-23T18:19:59.396502+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:19:59.396502+0000","last_undegraded":"2023-09-23T18:19:59.396502+0000","last_fullsized":"2023-09-23T18:19:59.396502+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":48,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.402765+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:19:59.402765+0000","last_peered":"2023-09-23T18:19:59.402765+0000","last_clean":"2023-09-23T18:19:59.402765+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:19:59.402765+0000","last_undegraded":"2023-09-23T18:19:59.402765+0000","last_fullsized":"2023-09-23T18:19:59.402765+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":134,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.385977+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:19:59.385977+0000","last_peered":"2023-09-23T18:19:59.385977+0000","last_clean":"2023-09-23T18:19:59.385977+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:19:59.385977+0000","last_undegraded":"2023-09-23T18:19:59.385977+0000","last_fullsized":"2023-09-23T18:19:59.385977+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":134,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386231+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:19:59.386231+0000","last_peered":"2023-09-23T18:19:59.386231+0000","last_clean":"2023-09-23T18:19:59.386231+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:19:59.386231+0000","last_undegraded":"2023-09-23T18:19:59.386231+0000","last_fullsized":"2023-09-23T18:19:59.386231+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"610'8522","reported_seq":8566,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:11.246600+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:20:11.246600+0000","last_peered":"2023-09-23T18:20:11.246600+0000","last_clean":"2023-09-23T18:20:11.246600+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:20:11.246600+0000","last_undegraded":"2023-09-23T18:20:11.246600+0000","last_fullsized":"2023-09-23T18:20:11.246600+0000","mapping_epoch":574,"log_start":"593'4100","ondisk_log_start":"593'4100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4422,"log_dups_size":0,"ondisk_log_size":4422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":283901952,"num_objects":4332,"num_object_clones":0,"num_object_copies":12996,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4332,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8522,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":908,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.398250+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:19:59.398250+0000","last_peered":"2023-09-23T18:19:59.398250+0000","last_clean":"2023-09-23T18:19:59.398250+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:19:59.398250+0000","last_undegraded":"2023-09-23T18:19:59.398250+0000","last_fullsized":"2023-09-23T18:19:59.398250+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":96,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386343+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:19:59.386343+0000","last_peered":"2023-09-23T18:19:59.386343+0000","last_clean":"2023-09-23T18:19:59.386343+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:19:59.386343+0000","last_undegraded":"2023-09-23T18:19:59.386343+0000","last_fullsized":"2023-09-23T18:19:59.386343+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":389,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396363+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:19:59.396363+0000","last_peered":"2023-09-23T18:19:59.396363+0000","last_clean":"2023-09-23T18:19:59.396363+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:19:59.396363+0000","last_undegraded":"2023-09-23T18:19:59.396363+0000","last_fullsized":"2023-09-23T18:19:59.396363+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":26,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.388964+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:19:59.388964+0000","last_peered":"2023-09-23T18:19:59.388964+0000","last_clean":"2023-09-23T18:19:59.388964+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:19:59.388964+0000","last_undegraded":"2023-09-23T18:19:59.388964+0000","last_fullsized":"2023-09-23T18:19:59.388964+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"610'8423","reported_seq":8467,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:11.253545+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:20:11.253545+0000","last_peered":"2023-09-23T18:20:11.253545+0000","last_clean":"2023-09-23T18:20:11.253545+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:20:11.253545+0000","last_undegraded":"2023-09-23T18:20:11.253545+0000","last_fullsized":"2023-09-23T18:20:11.253545+0000","mapping_epoch":574,"log_start":"593'4000","ondisk_log_start":"593'4000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4423,"log_dups_size":0,"ondisk_log_size":4423,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":287637504,"num_objects":4389,"num_object_clones":0,"num_object_copies":13167,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4389,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8423,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":742,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.389643+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:19:59.389643+0000","last_peered":"2023-09-23T18:19:59.389643+0000","last_clean":"2023-09-23T18:19:59.389643+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:19:59.389643+0000","last_undegraded":"2023-09-23T18:19:59.389643+0000","last_fullsized":"2023-09-23T18:19:59.389643+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1046,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.401233+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:19:59.401233+0000","last_peered":"2023-09-23T18:19:59.401233+0000","last_clean":"2023-09-23T18:19:59.401233+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:19:59.401233+0000","last_undegraded":"2023-09-23T18:19:59.401233+0000","last_fullsized":"2023-09-23T18:19:59.401233+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":36,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.399361+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:19:59.399361+0000","last_peered":"2023-09-23T18:19:59.399361+0000","last_clean":"2023-09-23T18:19:59.399361+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:19:59.399361+0000","last_undegraded":"2023-09-23T18:19:59.399361+0000","last_fullsized":"2023-09-23T18:19:59.399361+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":400,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396422+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:19:59.396422+0000","last_peered":"2023-09-23T18:19:59.396422+0000","last_clean":"2023-09-23T18:19:59.396422+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:19:59.396422+0000","last_undegraded":"2023-09-23T18:19:59.396422+0000","last_fullsized":"2023-09-23T18:19:59.396422+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":640,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.395672+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:19:59.395672+0000","last_peered":"2023-09-23T18:19:59.395672+0000","last_clean":"2023-09-23T18:19:59.395672+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:19:59.395672+0000","last_undegraded":"2023-09-23T18:19:59.395672+0000","last_fullsized":"2023-09-23T18:19:59.395672+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":32,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.398801+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:19:59.398801+0000","last_peered":"2023-09-23T18:19:59.398801+0000","last_clean":"2023-09-23T18:19:59.398801+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:19:59.398801+0000","last_undegraded":"2023-09-23T18:19:59.398801+0000","last_fullsized":"2023-09-23T18:19:59.398801+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":683,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.415244+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:19:59.415244+0000","last_peered":"2023-09-23T18:19:59.415244+0000","last_clean":"2023-09-23T18:19:59.415244+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:19:59.415244+0000","last_undegraded":"2023-09-23T18:19:59.415244+0000","last_fullsized":"2023-09-23T18:19:59.415244+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":640,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396226+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:19:59.396226+0000","last_peered":"2023-09-23T18:19:59.396226+0000","last_clean":"2023-09-23T18:19:59.396226+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:19:59.396226+0000","last_undegraded":"2023-09-23T18:19:59.396226+0000","last_fullsized":"2023-09-23T18:19:59.396226+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":128,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386053+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:19:59.386053+0000","last_peered":"2023-09-23T18:19:59.386053+0000","last_clean":"2023-09-23T18:19:59.386053+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:19:59.386053+0000","last_undegraded":"2023-09-23T18:19:59.386053+0000","last_fullsized":"2023-09-23T18:19:59.386053+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":244,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.407839+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:19:59.407839+0000","last_peered":"2023-09-23T18:19:59.407839+0000","last_clean":"2023-09-23T18:19:59.407839+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:19:59.407839+0000","last_undegraded":"2023-09-23T18:19:59.407839+0000","last_fullsized":"2023-09-23T18:19:59.407839+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":114,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.387967+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:19:59.387967+0000","last_peered":"2023-09-23T18:19:59.387967+0000","last_clean":"2023-09-23T18:19:59.387967+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:19:59.387967+0000","last_undegraded":"2023-09-23T18:19:59.387967+0000","last_fullsized":"2023-09-23T18:19:59.387967+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":227,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.407429+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:19:59.407429+0000","last_peered":"2023-09-23T18:19:59.407429+0000","last_clean":"2023-09-23T18:19:59.407429+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:19:59.407429+0000","last_undegraded":"2023-09-23T18:19:59.407429+0000","last_fullsized":"2023-09-23T18:19:59.407429+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1002,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.395848+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:19:59.395848+0000","last_peered":"2023-09-23T18:19:59.395848+0000","last_clean":"2023-09-23T18:19:59.395848+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:19:59.395848+0000","last_undegraded":"2023-09-23T18:19:59.395848+0000","last_fullsized":"2023-09-23T18:19:59.395848+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":507,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396433+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:19:59.396433+0000","last_peered":"2023-09-23T18:19:59.396433+0000","last_clean":"2023-09-23T18:19:59.396433+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:19:59.396433+0000","last_undegraded":"2023-09-23T18:19:59.396433+0000","last_fullsized":"2023-09-23T18:19:59.396433+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":144,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.396571+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:19:59.396571+0000","last_peered":"2023-09-23T18:19:59.396571+0000","last_clean":"2023-09-23T18:19:59.396571+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:19:59.396571+0000","last_undegraded":"2023-09-23T18:19:59.396571+0000","last_fullsized":"2023-09-23T18:19:59.396571+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":939,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.417465+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:19:59.417465+0000","last_peered":"2023-09-23T18:19:59.417465+0000","last_clean":"2023-09-23T18:19:59.417465+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:19:59.417465+0000","last_undegraded":"2023-09-23T18:19:59.417465+0000","last_fullsized":"2023-09-23T18:19:59.417465+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":132,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.386421+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:19:59.386421+0000","last_peered":"2023-09-23T18:19:59.386421+0000","last_clean":"2023-09-23T18:19:59.386421+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:19:59.386421+0000","last_undegraded":"2023-09-23T18:19:59.386421+0000","last_fullsized":"2023-09-23T18:19:59.386421+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":653,"reported_epoch":609,"state":"active+clean","last_fresh":"2023-09-23T18:19:59.397989+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:19:59.397989+0000","last_peered":"2023-09-23T18:19:59.397989+0000","last_clean":"2023-09-23T18:19:59.397989+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:19:59.397989+0000","last_undegraded":"2023-09-23T18:19:59.397989+0000","last_fullsized":"2023-09-23T18:19:59.397989+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4586405912,"num_objects":69984,"num_object_clones":0,"num_object_copies":209952,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16477,"num_objects_misplaced":1393,"num_objects_unfound":0,"num_objects_dirty":69984,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":136824,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5272,"num_bytes_recovered":345505792,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":14822027264,"data_stored":14822015048,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76924,"ondisk_log_size":76924,"up":48,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001571,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669507,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2888788,"kb_used_data":2272844,"kb_used_omap":7,"kb_used_meta":615864,"kb_avail":90856364,"statfs":{"total":95995035648,"available":93036916736,"internally_reserved":0,"allocated":2327392256,"data_stored":2325465870,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":630645540},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,5],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6160000000000001,"5min":1.599,"15min":1.2949999999999999},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":24.472999999999999,"5min":24.472999999999999,"15min":24.472999999999999},"last":0.60599999999999998},{"interface":"front","average":{"1min":3.6230000000000002,"5min":2.081,"15min":1.663},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":22.832999999999998,"5min":22.832999999999998,"15min":22.832999999999998},"last":3.008}]},{"osd":1,"last update":"Sat Sep 23 18:19:21 2023","interfaces":[{"interface":"back","average":{"1min":4.1740000000000004,"5min":1.8069999999999999,"15min":1.413},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":22.972000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":3.0459999999999998},{"interface":"front","average":{"1min":3.448,"5min":1.6559999999999999,"15min":1.357},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":22.664000000000001,"5min":22.664000000000001,"15min":22.664000000000001},"last":5.0279999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.52,"15min":1.3260000000000001},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":8.0559999999999992,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.861,"15min":1.492},"min":{"1min":0.34000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.1230000000000002,"5min":14.771000000000001,"15min":14.771000000000001},"last":3.0720000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":9.1259999999999994,"5min":3.1400000000000001,"15min":2.1059999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":103.348,"5min":103.348,"15min":103.348},"last":0.57099999999999995},{"interface":"front","average":{"1min":9.4610000000000003,"5min":3.6389999999999998,"15min":2.3610000000000002},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":103.40300000000001,"5min":103.40300000000001,"15min":103.40300000000001},"last":3.0990000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:19:13 2023","interfaces":[{"interface":"back","average":{"1min":2.2799999999999998,"5min":1.8140000000000001,"15min":1.3280000000000001},"min":{"1min":0.25700000000000001,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.2800000000000002,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":5.1210000000000004},{"interface":"front","average":{"1min":1.7569999999999999,"5min":1.9330000000000001,"15min":1.3200000000000001},"min":{"1min":0.36099999999999999,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":6.2030000000000003,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.63100000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":2.0609999999999999,"15min":1.9550000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":6.4710000000000001,"5min":13.502000000000001,"15min":13.502000000000001},"last":5.085},{"interface":"front","average":{"1min":1.3720000000000001,"5min":2.246,"15min":2.3660000000000001},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.1070000000000002,"5min":10.081,"15min":10.081},"last":4.9960000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":2.1459999999999999,"15min":1.5309999999999999},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.26800000000000002},"max":{"1min":6.4320000000000004,"5min":10.743,"15min":13.460000000000001},"last":5.0590000000000002},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.0710000000000002,"15min":1.5740000000000001},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.28499999999999998},"max":{"1min":6.2770000000000001,"5min":11.244,"15min":12.404},"last":3.222}]}]},{"osd":1,"up_from":545,"seq":2340757176356,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1969252,"kb_used_data":1749212,"kb_used_omap":13,"kb_used_meta":220018,"kb_avail":91775900,"statfs":{"total":95995035648,"available":93978521600,"internally_reserved":0,"allocated":1791193088,"data_stored":1788208112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":225299085},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.776,"5min":0.66800000000000004,"15min":0.48299999999999998},"min":{"1min":0.221,"5min":0.189,"15min":0.189},"max":{"1min":5.8079999999999998,"5min":5.8079999999999998,"15min":5.8079999999999998},"last":9.0299999999999994},{"interface":"front","average":{"1min":1.2430000000000001,"5min":0.64000000000000001,"15min":0.54000000000000004},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6159999999999997,"5min":4.6159999999999997,"15min":4.6159999999999997},"last":5.1189999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":0.67900000000000005,"15min":0.53300000000000003},"min":{"1min":0.28299999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9290000000000003,"5min":4.9290000000000003,"15min":4.9290000000000003},"last":9.0830000000000002},{"interface":"front","average":{"1min":1.272,"5min":0.65800000000000003,"15min":0.55600000000000005},"min":{"1min":0.29899999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2759999999999998,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":0.85899999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":0.746,"15min":0.50600000000000001},"min":{"1min":0.34300000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.3219999999999992,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":3.7000000000000002},{"interface":"front","average":{"1min":2.2429999999999999,"5min":0.85799999999999998,"15min":0.627},"min":{"1min":0.38500000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":8.3659999999999997},"last":19.762}]},{"osd":4,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":1.347,"15min":1.327},"min":{"1min":0.40200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.6609999999999996,"5min":12.663,"15min":12.663},"last":0.53100000000000003},{"interface":"front","average":{"1min":1.95,"5min":1.601,"15min":1.5429999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.1769999999999996,"5min":13.222,"15min":13.222},"last":3.5880000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":0.88200000000000001,"15min":0.77600000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":0.56799999999999995},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.0589999999999999,"15min":0.88900000000000001},"min":{"1min":0.46000000000000002,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":8.3190000000000008,"5min":8.3190000000000008,"15min":8.3190000000000008},"last":9.1210000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":0.997,"15min":0.83899999999999997},"min":{"1min":0.45400000000000001,"5min":0.435,"15min":0.435},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":0.58999999999999997},{"interface":"front","average":{"1min":2.6070000000000002,"5min":1.1419999999999999,"15min":0.89800000000000002},"min":{"1min":0.38900000000000001,"5min":0.378,"15min":0.378},"max":{"1min":9.2579999999999991,"5min":9.2579999999999991,"15min":9.2579999999999991},"last":3.637}]},{"osd":7,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":1.0109999999999999,"15min":0.79100000000000004},"min":{"1min":0.436,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":3.669},{"interface":"front","average":{"1min":2.6419999999999999,"5min":1.173,"15min":0.92800000000000005},"min":{"1min":0.48099999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":9.0640000000000001,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":9.1500000000000004}]}]},{"osd":0,"up_from":93,"seq":399431958852,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3878472,"kb_used_data":2965120,"kb_used_omap":7,"kb_used_meta":913336,"kb_avail":89866680,"statfs":{"total":95995035648,"available":92023480320,"internally_reserved":0,"allocated":3036282880,"data_stored":3033442547,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":935256804},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:19:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7229999999999999,"5min":0.80000000000000004,"15min":0.47999999999999998},"min":{"1min":0.214,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.2309999999999999,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.41099999999999998},{"interface":"front","average":{"1min":2.2010000000000001,"5min":0.85799999999999998,"15min":0.63400000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.714,"5min":13.714,"15min":13.714},"last":7.6630000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:20:04 2023","interfaces":[{"interface":"back","average":{"1min":3.911,"5min":1.71,"15min":1.3160000000000001},"min":{"1min":0.48999999999999999,"5min":0.185,"15min":0.185},"max":{"1min":15.002000000000001,"5min":18.550999999999998,"15min":18.550999999999998},"last":2.0760000000000001},{"interface":"front","average":{"1min":2.8530000000000002,"5min":1.258,"15min":0.93400000000000005},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":11.644,"5min":13.618,"15min":13.618},"last":2.4609999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7610000000000001,"5min":1.393,"15min":1.637},"min":{"1min":0.38400000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.539999999999999,"5min":13.539999999999999,"15min":13.539999999999999},"last":0.77800000000000002},{"interface":"front","average":{"1min":4.1390000000000002,"5min":1.4930000000000001,"15min":1.637},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.285,"5min":13.285,"15min":15.198},"last":5.3280000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.1469999999999998,"5min":2.2599999999999998,"15min":1.7789999999999999},"min":{"1min":0.32300000000000001,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":13.574999999999999,"5min":57.746000000000002,"15min":57.746000000000002},"last":17.741},{"interface":"front","average":{"1min":2.032,"5min":1.9770000000000001,"15min":1.6499999999999999},"min":{"1min":0.33900000000000002,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":5.8970000000000002,"5min":57.816000000000003,"15min":57.816000000000003},"last":2.117}]},{"osd":5,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.032,"15min":1.391},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":1.7370000000000001},{"interface":"front","average":{"1min":4.2859999999999996,"5min":2.2650000000000001,"15min":1.448},"min":{"1min":0.502,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":13.425000000000001},"last":2.1480000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:19:10 2023","interfaces":[{"interface":"back","average":{"1min":2.2450000000000001,"5min":1.8049999999999999,"15min":1.861},"min":{"1min":0.433,"5min":0.374,"15min":0.32000000000000001},"max":{"1min":7.1020000000000003,"5min":7.1020000000000003,"15min":58.457999999999998},"last":0.80900000000000005},{"interface":"front","average":{"1min":2.7210000000000001,"5min":1.798,"15min":1.7050000000000001},"min":{"1min":0.42099999999999999,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":13.672000000000001,"5min":13.672000000000001,"15min":58.354999999999997},"last":18.643999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":1.923,"15min":1.639},"min":{"1min":0.46899999999999997,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":13.644,"5min":13.644,"15min":13.644},"last":2.1960000000000002},{"interface":"front","average":{"1min":3.5179999999999998,"5min":1.821,"15min":1.8},"min":{"1min":0.55900000000000005,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.7519999999999998,"5min":10.404,"15min":10.404},"last":2.5329999999999999}]}]},{"osd":2,"up_from":0,"seq":2070174236738,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249367,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2283228,"kb_used_data":1566356,"kb_used_omap":3,"kb_used_meta":716796,"kb_avail":91461924,"statfs":{"total":95995035648,"available":93657010176,"internally_reserved":0,"allocated":1603948544,"data_stored":1600913513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":733999848},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3919999999999999,"5min":1.25,"15min":1.7050000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.183},"max":{"1min":17.036000000000001,"5min":17.036000000000001,"15min":195.69800000000001},"last":2.1480000000000001},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.1599999999999999,"15min":1.96},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.0980000000000008,"5min":9.0980000000000008,"15min":194.661},"last":0.23799999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":0.95799999999999996,"15min":0.66600000000000004},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":21.777000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.307},{"interface":"front","average":{"1min":6.5110000000000001,"5min":1.8660000000000001,"15min":1.0920000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":78.980000000000004,"5min":78.980000000000004,"15min":78.980000000000004},"last":5.5819999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":1.274,"15min":1.29},"min":{"1min":0.27400000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.151999999999999,"5min":12.151999999999999,"15min":12.151999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.298,"15min":1.3320000000000001},"min":{"1min":0.318,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.565,"5min":11.565,"15min":11.565},"last":6.9320000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.6140000000000001,"15min":1.659},"min":{"1min":0.31900000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2300000000000004,"5min":21.062999999999999,"15min":37.231000000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.9810000000000001,"15min":1.8939999999999999},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.2959999999999994,"5min":21.128,"15min":37.381},"last":0.37}]},{"osd":5,"last update":"Sat Sep 23 18:19:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.8009999999999999,"15min":1.4419999999999999},"min":{"1min":0.40999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":11.805999999999999,"5min":11.805999999999999,"15min":11.805999999999999},"last":0.35099999999999998},{"interface":"front","average":{"1min":2.028,"5min":1.601,"15min":1.6339999999999999},"min":{"1min":0.44500000000000001,"5min":0.33100000000000002,"15min":0.32000000000000001},"max":{"1min":8.9359999999999999,"5min":8.9359999999999999,"15min":8.9969999999999999},"last":2.206}]},{"osd":6,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":1.7809999999999999,"15min":1.847},"min":{"1min":0.432,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":17.192,"5min":17.192,"15min":115.54000000000001},"last":2.2400000000000002},{"interface":"front","average":{"1min":2.2530000000000001,"5min":1.7250000000000001,"15min":1.948},"min":{"1min":0.37,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":8.0350000000000001,"5min":14.321,"15min":115.691},"last":0.40400000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.5069999999999999,"15min":1.7110000000000001},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":8.2070000000000007,"5min":8.2070000000000007,"15min":15.972},"last":0.53700000000000003},{"interface":"front","average":{"1min":3.5920000000000001,"5min":1.7629999999999999,"15min":1.623},"min":{"1min":0.48799999999999999,"5min":0.41999999999999998,"15min":0.34899999999999998},"max":{"1min":17.164000000000001,"5min":17.164000000000001,"15min":17.164000000000001},"last":2.258}]}]},{"osd":4,"up_from":30,"seq":128849019317,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":4257364,"kb_used_data":3613388,"kb_used_omap":0,"kb_used_meta":643968,"kb_avail":89487788,"statfs":{"total":95995035648,"available":91635494912,"internally_reserved":0,"allocated":3700109312,"data_stored":3698182718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":659423232},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,1,1,4],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.534,"15min":1.268},"min":{"1min":0.255,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.6079999999999997,"5min":7.6079999999999997,"15min":13.375999999999999},"last":3.855},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.6819999999999999,"15min":1.546},"min":{"1min":0.26600000000000001,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":9.3710000000000004,"5min":10.890000000000001,"15min":10.890000000000001},"last":3.0920000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:19:22 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":1.47,"15min":0.95299999999999996},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":44.146999999999998,"5min":44.146999999999998,"15min":44.146999999999998},"last":1.716},{"interface":"front","average":{"1min":4.5419999999999998,"5min":1.4299999999999999,"15min":0.91100000000000003},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":43.497999999999998,"5min":43.497999999999998,"15min":43.497999999999998},"last":3.738}]},{"osd":2,"last update":"Sat Sep 23 18:19:56 2023","interfaces":[{"interface":"back","average":{"1min":2.621,"5min":1.6699999999999999,"15min":1.8180000000000001},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.896000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":3.6560000000000001},{"interface":"front","average":{"1min":1.6020000000000001,"5min":1.6200000000000001,"15min":1.6930000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":6.6890000000000001,"5min":10.938000000000001,"15min":10.938000000000001},"last":4.1440000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.5669999999999999,"15min":1.9350000000000001},"min":{"1min":0.33700000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.0609999999999999,"5min":11.130000000000001,"15min":29.154},"last":3.5299999999999998},{"interface":"front","average":{"1min":2.1800000000000002,"5min":1.8480000000000001,"15min":1.9390000000000001},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":13.470000000000001,"5min":13.470000000000001,"15min":29.216000000000001},"last":3.2090000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":2.4279999999999999,"15min":2.8559999999999999},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4950000000000001,"5min":27.675000000000001,"15min":27.675000000000001},"last":5.508},{"interface":"front","average":{"1min":1.401,"5min":1.827,"15min":2.7109999999999999},"min":{"1min":0.30499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.1040000000000001,"5min":13.938000000000001,"15min":32.640000000000001},"last":1.605}]},{"osd":6,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.873,"15min":1.347},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":6.1989999999999998,"5min":18.315000000000001,"15min":18.315000000000001},"last":1.6319999999999999},{"interface":"front","average":{"1min":1.859,"5min":1.9630000000000001,"15min":1.522},"min":{"1min":0.38700000000000001,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":9.5579999999999998,"5min":18.431000000000001,"15min":18.431000000000001},"last":5.6440000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":2.4849999999999999,"15min":1.774},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":5.9539999999999997,"5min":19.298999999999999,"15min":19.298999999999999},"last":3.1299999999999999},{"interface":"front","average":{"1min":1.722,"5min":1.8899999999999999,"15min":1.552},"min":{"1min":0.435,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.4730000000000008,"5min":12.800000000000001,"15min":12.800000000000001},"last":3.2370000000000001}]}]},{"osd":5,"up_from":420,"seq":1803886264413,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2802948,"kb_used_data":2470332,"kb_used_omap":9,"kb_used_meta":332406,"kb_avail":90942204,"statfs":{"total":95995035648,"available":93124816896,"internally_reserved":0,"allocated":2529619968,"data_stored":2527705718,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":340384189},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":1.2549999999999999,"15min":0.88100000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.6000000000000001,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.21299999999999999},{"interface":"front","average":{"1min":2.911,"5min":1.8149999999999999,"15min":1.6080000000000001},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":20.937999999999999,"5min":20.937999999999999,"15min":20.937999999999999},"last":1.012}]},{"osd":1,"last update":"Sat Sep 23 18:19:20 2023","interfaces":[{"interface":"back","average":{"1min":3.004,"5min":1.4770000000000001,"15min":1.2230000000000001},"min":{"1min":0.32100000000000001,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":23.946000000000002,"5min":23.946000000000002,"15min":23.946000000000002},"last":8.6649999999999991},{"interface":"front","average":{"1min":3.927,"5min":1.4630000000000001,"15min":1.052},"min":{"1min":0.26200000000000001,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":23.268999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":0.44800000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:19:52 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":1.629,"15min":1.615},"min":{"1min":0.28699999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":10.705,"5min":10.705,"15min":10.705},"last":0.29699999999999999},{"interface":"front","average":{"1min":2.5409999999999999,"5min":1.946,"15min":2.1600000000000001},"min":{"1min":0.373,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.8099999999999996,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.504}]},{"osd":3,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":1.8740000000000001,"15min":1.2789999999999999},"min":{"1min":0.443,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.8780000000000001,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":0.56699999999999995},{"interface":"front","average":{"1min":3.7320000000000002,"5min":2.0640000000000001,"15min":1.665},"min":{"1min":0.38100000000000001,"5min":0.315,"15min":0.313},"max":{"1min":20.035,"5min":20.035,"15min":20.035},"last":0.55200000000000005}]},{"osd":4,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.2770000000000001,"15min":2.0640000000000001},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":7.875,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.0899999999999999,"15min":1.6160000000000001},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7610000000000001,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":1.9650000000000001,"15min":1.2609999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2480000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.627},{"interface":"front","average":{"1min":2.343,"5min":2.2040000000000002,"15min":1.6819999999999999},"min":{"1min":0.34599999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.8410000000000002,"5min":12.504,"15min":12.504},"last":1.675}]},{"osd":7,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.3980000000000001,"15min":1.758},"min":{"1min":0.40699999999999997,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.528,"5min":13.506,"15min":13.506},"last":0.46300000000000002},{"interface":"front","average":{"1min":1.381,"5min":1.9630000000000001,"15min":1.542},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":5.0970000000000004,"5min":13.138,"15min":13.138},"last":0.51900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2996703232,"data_stored":2996699160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1770782720,"data_stored":1770782720,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1580597248,"data_stored":1580597248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3668709376,"data_stored":3668705304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2511273984,"data_stored":2511273984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2293960704,"data_stored":2293956632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:20:17.194 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:20:17.354 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:17.361 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:17.361 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:17.462 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:17.523 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:17.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:17.634 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:17.635 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:17.735 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:17.801 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:17 smithi099 ceph-mon[167076]: from='client.37263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:17.801 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:17 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:20:17.801 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:17 smithi099 ceph-mon[167076]: failed to encode map e611 with expected crc 2023-09-23T18:20:17.801 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3761510810' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:20:17.801 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:17 smithi099 ceph-mon[167076]: osdmap e611: 8 total, 8 up, 6 in 2023-09-23T18:20:17.802 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:17 smithi099 ceph-mon[167076]: failed to encode map e611 with expected crc 2023-09-23T18:20:17.806 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:17.918 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:17.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[184174]: from='client.37263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:17.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:20:17.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[184174]: failed to encode map e611 with expected crc 2023-09-23T18:20:17.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3761510810' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:20:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[184174]: osdmap e611: 8 total, 8 up, 6 in 2023-09-23T18:20:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[184174]: failed to encode map e611 with expected crc 2023-09-23T18:20:17.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[179668]: from='client.37263 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:17.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:20:17.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[179668]: failed to encode map e611 with expected crc 2023-09-23T18:20:17.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3761510810' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:20:17.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[179668]: osdmap e611: 8 total, 8 up, 6 in 2023-09-23T18:20:17.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:17 smithi055 ceph-mon[179668]: failed to encode map e611 with expected crc 2023-09-23T18:20:17.995 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:20:18.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:18.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:18.650 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:18.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[184174]: failed to encode map e611 with expected crc 2023-09-23T18:20:18.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[184174]: failed to encode map e611 with expected crc 2023-09-23T18:20:18.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[184174]: pgmap v1460: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 921 op/s; 16502/199098 objects degraded (8.288%); 1116/199098 objects misplaced (0.561%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T18:20:18.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:18.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:20:18.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:18.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16502/199098 objects degraded (8.288%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:18.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[179668]: failed to encode map e611 with expected crc 2023-09-23T18:20:18.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[179668]: failed to encode map e611 with expected crc 2023-09-23T18:20:18.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[179668]: pgmap v1460: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 921 op/s; 16502/199098 objects degraded (8.288%); 1116/199098 objects misplaced (0.561%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T18:20:18.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:18.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:20:18.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:18.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16502/199098 objects degraded (8.288%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:18.750 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:18.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:18 smithi099 ceph-mon[167076]: failed to encode map e611 with expected crc 2023-09-23T18:20:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:18 smithi099 ceph-mon[167076]: failed to encode map e611 with expected crc 2023-09-23T18:20:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:18 smithi099 ceph-mon[167076]: pgmap v1460: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.1 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 921 op/s; 16502/199098 objects degraded (8.288%); 1116/199098 objects misplaced (0.561%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T18:20:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:18 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:20:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:18 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:20:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:18 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:18.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16502/199098 objects degraded (8.288%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:18.834 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:18.918 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:18.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:18.924 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:18.956 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:19.026 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:19.083 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:19.196 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:19.212 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:19.213 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:19.314 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:19.375 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:19.385 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:20:19.461 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:19.468 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:19.468 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:19.569 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:19.598 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:19.676 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:19.681 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:19.681 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:19.782 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:19.840 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:19.966 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:19.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:19.973 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:20.073 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:20:20.073 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 16502/197589 objects degraded (8.352%), 7 pgs degraded, 6 pgs undersized","count":13},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1709,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":611,"num_osds":8,"num_up_osds":8,"osd_up_since":1695493039,"num_in_osds":6,"osd_in_since":1695493021,"num_remapped_pgs":6},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":112},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":5},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"active+recovery_wait+degraded","count":1}],"num_pgs":119,"num_pools":5,"num_objects":65863,"data_bytes":4319034044,"bytes_used":17743876096,"bytes_avail":558226337792,"bytes_total":575970213888,"degraded_objects":16502,"degraded_total":197589,"degraded_ratio":0.083516794963282376,"misplaced_objects":1116,"misplaced_total":197589,"misplaced_ratio":0.0056480876971896207,"recovering_objects_per_sec":52,"recovering_bytes_per_sec":3458491,"recovering_keys_per_sec":0,"num_objects_recovered":528,"num_bytes_recovered":34603008,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":823},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T18:20:09.153115+0000","services":{}},"progress_events":{"57c27436-1bc8-4f63-b41f-a80b88a05748":{"message":"Global Recovery Event (80s)\n [==========================..] (remaining: 5s)","progress":0.94117647409439087,"add_to_ceph_s":true},"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T18:20:20.073 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:20.078 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:20:20.078 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:20:20.122 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:20.431 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:20.435 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:20.435 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:20.512 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:20:20.536 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:20.560 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:20 smithi055 ceph-mon[179668]: pgmap v1461: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 823 op/s; 16502/197589 objects degraded (8.352%); 1116/197589 objects misplaced (0.565%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:20:20.560 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:20 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3493055425' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:20:20.561 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:20:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:20:20] "GET /metrics HTTP/1.1" 200 36687 "" "Prometheus/2.43.0" 2023-09-23T18:20:20.634 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:20.714 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:20.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:20.726 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:20.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:20 smithi099 ceph-mon[167076]: pgmap v1461: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 823 op/s; 16502/197589 objects degraded (8.352%); 1116/197589 objects misplaced (0.565%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:20:20.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:20 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3493055425' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:20:20.827 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:20.907 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:20.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:20 smithi055 ceph-mon[184174]: pgmap v1461: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 4.0 GiB data, 17 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 823 op/s; 16502/197589 objects degraded (8.352%); 1116/197589 objects misplaced (0.565%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:20:20.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:20 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3493055425' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:20:20.991 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:21.006 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:21.006 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:21.107 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:21.153 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:21.293 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:21.987 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:20:22.001 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:22.003 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:20:22.008 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:22.008 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:22.035 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:20:22.069 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:20:22.108 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:20:22.109 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:22.127 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:20:22.127 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:20:22.144 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:20:22.178 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:20:22.180 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:22.200 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:22.212 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:20:22.223 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:20:22.223 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:20:22.224 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:20:22.224 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:20:22.224 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:20:22.224 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:20:22.224 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:20:22.224 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:20:22.332 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:22.341 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:22.341 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:22.442 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:22.508 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:22.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:22.628 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:22.629 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:22.723 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:22 smithi055 ceph-mon[184174]: pgmap v1462: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 831 op/s; 16502/192627 objects degraded (8.567%); 1116/192627 objects misplaced (0.579%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:20:22.723 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:22.723 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:22.723 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2976489990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:22.725 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:22 smithi055 ceph-mon[179668]: pgmap v1462: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 831 op/s; 16502/192627 objects degraded (8.567%); 1116/192627 objects misplaced (0.579%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:20:22.725 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:22.726 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:22.726 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2976489990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:22.729 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:22.773 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:20:22.773 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:20:22.773 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:20:22.773 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:20:22.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:22 smithi099 ceph-mon[167076]: pgmap v1462: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.9 GiB data, 16 GiB used, 520 GiB / 536 GiB avail; 0 B/s wr, 831 op/s; 16502/192627 objects degraded (8.567%); 1116/192627 objects misplaced (0.579%); 3.3 MiB/s, 52 objects/s recovering 2023-09-23T18:20:22.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:22.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:22.780 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2976489990' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:22.783 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:22.854 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:22.858 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:22.859 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:22.959 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:23.030 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:23.148 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:23.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:23.159 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:23.191 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:20:23.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:23.326 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:23.371 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:23.458 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:23 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16502/192627 objects degraded (8.567%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:23.471 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:23.482 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:23.483 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:23.583 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:23.635 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:23.676 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:23 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16502/192627 objects degraded (8.567%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:23.676 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:23 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16502/192627 objects degraded (8.567%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:23.693 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:23.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:23.698 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:23.798 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:23.886 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:24.033 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:24.186 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:24.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:24.191 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:24.291 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:24.369 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:24.493 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:20:24.497 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:24.513 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:24.513 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:24.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:24 smithi099 ceph-mon[167076]: pgmap v1463: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 16502/185253 objects degraded (8.908%); 792/185253 objects misplaced (0.428%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T18:20:24.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1420392006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:20:24.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:24 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:20:24.614 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:24.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:24 smithi055 ceph-mon[179668]: pgmap v1463: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 16502/185253 objects degraded (8.908%); 792/185253 objects misplaced (0.428%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T18:20:24.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1420392006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:20:24.642 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:24 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:20:24.643 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:24 smithi055 ceph-mon[184174]: pgmap v1463: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 852 op/s; 16502/185253 objects degraded (8.908%); 792/185253 objects misplaced (0.428%); 3.8 MiB/s, 60 objects/s recovering 2023-09-23T18:20:24.643 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1420392006' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:20:24.643 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:24 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:20:24.681 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:24.784 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:20:24.876 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:24.885 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:24.885 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:24.986 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:25.067 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:25.156 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:25.163 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:25.163 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:25.264 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:25.330 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:25.412 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:25.415 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:25.416 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:25.484 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:20:25.513 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:20:25.516 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:25.545 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:25.583 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:25 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:25.583 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:25 smithi099 ceph-mon[167076]: failed to encode map e612 with expected crc 2023-09-23T18:20:25.583 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:25 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:20:25.583 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:25 smithi099 ceph-mon[167076]: osdmap e612: 8 total, 8 up, 6 in 2023-09-23T18:20:25.583 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:20:25.584 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:20:25.584 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:20:25.584 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:20:25.584 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:20:25.586 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:25.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:25 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:25.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:25 smithi055 ceph-mon[179668]: failed to encode map e612 with expected crc 2023-09-23T18:20:25.662 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:25 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:20:25.663 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:25 smithi055 ceph-mon[179668]: osdmap e612: 8 total, 8 up, 6 in 2023-09-23T18:20:25.664 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:25 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:25.664 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:25 smithi055 ceph-mon[184174]: failed to encode map e612 with expected crc 2023-09-23T18:20:25.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:25 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:20:25.665 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:25 smithi055 ceph-mon[184174]: osdmap e612: 8 total, 8 up, 6 in 2023-09-23T18:20:25.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:25.705 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:25.705 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:25.806 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:25.849 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:25.945 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:25.962 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:25.963 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:25.993 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1465,"stamp":"2023-09-23T18:20:25.160467+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4032838332,"num_objects":61496,"num_object_clones":0,"num_object_copies":184488,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":16502,"num_objects_misplaced":792,"num_objects_unfound":0,"num_objects_dirty":61496,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":146250,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6259,"num_bytes_recovered":400121481,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83261,"ondisk_log_size":83261,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":16611268,"kb_used_data":13146836,"kb_used_omap":40,"kb_used_meta":3463959,"kb_avail":545859644,"statfs":{"total":575970213888,"available":558960275456,"internally_reserved":0,"allocated":13462360064,"data_stored":13447701124,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3547094330},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-319160320,"num_objects":-4870,"num_object_clones":0,"num_object_copies":-14610,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-324,"num_objects_unfound":0,"num_objects_dirty":-4870,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4870,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":324,"num_bytes_recovered":21233664,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.225896"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":899,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.488575+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:20:08.488575+0000","last_peered":"2023-09-23T18:20:08.488575+0000","last_clean":"2023-09-23T18:20:08.488575+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:20:08.488575+0000","last_undegraded":"2023-09-23T18:20:08.488575+0000","last_fullsized":"2023-09-23T18:20:08.488575+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":270,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.484760+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:20:08.484760+0000","last_peered":"2023-09-23T18:20:08.484760+0000","last_clean":"2023-09-23T18:20:08.484760+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:20:08.484760+0000","last_undegraded":"2023-09-23T18:20:08.484760+0000","last_fullsized":"2023-09-23T18:20:08.484760+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":27,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.489564+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:20:08.489564+0000","last_peered":"2023-09-23T18:20:08.489564+0000","last_clean":"2023-09-23T18:20:08.489564+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:20:08.489564+0000","last_undegraded":"2023-09-23T18:20:08.489564+0000","last_fullsized":"2023-09-23T18:20:08.489564+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":905,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.485597+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:20:08.485597+0000","last_peered":"2023-09-23T18:20:08.485597+0000","last_clean":"2023-09-23T18:20:08.485597+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:20:08.485597+0000","last_undegraded":"2023-09-23T18:20:08.485597+0000","last_fullsized":"2023-09-23T18:20:08.485597+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":351,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.491984+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:20:08.491984+0000","last_peered":"2023-09-23T18:20:08.491984+0000","last_clean":"2023-09-23T18:20:08.491984+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:20:08.491984+0000","last_undegraded":"2023-09-23T18:20:08.491984+0000","last_fullsized":"2023-09-23T18:20:08.491984+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":671,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.490915+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:20:08.490915+0000","last_peered":"2023-09-23T18:20:08.490915+0000","last_clean":"2023-09-23T18:20:08.490915+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:20:08.490915+0000","last_undegraded":"2023-09-23T18:20:08.490915+0000","last_fullsized":"2023-09-23T18:20:08.490915+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":922,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478237+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:20:08.478237+0000","last_peered":"2023-09-23T18:20:08.478237+0000","last_clean":"2023-09-23T18:20:08.478237+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:08.478237+0000","last_undegraded":"2023-09-23T18:20:08.478237+0000","last_fullsized":"2023-09-23T18:20:08.478237+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":27,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.486959+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:08.486959+0000","last_peered":"2023-09-23T18:20:08.486959+0000","last_clean":"2023-09-23T18:20:08.486959+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:20:08.486959+0000","last_undegraded":"2023-09-23T18:20:08.486959+0000","last_fullsized":"2023-09-23T18:20:08.486959+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":187,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.504595+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:20:08.504595+0000","last_peered":"2023-09-23T18:20:08.504595+0000","last_clean":"2023-09-23T18:20:08.504595+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:20:08.504595+0000","last_undegraded":"2023-09-23T18:20:08.504595+0000","last_fullsized":"2023-09-23T18:20:08.504595+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1045,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478815+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:20:08.478815+0000","last_peered":"2023-09-23T18:20:08.478815+0000","last_clean":"2023-09-23T18:20:08.478815+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:20:08.478815+0000","last_undegraded":"2023-09-23T18:20:08.478815+0000","last_fullsized":"2023-09-23T18:20:08.478815+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":121,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477925+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:20:08.477925+0000","last_peered":"2023-09-23T18:20:08.477925+0000","last_clean":"2023-09-23T18:20:08.477925+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:20:08.477925+0000","last_undegraded":"2023-09-23T18:20:08.477925+0000","last_fullsized":"2023-09-23T18:20:08.477925+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":200,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.487941+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:20:08.487941+0000","last_peered":"2023-09-23T18:20:08.487941+0000","last_clean":"2023-09-23T18:20:08.487941+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:20:08.487941+0000","last_undegraded":"2023-09-23T18:20:08.487941+0000","last_fullsized":"2023-09-23T18:20:08.487941+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":822,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.498731+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:20:08.498731+0000","last_peered":"2023-09-23T18:20:08.498731+0000","last_clean":"2023-09-23T18:20:08.498731+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:20:08.498731+0000","last_undegraded":"2023-09-23T18:20:08.498731+0000","last_fullsized":"2023-09-23T18:20:08.498731+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":157,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477601+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:20:08.477601+0000","last_peered":"2023-09-23T18:20:08.477601+0000","last_clean":"2023-09-23T18:20:08.477601+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:20:08.477601+0000","last_undegraded":"2023-09-23T18:20:08.477601+0000","last_fullsized":"2023-09-23T18:20:08.477601+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":156,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.498874+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:20:08.498874+0000","last_peered":"2023-09-23T18:20:08.498874+0000","last_clean":"2023-09-23T18:20:08.498874+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:20:08.498874+0000","last_undegraded":"2023-09-23T18:20:08.498874+0000","last_fullsized":"2023-09-23T18:20:08.498874+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":956,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477997+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:20:08.477997+0000","last_peered":"2023-09-23T18:20:08.477997+0000","last_clean":"2023-09-23T18:20:08.477997+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:20:08.477997+0000","last_undegraded":"2023-09-23T18:20:08.477997+0000","last_fullsized":"2023-09-23T18:20:08.477997+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":405,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.489997+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:20:08.489997+0000","last_peered":"2023-09-23T18:20:08.489997+0000","last_clean":"2023-09-23T18:20:08.489997+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:20:08.489997+0000","last_undegraded":"2023-09-23T18:20:08.489997+0000","last_fullsized":"2023-09-23T18:20:08.489997+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":424,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.503051+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:20:08.503051+0000","last_peered":"2023-09-23T18:20:08.503051+0000","last_clean":"2023-09-23T18:20:08.503051+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:20:08.503051+0000","last_undegraded":"2023-09-23T18:20:08.503051+0000","last_fullsized":"2023-09-23T18:20:08.503051+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":985,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.494756+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:20:08.494756+0000","last_peered":"2023-09-23T18:20:08.494756+0000","last_clean":"2023-09-23T18:20:08.494756+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:20:08.494756+0000","last_undegraded":"2023-09-23T18:20:08.494756+0000","last_fullsized":"2023-09-23T18:20:08.494756+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":163,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.499102+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:20:08.499102+0000","last_peered":"2023-09-23T18:20:08.499102+0000","last_clean":"2023-09-23T18:20:08.499102+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:20:08.499102+0000","last_undegraded":"2023-09-23T18:20:08.499102+0000","last_fullsized":"2023-09-23T18:20:08.499102+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":430,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.501970+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:20:08.501970+0000","last_peered":"2023-09-23T18:20:08.501970+0000","last_clean":"2023-09-23T18:20:08.501970+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:20:08.501970+0000","last_undegraded":"2023-09-23T18:20:08.501970+0000","last_fullsized":"2023-09-23T18:20:08.501970+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":837,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.485023+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:20:08.485023+0000","last_peered":"2023-09-23T18:20:08.485023+0000","last_clean":"2023-09-23T18:20:08.485023+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:20:08.485023+0000","last_undegraded":"2023-09-23T18:20:08.485023+0000","last_fullsized":"2023-09-23T18:20:08.485023+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":430,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.501796+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:20:08.501796+0000","last_peered":"2023-09-23T18:20:08.501796+0000","last_clean":"2023-09-23T18:20:08.501796+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:20:08.501796+0000","last_undegraded":"2023-09-23T18:20:08.501796+0000","last_fullsized":"2023-09-23T18:20:08.501796+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":108,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477124+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:20:08.477124+0000","last_peered":"2023-09-23T18:20:08.477124+0000","last_clean":"2023-09-23T18:20:08.477124+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:20:08.477124+0000","last_undegraded":"2023-09-23T18:20:08.477124+0000","last_fullsized":"2023-09-23T18:20:08.477124+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":808,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.485053+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:20:08.485053+0000","last_peered":"2023-09-23T18:20:08.485053+0000","last_clean":"2023-09-23T18:20:08.485053+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:20:08.485053+0000","last_undegraded":"2023-09-23T18:20:08.485053+0000","last_fullsized":"2023-09-23T18:20:08.485053+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":126,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.490387+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:20:08.490387+0000","last_peered":"2023-09-23T18:20:08.490387+0000","last_clean":"2023-09-23T18:20:08.490387+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:20:08.490387+0000","last_undegraded":"2023-09-23T18:20:08.490387+0000","last_fullsized":"2023-09-23T18:20:08.490387+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":121,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.484723+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:20:08.484723+0000","last_peered":"2023-09-23T18:20:08.484723+0000","last_clean":"2023-09-23T18:20:08.484723+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:20:08.484723+0000","last_undegraded":"2023-09-23T18:20:08.484723+0000","last_fullsized":"2023-09-23T18:20:08.484723+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":746,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.488102+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:20:08.488102+0000","last_peered":"2023-09-23T18:20:08.488102+0000","last_clean":"2023-09-23T18:20:08.488102+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:20:08.488102+0000","last_undegraded":"2023-09-23T18:20:08.488102+0000","last_fullsized":"2023-09-23T18:20:08.488102+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":198,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.498732+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:20:08.498732+0000","last_peered":"2023-09-23T18:20:08.498732+0000","last_clean":"2023-09-23T18:20:08.498732+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:20:08.498732+0000","last_undegraded":"2023-09-23T18:20:08.498732+0000","last_fullsized":"2023-09-23T18:20:08.498732+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":209,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.489808+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:20:08.489808+0000","last_peered":"2023-09-23T18:20:08.489808+0000","last_clean":"2023-09-23T18:20:08.489808+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:20:08.489808+0000","last_undegraded":"2023-09-23T18:20:08.489808+0000","last_fullsized":"2023-09-23T18:20:08.489808+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":842,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.499034+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:20:08.499034+0000","last_peered":"2023-09-23T18:20:08.499034+0000","last_clean":"2023-09-23T18:20:08.499034+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:20:08.499034+0000","last_undegraded":"2023-09-23T18:20:08.499034+0000","last_fullsized":"2023-09-23T18:20:08.499034+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":751,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.499368+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:20:08.499368+0000","last_peered":"2023-09-23T18:20:08.499368+0000","last_clean":"2023-09-23T18:20:08.499368+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:20:08.499368+0000","last_undegraded":"2023-09-23T18:20:08.499368+0000","last_fullsized":"2023-09-23T18:20:08.499368+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":27,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.490587+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:20:08.490587+0000","last_peered":"2023-09-23T18:20:08.490587+0000","last_clean":"2023-09-23T18:20:08.490587+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:20:08.490587+0000","last_undegraded":"2023-09-23T18:20:08.490587+0000","last_fullsized":"2023-09-23T18:20:08.490587+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"611'9226","reported_seq":9271,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:21.144267+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:20:21.144267+0000","last_peered":"2023-09-23T18:20:21.144267+0000","last_clean":"2023-09-23T18:20:21.144267+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:20:21.144267+0000","last_undegraded":"2023-09-23T18:20:21.144267+0000","last_fullsized":"2023-09-23T18:20:21.144267+0000","mapping_epoch":574,"log_start":"592'3700","ondisk_log_start":"592'3700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5526,"log_dups_size":0,"ondisk_log_size":5526,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":252182528,"num_objects":3848,"num_object_clones":0,"num_object_copies":11544,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3848,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9226,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":876,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.502125+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:20:08.502125+0000","last_peered":"2023-09-23T18:20:08.502125+0000","last_clean":"2023-09-23T18:20:08.502125+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:20:08.502125+0000","last_undegraded":"2023-09-23T18:20:08.502125+0000","last_fullsized":"2023-09-23T18:20:08.502125+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":247,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.491573+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:20:08.491573+0000","last_peered":"2023-09-23T18:20:08.491573+0000","last_clean":"2023-09-23T18:20:08.491573+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:20:08.491573+0000","last_undegraded":"2023-09-23T18:20:08.491573+0000","last_fullsized":"2023-09-23T18:20:08.491573+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":121,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.485135+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:20:08.485135+0000","last_peered":"2023-09-23T18:20:08.485135+0000","last_clean":"2023-09-23T18:20:08.485135+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:20:08.485135+0000","last_undegraded":"2023-09-23T18:20:08.485135+0000","last_fullsized":"2023-09-23T18:20:08.485135+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"611'8915","reported_seq":8970,"reported_epoch":611,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:21.281615+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:20:21.281615+0000","last_peered":"2023-09-23T18:20:21.281615+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:20:21.281615+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"600'4500","ondisk_log_start":"600'4500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4415,"log_dups_size":0,"ondisk_log_size":4415,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":247529472,"num_objects":3777,"num_object_clones":0,"num_object_copies":11331,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3777,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8915,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":1579},{"shards":"5,6","objects":-371}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":730,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.487824+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:20:08.487824+0000","last_peered":"2023-09-23T18:20:08.487824+0000","last_clean":"2023-09-23T18:20:08.487824+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:20:08.487824+0000","last_undegraded":"2023-09-23T18:20:08.487824+0000","last_fullsized":"2023-09-23T18:20:08.487824+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":248,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.492214+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:20:08.492214+0000","last_peered":"2023-09-23T18:20:08.492214+0000","last_clean":"2023-09-23T18:20:08.492214+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:20:08.492214+0000","last_undegraded":"2023-09-23T18:20:08.492214+0000","last_fullsized":"2023-09-23T18:20:08.492214+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":227,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.487396+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:20:08.487396+0000","last_peered":"2023-09-23T18:20:08.487396+0000","last_clean":"2023-09-23T18:20:08.487396+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:20:08.487396+0000","last_undegraded":"2023-09-23T18:20:08.487396+0000","last_fullsized":"2023-09-23T18:20:08.487396+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"611'8963","reported_seq":15968,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:20.218575+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:20:20.218575+0000","last_peered":"2023-09-23T18:20:20.218575+0000","last_clean":"2023-09-23T18:20:20.218575+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:20:20.218575+0000","last_undegraded":"2023-09-23T18:20:20.218575+0000","last_fullsized":"2023-09-23T18:20:20.218575+0000","mapping_epoch":605,"log_start":"593'4100","ondisk_log_start":"593'4100","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4863,"log_dups_size":0,"ondisk_log_size":4863,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254345216,"num_objects":3881,"num_object_clones":0,"num_object_copies":11643,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3881,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8963,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":626,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.504512+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:20:08.504512+0000","last_peered":"2023-09-23T18:20:08.504512+0000","last_clean":"2023-09-23T18:20:08.504512+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:20:08.504512+0000","last_undegraded":"2023-09-23T18:20:08.504512+0000","last_fullsized":"2023-09-23T18:20:08.504512+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":536,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478892+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:20:08.478892+0000","last_peered":"2023-09-23T18:20:08.478892+0000","last_clean":"2023-09-23T18:20:08.478892+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:20:08.478892+0000","last_undegraded":"2023-09-23T18:20:08.478892+0000","last_fullsized":"2023-09-23T18:20:08.478892+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":187,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.502986+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:20:08.502986+0000","last_peered":"2023-09-23T18:20:08.502986+0000","last_clean":"2023-09-23T18:20:08.502986+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:20:08.502986+0000","last_undegraded":"2023-09-23T18:20:08.502986+0000","last_fullsized":"2023-09-23T18:20:08.502986+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"611'9164","reported_seq":9211,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:21.259229+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:20:21.259229+0000","last_peered":"2023-09-23T18:20:21.259229+0000","last_clean":"2023-09-23T18:20:21.259229+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:20:21.259229+0000","last_undegraded":"2023-09-23T18:20:21.259229+0000","last_fullsized":"2023-09-23T18:20:21.259229+0000","mapping_epoch":574,"log_start":"593'4000","ondisk_log_start":"593'4000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5164,"log_dups_size":0,"ondisk_log_size":5164,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257425408,"num_objects":3928,"num_object_clones":0,"num_object_copies":11784,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3928,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9164,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":642,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:11.242043+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:20:11.242043+0000","last_peered":"2023-09-23T18:20:11.242043+0000","last_clean":"2023-09-23T18:20:11.242043+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:20:11.242043+0000","last_undegraded":"2023-09-23T18:20:11.242043+0000","last_fullsized":"2023-09-23T18:20:11.242043+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":135,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477476+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:20:08.477476+0000","last_peered":"2023-09-23T18:20:08.477476+0000","last_clean":"2023-09-23T18:20:08.477476+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:20:08.477476+0000","last_undegraded":"2023-09-23T18:20:08.477476+0000","last_fullsized":"2023-09-23T18:20:08.477476+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":448,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:10.239508+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:20:10.239508+0000","last_peered":"2023-09-23T18:20:10.239508+0000","last_clean":"2023-09-23T18:20:10.239508+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:10.239508+0000","last_undegraded":"2023-09-23T18:20:10.239508+0000","last_fullsized":"2023-09-23T18:20:10.239508+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"611'9290","reported_seq":9337,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:23.849346+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:20:23.849346+0000","last_peered":"2023-09-23T18:20:23.849346+0000","last_clean":"2023-09-23T18:20:23.849346+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:20:23.849346+0000","last_undegraded":"2023-09-23T18:20:23.849346+0000","last_fullsized":"2023-09-23T18:20:23.849346+0000","mapping_epoch":574,"log_start":"600'4900","ondisk_log_start":"600'4900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4390,"log_dups_size":0,"ondisk_log_size":4390,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":246022144,"num_objects":3754,"num_object_clones":0,"num_object_copies":11262,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3754,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9290,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1093,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.484836+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:20:08.484836+0000","last_peered":"2023-09-23T18:20:08.484836+0000","last_clean":"2023-09-23T18:20:08.484836+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:20:08.484836+0000","last_undegraded":"2023-09-23T18:20:08.484836+0000","last_fullsized":"2023-09-23T18:20:08.484836+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":348,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:11.241994+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:20:11.241994+0000","last_peered":"2023-09-23T18:20:11.241994+0000","last_clean":"2023-09-23T18:20:11.241994+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:20:11.241994+0000","last_undegraded":"2023-09-23T18:20:11.241994+0000","last_fullsized":"2023-09-23T18:20:11.241994+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":155,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.498873+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:20:08.498873+0000","last_peered":"2023-09-23T18:20:08.498873+0000","last_clean":"2023-09-23T18:20:08.498873+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:20:08.498873+0000","last_undegraded":"2023-09-23T18:20:08.498873+0000","last_fullsized":"2023-09-23T18:20:08.498873+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"611'9209","reported_seq":9263,"reported_epoch":611,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:21.099171+0000","last_change":"2023-09-23T18:18:47.594759+0000","last_active":"2023-09-23T18:20:21.099171+0000","last_peered":"2023-09-23T18:20:21.099171+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:20:21.099171+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"592'3700","ondisk_log_start":"592'3700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5509,"log_dups_size":0,"ondisk_log_size":5509,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250413056,"num_objects":3821,"num_object_clones":0,"num_object_copies":11463,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3821,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9209,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1532},{"shards":"4,5","objects":-405}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":880,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.488487+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:20:08.488487+0000","last_peered":"2023-09-23T18:20:08.488487+0000","last_clean":"2023-09-23T18:20:08.488487+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:20:08.488487+0000","last_undegraded":"2023-09-23T18:20:08.488487+0000","last_fullsized":"2023-09-23T18:20:08.488487+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":163,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.502905+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:20:08.502905+0000","last_peered":"2023-09-23T18:20:08.502905+0000","last_clean":"2023-09-23T18:20:08.502905+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:20:08.502905+0000","last_undegraded":"2023-09-23T18:20:08.502905+0000","last_fullsized":"2023-09-23T18:20:08.502905+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":462,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.502288+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:20:08.502288+0000","last_peered":"2023-09-23T18:20:08.502288+0000","last_clean":"2023-09-23T18:20:08.502288+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:20:08.502288+0000","last_undegraded":"2023-09-23T18:20:08.502288+0000","last_fullsized":"2023-09-23T18:20:08.502288+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"611'9083","reported_seq":9128,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:21.270851+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:20:21.270851+0000","last_peered":"2023-09-23T18:20:21.270851+0000","last_clean":"2023-09-23T18:20:21.270851+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:20:21.270851+0000","last_undegraded":"2023-09-23T18:20:21.270851+0000","last_fullsized":"2023-09-23T18:20:21.270851+0000","mapping_epoch":574,"log_start":"600'4600","ondisk_log_start":"600'4600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4483,"log_dups_size":0,"ondisk_log_size":4483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":260112384,"num_objects":3969,"num_object_clones":0,"num_object_copies":11907,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3969,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9083,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":882,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.485685+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:20:08.485685+0000","last_peered":"2023-09-23T18:20:08.485685+0000","last_clean":"2023-09-23T18:20:08.485685+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:20:08.485685+0000","last_undegraded":"2023-09-23T18:20:08.485685+0000","last_fullsized":"2023-09-23T18:20:08.485685+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":447,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.484636+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:20:08.484636+0000","last_peered":"2023-09-23T18:20:08.484636+0000","last_clean":"2023-09-23T18:20:08.484636+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:20:08.484636+0000","last_undegraded":"2023-09-23T18:20:08.484636+0000","last_fullsized":"2023-09-23T18:20:08.484636+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":223,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.502815+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:20:08.502815+0000","last_peered":"2023-09-23T18:20:08.502815+0000","last_clean":"2023-09-23T18:20:08.502815+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:20:08.502815+0000","last_undegraded":"2023-09-23T18:20:08.502815+0000","last_fullsized":"2023-09-23T18:20:08.502815+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"611'9211","reported_seq":9256,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:22.336762+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:20:22.336762+0000","last_peered":"2023-09-23T18:20:22.336762+0000","last_clean":"2023-09-23T18:20:22.336762+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:20:22.336762+0000","last_undegraded":"2023-09-23T18:20:22.336762+0000","last_fullsized":"2023-09-23T18:20:22.336762+0000","mapping_epoch":574,"log_start":"600'4500","ondisk_log_start":"600'4500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4711,"log_dups_size":0,"ondisk_log_size":4711,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250544128,"num_objects":3823,"num_object_clones":0,"num_object_copies":11469,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3823,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9211,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":863,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478574+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:20:08.478574+0000","last_peered":"2023-09-23T18:20:08.478574+0000","last_clean":"2023-09-23T18:20:08.478574+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:20:08.478574+0000","last_undegraded":"2023-09-23T18:20:08.478574+0000","last_fullsized":"2023-09-23T18:20:08.478574+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":27,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.487089+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:20:08.487089+0000","last_peered":"2023-09-23T18:20:08.487089+0000","last_clean":"2023-09-23T18:20:08.487089+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:20:08.487089+0000","last_undegraded":"2023-09-23T18:20:08.487089+0000","last_fullsized":"2023-09-23T18:20:08.487089+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":339,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.495194+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:20:08.495194+0000","last_peered":"2023-09-23T18:20:08.495194+0000","last_clean":"2023-09-23T18:20:08.495194+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:20:08.495194+0000","last_undegraded":"2023-09-23T18:20:08.495194+0000","last_fullsized":"2023-09-23T18:20:08.495194+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"611'9137","reported_seq":9185,"reported_epoch":611,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:20.224977+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:20:20.224977+0000","last_peered":"2023-09-23T18:20:20.224977+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:20:20.224977+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"593'4200","ondisk_log_start":"593'4200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4937,"log_dups_size":0,"ondisk_log_size":4937,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":257622016,"num_objects":3931,"num_object_clones":0,"num_object_copies":11793,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3931,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9137,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-354},{"shards":"0,4,6","objects":1682}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":722,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.504664+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:20:08.504664+0000","last_peered":"2023-09-23T18:20:08.504664+0000","last_clean":"2023-09-23T18:20:08.504664+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:20:08.504664+0000","last_undegraded":"2023-09-23T18:20:08.504664+0000","last_fullsized":"2023-09-23T18:20:08.504664+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":961,"reported_epoch":610,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:08.493811+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:20:08.493811+0000","last_peered":"2023-09-23T18:20:08.493811+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:20:08.493811+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":145,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.485618+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:20:08.485618+0000","last_peered":"2023-09-23T18:20:08.485618+0000","last_clean":"2023-09-23T18:20:08.485618+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:20:08.485618+0000","last_undegraded":"2023-09-23T18:20:08.485618+0000","last_fullsized":"2023-09-23T18:20:08.485618+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":125,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478499+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:20:08.478499+0000","last_peered":"2023-09-23T18:20:08.478499+0000","last_clean":"2023-09-23T18:20:08.478499+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:20:08.478499+0000","last_undegraded":"2023-09-23T18:20:08.478499+0000","last_fullsized":"2023-09-23T18:20:08.478499+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"611'9071","reported_seq":9125,"reported_epoch":611,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:21.259975+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:20:21.259975+0000","last_peered":"2023-09-23T18:20:21.259975+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:20:21.259975+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"600'4600","ondisk_log_start":"600'4600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4471,"log_dups_size":0,"ondisk_log_size":4471,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253296640,"num_objects":3865,"num_object_clones":0,"num_object_copies":11595,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3865,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9071,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":955},{"shards":"5,6","objects":306}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":794,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.485141+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:20:08.485141+0000","last_peered":"2023-09-23T18:20:08.485141+0000","last_clean":"2023-09-23T18:20:08.485141+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:20:08.485141+0000","last_undegraded":"2023-09-23T18:20:08.485141+0000","last_fullsized":"2023-09-23T18:20:08.485141+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1495,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478434+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:20:08.478434+0000","last_peered":"2023-09-23T18:20:08.478434+0000","last_clean":"2023-09-23T18:20:08.478434+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:20:08.478434+0000","last_undegraded":"2023-09-23T18:20:08.478434+0000","last_fullsized":"2023-09-23T18:20:08.478434+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":135,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.501888+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:20:08.501888+0000","last_peered":"2023-09-23T18:20:08.501888+0000","last_clean":"2023-09-23T18:20:08.501888+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:20:08.501888+0000","last_undegraded":"2023-09-23T18:20:08.501888+0000","last_fullsized":"2023-09-23T18:20:08.501888+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":435,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.499181+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:20:08.499181+0000","last_peered":"2023-09-23T18:20:08.499181+0000","last_clean":"2023-09-23T18:20:08.499181+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:20:08.499181+0000","last_undegraded":"2023-09-23T18:20:08.499181+0000","last_fullsized":"2023-09-23T18:20:08.499181+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"611'8891","reported_seq":8936,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:21.131371+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:20:21.131371+0000","last_peered":"2023-09-23T18:20:21.131371+0000","last_clean":"2023-09-23T18:20:21.131371+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:20:21.131371+0000","last_undegraded":"2023-09-23T18:20:21.131371+0000","last_fullsized":"2023-09-23T18:20:21.131371+0000","mapping_epoch":574,"log_start":"587'3400","ondisk_log_start":"587'3400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5491,"log_dups_size":0,"ondisk_log_size":5491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":242155520,"num_objects":3695,"num_object_clones":0,"num_object_copies":11085,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3695,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8891,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":819,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478656+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:20:08.478656+0000","last_peered":"2023-09-23T18:20:08.478656+0000","last_clean":"2023-09-23T18:20:08.478656+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:20:08.478656+0000","last_undegraded":"2023-09-23T18:20:08.478656+0000","last_fullsized":"2023-09-23T18:20:08.478656+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":25,"reported_epoch":610,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:20:08.491871+0000","last_change":"2023-09-23T18:18:46.432707+0000","last_active":"2023-09-23T18:20:08.491871+0000","last_peered":"2023-09-23T18:20:08.491871+0000","last_clean":"2023-09-23T18:11:33.614822+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:20:08.491871+0000","last_undegraded":"2023-09-23T18:18:46.418246+0000","last_fullsized":"2023-09-23T18:20:08.491871+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":404,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.501493+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:20:08.501493+0000","last_peered":"2023-09-23T18:20:08.501493+0000","last_clean":"2023-09-23T18:20:08.501493+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:20:08.501493+0000","last_undegraded":"2023-09-23T18:20:08.501493+0000","last_fullsized":"2023-09-23T18:20:08.501493+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":147,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477859+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:20:08.477859+0000","last_peered":"2023-09-23T18:20:08.477859+0000","last_clean":"2023-09-23T18:20:08.477859+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:20:08.477859+0000","last_undegraded":"2023-09-23T18:20:08.477859+0000","last_fullsized":"2023-09-23T18:20:08.477859+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"611'9119","reported_seq":9164,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:21.268637+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:20:21.268637+0000","last_peered":"2023-09-23T18:20:21.268637+0000","last_clean":"2023-09-23T18:20:21.268637+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:20:21.268637+0000","last_undegraded":"2023-09-23T18:20:21.268637+0000","last_fullsized":"2023-09-23T18:20:21.268637+0000","mapping_epoch":574,"log_start":"600'4700","ondisk_log_start":"600'4700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4419,"log_dups_size":0,"ondisk_log_size":4419,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":250937344,"num_objects":3829,"num_object_clones":0,"num_object_copies":11487,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3829,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9119,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1508,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478816+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:20:08.478816+0000","last_peered":"2023-09-23T18:20:08.478816+0000","last_clean":"2023-09-23T18:20:08.478816+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:20:08.478816+0000","last_undegraded":"2023-09-23T18:20:08.478816+0000","last_fullsized":"2023-09-23T18:20:08.478816+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":561,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.499522+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:20:08.499522+0000","last_peered":"2023-09-23T18:20:08.499522+0000","last_clean":"2023-09-23T18:20:08.499522+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:20:08.499522+0000","last_undegraded":"2023-09-23T18:20:08.499522+0000","last_fullsized":"2023-09-23T18:20:08.499522+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":121,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478306+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:20:08.478306+0000","last_peered":"2023-09-23T18:20:08.478306+0000","last_clean":"2023-09-23T18:20:08.478306+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:20:08.478306+0000","last_undegraded":"2023-09-23T18:20:08.478306+0000","last_fullsized":"2023-09-23T18:20:08.478306+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":156,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.502636+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:20:08.502636+0000","last_peered":"2023-09-23T18:20:08.502636+0000","last_clean":"2023-09-23T18:20:08.502636+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:20:08.502636+0000","last_undegraded":"2023-09-23T18:20:08.502636+0000","last_fullsized":"2023-09-23T18:20:08.502636+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"611'8978","reported_seq":9023,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:21.099177+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:20:21.099177+0000","last_peered":"2023-09-23T18:20:21.099177+0000","last_clean":"2023-09-23T18:20:21.099177+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:20:21.099177+0000","last_undegraded":"2023-09-23T18:20:21.099177+0000","last_fullsized":"2023-09-23T18:20:21.099177+0000","mapping_epoch":574,"log_start":"591'3500","ondisk_log_start":"591'3500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5478,"log_dups_size":0,"ondisk_log_size":5478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":249430016,"num_objects":3806,"num_object_clones":0,"num_object_copies":11418,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3806,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8978,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":809,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.502043+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:20:08.502043+0000","last_peered":"2023-09-23T18:20:08.502043+0000","last_clean":"2023-09-23T18:20:08.502043+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:20:08.502043+0000","last_undegraded":"2023-09-23T18:20:08.502043+0000","last_fullsized":"2023-09-23T18:20:08.502043+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":124,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.487865+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:20:08.487865+0000","last_peered":"2023-09-23T18:20:08.487865+0000","last_clean":"2023-09-23T18:20:08.487865+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:20:08.487865+0000","last_undegraded":"2023-09-23T18:20:08.487865+0000","last_fullsized":"2023-09-23T18:20:08.487865+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":27,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.491828+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:20:08.491828+0000","last_peered":"2023-09-23T18:20:08.491828+0000","last_clean":"2023-09-23T18:20:08.491828+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:20:08.491828+0000","last_undegraded":"2023-09-23T18:20:08.491828+0000","last_fullsized":"2023-09-23T18:20:08.491828+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":626,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477672+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:20:08.477672+0000","last_peered":"2023-09-23T18:20:08.477672+0000","last_clean":"2023-09-23T18:20:08.477672+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:20:08.477672+0000","last_undegraded":"2023-09-23T18:20:08.477672+0000","last_fullsized":"2023-09-23T18:20:08.477672+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"611'9069","reported_seq":13930,"reported_epoch":611,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:21.281528+0000","last_change":"2023-09-23T18:19:32.858380+0000","last_active":"2023-09-23T18:20:21.281528+0000","last_peered":"2023-09-23T18:20:21.281528+0000","last_clean":"2023-09-23T18:18:32.699126+0000","last_became_active":"2023-09-23T18:18:34.716591+0000","last_became_peered":"2023-09-23T18:18:34.716591+0000","last_unstale":"2023-09-23T18:20:21.281528+0000","last_undegraded":"2023-09-23T18:18:34.351246+0000","last_fullsized":"2023-09-23T18:18:34.327168+0000","mapping_epoch":589,"log_start":"600'4600","ondisk_log_start":"600'4600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4469,"log_dups_size":0,"ondisk_log_size":4469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:13.611234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":254803992,"num_objects":3889,"num_object_clones":0,"num_object_copies":11667,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":259,"num_objects_misplaced":792,"num_objects_unfound":0,"num_objects_dirty":3889,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9069,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2402,"num_bytes_recovered":157417472,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":792},{"shards":"4,6","objects":2838}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":909,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.502141+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:20:08.502141+0000","last_peered":"2023-09-23T18:20:08.502141+0000","last_clean":"2023-09-23T18:20:08.502141+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:20:08.502141+0000","last_undegraded":"2023-09-23T18:20:08.502141+0000","last_fullsized":"2023-09-23T18:20:08.502141+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":49,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.490682+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:20:08.490682+0000","last_peered":"2023-09-23T18:20:08.490682+0000","last_clean":"2023-09-23T18:20:08.490682+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:20:08.490682+0000","last_undegraded":"2023-09-23T18:20:08.490682+0000","last_fullsized":"2023-09-23T18:20:08.490682+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":135,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477274+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:20:08.477274+0000","last_peered":"2023-09-23T18:20:08.477274+0000","last_clean":"2023-09-23T18:20:08.477274+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:20:08.477274+0000","last_undegraded":"2023-09-23T18:20:08.477274+0000","last_fullsized":"2023-09-23T18:20:08.477274+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":135,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478732+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:20:08.478732+0000","last_peered":"2023-09-23T18:20:08.478732+0000","last_clean":"2023-09-23T18:20:08.478732+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:20:08.478732+0000","last_undegraded":"2023-09-23T18:20:08.478732+0000","last_fullsized":"2023-09-23T18:20:08.478732+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"611'9068","reported_seq":9113,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:21.265626+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:20:21.265626+0000","last_peered":"2023-09-23T18:20:21.265626+0000","last_clean":"2023-09-23T18:20:21.265626+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:20:21.265626+0000","last_undegraded":"2023-09-23T18:20:21.265626+0000","last_fullsized":"2023-09-23T18:20:21.265626+0000","mapping_epoch":574,"log_start":"600'4600","ondisk_log_start":"600'4600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4468,"log_dups_size":0,"ondisk_log_size":4468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":248119296,"num_objects":3786,"num_object_clones":0,"num_object_copies":11358,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3786,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9068,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":909,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.502357+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:20:08.502357+0000","last_peered":"2023-09-23T18:20:08.502357+0000","last_clean":"2023-09-23T18:20:08.502357+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:20:08.502357+0000","last_undegraded":"2023-09-23T18:20:08.502357+0000","last_fullsized":"2023-09-23T18:20:08.502357+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":97,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477635+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:20:08.477635+0000","last_peered":"2023-09-23T18:20:08.477635+0000","last_clean":"2023-09-23T18:20:08.477635+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:20:08.477635+0000","last_undegraded":"2023-09-23T18:20:08.477635+0000","last_fullsized":"2023-09-23T18:20:08.477635+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":390,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.501965+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:20:08.501965+0000","last_peered":"2023-09-23T18:20:08.501965+0000","last_clean":"2023-09-23T18:20:08.501965+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:20:08.501965+0000","last_undegraded":"2023-09-23T18:20:08.501965+0000","last_fullsized":"2023-09-23T18:20:08.501965+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":27,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.486990+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:08.486990+0000","last_peered":"2023-09-23T18:20:08.486990+0000","last_clean":"2023-09-23T18:20:08.486990+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:20:08.486990+0000","last_undegraded":"2023-09-23T18:20:08.486990+0000","last_fullsized":"2023-09-23T18:20:08.486990+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"611'8945","reported_seq":8990,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:21.255165+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:20:21.255165+0000","last_peered":"2023-09-23T18:20:21.255165+0000","last_clean":"2023-09-23T18:20:21.255165+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:20:21.255165+0000","last_undegraded":"2023-09-23T18:20:21.255165+0000","last_fullsized":"2023-09-23T18:20:21.255165+0000","mapping_epoch":574,"log_start":"600'4500","ondisk_log_start":"600'4500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4445,"log_dups_size":0,"ondisk_log_size":4445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":253427712,"num_objects":3867,"num_object_clones":0,"num_object_copies":11601,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3867,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8945,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":743,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.487903+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:20:08.487903+0000","last_peered":"2023-09-23T18:20:08.487903+0000","last_clean":"2023-09-23T18:20:08.487903+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:20:08.487903+0000","last_undegraded":"2023-09-23T18:20:08.487903+0000","last_fullsized":"2023-09-23T18:20:08.487903+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1047,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.504856+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:20:08.504856+0000","last_peered":"2023-09-23T18:20:08.504856+0000","last_clean":"2023-09-23T18:20:08.504856+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:20:08.504856+0000","last_undegraded":"2023-09-23T18:20:08.504856+0000","last_fullsized":"2023-09-23T18:20:08.504856+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":37,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.491124+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:20:08.491124+0000","last_peered":"2023-09-23T18:20:08.491124+0000","last_clean":"2023-09-23T18:20:08.491124+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:20:08.491124+0000","last_undegraded":"2023-09-23T18:20:08.491124+0000","last_fullsized":"2023-09-23T18:20:08.491124+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":401,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.484915+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:20:08.484915+0000","last_peered":"2023-09-23T18:20:08.484915+0000","last_clean":"2023-09-23T18:20:08.484915+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:20:08.484915+0000","last_undegraded":"2023-09-23T18:20:08.484915+0000","last_fullsized":"2023-09-23T18:20:08.484915+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":641,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.499000+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:20:08.499000+0000","last_peered":"2023-09-23T18:20:08.499000+0000","last_clean":"2023-09-23T18:20:08.499000+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:20:08.499000+0000","last_undegraded":"2023-09-23T18:20:08.499000+0000","last_fullsized":"2023-09-23T18:20:08.499000+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":33,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.490530+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:20:08.490530+0000","last_peered":"2023-09-23T18:20:08.490530+0000","last_clean":"2023-09-23T18:20:08.490530+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:20:08.490530+0000","last_undegraded":"2023-09-23T18:20:08.490530+0000","last_fullsized":"2023-09-23T18:20:08.490530+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":684,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.493635+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:20:08.493635+0000","last_peered":"2023-09-23T18:20:08.493635+0000","last_clean":"2023-09-23T18:20:08.493635+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:20:08.493635+0000","last_undegraded":"2023-09-23T18:20:08.493635+0000","last_fullsized":"2023-09-23T18:20:08.493635+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":641,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.501774+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:20:08.501774+0000","last_peered":"2023-09-23T18:20:08.501774+0000","last_clean":"2023-09-23T18:20:08.501774+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:20:08.501774+0000","last_undegraded":"2023-09-23T18:20:08.501774+0000","last_fullsized":"2023-09-23T18:20:08.501774+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":129,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477348+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:20:08.477348+0000","last_peered":"2023-09-23T18:20:08.477348+0000","last_clean":"2023-09-23T18:20:08.477348+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:20:08.477348+0000","last_undegraded":"2023-09-23T18:20:08.477348+0000","last_fullsized":"2023-09-23T18:20:08.477348+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":245,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.485387+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:20:08.485387+0000","last_peered":"2023-09-23T18:20:08.485387+0000","last_clean":"2023-09-23T18:20:08.485387+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:20:08.485387+0000","last_undegraded":"2023-09-23T18:20:08.485387+0000","last_fullsized":"2023-09-23T18:20:08.485387+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":115,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.478901+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:20:08.478901+0000","last_peered":"2023-09-23T18:20:08.478901+0000","last_clean":"2023-09-23T18:20:08.478901+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:20:08.478901+0000","last_undegraded":"2023-09-23T18:20:08.478901+0000","last_fullsized":"2023-09-23T18:20:08.478901+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":228,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.484838+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:20:08.484838+0000","last_peered":"2023-09-23T18:20:08.484838+0000","last_clean":"2023-09-23T18:20:08.484838+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:20:08.484838+0000","last_undegraded":"2023-09-23T18:20:08.484838+0000","last_fullsized":"2023-09-23T18:20:08.484838+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1003,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.498816+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:20:08.498816+0000","last_peered":"2023-09-23T18:20:08.498816+0000","last_clean":"2023-09-23T18:20:08.498816+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:20:08.498816+0000","last_undegraded":"2023-09-23T18:20:08.498816+0000","last_fullsized":"2023-09-23T18:20:08.498816+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":508,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.502057+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:20:08.502057+0000","last_peered":"2023-09-23T18:20:08.502057+0000","last_clean":"2023-09-23T18:20:08.502057+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:20:08.502057+0000","last_undegraded":"2023-09-23T18:20:08.502057+0000","last_fullsized":"2023-09-23T18:20:08.502057+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":145,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.485107+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:20:08.485107+0000","last_peered":"2023-09-23T18:20:08.485107+0000","last_clean":"2023-09-23T18:20:08.485107+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:20:08.485107+0000","last_undegraded":"2023-09-23T18:20:08.485107+0000","last_fullsized":"2023-09-23T18:20:08.485107+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":940,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.494457+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:20:08.494457+0000","last_peered":"2023-09-23T18:20:08.494457+0000","last_clean":"2023-09-23T18:20:08.494457+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:20:08.494457+0000","last_undegraded":"2023-09-23T18:20:08.494457+0000","last_fullsized":"2023-09-23T18:20:08.494457+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":133,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.477794+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:20:08.477794+0000","last_peered":"2023-09-23T18:20:08.477794+0000","last_clean":"2023-09-23T18:20:08.477794+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:20:08.477794+0000","last_undegraded":"2023-09-23T18:20:08.477794+0000","last_fullsized":"2023-09-23T18:20:08.477794+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":654,"reported_epoch":610,"state":"active+clean","last_fresh":"2023-09-23T18:20:08.499443+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:20:08.499443+0000","last_peered":"2023-09-23T18:20:08.499443+0000","last_clean":"2023-09-23T18:20:08.499443+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:20:08.499443+0000","last_undegraded":"2023-09-23T18:20:08.499443+0000","last_fullsized":"2023-09-23T18:20:08.499443+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":4028366872,"num_objects":61469,"num_object_clones":0,"num_object_copies":184407,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16477,"num_objects_misplaced":792,"num_objects_unfound":0,"num_objects_dirty":61469,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":145339,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5873,"num_bytes_recovered":384892928,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":13329969152,"data_stored":13329956936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77239,"ondisk_log_size":77239,"up":48,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001573,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669509,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2648480,"kb_used_data":2033432,"kb_used_omap":7,"kb_used_meta":615032,"kb_avail":91096672,"statfs":{"total":95995035648,"available":93282992128,"internally_reserved":0,"allocated":2082234368,"data_stored":2080303999,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":629793572},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":1.7889999999999999,"15min":1.407},"min":{"1min":0.249,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":9.1600000000000001,"5min":24.472999999999999,"15min":24.472999999999999},"last":5.8399999999999999},{"interface":"front","average":{"1min":2.54,"5min":2.1960000000000002,"15min":1.726},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2029999999999994,"5min":22.832999999999998,"15min":22.832999999999998},"last":0.45000000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:19:21 2023","interfaces":[{"interface":"back","average":{"1min":4.1740000000000004,"5min":1.8069999999999999,"15min":1.413},"min":{"1min":0.34100000000000003,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":22.972000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":3.6190000000000002},{"interface":"front","average":{"1min":3.448,"5min":1.6559999999999999,"15min":1.357},"min":{"1min":0.30499999999999999,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":22.664000000000001,"5min":22.664000000000001,"15min":22.664000000000001},"last":1.724}]},{"osd":2,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.52,"15min":1.3260000000000001},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":8.0559999999999992,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":5.7919999999999998},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.861,"15min":1.492},"min":{"1min":0.34000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.1230000000000002,"5min":14.771000000000001,"15min":14.771000000000001},"last":3.5920000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":9.1259999999999994,"5min":3.1400000000000001,"15min":2.1059999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":103.348,"5min":103.348,"15min":103.348},"last":5.8170000000000002},{"interface":"front","average":{"1min":9.4610000000000003,"5min":3.6389999999999998,"15min":2.3610000000000002},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":103.40300000000001,"5min":103.40300000000001,"15min":103.40300000000001},"last":3.5600000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.8180000000000001,"15min":1.345},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.0730000000000004,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.776,"5min":1.8200000000000001,"15min":1.3700000000000001},"min":{"1min":0.45200000000000001,"5min":0.30399999999999999,"15min":0.27000000000000002},"max":{"1min":7.8319999999999999,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":5.7649999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:19:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":2.0609999999999999,"15min":1.9550000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":6.4710000000000001,"5min":13.502000000000001,"15min":13.502000000000001},"last":0.435},{"interface":"front","average":{"1min":1.3720000000000001,"5min":2.246,"15min":2.3660000000000001},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.1070000000000002,"5min":10.081,"15min":10.081},"last":0.98499999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":2.1459999999999999,"15min":1.5309999999999999},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.26800000000000002},"max":{"1min":6.4320000000000004,"5min":10.743,"15min":13.460000000000001},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.0710000000000002,"15min":1.5740000000000001},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.28499999999999998},"max":{"1min":6.2770000000000001,"5min":11.244,"15min":12.404},"last":5.7249999999999996}]}]},{"osd":1,"up_from":545,"seq":2340757176358,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1826740,"kb_used_data":1614892,"kb_used_omap":13,"kb_used_meta":211826,"kb_avail":91918412,"statfs":{"total":95995035648,"available":94124453888,"internally_reserved":0,"allocated":1653649408,"data_stored":1650656713,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":216910477},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.776,"5min":0.66800000000000004,"15min":0.48299999999999998},"min":{"1min":0.221,"5min":0.189,"15min":0.189},"max":{"1min":5.8079999999999998,"5min":5.8079999999999998,"15min":5.8079999999999998},"last":0.24399999999999999},{"interface":"front","average":{"1min":1.2430000000000001,"5min":0.64000000000000001,"15min":0.54000000000000004},"min":{"1min":0.20999999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":4.6159999999999997,"5min":4.6159999999999997,"15min":4.6159999999999997},"last":1.831}]},{"osd":2,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.5529999999999999,"5min":0.67900000000000005,"15min":0.53300000000000003},"min":{"1min":0.28299999999999997,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":4.9290000000000003,"5min":4.9290000000000003,"15min":4.9290000000000003},"last":0.49199999999999999},{"interface":"front","average":{"1min":1.272,"5min":0.65800000000000003,"15min":0.55600000000000005},"min":{"1min":0.29899999999999999,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2759999999999998,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":0.441}]},{"osd":3,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.1829999999999998,"5min":0.746,"15min":0.50600000000000001},"min":{"1min":0.34300000000000003,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":9.3219999999999992,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":1.8089999999999999},{"interface":"front","average":{"1min":2.2429999999999999,"5min":0.85799999999999998,"15min":0.627},"min":{"1min":0.38500000000000001,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":8.3659999999999997,"5min":8.3659999999999997,"15min":8.3659999999999997},"last":0.53500000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.4690000000000001,"5min":1.347,"15min":1.327},"min":{"1min":0.40200000000000002,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":7.6609999999999996,"5min":12.663,"15min":12.663},"last":0.41399999999999998},{"interface":"front","average":{"1min":1.95,"5min":1.601,"15min":1.5429999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":7.1769999999999996,"5min":13.222,"15min":13.222},"last":5.1269999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.52,"5min":0.88200000000000001,"15min":0.77600000000000002},"min":{"1min":0.308,"5min":0.308,"15min":0.308},"max":{"1min":5.093,"5min":5.093,"15min":5.093},"last":0.59899999999999998},{"interface":"front","average":{"1min":2.0830000000000002,"5min":1.0589999999999999,"15min":0.88900000000000001},"min":{"1min":0.46000000000000002,"5min":0.42299999999999999,"15min":0.42299999999999999},"max":{"1min":8.3190000000000008,"5min":8.3190000000000008,"15min":8.3190000000000008},"last":0.56399999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":1.948,"5min":0.997,"15min":0.83899999999999997},"min":{"1min":0.45400000000000001,"5min":0.435,"15min":0.435},"max":{"1min":13.134,"5min":13.134,"15min":13.134},"last":2.6419999999999999},{"interface":"front","average":{"1min":2.6070000000000002,"5min":1.1419999999999999,"15min":0.89800000000000002},"min":{"1min":0.38900000000000001,"5min":0.378,"15min":0.378},"max":{"1min":9.2579999999999991,"5min":9.2579999999999991,"15min":9.2579999999999991},"last":2.7450000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.335,"5min":1.0109999999999999,"15min":0.79100000000000004},"min":{"1min":0.436,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":8.2829999999999995,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":2.8279999999999998},{"interface":"front","average":{"1min":2.6419999999999999,"5min":1.173,"15min":0.92800000000000005},"min":{"1min":0.48099999999999998,"5min":0.44400000000000001,"15min":0.44400000000000001},"max":{"1min":9.0640000000000001,"5min":9.0640000000000001,"15min":9.0640000000000001},"last":3.899}]}]},{"osd":0,"up_from":93,"seq":399431958854,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3606036,"kb_used_data":2670284,"kb_used_omap":7,"kb_used_meta":935480,"kb_avail":90139116,"statfs":{"total":95995035648,"available":92302454784,"internally_reserved":0,"allocated":2734370816,"data_stored":2731526500,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":957932260},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:19:27 2023","interfaces":[{"interface":"back","average":{"1min":2.7229999999999999,"5min":0.80000000000000004,"15min":0.47999999999999998},"min":{"1min":0.214,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":8.2309999999999999,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":1.012},{"interface":"front","average":{"1min":2.2010000000000001,"5min":0.85799999999999998,"15min":0.63400000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":13.714,"5min":13.714,"15min":13.714},"last":1.6060000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:20:04 2023","interfaces":[{"interface":"back","average":{"1min":3.911,"5min":1.71,"15min":1.3160000000000001},"min":{"1min":0.48999999999999999,"5min":0.185,"15min":0.185},"max":{"1min":15.002000000000001,"5min":18.550999999999998,"15min":18.550999999999998},"last":1.871},{"interface":"front","average":{"1min":2.8530000000000002,"5min":1.258,"15min":0.93400000000000005},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":11.644,"5min":13.618,"15min":13.618},"last":1.577}]},{"osd":3,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7610000000000001,"5min":1.393,"15min":1.637},"min":{"1min":0.38400000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.539999999999999,"5min":13.539999999999999,"15min":13.539999999999999},"last":0.93799999999999994},{"interface":"front","average":{"1min":4.1390000000000002,"5min":1.4930000000000001,"15min":1.637},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.285,"5min":13.285,"15min":15.198},"last":0.88900000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7010000000000001,"5min":2.1309999999999998,"15min":1.8979999999999999},"min":{"1min":0.41799999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":17.741,"5min":17.741,"15min":57.746000000000002},"last":0.91500000000000004},{"interface":"front","average":{"1min":3.79,"5min":1.9159999999999999,"15min":1.7809999999999999},"min":{"1min":0.50900000000000001,"5min":0.26900000000000002,"15min":0.25600000000000001},"max":{"1min":15.079000000000001,"5min":15.079000000000001,"15min":57.816000000000003},"last":1.8340000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.032,"15min":1.391},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":0.81299999999999994},{"interface":"front","average":{"1min":4.2859999999999996,"5min":2.2650000000000001,"15min":1.448},"min":{"1min":0.502,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":13.425000000000001},"last":4.4909999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":1.948,"15min":1.9530000000000001},"min":{"1min":0.47499999999999998,"5min":0.377,"15min":0.32000000000000001},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":58.457999999999998},"last":0.85899999999999999},{"interface":"front","average":{"1min":4.6639999999999997,"5min":2.3159999999999998,"15min":1.9299999999999999},"min":{"1min":0.497,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":18.643999999999998,"5min":18.643999999999998,"15min":58.354999999999997},"last":2.891}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":1.923,"15min":1.639},"min":{"1min":0.46899999999999997,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":13.644,"5min":13.644,"15min":13.644},"last":1.5409999999999999},{"interface":"front","average":{"1min":3.5179999999999998,"5min":1.821,"15min":1.8},"min":{"1min":0.55900000000000005,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.7519999999999998,"5min":10.404,"15min":10.404},"last":1.748}]}]},{"osd":2,"up_from":0,"seq":2070174236740,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249369,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2137452,"kb_used_data":1420644,"kb_used_omap":3,"kb_used_meta":716796,"kb_avail":91607700,"statfs":{"total":95995035648,"available":93806284800,"internally_reserved":0,"allocated":1454739456,"data_stored":1451696706,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":733999848},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3919999999999999,"5min":1.25,"15min":1.7050000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.183},"max":{"1min":17.036000000000001,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.20799999999999999},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.1599999999999999,"15min":1.96},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.0980000000000008,"5min":9.0980000000000008,"15min":194.661},"last":1.2809999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:19:24 2023","interfaces":[{"interface":"back","average":{"1min":2.7109999999999999,"5min":0.95799999999999996,"15min":0.66600000000000004},"min":{"1min":0.28899999999999998,"5min":0.247,"15min":0.247},"max":{"1min":21.777000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":1.9730000000000001},{"interface":"front","average":{"1min":6.5110000000000001,"5min":1.8660000000000001,"15min":1.0920000000000001},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":78.980000000000004,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.56799999999999995}]},{"osd":2,"last update":"Sat Sep 23 18:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":1.274,"15min":1.29},"min":{"1min":0.27400000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.151999999999999,"5min":12.151999999999999,"15min":12.151999999999999},"last":2.1019999999999999},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.298,"15min":1.3320000000000001},"min":{"1min":0.318,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.565,"5min":11.565,"15min":11.565},"last":0.64600000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.6140000000000001,"15min":1.659},"min":{"1min":0.31900000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2300000000000004,"5min":21.062999999999999,"15min":37.231000000000002},"last":5.9169999999999998},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.9810000000000001,"15min":1.8939999999999999},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.2959999999999994,"5min":21.128,"15min":37.381},"last":1.6950000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.8009999999999999,"15min":1.4419999999999999},"min":{"1min":0.40999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":11.805999999999999,"5min":11.805999999999999,"15min":11.805999999999999},"last":1.7270000000000001},{"interface":"front","average":{"1min":2.028,"5min":1.601,"15min":1.6339999999999999},"min":{"1min":0.44500000000000001,"5min":0.33100000000000002,"15min":0.32000000000000001},"max":{"1min":8.9359999999999999,"5min":8.9359999999999999,"15min":8.9969999999999999},"last":0.61699999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":1.7809999999999999,"15min":1.847},"min":{"1min":0.432,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":17.192,"5min":17.192,"15min":115.54000000000001},"last":0.85399999999999998},{"interface":"front","average":{"1min":2.2530000000000001,"5min":1.7250000000000001,"15min":1.948},"min":{"1min":0.37,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":8.0350000000000001,"5min":14.321,"15min":115.691},"last":0.60199999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.5069999999999999,"15min":1.7110000000000001},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":8.2070000000000007,"5min":8.2070000000000007,"15min":15.972},"last":1.7869999999999999},{"interface":"front","average":{"1min":3.5920000000000001,"5min":1.7629999999999999,"15min":1.623},"min":{"1min":0.48799999999999999,"5min":0.41999999999999998,"15min":0.34899999999999998},"max":{"1min":17.164000000000001,"5min":17.164000000000001,"15min":17.164000000000001},"last":0.68000000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019319,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3862240,"kb_used_data":3209176,"kb_used_omap":0,"kb_used_meta":652992,"kb_avail":89882912,"statfs":{"total":95995035648,"available":92040101888,"internally_reserved":0,"allocated":3286196224,"data_stored":3284265647,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":668663808},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.534,"15min":1.268},"min":{"1min":0.255,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.6079999999999997,"5min":7.6079999999999997,"15min":13.375999999999999},"last":0.55100000000000005},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.6819999999999999,"15min":1.546},"min":{"1min":0.26600000000000001,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":9.3710000000000004,"5min":10.890000000000001,"15min":10.890000000000001},"last":1.1870000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:19:22 2023","interfaces":[{"interface":"back","average":{"1min":4.5739999999999998,"5min":1.47,"15min":0.95299999999999996},"min":{"1min":0.247,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":44.146999999999998,"5min":44.146999999999998,"15min":44.146999999999998},"last":6.8220000000000001},{"interface":"front","average":{"1min":4.5419999999999998,"5min":1.4299999999999999,"15min":0.91100000000000003},"min":{"1min":0.28000000000000003,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":43.497999999999998,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.61899999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:19:56 2023","interfaces":[{"interface":"back","average":{"1min":2.621,"5min":1.6699999999999999,"15min":1.8180000000000001},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.896000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":1.6200000000000001},{"interface":"front","average":{"1min":1.6020000000000001,"5min":1.6200000000000001,"15min":1.6930000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":6.6890000000000001,"5min":10.938000000000001,"15min":10.938000000000001},"last":1.2350000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.5669999999999999,"15min":1.9350000000000001},"min":{"1min":0.33700000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.0609999999999999,"5min":11.130000000000001,"15min":29.154},"last":0.59199999999999997},{"interface":"front","average":{"1min":2.1800000000000002,"5min":1.8480000000000001,"15min":1.9390000000000001},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":13.470000000000001,"5min":13.470000000000001,"15min":29.216000000000001},"last":1.2130000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":2.4279999999999999,"15min":2.8559999999999999},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4950000000000001,"5min":27.675000000000001,"15min":27.675000000000001},"last":1.157},{"interface":"front","average":{"1min":1.401,"5min":1.827,"15min":2.7109999999999999},"min":{"1min":0.30499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.1040000000000001,"5min":13.938000000000001,"15min":32.640000000000001},"last":3.1419999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.873,"15min":1.347},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":6.1989999999999998,"5min":18.315000000000001,"15min":18.315000000000001},"last":3.367},{"interface":"front","average":{"1min":1.859,"5min":1.9630000000000001,"15min":1.522},"min":{"1min":0.38700000000000001,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":9.5579999999999998,"5min":18.431000000000001,"15min":18.431000000000001},"last":3.0710000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":2.4849999999999999,"15min":1.774},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":5.9539999999999997,"5min":19.298999999999999,"15min":19.298999999999999},"last":1.3029999999999999},{"interface":"front","average":{"1min":1.722,"5min":1.8899999999999999,"15min":1.552},"min":{"1min":0.435,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.4730000000000008,"5min":12.800000000000001,"15min":12.800000000000001},"last":1.982}]}]},{"osd":5,"up_from":420,"seq":1803886264415,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2530320,"kb_used_data":2198408,"kb_used_omap":9,"kb_used_meta":331830,"kb_avail":91214832,"statfs":{"total":95995035648,"available":93403987968,"internally_reserved":0,"allocated":2251169792,"data_stored":2249251559,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":339794365},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":1.2549999999999999,"15min":0.88100000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.6000000000000001,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":2.9409999999999998},{"interface":"front","average":{"1min":2.911,"5min":1.8149999999999999,"15min":1.6080000000000001},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":20.937999999999999,"5min":20.937999999999999,"15min":20.937999999999999},"last":0.245}]},{"osd":1,"last update":"Sat Sep 23 18:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":1.7070000000000001,"15min":1.2989999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.6649999999999991,"5min":23.946000000000002,"15min":23.946000000000002},"last":2.843},{"interface":"front","average":{"1min":2.1840000000000002,"5min":1.73,"15min":1.141},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.5259999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":1.006}]},{"osd":2,"last update":"Sat Sep 23 18:19:52 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":1.629,"15min":1.615},"min":{"1min":0.28699999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":10.705,"5min":10.705,"15min":10.705},"last":2.8839999999999999},{"interface":"front","average":{"1min":2.5409999999999999,"5min":1.946,"15min":2.1600000000000001},"min":{"1min":0.373,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.8099999999999996,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.39600000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":1.8740000000000001,"15min":1.2789999999999999},"min":{"1min":0.443,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.8780000000000001,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":2.9119999999999999},{"interface":"front","average":{"1min":3.7320000000000002,"5min":2.0640000000000001,"15min":1.665},"min":{"1min":0.38100000000000001,"5min":0.315,"15min":0.313},"max":{"1min":20.035,"5min":20.035,"15min":20.035},"last":0.44400000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.2770000000000001,"15min":2.0640000000000001},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":7.875,"5min":12.747999999999999,"15min":12.747999999999999},"last":5.8449999999999998},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.0899999999999999,"15min":1.6160000000000001},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7610000000000001,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":3.1890000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":1.9650000000000001,"15min":1.2609999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2480000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":3.2189999999999999},{"interface":"front","average":{"1min":2.343,"5min":2.2040000000000002,"15min":1.6819999999999999},"min":{"1min":0.34599999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.8410000000000002,"5min":12.504,"15min":12.504},"last":0.42199999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.3980000000000001,"15min":1.758},"min":{"1min":0.40699999999999997,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.528,"5min":13.506,"15min":13.506},"last":5.7930000000000001},{"interface":"front","average":{"1min":1.381,"5min":1.9630000000000001,"15min":1.542},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":5.0970000000000004,"5min":13.138,"15min":13.138},"last":5.0839999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2720337920,"data_stored":2720333848,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1632632832,"data_stored":1632632832,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1444478976,"data_stored":1444478976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":3254390784,"data_stored":3254386712,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":2229272576,"data_stored":2229272576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":2048856064,"data_stored":2048851992,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:20:26.013 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:20:26.064 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:26.117 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:26.207 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:26.213 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:26.213 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:26.314 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:26.396 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:26.475 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:26.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:26.489 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:26.490 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:26 smithi055 ceph-mon[179668]: pgmap v1465: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 592 op/s; 16502/184488 objects degraded (8.945%); 792/184488 objects misplaced (0.429%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:20:26.490 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:26 smithi055 ceph-mon[179668]: from='client.37275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:26.590 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:26.664 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:26.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:26.729 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:26.730 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:26.819 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:26.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:26 smithi099 ceph-mon[167076]: pgmap v1465: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 592 op/s; 16502/184488 objects degraded (8.945%); 792/184488 objects misplaced (0.429%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:20:26.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:26 smithi099 ceph-mon[167076]: from='client.37275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:26.831 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:26.876 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:26 smithi055 ceph-mon[184174]: pgmap v1465: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.8 GiB data, 16 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 592 op/s; 16502/184488 objects degraded (8.945%); 792/184488 objects misplaced (0.429%); 2.5 MiB/s, 39 objects/s recovering 2023-09-23T18:20:26.876 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:26 smithi055 ceph-mon[184174]: from='client.37275 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:26.882 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:26.965 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:26.972 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:26.972 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:27.073 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:27.083 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:20:27.132 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:27.195 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:27.207 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:27.208 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:27.274 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:20:27.309 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:27.312 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:20:27.353 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:20:27.380 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:27.387 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:20:27.428 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:20:27.455 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:27.461 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:27.462 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:27.463 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:20:27.496 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:20:27.536 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:20:27.542 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:20:27.542 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:20:27.542 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:20:27.542 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:20:27.542 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:20:27.542 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:20:27.542 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:20:27.543 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:20:27.563 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:27.642 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:27.720 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1029549295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:27.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1029549295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:27.965 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1029549295' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:28.379 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:20:28.379 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:20:28.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:28 smithi099 ceph-mon[167076]: pgmap v1466: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 16502/173199 objects degraded (9.528%); 454/173199 objects misplaced (0.262%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:20:28.764 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:28 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16502/173199 objects degraded (9.528%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:28.797 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:20:28.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:28 smithi055 ceph-mon[184174]: pgmap v1466: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 16502/173199 objects degraded (9.528%); 454/173199 objects misplaced (0.262%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:20:28.921 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:28 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16502/173199 objects degraded (9.528%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:28.922 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:28 smithi055 ceph-mon[179668]: pgmap v1466: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 863 op/s; 16502/173199 objects degraded (9.528%); 454/173199 objects misplaced (0.262%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:20:28.922 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:28 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16502/173199 objects degraded (9.528%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:29.003 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:20:29.167 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:29.172 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:29.173 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:29.273 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:29.400 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:29.435 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:29.498 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:29.509 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:29.510 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:29.611 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:29.678 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:29.712 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:29.726 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:29.732 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:29.732 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:29.833 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:29.920 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:30.099 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:30.107 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:30.108 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:30.209 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:30.285 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:30.340 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:30.347 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:30.347 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:30.439 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:30.447 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:30.492 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:30.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:30.550 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:30.551 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:30.651 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:30.658 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:30 smithi055 ceph-mon[179668]: pgmap v1467: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 869 op/s; 16502/171486 objects degraded (9.623%); 454/171486 objects misplaced (0.265%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:20:30.659 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:20:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:20:30] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.43.0" 2023-09-23T18:20:30.659 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:30 smithi055 ceph-mon[184174]: pgmap v1467: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 869 op/s; 16502/171486 objects degraded (9.623%); 454/171486 objects misplaced (0.265%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:20:30.740 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:30.813 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:30.819 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:30.820 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:30.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:30 smithi099 ceph-mon[167076]: pgmap v1467: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.5 GiB data, 15 GiB used, 521 GiB / 536 GiB avail; 0 B/s wr, 869 op/s; 16502/171486 objects degraded (9.623%); 454/171486 objects misplaced (0.265%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:20:30.921 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:30.992 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:31.259 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:31.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:31.271 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:31.276 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:20:31.372 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:31.472 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:31.659 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:31.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:31.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:31.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:31.803 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:20:31.803 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:20:31.823 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:31.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:31.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:31.905 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:32.006 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:32.066 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:32.290 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:32.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:32.298 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:32.393 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:20:32.398 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:32.441 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:32.501 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:32.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:32.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:32.592 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:20:32.609 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:32.667 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:32.704 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:32.708 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:32 smithi099 ceph-mon[167076]: pgmap v1468: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 16502/166542 objects degraded (9.909%); 454/166542 objects misplaced (0.273%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:20:32.708 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:32 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2916383274' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:32.708 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:32 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:32.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:32.709 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:32.774 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:32.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:32.847 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:32.899 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:32.908 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:32.908 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:32 smithi055 ceph-mon[184174]: pgmap v1468: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 16502/166542 objects degraded (9.909%); 454/166542 objects misplaced (0.273%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:20:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:32 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2916383274' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:32.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:32 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:32.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:32 smithi055 ceph-mon[179668]: pgmap v1468: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.4 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 868 op/s; 16502/166542 objects degraded (9.909%); 454/166542 objects misplaced (0.273%); 4.1 MiB/s, 66 objects/s recovering 2023-09-23T18:20:32.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:32 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2916383274' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:32.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:32 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:33.009 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:33.070 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:33.148 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:33.158 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:33.159 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:33.207 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:20:33.227 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:20:33.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:33.302 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:33.353 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:33.356 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:33.357 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:33.458 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:33.514 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:33.599 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:33.605 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:33.605 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:33 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:33 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:20:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:33 smithi055 ceph-mon[184174]: osdmap e613: 8 total, 8 up, 6 in 2023-09-23T18:20:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:33 smithi055 ceph-mon[184174]: failed to encode map e613 with expected crc 2023-09-23T18:20:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 16502/166542 objects degraded (9.909%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:33.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:33 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:33.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:33 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:20:33.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:33 smithi055 ceph-mon[179668]: osdmap e613: 8 total, 8 up, 6 in 2023-09-23T18:20:33.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:33 smithi055 ceph-mon[179668]: failed to encode map e613 with expected crc 2023-09-23T18:20:33.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 16502/166542 objects degraded (9.909%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:33.706 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:33.760 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1470,"stamp":"2023-09-23T18:20:33.170745+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":3466934972,"num_objects":52861,"num_object_clones":0,"num_object_copies":158583,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":16502,"num_objects_misplaced":50,"num_objects_unfound":0,"num_objects_dirty":52861,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":154885,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7001,"num_bytes_recovered":448749193,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83296,"ondisk_log_size":83296,"up":357,"acting":351,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":15213976,"kb_used_data":11725288,"kb_used_omap":40,"kb_used_meta":3488471,"kb_avail":547256936,"statfs":{"total":575970213888,"available":560391102464,"internally_reserved":0,"allocated":12006694912,"data_stored":11992000866,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3572194618},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-565903360,"num_objects":-8635,"num_object_clones":0,"num_object_copies":-25905,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":-742,"num_objects_unfound":0,"num_objects_dirty":-8635,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8635,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":742,"num_bytes_recovered":48627712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.705811"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":900,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944493+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:20:16.944493+0000","last_peered":"2023-09-23T18:20:16.944493+0000","last_clean":"2023-09-23T18:20:16.944493+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:20:16.944493+0000","last_undegraded":"2023-09-23T18:20:16.944493+0000","last_fullsized":"2023-09-23T18:20:16.944493+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":271,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.951040+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:20:16.951040+0000","last_peered":"2023-09-23T18:20:16.951040+0000","last_clean":"2023-09-23T18:20:16.951040+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:20:16.951040+0000","last_undegraded":"2023-09-23T18:20:16.951040+0000","last_fullsized":"2023-09-23T18:20:16.951040+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":28,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.938331+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:20:16.938331+0000","last_peered":"2023-09-23T18:20:16.938331+0000","last_clean":"2023-09-23T18:20:16.938331+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:20:16.938331+0000","last_undegraded":"2023-09-23T18:20:16.938331+0000","last_fullsized":"2023-09-23T18:20:16.938331+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":906,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945946+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:20:16.945946+0000","last_peered":"2023-09-23T18:20:16.945946+0000","last_clean":"2023-09-23T18:20:16.945946+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:20:16.945946+0000","last_undegraded":"2023-09-23T18:20:16.945946+0000","last_fullsized":"2023-09-23T18:20:16.945946+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":353,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:24.476341+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:20:24.476341+0000","last_peered":"2023-09-23T18:20:24.476341+0000","last_clean":"2023-09-23T18:20:24.476341+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:20:24.476341+0000","last_undegraded":"2023-09-23T18:20:24.476341+0000","last_fullsized":"2023-09-23T18:20:24.476341+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":672,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.948123+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:20:16.948123+0000","last_peered":"2023-09-23T18:20:16.948123+0000","last_clean":"2023-09-23T18:20:16.948123+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:20:16.948123+0000","last_undegraded":"2023-09-23T18:20:16.948123+0000","last_fullsized":"2023-09-23T18:20:16.948123+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":923,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944794+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:20:16.944794+0000","last_peered":"2023-09-23T18:20:16.944794+0000","last_clean":"2023-09-23T18:20:16.944794+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:16.944794+0000","last_undegraded":"2023-09-23T18:20:16.944794+0000","last_fullsized":"2023-09-23T18:20:16.944794+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":28,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.936925+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:16.936925+0000","last_peered":"2023-09-23T18:20:16.936925+0000","last_clean":"2023-09-23T18:20:16.936925+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:20:16.936925+0000","last_undegraded":"2023-09-23T18:20:16.936925+0000","last_fullsized":"2023-09-23T18:20:16.936925+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":188,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.959956+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:20:16.959956+0000","last_peered":"2023-09-23T18:20:16.959956+0000","last_clean":"2023-09-23T18:20:16.959956+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:20:16.959956+0000","last_undegraded":"2023-09-23T18:20:16.959956+0000","last_fullsized":"2023-09-23T18:20:16.959956+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1046,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944694+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:20:16.944694+0000","last_peered":"2023-09-23T18:20:16.944694+0000","last_clean":"2023-09-23T18:20:16.944694+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:20:16.944694+0000","last_undegraded":"2023-09-23T18:20:16.944694+0000","last_fullsized":"2023-09-23T18:20:16.944694+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":122,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945526+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:20:16.945526+0000","last_peered":"2023-09-23T18:20:16.945526+0000","last_clean":"2023-09-23T18:20:16.945526+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:20:16.945526+0000","last_undegraded":"2023-09-23T18:20:16.945526+0000","last_fullsized":"2023-09-23T18:20:16.945526+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":201,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.937341+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:20:16.937341+0000","last_peered":"2023-09-23T18:20:16.937341+0000","last_clean":"2023-09-23T18:20:16.937341+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:20:16.937341+0000","last_undegraded":"2023-09-23T18:20:16.937341+0000","last_fullsized":"2023-09-23T18:20:16.937341+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":823,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.956964+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:20:16.956964+0000","last_peered":"2023-09-23T18:20:16.956964+0000","last_clean":"2023-09-23T18:20:16.956964+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:20:16.956964+0000","last_undegraded":"2023-09-23T18:20:16.956964+0000","last_fullsized":"2023-09-23T18:20:16.956964+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":158,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945910+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:20:16.945910+0000","last_peered":"2023-09-23T18:20:16.945910+0000","last_clean":"2023-09-23T18:20:16.945910+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:20:16.945910+0000","last_undegraded":"2023-09-23T18:20:16.945910+0000","last_fullsized":"2023-09-23T18:20:16.945910+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":157,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.950939+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:20:16.950939+0000","last_peered":"2023-09-23T18:20:16.950939+0000","last_clean":"2023-09-23T18:20:16.950939+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:20:16.950939+0000","last_undegraded":"2023-09-23T18:20:16.950939+0000","last_fullsized":"2023-09-23T18:20:16.950939+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":957,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945577+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:20:16.945577+0000","last_peered":"2023-09-23T18:20:16.945577+0000","last_clean":"2023-09-23T18:20:16.945577+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:20:16.945577+0000","last_undegraded":"2023-09-23T18:20:16.945577+0000","last_fullsized":"2023-09-23T18:20:16.945577+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":406,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.938646+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:20:16.938646+0000","last_peered":"2023-09-23T18:20:16.938646+0000","last_clean":"2023-09-23T18:20:16.938646+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:20:16.938646+0000","last_undegraded":"2023-09-23T18:20:16.938646+0000","last_fullsized":"2023-09-23T18:20:16.938646+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":425,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.952538+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:20:16.952538+0000","last_peered":"2023-09-23T18:20:16.952538+0000","last_clean":"2023-09-23T18:20:16.952538+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:20:16.952538+0000","last_undegraded":"2023-09-23T18:20:16.952538+0000","last_fullsized":"2023-09-23T18:20:16.952538+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":986,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.941101+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:20:16.941101+0000","last_peered":"2023-09-23T18:20:16.941101+0000","last_clean":"2023-09-23T18:20:16.941101+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:20:16.941101+0000","last_undegraded":"2023-09-23T18:20:16.941101+0000","last_fullsized":"2023-09-23T18:20:16.941101+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":164,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957474+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:20:16.957474+0000","last_peered":"2023-09-23T18:20:16.957474+0000","last_clean":"2023-09-23T18:20:16.957474+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:20:16.957474+0000","last_undegraded":"2023-09-23T18:20:16.957474+0000","last_fullsized":"2023-09-23T18:20:16.957474+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":431,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957206+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:20:16.957206+0000","last_peered":"2023-09-23T18:20:16.957206+0000","last_clean":"2023-09-23T18:20:16.957206+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:20:16.957206+0000","last_undegraded":"2023-09-23T18:20:16.957206+0000","last_fullsized":"2023-09-23T18:20:16.957206+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":838,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945034+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:20:16.945034+0000","last_peered":"2023-09-23T18:20:16.945034+0000","last_clean":"2023-09-23T18:20:16.945034+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:20:16.945034+0000","last_undegraded":"2023-09-23T18:20:16.945034+0000","last_fullsized":"2023-09-23T18:20:16.945034+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":431,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957047+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:20:16.957047+0000","last_peered":"2023-09-23T18:20:16.957047+0000","last_clean":"2023-09-23T18:20:16.957047+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:20:16.957047+0000","last_undegraded":"2023-09-23T18:20:16.957047+0000","last_fullsized":"2023-09-23T18:20:16.957047+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":109,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944316+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:20:16.944316+0000","last_peered":"2023-09-23T18:20:16.944316+0000","last_clean":"2023-09-23T18:20:16.944316+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:20:16.944316+0000","last_undegraded":"2023-09-23T18:20:16.944316+0000","last_fullsized":"2023-09-23T18:20:16.944316+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":809,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944154+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:20:16.944154+0000","last_peered":"2023-09-23T18:20:16.944154+0000","last_clean":"2023-09-23T18:20:16.944154+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:20:16.944154+0000","last_undegraded":"2023-09-23T18:20:16.944154+0000","last_fullsized":"2023-09-23T18:20:16.944154+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":127,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.948983+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:20:16.948983+0000","last_peered":"2023-09-23T18:20:16.948983+0000","last_clean":"2023-09-23T18:20:16.948983+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:20:16.948983+0000","last_undegraded":"2023-09-23T18:20:16.948983+0000","last_fullsized":"2023-09-23T18:20:16.948983+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":122,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944951+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:20:16.944951+0000","last_peered":"2023-09-23T18:20:16.944951+0000","last_clean":"2023-09-23T18:20:16.944951+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:20:16.944951+0000","last_undegraded":"2023-09-23T18:20:16.944951+0000","last_fullsized":"2023-09-23T18:20:16.944951+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":747,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.939050+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:20:16.939050+0000","last_peered":"2023-09-23T18:20:16.939050+0000","last_clean":"2023-09-23T18:20:16.939050+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:20:16.939050+0000","last_undegraded":"2023-09-23T18:20:16.939050+0000","last_fullsized":"2023-09-23T18:20:16.939050+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":199,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.950790+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:20:16.950790+0000","last_peered":"2023-09-23T18:20:16.950790+0000","last_clean":"2023-09-23T18:20:16.950790+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:20:16.950790+0000","last_undegraded":"2023-09-23T18:20:16.950790+0000","last_fullsized":"2023-09-23T18:20:16.950790+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":210,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.938327+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:20:16.938327+0000","last_peered":"2023-09-23T18:20:16.938327+0000","last_clean":"2023-09-23T18:20:16.938327+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:20:16.938327+0000","last_undegraded":"2023-09-23T18:20:16.938327+0000","last_fullsized":"2023-09-23T18:20:16.938327+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":843,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.951206+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:20:16.951206+0000","last_peered":"2023-09-23T18:20:16.951206+0000","last_clean":"2023-09-23T18:20:16.951206+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:20:16.951206+0000","last_undegraded":"2023-09-23T18:20:16.951206+0000","last_fullsized":"2023-09-23T18:20:16.951206+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":752,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.951799+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:20:16.951799+0000","last_peered":"2023-09-23T18:20:16.951799+0000","last_clean":"2023-09-23T18:20:16.951799+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:20:16.951799+0000","last_undegraded":"2023-09-23T18:20:16.951799+0000","last_fullsized":"2023-09-23T18:20:16.951799+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":28,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.942967+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:20:16.942967+0000","last_peered":"2023-09-23T18:20:16.942967+0000","last_clean":"2023-09-23T18:20:16.942967+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:20:16.942967+0000","last_undegraded":"2023-09-23T18:20:16.942967+0000","last_fullsized":"2023-09-23T18:20:16.942967+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"612'9740","reported_seq":9786,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:31.131827+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:20:31.131827+0000","last_peered":"2023-09-23T18:20:31.131827+0000","last_clean":"2023-09-23T18:20:31.131827+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:20:31.131827+0000","last_undegraded":"2023-09-23T18:20:31.131827+0000","last_fullsized":"2023-09-23T18:20:31.131827+0000","mapping_epoch":574,"log_start":"593'4200","ondisk_log_start":"593'4200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5540,"log_dups_size":0,"ondisk_log_size":5540,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218497024,"num_objects":3334,"num_object_clones":0,"num_object_copies":10002,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3334,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9740,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":877,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957650+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:20:16.957650+0000","last_peered":"2023-09-23T18:20:16.957650+0000","last_clean":"2023-09-23T18:20:16.957650+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:20:16.957650+0000","last_undegraded":"2023-09-23T18:20:16.957650+0000","last_fullsized":"2023-09-23T18:20:16.957650+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":248,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.950688+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:20:16.950688+0000","last_peered":"2023-09-23T18:20:16.950688+0000","last_clean":"2023-09-23T18:20:16.950688+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:20:16.950688+0000","last_undegraded":"2023-09-23T18:20:16.950688+0000","last_fullsized":"2023-09-23T18:20:16.950688+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":122,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944167+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:20:16.944167+0000","last_peered":"2023-09-23T18:20:16.944167+0000","last_clean":"2023-09-23T18:20:16.944167+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:20:16.944167+0000","last_undegraded":"2023-09-23T18:20:16.944167+0000","last_fullsized":"2023-09-23T18:20:16.944167+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"612'9470","reported_seq":9526,"reported_epoch":612,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:31.271057+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:20:31.271057+0000","last_peered":"2023-09-23T18:20:31.271057+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:20:31.271057+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"601'5000","ondisk_log_start":"601'5000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4470,"log_dups_size":0,"ondisk_log_size":4470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211156992,"num_objects":3222,"num_object_clones":0,"num_object_copies":9666,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3222,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9470,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":1024},{"shards":"5,6","objects":-926}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":731,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.937806+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:20:16.937806+0000","last_peered":"2023-09-23T18:20:16.937806+0000","last_clean":"2023-09-23T18:20:16.937806+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:20:16.937806+0000","last_undegraded":"2023-09-23T18:20:16.937806+0000","last_fullsized":"2023-09-23T18:20:16.937806+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":249,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.951363+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:20:16.951363+0000","last_peered":"2023-09-23T18:20:16.951363+0000","last_clean":"2023-09-23T18:20:16.951363+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:20:16.951363+0000","last_undegraded":"2023-09-23T18:20:16.951363+0000","last_fullsized":"2023-09-23T18:20:16.951363+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":228,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.950606+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:20:16.950606+0000","last_peered":"2023-09-23T18:20:16.950606+0000","last_clean":"2023-09-23T18:20:16.950606+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:20:16.950606+0000","last_undegraded":"2023-09-23T18:20:16.950606+0000","last_fullsized":"2023-09-23T18:20:16.950606+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"612'9513","reported_seq":16519,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:30.211539+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:20:30.211539+0000","last_peered":"2023-09-23T18:20:30.211539+0000","last_clean":"2023-09-23T18:20:30.211539+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:20:30.211539+0000","last_undegraded":"2023-09-23T18:20:30.211539+0000","last_fullsized":"2023-09-23T18:20:30.211539+0000","mapping_epoch":605,"log_start":"600'4600","ondisk_log_start":"600'4600","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4913,"log_dups_size":0,"ondisk_log_size":4913,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":218300416,"num_objects":3331,"num_object_clones":0,"num_object_copies":9993,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3331,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9513,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":627,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.959877+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:20:16.959877+0000","last_peered":"2023-09-23T18:20:16.959877+0000","last_clean":"2023-09-23T18:20:16.959877+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:20:16.959877+0000","last_undegraded":"2023-09-23T18:20:16.959877+0000","last_fullsized":"2023-09-23T18:20:16.959877+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":537,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944781+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:20:16.944781+0000","last_peered":"2023-09-23T18:20:16.944781+0000","last_clean":"2023-09-23T18:20:16.944781+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:20:16.944781+0000","last_undegraded":"2023-09-23T18:20:16.944781+0000","last_fullsized":"2023-09-23T18:20:16.944781+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":188,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.952447+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:20:16.952447+0000","last_peered":"2023-09-23T18:20:16.952447+0000","last_clean":"2023-09-23T18:20:16.952447+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:20:16.952447+0000","last_undegraded":"2023-09-23T18:20:16.952447+0000","last_fullsized":"2023-09-23T18:20:16.952447+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"612'9747","reported_seq":9795,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:31.288351+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:20:31.288351+0000","last_peered":"2023-09-23T18:20:31.288351+0000","last_clean":"2023-09-23T18:20:31.288351+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:20:31.288351+0000","last_undegraded":"2023-09-23T18:20:31.288351+0000","last_fullsized":"2023-09-23T18:20:31.288351+0000","mapping_epoch":574,"log_start":"600'4600","ondisk_log_start":"600'4600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5147,"log_dups_size":0,"ondisk_log_size":5147,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219217920,"num_objects":3345,"num_object_clones":0,"num_object_copies":10035,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3345,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9747,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":643,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.938375+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:20:16.938375+0000","last_peered":"2023-09-23T18:20:16.938375+0000","last_clean":"2023-09-23T18:20:16.938375+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:20:16.938375+0000","last_undegraded":"2023-09-23T18:20:16.938375+0000","last_fullsized":"2023-09-23T18:20:16.938375+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":136,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944695+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:20:16.944695+0000","last_peered":"2023-09-23T18:20:16.944695+0000","last_clean":"2023-09-23T18:20:16.944695+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:20:16.944695+0000","last_undegraded":"2023-09-23T18:20:16.944695+0000","last_fullsized":"2023-09-23T18:20:16.944695+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":449,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.948885+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:20:16.948885+0000","last_peered":"2023-09-23T18:20:16.948885+0000","last_clean":"2023-09-23T18:20:16.948885+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:16.948885+0000","last_undegraded":"2023-09-23T18:20:16.948885+0000","last_fullsized":"2023-09-23T18:20:16.948885+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"612'9584","reported_seq":9632,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:28.849924+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:20:28.849924+0000","last_peered":"2023-09-23T18:20:28.849924+0000","last_clean":"2023-09-23T18:20:28.849924+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:20:28.849924+0000","last_undegraded":"2023-09-23T18:20:28.849924+0000","last_fullsized":"2023-09-23T18:20:28.849924+0000","mapping_epoch":574,"log_start":"601'5200","ondisk_log_start":"601'5200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4384,"log_dups_size":0,"ondisk_log_size":4384,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":226754560,"num_objects":3460,"num_object_clones":0,"num_object_copies":10380,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3460,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9584,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1094,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944801+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:20:16.944801+0000","last_peered":"2023-09-23T18:20:16.944801+0000","last_clean":"2023-09-23T18:20:16.944801+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:20:16.944801+0000","last_undegraded":"2023-09-23T18:20:16.944801+0000","last_fullsized":"2023-09-23T18:20:16.944801+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":349,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.939373+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:20:16.939373+0000","last_peered":"2023-09-23T18:20:16.939373+0000","last_clean":"2023-09-23T18:20:16.939373+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:20:16.939373+0000","last_undegraded":"2023-09-23T18:20:16.939373+0000","last_fullsized":"2023-09-23T18:20:16.939373+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":156,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957116+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:20:16.957116+0000","last_peered":"2023-09-23T18:20:16.957116+0000","last_clean":"2023-09-23T18:20:16.957116+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:20:16.957116+0000","last_undegraded":"2023-09-23T18:20:16.957116+0000","last_fullsized":"2023-09-23T18:20:16.957116+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"612'9706","reported_seq":9761,"reported_epoch":612,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:31.071485+0000","last_change":"2023-09-23T18:18:47.594759+0000","last_active":"2023-09-23T18:20:31.071485+0000","last_peered":"2023-09-23T18:20:31.071485+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:20:31.071485+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"593'4200","ondisk_log_start":"593'4200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5506,"log_dups_size":0,"ondisk_log_size":5506,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217841664,"num_objects":3324,"num_object_clones":0,"num_object_copies":9972,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4226,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3324,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9706,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":1035},{"shards":"4,5","objects":-902}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":881,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944415+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:20:16.944415+0000","last_peered":"2023-09-23T18:20:16.944415+0000","last_clean":"2023-09-23T18:20:16.944415+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:20:16.944415+0000","last_undegraded":"2023-09-23T18:20:16.944415+0000","last_fullsized":"2023-09-23T18:20:16.944415+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":164,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.952371+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:20:16.952371+0000","last_peered":"2023-09-23T18:20:16.952371+0000","last_clean":"2023-09-23T18:20:16.952371+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:20:16.952371+0000","last_undegraded":"2023-09-23T18:20:16.952371+0000","last_fullsized":"2023-09-23T18:20:16.952371+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":463,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957782+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:20:16.957782+0000","last_peered":"2023-09-23T18:20:16.957782+0000","last_clean":"2023-09-23T18:20:16.957782+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:20:16.957782+0000","last_undegraded":"2023-09-23T18:20:16.957782+0000","last_fullsized":"2023-09-23T18:20:16.957782+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"612'9621","reported_seq":9667,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:31.241433+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:20:31.241433+0000","last_peered":"2023-09-23T18:20:31.241433+0000","last_clean":"2023-09-23T18:20:31.241433+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:20:31.241433+0000","last_undegraded":"2023-09-23T18:20:31.241433+0000","last_fullsized":"2023-09-23T18:20:31.241433+0000","mapping_epoch":574,"log_start":"601'5200","ondisk_log_start":"601'5200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4421,"log_dups_size":0,"ondisk_log_size":4421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":224854016,"num_objects":3431,"num_object_clones":0,"num_object_copies":10293,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3431,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9621,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":883,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.946027+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:20:16.946027+0000","last_peered":"2023-09-23T18:20:16.946027+0000","last_clean":"2023-09-23T18:20:16.946027+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:20:16.946027+0000","last_undegraded":"2023-09-23T18:20:16.946027+0000","last_fullsized":"2023-09-23T18:20:16.946027+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":448,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944672+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:20:16.944672+0000","last_peered":"2023-09-23T18:20:16.944672+0000","last_clean":"2023-09-23T18:20:16.944672+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:20:16.944672+0000","last_undegraded":"2023-09-23T18:20:16.944672+0000","last_fullsized":"2023-09-23T18:20:16.944672+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":224,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957598+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:20:16.957598+0000","last_peered":"2023-09-23T18:20:16.957598+0000","last_clean":"2023-09-23T18:20:16.957598+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:20:16.957598+0000","last_undegraded":"2023-09-23T18:20:16.957598+0000","last_fullsized":"2023-09-23T18:20:16.957598+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"612'9766","reported_seq":9812,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:32.352825+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:20:32.352825+0000","last_peered":"2023-09-23T18:20:32.352825+0000","last_clean":"2023-09-23T18:20:32.352825+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:20:32.352825+0000","last_undegraded":"2023-09-23T18:20:32.352825+0000","last_fullsized":"2023-09-23T18:20:32.352825+0000","mapping_epoch":574,"log_start":"601'5100","ondisk_log_start":"601'5100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4666,"log_dups_size":0,"ondisk_log_size":4666,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214171648,"num_objects":3268,"num_object_clones":0,"num_object_copies":9804,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3268,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9766,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":864,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945178+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:20:16.945178+0000","last_peered":"2023-09-23T18:20:16.945178+0000","last_clean":"2023-09-23T18:20:16.945178+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:20:16.945178+0000","last_undegraded":"2023-09-23T18:20:16.945178+0000","last_fullsized":"2023-09-23T18:20:16.945178+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":28,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.937069+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:20:16.937069+0000","last_peered":"2023-09-23T18:20:16.937069+0000","last_clean":"2023-09-23T18:20:16.937069+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:20:16.937069+0000","last_undegraded":"2023-09-23T18:20:16.937069+0000","last_fullsized":"2023-09-23T18:20:16.937069+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":340,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.940861+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:20:16.940861+0000","last_peered":"2023-09-23T18:20:16.940861+0000","last_clean":"2023-09-23T18:20:16.940861+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:20:16.940861+0000","last_undegraded":"2023-09-23T18:20:16.940861+0000","last_fullsized":"2023-09-23T18:20:16.940861+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"612'9754","reported_seq":9803,"reported_epoch":612,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:30.186521+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:20:30.186521+0000","last_peered":"2023-09-23T18:20:30.186521+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:20:30.186521+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"600'4900","ondisk_log_start":"600'4900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4854,"log_dups_size":0,"ondisk_log_size":4854,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":217186304,"num_objects":3314,"num_object_clones":0,"num_object_copies":9942,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3314,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9754,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-971},{"shards":"0,4,6","objects":1065}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":723,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.960040+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:20:16.960040+0000","last_peered":"2023-09-23T18:20:16.960040+0000","last_clean":"2023-09-23T18:20:16.960040+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:20:16.960040+0000","last_undegraded":"2023-09-23T18:20:16.960040+0000","last_fullsized":"2023-09-23T18:20:16.960040+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":962,"reported_epoch":611,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:16.940057+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:20:16.940057+0000","last_peered":"2023-09-23T18:20:16.940057+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:20:16.940057+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":146,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944960+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:20:16.944960+0000","last_peered":"2023-09-23T18:20:16.944960+0000","last_clean":"2023-09-23T18:20:16.944960+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:20:16.944960+0000","last_undegraded":"2023-09-23T18:20:16.944960+0000","last_fullsized":"2023-09-23T18:20:16.944960+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":126,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945097+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:20:16.945097+0000","last_peered":"2023-09-23T18:20:16.945097+0000","last_clean":"2023-09-23T18:20:16.945097+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:20:16.945097+0000","last_undegraded":"2023-09-23T18:20:16.945097+0000","last_fullsized":"2023-09-23T18:20:16.945097+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"612'9705","reported_seq":9760,"reported_epoch":612,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:31.282889+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:20:31.282889+0000","last_peered":"2023-09-23T18:20:31.282889+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:20:31.282889+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"601'5200","ondisk_log_start":"601'5200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4505,"log_dups_size":0,"ondisk_log_size":4505,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":211746816,"num_objects":3231,"num_object_clones":0,"num_object_copies":9693,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3231,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9705,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"1,5,6","objects":321},{"shards":"5,6","objects":-328}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":795,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945188+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:20:16.945188+0000","last_peered":"2023-09-23T18:20:16.945188+0000","last_clean":"2023-09-23T18:20:16.945188+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:20:16.945188+0000","last_undegraded":"2023-09-23T18:20:16.945188+0000","last_fullsized":"2023-09-23T18:20:16.945188+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1496,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945023+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:20:16.945023+0000","last_peered":"2023-09-23T18:20:16.945023+0000","last_clean":"2023-09-23T18:20:16.945023+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:20:16.945023+0000","last_undegraded":"2023-09-23T18:20:16.945023+0000","last_fullsized":"2023-09-23T18:20:16.945023+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":136,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957129+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:20:16.957129+0000","last_peered":"2023-09-23T18:20:16.957129+0000","last_clean":"2023-09-23T18:20:16.957129+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:20:16.957129+0000","last_undegraded":"2023-09-23T18:20:16.957129+0000","last_fullsized":"2023-09-23T18:20:16.957129+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":436,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957337+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:20:16.957337+0000","last_peered":"2023-09-23T18:20:16.957337+0000","last_clean":"2023-09-23T18:20:16.957337+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:20:16.957337+0000","last_undegraded":"2023-09-23T18:20:16.957337+0000","last_fullsized":"2023-09-23T18:20:16.957337+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"612'9415","reported_seq":9461,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:31.084691+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:20:31.084691+0000","last_peered":"2023-09-23T18:20:31.084691+0000","last_clean":"2023-09-23T18:20:31.084691+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:20:31.084691+0000","last_undegraded":"2023-09-23T18:20:31.084691+0000","last_fullsized":"2023-09-23T18:20:31.084691+0000","mapping_epoch":574,"log_start":"593'3900","ondisk_log_start":"593'3900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5515,"log_dups_size":0,"ondisk_log_size":5515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":207814656,"num_objects":3171,"num_object_clones":0,"num_object_copies":9513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9415,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":820,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945267+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:20:16.945267+0000","last_peered":"2023-09-23T18:20:16.945267+0000","last_clean":"2023-09-23T18:20:16.945267+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:20:16.945267+0000","last_undegraded":"2023-09-23T18:20:16.945267+0000","last_fullsized":"2023-09-23T18:20:16.945267+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":26,"reported_epoch":611,"state":"active+recovery_wait+degraded","last_fresh":"2023-09-23T18:20:16.937887+0000","last_change":"2023-09-23T18:18:46.432707+0000","last_active":"2023-09-23T18:20:16.937887+0000","last_peered":"2023-09-23T18:20:16.937887+0000","last_clean":"2023-09-23T18:11:33.614822+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:20:16.937887+0000","last_undegraded":"2023-09-23T18:18:46.418246+0000","last_fullsized":"2023-09-23T18:20:16.937887+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":12,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":36,"num_bytes_recovered":12582912,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":["0","1","6"],"object_location_counts":[{"shards":"0,1,6","objects":1}],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":405,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.952170+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:20:16.952170+0000","last_peered":"2023-09-23T18:20:16.952170+0000","last_clean":"2023-09-23T18:20:16.952170+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:20:16.952170+0000","last_undegraded":"2023-09-23T18:20:16.952170+0000","last_fullsized":"2023-09-23T18:20:16.952170+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":148,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945475+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:20:16.945475+0000","last_peered":"2023-09-23T18:20:16.945475+0000","last_clean":"2023-09-23T18:20:16.945475+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:20:16.945475+0000","last_undegraded":"2023-09-23T18:20:16.945475+0000","last_fullsized":"2023-09-23T18:20:16.945475+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"612'9681","reported_seq":9727,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:31.268639+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:20:31.268639+0000","last_peered":"2023-09-23T18:20:31.268639+0000","last_clean":"2023-09-23T18:20:31.268639+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:20:31.268639+0000","last_undegraded":"2023-09-23T18:20:31.268639+0000","last_fullsized":"2023-09-23T18:20:31.268639+0000","mapping_epoch":574,"log_start":"601'5200","ondisk_log_start":"601'5200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4481,"log_dups_size":0,"ondisk_log_size":4481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":214106112,"num_objects":3267,"num_object_clones":0,"num_object_copies":9801,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3267,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9681,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1509,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945853+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:20:16.945853+0000","last_peered":"2023-09-23T18:20:16.945853+0000","last_clean":"2023-09-23T18:20:16.945853+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:20:16.945853+0000","last_undegraded":"2023-09-23T18:20:16.945853+0000","last_fullsized":"2023-09-23T18:20:16.945853+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":562,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.951952+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:20:16.951952+0000","last_peered":"2023-09-23T18:20:16.951952+0000","last_clean":"2023-09-23T18:20:16.951952+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:20:16.951952+0000","last_undegraded":"2023-09-23T18:20:16.951952+0000","last_fullsized":"2023-09-23T18:20:16.951952+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":122,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944873+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:20:16.944873+0000","last_peered":"2023-09-23T18:20:16.944873+0000","last_clean":"2023-09-23T18:20:16.944873+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:20:16.944873+0000","last_undegraded":"2023-09-23T18:20:16.944873+0000","last_fullsized":"2023-09-23T18:20:16.944873+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":157,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.958101+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:20:16.958101+0000","last_peered":"2023-09-23T18:20:16.958101+0000","last_clean":"2023-09-23T18:20:16.958101+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:20:16.958101+0000","last_undegraded":"2023-09-23T18:20:16.958101+0000","last_fullsized":"2023-09-23T18:20:16.958101+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"612'9525","reported_seq":9571,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:31.119126+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:20:31.119126+0000","last_peered":"2023-09-23T18:20:31.119126+0000","last_clean":"2023-09-23T18:20:31.119126+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:20:31.119126+0000","last_undegraded":"2023-09-23T18:20:31.119126+0000","last_fullsized":"2023-09-23T18:20:31.119126+0000","mapping_epoch":574,"log_start":"593'4000","ondisk_log_start":"593'4000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5525,"log_dups_size":0,"ondisk_log_size":5525,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213581824,"num_objects":3259,"num_object_clones":0,"num_object_copies":9777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9525,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":810,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.950449+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:20:16.950449+0000","last_peered":"2023-09-23T18:20:16.950449+0000","last_clean":"2023-09-23T18:20:16.950449+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:20:16.950449+0000","last_undegraded":"2023-09-23T18:20:16.950449+0000","last_fullsized":"2023-09-23T18:20:16.950449+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":125,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.937243+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:20:16.937243+0000","last_peered":"2023-09-23T18:20:16.937243+0000","last_clean":"2023-09-23T18:20:16.937243+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:20:16.937243+0000","last_undegraded":"2023-09-23T18:20:16.937243+0000","last_fullsized":"2023-09-23T18:20:16.937243+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":28,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.937765+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:20:16.937765+0000","last_peered":"2023-09-23T18:20:16.937765+0000","last_clean":"2023-09-23T18:20:16.937765+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:20:16.937765+0000","last_undegraded":"2023-09-23T18:20:16.937765+0000","last_fullsized":"2023-09-23T18:20:16.937765+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":627,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945962+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:20:16.945962+0000","last_peered":"2023-09-23T18:20:16.945962+0000","last_clean":"2023-09-23T18:20:16.945962+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:20:16.945962+0000","last_undegraded":"2023-09-23T18:20:16.945962+0000","last_fullsized":"2023-09-23T18:20:16.945962+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"612'9614","reported_seq":15960,"reported_epoch":612,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:31.283083+0000","last_change":"2023-09-23T18:19:32.858380+0000","last_active":"2023-09-23T18:20:31.283083+0000","last_peered":"2023-09-23T18:20:31.283083+0000","last_clean":"2023-09-23T18:18:32.699126+0000","last_became_active":"2023-09-23T18:18:34.716591+0000","last_became_peered":"2023-09-23T18:18:34.716591+0000","last_unstale":"2023-09-23T18:20:31.283083+0000","last_undegraded":"2023-09-23T18:18:34.351246+0000","last_fullsized":"2023-09-23T18:18:34.327168+0000","mapping_epoch":589,"log_start":"602'5200","ondisk_log_start":"602'5200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4414,"log_dups_size":0,"ondisk_log_size":4414,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:27:13.611234+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":219086872,"num_objects":3344,"num_object_clones":0,"num_object_copies":10032,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":259,"num_objects_misplaced":50,"num_objects_unfound":0,"num_objects_dirty":3344,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":9614,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3144,"num_bytes_recovered":206045184,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":50},{"shards":"4,6","objects":3035}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":910,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.958006+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:20:16.958006+0000","last_peered":"2023-09-23T18:20:16.958006+0000","last_clean":"2023-09-23T18:20:16.958006+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:20:16.958006+0000","last_undegraded":"2023-09-23T18:20:16.958006+0000","last_fullsized":"2023-09-23T18:20:16.958006+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":50,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.949062+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:20:16.949062+0000","last_peered":"2023-09-23T18:20:16.949062+0000","last_clean":"2023-09-23T18:20:16.949062+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:20:16.949062+0000","last_undegraded":"2023-09-23T18:20:16.949062+0000","last_fullsized":"2023-09-23T18:20:16.949062+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":136,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944466+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:20:16.944466+0000","last_peered":"2023-09-23T18:20:16.944466+0000","last_clean":"2023-09-23T18:20:16.944466+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:20:16.944466+0000","last_undegraded":"2023-09-23T18:20:16.944466+0000","last_fullsized":"2023-09-23T18:20:16.944466+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":136,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945848+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:20:16.945848+0000","last_peered":"2023-09-23T18:20:16.945848+0000","last_clean":"2023-09-23T18:20:16.945848+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:20:16.945848+0000","last_undegraded":"2023-09-23T18:20:16.945848+0000","last_fullsized":"2023-09-23T18:20:16.945848+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"612'9602","reported_seq":9648,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:31.242278+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:20:31.242278+0000","last_peered":"2023-09-23T18:20:31.242278+0000","last_clean":"2023-09-23T18:20:31.242278+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:20:31.242278+0000","last_undegraded":"2023-09-23T18:20:31.242278+0000","last_fullsized":"2023-09-23T18:20:31.242278+0000","mapping_epoch":574,"log_start":"601'5100","ondisk_log_start":"601'5100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4502,"log_dups_size":0,"ondisk_log_size":4502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":213123072,"num_objects":3252,"num_object_clones":0,"num_object_copies":9756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9602,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":910,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957856+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:20:16.957856+0000","last_peered":"2023-09-23T18:20:16.957856+0000","last_clean":"2023-09-23T18:20:16.957856+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:20:16.957856+0000","last_undegraded":"2023-09-23T18:20:16.957856+0000","last_fullsized":"2023-09-23T18:20:16.957856+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":98,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945331+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:20:16.945331+0000","last_peered":"2023-09-23T18:20:16.945331+0000","last_clean":"2023-09-23T18:20:16.945331+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:20:16.945331+0000","last_undegraded":"2023-09-23T18:20:16.945331+0000","last_fullsized":"2023-09-23T18:20:16.945331+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":391,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957832+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:20:16.957832+0000","last_peered":"2023-09-23T18:20:16.957832+0000","last_clean":"2023-09-23T18:20:16.957832+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:20:16.957832+0000","last_undegraded":"2023-09-23T18:20:16.957832+0000","last_fullsized":"2023-09-23T18:20:16.957832+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":28,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.936973+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:16.936973+0000","last_peered":"2023-09-23T18:20:16.936973+0000","last_clean":"2023-09-23T18:20:16.936973+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:20:16.936973+0000","last_undegraded":"2023-09-23T18:20:16.936973+0000","last_fullsized":"2023-09-23T18:20:16.936973+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"612'9531","reported_seq":9576,"reported_epoch":612,"state":"active+clean","last_fresh":"2023-09-23T18:20:31.271188+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:20:31.271188+0000","last_peered":"2023-09-23T18:20:31.271188+0000","last_clean":"2023-09-23T18:20:31.271188+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:20:31.271188+0000","last_undegraded":"2023-09-23T18:20:31.271188+0000","last_fullsized":"2023-09-23T18:20:31.271188+0000","mapping_epoch":574,"log_start":"601'5100","ondisk_log_start":"601'5100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4431,"log_dups_size":0,"ondisk_log_size":4431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":215023616,"num_objects":3281,"num_object_clones":0,"num_object_copies":9843,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3281,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9531,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":744,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.937884+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:20:16.937884+0000","last_peered":"2023-09-23T18:20:16.937884+0000","last_clean":"2023-09-23T18:20:16.937884+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:20:16.937884+0000","last_undegraded":"2023-09-23T18:20:16.937884+0000","last_fullsized":"2023-09-23T18:20:16.937884+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1048,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.960297+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:20:16.960297+0000","last_peered":"2023-09-23T18:20:16.960297+0000","last_clean":"2023-09-23T18:20:16.960297+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:20:16.960297+0000","last_undegraded":"2023-09-23T18:20:16.960297+0000","last_fullsized":"2023-09-23T18:20:16.960297+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":38,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.948368+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:20:16.948368+0000","last_peered":"2023-09-23T18:20:16.948368+0000","last_clean":"2023-09-23T18:20:16.948368+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:20:16.948368+0000","last_undegraded":"2023-09-23T18:20:16.948368+0000","last_fullsized":"2023-09-23T18:20:16.948368+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":402,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944872+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:20:16.944872+0000","last_peered":"2023-09-23T18:20:16.944872+0000","last_clean":"2023-09-23T18:20:16.944872+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:20:16.944872+0000","last_undegraded":"2023-09-23T18:20:16.944872+0000","last_fullsized":"2023-09-23T18:20:16.944872+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":642,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.951019+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:20:16.951019+0000","last_peered":"2023-09-23T18:20:16.951019+0000","last_clean":"2023-09-23T18:20:16.951019+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:20:16.951019+0000","last_undegraded":"2023-09-23T18:20:16.951019+0000","last_fullsized":"2023-09-23T18:20:16.951019+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":34,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.948770+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:20:16.948770+0000","last_peered":"2023-09-23T18:20:16.948770+0000","last_clean":"2023-09-23T18:20:16.948770+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:20:16.948770+0000","last_undegraded":"2023-09-23T18:20:16.948770+0000","last_fullsized":"2023-09-23T18:20:16.948770+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":685,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.939870+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:20:16.939870+0000","last_peered":"2023-09-23T18:20:16.939870+0000","last_clean":"2023-09-23T18:20:16.939870+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:20:16.939870+0000","last_undegraded":"2023-09-23T18:20:16.939870+0000","last_fullsized":"2023-09-23T18:20:16.939870+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":642,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957681+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:20:16.957681+0000","last_peered":"2023-09-23T18:20:16.957681+0000","last_clean":"2023-09-23T18:20:16.957681+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:20:16.957681+0000","last_undegraded":"2023-09-23T18:20:16.957681+0000","last_fullsized":"2023-09-23T18:20:16.957681+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":130,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944546+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:20:16.944546+0000","last_peered":"2023-09-23T18:20:16.944546+0000","last_clean":"2023-09-23T18:20:16.944546+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:20:16.944546+0000","last_undegraded":"2023-09-23T18:20:16.944546+0000","last_fullsized":"2023-09-23T18:20:16.944546+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":246,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.950655+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:20:16.950655+0000","last_peered":"2023-09-23T18:20:16.950655+0000","last_clean":"2023-09-23T18:20:16.950655+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:20:16.950655+0000","last_undegraded":"2023-09-23T18:20:16.950655+0000","last_fullsized":"2023-09-23T18:20:16.950655+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":116,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.944862+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:20:16.944862+0000","last_peered":"2023-09-23T18:20:16.944862+0000","last_clean":"2023-09-23T18:20:16.944862+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:20:16.944862+0000","last_undegraded":"2023-09-23T18:20:16.944862+0000","last_fullsized":"2023-09-23T18:20:16.944862+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":229,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.951121+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:20:16.951121+0000","last_peered":"2023-09-23T18:20:16.951121+0000","last_clean":"2023-09-23T18:20:16.951121+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:20:16.951121+0000","last_undegraded":"2023-09-23T18:20:16.951121+0000","last_fullsized":"2023-09-23T18:20:16.951121+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1004,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957046+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:20:16.957046+0000","last_peered":"2023-09-23T18:20:16.957046+0000","last_clean":"2023-09-23T18:20:16.957046+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:20:16.957046+0000","last_undegraded":"2023-09-23T18:20:16.957046+0000","last_fullsized":"2023-09-23T18:20:16.957046+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":509,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.957921+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:20:16.957921+0000","last_peered":"2023-09-23T18:20:16.957921+0000","last_clean":"2023-09-23T18:20:16.957921+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:20:16.957921+0000","last_undegraded":"2023-09-23T18:20:16.957921+0000","last_fullsized":"2023-09-23T18:20:16.957921+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":146,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945108+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:20:16.945108+0000","last_peered":"2023-09-23T18:20:16.945108+0000","last_clean":"2023-09-23T18:20:16.945108+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:20:16.945108+0000","last_undegraded":"2023-09-23T18:20:16.945108+0000","last_fullsized":"2023-09-23T18:20:16.945108+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":941,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.940859+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:20:16.940859+0000","last_peered":"2023-09-23T18:20:16.940859+0000","last_clean":"2023-09-23T18:20:16.940859+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:20:16.940859+0000","last_undegraded":"2023-09-23T18:20:16.940859+0000","last_fullsized":"2023-09-23T18:20:16.940859+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":134,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.945427+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:20:16.945427+0000","last_peered":"2023-09-23T18:20:16.945427+0000","last_clean":"2023-09-23T18:20:16.945427+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:20:16.945427+0000","last_undegraded":"2023-09-23T18:20:16.945427+0000","last_fullsized":"2023-09-23T18:20:16.945427+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":655,"reported_epoch":611,"state":"active+clean","last_fresh":"2023-09-23T18:20:16.951874+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:20:16.951874+0000","last_peered":"2023-09-23T18:20:16.951874+0000","last_clean":"2023-09-23T18:20:16.951874+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:20:16.951874+0000","last_undegraded":"2023-09-23T18:20:16.951874+0000","last_fullsized":"2023-09-23T18:20:16.951874+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":3462463512,"num_objects":52834,"num_object_clones":0,"num_object_copies":158502,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":16477,"num_objects_misplaced":50,"num_objects_unfound":0,"num_objects_dirty":52834,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":153974,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6615,"num_bytes_recovered":433520640,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":11846561792,"data_stored":11846549576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77274,"ondisk_log_size":77274,"up":48,"acting":43,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":12,"num_objects_missing":12,"num_objects_degraded":25,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":286,"num_bytes_recovered":12582969,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001575,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669511,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2391660,"kb_used_data":1777252,"kb_used_omap":7,"kb_used_meta":614392,"kb_avail":91353492,"statfs":{"total":95995035648,"available":93545975808,"internally_reserved":0,"allocated":1819906048,"data_stored":1817971696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":629138212},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":1.7889999999999999,"15min":1.407},"min":{"1min":0.249,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":9.1600000000000001,"5min":24.472999999999999,"15min":24.472999999999999},"last":0.56699999999999995},{"interface":"front","average":{"1min":2.54,"5min":2.1960000000000002,"15min":1.726},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2029999999999994,"5min":22.832999999999998,"15min":22.832999999999998},"last":1.2490000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:20:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3570000000000002,"5min":2.0350000000000001,"15min":1.4890000000000001},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.726000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":1.4730000000000001},{"interface":"front","average":{"1min":2.423,"5min":1.899,"15min":1.4379999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.336,"5min":22.664000000000001,"15min":22.664000000000001},"last":8.016}]},{"osd":2,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.52,"15min":1.3260000000000001},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":8.0559999999999992,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.33500000000000002},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.861,"15min":1.492},"min":{"1min":0.34000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.1230000000000002,"5min":14.771000000000001,"15min":14.771000000000001},"last":1.2849999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":9.1259999999999994,"5min":3.1400000000000001,"15min":2.1059999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":103.348,"5min":103.348,"15min":103.348},"last":0.373},{"interface":"front","average":{"1min":9.4610000000000003,"5min":3.6389999999999998,"15min":2.3610000000000002},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":103.40300000000001,"5min":103.40300000000001,"15min":103.40300000000001},"last":1.3140000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.8180000000000001,"15min":1.345},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.0730000000000004,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.73699999999999999},{"interface":"front","average":{"1min":1.776,"5min":1.8200000000000001,"15min":1.3700000000000001},"min":{"1min":0.45200000000000001,"5min":0.30399999999999999,"15min":0.27000000000000002},"max":{"1min":7.8319999999999999,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.41099999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:19:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":2.0609999999999999,"15min":1.9550000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":6.4710000000000001,"5min":13.502000000000001,"15min":13.502000000000001},"last":1.7529999999999999},{"interface":"front","average":{"1min":1.3720000000000001,"5min":2.246,"15min":2.3660000000000001},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.1070000000000002,"5min":10.081,"15min":10.081},"last":0.77900000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":2.1459999999999999,"15min":1.5309999999999999},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.26800000000000002},"max":{"1min":6.4320000000000004,"5min":10.743,"15min":13.460000000000001},"last":0.70599999999999996},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.0710000000000002,"15min":1.5740000000000001},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.28499999999999998},"max":{"1min":6.2770000000000001,"5min":11.244,"15min":12.404},"last":0.59599999999999997}]}]},{"osd":1,"up_from":545,"seq":2340757176359,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1760260,"kb_used_data":1544316,"kb_used_omap":13,"kb_used_meta":215922,"kb_avail":91984892,"statfs":{"total":95995035648,"available":94192529408,"internally_reserved":0,"allocated":1581379584,"data_stored":1578379170,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":221104781},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.016,"15min":0.59899999999999998},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":9.0299999999999994,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.004,"15min":0.66100000000000003},"min":{"1min":0.34499999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.952999999999999,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.40200000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.0980000000000001,"15min":0.67300000000000004},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.6219999999999999,"5min":9.6219999999999999,"15min":9.6219999999999999},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.77200000000000002,"15min":0.59399999999999997},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2080000000000002,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":1.413}]},{"osd":3,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.0409999999999999,"15min":0.60499999999999998},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.3200000000000003,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":0.26700000000000002},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.2769999999999999,"15min":0.76700000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":19.762,"5min":19.762,"15min":19.762},"last":0.48199999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.3280000000000001,"15min":1.3200000000000001},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.8690000000000002,"5min":12.663,"15min":12.663},"last":1.873},{"interface":"front","average":{"1min":2.4049999999999998,"5min":1.7789999999999999,"15min":1.6020000000000001},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.317},"max":{"1min":6.0289999999999999,"5min":13.222,"15min":13.222},"last":3.7599999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.405,"15min":0.94999999999999996},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":23.190999999999999,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.59799999999999998},{"interface":"front","average":{"1min":4.1719999999999997,"5min":1.7330000000000001,"15min":1.113},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":23.204000000000001,"5min":23.204000000000001,"15min":23.204000000000001},"last":0.61899999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.121,"5min":1.4690000000000001,"15min":0.996},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":23.443000000000001,"5min":23.443000000000001,"15min":23.443000000000001},"last":2.161},{"interface":"front","average":{"1min":3.7970000000000002,"5min":1.746,"15min":1.099},"min":{"1min":0.42199999999999999,"5min":0.378,"15min":0.378},"max":{"1min":26.177,"5min":26.177,"15min":26.177},"last":3.5009999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.4340000000000002,"5min":1.3620000000000001,"15min":0.90800000000000003},"min":{"1min":0.42299999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.1050000000000004,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":1.3420000000000001},{"interface":"front","average":{"1min":2.3140000000000001,"5min":1.474,"15min":1.028},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":9.1500000000000004,"5min":9.1500000000000004,"15min":9.1500000000000004},"last":0.58799999999999997}]}]},{"osd":0,"up_from":93,"seq":399431958856,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3343588,"kb_used_data":2420316,"kb_used_omap":7,"kb_used_meta":923192,"kb_avail":90401564,"statfs":{"total":95995035648,"available":92571201536,"internally_reserved":0,"allocated":2478403584,"data_stored":2475551549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":945349348},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.044,"15min":0.56100000000000005},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.4779999999999998,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":3.3740000000000001},{"interface":"front","average":{"1min":2.6240000000000001,"5min":1.278,"15min":0.77400000000000002},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.335000000000001,"5min":15.335000000000001,"15min":15.335000000000001},"last":0.621}]},{"osd":2,"last update":"Sat Sep 23 18:20:04 2023","interfaces":[{"interface":"back","average":{"1min":3.911,"5min":1.71,"15min":1.3160000000000001},"min":{"1min":0.48999999999999999,"5min":0.185,"15min":0.185},"max":{"1min":15.002000000000001,"5min":18.550999999999998,"15min":18.550999999999998},"last":2.2229999999999999},{"interface":"front","average":{"1min":2.8530000000000002,"5min":1.258,"15min":0.93400000000000005},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":11.644,"5min":13.618,"15min":13.618},"last":2.8279999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7610000000000001,"5min":1.393,"15min":1.637},"min":{"1min":0.38400000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.539999999999999,"5min":13.539999999999999,"15min":13.539999999999999},"last":3.2919999999999998},{"interface":"front","average":{"1min":4.1390000000000002,"5min":1.4930000000000001,"15min":1.637},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.285,"5min":13.285,"15min":15.198},"last":3.4020000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7010000000000001,"5min":2.1309999999999998,"15min":1.8979999999999999},"min":{"1min":0.41799999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":17.741,"5min":17.741,"15min":57.746000000000002},"last":3.3180000000000001},{"interface":"front","average":{"1min":3.79,"5min":1.9159999999999999,"15min":1.7809999999999999},"min":{"1min":0.50900000000000001,"5min":0.26900000000000002,"15min":0.25600000000000001},"max":{"1min":15.079000000000001,"5min":15.079000000000001,"15min":57.816000000000003},"last":2.3530000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.032,"15min":1.391},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":3.3450000000000002},{"interface":"front","average":{"1min":4.2859999999999996,"5min":2.2650000000000001,"15min":1.448},"min":{"1min":0.502,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":13.425000000000001},"last":2.2770000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":1.948,"15min":1.9530000000000001},"min":{"1min":0.47499999999999998,"5min":0.377,"15min":0.32000000000000001},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":58.457999999999998},"last":5.0590000000000002},{"interface":"front","average":{"1min":4.6639999999999997,"5min":2.3159999999999998,"15min":1.9299999999999999},"min":{"1min":0.497,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":18.643999999999998,"5min":18.643999999999998,"15min":58.354999999999997},"last":1.0149999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":1.923,"15min":1.639},"min":{"1min":0.46899999999999997,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":13.644,"5min":13.644,"15min":13.644},"last":1.155},{"interface":"front","average":{"1min":3.5179999999999998,"5min":1.821,"15min":1.8},"min":{"1min":0.55900000000000005,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.7519999999999998,"5min":10.404,"15min":10.404},"last":3.2480000000000002}]}]},{"osd":2,"up_from":0,"seq":2070174236741,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249371,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1990204,"kb_used_data":1277492,"kb_used_omap":3,"kb_used_meta":712700,"kb_avail":91754948,"statfs":{"total":95995035648,"available":93957066752,"internally_reserved":0,"allocated":1308151808,"data_stored":1305101339,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":729805544},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3919999999999999,"5min":1.25,"15min":1.7050000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.183},"max":{"1min":17.036000000000001,"5min":17.036000000000001,"15min":195.69800000000001},"last":9.8930000000000007},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.1599999999999999,"15min":1.96},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.0980000000000008,"5min":9.0980000000000008,"15min":194.661},"last":2.0379999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:20:26 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":1.4590000000000001,"15min":0.83299999999999996},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.079000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":2.9209999999999998},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.5129999999999999,"15min":1.3069999999999999},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.081,"5min":78.980000000000004,"15min":78.980000000000004},"last":4.9290000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":1.274,"15min":1.29},"min":{"1min":0.27400000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.151999999999999,"5min":12.151999999999999,"15min":12.151999999999999},"last":2.956},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.298,"15min":1.3320000000000001},"min":{"1min":0.318,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.565,"5min":11.565,"15min":11.565},"last":4.9900000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.6140000000000001,"15min":1.659},"min":{"1min":0.31900000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2300000000000004,"5min":21.062999999999999,"15min":37.231000000000002},"last":1.857},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.9810000000000001,"15min":1.8939999999999999},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.2959999999999994,"5min":21.128,"15min":37.381},"last":0.50600000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:36 2023","interfaces":[{"interface":"back","average":{"1min":2.8340000000000001,"5min":1.8009999999999999,"15min":1.4419999999999999},"min":{"1min":0.40999999999999998,"5min":0.316,"15min":0.316},"max":{"1min":11.805999999999999,"5min":11.805999999999999,"15min":11.805999999999999},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.028,"5min":1.601,"15min":1.6339999999999999},"min":{"1min":0.44500000000000001,"5min":0.33100000000000002,"15min":0.32000000000000001},"max":{"1min":8.9359999999999999,"5min":8.9359999999999999,"15min":8.9969999999999999},"last":4.8200000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":1.7809999999999999,"15min":1.847},"min":{"1min":0.432,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":17.192,"5min":17.192,"15min":115.54000000000001},"last":4.8710000000000004},{"interface":"front","average":{"1min":2.2530000000000001,"5min":1.7250000000000001,"15min":1.948},"min":{"1min":0.37,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":8.0350000000000001,"5min":14.321,"15min":115.691},"last":1.8919999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.5069999999999999,"15min":1.7110000000000001},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":8.2070000000000007,"5min":8.2070000000000007,"15min":15.972},"last":1.8129999999999999},{"interface":"front","average":{"1min":3.5920000000000001,"5min":1.7629999999999999,"15min":1.623},"min":{"1min":0.48799999999999999,"5min":0.41999999999999998,"15min":0.34899999999999998},"max":{"1min":17.164000000000001,"5min":17.164000000000001,"15min":17.164000000000001},"last":4.9009999999999998}]}]},{"osd":4,"up_from":30,"seq":128849019321,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3457772,"kb_used_data":2787300,"kb_used_omap":0,"kb_used_meta":670400,"kb_avail":90287380,"statfs":{"total":95995035648,"available":92454277120,"internally_reserved":0,"allocated":2854195200,"data_stored":2852260640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":686489600},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.534,"15min":1.268},"min":{"1min":0.255,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.6079999999999997,"5min":7.6079999999999997,"15min":13.375999999999999},"last":3.6080000000000001},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.6819999999999999,"15min":1.546},"min":{"1min":0.26600000000000001,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":9.3710000000000004,"5min":10.890000000000001,"15min":10.890000000000001},"last":1.071}]},{"osd":1,"last update":"Sat Sep 23 18:20:22 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":1.7170000000000001,"15min":1.0349999999999999},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8220000000000001,"5min":44.146999999999998,"15min":44.146999999999998},"last":6.8499999999999996},{"interface":"front","average":{"1min":1.8420000000000001,"5min":1.6679999999999999,"15min":0.98999999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.5700000000000003,"5min":43.497999999999998,"15min":43.497999999999998},"last":3.6520000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:19:56 2023","interfaces":[{"interface":"back","average":{"1min":2.621,"5min":1.6699999999999999,"15min":1.8180000000000001},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.896000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":3.7400000000000002},{"interface":"front","average":{"1min":1.6020000000000001,"5min":1.6200000000000001,"15min":1.6930000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":6.6890000000000001,"5min":10.938000000000001,"15min":10.938000000000001},"last":1.093}]},{"osd":3,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.5669999999999999,"15min":1.9350000000000001},"min":{"1min":0.33700000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.0609999999999999,"5min":11.130000000000001,"15min":29.154},"last":6.7519999999999998},{"interface":"front","average":{"1min":2.1800000000000002,"5min":1.8480000000000001,"15min":1.9390000000000001},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":13.470000000000001,"5min":13.470000000000001,"15min":29.216000000000001},"last":3.754}]},{"osd":5,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":2.4279999999999999,"15min":2.8559999999999999},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4950000000000001,"5min":27.675000000000001,"15min":27.675000000000001},"last":1.0409999999999999},{"interface":"front","average":{"1min":1.401,"5min":1.827,"15min":2.7109999999999999},"min":{"1min":0.30499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.1040000000000001,"5min":13.938000000000001,"15min":32.640000000000001},"last":3.8359999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.873,"15min":1.347},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":6.1989999999999998,"5min":18.315000000000001,"15min":18.315000000000001},"last":3.8090000000000002},{"interface":"front","average":{"1min":1.859,"5min":1.9630000000000001,"15min":1.522},"min":{"1min":0.38700000000000001,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":9.5579999999999998,"5min":18.431000000000001,"15min":18.431000000000001},"last":7.4969999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":2.4849999999999999,"15min":1.774},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":5.9539999999999997,"5min":19.298999999999999,"15min":19.298999999999999},"last":6.6970000000000001},{"interface":"front","average":{"1min":1.722,"5min":1.8899999999999999,"15min":1.552},"min":{"1min":0.435,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.4730000000000008,"5min":12.800000000000001,"15min":12.800000000000001},"last":3.786}]}]},{"osd":5,"up_from":420,"seq":1803886264417,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2270492,"kb_used_data":1918612,"kb_used_omap":9,"kb_used_meta":351862,"kb_avail":91474660,"statfs":{"total":95995035648,"available":93670051840,"internally_reserved":0,"allocated":1964658688,"data_stored":1962736472,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":360307133},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":1.2549999999999999,"15min":0.88100000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.6000000000000001,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.41499999999999998},{"interface":"front","average":{"1min":2.911,"5min":1.8149999999999999,"15min":1.6080000000000001},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":20.937999999999999,"5min":20.937999999999999,"15min":20.937999999999999},"last":5.7460000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":1.7070000000000001,"15min":1.2989999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.6649999999999991,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.49299999999999999},{"interface":"front","average":{"1min":2.1840000000000002,"5min":1.73,"15min":1.141},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.5259999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":1.8300000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:19:52 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":1.629,"15min":1.615},"min":{"1min":0.28699999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":10.705,"5min":10.705,"15min":10.705},"last":0.51700000000000002},{"interface":"front","average":{"1min":2.5409999999999999,"5min":1.946,"15min":2.1600000000000001},"min":{"1min":0.373,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.8099999999999996,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":1.7410000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":1.8740000000000001,"15min":1.2789999999999999},"min":{"1min":0.443,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.8780000000000001,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":6.0060000000000002},{"interface":"front","average":{"1min":3.7320000000000002,"5min":2.0640000000000001,"15min":1.665},"min":{"1min":0.38100000000000001,"5min":0.315,"15min":0.313},"max":{"1min":20.035,"5min":20.035,"15min":20.035},"last":1.228}]},{"osd":4,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.2770000000000001,"15min":2.0640000000000001},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":7.875,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.27600000000000002},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.0899999999999999,"15min":1.6160000000000001},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7610000000000001,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.53800000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":1.9650000000000001,"15min":1.2609999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2480000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":2.343,"5min":2.2040000000000002,"15min":1.6819999999999999},"min":{"1min":0.34599999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.8410000000000002,"5min":12.504,"15min":12.504},"last":1.1060000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.3980000000000001,"15min":1.758},"min":{"1min":0.40699999999999997,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.528,"5min":13.506,"15min":13.506},"last":0.34599999999999997},{"interface":"front","average":{"1min":1.381,"5min":1.9630000000000001,"15min":1.542},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":5.0970000000000004,"5min":13.138,"15min":13.138},"last":0.45800000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2440237056,"data_stored":2440232984,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1559166976,"data_stored":1559166976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1297678336,"data_stored":1297678336,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2817921024,"data_stored":2817916952,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1941766144,"data_stored":1941766144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1789792256,"data_stored":1789788184,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:20:33.767 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:20:33.795 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:33.976 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:33.984 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:33.984 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:33.994 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:34.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:33 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:34.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:33 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:20:34.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:33 smithi099 ceph-mon[167076]: osdmap e613: 8 total, 8 up, 6 in 2023-09-23T18:20:34.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:33 smithi099 ceph-mon[167076]: failed to encode map e613 with expected crc 2023-09-23T18:20:34.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 16502/166542 objects degraded (9.909%), 7 pgs degraded, 6 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:34.086 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:34.133 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:34.185 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:34.190 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:34.190 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:34.291 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:34.346 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:34.389 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:34.401 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:34.401 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:34.502 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:34.580 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:34.788 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:20:34.831 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:34 smithi099 ceph-mon[167076]: pgmap v1470: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 991 op/s; 16502/158583 objects degraded (10.406%); 50/158583 objects misplaced (0.032%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T18:20:34.832 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:34 smithi099 ceph-mon[167076]: from='client.47063 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:34.832 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:20:34.872 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:20:34.880 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:34.895 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:34.896 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:34 smithi055 ceph-mon[179668]: pgmap v1470: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 991 op/s; 16502/158583 objects degraded (10.406%); 50/158583 objects misplaced (0.032%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T18:20:34.896 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:34 smithi055 ceph-mon[179668]: from='client.47063 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:34.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:34 smithi055 ceph-mon[184174]: pgmap v1470: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 15 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 991 op/s; 16502/158583 objects degraded (10.406%); 50/158583 objects misplaced (0.032%); 5.3 MiB/s, 85 objects/s recovering 2023-09-23T18:20:34.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:34 smithi055 ceph-mon[184174]: from='client.47063 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:34.897 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:34.924 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:20:34.971 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:20:34.998 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:35.016 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:20:35.067 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:20:35.067 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:35.094 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:35.100 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:20:35.134 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:20:35.134 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:20:35.134 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:20:35.134 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:20:35.134 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:20:35.134 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:20:35.134 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:20:35.134 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:20:35.181 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:35.183 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:20:35.188 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:35.189 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:35.290 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:35.397 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:35.745 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:20:35.745 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:20:35.745 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:20:35.746 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:20:35.746 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:20:35.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:35 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4013285003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:35.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:35 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4013285003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:36.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:35 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4013285003' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:36.190 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:36.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:36.194 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:36.296 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:36.324 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:36.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[179668]: pgmap v1471: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 891 op/s; 16502/157701 objects degraded (10.464%); 50/157701 objects misplaced (0.032%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T18:20:36.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[179668]: failed to encode map e614 with expected crc 2023-09-23T18:20:36.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[179668]: osdmap e614: 8 total, 8 up, 6 in 2023-09-23T18:20:36.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[179668]: failed to encode map e614 with expected crc 2023-09-23T18:20:36.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:36.933 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:36.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[184174]: pgmap v1471: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 891 op/s; 16502/157701 objects degraded (10.464%); 50/157701 objects misplaced (0.032%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T18:20:36.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[184174]: failed to encode map e614 with expected crc 2023-09-23T18:20:36.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[184174]: osdmap e614: 8 total, 8 up, 6 in 2023-09-23T18:20:36.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[184174]: failed to encode map e614 with expected crc 2023-09-23T18:20:36.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:36.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:37.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:36 smithi099 ceph-mon[167076]: pgmap v1471: 119 pgs: 1 active+recovery_wait+degraded, 1 active+recovering+undersized+degraded+remapped, 5 active+recovery_wait+undersized+degraded+remapped, 112 active+clean; 3.2 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 891 op/s; 16502/157701 objects degraded (10.464%); 50/157701 objects misplaced (0.032%); 4.6 MiB/s, 74 objects/s recovering 2023-09-23T18:20:37.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:36 smithi099 ceph-mon[167076]: failed to encode map e614 with expected crc 2023-09-23T18:20:37.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:36 smithi099 ceph-mon[167076]: osdmap e614: 8 total, 8 up, 6 in 2023-09-23T18:20:37.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:36 smithi099 ceph-mon[167076]: failed to encode map e614 with expected crc 2023-09-23T18:20:37.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:37.082 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:37.318 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:37.322 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:37.322 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:37.423 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:37.487 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:37.537 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:20:37.673 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:37.678 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:37.678 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:37.779 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:37.866 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:37 smithi055 ceph-mon[184174]: failed to encode map e615 with expected crc 2023-09-23T18:20:37.866 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:37 smithi055 ceph-mon[184174]: osdmap e615: 8 total, 8 up, 6 in 2023-09-23T18:20:37.866 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:37 smithi055 ceph-mon[184174]: failed to encode map e615 with expected crc 2023-09-23T18:20:37.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:37 smithi055 ceph-mon[179668]: failed to encode map e615 with expected crc 2023-09-23T18:20:37.866 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:37 smithi055 ceph-mon[179668]: osdmap e615: 8 total, 8 up, 6 in 2023-09-23T18:20:37.867 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:37 smithi055 ceph-mon[179668]: failed to encode map e615 with expected crc 2023-09-23T18:20:37.876 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:37.951 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:37.955 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:37.955 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:37.955 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:38.056 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:38.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:37 smithi099 ceph-mon[167076]: failed to encode map e615 with expected crc 2023-09-23T18:20:38.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:37 smithi099 ceph-mon[167076]: osdmap e615: 8 total, 8 up, 6 in 2023-09-23T18:20:38.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:37 smithi099 ceph-mon[167076]: failed to encode map e615 with expected crc 2023-09-23T18:20:38.136 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:38.203 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:38.210 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:38.210 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:38.219 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:20:38.311 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:38.343 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:38.401 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:38.409 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:38.410 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:38.510 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:38.635 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:38.765 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:38.790 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:38.798 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:38.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:38 smithi055 ceph-mon[179668]: pgmap v1474: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 15418/146841 objects degraded (10.500%); 748/146841 objects misplaced (0.509%); 6.1 MiB/s, 131 objects/s recovering 2023-09-23T18:20:38.799 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15418/146841 objects degraded (10.500%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:38.799 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:38 smithi055 ceph-mon[184174]: pgmap v1474: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 15418/146841 objects degraded (10.500%); 748/146841 objects misplaced (0.509%); 6.1 MiB/s, 131 objects/s recovering 2023-09-23T18:20:38.799 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15418/146841 objects degraded (10.500%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:38.800 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:38.901 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:38.960 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:38 smithi099 ceph-mon[167076]: pgmap v1474: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 14 GiB used, 522 GiB / 536 GiB avail; 0 B/s wr, 1.09k op/s; 15418/146841 objects degraded (10.500%); 748/146841 objects misplaced (0.509%); 6.1 MiB/s, 131 objects/s recovering 2023-09-23T18:20:38.960 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15418/146841 objects degraded (10.500%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:38.969 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:39.035 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:20:39.036 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15418/146841 objects degraded (10.500%), 5 pgs degraded, 5 pgs undersized","count":10},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1728,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":615,"num_osds":8,"num_up_osds":8,"osd_up_since":1695493039,"num_in_osds":6,"osd_in_since":1695493021,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":113},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+recovering+undersized+degraded+remapped","count":1},{"state_name":"peering","count":1}],"num_pgs":119,"num_pools":5,"num_objects":48947,"data_bytes":3210427068,"bytes_used":15036706816,"bytes_avail":560933507072,"bytes_total":575970213888,"inactive_pgs_ratio":0.0084033617749810219,"degraded_objects":15418,"degraded_total":146841,"degraded_ratio":0.1049979229234342,"misplaced_objects":748,"misplaced_total":146841,"misplaced_ratio":0.0050939451515584887,"recovering_objects_per_sec":131,"recovering_bytes_per_sec":6362762,"recovering_keys_per_sec":0,"num_objects_recovered":790,"num_bytes_recovered":38207488,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":1093},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T18:20:09.153115+0000","services":{}},"progress_events":{"57c27436-1bc8-4f63-b41f-a80b88a05748":{"message":"Global Recovery Event (100s)\n [==========================..] (remaining: 5s)","progress":0.94957983493804932,"add_to_ceph_s":true},"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T18:20:39.050 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:39.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:39.056 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:39.157 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:39.236 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:39.735 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:20:39.736 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:20:39.898 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:39.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:39.904 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:39.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2636780240' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:20:39.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2636780240' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:20:40.005 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:40.063 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2636780240' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:20:40.071 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:40.112 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:40.121 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:40.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:40.127 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:40.228 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:40.300 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:40.457 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:40.465 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:40.466 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:40.559 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:20:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:20:40] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.43.0" 2023-09-23T18:20:40.567 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:40.638 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:40.697 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:40.706 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:40.707 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:40.784 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:20:40.808 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:40.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:40 smithi055 ceph-mon[184174]: pgmap v1475: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 668 op/s; 15418/145329 objects degraded (10.609%); 748/145329 objects misplaced (0.515%); 1.7 MiB/s, 58 objects/s recovering 2023-09-23T18:20:40.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3222792220' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:40.833 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:40 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:40.833 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:40 smithi055 ceph-mon[179668]: pgmap v1475: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 668 op/s; 15418/145329 objects degraded (10.609%); 748/145329 objects misplaced (0.515%); 1.7 MiB/s, 58 objects/s recovering 2023-09-23T18:20:40.833 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3222792220' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:40.833 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:40 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:40.888 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:41.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:40 smithi099 ceph-mon[167076]: pgmap v1475: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 3.0 GiB data, 14 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 668 op/s; 15418/145329 objects degraded (10.609%); 748/145329 objects misplaced (0.515%); 1.7 MiB/s, 58 objects/s recovering 2023-09-23T18:20:41.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3222792220' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:41.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:40 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:20:41.134 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:20:41.134 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:20:41.134 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:20:41.134 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:126230 8.8 8:1237b1d1:::benchmark_data_smithi055_346788_object15638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:24.368296+0000", 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 556.51404515800004, 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.418939742, 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 126230 2023-09-23T18:20:41.135 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368296+0000", 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368296+0000", 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.136 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368294+0000", 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368300+0000", 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9560000000000002e-06 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.137 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368318+0000", 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7795000000000002e-05 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.368328+0000", 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4790000000000002e-06 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.138 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.457382+0000", 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.089054340999999995 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.457459+0000", 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6725999999999997e-05 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.692919+0000", 2023-09-23T18:20:41.139 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.235460265 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.693016+0000", 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6869999999999999e-05 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.693104+0000", 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7700000000000004e-05 2023-09-23T18:20:41.140 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.718266+0000", 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.025162766 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.783487+0000", 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.065220847999999998 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.141 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.783537+0000", 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0043000000000002e-05 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.783574+0000", 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6875000000000002e-05 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.142 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.787236+0000", 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003661756 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:126237 8.8 8:1cdbad3c:::benchmark_data_smithi055_346788_object15645:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:24.377458+0000", 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 556.50488375899999, 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.48202789600000001, 2023-09-23T18:20:41.143 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 126237 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377458+0000", 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.144 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377458+0000", 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377456+0000", 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.145 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377463+0000", 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.7309999999999996e-06 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377476+0000", 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2894999999999999e-05 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.146 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.377483+0000", 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.374e-06 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.517115+0000", 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.13963230200000001 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.517177+0000", 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1897999999999996e-05 2023-09-23T18:20:41.147 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.825406+0000", 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.30822874099999997 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.825506+0000", 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010000099999999999 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.825640+0000", 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013374700000000001 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.830736+0000", 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.005096125 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.855124+0000", 2023-09-23T18:20:41.149 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.024388184 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.855433+0000", 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00030889699999999998 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.855497+0000", 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4228999999999998e-05 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.150 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.859485+0000", 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039881949999999999 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:126259 8.8 8:15b6e0af:::benchmark_data_smithi055_346788_object15667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e395)", 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:24.416937+0000", 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 556.46540418100005, 2023-09-23T18:20:41.151 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.48747808799999998, 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 126259 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416937+0000", 2023-09-23T18:20:41.152 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416937+0000", 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416936+0000", 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.153 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416943+0000", 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4170000000000003e-06 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416971+0000", 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7552e-05 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.154 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.416979+0000", 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9589999999999995e-06 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.585139+0000", 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.16816076399999999 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.585243+0000", 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000103185 2023-09-23T18:20:41.155 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.840971+0000", 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.25572825999999999 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.841051+0000", 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9952999999999998e-05 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:41.156 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.841149+0000", 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8142999999999999e-05 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.855376+0000", 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014226664999999999 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.855389+0000", 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3842999999999999e-05 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.157 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.900533+0000", 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.045143100999999998 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.900571+0000", 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8325999999999999e-05 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.158 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.904415+0000", 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0038443079999999998 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPushReply(8.8 396/394 [PushReplyOp(8:1cdbad3c:::benchmark_data_smithi055_346788_object15645:head)])", 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:24.520324+0000", 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 556.36201716200003, 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.30352263200000001, 2023-09-23T18:20:41.159 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520324+0000", 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520324+0000", 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.160 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520323+0000", 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520335+0000", 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1651000000000001e-05 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.161 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520354+0000", 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9352e-05 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.520364+0000", 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5079999999999994e-06 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.815627+0000", 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.29526276000000001 2023-09-23T18:20:41.162 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.823847+0000", 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0082201010000000005 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:126427 8.8 8:11ddd72a:::benchmark_data_smithi055_346788_object15835:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:20:41.163 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:24.716103+0000", 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 556.16623837700001, 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.28519597000000002, 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 126427 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.164 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716103+0000", 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716103+0000", 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716101+0000", 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:20:41.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716110+0000", 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1800000000000002e-06 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716133+0000", 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2909999999999999e-05 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.716143+0000", 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9180000000000006e-06 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.815517+0000", 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.099374219999999999 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:20:41.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.815593+0000", 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5104999999999993e-05 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.961822+0000", 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.14622923400000001 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.961937+0000", 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114757 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.962090+0000", 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000153661 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.997809+0000", 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.035718911999999998 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.997828+0000", 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9052e-05 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.997869+0000", 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0373999999999997e-05 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:24.997928+0000", 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9286999999999997e-05 2023-09-23T18:20:41.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.001299+0000", 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033710820000000001 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:126952 8.8 8:18980ef4:::benchmark_data_smithi055_346788_object16360:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:25.392938+0000", 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 555.48940293999999, 2023-09-23T18:20:41.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.28906703900000003, 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 126952 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392938+0000", 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392938+0000", 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392937+0000", 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392942+0000", 2023-09-23T18:20:41.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2089999999999996e-06 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392954+0000", 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2072999999999999e-05 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.392962+0000", 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8159999999999994e-06 2023-09-23T18:20:41.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.449589+0000", 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.056626870000000003 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.449675+0000", 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6088999999999995e-05 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.175 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.608823+0000", 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.159147594 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.608928+0000", 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000105678 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.609073+0000", 2023-09-23T18:20:41.176 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014435500000000001 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.627538+0000", 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018464960999999998 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.627548+0000", 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0257e-05 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.177 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.178 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.678648+0000", 2023-09-23T18:20:41.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.051100356 2023-09-23T18:20:41.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.178 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.178 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.678689+0000", 2023-09-23T18:20:41.178 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1037e-05 2023-09-23T18:20:41.178 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.178 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.181 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.181 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.682005+0000", 2023-09-23T18:20:41.181 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003316041 2023-09-23T18:20:41.181 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.181 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.181 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.181 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.181 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.181 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:127014 8.8 8:1461082a:::benchmark_data_smithi055_346788_object16422:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:25.470801+0000", 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 555.411540124, 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.32388431200000001, 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 127014 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.182 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470801+0000", 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470801+0000", 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.183 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470799+0000", 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470809+0000", 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1560000000000001e-06 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470824+0000", 2023-09-23T18:20:41.184 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5006e-05 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.470838+0000", 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.446e-05 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.533922+0000", 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.063083976 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.185 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.533990+0000", 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8493999999999998e-05 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.752999+0000", 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.21900887999999999 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.753123+0000", 2023-09-23T18:20:41.186 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012370099999999999 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.753310+0000", 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000186629 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.779553+0000", 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026243391000000001 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.187 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.791116+0000", 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011562862 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.791169+0000", 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2855999999999997e-05 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.791227+0000", 2023-09-23T18:20:41.188 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7828000000000003e-05 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.794685+0000", 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0034588449999999999 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "MOSDPGPushReply(8.8 396/394 [PushReplyOp(8:1461082a:::benchmark_data_smithi055_346788_object16422:head)])", 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:25.477212+0000", 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 555.40512932000001, 2023-09-23T18:20:41.189 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.27574391999999998, 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477212+0000", 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477212+0000", 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.190 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477211+0000", 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477219+0000", 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7479999999999993e-06 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.191 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477230+0000", 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1585e-05 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.477239+0000", 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.3669999999999997e-06 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.745071+0000", 2023-09-23T18:20:41.192 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.26783221600000001 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.752956+0000", 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0078849369999999998 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.193 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:127070 8.8 8:14141a6a:::benchmark_data_smithi055_346788_object16478:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:25.535150+0000", 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 555.34719112000005, 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.30308834499999998, 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 127070 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.194 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535150+0000", 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535150+0000", 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.195 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535149+0000", 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535157+0000", 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7619999999999995e-06 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535173+0000", 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.683e-05 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.196 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.535199+0000", 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5420999999999999e-05 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.549349+0000", 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014150107 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.549399+0000", 2023-09-23T18:20:41.197 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0049000000000001e-05 2023-09-23T18:20:41.198 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.198 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.198 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.198 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.810675+0000", 2023-09-23T18:20:41.198 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.26127647399999998 2023-09-23T18:20:41.198 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.198 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.198 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.198 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.810746+0000", 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0378999999999998e-05 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.810830+0000", 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4566000000000003e-05 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.820368+0000", 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0095380090000000001 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.199 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.820382+0000", 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.343e-05 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.834163+0000", 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013781411 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.200 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.834233+0000", 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9759999999999996e-05 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:25.838239+0000", 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040055519999999999 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:128395 8.8 8:10f4d9ca:::benchmark_data_smithi055_346788_object17803:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e396)", 2023-09-23T18:20:41.201 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:11:27.281591+0000", 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 553.60075067299999, 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.29826881199999999, 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 128395 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.202 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281591+0000", 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281591+0000", 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281590+0000", 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.203 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281595+0000", 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5439999999999998e-06 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281608+0000", 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3247999999999999e-05 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.281616+0000", 2023-09-23T18:20:41.204 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7559999999999999e-06 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.482164+0000", 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.20054759799999999 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.482252+0000", 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8307999999999995e-05 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.205 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.508276+0000", 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026023748999999999 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.508349+0000", 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3506999999999999e-05 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.508453+0000", 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000103609 2023-09-23T18:20:41.206 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.527324+0000", 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018870735 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.540632+0000", 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.013307941 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.207 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.576091+0000", 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.035459813 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.576138+0000", 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6461999999999997e-05 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:11:27.579859+0000", 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037215429999999999 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.208 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:173525 8.8 8:11c377cb:::benchmark_data_smithi055_346788_object62933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:27.592880+0000", 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 493.289460894, 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.27587297500000002, 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 173525 2023-09-23T18:20:41.209 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592880+0000", 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592880+0000", 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.210 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592879+0000", 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592884+0000", 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4500000000000003e-06 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592895+0000", 2023-09-23T18:20:41.211 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0366e-05 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592901+0000", 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1600000000000003e-06 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.653327+0000", 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.060425660999999999 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.212 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.653398+0000", 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1755000000000007e-05 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.758477+0000", 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10507913000000001 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.758540+0000", 2023-09-23T18:20:41.213 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2232999999999999e-05 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.758626+0000", 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6231999999999997e-05 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.764380+0000", 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0057542340000000004 2023-09-23T18:20:41.214 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.765155+0000", 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00077486000000000005 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.866705+0000", 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10154997 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.215 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.866758+0000", 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3186000000000002e-05 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.868753+0000", 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019952580000000002 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.216 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:35553 10.5 10:a3327171:::benchmark_data_smithi055_425050_object35552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:17.646314+0000", 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 143.23602722800001, 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18390158100000001, 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 35553 2023-09-23T18:20:41.217 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646314+0000", 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646314+0000", 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.218 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646314+0000", 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646337+0000", 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3941999999999999e-05 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.219 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646345+0000", 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9670000000000001e-06 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646351+0000", 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6500000000000001e-06 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.220 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647569+0000", 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0012183230000000001 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647780+0000", 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021045300000000001 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647914+0000", 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013460100000000001 2023-09-23T18:20:41.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648993+0000", 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001078411 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.653792+0000", 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047994989999999996 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830093+0000", 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.176300967 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830163+0000", 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9437999999999995e-05 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830216+0000", 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2889999999999997e-05 2023-09-23T18:20:41.223 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:35557 10.5 10:a4583da5:::benchmark_data_smithi055_425050_object35556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:17.651246+0000", 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 143.23109507699999, 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.17918794399999999, 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.224 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 35557 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651246+0000", 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651246+0000", 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651245+0000", 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651268+0000", 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3459999999999999e-05 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651278+0000", 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5980000000000003e-06 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651284+0000", 2023-09-23T18:20:41.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.003e-06 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651342+0000", 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8736000000000001e-05 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651537+0000", 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019446700000000001 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.228 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651716+0000", 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017865599999999999 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.653287+0000", 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015712199999999999 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.655801+0000", 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002514077 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.229 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830339+0000", 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.174538633 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830395+0000", 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5553999999999997e-05 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830434+0000", 2023-09-23T18:20:41.230 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9175999999999999e-05 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67163 10.c 10.eeb40c6c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.431909+0000", 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 115.45043206, 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99054996900000003, 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.231 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67163 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431909+0000", 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431909+0000", 2023-09-23T18:20:41.232 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431908+0000", 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431935+0000", 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6780000000000001e-05 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431946+0000", 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1302e-05 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431954+0000", 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4279999999999996e-06 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.432168+0000", 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021302499999999999 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.432177+0000", 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7329999999999998e-06 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.422443+0000", 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99026528199999997 2023-09-23T18:20:41.235 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.422459+0000", 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6671999999999999e-05 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67177 10.c 10.f65fed0c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.442552+0000", 2023-09-23T18:20:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 115.439789639, 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98041694300000004, 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67177 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442552+0000", 2023-09-23T18:20:41.237 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442552+0000", 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442551+0000", 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.238 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442575+0000", 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4281999999999999e-05 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442583+0000", 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3440000000000003e-06 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442588+0000", 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5520000000000004e-06 2023-09-23T18:20:41.239 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.444556+0000", 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019680460000000002 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.444566+0000", 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9280000000000005e-06 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.240 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.422959+0000", 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97839311500000004 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.422969+0000", 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1409999999999994e-06 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.241 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67189 10.c 10.7b7f43bc (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.452426+0000", 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 115.429915588, 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97057495000000005, 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67189 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.242 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452426+0000", 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452426+0000", 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452424+0000", 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:41.243 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452447+0000", 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3203000000000001e-05 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452465+0000", 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7722000000000001e-05 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.244 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452474+0000", 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7290000000000006e-06 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452525+0000", 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0986000000000003e-05 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452537+0000", 2023-09-23T18:20:41.245 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.206e-05 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.422993+0000", 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97045598 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.423001+0000", 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8860000000000004e-06 2023-09-23T18:20:41.246 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67195 10.c 10.e21a187c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.459821+0000", 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 115.42252074300001, 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96321096799999995, 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:20:41.247 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67195 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459821+0000", 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459821+0000", 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.248 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459819+0000", 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459840+0000", 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1480000000000001e-05 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.249 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459855+0000", 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4151000000000001e-05 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459867+0000", 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2549e-05 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459924+0000", 2023-09-23T18:20:41.250 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6986e-05 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459938+0000", 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3679000000000001e-05 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.423024+0000", 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96308652900000002 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.251 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.423032+0000", 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.199e-06 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67196 10.c 10.cbe8354c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.460546+0000", 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 115.421795641, 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96306485200000003, 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.252 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67196 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460546+0000", 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.253 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460546+0000", 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460545+0000", 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.254 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460590+0000", 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5454999999999998e-05 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460608+0000", 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7227999999999998e-05 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460613+0000", 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5670000000000002e-06 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.255 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460655+0000", 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2304e-05 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460662+0000", 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0920000000000003e-06 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.423601+0000", 2023-09-23T18:20:41.256 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96293968399999996 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.423611+0000", 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2909999999999996e-06 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:111865 10.5 10:ad610951:::benchmark_data_smithi055_425050_object8459:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:19:42.351369+0000", 2023-09-23T18:20:41.257 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 58.530972351000003, 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18157957399999999, 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 111865 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351369+0000", 2023-09-23T18:20:41.258 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351369+0000", 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351367+0000", 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.259 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351375+0000", 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4490000000000002e-06 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351390+0000", 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5296999999999999e-05 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351400+0000", 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8679999999999994e-06 2023-09-23T18:20:41.260 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351455+0000", 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5569000000000003e-05 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351570+0000", 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011479199999999999 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:20:41.261 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351671+0000", 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000101233 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.363540+0000", 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011868408 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.364077+0000", 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00053702899999999996 2023-09-23T18:20:41.262 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.532870+0000", 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.16879335200000001 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.532909+0000", 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8705000000000003e-05 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.532949+0000", 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9758000000000001e-05 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:111868 10.5 10:a6bcee6d:::benchmark_data_smithi055_425050_object8462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:19:42.351434+0000", 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 58.530907562000003, 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.181727735, 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:41.264 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 111868 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351434+0000", 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351434+0000", 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351433+0000", 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351437+0000", 2023-09-23T18:20:41.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1520000000000002e-06 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351446+0000", 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2210000000000003e-06 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351454+0000", 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8769999999999993e-06 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.354487+0000", 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030321990000000002 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.354608+0000", 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012146699999999999 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.354723+0000", 2023-09-23T18:20:41.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114887 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.364081+0000", 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093583480000000007 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.526329+0000", 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.162247373 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:41.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.533101+0000", 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006772246 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.533132+0000", 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0898999999999997e-05 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.533161+0000", 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9728000000000001e-05 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.270 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.271 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.271 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:20:41.271 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:20:41.271 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:20:41.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:41.272 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:41.272 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:41.356 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:41.373 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:41.452 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:41.673 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:41.684 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:41.685 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:41.786 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:41.858 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:20:41.868 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:41.965 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:41 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:41.965 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:41 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:20:41.965 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:41 smithi055 ceph-mon[179668]: osdmap e616: 8 total, 8 up, 6 in 2023-09-23T18:20:41.965 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:41 smithi055 ceph-mon[179668]: failed to encode map e616 with expected crc 2023-09-23T18:20:41.965 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:41 smithi055 ceph-mon[179668]: pgmap v1477: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 933 op/s; 15418/140889 objects degraded (10.943%); 498/140889 objects misplaced (0.353%); 4.5 MiB/s, 105 objects/s recovering 2023-09-23T18:20:41.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:41 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:41.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:41 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:20:41.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:41 smithi055 ceph-mon[184174]: osdmap e616: 8 total, 8 up, 6 in 2023-09-23T18:20:41.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:41 smithi055 ceph-mon[184174]: failed to encode map e616 with expected crc 2023-09-23T18:20:41.966 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:41 smithi055 ceph-mon[184174]: pgmap v1477: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 933 op/s; 15418/140889 objects degraded (10.943%); 498/140889 objects misplaced (0.353%); 4.5 MiB/s, 105 objects/s recovering 2023-09-23T18:20:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:41 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:41 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:20:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:41 smithi099 ceph-mon[167076]: osdmap e616: 8 total, 8 up, 6 in 2023-09-23T18:20:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:41 smithi099 ceph-mon[167076]: failed to encode map e616 with expected crc 2023-09-23T18:20:42.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:41 smithi099 ceph-mon[167076]: pgmap v1477: 119 pgs: 1 peering, 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 113 active+clean; 2.9 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 933 op/s; 15418/140889 objects degraded (10.943%); 498/140889 objects misplaced (0.353%); 4.5 MiB/s, 105 objects/s recovering 2023-09-23T18:20:42.088 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:42.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:42.101 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:42.202 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:42.275 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:42.371 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:42.378 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:42.379 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:42.480 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:42.505 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:20:42.544 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:20:42.573 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:20:42.606 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:20:42.641 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:20:42.678 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:42.681 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:20:42.689 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:42.717 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:20:42.750 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:20:42.755 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:20:42.755 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:20:42.755 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:20:42.755 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:20:42.756 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:20:42.756 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:20:42.756 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:20:42.756 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:20:43.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:42 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1250146790' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:43.117 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:42 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1250146790' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:43.119 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:42 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1250146790' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:43.368 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:20:43.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:43 smithi055 ceph-mon[179668]: pgmap v1478: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 983 op/s; 15418/133809 objects degraded (11.522%); 498/133809 objects misplaced (0.372%); 3.6 MiB/s, 84 objects/s recovering 2023-09-23T18:20:43.936 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15418/140889 objects degraded (10.943%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:43.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:43 smithi055 ceph-mon[184174]: pgmap v1478: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 983 op/s; 15418/133809 objects degraded (11.522%); 498/133809 objects misplaced (0.372%); 3.6 MiB/s, 84 objects/s recovering 2023-09-23T18:20:43.937 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15418/140889 objects degraded (10.943%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:43.949 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:43.954 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:43.954 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:44.055 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:44.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:43 smithi099 ceph-mon[167076]: pgmap v1478: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 983 op/s; 15418/133809 objects degraded (11.522%); 498/133809 objects misplaced (0.372%); 3.6 MiB/s, 84 objects/s recovering 2023-09-23T18:20:44.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15418/140889 objects degraded (10.943%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:44.119 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:44.156 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:44.159 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:44.160 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:44.168 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:44.260 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:20:44.275 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:20:44.277 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:20:44.317 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:44.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:44 smithi055 ceph-mon[179668]: from='client.47084 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:44.933 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:44 smithi055 ceph-mon[184174]: from='client.47084 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:44.936 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:20:44.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:44.945 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:20:45.038 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:44 smithi099 ceph-mon[167076]: from='client.47084 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:45.046 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:45.084 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1478,"stamp":"2023-09-23T18:20:43.176828+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2925738684,"num_objects":44603,"num_object_clones":0,"num_object_copies":133809,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15418,"num_objects_misplaced":498,"num_objects_unfound":0,"num_objects_dirty":44603,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":163143,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7637,"num_bytes_recovered":476864137,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83254,"ondisk_log_size":83254,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":13756968,"kb_used_data":10268536,"kb_used_omap":40,"kb_used_meta":3487767,"kb_avail":548713944,"statfs":{"total":575970213888,"available":561883078656,"internally_reserved":0,"allocated":10514980864,"data_stored":10500184163,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3571473722},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,3,4,10,4,10],"upper_bound":64},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-486277120,"num_objects":-7420,"num_object_clones":0,"num_object_copies":-22260,"num_objects_missing_on_primary":-12,"num_objects_missing":-12,"num_objects_degraded":-1084,"num_objects_misplaced":448,"num_objects_unfound":0,"num_objects_dirty":-7420,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7420,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":636,"num_bytes_recovered":28114944,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"7.541259"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":904,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.645966+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:20:36.645966+0000","last_peered":"2023-09-23T18:20:36.645966+0000","last_clean":"2023-09-23T18:20:36.645966+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:20:36.645966+0000","last_undegraded":"2023-09-23T18:20:36.645966+0000","last_fullsized":"2023-09-23T18:20:36.645966+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":275,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650621+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:20:36.650621+0000","last_peered":"2023-09-23T18:20:36.650621+0000","last_clean":"2023-09-23T18:20:36.650621+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:20:36.650621+0000","last_undegraded":"2023-09-23T18:20:36.650621+0000","last_fullsized":"2023-09-23T18:20:36.650621+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":32,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649979+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:20:36.649979+0000","last_peered":"2023-09-23T18:20:36.649979+0000","last_clean":"2023-09-23T18:20:36.649979+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:20:36.649979+0000","last_undegraded":"2023-09-23T18:20:36.649979+0000","last_fullsized":"2023-09-23T18:20:36.649979+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":910,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646385+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:20:36.646385+0000","last_peered":"2023-09-23T18:20:36.646385+0000","last_clean":"2023-09-23T18:20:36.646385+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:20:36.646385+0000","last_undegraded":"2023-09-23T18:20:36.646385+0000","last_fullsized":"2023-09-23T18:20:36.646385+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":357,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649916+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:20:36.649916+0000","last_peered":"2023-09-23T18:20:36.649916+0000","last_clean":"2023-09-23T18:20:36.649916+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:20:36.649916+0000","last_undegraded":"2023-09-23T18:20:36.649916+0000","last_fullsized":"2023-09-23T18:20:36.649916+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":675,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T18:20:35.649346+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:20:35.649346+0000","last_peered":"2023-09-23T18:20:35.649346+0000","last_clean":"2023-09-23T18:20:35.649346+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:20:35.649346+0000","last_undegraded":"2023-09-23T18:20:35.649346+0000","last_fullsized":"2023-09-23T18:20:35.649346+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":927,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.645884+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:20:36.645884+0000","last_peered":"2023-09-23T18:20:36.645884+0000","last_clean":"2023-09-23T18:20:36.645884+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:36.645884+0000","last_undegraded":"2023-09-23T18:20:36.645884+0000","last_fullsized":"2023-09-23T18:20:36.645884+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":32,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649533+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:36.649533+0000","last_peered":"2023-09-23T18:20:36.649533+0000","last_clean":"2023-09-23T18:20:36.649533+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:20:36.649533+0000","last_undegraded":"2023-09-23T18:20:36.649533+0000","last_fullsized":"2023-09-23T18:20:36.649533+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":192,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649942+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:20:36.649942+0000","last_peered":"2023-09-23T18:20:36.649942+0000","last_clean":"2023-09-23T18:20:36.649942+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:20:36.649942+0000","last_undegraded":"2023-09-23T18:20:36.649942+0000","last_fullsized":"2023-09-23T18:20:36.649942+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1050,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646334+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:20:36.646334+0000","last_peered":"2023-09-23T18:20:36.646334+0000","last_clean":"2023-09-23T18:20:36.646334+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:20:36.646334+0000","last_undegraded":"2023-09-23T18:20:36.646334+0000","last_fullsized":"2023-09-23T18:20:36.646334+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":126,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646779+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:20:36.646779+0000","last_peered":"2023-09-23T18:20:36.646779+0000","last_clean":"2023-09-23T18:20:36.646779+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:20:36.646779+0000","last_undegraded":"2023-09-23T18:20:36.646779+0000","last_fullsized":"2023-09-23T18:20:36.646779+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":206,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.759300+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:20:40.759300+0000","last_peered":"2023-09-23T18:20:40.759300+0000","last_clean":"2023-09-23T18:20:40.759300+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:20:40.759300+0000","last_undegraded":"2023-09-23T18:20:40.759300+0000","last_fullsized":"2023-09-23T18:20:40.759300+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":827,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647731+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:20:36.647731+0000","last_peered":"2023-09-23T18:20:36.647731+0000","last_clean":"2023-09-23T18:20:36.647731+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:20:36.647731+0000","last_undegraded":"2023-09-23T18:20:36.647731+0000","last_fullsized":"2023-09-23T18:20:36.647731+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":162,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646642+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:20:36.646642+0000","last_peered":"2023-09-23T18:20:36.646642+0000","last_clean":"2023-09-23T18:20:36.646642+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:20:36.646642+0000","last_undegraded":"2023-09-23T18:20:36.646642+0000","last_fullsized":"2023-09-23T18:20:36.646642+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":161,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647908+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:20:36.647908+0000","last_peered":"2023-09-23T18:20:36.647908+0000","last_clean":"2023-09-23T18:20:36.647908+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:20:36.647908+0000","last_undegraded":"2023-09-23T18:20:36.647908+0000","last_fullsized":"2023-09-23T18:20:36.647908+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":961,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646830+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:20:36.646830+0000","last_peered":"2023-09-23T18:20:36.646830+0000","last_clean":"2023-09-23T18:20:36.646830+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:20:36.646830+0000","last_undegraded":"2023-09-23T18:20:36.646830+0000","last_fullsized":"2023-09-23T18:20:36.646830+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":410,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650358+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:20:36.650358+0000","last_peered":"2023-09-23T18:20:36.650358+0000","last_clean":"2023-09-23T18:20:36.650358+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:20:36.650358+0000","last_undegraded":"2023-09-23T18:20:36.650358+0000","last_fullsized":"2023-09-23T18:20:36.650358+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":429,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648719+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:20:36.648719+0000","last_peered":"2023-09-23T18:20:36.648719+0000","last_clean":"2023-09-23T18:20:36.648719+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:20:36.648719+0000","last_undegraded":"2023-09-23T18:20:36.648719+0000","last_fullsized":"2023-09-23T18:20:36.648719+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":989,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T18:20:35.657698+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:20:35.657698+0000","last_peered":"2023-09-23T18:20:35.657698+0000","last_clean":"2023-09-23T18:20:35.657698+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:20:35.657698+0000","last_undegraded":"2023-09-23T18:20:35.657698+0000","last_fullsized":"2023-09-23T18:20:35.657698+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":168,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648360+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:20:36.648360+0000","last_peered":"2023-09-23T18:20:36.648360+0000","last_clean":"2023-09-23T18:20:36.648360+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:20:36.648360+0000","last_undegraded":"2023-09-23T18:20:36.648360+0000","last_fullsized":"2023-09-23T18:20:36.648360+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":435,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648771+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:20:36.648771+0000","last_peered":"2023-09-23T18:20:36.648771+0000","last_clean":"2023-09-23T18:20:36.648771+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:20:36.648771+0000","last_undegraded":"2023-09-23T18:20:36.648771+0000","last_fullsized":"2023-09-23T18:20:36.648771+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":842,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646061+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:20:36.646061+0000","last_peered":"2023-09-23T18:20:36.646061+0000","last_clean":"2023-09-23T18:20:36.646061+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:20:36.646061+0000","last_undegraded":"2023-09-23T18:20:36.646061+0000","last_fullsized":"2023-09-23T18:20:36.646061+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":435,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648694+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:20:36.648694+0000","last_peered":"2023-09-23T18:20:36.648694+0000","last_clean":"2023-09-23T18:20:36.648694+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:20:36.648694+0000","last_undegraded":"2023-09-23T18:20:36.648694+0000","last_fullsized":"2023-09-23T18:20:36.648694+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":113,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646043+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:20:36.646043+0000","last_peered":"2023-09-23T18:20:36.646043+0000","last_clean":"2023-09-23T18:20:36.646043+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:20:36.646043+0000","last_undegraded":"2023-09-23T18:20:36.646043+0000","last_fullsized":"2023-09-23T18:20:36.646043+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":813,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.645684+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:20:36.645684+0000","last_peered":"2023-09-23T18:20:36.645684+0000","last_clean":"2023-09-23T18:20:36.645684+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:20:36.645684+0000","last_undegraded":"2023-09-23T18:20:36.645684+0000","last_fullsized":"2023-09-23T18:20:36.645684+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":130,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T18:20:35.650014+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:20:35.650014+0000","last_peered":"2023-09-23T18:20:35.650014+0000","last_clean":"2023-09-23T18:20:35.650014+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:20:35.650014+0000","last_undegraded":"2023-09-23T18:20:35.650014+0000","last_fullsized":"2023-09-23T18:20:35.650014+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":126,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646618+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:20:36.646618+0000","last_peered":"2023-09-23T18:20:36.646618+0000","last_clean":"2023-09-23T18:20:36.646618+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:20:36.646618+0000","last_undegraded":"2023-09-23T18:20:36.646618+0000","last_fullsized":"2023-09-23T18:20:36.646618+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":751,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649889+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:20:36.649889+0000","last_peered":"2023-09-23T18:20:36.649889+0000","last_clean":"2023-09-23T18:20:36.649889+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:20:36.649889+0000","last_undegraded":"2023-09-23T18:20:36.649889+0000","last_fullsized":"2023-09-23T18:20:36.649889+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":203,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647778+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:20:36.647778+0000","last_peered":"2023-09-23T18:20:36.647778+0000","last_clean":"2023-09-23T18:20:36.647778+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:20:36.647778+0000","last_undegraded":"2023-09-23T18:20:36.647778+0000","last_fullsized":"2023-09-23T18:20:36.647778+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":214,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650779+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:20:36.650779+0000","last_peered":"2023-09-23T18:20:36.650779+0000","last_clean":"2023-09-23T18:20:36.650779+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:20:36.650779+0000","last_undegraded":"2023-09-23T18:20:36.650779+0000","last_fullsized":"2023-09-23T18:20:36.650779+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":847,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647641+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:20:36.647641+0000","last_peered":"2023-09-23T18:20:36.647641+0000","last_clean":"2023-09-23T18:20:36.647641+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:20:36.647641+0000","last_undegraded":"2023-09-23T18:20:36.647641+0000","last_fullsized":"2023-09-23T18:20:36.647641+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":756,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647535+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:20:36.647535+0000","last_peered":"2023-09-23T18:20:36.647535+0000","last_clean":"2023-09-23T18:20:36.647535+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:20:36.647535+0000","last_undegraded":"2023-09-23T18:20:36.647535+0000","last_fullsized":"2023-09-23T18:20:36.647535+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":32,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650148+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:20:36.650148+0000","last_peered":"2023-09-23T18:20:36.650148+0000","last_clean":"2023-09-23T18:20:36.650148+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:20:36.650148+0000","last_undegraded":"2023-09-23T18:20:36.650148+0000","last_fullsized":"2023-09-23T18:20:36.650148+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"616'10278","reported_seq":10327,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:41.153548+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:20:41.153548+0000","last_peered":"2023-09-23T18:20:41.153548+0000","last_clean":"2023-09-23T18:20:41.153548+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:20:41.153548+0000","last_undegraded":"2023-09-23T18:20:41.153548+0000","last_fullsized":"2023-09-23T18:20:41.153548+0000","mapping_epoch":574,"log_start":"600'4800","ondisk_log_start":"600'4800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5478,"log_dups_size":0,"ondisk_log_size":5478,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183238656,"num_objects":2796,"num_object_clones":0,"num_object_copies":8388,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2796,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10278,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":881,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648003+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:20:36.648003+0000","last_peered":"2023-09-23T18:20:36.648003+0000","last_clean":"2023-09-23T18:20:36.648003+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:20:36.648003+0000","last_undegraded":"2023-09-23T18:20:36.648003+0000","last_fullsized":"2023-09-23T18:20:36.648003+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":252,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650819+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:20:36.650819+0000","last_peered":"2023-09-23T18:20:36.650819+0000","last_clean":"2023-09-23T18:20:36.650819+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:20:36.650819+0000","last_undegraded":"2023-09-23T18:20:36.650819+0000","last_fullsized":"2023-09-23T18:20:36.650819+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":126,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.645684+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:20:36.645684+0000","last_peered":"2023-09-23T18:20:36.645684+0000","last_clean":"2023-09-23T18:20:36.645684+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:20:36.645684+0000","last_undegraded":"2023-09-23T18:20:36.645684+0000","last_fullsized":"2023-09-23T18:20:36.645684+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"616'10002","reported_seq":10062,"reported_epoch":616,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:41.283650+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:20:41.283650+0000","last_peered":"2023-09-23T18:20:41.283650+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:20:41.283650+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"602'5500","ondisk_log_start":"602'5500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4502,"log_dups_size":0,"ondisk_log_size":4502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":176291840,"num_objects":2690,"num_object_clones":0,"num_object_copies":8070,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2690,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10002,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"4,5,6","objects":492},{"shards":"5,6","objects":-1458}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":735,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650024+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:20:36.650024+0000","last_peered":"2023-09-23T18:20:36.650024+0000","last_clean":"2023-09-23T18:20:36.650024+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:20:36.650024+0000","last_undegraded":"2023-09-23T18:20:36.650024+0000","last_fullsized":"2023-09-23T18:20:36.650024+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":253,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.651399+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:20:36.651399+0000","last_peered":"2023-09-23T18:20:36.651399+0000","last_clean":"2023-09-23T18:20:36.651399+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:20:36.651399+0000","last_undegraded":"2023-09-23T18:20:36.651399+0000","last_fullsized":"2023-09-23T18:20:36.651399+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":232,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650505+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:20:36.650505+0000","last_peered":"2023-09-23T18:20:36.650505+0000","last_clean":"2023-09-23T18:20:36.650505+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:20:36.650505+0000","last_undegraded":"2023-09-23T18:20:36.650505+0000","last_fullsized":"2023-09-23T18:20:36.650505+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"615'10040","reported_seq":17049,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.244728+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:20:40.244728+0000","last_peered":"2023-09-23T18:20:40.244728+0000","last_clean":"2023-09-23T18:20:40.244728+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:20:40.244728+0000","last_undegraded":"2023-09-23T18:20:40.244728+0000","last_fullsized":"2023-09-23T18:20:40.244728+0000","mapping_epoch":605,"log_start":"602'5200","ondisk_log_start":"602'5200","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4840,"log_dups_size":0,"ondisk_log_size":4840,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183762944,"num_objects":2804,"num_object_clones":0,"num_object_copies":8412,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2804,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10040,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":631,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649865+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:20:36.649865+0000","last_peered":"2023-09-23T18:20:36.649865+0000","last_clean":"2023-09-23T18:20:36.649865+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:20:36.649865+0000","last_undegraded":"2023-09-23T18:20:36.649865+0000","last_fullsized":"2023-09-23T18:20:36.649865+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":541,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646433+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:20:36.646433+0000","last_peered":"2023-09-23T18:20:36.646433+0000","last_clean":"2023-09-23T18:20:36.646433+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:20:36.646433+0000","last_undegraded":"2023-09-23T18:20:36.646433+0000","last_fullsized":"2023-09-23T18:20:36.646433+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":192,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648608+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:20:36.648608+0000","last_peered":"2023-09-23T18:20:36.648608+0000","last_clean":"2023-09-23T18:20:36.648608+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:20:36.648608+0000","last_undegraded":"2023-09-23T18:20:36.648608+0000","last_fullsized":"2023-09-23T18:20:36.648608+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"616'10294","reported_seq":10346,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:41.299993+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:20:41.299993+0000","last_peered":"2023-09-23T18:20:41.299993+0000","last_clean":"2023-09-23T18:20:41.299993+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:20:41.299993+0000","last_undegraded":"2023-09-23T18:20:41.299993+0000","last_fullsized":"2023-09-23T18:20:41.299993+0000","mapping_epoch":574,"log_start":"601'5200","ondisk_log_start":"601'5200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5094,"log_dups_size":0,"ondisk_log_size":5094,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183369728,"num_objects":2798,"num_object_clones":0,"num_object_copies":8394,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2798,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10294,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":647,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650864+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:20:36.650864+0000","last_peered":"2023-09-23T18:20:36.650864+0000","last_clean":"2023-09-23T18:20:36.650864+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:20:36.650864+0000","last_undegraded":"2023-09-23T18:20:36.650864+0000","last_fullsized":"2023-09-23T18:20:36.650864+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":140,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646347+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:20:36.646347+0000","last_peered":"2023-09-23T18:20:36.646347+0000","last_clean":"2023-09-23T18:20:36.646347+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:20:36.646347+0000","last_undegraded":"2023-09-23T18:20:36.646347+0000","last_fullsized":"2023-09-23T18:20:36.646347+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":452,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T18:20:35.649368+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:20:35.649368+0000","last_peered":"2023-09-23T18:20:35.649368+0000","last_clean":"2023-09-23T18:20:35.649368+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:35.649368+0000","last_undegraded":"2023-09-23T18:20:35.649368+0000","last_fullsized":"2023-09-23T18:20:35.649368+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"615'10138","reported_seq":10185,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:38.875373+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:20:38.875373+0000","last_peered":"2023-09-23T18:20:38.875373+0000","last_clean":"2023-09-23T18:20:38.875373+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:20:38.875373+0000","last_undegraded":"2023-09-23T18:20:38.875373+0000","last_fullsized":"2023-09-23T18:20:38.875373+0000","mapping_epoch":574,"log_start":"603'5700","ondisk_log_start":"603'5700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4438,"log_dups_size":0,"ondisk_log_size":4438,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190447616,"num_objects":2906,"num_object_clones":0,"num_object_copies":8718,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2906,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10138,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1098,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.645875+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:20:36.645875+0000","last_peered":"2023-09-23T18:20:36.645875+0000","last_clean":"2023-09-23T18:20:36.645875+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:20:36.645875+0000","last_undegraded":"2023-09-23T18:20:36.645875+0000","last_fullsized":"2023-09-23T18:20:36.645875+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":353,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650084+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:20:36.650084+0000","last_peered":"2023-09-23T18:20:36.650084+0000","last_clean":"2023-09-23T18:20:36.650084+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:20:36.650084+0000","last_undegraded":"2023-09-23T18:20:36.650084+0000","last_fullsized":"2023-09-23T18:20:36.650084+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":160,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648107+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:20:36.648107+0000","last_peered":"2023-09-23T18:20:36.648107+0000","last_clean":"2023-09-23T18:20:36.648107+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:20:36.648107+0000","last_undegraded":"2023-09-23T18:20:36.648107+0000","last_fullsized":"2023-09-23T18:20:36.648107+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"616'10231","reported_seq":10921,"reported_epoch":616,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:41.154059+0000","last_change":"2023-09-23T18:20:34.894639+0000","last_active":"2023-09-23T18:20:41.154059+0000","last_peered":"2023-09-23T18:20:41.154059+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:20:41.154059+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"600'4700","ondisk_log_start":"600'4700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5531,"log_dups_size":0,"ondisk_log_size":5531,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":183435264,"num_objects":2799,"num_object_clones":0,"num_object_copies":8397,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3413,"num_objects_misplaced":498,"num_objects_unfound":0,"num_objects_dirty":2799,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10231,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":315,"num_bytes_recovered":20643840,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"0,4,5","objects":498},{"shards":"4,5","objects":-1112}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":885,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.645869+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:20:36.645869+0000","last_peered":"2023-09-23T18:20:36.645869+0000","last_clean":"2023-09-23T18:20:36.645869+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:20:36.645869+0000","last_undegraded":"2023-09-23T18:20:36.645869+0000","last_fullsized":"2023-09-23T18:20:36.645869+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":168,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648540+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:20:36.648540+0000","last_peered":"2023-09-23T18:20:36.648540+0000","last_clean":"2023-09-23T18:20:36.648540+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:20:36.648540+0000","last_undegraded":"2023-09-23T18:20:36.648540+0000","last_fullsized":"2023-09-23T18:20:36.648540+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":467,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648129+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:20:36.648129+0000","last_peered":"2023-09-23T18:20:36.648129+0000","last_clean":"2023-09-23T18:20:36.648129+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:20:36.648129+0000","last_undegraded":"2023-09-23T18:20:36.648129+0000","last_fullsized":"2023-09-23T18:20:36.648129+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"616'10150","reported_seq":10200,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:41.283823+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:20:41.283823+0000","last_peered":"2023-09-23T18:20:41.283823+0000","last_clean":"2023-09-23T18:20:41.283823+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:20:41.283823+0000","last_undegraded":"2023-09-23T18:20:41.283823+0000","last_fullsized":"2023-09-23T18:20:41.283823+0000","mapping_epoch":574,"log_start":"603'5700","ondisk_log_start":"603'5700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4450,"log_dups_size":0,"ondisk_log_size":4450,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":190185472,"num_objects":2902,"num_object_clones":0,"num_object_copies":8706,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2902,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10150,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":887,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646470+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:20:36.646470+0000","last_peered":"2023-09-23T18:20:36.646470+0000","last_clean":"2023-09-23T18:20:36.646470+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:20:36.646470+0000","last_undegraded":"2023-09-23T18:20:36.646470+0000","last_fullsized":"2023-09-23T18:20:36.646470+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":452,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.645771+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:20:36.645771+0000","last_peered":"2023-09-23T18:20:36.645771+0000","last_clean":"2023-09-23T18:20:36.645771+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:20:36.645771+0000","last_undegraded":"2023-09-23T18:20:36.645771+0000","last_fullsized":"2023-09-23T18:20:36.645771+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":228,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647883+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:20:36.647883+0000","last_peered":"2023-09-23T18:20:36.647883+0000","last_clean":"2023-09-23T18:20:36.647883+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:20:36.647883+0000","last_undegraded":"2023-09-23T18:20:36.647883+0000","last_fullsized":"2023-09-23T18:20:36.647883+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"616'10277","reported_seq":10327,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:42.281058+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:20:42.281058+0000","last_peered":"2023-09-23T18:20:42.281058+0000","last_clean":"2023-09-23T18:20:42.281058+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:20:42.281058+0000","last_undegraded":"2023-09-23T18:20:42.281058+0000","last_fullsized":"2023-09-23T18:20:42.281058+0000","mapping_epoch":574,"log_start":"602'5600","ondisk_log_start":"602'5600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4677,"log_dups_size":0,"ondisk_log_size":4677,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180682752,"num_objects":2757,"num_object_clones":0,"num_object_copies":8271,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2757,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10277,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":868,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646204+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:20:36.646204+0000","last_peered":"2023-09-23T18:20:36.646204+0000","last_clean":"2023-09-23T18:20:36.646204+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:20:36.646204+0000","last_undegraded":"2023-09-23T18:20:36.646204+0000","last_fullsized":"2023-09-23T18:20:36.646204+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":32,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649648+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:20:36.649648+0000","last_peered":"2023-09-23T18:20:36.649648+0000","last_clean":"2023-09-23T18:20:36.649648+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:20:36.649648+0000","last_undegraded":"2023-09-23T18:20:36.649648+0000","last_fullsized":"2023-09-23T18:20:36.649648+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":343,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T18:20:35.657566+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:20:35.657566+0000","last_peered":"2023-09-23T18:20:35.657566+0000","last_clean":"2023-09-23T18:20:35.657566+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:20:35.657566+0000","last_undegraded":"2023-09-23T18:20:35.657566+0000","last_fullsized":"2023-09-23T18:20:35.657566+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"615'10246","reported_seq":10298,"reported_epoch":615,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:40.229345+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:20:40.229345+0000","last_peered":"2023-09-23T18:20:40.229345+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:20:40.229345+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"602'5400","ondisk_log_start":"602'5400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4846,"log_dups_size":0,"ondisk_log_size":4846,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":184942592,"num_objects":2822,"num_object_clones":0,"num_object_copies":8466,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2822,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10246,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-1463},{"shards":"0,4,6","objects":573}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":727,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650031+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:20:36.650031+0000","last_peered":"2023-09-23T18:20:36.650031+0000","last_clean":"2023-09-23T18:20:36.650031+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:20:36.650031+0000","last_undegraded":"2023-09-23T18:20:36.650031+0000","last_fullsized":"2023-09-23T18:20:36.650031+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":965,"reported_epoch":614,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:35.657098+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:20:35.657098+0000","last_peered":"2023-09-23T18:20:35.657098+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:20:35.657098+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":150,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646703+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:20:36.646703+0000","last_peered":"2023-09-23T18:20:36.646703+0000","last_clean":"2023-09-23T18:20:36.646703+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:20:36.646703+0000","last_undegraded":"2023-09-23T18:20:36.646703+0000","last_fullsized":"2023-09-23T18:20:36.646703+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":130,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646122+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:20:36.646122+0000","last_peered":"2023-09-23T18:20:36.646122+0000","last_clean":"2023-09-23T18:20:36.646122+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:20:36.646122+0000","last_undegraded":"2023-09-23T18:20:36.646122+0000","last_fullsized":"2023-09-23T18:20:36.646122+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"616'10202","reported_seq":10261,"reported_epoch":616,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:41.256480+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:20:41.256480+0000","last_peered":"2023-09-23T18:20:41.256480+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:20:41.256480+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"603'5700","ondisk_log_start":"603'5700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4502,"log_dups_size":0,"ondisk_log_size":4502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":179175424,"num_objects":2734,"num_object_clones":0,"num_object_copies":8202,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2734,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10202,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"5,6","objects":-825}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":799,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646234+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:20:36.646234+0000","last_peered":"2023-09-23T18:20:36.646234+0000","last_clean":"2023-09-23T18:20:36.646234+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:20:36.646234+0000","last_undegraded":"2023-09-23T18:20:36.646234+0000","last_fullsized":"2023-09-23T18:20:36.646234+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1500,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646043+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:20:36.646043+0000","last_peered":"2023-09-23T18:20:36.646043+0000","last_clean":"2023-09-23T18:20:36.646043+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:20:36.646043+0000","last_undegraded":"2023-09-23T18:20:36.646043+0000","last_fullsized":"2023-09-23T18:20:36.646043+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":140,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648695+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:20:36.648695+0000","last_peered":"2023-09-23T18:20:36.648695+0000","last_clean":"2023-09-23T18:20:36.648695+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:20:36.648695+0000","last_undegraded":"2023-09-23T18:20:36.648695+0000","last_fullsized":"2023-09-23T18:20:36.648695+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":440,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648233+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:20:36.648233+0000","last_peered":"2023-09-23T18:20:36.648233+0000","last_clean":"2023-09-23T18:20:36.648233+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:20:36.648233+0000","last_undegraded":"2023-09-23T18:20:36.648233+0000","last_fullsized":"2023-09-23T18:20:36.648233+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"616'9920","reported_seq":9970,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:41.153513+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:20:41.153513+0000","last_peered":"2023-09-23T18:20:41.153513+0000","last_clean":"2023-09-23T18:20:41.153513+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:20:41.153513+0000","last_undegraded":"2023-09-23T18:20:41.153513+0000","last_fullsized":"2023-09-23T18:20:41.153513+0000","mapping_epoch":574,"log_start":"600'4400","ondisk_log_start":"600'4400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5520,"log_dups_size":0,"ondisk_log_size":5520,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":174718976,"num_objects":2666,"num_object_clones":0,"num_object_copies":7998,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2666,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9920,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":824,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646297+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:20:36.646297+0000","last_peered":"2023-09-23T18:20:36.646297+0000","last_clean":"2023-09-23T18:20:36.646297+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:20:36.646297+0000","last_undegraded":"2023-09-23T18:20:36.646297+0000","last_fullsized":"2023-09-23T18:20:36.646297+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":58,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649784+0000","last_change":"2023-09-23T18:20:35.016554+0000","last_active":"2023-09-23T18:20:36.649784+0000","last_peered":"2023-09-23T18:20:36.649784+0000","last_clean":"2023-09-23T18:20:36.649784+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:20:36.649784+0000","last_undegraded":"2023-09-23T18:20:36.649784+0000","last_fullsized":"2023-09-23T18:20:36.649784+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":409,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648324+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:20:36.648324+0000","last_peered":"2023-09-23T18:20:36.648324+0000","last_clean":"2023-09-23T18:20:36.648324+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:20:36.648324+0000","last_undegraded":"2023-09-23T18:20:36.648324+0000","last_fullsized":"2023-09-23T18:20:36.648324+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":152,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646758+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:20:36.646758+0000","last_peered":"2023-09-23T18:20:36.646758+0000","last_clean":"2023-09-23T18:20:36.646758+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:20:36.646758+0000","last_undegraded":"2023-09-23T18:20:36.646758+0000","last_fullsized":"2023-09-23T18:20:36.646758+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"616'10183","reported_seq":10233,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:41.287397+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:20:41.287397+0000","last_peered":"2023-09-23T18:20:41.287397+0000","last_clean":"2023-09-23T18:20:41.287397+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:20:41.287397+0000","last_undegraded":"2023-09-23T18:20:41.287397+0000","last_fullsized":"2023-09-23T18:20:41.287397+0000","mapping_epoch":574,"log_start":"603'5700","ondisk_log_start":"603'5700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4483,"log_dups_size":0,"ondisk_log_size":4483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181207040,"num_objects":2765,"num_object_clones":0,"num_object_copies":8295,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2765,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10183,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1513,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646286+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:20:36.646286+0000","last_peered":"2023-09-23T18:20:36.646286+0000","last_clean":"2023-09-23T18:20:36.646286+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:20:36.646286+0000","last_undegraded":"2023-09-23T18:20:36.646286+0000","last_fullsized":"2023-09-23T18:20:36.646286+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":566,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647631+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:20:36.647631+0000","last_peered":"2023-09-23T18:20:36.647631+0000","last_clean":"2023-09-23T18:20:36.647631+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:20:36.647631+0000","last_undegraded":"2023-09-23T18:20:36.647631+0000","last_fullsized":"2023-09-23T18:20:36.647631+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":126,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.645964+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:20:36.645964+0000","last_peered":"2023-09-23T18:20:36.645964+0000","last_clean":"2023-09-23T18:20:36.645964+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:20:36.645964+0000","last_undegraded":"2023-09-23T18:20:36.645964+0000","last_fullsized":"2023-09-23T18:20:36.645964+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":161,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648418+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:20:36.648418+0000","last_peered":"2023-09-23T18:20:36.648418+0000","last_clean":"2023-09-23T18:20:36.648418+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:20:36.648418+0000","last_undegraded":"2023-09-23T18:20:36.648418+0000","last_fullsized":"2023-09-23T18:20:36.648418+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"616'9993","reported_seq":10042,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:41.158822+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:20:41.158822+0000","last_peered":"2023-09-23T18:20:41.158822+0000","last_clean":"2023-09-23T18:20:41.158822+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:20:41.158822+0000","last_undegraded":"2023-09-23T18:20:41.158822+0000","last_fullsized":"2023-09-23T18:20:41.158822+0000","mapping_epoch":574,"log_start":"600'4500","ondisk_log_start":"600'4500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5493,"log_dups_size":0,"ondisk_log_size":5493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":182910976,"num_objects":2791,"num_object_clones":0,"num_object_copies":8373,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2791,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":9993,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":814,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648819+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:20:36.648819+0000","last_peered":"2023-09-23T18:20:36.648819+0000","last_clean":"2023-09-23T18:20:36.648819+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:20:36.648819+0000","last_undegraded":"2023-09-23T18:20:36.648819+0000","last_fullsized":"2023-09-23T18:20:36.648819+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":129,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649713+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:20:36.649713+0000","last_peered":"2023-09-23T18:20:36.649713+0000","last_clean":"2023-09-23T18:20:36.649713+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:20:36.649713+0000","last_undegraded":"2023-09-23T18:20:36.649713+0000","last_fullsized":"2023-09-23T18:20:36.649713+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":32,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649829+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:20:36.649829+0000","last_peered":"2023-09-23T18:20:36.649829+0000","last_clean":"2023-09-23T18:20:36.649829+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:20:36.649829+0000","last_undegraded":"2023-09-23T18:20:36.649829+0000","last_fullsized":"2023-09-23T18:20:36.649829+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":631,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646718+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:20:36.646718+0000","last_peered":"2023-09-23T18:20:36.646718+0000","last_clean":"2023-09-23T18:20:36.646718+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:20:36.646718+0000","last_undegraded":"2023-09-23T18:20:36.646718+0000","last_fullsized":"2023-09-23T18:20:36.646718+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"616'10128","reported_seq":17107,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:41.251344+0000","last_change":"2023-09-23T18:20:36.673759+0000","last_active":"2023-09-23T18:20:41.251344+0000","last_peered":"2023-09-23T18:20:41.251344+0000","last_clean":"2023-09-23T18:20:41.251344+0000","last_became_active":"2023-09-23T18:20:36.651913+0000","last_became_peered":"2023-09-23T18:20:36.651913+0000","last_unstale":"2023-09-23T18:20:41.251344+0000","last_undegraded":"2023-09-23T18:20:41.251344+0000","last_fullsized":"2023-09-23T18:20:41.251344+0000","mapping_epoch":614,"log_start":"603'5700","ondisk_log_start":"603'5700","created":574,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4428,"log_dups_size":0,"ondisk_log_size":4428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:47.765348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":185401368,"num_objects":2830,"num_object_clones":0,"num_object_copies":8490,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2830,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10128,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3453,"num_bytes_recovered":209321984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":914,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648286+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:20:36.648286+0000","last_peered":"2023-09-23T18:20:36.648286+0000","last_clean":"2023-09-23T18:20:36.648286+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:20:36.648286+0000","last_undegraded":"2023-09-23T18:20:36.648286+0000","last_fullsized":"2023-09-23T18:20:36.648286+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":54,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.651049+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:20:36.651049+0000","last_peered":"2023-09-23T18:20:36.651049+0000","last_clean":"2023-09-23T18:20:36.651049+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:20:36.651049+0000","last_undegraded":"2023-09-23T18:20:36.651049+0000","last_fullsized":"2023-09-23T18:20:36.651049+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":140,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646206+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:20:36.646206+0000","last_peered":"2023-09-23T18:20:36.646206+0000","last_clean":"2023-09-23T18:20:36.646206+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:20:36.646206+0000","last_undegraded":"2023-09-23T18:20:36.646206+0000","last_fullsized":"2023-09-23T18:20:36.646206+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":140,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.645887+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:20:36.645887+0000","last_peered":"2023-09-23T18:20:36.645887+0000","last_clean":"2023-09-23T18:20:36.645887+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:20:36.645887+0000","last_undegraded":"2023-09-23T18:20:36.645887+0000","last_fullsized":"2023-09-23T18:20:36.645887+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"616'10101","reported_seq":10151,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:41.284228+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:20:41.284228+0000","last_peered":"2023-09-23T18:20:41.284228+0000","last_clean":"2023-09-23T18:20:41.284228+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:20:41.284228+0000","last_undegraded":"2023-09-23T18:20:41.284228+0000","last_fullsized":"2023-09-23T18:20:41.284228+0000","mapping_epoch":574,"log_start":"603'5600","ondisk_log_start":"603'5600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4501,"log_dups_size":0,"ondisk_log_size":4501,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":180420608,"num_objects":2753,"num_object_clones":0,"num_object_copies":8259,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2753,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10101,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":914,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648206+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:20:36.648206+0000","last_peered":"2023-09-23T18:20:36.648206+0000","last_clean":"2023-09-23T18:20:36.648206+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:20:36.648206+0000","last_undegraded":"2023-09-23T18:20:36.648206+0000","last_fullsized":"2023-09-23T18:20:36.648206+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":102,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646595+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:20:36.646595+0000","last_peered":"2023-09-23T18:20:36.646595+0000","last_clean":"2023-09-23T18:20:36.646595+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:20:36.646595+0000","last_undegraded":"2023-09-23T18:20:36.646595+0000","last_fullsized":"2023-09-23T18:20:36.646595+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":395,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648122+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:20:36.648122+0000","last_peered":"2023-09-23T18:20:36.648122+0000","last_clean":"2023-09-23T18:20:36.648122+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:20:36.648122+0000","last_undegraded":"2023-09-23T18:20:36.648122+0000","last_fullsized":"2023-09-23T18:20:36.648122+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":32,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.649537+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:36.649537+0000","last_peered":"2023-09-23T18:20:36.649537+0000","last_clean":"2023-09-23T18:20:36.649537+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:20:36.649537+0000","last_undegraded":"2023-09-23T18:20:36.649537+0000","last_fullsized":"2023-09-23T18:20:36.649537+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"616'10049","reported_seq":10099,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:41.281154+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:20:41.281154+0000","last_peered":"2023-09-23T18:20:41.281154+0000","last_clean":"2023-09-23T18:20:41.281154+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:20:41.281154+0000","last_undegraded":"2023-09-23T18:20:41.281154+0000","last_fullsized":"2023-09-23T18:20:41.281154+0000","mapping_epoch":574,"log_start":"603'5600","ondisk_log_start":"603'5600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4449,"log_dups_size":0,"ondisk_log_size":4449,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":181075968,"num_objects":2763,"num_object_clones":0,"num_object_copies":8289,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2763,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10049,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":748,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650112+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:20:36.650112+0000","last_peered":"2023-09-23T18:20:36.650112+0000","last_clean":"2023-09-23T18:20:36.650112+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:20:36.650112+0000","last_undegraded":"2023-09-23T18:20:36.650112+0000","last_fullsized":"2023-09-23T18:20:36.650112+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1052,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650526+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:20:36.650526+0000","last_peered":"2023-09-23T18:20:36.650526+0000","last_clean":"2023-09-23T18:20:36.650526+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:20:36.650526+0000","last_undegraded":"2023-09-23T18:20:36.650526+0000","last_fullsized":"2023-09-23T18:20:36.650526+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":41,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T18:20:35.649747+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:20:35.649747+0000","last_peered":"2023-09-23T18:20:35.649747+0000","last_clean":"2023-09-23T18:20:35.649747+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:20:35.649747+0000","last_undegraded":"2023-09-23T18:20:35.649747+0000","last_fullsized":"2023-09-23T18:20:35.649747+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":406,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.645970+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:20:36.645970+0000","last_peered":"2023-09-23T18:20:36.645970+0000","last_clean":"2023-09-23T18:20:36.645970+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:20:36.645970+0000","last_undegraded":"2023-09-23T18:20:36.645970+0000","last_fullsized":"2023-09-23T18:20:36.645970+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":646,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647983+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:20:36.647983+0000","last_peered":"2023-09-23T18:20:36.647983+0000","last_clean":"2023-09-23T18:20:36.647983+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:20:36.647983+0000","last_undegraded":"2023-09-23T18:20:36.647983+0000","last_fullsized":"2023-09-23T18:20:36.647983+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":37,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T18:20:35.649228+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:20:35.649228+0000","last_peered":"2023-09-23T18:20:35.649228+0000","last_clean":"2023-09-23T18:20:35.649228+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:20:35.649228+0000","last_undegraded":"2023-09-23T18:20:35.649228+0000","last_fullsized":"2023-09-23T18:20:35.649228+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":688,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T18:20:35.656920+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:20:35.656920+0000","last_peered":"2023-09-23T18:20:35.656920+0000","last_clean":"2023-09-23T18:20:35.656920+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:20:35.656920+0000","last_undegraded":"2023-09-23T18:20:35.656920+0000","last_fullsized":"2023-09-23T18:20:35.656920+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":646,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647960+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:20:36.647960+0000","last_peered":"2023-09-23T18:20:36.647960+0000","last_clean":"2023-09-23T18:20:36.647960+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:20:36.647960+0000","last_undegraded":"2023-09-23T18:20:36.647960+0000","last_fullsized":"2023-09-23T18:20:36.647960+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":134,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646210+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:20:36.646210+0000","last_peered":"2023-09-23T18:20:36.646210+0000","last_clean":"2023-09-23T18:20:36.646210+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:20:36.646210+0000","last_undegraded":"2023-09-23T18:20:36.646210+0000","last_fullsized":"2023-09-23T18:20:36.646210+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":250,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650761+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:20:36.650761+0000","last_peered":"2023-09-23T18:20:36.650761+0000","last_clean":"2023-09-23T18:20:36.650761+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:20:36.650761+0000","last_undegraded":"2023-09-23T18:20:36.650761+0000","last_fullsized":"2023-09-23T18:20:36.650761+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":120,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646529+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:20:36.646529+0000","last_peered":"2023-09-23T18:20:36.646529+0000","last_clean":"2023-09-23T18:20:36.646529+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:20:36.646529+0000","last_undegraded":"2023-09-23T18:20:36.646529+0000","last_fullsized":"2023-09-23T18:20:36.646529+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":233,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.650680+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:20:36.650680+0000","last_peered":"2023-09-23T18:20:36.650680+0000","last_clean":"2023-09-23T18:20:36.650680+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:20:36.650680+0000","last_undegraded":"2023-09-23T18:20:36.650680+0000","last_fullsized":"2023-09-23T18:20:36.650680+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1008,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647787+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:20:36.647787+0000","last_peered":"2023-09-23T18:20:36.647787+0000","last_clean":"2023-09-23T18:20:36.647787+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:20:36.647787+0000","last_undegraded":"2023-09-23T18:20:36.647787+0000","last_fullsized":"2023-09-23T18:20:36.647787+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":513,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.648204+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:20:36.648204+0000","last_peered":"2023-09-23T18:20:36.648204+0000","last_clean":"2023-09-23T18:20:36.648204+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:20:36.648204+0000","last_undegraded":"2023-09-23T18:20:36.648204+0000","last_fullsized":"2023-09-23T18:20:36.648204+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":150,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646143+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:20:36.646143+0000","last_peered":"2023-09-23T18:20:36.646143+0000","last_clean":"2023-09-23T18:20:36.646143+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:20:36.646143+0000","last_undegraded":"2023-09-23T18:20:36.646143+0000","last_fullsized":"2023-09-23T18:20:36.646143+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":944,"reported_epoch":614,"state":"active+clean","last_fresh":"2023-09-23T18:20:35.659302+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:20:35.659302+0000","last_peered":"2023-09-23T18:20:35.659302+0000","last_clean":"2023-09-23T18:20:35.659302+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:20:35.659302+0000","last_undegraded":"2023-09-23T18:20:35.659302+0000","last_fullsized":"2023-09-23T18:20:35.659302+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":138,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.646724+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:20:36.646724+0000","last_peered":"2023-09-23T18:20:36.646724+0000","last_clean":"2023-09-23T18:20:36.646724+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:20:36.646724+0000","last_undegraded":"2023-09-23T18:20:36.646724+0000","last_fullsized":"2023-09-23T18:20:36.646724+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":659,"reported_epoch":615,"state":"active+clean","last_fresh":"2023-09-23T18:20:36.647532+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:20:36.647532+0000","last_peered":"2023-09-23T18:20:36.647532+0000","last_clean":"2023-09-23T18:20:36.647532+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:20:36.647532+0000","last_undegraded":"2023-09-23T18:20:36.647532+0000","last_fullsized":"2023-09-23T18:20:36.647532+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2921267224,"num_objects":44576,"num_object_clones":0,"num_object_copies":133728,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15405,"num_objects_misplaced":498,"num_objects_unfound":0,"num_objects_dirty":44576,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":162232,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7239,"num_bytes_recovered":457441280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":10353913856,"data_stored":10353901640,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77232,"ondisk_log_size":77232,"up":48,"acting":44,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20983808,"data_stored":20971577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001577,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669513,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2157520,"kb_used_data":1543944,"kb_used_omap":7,"kb_used_meta":613432,"kb_avail":91587632,"statfs":{"total":95995035648,"available":93785735168,"internally_reserved":0,"allocated":1580998656,"data_stored":1579052297,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":628155172},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[3,1,0,3,1,5],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":1.7889999999999999,"15min":1.407},"min":{"1min":0.249,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":9.1600000000000001,"5min":24.472999999999999,"15min":24.472999999999999},"last":1.1859999999999999},{"interface":"front","average":{"1min":2.54,"5min":2.1960000000000002,"15min":1.726},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2029999999999994,"5min":22.832999999999998,"15min":22.832999999999998},"last":3.2189999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:20:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3570000000000002,"5min":2.0350000000000001,"15min":1.4890000000000001},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.726000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":3.109},{"interface":"front","average":{"1min":2.423,"5min":1.899,"15min":1.4379999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.336,"5min":22.664000000000001,"15min":22.664000000000001},"last":2.0760000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.52,"15min":1.3260000000000001},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":8.0559999999999992,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":1.202},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.861,"15min":1.492},"min":{"1min":0.34000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.1230000000000002,"5min":14.771000000000001,"15min":14.771000000000001},"last":4.5730000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":9.1259999999999994,"5min":3.1400000000000001,"15min":2.1059999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":103.348,"5min":103.348,"15min":103.348},"last":2.1539999999999999},{"interface":"front","average":{"1min":9.4610000000000003,"5min":3.6389999999999998,"15min":2.3610000000000002},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":103.40300000000001,"5min":103.40300000000001,"15min":103.40300000000001},"last":3.1400000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.8180000000000001,"15min":1.345},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.0730000000000004,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":1.994},{"interface":"front","average":{"1min":1.776,"5min":1.8200000000000001,"15min":1.3700000000000001},"min":{"1min":0.45200000000000001,"5min":0.30399999999999999,"15min":0.27000000000000002},"max":{"1min":7.8319999999999999,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":1.1240000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:19:46 2023","interfaces":[{"interface":"back","average":{"1min":1.3839999999999999,"5min":2.0609999999999999,"15min":1.9550000000000001},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":6.4710000000000001,"5min":13.502000000000001,"15min":13.502000000000001},"last":2.1909999999999998},{"interface":"front","average":{"1min":1.3720000000000001,"5min":2.246,"15min":2.3660000000000001},"min":{"1min":0.34899999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":6.1070000000000002,"5min":10.081,"15min":10.081},"last":2.2170000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":2.1459999999999999,"15min":1.5309999999999999},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.26800000000000002},"max":{"1min":6.4320000000000004,"5min":10.743,"15min":13.460000000000001},"last":3.0910000000000002},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.0710000000000002,"15min":1.5740000000000001},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.28499999999999998},"max":{"1min":6.2770000000000001,"5min":11.244,"15min":12.404},"last":1.1599999999999999}]}]},{"osd":1,"up_from":545,"seq":2340757176361,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1612144,"kb_used_data":1408488,"kb_used_omap":13,"kb_used_meta":203634,"kb_avail":92133008,"statfs":{"total":95995035648,"available":94344200192,"internally_reserved":0,"allocated":1442291712,"data_stored":1439271856,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":208521869},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.016,"15min":0.59899999999999998},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":9.0299999999999994,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.004,"15min":0.66100000000000003},"min":{"1min":0.34499999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.952999999999999,"5min":14.952999999999999,"15min":14.952999999999999},"last":2.7949999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.0980000000000001,"15min":0.67300000000000004},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.6219999999999999,"5min":9.6219999999999999,"15min":9.6219999999999999},"last":1.4530000000000001},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.77200000000000002,"15min":0.59399999999999997},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2080000000000002,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":2.7320000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.0409999999999999,"15min":0.60499999999999998},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.3200000000000003,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":1.3799999999999999},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.2769999999999999,"15min":0.76700000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":19.762,"5min":19.762,"15min":19.762},"last":8.3399999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.3280000000000001,"15min":1.3200000000000001},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.8690000000000002,"5min":12.663,"15min":12.663},"last":2.6909999999999998},{"interface":"front","average":{"1min":2.4049999999999998,"5min":1.7789999999999999,"15min":1.6020000000000001},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.317},"max":{"1min":6.0289999999999999,"5min":13.222,"15min":13.222},"last":1.4259999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.405,"15min":0.94999999999999996},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":23.190999999999999,"5min":23.190999999999999,"15min":23.190999999999999},"last":2.7690000000000001},{"interface":"front","average":{"1min":4.1719999999999997,"5min":1.7330000000000001,"15min":1.113},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":23.204000000000001,"5min":23.204000000000001,"15min":23.204000000000001},"last":5.6340000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.121,"5min":1.4690000000000001,"15min":0.996},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":23.443000000000001,"5min":23.443000000000001,"15min":23.443000000000001},"last":2.819},{"interface":"front","average":{"1min":3.7970000000000002,"5min":1.746,"15min":1.099},"min":{"1min":0.42199999999999999,"5min":0.378,"15min":0.378},"max":{"1min":26.177,"5min":26.177,"15min":26.177},"last":1.4770000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.4340000000000002,"5min":1.3620000000000001,"15min":0.90800000000000003},"min":{"1min":0.42299999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.1050000000000004,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":2.157},{"interface":"front","average":{"1min":2.3140000000000001,"5min":1.474,"15min":1.028},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":9.1500000000000004,"5min":9.1500000000000004,"15min":9.1500000000000004},"last":0.45200000000000001}]}]},{"osd":0,"up_from":93,"seq":399431958858,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3054860,"kb_used_data":2131396,"kb_used_omap":7,"kb_used_meta":923192,"kb_avail":90690292,"statfs":{"total":95995035648,"available":92866859008,"internally_reserved":0,"allocated":2182549504,"data_stored":2179681755,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":945349348},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,3,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.044,"15min":0.56100000000000005},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.4779999999999998,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.6240000000000001,"5min":1.278,"15min":0.77400000000000002},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.335000000000001,"5min":15.335000000000001,"15min":15.335000000000001},"last":3.839}]},{"osd":2,"last update":"Sat Sep 23 18:20:04 2023","interfaces":[{"interface":"back","average":{"1min":3.911,"5min":1.71,"15min":1.3160000000000001},"min":{"1min":0.48999999999999999,"5min":0.185,"15min":0.185},"max":{"1min":15.002000000000001,"5min":18.550999999999998,"15min":18.550999999999998},"last":4.3120000000000003},{"interface":"front","average":{"1min":2.8530000000000002,"5min":1.258,"15min":0.93400000000000005},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":11.644,"5min":13.618,"15min":13.618},"last":0.53200000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":3.7610000000000001,"5min":1.393,"15min":1.637},"min":{"1min":0.38400000000000001,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":13.539999999999999,"5min":13.539999999999999,"15min":13.539999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":4.1390000000000002,"5min":1.4930000000000001,"15min":1.637},"min":{"1min":0.28699999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":13.285,"5min":13.285,"15min":15.198},"last":3.887}]},{"osd":4,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7010000000000001,"5min":2.1309999999999998,"15min":1.8979999999999999},"min":{"1min":0.41799999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":17.741,"5min":17.741,"15min":57.746000000000002},"last":3.9969999999999999},{"interface":"front","average":{"1min":3.79,"5min":1.9159999999999999,"15min":1.7809999999999999},"min":{"1min":0.50900000000000001,"5min":0.26900000000000002,"15min":0.25600000000000001},"max":{"1min":15.079000000000001,"5min":15.079000000000001,"15min":57.816000000000003},"last":4.359}]},{"osd":5,"last update":"Sat Sep 23 18:19:44 2023","interfaces":[{"interface":"back","average":{"1min":2.9300000000000002,"5min":2.032,"15min":1.391},"min":{"1min":0.35599999999999998,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":7.5999999999999996,"5min":7.9660000000000002,"15min":7.9660000000000002},"last":3.968},{"interface":"front","average":{"1min":4.2859999999999996,"5min":2.2650000000000001,"15min":1.448},"min":{"1min":0.502,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":13.425000000000001,"5min":13.425000000000001,"15min":13.425000000000001},"last":4.3879999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":1.948,"15min":1.9530000000000001},"min":{"1min":0.47499999999999998,"5min":0.377,"15min":0.32000000000000001},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":58.457999999999998},"last":4.024},{"interface":"front","average":{"1min":4.6639999999999997,"5min":2.3159999999999998,"15min":1.9299999999999999},"min":{"1min":0.497,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":18.643999999999998,"5min":18.643999999999998,"15min":58.354999999999997},"last":3.7240000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":1.923,"15min":1.639},"min":{"1min":0.46899999999999997,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":13.644,"5min":13.644,"15min":13.644},"last":3.758},{"interface":"front","average":{"1min":3.5179999999999998,"5min":1.821,"15min":1.8},"min":{"1min":0.55900000000000005,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.7519999999999998,"5min":10.404,"15min":10.404},"last":3.927}]}]},{"osd":2,"up_from":0,"seq":2070174236743,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249373,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1878004,"kb_used_data":1148780,"kb_used_omap":3,"kb_used_meta":729212,"kb_avail":91867148,"statfs":{"total":95995035648,"available":94071959552,"internally_reserved":0,"allocated":1176350720,"data_stored":1173276797,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":746713832},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,2],"upper_bound":4},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.3919999999999999,"5min":1.25,"15min":1.7050000000000001},"min":{"1min":0.192,"5min":0.192,"15min":0.183},"max":{"1min":17.036000000000001,"5min":17.036000000000001,"15min":195.69800000000001},"last":1.204},{"interface":"front","average":{"1min":2.6909999999999998,"5min":1.1599999999999999,"15min":1.96},"min":{"1min":0.217,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":9.0980000000000008,"5min":9.0980000000000008,"15min":194.661},"last":5.3479999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:20:26 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":1.4590000000000001,"15min":0.83299999999999996},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.079000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":2.823},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.5129999999999999,"15min":1.3069999999999999},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.081,"5min":78.980000000000004,"15min":78.980000000000004},"last":1.9530000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":1.274,"15min":1.29},"min":{"1min":0.27400000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.151999999999999,"5min":12.151999999999999,"15min":12.151999999999999},"last":2.8849999999999998},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.298,"15min":1.3320000000000001},"min":{"1min":0.318,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.565,"5min":11.565,"15min":11.565},"last":1.2410000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.0459999999999998,"5min":1.6140000000000001,"15min":1.659},"min":{"1min":0.31900000000000001,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":8.2300000000000004,"5min":21.062999999999999,"15min":37.231000000000002},"last":3.0760000000000001},{"interface":"front","average":{"1min":2.2549999999999999,"5min":1.9810000000000001,"15min":1.8939999999999999},"min":{"1min":0.35299999999999998,"5min":0.29199999999999998,"15min":0.29199999999999998},"max":{"1min":8.2959999999999994,"5min":21.128,"15min":37.381},"last":2.9169999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:20:37 2023","interfaces":[{"interface":"back","average":{"1min":3.452,"5min":2.048,"15min":1.587},"min":{"1min":0.35099999999999998,"5min":0.316,"15min":0.316},"max":{"1min":9.7430000000000003,"5min":11.805999999999999,"15min":11.805999999999999},"last":3.0569999999999999},{"interface":"front","average":{"1min":3.0339999999999998,"5min":1.74,"15min":1.722},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":17.135999999999999,"5min":17.135999999999999,"15min":17.135999999999999},"last":1.355}]},{"osd":6,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":3.0459999999999998,"5min":1.7809999999999999,"15min":1.847},"min":{"1min":0.432,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":17.192,"5min":17.192,"15min":115.54000000000001},"last":1.28},{"interface":"front","average":{"1min":2.2530000000000001,"5min":1.7250000000000001,"15min":1.948},"min":{"1min":0.37,"5min":0.34200000000000003,"15min":0.30499999999999999},"max":{"1min":8.0350000000000001,"5min":14.321,"15min":115.691},"last":5.6790000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:19:47 2023","interfaces":[{"interface":"back","average":{"1min":2.2490000000000001,"5min":1.5069999999999999,"15min":1.7110000000000001},"min":{"1min":0.38700000000000001,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":8.2070000000000007,"5min":8.2070000000000007,"15min":15.972},"last":2.992},{"interface":"front","average":{"1min":3.5920000000000001,"5min":1.7629999999999999,"15min":1.623},"min":{"1min":0.48799999999999999,"5min":0.41999999999999998,"15min":0.34899999999999998},"max":{"1min":17.164000000000001,"5min":17.164000000000001,"15min":17.164000000000001},"last":1.3140000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019323,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":3052188,"kb_used_data":2385044,"kb_used_omap":0,"kb_used_meta":666944,"kb_avail":90692964,"statfs":{"total":95995035648,"available":92869595136,"internally_reserved":0,"allocated":2442285056,"data_stored":2440334477,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":682950656},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,2,2],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.534,"15min":1.268},"min":{"1min":0.255,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.6079999999999997,"5min":7.6079999999999997,"15min":13.375999999999999},"last":4.2729999999999997},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.6819999999999999,"15min":1.546},"min":{"1min":0.26600000000000001,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":9.3710000000000004,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.28100000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:20:22 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":1.7170000000000001,"15min":1.0349999999999999},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8220000000000001,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.40000000000000002},{"interface":"front","average":{"1min":1.8420000000000001,"5min":1.6679999999999999,"15min":0.98999999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.5700000000000003,"5min":43.497999999999998,"15min":43.497999999999998},"last":1.659}]},{"osd":2,"last update":"Sat Sep 23 18:19:56 2023","interfaces":[{"interface":"back","average":{"1min":2.621,"5min":1.6699999999999999,"15min":1.8180000000000001},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.896000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":1.6990000000000001},{"interface":"front","average":{"1min":1.6020000000000001,"5min":1.6200000000000001,"15min":1.6930000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":6.6890000000000001,"5min":10.938000000000001,"15min":10.938000000000001},"last":0.80000000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.5049999999999999,"5min":1.5669999999999999,"15min":1.9350000000000001},"min":{"1min":0.33700000000000002,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.0609999999999999,"5min":11.130000000000001,"15min":29.154},"last":1.7270000000000001},{"interface":"front","average":{"1min":2.1800000000000002,"5min":1.8480000000000001,"15min":1.9390000000000001},"min":{"1min":0.35399999999999998,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":13.470000000000001,"5min":13.470000000000001,"15min":29.216000000000001},"last":4.298}]},{"osd":5,"last update":"Sat Sep 23 18:19:41 2023","interfaces":[{"interface":"back","average":{"1min":1.0489999999999999,"5min":2.4279999999999999,"15min":2.8559999999999999},"min":{"1min":0.318,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":5.4950000000000001,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.379},{"interface":"front","average":{"1min":1.401,"5min":1.827,"15min":2.7109999999999999},"min":{"1min":0.30499999999999999,"5min":0.26300000000000001,"15min":0.26300000000000001},"max":{"1min":6.1040000000000001,"5min":13.938000000000001,"15min":32.640000000000001},"last":0.76000000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.873,"15min":1.347},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":6.1989999999999998,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.69699999999999995},{"interface":"front","average":{"1min":1.859,"5min":1.9630000000000001,"15min":1.522},"min":{"1min":0.38700000000000001,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":9.5579999999999998,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.72999999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":2.4849999999999999,"15min":1.774},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":5.9539999999999997,"5min":19.298999999999999,"15min":19.298999999999999},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.722,"5min":1.8899999999999999,"15min":1.552},"min":{"1min":0.435,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.4730000000000008,"5min":12.800000000000001,"15min":12.800000000000001},"last":1.9079999999999999}]}]},{"osd":5,"up_from":420,"seq":1803886264419,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2002252,"kb_used_data":1650884,"kb_used_omap":9,"kb_used_meta":351350,"kb_avail":91742900,"statfs":{"total":95995035648,"available":93944729600,"internally_reserved":0,"allocated":1690505216,"data_stored":1688566981,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":359782845},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,2,2,1,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.2809999999999999,"5min":1.2549999999999999,"15min":0.88100000000000001},"min":{"1min":0.26300000000000001,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":3.6000000000000001,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.25},{"interface":"front","average":{"1min":2.911,"5min":1.8149999999999999,"15min":1.6080000000000001},"min":{"1min":0.27400000000000002,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":20.937999999999999,"5min":20.937999999999999,"15min":20.937999999999999},"last":4.109}]},{"osd":1,"last update":"Sat Sep 23 18:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":1.7070000000000001,"15min":1.2989999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.6649999999999991,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.32100000000000001},{"interface":"front","average":{"1min":2.1840000000000002,"5min":1.73,"15min":1.141},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.5259999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":4.1600000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:19:52 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":1.629,"15min":1.615},"min":{"1min":0.28699999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":10.705,"5min":10.705,"15min":10.705},"last":0.35099999999999998},{"interface":"front","average":{"1min":2.5409999999999999,"5min":1.946,"15min":2.1600000000000001},"min":{"1min":0.373,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.8099999999999996,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":4.2910000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.8420000000000001,"5min":1.8740000000000001,"15min":1.2789999999999999},"min":{"1min":0.443,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":8.8780000000000001,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":1.3260000000000001},{"interface":"front","average":{"1min":3.7320000000000002,"5min":2.0640000000000001,"15min":1.665},"min":{"1min":0.38100000000000001,"5min":0.315,"15min":0.313},"max":{"1min":20.035,"5min":20.035,"15min":20.035},"last":4.1360000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.3849999999999998,"5min":2.2770000000000001,"15min":2.0640000000000001},"min":{"1min":0.33000000000000002,"5min":0.28199999999999997,"15min":0.251},"max":{"1min":7.875,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.628},{"interface":"front","average":{"1min":2.2530000000000001,"5min":2.0899999999999999,"15min":1.6160000000000001},"min":{"1min":0.32400000000000001,"5min":0.30499999999999999,"15min":0.30499999999999999},"max":{"1min":6.7610000000000001,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":6.4610000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":1.9119999999999999,"5min":1.9650000000000001,"15min":1.2609999999999999},"min":{"1min":0.28100000000000003,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.2480000000000002,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.52500000000000002},{"interface":"front","average":{"1min":2.343,"5min":2.2040000000000002,"15min":1.6819999999999999},"min":{"1min":0.34599999999999997,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.8410000000000002,"5min":12.504,"15min":12.504},"last":4.0700000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:19:50 2023","interfaces":[{"interface":"back","average":{"1min":2.2879999999999998,"5min":2.3980000000000001,"15min":1.758},"min":{"1min":0.40699999999999997,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.528,"5min":13.506,"15min":13.506},"last":0.58599999999999997},{"interface":"front","average":{"1min":1.381,"5min":1.9630000000000001,"15min":1.542},"min":{"1min":0.36099999999999999,"5min":0.35199999999999998,"15min":0.29799999999999999},"max":{"1min":5.0970000000000004,"5min":13.138,"15min":13.138},"last":0.49099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":2135232512,"data_stored":2135228440,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1422131200,"data_stored":1422131200,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1160904704,"data_stored":1160904704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":2408845312,"data_stored":2408841240,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1672282112,"data_stored":1672282112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1554518016,"data_stored":1554513944,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:20:45.103 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:45.261 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:20:45.283 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:45.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:45.294 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:45.395 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:45.493 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:45.676 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:45.680 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:45.680 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:45.781 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:45.856 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:45.926 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:45 smithi099 ceph-mon[167076]: pgmap v1479: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 15418/132912 objects degraded (11.600%); 498/132912 objects misplaced (0.375%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T18:20:45.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:45 smithi055 ceph-mon[184174]: pgmap v1479: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 15418/132912 objects degraded (11.600%); 498/132912 objects misplaced (0.375%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T18:20:45.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:45 smithi055 ceph-mon[179668]: pgmap v1479: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.7 GiB data, 13 GiB used, 523 GiB / 536 GiB avail; 0 B/s wr, 544 op/s; 15418/132912 objects degraded (11.600%); 498/132912 objects misplaced (0.375%); 1.8 MiB/s, 29 objects/s recovering 2023-09-23T18:20:45.934 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:45.941 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:45.941 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:46.042 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:46.104 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:46.163 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:46.202 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:46.205 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:46.206 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:46.237 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:20:46.237 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:20:46.308 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:46.409 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:46.608 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:46.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:46.611 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:46.661 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:20:46.713 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:46.786 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2835309390' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:46.786 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2835309390' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:46.809 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:20:46.868 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:46.993 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:46.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:46.996 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:47.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2835309390' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:20:47.097 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:47.163 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:47.411 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:47.688 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:47.701 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:47.701 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:47.802 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:47.853 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:20:47.853 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[184174]: failed to encode map e617 with expected crc 2023-09-23T18:20:47.853 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[184174]: failed to encode map e617 with expected crc 2023-09-23T18:20:47.853 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2835309390' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:20:47.853 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[184174]: osdmap e617: 8 total, 8 up, 6 in 2023-09-23T18:20:47.853 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[184174]: failed to encode map e617 with expected crc 2023-09-23T18:20:47.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[184174]: failed to encode map e617 with expected crc 2023-09-23T18:20:47.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[184174]: pgmap v1481: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 15502/120957 objects degraded (12.816%); 211/120957 objects misplaced (0.174%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:20:47.854 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:20:47.854 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[179668]: failed to encode map e617 with expected crc 2023-09-23T18:20:47.854 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[179668]: failed to encode map e617 with expected crc 2023-09-23T18:20:47.854 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2835309390' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:20:47.854 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[179668]: osdmap e617: 8 total, 8 up, 6 in 2023-09-23T18:20:47.854 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[179668]: failed to encode map e617 with expected crc 2023-09-23T18:20:47.854 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[179668]: failed to encode map e617 with expected crc 2023-09-23T18:20:47.855 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:47 smithi055 ceph-mon[179668]: pgmap v1481: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 15502/120957 objects degraded (12.816%); 211/120957 objects misplaced (0.174%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:20:47.870 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:48.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:47 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:20:48.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:47 smithi099 ceph-mon[167076]: failed to encode map e617 with expected crc 2023-09-23T18:20:48.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:47 smithi099 ceph-mon[167076]: failed to encode map e617 with expected crc 2023-09-23T18:20:48.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:47 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2835309390' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:20:48.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:47 smithi099 ceph-mon[167076]: osdmap e617: 8 total, 8 up, 6 in 2023-09-23T18:20:48.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:47 smithi099 ceph-mon[167076]: failed to encode map e617 with expected crc 2023-09-23T18:20:48.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:47 smithi099 ceph-mon[167076]: failed to encode map e617 with expected crc 2023-09-23T18:20:48.021 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:47 smithi099 ceph-mon[167076]: pgmap v1481: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.5 GiB data, 13 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 1.01k op/s; 15502/120957 objects degraded (12.816%); 211/120957 objects misplaced (0.174%); 3.5 MiB/s, 56 objects/s recovering 2023-09-23T18:20:48.091 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:20:48.817 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15502/120957 objects degraded (12.816%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:48.821 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:20:48.821 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:20:49.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15502/120957 objects degraded (12.816%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:49.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15502/120957 objects degraded (12.816%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:49.272 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:20:49.834 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:49 smithi055 ceph-mon[184174]: pgmap v1482: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 974 op/s; 15502/119253 objects degraded (12.999%); 211/119253 objects misplaced (0.177%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T18:20:49.835 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:49 smithi055 ceph-mon[179668]: pgmap v1482: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 974 op/s; 15502/119253 objects degraded (12.999%); 211/119253 objects misplaced (0.177%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T18:20:49.896 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:49.904 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:49.904 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:50.005 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:50.055 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:50.119 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:20:50.141 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:50.143 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:49 smithi099 ceph-mon[167076]: pgmap v1482: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 974 op/s; 15502/119253 objects degraded (12.999%); 211/119253 objects misplaced (0.177%); 3.4 MiB/s, 53 objects/s recovering 2023-09-23T18:20:50.144 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:50.144 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:50.245 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:50.283 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:50.389 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:50.392 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:50.392 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:50.493 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:50.556 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:20:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:20:50] "GET /metrics HTTP/1.1" 200 36685 "" "Prometheus/2.43.0" 2023-09-23T18:20:50.565 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:50.710 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:50.710 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:50.725 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:50.725 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:50.826 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:50.902 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:51.001 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:51.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:51.005 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:51.101 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:20:51.106 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:51.148 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:20:51.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3755594629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:51.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3755594629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:51.192 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:20:51.194 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:51.231 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:20:51.266 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:51.271 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:51.272 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:51.272 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:20:51.310 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:20:51.344 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:20:51.373 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:51.384 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:20:51.393 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:20:51.393 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:20:51.393 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:20:51.393 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:20:51.393 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:20:51.394 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:20:51.394 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:20:51.394 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:20:51.433 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3755594629' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:51.440 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:51.482 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:20:51.482 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:20:51.482 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:140311 9.f 9:ff5e59f6:::benchmark_data_smithi055_386467_object35651:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:16:08.274063+0000", 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 283.19999071500001, 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.081975385, 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.483 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 140311 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274063+0000", 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274063+0000", 2023-09-23T18:20:51.484 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274063+0000", 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274065+0000", 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9539999999999998e-06 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.485 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274070+0000", 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6969999999999998e-06 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274074+0000", 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7419999999999999e-06 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.486 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.279237+0000", 2023-09-23T18:20:51.487 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051623440000000001 2023-09-23T18:20:51.487 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.487 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.487 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:20:51.487 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.279244+0000", 2023-09-23T18:20:51.487 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5429999999999996e-06 2023-09-23T18:20:51.487 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.283291+0000", 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040470569999999997 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.283299+0000", 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3560000000000004e-06 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.489 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.339507+0000", 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0562083 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.339618+0000", 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111382 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.339717+0000", 2023-09-23T18:20:51.490 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8845999999999995e-05 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.348179+0000", 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0084614519999999995 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.353282+0000", 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051035469999999999 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.491 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.355898+0000", 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026161159999999999 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.355945+0000", 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6841000000000001e-05 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.356039+0000", 2023-09-23T18:20:51.492 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3576e-05 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:140325 9.f 9:f04bcc13:::benchmark_data_smithi055_386467_object35665:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e510)", 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:16:08.274424+0000", 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 283.19962945999998, 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0817564, 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.493 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 140325 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274424+0000", 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.494 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274424+0000", 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274424+0000", 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274428+0000", 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1579999999999998e-06 2023-09-23T18:20:51.495 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274437+0000", 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7469999999999994e-06 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.274442+0000", 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6319999999999996e-06 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.496 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.279295+0000", 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048526280000000003 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting_for_map not empty", 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.279302+0000", 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2400000000000001e-06 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.283320+0000", 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040175460000000003 2023-09-23T18:20:51.497 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:08.283327+0000", 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2640000000000002e-06 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.342129+0000", 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0588019900000001 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.498 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.342226+0000", 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7421000000000001e-05 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.342329+0000", 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102264 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.353252+0000", 2023-09-23T18:20:51.499 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010923042000000001 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.355890+0000", 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026384709999999999 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.356046+0000", 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000155501 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.500 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.356085+0000", 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8974000000000003e-05 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:16:09.356181+0000", 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6189999999999999e-05 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.501 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199290 9.3 9:cbcdf350:::benchmark_data_smithi055_386467_object94630:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.715714+0000", 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.75833963599999, 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2707276919999999, 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199290 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.502 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715714+0000", 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715714+0000", 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.503 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715714+0000", 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715717+0000", 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0599999999999999e-06 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715723+0000", 2023-09-23T18:20:51.504 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5889999999999999e-06 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.715728+0000", 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5909999999999998e-06 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716971+0000", 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012426220000000001 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.505 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717081+0000", 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000110571 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,7", 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717196+0000", 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115203 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.506 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.722675+0000", 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0054787109999999998 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.723307+0000", 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00063171099999999997 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.755425+0000", 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032118345999999999 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.507 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.755435+0000", 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6560000000000004e-06 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.986367+0000", 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2309317420000001 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.508 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.986442+0000", 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5337000000000001e-05 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199294 9.8 9:12efe5c6:::benchmark_data_smithi055_386467_object94634:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.716893+0000", 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.75716125599999, 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2851745619999999, 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.509 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199294 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716893+0000", 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.510 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716893+0000", 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716892+0000", 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716897+0000", 2023-09-23T18:20:51.511 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4859999999999997e-06 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716912+0000", 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5034e-05 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716919+0000", 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3840000000000002e-06 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.512 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716961+0000", 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2438999999999998e-05 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717078+0000", 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117118 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717194+0000", 2023-09-23T18:20:51.513 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011586699999999999 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.722355+0000", 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051609560000000004 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.753140+0000", 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.030785605000000001 2023-09-23T18:20:51.514 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.753151+0000", 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0198e-05 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.001993+0000", 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2488427980000001 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.515 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.002067+0000", 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3726999999999994e-05 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199295 9.5 9:ad8338c2:::benchmark_data_smithi055_386467_object94635:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.716932+0000", 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.75712197799999, 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.305777551, 2023-09-23T18:20:51.516 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199295 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716932+0000", 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.517 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716932+0000", 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716931+0000", 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.518 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716934+0000", 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0259999999999999e-06 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716941+0000", 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0599999999999996e-06 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716952+0000", 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1178e-05 2023-09-23T18:20:51.519 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.716992+0000", 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0167999999999999e-05 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717094+0000", 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000101951 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.520 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,7", 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.717198+0000", 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104474 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.723586+0000", 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0063876339999999997 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.724705+0000", 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001118775 2023-09-23T18:20:51.521 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771588+0000", 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.046883263000000001 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771596+0000", 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2490000000000001e-06 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.522 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022627+0000", 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2510307110000001 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022709+0000", 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2565999999999995e-05 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.523 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199297 9.6 9:6d14a21c:::benchmark_data_smithi055_386467_object94637:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.742942+0000", 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.731111545, 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1789063959999999, 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199297 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.524 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742942+0000", 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742942+0000", 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.525 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742941+0000", 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742948+0000", 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6979999999999998e-06 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742960+0000", 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1433999999999999e-05 2023-09-23T18:20:51.526 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742968+0000", 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7980000000000006e-06 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.751116+0000", 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.008147886 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.527 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.751128+0000", 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2094e-05 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.759543+0000", 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0084149099999999 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.759560+0000", 2023-09-23T18:20:51.528 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7638e-05 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.903201+0000", 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1436407420000001 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.903324+0000", 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123025 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.529 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.904060+0000", 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00073632600000000004 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.916453+0000", 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012393131 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.530 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921602+0000", 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051480709999999997 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921664+0000", 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2304999999999994e-05 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921726+0000", 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1891999999999996e-05 2023-09-23T18:20:51.531 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921849+0000", 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123044 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199298 9.7 9:e7115a81:::benchmark_data_smithi055_386467_object94638:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.742975+0000", 2023-09-23T18:20:51.532 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.73107907400001, 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3049147640000001, 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199298 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.533 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742975+0000", 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742975+0000", 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742974+0000", 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:51.534 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742977+0000", 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0570000000000001e-06 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742983+0000", 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0009999999999998e-06 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.535 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742987+0000", 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.9190000000000001e-06 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.772191+0000", 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.029203289 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.772200+0000", 2023-09-23T18:20:51.536 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1079999999999999e-06 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.015514+0000", 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.243313772 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.015648+0000", 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013492800000000001 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.537 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6,7", 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.015776+0000", 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012704 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.018114+0000", 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023388879999999999 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.538 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047670+0000", 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.029555352999999999 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047711+0000", 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0816999999999999e-05 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047773+0000", 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1932000000000003e-05 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.539 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047820+0000", 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7136999999999999e-05 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047890+0000", 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9951999999999997e-05 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.540 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199299 9.d 9:b372b2c2:::benchmark_data_smithi055_386467_object94639:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.742992+0000", 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.73106168800001, 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.280050796, 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199299 2023-09-23T18:20:51.541 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742992+0000", 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742992+0000", 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.542 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742992+0000", 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.742994+0000", 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6060000000000001e-06 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.743000+0000", 2023-09-23T18:20:51.543 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8429999999999998e-06 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.743004+0000", 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0570000000000003e-06 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771616+0000", 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028612047000000002 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.544 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771622+0000", 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9630000000000003e-06 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.985340+0000", 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2137177180000001 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.545 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.985469+0000", 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012850700000000001 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.985617+0000", 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000148553 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.987979+0000", 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023620759999999998 2023-09-23T18:20:51.546 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022765+0000", 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.034785592999999997 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022835+0000", 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0127000000000001e-05 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.547 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022868+0000", 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3479999999999998e-05 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022920+0000", 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1329000000000003e-05 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023043+0000", 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012324699999999999 2023-09-23T18:20:51.548 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199303 9.f 9:f765a439:::benchmark_data_smithi055_386467_object94643:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.756527+0000", 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.71752695800001, 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2992920190000001, 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.549 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199303 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.756527+0000", 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.550 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.756527+0000", 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.756526+0000", 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.756531+0000", 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1900000000000003e-06 2023-09-23T18:20:51.551 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757137+0000", 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00060600400000000005 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757147+0000", 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8700000000000004e-06 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.552 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.772195+0000", 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.015048598 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.772201+0000", 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7409999999999998e-06 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.045405+0000", 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2732037970000001 2023-09-23T18:20:51.553 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.045497+0000", 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1938e-05 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.045590+0000", 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2592999999999999e-05 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.554 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.047756+0000", 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0021669079999999999 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.048160+0000", 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00040372800000000001 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.048189+0000", 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9183000000000002e-05 2023-09-23T18:20:51.555 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.055687+0000", 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0074980619999999998 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.055734+0000", 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6418000000000002e-05 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.556 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.055819+0000", 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5177000000000001e-05 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199304 9.d 9:bc343161:::benchmark_data_smithi055_386467_object94644:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.757155+0000", 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.71689910800001, 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.266180844, 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.557 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199304 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757155+0000", 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.558 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757155+0000", 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757154+0000", 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757159+0000", 2023-09-23T18:20:51.559 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8539999999999996e-06 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757168+0000", 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6819999999999992e-06 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.757173+0000", 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9710000000000003e-06 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.560 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771640+0000", 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014467617 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771648+0000", 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7659999999999998e-06 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.561 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.988003+0000", 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.216355106 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.988116+0000", 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011312899999999999 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,4,7", 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.988249+0000", 2023-09-23T18:20:51.562 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000133106 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.992664+0000", 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.004414327 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.022771+0000", 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.030107643999999999 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.563 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023122+0000", 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00035060100000000003 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023168+0000", 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5799e-05 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023219+0000", 2023-09-23T18:20:51.564 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1851999999999998e-05 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023336+0000", 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000116197 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199306 9.5 9:a2b69264:::benchmark_data_smithi055_386467_object94646:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.565 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.765909+0000", 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.708144748, 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2612426189999999, 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199306 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.566 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765909+0000", 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765909+0000", 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765908+0000", 2023-09-23T18:20:51.567 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765914+0000", 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3640000000000004e-06 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765926+0000", 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1460999999999999e-05 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.568 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765935+0000", 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7630000000000006e-06 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771666+0000", 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0057316600000000004 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.771672+0000", 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8379999999999998e-06 2023-09-23T18:20:51.569 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023404+0000", 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.251732098 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023529+0000", 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012490900000000001 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.570 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.023682+0000", 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015258099999999999 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.026185+0000", 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025030669999999999 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.026377+0000", 2023-09-23T18:20:51.571 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019275800000000001 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.026900+0000", 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00052258399999999998 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.026994+0000", 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3902999999999999e-05 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.572 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.027043+0000", 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8639e-05 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.027152+0000", 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000109197 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.573 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199307 9.6 9:60908162:::benchmark_data_smithi055_386467_object94647:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.765940+0000", 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.70811382299999, 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1561716029999998, 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199307 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.574 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765940+0000", 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765940+0000", 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.575 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765940+0000", 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765943+0000", 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1099999999999999e-06 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.576 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765949+0000", 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2169999999999999e-06 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765953+0000", 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.788e-06 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.767010+0000", 2023-09-23T18:20:51.577 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001057058 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.767020+0000", 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9089999999999995e-06 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.759589+0000", 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99256918599999999 2023-09-23T18:20:51.578 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.759597+0000", 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2090000000000005e-06 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.906374+0000", 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.146776995 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.579 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.906482+0000", 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000107633 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,6", 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.906617+0000", 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000135341 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.916497+0000", 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.009879489 2023-09-23T18:20:51.580 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921610+0000", 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051130739999999996 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.921949+0000", 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00033926300000000002 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.581 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.922000+0000", 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0912999999999999e-05 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.922112+0000", 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111897 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.582 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199308 9.8 9:1df43a87:::benchmark_data_smithi055_386467_object94648:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.765958+0000", 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 211.70809636499999, 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.244589932, 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199308 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.583 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765958+0000", 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765958+0000", 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.584 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765957+0000", 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765960+0000", 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.6749999999999998e-06 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765965+0000", 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6080000000000003e-06 2023-09-23T18:20:51.585 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.765969+0000", 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8550000000000004e-06 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.770519+0000", 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045492950000000001 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.586 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.770535+0000", 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6084000000000001e-05 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.002452+0000", 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2319171680000001 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.004743+0000", 2023-09-23T18:20:51.587 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022912919999999999 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1,7", 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.004876+0000", 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013298100000000001 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.007521+0000", 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026448959999999999 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.588 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.007529+0000", 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8650000000000001e-06 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.007964+0000", 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000434976 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.010368+0000", 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002403793 2023-09-23T18:20:51.589 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.010420+0000", 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2784999999999997e-05 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:21.010547+0000", 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127081 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.590 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67151 10.3 10:c85af3e6:::benchmark_data_smithi055_425050_object67150:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.355657+0000", 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 126.118396813, 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2389400689999999, 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:51.591 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67151 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355657+0000", 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355657+0000", 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.592 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355655+0000", 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355688+0000", 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2824999999999998e-05 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.593 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355702+0000", 2023-09-23T18:20:51.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3193999999999999e-05 2023-09-23T18:20:51.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355707+0000", 2023-09-23T18:20:51.594 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.976e-06 2023-09-23T18:20:51.594 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.594 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.594 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.594 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355770+0000", 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2339000000000001e-05 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.355958+0000", 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00018780000000000001 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.356087+0000", 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000129383 2023-09-23T18:20:51.595 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.358394+0000", 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0023073450000000001 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.361318+0000", 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029238490000000001 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.596 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.424224+0000", 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.062906161000000002 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.424237+0000", 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3063e-05 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426092+0000", 2023-09-23T18:20:51.597 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0018549569999999 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426106+0000", 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3603e-05 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.594476+0000", 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.16837009 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.598 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.594597+0000", 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000121097 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67164 10.3 10:c93b5b80:::benchmark_data_smithi055_425050_object67163:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.428830+0000", 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 126.04522406, 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2067956679999998, 2023-09-23T18:20:51.599 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67164 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428830+0000", 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.600 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428830+0000", 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428829+0000", 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.601 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428877+0000", 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8142000000000001e-05 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428889+0000", 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.253e-05 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428895+0000", 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9349999999999999e-06 2023-09-23T18:20:51.602 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428931+0000", 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5407999999999997e-05 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.428938+0000", 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1430000000000001e-06 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.603 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426131+0000", 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99719329800000001 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426138+0000", 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8120000000000002e-06 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.597584+0000", 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1714459260000001 2023-09-23T18:20:51.604 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.597724+0000", 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014053199999999999 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.597929+0000", 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000204448 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.605 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604471+0000", 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0065424530000000002 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604486+0000", 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4440000000000001e-05 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.632642+0000", 2023-09-23T18:20:51.606 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.028155961 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.632690+0000", 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7898000000000001e-05 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.635626+0000", 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029359920000000001 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.607 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67171 10.3 10:cede541a:::benchmark_data_smithi055_425050_object67170:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.435663+0000", 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 126.038391264, 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2024671709999999, 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:51.608 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67171 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435663+0000", 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435663+0000", 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.609 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435662+0000", 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435759+0000", 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6586e-05 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.610 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435763+0000", 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4680000000000003e-06 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435766+0000", 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0400000000000001e-06 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435799+0000", 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2561000000000002e-05 2023-09-23T18:20:51.611 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435808+0000", 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3749999999999992e-06 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426159+0000", 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99035037599999998 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.612 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426179+0000", 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0211999999999999e-05 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.600032+0000", 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.173853203 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.600141+0000", 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0001093 2023-09-23T18:20:51.613 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.600325+0000", 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000183265 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604495+0000", 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041700979999999997 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.614 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604538+0000", 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2914000000000003e-05 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.635675+0000", 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.031136938999999999 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.635729+0000", 2023-09-23T18:20:51.615 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4243000000000002e-05 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.638130+0000", 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002400976 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67175 10.3 10:c0e0e754:::benchmark_data_smithi055_425050_object67174:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:51.616 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.435944+0000", 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 126.03810955900001, 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.2040770759999999, 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67175 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.617 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435944+0000", 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435944+0000", 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.435944+0000", 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:20:51.618 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.436036+0000", 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.2362999999999996e-05 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.436045+0000", 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.4010000000000003e-06 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.619 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.436050+0000", 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1229999999999999e-06 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.436098+0000", 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7287000000000003e-05 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.436111+0000", 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3691e-05 2023-09-23T18:20:51.620 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426201+0000", 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99008958800000002 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426209+0000", 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5310000000000004e-06 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.621 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.602179+0000", 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.17597065 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.602335+0000", 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015603000000000001 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.602509+0000", 2023-09-23T18:20:51.622 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00017391300000000001 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604574+0000", 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020652299999999999 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604583+0000", 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8980000000000002e-06 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.623 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.624 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.624 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.638159+0000", 2023-09-23T18:20:51.624 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.033576175 2023-09-23T18:20:51.624 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.638195+0000", 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5228999999999998e-05 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.640021+0000", 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001826725 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.625 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67180 10.3 10:cb0581d8:::benchmark_data_smithi055_425050_object67179:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.445326+0000", 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 126.028728161, 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1972762810000002, 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:51.626 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67180 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445326+0000", 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445326+0000", 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.627 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445325+0000", 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445438+0000", 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113678 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.628 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445449+0000", 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0993e-05 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445454+0000", 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1089999999999997e-06 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445502+0000", 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7259999999999998e-05 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.629 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.445508+0000", 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.297e-06 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426229+0000", 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.98072124400000005 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.630 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426236+0000", 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9489999999999999e-06 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604614+0000", 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1783773929999999 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604709+0000", 2023-09-23T18:20:51.631 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5921999999999994e-05 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.604853+0000", 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014312299999999999 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.611827+0000", 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0069741430000000004 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.632 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.612776+0000", 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00094902099999999998 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.640063+0000", 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.027287227000000001 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.640112+0000", 2023-09-23T18:20:51.633 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9088000000000002e-05 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.642602+0000", 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024899879999999998 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67205 10.3 10:ce268af2:::benchmark_data_smithi055_425050_object67204:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:51.634 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.474799+0000", 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 125.999254647, 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.170461601, 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67205 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.635 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474799+0000", 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474799+0000", 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474797+0000", 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999971 2023-09-23T18:20:51.636 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474893+0000", 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.6348000000000007e-05 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474902+0000", 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0440000000000001e-06 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.637 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474906+0000", 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0759999999999996e-06 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474968+0000", 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1721000000000001e-05 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.474981+0000", 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.289e-05 2023-09-23T18:20:51.638 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426255+0000", 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.95127468599999998 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426262+0000", 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8589999999999999e-06 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.639 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.606693+0000", 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1804307679999999 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.606822+0000", 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012950400000000001 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.607021+0000", 2023-09-23T18:20:51.640 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019809800000000001 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.611833+0000", 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0048126979999999998 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.612807+0000", 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00097327799999999995 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.641 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.642656+0000", 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.02984966 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.642717+0000", 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0656000000000003e-05 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.645261+0000", 2023-09-23T18:20:51.642 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025439849999999999 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:67218 10.3 10:c5b8088f:::benchmark_data_smithi055_425050_object67217:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:18:45.502196+0000", 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 125.97185813199999, 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1458930650000001, 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:20:51.643 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 67218 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502196+0000", 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.644 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502196+0000", 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502194+0000", 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502247+0000", 2023-09-23T18:20:51.645 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2800000000000003e-05 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502262+0000", 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4459e-05 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502267+0000", 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2599999999999996e-06 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.646 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502317+0000", 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0407e-05 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:45.502324+0000", 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7889999999999997e-06 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426283+0000", 2023-09-23T18:20:51.647 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.92395870300000005 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:46.426290+0000", 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8480000000000003e-06 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.609467+0000", 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.183177476 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.648 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.609609+0000", 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000141927 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.609777+0000", 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016761200000000001 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.612002+0000", 2023-09-23T18:20:51.649 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022256839999999999 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.612942+0000", 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00093938799999999999 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.645322+0000", 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032380474999999999 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.650 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.645382+0000", 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0148000000000001e-05 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:18:47.648089+0000", 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027063980000000001 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:20:51.651 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:20:51.652 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:20:51.652 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:51.652 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:51.653 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:51.753 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:51.868 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:51.929 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:20:52.059 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:20:52.708 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:52 smithi055 ceph-mon[179668]: pgmap v1483: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 15502/117669 objects degraded (13.174%); 211/117669 objects misplaced (0.179%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:20:52.709 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:52.709 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:52.709 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:52 smithi055 ceph-mon[184174]: pgmap v1483: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 15502/117669 objects degraded (13.174%); 211/117669 objects misplaced (0.179%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:20:52.710 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:52.710 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:52.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:52 smithi099 ceph-mon[167076]: pgmap v1483: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.4 GiB data, 12 GiB used, 524 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 15502/117669 objects degraded (13.174%); 211/117669 objects misplaced (0.179%); 1.3 MiB/s, 20 objects/s recovering 2023-09-23T18:20:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:20:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:20:52.837 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:52.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:52.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:52.951 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:53.060 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:53.096 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:53.186 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:53.194 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:53.204 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:53.204 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:53.305 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:53.385 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:53.650 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15502/117669 objects degraded (13.174%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:53.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15502/117669 objects degraded (13.174%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:53.656 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:53.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:53.669 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:53.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:53.814 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15502/117669 objects degraded (13.174%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:53.817 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:53.857 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:53.860 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:53.860 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:53.961 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:54.072 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:54.215 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:20:54.223 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:20:54.403 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:54.407 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:54.407 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:54.509 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:54.615 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:54 smithi055 ceph-mon[184174]: pgmap v1484: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 883 op/s; 15487/107301 objects degraded (14.433%); 2.4 MiB/s, 42 objects/s recovering 2023-09-23T18:20:54.615 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:54 smithi055 ceph-mon[179668]: pgmap v1484: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 883 op/s; 15487/107301 objects degraded (14.433%); 2.4 MiB/s, 42 objects/s recovering 2023-09-23T18:20:54.627 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:20:54.637 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:54.797 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:54.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:54.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:54.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:54 smithi099 ceph-mon[167076]: pgmap v1484: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 883 op/s; 15487/107301 objects degraded (14.433%); 2.4 MiB/s, 42 objects/s recovering 2023-09-23T18:20:54.902 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:20:54.976 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1484,"stamp":"2023-09-23T18:20:53.182209+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":2346662588,"num_objects":35767,"num_object_clones":0,"num_object_copies":107301,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15487,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35767,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":171979,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8066,"num_bytes_recovered":502226569,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83190,"ondisk_log_size":83190,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":12193816,"kb_used_data":8665576,"kb_used_omap":40,"kb_used_meta":3528087,"kb_avail":550277096,"statfs":{"total":575970213888,"available":563483746304,"internally_reserved":0,"allocated":8873549824,"data_stored":8858713279,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3612761402},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":6,"apply_latency_ms":6,"commit_latency_ns":6000000,"apply_latency_ns":6000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-579076096,"num_objects":-8836,"num_object_clones":0,"num_object_copies":-26508,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":69,"num_objects_misplaced":-498,"num_objects_unfound":0,"num_objects_dirty":-8836,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8836,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":429,"num_bytes_recovered":25362432,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.005381"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":905,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749332+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:20:40.749332+0000","last_peered":"2023-09-23T18:20:40.749332+0000","last_clean":"2023-09-23T18:20:40.749332+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:20:40.749332+0000","last_undegraded":"2023-09-23T18:20:40.749332+0000","last_fullsized":"2023-09-23T18:20:40.749332+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":276,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.767901+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:20:40.767901+0000","last_peered":"2023-09-23T18:20:40.767901+0000","last_clean":"2023-09-23T18:20:40.767901+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:20:40.767901+0000","last_undegraded":"2023-09-23T18:20:40.767901+0000","last_fullsized":"2023-09-23T18:20:40.767901+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":33,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.759741+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:20:40.759741+0000","last_peered":"2023-09-23T18:20:40.759741+0000","last_clean":"2023-09-23T18:20:40.759741+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:20:40.759741+0000","last_undegraded":"2023-09-23T18:20:40.759741+0000","last_fullsized":"2023-09-23T18:20:40.759741+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":911,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749522+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:20:40.749522+0000","last_peered":"2023-09-23T18:20:40.749522+0000","last_clean":"2023-09-23T18:20:40.749522+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:20:40.749522+0000","last_undegraded":"2023-09-23T18:20:40.749522+0000","last_fullsized":"2023-09-23T18:20:40.749522+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":358,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.759776+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:20:40.759776+0000","last_peered":"2023-09-23T18:20:40.759776+0000","last_clean":"2023-09-23T18:20:40.759776+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:20:40.759776+0000","last_undegraded":"2023-09-23T18:20:40.759776+0000","last_fullsized":"2023-09-23T18:20:40.759776+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":678,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.819124+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:20:46.819124+0000","last_peered":"2023-09-23T18:20:46.819124+0000","last_clean":"2023-09-23T18:20:46.819124+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:20:46.819124+0000","last_undegraded":"2023-09-23T18:20:46.819124+0000","last_fullsized":"2023-09-23T18:20:46.819124+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":928,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748612+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:20:40.748612+0000","last_peered":"2023-09-23T18:20:40.748612+0000","last_clean":"2023-09-23T18:20:40.748612+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:40.748612+0000","last_undegraded":"2023-09-23T18:20:40.748612+0000","last_fullsized":"2023-09-23T18:20:40.748612+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":33,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.758945+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:40.758945+0000","last_peered":"2023-09-23T18:20:40.758945+0000","last_clean":"2023-09-23T18:20:40.758945+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:20:40.758945+0000","last_undegraded":"2023-09-23T18:20:40.758945+0000","last_fullsized":"2023-09-23T18:20:40.758945+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":193,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762605+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:20:40.762605+0000","last_peered":"2023-09-23T18:20:40.762605+0000","last_clean":"2023-09-23T18:20:40.762605+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:20:40.762605+0000","last_undegraded":"2023-09-23T18:20:40.762605+0000","last_fullsized":"2023-09-23T18:20:40.762605+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1051,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749126+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:20:40.749126+0000","last_peered":"2023-09-23T18:20:40.749126+0000","last_clean":"2023-09-23T18:20:40.749126+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:20:40.749126+0000","last_undegraded":"2023-09-23T18:20:40.749126+0000","last_fullsized":"2023-09-23T18:20:40.749126+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":127,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749160+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:20:40.749160+0000","last_peered":"2023-09-23T18:20:40.749160+0000","last_clean":"2023-09-23T18:20:40.749160+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:20:40.749160+0000","last_undegraded":"2023-09-23T18:20:40.749160+0000","last_fullsized":"2023-09-23T18:20:40.749160+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":207,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:41.335466+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:20:41.335466+0000","last_peered":"2023-09-23T18:20:41.335466+0000","last_clean":"2023-09-23T18:20:41.335466+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:20:41.335466+0000","last_undegraded":"2023-09-23T18:20:41.335466+0000","last_fullsized":"2023-09-23T18:20:41.335466+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":828,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.763664+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:20:40.763664+0000","last_peered":"2023-09-23T18:20:40.763664+0000","last_clean":"2023-09-23T18:20:40.763664+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:20:40.763664+0000","last_undegraded":"2023-09-23T18:20:40.763664+0000","last_fullsized":"2023-09-23T18:20:40.763664+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":163,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749077+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:20:40.749077+0000","last_peered":"2023-09-23T18:20:40.749077+0000","last_clean":"2023-09-23T18:20:40.749077+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:20:40.749077+0000","last_undegraded":"2023-09-23T18:20:40.749077+0000","last_fullsized":"2023-09-23T18:20:40.749077+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":162,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760577+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:20:40.760577+0000","last_peered":"2023-09-23T18:20:40.760577+0000","last_clean":"2023-09-23T18:20:40.760577+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:20:40.760577+0000","last_undegraded":"2023-09-23T18:20:40.760577+0000","last_fullsized":"2023-09-23T18:20:40.760577+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":962,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749205+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:20:40.749205+0000","last_peered":"2023-09-23T18:20:40.749205+0000","last_clean":"2023-09-23T18:20:40.749205+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:20:40.749205+0000","last_undegraded":"2023-09-23T18:20:40.749205+0000","last_fullsized":"2023-09-23T18:20:40.749205+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":411,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760120+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:20:40.760120+0000","last_peered":"2023-09-23T18:20:40.760120+0000","last_clean":"2023-09-23T18:20:40.760120+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:20:40.760120+0000","last_undegraded":"2023-09-23T18:20:40.760120+0000","last_fullsized":"2023-09-23T18:20:40.760120+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":430,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762028+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:20:40.762028+0000","last_peered":"2023-09-23T18:20:40.762028+0000","last_clean":"2023-09-23T18:20:40.762028+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:20:40.762028+0000","last_undegraded":"2023-09-23T18:20:40.762028+0000","last_fullsized":"2023-09-23T18:20:40.762028+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":991,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.765675+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:20:40.765675+0000","last_peered":"2023-09-23T18:20:40.765675+0000","last_clean":"2023-09-23T18:20:40.765675+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:20:40.765675+0000","last_undegraded":"2023-09-23T18:20:40.765675+0000","last_fullsized":"2023-09-23T18:20:40.765675+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":169,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.764162+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:20:40.764162+0000","last_peered":"2023-09-23T18:20:40.764162+0000","last_clean":"2023-09-23T18:20:40.764162+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:20:40.764162+0000","last_undegraded":"2023-09-23T18:20:40.764162+0000","last_fullsized":"2023-09-23T18:20:40.764162+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":436,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761272+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:20:40.761272+0000","last_peered":"2023-09-23T18:20:40.761272+0000","last_clean":"2023-09-23T18:20:40.761272+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:20:40.761272+0000","last_undegraded":"2023-09-23T18:20:40.761272+0000","last_fullsized":"2023-09-23T18:20:40.761272+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":843,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748926+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:20:40.748926+0000","last_peered":"2023-09-23T18:20:40.748926+0000","last_clean":"2023-09-23T18:20:40.748926+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:20:40.748926+0000","last_undegraded":"2023-09-23T18:20:40.748926+0000","last_fullsized":"2023-09-23T18:20:40.748926+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":436,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761098+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:20:40.761098+0000","last_peered":"2023-09-23T18:20:40.761098+0000","last_clean":"2023-09-23T18:20:40.761098+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:20:40.761098+0000","last_undegraded":"2023-09-23T18:20:40.761098+0000","last_fullsized":"2023-09-23T18:20:40.761098+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":114,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748826+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:20:40.748826+0000","last_peered":"2023-09-23T18:20:40.748826+0000","last_clean":"2023-09-23T18:20:40.748826+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:20:40.748826+0000","last_undegraded":"2023-09-23T18:20:40.748826+0000","last_fullsized":"2023-09-23T18:20:40.748826+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":814,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748501+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:20:40.748501+0000","last_peered":"2023-09-23T18:20:40.748501+0000","last_clean":"2023-09-23T18:20:40.748501+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:20:40.748501+0000","last_undegraded":"2023-09-23T18:20:40.748501+0000","last_fullsized":"2023-09-23T18:20:40.748501+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":132,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.773499+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:20:40.773499+0000","last_peered":"2023-09-23T18:20:40.773499+0000","last_clean":"2023-09-23T18:20:40.773499+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:20:40.773499+0000","last_undegraded":"2023-09-23T18:20:40.773499+0000","last_fullsized":"2023-09-23T18:20:40.773499+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":127,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749294+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:20:40.749294+0000","last_peered":"2023-09-23T18:20:40.749294+0000","last_clean":"2023-09-23T18:20:40.749294+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:20:40.749294+0000","last_undegraded":"2023-09-23T18:20:40.749294+0000","last_fullsized":"2023-09-23T18:20:40.749294+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":752,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762882+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:20:40.762882+0000","last_peered":"2023-09-23T18:20:40.762882+0000","last_clean":"2023-09-23T18:20:40.762882+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:20:40.762882+0000","last_undegraded":"2023-09-23T18:20:40.762882+0000","last_fullsized":"2023-09-23T18:20:40.762882+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":204,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760484+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:20:40.760484+0000","last_peered":"2023-09-23T18:20:40.760484+0000","last_clean":"2023-09-23T18:20:40.760484+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:20:40.760484+0000","last_undegraded":"2023-09-23T18:20:40.760484+0000","last_fullsized":"2023-09-23T18:20:40.760484+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":215,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760188+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:20:40.760188+0000","last_peered":"2023-09-23T18:20:40.760188+0000","last_clean":"2023-09-23T18:20:40.760188+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:20:40.760188+0000","last_undegraded":"2023-09-23T18:20:40.760188+0000","last_fullsized":"2023-09-23T18:20:40.760188+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":848,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760581+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:20:40.760581+0000","last_peered":"2023-09-23T18:20:40.760581+0000","last_clean":"2023-09-23T18:20:40.760581+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:20:40.760581+0000","last_undegraded":"2023-09-23T18:20:40.760581+0000","last_fullsized":"2023-09-23T18:20:40.760581+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":757,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760511+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:20:40.760511+0000","last_peered":"2023-09-23T18:20:40.760511+0000","last_clean":"2023-09-23T18:20:40.760511+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:20:40.760511+0000","last_undegraded":"2023-09-23T18:20:40.760511+0000","last_fullsized":"2023-09-23T18:20:40.760511+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":33,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762555+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:20:40.762555+0000","last_peered":"2023-09-23T18:20:40.762555+0000","last_clean":"2023-09-23T18:20:40.762555+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:20:40.762555+0000","last_undegraded":"2023-09-23T18:20:40.762555+0000","last_fullsized":"2023-09-23T18:20:40.762555+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"617'10834","reported_seq":10885,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:51.105822+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:20:51.105822+0000","last_peered":"2023-09-23T18:20:51.105822+0000","last_clean":"2023-09-23T18:20:51.105822+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:20:51.105822+0000","last_undegraded":"2023-09-23T18:20:51.105822+0000","last_fullsized":"2023-09-23T18:20:51.105822+0000","mapping_epoch":574,"log_start":"602'5300","ondisk_log_start":"602'5300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5534,"log_dups_size":0,"ondisk_log_size":5534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":146800640,"num_objects":2240,"num_object_clones":0,"num_object_copies":6720,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2240,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10834,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":882,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762249+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:20:40.762249+0000","last_peered":"2023-09-23T18:20:40.762249+0000","last_clean":"2023-09-23T18:20:40.762249+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:20:40.762249+0000","last_undegraded":"2023-09-23T18:20:40.762249+0000","last_fullsized":"2023-09-23T18:20:40.762249+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":253,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.767469+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:20:40.767469+0000","last_peered":"2023-09-23T18:20:40.767469+0000","last_clean":"2023-09-23T18:20:40.767469+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:20:40.767469+0000","last_undegraded":"2023-09-23T18:20:40.767469+0000","last_fullsized":"2023-09-23T18:20:40.767469+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":127,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748600+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:20:40.748600+0000","last_peered":"2023-09-23T18:20:40.748600+0000","last_clean":"2023-09-23T18:20:40.748600+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:20:40.748600+0000","last_undegraded":"2023-09-23T18:20:40.748600+0000","last_fullsized":"2023-09-23T18:20:40.748600+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"617'10542","reported_seq":10603,"reported_epoch":617,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:51.205643+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:20:51.205643+0000","last_peered":"2023-09-23T18:20:51.205643+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:20:51.205643+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"604'6100","ondisk_log_start":"604'6100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4442,"log_dups_size":0,"ondisk_log_size":4442,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140902400,"num_objects":2150,"num_object_clones":0,"num_object_copies":6450,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2150,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10542,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"5,6","objects":-1998}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":736,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760570+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:20:40.760570+0000","last_peered":"2023-09-23T18:20:40.760570+0000","last_clean":"2023-09-23T18:20:40.760570+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:20:40.760570+0000","last_undegraded":"2023-09-23T18:20:40.760570+0000","last_fullsized":"2023-09-23T18:20:40.760570+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":254,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.767466+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:20:40.767466+0000","last_peered":"2023-09-23T18:20:40.767466+0000","last_clean":"2023-09-23T18:20:40.767466+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:20:40.767466+0000","last_undegraded":"2023-09-23T18:20:40.767466+0000","last_fullsized":"2023-09-23T18:20:40.767466+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":233,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.766345+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:20:40.766345+0000","last_peered":"2023-09-23T18:20:40.766345+0000","last_clean":"2023-09-23T18:20:40.766345+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:20:40.766345+0000","last_undegraded":"2023-09-23T18:20:40.766345+0000","last_fullsized":"2023-09-23T18:20:40.766345+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"617'10590","reported_seq":17601,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:50.258692+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:20:50.258692+0000","last_peered":"2023-09-23T18:20:50.258692+0000","last_clean":"2023-09-23T18:20:50.258692+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:20:50.258692+0000","last_undegraded":"2023-09-23T18:20:50.258692+0000","last_fullsized":"2023-09-23T18:20:50.258692+0000","mapping_epoch":605,"log_start":"603'5700","ondisk_log_start":"603'5700","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4890,"log_dups_size":0,"ondisk_log_size":4890,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147718144,"num_objects":2254,"num_object_clones":0,"num_object_copies":6762,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2254,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10590,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":632,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762506+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:20:40.762506+0000","last_peered":"2023-09-23T18:20:40.762506+0000","last_clean":"2023-09-23T18:20:40.762506+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:20:40.762506+0000","last_undegraded":"2023-09-23T18:20:40.762506+0000","last_fullsized":"2023-09-23T18:20:40.762506+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":542,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749215+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:20:40.749215+0000","last_peered":"2023-09-23T18:20:40.749215+0000","last_clean":"2023-09-23T18:20:40.749215+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:20:40.749215+0000","last_undegraded":"2023-09-23T18:20:40.749215+0000","last_fullsized":"2023-09-23T18:20:40.749215+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":193,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761893+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:20:40.761893+0000","last_peered":"2023-09-23T18:20:40.761893+0000","last_clean":"2023-09-23T18:20:40.761893+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:20:40.761893+0000","last_undegraded":"2023-09-23T18:20:40.761893+0000","last_fullsized":"2023-09-23T18:20:40.761893+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"617'10832","reported_seq":10882,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:51.372612+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:20:51.372612+0000","last_peered":"2023-09-23T18:20:51.372612+0000","last_clean":"2023-09-23T18:20:51.372612+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:20:51.372612+0000","last_undegraded":"2023-09-23T18:20:51.372612+0000","last_fullsized":"2023-09-23T18:20:51.372612+0000","mapping_epoch":574,"log_start":"603'5700","ondisk_log_start":"603'5700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5132,"log_dups_size":0,"ondisk_log_size":5132,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148111360,"num_objects":2260,"num_object_clones":0,"num_object_copies":6780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10832,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":648,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760276+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:20:40.760276+0000","last_peered":"2023-09-23T18:20:40.760276+0000","last_clean":"2023-09-23T18:20:40.760276+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:20:40.760276+0000","last_undegraded":"2023-09-23T18:20:40.760276+0000","last_fullsized":"2023-09-23T18:20:40.760276+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":141,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749142+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:20:40.749142+0000","last_peered":"2023-09-23T18:20:40.749142+0000","last_clean":"2023-09-23T18:20:40.749142+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:20:40.749142+0000","last_undegraded":"2023-09-23T18:20:40.749142+0000","last_fullsized":"2023-09-23T18:20:40.749142+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":454,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.770344+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:20:40.770344+0000","last_peered":"2023-09-23T18:20:40.770344+0000","last_clean":"2023-09-23T18:20:40.770344+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:40.770344+0000","last_undegraded":"2023-09-23T18:20:40.770344+0000","last_fullsized":"2023-09-23T18:20:40.770344+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"617'10725","reported_seq":10778,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:48.904523+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:20:48.904523+0000","last_peered":"2023-09-23T18:20:48.904523+0000","last_clean":"2023-09-23T18:20:48.904523+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:20:48.904523+0000","last_undegraded":"2023-09-23T18:20:48.904523+0000","last_fullsized":"2023-09-23T18:20:48.904523+0000","mapping_epoch":574,"log_start":"604'6300","ondisk_log_start":"604'6300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4425,"log_dups_size":0,"ondisk_log_size":4425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":151977984,"num_objects":2319,"num_object_clones":0,"num_object_copies":6957,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2319,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10725,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1099,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748767+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:20:40.748767+0000","last_peered":"2023-09-23T18:20:40.748767+0000","last_clean":"2023-09-23T18:20:40.748767+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:20:40.748767+0000","last_undegraded":"2023-09-23T18:20:40.748767+0000","last_fullsized":"2023-09-23T18:20:40.748767+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":354,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.763089+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:20:40.763089+0000","last_peered":"2023-09-23T18:20:40.763089+0000","last_clean":"2023-09-23T18:20:40.763089+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:20:40.763089+0000","last_undegraded":"2023-09-23T18:20:40.763089+0000","last_fullsized":"2023-09-23T18:20:40.763089+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":161,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.763818+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:20:40.763818+0000","last_peered":"2023-09-23T18:20:40.763818+0000","last_clean":"2023-09-23T18:20:40.763818+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:20:40.763818+0000","last_undegraded":"2023-09-23T18:20:40.763818+0000","last_fullsized":"2023-09-23T18:20:40.763818+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"617'10770","reported_seq":12319,"reported_epoch":617,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:51.192641+0000","last_change":"2023-09-23T18:20:34.894639+0000","last_active":"2023-09-23T18:20:51.192641+0000","last_peered":"2023-09-23T18:20:51.192641+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:20:51.192641+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"602'5300","ondisk_log_start":"602'5300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5470,"log_dups_size":0,"ondisk_log_size":5470,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148111360,"num_objects":2260,"num_object_clones":0,"num_object_copies":6780,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3482,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2260,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10770,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":744,"num_bytes_recovered":46006272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"4,5","objects":-1222}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":886,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748850+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:20:40.748850+0000","last_peered":"2023-09-23T18:20:40.748850+0000","last_clean":"2023-09-23T18:20:40.748850+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:20:40.748850+0000","last_undegraded":"2023-09-23T18:20:40.748850+0000","last_fullsized":"2023-09-23T18:20:40.748850+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":169,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761809+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:20:40.761809+0000","last_peered":"2023-09-23T18:20:40.761809+0000","last_clean":"2023-09-23T18:20:40.761809+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:20:40.761809+0000","last_undegraded":"2023-09-23T18:20:40.761809+0000","last_fullsized":"2023-09-23T18:20:40.761809+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":468,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762387+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:20:40.762387+0000","last_peered":"2023-09-23T18:20:40.762387+0000","last_clean":"2023-09-23T18:20:40.762387+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:20:40.762387+0000","last_undegraded":"2023-09-23T18:20:40.762387+0000","last_fullsized":"2023-09-23T18:20:40.762387+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"617'10722","reported_seq":10773,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:51.223392+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:20:51.223392+0000","last_peered":"2023-09-23T18:20:51.223392+0000","last_clean":"2023-09-23T18:20:51.223392+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:20:51.223392+0000","last_undegraded":"2023-09-23T18:20:51.223392+0000","last_fullsized":"2023-09-23T18:20:51.223392+0000","mapping_epoch":574,"log_start":"604'6300","ondisk_log_start":"604'6300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4422,"log_dups_size":0,"ondisk_log_size":4422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":152698880,"num_objects":2330,"num_object_clones":0,"num_object_copies":6990,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2330,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10722,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":888,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749600+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:20:40.749600+0000","last_peered":"2023-09-23T18:20:40.749600+0000","last_clean":"2023-09-23T18:20:40.749600+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:20:40.749600+0000","last_undegraded":"2023-09-23T18:20:40.749600+0000","last_fullsized":"2023-09-23T18:20:40.749600+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":453,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748616+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:20:40.748616+0000","last_peered":"2023-09-23T18:20:40.748616+0000","last_clean":"2023-09-23T18:20:40.748616+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:20:40.748616+0000","last_undegraded":"2023-09-23T18:20:40.748616+0000","last_fullsized":"2023-09-23T18:20:40.748616+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":229,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761389+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:20:40.761389+0000","last_peered":"2023-09-23T18:20:40.761389+0000","last_clean":"2023-09-23T18:20:40.761389+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:20:40.761389+0000","last_undegraded":"2023-09-23T18:20:40.761389+0000","last_fullsized":"2023-09-23T18:20:40.761389+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"617'10817","reported_seq":10868,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:52.230082+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:20:52.230082+0000","last_peered":"2023-09-23T18:20:52.230082+0000","last_clean":"2023-09-23T18:20:52.230082+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:20:52.230082+0000","last_undegraded":"2023-09-23T18:20:52.230082+0000","last_fullsized":"2023-09-23T18:20:52.230082+0000","mapping_epoch":574,"log_start":"604'6200","ondisk_log_start":"604'6200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4617,"log_dups_size":0,"ondisk_log_size":4617,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145293312,"num_objects":2217,"num_object_clones":0,"num_object_copies":6651,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2217,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10817,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":869,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748873+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:20:40.748873+0000","last_peered":"2023-09-23T18:20:40.748873+0000","last_clean":"2023-09-23T18:20:40.748873+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:20:40.748873+0000","last_undegraded":"2023-09-23T18:20:40.748873+0000","last_fullsized":"2023-09-23T18:20:40.748873+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":33,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.759092+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:20:40.759092+0000","last_peered":"2023-09-23T18:20:40.759092+0000","last_clean":"2023-09-23T18:20:40.759092+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:20:40.759092+0000","last_undegraded":"2023-09-23T18:20:40.759092+0000","last_fullsized":"2023-09-23T18:20:40.759092+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":345,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.765532+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:20:40.765532+0000","last_peered":"2023-09-23T18:20:40.765532+0000","last_clean":"2023-09-23T18:20:40.765532+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:20:40.765532+0000","last_undegraded":"2023-09-23T18:20:40.765532+0000","last_fullsized":"2023-09-23T18:20:40.765532+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"617'10816","reported_seq":10870,"reported_epoch":617,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:50.238597+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:20:50.238597+0000","last_peered":"2023-09-23T18:20:50.238597+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:20:50.238597+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"603'5900","ondisk_log_start":"603'5900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4916,"log_dups_size":0,"ondisk_log_size":4916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147587072,"num_objects":2252,"num_object_clones":0,"num_object_copies":6756,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2252,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10816,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-2033},{"shards":"0,4,6","objects":3}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":728,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762695+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:20:40.762695+0000","last_peered":"2023-09-23T18:20:40.762695+0000","last_clean":"2023-09-23T18:20:40.762695+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:20:40.762695+0000","last_undegraded":"2023-09-23T18:20:40.762695+0000","last_fullsized":"2023-09-23T18:20:40.762695+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":967,"reported_epoch":616,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:40.765327+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:20:40.765327+0000","last_peered":"2023-09-23T18:20:40.765327+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:20:40.765327+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":151,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749326+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:20:40.749326+0000","last_peered":"2023-09-23T18:20:40.749326+0000","last_clean":"2023-09-23T18:20:40.749326+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:20:40.749326+0000","last_undegraded":"2023-09-23T18:20:40.749326+0000","last_fullsized":"2023-09-23T18:20:40.749326+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":131,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748787+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:20:40.748787+0000","last_peered":"2023-09-23T18:20:40.748787+0000","last_clean":"2023-09-23T18:20:40.748787+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:20:40.748787+0000","last_undegraded":"2023-09-23T18:20:40.748787+0000","last_fullsized":"2023-09-23T18:20:40.748787+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"617'10764","reported_seq":10824,"reported_epoch":617,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:51.282762+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:20:51.282762+0000","last_peered":"2023-09-23T18:20:51.282762+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:20:51.282762+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"604'6300","ondisk_log_start":"604'6300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4464,"log_dups_size":0,"ondisk_log_size":4464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":142344192,"num_objects":2172,"num_object_clones":0,"num_object_copies":6516,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2172,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10764,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"5,6","objects":-1387}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":800,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749095+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:20:40.749095+0000","last_peered":"2023-09-23T18:20:40.749095+0000","last_clean":"2023-09-23T18:20:40.749095+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:20:40.749095+0000","last_undegraded":"2023-09-23T18:20:40.749095+0000","last_fullsized":"2023-09-23T18:20:40.749095+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1501,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748742+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:20:40.748742+0000","last_peered":"2023-09-23T18:20:40.748742+0000","last_clean":"2023-09-23T18:20:40.748742+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:20:40.748742+0000","last_undegraded":"2023-09-23T18:20:40.748742+0000","last_fullsized":"2023-09-23T18:20:40.748742+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":141,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761190+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:20:40.761190+0000","last_peered":"2023-09-23T18:20:40.761190+0000","last_clean":"2023-09-23T18:20:40.761190+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:20:40.761190+0000","last_undegraded":"2023-09-23T18:20:40.761190+0000","last_fullsized":"2023-09-23T18:20:40.761190+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":441,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.764029+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:20:40.764029+0000","last_peered":"2023-09-23T18:20:40.764029+0000","last_clean":"2023-09-23T18:20:40.764029+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:20:40.764029+0000","last_undegraded":"2023-09-23T18:20:40.764029+0000","last_fullsized":"2023-09-23T18:20:40.764029+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"617'10447","reported_seq":10498,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:51.141142+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:20:51.141142+0000","last_peered":"2023-09-23T18:20:51.141142+0000","last_clean":"2023-09-23T18:20:51.141142+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:20:51.141142+0000","last_undegraded":"2023-09-23T18:20:51.141142+0000","last_fullsized":"2023-09-23T18:20:51.141142+0000","mapping_epoch":574,"log_start":"601'4900","ondisk_log_start":"601'4900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5547,"log_dups_size":0,"ondisk_log_size":5547,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":140181504,"num_objects":2139,"num_object_clones":0,"num_object_copies":6417,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2139,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10447,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":825,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748972+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:20:40.748972+0000","last_peered":"2023-09-23T18:20:40.748972+0000","last_clean":"2023-09-23T18:20:40.748972+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:20:40.748972+0000","last_undegraded":"2023-09-23T18:20:40.748972+0000","last_fullsized":"2023-09-23T18:20:40.748972+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":60,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.824215+0000","last_change":"2023-09-23T18:20:35.016554+0000","last_active":"2023-09-23T18:20:46.824215+0000","last_peered":"2023-09-23T18:20:46.824215+0000","last_clean":"2023-09-23T18:20:46.824215+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:20:46.824215+0000","last_undegraded":"2023-09-23T18:20:46.824215+0000","last_fullsized":"2023-09-23T18:20:46.824215+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":410,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760838+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:20:40.760838+0000","last_peered":"2023-09-23T18:20:40.760838+0000","last_clean":"2023-09-23T18:20:40.760838+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:20:40.760838+0000","last_undegraded":"2023-09-23T18:20:40.760838+0000","last_fullsized":"2023-09-23T18:20:40.760838+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":153,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749094+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:20:40.749094+0000","last_peered":"2023-09-23T18:20:40.749094+0000","last_clean":"2023-09-23T18:20:40.749094+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:20:40.749094+0000","last_undegraded":"2023-09-23T18:20:40.749094+0000","last_fullsized":"2023-09-23T18:20:40.749094+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"617'10722","reported_seq":10773,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:51.257553+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:20:51.257553+0000","last_peered":"2023-09-23T18:20:51.257553+0000","last_clean":"2023-09-23T18:20:51.257553+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:20:51.257553+0000","last_undegraded":"2023-09-23T18:20:51.257553+0000","last_fullsized":"2023-09-23T18:20:51.257553+0000","mapping_epoch":574,"log_start":"604'6300","ondisk_log_start":"604'6300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4422,"log_dups_size":0,"ondisk_log_size":4422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145883136,"num_objects":2226,"num_object_clones":0,"num_object_copies":6678,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2226,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10722,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1514,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749445+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:20:40.749445+0000","last_peered":"2023-09-23T18:20:40.749445+0000","last_clean":"2023-09-23T18:20:40.749445+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:20:40.749445+0000","last_undegraded":"2023-09-23T18:20:40.749445+0000","last_fullsized":"2023-09-23T18:20:40.749445+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":567,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760705+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:20:40.760705+0000","last_peered":"2023-09-23T18:20:40.760705+0000","last_clean":"2023-09-23T18:20:40.760705+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:20:40.760705+0000","last_undegraded":"2023-09-23T18:20:40.760705+0000","last_fullsized":"2023-09-23T18:20:40.760705+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":127,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748652+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:20:40.748652+0000","last_peered":"2023-09-23T18:20:40.748652+0000","last_clean":"2023-09-23T18:20:40.748652+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:20:40.748652+0000","last_undegraded":"2023-09-23T18:20:40.748652+0000","last_fullsized":"2023-09-23T18:20:40.748652+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":162,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762773+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:20:40.762773+0000","last_peered":"2023-09-23T18:20:40.762773+0000","last_clean":"2023-09-23T18:20:40.762773+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:20:40.762773+0000","last_undegraded":"2023-09-23T18:20:40.762773+0000","last_fullsized":"2023-09-23T18:20:40.762773+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"617'10569","reported_seq":10620,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:51.181044+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:20:51.181044+0000","last_peered":"2023-09-23T18:20:51.181044+0000","last_clean":"2023-09-23T18:20:51.181044+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:20:51.181044+0000","last_undegraded":"2023-09-23T18:20:51.181044+0000","last_fullsized":"2023-09-23T18:20:51.181044+0000","mapping_epoch":574,"log_start":"601'5100","ondisk_log_start":"601'5100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5469,"log_dups_size":0,"ondisk_log_size":5469,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":145162240,"num_objects":2215,"num_object_clones":0,"num_object_copies":6645,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2215,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10569,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":815,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761361+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:20:40.761361+0000","last_peered":"2023-09-23T18:20:40.761361+0000","last_clean":"2023-09-23T18:20:40.761361+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:20:40.761361+0000","last_undegraded":"2023-09-23T18:20:40.761361+0000","last_fullsized":"2023-09-23T18:20:40.761361+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":130,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.759222+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:20:40.759222+0000","last_peered":"2023-09-23T18:20:40.759222+0000","last_clean":"2023-09-23T18:20:40.759222+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:20:40.759222+0000","last_undegraded":"2023-09-23T18:20:40.759222+0000","last_fullsized":"2023-09-23T18:20:40.759222+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":34,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.824144+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:20:46.824144+0000","last_peered":"2023-09-23T18:20:46.824144+0000","last_clean":"2023-09-23T18:20:46.824144+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:20:46.824144+0000","last_undegraded":"2023-09-23T18:20:46.824144+0000","last_fullsized":"2023-09-23T18:20:46.824144+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":632,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749156+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:20:40.749156+0000","last_peered":"2023-09-23T18:20:40.749156+0000","last_clean":"2023-09-23T18:20:40.749156+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:20:40.749156+0000","last_undegraded":"2023-09-23T18:20:40.749156+0000","last_fullsized":"2023-09-23T18:20:40.749156+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"617'10687","reported_seq":17667,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:51.241538+0000","last_change":"2023-09-23T18:20:36.673759+0000","last_active":"2023-09-23T18:20:51.241538+0000","last_peered":"2023-09-23T18:20:51.241538+0000","last_clean":"2023-09-23T18:20:51.241538+0000","last_became_active":"2023-09-23T18:20:36.651913+0000","last_became_peered":"2023-09-23T18:20:36.651913+0000","last_unstale":"2023-09-23T18:20:51.241538+0000","last_undegraded":"2023-09-23T18:20:51.241538+0000","last_fullsized":"2023-09-23T18:20:51.241538+0000","mapping_epoch":614,"log_start":"604'6200","ondisk_log_start":"604'6200","created":574,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4487,"log_dups_size":0,"ondisk_log_size":4487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:47.765348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":148766744,"num_objects":2271,"num_object_clones":0,"num_object_copies":6813,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2271,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":10687,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3453,"num_bytes_recovered":209321984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":915,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761836+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:20:40.761836+0000","last_peered":"2023-09-23T18:20:40.761836+0000","last_clean":"2023-09-23T18:20:40.761836+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:20:40.761836+0000","last_undegraded":"2023-09-23T18:20:40.761836+0000","last_fullsized":"2023-09-23T18:20:40.761836+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":56,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.778720+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:20:40.778720+0000","last_peered":"2023-09-23T18:20:40.778720+0000","last_clean":"2023-09-23T18:20:40.778720+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:20:40.778720+0000","last_undegraded":"2023-09-23T18:20:40.778720+0000","last_fullsized":"2023-09-23T18:20:40.778720+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":141,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749015+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:20:40.749015+0000","last_peered":"2023-09-23T18:20:40.749015+0000","last_clean":"2023-09-23T18:20:40.749015+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:20:40.749015+0000","last_undegraded":"2023-09-23T18:20:40.749015+0000","last_fullsized":"2023-09-23T18:20:40.749015+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":141,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749418+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:20:40.749418+0000","last_peered":"2023-09-23T18:20:40.749418+0000","last_clean":"2023-09-23T18:20:40.749418+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:20:40.749418+0000","last_undegraded":"2023-09-23T18:20:40.749418+0000","last_fullsized":"2023-09-23T18:20:40.749418+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"617'10665","reported_seq":10716,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:51.227675+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:20:51.227675+0000","last_peered":"2023-09-23T18:20:51.227675+0000","last_clean":"2023-09-23T18:20:51.227675+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:20:51.227675+0000","last_undegraded":"2023-09-23T18:20:51.227675+0000","last_fullsized":"2023-09-23T18:20:51.227675+0000","mapping_epoch":574,"log_start":"604'6200","ondisk_log_start":"604'6200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4465,"log_dups_size":0,"ondisk_log_size":4465,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":143458304,"num_objects":2189,"num_object_clones":0,"num_object_copies":6567,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2189,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10665,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":915,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762475+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:20:40.762475+0000","last_peered":"2023-09-23T18:20:40.762475+0000","last_clean":"2023-09-23T18:20:40.762475+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:20:40.762475+0000","last_undegraded":"2023-09-23T18:20:40.762475+0000","last_fullsized":"2023-09-23T18:20:40.762475+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":103,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748912+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:20:40.748912+0000","last_peered":"2023-09-23T18:20:40.748912+0000","last_clean":"2023-09-23T18:20:40.748912+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:20:40.748912+0000","last_undegraded":"2023-09-23T18:20:40.748912+0000","last_fullsized":"2023-09-23T18:20:40.748912+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":396,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761659+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:20:40.761659+0000","last_peered":"2023-09-23T18:20:40.761659+0000","last_clean":"2023-09-23T18:20:40.761659+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:20:40.761659+0000","last_undegraded":"2023-09-23T18:20:40.761659+0000","last_fullsized":"2023-09-23T18:20:40.761659+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":34,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.355382+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:46.355382+0000","last_peered":"2023-09-23T18:20:46.355382+0000","last_clean":"2023-09-23T18:20:46.355382+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:20:46.355382+0000","last_undegraded":"2023-09-23T18:20:46.355382+0000","last_fullsized":"2023-09-23T18:20:46.355382+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"617'10566","reported_seq":10617,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:51.171094+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:20:51.171094+0000","last_peered":"2023-09-23T18:20:51.171094+0000","last_clean":"2023-09-23T18:20:51.171094+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:20:51.171094+0000","last_undegraded":"2023-09-23T18:20:51.171094+0000","last_fullsized":"2023-09-23T18:20:51.171094+0000","mapping_epoch":574,"log_start":"604'6100","ondisk_log_start":"604'6100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4466,"log_dups_size":0,"ondisk_log_size":4466,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":147193856,"num_objects":2246,"num_object_clones":0,"num_object_copies":6738,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2246,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":10566,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":750,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.824504+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:20:46.824504+0000","last_peered":"2023-09-23T18:20:46.824504+0000","last_clean":"2023-09-23T18:20:46.824504+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:20:46.824504+0000","last_undegraded":"2023-09-23T18:20:46.824504+0000","last_fullsized":"2023-09-23T18:20:46.824504+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1053,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.762932+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:20:40.762932+0000","last_peered":"2023-09-23T18:20:40.762932+0000","last_clean":"2023-09-23T18:20:40.762932+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:20:40.762932+0000","last_undegraded":"2023-09-23T18:20:40.762932+0000","last_fullsized":"2023-09-23T18:20:40.762932+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":44,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:45.275197+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:20:45.275197+0000","last_peered":"2023-09-23T18:20:45.275197+0000","last_clean":"2023-09-23T18:20:45.275197+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:20:45.275197+0000","last_undegraded":"2023-09-23T18:20:45.275197+0000","last_fullsized":"2023-09-23T18:20:45.275197+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":407,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.748840+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:20:40.748840+0000","last_peered":"2023-09-23T18:20:40.748840+0000","last_clean":"2023-09-23T18:20:40.748840+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:20:40.748840+0000","last_undegraded":"2023-09-23T18:20:40.748840+0000","last_fullsized":"2023-09-23T18:20:40.748840+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":647,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760599+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:20:40.760599+0000","last_peered":"2023-09-23T18:20:40.760599+0000","last_clean":"2023-09-23T18:20:40.760599+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:20:40.760599+0000","last_undegraded":"2023-09-23T18:20:40.760599+0000","last_fullsized":"2023-09-23T18:20:40.760599+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":39,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.770158+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:20:40.770158+0000","last_peered":"2023-09-23T18:20:40.770158+0000","last_clean":"2023-09-23T18:20:40.770158+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:20:40.770158+0000","last_undegraded":"2023-09-23T18:20:40.770158+0000","last_fullsized":"2023-09-23T18:20:40.770158+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":690,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.765151+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:20:40.765151+0000","last_peered":"2023-09-23T18:20:40.765151+0000","last_clean":"2023-09-23T18:20:40.765151+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:20:40.765151+0000","last_undegraded":"2023-09-23T18:20:40.765151+0000","last_fullsized":"2023-09-23T18:20:40.765151+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":647,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761559+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:20:40.761559+0000","last_peered":"2023-09-23T18:20:40.761559+0000","last_clean":"2023-09-23T18:20:40.761559+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:20:40.761559+0000","last_undegraded":"2023-09-23T18:20:40.761559+0000","last_fullsized":"2023-09-23T18:20:40.761559+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":135,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749058+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:20:40.749058+0000","last_peered":"2023-09-23T18:20:40.749058+0000","last_clean":"2023-09-23T18:20:40.749058+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:20:40.749058+0000","last_undegraded":"2023-09-23T18:20:40.749058+0000","last_fullsized":"2023-09-23T18:20:40.749058+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":251,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.766495+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:20:40.766495+0000","last_peered":"2023-09-23T18:20:40.766495+0000","last_clean":"2023-09-23T18:20:40.766495+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:20:40.766495+0000","last_undegraded":"2023-09-23T18:20:40.766495+0000","last_fullsized":"2023-09-23T18:20:40.766495+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":121,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749244+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:20:40.749244+0000","last_peered":"2023-09-23T18:20:40.749244+0000","last_clean":"2023-09-23T18:20:40.749244+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:20:40.749244+0000","last_undegraded":"2023-09-23T18:20:40.749244+0000","last_fullsized":"2023-09-23T18:20:40.749244+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":234,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.767983+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:20:40.767983+0000","last_peered":"2023-09-23T18:20:40.767983+0000","last_clean":"2023-09-23T18:20:40.767983+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:20:40.767983+0000","last_undegraded":"2023-09-23T18:20:40.767983+0000","last_fullsized":"2023-09-23T18:20:40.767983+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1009,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.763736+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:20:40.763736+0000","last_peered":"2023-09-23T18:20:40.763736+0000","last_clean":"2023-09-23T18:20:40.763736+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:20:40.763736+0000","last_undegraded":"2023-09-23T18:20:40.763736+0000","last_fullsized":"2023-09-23T18:20:40.763736+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":514,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.761751+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:20:40.761751+0000","last_peered":"2023-09-23T18:20:40.761751+0000","last_clean":"2023-09-23T18:20:40.761751+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:20:40.761751+0000","last_undegraded":"2023-09-23T18:20:40.761751+0000","last_fullsized":"2023-09-23T18:20:40.761751+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":151,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749008+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:20:40.749008+0000","last_peered":"2023-09-23T18:20:40.749008+0000","last_clean":"2023-09-23T18:20:40.749008+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:20:40.749008+0000","last_undegraded":"2023-09-23T18:20:40.749008+0000","last_fullsized":"2023-09-23T18:20:40.749008+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":946,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.766260+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:20:40.766260+0000","last_peered":"2023-09-23T18:20:40.766260+0000","last_clean":"2023-09-23T18:20:40.766260+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:20:40.766260+0000","last_undegraded":"2023-09-23T18:20:40.766260+0000","last_fullsized":"2023-09-23T18:20:40.766260+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":139,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.749029+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:20:40.749029+0000","last_peered":"2023-09-23T18:20:40.749029+0000","last_clean":"2023-09-23T18:20:40.749029+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:20:40.749029+0000","last_undegraded":"2023-09-23T18:20:40.749029+0000","last_fullsized":"2023-09-23T18:20:40.749029+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":660,"reported_epoch":616,"state":"active+clean","last_fresh":"2023-09-23T18:20:40.760614+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:20:40.760614+0000","last_peered":"2023-09-23T18:20:40.760614+0000","last_clean":"2023-09-23T18:20:40.760614+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:20:40.760614+0000","last_undegraded":"2023-09-23T18:20:40.760614+0000","last_fullsized":"2023-09-23T18:20:40.760614+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":2342191128,"num_objects":35740,"num_object_clones":0,"num_object_copies":107220,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":15474,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":35740,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":171068,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7668,"num_bytes_recovered":482803712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":8725016576,"data_stored":8725004360,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77168,"ondisk_log_size":77168,"up":48,"acting":44,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20983808,"data_stored":20971577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001579,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669515,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1921320,"kb_used_data":1292512,"kb_used_omap":7,"kb_used_meta":628792,"kb_avail":91823832,"statfs":{"total":95995035648,"available":94027603968,"internally_reserved":0,"allocated":1323532288,"data_stored":1321577905,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":643883812},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":1.7889999999999999,"15min":1.407},"min":{"1min":0.249,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":9.1600000000000001,"5min":24.472999999999999,"15min":24.472999999999999},"last":1.448},{"interface":"front","average":{"1min":2.54,"5min":2.1960000000000002,"15min":1.726},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2029999999999994,"5min":22.832999999999998,"15min":22.832999999999998},"last":0.55900000000000005}]},{"osd":1,"last update":"Sat Sep 23 18:20:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3570000000000002,"5min":2.0350000000000001,"15min":1.4890000000000001},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.726000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":2.423,"5min":1.899,"15min":1.4379999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.336,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.39300000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.8129999999999999,"5min":1.52,"15min":1.3260000000000001},"min":{"1min":0.32900000000000001,"5min":0.25700000000000001,"15min":0.25700000000000001},"max":{"1min":8.0559999999999992,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":1.4730000000000001},{"interface":"front","average":{"1min":1.8759999999999999,"5min":1.861,"15min":1.492},"min":{"1min":0.34000000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.1230000000000002,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.435}]},{"osd":3,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":9.1259999999999994,"5min":3.1400000000000001,"15min":2.1059999999999999},"min":{"1min":0.33500000000000002,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":103.348,"5min":103.348,"15min":103.348},"last":1.49},{"interface":"front","average":{"1min":9.4610000000000003,"5min":3.6389999999999998,"15min":2.3610000000000002},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":103.40300000000001,"5min":103.40300000000001,"15min":103.40300000000001},"last":1.1699999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.8180000000000001,"15min":1.345},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.0730000000000004,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":1.776,"5min":1.8200000000000001,"15min":1.3700000000000001},"min":{"1min":0.45200000000000001,"5min":0.30399999999999999,"15min":0.27000000000000002},"max":{"1min":7.8319999999999999,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":1.5189999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":2.1099999999999999,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":18.417000000000002,"5min":18.417000000000002,"15min":18.417000000000002},"last":1.6950000000000001},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.3399999999999999,"15min":2.3780000000000001},"min":{"1min":0.36299999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":18.454999999999998,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.36499999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":1.4970000000000001,"5min":2.1459999999999999,"15min":1.5309999999999999},"min":{"1min":0.41799999999999998,"5min":0.374,"15min":0.26800000000000002},"max":{"1min":6.4320000000000004,"5min":10.743,"15min":13.460000000000001},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.5940000000000001,"5min":2.0710000000000002,"15min":1.5740000000000001},"min":{"1min":0.38500000000000001,"5min":0.36399999999999999,"15min":0.28499999999999998},"max":{"1min":6.2770000000000001,"5min":11.244,"15min":12.404},"last":1.5049999999999999}]}]},{"osd":1,"up_from":545,"seq":2340757176363,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1476940,"kb_used_data":1265092,"kb_used_omap":13,"kb_used_meta":211826,"kb_avail":92268212,"statfs":{"total":95995035648,"available":94482649088,"internally_reserved":0,"allocated":1295454208,"data_stored":1292422592,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":216910477},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.016,"15min":0.59899999999999998},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":9.0299999999999994,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":1.381},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.004,"15min":0.66100000000000003},"min":{"1min":0.34499999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.952999999999999,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.79100000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.0980000000000001,"15min":0.67300000000000004},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.6219999999999999,"5min":9.6219999999999999,"15min":9.6219999999999999},"last":0.97999999999999998},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.77200000000000002,"15min":0.59399999999999997},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2080000000000002,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":0.26100000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.0409999999999999,"15min":0.60499999999999998},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.3200000000000003,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":6.0940000000000003},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.2769999999999999,"15min":0.76700000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":19.762,"5min":19.762,"15min":19.762},"last":1.1719999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.3280000000000001,"15min":1.3200000000000001},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.8690000000000002,"5min":12.663,"15min":12.663},"last":1.4430000000000001},{"interface":"front","average":{"1min":2.4049999999999998,"5min":1.7789999999999999,"15min":1.6020000000000001},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.317},"max":{"1min":6.0289999999999999,"5min":13.222,"15min":13.222},"last":6.1760000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.405,"15min":0.94999999999999996},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":23.190999999999999,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.38100000000000001},{"interface":"front","average":{"1min":4.1719999999999997,"5min":1.7330000000000001,"15min":1.113},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":23.204000000000001,"5min":23.204000000000001,"15min":23.204000000000001},"last":1.02}]},{"osd":6,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.121,"5min":1.4690000000000001,"15min":0.996},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":23.443000000000001,"5min":23.443000000000001,"15min":23.443000000000001},"last":0.96899999999999997},{"interface":"front","average":{"1min":3.7970000000000002,"5min":1.746,"15min":1.099},"min":{"1min":0.42199999999999999,"5min":0.378,"15min":0.378},"max":{"1min":26.177,"5min":26.177,"15min":26.177},"last":6.1440000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.4340000000000002,"5min":1.3620000000000001,"15min":0.90800000000000003},"min":{"1min":0.42299999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.1050000000000004,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":10.089},{"interface":"front","average":{"1min":2.3140000000000001,"5min":1.474,"15min":1.028},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":9.1500000000000004,"5min":9.1500000000000004,"15min":9.1500000000000004},"last":1.081}]}]},{"osd":0,"up_from":93,"seq":399431958860,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2695716,"kb_used_data":1768348,"kb_used_omap":7,"kb_used_meta":927288,"kb_avail":91049436,"statfs":{"total":95995035648,"available":93234622464,"internally_reserved":0,"allocated":1810788352,"data_stored":1807912579,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":949543652},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.044,"15min":0.56100000000000005},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.4779999999999998,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.41899999999999998},{"interface":"front","average":{"1min":2.6240000000000001,"5min":1.278,"15min":0.77400000000000002},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.335000000000001,"5min":15.335000000000001,"15min":15.335000000000001},"last":0.59399999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:20:04 2023","interfaces":[{"interface":"back","average":{"1min":3.911,"5min":1.71,"15min":1.3160000000000001},"min":{"1min":0.48999999999999999,"5min":0.185,"15min":0.185},"max":{"1min":15.002000000000001,"5min":18.550999999999998,"15min":18.550999999999998},"last":1.3560000000000001},{"interface":"front","average":{"1min":2.8530000000000002,"5min":1.258,"15min":0.93400000000000005},"min":{"1min":0.29799999999999999,"5min":0.22,"15min":0.22},"max":{"1min":11.644,"5min":13.618,"15min":13.618},"last":0.65700000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.4199999999999999,"15min":1.5940000000000001},"min":{"1min":0.313,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.8460000000000001,"5min":13.539999999999999,"15min":13.539999999999999},"last":15.173999999999999},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.6619999999999999,"15min":1.7010000000000001},"min":{"1min":0.36299999999999999,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":6.8879999999999999,"5min":13.285,"15min":15.198},"last":15.135}]},{"osd":4,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7010000000000001,"5min":2.1309999999999998,"15min":1.8979999999999999},"min":{"1min":0.41799999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":17.741,"5min":17.741,"15min":57.746000000000002},"last":0.49399999999999999},{"interface":"front","average":{"1min":3.79,"5min":1.9159999999999999,"15min":1.7809999999999999},"min":{"1min":0.50900000000000001,"5min":0.26900000000000002,"15min":0.25600000000000001},"max":{"1min":15.079000000000001,"5min":15.079000000000001,"15min":57.816000000000003},"last":1.4179999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":1.8220000000000001,"15min":1.4410000000000001},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":5.5970000000000004,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.45000000000000001},{"interface":"front","average":{"1min":3.2909999999999999,"5min":2.4630000000000001,"15min":1.599},"min":{"1min":0.40100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":15.047000000000001,"5min":15.047000000000001,"15min":15.047000000000001},"last":1.391}]},{"osd":6,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":1.948,"15min":1.9530000000000001},"min":{"1min":0.47499999999999998,"5min":0.377,"15min":0.32000000000000001},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":58.457999999999998},"last":0.47199999999999998},{"interface":"front","average":{"1min":4.6639999999999997,"5min":2.3159999999999998,"15min":1.9299999999999999},"min":{"1min":0.497,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":18.643999999999998,"5min":18.643999999999998,"15min":58.354999999999997},"last":0.68200000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:19:53 2023","interfaces":[{"interface":"back","average":{"1min":3.4340000000000002,"5min":1.923,"15min":1.639},"min":{"1min":0.46899999999999997,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":13.644,"5min":13.644,"15min":13.644},"last":0.63300000000000001},{"interface":"front","average":{"1min":3.5179999999999998,"5min":1.821,"15min":1.8},"min":{"1min":0.55900000000000005,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":7.7519999999999998,"5min":10.404,"15min":10.404},"last":1.5580000000000001}]}]},{"osd":2,"up_from":0,"seq":2070174236745,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249375,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1738048,"kb_used_data":1008824,"kb_used_omap":3,"kb_used_meta":729212,"kb_avail":92007104,"statfs":{"total":95995035648,"available":94215274496,"internally_reserved":0,"allocated":1033035776,"data_stored":1029957841,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":746713832},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":1.5800000000000001,"15min":1.8089999999999999},"min":{"1min":0.20799999999999999,"5min":0.192,"15min":0.183},"max":{"1min":12.472,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.405,"15min":2.004},"min":{"1min":0.23799999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.468,"5min":11.468,"15min":194.661},"last":0.27700000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:20:26 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":1.4590000000000001,"15min":0.83299999999999996},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.079000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.249},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.5129999999999999,"15min":1.3069999999999999},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.081,"5min":78.980000000000004,"15min":78.980000000000004},"last":3.419}]},{"osd":2,"last update":"Sat Sep 23 18:19:58 2023","interfaces":[{"interface":"back","average":{"1min":2.956,"5min":1.274,"15min":1.29},"min":{"1min":0.27400000000000002,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":12.151999999999999,"5min":12.151999999999999,"15min":12.151999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":3.1850000000000001,"5min":1.298,"15min":1.3320000000000001},"min":{"1min":0.318,"5min":0.19600000000000001,"15min":0.19600000000000001},"max":{"1min":11.565,"5min":11.565,"15min":11.565},"last":0.39500000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":1.8839999999999999,"15min":1.706},"min":{"1min":0.34200000000000003,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":11.4,"5min":21.062999999999999,"15min":37.231000000000002},"last":1.5600000000000001},{"interface":"front","average":{"1min":2.532,"5min":2.1240000000000001,"15min":1.885},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":9.1630000000000003,"5min":21.128,"15min":37.381},"last":0.379}]},{"osd":5,"last update":"Sat Sep 23 18:20:37 2023","interfaces":[{"interface":"back","average":{"1min":3.452,"5min":2.048,"15min":1.587},"min":{"1min":0.35099999999999998,"5min":0.316,"15min":0.316},"max":{"1min":9.7430000000000003,"5min":11.805999999999999,"15min":11.805999999999999},"last":3.3570000000000002},{"interface":"front","average":{"1min":3.0339999999999998,"5min":1.74,"15min":1.722},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":17.135999999999999,"5min":17.135999999999999,"15min":17.135999999999999},"last":0.70799999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.9390000000000001,"15min":1.9379999999999999},"min":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":10.303000000000001,"5min":17.192,"15min":115.54000000000001},"last":0.748},{"interface":"front","average":{"1min":2.9889999999999999,"5min":1.976,"15min":1.9810000000000001},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.30499999999999999},"max":{"1min":11.442,"5min":14.321,"15min":115.691},"last":0.58899999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0939999999999999,"5min":1.744,"15min":1.7609999999999999},"min":{"1min":0.42699999999999999,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":15.972},"last":3.3929999999999998},{"interface":"front","average":{"1min":1.974,"5min":1.829,"15min":1.6240000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":6.9749999999999996,"5min":17.164000000000001,"15min":17.164000000000001},"last":0.499}]}]},{"osd":4,"up_from":30,"seq":128849019325,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2641640,"kb_used_data":1965792,"kb_used_omap":0,"kb_used_meta":675840,"kb_avail":91103512,"statfs":{"total":95995035648,"available":93289996288,"internally_reserved":0,"allocated":2012971008,"data_stored":2011016417,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":692060160},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.534,"15min":1.268},"min":{"1min":0.255,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.6079999999999997,"5min":7.6079999999999997,"15min":13.375999999999999},"last":1.7190000000000001},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.6819999999999999,"15min":1.546},"min":{"1min":0.26600000000000001,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":9.3710000000000004,"5min":10.890000000000001,"15min":10.890000000000001},"last":1.363}]},{"osd":1,"last update":"Sat Sep 23 18:20:22 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":1.7170000000000001,"15min":1.0349999999999999},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8220000000000001,"5min":44.146999999999998,"15min":44.146999999999998},"last":10.619},{"interface":"front","average":{"1min":1.8420000000000001,"5min":1.6679999999999999,"15min":0.98999999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.5700000000000003,"5min":43.497999999999998,"15min":43.497999999999998},"last":1.772}]},{"osd":2,"last update":"Sat Sep 23 18:19:56 2023","interfaces":[{"interface":"back","average":{"1min":2.621,"5min":1.6699999999999999,"15min":1.8180000000000001},"min":{"1min":0.29799999999999999,"5min":0.255,"15min":0.255},"max":{"1min":11.896000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":3.097},{"interface":"front","average":{"1min":1.6020000000000001,"5min":1.6200000000000001,"15min":1.6930000000000001},"min":{"1min":0.30299999999999999,"5min":0.253,"15min":0.253},"max":{"1min":6.6890000000000001,"5min":10.938000000000001,"15min":10.938000000000001},"last":17.190999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.395,"5min":1.5529999999999999,"15min":1.9630000000000001},"min":{"1min":0.379,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.7519999999999998,"5min":11.130000000000001,"15min":29.154},"last":8.766},{"interface":"front","average":{"1min":2.6230000000000002,"5min":1.8300000000000001,"15min":1.9930000000000001},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.099,"5min":13.470000000000001,"15min":29.216000000000001},"last":1.415}]},{"osd":5,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.3170000000000002,"15min":2.7829999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.5009999999999994,"5min":27.675000000000001,"15min":27.675000000000001},"last":1.444},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.6499999999999999,"15min":2.5609999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":3.8359999999999999,"5min":13.938000000000001,"15min":32.640000000000001},"last":2.6379999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.873,"15min":1.347},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":6.1989999999999998,"5min":18.315000000000001,"15min":18.315000000000001},"last":3.0840000000000001},{"interface":"front","average":{"1min":1.859,"5min":1.9630000000000001,"15min":1.522},"min":{"1min":0.38700000000000001,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":9.5579999999999998,"5min":18.431000000000001,"15min":18.431000000000001},"last":2.6739999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":2.4849999999999999,"15min":1.774},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":5.9539999999999997,"5min":19.298999999999999,"15min":19.298999999999999},"last":1.754},{"interface":"front","average":{"1min":1.722,"5min":1.8899999999999999,"15min":1.552},"min":{"1min":0.435,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.4730000000000008,"5min":12.800000000000001,"15min":12.800000000000001},"last":1.393}]}]},{"osd":5,"up_from":420,"seq":1803886264421,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1720152,"kb_used_data":1365008,"kb_used_omap":9,"kb_used_meta":355126,"kb_avail":92025000,"statfs":{"total":95995035648,"available":94233600000,"internally_reserved":0,"allocated":1397768192,"data_stored":1395825945,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":363649469},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.2290000000000001,"15min":0.93200000000000005},"min":{"1min":0.21299999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.867,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.22700000000000001},{"interface":"front","average":{"1min":2.137,"5min":1.8,"15min":1.6459999999999999},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.621,"5min":20.937999999999999,"15min":20.937999999999999},"last":10.621}]},{"osd":1,"last update":"Sat Sep 23 18:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":1.7070000000000001,"15min":1.2989999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.6649999999999991,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.28999999999999998},{"interface":"front","average":{"1min":2.1840000000000002,"5min":1.73,"15min":1.141},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.5259999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":5.633}]},{"osd":2,"last update":"Sat Sep 23 18:19:52 2023","interfaces":[{"interface":"back","average":{"1min":2.46,"5min":1.629,"15min":1.615},"min":{"1min":0.28699999999999998,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":10.705,"5min":10.705,"15min":10.705},"last":0.40500000000000003},{"interface":"front","average":{"1min":2.5409999999999999,"5min":1.946,"15min":2.1600000000000001},"min":{"1min":0.373,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":7.8099999999999996,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":5.7560000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.762,"15min":1.359},"min":{"1min":0.35599999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.0060000000000002,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":0.377},{"interface":"front","average":{"1min":2.2200000000000002,"5min":2.1150000000000002,"15min":1.7130000000000001},"min":{"1min":0.372,"5min":0.33500000000000002,"15min":0.313},"max":{"1min":10.664999999999999,"5min":20.035,"15min":20.035},"last":10.664999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":2.1480000000000001,"15min":2.0499999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.2270000000000003,"5min":12.747999999999999,"15min":12.747999999999999},"last":8.2270000000000003},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1179999999999999,"15min":1.6639999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7919999999999998,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":7.7919999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.8340000000000001,"15min":1.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.2189999999999999,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.33400000000000002},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.3319999999999999,"15min":1.7370000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.6920000000000002,"5min":12.504,"15min":12.504},"last":6.8090000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.23,"15min":1.804},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.195,"5min":12.641,"15min":13.506},"last":6.0499999999999998},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.8500000000000001,"15min":1.5600000000000001},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":5.0839999999999996,"5min":13.138,"15min":13.138},"last":0.47099999999999997}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1765150720,"data_stored":1765146648,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1271398400,"data_stored":1271398400,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":1013121024,"data_stored":1013121024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1990987776,"data_stored":1990983704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1382547456,"data_stored":1382547456,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":1301811200,"data_stored":1301807128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:20:54.983 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:20:55.059 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:55.191 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:20:55.208 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:55.209 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:20:55.310 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:20:55.390 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:55.441 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:20:55.447 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:55.447 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:20:55.485 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:55 smithi055 ceph-mon[179668]: from='client.47096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:55.548 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:55.549 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:20:55.549 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:20:55.549 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:20:55.549 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:20:55.604 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:55.673 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:55.811 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:55 smithi055 ceph-mon[184174]: from='client.47096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:55.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:55 smithi099 ceph-mon[167076]: from='client.47096 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:20:56.032 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:56.046 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:56.047 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:56.148 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:56.162 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:20:56.233 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:56.331 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:56.336 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:56.337 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:56.429 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:56.438 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:56.472 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:56.519 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:56.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:56.523 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:56.584 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:20:56.625 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:56.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:56 smithi055 ceph-mon[179668]: pgmap v1485: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 874 op/s; 15487/106653 objects degraded (14.521%); 2.4 MiB/s, 42 objects/s recovering 2023-09-23T18:20:56.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:56 smithi055 ceph-mon[184174]: pgmap v1485: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 874 op/s; 15487/106653 objects degraded (14.521%); 2.4 MiB/s, 42 objects/s recovering 2023-09-23T18:20:56.696 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:56.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:56 smithi099 ceph-mon[167076]: pgmap v1485: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.2 GiB data, 12 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 874 op/s; 15487/106653 objects degraded (14.521%); 2.4 MiB/s, 42 objects/s recovering 2023-09-23T18:20:57.084 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:57.088 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:57.089 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:57.190 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:57.271 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:57.468 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:57.471 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:20:57.472 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:57.472 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:57.573 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:57.616 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:57.681 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:57.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3983153953' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:20:57.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:57.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3983153953' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:20:57.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[184174]: failed to encode map e618 with expected crc 2023-09-23T18:20:57.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[184174]: osdmap e618: 8 total, 8 up, 6 in 2023-09-23T18:20:57.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[184174]: failed to encode map e618 with expected crc 2023-09-23T18:20:57.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[184174]: failed to encode map e618 with expected crc 2023-09-23T18:20:57.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3983153953' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:20:57.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:57.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3983153953' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:20:57.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[179668]: failed to encode map e618 with expected crc 2023-09-23T18:20:57.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[179668]: osdmap e618: 8 total, 8 up, 6 in 2023-09-23T18:20:57.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[179668]: failed to encode map e618 with expected crc 2023-09-23T18:20:57.683 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:57 smithi055 ceph-mon[179668]: failed to encode map e618 with expected crc 2023-09-23T18:20:57.685 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:57.685 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:57.741 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:20:57.775 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:20:57.786 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:20:57.815 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:20:57.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3983153953' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:20:57.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:57 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:20:57.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3983153953' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:20:57.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:57 smithi099 ceph-mon[167076]: failed to encode map e618 with expected crc 2023-09-23T18:20:57.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:57 smithi099 ceph-mon[167076]: osdmap e618: 8 total, 8 up, 6 in 2023-09-23T18:20:57.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:57 smithi099 ceph-mon[167076]: failed to encode map e618 with expected crc 2023-09-23T18:20:57.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:57 smithi099 ceph-mon[167076]: failed to encode map e618 with expected crc 2023-09-23T18:20:57.863 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:20:57.874 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:57.907 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:20:57.943 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:20:57.982 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:20:58.028 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:20:58.031 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:20:58.031 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:20:58.031 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:20:58.031 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:20:58.032 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:20:58.032 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:20:58.032 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:20:58.032 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:20:58.093 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:20:58.100 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:58.100 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:20:58.201 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:58.286 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:58.545 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:58.548 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:58.549 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:58.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:20:58.737 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:58.795 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:20:58.803 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:58.803 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:20:58.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:58 smithi099 ceph-mon[167076]: pgmap v1487: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 792 op/s; 15290/97179 objects degraded (15.734%); 1.1 MiB/s, 42 objects/s recovering 2023-09-23T18:20:58.830 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:58 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2747228665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:58.831 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:20:58 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15290/97179 objects degraded (15.734%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:58.851 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:58 smithi055 ceph-mon[179668]: pgmap v1487: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 792 op/s; 15290/97179 objects degraded (15.734%); 1.1 MiB/s, 42 objects/s recovering 2023-09-23T18:20:58.851 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:58 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2747228665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:58.852 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:20:58 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15290/97179 objects degraded (15.734%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:58.852 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:58 smithi055 ceph-mon[184174]: pgmap v1487: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.0 GiB data, 11 GiB used, 525 GiB / 536 GiB avail; 0 B/s wr, 792 op/s; 15290/97179 objects degraded (15.734%); 1.1 MiB/s, 42 objects/s recovering 2023-09-23T18:20:58.852 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:58 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2747228665' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:20:58.852 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:20:58 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15290/97179 objects degraded (15.734%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:20:58.904 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:20:58.945 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:58.998 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:20:59.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:59.004 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:20:59.104 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:20:59.154 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:20:59.224 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:59.292 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:20:59.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:59.299 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:20:59.400 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:59.524 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:59.609 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:20:59.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:20:59.620 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:20:59.645 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:20:59.645 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:20:59.721 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:20:59.840 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:20:59.852 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:20:59.852 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub flag(s) set","count":7},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 15290/95796 objects degraded (15.961%), 5 pgs degraded, 5 pgs undersized","count":10},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1749,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":618,"num_osds":8,"num_up_osds":8,"osd_up_since":1695493039,"num_in_osds":6,"osd_in_since":1695493021,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":114},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":31932,"data_bytes":2095332028,"bytes_used":11711201280,"bytes_avail":564259012608,"bytes_total":575970213888,"degraded_objects":15290,"degraded_total":95796,"degraded_ratio":0.15961000459309366,"recovering_objects_per_sec":42,"recovering_bytes_per_sec":1204661,"recovering_keys_per_sec":0,"num_objects_recovered":423,"num_bytes_recovered":12058624,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":781},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T18:20:09.153115+0000","services":{}},"progress_events":{"57c27436-1bc8-4f63-b41f-a80b88a05748":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 5s)","progress":0.95798319578170776,"add_to_ceph_s":true},"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T18:21:00.020 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:00.036 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:00.037 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:00.057 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:21:00.138 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:00.173 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:00.217 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:00.224 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:00.224 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:00.325 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:00.361 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:21:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:21:00] "GET /metrics HTTP/1.1" 200 36683 "" "Prometheus/2.43.0" 2023-09-23T18:21:00.422 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:00.468 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:21:00.468 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:21:00.484 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:00.491 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:00.491 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:00.507 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:00.593 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:00.648 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:21:00.659 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:00 smithi055 ceph-mon[184174]: pgmap v1488: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 781 op/s; 15290/95796 objects degraded (15.961%); 1.1 MiB/s, 42 objects/s recovering 2023-09-23T18:21:00.659 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:00 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2964691312' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:21:00.661 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:00 smithi055 ceph-mon[179668]: pgmap v1488: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 781 op/s; 15290/95796 objects degraded (15.961%); 1.1 MiB/s, 42 objects/s recovering 2023-09-23T18:21:00.661 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:00 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2964691312' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:21:00.666 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:00.777 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:00.782 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:00.782 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:00.883 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:01.047 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:00 smithi099 ceph-mon[167076]: pgmap v1488: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 2.0 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 781 op/s; 15290/95796 objects degraded (15.961%); 1.1 MiB/s, 42 objects/s recovering 2023-09-23T18:21:01.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:00 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2964691312' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:21:01.219 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:01.225 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:01.226 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:01.326 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:01.383 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:01.450 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:01.455 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:01.455 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:01.556 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:01.593 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:01.594 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:01.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:01.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:01.649 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:01.652 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:01.749 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:01.815 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:01.817 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:01.905 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:01.912 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:01.912 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:02.013 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:02.105 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:02.276 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:02.283 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:02.284 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:02.385 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:02.436 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:02.475 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:02.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:02.480 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:02.581 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:02.694 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:02 smithi099 ceph-mon[167076]: pgmap v1489: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 776 op/s; 15290/94350 objects degraded (16.206%); 1.1 MiB/s, 42 objects/s recovering 2023-09-23T18:21:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:02 smithi055 ceph-mon[184174]: pgmap v1489: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 776 op/s; 15290/94350 objects degraded (16.206%); 1.1 MiB/s, 42 objects/s recovering 2023-09-23T18:21:02.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:02 smithi055 ceph-mon[179668]: pgmap v1489: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.9 GiB data, 11 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 776 op/s; 15290/94350 objects degraded (16.206%); 1.1 MiB/s, 42 objects/s recovering 2023-09-23T18:21:03.366 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:03.371 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:03.372 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:03.473 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:03.526 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:03.586 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:03.597 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:03.598 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:03.621 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3048517879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:03.622 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:03 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 15290/94350 objects degraded (16.206%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:03.664 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:21:03.698 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:03.764 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:03.823 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:03.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3048517879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:03.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:03 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 15290/94350 objects degraded (16.206%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:03.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:03.829 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:03.929 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:03.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3048517879' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:03.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:03 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 15290/94350 objects degraded (16.206%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:03.956 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:04.027 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:04.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:04.038 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:04.139 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:04.198 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:04.265 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:04.268 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:04.268 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:04.369 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:04.463 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:04.788 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[179668]: pgmap v1490: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 743 op/s; 15096/84990 objects degraded (17.762%); 0 B/s, 39 objects/s recovering 2023-09-23T18:21:04.788 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:04.788 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[179668]: failed to encode map e619 with expected crc 2023-09-23T18:21:04.788 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3048517879' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:04.789 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[179668]: osdmap e619: 8 total, 8 up, 6 in 2023-09-23T18:21:04.789 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[179668]: failed to encode map e619 with expected crc 2023-09-23T18:21:04.789 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[184174]: pgmap v1490: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 743 op/s; 15096/84990 objects degraded (17.762%); 0 B/s, 39 objects/s recovering 2023-09-23T18:21:04.789 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:04.789 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[184174]: failed to encode map e619 with expected crc 2023-09-23T18:21:04.789 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3048517879' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:04.789 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[184174]: osdmap e619: 8 total, 8 up, 6 in 2023-09-23T18:21:04.789 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:04 smithi055 ceph-mon[184174]: failed to encode map e619 with expected crc 2023-09-23T18:21:04.970 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:04.977 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:04.977 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:05.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:04 smithi099 ceph-mon[167076]: pgmap v1490: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 743 op/s; 15096/84990 objects degraded (17.762%); 0 B/s, 39 objects/s recovering 2023-09-23T18:21:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:04 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:04 smithi099 ceph-mon[167076]: failed to encode map e619 with expected crc 2023-09-23T18:21:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:04 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3048517879' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:04 smithi099 ceph-mon[167076]: osdmap e619: 8 total, 8 up, 6 in 2023-09-23T18:21:05.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:04 smithi099 ceph-mon[167076]: failed to encode map e619 with expected crc 2023-09-23T18:21:05.078 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:05.118 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:05.193 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:05.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:05.200 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:05.301 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:05.373 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:05.439 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:05.444 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:05.445 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:05.545 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:05.649 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:05.828 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:05.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:05.850 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:05.950 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:05.994 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:06.056 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:06.060 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:06.061 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:06.161 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:06.206 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:06.275 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:06.278 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:06.278 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:06.366 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:21:06.366 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:21:06.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:06.425 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:06.479 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:06.487 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:06.487 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:06.588 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:06.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:06 smithi055 ceph-mon[179668]: pgmap v1492: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 15096/84213 objects degraded (17.926%); 0 B/s, 22 objects/s recovering 2023-09-23T18:21:06.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:06.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:06.635 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:06 smithi055 ceph-mon[184174]: pgmap v1492: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 15096/84213 objects degraded (17.926%); 0 B/s, 22 objects/s recovering 2023-09-23T18:21:06.635 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:06.636 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:06.650 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:06.823 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:06.850 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:06.850 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:06.869 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:21:06.951 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:06.956 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:21:06.983 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:06 smithi099 ceph-mon[167076]: pgmap v1492: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.7 GiB data, 10 GiB used, 526 GiB / 536 GiB avail; 0 B/s wr, 501 op/s; 15096/84213 objects degraded (17.926%); 0 B/s, 22 objects/s recovering 2023-09-23T18:21:06.983 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:06.983 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:06.989 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:07.021 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:21:07.037 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:07.039 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:07.040 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:07.072 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:21:07.134 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:21:07.141 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:07.179 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:21:07.235 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:21:07.262 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:07.299 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:21:07.329 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:07.358 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:07.359 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:07.361 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:21:07.405 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:21:07.405 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:21:07.405 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:21:07.406 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:21:07.406 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:21:07.406 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:21:07.406 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:21:07.406 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:21:07.445 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:21:07.460 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:07.496 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:07.503 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:21:07.525 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:21:07.544 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:07.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:07.550 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:07.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:07.680 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:07.725 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:07.734 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:07.734 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:07.815 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:07 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4080018907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:07.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:07 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4080018907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:07.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:07 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4080018907' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:07.836 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:07.877 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:07.933 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:07.940 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:07.940 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:08.041 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:08.060 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1493,"stamp":"2023-09-23T18:21:07.191572+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1603418812,"num_objects":24426,"num_object_clones":0,"num_object_copies":73278,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14838,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24426,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":183320,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8715,"num_bytes_recovered":502226569,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83231,"ondisk_log_size":83231,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":10251360,"kb_used_data":6648432,"kb_used_omap":40,"kb_used_meta":3602775,"kb_avail":552219552,"statfs":{"total":575970213888,"available":565472821248,"internally_reserved":0,"allocated":6807994368,"data_stored":6793102759,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3689241914},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-522125312,"num_objects":-7967,"num_object_clones":0,"num_object_copies":-23901,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-452,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7967,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7967,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":452,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.006911"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":907,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582135+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:20:56.582135+0000","last_peered":"2023-09-23T18:20:56.582135+0000","last_clean":"2023-09-23T18:20:56.582135+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:20:56.582135+0000","last_undegraded":"2023-09-23T18:20:56.582135+0000","last_fullsized":"2023-09-23T18:20:56.582135+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":277,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.817577+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:20:46.817577+0000","last_peered":"2023-09-23T18:20:46.817577+0000","last_clean":"2023-09-23T18:20:46.817577+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:20:46.817577+0000","last_undegraded":"2023-09-23T18:20:46.817577+0000","last_fullsized":"2023-09-23T18:20:46.817577+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":35,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585740+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:20:56.585740+0000","last_peered":"2023-09-23T18:20:56.585740+0000","last_clean":"2023-09-23T18:20:56.585740+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:20:56.585740+0000","last_undegraded":"2023-09-23T18:20:56.585740+0000","last_fullsized":"2023-09-23T18:20:56.585740+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":913,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583809+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:20:56.583809+0000","last_peered":"2023-09-23T18:20:56.583809+0000","last_clean":"2023-09-23T18:20:56.583809+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:20:56.583809+0000","last_undegraded":"2023-09-23T18:20:56.583809+0000","last_fullsized":"2023-09-23T18:20:56.583809+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":360,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577982+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:20:56.577982+0000","last_peered":"2023-09-23T18:20:56.577982+0000","last_clean":"2023-09-23T18:20:56.577982+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:20:56.577982+0000","last_undegraded":"2023-09-23T18:20:56.577982+0000","last_fullsized":"2023-09-23T18:20:56.577982+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":680,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.580059+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:20:56.580059+0000","last_peered":"2023-09-23T18:20:56.580059+0000","last_clean":"2023-09-23T18:20:56.580059+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:20:56.580059+0000","last_undegraded":"2023-09-23T18:20:56.580059+0000","last_fullsized":"2023-09-23T18:20:56.580059+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":930,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582996+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:20:56.582996+0000","last_peered":"2023-09-23T18:20:56.582996+0000","last_clean":"2023-09-23T18:20:56.582996+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:56.582996+0000","last_undegraded":"2023-09-23T18:20:56.582996+0000","last_fullsized":"2023-09-23T18:20:56.582996+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":35,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577390+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:56.577390+0000","last_peered":"2023-09-23T18:20:56.577390+0000","last_clean":"2023-09-23T18:20:56.577390+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:20:56.577390+0000","last_undegraded":"2023-09-23T18:20:56.577390+0000","last_fullsized":"2023-09-23T18:20:56.577390+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":195,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585783+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:20:56.585783+0000","last_peered":"2023-09-23T18:20:56.585783+0000","last_clean":"2023-09-23T18:20:56.585783+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:20:56.585783+0000","last_undegraded":"2023-09-23T18:20:56.585783+0000","last_fullsized":"2023-09-23T18:20:56.585783+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1053,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581993+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:20:56.581993+0000","last_peered":"2023-09-23T18:20:56.581993+0000","last_clean":"2023-09-23T18:20:56.581993+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:20:56.581993+0000","last_undegraded":"2023-09-23T18:20:56.581993+0000","last_fullsized":"2023-09-23T18:20:56.581993+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":129,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586070+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:20:56.586070+0000","last_peered":"2023-09-23T18:20:56.586070+0000","last_clean":"2023-09-23T18:20:56.586070+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:20:56.586070+0000","last_undegraded":"2023-09-23T18:20:56.586070+0000","last_fullsized":"2023-09-23T18:20:56.586070+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":209,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577734+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:20:56.577734+0000","last_peered":"2023-09-23T18:20:56.577734+0000","last_clean":"2023-09-23T18:20:56.577734+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:20:56.577734+0000","last_undegraded":"2023-09-23T18:20:56.577734+0000","last_fullsized":"2023-09-23T18:20:56.577734+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":830,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586286+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:20:56.586286+0000","last_peered":"2023-09-23T18:20:56.586286+0000","last_clean":"2023-09-23T18:20:56.586286+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:20:56.586286+0000","last_undegraded":"2023-09-23T18:20:56.586286+0000","last_fullsized":"2023-09-23T18:20:56.586286+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":165,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581954+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:20:56.581954+0000","last_peered":"2023-09-23T18:20:56.581954+0000","last_clean":"2023-09-23T18:20:56.581954+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:20:56.581954+0000","last_undegraded":"2023-09-23T18:20:56.581954+0000","last_fullsized":"2023-09-23T18:20:56.581954+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":164,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.584467+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:20:56.584467+0000","last_peered":"2023-09-23T18:20:56.584467+0000","last_clean":"2023-09-23T18:20:56.584467+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:20:56.584467+0000","last_undegraded":"2023-09-23T18:20:56.584467+0000","last_fullsized":"2023-09-23T18:20:56.584467+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":964,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586154+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:20:56.586154+0000","last_peered":"2023-09-23T18:20:56.586154+0000","last_clean":"2023-09-23T18:20:56.586154+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:20:56.586154+0000","last_undegraded":"2023-09-23T18:20:56.586154+0000","last_fullsized":"2023-09-23T18:20:56.586154+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":413,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586051+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:20:56.586051+0000","last_peered":"2023-09-23T18:20:56.586051+0000","last_clean":"2023-09-23T18:20:56.586051+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:20:56.586051+0000","last_undegraded":"2023-09-23T18:20:56.586051+0000","last_fullsized":"2023-09-23T18:20:56.586051+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":432,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582972+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:20:56.582972+0000","last_peered":"2023-09-23T18:20:56.582972+0000","last_clean":"2023-09-23T18:20:56.582972+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:20:56.582972+0000","last_undegraded":"2023-09-23T18:20:56.582972+0000","last_fullsized":"2023-09-23T18:20:56.582972+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":993,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.627356+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:20:56.627356+0000","last_peered":"2023-09-23T18:20:56.627356+0000","last_clean":"2023-09-23T18:20:56.627356+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:20:56.627356+0000","last_undegraded":"2023-09-23T18:20:56.627356+0000","last_fullsized":"2023-09-23T18:20:56.627356+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":171,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586698+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:20:56.586698+0000","last_peered":"2023-09-23T18:20:56.586698+0000","last_clean":"2023-09-23T18:20:56.586698+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:20:56.586698+0000","last_undegraded":"2023-09-23T18:20:56.586698+0000","last_fullsized":"2023-09-23T18:20:56.586698+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":438,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586591+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:20:56.586591+0000","last_peered":"2023-09-23T18:20:56.586591+0000","last_clean":"2023-09-23T18:20:56.586591+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:20:56.586591+0000","last_undegraded":"2023-09-23T18:20:56.586591+0000","last_fullsized":"2023-09-23T18:20:56.586591+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":845,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583543+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:20:56.583543+0000","last_peered":"2023-09-23T18:20:56.583543+0000","last_clean":"2023-09-23T18:20:56.583543+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:20:56.583543+0000","last_undegraded":"2023-09-23T18:20:56.583543+0000","last_fullsized":"2023-09-23T18:20:56.583543+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":438,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586433+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:20:56.586433+0000","last_peered":"2023-09-23T18:20:56.586433+0000","last_clean":"2023-09-23T18:20:56.586433+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:20:56.586433+0000","last_undegraded":"2023-09-23T18:20:56.586433+0000","last_fullsized":"2023-09-23T18:20:56.586433+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":116,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581716+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:20:56.581716+0000","last_peered":"2023-09-23T18:20:56.581716+0000","last_clean":"2023-09-23T18:20:56.581716+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:20:56.581716+0000","last_undegraded":"2023-09-23T18:20:56.581716+0000","last_fullsized":"2023-09-23T18:20:56.581716+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":816,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581472+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:20:56.581472+0000","last_peered":"2023-09-23T18:20:56.581472+0000","last_clean":"2023-09-23T18:20:56.581472+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:20:56.581472+0000","last_undegraded":"2023-09-23T18:20:56.581472+0000","last_fullsized":"2023-09-23T18:20:56.581472+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":134,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.580549+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:20:56.580549+0000","last_peered":"2023-09-23T18:20:56.580549+0000","last_clean":"2023-09-23T18:20:56.580549+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:20:56.580549+0000","last_undegraded":"2023-09-23T18:20:56.580549+0000","last_fullsized":"2023-09-23T18:20:56.580549+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":129,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582258+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:20:56.582258+0000","last_peered":"2023-09-23T18:20:56.582258+0000","last_clean":"2023-09-23T18:20:56.582258+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:20:56.582258+0000","last_undegraded":"2023-09-23T18:20:56.582258+0000","last_fullsized":"2023-09-23T18:20:56.582258+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":755,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.643383+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:21:03.643383+0000","last_peered":"2023-09-23T18:21:03.643383+0000","last_clean":"2023-09-23T18:21:03.643383+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:21:03.643383+0000","last_undegraded":"2023-09-23T18:21:03.643383+0000","last_fullsized":"2023-09-23T18:21:03.643383+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":206,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.584319+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:20:56.584319+0000","last_peered":"2023-09-23T18:20:56.584319+0000","last_clean":"2023-09-23T18:20:56.584319+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:20:56.584319+0000","last_undegraded":"2023-09-23T18:20:56.584319+0000","last_fullsized":"2023-09-23T18:20:56.584319+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":218,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586174+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:20:56.586174+0000","last_peered":"2023-09-23T18:20:56.586174+0000","last_clean":"2023-09-23T18:20:56.586174+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:20:56.586174+0000","last_undegraded":"2023-09-23T18:20:56.586174+0000","last_fullsized":"2023-09-23T18:20:56.586174+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":850,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582422+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:20:56.582422+0000","last_peered":"2023-09-23T18:20:56.582422+0000","last_clean":"2023-09-23T18:20:56.582422+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:20:56.582422+0000","last_undegraded":"2023-09-23T18:20:56.582422+0000","last_fullsized":"2023-09-23T18:20:56.582422+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":759,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582669+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:20:56.582669+0000","last_peered":"2023-09-23T18:20:56.582669+0000","last_clean":"2023-09-23T18:20:56.582669+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:20:56.582669+0000","last_undegraded":"2023-09-23T18:20:56.582669+0000","last_fullsized":"2023-09-23T18:20:56.582669+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":35,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.590243+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:20:56.590243+0000","last_peered":"2023-09-23T18:20:56.590243+0000","last_clean":"2023-09-23T18:20:56.590243+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:20:56.590243+0000","last_undegraded":"2023-09-23T18:20:56.590243+0000","last_fullsized":"2023-09-23T18:20:56.590243+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"619'11532","reported_seq":11585,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:06.147385+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:21:06.147385+0000","last_peered":"2023-09-23T18:21:06.147385+0000","last_clean":"2023-09-23T18:21:06.147385+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:21:06.147385+0000","last_undegraded":"2023-09-23T18:21:06.147385+0000","last_fullsized":"2023-09-23T18:21:06.147385+0000","mapping_epoch":574,"log_start":"603'6000","ondisk_log_start":"603'6000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5532,"log_dups_size":0,"ondisk_log_size":5532,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":101056512,"num_objects":1542,"num_object_clones":0,"num_object_copies":4626,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1542,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11532,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":884,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.588317+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:20:56.588317+0000","last_peered":"2023-09-23T18:20:56.588317+0000","last_clean":"2023-09-23T18:20:56.588317+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:20:56.588317+0000","last_undegraded":"2023-09-23T18:20:56.588317+0000","last_fullsized":"2023-09-23T18:20:56.588317+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":254,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.818641+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:20:46.818641+0000","last_peered":"2023-09-23T18:20:46.818641+0000","last_clean":"2023-09-23T18:20:46.818641+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:20:46.818641+0000","last_undegraded":"2023-09-23T18:20:46.818641+0000","last_fullsized":"2023-09-23T18:20:46.818641+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":129,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581571+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:20:56.581571+0000","last_peered":"2023-09-23T18:20:56.581571+0000","last_clean":"2023-09-23T18:20:56.581571+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:20:56.581571+0000","last_undegraded":"2023-09-23T18:20:56.581571+0000","last_fullsized":"2023-09-23T18:20:56.581571+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"619'11228","reported_seq":11291,"reported_epoch":619,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:06.264819+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:21:06.264819+0000","last_peered":"2023-09-23T18:21:06.264819+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:21:06.264819+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"606'6800","ondisk_log_start":"606'6800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4428,"log_dups_size":0,"ondisk_log_size":4428,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":95944704,"num_objects":1464,"num_object_clones":0,"num_object_copies":4392,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1464,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11228,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"5,6","objects":-2684}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":739,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:21:01.401541+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:21:01.401541+0000","last_peered":"2023-09-23T18:21:01.401541+0000","last_clean":"2023-09-23T18:21:01.401541+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:21:01.401541+0000","last_undegraded":"2023-09-23T18:21:01.401541+0000","last_fullsized":"2023-09-23T18:21:01.401541+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":255,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.818702+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:20:46.818702+0000","last_peered":"2023-09-23T18:20:46.818702+0000","last_clean":"2023-09-23T18:20:46.818702+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:20:46.818702+0000","last_undegraded":"2023-09-23T18:20:46.818702+0000","last_fullsized":"2023-09-23T18:20:46.818702+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":234,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.818777+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:20:46.818777+0000","last_peered":"2023-09-23T18:20:46.818777+0000","last_clean":"2023-09-23T18:20:46.818777+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:20:46.818777+0000","last_undegraded":"2023-09-23T18:20:46.818777+0000","last_fullsized":"2023-09-23T18:20:46.818777+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"619'11284","reported_seq":18297,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:05.180065+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:21:05.180065+0000","last_peered":"2023-09-23T18:21:05.180065+0000","last_clean":"2023-09-23T18:21:05.180065+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:21:05.180065+0000","last_undegraded":"2023-09-23T18:21:05.180065+0000","last_fullsized":"2023-09-23T18:21:05.180065+0000","mapping_epoch":605,"log_start":"604'6400","ondisk_log_start":"604'6400","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4884,"log_dups_size":0,"ondisk_log_size":4884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102236160,"num_objects":1560,"num_object_clones":0,"num_object_copies":4680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11284,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":634,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585708+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:20:56.585708+0000","last_peered":"2023-09-23T18:20:56.585708+0000","last_clean":"2023-09-23T18:20:56.585708+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:20:56.585708+0000","last_undegraded":"2023-09-23T18:20:56.585708+0000","last_fullsized":"2023-09-23T18:20:56.585708+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":544,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582088+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:20:56.582088+0000","last_peered":"2023-09-23T18:20:56.582088+0000","last_clean":"2023-09-23T18:20:56.582088+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:20:56.582088+0000","last_undegraded":"2023-09-23T18:20:56.582088+0000","last_fullsized":"2023-09-23T18:20:56.582088+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":195,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582850+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:20:56.582850+0000","last_peered":"2023-09-23T18:20:56.582850+0000","last_clean":"2023-09-23T18:20:56.582850+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:20:56.582850+0000","last_undegraded":"2023-09-23T18:20:56.582850+0000","last_fullsized":"2023-09-23T18:20:56.582850+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"619'11600","reported_seq":11655,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:06.352244+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:21:06.352244+0000","last_peered":"2023-09-23T18:21:06.352244+0000","last_clean":"2023-09-23T18:21:06.352244+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:21:06.352244+0000","last_undegraded":"2023-09-23T18:21:06.352244+0000","last_fullsized":"2023-09-23T18:21:06.352244+0000","mapping_epoch":574,"log_start":"604'6500","ondisk_log_start":"604'6500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5100,"log_dups_size":0,"ondisk_log_size":5100,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":97779712,"num_objects":1492,"num_object_clones":0,"num_object_copies":4476,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1492,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11600,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":650,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586246+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:20:56.586246+0000","last_peered":"2023-09-23T18:20:56.586246+0000","last_clean":"2023-09-23T18:20:56.586246+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:20:56.586246+0000","last_undegraded":"2023-09-23T18:20:56.586246+0000","last_fullsized":"2023-09-23T18:20:56.586246+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":143,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582108+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:20:56.582108+0000","last_peered":"2023-09-23T18:20:56.582108+0000","last_clean":"2023-09-23T18:20:56.582108+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:20:56.582108+0000","last_undegraded":"2023-09-23T18:20:56.582108+0000","last_fullsized":"2023-09-23T18:20:56.582108+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":456,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.584177+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:20:56.584177+0000","last_peered":"2023-09-23T18:20:56.584177+0000","last_clean":"2023-09-23T18:20:56.584177+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:56.584177+0000","last_undegraded":"2023-09-23T18:20:56.584177+0000","last_fullsized":"2023-09-23T18:20:56.584177+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"619'11436","reported_seq":11491,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.879066+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:21:03.879066+0000","last_peered":"2023-09-23T18:21:03.879066+0000","last_clean":"2023-09-23T18:21:03.879066+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:21:03.879066+0000","last_undegraded":"2023-09-23T18:21:03.879066+0000","last_fullsized":"2023-09-23T18:21:03.879066+0000","mapping_epoch":574,"log_start":"606'7000","ondisk_log_start":"606'7000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4436,"log_dups_size":0,"ondisk_log_size":4436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":105381888,"num_objects":1608,"num_object_clones":0,"num_object_copies":4824,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1608,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11436,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1101,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583397+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:20:56.583397+0000","last_peered":"2023-09-23T18:20:56.583397+0000","last_clean":"2023-09-23T18:20:56.583397+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:20:56.583397+0000","last_undegraded":"2023-09-23T18:20:56.583397+0000","last_fullsized":"2023-09-23T18:20:56.583397+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":356,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581041+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:20:56.581041+0000","last_peered":"2023-09-23T18:20:56.581041+0000","last_clean":"2023-09-23T18:20:56.581041+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:20:56.581041+0000","last_undegraded":"2023-09-23T18:20:56.581041+0000","last_fullsized":"2023-09-23T18:20:56.581041+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586358+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:20:56.586358+0000","last_peered":"2023-09-23T18:20:56.586358+0000","last_clean":"2023-09-23T18:20:56.586358+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:20:56.586358+0000","last_undegraded":"2023-09-23T18:20:56.586358+0000","last_fullsized":"2023-09-23T18:20:56.586358+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"619'11510","reported_seq":14359,"reported_epoch":619,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:06.193436+0000","last_change":"2023-09-23T18:20:34.894639+0000","last_active":"2023-09-23T18:21:06.193436+0000","last_peered":"2023-09-23T18:21:06.193436+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:21:06.193436+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"603'6000","ondisk_log_start":"603'6000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":99614720,"num_objects":1520,"num_object_clones":0,"num_object_copies":4560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2833,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11510,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1393,"num_bytes_recovered":46006272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"4,5","objects":-1313}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":888,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582041+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:20:56.582041+0000","last_peered":"2023-09-23T18:20:56.582041+0000","last_clean":"2023-09-23T18:20:56.582041+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:20:56.582041+0000","last_undegraded":"2023-09-23T18:20:56.582041+0000","last_fullsized":"2023-09-23T18:20:56.582041+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":171,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582779+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:20:56.582779+0000","last_peered":"2023-09-23T18:20:56.582779+0000","last_clean":"2023-09-23T18:20:56.582779+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:20:56.582779+0000","last_undegraded":"2023-09-23T18:20:56.582779+0000","last_fullsized":"2023-09-23T18:20:56.582779+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":470,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.588407+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:20:56.588407+0000","last_peered":"2023-09-23T18:20:56.588407+0000","last_clean":"2023-09-23T18:20:56.588407+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:20:56.588407+0000","last_undegraded":"2023-09-23T18:20:56.588407+0000","last_fullsized":"2023-09-23T18:20:56.588407+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"619'11487","reported_seq":11540,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:06.303604+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:21:06.303604+0000","last_peered":"2023-09-23T18:21:06.303604+0000","last_clean":"2023-09-23T18:21:06.303604+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:21:06.303604+0000","last_undegraded":"2023-09-23T18:21:06.303604+0000","last_fullsized":"2023-09-23T18:21:06.303604+0000","mapping_epoch":574,"log_start":"606'7000","ondisk_log_start":"606'7000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4487,"log_dups_size":0,"ondisk_log_size":4487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102563840,"num_objects":1565,"num_object_clones":0,"num_object_copies":4695,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1565,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11487,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":890,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583889+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:20:56.583889+0000","last_peered":"2023-09-23T18:20:56.583889+0000","last_clean":"2023-09-23T18:20:56.583889+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:20:56.583889+0000","last_undegraded":"2023-09-23T18:20:56.583889+0000","last_fullsized":"2023-09-23T18:20:56.583889+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":455,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583255+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:20:56.583255+0000","last_peered":"2023-09-23T18:20:56.583255+0000","last_clean":"2023-09-23T18:20:56.583255+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:20:56.583255+0000","last_undegraded":"2023-09-23T18:20:56.583255+0000","last_fullsized":"2023-09-23T18:20:56.583255+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":231,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582506+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:20:56.582506+0000","last_peered":"2023-09-23T18:20:56.582506+0000","last_clean":"2023-09-23T18:20:56.582506+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:20:56.582506+0000","last_undegraded":"2023-09-23T18:20:56.582506+0000","last_fullsized":"2023-09-23T18:20:56.582506+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"618'11297","reported_seq":11349,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:21:02.306401+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:21:02.306401+0000","last_peered":"2023-09-23T18:21:02.306401+0000","last_clean":"2023-09-23T18:21:02.306401+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:21:02.306401+0000","last_undegraded":"2023-09-23T18:21:02.306401+0000","last_fullsized":"2023-09-23T18:21:02.306401+0000","mapping_epoch":574,"log_start":"605'6600","ondisk_log_start":"605'6600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4697,"log_dups_size":0,"ondisk_log_size":4697,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":113836032,"num_objects":1737,"num_object_clones":0,"num_object_copies":5211,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1737,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11297,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":871,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583302+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:20:56.583302+0000","last_peered":"2023-09-23T18:20:56.583302+0000","last_clean":"2023-09-23T18:20:56.583302+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:20:56.583302+0000","last_undegraded":"2023-09-23T18:20:56.583302+0000","last_fullsized":"2023-09-23T18:20:56.583302+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":36,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577543+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:20:56.577543+0000","last_peered":"2023-09-23T18:20:56.577543+0000","last_clean":"2023-09-23T18:20:56.577543+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:20:56.577543+0000","last_undegraded":"2023-09-23T18:20:56.577543+0000","last_fullsized":"2023-09-23T18:20:56.577543+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":347,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.627882+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:20:56.627882+0000","last_peered":"2023-09-23T18:20:56.627882+0000","last_clean":"2023-09-23T18:20:56.627882+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:20:56.627882+0000","last_undegraded":"2023-09-23T18:20:56.627882+0000","last_fullsized":"2023-09-23T18:20:56.627882+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"619'11539","reported_seq":11594,"reported_epoch":619,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:05.283960+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:21:05.283960+0000","last_peered":"2023-09-23T18:21:05.283960+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:21:05.283960+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"606'6700","ondisk_log_start":"606'6700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4839,"log_dups_size":0,"ondisk_log_size":4839,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":100204544,"num_objects":1529,"num_object_clones":0,"num_object_copies":4587,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1529,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11539,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-2756}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":730,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585862+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:20:56.585862+0000","last_peered":"2023-09-23T18:20:56.585862+0000","last_clean":"2023-09-23T18:20:56.585862+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:20:56.585862+0000","last_undegraded":"2023-09-23T18:20:56.585862+0000","last_fullsized":"2023-09-23T18:20:56.585862+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":969,"reported_epoch":618,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:20:56.626629+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:20:56.626629+0000","last_peered":"2023-09-23T18:20:56.626629+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:20:56.626629+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":153,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582332+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:20:56.582332+0000","last_peered":"2023-09-23T18:20:56.582332+0000","last_clean":"2023-09-23T18:20:56.582332+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:20:56.582332+0000","last_undegraded":"2023-09-23T18:20:56.582332+0000","last_fullsized":"2023-09-23T18:20:56.582332+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":133,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583238+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:20:56.583238+0000","last_peered":"2023-09-23T18:20:56.583238+0000","last_clean":"2023-09-23T18:20:56.583238+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:20:56.583238+0000","last_undegraded":"2023-09-23T18:20:56.583238+0000","last_fullsized":"2023-09-23T18:20:56.583238+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"619'11500","reported_seq":11562,"reported_epoch":619,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:06.244208+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:21:06.244208+0000","last_peered":"2023-09-23T18:21:06.244208+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:21:06.244208+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"607'7000","ondisk_log_start":"607'7000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4500,"log_dups_size":0,"ondisk_log_size":4500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94109696,"num_objects":1436,"num_object_clones":0,"num_object_copies":4308,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1436,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11500,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"5,6","objects":-2123}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":802,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583703+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:20:56.583703+0000","last_peered":"2023-09-23T18:20:56.583703+0000","last_clean":"2023-09-23T18:20:56.583703+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:20:56.583703+0000","last_undegraded":"2023-09-23T18:20:56.583703+0000","last_fullsized":"2023-09-23T18:20:56.583703+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1503,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583178+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:20:56.583178+0000","last_peered":"2023-09-23T18:20:56.583178+0000","last_clean":"2023-09-23T18:20:56.583178+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:20:56.583178+0000","last_undegraded":"2023-09-23T18:20:56.583178+0000","last_fullsized":"2023-09-23T18:20:56.583178+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":143,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586513+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:20:56.586513+0000","last_peered":"2023-09-23T18:20:56.586513+0000","last_clean":"2023-09-23T18:20:56.586513+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:20:56.586513+0000","last_undegraded":"2023-09-23T18:20:56.586513+0000","last_fullsized":"2023-09-23T18:20:56.586513+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":443,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586568+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:20:56.586568+0000","last_peered":"2023-09-23T18:20:56.586568+0000","last_clean":"2023-09-23T18:20:56.586568+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:20:56.586568+0000","last_undegraded":"2023-09-23T18:20:56.586568+0000","last_fullsized":"2023-09-23T18:20:56.586568+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"619'11158","reported_seq":11211,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:06.180772+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:21:06.180772+0000","last_peered":"2023-09-23T18:21:06.180772+0000","last_clean":"2023-09-23T18:21:06.180772+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:21:06.180772+0000","last_undegraded":"2023-09-23T18:21:06.180772+0000","last_fullsized":"2023-09-23T18:21:06.180772+0000","mapping_epoch":574,"log_start":"603'5700","ondisk_log_start":"603'5700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5458,"log_dups_size":0,"ondisk_log_size":5458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":93585408,"num_objects":1428,"num_object_clones":0,"num_object_copies":4284,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1428,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11158,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":827,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583373+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:20:56.583373+0000","last_peered":"2023-09-23T18:20:56.583373+0000","last_clean":"2023-09-23T18:20:56.583373+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:20:56.583373+0000","last_undegraded":"2023-09-23T18:20:56.583373+0000","last_fullsized":"2023-09-23T18:20:56.583373+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":62,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.575602+0000","last_change":"2023-09-23T18:20:35.016554+0000","last_active":"2023-09-23T18:20:56.575602+0000","last_peered":"2023-09-23T18:20:56.575602+0000","last_clean":"2023-09-23T18:20:56.575602+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:20:56.575602+0000","last_undegraded":"2023-09-23T18:20:56.575602+0000","last_fullsized":"2023-09-23T18:20:56.575602+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":412,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582939+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:20:56.582939+0000","last_peered":"2023-09-23T18:20:56.582939+0000","last_clean":"2023-09-23T18:20:56.582939+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:20:56.582939+0000","last_undegraded":"2023-09-23T18:20:56.582939+0000","last_fullsized":"2023-09-23T18:20:56.582939+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":155,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585992+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:20:56.585992+0000","last_peered":"2023-09-23T18:20:56.585992+0000","last_clean":"2023-09-23T18:20:56.585992+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:20:56.585992+0000","last_undegraded":"2023-09-23T18:20:56.585992+0000","last_fullsized":"2023-09-23T18:20:56.585992+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"619'11468","reported_seq":11521,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:06.284330+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:21:06.284330+0000","last_peered":"2023-09-23T18:21:06.284330+0000","last_clean":"2023-09-23T18:21:06.284330+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:21:06.284330+0000","last_undegraded":"2023-09-23T18:21:06.284330+0000","last_fullsized":"2023-09-23T18:21:06.284330+0000","mapping_epoch":574,"log_start":"607'7000","ondisk_log_start":"607'7000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4468,"log_dups_size":0,"ondisk_log_size":4468,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96993280,"num_objects":1480,"num_object_clones":0,"num_object_copies":4440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11468,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1516,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582292+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:20:56.582292+0000","last_peered":"2023-09-23T18:20:56.582292+0000","last_clean":"2023-09-23T18:20:56.582292+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:20:56.582292+0000","last_undegraded":"2023-09-23T18:20:56.582292+0000","last_fullsized":"2023-09-23T18:20:56.582292+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":569,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582815+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:20:56.582815+0000","last_peered":"2023-09-23T18:20:56.582815+0000","last_clean":"2023-09-23T18:20:56.582815+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:20:56.582815+0000","last_undegraded":"2023-09-23T18:20:56.582815+0000","last_fullsized":"2023-09-23T18:20:56.582815+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":129,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583061+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:20:56.583061+0000","last_peered":"2023-09-23T18:20:56.583061+0000","last_clean":"2023-09-23T18:20:56.583061+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:20:56.583061+0000","last_undegraded":"2023-09-23T18:20:56.583061+0000","last_fullsized":"2023-09-23T18:20:56.583061+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":164,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.588646+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:20:56.588646+0000","last_peered":"2023-09-23T18:20:56.588646+0000","last_clean":"2023-09-23T18:20:56.588646+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:20:56.588646+0000","last_undegraded":"2023-09-23T18:20:56.588646+0000","last_fullsized":"2023-09-23T18:20:56.588646+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"619'11283","reported_seq":11336,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:06.180689+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:21:06.180689+0000","last_peered":"2023-09-23T18:21:06.180689+0000","last_clean":"2023-09-23T18:21:06.180689+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:21:06.180689+0000","last_undegraded":"2023-09-23T18:21:06.180689+0000","last_fullsized":"2023-09-23T18:21:06.180689+0000","mapping_epoch":574,"log_start":"603'5800","ondisk_log_start":"603'5800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5483,"log_dups_size":0,"ondisk_log_size":5483,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98369536,"num_objects":1501,"num_object_clones":0,"num_object_copies":4503,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1501,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11283,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":817,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586664+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:20:56.586664+0000","last_peered":"2023-09-23T18:20:56.586664+0000","last_clean":"2023-09-23T18:20:56.586664+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:20:56.586664+0000","last_undegraded":"2023-09-23T18:20:56.586664+0000","last_fullsized":"2023-09-23T18:20:56.586664+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":133,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577664+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:20:56.577664+0000","last_peered":"2023-09-23T18:20:56.577664+0000","last_clean":"2023-09-23T18:20:56.577664+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:20:56.577664+0000","last_undegraded":"2023-09-23T18:20:56.577664+0000","last_fullsized":"2023-09-23T18:20:56.577664+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":36,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.575523+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:20:56.575523+0000","last_peered":"2023-09-23T18:20:56.575523+0000","last_clean":"2023-09-23T18:20:56.575523+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:20:56.575523+0000","last_undegraded":"2023-09-23T18:20:56.575523+0000","last_fullsized":"2023-09-23T18:20:56.575523+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":634,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582025+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:20:56.582025+0000","last_peered":"2023-09-23T18:20:56.582025+0000","last_clean":"2023-09-23T18:20:56.582025+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:20:56.582025+0000","last_undegraded":"2023-09-23T18:20:56.582025+0000","last_fullsized":"2023-09-23T18:20:56.582025+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"619'11391","reported_seq":18372,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:06.281905+0000","last_change":"2023-09-23T18:20:36.673759+0000","last_active":"2023-09-23T18:21:06.281905+0000","last_peered":"2023-09-23T18:21:06.281905+0000","last_clean":"2023-09-23T18:21:06.281905+0000","last_became_active":"2023-09-23T18:20:36.651913+0000","last_became_peered":"2023-09-23T18:20:36.651913+0000","last_unstale":"2023-09-23T18:21:06.281905+0000","last_undegraded":"2023-09-23T18:21:06.281905+0000","last_fullsized":"2023-09-23T18:21:06.281905+0000","mapping_epoch":614,"log_start":"606'6900","ondisk_log_start":"606'6900","created":574,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4491,"log_dups_size":0,"ondisk_log_size":4491,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:47.765348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":102629400,"num_objects":1567,"num_object_clones":0,"num_object_copies":4701,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1567,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11391,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3453,"num_bytes_recovered":209321984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":917,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.589325+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:20:56.589325+0000","last_peered":"2023-09-23T18:20:56.589325+0000","last_clean":"2023-09-23T18:20:56.589325+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:20:56.589325+0000","last_undegraded":"2023-09-23T18:20:56.589325+0000","last_fullsized":"2023-09-23T18:20:56.589325+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":58,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586563+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:20:56.586563+0000","last_peered":"2023-09-23T18:20:56.586563+0000","last_clean":"2023-09-23T18:20:56.586563+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:20:56.586563+0000","last_undegraded":"2023-09-23T18:20:56.586563+0000","last_fullsized":"2023-09-23T18:20:56.586563+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581871+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:20:56.581871+0000","last_peered":"2023-09-23T18:20:56.581871+0000","last_clean":"2023-09-23T18:20:56.581871+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:20:56.581871+0000","last_undegraded":"2023-09-23T18:20:56.581871+0000","last_fullsized":"2023-09-23T18:20:56.581871+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":143,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582209+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:20:56.582209+0000","last_peered":"2023-09-23T18:20:56.582209+0000","last_clean":"2023-09-23T18:20:56.582209+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:20:56.582209+0000","last_undegraded":"2023-09-23T18:20:56.582209+0000","last_fullsized":"2023-09-23T18:20:56.582209+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"619'11355","reported_seq":11408,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:06.237341+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:21:06.237341+0000","last_peered":"2023-09-23T18:21:06.237341+0000","last_clean":"2023-09-23T18:21:06.237341+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:21:06.237341+0000","last_undegraded":"2023-09-23T18:21:06.237341+0000","last_fullsized":"2023-09-23T18:21:06.237341+0000","mapping_epoch":574,"log_start":"606'6900","ondisk_log_start":"606'6900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4455,"log_dups_size":0,"ondisk_log_size":4455,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":98238464,"num_objects":1499,"num_object_clones":0,"num_object_copies":4497,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1499,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11355,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":917,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.588452+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:20:56.588452+0000","last_peered":"2023-09-23T18:20:56.588452+0000","last_clean":"2023-09-23T18:20:56.588452+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:20:56.588452+0000","last_undegraded":"2023-09-23T18:20:56.588452+0000","last_fullsized":"2023-09-23T18:20:56.588452+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":105,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585775+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:20:56.585775+0000","last_peered":"2023-09-23T18:20:56.585775+0000","last_clean":"2023-09-23T18:20:56.585775+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:20:56.585775+0000","last_undegraded":"2023-09-23T18:20:56.585775+0000","last_fullsized":"2023-09-23T18:20:56.585775+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":398,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.589138+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:20:56.589138+0000","last_peered":"2023-09-23T18:20:56.589138+0000","last_clean":"2023-09-23T18:20:56.589138+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:20:56.589138+0000","last_undegraded":"2023-09-23T18:20:56.589138+0000","last_fullsized":"2023-09-23T18:20:56.589138+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":36,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577470+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:56.577470+0000","last_peered":"2023-09-23T18:20:56.577470+0000","last_clean":"2023-09-23T18:20:56.577470+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:20:56.577470+0000","last_undegraded":"2023-09-23T18:20:56.577470+0000","last_fullsized":"2023-09-23T18:20:56.577470+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"619'11341","reported_seq":11394,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:06.276608+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:21:06.276608+0000","last_peered":"2023-09-23T18:21:06.276608+0000","last_clean":"2023-09-23T18:21:06.276608+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:21:06.276608+0000","last_undegraded":"2023-09-23T18:21:06.276608+0000","last_fullsized":"2023-09-23T18:21:06.276608+0000","mapping_epoch":574,"log_start":"606'6900","ondisk_log_start":"606'6900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4441,"log_dups_size":0,"ondisk_log_size":4441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":96403456,"num_objects":1471,"num_object_clones":0,"num_object_copies":4413,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1471,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11341,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":752,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581684+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:20:56.581684+0000","last_peered":"2023-09-23T18:20:56.581684+0000","last_clean":"2023-09-23T18:20:56.581684+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:20:56.581684+0000","last_undegraded":"2023-09-23T18:20:56.581684+0000","last_fullsized":"2023-09-23T18:20:56.581684+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1055,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586115+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:20:56.586115+0000","last_peered":"2023-09-23T18:20:56.586115+0000","last_clean":"2023-09-23T18:20:56.586115+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:20:56.586115+0000","last_undegraded":"2023-09-23T18:20:56.586115+0000","last_fullsized":"2023-09-23T18:20:56.586115+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":46,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.580303+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:20:56.580303+0000","last_peered":"2023-09-23T18:20:56.580303+0000","last_clean":"2023-09-23T18:20:56.580303+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:20:56.580303+0000","last_undegraded":"2023-09-23T18:20:56.580303+0000","last_fullsized":"2023-09-23T18:20:56.580303+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":409,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583468+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:20:56.583468+0000","last_peered":"2023-09-23T18:20:56.583468+0000","last_clean":"2023-09-23T18:20:56.583468+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:20:56.583468+0000","last_undegraded":"2023-09-23T18:20:56.583468+0000","last_fullsized":"2023-09-23T18:20:56.583468+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":649,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.584553+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:20:56.584553+0000","last_peered":"2023-09-23T18:20:56.584553+0000","last_clean":"2023-09-23T18:20:56.584553+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:20:56.584553+0000","last_undegraded":"2023-09-23T18:20:56.584553+0000","last_fullsized":"2023-09-23T18:20:56.584553+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":42,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.647830+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:21:03.647830+0000","last_peered":"2023-09-23T18:21:03.647830+0000","last_clean":"2023-09-23T18:21:03.647830+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:21:03.647830+0000","last_undegraded":"2023-09-23T18:21:03.647830+0000","last_fullsized":"2023-09-23T18:21:03.647830+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":692,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.626421+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:20:56.626421+0000","last_peered":"2023-09-23T18:20:56.626421+0000","last_clean":"2023-09-23T18:20:56.626421+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:20:56.626421+0000","last_undegraded":"2023-09-23T18:20:56.626421+0000","last_fullsized":"2023-09-23T18:20:56.626421+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":649,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582590+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:20:56.582590+0000","last_peered":"2023-09-23T18:20:56.582590+0000","last_clean":"2023-09-23T18:20:56.582590+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:20:56.582590+0000","last_undegraded":"2023-09-23T18:20:56.582590+0000","last_fullsized":"2023-09-23T18:20:56.582590+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":137,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581957+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:20:56.581957+0000","last_peered":"2023-09-23T18:20:56.581957+0000","last_clean":"2023-09-23T18:20:56.581957+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:20:56.581957+0000","last_undegraded":"2023-09-23T18:20:56.581957+0000","last_fullsized":"2023-09-23T18:20:56.581957+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":252,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.818135+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:20:46.818135+0000","last_peered":"2023-09-23T18:20:46.818135+0000","last_clean":"2023-09-23T18:20:46.818135+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:20:46.818135+0000","last_undegraded":"2023-09-23T18:20:46.818135+0000","last_fullsized":"2023-09-23T18:20:46.818135+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":123,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582182+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:20:56.582182+0000","last_peered":"2023-09-23T18:20:56.582182+0000","last_clean":"2023-09-23T18:20:56.582182+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:20:56.582182+0000","last_undegraded":"2023-09-23T18:20:56.582182+0000","last_fullsized":"2023-09-23T18:20:56.582182+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":235,"reported_epoch":617,"state":"active+clean","last_fresh":"2023-09-23T18:20:46.817650+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:20:46.817650+0000","last_peered":"2023-09-23T18:20:46.817650+0000","last_clean":"2023-09-23T18:20:46.817650+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:20:46.817650+0000","last_undegraded":"2023-09-23T18:20:46.817650+0000","last_fullsized":"2023-09-23T18:20:46.817650+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1011,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586776+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:20:56.586776+0000","last_peered":"2023-09-23T18:20:56.586776+0000","last_clean":"2023-09-23T18:20:56.586776+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:20:56.586776+0000","last_undegraded":"2023-09-23T18:20:56.586776+0000","last_fullsized":"2023-09-23T18:20:56.586776+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":516,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.589238+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:20:56.589238+0000","last_peered":"2023-09-23T18:20:56.589238+0000","last_clean":"2023-09-23T18:20:56.589238+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:20:56.589238+0000","last_undegraded":"2023-09-23T18:20:56.589238+0000","last_fullsized":"2023-09-23T18:20:56.589238+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":153,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583621+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:20:56.583621+0000","last_peered":"2023-09-23T18:20:56.583621+0000","last_clean":"2023-09-23T18:20:56.583621+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:20:56.583621+0000","last_undegraded":"2023-09-23T18:20:56.583621+0000","last_fullsized":"2023-09-23T18:20:56.583621+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":948,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.628482+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:20:56.628482+0000","last_peered":"2023-09-23T18:20:56.628482+0000","last_clean":"2023-09-23T18:20:56.628482+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:20:56.628482+0000","last_undegraded":"2023-09-23T18:20:56.628482+0000","last_fullsized":"2023-09-23T18:20:56.628482+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":141,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585916+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:20:56.585916+0000","last_peered":"2023-09-23T18:20:56.585916+0000","last_clean":"2023-09-23T18:20:56.585916+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:20:56.585916+0000","last_undegraded":"2023-09-23T18:20:56.585916+0000","last_fullsized":"2023-09-23T18:20:56.585916+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":662,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582738+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:20:56.582738+0000","last_peered":"2023-09-23T18:20:56.582738+0000","last_clean":"2023-09-23T18:20:56.582738+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:20:56.582738+0000","last_undegraded":"2023-09-23T18:20:56.582738+0000","last_fullsized":"2023-09-23T18:20:56.582738+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1598947352,"num_objects":24399,"num_object_clones":0,"num_object_copies":73197,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14825,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24399,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":182409,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8317,"num_bytes_recovered":482803712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":6641430528,"data_stored":6641418312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77209,"ondisk_log_size":77209,"up":48,"acting":44,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20983808,"data_stored":20971577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001581,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669518,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1607424,"kb_used_data":974776,"kb_used_omap":7,"kb_used_meta":632632,"kb_avail":92137728,"statfs":{"total":95995035648,"available":94349033472,"internally_reserved":0,"allocated":998170624,"data_stored":996208217,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":647815972},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":1.7889999999999999,"15min":1.407},"min":{"1min":0.249,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":9.1600000000000001,"5min":24.472999999999999,"15min":24.472999999999999},"last":1.0089999999999999},{"interface":"front","average":{"1min":2.54,"5min":2.1960000000000002,"15min":1.726},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2029999999999994,"5min":22.832999999999998,"15min":22.832999999999998},"last":0.38300000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:20:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3570000000000002,"5min":2.0350000000000001,"15min":1.4890000000000001},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.726000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":2.9990000000000001},{"interface":"front","average":{"1min":2.423,"5min":1.899,"15min":1.4379999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.336,"5min":22.664000000000001,"15min":22.664000000000001},"last":12.273}]},{"osd":2,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.579,"15min":1.345},"min":{"1min":0.33500000000000002,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":5.7919999999999998,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":3.1600000000000001},{"interface":"front","average":{"1min":2.3029999999999999,"5min":2.0600000000000001,"15min":1.5580000000000001},"min":{"1min":0.39700000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0730000000000004,"5min":14.771000000000001,"15min":14.771000000000001},"last":3.1920000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":3.0419999999999998,"15min":2.1150000000000002},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.24099999999999999},"max":{"1min":5.8170000000000002,"5min":103.348,"15min":103.348},"last":0.51300000000000001},{"interface":"front","average":{"1min":2.5600000000000001,"5min":3.6190000000000002,"15min":2.4209999999999998},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":7.4530000000000003,"5min":103.40300000000001,"15min":103.40300000000001},"last":0.45300000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.8180000000000001,"15min":1.345},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.0730000000000004,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.42999999999999999},{"interface":"front","average":{"1min":1.776,"5min":1.8200000000000001,"15min":1.3700000000000001},"min":{"1min":0.45200000000000001,"5min":0.30399999999999999,"15min":0.27000000000000002},"max":{"1min":7.8319999999999999,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.33200000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":2.1099999999999999,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":18.417000000000002,"5min":18.417000000000002,"15min":18.417000000000002},"last":2.109},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.3399999999999999,"15min":2.3780000000000001},"min":{"1min":0.36299999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":18.454999999999998,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.39400000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":2.1059999999999999,"15min":1.5580000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.26800000000000002},"max":{"1min":6.1500000000000004,"5min":10.743,"15min":13.460000000000001},"last":0.47999999999999998},{"interface":"front","average":{"1min":2.0830000000000002,"5min":2.0529999999999999,"15min":1.603},"min":{"1min":0.40300000000000002,"5min":0.38500000000000001,"15min":0.28499999999999998},"max":{"1min":8.6890000000000001,"5min":11.244,"15min":12.404},"last":0.41099999999999998}]}]},{"osd":1,"up_from":545,"seq":2340757176366,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1303704,"kb_used_data":1077904,"kb_used_omap":13,"kb_used_meta":225778,"kb_avail":92441448,"statfs":{"total":95995035648,"available":94660042752,"internally_reserved":0,"allocated":1103773696,"data_stored":1100738068,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":231197325},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.016,"15min":0.59899999999999998},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":9.0299999999999994,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":5.0359999999999996},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.004,"15min":0.66100000000000003},"min":{"1min":0.34499999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.952999999999999,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.41799999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.0980000000000001,"15min":0.67300000000000004},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.6219999999999999,"5min":9.6219999999999999,"15min":9.6219999999999999},"last":2.8530000000000002},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.77200000000000002,"15min":0.59399999999999997},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2080000000000002,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":1.891}]},{"osd":3,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.0409999999999999,"15min":0.60499999999999998},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.3200000000000003,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":1.996},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.2769999999999999,"15min":0.76700000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":19.762,"5min":19.762,"15min":19.762},"last":2.9710000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.3280000000000001,"15min":1.3200000000000001},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.8690000000000002,"5min":12.663,"15min":12.663},"last":0.48099999999999998},{"interface":"front","average":{"1min":2.4049999999999998,"5min":1.7789999999999999,"15min":1.6020000000000001},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.317},"max":{"1min":6.0289999999999999,"5min":13.222,"15min":13.222},"last":1.7210000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.405,"15min":0.94999999999999996},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":23.190999999999999,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.502},{"interface":"front","average":{"1min":4.1719999999999997,"5min":1.7330000000000001,"15min":1.113},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":23.204000000000001,"5min":23.204000000000001,"15min":23.204000000000001},"last":2.8969999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.121,"5min":1.4690000000000001,"15min":0.996},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":23.443000000000001,"5min":23.443000000000001,"15min":23.443000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":3.7970000000000002,"5min":1.746,"15min":1.099},"min":{"1min":0.42199999999999999,"5min":0.378,"15min":0.378},"max":{"1min":26.177,"5min":26.177,"15min":26.177},"last":1.6759999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.4340000000000002,"5min":1.3620000000000001,"15min":0.90800000000000003},"min":{"1min":0.42299999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.1050000000000004,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":1.754},{"interface":"front","average":{"1min":2.3140000000000001,"5min":1.474,"15min":1.028},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":9.1500000000000004,"5min":9.1500000000000004,"15min":9.1500000000000004},"last":2.9369999999999998}]}]},{"osd":0,"up_from":93,"seq":399431958863,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2259524,"kb_used_data":1310076,"kb_used_omap":7,"kb_used_meta":949432,"kb_avail":91485628,"statfs":{"total":95995035648,"available":93681283072,"internally_reserved":0,"allocated":1341517824,"data_stored":1338626555,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":972219108},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.044,"15min":0.56100000000000005},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.4779999999999998,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.23699999999999999},{"interface":"front","average":{"1min":2.6240000000000001,"5min":1.278,"15min":0.77400000000000002},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.335000000000001,"5min":15.335000000000001,"15min":15.335000000000001},"last":5.3769999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:21:04 2023","interfaces":[{"interface":"back","average":{"1min":3.008,"5min":2.0880000000000001,"15min":1.4419999999999999},"min":{"1min":0.40200000000000002,"5min":0.185,"15min":0.185},"max":{"1min":10.013,"5min":18.550999999999998,"15min":18.550999999999998},"last":4.4500000000000002},{"interface":"front","average":{"1min":3.2160000000000002,"5min":1.7470000000000001,"15min":1.097},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":23.161999999999999,"5min":23.161999999999999,"15min":23.161999999999999},"last":5.3460000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.4199999999999999,"15min":1.5940000000000001},"min":{"1min":0.313,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.8460000000000001,"5min":13.539999999999999,"15min":13.539999999999999},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.6619999999999999,"15min":1.7010000000000001},"min":{"1min":0.36299999999999999,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":6.8879999999999999,"5min":13.285,"15min":15.198},"last":0.376}]},{"osd":4,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7010000000000001,"5min":2.1309999999999998,"15min":1.8979999999999999},"min":{"1min":0.41799999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":17.741,"5min":17.741,"15min":57.746000000000002},"last":0.41699999999999998},{"interface":"front","average":{"1min":3.79,"5min":1.9159999999999999,"15min":1.7809999999999999},"min":{"1min":0.50900000000000001,"5min":0.26900000000000002,"15min":0.25600000000000001},"max":{"1min":15.079000000000001,"5min":15.079000000000001,"15min":57.816000000000003},"last":4.4139999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":1.8220000000000001,"15min":1.4410000000000001},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":5.5970000000000004,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.44},{"interface":"front","average":{"1min":3.2909999999999999,"5min":2.4630000000000001,"15min":1.599},"min":{"1min":0.40100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":15.047000000000001,"5min":15.047000000000001,"15min":15.047000000000001},"last":4.3609999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":1.948,"15min":1.9530000000000001},"min":{"1min":0.47499999999999998,"5min":0.377,"15min":0.32000000000000001},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":58.457999999999998},"last":2.4900000000000002},{"interface":"front","average":{"1min":4.6639999999999997,"5min":2.3159999999999998,"15min":1.9299999999999999},"min":{"1min":0.497,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":18.643999999999998,"5min":18.643999999999998,"15min":58.354999999999997},"last":5.2770000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":3.431,"5min":2.2480000000000002,"15min":1.7889999999999999},"min":{"1min":0.49199999999999999,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":22.945,"5min":22.945,"15min":22.945},"last":5.3159999999999998},{"interface":"front","average":{"1min":2.2330000000000001,"5min":1.714,"15min":1.851},"min":{"1min":0.47099999999999997,"5min":0.371,"15min":0.36199999999999999},"max":{"1min":6.9249999999999998,"5min":7.7519999999999998,"15min":10.404},"last":0.49199999999999999}]}]},{"osd":2,"up_from":0,"seq":2070174236748,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249378,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1433824,"kb_used_data":682392,"kb_used_omap":3,"kb_used_meta":751356,"kb_avail":92311328,"statfs":{"total":95995035648,"available":94526799872,"internally_reserved":0,"allocated":698769408,"data_stored":695675977,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":769389288},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":1.5800000000000001,"15min":1.8089999999999999},"min":{"1min":0.20799999999999999,"5min":0.192,"15min":0.183},"max":{"1min":12.472,"5min":17.036000000000001,"15min":195.69800000000001},"last":2.1440000000000001},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.405,"15min":2.004},"min":{"1min":0.23799999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.468,"5min":11.468,"15min":194.661},"last":0.434}]},{"osd":1,"last update":"Sat Sep 23 18:20:26 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":1.4590000000000001,"15min":0.83299999999999996},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.079000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.48599999999999999},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.5129999999999999,"15min":1.3069999999999999},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.081,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.81999999999999995}]},{"osd":2,"last update":"Sat Sep 23 18:21:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5569999999999999,"5min":1.526,"15min":1.3740000000000001},"min":{"1min":0.29899999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.2430000000000003,"5min":12.151999999999999,"15min":12.151999999999999},"last":0.51200000000000001},{"interface":"front","average":{"1min":1.78,"5min":1.3839999999999999,"15min":1.361},"min":{"1min":0.32300000000000001,"5min":0.27900000000000003,"15min":0.19600000000000001},"max":{"1min":7.0640000000000001,"5min":11.565,"15min":11.565},"last":1.9159999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":1.8839999999999999,"15min":1.706},"min":{"1min":0.34200000000000003,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":11.4,"5min":21.062999999999999,"15min":37.231000000000002},"last":2.2450000000000001},{"interface":"front","average":{"1min":2.532,"5min":2.1240000000000001,"15min":1.885},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":9.1630000000000003,"5min":21.128,"15min":37.381},"last":1.169}]},{"osd":5,"last update":"Sat Sep 23 18:20:37 2023","interfaces":[{"interface":"back","average":{"1min":3.452,"5min":2.048,"15min":1.587},"min":{"1min":0.35099999999999998,"5min":0.316,"15min":0.316},"max":{"1min":9.7430000000000003,"5min":11.805999999999999,"15min":11.805999999999999},"last":1.137},{"interface":"front","average":{"1min":3.0339999999999998,"5min":1.74,"15min":1.722},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":17.135999999999999,"5min":17.135999999999999,"15min":17.135999999999999},"last":0.89000000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.9390000000000001,"15min":1.9379999999999999},"min":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":10.303000000000001,"5min":17.192,"15min":115.54000000000001},"last":0.96599999999999997},{"interface":"front","average":{"1min":2.9889999999999999,"5min":1.976,"15min":1.9810000000000001},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.30499999999999999},"max":{"1min":11.442,"5min":14.321,"15min":115.691},"last":0.47299999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0939999999999999,"5min":1.744,"15min":1.7609999999999999},"min":{"1min":0.42699999999999999,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":15.972},"last":3.4409999999999998},{"interface":"front","average":{"1min":1.974,"5min":1.829,"15min":1.6240000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":6.9749999999999996,"5min":17.164000000000001,"15min":17.164000000000001},"last":2.169}]}]},{"osd":4,"up_from":30,"seq":128849019327,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2286324,"kb_used_data":1606444,"kb_used_omap":0,"kb_used_meta":679872,"kb_avail":91458828,"statfs":{"total":95995035648,"available":93653839872,"internally_reserved":0,"allocated":1644998656,"data_stored":1643040053,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":696188928},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":2.0609999999999999,"5min":1.534,"15min":1.268},"min":{"1min":0.255,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":7.6079999999999997,"5min":7.6079999999999997,"15min":13.375999999999999},"last":3.7719999999999998},{"interface":"front","average":{"1min":2.1890000000000001,"5min":1.6819999999999999,"15min":1.546},"min":{"1min":0.26600000000000001,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":9.3710000000000004,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.46000000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:20:22 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":1.7170000000000001,"15min":1.0349999999999999},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8220000000000001,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.8420000000000001,"5min":1.6679999999999999,"15min":0.98999999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.5700000000000003,"5min":43.497999999999998,"15min":43.497999999999998},"last":3.8079999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3839999999999999,"5min":1.768,"15min":1.8500000000000001},"min":{"1min":0.34300000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.1749999999999998,"5min":11.896000000000001,"15min":11.896000000000001},"last":3.8610000000000002},{"interface":"front","average":{"1min":2.8079999999999998,"5min":1.8360000000000001,"15min":1.7649999999999999},"min":{"1min":0.36199999999999999,"5min":0.29899999999999999,"15min":0.253},"max":{"1min":17.190999999999999,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.51600000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.395,"5min":1.5529999999999999,"15min":1.9630000000000001},"min":{"1min":0.379,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.7519999999999998,"5min":11.130000000000001,"15min":29.154},"last":3.8340000000000001},{"interface":"front","average":{"1min":2.6230000000000002,"5min":1.8300000000000001,"15min":1.9930000000000001},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.099,"5min":13.470000000000001,"15min":29.216000000000001},"last":3.5230000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.3170000000000002,"15min":2.7829999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.5009999999999994,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.81100000000000005},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.6499999999999999,"15min":2.5609999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":3.8359999999999999,"5min":13.938000000000001,"15min":32.640000000000001},"last":0.58799999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.3899999999999999,"5min":1.873,"15min":1.347},"min":{"1min":0.32000000000000001,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":6.1989999999999998,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.61299999999999999},{"interface":"front","average":{"1min":1.859,"5min":1.9630000000000001,"15min":1.522},"min":{"1min":0.38700000000000001,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":9.5579999999999998,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.64000000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:20:07 2023","interfaces":[{"interface":"back","average":{"1min":1.875,"5min":2.4849999999999999,"15min":1.774},"min":{"1min":0.39000000000000001,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":5.9539999999999997,"5min":19.298999999999999,"15min":19.298999999999999},"last":0.49299999999999999},{"interface":"front","average":{"1min":1.722,"5min":1.8899999999999999,"15min":1.552},"min":{"1min":0.435,"5min":0.42499999999999999,"15min":0.28399999999999997},"max":{"1min":9.4730000000000008,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.53600000000000003}]}]},{"osd":5,"up_from":420,"seq":1803886264424,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1360560,"kb_used_data":996840,"kb_used_omap":9,"kb_used_meta":363702,"kb_avail":92384592,"statfs":{"total":95995035648,"available":94601822208,"internally_reserved":0,"allocated":1020764160,"data_stored":1018813889,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":372431293},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.2290000000000001,"15min":0.93200000000000005},"min":{"1min":0.21299999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.867,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.20499999999999999},{"interface":"front","average":{"1min":2.137,"5min":1.8,"15min":1.6459999999999999},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.621,"5min":20.937999999999999,"15min":20.937999999999999},"last":0.36899999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":1.7070000000000001,"15min":1.2989999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.6649999999999991,"5min":23.946000000000002,"15min":23.946000000000002},"last":8.7390000000000008},{"interface":"front","average":{"1min":2.1840000000000002,"5min":1.73,"15min":1.141},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.5259999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":0.52700000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.6160000000000001,"15min":1.611},"min":{"1min":0.29699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.1150000000000002,"5min":10.705,"15min":10.705},"last":1.498},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.8520000000000001,"15min":2.1280000000000001},"min":{"1min":0.39600000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.7560000000000002,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.55600000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.762,"15min":1.359},"min":{"1min":0.35599999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.0060000000000002,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":1.5209999999999999},{"interface":"front","average":{"1min":2.2200000000000002,"5min":2.1150000000000002,"15min":1.7130000000000001},"min":{"1min":0.372,"5min":0.33500000000000002,"15min":0.313},"max":{"1min":10.664999999999999,"5min":20.035,"15min":20.035},"last":0.46200000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":2.1480000000000001,"15min":2.0499999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.2270000000000003,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1179999999999999,"15min":1.6639999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7919999999999998,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":1.5660000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.8340000000000001,"15min":1.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.2189999999999999,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":1.5369999999999999},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.3319999999999999,"15min":1.7370000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.6920000000000002,"5min":12.504,"15min":12.504},"last":4.2190000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.23,"15min":1.804},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.195,"5min":12.641,"15min":13.506},"last":0.38900000000000001},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.8500000000000001,"15min":1.5600000000000001},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":5.0839999999999996,"5min":13.138,"15min":13.138},"last":1.552}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1296896000,"data_stored":1296891928,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":1081606144,"data_stored":1081606144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":674758656,"data_stored":674758656,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1614811136,"data_stored":1614807064,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":1006239744,"data_stored":1006239744,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":967118848,"data_stored":967114776,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:21:08.099 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:08.155 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:08.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:08.160 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:08.261 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:08.286 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:08.359 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:08.560 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:08.566 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:08.566 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:08.567 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:08.668 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:08.735 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:08.817 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:08 smithi055 ceph-mon[179668]: pgmap v1493: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.5 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 796 op/s; 14838/73278 objects degraded (20.249%); 0 B/s, 45 objects/s recovering 2023-09-23T18:21:08.818 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:08 smithi055 ceph-mon[179668]: from='client.56716 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:08.818 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:08 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 14838/73278 objects degraded (20.249%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:08.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:08 smithi055 ceph-mon[184174]: pgmap v1493: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.5 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 796 op/s; 14838/73278 objects degraded (20.249%); 0 B/s, 45 objects/s recovering 2023-09-23T18:21:08.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:08 smithi055 ceph-mon[184174]: from='client.56716 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:08.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:08 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 14838/73278 objects degraded (20.249%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:08.821 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:08.828 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:08.828 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:08.929 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:09.003 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:09.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:08 smithi099 ceph-mon[167076]: pgmap v1493: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.5 GiB data, 9.8 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 796 op/s; 14838/73278 objects degraded (20.249%); 0 B/s, 45 objects/s recovering 2023-09-23T18:21:09.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:08 smithi099 ceph-mon[167076]: from='client.56716 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:09.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:08 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 14838/73278 objects degraded (20.249%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:09.501 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:09.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:09.512 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:09.612 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:09.654 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:09.708 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:09.714 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:09.714 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:09.815 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:09.894 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:09.968 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:09.973 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:09.973 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:10.074 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:10.126 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:10.132 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:21:10.173 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:10.179 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:10.180 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:10.219 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:21:10.220 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:21:10.220 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.220 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:186064 10.0 e619/574 10:053c3806:::benchmark_data_smithi055_425050_object82658:head v 619'11702, mlcod=619'11700)", 2023-09-23T18:21:10.220 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:10.217338+0000", 2023-09-23T18:21:10.220 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0018102299999999999, 2023-09-23T18:21:10.220 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018393369999999999, 2023-09-23T18:21:10.220 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:10.220 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:21:10.220 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217338+0000", 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217338+0000", 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.221 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217337+0000", 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217347+0000", 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0438000000000001e-05 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:10.222 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217368+0000", 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0472999999999999e-05 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217377+0000", 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.0180000000000007e-06 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217534+0000", 2023-09-23T18:21:10.223 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015732499999999999 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217555+0000", 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0871000000000001e-05 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.224 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:186063 10.b e619/574)", 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:10.217391+0000", 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0017568659999999999, 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018564219999999999, 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217391+0000", 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:10.225 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217391+0000", 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217391+0000", 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.226 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217396+0000", 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1259999999999997e-06 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217402+0000", 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8510000000000004e-06 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:10.227 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.217406+0000", 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.143e-06 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:186068 10.2 10:4f1c8b63:::benchmark_data_smithi055_425050_object82662:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e619)", 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:10.210942+0000", 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0082060469999999993, 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0083675320000000004, 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:10.228 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 186068 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.210942+0000", 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:10.229 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.210942+0000", 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.210940+0000", 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.230 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.210946+0000", 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3980000000000004e-06 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.210967+0000", 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0224000000000002e-05 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.210975+0000", 2023-09-23T18:21:10.231 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.5609999999999999e-06 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.211032+0000", 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.7061999999999997e-05 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.211180+0000", 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014745000000000001 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.232 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.211339+0000", 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000159691 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.214054+0000", 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027150719999999998 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:10.233 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:10.234 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:10.214940+0000", 2023-09-23T18:21:10.234 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00088581200000000004 2023-09-23T18:21:10.234 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:10.234 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:10.234 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:10.234 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:10.234 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:21:10.234 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 3 2023-09-23T18:21:10.234 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:21:10.281 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:10.469 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:21:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:21:10] "GET /metrics HTTP/1.1" 200 36666 "" "Prometheus/2.43.0" 2023-09-23T18:21:10.478 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:10.768 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:10.776 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:10.777 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:10.777 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:10 smithi055 ceph-mon[179668]: pgmap v1494: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.5 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 800 op/s; 14838/71784 objects degraded (20.670%); 0 B/s, 45 objects/s recovering 2023-09-23T18:21:10.777 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:10 smithi055 ceph-mon[184174]: pgmap v1494: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.5 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 800 op/s; 14838/71784 objects degraded (20.670%); 0 B/s, 45 objects/s recovering 2023-09-23T18:21:10.842 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:21:10.878 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:10.959 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:11.068 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:21:11.075 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:11.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:10 smithi099 ceph-mon[167076]: pgmap v1494: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.5 GiB data, 9.5 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 800 op/s; 14838/71784 objects degraded (20.670%); 0 B/s, 45 objects/s recovering 2023-09-23T18:21:11.081 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:11.082 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:11.182 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:11.213 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:11.215 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:11.254 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:11.261 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:11.262 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:11.362 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:11.427 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:11.484 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:11.493 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:11.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:11.594 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:11.704 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:11.756 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:21:11.788 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:11 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3598943800' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:21:11.788 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:11 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:21:11.788 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:11 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3598943800' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:21:11.788 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:11 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:21:11.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:11 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3598943800' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:21:11.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:11 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:21:11.832 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:11.840 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:11.840 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:11.941 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:12.025 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:12.508 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:12.511 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:12.512 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:12.613 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:12.672 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:12.739 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:12.749 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:12.750 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:12.851 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:12.915 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[184174]: pgmap v1495: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.4 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 14838/70203 objects degraded (21.136%); 0 B/s, 45 objects/s recovering 2023-09-23T18:21:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[184174]: failed to encode map e620 with expected crc 2023-09-23T18:21:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:21:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[184174]: osdmap e620: 8 total, 8 up, 6 in 2023-09-23T18:21:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[184174]: failed to encode map e620 with expected crc 2023-09-23T18:21:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[179668]: pgmap v1495: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.4 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 14838/70203 objects degraded (21.136%); 0 B/s, 45 objects/s recovering 2023-09-23T18:21:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[179668]: failed to encode map e620 with expected crc 2023-09-23T18:21:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:21:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[179668]: osdmap e620: 8 total, 8 up, 6 in 2023-09-23T18:21:12.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:12 smithi055 ceph-mon[179668]: failed to encode map e620 with expected crc 2023-09-23T18:21:12.984 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:12.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:12 smithi099 ceph-mon[167076]: pgmap v1495: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.4 GiB data, 9.4 GiB used, 527 GiB / 536 GiB avail; 0 B/s wr, 804 op/s; 14838/70203 objects degraded (21.136%); 0 B/s, 45 objects/s recovering 2023-09-23T18:21:12.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:12 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:12.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:12 smithi099 ceph-mon[167076]: failed to encode map e620 with expected crc 2023-09-23T18:21:12.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:12 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:21:12.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:12 smithi099 ceph-mon[167076]: osdmap e620: 8 total, 8 up, 6 in 2023-09-23T18:21:12.995 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:12 smithi099 ceph-mon[167076]: failed to encode map e620 with expected crc 2023-09-23T18:21:12.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:12.996 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:13.097 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:13.159 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:13.412 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:13.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:13.417 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:13.518 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:13.615 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:13.757 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:13.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:13.761 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:13.766 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:13.766 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:13.862 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:13.926 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:13 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 14838/70203 objects degraded (21.136%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:13.927 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:13 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 14838/70203 objects degraded (21.136%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:13.939 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:13 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 14838/70203 objects degraded (21.136%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:14.303 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:14.310 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:14.311 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:14.384 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:21:14.412 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:14.446 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:21:14.455 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:14.493 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:21:14.517 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:14.527 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:14.527 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:14.530 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:21:14.577 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:21:14.615 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:21:14.628 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:14.660 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:21:14.669 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:14.706 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:21:14.714 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:21:14.714 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:21:14.714 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:21:14.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:21:14.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:21:14.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:21:14.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:21:14.715 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:21:14.724 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:14.730 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:14.730 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:14.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:14 smithi055 ceph-mon[184174]: pgmap v1497: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 854 op/s; 14616/60474 objects degraded (24.169%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:14.818 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3304111889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:14.818 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:14 smithi055 ceph-mon[179668]: pgmap v1497: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 854 op/s; 14616/60474 objects degraded (24.169%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:14.818 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3304111889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:14.831 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:14.878 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:14.935 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:14.944 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:14.944 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:15.045 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:15.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:14 smithi099 ceph-mon[167076]: pgmap v1497: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.2 GiB data, 8.9 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 854 op/s; 14616/60474 objects degraded (24.169%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:15.080 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3304111889' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:15.095 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:15.160 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:15.166 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:15.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:15.267 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:15.353 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:15.466 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:15.471 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:15.471 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:15.572 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:15.659 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:15.675 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:21:15.874 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:15.884 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:15.884 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:15.985 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:16.083 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:16.150 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:21:16.151 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:21:16.188 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:16.194 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:16.195 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:16.295 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:16.355 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:16.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:16.431 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:16.431 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:16.532 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:16.610 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:16.677 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:16.689 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:16.690 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:16.720 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:21:16.791 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:16.813 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:21:16.834 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:21:16.871 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:16 smithi055 ceph-mon[179668]: pgmap v1498: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.2 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 14616/59700 objects degraded (24.482%); 0 B/s, 47 objects/s recovering 2023-09-23T18:21:16.875 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:16 smithi055 ceph-mon[184174]: pgmap v1498: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.2 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 14616/59700 objects degraded (24.482%); 0 B/s, 47 objects/s recovering 2023-09-23T18:21:16.879 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:16.929 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:16 smithi099 ceph-mon[167076]: pgmap v1498: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1.2 GiB data, 8.8 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 816 op/s; 14616/59700 objects degraded (24.482%); 0 B/s, 47 objects/s recovering 2023-09-23T18:21:17.323 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:21:17.378 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:17.388 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:17.389 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:17.473 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1498,"stamp":"2023-09-23T18:21:15.193983+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1306802876,"num_objects":19900,"num_object_clones":0,"num_object_copies":59700,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14616,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19900,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":187846,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8937,"num_bytes_recovered":502226569,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83257,"ondisk_log_size":83257,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":9242264,"kb_used_data":5603880,"kb_used_omap":40,"kb_used_meta":3638103,"kb_avail":553228648,"statfs":{"total":575970213888,"available":566506135552,"internally_reserved":0,"allocated":5738373120,"data_stored":5723457991,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3725417786},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[4,1,11,6,4,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-535494656,"num_objects":-8171,"num_object_clones":0,"num_object_copies":-24513,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-480,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-8171,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":8171,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":480,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.003091"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":907,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582135+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:20:56.582135+0000","last_peered":"2023-09-23T18:20:56.582135+0000","last_clean":"2023-09-23T18:20:56.582135+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:20:56.582135+0000","last_undegraded":"2023-09-23T18:20:56.582135+0000","last_fullsized":"2023-09-23T18:20:56.582135+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":279,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.645861+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:21:03.645861+0000","last_peered":"2023-09-23T18:21:03.645861+0000","last_clean":"2023-09-23T18:21:03.645861+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:21:03.645861+0000","last_undegraded":"2023-09-23T18:21:03.645861+0000","last_fullsized":"2023-09-23T18:21:03.645861+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":35,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585740+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:20:56.585740+0000","last_peered":"2023-09-23T18:20:56.585740+0000","last_clean":"2023-09-23T18:20:56.585740+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:20:56.585740+0000","last_undegraded":"2023-09-23T18:20:56.585740+0000","last_fullsized":"2023-09-23T18:20:56.585740+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":913,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583809+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:20:56.583809+0000","last_peered":"2023-09-23T18:20:56.583809+0000","last_clean":"2023-09-23T18:20:56.583809+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:20:56.583809+0000","last_undegraded":"2023-09-23T18:20:56.583809+0000","last_fullsized":"2023-09-23T18:20:56.583809+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":360,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577982+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:20:56.577982+0000","last_peered":"2023-09-23T18:20:56.577982+0000","last_clean":"2023-09-23T18:20:56.577982+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:20:56.577982+0000","last_undegraded":"2023-09-23T18:20:56.577982+0000","last_fullsized":"2023-09-23T18:20:56.577982+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":680,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.580059+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:20:56.580059+0000","last_peered":"2023-09-23T18:20:56.580059+0000","last_clean":"2023-09-23T18:20:56.580059+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:20:56.580059+0000","last_undegraded":"2023-09-23T18:20:56.580059+0000","last_fullsized":"2023-09-23T18:20:56.580059+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":930,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582996+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:20:56.582996+0000","last_peered":"2023-09-23T18:20:56.582996+0000","last_clean":"2023-09-23T18:20:56.582996+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:56.582996+0000","last_undegraded":"2023-09-23T18:20:56.582996+0000","last_fullsized":"2023-09-23T18:20:56.582996+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":36,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.641831+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:21:03.641831+0000","last_peered":"2023-09-23T18:21:03.641831+0000","last_clean":"2023-09-23T18:21:03.641831+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:21:03.641831+0000","last_undegraded":"2023-09-23T18:21:03.641831+0000","last_fullsized":"2023-09-23T18:21:03.641831+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":195,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585783+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:20:56.585783+0000","last_peered":"2023-09-23T18:20:56.585783+0000","last_clean":"2023-09-23T18:20:56.585783+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:20:56.585783+0000","last_undegraded":"2023-09-23T18:20:56.585783+0000","last_fullsized":"2023-09-23T18:20:56.585783+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1053,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581993+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:20:56.581993+0000","last_peered":"2023-09-23T18:20:56.581993+0000","last_clean":"2023-09-23T18:20:56.581993+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:20:56.581993+0000","last_undegraded":"2023-09-23T18:20:56.581993+0000","last_fullsized":"2023-09-23T18:20:56.581993+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":129,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586070+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:20:56.586070+0000","last_peered":"2023-09-23T18:20:56.586070+0000","last_clean":"2023-09-23T18:20:56.586070+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:20:56.586070+0000","last_undegraded":"2023-09-23T18:20:56.586070+0000","last_fullsized":"2023-09-23T18:20:56.586070+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":209,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577734+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:20:56.577734+0000","last_peered":"2023-09-23T18:20:56.577734+0000","last_clean":"2023-09-23T18:20:56.577734+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:20:56.577734+0000","last_undegraded":"2023-09-23T18:20:56.577734+0000","last_fullsized":"2023-09-23T18:20:56.577734+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":830,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586286+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:20:56.586286+0000","last_peered":"2023-09-23T18:20:56.586286+0000","last_clean":"2023-09-23T18:20:56.586286+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:20:56.586286+0000","last_undegraded":"2023-09-23T18:20:56.586286+0000","last_fullsized":"2023-09-23T18:20:56.586286+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":165,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581954+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:20:56.581954+0000","last_peered":"2023-09-23T18:20:56.581954+0000","last_clean":"2023-09-23T18:20:56.581954+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:20:56.581954+0000","last_undegraded":"2023-09-23T18:20:56.581954+0000","last_fullsized":"2023-09-23T18:20:56.581954+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":164,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.584467+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:20:56.584467+0000","last_peered":"2023-09-23T18:20:56.584467+0000","last_clean":"2023-09-23T18:20:56.584467+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:20:56.584467+0000","last_undegraded":"2023-09-23T18:20:56.584467+0000","last_fullsized":"2023-09-23T18:20:56.584467+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":964,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586154+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:20:56.586154+0000","last_peered":"2023-09-23T18:20:56.586154+0000","last_clean":"2023-09-23T18:20:56.586154+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:20:56.586154+0000","last_undegraded":"2023-09-23T18:20:56.586154+0000","last_fullsized":"2023-09-23T18:20:56.586154+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":413,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586051+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:20:56.586051+0000","last_peered":"2023-09-23T18:20:56.586051+0000","last_clean":"2023-09-23T18:20:56.586051+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:20:56.586051+0000","last_undegraded":"2023-09-23T18:20:56.586051+0000","last_fullsized":"2023-09-23T18:20:56.586051+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":432,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582972+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:20:56.582972+0000","last_peered":"2023-09-23T18:20:56.582972+0000","last_clean":"2023-09-23T18:20:56.582972+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:20:56.582972+0000","last_undegraded":"2023-09-23T18:20:56.582972+0000","last_fullsized":"2023-09-23T18:20:56.582972+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":994,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.643389+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:21:03.643389+0000","last_peered":"2023-09-23T18:21:03.643389+0000","last_clean":"2023-09-23T18:21:03.643389+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:21:03.643389+0000","last_undegraded":"2023-09-23T18:21:03.643389+0000","last_fullsized":"2023-09-23T18:21:03.643389+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":171,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586698+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:20:56.586698+0000","last_peered":"2023-09-23T18:20:56.586698+0000","last_clean":"2023-09-23T18:20:56.586698+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:20:56.586698+0000","last_undegraded":"2023-09-23T18:20:56.586698+0000","last_fullsized":"2023-09-23T18:20:56.586698+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":438,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586591+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:20:56.586591+0000","last_peered":"2023-09-23T18:20:56.586591+0000","last_clean":"2023-09-23T18:20:56.586591+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:20:56.586591+0000","last_undegraded":"2023-09-23T18:20:56.586591+0000","last_fullsized":"2023-09-23T18:20:56.586591+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":845,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583543+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:20:56.583543+0000","last_peered":"2023-09-23T18:20:56.583543+0000","last_clean":"2023-09-23T18:20:56.583543+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:20:56.583543+0000","last_undegraded":"2023-09-23T18:20:56.583543+0000","last_fullsized":"2023-09-23T18:20:56.583543+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":438,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586433+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:20:56.586433+0000","last_peered":"2023-09-23T18:20:56.586433+0000","last_clean":"2023-09-23T18:20:56.586433+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:20:56.586433+0000","last_undegraded":"2023-09-23T18:20:56.586433+0000","last_fullsized":"2023-09-23T18:20:56.586433+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":116,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581716+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:20:56.581716+0000","last_peered":"2023-09-23T18:20:56.581716+0000","last_clean":"2023-09-23T18:20:56.581716+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:20:56.581716+0000","last_undegraded":"2023-09-23T18:20:56.581716+0000","last_fullsized":"2023-09-23T18:20:56.581716+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":816,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581472+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:20:56.581472+0000","last_peered":"2023-09-23T18:20:56.581472+0000","last_clean":"2023-09-23T18:20:56.581472+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:20:56.581472+0000","last_undegraded":"2023-09-23T18:20:56.581472+0000","last_fullsized":"2023-09-23T18:20:56.581472+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":135,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.648153+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:21:03.648153+0000","last_peered":"2023-09-23T18:21:03.648153+0000","last_clean":"2023-09-23T18:21:03.648153+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:21:03.648153+0000","last_undegraded":"2023-09-23T18:21:03.648153+0000","last_fullsized":"2023-09-23T18:21:03.648153+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":129,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582258+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:20:56.582258+0000","last_peered":"2023-09-23T18:20:56.582258+0000","last_clean":"2023-09-23T18:20:56.582258+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:20:56.582258+0000","last_undegraded":"2023-09-23T18:20:56.582258+0000","last_fullsized":"2023-09-23T18:20:56.582258+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":755,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.643383+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:21:03.643383+0000","last_peered":"2023-09-23T18:21:03.643383+0000","last_clean":"2023-09-23T18:21:03.643383+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:21:03.643383+0000","last_undegraded":"2023-09-23T18:21:03.643383+0000","last_fullsized":"2023-09-23T18:21:03.643383+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":206,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.584319+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:20:56.584319+0000","last_peered":"2023-09-23T18:20:56.584319+0000","last_clean":"2023-09-23T18:20:56.584319+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:20:56.584319+0000","last_undegraded":"2023-09-23T18:20:56.584319+0000","last_fullsized":"2023-09-23T18:20:56.584319+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":218,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586174+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:20:56.586174+0000","last_peered":"2023-09-23T18:20:56.586174+0000","last_clean":"2023-09-23T18:20:56.586174+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:20:56.586174+0000","last_undegraded":"2023-09-23T18:20:56.586174+0000","last_fullsized":"2023-09-23T18:20:56.586174+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":850,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582422+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:20:56.582422+0000","last_peered":"2023-09-23T18:20:56.582422+0000","last_clean":"2023-09-23T18:20:56.582422+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:20:56.582422+0000","last_undegraded":"2023-09-23T18:20:56.582422+0000","last_fullsized":"2023-09-23T18:20:56.582422+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":759,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582669+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:20:56.582669+0000","last_peered":"2023-09-23T18:20:56.582669+0000","last_clean":"2023-09-23T18:20:56.582669+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:20:56.582669+0000","last_undegraded":"2023-09-23T18:20:56.582669+0000","last_fullsized":"2023-09-23T18:20:56.582669+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":36,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.643882+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:21:03.643882+0000","last_peered":"2023-09-23T18:21:03.643882+0000","last_clean":"2023-09-23T18:21:03.643882+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:21:03.643882+0000","last_undegraded":"2023-09-23T18:21:03.643882+0000","last_fullsized":"2023-09-23T18:21:03.643882+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"619'11786","reported_seq":11839,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.170414+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:21:11.170414+0000","last_peered":"2023-09-23T18:21:11.170414+0000","last_clean":"2023-09-23T18:21:11.170414+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:21:11.170414+0000","last_undegraded":"2023-09-23T18:21:11.170414+0000","last_fullsized":"2023-09-23T18:21:11.170414+0000","mapping_epoch":574,"log_start":"604'6300","ondisk_log_start":"604'6300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84410368,"num_objects":1288,"num_object_clones":0,"num_object_copies":3864,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1288,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11786,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":884,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.588317+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:20:56.588317+0000","last_peered":"2023-09-23T18:20:56.588317+0000","last_clean":"2023-09-23T18:20:56.588317+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:20:56.588317+0000","last_undegraded":"2023-09-23T18:20:56.588317+0000","last_fullsized":"2023-09-23T18:20:56.588317+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":256,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.645821+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:21:03.645821+0000","last_peered":"2023-09-23T18:21:03.645821+0000","last_clean":"2023-09-23T18:21:03.645821+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:21:03.645821+0000","last_undegraded":"2023-09-23T18:21:03.645821+0000","last_fullsized":"2023-09-23T18:21:03.645821+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":129,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581571+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:20:56.581571+0000","last_peered":"2023-09-23T18:20:56.581571+0000","last_clean":"2023-09-23T18:20:56.581571+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:20:56.581571+0000","last_undegraded":"2023-09-23T18:20:56.581571+0000","last_fullsized":"2023-09-23T18:20:56.581571+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"619'11479","reported_seq":11542,"reported_epoch":619,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:11.307884+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:21:11.307884+0000","last_peered":"2023-09-23T18:21:11.307884+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:21:11.307884+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"607'7000","ondisk_log_start":"607'7000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4479,"log_dups_size":0,"ondisk_log_size":4479,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79495168,"num_objects":1213,"num_object_clones":0,"num_object_copies":3639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11479,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"5,6","objects":-2935}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":739,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:21:01.401541+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:21:01.401541+0000","last_peered":"2023-09-23T18:21:01.401541+0000","last_clean":"2023-09-23T18:21:01.401541+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:21:01.401541+0000","last_undegraded":"2023-09-23T18:21:01.401541+0000","last_fullsized":"2023-09-23T18:21:01.401541+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":257,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.646362+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:21:03.646362+0000","last_peered":"2023-09-23T18:21:03.646362+0000","last_clean":"2023-09-23T18:21:03.646362+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:21:03.646362+0000","last_undegraded":"2023-09-23T18:21:03.646362+0000","last_fullsized":"2023-09-23T18:21:03.646362+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":236,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.645638+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:21:03.645638+0000","last_peered":"2023-09-23T18:21:03.645638+0000","last_clean":"2023-09-23T18:21:03.645638+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:21:03.645638+0000","last_undegraded":"2023-09-23T18:21:03.645638+0000","last_fullsized":"2023-09-23T18:21:03.645638+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"619'11552","reported_seq":18565,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:10.290339+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:21:10.290339+0000","last_peered":"2023-09-23T18:21:10.290339+0000","last_clean":"2023-09-23T18:21:10.290339+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:21:10.290339+0000","last_undegraded":"2023-09-23T18:21:10.290339+0000","last_fullsized":"2023-09-23T18:21:10.290339+0000","mapping_epoch":605,"log_start":"606'6700","ondisk_log_start":"606'6700","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4852,"log_dups_size":0,"ondisk_log_size":4852,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":84672512,"num_objects":1292,"num_object_clones":0,"num_object_copies":3876,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1292,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11552,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":634,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585708+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:20:56.585708+0000","last_peered":"2023-09-23T18:20:56.585708+0000","last_clean":"2023-09-23T18:20:56.585708+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:20:56.585708+0000","last_undegraded":"2023-09-23T18:20:56.585708+0000","last_fullsized":"2023-09-23T18:20:56.585708+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":544,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582088+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:20:56.582088+0000","last_peered":"2023-09-23T18:20:56.582088+0000","last_clean":"2023-09-23T18:20:56.582088+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:20:56.582088+0000","last_undegraded":"2023-09-23T18:20:56.582088+0000","last_fullsized":"2023-09-23T18:20:56.582088+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":195,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582850+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:20:56.582850+0000","last_peered":"2023-09-23T18:20:56.582850+0000","last_clean":"2023-09-23T18:20:56.582850+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:20:56.582850+0000","last_undegraded":"2023-09-23T18:20:56.582850+0000","last_fullsized":"2023-09-23T18:20:56.582850+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"619'11826","reported_seq":11880,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.321056+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:21:11.321056+0000","last_peered":"2023-09-23T18:21:11.321056+0000","last_clean":"2023-09-23T18:21:11.321056+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:21:11.321056+0000","last_undegraded":"2023-09-23T18:21:11.321056+0000","last_fullsized":"2023-09-23T18:21:11.321056+0000","mapping_epoch":574,"log_start":"606'6700","ondisk_log_start":"606'6700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5126,"log_dups_size":0,"ondisk_log_size":5126,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82968576,"num_objects":1266,"num_object_clones":0,"num_object_copies":3798,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1266,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11826,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":650,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586246+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:20:56.586246+0000","last_peered":"2023-09-23T18:20:56.586246+0000","last_clean":"2023-09-23T18:20:56.586246+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:20:56.586246+0000","last_undegraded":"2023-09-23T18:20:56.586246+0000","last_fullsized":"2023-09-23T18:20:56.586246+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":143,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582108+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:20:56.582108+0000","last_peered":"2023-09-23T18:20:56.582108+0000","last_clean":"2023-09-23T18:20:56.582108+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:20:56.582108+0000","last_undegraded":"2023-09-23T18:20:56.582108+0000","last_fullsized":"2023-09-23T18:20:56.582108+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":456,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.584177+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:20:56.584177+0000","last_peered":"2023-09-23T18:20:56.584177+0000","last_clean":"2023-09-23T18:20:56.584177+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:20:56.584177+0000","last_undegraded":"2023-09-23T18:20:56.584177+0000","last_fullsized":"2023-09-23T18:20:56.584177+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"620'11945","reported_seq":12001,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:13.913377+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:21:13.913377+0000","last_peered":"2023-09-23T18:21:13.913377+0000","last_clean":"2023-09-23T18:21:13.913377+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:21:13.913377+0000","last_undegraded":"2023-09-23T18:21:13.913377+0000","last_fullsized":"2023-09-23T18:21:13.913377+0000","mapping_epoch":574,"log_start":"607'7500","ondisk_log_start":"607'7500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4445,"log_dups_size":0,"ondisk_log_size":4445,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72024064,"num_objects":1099,"num_object_clones":0,"num_object_copies":3297,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1099,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11945,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1101,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583397+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:20:56.583397+0000","last_peered":"2023-09-23T18:20:56.583397+0000","last_clean":"2023-09-23T18:20:56.583397+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:20:56.583397+0000","last_undegraded":"2023-09-23T18:20:56.583397+0000","last_fullsized":"2023-09-23T18:20:56.583397+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":356,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581041+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:20:56.581041+0000","last_peered":"2023-09-23T18:20:56.581041+0000","last_clean":"2023-09-23T18:20:56.581041+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:20:56.581041+0000","last_undegraded":"2023-09-23T18:20:56.581041+0000","last_fullsized":"2023-09-23T18:20:56.581041+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":163,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586358+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:20:56.586358+0000","last_peered":"2023-09-23T18:20:56.586358+0000","last_clean":"2023-09-23T18:20:56.586358+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:20:56.586358+0000","last_undegraded":"2023-09-23T18:20:56.586358+0000","last_fullsized":"2023-09-23T18:20:56.586358+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"619'11780","reported_seq":15073,"reported_epoch":619,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:11.208159+0000","last_change":"2023-09-23T18:20:34.894639+0000","last_active":"2023-09-23T18:21:11.208159+0000","last_peered":"2023-09-23T18:21:11.208159+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:21:11.208159+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"604'6300","ondisk_log_start":"604'6300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5480,"log_dups_size":0,"ondisk_log_size":5480,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81920000,"num_objects":1250,"num_object_clones":0,"num_object_copies":3750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":2611,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11780,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1615,"num_bytes_recovered":46006272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"4,5","objects":-1361}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":888,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582041+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:20:56.582041+0000","last_peered":"2023-09-23T18:20:56.582041+0000","last_clean":"2023-09-23T18:20:56.582041+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:20:56.582041+0000","last_undegraded":"2023-09-23T18:20:56.582041+0000","last_fullsized":"2023-09-23T18:20:56.582041+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":171,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582779+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:20:56.582779+0000","last_peered":"2023-09-23T18:20:56.582779+0000","last_clean":"2023-09-23T18:20:56.582779+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:20:56.582779+0000","last_undegraded":"2023-09-23T18:20:56.582779+0000","last_fullsized":"2023-09-23T18:20:56.582779+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":470,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.588407+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:20:56.588407+0000","last_peered":"2023-09-23T18:20:56.588407+0000","last_clean":"2023-09-23T18:20:56.588407+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:20:56.588407+0000","last_undegraded":"2023-09-23T18:20:56.588407+0000","last_fullsized":"2023-09-23T18:20:56.588407+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"619'11741","reported_seq":11794,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.173147+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:21:11.173147+0000","last_peered":"2023-09-23T18:21:11.173147+0000","last_clean":"2023-09-23T18:21:11.173147+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:21:11.173147+0000","last_undegraded":"2023-09-23T18:21:11.173147+0000","last_fullsized":"2023-09-23T18:21:11.173147+0000","mapping_epoch":574,"log_start":"607'7300","ondisk_log_start":"607'7300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4441,"log_dups_size":0,"ondisk_log_size":4441,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85917696,"num_objects":1311,"num_object_clones":0,"num_object_copies":3933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11741,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":890,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583889+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:20:56.583889+0000","last_peered":"2023-09-23T18:20:56.583889+0000","last_clean":"2023-09-23T18:20:56.583889+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:20:56.583889+0000","last_undegraded":"2023-09-23T18:20:56.583889+0000","last_fullsized":"2023-09-23T18:20:56.583889+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":455,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583255+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:20:56.583255+0000","last_peered":"2023-09-23T18:20:56.583255+0000","last_clean":"2023-09-23T18:20:56.583255+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:20:56.583255+0000","last_undegraded":"2023-09-23T18:20:56.583255+0000","last_fullsized":"2023-09-23T18:20:56.583255+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":231,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582506+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:20:56.582506+0000","last_peered":"2023-09-23T18:20:56.582506+0000","last_clean":"2023-09-23T18:20:56.582506+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:20:56.582506+0000","last_undegraded":"2023-09-23T18:20:56.582506+0000","last_fullsized":"2023-09-23T18:20:56.582506+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"620'11813","reported_seq":11867,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:12.373075+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:21:12.373075+0000","last_peered":"2023-09-23T18:21:12.373075+0000","last_clean":"2023-09-23T18:21:12.373075+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:21:12.373075+0000","last_undegraded":"2023-09-23T18:21:12.373075+0000","last_fullsized":"2023-09-23T18:21:12.373075+0000","mapping_epoch":574,"log_start":"607'7100","ondisk_log_start":"607'7100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4713,"log_dups_size":0,"ondisk_log_size":4713,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80019456,"num_objects":1221,"num_object_clones":0,"num_object_copies":3663,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1221,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11813,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":871,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583302+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:20:56.583302+0000","last_peered":"2023-09-23T18:20:56.583302+0000","last_clean":"2023-09-23T18:20:56.583302+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:20:56.583302+0000","last_undegraded":"2023-09-23T18:20:56.583302+0000","last_fullsized":"2023-09-23T18:20:56.583302+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":36,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577543+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:20:56.577543+0000","last_peered":"2023-09-23T18:20:56.577543+0000","last_clean":"2023-09-23T18:20:56.577543+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:20:56.577543+0000","last_undegraded":"2023-09-23T18:20:56.577543+0000","last_fullsized":"2023-09-23T18:20:56.577543+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":348,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.643884+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:21:03.643884+0000","last_peered":"2023-09-23T18:21:03.643884+0000","last_clean":"2023-09-23T18:21:03.643884+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:21:03.643884+0000","last_undegraded":"2023-09-23T18:21:03.643884+0000","last_fullsized":"2023-09-23T18:21:03.643884+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"619'11798","reported_seq":11855,"reported_epoch":619,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:10.268502+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:21:10.268502+0000","last_peered":"2023-09-23T18:21:10.268502+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:21:10.268502+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"606'6900","ondisk_log_start":"606'6900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4898,"log_dups_size":0,"ondisk_log_size":4898,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83230720,"num_objects":1270,"num_object_clones":0,"num_object_copies":3810,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1270,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11798,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-3015}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":730,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585862+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:20:56.585862+0000","last_peered":"2023-09-23T18:20:56.585862+0000","last_clean":"2023-09-23T18:20:56.585862+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:20:56.585862+0000","last_undegraded":"2023-09-23T18:20:56.585862+0000","last_fullsized":"2023-09-23T18:20:56.585862+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":970,"reported_epoch":619,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:03.642325+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:21:03.642325+0000","last_peered":"2023-09-23T18:21:03.642325+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:21:03.642325+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":153,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582332+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:20:56.582332+0000","last_peered":"2023-09-23T18:20:56.582332+0000","last_clean":"2023-09-23T18:20:56.582332+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:20:56.582332+0000","last_undegraded":"2023-09-23T18:20:56.582332+0000","last_fullsized":"2023-09-23T18:20:56.582332+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":133,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583238+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:20:56.583238+0000","last_peered":"2023-09-23T18:20:56.583238+0000","last_clean":"2023-09-23T18:20:56.583238+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:20:56.583238+0000","last_undegraded":"2023-09-23T18:20:56.583238+0000","last_fullsized":"2023-09-23T18:20:56.583238+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"619'11735","reported_seq":11797,"reported_epoch":619,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:11.254310+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:21:11.254310+0000","last_peered":"2023-09-23T18:21:11.254310+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:21:11.254310+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"607'7300","ondisk_log_start":"607'7300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4435,"log_dups_size":0,"ondisk_log_size":4435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78708736,"num_objects":1201,"num_object_clones":0,"num_object_copies":3603,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1201,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11735,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"5,6","objects":-2358}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":802,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583703+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:20:56.583703+0000","last_peered":"2023-09-23T18:20:56.583703+0000","last_clean":"2023-09-23T18:20:56.583703+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:20:56.583703+0000","last_undegraded":"2023-09-23T18:20:56.583703+0000","last_fullsized":"2023-09-23T18:20:56.583703+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1503,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583178+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:20:56.583178+0000","last_peered":"2023-09-23T18:20:56.583178+0000","last_clean":"2023-09-23T18:20:56.583178+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:20:56.583178+0000","last_undegraded":"2023-09-23T18:20:56.583178+0000","last_fullsized":"2023-09-23T18:20:56.583178+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":143,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586513+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:20:56.586513+0000","last_peered":"2023-09-23T18:20:56.586513+0000","last_clean":"2023-09-23T18:20:56.586513+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:20:56.586513+0000","last_undegraded":"2023-09-23T18:20:56.586513+0000","last_fullsized":"2023-09-23T18:20:56.586513+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":443,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586568+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:20:56.586568+0000","last_peered":"2023-09-23T18:20:56.586568+0000","last_clean":"2023-09-23T18:20:56.586568+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:20:56.586568+0000","last_undegraded":"2023-09-23T18:20:56.586568+0000","last_fullsized":"2023-09-23T18:20:56.586568+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"619'11390","reported_seq":11443,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.120792+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:21:11.120792+0000","last_peered":"2023-09-23T18:21:11.120792+0000","last_clean":"2023-09-23T18:21:11.120792+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:21:11.120792+0000","last_undegraded":"2023-09-23T18:21:11.120792+0000","last_fullsized":"2023-09-23T18:21:11.120792+0000","mapping_epoch":574,"log_start":"604'5900","ondisk_log_start":"604'5900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5490,"log_dups_size":0,"ondisk_log_size":5490,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78381056,"num_objects":1196,"num_object_clones":0,"num_object_copies":3588,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1196,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11390,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":827,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583373+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:20:56.583373+0000","last_peered":"2023-09-23T18:20:56.583373+0000","last_clean":"2023-09-23T18:20:56.583373+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:20:56.583373+0000","last_undegraded":"2023-09-23T18:20:56.583373+0000","last_fullsized":"2023-09-23T18:20:56.583373+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":62,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.575602+0000","last_change":"2023-09-23T18:20:35.016554+0000","last_active":"2023-09-23T18:20:56.575602+0000","last_peered":"2023-09-23T18:20:56.575602+0000","last_clean":"2023-09-23T18:20:56.575602+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:20:56.575602+0000","last_undegraded":"2023-09-23T18:20:56.575602+0000","last_fullsized":"2023-09-23T18:20:56.575602+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":412,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582939+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:20:56.582939+0000","last_peered":"2023-09-23T18:20:56.582939+0000","last_clean":"2023-09-23T18:20:56.582939+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:20:56.582939+0000","last_undegraded":"2023-09-23T18:20:56.582939+0000","last_fullsized":"2023-09-23T18:20:56.582939+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":155,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585992+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:20:56.585992+0000","last_peered":"2023-09-23T18:20:56.585992+0000","last_clean":"2023-09-23T18:20:56.585992+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:20:56.585992+0000","last_undegraded":"2023-09-23T18:20:56.585992+0000","last_fullsized":"2023-09-23T18:20:56.585992+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"619'11735","reported_seq":11788,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.272563+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:21:11.272563+0000","last_peered":"2023-09-23T18:21:11.272563+0000","last_clean":"2023-09-23T18:21:11.272563+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:21:11.272563+0000","last_undegraded":"2023-09-23T18:21:11.272563+0000","last_fullsized":"2023-09-23T18:21:11.272563+0000","mapping_epoch":574,"log_start":"607'7300","ondisk_log_start":"607'7300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4435,"log_dups_size":0,"ondisk_log_size":4435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":79495168,"num_objects":1213,"num_object_clones":0,"num_object_copies":3639,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1213,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11735,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1516,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582292+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:20:56.582292+0000","last_peered":"2023-09-23T18:20:56.582292+0000","last_clean":"2023-09-23T18:20:56.582292+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:20:56.582292+0000","last_undegraded":"2023-09-23T18:20:56.582292+0000","last_fullsized":"2023-09-23T18:20:56.582292+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":569,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582815+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:20:56.582815+0000","last_peered":"2023-09-23T18:20:56.582815+0000","last_clean":"2023-09-23T18:20:56.582815+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:20:56.582815+0000","last_undegraded":"2023-09-23T18:20:56.582815+0000","last_fullsized":"2023-09-23T18:20:56.582815+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":129,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583061+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:20:56.583061+0000","last_peered":"2023-09-23T18:20:56.583061+0000","last_clean":"2023-09-23T18:20:56.583061+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:20:56.583061+0000","last_undegraded":"2023-09-23T18:20:56.583061+0000","last_fullsized":"2023-09-23T18:20:56.583061+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":164,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.588646+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:20:56.588646+0000","last_peered":"2023-09-23T18:20:56.588646+0000","last_clean":"2023-09-23T18:20:56.588646+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:20:56.588646+0000","last_undegraded":"2023-09-23T18:20:56.588646+0000","last_fullsized":"2023-09-23T18:20:56.588646+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"619'11534","reported_seq":11587,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.173729+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:21:11.173729+0000","last_peered":"2023-09-23T18:21:11.173729+0000","last_clean":"2023-09-23T18:21:11.173729+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:21:11.173729+0000","last_undegraded":"2023-09-23T18:21:11.173729+0000","last_fullsized":"2023-09-23T18:21:11.173729+0000","mapping_epoch":574,"log_start":"604'6000","ondisk_log_start":"604'6000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5534,"log_dups_size":0,"ondisk_log_size":5534,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":81920000,"num_objects":1250,"num_object_clones":0,"num_object_copies":3750,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1250,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11534,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":817,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586664+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:20:56.586664+0000","last_peered":"2023-09-23T18:20:56.586664+0000","last_clean":"2023-09-23T18:20:56.586664+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:20:56.586664+0000","last_undegraded":"2023-09-23T18:20:56.586664+0000","last_fullsized":"2023-09-23T18:20:56.586664+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":133,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577664+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:20:56.577664+0000","last_peered":"2023-09-23T18:20:56.577664+0000","last_clean":"2023-09-23T18:20:56.577664+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:20:56.577664+0000","last_undegraded":"2023-09-23T18:20:56.577664+0000","last_fullsized":"2023-09-23T18:20:56.577664+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":36,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.575523+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:20:56.575523+0000","last_peered":"2023-09-23T18:20:56.575523+0000","last_clean":"2023-09-23T18:20:56.575523+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:20:56.575523+0000","last_undegraded":"2023-09-23T18:20:56.575523+0000","last_fullsized":"2023-09-23T18:20:56.575523+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":634,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582025+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:20:56.582025+0000","last_peered":"2023-09-23T18:20:56.582025+0000","last_clean":"2023-09-23T18:20:56.582025+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:20:56.582025+0000","last_undegraded":"2023-09-23T18:20:56.582025+0000","last_fullsized":"2023-09-23T18:20:56.582025+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"619'11652","reported_seq":18634,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.269631+0000","last_change":"2023-09-23T18:20:36.673759+0000","last_active":"2023-09-23T18:21:11.269631+0000","last_peered":"2023-09-23T18:21:11.269631+0000","last_clean":"2023-09-23T18:21:11.269631+0000","last_became_active":"2023-09-23T18:20:36.651913+0000","last_became_peered":"2023-09-23T18:20:36.651913+0000","last_unstale":"2023-09-23T18:21:11.269631+0000","last_undegraded":"2023-09-23T18:21:11.269631+0000","last_fullsized":"2023-09-23T18:21:11.269631+0000","mapping_epoch":614,"log_start":"607'7200","ondisk_log_start":"607'7200","created":574,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4452,"log_dups_size":0,"ondisk_log_size":4452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:47.765348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":85524504,"num_objects":1306,"num_object_clones":0,"num_object_copies":3918,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1306,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":11652,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3453,"num_bytes_recovered":209321984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":917,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.589325+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:20:56.589325+0000","last_peered":"2023-09-23T18:20:56.589325+0000","last_clean":"2023-09-23T18:20:56.589325+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:20:56.589325+0000","last_undegraded":"2023-09-23T18:20:56.589325+0000","last_fullsized":"2023-09-23T18:20:56.589325+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":58,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586563+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:20:56.586563+0000","last_peered":"2023-09-23T18:20:56.586563+0000","last_clean":"2023-09-23T18:20:56.586563+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:20:56.586563+0000","last_undegraded":"2023-09-23T18:20:56.586563+0000","last_fullsized":"2023-09-23T18:20:56.586563+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":143,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581871+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:20:56.581871+0000","last_peered":"2023-09-23T18:20:56.581871+0000","last_clean":"2023-09-23T18:20:56.581871+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:20:56.581871+0000","last_undegraded":"2023-09-23T18:20:56.581871+0000","last_fullsized":"2023-09-23T18:20:56.581871+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":143,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582209+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:20:56.582209+0000","last_peered":"2023-09-23T18:20:56.582209+0000","last_clean":"2023-09-23T18:20:56.582209+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:20:56.582209+0000","last_undegraded":"2023-09-23T18:20:56.582209+0000","last_fullsized":"2023-09-23T18:20:56.582209+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"619'11582","reported_seq":11635,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.278845+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:21:11.278845+0000","last_peered":"2023-09-23T18:21:11.278845+0000","last_clean":"2023-09-23T18:21:11.278845+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:21:11.278845+0000","last_undegraded":"2023-09-23T18:21:11.278845+0000","last_fullsized":"2023-09-23T18:21:11.278845+0000","mapping_epoch":574,"log_start":"607'7100","ondisk_log_start":"607'7100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4482,"log_dups_size":0,"ondisk_log_size":4482,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":83361792,"num_objects":1272,"num_object_clones":0,"num_object_copies":3816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11582,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":917,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.588452+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:20:56.588452+0000","last_peered":"2023-09-23T18:20:56.588452+0000","last_clean":"2023-09-23T18:20:56.588452+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:20:56.588452+0000","last_undegraded":"2023-09-23T18:20:56.588452+0000","last_fullsized":"2023-09-23T18:20:56.588452+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":105,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585775+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:20:56.585775+0000","last_peered":"2023-09-23T18:20:56.585775+0000","last_clean":"2023-09-23T18:20:56.585775+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:20:56.585775+0000","last_undegraded":"2023-09-23T18:20:56.585775+0000","last_fullsized":"2023-09-23T18:20:56.585775+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":398,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.589138+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:20:56.589138+0000","last_peered":"2023-09-23T18:20:56.589138+0000","last_clean":"2023-09-23T18:20:56.589138+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:20:56.589138+0000","last_undegraded":"2023-09-23T18:20:56.589138+0000","last_fullsized":"2023-09-23T18:20:56.589138+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":36,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.577470+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:20:56.577470+0000","last_peered":"2023-09-23T18:20:56.577470+0000","last_clean":"2023-09-23T18:20:56.577470+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:20:56.577470+0000","last_undegraded":"2023-09-23T18:20:56.577470+0000","last_fullsized":"2023-09-23T18:20:56.577470+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"619'11587","reported_seq":11640,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.298069+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:21:11.298069+0000","last_peered":"2023-09-23T18:21:11.298069+0000","last_clean":"2023-09-23T18:21:11.298069+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:21:11.298069+0000","last_undegraded":"2023-09-23T18:21:11.298069+0000","last_fullsized":"2023-09-23T18:21:11.298069+0000","mapping_epoch":574,"log_start":"607'7100","ondisk_log_start":"607'7100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4487,"log_dups_size":0,"ondisk_log_size":4487,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":80281600,"num_objects":1225,"num_object_clones":0,"num_object_copies":3675,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1225,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":11587,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":752,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581684+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:20:56.581684+0000","last_peered":"2023-09-23T18:20:56.581684+0000","last_clean":"2023-09-23T18:20:56.581684+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:20:56.581684+0000","last_undegraded":"2023-09-23T18:20:56.581684+0000","last_fullsized":"2023-09-23T18:20:56.581684+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1055,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586115+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:20:56.586115+0000","last_peered":"2023-09-23T18:20:56.586115+0000","last_clean":"2023-09-23T18:20:56.586115+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:20:56.586115+0000","last_undegraded":"2023-09-23T18:20:56.586115+0000","last_fullsized":"2023-09-23T18:20:56.586115+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":46,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.580303+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:20:56.580303+0000","last_peered":"2023-09-23T18:20:56.580303+0000","last_clean":"2023-09-23T18:20:56.580303+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:20:56.580303+0000","last_undegraded":"2023-09-23T18:20:56.580303+0000","last_fullsized":"2023-09-23T18:20:56.580303+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":409,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583468+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:20:56.583468+0000","last_peered":"2023-09-23T18:20:56.583468+0000","last_clean":"2023-09-23T18:20:56.583468+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:20:56.583468+0000","last_undegraded":"2023-09-23T18:20:56.583468+0000","last_fullsized":"2023-09-23T18:20:56.583468+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":649,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.584553+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:20:56.584553+0000","last_peered":"2023-09-23T18:20:56.584553+0000","last_clean":"2023-09-23T18:20:56.584553+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:20:56.584553+0000","last_undegraded":"2023-09-23T18:20:56.584553+0000","last_fullsized":"2023-09-23T18:20:56.584553+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":42,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.647830+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:21:03.647830+0000","last_peered":"2023-09-23T18:21:03.647830+0000","last_clean":"2023-09-23T18:21:03.647830+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:21:03.647830+0000","last_undegraded":"2023-09-23T18:21:03.647830+0000","last_fullsized":"2023-09-23T18:21:03.647830+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":693,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.642131+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:21:03.642131+0000","last_peered":"2023-09-23T18:21:03.642131+0000","last_clean":"2023-09-23T18:21:03.642131+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:21:03.642131+0000","last_undegraded":"2023-09-23T18:21:03.642131+0000","last_fullsized":"2023-09-23T18:21:03.642131+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":649,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582590+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:20:56.582590+0000","last_peered":"2023-09-23T18:20:56.582590+0000","last_clean":"2023-09-23T18:20:56.582590+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:20:56.582590+0000","last_undegraded":"2023-09-23T18:20:56.582590+0000","last_fullsized":"2023-09-23T18:20:56.582590+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":137,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.581957+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:20:56.581957+0000","last_peered":"2023-09-23T18:20:56.581957+0000","last_clean":"2023-09-23T18:20:56.581957+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:20:56.581957+0000","last_undegraded":"2023-09-23T18:20:56.581957+0000","last_fullsized":"2023-09-23T18:20:56.581957+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":254,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.645729+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:21:03.645729+0000","last_peered":"2023-09-23T18:21:03.645729+0000","last_clean":"2023-09-23T18:21:03.645729+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:21:03.645729+0000","last_undegraded":"2023-09-23T18:21:03.645729+0000","last_fullsized":"2023-09-23T18:21:03.645729+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":123,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582182+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:20:56.582182+0000","last_peered":"2023-09-23T18:20:56.582182+0000","last_clean":"2023-09-23T18:20:56.582182+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:20:56.582182+0000","last_undegraded":"2023-09-23T18:20:56.582182+0000","last_fullsized":"2023-09-23T18:20:56.582182+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":237,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.645935+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:21:03.645935+0000","last_peered":"2023-09-23T18:21:03.645935+0000","last_clean":"2023-09-23T18:21:03.645935+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:21:03.645935+0000","last_undegraded":"2023-09-23T18:21:03.645935+0000","last_fullsized":"2023-09-23T18:21:03.645935+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1011,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.586776+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:20:56.586776+0000","last_peered":"2023-09-23T18:20:56.586776+0000","last_clean":"2023-09-23T18:20:56.586776+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:20:56.586776+0000","last_undegraded":"2023-09-23T18:20:56.586776+0000","last_fullsized":"2023-09-23T18:20:56.586776+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":516,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.589238+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:20:56.589238+0000","last_peered":"2023-09-23T18:20:56.589238+0000","last_clean":"2023-09-23T18:20:56.589238+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:20:56.589238+0000","last_undegraded":"2023-09-23T18:20:56.589238+0000","last_fullsized":"2023-09-23T18:20:56.589238+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":153,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.583621+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:20:56.583621+0000","last_peered":"2023-09-23T18:20:56.583621+0000","last_clean":"2023-09-23T18:20:56.583621+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:20:56.583621+0000","last_undegraded":"2023-09-23T18:20:56.583621+0000","last_fullsized":"2023-09-23T18:20:56.583621+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":949,"reported_epoch":619,"state":"active+clean","last_fresh":"2023-09-23T18:21:03.643037+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:21:03.643037+0000","last_peered":"2023-09-23T18:21:03.643037+0000","last_clean":"2023-09-23T18:21:03.643037+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:21:03.643037+0000","last_undegraded":"2023-09-23T18:21:03.643037+0000","last_fullsized":"2023-09-23T18:21:03.643037+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":141,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.585916+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:20:56.585916+0000","last_peered":"2023-09-23T18:20:56.585916+0000","last_clean":"2023-09-23T18:20:56.585916+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:20:56.585916+0000","last_undegraded":"2023-09-23T18:20:56.585916+0000","last_fullsized":"2023-09-23T18:20:56.585916+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":662,"reported_epoch":618,"state":"active+clean","last_fresh":"2023-09-23T18:20:56.582738+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:20:56.582738+0000","last_peered":"2023-09-23T18:20:56.582738+0000","last_clean":"2023-09-23T18:20:56.582738+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:20:56.582738+0000","last_undegraded":"2023-09-23T18:20:56.582738+0000","last_fullsized":"2023-09-23T18:20:56.582738+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":1302331416,"num_objects":19873,"num_object_clones":0,"num_object_copies":59619,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":14603,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":19873,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":186935,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8539,"num_bytes_recovered":482803712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":5576273920,"data_stored":5576261704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77235,"ondisk_log_size":77235,"up":48,"acting":44,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20983808,"data_stored":20971577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001583,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669519,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1484736,"kb_used_data":858872,"kb_used_omap":7,"kb_used_meta":625848,"kb_avail":92260416,"statfs":{"total":95995035648,"available":94474665984,"internally_reserved":0,"allocated":879484928,"data_stored":877522521,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":640869156},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,5,2,3],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":2.6139999999999999,"5min":1.7889999999999999,"15min":1.407},"min":{"1min":0.249,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":9.1600000000000001,"5min":24.472999999999999,"15min":24.472999999999999},"last":0.255},{"interface":"front","average":{"1min":2.54,"5min":2.1960000000000002,"15min":1.726},"min":{"1min":0.26000000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":8.2029999999999994,"5min":22.832999999999998,"15min":22.832999999999998},"last":2.7749999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:20:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3570000000000002,"5min":2.0350000000000001,"15min":1.4890000000000001},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.726000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":2.8149999999999999},{"interface":"front","average":{"1min":2.423,"5min":1.899,"15min":1.4379999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.336,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.498}]},{"osd":2,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.579,"15min":1.345},"min":{"1min":0.33500000000000002,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":5.7919999999999998,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":1.7030000000000001},{"interface":"front","average":{"1min":2.3029999999999999,"5min":2.0600000000000001,"15min":1.5580000000000001},"min":{"1min":0.39700000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0730000000000004,"5min":14.771000000000001,"15min":14.771000000000001},"last":8.2690000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":3.0419999999999998,"15min":2.1150000000000002},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.24099999999999999},"max":{"1min":5.8170000000000002,"5min":103.348,"15min":103.348},"last":5.3449999999999998},{"interface":"front","average":{"1min":2.5600000000000001,"5min":3.6190000000000002,"15min":2.4209999999999998},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":7.4530000000000003,"5min":103.40300000000001,"15min":103.40300000000001},"last":7.0970000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:20:13 2023","interfaces":[{"interface":"back","average":{"1min":1.4430000000000001,"5min":1.8180000000000001,"15min":1.345},"min":{"1min":0.29299999999999998,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.0730000000000004,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.47099999999999997},{"interface":"front","average":{"1min":1.776,"5min":1.8200000000000001,"15min":1.3700000000000001},"min":{"1min":0.45200000000000001,"5min":0.30399999999999999,"15min":0.27000000000000002},"max":{"1min":7.8319999999999999,"5min":9.2260000000000009,"15min":9.2260000000000009},"last":0.60599999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":2.1099999999999999,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":18.417000000000002,"5min":18.417000000000002,"15min":18.417000000000002},"last":2.992},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.3399999999999999,"15min":2.3780000000000001},"min":{"1min":0.36299999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":18.454999999999998,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.52400000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":2.1059999999999999,"15min":1.5580000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.26800000000000002},"max":{"1min":6.1500000000000004,"5min":10.743,"15min":13.460000000000001},"last":3.0219999999999998},{"interface":"front","average":{"1min":2.0830000000000002,"5min":2.0529999999999999,"15min":1.603},"min":{"1min":0.40300000000000002,"5min":0.38500000000000001,"15min":0.28499999999999998},"max":{"1min":8.6890000000000001,"5min":11.244,"15min":12.404},"last":5.0990000000000002}]}]},{"osd":1,"up_from":545,"seq":2340757176368,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1192632,"kb_used_data":940592,"kb_used_omap":13,"kb_used_meta":252018,"kb_avail":92552520,"statfs":{"total":95995035648,"available":94773780480,"internally_reserved":0,"allocated":963166208,"data_stored":960115084,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":258067085},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.016,"15min":0.59899999999999998},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":9.0299999999999994,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":1.4750000000000001},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.004,"15min":0.66100000000000003},"min":{"1min":0.34499999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.952999999999999,"5min":14.952999999999999,"15min":14.952999999999999},"last":3.3519999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.0980000000000001,"15min":0.67300000000000004},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.6219999999999999,"5min":9.6219999999999999,"15min":9.6219999999999999},"last":4.5800000000000001},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.77200000000000002,"15min":0.59399999999999997},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2080000000000002,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":3.157}]},{"osd":3,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.0409999999999999,"15min":0.60499999999999998},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.3200000000000003,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":2.129},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.2769999999999999,"15min":0.76700000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":19.762,"5min":19.762,"15min":19.762},"last":0.624}]},{"osd":4,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.3280000000000001,"15min":1.3200000000000001},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.8690000000000002,"5min":12.663,"15min":12.663},"last":3.0950000000000002},{"interface":"front","average":{"1min":2.4049999999999998,"5min":1.7789999999999999,"15min":1.6020000000000001},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.317},"max":{"1min":6.0289999999999999,"5min":13.222,"15min":13.222},"last":1.845}]},{"osd":5,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.405,"15min":0.94999999999999996},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":23.190999999999999,"5min":23.190999999999999,"15min":23.190999999999999},"last":3.1320000000000001},{"interface":"front","average":{"1min":4.1719999999999997,"5min":1.7330000000000001,"15min":1.113},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":23.204000000000001,"5min":23.204000000000001,"15min":23.204000000000001},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.121,"5min":1.4690000000000001,"15min":0.996},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":23.443000000000001,"5min":23.443000000000001,"15min":23.443000000000001},"last":5.0780000000000003},{"interface":"front","average":{"1min":3.7970000000000002,"5min":1.746,"15min":1.099},"min":{"1min":0.42199999999999999,"5min":0.378,"15min":0.378},"max":{"1min":26.177,"5min":26.177,"15min":26.177},"last":1.9239999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.4340000000000002,"5min":1.3620000000000001,"15min":0.90800000000000003},"min":{"1min":0.42299999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.1050000000000004,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":1.883},{"interface":"front","average":{"1min":2.3140000000000001,"5min":1.474,"15min":1.028},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":9.1500000000000004,"5min":9.1500000000000004,"15min":9.1500000000000004},"last":0.65100000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958864,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":2088644,"kb_used_data":1147388,"kb_used_omap":7,"kb_used_meta":941240,"kb_avail":91656508,"statfs":{"total":95995035648,"available":93856264192,"internally_reserved":0,"allocated":1174925312,"data_stored":1172034043,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":963830500},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,1,2,2],"upper_bound":16},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.044,"15min":0.56100000000000005},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.4779999999999998,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":5.5090000000000003},{"interface":"front","average":{"1min":2.6240000000000001,"5min":1.278,"15min":0.77400000000000002},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.335000000000001,"5min":15.335000000000001,"15min":15.335000000000001},"last":6.1609999999999996}]},{"osd":2,"last update":"Sat Sep 23 18:21:04 2023","interfaces":[{"interface":"back","average":{"1min":3.008,"5min":2.0880000000000001,"15min":1.4419999999999999},"min":{"1min":0.40200000000000002,"5min":0.185,"15min":0.185},"max":{"1min":10.013,"5min":18.550999999999998,"15min":18.550999999999998},"last":4.875},{"interface":"front","average":{"1min":3.2160000000000002,"5min":1.7470000000000001,"15min":1.097},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":23.161999999999999,"5min":23.161999999999999,"15min":23.161999999999999},"last":6.2220000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.4199999999999999,"15min":1.5940000000000001},"min":{"1min":0.313,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.8460000000000001,"5min":13.539999999999999,"15min":13.539999999999999},"last":5.5359999999999996},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.6619999999999999,"15min":1.7010000000000001},"min":{"1min":0.36299999999999999,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":6.8879999999999999,"5min":13.285,"15min":15.198},"last":5.4850000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":3.7010000000000001,"5min":2.1309999999999998,"15min":1.8979999999999999},"min":{"1min":0.41799999999999998,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":17.741,"5min":17.741,"15min":57.746000000000002},"last":5.3550000000000004},{"interface":"front","average":{"1min":3.79,"5min":1.9159999999999999,"15min":1.7809999999999999},"min":{"1min":0.50900000000000001,"5min":0.26900000000000002,"15min":0.25600000000000001},"max":{"1min":15.079000000000001,"5min":15.079000000000001,"15min":57.816000000000003},"last":4.9089999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":1.8220000000000001,"15min":1.4410000000000001},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":5.5970000000000004,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":5.3979999999999997},{"interface":"front","average":{"1min":3.2909999999999999,"5min":2.4630000000000001,"15min":1.599},"min":{"1min":0.40100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":15.047000000000001,"5min":15.047000000000001,"15min":15.047000000000001},"last":4.9329999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:20:12 2023","interfaces":[{"interface":"back","average":{"1min":2.6269999999999998,"5min":1.948,"15min":1.9530000000000001},"min":{"1min":0.47499999999999998,"5min":0.377,"15min":0.32000000000000001},"max":{"1min":7.6900000000000004,"5min":7.6900000000000004,"15min":58.457999999999998},"last":5.4610000000000003},{"interface":"front","average":{"1min":4.6639999999999997,"5min":2.3159999999999998,"15min":1.9299999999999999},"min":{"1min":0.497,"5min":0.36299999999999999,"15min":0.315},"max":{"1min":18.643999999999998,"5min":18.643999999999998,"15min":58.354999999999997},"last":6.1920000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":3.431,"5min":2.2480000000000002,"15min":1.7889999999999999},"min":{"1min":0.49199999999999999,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":22.945,"5min":22.945,"15min":22.945},"last":6.1200000000000001},{"interface":"front","average":{"1min":2.2330000000000001,"5min":1.714,"15min":1.851},"min":{"1min":0.47099999999999997,"5min":0.371,"15min":0.36199999999999999},"max":{"1min":6.9249999999999998,"5min":7.7519999999999998,"15min":10.404},"last":5.4320000000000004}]}]},{"osd":2,"up_from":0,"seq":2070174236750,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249379,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1343456,"kb_used_data":569816,"kb_used_omap":3,"kb_used_meta":773500,"kb_avail":92401696,"statfs":{"total":95995035648,"available":94619336704,"internally_reserved":0,"allocated":583491584,"data_stored":580398153,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":792064744},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,0,1],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":1.5800000000000001,"15min":1.8089999999999999},"min":{"1min":0.20799999999999999,"5min":0.192,"15min":0.183},"max":{"1min":12.472,"5min":17.036000000000001,"15min":195.69800000000001},"last":4.702},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.405,"15min":2.004},"min":{"1min":0.23799999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.468,"5min":11.468,"15min":194.661},"last":3.5289999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:20:26 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":1.4590000000000001,"15min":0.83299999999999996},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.079000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":4.8700000000000001},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.5129999999999999,"15min":1.3069999999999999},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.081,"5min":78.980000000000004,"15min":78.980000000000004},"last":3.1019999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:21:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5569999999999999,"5min":1.526,"15min":1.3740000000000001},"min":{"1min":0.29899999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.2430000000000003,"5min":12.151999999999999,"15min":12.151999999999999},"last":7.1699999999999999},{"interface":"front","average":{"1min":1.78,"5min":1.3839999999999999,"15min":1.361},"min":{"1min":0.32300000000000001,"5min":0.27900000000000003,"15min":0.19600000000000001},"max":{"1min":7.0640000000000001,"5min":11.565,"15min":11.565},"last":4.3799999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":1.8839999999999999,"15min":1.706},"min":{"1min":0.34200000000000003,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":11.4,"5min":21.062999999999999,"15min":37.231000000000002},"last":3.1850000000000001},{"interface":"front","average":{"1min":2.532,"5min":2.1240000000000001,"15min":1.885},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":9.1630000000000003,"5min":21.128,"15min":37.381},"last":4.8410000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:20:37 2023","interfaces":[{"interface":"back","average":{"1min":3.452,"5min":2.048,"15min":1.587},"min":{"1min":0.35099999999999998,"5min":0.316,"15min":0.316},"max":{"1min":9.7430000000000003,"5min":11.805999999999999,"15min":11.805999999999999},"last":4.7699999999999996},{"interface":"front","average":{"1min":3.0339999999999998,"5min":1.74,"15min":1.722},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":17.135999999999999,"5min":17.135999999999999,"15min":17.135999999999999},"last":3.036}]},{"osd":6,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.9390000000000001,"15min":1.9379999999999999},"min":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":10.303000000000001,"5min":17.192,"15min":115.54000000000001},"last":2.9969999999999999},{"interface":"front","average":{"1min":2.9889999999999999,"5min":1.976,"15min":1.9810000000000001},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.30499999999999999},"max":{"1min":11.442,"5min":14.321,"15min":115.691},"last":3.145}]},{"osd":7,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0939999999999999,"5min":1.744,"15min":1.7609999999999999},"min":{"1min":0.42699999999999999,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":15.972},"last":4.8090000000000002},{"interface":"front","average":{"1min":1.974,"5min":1.829,"15min":1.6240000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":6.9749999999999996,"5min":17.164000000000001,"15min":17.164000000000001},"last":3.0680000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019329,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1908236,"kb_used_data":1215684,"kb_used_omap":0,"kb_used_meta":692480,"kb_avail":91836916,"statfs":{"total":95995035648,"available":94041001984,"internally_reserved":0,"allocated":1244860416,"data_stored":1242893789,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":709099520},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,4,0,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5449999999999999,"5min":1.7589999999999999,"15min":1.387},"min":{"1min":0.443,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.0960000000000001,"5min":7.6079999999999997,"15min":13.375999999999999},"last":0.23899999999999999},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.919,"15min":1.6379999999999999},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":7.6440000000000001,"5min":10.890000000000001,"15min":10.890000000000001},"last":4.2779999999999996}]},{"osd":1,"last update":"Sat Sep 23 18:20:22 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":1.7170000000000001,"15min":1.0349999999999999},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8220000000000001,"5min":44.146999999999998,"15min":44.146999999999998},"last":1.994},{"interface":"front","average":{"1min":1.8420000000000001,"5min":1.6679999999999999,"15min":0.98999999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.5700000000000003,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.34899999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3839999999999999,"5min":1.768,"15min":1.8500000000000001},"min":{"1min":0.34300000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.1749999999999998,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.49199999999999999},{"interface":"front","average":{"1min":2.8079999999999998,"5min":1.8360000000000001,"15min":1.7649999999999999},"min":{"1min":0.36199999999999999,"5min":0.29899999999999999,"15min":0.253},"max":{"1min":17.190999999999999,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.86199999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.395,"5min":1.5529999999999999,"15min":1.9630000000000001},"min":{"1min":0.379,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.7519999999999998,"5min":11.130000000000001,"15min":29.154},"last":2.5419999999999998},{"interface":"front","average":{"1min":2.6230000000000002,"5min":1.8300000000000001,"15min":1.9930000000000001},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.099,"5min":13.470000000000001,"15min":29.216000000000001},"last":2.3820000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.3170000000000002,"15min":2.7829999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.5009999999999994,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.82699999999999996},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.6499999999999999,"15min":2.5609999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":3.8359999999999999,"5min":13.938000000000001,"15min":32.640000000000001},"last":0.55000000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.8819999999999999,"15min":1.4099999999999999},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":3.9750000000000001,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.307},{"interface":"front","average":{"1min":2.1890000000000001,"5min":2.0499999999999998,"15min":1.619},"min":{"1min":0.311,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.4969999999999999,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.38600000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.7770000000000001,"5min":2.5430000000000001,"15min":1.8360000000000001},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":18.283000000000001,"5min":19.298999999999999,"15min":19.298999999999999},"last":1.087},{"interface":"front","average":{"1min":1.5760000000000001,"5min":1.7869999999999999,"15min":1.5409999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.28399999999999997},"max":{"1min":3.786,"5min":12.800000000000001,"15min":12.800000000000001},"last":1.149}]}]},{"osd":5,"up_from":420,"seq":1803886264425,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1224560,"kb_used_data":871528,"kb_used_omap":9,"kb_used_meta":353014,"kb_avail":92520592,"statfs":{"total":95995035648,"available":94741086208,"internally_reserved":0,"allocated":892444672,"data_stored":890494401,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":361486781},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1],"upper_bound":16},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.2290000000000001,"15min":0.93200000000000005},"min":{"1min":0.21299999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.867,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":1.01},{"interface":"front","average":{"1min":2.137,"5min":1.8,"15min":1.6459999999999999},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.621,"5min":20.937999999999999,"15min":20.937999999999999},"last":0.23999999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:20:20 2023","interfaces":[{"interface":"back","average":{"1min":2.2429999999999999,"5min":1.7070000000000001,"15min":1.2989999999999999},"min":{"1min":0.23200000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":8.6649999999999991,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.374},{"interface":"front","average":{"1min":2.1840000000000002,"5min":1.73,"15min":1.141},"min":{"1min":0.315,"5min":0.26200000000000001,"15min":0.26200000000000001},"max":{"1min":8.5259999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":1.722}]},{"osd":2,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.6160000000000001,"15min":1.611},"min":{"1min":0.29699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.1150000000000002,"5min":10.705,"15min":10.705},"last":2.0979999999999999},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.8520000000000001,"15min":2.1280000000000001},"min":{"1min":0.39600000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.7560000000000002,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":4.3390000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.762,"15min":1.359},"min":{"1min":0.35599999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.0060000000000002,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":4.4870000000000001},{"interface":"front","average":{"1min":2.2200000000000002,"5min":2.1150000000000002,"15min":1.7130000000000001},"min":{"1min":0.372,"5min":0.33500000000000002,"15min":0.313},"max":{"1min":10.664999999999999,"5min":20.035,"15min":20.035},"last":1.6899999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":2.1480000000000001,"15min":2.0499999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.2270000000000003,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.29599999999999999},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1179999999999999,"15min":1.6639999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7919999999999998,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.34599999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.8340000000000001,"15min":1.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.2189999999999999,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.32900000000000001},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.3319999999999999,"15min":1.7370000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.6920000000000002,"5min":12.504,"15min":12.504},"last":0.38300000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.23,"15min":1.804},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.195,"5min":12.641,"15min":13.506},"last":1.335},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.8500000000000001,"15min":1.5600000000000001},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":5.0839999999999996,"5min":13.138,"15min":13.138},"last":0.39900000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":1127747584,"data_stored":1127743512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":941883392,"data_stored":941883392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":561840128,"data_stored":561840128,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":1212682240,"data_stored":1212678168,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":878706688,"data_stored":878706688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":853413888,"data_stored":853409816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:21:17.480 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:21:17.490 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:17.526 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:17.572 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:17.575 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:17.575 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:17.676 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:17.769 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:17.860 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:17.869 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:17.869 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:17.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:17 smithi055 ceph-mon[179668]: from='client.37353 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:17.934 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:17 smithi055 ceph-mon[179668]: pgmap v1499: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1010 MiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 829 op/s; 14371/48378 objects degraded (29.706%); 0 B/s, 46 objects/s recovering 2023-09-23T18:21:17.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:17 smithi055 ceph-mon[184174]: from='client.37353 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:17.934 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:17 smithi055 ceph-mon[184174]: pgmap v1499: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1010 MiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 829 op/s; 14371/48378 objects degraded (29.706%); 0 B/s, 46 objects/s recovering 2023-09-23T18:21:17.970 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:18.056 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:17 smithi099 ceph-mon[167076]: from='client.37353 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:18.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:17 smithi099 ceph-mon[167076]: pgmap v1499: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 1010 MiB data, 8.3 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 829 op/s; 14371/48378 objects degraded (29.706%); 0 B/s, 46 objects/s recovering 2023-09-23T18:21:18.138 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:18.142 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:18.143 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:18.243 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:18.300 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:18.359 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:18.988 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:19.037 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:19.059 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:19.059 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:19.152 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:19.160 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:19.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:18 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:19.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:18 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 14371/48378 objects degraded (29.706%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:19.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:18 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:19.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:18 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 14371/48378 objects degraded (29.706%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:19.208 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:18 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:21:19.208 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:18 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 14371/48378 objects degraded (29.706%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:19.221 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:19.285 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:19.294 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:19.294 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:19.395 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:19.438 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:19.883 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:19 smithi055 ceph-mon[179668]: pgmap v1500: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 976 MiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 834 op/s; 14371/46719 objects degraded (30.761%); 0 B/s, 46 objects/s recovering 2023-09-23T18:21:20.148 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:20.150 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:19 smithi099 ceph-mon[167076]: pgmap v1500: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 976 MiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 834 op/s; 14371/46719 objects degraded (30.761%); 0 B/s, 46 objects/s recovering 2023-09-23T18:21:20.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:20.151 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:20.160 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:19 smithi055 ceph-mon[184174]: pgmap v1500: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 976 MiB data, 8.0 GiB used, 528 GiB / 536 GiB avail; 0 B/s wr, 834 op/s; 14371/46719 objects degraded (30.761%); 0 B/s, 46 objects/s recovering 2023-09-23T18:21:20.252 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:20.324 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:20.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:21:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:21:20] "GET /metrics HTTP/1.1" 200 36666 "" "Prometheus/2.43.0" 2023-09-23T18:21:20.795 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:20.801 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:20.802 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:20.902 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:20.952 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:21:20.985 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:21.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:20 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1664723309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:21.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:20 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:21.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:20 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1664723309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:21.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:20 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:21.309 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:21.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:20 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1664723309' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:21.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:20 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:21.330 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:21.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:21.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:21.491 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:21.805 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:21.810 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:21.811 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:21.894 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:21:21.894 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[179668]: failed to encode map e621 with expected crc 2023-09-23T18:21:21.894 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:21.895 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[179668]: osdmap e621: 8 total, 8 up, 6 in 2023-09-23T18:21:21.895 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[179668]: failed to encode map e621 with expected crc 2023-09-23T18:21:21.895 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[179668]: pgmap v1502: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 948 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 14371/45372 objects degraded (31.674%); 0 B/s, 25 objects/s recovering 2023-09-23T18:21:21.895 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:21.895 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:21.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:21:21.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[184174]: failed to encode map e621 with expected crc 2023-09-23T18:21:21.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:21.900 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[184174]: osdmap e621: 8 total, 8 up, 6 in 2023-09-23T18:21:21.900 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[184174]: failed to encode map e621 with expected crc 2023-09-23T18:21:21.900 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[184174]: pgmap v1502: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 948 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 14371/45372 objects degraded (31.674%); 0 B/s, 25 objects/s recovering 2023-09-23T18:21:21.900 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:21.900 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:21.911 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:21.953 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:22.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:21 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:21:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:21 smithi099 ceph-mon[167076]: failed to encode map e621 with expected crc 2023-09-23T18:21:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:21 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:21 smithi099 ceph-mon[167076]: osdmap e621: 8 total, 8 up, 6 in 2023-09-23T18:21:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:21 smithi099 ceph-mon[167076]: failed to encode map e621 with expected crc 2023-09-23T18:21:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:21 smithi099 ceph-mon[167076]: pgmap v1502: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 948 MiB data, 7.8 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 559 op/s; 14371/45372 objects degraded (31.674%); 0 B/s, 25 objects/s recovering 2023-09-23T18:21:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:22.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:22.570 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:21:22.570 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:21:22.758 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:22.762 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:22.762 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:22.863 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:22.918 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:22.954 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:22.957 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:22.957 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:22.999 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:21:22.999 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 14371/45372 objects degraded (31.674%), 5 pgs degraded, 5 pgs undersized","count":10},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1772,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":621,"num_osds":8,"num_up_osds":8,"osd_up_since":1695493039,"num_in_osds":6,"osd_in_since":1695493021,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":114},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":15124,"data_bytes":993802940,"bytes_used":8422244352,"bytes_avail":567547969536,"bytes_total":575970213888,"degraded_objects":14371,"degraded_total":45372,"degraded_ratio":0.31673719474565809,"recovering_objects_per_sec":25,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":245,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":559},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":33,"modified":"2023-09-23T18:20:09.153115+0000","services":{}},"progress_events":{"57c27436-1bc8-4f63-b41f-a80b88a05748":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 6s)","progress":0.95798319578170776,"add_to_ceph_s":true},"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T18:21:23.058 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:23.058 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:23 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3443493628' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:21:23.125 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_ops_in_flight 2023-09-23T18:21:23.166 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:23.388 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:23 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3443493628' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:21:23.389 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:23 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3443493628' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:21:23.435 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:23.443 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:23.444 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:23.545 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:23.546 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:21:23.546 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:21:23.585 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:23.642 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:23.647 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:23.647 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:23.662 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:21:23.748 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:23.837 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:21:23.877 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:23.885 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:21:23.926 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:21:23.994 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:21:24.010 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:24.028 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:24.028 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:24.046 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:21:24.067 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:24.120 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:21:24.129 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:24.167 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:21:24.221 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:21:24.227 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:24.238 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:21:24.238 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:21:24.238 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:21:24.238 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:21:24.238 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:21:24.239 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:21:24.239 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:21:24.239 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:21:24.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:24 smithi099 ceph-mon[167076]: pgmap v1503: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 751 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 14131/35904 objects degraded (39.358%); 0 B/s, 48 objects/s recovering 2023-09-23T18:21:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:24 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 14371/45372 objects degraded (31.674%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:24.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/752524694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:24.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:24 smithi055 ceph-mon[184174]: pgmap v1503: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 751 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 14131/35904 objects degraded (39.358%); 0 B/s, 48 objects/s recovering 2023-09-23T18:21:24.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:24 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 14371/45372 objects degraded (31.674%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:24.419 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/752524694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:24.419 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:24 smithi055 ceph-mon[179668]: pgmap v1503: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 751 MiB data, 7.4 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 817 op/s; 14131/35904 objects degraded (39.358%); 0 B/s, 48 objects/s recovering 2023-09-23T18:21:24.419 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:24 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 14371/45372 objects degraded (31.674%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:24.420 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/752524694' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:24.703 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:24.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:24.709 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:24.810 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:24.901 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:24.935 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:25.267 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:25.276 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:25.276 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:25.276 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:25.377 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:25.479 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:25.601 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:25.611 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:25.611 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:25.712 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:25.784 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:25.897 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:25.910 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:25.910 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:26.011 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:26.066 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:26.130 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:26.141 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:26.142 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:26.242 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:26.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:26 smithi099 ceph-mon[167076]: pgmap v1504: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 736 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 14131/35184 objects degraded (40.163%); 0 B/s, 48 objects/s recovering 2023-09-23T18:21:26.337 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:26.429 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:26 smithi055 ceph-mon[184174]: pgmap v1504: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 736 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 14131/35184 objects degraded (40.163%); 0 B/s, 48 objects/s recovering 2023-09-23T18:21:26.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:26 smithi055 ceph-mon[179668]: pgmap v1504: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 736 MiB data, 7.3 GiB used, 529 GiB / 536 GiB avail; 0 B/s wr, 815 op/s; 14131/35184 objects degraded (40.163%); 0 B/s, 48 objects/s recovering 2023-09-23T18:21:26.638 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:26.651 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:26.651 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:26.752 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:26.848 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:26.859 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:21:26.908 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:21:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:21:26.908 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:199370 10.c 10:3e6c51a1:::benchmark_data_smithi055_425050_object95964:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T18:21:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.904675+0000", 2023-09-23T18:21:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0028685360000000001, 2023-09-23T18:21:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029020729999999998, 2023-09-23T18:21:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:21:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:21:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:21:26.909 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 199370 2023-09-23T18:21:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.904675+0000", 2023-09-23T18:21:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.910 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.904675+0000", 2023-09-23T18:21:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.904673+0000", 2023-09-23T18:21:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.911 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.904682+0000", 2023-09-23T18:21:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2679999999999994e-06 2023-09-23T18:21:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.904701+0000", 2023-09-23T18:21:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9508999999999999e-05 2023-09-23T18:21:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.912 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.904711+0000", 2023-09-23T18:21:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.781e-06 2023-09-23T18:21:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:21:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.904766+0000", 2023-09-23T18:21:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5359000000000003e-05 2023-09-23T18:21:26.913 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:21:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.904965+0000", 2023-09-23T18:21:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019862800000000001 2023-09-23T18:21:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:21:26.914 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905138+0000", 2023-09-23T18:21:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017291299999999999 2023-09-23T18:21:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.46799.0:199340 10.4 e621/605)", 2023-09-23T18:21:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.905454+0000", 2023-09-23T18:21:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0020896410000000002, 2023-09-23T18:21:26.915 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002198925, 2023-09-23T18:21:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:21:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905454+0000", 2023-09-23T18:21:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.916 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905454+0000", 2023-09-23T18:21:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905453+0000", 2023-09-23T18:21:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.917 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905459+0000", 2023-09-23T18:21:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5860000000000004e-06 2023-09-23T18:21:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905470+0000", 2023-09-23T18:21:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1569999999999999e-05 2023-09-23T18:21:26.918 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905478+0000", 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.7530000000000001e-06 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.46799.0:199341 10.c e621/595)", 2023-09-23T18:21:26.919 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.905485+0000", 2023-09-23T18:21:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0020585439999999998, 2023-09-23T18:21:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022025859999999999, 2023-09-23T18:21:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:21:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905485+0000", 2023-09-23T18:21:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.920 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905485+0000", 2023-09-23T18:21:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905485+0000", 2023-09-23T18:21:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:26.921 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905487+0000", 2023-09-23T18:21:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6589999999999999e-06 2023-09-23T18:21:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905493+0000", 2023-09-23T18:21:26.922 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0290000000000002e-06 2023-09-23T18:21:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905498+0000", 2023-09-23T18:21:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5600000000000004e-06 2023-09-23T18:21:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.923 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.46799.0:199348 10.4 e621/605)", 2023-09-23T18:21:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.905502+0000", 2023-09-23T18:21:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.002041653, 2023-09-23T18:21:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022124559999999998, 2023-09-23T18:21:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:21:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.924 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905502+0000", 2023-09-23T18:21:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905502+0000", 2023-09-23T18:21:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.925 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905502+0000", 2023-09-23T18:21:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:21:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905504+0000", 2023-09-23T18:21:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9190000000000002e-06 2023-09-23T18:21:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.926 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905509+0000", 2023-09-23T18:21:26.927 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3369999999999999e-06 2023-09-23T18:21:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905513+0000", 2023-09-23T18:21:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3880000000000002e-06 2023-09-23T18:21:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.929 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.46799.0:199356 10.4 e621/605)", 2023-09-23T18:21:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.905517+0000", 2023-09-23T18:21:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0020266419999999999, 2023-09-23T18:21:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022235990000000001, 2023-09-23T18:21:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:21:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.930 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905517+0000", 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905517+0000", 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.931 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905517+0000", 2023-09-23T18:21:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:21:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905519+0000", 2023-09-23T18:21:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8059999999999999e-06 2023-09-23T18:21:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.932 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905524+0000", 2023-09-23T18:21:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1329999999999998e-06 2023-09-23T18:21:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905528+0000", 2023-09-23T18:21:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0409999999999999e-06 2023-09-23T18:21:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.933 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.46799.0:199360 10.4 e621/605)", 2023-09-23T18:21:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.905531+0000", 2023-09-23T18:21:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0020122030000000002, 2023-09-23T18:21:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022405630000000001, 2023-09-23T18:21:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:21:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.934 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905531+0000", 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905531+0000", 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.935 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905531+0000", 2023-09-23T18:21:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:21:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905533+0000", 2023-09-23T18:21:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0260000000000002e-06 2023-09-23T18:21:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.936 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905538+0000", 2023-09-23T18:21:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2140000000000004e-06 2023-09-23T18:21:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905542+0000", 2023-09-23T18:21:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8890000000000004e-06 2023-09-23T18:21:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.937 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.46799.0:199365 10.4 e621/605)", 2023-09-23T18:21:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.905546+0000", 2023-09-23T18:21:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.0019976999999999998, 2023-09-23T18:21:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002251959, 2023-09-23T18:21:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "queued for pg", 2023-09-23T18:21:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.938 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905546+0000", 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905546+0000", 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.939 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905546+0000", 2023-09-23T18:21:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967296 2023-09-23T18:21:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905548+0000", 2023-09-23T18:21:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.8059999999999999e-06 2023-09-23T18:21:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.940 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905553+0000", 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1619999999999999e-06 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.905557+0000", 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8550000000000004e-06 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.941 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:199340 10.4 10:22e27c4a:::benchmark_data_smithi055_425050_object95934:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T18:21:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.854401+0000", 2023-09-23T18:21:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.053142446000000003, 2023-09-23T18:21:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.053432036000000002, 2023-09-23T18:21:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:21:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:21:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:21:26.942 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 199340 2023-09-23T18:21:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.854401+0000", 2023-09-23T18:21:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.943 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.854401+0000", 2023-09-23T18:21:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.854399+0000", 2023-09-23T18:21:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:21:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.944 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.854408+0000", 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8699999999999998e-06 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.854420+0000", 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2037e-05 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.945 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.854427+0000", 2023-09-23T18:21:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.878e-06 2023-09-23T18:21:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:21:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.866226+0000", 2023-09-23T18:21:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011798774999999999 2023-09-23T18:21:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:21:26.946 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.866334+0000", 2023-09-23T18:21:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000108242 2023-09-23T18:21:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:21:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.866474+0000", 2023-09-23T18:21:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013958300000000001 2023-09-23T18:21:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:26.947 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.884626+0000", 2023-09-23T18:21:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018152007000000001 2023-09-23T18:21:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:21:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892067+0000", 2023-09-23T18:21:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0074417270000000004 2023-09-23T18:21:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.948 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:199341 10.c 10:3f72ee75:::benchmark_data_smithi055_425050_object95935:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.865212+0000", 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.042331727999999999, 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.042705161999999998, 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:26.949 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 199341 2023-09-23T18:21:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.865212+0000", 2023-09-23T18:21:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.950 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.865212+0000", 2023-09-23T18:21:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.865210+0000", 2023-09-23T18:21:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:21:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.951 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.865220+0000", 2023-09-23T18:21:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0419e-05 2023-09-23T18:21:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.865235+0000", 2023-09-23T18:21:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5369000000000001e-05 2023-09-23T18:21:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.952 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.865247+0000", 2023-09-23T18:21:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1423999999999999e-05 2023-09-23T18:21:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:21:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.869012+0000", 2023-09-23T18:21:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037648220000000001 2023-09-23T18:21:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:21:26.953 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.869108+0000", 2023-09-23T18:21:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5816999999999994e-05 2023-09-23T18:21:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:21:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.869202+0000", 2023-09-23T18:21:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.4369000000000004e-05 2023-09-23T18:21:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:26.954 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.884667+0000", 2023-09-23T18:21:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01546458 2023-09-23T18:21:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:21:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892082+0000", 2023-09-23T18:21:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0074151429999999999 2023-09-23T18:21:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.955 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:199348 10.4 10:2d73876d:::benchmark_data_smithi055_425050_object95942:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.867325+0000", 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.040218263999999997, 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.040677921999999998, 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:26.956 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 199348 2023-09-23T18:21:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.867325+0000", 2023-09-23T18:21:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.957 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.867325+0000", 2023-09-23T18:21:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.867324+0000", 2023-09-23T18:21:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.958 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.867333+0000", 2023-09-23T18:21:26.959 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7940000000000008e-06 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.867347+0000", 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4809e-05 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.867357+0000", 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.662e-06 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.960 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.871930+0000", 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045732639999999996 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.872066+0000", 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013563799999999999 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:21:26.961 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.872204+0000", 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013841999999999999 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.890438+0000", 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.018233198999999999 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892088+0000", 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001650606 2023-09-23T18:21:26.962 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:199356 10.4 10:22cbda17:::benchmark_data_smithi055_425050_object95950:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.881743+0000", 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.025800946000000002, 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.026314039000000001, 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:21:26.963 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 199356 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.881743+0000", 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.964 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.881743+0000", 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.881741+0000", 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.965 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.881754+0000", 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3930999999999999e-05 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.881776+0000", 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.126e-05 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.881788+0000", 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2672e-05 2023-09-23T18:21:26.966 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.881844+0000", 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5457000000000001e-05 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.881999+0000", 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015554499999999999 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.967 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.882170+0000", 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017065099999999999 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.890557+0000", 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0083864979999999992 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.898110+0000", 2023-09-23T18:21:26.968 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0075530329999999998 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:199360 10.4 10:23f36662:::benchmark_data_smithi055_425050_object95954:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.892141+0000", 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.015402813, 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015965285999999999, 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:21:26.969 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:21:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 199360 2023-09-23T18:21:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892141+0000", 2023-09-23T18:21:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.970 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892141+0000", 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892139+0000", 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.971 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892148+0000", 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7600000000000008e-06 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892173+0000", 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5145000000000001e-05 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892203+0000", 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0392e-05 2023-09-23T18:21:26.972 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892360+0000", 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015714799999999999 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892507+0000", 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014646400000000001 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.973 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.892661+0000", 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015418100000000001 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.898590+0000", 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0059291630000000003 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:26.974 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.900805+0000", 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022148129999999999 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:199365 10.4 10:2be8a31d:::benchmark_data_smithi055_425050_object95959:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:26.894093+0000", 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 0.013451029999999999, 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014076155, 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:21:26.975 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "waiting for sub ops", 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 199365 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.894093+0000", 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.976 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.894093+0000", 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.894091+0000", 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.894098+0000", 2023-09-23T18:21:26.977 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.6000000000000003e-06 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.894115+0000", 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.713e-05 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.894123+0000", 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9540000000000004e-06 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.978 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.895558+0000", 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014354750000000001 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.895685+0000", 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012672700000000001 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 3,4", 2023-09-23T18:21:26.979 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.895805+0000", 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119636 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.898596+0000", 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0027914509999999999 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:26.900839+0000", 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00224326 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.980 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:21:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:21:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: ], 2023-09-23T18:21:26.981 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 13 2023-09-23T18:21:26.981 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:21:27.040 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:27.054 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:27.054 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:27.155 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:27.283 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:27.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:27.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:27.434 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:27.435 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:27.729 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_blocked_ops 2023-09-23T18:21:27.764 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:27.770 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:27.771 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:27.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:27.871 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:27.908 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:21:27.923 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:21:27.988 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:28.099 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:28.105 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:28.106 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:28.206 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:28.249 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:28.323 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:28.325 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:28.326 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:28.417 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:28 smithi055 ceph-mon[179668]: pgmap v1505: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 525 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 774 op/s; 13868/25095 objects degraded (55.262%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:28.426 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:28.461 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:28.518 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:28.520 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:28 smithi099 ceph-mon[167076]: pgmap v1505: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 525 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 774 op/s; 13868/25095 objects degraded (55.262%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:28.523 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:28.523 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:28.624 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:28.662 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:28.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:28 smithi055 ceph-mon[184174]: pgmap v1505: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 525 MiB data, 6.9 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 774 op/s; 13868/25095 objects degraded (55.262%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:28.709 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:28.716 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:28.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:28.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:28.857 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:28.914 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:28.918 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:28.919 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:29.019 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:29.122 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:29.420 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:29 smithi055 ceph-mon[179668]: from='client.56737 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:29.421 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:29 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 13868/25095 objects degraded (55.262%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:29.421 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3751033268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:29.421 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:29 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:29.421 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:29 smithi055 ceph-mon[184174]: from='client.56737 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:29.421 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:29 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 13868/25095 objects degraded (55.262%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:29.421 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3751033268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:29.421 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:29 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:29.505 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:29.507 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:21:29.521 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:29.522 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:29.622 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:29.677 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:29.754 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:29.760 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:29.761 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:29.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:29 smithi099 ceph-mon[167076]: from='client.56737 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:29.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:29 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 13868/25095 objects degraded (55.262%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:29.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3751033268' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:29.762 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:29 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:29.862 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:29.938 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:29.994 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1505,"stamp":"2023-09-23T18:21:27.214292+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":550845116,"num_objects":8365,"num_object_clones":0,"num_object_copies":25095,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13868,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8365,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":199381,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9685,"num_bytes_recovered":502226569,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":82992,"ondisk_log_size":82992,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":7194588,"kb_used_data":3526572,"kb_used_omap":40,"kb_used_meta":3667223,"kb_avail":555276324,"statfs":{"total":575970213888,"available":568602955776,"internally_reserved":0,"allocated":3611209728,"data_stored":3596243271,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3755236666},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2,0,0,0,3,7,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":9,"apply_latency_ms":9,"commit_latency_ns":9000000,"apply_latency_ns":9000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-508624896,"num_objects":-7761,"num_object_clones":0,"num_object_copies":-23283,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-503,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7761,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7761,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":503,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.019686"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":909,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723363+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:21:11.723363+0000","last_peered":"2023-09-23T18:21:11.723363+0000","last_clean":"2023-09-23T18:21:11.723363+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:21:11.723363+0000","last_undegraded":"2023-09-23T18:21:11.723363+0000","last_fullsized":"2023-09-23T18:21:11.723363+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":280,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.733329+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:21:11.733329+0000","last_peered":"2023-09-23T18:21:11.733329+0000","last_clean":"2023-09-23T18:21:11.733329+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:21:11.733329+0000","last_undegraded":"2023-09-23T18:21:11.733329+0000","last_fullsized":"2023-09-23T18:21:11.733329+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":38,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:16.418668+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:21:16.418668+0000","last_peered":"2023-09-23T18:21:16.418668+0000","last_clean":"2023-09-23T18:21:16.418668+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:21:16.418668+0000","last_undegraded":"2023-09-23T18:21:16.418668+0000","last_fullsized":"2023-09-23T18:21:16.418668+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":915,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724635+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:21:11.724635+0000","last_peered":"2023-09-23T18:21:11.724635+0000","last_clean":"2023-09-23T18:21:11.724635+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:21:11.724635+0000","last_undegraded":"2023-09-23T18:21:11.724635+0000","last_fullsized":"2023-09-23T18:21:11.724635+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":362,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.736231+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:21:11.736231+0000","last_peered":"2023-09-23T18:21:11.736231+0000","last_clean":"2023-09-23T18:21:11.736231+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:21:11.736231+0000","last_undegraded":"2023-09-23T18:21:11.736231+0000","last_fullsized":"2023-09-23T18:21:11.736231+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":682,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.742203+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:21:11.742203+0000","last_peered":"2023-09-23T18:21:11.742203+0000","last_clean":"2023-09-23T18:21:11.742203+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:21:11.742203+0000","last_undegraded":"2023-09-23T18:21:11.742203+0000","last_fullsized":"2023-09-23T18:21:11.742203+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":932,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723456+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:21:11.723456+0000","last_peered":"2023-09-23T18:21:11.723456+0000","last_clean":"2023-09-23T18:21:11.723456+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:21:11.723456+0000","last_undegraded":"2023-09-23T18:21:11.723456+0000","last_fullsized":"2023-09-23T18:21:11.723456+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":38,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.733580+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:21:11.733580+0000","last_peered":"2023-09-23T18:21:11.733580+0000","last_clean":"2023-09-23T18:21:11.733580+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:21:11.733580+0000","last_undegraded":"2023-09-23T18:21:11.733580+0000","last_fullsized":"2023-09-23T18:21:11.733580+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":197,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729514+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:21:11.729514+0000","last_peered":"2023-09-23T18:21:11.729514+0000","last_clean":"2023-09-23T18:21:11.729514+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:21:11.729514+0000","last_undegraded":"2023-09-23T18:21:11.729514+0000","last_fullsized":"2023-09-23T18:21:11.729514+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1055,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723912+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:21:11.723912+0000","last_peered":"2023-09-23T18:21:11.723912+0000","last_clean":"2023-09-23T18:21:11.723912+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:21:11.723912+0000","last_undegraded":"2023-09-23T18:21:11.723912+0000","last_fullsized":"2023-09-23T18:21:11.723912+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":131,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724750+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:21:11.724750+0000","last_peered":"2023-09-23T18:21:11.724750+0000","last_clean":"2023-09-23T18:21:11.724750+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:21:11.724750+0000","last_undegraded":"2023-09-23T18:21:11.724750+0000","last_fullsized":"2023-09-23T18:21:11.724750+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":211,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.733936+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:21:11.733936+0000","last_peered":"2023-09-23T18:21:11.733936+0000","last_clean":"2023-09-23T18:21:11.733936+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:21:11.733936+0000","last_undegraded":"2023-09-23T18:21:11.733936+0000","last_fullsized":"2023-09-23T18:21:11.733936+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":832,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728228+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:21:11.728228+0000","last_peered":"2023-09-23T18:21:11.728228+0000","last_clean":"2023-09-23T18:21:11.728228+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:21:11.728228+0000","last_undegraded":"2023-09-23T18:21:11.728228+0000","last_fullsized":"2023-09-23T18:21:11.728228+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":167,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.725839+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:21:11.725839+0000","last_peered":"2023-09-23T18:21:11.725839+0000","last_clean":"2023-09-23T18:21:11.725839+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:21:11.725839+0000","last_undegraded":"2023-09-23T18:21:11.725839+0000","last_fullsized":"2023-09-23T18:21:11.725839+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":166,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.727736+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:21:11.727736+0000","last_peered":"2023-09-23T18:21:11.727736+0000","last_clean":"2023-09-23T18:21:11.727736+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:21:11.727736+0000","last_undegraded":"2023-09-23T18:21:11.727736+0000","last_fullsized":"2023-09-23T18:21:11.727736+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":966,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724837+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:21:11.724837+0000","last_peered":"2023-09-23T18:21:11.724837+0000","last_clean":"2023-09-23T18:21:11.724837+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:21:11.724837+0000","last_undegraded":"2023-09-23T18:21:11.724837+0000","last_fullsized":"2023-09-23T18:21:11.724837+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":415,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.737981+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:21:11.737981+0000","last_peered":"2023-09-23T18:21:11.737981+0000","last_clean":"2023-09-23T18:21:11.737981+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:21:11.737981+0000","last_undegraded":"2023-09-23T18:21:11.737981+0000","last_fullsized":"2023-09-23T18:21:11.737981+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":434,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729955+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:21:11.729955+0000","last_peered":"2023-09-23T18:21:11.729955+0000","last_clean":"2023-09-23T18:21:11.729955+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:21:11.729955+0000","last_undegraded":"2023-09-23T18:21:11.729955+0000","last_fullsized":"2023-09-23T18:21:11.729955+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":995,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724325+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:21:11.724325+0000","last_peered":"2023-09-23T18:21:11.724325+0000","last_clean":"2023-09-23T18:21:11.724325+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:21:11.724325+0000","last_undegraded":"2023-09-23T18:21:11.724325+0000","last_fullsized":"2023-09-23T18:21:11.724325+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":173,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728683+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:21:11.728683+0000","last_peered":"2023-09-23T18:21:11.728683+0000","last_clean":"2023-09-23T18:21:11.728683+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:21:11.728683+0000","last_undegraded":"2023-09-23T18:21:11.728683+0000","last_fullsized":"2023-09-23T18:21:11.728683+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":440,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728355+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:21:11.728355+0000","last_peered":"2023-09-23T18:21:11.728355+0000","last_clean":"2023-09-23T18:21:11.728355+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:21:11.728355+0000","last_undegraded":"2023-09-23T18:21:11.728355+0000","last_fullsized":"2023-09-23T18:21:11.728355+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":847,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723449+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:21:11.723449+0000","last_peered":"2023-09-23T18:21:11.723449+0000","last_clean":"2023-09-23T18:21:11.723449+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:21:11.723449+0000","last_undegraded":"2023-09-23T18:21:11.723449+0000","last_fullsized":"2023-09-23T18:21:11.723449+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":440,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728369+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:21:11.728369+0000","last_peered":"2023-09-23T18:21:11.728369+0000","last_clean":"2023-09-23T18:21:11.728369+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:21:11.728369+0000","last_undegraded":"2023-09-23T18:21:11.728369+0000","last_fullsized":"2023-09-23T18:21:11.728369+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":118,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723334+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:21:11.723334+0000","last_peered":"2023-09-23T18:21:11.723334+0000","last_clean":"2023-09-23T18:21:11.723334+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:21:11.723334+0000","last_undegraded":"2023-09-23T18:21:11.723334+0000","last_fullsized":"2023-09-23T18:21:11.723334+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":818,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723103+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:21:11.723103+0000","last_peered":"2023-09-23T18:21:11.723103+0000","last_clean":"2023-09-23T18:21:11.723103+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:21:11.723103+0000","last_undegraded":"2023-09-23T18:21:11.723103+0000","last_fullsized":"2023-09-23T18:21:11.723103+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":137,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.739286+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:21:11.739286+0000","last_peered":"2023-09-23T18:21:11.739286+0000","last_clean":"2023-09-23T18:21:11.739286+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:21:11.739286+0000","last_undegraded":"2023-09-23T18:21:11.739286+0000","last_fullsized":"2023-09-23T18:21:11.739286+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":131,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724195+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:21:11.724195+0000","last_peered":"2023-09-23T18:21:11.724195+0000","last_clean":"2023-09-23T18:21:11.724195+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:21:11.724195+0000","last_undegraded":"2023-09-23T18:21:11.724195+0000","last_fullsized":"2023-09-23T18:21:11.724195+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":757,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.747276+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:21:11.747276+0000","last_peered":"2023-09-23T18:21:11.747276+0000","last_clean":"2023-09-23T18:21:11.747276+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:21:11.747276+0000","last_undegraded":"2023-09-23T18:21:11.747276+0000","last_fullsized":"2023-09-23T18:21:11.747276+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":208,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.727603+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:21:11.727603+0000","last_peered":"2023-09-23T18:21:11.727603+0000","last_clean":"2023-09-23T18:21:11.727603+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:21:11.727603+0000","last_undegraded":"2023-09-23T18:21:11.727603+0000","last_fullsized":"2023-09-23T18:21:11.727603+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":220,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.734922+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:21:11.734922+0000","last_peered":"2023-09-23T18:21:11.734922+0000","last_clean":"2023-09-23T18:21:11.734922+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:21:11.734922+0000","last_undegraded":"2023-09-23T18:21:11.734922+0000","last_fullsized":"2023-09-23T18:21:11.734922+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":852,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.727408+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:21:11.727408+0000","last_peered":"2023-09-23T18:21:11.727408+0000","last_clean":"2023-09-23T18:21:11.727408+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:21:11.727408+0000","last_undegraded":"2023-09-23T18:21:11.727408+0000","last_fullsized":"2023-09-23T18:21:11.727408+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":761,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.727457+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:21:11.727457+0000","last_peered":"2023-09-23T18:21:11.727457+0000","last_clean":"2023-09-23T18:21:11.727457+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:21:11.727457+0000","last_undegraded":"2023-09-23T18:21:11.727457+0000","last_fullsized":"2023-09-23T18:21:11.727457+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":38,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.734029+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:21:11.734029+0000","last_peered":"2023-09-23T18:21:11.734029+0000","last_clean":"2023-09-23T18:21:11.734029+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:21:11.734029+0000","last_undegraded":"2023-09-23T18:21:11.734029+0000","last_fullsized":"2023-09-23T18:21:11.734029+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"621'12581","reported_seq":12636,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:26.184657+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:21:26.184657+0000","last_peered":"2023-09-23T18:21:26.184657+0000","last_clean":"2023-09-23T18:21:26.184657+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:21:26.184657+0000","last_undegraded":"2023-09-23T18:21:26.184657+0000","last_fullsized":"2023-09-23T18:21:26.184657+0000","mapping_epoch":574,"log_start":"607'7100","ondisk_log_start":"607'7100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5481,"log_dups_size":0,"ondisk_log_size":5481,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32309248,"num_objects":493,"num_object_clones":0,"num_object_copies":1479,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":493,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12581,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":886,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728732+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:21:11.728732+0000","last_peered":"2023-09-23T18:21:11.728732+0000","last_clean":"2023-09-23T18:21:11.728732+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:21:11.728732+0000","last_undegraded":"2023-09-23T18:21:11.728732+0000","last_fullsized":"2023-09-23T18:21:11.728732+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":257,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.734919+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:21:11.734919+0000","last_peered":"2023-09-23T18:21:11.734919+0000","last_clean":"2023-09-23T18:21:11.734919+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:21:11.734919+0000","last_undegraded":"2023-09-23T18:21:11.734919+0000","last_fullsized":"2023-09-23T18:21:11.734919+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":131,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723103+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:21:11.723103+0000","last_peered":"2023-09-23T18:21:11.723103+0000","last_clean":"2023-09-23T18:21:11.723103+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:21:11.723103+0000","last_undegraded":"2023-09-23T18:21:11.723103+0000","last_fullsized":"2023-09-23T18:21:11.723103+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"621'12207","reported_seq":12272,"reported_epoch":621,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:26.231007+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:21:26.231007+0000","last_peered":"2023-09-23T18:21:26.231007+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:21:26.231007+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"608'7700","ondisk_log_start":"608'7700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4507,"log_dups_size":0,"ondisk_log_size":4507,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31784960,"num_objects":485,"num_object_clones":0,"num_object_copies":1455,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":485,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12207,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"5,6","objects":-3663}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":741,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.736766+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:21:11.736766+0000","last_peered":"2023-09-23T18:21:11.736766+0000","last_clean":"2023-09-23T18:21:11.736766+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:21:11.736766+0000","last_undegraded":"2023-09-23T18:21:11.736766+0000","last_fullsized":"2023-09-23T18:21:11.736766+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":258,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.736672+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:21:11.736672+0000","last_peered":"2023-09-23T18:21:11.736672+0000","last_clean":"2023-09-23T18:21:11.736672+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:21:11.736672+0000","last_undegraded":"2023-09-23T18:21:11.736672+0000","last_fullsized":"2023-09-23T18:21:11.736672+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":237,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.733656+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:21:11.733656+0000","last_peered":"2023-09-23T18:21:11.733656+0000","last_clean":"2023-09-23T18:21:11.733656+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:21:11.733656+0000","last_undegraded":"2023-09-23T18:21:11.733656+0000","last_fullsized":"2023-09-23T18:21:11.733656+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"621'12284","reported_seq":19299,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:25.282172+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:21:25.282172+0000","last_peered":"2023-09-23T18:21:25.282172+0000","last_clean":"2023-09-23T18:21:25.282172+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:21:25.282172+0000","last_undegraded":"2023-09-23T18:21:25.282172+0000","last_fullsized":"2023-09-23T18:21:25.282172+0000","mapping_epoch":605,"log_start":"607'7400","ondisk_log_start":"607'7400","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4884,"log_dups_size":0,"ondisk_log_size":4884,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":36700160,"num_objects":560,"num_object_clones":0,"num_object_copies":1680,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":560,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12284,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":636,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729450+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:21:11.729450+0000","last_peered":"2023-09-23T18:21:11.729450+0000","last_clean":"2023-09-23T18:21:11.729450+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:21:11.729450+0000","last_undegraded":"2023-09-23T18:21:11.729450+0000","last_fullsized":"2023-09-23T18:21:11.729450+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":546,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724102+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:21:11.724102+0000","last_peered":"2023-09-23T18:21:11.724102+0000","last_clean":"2023-09-23T18:21:11.724102+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:21:11.724102+0000","last_undegraded":"2023-09-23T18:21:11.724102+0000","last_fullsized":"2023-09-23T18:21:11.724102+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":197,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729851+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:21:11.729851+0000","last_peered":"2023-09-23T18:21:11.729851+0000","last_clean":"2023-09-23T18:21:11.729851+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:21:11.729851+0000","last_undegraded":"2023-09-23T18:21:11.729851+0000","last_fullsized":"2023-09-23T18:21:11.729851+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"621'12614","reported_seq":12671,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:26.377567+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:21:26.377567+0000","last_peered":"2023-09-23T18:21:26.377567+0000","last_clean":"2023-09-23T18:21:26.377567+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:21:26.377567+0000","last_undegraded":"2023-09-23T18:21:26.377567+0000","last_fullsized":"2023-09-23T18:21:26.377567+0000","mapping_epoch":574,"log_start":"607'7500","ondisk_log_start":"607'7500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5114,"log_dups_size":0,"ondisk_log_size":5114,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31326208,"num_objects":478,"num_object_clones":0,"num_object_copies":1434,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":478,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12614,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":652,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.735014+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:21:11.735014+0000","last_peered":"2023-09-23T18:21:11.735014+0000","last_clean":"2023-09-23T18:21:11.735014+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:21:11.735014+0000","last_undegraded":"2023-09-23T18:21:11.735014+0000","last_fullsized":"2023-09-23T18:21:11.735014+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":145,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723727+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:21:11.723727+0000","last_peered":"2023-09-23T18:21:11.723727+0000","last_clean":"2023-09-23T18:21:11.723727+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:21:11.723727+0000","last_undegraded":"2023-09-23T18:21:11.723727+0000","last_fullsized":"2023-09-23T18:21:11.723727+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":458,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.740140+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:21:11.740140+0000","last_peered":"2023-09-23T18:21:11.740140+0000","last_clean":"2023-09-23T18:21:11.740140+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:21:11.740140+0000","last_undegraded":"2023-09-23T18:21:11.740140+0000","last_fullsized":"2023-09-23T18:21:11.740140+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"621'12460","reported_seq":12517,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:23.841494+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:21:23.841494+0000","last_peered":"2023-09-23T18:21:23.841494+0000","last_clean":"2023-09-23T18:21:23.841494+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:21:23.841494+0000","last_undegraded":"2023-09-23T18:21:23.841494+0000","last_fullsized":"2023-09-23T18:21:23.841494+0000","mapping_epoch":574,"log_start":"609'8100","ondisk_log_start":"609'8100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4360,"log_dups_size":0,"ondisk_log_size":4360,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":38273024,"num_objects":584,"num_object_clones":0,"num_object_copies":1752,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":584,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12460,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1103,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723290+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:21:11.723290+0000","last_peered":"2023-09-23T18:21:11.723290+0000","last_clean":"2023-09-23T18:21:11.723290+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:21:11.723290+0000","last_undegraded":"2023-09-23T18:21:11.723290+0000","last_fullsized":"2023-09-23T18:21:11.723290+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":358,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.747555+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:21:11.747555+0000","last_peered":"2023-09-23T18:21:11.747555+0000","last_clean":"2023-09-23T18:21:11.747555+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:21:11.747555+0000","last_undegraded":"2023-09-23T18:21:11.747555+0000","last_fullsized":"2023-09-23T18:21:11.747555+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":165,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728306+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:21:11.728306+0000","last_peered":"2023-09-23T18:21:11.728306+0000","last_clean":"2023-09-23T18:21:11.728306+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:21:11.728306+0000","last_undegraded":"2023-09-23T18:21:11.728306+0000","last_fullsized":"2023-09-23T18:21:11.728306+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"621'12557","reported_seq":17348,"reported_epoch":621,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:26.221700+0000","last_change":"2023-09-23T18:20:34.894639+0000","last_active":"2023-09-23T18:21:26.221700+0000","last_peered":"2023-09-23T18:21:26.221700+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:21:26.221700+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"607'7100","ondisk_log_start":"607'7100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5457,"log_dups_size":0,"ondisk_log_size":5457,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":30998528,"num_objects":473,"num_object_clones":0,"num_object_copies":1419,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1863,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":473,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12557,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2363,"num_bytes_recovered":46006272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"4,5","objects":-1390}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":890,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723281+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:21:11.723281+0000","last_peered":"2023-09-23T18:21:11.723281+0000","last_clean":"2023-09-23T18:21:11.723281+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:21:11.723281+0000","last_undegraded":"2023-09-23T18:21:11.723281+0000","last_fullsized":"2023-09-23T18:21:11.723281+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":173,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729791+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:21:11.729791+0000","last_peered":"2023-09-23T18:21:11.729791+0000","last_clean":"2023-09-23T18:21:11.729791+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:21:11.729791+0000","last_undegraded":"2023-09-23T18:21:11.729791+0000","last_fullsized":"2023-09-23T18:21:11.729791+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":472,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728857+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:21:11.728857+0000","last_peered":"2023-09-23T18:21:11.728857+0000","last_clean":"2023-09-23T18:21:11.728857+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:21:11.728857+0000","last_undegraded":"2023-09-23T18:21:11.728857+0000","last_fullsized":"2023-09-23T18:21:11.728857+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"621'12544","reported_seq":12599,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:26.265940+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:21:26.265940+0000","last_peered":"2023-09-23T18:21:26.265940+0000","last_clean":"2023-09-23T18:21:26.265940+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:21:26.265940+0000","last_undegraded":"2023-09-23T18:21:26.265940+0000","last_fullsized":"2023-09-23T18:21:26.265940+0000","mapping_epoch":574,"log_start":"609'8100","ondisk_log_start":"609'8100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4444,"log_dups_size":0,"ondisk_log_size":4444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":33292288,"num_objects":508,"num_object_clones":0,"num_object_copies":1524,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":508,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12544,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":892,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724720+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:21:11.724720+0000","last_peered":"2023-09-23T18:21:11.724720+0000","last_clean":"2023-09-23T18:21:11.724720+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:21:11.724720+0000","last_undegraded":"2023-09-23T18:21:11.724720+0000","last_fullsized":"2023-09-23T18:21:11.724720+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":457,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723178+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:21:11.723178+0000","last_peered":"2023-09-23T18:21:11.723178+0000","last_clean":"2023-09-23T18:21:11.723178+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:21:11.723178+0000","last_undegraded":"2023-09-23T18:21:11.723178+0000","last_fullsized":"2023-09-23T18:21:11.723178+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":233,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728816+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:21:11.728816+0000","last_peered":"2023-09-23T18:21:11.728816+0000","last_clean":"2023-09-23T18:21:11.728816+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:21:11.728816+0000","last_undegraded":"2023-09-23T18:21:11.728816+0000","last_fullsized":"2023-09-23T18:21:11.728816+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"621'12342","reported_seq":12397,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:22.346392+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:21:22.346392+0000","last_peered":"2023-09-23T18:21:22.346392+0000","last_clean":"2023-09-23T18:21:22.346392+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:21:22.346392+0000","last_undegraded":"2023-09-23T18:21:22.346392+0000","last_fullsized":"2023-09-23T18:21:22.346392+0000","mapping_epoch":574,"log_start":"608'7700","ondisk_log_start":"608'7700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4642,"log_dups_size":0,"ondisk_log_size":4642,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":45350912,"num_objects":692,"num_object_clones":0,"num_object_copies":2076,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":692,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12342,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":873,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723675+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:21:11.723675+0000","last_peered":"2023-09-23T18:21:11.723675+0000","last_clean":"2023-09-23T18:21:11.723675+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:21:11.723675+0000","last_undegraded":"2023-09-23T18:21:11.723675+0000","last_fullsized":"2023-09-23T18:21:11.723675+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":38,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.733726+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:21:11.733726+0000","last_peered":"2023-09-23T18:21:11.733726+0000","last_clean":"2023-09-23T18:21:11.733726+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:21:11.733726+0000","last_undegraded":"2023-09-23T18:21:11.733726+0000","last_fullsized":"2023-09-23T18:21:11.733726+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":349,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.722887+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:21:11.722887+0000","last_peered":"2023-09-23T18:21:11.722887+0000","last_clean":"2023-09-23T18:21:11.722887+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:21:11.722887+0000","last_undegraded":"2023-09-23T18:21:11.722887+0000","last_fullsized":"2023-09-23T18:21:11.722887+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"621'12520","reported_seq":12578,"reported_epoch":621,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:25.285670+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:21:25.285670+0000","last_peered":"2023-09-23T18:21:25.285670+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:21:25.285670+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"608'7600","ondisk_log_start":"608'7600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4920,"log_dups_size":0,"ondisk_log_size":4920,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":35913728,"num_objects":548,"num_object_clones":0,"num_object_copies":1644,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":548,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12520,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-3737}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":732,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729597+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:21:11.729597+0000","last_peered":"2023-09-23T18:21:11.729597+0000","last_clean":"2023-09-23T18:21:11.729597+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:21:11.729597+0000","last_undegraded":"2023-09-23T18:21:11.729597+0000","last_fullsized":"2023-09-23T18:21:11.729597+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":971,"reported_epoch":620,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:11.722993+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:21:11.722993+0000","last_peered":"2023-09-23T18:21:11.722993+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:21:11.722993+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":155,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724283+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:21:11.724283+0000","last_peered":"2023-09-23T18:21:11.724283+0000","last_clean":"2023-09-23T18:21:11.724283+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:21:11.724283+0000","last_undegraded":"2023-09-23T18:21:11.724283+0000","last_fullsized":"2023-09-23T18:21:11.724283+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":135,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723623+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:21:11.723623+0000","last_peered":"2023-09-23T18:21:11.723623+0000","last_clean":"2023-09-23T18:21:11.723623+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:21:11.723623+0000","last_undegraded":"2023-09-23T18:21:11.723623+0000","last_fullsized":"2023-09-23T18:21:11.723623+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"621'12456","reported_seq":12520,"reported_epoch":621,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:26.270795+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:21:26.270795+0000","last_peered":"2023-09-23T18:21:26.270795+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:21:26.270795+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"609'8000","ondisk_log_start":"609'8000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4456,"log_dups_size":0,"ondisk_log_size":4456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31457280,"num_objects":480,"num_object_clones":0,"num_object_copies":1440,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":480,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12456,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"5,6","objects":-3079}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":804,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723613+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:21:11.723613+0000","last_peered":"2023-09-23T18:21:11.723613+0000","last_clean":"2023-09-23T18:21:11.723613+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:21:11.723613+0000","last_undegraded":"2023-09-23T18:21:11.723613+0000","last_fullsized":"2023-09-23T18:21:11.723613+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1505,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723595+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:21:11.723595+0000","last_peered":"2023-09-23T18:21:11.723595+0000","last_clean":"2023-09-23T18:21:11.723595+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:21:11.723595+0000","last_undegraded":"2023-09-23T18:21:11.723595+0000","last_fullsized":"2023-09-23T18:21:11.723595+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":145,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728282+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:21:11.728282+0000","last_peered":"2023-09-23T18:21:11.728282+0000","last_clean":"2023-09-23T18:21:11.728282+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:21:11.728282+0000","last_undegraded":"2023-09-23T18:21:11.728282+0000","last_fullsized":"2023-09-23T18:21:11.728282+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":445,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728517+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:21:11.728517+0000","last_peered":"2023-09-23T18:21:11.728517+0000","last_clean":"2023-09-23T18:21:11.728517+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:21:11.728517+0000","last_undegraded":"2023-09-23T18:21:11.728517+0000","last_fullsized":"2023-09-23T18:21:11.728517+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"621'12110","reported_seq":12165,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:26.201311+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:21:26.201311+0000","last_peered":"2023-09-23T18:21:26.201311+0000","last_clean":"2023-09-23T18:21:26.201311+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:21:26.201311+0000","last_undegraded":"2023-09-23T18:21:26.201311+0000","last_fullsized":"2023-09-23T18:21:26.201311+0000","mapping_epoch":574,"log_start":"606'6600","ondisk_log_start":"606'6600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5510,"log_dups_size":0,"ondisk_log_size":5510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":31195136,"num_objects":476,"num_object_clones":0,"num_object_copies":1428,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":476,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12110,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":829,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723718+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:21:11.723718+0000","last_peered":"2023-09-23T18:21:11.723718+0000","last_clean":"2023-09-23T18:21:11.723718+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:21:11.723718+0000","last_undegraded":"2023-09-23T18:21:11.723718+0000","last_fullsized":"2023-09-23T18:21:11.723718+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":64,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.736074+0000","last_change":"2023-09-23T18:20:35.016554+0000","last_active":"2023-09-23T18:21:11.736074+0000","last_peered":"2023-09-23T18:21:11.736074+0000","last_clean":"2023-09-23T18:21:11.736074+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:21:11.736074+0000","last_undegraded":"2023-09-23T18:21:11.736074+0000","last_fullsized":"2023-09-23T18:21:11.736074+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":414,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.727595+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:21:11.727595+0000","last_peered":"2023-09-23T18:21:11.727595+0000","last_clean":"2023-09-23T18:21:11.727595+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:21:11.727595+0000","last_undegraded":"2023-09-23T18:21:11.727595+0000","last_fullsized":"2023-09-23T18:21:11.727595+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":157,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724668+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:21:11.724668+0000","last_peered":"2023-09-23T18:21:11.724668+0000","last_clean":"2023-09-23T18:21:11.724668+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:21:11.724668+0000","last_undegraded":"2023-09-23T18:21:11.724668+0000","last_fullsized":"2023-09-23T18:21:11.724668+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"621'12446","reported_seq":12501,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:26.314926+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:21:26.314926+0000","last_peered":"2023-09-23T18:21:26.314926+0000","last_clean":"2023-09-23T18:21:26.314926+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:21:26.314926+0000","last_undegraded":"2023-09-23T18:21:26.314926+0000","last_fullsized":"2023-09-23T18:21:26.314926+0000","mapping_epoch":574,"log_start":"609'8000","ondisk_log_start":"609'8000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4446,"log_dups_size":0,"ondisk_log_size":4446,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32899072,"num_objects":502,"num_object_clones":0,"num_object_copies":1506,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":502,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12446,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1518,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723379+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:21:11.723379+0000","last_peered":"2023-09-23T18:21:11.723379+0000","last_clean":"2023-09-23T18:21:11.723379+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:21:11.723379+0000","last_undegraded":"2023-09-23T18:21:11.723379+0000","last_fullsized":"2023-09-23T18:21:11.723379+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":571,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.727521+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:21:11.727521+0000","last_peered":"2023-09-23T18:21:11.727521+0000","last_clean":"2023-09-23T18:21:11.727521+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:21:11.727521+0000","last_undegraded":"2023-09-23T18:21:11.727521+0000","last_fullsized":"2023-09-23T18:21:11.727521+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":131,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723508+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:21:11.723508+0000","last_peered":"2023-09-23T18:21:11.723508+0000","last_clean":"2023-09-23T18:21:11.723508+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:21:11.723508+0000","last_undegraded":"2023-09-23T18:21:11.723508+0000","last_fullsized":"2023-09-23T18:21:11.723508+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":166,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729198+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:21:11.729198+0000","last_peered":"2023-09-23T18:21:11.729198+0000","last_clean":"2023-09-23T18:21:11.729198+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:21:11.729198+0000","last_undegraded":"2023-09-23T18:21:11.729198+0000","last_fullsized":"2023-09-23T18:21:11.729198+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"621'12264","reported_seq":12319,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:26.201123+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:21:26.201123+0000","last_peered":"2023-09-23T18:21:26.201123+0000","last_clean":"2023-09-23T18:21:26.201123+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:21:26.201123+0000","last_undegraded":"2023-09-23T18:21:26.201123+0000","last_fullsized":"2023-09-23T18:21:26.201123+0000","mapping_epoch":574,"log_start":"606'6800","ondisk_log_start":"606'6800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5464,"log_dups_size":0,"ondisk_log_size":5464,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34078720,"num_objects":520,"num_object_clones":0,"num_object_copies":1560,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":520,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12264,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":819,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728435+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:21:11.728435+0000","last_peered":"2023-09-23T18:21:11.728435+0000","last_clean":"2023-09-23T18:21:11.728435+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:21:11.728435+0000","last_undegraded":"2023-09-23T18:21:11.728435+0000","last_fullsized":"2023-09-23T18:21:11.728435+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":135,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.733868+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:21:11.733868+0000","last_peered":"2023-09-23T18:21:11.733868+0000","last_clean":"2023-09-23T18:21:11.733868+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:21:11.733868+0000","last_undegraded":"2023-09-23T18:21:11.733868+0000","last_fullsized":"2023-09-23T18:21:11.733868+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":38,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.735995+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:21:11.735995+0000","last_peered":"2023-09-23T18:21:11.735995+0000","last_clean":"2023-09-23T18:21:11.735995+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:21:11.735995+0000","last_undegraded":"2023-09-23T18:21:11.735995+0000","last_fullsized":"2023-09-23T18:21:11.735995+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":636,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.725907+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:21:11.725907+0000","last_peered":"2023-09-23T18:21:11.725907+0000","last_clean":"2023-09-23T18:21:11.725907+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:21:11.725907+0000","last_undegraded":"2023-09-23T18:21:11.725907+0000","last_fullsized":"2023-09-23T18:21:11.725907+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"621'12431","reported_seq":19415,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:26.262754+0000","last_change":"2023-09-23T18:20:36.673759+0000","last_active":"2023-09-23T18:21:26.262754+0000","last_peered":"2023-09-23T18:21:26.262754+0000","last_clean":"2023-09-23T18:21:26.262754+0000","last_became_active":"2023-09-23T18:20:36.651913+0000","last_became_peered":"2023-09-23T18:20:36.651913+0000","last_unstale":"2023-09-23T18:21:26.262754+0000","last_undegraded":"2023-09-23T18:21:26.262754+0000","last_fullsized":"2023-09-23T18:21:26.262754+0000","mapping_epoch":614,"log_start":"609'8000","ondisk_log_start":"609'8000","created":574,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4431,"log_dups_size":0,"ondisk_log_size":4431,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:47.765348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34471960,"num_objects":527,"num_object_clones":0,"num_object_copies":1581,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":527,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12431,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3453,"num_bytes_recovered":209321984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":919,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729195+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:21:11.729195+0000","last_peered":"2023-09-23T18:21:11.729195+0000","last_clean":"2023-09-23T18:21:11.729195+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:21:11.729195+0000","last_undegraded":"2023-09-23T18:21:11.729195+0000","last_fullsized":"2023-09-23T18:21:11.729195+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":60,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.744592+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:21:11.744592+0000","last_peered":"2023-09-23T18:21:11.744592+0000","last_clean":"2023-09-23T18:21:11.744592+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:21:11.744592+0000","last_undegraded":"2023-09-23T18:21:11.744592+0000","last_fullsized":"2023-09-23T18:21:11.744592+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":145,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723494+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:21:11.723494+0000","last_peered":"2023-09-23T18:21:11.723494+0000","last_clean":"2023-09-23T18:21:11.723494+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:21:11.723494+0000","last_undegraded":"2023-09-23T18:21:11.723494+0000","last_fullsized":"2023-09-23T18:21:11.723494+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":145,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723284+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:21:11.723284+0000","last_peered":"2023-09-23T18:21:11.723284+0000","last_clean":"2023-09-23T18:21:11.723284+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:21:11.723284+0000","last_undegraded":"2023-09-23T18:21:11.723284+0000","last_fullsized":"2023-09-23T18:21:11.723284+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"621'12333","reported_seq":12388,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:26.296767+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:21:26.296767+0000","last_peered":"2023-09-23T18:21:26.296767+0000","last_clean":"2023-09-23T18:21:26.296767+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:21:26.296767+0000","last_undegraded":"2023-09-23T18:21:26.296767+0000","last_fullsized":"2023-09-23T18:21:26.296767+0000","mapping_epoch":574,"log_start":"609'7900","ondisk_log_start":"609'7900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4433,"log_dups_size":0,"ondisk_log_size":4433,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34144256,"num_objects":521,"num_object_clones":0,"num_object_copies":1563,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":521,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12333,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":919,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728939+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:21:11.728939+0000","last_peered":"2023-09-23T18:21:11.728939+0000","last_clean":"2023-09-23T18:21:11.728939+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:21:11.728939+0000","last_undegraded":"2023-09-23T18:21:11.728939+0000","last_fullsized":"2023-09-23T18:21:11.728939+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":107,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724445+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:21:11.724445+0000","last_peered":"2023-09-23T18:21:11.724445+0000","last_clean":"2023-09-23T18:21:11.724445+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:21:11.724445+0000","last_undegraded":"2023-09-23T18:21:11.724445+0000","last_fullsized":"2023-09-23T18:21:11.724445+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":400,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729038+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:21:11.729038+0000","last_peered":"2023-09-23T18:21:11.729038+0000","last_clean":"2023-09-23T18:21:11.729038+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:21:11.729038+0000","last_undegraded":"2023-09-23T18:21:11.729038+0000","last_fullsized":"2023-09-23T18:21:11.729038+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":38,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.733656+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:21:11.733656+0000","last_peered":"2023-09-23T18:21:11.733656+0000","last_clean":"2023-09-23T18:21:11.733656+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:21:11.733656+0000","last_undegraded":"2023-09-23T18:21:11.733656+0000","last_fullsized":"2023-09-23T18:21:11.733656+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"621'12321","reported_seq":12376,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:26.269457+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:21:26.269457+0000","last_peered":"2023-09-23T18:21:26.269457+0000","last_clean":"2023-09-23T18:21:26.269457+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:21:26.269457+0000","last_undegraded":"2023-09-23T18:21:26.269457+0000","last_fullsized":"2023-09-23T18:21:26.269457+0000","mapping_epoch":574,"log_start":"609'7900","ondisk_log_start":"609'7900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4421,"log_dups_size":0,"ondisk_log_size":4421,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":32178176,"num_objects":491,"num_object_clones":0,"num_object_copies":1473,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":491,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12321,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":754,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.736848+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:21:11.736848+0000","last_peered":"2023-09-23T18:21:11.736848+0000","last_clean":"2023-09-23T18:21:11.736848+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:21:11.736848+0000","last_undegraded":"2023-09-23T18:21:11.736848+0000","last_fullsized":"2023-09-23T18:21:11.736848+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1057,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729809+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:21:11.729809+0000","last_peered":"2023-09-23T18:21:11.729809+0000","last_clean":"2023-09-23T18:21:11.729809+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:21:11.729809+0000","last_undegraded":"2023-09-23T18:21:11.729809+0000","last_fullsized":"2023-09-23T18:21:11.729809+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":48,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.742393+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:21:11.742393+0000","last_peered":"2023-09-23T18:21:11.742393+0000","last_clean":"2023-09-23T18:21:11.742393+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:21:11.742393+0000","last_undegraded":"2023-09-23T18:21:11.742393+0000","last_fullsized":"2023-09-23T18:21:11.742393+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":411,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723364+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:21:11.723364+0000","last_peered":"2023-09-23T18:21:11.723364+0000","last_clean":"2023-09-23T18:21:11.723364+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:21:11.723364+0000","last_undegraded":"2023-09-23T18:21:11.723364+0000","last_fullsized":"2023-09-23T18:21:11.723364+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":651,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.727808+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:21:11.727808+0000","last_peered":"2023-09-23T18:21:11.727808+0000","last_clean":"2023-09-23T18:21:11.727808+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:21:11.727808+0000","last_undegraded":"2023-09-23T18:21:11.727808+0000","last_fullsized":"2023-09-23T18:21:11.727808+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":44,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.740019+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:21:11.740019+0000","last_peered":"2023-09-23T18:21:11.740019+0000","last_clean":"2023-09-23T18:21:11.740019+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:21:11.740019+0000","last_undegraded":"2023-09-23T18:21:11.740019+0000","last_fullsized":"2023-09-23T18:21:11.740019+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":694,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.722785+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:21:11.722785+0000","last_peered":"2023-09-23T18:21:11.722785+0000","last_clean":"2023-09-23T18:21:11.722785+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:21:11.722785+0000","last_undegraded":"2023-09-23T18:21:11.722785+0000","last_fullsized":"2023-09-23T18:21:11.722785+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":651,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728890+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:21:11.728890+0000","last_peered":"2023-09-23T18:21:11.728890+0000","last_clean":"2023-09-23T18:21:11.728890+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:21:11.728890+0000","last_undegraded":"2023-09-23T18:21:11.728890+0000","last_fullsized":"2023-09-23T18:21:11.728890+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":139,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723582+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:21:11.723582+0000","last_peered":"2023-09-23T18:21:11.723582+0000","last_clean":"2023-09-23T18:21:11.723582+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:21:11.723582+0000","last_undegraded":"2023-09-23T18:21:11.723582+0000","last_fullsized":"2023-09-23T18:21:11.723582+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":255,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.735676+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:21:11.735676+0000","last_peered":"2023-09-23T18:21:11.735676+0000","last_clean":"2023-09-23T18:21:11.735676+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:21:11.735676+0000","last_undegraded":"2023-09-23T18:21:11.735676+0000","last_fullsized":"2023-09-23T18:21:11.735676+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":125,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724212+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:21:11.724212+0000","last_peered":"2023-09-23T18:21:11.724212+0000","last_clean":"2023-09-23T18:21:11.724212+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:21:11.724212+0000","last_undegraded":"2023-09-23T18:21:11.724212+0000","last_fullsized":"2023-09-23T18:21:11.724212+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":238,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.733401+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:21:11.733401+0000","last_peered":"2023-09-23T18:21:11.733401+0000","last_clean":"2023-09-23T18:21:11.733401+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:21:11.733401+0000","last_undegraded":"2023-09-23T18:21:11.733401+0000","last_fullsized":"2023-09-23T18:21:11.733401+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1013,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.728294+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:21:11.728294+0000","last_peered":"2023-09-23T18:21:11.728294+0000","last_clean":"2023-09-23T18:21:11.728294+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:21:11.728294+0000","last_undegraded":"2023-09-23T18:21:11.728294+0000","last_fullsized":"2023-09-23T18:21:11.728294+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":518,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.729117+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:21:11.729117+0000","last_peered":"2023-09-23T18:21:11.729117+0000","last_clean":"2023-09-23T18:21:11.729117+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:21:11.729117+0000","last_undegraded":"2023-09-23T18:21:11.729117+0000","last_fullsized":"2023-09-23T18:21:11.729117+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":155,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723526+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:21:11.723526+0000","last_peered":"2023-09-23T18:21:11.723526+0000","last_clean":"2023-09-23T18:21:11.723526+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:21:11.723526+0000","last_undegraded":"2023-09-23T18:21:11.723526+0000","last_fullsized":"2023-09-23T18:21:11.723526+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":950,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.723269+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:21:11.723269+0000","last_peered":"2023-09-23T18:21:11.723269+0000","last_clean":"2023-09-23T18:21:11.723269+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:21:11.723269+0000","last_undegraded":"2023-09-23T18:21:11.723269+0000","last_fullsized":"2023-09-23T18:21:11.723269+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":143,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.724588+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:21:11.724588+0000","last_peered":"2023-09-23T18:21:11.724588+0000","last_clean":"2023-09-23T18:21:11.724588+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:21:11.724588+0000","last_undegraded":"2023-09-23T18:21:11.724588+0000","last_fullsized":"2023-09-23T18:21:11.724588+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":664,"reported_epoch":620,"state":"active+clean","last_fresh":"2023-09-23T18:21:11.727462+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:21:11.727462+0000","last_peered":"2023-09-23T18:21:11.727462+0000","last_clean":"2023-09-23T18:21:11.727462+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:21:11.727462+0000","last_undegraded":"2023-09-23T18:21:11.727462+0000","last_fullsized":"2023-09-23T18:21:11.727462+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":546373656,"num_objects":8338,"num_object_clones":0,"num_object_copies":25014,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13855,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":8338,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":198470,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9287,"num_bytes_recovered":482803712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":3467784192,"data_stored":3467771976,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":76970,"ondisk_log_size":76970,"up":48,"acting":44,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20983808,"data_stored":20971577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001585,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669522,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1158488,"kb_used_data":528592,"kb_used_omap":7,"kb_used_meta":629880,"kb_avail":92586664,"statfs":{"total":95995035648,"available":94808743936,"internally_reserved":0,"allocated":541278208,"data_stored":539307777,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":644997924},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0169999999999999,"5min":2.194,"15min":1.5620000000000001},"min":{"1min":0.255,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":12.207000000000001,"5min":24.472999999999999,"15min":24.472999999999999},"last":0.96999999999999997},{"interface":"front","average":{"1min":2.375,"5min":2.4220000000000002,"15min":1.833},"min":{"1min":0.255,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.255000000000001,"5min":22.832999999999998,"15min":22.832999999999998},"last":0.377}]},{"osd":1,"last update":"Sat Sep 23 18:20:25 2023","interfaces":[{"interface":"back","average":{"1min":2.3570000000000002,"5min":2.0350000000000001,"15min":1.4890000000000001},"min":{"1min":0.29099999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":11.726000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":5.7679999999999998},{"interface":"front","average":{"1min":2.423,"5min":1.899,"15min":1.4379999999999999},"min":{"1min":0.29899999999999999,"5min":0.29899999999999999,"15min":0.29899999999999999},"max":{"1min":12.336,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.579,"15min":1.345},"min":{"1min":0.33500000000000002,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":5.7919999999999998,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.94399999999999995},{"interface":"front","average":{"1min":2.3029999999999999,"5min":2.0600000000000001,"15min":1.5580000000000001},"min":{"1min":0.39700000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0730000000000004,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.42399999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":3.0419999999999998,"15min":2.1150000000000002},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.24099999999999999},"max":{"1min":5.8170000000000002,"5min":103.348,"15min":103.348},"last":0.99399999999999999},{"interface":"front","average":{"1min":2.5600000000000001,"5min":3.6190000000000002,"15min":2.4209999999999998},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":7.4530000000000003,"5min":103.40300000000001,"15min":103.40300000000001},"last":3.3250000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:21:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.7390000000000001,"15min":1.341},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.0300000000000002,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.33000000000000002},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.661,"15min":1.389},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.27000000000000002},"max":{"1min":5.7649999999999997,"5min":7.8319999999999999,"15min":9.2260000000000009},"last":0.91200000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":2.1099999999999999,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":18.417000000000002,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.29999999999999999},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.3399999999999999,"15min":2.3780000000000001},"min":{"1min":0.36299999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":18.454999999999998,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.35399999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":2.1059999999999999,"15min":1.5580000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.26800000000000002},"max":{"1min":6.1500000000000004,"5min":10.743,"15min":13.460000000000001},"last":6.1799999999999997},{"interface":"front","average":{"1min":2.0830000000000002,"5min":2.0529999999999999,"15min":1.603},"min":{"1min":0.40300000000000002,"5min":0.38500000000000001,"15min":0.28499999999999998},"max":{"1min":8.6890000000000001,"5min":11.244,"15min":12.404},"last":1.1439999999999999}]}]},{"osd":1,"up_from":545,"seq":2340757176370,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1054152,"kb_used_data":810176,"kb_used_omap":13,"kb_used_meta":243826,"kb_avail":92691000,"statfs":{"total":95995035648,"available":94915584000,"internally_reserved":0,"allocated":829620224,"data_stored":826561352,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":249678477},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.1339999999999999,"5min":1.016,"15min":0.59899999999999998},"min":{"1min":0.19900000000000001,"5min":0.189,"15min":0.189},"max":{"1min":9.0299999999999994,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.24299999999999999},{"interface":"front","average":{"1min":2.3109999999999999,"5min":1.004,"15min":0.66100000000000003},"min":{"1min":0.34499999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":14.952999999999999,"5min":14.952999999999999,"15min":14.952999999999999},"last":4.9320000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.5579999999999998,"5min":1.0980000000000001,"15min":0.67300000000000004},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":9.6219999999999999,"5min":9.6219999999999999,"15min":9.6219999999999999},"last":6.9710000000000001},{"interface":"front","average":{"1min":1.0740000000000001,"5min":0.77200000000000002,"15min":0.59399999999999997},"min":{"1min":0.28599999999999998,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":4.2080000000000002,"5min":4.2759999999999998,"15min":4.2759999999999998},"last":0.27500000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.861,"5min":1.0409999999999999,"15min":0.60499999999999998},"min":{"1min":0.26700000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":6.3200000000000003,"5min":9.3219999999999992,"15min":9.3219999999999992},"last":0.51400000000000001},{"interface":"front","average":{"1min":2.6080000000000001,"5min":1.2769999999999999,"15min":0.76700000000000002},"min":{"1min":0.33300000000000002,"5min":0.33300000000000002,"15min":0.33300000000000002},"max":{"1min":19.762,"5min":19.762,"15min":19.762},"last":0.54800000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.224,"5min":1.3280000000000001,"15min":1.3200000000000001},"min":{"1min":0.311,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.8690000000000002,"5min":12.663,"15min":12.663},"last":0.56599999999999995},{"interface":"front","average":{"1min":2.4049999999999998,"5min":1.7789999999999999,"15min":1.6020000000000001},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.317},"max":{"1min":6.0289999999999999,"5min":13.222,"15min":13.222},"last":7.0830000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.3359999999999999,"5min":1.405,"15min":0.94999999999999996},"min":{"1min":0.32200000000000001,"5min":0.308,"15min":0.308},"max":{"1min":23.190999999999999,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.998},{"interface":"front","average":{"1min":4.1719999999999997,"5min":1.7330000000000001,"15min":1.113},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":23.204000000000001,"5min":23.204000000000001,"15min":23.204000000000001},"last":1.022}]},{"osd":6,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":3.121,"5min":1.4690000000000001,"15min":0.996},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":23.443000000000001,"5min":23.443000000000001,"15min":23.443000000000001},"last":2.0070000000000001},{"interface":"front","average":{"1min":3.7970000000000002,"5min":1.746,"15min":1.099},"min":{"1min":0.42199999999999999,"5min":0.378,"15min":0.378},"max":{"1min":26.177,"5min":26.177,"15min":26.177},"last":1.0089999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":2.4340000000000002,"5min":1.3620000000000001,"15min":0.90800000000000003},"min":{"1min":0.42299999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":7.1050000000000004,"5min":8.2829999999999995,"15min":8.2829999999999995},"last":1.1419999999999999},{"interface":"front","average":{"1min":2.3140000000000001,"5min":1.474,"15min":1.028},"min":{"1min":0.42399999999999999,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":9.1500000000000004,"5min":9.1500000000000004,"15min":9.1500000000000004},"last":7.0140000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958867,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1586976,"kb_used_data":637272,"kb_used_omap":7,"kb_used_meta":949432,"kb_avail":92158176,"statfs":{"total":95995035648,"available":94369972224,"internally_reserved":0,"allocated":652566528,"data_stored":649663499,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":972219108},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,3,7,2],"upper_bound":128},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:20:28 2023","interfaces":[{"interface":"back","average":{"1min":1.5369999999999999,"5min":1.044,"15min":0.56100000000000005},"min":{"1min":0.21299999999999999,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.4779999999999998,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":1.8779999999999999},{"interface":"front","average":{"1min":2.6240000000000001,"5min":1.278,"15min":0.77400000000000002},"min":{"1min":0.247,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":15.335000000000001,"5min":15.335000000000001,"15min":15.335000000000001},"last":4.3579999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:21:04 2023","interfaces":[{"interface":"back","average":{"1min":3.008,"5min":2.0880000000000001,"15min":1.4419999999999999},"min":{"1min":0.40200000000000002,"5min":0.185,"15min":0.185},"max":{"1min":10.013,"5min":18.550999999999998,"15min":18.550999999999998},"last":3.9590000000000001},{"interface":"front","average":{"1min":3.2160000000000002,"5min":1.7470000000000001,"15min":1.097},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":23.161999999999999,"5min":23.161999999999999,"15min":23.161999999999999},"last":4.2599999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.4199999999999999,"15min":1.5940000000000001},"min":{"1min":0.313,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.8460000000000001,"5min":13.539999999999999,"15min":13.539999999999999},"last":7.75},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.6619999999999999,"15min":1.7010000000000001},"min":{"1min":0.36299999999999999,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":6.8879999999999999,"5min":13.285,"15min":15.198},"last":5.0800000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:21:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":2.1019999999999999,"15min":1.982},"min":{"1min":0.39900000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.1379999999999999,"5min":17.741,"15min":57.746000000000002},"last":1.9159999999999999},{"interface":"front","average":{"1min":1.966,"5min":1.9099999999999999,"15min":1.792},"min":{"1min":0.30099999999999999,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":4.9089999999999998,"5min":15.079000000000001,"15min":57.816000000000003},"last":1.9650000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":1.8220000000000001,"15min":1.4410000000000001},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":5.5970000000000004,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":1.9450000000000001},{"interface":"front","average":{"1min":3.2909999999999999,"5min":2.4630000000000001,"15min":1.599},"min":{"1min":0.40100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":15.047000000000001,"5min":15.047000000000001,"15min":15.047000000000001},"last":2.0219999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:21:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":1.9379999999999999,"15min":2.0299999999999998},"min":{"1min":0.40300000000000002,"5min":0.377,"15min":0.32000000000000001},"max":{"1min":5.4610000000000003,"5min":7.6900000000000004,"15min":58.457999999999998},"last":2.0430000000000001},{"interface":"front","average":{"1min":3.7629999999999999,"5min":2.657,"15min":2.1190000000000002},"min":{"1min":0.54100000000000004,"5min":0.373,"15min":0.315},"max":{"1min":22.893000000000001,"5min":22.893000000000001,"15min":58.354999999999997},"last":4.29}]},{"osd":7,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":3.431,"5min":2.2480000000000002,"15min":1.7889999999999999},"min":{"1min":0.49199999999999999,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":22.945,"5min":22.945,"15min":22.945},"last":4.3200000000000003},{"interface":"front","average":{"1min":2.2330000000000001,"5min":1.714,"15min":1.851},"min":{"1min":0.47099999999999997,"5min":0.371,"15min":0.36199999999999999},"max":{"1min":6.9249999999999998,"5min":7.7519999999999998,"15min":10.404},"last":1.9930000000000001}]}]},{"osd":2,"up_from":0,"seq":2070174236752,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249382,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1038780,"kb_used_data":241204,"kb_used_omap":3,"kb_used_meta":797308,"kb_avail":92706372,"statfs":{"total":95995035648,"available":94931324928,"internally_reserved":0,"allocated":246992896,"data_stored":243887705,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":816444136},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":1.5800000000000001,"15min":1.8089999999999999},"min":{"1min":0.20799999999999999,"5min":0.192,"15min":0.183},"max":{"1min":12.472,"5min":17.036000000000001,"15min":195.69800000000001},"last":3.589},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.405,"15min":2.004},"min":{"1min":0.23799999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.468,"5min":11.468,"15min":194.661},"last":4.3460000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:20:26 2023","interfaces":[{"interface":"back","average":{"1min":3.028,"5min":1.4590000000000001,"15min":0.83299999999999996},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":12.079000000000001,"5min":21.777000000000001,"15min":21.777000000000001},"last":1.1060000000000001},{"interface":"front","average":{"1min":3.9390000000000001,"5min":2.5129999999999999,"15min":1.3069999999999999},"min":{"1min":0.28599999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":17.081,"5min":78.980000000000004,"15min":78.980000000000004},"last":3.637}]},{"osd":2,"last update":"Sat Sep 23 18:21:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5569999999999999,"5min":1.526,"15min":1.3740000000000001},"min":{"1min":0.29899999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.2430000000000003,"5min":12.151999999999999,"15min":12.151999999999999},"last":3.2970000000000002},{"interface":"front","average":{"1min":1.78,"5min":1.3839999999999999,"15min":1.361},"min":{"1min":0.32300000000000001,"5min":0.27900000000000003,"15min":0.19600000000000001},"max":{"1min":7.0640000000000001,"5min":11.565,"15min":11.565},"last":3.6819999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":1.8839999999999999,"15min":1.706},"min":{"1min":0.34200000000000003,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":11.4,"5min":21.062999999999999,"15min":37.231000000000002},"last":4.4089999999999998},{"interface":"front","average":{"1min":2.532,"5min":2.1240000000000001,"15min":1.885},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":9.1630000000000003,"5min":21.128,"15min":37.381},"last":0.34399999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:20:37 2023","interfaces":[{"interface":"back","average":{"1min":3.452,"5min":2.048,"15min":1.587},"min":{"1min":0.35099999999999998,"5min":0.316,"15min":0.316},"max":{"1min":9.7430000000000003,"5min":11.805999999999999,"15min":11.805999999999999},"last":2.0819999999999999},{"interface":"front","average":{"1min":3.0339999999999998,"5min":1.74,"15min":1.722},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":17.135999999999999,"5min":17.135999999999999,"15min":17.135999999999999},"last":3.7160000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.9390000000000001,"15min":1.9379999999999999},"min":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":10.303000000000001,"5min":17.192,"15min":115.54000000000001},"last":3.7469999999999999},{"interface":"front","average":{"1min":2.9889999999999999,"5min":1.976,"15min":1.9810000000000001},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.30499999999999999},"max":{"1min":11.442,"5min":14.321,"15min":115.691},"last":4.3840000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0939999999999999,"5min":1.744,"15min":1.7609999999999999},"min":{"1min":0.42699999999999999,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":15.972},"last":0.39500000000000002},{"interface":"front","average":{"1min":1.974,"5min":1.829,"15min":1.6240000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":6.9749999999999996,"5min":17.164000000000001,"15min":17.164000000000001},"last":3.806}]}]},{"osd":4,"up_from":30,"seq":128849019331,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1505432,"kb_used_data":820752,"kb_used_omap":0,"kb_used_meta":684672,"kb_avail":92239720,"statfs":{"total":95995035648,"available":94453473280,"internally_reserved":0,"allocated":840450048,"data_stored":838479409,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":701104128},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5449999999999999,"5min":1.7589999999999999,"15min":1.387},"min":{"1min":0.443,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.0960000000000001,"5min":7.6079999999999997,"15min":13.375999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.919,"15min":1.6379999999999999},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":7.6440000000000001,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.23899999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:20:22 2023","interfaces":[{"interface":"back","average":{"1min":1.929,"5min":1.7170000000000001,"15min":1.0349999999999999},"min":{"1min":0.27700000000000002,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":6.8220000000000001,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.255},{"interface":"front","average":{"1min":1.8420000000000001,"5min":1.6679999999999999,"15min":0.98999999999999999},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":7.5700000000000003,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.28399999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3839999999999999,"5min":1.768,"15min":1.8500000000000001},"min":{"1min":0.34300000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.1749999999999998,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.34399999999999997},{"interface":"front","average":{"1min":2.8079999999999998,"5min":1.8360000000000001,"15min":1.7649999999999999},"min":{"1min":0.36199999999999999,"5min":0.29899999999999999,"15min":0.253},"max":{"1min":17.190999999999999,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.33200000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.395,"5min":1.5529999999999999,"15min":1.9630000000000001},"min":{"1min":0.379,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.7519999999999998,"5min":11.130000000000001,"15min":29.154},"last":0.32000000000000001},{"interface":"front","average":{"1min":2.6230000000000002,"5min":1.8300000000000001,"15min":1.9930000000000001},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.099,"5min":13.470000000000001,"15min":29.216000000000001},"last":0.307}]},{"osd":5,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.3170000000000002,"15min":2.7829999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.5009999999999994,"5min":27.675000000000001,"15min":27.675000000000001},"last":91.432000000000002},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.6499999999999999,"15min":2.5609999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":3.8359999999999999,"5min":13.938000000000001,"15min":32.640000000000001},"last":91.626999999999995}]},{"osd":6,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.8819999999999999,"15min":1.4099999999999999},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":3.9750000000000001,"5min":18.315000000000001,"15min":18.315000000000001},"last":91.688999999999993},{"interface":"front","average":{"1min":2.1890000000000001,"5min":2.0499999999999998,"15min":1.619},"min":{"1min":0.311,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.4969999999999999,"5min":18.431000000000001,"15min":18.431000000000001},"last":91.837000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.7770000000000001,"5min":2.5430000000000001,"15min":1.8360000000000001},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":18.283000000000001,"5min":19.298999999999999,"15min":19.298999999999999},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.5760000000000001,"5min":1.7869999999999999,"15min":1.5409999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.28399999999999997},"max":{"1min":3.786,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.36399999999999999}]}]},{"osd":5,"up_from":420,"seq":1803886264428,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":850760,"kb_used_data":488576,"kb_used_omap":9,"kb_used_meta":362102,"kb_avail":92894392,"statfs":{"total":95995035648,"available":95123857408,"internally_reserved":0,"allocated":500301824,"data_stored":498343529,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":370792893},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[2],"upper_bound":2},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.2290000000000001,"15min":0.93200000000000005},"min":{"1min":0.21299999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.867,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":4.3449999999999998},{"interface":"front","average":{"1min":2.137,"5min":1.8,"15min":1.6459999999999999},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.621,"5min":20.937999999999999,"15min":20.937999999999999},"last":1.7010000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:21:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":1.9570000000000001,"15min":1.383},"min":{"1min":0.28999999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.752000000000001,"5min":23.946000000000002,"15min":23.946000000000002},"last":4.3879999999999999},{"interface":"front","average":{"1min":2.1640000000000001,"5min":1.9930000000000001,"15min":1.2290000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.4969999999999999,"5min":23.268999999999998,"15min":23.268999999999998},"last":5.1239999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.6160000000000001,"15min":1.611},"min":{"1min":0.29699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.1150000000000002,"5min":10.705,"15min":10.705},"last":8.6039999999999992},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.8520000000000001,"15min":2.1280000000000001},"min":{"1min":0.39600000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.7560000000000002,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":1.7310000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.762,"15min":1.359},"min":{"1min":0.35599999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.0060000000000002,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":5.1600000000000001},{"interface":"front","average":{"1min":2.2200000000000002,"5min":2.1150000000000002,"15min":1.7130000000000001},"min":{"1min":0.372,"5min":0.33500000000000002,"15min":0.313},"max":{"1min":10.664999999999999,"5min":20.035,"15min":20.035},"last":5.2220000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":2.1480000000000001,"15min":2.0499999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.2270000000000003,"5min":12.747999999999999,"15min":12.747999999999999},"last":5.7210000000000001},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1179999999999999,"15min":1.6639999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7919999999999998,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":8.5559999999999992}]},{"osd":6,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.8340000000000001,"15min":1.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.2189999999999999,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":5.444},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.3319999999999999,"15min":1.7370000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.6920000000000002,"5min":12.504,"15min":12.504},"last":5.0890000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.23,"15min":1.804},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.195,"5min":12.641,"15min":13.506},"last":4.5110000000000001},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.8500000000000001,"15min":1.5600000000000001},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":5.0839999999999996,"5min":13.138,"15min":13.138},"last":5.3499999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":635441152,"data_stored":635437080,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":808845312,"data_stored":808845312,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":221315072,"data_stored":221315072,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":806621184,"data_stored":806617112,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":484704256,"data_stored":484704256,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":510857216,"data_stored":510853144,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:21:30.366 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:30.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:30.371 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:30.391 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:30.472 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:30.535 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:30.555 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:30.603 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:30.607 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:30.608 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:30.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:21:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:21:30] "GET /metrics HTTP/1.1" 200 36661 "" "Prometheus/2.43.0" 2023-09-23T18:21:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:30 smithi055 ceph-mon[184174]: pgmap v1506: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 494 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 13868/23601 objects degraded (58.760%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:30 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:30 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:21:30.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:30 smithi055 ceph-mon[184174]: osdmap e622: 8 total, 8 up, 6 in 2023-09-23T18:21:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:30 smithi055 ceph-mon[179668]: pgmap v1506: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 494 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 13868/23601 objects degraded (58.760%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:30 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:30 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:21:30.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:30 smithi055 ceph-mon[179668]: osdmap e622: 8 total, 8 up, 6 in 2023-09-23T18:21:30.708 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:30.754 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:30 smithi099 ceph-mon[167076]: pgmap v1506: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 494 MiB data, 6.6 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 769 op/s; 13868/23601 objects degraded (58.760%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:30.755 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:30 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:30.755 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:30 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:21:30.755 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:30 smithi099 ceph-mon[167076]: osdmap e622: 8 total, 8 up, 6 in 2023-09-23T18:21:30.759 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:30.814 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:30.822 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:30.823 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:30.923 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:30.965 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:31.033 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:31.045 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:31.045 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:31.146 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:31.237 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:31.321 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:31.327 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:31.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:31.428 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:31.535 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:31.795 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:31 smithi055 ceph-mon[179668]: failed to encode map e622 with expected crc 2023-09-23T18:21:31.795 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:31 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:31.795 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:31 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:31.795 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:31 smithi055 ceph-mon[184174]: failed to encode map e622 with expected crc 2023-09-23T18:21:31.795 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:31 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:31.795 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:31 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:31.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:31 smithi099 ceph-mon[167076]: failed to encode map e622 with expected crc 2023-09-23T18:21:31.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:31 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:31.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:31 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:31.842 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:31.848 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:31.849 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:31.950 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:32.053 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:32.171 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:32.176 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:32.177 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:32.277 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:32.394 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:32.468 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:21:32.573 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:32.581 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:32.582 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:32.582 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:32 smithi055 ceph-mon[184174]: pgmap v1508: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 465 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 13868/22194 objects degraded (62.485%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:32.582 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:32 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:32.582 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:32 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:21:32.583 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:32 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:32.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:32 smithi055 ceph-mon[179668]: pgmap v1508: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 465 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 13868/22194 objects degraded (62.485%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:32.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:32 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:32.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:32 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:21:32.583 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:32 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:32.683 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:32.700 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:21:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:21:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:21:32.700 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:21:32.700 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:21:32.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:32 smithi099 ceph-mon[167076]: pgmap v1508: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 465 MiB data, 6.5 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 13868/22194 objects degraded (62.485%); 0 B/s, 50 objects/s recovering 2023-09-23T18:21:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:32 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:21:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:32 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:21:32.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:32 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:32.842 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:33.002 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:21:33.043 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:21:33.044 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:21:33.082 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:21:33.126 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:21:33.126 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:33.150 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:33.151 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:33.172 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:21:33.235 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:21:33.247 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.0 dump_historic_ops 2023-09-23T18:21:33.252 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:33.280 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:21:33.370 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:21:33.399 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:33.401 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:21:33.401 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:21:33.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:21:33.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:21:33.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:21:33.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:21:33.402 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:21:33.403 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:21:33.493 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:33.582 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:33.600 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:33.600 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3086349925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:33.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:33 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 13868/22194 objects degraded (62.485%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:33.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3086349925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:33.682 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:33 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 13868/22194 objects degraded (62.485%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:33.701 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:33.802 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:33.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:33 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3086349925' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:33.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:33 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 13868/22194 objects degraded (62.485%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:33.937 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:33.945 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:33.945 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:34.046 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:34.067 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:34.150 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:34.292 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:34.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:34.300 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:34.400 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:34.446 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:34.484 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:34.501 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:34.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:34.507 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:34.607 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:34.663 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:34 smithi055 ceph-mon[184174]: pgmap v1509: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 267 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 13600/12690 objects degraded (107.171%); 0 B/s, 53 objects/s recovering 2023-09-23T18:21:34.663 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:34 smithi055 ceph-mon[179668]: pgmap v1509: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 267 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 13600/12690 objects degraded (107.171%); 0 B/s, 53 objects/s recovering 2023-09-23T18:21:34.666 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:34.784 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:34.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:34.795 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:34.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:34 smithi099 ceph-mon[167076]: pgmap v1509: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 267 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 773 op/s; 13600/12690 objects degraded (107.171%); 0 B/s, 53 objects/s recovering 2023-09-23T18:21:34.896 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:34.958 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:35.112 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:35.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:35.122 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:35.223 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:35.344 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:35.408 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:35.420 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:35.421 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:35.522 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:35.588 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:35.875 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:21:36.079 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:36.091 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:36.091 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:36.192 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:36.226 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:36.289 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:36.292 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:36.293 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:36.393 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:36.532 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:36.559 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Removed 103403 objects 2023-09-23T18:21:36.559 INFO:tasks.radosbench.radosbench.0.smithi055.stdout:Clean up completed and total clean up time :124.56 2023-09-23T18:21:36.590 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:36.595 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:36.595 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:36.696 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:36.726 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:36.761 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:21:36.763 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:21:36.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:36 smithi099 ceph-mon[167076]: pgmap v1510: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 253 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 13600/12030 objects degraded (113.051%); 0 B/s, 53 objects/s recovering 2023-09-23T18:21:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:36.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3221076945' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:36.834 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:36.841 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:36 smithi055 ceph-mon[179668]: pgmap v1510: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 253 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 13600/12030 objects degraded (113.051%); 0 B/s, 53 objects/s recovering 2023-09-23T18:21:36.841 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:36.841 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:36.841 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3221076945' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:36.842 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:36 smithi055 ceph-mon[184174]: pgmap v1510: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 253 MiB data, 6.0 GiB used, 530 GiB / 536 GiB avail; 0 B/s wr, 771 op/s; 13600/12030 objects degraded (113.051%); 0 B/s, 53 objects/s recovering 2023-09-23T18:21:36.842 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:21:36.842 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:36.842 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3221076945' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:36.843 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:36.843 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:36.944 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:36.973 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:37.457 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:21:37.562 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:37.565 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:37.565 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:37.666 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:37.691 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:21:37.691 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:21:37.692 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:37.734 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:37.737 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:37.738 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:37.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:37 smithi099 ceph-mon[167076]: from='client.56752 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:37 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:37 smithi099 ceph-mon[167076]: failed to encode map e623 with expected crc 2023-09-23T18:21:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:37 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3221076945' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:37.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:37 smithi099 ceph-mon[167076]: osdmap e623: 8 total, 8 up, 6 in 2023-09-23T18:21:37.839 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:37.866 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:37.900 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:37.903 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:37.903 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:37.928 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:37 smithi055 ceph-mon[184174]: from='client.56752 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:37.928 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:37 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:37.928 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:37 smithi055 ceph-mon[184174]: failed to encode map e623 with expected crc 2023-09-23T18:21:37.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:37 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3221076945' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:37.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:37 smithi055 ceph-mon[184174]: osdmap e623: 8 total, 8 up, 6 in 2023-09-23T18:21:37.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:37 smithi055 ceph-mon[179668]: from='client.56752 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:37.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:37 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:37.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:37 smithi055 ceph-mon[179668]: failed to encode map e623 with expected crc 2023-09-23T18:21:37.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:37 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3221076945' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:37.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:37 smithi055 ceph-mon[179668]: osdmap e623: 8 total, 8 up, 6 in 2023-09-23T18:21:38.004 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:38.035 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:38.135 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:38.140 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:38.140 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:38.236 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1510,"stamp":"2023-09-23T18:21:35.217427+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":265435836,"num_objects":4010,"num_object_clones":0,"num_object_copies":12030,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13600,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4010,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":203736,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9953,"num_bytes_recovered":502226569,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83347,"ondisk_log_size":83347,"up":357,"acting":352,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":378,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":6251308,"kb_used_data":2540028,"kb_used_omap":40,"kb_used_meta":3710935,"kb_avail":556219604,"statfs":{"total":575970213888,"available":569568874496,"internally_reserved":0,"allocated":2600988672,"data_stored":2585990671,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3799997754},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,2,4,4,10],"upper_bound":64},"perf_stat":{"commit_latency_ms":8,"apply_latency_ms":8,"commit_latency_ns":8000000,"apply_latency_ns":8000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-505806848,"num_objects":-7718,"num_object_clones":0,"num_object_copies":-23154,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-531,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-7718,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":7718,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":531,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"10.007162"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":910,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913684+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:21:20.913684+0000","last_peered":"2023-09-23T18:21:20.913684+0000","last_clean":"2023-09-23T18:21:20.913684+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:21:20.913684+0000","last_undegraded":"2023-09-23T18:21:20.913684+0000","last_fullsized":"2023-09-23T18:21:20.913684+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":281,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.934270+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:21:20.934270+0000","last_peered":"2023-09-23T18:21:20.934270+0000","last_clean":"2023-09-23T18:21:20.934270+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:21:20.934270+0000","last_undegraded":"2023-09-23T18:21:20.934270+0000","last_fullsized":"2023-09-23T18:21:20.934270+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":39,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.921465+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:21:20.921465+0000","last_peered":"2023-09-23T18:21:20.921465+0000","last_clean":"2023-09-23T18:21:20.921465+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:21:20.921465+0000","last_undegraded":"2023-09-23T18:21:20.921465+0000","last_fullsized":"2023-09-23T18:21:20.921465+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":916,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914168+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:21:20.914168+0000","last_peered":"2023-09-23T18:21:20.914168+0000","last_clean":"2023-09-23T18:21:20.914168+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:21:20.914168+0000","last_undegraded":"2023-09-23T18:21:20.914168+0000","last_fullsized":"2023-09-23T18:21:20.914168+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":363,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.921052+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:21:20.921052+0000","last_peered":"2023-09-23T18:21:20.921052+0000","last_clean":"2023-09-23T18:21:20.921052+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:21:20.921052+0000","last_undegraded":"2023-09-23T18:21:20.921052+0000","last_fullsized":"2023-09-23T18:21:20.921052+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":683,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.924929+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:21:20.924929+0000","last_peered":"2023-09-23T18:21:20.924929+0000","last_clean":"2023-09-23T18:21:20.924929+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:21:20.924929+0000","last_undegraded":"2023-09-23T18:21:20.924929+0000","last_fullsized":"2023-09-23T18:21:20.924929+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":933,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913678+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:21:20.913678+0000","last_peered":"2023-09-23T18:21:20.913678+0000","last_clean":"2023-09-23T18:21:20.913678+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:21:20.913678+0000","last_undegraded":"2023-09-23T18:21:20.913678+0000","last_fullsized":"2023-09-23T18:21:20.913678+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":39,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.920014+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:21:20.920014+0000","last_peered":"2023-09-23T18:21:20.920014+0000","last_clean":"2023-09-23T18:21:20.920014+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:21:20.920014+0000","last_undegraded":"2023-09-23T18:21:20.920014+0000","last_fullsized":"2023-09-23T18:21:20.920014+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":198,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.932657+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:21:20.932657+0000","last_peered":"2023-09-23T18:21:20.932657+0000","last_clean":"2023-09-23T18:21:20.932657+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:21:20.932657+0000","last_undegraded":"2023-09-23T18:21:20.932657+0000","last_fullsized":"2023-09-23T18:21:20.932657+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1056,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914455+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:21:20.914455+0000","last_peered":"2023-09-23T18:21:20.914455+0000","last_clean":"2023-09-23T18:21:20.914455+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:21:20.914455+0000","last_undegraded":"2023-09-23T18:21:20.914455+0000","last_fullsized":"2023-09-23T18:21:20.914455+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":132,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914898+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:21:20.914898+0000","last_peered":"2023-09-23T18:21:20.914898+0000","last_clean":"2023-09-23T18:21:20.914898+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:21:20.914898+0000","last_undegraded":"2023-09-23T18:21:20.914898+0000","last_fullsized":"2023-09-23T18:21:20.914898+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":212,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.920418+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:21:20.920418+0000","last_peered":"2023-09-23T18:21:20.920418+0000","last_clean":"2023-09-23T18:21:20.920418+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:21:20.920418+0000","last_undegraded":"2023-09-23T18:21:20.920418+0000","last_fullsized":"2023-09-23T18:21:20.920418+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":833,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928136+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:21:20.928136+0000","last_peered":"2023-09-23T18:21:20.928136+0000","last_clean":"2023-09-23T18:21:20.928136+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:21:20.928136+0000","last_undegraded":"2023-09-23T18:21:20.928136+0000","last_fullsized":"2023-09-23T18:21:20.928136+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":168,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914575+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:21:20.914575+0000","last_peered":"2023-09-23T18:21:20.914575+0000","last_clean":"2023-09-23T18:21:20.914575+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:21:20.914575+0000","last_undegraded":"2023-09-23T18:21:20.914575+0000","last_fullsized":"2023-09-23T18:21:20.914575+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":167,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928133+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:21:20.928133+0000","last_peered":"2023-09-23T18:21:20.928133+0000","last_clean":"2023-09-23T18:21:20.928133+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:21:20.928133+0000","last_undegraded":"2023-09-23T18:21:20.928133+0000","last_fullsized":"2023-09-23T18:21:20.928133+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":967,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914941+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:21:20.914941+0000","last_peered":"2023-09-23T18:21:20.914941+0000","last_clean":"2023-09-23T18:21:20.914941+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:21:20.914941+0000","last_undegraded":"2023-09-23T18:21:20.914941+0000","last_fullsized":"2023-09-23T18:21:20.914941+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":416,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.921844+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:21:20.921844+0000","last_peered":"2023-09-23T18:21:20.921844+0000","last_clean":"2023-09-23T18:21:20.921844+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:21:20.921844+0000","last_undegraded":"2023-09-23T18:21:20.921844+0000","last_fullsized":"2023-09-23T18:21:20.921844+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":435,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.929975+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:21:20.929975+0000","last_peered":"2023-09-23T18:21:20.929975+0000","last_clean":"2023-09-23T18:21:20.929975+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:21:20.929975+0000","last_undegraded":"2023-09-23T18:21:20.929975+0000","last_fullsized":"2023-09-23T18:21:20.929975+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":996,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.912223+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:21:20.912223+0000","last_peered":"2023-09-23T18:21:20.912223+0000","last_clean":"2023-09-23T18:21:20.912223+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:21:20.912223+0000","last_undegraded":"2023-09-23T18:21:20.912223+0000","last_fullsized":"2023-09-23T18:21:20.912223+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":174,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928668+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:21:20.928668+0000","last_peered":"2023-09-23T18:21:20.928668+0000","last_clean":"2023-09-23T18:21:20.928668+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:21:20.928668+0000","last_undegraded":"2023-09-23T18:21:20.928668+0000","last_fullsized":"2023-09-23T18:21:20.928668+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":441,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.929040+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:21:20.929040+0000","last_peered":"2023-09-23T18:21:20.929040+0000","last_clean":"2023-09-23T18:21:20.929040+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:21:20.929040+0000","last_undegraded":"2023-09-23T18:21:20.929040+0000","last_fullsized":"2023-09-23T18:21:20.929040+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":848,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913849+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:21:20.913849+0000","last_peered":"2023-09-23T18:21:20.913849+0000","last_clean":"2023-09-23T18:21:20.913849+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:21:20.913849+0000","last_undegraded":"2023-09-23T18:21:20.913849+0000","last_fullsized":"2023-09-23T18:21:20.913849+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":441,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928958+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:21:20.928958+0000","last_peered":"2023-09-23T18:21:20.928958+0000","last_clean":"2023-09-23T18:21:20.928958+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:21:20.928958+0000","last_undegraded":"2023-09-23T18:21:20.928958+0000","last_fullsized":"2023-09-23T18:21:20.928958+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":119,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914084+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:21:20.914084+0000","last_peered":"2023-09-23T18:21:20.914084+0000","last_clean":"2023-09-23T18:21:20.914084+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:21:20.914084+0000","last_undegraded":"2023-09-23T18:21:20.914084+0000","last_fullsized":"2023-09-23T18:21:20.914084+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":819,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913481+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:21:20.913481+0000","last_peered":"2023-09-23T18:21:20.913481+0000","last_clean":"2023-09-23T18:21:20.913481+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:21:20.913481+0000","last_undegraded":"2023-09-23T18:21:20.913481+0000","last_fullsized":"2023-09-23T18:21:20.913481+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":138,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.925845+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:21:20.925845+0000","last_peered":"2023-09-23T18:21:20.925845+0000","last_clean":"2023-09-23T18:21:20.925845+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:21:20.925845+0000","last_undegraded":"2023-09-23T18:21:20.925845+0000","last_fullsized":"2023-09-23T18:21:20.925845+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":132,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914611+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:21:20.914611+0000","last_peered":"2023-09-23T18:21:20.914611+0000","last_clean":"2023-09-23T18:21:20.914611+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:21:20.914611+0000","last_undegraded":"2023-09-23T18:21:20.914611+0000","last_fullsized":"2023-09-23T18:21:20.914611+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":758,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.922429+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:21:20.922429+0000","last_peered":"2023-09-23T18:21:20.922429+0000","last_clean":"2023-09-23T18:21:20.922429+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:21:20.922429+0000","last_undegraded":"2023-09-23T18:21:20.922429+0000","last_fullsized":"2023-09-23T18:21:20.922429+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":209,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.927964+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:21:20.927964+0000","last_peered":"2023-09-23T18:21:20.927964+0000","last_clean":"2023-09-23T18:21:20.927964+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:21:20.927964+0000","last_undegraded":"2023-09-23T18:21:20.927964+0000","last_fullsized":"2023-09-23T18:21:20.927964+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":221,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.920843+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:21:20.920843+0000","last_peered":"2023-09-23T18:21:20.920843+0000","last_clean":"2023-09-23T18:21:20.920843+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:21:20.920843+0000","last_undegraded":"2023-09-23T18:21:20.920843+0000","last_fullsized":"2023-09-23T18:21:20.920843+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":853,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928154+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:21:20.928154+0000","last_peered":"2023-09-23T18:21:20.928154+0000","last_clean":"2023-09-23T18:21:20.928154+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:21:20.928154+0000","last_undegraded":"2023-09-23T18:21:20.928154+0000","last_fullsized":"2023-09-23T18:21:20.928154+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":762,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.927875+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:21:20.927875+0000","last_peered":"2023-09-23T18:21:20.927875+0000","last_clean":"2023-09-23T18:21:20.927875+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:21:20.927875+0000","last_undegraded":"2023-09-23T18:21:20.927875+0000","last_fullsized":"2023-09-23T18:21:20.927875+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":39,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.922237+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:21:20.922237+0000","last_peered":"2023-09-23T18:21:20.922237+0000","last_clean":"2023-09-23T18:21:20.922237+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:21:20.922237+0000","last_undegraded":"2023-09-23T18:21:20.922237+0000","last_fullsized":"2023-09-23T18:21:20.922237+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"622'12802","reported_seq":12858,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:31.212449+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:21:31.212449+0000","last_peered":"2023-09-23T18:21:31.212449+0000","last_clean":"2023-09-23T18:21:31.212449+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:21:31.212449+0000","last_undegraded":"2023-09-23T18:21:31.212449+0000","last_fullsized":"2023-09-23T18:21:31.212449+0000","mapping_epoch":574,"log_start":"607'7300","ondisk_log_start":"607'7300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5502,"log_dups_size":0,"ondisk_log_size":5502,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17825792,"num_objects":272,"num_object_clones":0,"num_object_copies":816,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":272,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12802,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":887,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928437+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:21:20.928437+0000","last_peered":"2023-09-23T18:21:20.928437+0000","last_clean":"2023-09-23T18:21:20.928437+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:21:20.928437+0000","last_undegraded":"2023-09-23T18:21:20.928437+0000","last_fullsized":"2023-09-23T18:21:20.928437+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":258,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.934302+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:21:20.934302+0000","last_peered":"2023-09-23T18:21:20.934302+0000","last_clean":"2023-09-23T18:21:20.934302+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:21:20.934302+0000","last_undegraded":"2023-09-23T18:21:20.934302+0000","last_fullsized":"2023-09-23T18:21:20.934302+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":132,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913479+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:21:20.913479+0000","last_peered":"2023-09-23T18:21:20.913479+0000","last_clean":"2023-09-23T18:21:20.913479+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:21:20.913479+0000","last_undegraded":"2023-09-23T18:21:20.913479+0000","last_fullsized":"2023-09-23T18:21:20.913479+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"622'12456","reported_seq":12522,"reported_epoch":622,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:31.296736+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:21:31.296736+0000","last_peered":"2023-09-23T18:21:31.296736+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:21:31.296736+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"609'8000","ondisk_log_start":"609'8000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4456,"log_dups_size":0,"ondisk_log_size":4456,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15466496,"num_objects":236,"num_object_clones":0,"num_object_copies":708,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":236,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12456,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"5,6","objects":-3912}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":742,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.922143+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:21:20.922143+0000","last_peered":"2023-09-23T18:21:20.922143+0000","last_clean":"2023-09-23T18:21:20.922143+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:21:20.922143+0000","last_undegraded":"2023-09-23T18:21:20.922143+0000","last_fullsized":"2023-09-23T18:21:20.922143+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":259,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.934684+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:21:20.934684+0000","last_peered":"2023-09-23T18:21:20.934684+0000","last_clean":"2023-09-23T18:21:20.934684+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:21:20.934684+0000","last_undegraded":"2023-09-23T18:21:20.934684+0000","last_fullsized":"2023-09-23T18:21:20.934684+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":238,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.934223+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:21:20.934223+0000","last_peered":"2023-09-23T18:21:20.934223+0000","last_clean":"2023-09-23T18:21:20.934223+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:21:20.934223+0000","last_undegraded":"2023-09-23T18:21:20.934223+0000","last_fullsized":"2023-09-23T18:21:20.934223+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"622'12516","reported_seq":19532,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:30.316545+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:21:30.316545+0000","last_peered":"2023-09-23T18:21:30.316545+0000","last_clean":"2023-09-23T18:21:30.316545+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:21:30.316545+0000","last_undegraded":"2023-09-23T18:21:30.316545+0000","last_fullsized":"2023-09-23T18:21:30.316545+0000","mapping_epoch":605,"log_start":"608'7600","ondisk_log_start":"608'7600","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4916,"log_dups_size":0,"ondisk_log_size":4916,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":21495808,"num_objects":328,"num_object_clones":0,"num_object_copies":984,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":328,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12516,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":637,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.932584+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:21:20.932584+0000","last_peered":"2023-09-23T18:21:20.932584+0000","last_clean":"2023-09-23T18:21:20.932584+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:21:20.932584+0000","last_undegraded":"2023-09-23T18:21:20.932584+0000","last_fullsized":"2023-09-23T18:21:20.932584+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":547,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914496+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:21:20.914496+0000","last_peered":"2023-09-23T18:21:20.914496+0000","last_clean":"2023-09-23T18:21:20.914496+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:21:20.914496+0000","last_undegraded":"2023-09-23T18:21:20.914496+0000","last_fullsized":"2023-09-23T18:21:20.914496+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":198,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.929830+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:21:20.929830+0000","last_peered":"2023-09-23T18:21:20.929830+0000","last_clean":"2023-09-23T18:21:20.929830+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:21:20.929830+0000","last_undegraded":"2023-09-23T18:21:20.929830+0000","last_fullsized":"2023-09-23T18:21:20.929830+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"622'12836","reported_seq":12894,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:31.393301+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:21:31.393301+0000","last_peered":"2023-09-23T18:21:31.393301+0000","last_clean":"2023-09-23T18:21:31.393301+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:21:31.393301+0000","last_undegraded":"2023-09-23T18:21:31.393301+0000","last_fullsized":"2023-09-23T18:21:31.393301+0000","mapping_epoch":574,"log_start":"608'7700","ondisk_log_start":"608'7700","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5136,"log_dups_size":0,"ondisk_log_size":5136,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16777216,"num_objects":256,"num_object_clones":0,"num_object_copies":768,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":256,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12836,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":653,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.920922+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:21:20.920922+0000","last_peered":"2023-09-23T18:21:20.920922+0000","last_clean":"2023-09-23T18:21:20.920922+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:21:20.920922+0000","last_undegraded":"2023-09-23T18:21:20.920922+0000","last_fullsized":"2023-09-23T18:21:20.920922+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":146,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914277+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:21:20.914277+0000","last_peered":"2023-09-23T18:21:20.914277+0000","last_clean":"2023-09-23T18:21:20.914277+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:21:20.914277+0000","last_undegraded":"2023-09-23T18:21:20.914277+0000","last_fullsized":"2023-09-23T18:21:20.914277+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":459,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.929048+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:21:20.929048+0000","last_peered":"2023-09-23T18:21:20.929048+0000","last_clean":"2023-09-23T18:21:20.929048+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:21:20.929048+0000","last_undegraded":"2023-09-23T18:21:20.929048+0000","last_fullsized":"2023-09-23T18:21:20.929048+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"622'12904","reported_seq":12962,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:33.856181+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:21:33.856181+0000","last_peered":"2023-09-23T18:21:33.856181+0000","last_clean":"2023-09-23T18:21:33.856181+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:21:33.856181+0000","last_undegraded":"2023-09-23T18:21:33.856181+0000","last_fullsized":"2023-09-23T18:21:33.856181+0000","mapping_epoch":574,"log_start":"610'8500","ondisk_log_start":"610'8500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4404,"log_dups_size":0,"ondisk_log_size":4404,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":9175040,"num_objects":140,"num_object_clones":0,"num_object_copies":420,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":140,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12904,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1104,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914058+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:21:20.914058+0000","last_peered":"2023-09-23T18:21:20.914058+0000","last_clean":"2023-09-23T18:21:20.914058+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:21:20.914058+0000","last_undegraded":"2023-09-23T18:21:20.914058+0000","last_fullsized":"2023-09-23T18:21:20.914058+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":359,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.922539+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:21:20.922539+0000","last_peered":"2023-09-23T18:21:20.922539+0000","last_clean":"2023-09-23T18:21:20.922539+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:21:20.922539+0000","last_undegraded":"2023-09-23T18:21:20.922539+0000","last_fullsized":"2023-09-23T18:21:20.922539+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":166,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928295+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:21:20.928295+0000","last_peered":"2023-09-23T18:21:20.928295+0000","last_clean":"2023-09-23T18:21:20.928295+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:21:20.928295+0000","last_undegraded":"2023-09-23T18:21:20.928295+0000","last_fullsized":"2023-09-23T18:21:20.928295+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"622'12786","reported_seq":18114,"reported_epoch":622,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:31.205828+0000","last_change":"2023-09-23T18:20:34.894639+0000","last_active":"2023-09-23T18:21:31.205828+0000","last_peered":"2023-09-23T18:21:31.205828+0000","last_clean":"2023-09-23T18:18:45.352246+0000","last_became_active":"2023-09-23T18:18:47.586932+0000","last_became_peered":"2023-09-23T18:18:47.586932+0000","last_unstale":"2023-09-23T18:21:31.205828+0000","last_undegraded":"2023-09-23T18:18:47.441955+0000","last_fullsized":"2023-09-23T18:18:47.412772+0000","mapping_epoch":595,"log_start":"607'7300","ondisk_log_start":"607'7300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:32:37.458788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15990784,"num_objects":244,"num_object_clones":0,"num_object_copies":732,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":1595,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":244,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12786,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2631,"num_bytes_recovered":46006272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,4],"avail_no_missing":["5","0","4"],"object_location_counts":[{"shards":"4,5","objects":-1351}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":891,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913684+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:21:20.913684+0000","last_peered":"2023-09-23T18:21:20.913684+0000","last_clean":"2023-09-23T18:21:20.913684+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:21:20.913684+0000","last_undegraded":"2023-09-23T18:21:20.913684+0000","last_fullsized":"2023-09-23T18:21:20.913684+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":174,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.929757+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:21:20.929757+0000","last_peered":"2023-09-23T18:21:20.929757+0000","last_clean":"2023-09-23T18:21:20.929757+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:21:20.929757+0000","last_undegraded":"2023-09-23T18:21:20.929757+0000","last_fullsized":"2023-09-23T18:21:20.929757+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":473,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928577+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:21:20.928577+0000","last_peered":"2023-09-23T18:21:20.928577+0000","last_clean":"2023-09-23T18:21:20.928577+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:21:20.928577+0000","last_undegraded":"2023-09-23T18:21:20.928577+0000","last_fullsized":"2023-09-23T18:21:20.928577+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"622'12803","reported_seq":12859,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:31.303222+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:21:31.303222+0000","last_peered":"2023-09-23T18:21:31.303222+0000","last_clean":"2023-09-23T18:21:31.303222+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:21:31.303222+0000","last_undegraded":"2023-09-23T18:21:31.303222+0000","last_fullsized":"2023-09-23T18:21:31.303222+0000","mapping_epoch":574,"log_start":"609'8300","ondisk_log_start":"609'8300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4503,"log_dups_size":0,"ondisk_log_size":4503,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16318464,"num_objects":249,"num_object_clones":0,"num_object_copies":747,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":249,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12803,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":893,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914251+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:21:20.914251+0000","last_peered":"2023-09-23T18:21:20.914251+0000","last_clean":"2023-09-23T18:21:20.914251+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:21:20.914251+0000","last_undegraded":"2023-09-23T18:21:20.914251+0000","last_fullsized":"2023-09-23T18:21:20.914251+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":458,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913571+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:21:20.913571+0000","last_peered":"2023-09-23T18:21:20.913571+0000","last_clean":"2023-09-23T18:21:20.913571+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:21:20.913571+0000","last_undegraded":"2023-09-23T18:21:20.913571+0000","last_fullsized":"2023-09-23T18:21:20.913571+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":234,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.930353+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:21:20.930353+0000","last_peered":"2023-09-23T18:21:20.930353+0000","last_clean":"2023-09-23T18:21:20.930353+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:21:20.930353+0000","last_undegraded":"2023-09-23T18:21:20.930353+0000","last_fullsized":"2023-09-23T18:21:20.930353+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"622'12848","reported_seq":12904,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:32.374083+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:21:32.374083+0000","last_peered":"2023-09-23T18:21:32.374083+0000","last_clean":"2023-09-23T18:21:32.374083+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:21:32.374083+0000","last_undegraded":"2023-09-23T18:21:32.374083+0000","last_fullsized":"2023-09-23T18:21:32.374083+0000","mapping_epoch":574,"log_start":"609'8200","ondisk_log_start":"609'8200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4648,"log_dups_size":0,"ondisk_log_size":4648,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":12189696,"num_objects":186,"num_object_clones":0,"num_object_copies":558,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":186,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12848,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":874,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913841+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:21:20.913841+0000","last_peered":"2023-09-23T18:21:20.913841+0000","last_clean":"2023-09-23T18:21:20.913841+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:21:20.913841+0000","last_undegraded":"2023-09-23T18:21:20.913841+0000","last_fullsized":"2023-09-23T18:21:20.913841+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":39,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.920169+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:21:20.920169+0000","last_peered":"2023-09-23T18:21:20.920169+0000","last_clean":"2023-09-23T18:21:20.920169+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:21:20.920169+0000","last_undegraded":"2023-09-23T18:21:20.920169+0000","last_fullsized":"2023-09-23T18:21:20.920169+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":350,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.915514+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:21:20.915514+0000","last_peered":"2023-09-23T18:21:20.915514+0000","last_clean":"2023-09-23T18:21:20.915514+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:21:20.915514+0000","last_undegraded":"2023-09-23T18:21:20.915514+0000","last_fullsized":"2023-09-23T18:21:20.915514+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"622'12757","reported_seq":12817,"reported_epoch":622,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:30.276019+0000","last_change":"2023-09-23T18:18:47.583069+0000","last_active":"2023-09-23T18:21:30.276019+0000","last_peered":"2023-09-23T18:21:30.276019+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:21:30.276019+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"608'7900","ondisk_log_start":"608'7900","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4857,"log_dups_size":0,"ondisk_log_size":4857,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":20381696,"num_objects":311,"num_object_clones":0,"num_object_copies":933,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4285,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":311,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12757,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-3974}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":733,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.932729+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:21:20.932729+0000","last_peered":"2023-09-23T18:21:20.932729+0000","last_clean":"2023-09-23T18:21:20.932729+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:21:20.932729+0000","last_undegraded":"2023-09-23T18:21:20.932729+0000","last_fullsized":"2023-09-23T18:21:20.932729+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":973,"reported_epoch":621,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:28.932458+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:21:28.932458+0000","last_peered":"2023-09-23T18:21:28.932458+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:21:28.932458+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":156,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914646+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:21:20.914646+0000","last_peered":"2023-09-23T18:21:20.914646+0000","last_clean":"2023-09-23T18:21:20.914646+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:21:20.914646+0000","last_undegraded":"2023-09-23T18:21:20.914646+0000","last_fullsized":"2023-09-23T18:21:20.914646+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":136,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913806+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:21:20.913806+0000","last_peered":"2023-09-23T18:21:20.913806+0000","last_clean":"2023-09-23T18:21:20.913806+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:21:20.913806+0000","last_undegraded":"2023-09-23T18:21:20.913806+0000","last_fullsized":"2023-09-23T18:21:20.913806+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"622'12693","reported_seq":12758,"reported_epoch":622,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:31.255341+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:21:31.255341+0000","last_peered":"2023-09-23T18:21:31.255341+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:21:31.255341+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"609'8200","ondisk_log_start":"609'8200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4493,"log_dups_size":0,"ondisk_log_size":4493,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15925248,"num_objects":243,"num_object_clones":0,"num_object_copies":729,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":243,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12693,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"5,6","objects":-3316}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":805,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914011+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:21:20.914011+0000","last_peered":"2023-09-23T18:21:20.914011+0000","last_clean":"2023-09-23T18:21:20.914011+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:21:20.914011+0000","last_undegraded":"2023-09-23T18:21:20.914011+0000","last_fullsized":"2023-09-23T18:21:20.914011+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1506,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913791+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:21:20.913791+0000","last_peered":"2023-09-23T18:21:20.913791+0000","last_clean":"2023-09-23T18:21:20.913791+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:21:20.913791+0000","last_undegraded":"2023-09-23T18:21:20.913791+0000","last_fullsized":"2023-09-23T18:21:20.913791+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":146,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.929000+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:21:20.929000+0000","last_peered":"2023-09-23T18:21:20.929000+0000","last_clean":"2023-09-23T18:21:20.929000+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:21:20.929000+0000","last_undegraded":"2023-09-23T18:21:20.929000+0000","last_fullsized":"2023-09-23T18:21:20.929000+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":446,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928517+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:21:20.928517+0000","last_peered":"2023-09-23T18:21:20.928517+0000","last_clean":"2023-09-23T18:21:20.928517+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:21:20.928517+0000","last_undegraded":"2023-09-23T18:21:20.928517+0000","last_fullsized":"2023-09-23T18:21:20.928517+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"622'12327","reported_seq":12383,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:31.202088+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:21:31.202088+0000","last_peered":"2023-09-23T18:21:31.202088+0000","last_clean":"2023-09-23T18:21:31.202088+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:21:31.202088+0000","last_undegraded":"2023-09-23T18:21:31.202088+0000","last_fullsized":"2023-09-23T18:21:31.202088+0000","mapping_epoch":574,"log_start":"606'6800","ondisk_log_start":"606'6800","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5527,"log_dups_size":0,"ondisk_log_size":5527,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16973824,"num_objects":259,"num_object_clones":0,"num_object_copies":777,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":259,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12327,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":830,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913887+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:21:20.913887+0000","last_peered":"2023-09-23T18:21:20.913887+0000","last_clean":"2023-09-23T18:21:20.913887+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:21:20.913887+0000","last_undegraded":"2023-09-23T18:21:20.913887+0000","last_fullsized":"2023-09-23T18:21:20.913887+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":65,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.920899+0000","last_change":"2023-09-23T18:20:35.016554+0000","last_active":"2023-09-23T18:21:20.920899+0000","last_peered":"2023-09-23T18:21:20.920899+0000","last_clean":"2023-09-23T18:21:20.920899+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:21:20.920899+0000","last_undegraded":"2023-09-23T18:21:20.920899+0000","last_fullsized":"2023-09-23T18:21:20.920899+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":415,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928052+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:21:20.928052+0000","last_peered":"2023-09-23T18:21:20.928052+0000","last_clean":"2023-09-23T18:21:20.928052+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:21:20.928052+0000","last_undegraded":"2023-09-23T18:21:20.928052+0000","last_fullsized":"2023-09-23T18:21:20.928052+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":158,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914847+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:21:20.914847+0000","last_peered":"2023-09-23T18:21:20.914847+0000","last_clean":"2023-09-23T18:21:20.914847+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:21:20.914847+0000","last_undegraded":"2023-09-23T18:21:20.914847+0000","last_fullsized":"2023-09-23T18:21:20.914847+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"622'12700","reported_seq":12756,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:31.270284+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:21:31.270284+0000","last_peered":"2023-09-23T18:21:31.270284+0000","last_clean":"2023-09-23T18:21:31.270284+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:21:31.270284+0000","last_undegraded":"2023-09-23T18:21:31.270284+0000","last_fullsized":"2023-09-23T18:21:31.270284+0000","mapping_epoch":574,"log_start":"609'8200","ondisk_log_start":"609'8200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4500,"log_dups_size":0,"ondisk_log_size":4500,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":16252928,"num_objects":248,"num_object_clones":0,"num_object_copies":744,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":248,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12700,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1519,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914052+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:21:20.914052+0000","last_peered":"2023-09-23T18:21:20.914052+0000","last_clean":"2023-09-23T18:21:20.914052+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:21:20.914052+0000","last_undegraded":"2023-09-23T18:21:20.914052+0000","last_fullsized":"2023-09-23T18:21:20.914052+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":572,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.927975+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:21:20.927975+0000","last_peered":"2023-09-23T18:21:20.927975+0000","last_clean":"2023-09-23T18:21:20.927975+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:21:20.927975+0000","last_undegraded":"2023-09-23T18:21:20.927975+0000","last_fullsized":"2023-09-23T18:21:20.927975+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":132,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913728+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:21:20.913728+0000","last_peered":"2023-09-23T18:21:20.913728+0000","last_clean":"2023-09-23T18:21:20.913728+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:21:20.913728+0000","last_undegraded":"2023-09-23T18:21:20.913728+0000","last_fullsized":"2023-09-23T18:21:20.913728+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":167,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.929338+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:21:20.929338+0000","last_peered":"2023-09-23T18:21:20.929338+0000","last_clean":"2023-09-23T18:21:20.929338+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:21:20.929338+0000","last_undegraded":"2023-09-23T18:21:20.929338+0000","last_fullsized":"2023-09-23T18:21:20.929338+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"622'12515","reported_seq":12571,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:31.204690+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:21:31.204690+0000","last_peered":"2023-09-23T18:21:31.204690+0000","last_clean":"2023-09-23T18:21:31.204690+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:21:31.204690+0000","last_undegraded":"2023-09-23T18:21:31.204690+0000","last_fullsized":"2023-09-23T18:21:31.204690+0000","mapping_epoch":574,"log_start":"607'7000","ondisk_log_start":"607'7000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5515,"log_dups_size":0,"ondisk_log_size":5515,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17629184,"num_objects":269,"num_object_clones":0,"num_object_copies":807,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":269,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12515,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":820,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.929082+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:21:20.929082+0000","last_peered":"2023-09-23T18:21:20.929082+0000","last_clean":"2023-09-23T18:21:20.929082+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:21:20.929082+0000","last_undegraded":"2023-09-23T18:21:20.929082+0000","last_fullsized":"2023-09-23T18:21:20.929082+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":136,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.920346+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:21:20.920346+0000","last_peered":"2023-09-23T18:21:20.920346+0000","last_clean":"2023-09-23T18:21:20.920346+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:21:20.920346+0000","last_undegraded":"2023-09-23T18:21:20.920346+0000","last_fullsized":"2023-09-23T18:21:20.920346+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":39,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.920808+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:21:20.920808+0000","last_peered":"2023-09-23T18:21:20.920808+0000","last_clean":"2023-09-23T18:21:20.920808+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:21:20.920808+0000","last_undegraded":"2023-09-23T18:21:20.920808+0000","last_fullsized":"2023-09-23T18:21:20.920808+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":637,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914605+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:21:20.914605+0000","last_peered":"2023-09-23T18:21:20.914605+0000","last_clean":"2023-09-23T18:21:20.914605+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:21:20.914605+0000","last_undegraded":"2023-09-23T18:21:20.914605+0000","last_fullsized":"2023-09-23T18:21:20.914605+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"622'12716","reported_seq":19700,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:31.314619+0000","last_change":"2023-09-23T18:20:36.673759+0000","last_active":"2023-09-23T18:21:31.314619+0000","last_peered":"2023-09-23T18:21:31.314619+0000","last_clean":"2023-09-23T18:21:31.314619+0000","last_became_active":"2023-09-23T18:20:36.651913+0000","last_became_peered":"2023-09-23T18:20:36.651913+0000","last_unstale":"2023-09-23T18:21:31.314619+0000","last_undegraded":"2023-09-23T18:21:31.314619+0000","last_fullsized":"2023-09-23T18:21:31.314619+0000","mapping_epoch":614,"log_start":"609'8300","ondisk_log_start":"609'8300","created":574,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4416,"log_dups_size":0,"ondisk_log_size":4416,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:47.765348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15794200,"num_objects":242,"num_object_clones":0,"num_object_copies":726,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":242,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12716,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3453,"num_bytes_recovered":209321984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":920,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.930828+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:21:20.930828+0000","last_peered":"2023-09-23T18:21:20.930828+0000","last_clean":"2023-09-23T18:21:20.930828+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:21:20.930828+0000","last_undegraded":"2023-09-23T18:21:20.930828+0000","last_fullsized":"2023-09-23T18:21:20.930828+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":61,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.926362+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:21:20.926362+0000","last_peered":"2023-09-23T18:21:20.926362+0000","last_clean":"2023-09-23T18:21:20.926362+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:21:20.926362+0000","last_undegraded":"2023-09-23T18:21:20.926362+0000","last_fullsized":"2023-09-23T18:21:20.926362+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":146,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914193+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:21:20.914193+0000","last_peered":"2023-09-23T18:21:20.914193+0000","last_clean":"2023-09-23T18:21:20.914193+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:21:20.914193+0000","last_undegraded":"2023-09-23T18:21:20.914193+0000","last_fullsized":"2023-09-23T18:21:20.914193+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":146,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913664+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:21:20.913664+0000","last_peered":"2023-09-23T18:21:20.913664+0000","last_clean":"2023-09-23T18:21:20.913664+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:21:20.913664+0000","last_undegraded":"2023-09-23T18:21:20.913664+0000","last_fullsized":"2023-09-23T18:21:20.913664+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"622'12592","reported_seq":12648,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:31.296259+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:21:31.296259+0000","last_peered":"2023-09-23T18:21:31.296259+0000","last_clean":"2023-09-23T18:21:31.296259+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:21:31.296259+0000","last_undegraded":"2023-09-23T18:21:31.296259+0000","last_fullsized":"2023-09-23T18:21:31.296259+0000","mapping_epoch":574,"log_start":"609'8100","ondisk_log_start":"609'8100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4492,"log_dups_size":0,"ondisk_log_size":4492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":17170432,"num_objects":262,"num_object_clones":0,"num_object_copies":786,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":262,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12592,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":920,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928659+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:21:20.928659+0000","last_peered":"2023-09-23T18:21:20.928659+0000","last_clean":"2023-09-23T18:21:20.928659+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:21:20.928659+0000","last_undegraded":"2023-09-23T18:21:20.928659+0000","last_fullsized":"2023-09-23T18:21:20.928659+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":108,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914677+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:21:20.914677+0000","last_peered":"2023-09-23T18:21:20.914677+0000","last_clean":"2023-09-23T18:21:20.914677+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:21:20.914677+0000","last_undegraded":"2023-09-23T18:21:20.914677+0000","last_fullsized":"2023-09-23T18:21:20.914677+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":401,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.930677+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:21:20.930677+0000","last_peered":"2023-09-23T18:21:20.930677+0000","last_clean":"2023-09-23T18:21:20.930677+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:21:20.930677+0000","last_undegraded":"2023-09-23T18:21:20.930677+0000","last_fullsized":"2023-09-23T18:21:20.930677+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":39,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.920095+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:21:20.920095+0000","last_peered":"2023-09-23T18:21:20.920095+0000","last_clean":"2023-09-23T18:21:20.920095+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:21:20.920095+0000","last_undegraded":"2023-09-23T18:21:20.920095+0000","last_fullsized":"2023-09-23T18:21:20.920095+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"622'12574","reported_seq":12630,"reported_epoch":622,"state":"active+clean","last_fresh":"2023-09-23T18:21:31.324662+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:21:31.324662+0000","last_peered":"2023-09-23T18:21:31.324662+0000","last_clean":"2023-09-23T18:21:31.324662+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:21:31.324662+0000","last_undegraded":"2023-09-23T18:21:31.324662+0000","last_fullsized":"2023-09-23T18:21:31.324662+0000","mapping_epoch":574,"log_start":"609'8100","ondisk_log_start":"609'8100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4474,"log_dups_size":0,"ondisk_log_size":4474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":15597568,"num_objects":238,"num_object_clones":0,"num_object_copies":714,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":238,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12574,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":755,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.922237+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:21:20.922237+0000","last_peered":"2023-09-23T18:21:20.922237+0000","last_clean":"2023-09-23T18:21:20.922237+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:21:20.922237+0000","last_undegraded":"2023-09-23T18:21:20.922237+0000","last_fullsized":"2023-09-23T18:21:20.922237+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1058,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.932945+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:21:20.932945+0000","last_peered":"2023-09-23T18:21:20.932945+0000","last_clean":"2023-09-23T18:21:20.932945+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:21:20.932945+0000","last_undegraded":"2023-09-23T18:21:20.932945+0000","last_fullsized":"2023-09-23T18:21:20.932945+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":49,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.925130+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:21:20.925130+0000","last_peered":"2023-09-23T18:21:20.925130+0000","last_clean":"2023-09-23T18:21:20.925130+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:21:20.925130+0000","last_undegraded":"2023-09-23T18:21:20.925130+0000","last_fullsized":"2023-09-23T18:21:20.925130+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":412,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913765+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:21:20.913765+0000","last_peered":"2023-09-23T18:21:20.913765+0000","last_clean":"2023-09-23T18:21:20.913765+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:21:20.913765+0000","last_undegraded":"2023-09-23T18:21:20.913765+0000","last_fullsized":"2023-09-23T18:21:20.913765+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":652,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928189+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:21:20.928189+0000","last_peered":"2023-09-23T18:21:20.928189+0000","last_clean":"2023-09-23T18:21:20.928189+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:21:20.928189+0000","last_undegraded":"2023-09-23T18:21:20.928189+0000","last_fullsized":"2023-09-23T18:21:20.928189+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":45,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928969+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:21:20.928969+0000","last_peered":"2023-09-23T18:21:20.928969+0000","last_clean":"2023-09-23T18:21:20.928969+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:21:20.928969+0000","last_undegraded":"2023-09-23T18:21:20.928969+0000","last_fullsized":"2023-09-23T18:21:20.928969+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":695,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913505+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:21:20.913505+0000","last_peered":"2023-09-23T18:21:20.913505+0000","last_clean":"2023-09-23T18:21:20.913505+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:21:20.913505+0000","last_undegraded":"2023-09-23T18:21:20.913505+0000","last_fullsized":"2023-09-23T18:21:20.913505+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":652,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.930501+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:21:20.930501+0000","last_peered":"2023-09-23T18:21:20.930501+0000","last_clean":"2023-09-23T18:21:20.930501+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:21:20.930501+0000","last_undegraded":"2023-09-23T18:21:20.930501+0000","last_fullsized":"2023-09-23T18:21:20.930501+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":140,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914233+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:21:20.914233+0000","last_peered":"2023-09-23T18:21:20.914233+0000","last_clean":"2023-09-23T18:21:20.914233+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:21:20.914233+0000","last_undegraded":"2023-09-23T18:21:20.914233+0000","last_fullsized":"2023-09-23T18:21:20.914233+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":256,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.935302+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:21:20.935302+0000","last_peered":"2023-09-23T18:21:20.935302+0000","last_clean":"2023-09-23T18:21:20.935302+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:21:20.935302+0000","last_undegraded":"2023-09-23T18:21:20.935302+0000","last_fullsized":"2023-09-23T18:21:20.935302+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":126,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914591+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:21:20.914591+0000","last_peered":"2023-09-23T18:21:20.914591+0000","last_clean":"2023-09-23T18:21:20.914591+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:21:20.914591+0000","last_undegraded":"2023-09-23T18:21:20.914591+0000","last_fullsized":"2023-09-23T18:21:20.914591+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":239,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.934350+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:21:20.934350+0000","last_peered":"2023-09-23T18:21:20.934350+0000","last_clean":"2023-09-23T18:21:20.934350+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:21:20.934350+0000","last_undegraded":"2023-09-23T18:21:20.934350+0000","last_fullsized":"2023-09-23T18:21:20.934350+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1014,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.928219+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:21:20.928219+0000","last_peered":"2023-09-23T18:21:20.928219+0000","last_clean":"2023-09-23T18:21:20.928219+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:21:20.928219+0000","last_undegraded":"2023-09-23T18:21:20.928219+0000","last_fullsized":"2023-09-23T18:21:20.928219+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":519,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.930755+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:21:20.930755+0000","last_peered":"2023-09-23T18:21:20.930755+0000","last_clean":"2023-09-23T18:21:20.930755+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:21:20.930755+0000","last_undegraded":"2023-09-23T18:21:20.930755+0000","last_fullsized":"2023-09-23T18:21:20.930755+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":156,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913929+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:21:20.913929+0000","last_peered":"2023-09-23T18:21:20.913929+0000","last_clean":"2023-09-23T18:21:20.913929+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:21:20.913929+0000","last_undegraded":"2023-09-23T18:21:20.913929+0000","last_fullsized":"2023-09-23T18:21:20.913929+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":951,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.913209+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:21:20.913209+0000","last_peered":"2023-09-23T18:21:20.913209+0000","last_clean":"2023-09-23T18:21:20.913209+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:21:20.913209+0000","last_undegraded":"2023-09-23T18:21:20.913209+0000","last_fullsized":"2023-09-23T18:21:20.913209+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":144,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.914788+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:21:20.914788+0000","last_peered":"2023-09-23T18:21:20.914788+0000","last_clean":"2023-09-23T18:21:20.914788+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:21:20.914788+0000","last_undegraded":"2023-09-23T18:21:20.914788+0000","last_fullsized":"2023-09-23T18:21:20.914788+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":665,"reported_epoch":621,"state":"active+clean","last_fresh":"2023-09-23T18:21:20.927882+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:21:20.927882+0000","last_peered":"2023-09-23T18:21:20.927882+0000","last_clean":"2023-09-23T18:21:20.927882+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:21:20.927882+0000","last_undegraded":"2023-09-23T18:21:20.927882+0000","last_fullsized":"2023-09-23T18:21:20.927882+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":260964376,"num_objects":3983,"num_object_clones":0,"num_object_copies":11949,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13587,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3983,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":202825,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":9555,"num_bytes_recovered":482803712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2457481216,"data_stored":2457469000,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77325,"ondisk_log_size":77325,"up":48,"acting":44,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20983808,"data_stored":20971577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001587,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669523,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1068068,"kb_used_data":412380,"kb_used_omap":7,"kb_used_meta":655608,"kb_avail":92677084,"statfs":{"total":95995035648,"available":94901334016,"internally_reserved":0,"allocated":422277120,"data_stored":420302677,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":671343396},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0169999999999999,"5min":2.194,"15min":1.5620000000000001},"min":{"1min":0.255,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":12.207000000000001,"5min":24.472999999999999,"15min":24.472999999999999},"last":79.203999999999994},{"interface":"front","average":{"1min":2.375,"5min":2.4220000000000002,"15min":1.833},"min":{"1min":0.255,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.255000000000001,"5min":22.832999999999998,"15min":22.832999999999998},"last":78.034000000000006}]},{"osd":1,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":2.258,"15min":1.5629999999999999},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.7309999999999999,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.307},{"interface":"front","average":{"1min":2.9580000000000002,"5min":2.2490000000000001,"15min":1.5549999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.273,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.35299999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.579,"15min":1.345},"min":{"1min":0.33500000000000002,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":5.7919999999999998,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.40999999999999998},{"interface":"front","average":{"1min":2.3029999999999999,"5min":2.0600000000000001,"15min":1.5580000000000001},"min":{"1min":0.39700000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0730000000000004,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.38300000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":3.0419999999999998,"15min":2.1150000000000002},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.24099999999999999},"max":{"1min":5.8170000000000002,"5min":103.348,"15min":103.348},"last":78.643000000000001},{"interface":"front","average":{"1min":2.5600000000000001,"5min":3.6190000000000002,"15min":2.4209999999999998},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":7.4530000000000003,"5min":103.40300000000001,"15min":103.40300000000001},"last":78.347999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:21:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.7390000000000001,"15min":1.341},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.0300000000000002,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.661,"15min":1.389},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.27000000000000002},"max":{"1min":5.7649999999999997,"5min":7.8319999999999999,"15min":9.2260000000000009},"last":0.32300000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":2.1099999999999999,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":18.417000000000002,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.42699999999999999},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.3399999999999999,"15min":2.3780000000000001},"min":{"1min":0.36299999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":18.454999999999998,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.44500000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":2.1059999999999999,"15min":1.5580000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.26800000000000002},"max":{"1min":6.1500000000000004,"5min":10.743,"15min":13.460000000000001},"last":0.39900000000000002},{"interface":"front","average":{"1min":2.0830000000000002,"5min":2.0529999999999999,"15min":1.603},"min":{"1min":0.40300000000000002,"5min":0.38500000000000001,"15min":0.28499999999999998},"max":{"1min":8.6890000000000001,"5min":11.244,"15min":12.404},"last":0.372}]}]},{"osd":1,"up_from":545,"seq":2340757176372,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":934868,"kb_used_data":686924,"kb_used_omap":13,"kb_used_meta":247922,"kb_avail":92810284,"statfs":{"total":95995035648,"available":95037730816,"internally_reserved":0,"allocated":703410176,"data_stored":700347292,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":253872781},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1],"upper_bound":8},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.286,"15min":0.68899999999999995},"min":{"1min":0.23400000000000001,"5min":0.189,"15min":0.189},"max":{"1min":5.0359999999999996,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":4.5030000000000001},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.3280000000000001,"15min":0.76900000000000002},"min":{"1min":0.372,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.5650000000000004,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.50900000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.4450000000000001,"15min":0.78900000000000003},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.02,"5min":11.02,"15min":11.02},"last":0.61899999999999999},{"interface":"front","average":{"1min":1.5249999999999999,"5min":0.97599999999999998,"15min":0.66200000000000003},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.1440000000000001,"5min":5.1440000000000001,"15min":5.1440000000000001},"last":0.86299999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":3.9870000000000001,"5min":1.7609999999999999,"15min":0.84499999999999997},"min":{"1min":0.51400000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":14.962,"5min":14.962,"15min":14.962},"last":1.0980000000000001},{"interface":"front","average":{"1min":2.008,"5min":1.5760000000000001,"15min":0.86599999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":8.3399999999999999,"5min":19.762,"15min":19.762},"last":0.82199999999999995}]},{"osd":4,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.421,"15min":1.351},"min":{"1min":0.41899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.0759999999999996,"5min":12.663,"15min":12.663},"last":3.532},{"interface":"front","average":{"1min":3.952,"5min":2.2669999999999999,"15min":1.7649999999999999},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.317},"max":{"1min":11.353999999999999,"5min":13.222,"15min":13.222},"last":4.7320000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.359,"5min":1.732,"15min":1.0589999999999999},"min":{"1min":0.36799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":6.3460000000000001,"5min":23.190999999999999,"15min":23.190999999999999},"last":3.6760000000000002},{"interface":"front","average":{"1min":2.3450000000000002,"5min":2.0409999999999999,"15min":1.216},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":10.926,"5min":23.204000000000001,"15min":23.204000000000001},"last":4.556}]},{"osd":6,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":1.7270000000000001,"15min":1.0820000000000001},"min":{"1min":0.45700000000000002,"5min":0.372,"15min":0.372},"max":{"1min":5.1680000000000001,"5min":23.443000000000001,"15min":23.443000000000001},"last":3.5659999999999998},{"interface":"front","average":{"1min":3.5059999999999998,"5min":2.2919999999999998,"15min":1.2809999999999999},"min":{"1min":0.49399999999999999,"5min":0.378,"15min":0.378},"max":{"1min":11.307,"5min":26.177,"15min":26.177},"last":4.8959999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":3.649,"5min":1.956,"15min":1.1060000000000001},"min":{"1min":0.42799999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":15.01,"5min":15.01,"15min":15.01},"last":1.052},{"interface":"front","average":{"1min":2.931,"5min":1.899,"15min":1.1699999999999999},"min":{"1min":0.45200000000000001,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":10.988,"5min":10.988,"15min":10.988},"last":0.78200000000000003}]}]},{"osd":0,"up_from":93,"seq":399431958868,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1435376,"kb_used_data":481768,"kb_used_omap":7,"kb_used_meta":953528,"kb_avail":92309776,"statfs":{"total":95995035648,"available":94525210624,"internally_reserved":0,"allocated":493330432,"data_stored":490419655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":976413412},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,1,3,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.363,"15min":0.66700000000000004},"min":{"1min":0.19800000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.5090000000000003,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":4.4039999999999999},{"interface":"front","average":{"1min":3.3130000000000002,"5min":1.8360000000000001,"15min":0.95999999999999996},"min":{"1min":0.33800000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":22.978000000000002,"5min":22.978000000000002,"15min":22.978000000000002},"last":4.0220000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:21:04 2023","interfaces":[{"interface":"back","average":{"1min":3.008,"5min":2.0880000000000001,"15min":1.4419999999999999},"min":{"1min":0.40200000000000002,"5min":0.185,"15min":0.185},"max":{"1min":10.013,"5min":18.550999999999998,"15min":18.550999999999998},"last":4.6749999999999998},{"interface":"front","average":{"1min":3.2160000000000002,"5min":1.7470000000000001,"15min":1.097},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":23.161999999999999,"5min":23.161999999999999,"15min":23.161999999999999},"last":3.6970000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.4199999999999999,"15min":1.5940000000000001},"min":{"1min":0.313,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.8460000000000001,"5min":13.539999999999999,"15min":13.539999999999999},"last":0.24199999999999999},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.6619999999999999,"15min":1.7010000000000001},"min":{"1min":0.36299999999999999,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":6.8879999999999999,"5min":13.285,"15min":15.198},"last":0.39000000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:21:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":2.1019999999999999,"15min":1.982},"min":{"1min":0.39900000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.1379999999999999,"5min":17.741,"15min":57.746000000000002},"last":0.33300000000000002},{"interface":"front","average":{"1min":1.966,"5min":1.9099999999999999,"15min":1.792},"min":{"1min":0.30099999999999999,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":4.9089999999999998,"5min":15.079000000000001,"15min":57.816000000000003},"last":0.60999999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":1.8220000000000001,"15min":1.4410000000000001},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":5.5970000000000004,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.30199999999999999},{"interface":"front","average":{"1min":3.2909999999999999,"5min":2.4630000000000001,"15min":1.599},"min":{"1min":0.40100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":15.047000000000001,"5min":15.047000000000001,"15min":15.047000000000001},"last":0.64500000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:21:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":1.9379999999999999,"15min":2.0299999999999998},"min":{"1min":0.40300000000000002,"5min":0.377,"15min":0.32000000000000001},"max":{"1min":5.4610000000000003,"5min":7.6900000000000004,"15min":58.457999999999998},"last":0.36099999999999999},{"interface":"front","average":{"1min":3.7629999999999999,"5min":2.657,"15min":2.1190000000000002},"min":{"1min":0.54100000000000004,"5min":0.373,"15min":0.315},"max":{"1min":22.893000000000001,"5min":22.893000000000001,"15min":58.354999999999997},"last":3.778}]},{"osd":7,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":3.431,"5min":2.2480000000000002,"15min":1.7889999999999999},"min":{"1min":0.49199999999999999,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":22.945,"5min":22.945,"15min":22.945},"last":3.7410000000000001},{"interface":"front","average":{"1min":2.2330000000000001,"5min":1.714,"15min":1.851},"min":{"1min":0.47099999999999997,"5min":0.371,"15min":0.36199999999999999},"max":{"1min":6.9249999999999998,"5min":7.7519999999999998,"15min":10.404},"last":0.42799999999999999}]}]},{"osd":2,"up_from":0,"seq":2070174236754,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249383,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":942156,"kb_used_data":136516,"kb_used_omap":3,"kb_used_meta":805500,"kb_avail":92802996,"statfs":{"total":95995035648,"available":95030267904,"internally_reserved":0,"allocated":139792384,"data_stored":136679445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":824832744},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,1,2],"upper_bound":64},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":1.5800000000000001,"15min":1.8089999999999999},"min":{"1min":0.20799999999999999,"5min":0.192,"15min":0.183},"max":{"1min":12.472,"5min":17.036000000000001,"15min":195.69800000000001},"last":2.923},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.405,"15min":2.004},"min":{"1min":0.23799999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.468,"5min":11.468,"15min":194.661},"last":2.8359999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:21:27 2023","interfaces":[{"interface":"back","average":{"1min":1.994,"5min":1.754,"15min":0.93100000000000005},"min":{"1min":0.249,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.1189999999999998,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.46100000000000002},{"interface":"front","average":{"1min":1.9710000000000001,"5min":2.766,"15min":1.3919999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.9050000000000002,"5min":78.980000000000004,"15min":78.980000000000004},"last":2.6419999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:21:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5569999999999999,"5min":1.526,"15min":1.3740000000000001},"min":{"1min":0.29899999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.2430000000000003,"5min":12.151999999999999,"15min":12.151999999999999},"last":2.5720000000000001},{"interface":"front","average":{"1min":1.78,"5min":1.3839999999999999,"15min":1.361},"min":{"1min":0.32300000000000001,"5min":0.27900000000000003,"15min":0.19600000000000001},"max":{"1min":7.0640000000000001,"5min":11.565,"15min":11.565},"last":7.0579999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":1.8839999999999999,"15min":1.706},"min":{"1min":0.34200000000000003,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":11.4,"5min":21.062999999999999,"15min":37.231000000000002},"last":2.6190000000000002},{"interface":"front","average":{"1min":2.532,"5min":2.1240000000000001,"15min":1.885},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":9.1630000000000003,"5min":21.128,"15min":37.381},"last":0.52800000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:20:37 2023","interfaces":[{"interface":"back","average":{"1min":3.452,"5min":2.048,"15min":1.587},"min":{"1min":0.35099999999999998,"5min":0.316,"15min":0.316},"max":{"1min":9.7430000000000003,"5min":11.805999999999999,"15min":11.805999999999999},"last":0.5},{"interface":"front","average":{"1min":3.0339999999999998,"5min":1.74,"15min":1.722},"min":{"1min":0.32800000000000001,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":17.135999999999999,"5min":17.135999999999999,"15min":17.135999999999999},"last":2.5840000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.9390000000000001,"15min":1.9379999999999999},"min":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":10.303000000000001,"5min":17.192,"15min":115.54000000000001},"last":2.6840000000000002},{"interface":"front","average":{"1min":2.9889999999999999,"5min":1.976,"15min":1.9810000000000001},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.30499999999999999},"max":{"1min":11.442,"5min":14.321,"15min":115.691},"last":2.669}]},{"osd":7,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0939999999999999,"5min":1.744,"15min":1.7609999999999999},"min":{"1min":0.42699999999999999,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":15.972},"last":0.55500000000000005},{"interface":"front","average":{"1min":1.974,"5min":1.829,"15min":1.6240000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":6.9749999999999996,"5min":17.164000000000001,"15min":17.164000000000001},"last":2.7170000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019333,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1146084,"kb_used_data":453148,"kb_used_omap":0,"kb_used_meta":692928,"kb_avail":92599068,"statfs":{"total":95995035648,"available":94821445632,"internally_reserved":0,"allocated":464023552,"data_stored":462048901,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":709558272},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[1,0,0,1,1],"upper_bound":32},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5449999999999999,"5min":1.7589999999999999,"15min":1.387},"min":{"1min":0.443,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.0960000000000001,"5min":7.6079999999999997,"15min":13.375999999999999},"last":0.26000000000000001},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.919,"15min":1.6379999999999999},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":7.6440000000000001,"5min":10.890000000000001,"15min":10.890000000000001},"last":3.056}]},{"osd":1,"last update":"Sat Sep 23 18:21:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":2.048,"15min":1.1459999999999999},"min":{"1min":0.255,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.619,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.38100000000000001},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.8899999999999999,"15min":1.0640000000000001},"min":{"1min":0.28399999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.1470000000000002,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.29999999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3839999999999999,"5min":1.768,"15min":1.8500000000000001},"min":{"1min":0.34300000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.1749999999999998,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.8079999999999998,"5min":1.8360000000000001,"15min":1.7649999999999999},"min":{"1min":0.36199999999999999,"5min":0.29899999999999999,"15min":0.253},"max":{"1min":17.190999999999999,"5min":17.190999999999999,"15min":17.190999999999999},"last":3.3839999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.395,"5min":1.5529999999999999,"15min":1.9630000000000001},"min":{"1min":0.379,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.7519999999999998,"5min":11.130000000000001,"15min":29.154},"last":4.0949999999999998},{"interface":"front","average":{"1min":2.6230000000000002,"5min":1.8300000000000001,"15min":1.9930000000000001},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.099,"5min":13.470000000000001,"15min":29.216000000000001},"last":0.39700000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.3170000000000002,"15min":2.7829999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.5009999999999994,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.6499999999999999,"15min":2.5609999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":3.8359999999999999,"5min":13.938000000000001,"15min":32.640000000000001},"last":7.6020000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.8819999999999999,"15min":1.4099999999999999},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":3.9750000000000001,"5min":18.315000000000001,"15min":18.315000000000001},"last":7.423},{"interface":"front","average":{"1min":2.1890000000000001,"5min":2.0499999999999998,"15min":1.619},"min":{"1min":0.311,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.4969999999999999,"5min":18.431000000000001,"15min":18.431000000000001},"last":7.4640000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.7770000000000001,"5min":2.5430000000000001,"15min":1.8360000000000001},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":18.283000000000001,"5min":19.298999999999999,"15min":19.298999999999999},"last":3.6800000000000002},{"interface":"front","average":{"1min":1.5760000000000001,"5min":1.7869999999999999,"15min":1.5409999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.28399999999999997},"max":{"1min":3.786,"5min":12.800000000000001,"15min":12.800000000000001},"last":3.109}]}]},{"osd":5,"up_from":420,"seq":1803886264429,"num_pgs":55,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":724756,"kb_used_data":369292,"kb_used_omap":9,"kb_used_meta":355446,"kb_avail":93020396,"statfs":{"total":95995035648,"available":95252885504,"internally_reserved":0,"allocated":378155008,"data_stored":376192701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":363977149},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[0,0,0,0,0,4],"upper_bound":64},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.2290000000000001,"15min":0.93200000000000005},"min":{"1min":0.21299999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.867,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":77.162000000000006},{"interface":"front","average":{"1min":2.137,"5min":1.8,"15min":1.6459999999999999},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.621,"5min":20.937999999999999,"15min":20.937999999999999},"last":75.784999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:21:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":1.9570000000000001,"15min":1.383},"min":{"1min":0.28999999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.752000000000001,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.23300000000000001},{"interface":"front","average":{"1min":2.1640000000000001,"5min":1.9930000000000001,"15min":1.2290000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.4969999999999999,"5min":23.268999999999998,"15min":23.268999999999998},"last":0.27300000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.6160000000000001,"15min":1.611},"min":{"1min":0.29699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.1150000000000002,"5min":10.705,"15min":10.705},"last":0.31},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.8520000000000001,"15min":2.1280000000000001},"min":{"1min":0.39600000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.7560000000000002,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.32300000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.762,"15min":1.359},"min":{"1min":0.35599999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.0060000000000002,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":79.016999999999996},{"interface":"front","average":{"1min":2.2200000000000002,"5min":2.1150000000000002,"15min":1.7130000000000001},"min":{"1min":0.372,"5min":0.33500000000000002,"15min":0.313},"max":{"1min":10.664999999999999,"5min":20.035,"15min":20.035},"last":76.188999999999993}]},{"osd":4,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":2.1480000000000001,"15min":2.0499999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.2270000000000003,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.29199999999999998},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1179999999999999,"15min":1.6639999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7919999999999998,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.36499999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.8340000000000001,"15min":1.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.2189999999999999,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.40300000000000002},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.3319999999999999,"15min":1.7370000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.6920000000000002,"5min":12.504,"15min":12.504},"last":0.35499999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.23,"15min":1.804},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.195,"5min":12.641,"15min":13.506},"last":0.33700000000000002},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.8500000000000001,"15min":1.5600000000000001},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":5.0839999999999996,"5min":13.138,"15min":13.138},"last":0.42599999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":477892608,"data_stored":477888536,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":681967616,"data_stored":681967616,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":113442816,"data_stored":113442816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":429527040,"data_stored":429522968,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":361431040,"data_stored":361431040,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":393220096,"data_stored":393216024,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:21:38.243 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:21:38.244 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:38.244 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:38.270 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:38.303 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:38.306 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:38.307 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:38.316 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_ops_in_flight 2023-09-23T18:21:38.408 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:38.439 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:38.484 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:38.488 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:38.489 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:38.589 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:38.617 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:38.665 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:38.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:38.670 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:38.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:38 smithi055 ceph-mon[184174]: pgmap v1511: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 39 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 778 op/s; 13318/1746 objects degraded (762.772%); 0 B/s, 54 objects/s recovering 2023-09-23T18:21:38.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:38 smithi055 ceph-mon[184174]: failed to encode map e623 with expected crc 2023-09-23T18:21:38.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:38 smithi055 ceph-mon[184174]: failed to encode map e623 with expected crc 2023-09-23T18:21:38.702 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:38 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 13318/1746 objects degraded (762.772%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:38.702 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:38 smithi055 ceph-mon[179668]: pgmap v1511: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 39 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 778 op/s; 13318/1746 objects degraded (762.772%); 0 B/s, 54 objects/s recovering 2023-09-23T18:21:38.702 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:38 smithi055 ceph-mon[179668]: failed to encode map e623 with expected crc 2023-09-23T18:21:38.703 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:38 smithi055 ceph-mon[179668]: failed to encode map e623 with expected crc 2023-09-23T18:21:38.703 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:38 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 13318/1746 objects degraded (762.772%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:38.771 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:38.800 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:38.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:38 smithi099 ceph-mon[167076]: pgmap v1511: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 39 MiB data, 5.5 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 778 op/s; 13318/1746 objects degraded (762.772%); 0 B/s, 54 objects/s recovering 2023-09-23T18:21:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:38 smithi099 ceph-mon[167076]: failed to encode map e623 with expected crc 2023-09-23T18:21:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:38 smithi099 ceph-mon[167076]: failed to encode map e623 with expected crc 2023-09-23T18:21:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:38 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 13318/1746 objects degraded (762.772%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:38.835 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:38.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:38.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:38.940 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:38.950 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:21:38.980 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:39.015 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:39.018 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:39.018 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:39.055 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:39.120 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:39.146 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:39.181 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:39.186 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:39.187 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:39.287 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:39.319 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:39.570 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:39.574 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:39.574 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:39.675 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:39.713 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:39.853 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:39.857 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:39.858 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:39.959 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:39.990 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:40.030 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:40.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:40.035 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:40.130 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:21:40.135 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:40.150 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:21:40.167 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:40.177 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:21:40.207 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:21:40.219 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:40.221 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:40.222 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:40.242 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:21:40.268 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:21:40.294 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:21:40.321 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:21:40.322 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:21:40.323 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:21:40.323 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:21:40.323 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:21:40.323 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:21:40.323 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:21:40.323 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:21:40.323 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:21:40.324 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:40.343 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:21:40.343 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:21:40.343 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:21:40.344 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:21:40.354 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:40.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:21:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:21:40] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.43.0" 2023-09-23T18:21:40.447 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:40.451 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:40.451 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:40.552 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:40.584 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:40.593 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:21:40.706 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:40.709 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:40.710 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:40.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_blocked_ops 2023-09-23T18:21:40.779 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:40 smithi099 ceph-mon[167076]: pgmap v1513: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 19 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 13318/768 objects degraded (1734.115%); 0 B/s, 56 objects/s recovering 2023-09-23T18:21:40.779 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/141323929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:40.811 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:40.837 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:40.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:40.878 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:40.878 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:40.919 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:40 smithi055 ceph-mon[184174]: pgmap v1513: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 19 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 13318/768 objects degraded (1734.115%); 0 B/s, 56 objects/s recovering 2023-09-23T18:21:40.919 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/141323929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:40.919 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:40 smithi055 ceph-mon[179668]: pgmap v1513: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 19 MiB data, 5.3 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 782 op/s; 13318/768 objects degraded (1734.115%); 0 B/s, 56 objects/s recovering 2023-09-23T18:21:40.919 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/141323929' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:40.979 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:41.019 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:41.061 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:41.064 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:41.064 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:41.165 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:41.196 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:41.236 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:21:41.236 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_WARN","checks":{"OSDMAP_FLAGS":{"severity":"HEALTH_WARN","summary":{"message":"noscrub,nodeep-scrub flag(s) set","count":20},"muted":false},"PG_DEGRADED":{"severity":"HEALTH_WARN","summary":{"message":"Degraded data redundancy: 13318/768 objects degraded (1734.115%), 5 pgs degraded, 5 pgs undersized","count":10},"muted":false}},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1790,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":623,"num_osds":8,"num_up_osds":8,"osd_up_since":1695493039,"num_in_osds":6,"osd_in_since":1695493021,"num_remapped_pgs":5},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":114},{"state_name":"active+recovery_wait+undersized+degraded+remapped","count":4},{"state_name":"active+recovering+undersized+degraded+remapped","count":1}],"num_pgs":119,"num_pools":5,"num_objects":256,"data_bytes":19413692,"bytes_used":5659873280,"bytes_avail":570310340608,"bytes_total":575970213888,"degraded_objects":13318,"degraded_total":768,"degraded_ratio":17.341145833333332,"recovering_objects_per_sec":56,"recovering_bytes_per_sec":0,"recovering_keys_per_sec":0,"num_objects_recovered":550,"num_bytes_recovered":0,"num_keys_recovered":0,"write_bytes_sec":0,"read_op_per_sec":0,"write_op_per_sec":782},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":34,"modified":"2023-09-23T18:21:27.214787+0000","services":{}},"progress_events":{"57c27436-1bc8-4f63-b41f-a80b88a05748":{"message":"Global Recovery Event (2m)\n [==========================..] (remaining: 7s)","progress":0.95798319578170776,"add_to_ceph_s":true},"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T18:21:41.238 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:41.242 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:41.243 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:41.252 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:41.344 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:41.366 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:21:41.385 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:41.427 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:41.430 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:41.430 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:41.531 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:41.558 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:41.598 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:41.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:41.602 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:41.655 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_5 2023-09-23T18:21:41.655 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool rm unique_pool_5 unique_pool_5 --yes-i-really-really-mean-it 2023-09-23T18:21:41.703 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:41.735 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:41 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1149607809' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:21:41.735 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:41 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1149607809' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:21:41.738 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:41.783 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:41.786 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:41.787 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:41.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:41 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1149607809' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:21:41.888 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:41.902 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:21:41.903 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:21:41.916 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:42.076 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:42.079 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:42.080 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:42.180 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:42.211 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:42.268 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:42.273 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:42.273 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:42.374 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:42.410 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:42.546 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:42.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:42.549 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:42.574 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:42.625 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:21:42.625 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:21:42.625 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:21:42.625 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:21:42.625 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:21:42.649 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:42 smithi099 ceph-mon[167076]: pgmap v1514: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 11 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 725 op/s; 13318/429 objects degraded (3104.429%); 0 B/s, 54 objects/s recovering 2023-09-23T18:21:42.649 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:42 smithi099 ceph-mon[167076]: failed to encode map e624 with expected crc 2023-09-23T18:21:42.649 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:42 smithi099 ceph-mon[167076]: osdmap e624: 8 total, 8 up, 6 in 2023-09-23T18:21:42.649 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:42 smithi099 ceph-mon[167076]: failed to encode map e624 with expected crc 2023-09-23T18:21:42.649 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:42 smithi099 ceph-mon[167076]: failed to encode map e624 with expected crc 2023-09-23T18:21:42.649 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:42.676 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:42.716 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:42.720 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:42.720 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:42.821 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:42.848 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:42.922 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:21:42.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:42 smithi055 ceph-mon[179668]: pgmap v1514: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 11 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 725 op/s; 13318/429 objects degraded (3104.429%); 0 B/s, 54 objects/s recovering 2023-09-23T18:21:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:42 smithi055 ceph-mon[179668]: failed to encode map e624 with expected crc 2023-09-23T18:21:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:42 smithi055 ceph-mon[179668]: osdmap e624: 8 total, 8 up, 6 in 2023-09-23T18:21:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:42 smithi055 ceph-mon[179668]: failed to encode map e624 with expected crc 2023-09-23T18:21:42.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:42 smithi055 ceph-mon[179668]: failed to encode map e624 with expected crc 2023-09-23T18:21:42.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:42 smithi055 ceph-mon[184174]: pgmap v1514: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 11 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 725 op/s; 13318/429 objects degraded (3104.429%); 0 B/s, 54 objects/s recovering 2023-09-23T18:21:42.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:42 smithi055 ceph-mon[184174]: failed to encode map e624 with expected crc 2023-09-23T18:21:42.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:42 smithi055 ceph-mon[184174]: osdmap e624: 8 total, 8 up, 6 in 2023-09-23T18:21:42.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:42 smithi055 ceph-mon[184174]: failed to encode map e624 with expected crc 2023-09-23T18:21:42.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:42 smithi055 ceph-mon[184174]: failed to encode map e624 with expected crc 2023-09-23T18:21:42.948 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:42.951 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:42.951 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:43.018 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.4 dump_historic_ops 2023-09-23T18:21:43.053 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:43.084 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:43.451 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:43.453 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:43.454 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:43.454 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:43.555 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:43.582 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:43.612 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:21:43.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:43.622 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:43.623 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:43.724 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:43.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:43 smithi055 ceph-mon[184174]: failed to encode map e625 with expected crc 2023-09-23T18:21:43.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:43 smithi055 ceph-mon[184174]: osdmap e625: 8 total, 8 up, 6 in 2023-09-23T18:21:43.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:43 smithi055 ceph-mon[184174]: failed to encode map e625 with expected crc 2023-09-23T18:21:43.751 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:43 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 13318/429 objects degraded (3104.429%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:43.752 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3318270707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:21:43.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:43 smithi055 ceph-mon[179668]: failed to encode map e625 with expected crc 2023-09-23T18:21:43.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:43 smithi055 ceph-mon[179668]: osdmap e625: 8 total, 8 up, 6 in 2023-09-23T18:21:43.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:43 smithi055 ceph-mon[179668]: failed to encode map e625 with expected crc 2023-09-23T18:21:43.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:43 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 13318/429 objects degraded (3104.429%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:43.752 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3318270707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:21:43.754 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:43.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:43 smithi099 ceph-mon[167076]: failed to encode map e625 with expected crc 2023-09-23T18:21:43.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:43 smithi099 ceph-mon[167076]: osdmap e625: 8 total, 8 up, 6 in 2023-09-23T18:21:43.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:43 smithi099 ceph-mon[167076]: failed to encode map e625 with expected crc 2023-09-23T18:21:43.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:43 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 13318/429 objects degraded (3104.429%), 5 pgs degraded, 5 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:43.848 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3318270707' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:21:44.243 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:44.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:44.248 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:44.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:44.379 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:44.421 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:44.425 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:44.426 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:44.446 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:21:44.526 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:44.555 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:44.640 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:44.644 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:44.644 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:44.691 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:44 smithi099 ceph-mon[167076]: failed to encode map e625 with expected crc 2023-09-23T18:21:44.691 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:44 smithi099 ceph-mon[167076]: pgmap v1517: 119 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 12005/81 objects degraded (14820.988%); 0 B/s, 218 objects/s recovering 2023-09-23T18:21:44.691 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:44 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:44.691 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3318270707' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:21:44.691 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:44 smithi099 ceph-mon[167076]: osdmap e626: 8 total, 8 up, 6 in 2023-09-23T18:21:44.727 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:44 smithi055 ceph-mon[184174]: failed to encode map e625 with expected crc 2023-09-23T18:21:44.728 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:44 smithi055 ceph-mon[184174]: pgmap v1517: 119 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 12005/81 objects degraded (14820.988%); 0 B/s, 218 objects/s recovering 2023-09-23T18:21:44.728 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:44 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:44.728 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3318270707' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:21:44.728 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:44 smithi055 ceph-mon[184174]: osdmap e626: 8 total, 8 up, 6 in 2023-09-23T18:21:44.728 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:44 smithi055 ceph-mon[179668]: failed to encode map e625 with expected crc 2023-09-23T18:21:44.728 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:44 smithi055 ceph-mon[179668]: pgmap v1517: 119 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 92 op/s; 12005/81 objects degraded (14820.988%); 0 B/s, 218 objects/s recovering 2023-09-23T18:21:44.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:44 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:44.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3318270707' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:21:44.729 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:44 smithi055 ceph-mon[179668]: osdmap e626: 8 total, 8 up, 6 in 2023-09-23T18:21:44.745 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:44.778 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:44.847 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:21:44.847 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:21:44.847 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:21:44.848 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:21:44.848 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.848 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177038 8.2 8:4ced54d7:::benchmark_data_smithi055_346788_object66446:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T18:21:44.848 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:21:44.848 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 553.56974742499995, 2023-09-23T18:21:44.848 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1583805069999999, 2023-09-23T18:21:44.848 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.848 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.849 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.849 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:21:44.849 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:21:44.849 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177038 2023-09-23T18:21:44.849 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.849 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.849 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.849 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.849 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:21:44.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.850 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275128+0000", 2023-09-23T18:21:44.850 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.850 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.850 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.850 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275127+0000", 2023-09-23T18:21:44.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:44.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.851 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.851 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275133+0000", 2023-09-23T18:21:44.851 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5910000000000001e-06 2023-09-23T18:21:44.851 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.851 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275146+0000", 2023-09-23T18:21:44.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2302e-05 2023-09-23T18:21:44.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.852 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.852 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.852 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.275163+0000", 2023-09-23T18:21:44.852 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.7689e-05 2023-09-23T18:21:44.852 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.282482+0000", 2023-09-23T18:21:44.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0073186709999999997 2023-09-23T18:21:44.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.853 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.853 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.853 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.282498+0000", 2023-09-23T18:21:44.853 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5841999999999998e-05 2023-09-23T18:21:44.853 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430240+0000", 2023-09-23T18:21:44.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1477423680000001 2023-09-23T18:21:44.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.854 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.854 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.854 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430343+0000", 2023-09-23T18:21:44.854 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102289 2023-09-23T18:21:44.854 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,5,6", 2023-09-23T18:21:44.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.430425+0000", 2023-09-23T18:21:44.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2150000000000005e-05 2023-09-23T18:21:44.855 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.855 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.855 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.855 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.432859+0000", 2023-09-23T18:21:44.855 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024338519999999998 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.432958+0000", 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9440999999999999e-05 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433088+0000", 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013031100000000001 2023-09-23T18:21:44.856 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433396+0000", 2023-09-23T18:21:44.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00030761699999999999 2023-09-23T18:21:44.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.857 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.857 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.857 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433437+0000", 2023-09-23T18:21:44.857 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1242999999999998e-05 2023-09-23T18:21:44.857 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.858 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.858 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.433509+0000", 2023-09-23T18:21:44.858 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1559999999999999e-05 2023-09-23T18:21:44.858 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.858 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.858 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.858 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.858 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.859 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:213401 8.e 8:7ecd3558:::benchmark_data_smithi055_346788_object102809:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e444)", 2023-09-23T18:21:44.859 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:21:44.859 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 504.60125758100003, 2023-09-23T18:21:44.859 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0290741050000001, 2023-09-23T18:21:44.859 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.859 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.859 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.859 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:21:44.859 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:21:44.859 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 213401 2023-09-23T18:21:44.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.860 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:21:44.860 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.860 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.860 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.860 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.860 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:21:44.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243618+0000", 2023-09-23T18:21:44.861 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:44.861 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.861 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.861 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.861 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243620+0000", 2023-09-23T18:21:44.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.9649999999999998e-06 2023-09-23T18:21:44.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.862 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243623+0000", 2023-09-23T18:21:44.862 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6179999999999999e-06 2023-09-23T18:21:44.862 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.862 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.862 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243626+0000", 2023-09-23T18:21:44.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.294e-06 2023-09-23T18:21:44.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.863 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.863 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243677+0000", 2023-09-23T18:21:44.863 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1668000000000001e-05 2023-09-23T18:21:44.863 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.863 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:20.243686+0000", 2023-09-23T18:21:44.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7730000000000005e-06 2023-09-23T18:21:44.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.864 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.864 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.235935+0000", 2023-09-23T18:21:44.864 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99224838100000001 2023-09-23T18:21:44.864 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.864 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.236043+0000", 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108855 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 2,7", 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.236169+0000", 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012573700000000001 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.865 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.253075+0000", 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016906304 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272511+0000", 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019435859 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272626+0000", 2023-09-23T18:21:44.866 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115215 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272667+0000", 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0590000000000003e-05 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:21.272692+0000", 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5111999999999999e-05 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.867 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121636 9.f 9:f1d044dd:::benchmark_data_smithi055_386467_object16976:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.695760+0000", 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 368.14911591100002, 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0409455569999999, 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.868 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:21:44.869 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:21:44.869 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121636 2023-09-23T18:21:44.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.869 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.869 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.869 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695760+0000", 2023-09-23T18:21:44.869 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.869 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.869 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695760+0000", 2023-09-23T18:21:44.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.870 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.870 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695759+0000", 2023-09-23T18:21:44.870 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:44.870 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.870 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695762+0000", 2023-09-23T18:21:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.4040000000000002e-06 2023-09-23T18:21:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695768+0000", 2023-09-23T18:21:44.871 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1099999999999999e-06 2023-09-23T18:21:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695772+0000", 2023-09-23T18:21:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0579999999999999e-06 2023-09-23T18:21:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.872 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.698805+0000", 2023-09-23T18:21:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030330489999999999 2023-09-23T18:21:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.698810+0000", 2023-09-23T18:21:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5439999999999998e-06 2023-09-23T18:21:44.873 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.717639+0000", 2023-09-23T18:21:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.018828426 2023-09-23T18:21:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.874 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.718020+0000", 2023-09-23T18:21:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00038104099999999999 2023-09-23T18:21:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:21:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.718155+0000", 2023-09-23T18:21:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000134618 2023-09-23T18:21:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.875 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.725686+0000", 2023-09-23T18:21:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0075315429999999999 2023-09-23T18:21:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.735765+0000", 2023-09-23T18:21:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010079210999999999 2023-09-23T18:21:44.876 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.736527+0000", 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00076177799999999996 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.736593+0000", 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6124000000000006e-05 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.877 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.736705+0000", 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011207 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:122388 9.f 9:fdc2bcfb:::benchmark_data_smithi055_386467_object17728:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:21:44.878 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 356.583697374, 2023-09-23T18:21:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0337659729999999, 2023-09-23T18:21:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:21:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:21:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122388 2023-09-23T18:21:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.879 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:21:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.880 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:21:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261178+0000", 2023-09-23T18:21:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.881 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261180+0000", 2023-09-23T18:21:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.057e-06 2023-09-23T18:21:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261187+0000", 2023-09-23T18:21:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2559999999999996e-06 2023-09-23T18:21:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.882 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261191+0000", 2023-09-23T18:21:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4089999999999999e-06 2023-09-23T18:21:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261250+0000", 2023-09-23T18:21:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9678e-05 2023-09-23T18:21:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.883 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op must wait for map", 2023-09-23T18:21:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.261258+0000", 2023-09-23T18:21:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3209999999999999e-06 2023-09-23T18:21:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.278415+0000", 2023-09-23T18:21:44.884 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.01715761 2023-09-23T18:21:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.278423+0000", 2023-09-23T18:21:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.8089999999999993e-06 2023-09-23T18:21:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.287981+0000", 2023-09-23T18:21:44.885 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0095577870000001 2023-09-23T18:21:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.288127+0000", 2023-09-23T18:21:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014562099999999999 2023-09-23T18:21:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:21:44.886 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.288257+0000", 2023-09-23T18:21:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000129982 2023-09-23T18:21:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294755+0000", 2023-09-23T18:21:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064986159999999996 2023-09-23T18:21:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.887 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294764+0000", 2023-09-23T18:21:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.4689999999999997e-06 2023-09-23T18:21:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294789+0000", 2023-09-23T18:21:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5245e-05 2023-09-23T18:21:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.888 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294838+0000", 2023-09-23T18:21:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9027999999999999e-05 2023-09-23T18:21:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294944+0000", 2023-09-23T18:21:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000106378 2023-09-23T18:21:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.889 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:122391 9.f 9:fd5978e6:::benchmark_data_smithi055_386467_object17731:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e504)", 2023-09-23T18:21:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:48.272754+0000", 2023-09-23T18:21:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 356.572121396, 2023-09-23T18:21:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0225196940000001, 2023-09-23T18:21:44.890 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:21:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:21:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 122391 2023-09-23T18:21:44.893 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272754+0000", 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272754+0000", 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.894 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272753+0000", 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.272758+0000", 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9169999999999997e-06 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.895 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.275901+0000", 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031430659999999999 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.275910+0000", 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0569999999999999e-06 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.279182+0000", 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0032713920000000001 2023-09-23T18:21:44.896 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:48.279188+0000", 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.302e-06 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.290722+0000", 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.011533781 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.897 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.290834+0000", 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011197299999999999 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,7", 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.290966+0000", 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000132531 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.294997+0000", 2023-09-23T18:21:44.898 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0040308899999999996 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295070+0000", 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.3144000000000003e-05 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295128+0000", 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8236000000000003e-05 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.899 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295169+0000", 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0176000000000003e-05 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:49.295274+0000", 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010541 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.900 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199300 9.c 9:3f680051:::benchmark_data_smithi055_386467_object94640:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.729844+0000", 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.115032068, 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.107413183, 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199300 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.901 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729844+0000", 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729844+0000", 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.902 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729842+0000", 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729848+0000", 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4229999999999996e-06 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.903 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729858+0000", 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0064999999999999e-05 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729864+0000", 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2550000000000003e-06 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.732111+0000", 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0022468330000000002 2023-09-23T18:21:44.904 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.732247+0000", 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013641200000000001 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,6,7", 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.732401+0000", 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015389900000000001 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.905 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.764773+0000", 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.032371687000000003 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.764782+0000", 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7350000000000002e-06 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.906 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.820636+0000", 2023-09-23T18:21:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0558545260000001 2023-09-23T18:21:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:21:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.820762+0000", 2023-09-23T18:21:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000125988 2023-09-23T18:21:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.907 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.837194+0000", 2023-09-23T18:21:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.016432303999999998 2023-09-23T18:21:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.837257+0000", 2023-09-23T18:21:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2335000000000005e-05 2023-09-23T18:21:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.908 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:199301 9.c 9:3dcb5cd4:::benchmark_data_smithi055_386467_object94641:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e544)", 2023-09-23T18:21:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:19.729873+0000", 2023-09-23T18:21:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 265.11500220900001, 2023-09-23T18:21:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.107041538, 2023-09-23T18:21:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.909 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:21:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:21:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 199301 2023-09-23T18:21:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729873+0000", 2023-09-23T18:21:44.910 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729873+0000", 2023-09-23T18:21:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.911 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729873+0000", 2023-09-23T18:21:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729876+0000", 2023-09-23T18:21:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.813e-06 2023-09-23T18:21:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.912 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729882+0000", 2023-09-23T18:21:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7059999999999998e-06 2023-09-23T18:21:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.729887+0000", 2023-09-23T18:21:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2910000000000001e-06 2023-09-23T18:21:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.913 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.764844+0000", 2023-09-23T18:21:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.034956813000000003 2023-09-23T18:21:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:19.764847+0000", 2023-09-23T18:21:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8670000000000004e-06 2023-09-23T18:21:44.914 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.820787+0000", 2023-09-23T18:21:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.055939384 2023-09-23T18:21:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.820905+0000", 2023-09-23T18:21:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117979 2023-09-23T18:21:44.915 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,6", 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.821043+0000", 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013790800000000001 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.835902+0000", 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.014859659000000001 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.916 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.836434+0000", 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00053201699999999997 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.836466+0000", 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1909999999999998e-05 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.836520+0000", 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3516000000000001e-05 2023-09-23T18:21:44.917 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:20.836915+0000", 2023-09-23T18:21:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00039519400000000002 2023-09-23T18:21:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.918 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:201222 9.4 9:27ad1e07:::benchmark_data_smithi055_386467_object96562:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e550)", 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:25.790582+0000", 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 259.05429317800002, 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0766256249999999, 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201222 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.919 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790582+0000", 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790582+0000", 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790581+0000", 2023-09-23T18:21:44.920 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790589+0000", 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6750000000000002e-06 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790601+0000", 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2081e-05 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.921 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790608+0000", 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9450000000000004e-06 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790662+0000", 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4673000000000003e-05 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.922 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790781+0000", 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000118943 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.790909+0000", 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012749300000000001 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.794044+0000", 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0031351450000000002 2023-09-23T18:21:44.923 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.802883+0000", 2023-09-23T18:21:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0088395729999999999 2023-09-23T18:21:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.802892+0000", 2023-09-23T18:21:44.924 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.5140000000000001e-06 2023-09-23T18:21:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.848461+0000", 2023-09-23T18:21:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.045569451 2023-09-23T18:21:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:21:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.848575+0000", 2023-09-23T18:21:44.925 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113906 2023-09-23T18:21:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.867156+0000", 2023-09-23T18:21:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.018581033 2023-09-23T18:21:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.926 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.867208+0000", 2023-09-23T18:21:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1820999999999997e-05 2023-09-23T18:21:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:201232 9.4 9:23829d9b:::benchmark_data_smithi055_386467_object96572:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T18:21:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:25.806549+0000", 2023-09-23T18:21:44.927 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 259.03832632500001, 2023-09-23T18:21:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.05999021, 2023-09-23T18:21:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:21:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:21:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201232 2023-09-23T18:21:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.928 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806549+0000", 2023-09-23T18:21:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806549+0000", 2023-09-23T18:21:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.929 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806549+0000", 2023-09-23T18:21:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806552+0000", 2023-09-23T18:21:44.930 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9620000000000002e-06 2023-09-23T18:21:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806567+0000", 2023-09-23T18:21:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5602000000000001e-05 2023-09-23T18:21:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.931 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806572+0000", 2023-09-23T18:21:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0170000000000004e-06 2023-09-23T18:21:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806696+0000", 2023-09-23T18:21:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000123621 2023-09-23T18:21:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.932 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.806703+0000", 2023-09-23T18:21:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0559999999999999e-06 2023-09-23T18:21:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.848619+0000", 2023-09-23T18:21:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0419157489999999 2023-09-23T18:21:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.933 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.848707+0000", 2023-09-23T18:21:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.7841000000000001e-05 2023-09-23T18:21:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:21:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.848795+0000", 2023-09-23T18:21:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8675000000000001e-05 2023-09-23T18:21:44.934 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.859785+0000", 2023-09-23T18:21:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010989103 2023-09-23T18:21:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.865959+0000", 2023-09-23T18:21:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0061744790000000001 2023-09-23T18:21:44.935 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866346+0000", 2023-09-23T18:21:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00038720099999999999 2023-09-23T18:21:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866412+0000", 2023-09-23T18:21:44.936 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6030000000000003e-05 2023-09-23T18:21:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866540+0000", 2023-09-23T18:21:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012732299999999999 2023-09-23T18:21:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.937 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:201265 9.4 9:295971ad:::benchmark_data_smithi055_386467_object96605:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:25.842742+0000", 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 259.002133249, 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0239629960000001, 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201265 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.938 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842742+0000", 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842742+0000", 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.939 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842741+0000", 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842748+0000", 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.5810000000000002e-06 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842760+0000", 2023-09-23T18:21:44.940 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1878999999999999e-05 2023-09-23T18:21:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842767+0000", 2023-09-23T18:21:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9820000000000002e-06 2023-09-23T18:21:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842814+0000", 2023-09-23T18:21:44.941 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7670000000000003e-05 2023-09-23T18:21:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.842821+0000", 2023-09-23T18:21:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9979999999999997e-06 2023-09-23T18:21:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.942 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.850540+0000", 2023-09-23T18:21:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0077192370000001 2023-09-23T18:21:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.850617+0000", 2023-09-23T18:21:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.6699000000000005e-05 2023-09-23T18:21:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.943 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:21:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.850688+0000", 2023-09-23T18:21:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0770000000000002e-05 2023-09-23T18:21:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.862765+0000", 2023-09-23T18:21:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012077044 2023-09-23T18:21:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.944 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866147+0000", 2023-09-23T18:21:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033823690000000001 2023-09-23T18:21:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866574+0000", 2023-09-23T18:21:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00042635999999999998 2023-09-23T18:21:44.945 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866606+0000", 2023-09-23T18:21:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.2583000000000003e-05 2023-09-23T18:21:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866705+0000", 2023-09-23T18:21:44.946 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9137999999999998e-05 2023-09-23T18:21:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:201274 9.4 9:29d51ac9:::benchmark_data_smithi055_386467_object96614:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T18:21:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:25.853318+0000", 2023-09-23T18:21:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 258.99155743300003, 2023-09-23T18:21:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.014229311, 2023-09-23T18:21:44.947 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:21:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:21:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201274 2023-09-23T18:21:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.948 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853318+0000", 2023-09-23T18:21:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853318+0000", 2023-09-23T18:21:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.949 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853318+0000", 2023-09-23T18:21:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853321+0000", 2023-09-23T18:21:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3160000000000001e-06 2023-09-23T18:21:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.950 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853328+0000", 2023-09-23T18:21:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9179999999999997e-06 2023-09-23T18:21:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853333+0000", 2023-09-23T18:21:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6149999999999999e-06 2023-09-23T18:21:44.951 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853384+0000", 2023-09-23T18:21:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1100999999999997e-05 2023-09-23T18:21:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.853392+0000", 2023-09-23T18:21:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6629999999999999e-06 2023-09-23T18:21:44.952 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.851766+0000", 2023-09-23T18:21:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99837366299999997 2023-09-23T18:21:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.851835+0000", 2023-09-23T18:21:44.953 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9090000000000004e-05 2023-09-23T18:21:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:21:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.851910+0000", 2023-09-23T18:21:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4946999999999999e-05 2023-09-23T18:21:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.954 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.862769+0000", 2023-09-23T18:21:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010859291 2023-09-23T18:21:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866196+0000", 2023-09-23T18:21:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034264439999999998 2023-09-23T18:21:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.955 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.867401+0000", 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0012055989999999999 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.867456+0000", 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4262000000000003e-05 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.956 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.867548+0000", 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1908000000000002e-05 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:201278 9.4 9:26c0438d:::benchmark_data_smithi055_386467_object96618:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e551)", 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:17:25.859092+0000", 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 258.98578368099999, 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0122590970000001, 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.957 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:21:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:21:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201278 2023-09-23T18:21:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.958 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859092+0000", 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859092+0000", 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859091+0000", 2023-09-23T18:21:44.959 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859097+0000", 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4430000000000002e-06 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859111+0000", 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4606e-05 2023-09-23T18:21:44.960 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859118+0000", 2023-09-23T18:21:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8539999999999999e-06 2023-09-23T18:21:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859164+0000", 2023-09-23T18:21:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.6180000000000002e-05 2023-09-23T18:21:44.961 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:21:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:25.859170+0000", 2023-09-23T18:21:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1410000000000002e-06 2023-09-23T18:21:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.854090+0000", 2023-09-23T18:21:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.99491981600000001 2023-09-23T18:21:44.962 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.854152+0000", 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1598000000000002e-05 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.854226+0000", 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.4345999999999996e-05 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.963 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.862777+0000", 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085508010000000002 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.866272+0000", 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0034955300000000002 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.871174+0000", 2023-09-23T18:21:44.964 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0049011030000000004 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.871234+0000", 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0041000000000003e-05 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:17:26.871351+0000", 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117516 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.965 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:81799 10.1 10:83581c68:::benchmark_data_smithi055_425050_object81798:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e601)", 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:19:05.634939+0000", 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 159.20993638300001, 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.058091700000000003, 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:21:44.966 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 81799 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.634939+0000", 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.634939+0000", 2023-09-23T18:21:44.967 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.634939+0000", 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.634974+0000", 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.5546000000000002e-05 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.968 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.634982+0000", 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9719999999999992e-06 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.634987+0000", 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0359999999999997e-06 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.637965+0000", 2023-09-23T18:21:44.969 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0029773600000000001 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.638062+0000", 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.7757000000000006e-05 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.638172+0000", 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010997000000000001 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.970 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.646055+0000", 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0078822609999999998 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.665633+0000", 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.019578755 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.971 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.692939+0000", 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.027305407 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.692990+0000", 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1177000000000001e-05 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.693031+0000", 2023-09-23T18:21:44.972 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1013999999999997e-05 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:81807 10.1 10:83001627:::benchmark_data_smithi055_425050_object81806:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e601)", 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:19:05.634993+0000", 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 159.20988270500001, 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.058191947000000001, 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.973 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 81807 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.634993+0000", 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.974 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.634993+0000", 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.634992+0000", 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.636954+0000", 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0019619289999999998 2023-09-23T18:21:44.975 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.636970+0000", 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5265e-05 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.636976+0000", 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6390000000000003e-06 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.976 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.640379+0000", 2023-09-23T18:21:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00340296 2023-09-23T18:21:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.640492+0000", 2023-09-23T18:21:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000113089 2023-09-23T18:21:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.977 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:21:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.640636+0000", 2023-09-23T18:21:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014394700000000001 2023-09-23T18:21:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.648090+0000", 2023-09-23T18:21:44.978 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0074538920000000002 2023-09-23T18:21:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.668423+0000", 2023-09-23T18:21:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020333169000000002 2023-09-23T18:21:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.979 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.693089+0000", 2023-09-23T18:21:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.024665840000000001 2023-09-23T18:21:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.693139+0000", 2023-09-23T18:21:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0281999999999997e-05 2023-09-23T18:21:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.980 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:05.693185+0000", 2023-09-23T18:21:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5478000000000001e-05 2023-09-23T18:21:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:83547 10.1 10:8bcb2db3:::benchmark_data_smithi055_425050_object83546:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T18:21:44.981 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:19:07.822990+0000", 2023-09-23T18:21:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 157.021885443, 2023-09-23T18:21:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.068002447999999993, 2023-09-23T18:21:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:21:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 83547 2023-09-23T18:21:44.982 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.822990+0000", 2023-09-23T18:21:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.822990+0000", 2023-09-23T18:21:44.983 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.822989+0000", 2023-09-23T18:21:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.984 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.823043+0000", 2023-09-23T18:21:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.4177e-05 2023-09-23T18:21:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.823056+0000", 2023-09-23T18:21:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3601000000000001e-05 2023-09-23T18:21:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.985 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.823063+0000", 2023-09-23T18:21:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2670000000000003e-06 2023-09-23T18:21:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.825538+0000", 2023-09-23T18:21:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.002474871 2023-09-23T18:21:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.986 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.825664+0000", 2023-09-23T18:21:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012648299999999999 2023-09-23T18:21:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:21:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.825804+0000", 2023-09-23T18:21:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014033200000000001 2023-09-23T18:21:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.987 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.828508+0000", 2023-09-23T18:21:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0027036230000000001 2023-09-23T18:21:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.833779+0000", 2023-09-23T18:21:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0052715069999999999 2023-09-23T18:21:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.988 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.890888+0000", 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.057108881 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.890948+0000", 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.0102999999999997e-05 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.890993+0000", 2023-09-23T18:21:44.989 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4211000000000001e-05 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:83559 10.1 10:8df92322:::benchmark_data_smithi055_425050_object83558:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e602)", 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:19:07.838667+0000", 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 157.006208953, 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.056052562, 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.990 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 83559 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.838667+0000", 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.991 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.838667+0000", 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.838665+0000", 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.838709+0000", 2023-09-23T18:21:44.992 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4385999999999998e-05 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.838721+0000", 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1813e-05 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.838728+0000", 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.3249999999999996e-06 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.993 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.838772+0000", 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4864999999999998e-05 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.838880+0000", 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000107798 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:21:44.994 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.838980+0000", 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000100227 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.844161+0000", 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051804980000000004 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.886996+0000", 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.042835520000000002 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.995 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.894619+0000", 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0076227689999999997 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.894676+0000", 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7188999999999997e-05 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:44.996 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:19:07.894719+0000", 2023-09-23T18:21:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2833000000000001e-05 2023-09-23T18:21:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:138265 10.1 10:89ae1573:::benchmark_data_smithi055_425050_object34859:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e610)", 2023-09-23T18:21:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:20:13.020084+0000", 2023-09-23T18:21:44.997 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 91.824792032000005, 2023-09-23T18:21:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.117597984, 2023-09-23T18:21:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:21:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 138265 2023-09-23T18:21:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.998 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.020084+0000", 2023-09-23T18:21:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.020084+0000", 2023-09-23T18:21:44.999 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.020082+0000", 2023-09-23T18:21:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:21:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.020090+0000", 2023-09-23T18:21:45.000 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9680000000000006e-06 2023-09-23T18:21:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.020105+0000", 2023-09-23T18:21:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5257e-05 2023-09-23T18:21:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.020112+0000", 2023-09-23T18:21:45.001 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6259999999999997e-06 2023-09-23T18:21:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.020468+0000", 2023-09-23T18:21:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00035647700000000001 2023-09-23T18:21:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:45.002 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.020600+0000", 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013196900000000001 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.020696+0000", 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5958999999999994e-05 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.023165+0000", 2023-09-23T18:21:45.003 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024691169999999998 2023-09-23T18:21:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.023359+0000", 2023-09-23T18:21:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019400799999999999 2023-09-23T18:21:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:45.004 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.137596+0000", 2023-09-23T18:21:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.114237143 2023-09-23T18:21:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.137645+0000", 2023-09-23T18:21:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.8656000000000003e-05 2023-09-23T18:21:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.005 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:20:13.137682+0000", 2023-09-23T18:21:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6538999999999997e-05 2023-09-23T18:21:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:201067 10.1 10:811531a6:::benchmark_data_smithi055_425050_object97661:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T18:21:45.006 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:21:29.185256+0000", 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.659619600999999, 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.098691982999999997, 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201067 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:45.007 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.185256+0000", 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.185256+0000", 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:45.008 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.185255+0000", 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.185261+0000", 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2009999999999996e-06 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.185278+0000", 2023-09-23T18:21:45.009 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.6877000000000001e-05 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.185284+0000", 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.652e-06 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.185330+0000", 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5707000000000001e-05 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.010 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.185434+0000", 2023-09-23T18:21:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000104471 2023-09-23T18:21:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.011 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:21:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.185534+0000", 2023-09-23T18:21:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.9216999999999996e-05 2023-09-23T18:21:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.188234+0000", 2023-09-23T18:21:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026999210000000001 2023-09-23T18:21:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.012 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.192718+0000", 2023-09-23T18:21:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044847369999999999 2023-09-23T18:21:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.283872+0000", 2023-09-23T18:21:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.091153333000000003 2023-09-23T18:21:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.013 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.283919+0000", 2023-09-23T18:21:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7800000000000003e-05 2023-09-23T18:21:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.283948+0000", 2023-09-23T18:21:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.8606e-05 2023-09-23T18:21:45.014 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:201549 10.1 10:8b214139:::benchmark_data_smithi055_425050_object98143:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T18:21:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:21:29.829612+0000", 2023-09-23T18:21:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 15.015263998, 2023-09-23T18:21:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.048612992000000001, 2023-09-23T18:21:45.015 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:21:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 201549 2023-09-23T18:21:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:45.016 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.829612+0000", 2023-09-23T18:21:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.829612+0000", 2023-09-23T18:21:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.017 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.829610+0000", 2023-09-23T18:21:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:21:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.829616+0000", 2023-09-23T18:21:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6590000000000004e-06 2023-09-23T18:21:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.018 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.829627+0000", 2023-09-23T18:21:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1049999999999999e-05 2023-09-23T18:21:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.829633+0000", 2023-09-23T18:21:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7620000000000001e-06 2023-09-23T18:21:45.019 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.829736+0000", 2023-09-23T18:21:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000102917 2023-09-23T18:21:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.829841+0000", 2023-09-23T18:21:45.020 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000105886 2023-09-23T18:21:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:21:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.829947+0000", 2023-09-23T18:21:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010526000000000001 2023-09-23T18:21:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.832362+0000", 2023-09-23T18:21:45.021 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0024155489999999999 2023-09-23T18:21:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.833593+0000", 2023-09-23T18:21:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001231132 2023-09-23T18:21:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:45.022 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.878143+0000", 2023-09-23T18:21:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.044549547000000002 2023-09-23T18:21:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.878186+0000", 2023-09-23T18:21:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3443999999999999e-05 2023-09-23T18:21:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.023 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:29.878225+0000", 2023-09-23T18:21:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.8296000000000001e-05 2023-09-23T18:21:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.46799.0:203455 10.1 10:88017b21:::benchmark_data_smithi055_425050_object100049:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T18:21:45.024 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:21:32.132544+0000", 2023-09-23T18:21:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 12.712331352, 2023-09-23T18:21:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.041694653999999998, 2023-09-23T18:21:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:21:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:21:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:21:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.46799", 2023-09-23T18:21:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:21:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 203455 2023-09-23T18:21:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.025 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.132544+0000", 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.132544+0000", 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.026 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.132544+0000", 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.132547+0000", 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.9299999999999999e-06 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.027 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:21:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.132562+0000", 2023-09-23T18:21:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.4899000000000001e-05 2023-09-23T18:21:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:21:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.132566+0000", 2023-09-23T18:21:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5020000000000003e-06 2023-09-23T18:21:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.028 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:21:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.135164+0000", 2023-09-23T18:21:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0025976929999999999 2023-09-23T18:21:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:21:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.135276+0000", 2023-09-23T18:21:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000111774 2023-09-23T18:21:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.029 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1", 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.135384+0000", 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010846200000000001 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.137471+0000", 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0020864579999999998 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:45.030 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.147421+0000", 2023-09-23T18:21:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0099506070000000002 2023-09-23T18:21:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:21:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.174168+0000", 2023-09-23T18:21:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.026746748000000001 2023-09-23T18:21:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:21:45.031 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.174202+0000", 2023-09-23T18:21:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.4416999999999999e-05 2023-09-23T18:21:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:21:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:21:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:21:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:21:32.174239+0000", 2023-09-23T18:21:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6538999999999997e-05 2023-09-23T18:21:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:45.032 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:21:45.033 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:21:45.033 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:21:45.033 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:45.034 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:45.034 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:45.135 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:45.169 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:45.266 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_ops_in_flight 2023-09-23T18:21:45.305 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:45.314 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:45.319 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:45.319 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:45.420 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:45.458 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:45.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:45 smithi055 ceph-mon[179668]: failed to encode map e626 with expected crc 2023-09-23T18:21:45.865 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:21:45.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:45 smithi055 ceph-mon[184174]: failed to encode map e626 with expected crc 2023-09-23T18:21:46.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:45 smithi099 ceph-mon[167076]: failed to encode map e626 with expected crc 2023-09-23T18:21:46.658 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:46 smithi099 ceph-mon[167076]: pgmap v1519: 119 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 38 op/s; 12005/81 objects degraded (14820.988%); 0 B/s, 218 objects/s recovering 2023-09-23T18:21:46.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:46 smithi055 ceph-mon[184174]: pgmap v1519: 119 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 38 op/s; 12005/81 objects degraded (14820.988%); 0 B/s, 218 objects/s recovering 2023-09-23T18:21:46.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:46 smithi055 ceph-mon[179668]: pgmap v1519: 119 pgs: 1 active+undersized+remapped, 4 active+recovery_wait+undersized+degraded+remapped, 114 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 38 op/s; 12005/81 objects degraded (14820.988%); 0 B/s, 218 objects/s recovering 2023-09-23T18:21:46.965 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:46.969 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:46.969 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:47.059 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:21:47.059 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:21:47.059 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:21:47.059 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:21:47.070 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:47.072 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:21:47.105 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:47.474 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:47.479 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:47.479 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:47.510 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_blocked_ops 2023-09-23T18:21:47.580 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:47.608 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:47.643 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:47.646 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:47.646 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:47.747 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:47.773 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:47.806 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:47.809 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:47.809 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:47.910 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:47.958 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:48.113 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:21:48.371 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:48.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:48.376 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:48.477 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:48.548 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:48.745 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:48.750 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:48.751 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:48.751 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:48.852 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:48.878 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:48 smithi099 ceph-mon[167076]: pgmap v1520: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 19 op/s; 11521/81 objects degraded (14223.457%); 0 B/s, 299 objects/s recovering 2023-09-23T18:21:48.879 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:48 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 11521/81 objects degraded (14223.457%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:48.882 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:48.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:48 smithi055 ceph-mon[179668]: pgmap v1520: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 19 op/s; 11521/81 objects degraded (14223.457%); 0 B/s, 299 objects/s recovering 2023-09-23T18:21:48.900 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:48 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 11521/81 objects degraded (14223.457%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:48.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:48 smithi055 ceph-mon[184174]: pgmap v1520: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 0 B/s wr, 19 op/s; 11521/81 objects degraded (14223.457%); 0 B/s, 299 objects/s recovering 2023-09-23T18:21:48.901 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:48 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 11521/81 objects degraded (14223.457%), 4 pgs degraded, 4 pgs undersized (PG_DEGRADED) 2023-09-23T18:21:48.917 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:48.920 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:48.920 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:49.021 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:49.039 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:21:49.041 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:21:49.100 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:49.121 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:21:49.159 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:21:49.163 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:49.168 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:49.168 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:49.184 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:21:49.208 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:21:49.228 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:21:49.252 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:21:49.269 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:49.276 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:21:49.303 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:49.303 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:21:49.307 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:21:49.307 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:21:49.307 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:21:49.307 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:21:49.307 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:21:49.307 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:21:49.307 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:21:49.308 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:21:49.328 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:21:49.328 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:21:49.555 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:49.558 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:49.558 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:49.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1690996435' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:21:49.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:49 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:21:49.638 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:49 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2938541316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:49.654 INFO:teuthology.orchestra.run.smithi055.stderr:pool 'unique_pool_5' removed 2023-09-23T18:21:49.656 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1520,"stamp":"2023-09-23T18:21:47.220893+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4471460,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11521,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":5960,"num_read_kb":12161,"num_write":207719,"num_write_kb":6630555,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12032,"num_bytes_recovered":502226569,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":83030,"ondisk_log_size":83030,"up":357,"acting":353,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":377,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5411576,"kb_used_data":1718152,"kb_used_omap":40,"kb_used_meta":3693335,"kb_avail":557059336,"statfs":{"total":575970213888,"available":570428760064,"internally_reserved":0,"allocated":1759387648,"data_stored":1744291098,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3781975354},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":-7536664,"num_objects":-116,"num_object_clones":0,"num_object_copies":-348,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-1797,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":-116,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":116,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1797,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001490"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":914,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590521+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:21:42.590521+0000","last_peered":"2023-09-23T18:21:42.590521+0000","last_clean":"2023-09-23T18:21:42.590521+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:21:42.590521+0000","last_undegraded":"2023-09-23T18:21:42.590521+0000","last_fullsized":"2023-09-23T18:21:42.590521+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":283,"reported_epoch":623,"state":"active+clean","last_fresh":"2023-09-23T18:21:37.698176+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:21:37.698176+0000","last_peered":"2023-09-23T18:21:37.698176+0000","last_clean":"2023-09-23T18:21:37.698176+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:21:37.698176+0000","last_undegraded":"2023-09-23T18:21:37.698176+0000","last_fullsized":"2023-09-23T18:21:37.698176+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":43,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.596087+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:21:42.596087+0000","last_peered":"2023-09-23T18:21:42.596087+0000","last_clean":"2023-09-23T18:21:42.596087+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:21:42.596087+0000","last_undegraded":"2023-09-23T18:21:42.596087+0000","last_fullsized":"2023-09-23T18:21:42.596087+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":920,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590764+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:21:42.590764+0000","last_peered":"2023-09-23T18:21:42.590764+0000","last_clean":"2023-09-23T18:21:42.590764+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:21:42.590764+0000","last_undegraded":"2023-09-23T18:21:42.590764+0000","last_fullsized":"2023-09-23T18:21:42.590764+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":367,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.594951+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:21:42.594951+0000","last_peered":"2023-09-23T18:21:42.594951+0000","last_clean":"2023-09-23T18:21:42.594951+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:21:42.594951+0000","last_undegraded":"2023-09-23T18:21:42.594951+0000","last_fullsized":"2023-09-23T18:21:42.594951+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":687,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.599361+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:21:42.599361+0000","last_peered":"2023-09-23T18:21:42.599361+0000","last_clean":"2023-09-23T18:21:42.599361+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:21:42.599361+0000","last_undegraded":"2023-09-23T18:21:42.599361+0000","last_fullsized":"2023-09-23T18:21:42.599361+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":937,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590575+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:21:42.590575+0000","last_peered":"2023-09-23T18:21:42.590575+0000","last_clean":"2023-09-23T18:21:42.590575+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:21:42.590575+0000","last_undegraded":"2023-09-23T18:21:42.590575+0000","last_fullsized":"2023-09-23T18:21:42.590575+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":43,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.593989+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:21:42.593989+0000","last_peered":"2023-09-23T18:21:42.593989+0000","last_clean":"2023-09-23T18:21:42.593989+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:21:42.593989+0000","last_undegraded":"2023-09-23T18:21:42.593989+0000","last_fullsized":"2023-09-23T18:21:42.593989+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":202,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093451+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:21:43.093451+0000","last_peered":"2023-09-23T18:21:43.093451+0000","last_clean":"2023-09-23T18:21:43.093451+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:21:43.093451+0000","last_undegraded":"2023-09-23T18:21:43.093451+0000","last_fullsized":"2023-09-23T18:21:43.093451+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1060,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591020+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:21:42.591020+0000","last_peered":"2023-09-23T18:21:42.591020+0000","last_clean":"2023-09-23T18:21:42.591020+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:21:42.591020+0000","last_undegraded":"2023-09-23T18:21:42.591020+0000","last_fullsized":"2023-09-23T18:21:42.591020+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":136,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591301+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:21:42.591301+0000","last_peered":"2023-09-23T18:21:42.591301+0000","last_clean":"2023-09-23T18:21:42.591301+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:21:42.591301+0000","last_undegraded":"2023-09-23T18:21:42.591301+0000","last_fullsized":"2023-09-23T18:21:42.591301+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":216,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.594288+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:21:42.594288+0000","last_peered":"2023-09-23T18:21:42.594288+0000","last_clean":"2023-09-23T18:21:42.594288+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:21:42.594288+0000","last_undegraded":"2023-09-23T18:21:42.594288+0000","last_fullsized":"2023-09-23T18:21:42.594288+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":837,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092399+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:21:43.092399+0000","last_peered":"2023-09-23T18:21:43.092399+0000","last_clean":"2023-09-23T18:21:43.092399+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:21:43.092399+0000","last_undegraded":"2023-09-23T18:21:43.092399+0000","last_fullsized":"2023-09-23T18:21:43.092399+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":172,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591186+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:21:42.591186+0000","last_peered":"2023-09-23T18:21:42.591186+0000","last_clean":"2023-09-23T18:21:42.591186+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:21:42.591186+0000","last_undegraded":"2023-09-23T18:21:42.591186+0000","last_fullsized":"2023-09-23T18:21:42.591186+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":171,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092305+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:21:43.092305+0000","last_peered":"2023-09-23T18:21:43.092305+0000","last_clean":"2023-09-23T18:21:43.092305+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:21:43.092305+0000","last_undegraded":"2023-09-23T18:21:43.092305+0000","last_fullsized":"2023-09-23T18:21:43.092305+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":971,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591195+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:21:42.591195+0000","last_peered":"2023-09-23T18:21:42.591195+0000","last_clean":"2023-09-23T18:21:42.591195+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:21:42.591195+0000","last_undegraded":"2023-09-23T18:21:42.591195+0000","last_fullsized":"2023-09-23T18:21:42.591195+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":421,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.596345+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:21:42.596345+0000","last_peered":"2023-09-23T18:21:42.596345+0000","last_clean":"2023-09-23T18:21:42.596345+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:21:42.596345+0000","last_undegraded":"2023-09-23T18:21:42.596345+0000","last_fullsized":"2023-09-23T18:21:42.596345+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":439,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092592+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:21:43.092592+0000","last_peered":"2023-09-23T18:21:43.092592+0000","last_clean":"2023-09-23T18:21:43.092592+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:21:43.092592+0000","last_undegraded":"2023-09-23T18:21:43.092592+0000","last_fullsized":"2023-09-23T18:21:43.092592+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":999,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T18:21:41.585529+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:21:41.585529+0000","last_peered":"2023-09-23T18:21:41.585529+0000","last_clean":"2023-09-23T18:21:41.585529+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:21:41.585529+0000","last_undegraded":"2023-09-23T18:21:41.585529+0000","last_fullsized":"2023-09-23T18:21:41.585529+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":178,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093152+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:21:43.093152+0000","last_peered":"2023-09-23T18:21:43.093152+0000","last_clean":"2023-09-23T18:21:43.093152+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:21:43.093152+0000","last_undegraded":"2023-09-23T18:21:43.093152+0000","last_fullsized":"2023-09-23T18:21:43.093152+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":445,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092717+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:21:43.092717+0000","last_peered":"2023-09-23T18:21:43.092717+0000","last_clean":"2023-09-23T18:21:43.092717+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:21:43.092717+0000","last_undegraded":"2023-09-23T18:21:43.092717+0000","last_fullsized":"2023-09-23T18:21:43.092717+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":852,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590653+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:21:42.590653+0000","last_peered":"2023-09-23T18:21:42.590653+0000","last_clean":"2023-09-23T18:21:42.590653+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:21:42.590653+0000","last_undegraded":"2023-09-23T18:21:42.590653+0000","last_fullsized":"2023-09-23T18:21:42.590653+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":445,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092644+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:21:43.092644+0000","last_peered":"2023-09-23T18:21:43.092644+0000","last_clean":"2023-09-23T18:21:43.092644+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:21:43.092644+0000","last_undegraded":"2023-09-23T18:21:43.092644+0000","last_fullsized":"2023-09-23T18:21:43.092644+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":123,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590980+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:21:42.590980+0000","last_peered":"2023-09-23T18:21:42.590980+0000","last_clean":"2023-09-23T18:21:42.590980+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:21:42.590980+0000","last_undegraded":"2023-09-23T18:21:42.590980+0000","last_fullsized":"2023-09-23T18:21:42.590980+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":823,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590334+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:21:42.590334+0000","last_peered":"2023-09-23T18:21:42.590334+0000","last_clean":"2023-09-23T18:21:42.590334+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:21:42.590334+0000","last_undegraded":"2023-09-23T18:21:42.590334+0000","last_fullsized":"2023-09-23T18:21:42.590334+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":142,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.598519+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:21:42.598519+0000","last_peered":"2023-09-23T18:21:42.598519+0000","last_clean":"2023-09-23T18:21:42.598519+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:21:42.598519+0000","last_undegraded":"2023-09-23T18:21:42.598519+0000","last_fullsized":"2023-09-23T18:21:42.598519+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":136,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591215+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:21:42.591215+0000","last_peered":"2023-09-23T18:21:42.591215+0000","last_clean":"2023-09-23T18:21:42.591215+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:21:42.591215+0000","last_undegraded":"2023-09-23T18:21:42.591215+0000","last_fullsized":"2023-09-23T18:21:42.591215+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":762,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.594302+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:21:42.594302+0000","last_peered":"2023-09-23T18:21:42.594302+0000","last_clean":"2023-09-23T18:21:42.594302+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:21:42.594302+0000","last_undegraded":"2023-09-23T18:21:42.594302+0000","last_fullsized":"2023-09-23T18:21:42.594302+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":213,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092225+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:21:43.092225+0000","last_peered":"2023-09-23T18:21:43.092225+0000","last_clean":"2023-09-23T18:21:43.092225+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:21:43.092225+0000","last_undegraded":"2023-09-23T18:21:43.092225+0000","last_fullsized":"2023-09-23T18:21:43.092225+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":225,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.596222+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:21:42.596222+0000","last_peered":"2023-09-23T18:21:42.596222+0000","last_clean":"2023-09-23T18:21:42.596222+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:21:42.596222+0000","last_undegraded":"2023-09-23T18:21:42.596222+0000","last_fullsized":"2023-09-23T18:21:42.596222+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":857,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092191+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:21:43.092191+0000","last_peered":"2023-09-23T18:21:43.092191+0000","last_clean":"2023-09-23T18:21:43.092191+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:21:43.092191+0000","last_undegraded":"2023-09-23T18:21:43.092191+0000","last_fullsized":"2023-09-23T18:21:43.092191+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":766,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092205+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:21:43.092205+0000","last_peered":"2023-09-23T18:21:43.092205+0000","last_clean":"2023-09-23T18:21:43.092205+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:21:43.092205+0000","last_undegraded":"2023-09-23T18:21:43.092205+0000","last_fullsized":"2023-09-23T18:21:43.092205+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":43,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.596643+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:21:42.596643+0000","last_peered":"2023-09-23T18:21:42.596643+0000","last_clean":"2023-09-23T18:21:42.596643+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:21:42.596643+0000","last_undegraded":"2023-09-23T18:21:42.596643+0000","last_fullsized":"2023-09-23T18:21:42.596643+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.6","version":"622'13074","reported_seq":13134,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590843+0000","last_change":"2023-09-23T18:17:57.629275+0000","last_active":"2023-09-23T18:21:42.590843+0000","last_peered":"2023-09-23T18:21:42.590843+0000","last_clean":"2023-09-23T18:21:42.590843+0000","last_became_active":"2023-09-23T18:17:57.628937+0000","last_became_peered":"2023-09-23T18:17:57.628937+0000","last_unstale":"2023-09-23T18:21:42.590843+0000","last_undegraded":"2023-09-23T18:21:42.590843+0000","last_fullsized":"2023-09-23T18:21:42.590843+0000","mapping_epoch":574,"log_start":"608'7600","ondisk_log_start":"608'7600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5474,"log_dups_size":0,"ondisk_log_size":5474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:09:41.550809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13074,"num_write_kb":418368,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":891,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092922+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:21:43.092922+0000","last_peered":"2023-09-23T18:21:43.092922+0000","last_clean":"2023-09-23T18:21:43.092922+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:21:43.092922+0000","last_undegraded":"2023-09-23T18:21:43.092922+0000","last_fullsized":"2023-09-23T18:21:43.092922+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":260,"reported_epoch":623,"state":"active+clean","last_fresh":"2023-09-23T18:21:37.699032+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:21:37.699032+0000","last_peered":"2023-09-23T18:21:37.699032+0000","last_clean":"2023-09-23T18:21:37.699032+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:21:37.699032+0000","last_undegraded":"2023-09-23T18:21:37.699032+0000","last_fullsized":"2023-09-23T18:21:37.699032+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":136,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590337+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:21:42.590337+0000","last_peered":"2023-09-23T18:21:42.590337+0000","last_clean":"2023-09-23T18:21:42.590337+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:21:42.590337+0000","last_undegraded":"2023-09-23T18:21:42.590337+0000","last_fullsized":"2023-09-23T18:21:42.590337+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.7","version":"622'12692","reported_seq":12762,"reported_epoch":625,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:43.093237+0000","last_change":"2023-09-23T18:18:47.587098+0000","last_active":"2023-09-23T18:21:43.093237+0000","last_peered":"2023-09-23T18:21:43.093237+0000","last_clean":"2023-09-23T18:18:45.358166+0000","last_became_active":"2023-09-23T18:18:47.578360+0000","last_became_peered":"2023-09-23T18:18:47.578360+0000","last_unstale":"2023-09-23T18:21:43.093237+0000","last_undegraded":"2023-09-23T18:18:47.441486+0000","last_fullsized":"2023-09-23T18:18:47.417156+0000","mapping_epoch":595,"log_start":"609'8200","ondisk_log_start":"609'8200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4492,"log_dups_size":0,"ondisk_log_size":4492,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:05:01.135419+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":4148,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12692,"num_write_kb":406144,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,5],"avail_no_missing":["6","4","5"],"object_location_counts":[{"shards":"5,6","objects":-4148}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":746,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.595242+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:21:42.595242+0000","last_peered":"2023-09-23T18:21:42.595242+0000","last_clean":"2023-09-23T18:21:42.595242+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:21:42.595242+0000","last_undegraded":"2023-09-23T18:21:42.595242+0000","last_fullsized":"2023-09-23T18:21:42.595242+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":261,"reported_epoch":623,"state":"active+clean","last_fresh":"2023-09-23T18:21:37.699646+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:21:37.699646+0000","last_peered":"2023-09-23T18:21:37.699646+0000","last_clean":"2023-09-23T18:21:37.699646+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:21:37.699646+0000","last_undegraded":"2023-09-23T18:21:37.699646+0000","last_fullsized":"2023-09-23T18:21:37.699646+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":240,"reported_epoch":623,"state":"active+clean","last_fresh":"2023-09-23T18:21:37.698197+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:21:37.698197+0000","last_peered":"2023-09-23T18:21:37.698197+0000","last_clean":"2023-09-23T18:21:37.698197+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:21:37.698197+0000","last_undegraded":"2023-09-23T18:21:37.698197+0000","last_fullsized":"2023-09-23T18:21:37.698197+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"10.4","version":"622'12844","reported_seq":19865,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.600095+0000","last_change":"2023-09-23T18:19:34.407297+0000","last_active":"2023-09-23T18:21:42.600095+0000","last_peered":"2023-09-23T18:21:42.600095+0000","last_clean":"2023-09-23T18:21:42.600095+0000","last_became_active":"2023-09-23T18:19:34.401050+0000","last_became_peered":"2023-09-23T18:19:34.401050+0000","last_unstale":"2023-09-23T18:21:42.600095+0000","last_undegraded":"2023-09-23T18:21:42.600095+0000","last_fullsized":"2023-09-23T18:21:42.600095+0000","mapping_epoch":605,"log_start":"609'8000","ondisk_log_start":"609'8000","created":574,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4844,"log_dups_size":0,"ondisk_log_size":4844,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:38:04.862169+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12844,"num_write_kb":411008,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3471,"num_bytes_recovered":227475456,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":641,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093410+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:21:43.093410+0000","last_peered":"2023-09-23T18:21:43.093410+0000","last_clean":"2023-09-23T18:21:43.093410+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:21:43.093410+0000","last_undegraded":"2023-09-23T18:21:43.093410+0000","last_fullsized":"2023-09-23T18:21:43.093410+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":551,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591116+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:21:42.591116+0000","last_peered":"2023-09-23T18:21:42.591116+0000","last_clean":"2023-09-23T18:21:42.591116+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:21:42.591116+0000","last_undegraded":"2023-09-23T18:21:42.591116+0000","last_fullsized":"2023-09-23T18:21:42.591116+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":202,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092511+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:21:43.092511+0000","last_peered":"2023-09-23T18:21:43.092511+0000","last_clean":"2023-09-23T18:21:43.092511+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:21:43.092511+0000","last_undegraded":"2023-09-23T18:21:43.092511+0000","last_fullsized":"2023-09-23T18:21:43.092511+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.5","version":"622'13092","reported_seq":13154,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.596163+0000","last_change":"2023-09-23T18:17:57.629152+0000","last_active":"2023-09-23T18:21:42.596163+0000","last_peered":"2023-09-23T18:21:42.596163+0000","last_clean":"2023-09-23T18:21:42.596163+0000","last_became_active":"2023-09-23T18:17:57.628838+0000","last_became_peered":"2023-09-23T18:17:57.628838+0000","last_unstale":"2023-09-23T18:21:42.596163+0000","last_undegraded":"2023-09-23T18:21:42.596163+0000","last_fullsized":"2023-09-23T18:21:42.596163+0000","mapping_epoch":574,"log_start":"608'8000","ondisk_log_start":"608'8000","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5092,"log_dups_size":0,"ondisk_log_size":5092,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:31:01.367137+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13092,"num_write_kb":418944,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,5],"acting":[3,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":657,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.596318+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:21:42.596318+0000","last_peered":"2023-09-23T18:21:42.596318+0000","last_clean":"2023-09-23T18:21:42.596318+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:21:42.596318+0000","last_undegraded":"2023-09-23T18:21:42.596318+0000","last_fullsized":"2023-09-23T18:21:42.596318+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":150,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591227+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:21:42.591227+0000","last_peered":"2023-09-23T18:21:42.591227+0000","last_clean":"2023-09-23T18:21:42.591227+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:21:42.591227+0000","last_undegraded":"2023-09-23T18:21:42.591227+0000","last_fullsized":"2023-09-23T18:21:42.591227+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":463,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.598829+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:21:42.598829+0000","last_peered":"2023-09-23T18:21:42.598829+0000","last_clean":"2023-09-23T18:21:42.598829+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:21:42.598829+0000","last_undegraded":"2023-09-23T18:21:42.598829+0000","last_fullsized":"2023-09-23T18:21:42.598829+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"10.2","version":"622'13044","reported_seq":13105,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T18:21:41.583775+0000","last_change":"2023-09-23T18:17:57.629069+0000","last_active":"2023-09-23T18:21:41.583775+0000","last_peered":"2023-09-23T18:21:41.583775+0000","last_clean":"2023-09-23T18:21:41.583775+0000","last_became_active":"2023-09-23T18:17:57.628671+0000","last_became_peered":"2023-09-23T18:17:57.628671+0000","last_unstale":"2023-09-23T18:21:41.583775+0000","last_undegraded":"2023-09-23T18:21:41.583775+0000","last_fullsized":"2023-09-23T18:21:41.583775+0000","mapping_epoch":574,"log_start":"610'8600","ondisk_log_start":"610'8600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4444,"log_dups_size":0,"ondisk_log_size":4444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:43:05.448347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13044,"num_write_kb":417408,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1108,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590548+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:21:42.590548+0000","last_peered":"2023-09-23T18:21:42.590548+0000","last_clean":"2023-09-23T18:21:42.590548+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:21:42.590548+0000","last_undegraded":"2023-09-23T18:21:42.590548+0000","last_fullsized":"2023-09-23T18:21:42.590548+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":363,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.594544+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:21:42.594544+0000","last_peered":"2023-09-23T18:21:42.594544+0000","last_clean":"2023-09-23T18:21:42.594544+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:21:42.594544+0000","last_undegraded":"2023-09-23T18:21:42.594544+0000","last_fullsized":"2023-09-23T18:21:42.594544+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":170,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092701+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:21:43.092701+0000","last_peered":"2023-09-23T18:21:43.092701+0000","last_clean":"2023-09-23T18:21:43.092701+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:21:43.092701+0000","last_undegraded":"2023-09-23T18:21:43.092701+0000","last_fullsized":"2023-09-23T18:21:43.092701+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.3","version":"622'13030","reported_seq":21563,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.623293+0000","last_change":"2023-09-23T18:21:42.623042+0000","last_active":"2023-09-23T18:21:42.623293+0000","last_peered":"2023-09-23T18:21:42.623293+0000","last_clean":"2023-09-23T18:21:42.623293+0000","last_became_active":"2023-09-23T18:21:42.595312+0000","last_became_peered":"2023-09-23T18:21:42.595312+0000","last_unstale":"2023-09-23T18:21:42.623293+0000","last_undegraded":"2023-09-23T18:21:42.623293+0000","last_fullsized":"2023-09-23T18:21:42.623293+0000","mapping_epoch":624,"log_start":"607'7500","ondisk_log_start":"607'7500","created":574,"last_epoch_clean":625,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5530,"log_dups_size":0,"ondisk_log_size":5530,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:25:31.793542+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13030,"num_write_kb":416960,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4226,"num_bytes_recovered":46006272,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,4],"acting":[5,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":895,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590509+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:21:42.590509+0000","last_peered":"2023-09-23T18:21:42.590509+0000","last_clean":"2023-09-23T18:21:42.590509+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:21:42.590509+0000","last_undegraded":"2023-09-23T18:21:42.590509+0000","last_fullsized":"2023-09-23T18:21:42.590509+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":178,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092468+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:21:43.092468+0000","last_peered":"2023-09-23T18:21:43.092468+0000","last_clean":"2023-09-23T18:21:43.092468+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:21:43.092468+0000","last_undegraded":"2023-09-23T18:21:43.092468+0000","last_fullsized":"2023-09-23T18:21:43.092468+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":477,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093045+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:21:43.093045+0000","last_peered":"2023-09-23T18:21:43.093045+0000","last_clean":"2023-09-23T18:21:43.093045+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:21:43.093045+0000","last_undegraded":"2023-09-23T18:21:43.093045+0000","last_fullsized":"2023-09-23T18:21:43.093045+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.0","version":"622'13052","reported_seq":13112,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092062+0000","last_change":"2023-09-23T18:17:57.426750+0000","last_active":"2023-09-23T18:21:43.092062+0000","last_peered":"2023-09-23T18:21:43.092062+0000","last_clean":"2023-09-23T18:21:43.092062+0000","last_became_active":"2023-09-23T18:17:57.426426+0000","last_became_peered":"2023-09-23T18:17:57.426426+0000","last_unstale":"2023-09-23T18:21:43.092062+0000","last_undegraded":"2023-09-23T18:21:43.092062+0000","last_fullsized":"2023-09-23T18:21:43.092062+0000","mapping_epoch":574,"log_start":"610'8600","ondisk_log_start":"610'8600","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4452,"log_dups_size":0,"ondisk_log_size":4452,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:47.465119+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13052,"num_write_kb":417664,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":897,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590847+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:21:42.590847+0000","last_peered":"2023-09-23T18:21:42.590847+0000","last_clean":"2023-09-23T18:21:42.590847+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:21:42.590847+0000","last_undegraded":"2023-09-23T18:21:42.590847+0000","last_fullsized":"2023-09-23T18:21:42.590847+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":462,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590428+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:21:42.590428+0000","last_peered":"2023-09-23T18:21:42.590428+0000","last_clean":"2023-09-23T18:21:42.590428+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:21:42.590428+0000","last_undegraded":"2023-09-23T18:21:42.590428+0000","last_fullsized":"2023-09-23T18:21:42.590428+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":238,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092869+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:21:43.092869+0000","last_peered":"2023-09-23T18:21:43.092869+0000","last_clean":"2023-09-23T18:21:43.092869+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:21:43.092869+0000","last_undegraded":"2023-09-23T18:21:43.092869+0000","last_fullsized":"2023-09-23T18:21:43.092869+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.1","version":"622'13034","reported_seq":13092,"reported_epoch":623,"state":"active+clean","last_fresh":"2023-09-23T18:21:37.698051+0000","last_change":"2023-09-23T18:17:57.629164+0000","last_active":"2023-09-23T18:21:37.698051+0000","last_peered":"2023-09-23T18:21:37.698051+0000","last_clean":"2023-09-23T18:21:37.698051+0000","last_became_active":"2023-09-23T18:17:57.628839+0000","last_became_peered":"2023-09-23T18:17:57.628839+0000","last_unstale":"2023-09-23T18:21:37.698051+0000","last_undegraded":"2023-09-23T18:21:37.698051+0000","last_fullsized":"2023-09-23T18:21:37.698051+0000","mapping_epoch":574,"log_start":"609'8400","ondisk_log_start":"609'8400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4634,"log_dups_size":0,"ondisk_log_size":4634,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:03.443000+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13034,"num_write_kb":417088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":878,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591025+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:21:42.591025+0000","last_peered":"2023-09-23T18:21:42.591025+0000","last_clean":"2023-09-23T18:21:42.591025+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:21:42.591025+0000","last_undegraded":"2023-09-23T18:21:42.591025+0000","last_fullsized":"2023-09-23T18:21:42.591025+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":43,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.594094+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:21:42.594094+0000","last_peered":"2023-09-23T18:21:42.594094+0000","last_clean":"2023-09-23T18:21:42.594094+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:21:42.594094+0000","last_undegraded":"2023-09-23T18:21:42.594094+0000","last_fullsized":"2023-09-23T18:21:42.594094+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":353,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T18:21:41.593299+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:21:41.593299+0000","last_peered":"2023-09-23T18:21:41.593299+0000","last_clean":"2023-09-23T18:21:41.593299+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:21:41.593299+0000","last_undegraded":"2023-09-23T18:21:41.593299+0000","last_fullsized":"2023-09-23T18:21:41.593299+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"10.c","version":"622'13068","reported_seq":14101,"reported_epoch":626,"state":"active+recovering+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:45.331669+0000","last_change":"2023-09-23T18:21:41.132429+0000","last_active":"2023-09-23T18:21:45.331669+0000","last_peered":"2023-09-23T18:21:45.331669+0000","last_clean":"2023-09-23T18:18:45.234971+0000","last_became_active":"2023-09-23T18:18:47.573145+0000","last_became_peered":"2023-09-23T18:18:47.573145+0000","last_unstale":"2023-09-23T18:21:45.331669+0000","last_undegraded":"2023-09-23T18:18:47.439404+0000","last_fullsized":"2023-09-23T18:18:47.413261+0000","mapping_epoch":595,"log_start":"609'8200","ondisk_log_start":"609'8200","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4868,"log_dups_size":0,"ondisk_log_size":4868,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T06:12:52.216199+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3801,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":13068,"num_write_kb":418176,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":484,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[0,4],"avail_no_missing":["0","4","6"],"object_location_counts":[{"shards":"0,4","objects":-3801}],"blocked_by":[],"up_primary":3,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":737,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093493+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:21:43.093493+0000","last_peered":"2023-09-23T18:21:43.093493+0000","last_clean":"2023-09-23T18:21:43.093493+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:21:43.093493+0000","last_undegraded":"2023-09-23T18:21:43.093493+0000","last_fullsized":"2023-09-23T18:21:43.093493+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":976,"reported_epoch":624,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:41.585352+0000","last_change":"2023-09-23T18:18:47.445313+0000","last_active":"2023-09-23T18:21:41.585352+0000","last_peered":"2023-09-23T18:21:41.585352+0000","last_clean":"2023-09-23T18:18:37.463519+0000","last_became_active":"2023-09-23T18:18:47.444889+0000","last_became_peered":"2023-09-23T18:18:47.444889+0000","last_unstale":"2023-09-23T18:21:41.585352+0000","last_undegraded":"2023-09-23T18:18:47.421365+0000","last_fullsized":"2023-09-23T18:18:47.420863+0000","mapping_epoch":595,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:25:33.015771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":52,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0],"avail_no_missing":["1","0","4"],"object_location_counts":[{"shards":"0,1,4","objects":1}],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":160,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591264+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:21:42.591264+0000","last_peered":"2023-09-23T18:21:42.591264+0000","last_clean":"2023-09-23T18:21:42.591264+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:21:42.591264+0000","last_undegraded":"2023-09-23T18:21:42.591264+0000","last_fullsized":"2023-09-23T18:21:42.591264+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":140,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590839+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:21:42.590839+0000","last_peered":"2023-09-23T18:21:42.590839+0000","last_clean":"2023-09-23T18:21:42.590839+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:21:42.590839+0000","last_undegraded":"2023-09-23T18:21:42.590839+0000","last_fullsized":"2023-09-23T18:21:42.590839+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.a","version":"622'12936","reported_seq":13005,"reported_epoch":625,"state":"active+recovery_wait+undersized+degraded+remapped","last_fresh":"2023-09-23T18:21:43.092600+0000","last_change":"2023-09-23T18:18:34.740705+0000","last_active":"2023-09-23T18:21:43.092600+0000","last_peered":"2023-09-23T18:21:43.092600+0000","last_clean":"2023-09-23T18:18:32.703184+0000","last_became_active":"2023-09-23T18:18:34.721542+0000","last_became_peered":"2023-09-23T18:18:34.721542+0000","last_unstale":"2023-09-23T18:21:43.092600+0000","last_undegraded":"2023-09-23T18:18:34.351725+0000","last_fullsized":"2023-09-23T18:18:34.327235+0000","mapping_epoch":589,"log_start":"610'8500","ondisk_log_start":"610'8500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4436,"log_dups_size":0,"ondisk_log_size":4436,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:32:54.937849+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":3559,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12936,"num_write_kb":413952,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,5],"avail_no_missing":["6","1","5"],"object_location_counts":[{"shards":"5,6","objects":-3559}],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":809,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590755+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:21:42.590755+0000","last_peered":"2023-09-23T18:21:42.590755+0000","last_clean":"2023-09-23T18:21:42.590755+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:21:42.590755+0000","last_undegraded":"2023-09-23T18:21:42.590755+0000","last_fullsized":"2023-09-23T18:21:42.590755+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1510,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590751+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:21:42.590751+0000","last_peered":"2023-09-23T18:21:42.590751+0000","last_clean":"2023-09-23T18:21:42.590751+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:21:42.590751+0000","last_undegraded":"2023-09-23T18:21:42.590751+0000","last_fullsized":"2023-09-23T18:21:42.590751+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":150,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092662+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:21:43.092662+0000","last_peered":"2023-09-23T18:21:43.092662+0000","last_clean":"2023-09-23T18:21:43.092662+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:21:43.092662+0000","last_undegraded":"2023-09-23T18:21:43.092662+0000","last_fullsized":"2023-09-23T18:21:43.092662+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":450,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092993+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:21:43.092993+0000","last_peered":"2023-09-23T18:21:43.092993+0000","last_clean":"2023-09-23T18:21:43.092993+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:21:43.092993+0000","last_undegraded":"2023-09-23T18:21:43.092993+0000","last_fullsized":"2023-09-23T18:21:43.092993+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.9","version":"622'12586","reported_seq":12646,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590422+0000","last_change":"2023-09-23T18:17:57.425319+0000","last_active":"2023-09-23T18:21:42.590422+0000","last_peered":"2023-09-23T18:21:42.590422+0000","last_clean":"2023-09-23T18:21:42.590422+0000","last_became_active":"2023-09-23T18:17:57.424990+0000","last_became_peered":"2023-09-23T18:17:57.424990+0000","last_unstale":"2023-09-23T18:21:42.590422+0000","last_undegraded":"2023-09-23T18:21:42.590422+0000","last_fullsized":"2023-09-23T18:21:42.590422+0000","mapping_epoch":574,"log_start":"607'7100","ondisk_log_start":"607'7100","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5486,"log_dups_size":0,"ondisk_log_size":5486,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:26.097926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12586,"num_write_kb":402752,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,3],"acting":[5,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":834,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590955+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:21:42.590955+0000","last_peered":"2023-09-23T18:21:42.590955+0000","last_clean":"2023-09-23T18:21:42.590955+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:21:42.590955+0000","last_undegraded":"2023-09-23T18:21:42.590955+0000","last_fullsized":"2023-09-23T18:21:42.590955+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":69,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.594791+0000","last_change":"2023-09-23T18:20:35.016554+0000","last_active":"2023-09-23T18:21:42.594791+0000","last_peered":"2023-09-23T18:21:42.594791+0000","last_clean":"2023-09-23T18:21:42.594791+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:21:42.594791+0000","last_undegraded":"2023-09-23T18:21:42.594791+0000","last_fullsized":"2023-09-23T18:21:42.594791+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":419,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092380+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:21:43.092380+0000","last_peered":"2023-09-23T18:21:43.092380+0000","last_clean":"2023-09-23T18:21:43.092380+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:21:43.092380+0000","last_undegraded":"2023-09-23T18:21:43.092380+0000","last_fullsized":"2023-09-23T18:21:43.092380+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":162,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591108+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:21:42.591108+0000","last_peered":"2023-09-23T18:21:42.591108+0000","last_clean":"2023-09-23T18:21:42.591108+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:21:42.591108+0000","last_undegraded":"2023-09-23T18:21:42.591108+0000","last_fullsized":"2023-09-23T18:21:42.591108+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.b","version":"622'12948","reported_seq":13008,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092507+0000","last_change":"2023-09-23T18:17:57.629252+0000","last_active":"2023-09-23T18:21:43.092507+0000","last_peered":"2023-09-23T18:21:43.092507+0000","last_clean":"2023-09-23T18:21:43.092507+0000","last_became_active":"2023-09-23T18:17:57.628942+0000","last_became_peered":"2023-09-23T18:17:57.628942+0000","last_unstale":"2023-09-23T18:21:43.092507+0000","last_undegraded":"2023-09-23T18:21:43.092507+0000","last_fullsized":"2023-09-23T18:21:43.092507+0000","mapping_epoch":574,"log_start":"610'8500","ondisk_log_start":"610'8500","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4448,"log_dups_size":0,"ondisk_log_size":4448,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:47:56.771295+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12948,"num_write_kb":414336,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1523,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590674+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:21:42.590674+0000","last_peered":"2023-09-23T18:21:42.590674+0000","last_clean":"2023-09-23T18:21:42.590674+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:21:42.590674+0000","last_undegraded":"2023-09-23T18:21:42.590674+0000","last_fullsized":"2023-09-23T18:21:42.590674+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":576,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092288+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:21:43.092288+0000","last_peered":"2023-09-23T18:21:43.092288+0000","last_clean":"2023-09-23T18:21:43.092288+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:21:43.092288+0000","last_undegraded":"2023-09-23T18:21:43.092288+0000","last_fullsized":"2023-09-23T18:21:43.092288+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":136,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590577+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:21:42.590577+0000","last_peered":"2023-09-23T18:21:42.590577+0000","last_clean":"2023-09-23T18:21:42.590577+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:21:42.590577+0000","last_undegraded":"2023-09-23T18:21:42.590577+0000","last_fullsized":"2023-09-23T18:21:42.590577+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":171,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093315+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:21:43.093315+0000","last_peered":"2023-09-23T18:21:43.093315+0000","last_clean":"2023-09-23T18:21:43.093315+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:21:43.093315+0000","last_undegraded":"2023-09-23T18:21:43.093315+0000","last_fullsized":"2023-09-23T18:21:43.093315+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"10.8","version":"622'12784","reported_seq":12844,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590411+0000","last_change":"2023-09-23T18:17:57.629994+0000","last_active":"2023-09-23T18:21:42.590411+0000","last_peered":"2023-09-23T18:21:42.590411+0000","last_clean":"2023-09-23T18:21:42.590411+0000","last_became_active":"2023-09-23T18:17:57.629695+0000","last_became_peered":"2023-09-23T18:17:57.629695+0000","last_unstale":"2023-09-23T18:21:42.590411+0000","last_undegraded":"2023-09-23T18:21:42.590411+0000","last_fullsized":"2023-09-23T18:21:42.590411+0000","mapping_epoch":574,"log_start":"607'7300","ondisk_log_start":"607'7300","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":5484,"log_dups_size":0,"ondisk_log_size":5484,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:51.921601+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12784,"num_write_kb":409088,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":824,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092754+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:21:43.092754+0000","last_peered":"2023-09-23T18:21:43.092754+0000","last_clean":"2023-09-23T18:21:43.092754+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:21:43.092754+0000","last_undegraded":"2023-09-23T18:21:43.092754+0000","last_fullsized":"2023-09-23T18:21:43.092754+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":140,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.594206+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:21:42.594206+0000","last_peered":"2023-09-23T18:21:42.594206+0000","last_clean":"2023-09-23T18:21:42.594206+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:21:42.594206+0000","last_undegraded":"2023-09-23T18:21:42.594206+0000","last_fullsized":"2023-09-23T18:21:42.594206+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":43,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.594703+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:21:42.594703+0000","last_peered":"2023-09-23T18:21:42.594703+0000","last_clean":"2023-09-23T18:21:42.594703+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:21:42.594703+0000","last_undegraded":"2023-09-23T18:21:42.594703+0000","last_fullsized":"2023-09-23T18:21:42.594703+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":641,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591288+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:21:42.591288+0000","last_peered":"2023-09-23T18:21:42.591288+0000","last_clean":"2023-09-23T18:21:42.591288+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:21:42.591288+0000","last_undegraded":"2023-09-23T18:21:42.591288+0000","last_fullsized":"2023-09-23T18:21:42.591288+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.d","version":"622'12958","reported_seq":19947,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092819+0000","last_change":"2023-09-23T18:20:36.673759+0000","last_active":"2023-09-23T18:21:43.092819+0000","last_peered":"2023-09-23T18:21:43.092819+0000","last_clean":"2023-09-23T18:21:43.092819+0000","last_became_active":"2023-09-23T18:20:36.651913+0000","last_became_peered":"2023-09-23T18:20:36.651913+0000","last_unstale":"2023-09-23T18:21:43.092819+0000","last_undegraded":"2023-09-23T18:21:43.092819+0000","last_fullsized":"2023-09-23T18:21:43.092819+0000","mapping_epoch":614,"log_start":"610'8500","ondisk_log_start":"610'8500","created":574,"last_epoch_clean":615,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4458,"log_dups_size":0,"ondisk_log_size":4458,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:55:47.765348+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":12958,"num_write_kb":414593,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3453,"num_bytes_recovered":209321984,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":924,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093048+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:21:43.093048+0000","last_peered":"2023-09-23T18:21:43.093048+0000","last_clean":"2023-09-23T18:21:43.093048+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:21:43.093048+0000","last_undegraded":"2023-09-23T18:21:43.093048+0000","last_fullsized":"2023-09-23T18:21:43.093048+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":65,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.598933+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:21:42.598933+0000","last_peered":"2023-09-23T18:21:42.598933+0000","last_clean":"2023-09-23T18:21:42.598933+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:21:42.598933+0000","last_undegraded":"2023-09-23T18:21:42.598933+0000","last_fullsized":"2023-09-23T18:21:42.598933+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":150,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591051+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:21:42.591051+0000","last_peered":"2023-09-23T18:21:42.591051+0000","last_clean":"2023-09-23T18:21:42.591051+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:21:42.591051+0000","last_undegraded":"2023-09-23T18:21:42.591051+0000","last_fullsized":"2023-09-23T18:21:42.591051+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":150,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590641+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:21:42.590641+0000","last_peered":"2023-09-23T18:21:42.590641+0000","last_clean":"2023-09-23T18:21:42.590641+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:21:42.590641+0000","last_undegraded":"2023-09-23T18:21:42.590641+0000","last_fullsized":"2023-09-23T18:21:42.590641+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"10.e","version":"622'12854","reported_seq":12914,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092939+0000","last_change":"2023-09-23T18:17:57.426731+0000","last_active":"2023-09-23T18:21:43.092939+0000","last_peered":"2023-09-23T18:21:43.092939+0000","last_clean":"2023-09-23T18:21:43.092939+0000","last_became_active":"2023-09-23T18:17:57.426379+0000","last_became_peered":"2023-09-23T18:17:57.426379+0000","last_unstale":"2023-09-23T18:21:43.092939+0000","last_undegraded":"2023-09-23T18:21:43.092939+0000","last_fullsized":"2023-09-23T18:21:43.092939+0000","mapping_epoch":574,"log_start":"610'8400","ondisk_log_start":"610'8400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4454,"log_dups_size":0,"ondisk_log_size":4454,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:53.262771+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12854,"num_write_kb":411328,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":924,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093123+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:21:43.093123+0000","last_peered":"2023-09-23T18:21:43.093123+0000","last_clean":"2023-09-23T18:21:43.093123+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:21:43.093123+0000","last_undegraded":"2023-09-23T18:21:43.093123+0000","last_fullsized":"2023-09-23T18:21:43.093123+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":112,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590911+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:21:42.590911+0000","last_peered":"2023-09-23T18:21:42.590911+0000","last_clean":"2023-09-23T18:21:42.590911+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:21:42.590911+0000","last_undegraded":"2023-09-23T18:21:42.590911+0000","last_fullsized":"2023-09-23T18:21:42.590911+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":405,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092969+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:21:43.092969+0000","last_peered":"2023-09-23T18:21:43.092969+0000","last_clean":"2023-09-23T18:21:43.092969+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:21:43.092969+0000","last_undegraded":"2023-09-23T18:21:43.092969+0000","last_fullsized":"2023-09-23T18:21:43.092969+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":43,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.594029+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:21:42.594029+0000","last_peered":"2023-09-23T18:21:42.594029+0000","last_clean":"2023-09-23T18:21:42.594029+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:21:42.594029+0000","last_undegraded":"2023-09-23T18:21:42.594029+0000","last_fullsized":"2023-09-23T18:21:42.594029+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"10.f","version":"622'12812","reported_seq":12872,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093535+0000","last_change":"2023-09-23T18:17:57.427735+0000","last_active":"2023-09-23T18:21:43.093535+0000","last_peered":"2023-09-23T18:21:43.093535+0000","last_clean":"2023-09-23T18:21:43.093535+0000","last_became_active":"2023-09-23T18:17:57.427409+0000","last_became_peered":"2023-09-23T18:17:57.427409+0000","last_unstale":"2023-09-23T18:21:43.093535+0000","last_undegraded":"2023-09-23T18:21:43.093535+0000","last_fullsized":"2023-09-23T18:21:43.093535+0000","mapping_epoch":574,"log_start":"610'8400","ondisk_log_start":"610'8400","created":574,"last_epoch_clean":575,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T18:17:56.382403+0000","last_clean_scrub_stamp":"2023-09-23T18:17:56.382403+0000","objects_scrubbed":0,"log_size":4412,"log_dups_size":0,"ondisk_log_size":4412,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:06:44.288387+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12812,"num_write_kb":409984,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":759,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.595776+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:21:42.595776+0000","last_peered":"2023-09-23T18:21:42.595776+0000","last_clean":"2023-09-23T18:21:42.595776+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:21:42.595776+0000","last_undegraded":"2023-09-23T18:21:42.595776+0000","last_fullsized":"2023-09-23T18:21:42.595776+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1062,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093622+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:21:43.093622+0000","last_peered":"2023-09-23T18:21:43.093622+0000","last_clean":"2023-09-23T18:21:43.093622+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:21:43.093622+0000","last_undegraded":"2023-09-23T18:21:43.093622+0000","last_fullsized":"2023-09-23T18:21:43.093622+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":53,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.599537+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:21:42.599537+0000","last_peered":"2023-09-23T18:21:42.599537+0000","last_clean":"2023-09-23T18:21:42.599537+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:21:42.599537+0000","last_undegraded":"2023-09-23T18:21:42.599537+0000","last_fullsized":"2023-09-23T18:21:42.599537+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":416,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590596+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:21:42.590596+0000","last_peered":"2023-09-23T18:21:42.590596+0000","last_clean":"2023-09-23T18:21:42.590596+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:21:42.590596+0000","last_undegraded":"2023-09-23T18:21:42.590596+0000","last_fullsized":"2023-09-23T18:21:42.590596+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":656,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092326+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:21:43.092326+0000","last_peered":"2023-09-23T18:21:43.092326+0000","last_clean":"2023-09-23T18:21:43.092326+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:21:43.092326+0000","last_undegraded":"2023-09-23T18:21:43.092326+0000","last_fullsized":"2023-09-23T18:21:43.092326+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":49,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.598779+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:21:42.598779+0000","last_peered":"2023-09-23T18:21:42.598779+0000","last_clean":"2023-09-23T18:21:42.598779+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:21:42.598779+0000","last_undegraded":"2023-09-23T18:21:42.598779+0000","last_fullsized":"2023-09-23T18:21:42.598779+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":698,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T18:21:41.584992+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:21:41.584992+0000","last_peered":"2023-09-23T18:21:41.584992+0000","last_clean":"2023-09-23T18:21:41.584992+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:21:41.584992+0000","last_undegraded":"2023-09-23T18:21:41.584992+0000","last_fullsized":"2023-09-23T18:21:41.584992+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":656,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092889+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:21:43.092889+0000","last_peered":"2023-09-23T18:21:43.092889+0000","last_clean":"2023-09-23T18:21:43.092889+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:21:43.092889+0000","last_undegraded":"2023-09-23T18:21:43.092889+0000","last_fullsized":"2023-09-23T18:21:43.092889+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":144,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591062+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:21:42.591062+0000","last_peered":"2023-09-23T18:21:42.591062+0000","last_clean":"2023-09-23T18:21:42.591062+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:21:42.591062+0000","last_undegraded":"2023-09-23T18:21:42.591062+0000","last_fullsized":"2023-09-23T18:21:42.591062+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":258,"reported_epoch":623,"state":"active+clean","last_fresh":"2023-09-23T18:21:37.698372+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:21:37.698372+0000","last_peered":"2023-09-23T18:21:37.698372+0000","last_clean":"2023-09-23T18:21:37.698372+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:21:37.698372+0000","last_undegraded":"2023-09-23T18:21:37.698372+0000","last_fullsized":"2023-09-23T18:21:37.698372+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":130,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591178+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:21:42.591178+0000","last_peered":"2023-09-23T18:21:42.591178+0000","last_clean":"2023-09-23T18:21:42.591178+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:21:42.591178+0000","last_undegraded":"2023-09-23T18:21:42.591178+0000","last_fullsized":"2023-09-23T18:21:42.591178+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":241,"reported_epoch":623,"state":"active+clean","last_fresh":"2023-09-23T18:21:37.698254+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:21:37.698254+0000","last_peered":"2023-09-23T18:21:37.698254+0000","last_clean":"2023-09-23T18:21:37.698254+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:21:37.698254+0000","last_undegraded":"2023-09-23T18:21:37.698254+0000","last_fullsized":"2023-09-23T18:21:37.698254+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1018,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092700+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:21:43.092700+0000","last_peered":"2023-09-23T18:21:43.092700+0000","last_clean":"2023-09-23T18:21:43.092700+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:21:43.092700+0000","last_undegraded":"2023-09-23T18:21:43.092700+0000","last_fullsized":"2023-09-23T18:21:43.092700+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":523,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.093019+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:21:43.093019+0000","last_peered":"2023-09-23T18:21:43.093019+0000","last_clean":"2023-09-23T18:21:43.093019+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:21:43.093019+0000","last_undegraded":"2023-09-23T18:21:43.093019+0000","last_fullsized":"2023-09-23T18:21:43.093019+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":160,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.590673+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:21:42.590673+0000","last_peered":"2023-09-23T18:21:42.590673+0000","last_clean":"2023-09-23T18:21:42.590673+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:21:42.590673+0000","last_undegraded":"2023-09-23T18:21:42.590673+0000","last_fullsized":"2023-09-23T18:21:42.590673+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":954,"reported_epoch":624,"state":"active+clean","last_fresh":"2023-09-23T18:21:41.585293+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:21:41.585293+0000","last_peered":"2023-09-23T18:21:41.585293+0000","last_clean":"2023-09-23T18:21:41.585293+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:21:41.585293+0000","last_undegraded":"2023-09-23T18:21:41.585293+0000","last_fullsized":"2023-09-23T18:21:41.585293+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":148,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:42.591068+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:21:42.591068+0000","last_peered":"2023-09-23T18:21:42.591068+0000","last_clean":"2023-09-23T18:21:42.591068+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:21:42.591068+0000","last_undegraded":"2023-09-23T18:21:42.591068+0000","last_fullsized":"2023-09-23T18:21:42.591068+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":669,"reported_epoch":625,"state":"active+clean","last_fresh":"2023-09-23T18:21:43.092223+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:21:43.092223+0000","last_peered":"2023-09-23T18:21:43.092223+0000","last_clean":"2023-09-23T18:21:43.092223+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:21:43.092223+0000","last_undegraded":"2023-09-23T18:21:43.092223+0000","last_fullsized":"2023-09-23T18:21:43.092223+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":10,"num_pg":16,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":11508,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1,"num_read_kb":1,"num_write":206808,"num_write_kb":6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":11634,"num_bytes_recovered":482803712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1693188096,"data_stored":1693188096,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":77008,"ondisk_log_size":77008,"up":48,"acting":45,"num_store_stats":6},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":13,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":298,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":20983808,"data_stored":20971577,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001589,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669526,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":911376,"kb_used_data":285512,"kb_used_omap":7,"kb_used_meta":625848,"kb_avail":92833776,"statfs":{"total":95995035648,"available":95061786624,"internally_reserved":0,"allocated":292364288,"data_stored":290377782,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":640869156},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0169999999999999,"5min":2.194,"15min":1.5620000000000001},"min":{"1min":0.255,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":12.207000000000001,"5min":24.472999999999999,"15min":24.472999999999999},"last":0.28799999999999998},{"interface":"front","average":{"1min":2.375,"5min":2.4220000000000002,"15min":1.833},"min":{"1min":0.255,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.255000000000001,"5min":22.832999999999998,"15min":22.832999999999998},"last":0.44}]},{"osd":1,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":2.258,"15min":1.5629999999999999},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.7309999999999999,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.38800000000000001},{"interface":"front","average":{"1min":2.9580000000000002,"5min":2.2490000000000001,"15min":1.5549999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.273,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.51100000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.579,"15min":1.345},"min":{"1min":0.33500000000000002,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":5.7919999999999998,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.42499999999999999},{"interface":"front","average":{"1min":2.3029999999999999,"5min":2.0600000000000001,"15min":1.5580000000000001},"min":{"1min":0.39700000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0730000000000004,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.5}]},{"osd":3,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":3.0419999999999998,"15min":2.1150000000000002},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.24099999999999999},"max":{"1min":5.8170000000000002,"5min":103.348,"15min":103.348},"last":0.372},{"interface":"front","average":{"1min":2.5600000000000001,"5min":3.6190000000000002,"15min":2.4209999999999998},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":7.4530000000000003,"5min":103.40300000000001,"15min":103.40300000000001},"last":0.52200000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:21:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.7390000000000001,"15min":1.341},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.0300000000000002,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.40600000000000003},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.661,"15min":1.389},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.27000000000000002},"max":{"1min":5.7649999999999997,"5min":7.8319999999999999,"15min":9.2260000000000009},"last":0.48499999999999999}]},{"osd":5,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":2.1099999999999999,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":18.417000000000002,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.46999999999999997},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.3399999999999999,"15min":2.3780000000000001},"min":{"1min":0.36299999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":18.454999999999998,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.55600000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":2.1059999999999999,"15min":1.5580000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.26800000000000002},"max":{"1min":6.1500000000000004,"5min":10.743,"15min":13.460000000000001},"last":0.57699999999999996},{"interface":"front","average":{"1min":2.0830000000000002,"5min":2.0529999999999999,"15min":1.603},"min":{"1min":0.40300000000000002,"5min":0.38500000000000001,"15min":0.28499999999999998},"max":{"1min":8.6890000000000001,"5min":11.244,"15min":12.404},"last":0.53100000000000003}]}]},{"osd":1,"up_from":545,"seq":2340757176374,"num_pgs":69,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":898240,"kb_used_data":646200,"kb_used_omap":13,"kb_used_meta":252018,"kb_avail":92846912,"statfs":{"total":95995035648,"available":95075237888,"internally_reserved":0,"allocated":661708800,"data_stored":658626387,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":258067085},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.286,"15min":0.68899999999999995},"min":{"1min":0.23400000000000001,"5min":0.189,"15min":0.189},"max":{"1min":5.0359999999999996,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.437},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.3280000000000001,"15min":0.76900000000000002},"min":{"1min":0.372,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.5650000000000004,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.34100000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.4450000000000001,"15min":0.78900000000000003},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.02,"5min":11.02,"15min":11.02},"last":0.45800000000000002},{"interface":"front","average":{"1min":1.5249999999999999,"5min":0.97599999999999998,"15min":0.66200000000000003},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.1440000000000001,"5min":5.1440000000000001,"15min":5.1440000000000001},"last":0.249}]},{"osd":3,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":3.9870000000000001,"5min":1.7609999999999999,"15min":0.84499999999999997},"min":{"1min":0.51400000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":14.962,"5min":14.962,"15min":14.962},"last":0.375},{"interface":"front","average":{"1min":2.008,"5min":1.5760000000000001,"15min":0.86599999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":8.3399999999999999,"5min":19.762,"15min":19.762},"last":0.48599999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.421,"15min":1.351},"min":{"1min":0.41899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.0759999999999996,"5min":12.663,"15min":12.663},"last":0.53600000000000003},{"interface":"front","average":{"1min":3.952,"5min":2.2669999999999999,"15min":1.7649999999999999},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.317},"max":{"1min":11.353999999999999,"5min":13.222,"15min":13.222},"last":0.59099999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.359,"5min":1.732,"15min":1.0589999999999999},"min":{"1min":0.36799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":6.3460000000000001,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.497},{"interface":"front","average":{"1min":2.3450000000000002,"5min":2.0409999999999999,"15min":1.216},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":10.926,"5min":23.204000000000001,"15min":23.204000000000001},"last":0.64600000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":1.7270000000000001,"15min":1.0820000000000001},"min":{"1min":0.45700000000000002,"5min":0.372,"15min":0.372},"max":{"1min":5.1680000000000001,"5min":23.443000000000001,"15min":23.443000000000001},"last":0.56899999999999995},{"interface":"front","average":{"1min":3.5059999999999998,"5min":2.2919999999999998,"15min":1.2809999999999999},"min":{"1min":0.49399999999999999,"5min":0.378,"15min":0.378},"max":{"1min":11.307,"5min":26.177,"15min":26.177},"last":0.69999999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":3.649,"5min":1.956,"15min":1.1060000000000001},"min":{"1min":0.42799999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":15.01,"5min":15.01,"15min":15.01},"last":0.66200000000000003},{"interface":"front","average":{"1min":2.931,"5min":1.899,"15min":1.1699999999999999},"min":{"1min":0.45200000000000001,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":10.988,"5min":10.988,"15min":10.988},"last":0.67300000000000004}]}]},{"osd":0,"up_from":93,"seq":399431958871,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1232488,"kb_used_data":283040,"kb_used_omap":7,"kb_used_meta":949432,"kb_avail":92512664,"statfs":{"total":95995035648,"available":94732967936,"internally_reserved":0,"allocated":289832960,"data_stored":286898920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":972219108},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.363,"15min":0.66700000000000004},"min":{"1min":0.19800000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.5090000000000003,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.42899999999999999},{"interface":"front","average":{"1min":3.3130000000000002,"5min":1.8360000000000001,"15min":0.95999999999999996},"min":{"1min":0.33800000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":22.978000000000002,"5min":22.978000000000002,"15min":22.978000000000002},"last":0.40699999999999997}]},{"osd":2,"last update":"Sat Sep 23 18:21:04 2023","interfaces":[{"interface":"back","average":{"1min":3.008,"5min":2.0880000000000001,"15min":1.4419999999999999},"min":{"1min":0.40200000000000002,"5min":0.185,"15min":0.185},"max":{"1min":10.013,"5min":18.550999999999998,"15min":18.550999999999998},"last":1.2969999999999999},{"interface":"front","average":{"1min":3.2160000000000002,"5min":1.7470000000000001,"15min":1.097},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":23.161999999999999,"5min":23.161999999999999,"15min":23.161999999999999},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.643,"5min":1.4199999999999999,"15min":1.5940000000000001},"min":{"1min":0.313,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":6.8460000000000001,"5min":13.539999999999999,"15min":13.539999999999999},"last":1.3180000000000001},{"interface":"front","average":{"1min":2.4399999999999999,"5min":1.6619999999999999,"15min":1.7010000000000001},"min":{"1min":0.36299999999999999,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":6.8879999999999999,"5min":13.285,"15min":15.198},"last":0.48399999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:21:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":2.1019999999999999,"15min":1.982},"min":{"1min":0.39900000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.1379999999999999,"5min":17.741,"15min":57.746000000000002},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.966,"5min":1.9099999999999999,"15min":1.792},"min":{"1min":0.30099999999999999,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":4.9089999999999998,"5min":15.079000000000001,"15min":57.816000000000003},"last":0.51600000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:20:46 2023","interfaces":[{"interface":"back","average":{"1min":1.726,"5min":1.8220000000000001,"15min":1.4410000000000001},"min":{"1min":0.375,"5min":0.35299999999999998,"15min":0.33800000000000002},"max":{"1min":5.5970000000000004,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.70999999999999996},{"interface":"front","average":{"1min":3.2909999999999999,"5min":2.4630000000000001,"15min":1.599},"min":{"1min":0.40100000000000002,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":15.047000000000001,"5min":15.047000000000001,"15min":15.047000000000001},"last":0.45500000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:21:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":1.9379999999999999,"15min":2.0299999999999998},"min":{"1min":0.40300000000000002,"5min":0.377,"15min":0.32000000000000001},"max":{"1min":5.4610000000000003,"5min":7.6900000000000004,"15min":58.457999999999998},"last":0.59499999999999997},{"interface":"front","average":{"1min":3.7629999999999999,"5min":2.657,"15min":2.1190000000000002},"min":{"1min":0.54100000000000004,"5min":0.373,"15min":0.315},"max":{"1min":22.893000000000001,"5min":22.893000000000001,"15min":58.354999999999997},"last":0.53900000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":3.431,"5min":2.2480000000000002,"15min":1.7889999999999999},"min":{"1min":0.49199999999999999,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":22.945,"5min":22.945,"15min":22.945},"last":0.66500000000000004},{"interface":"front","average":{"1min":2.2330000000000001,"5min":1.714,"15min":1.851},"min":{"1min":0.47099999999999997,"5min":0.371,"15min":0.36199999999999999},"max":{"1min":6.9249999999999998,"5min":7.7519999999999998,"15min":10.404},"last":0.77300000000000002}]}]},{"osd":2,"up_from":0,"seq":2070174236756,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249386,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":826244,"kb_used_data":12540,"kb_used_omap":3,"kb_used_meta":813692,"kb_avail":92918908,"statfs":{"total":95995035648,"available":95148961792,"internally_reserved":0,"allocated":12840960,"data_stored":9704443,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":833221352},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":1.5800000000000001,"15min":1.8089999999999999},"min":{"1min":0.20799999999999999,"5min":0.192,"15min":0.183},"max":{"1min":12.472,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.223},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.405,"15min":2.004},"min":{"1min":0.23799999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.468,"5min":11.468,"15min":194.661},"last":0.43099999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:21:27 2023","interfaces":[{"interface":"back","average":{"1min":1.994,"5min":1.754,"15min":0.93100000000000005},"min":{"1min":0.249,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.1189999999999998,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.9710000000000001,"5min":2.766,"15min":1.3919999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.9050000000000002,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.30299999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:21:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5569999999999999,"5min":1.526,"15min":1.3740000000000001},"min":{"1min":0.29899999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.2430000000000003,"5min":12.151999999999999,"15min":12.151999999999999},"last":0.47699999999999998},{"interface":"front","average":{"1min":1.78,"5min":1.3839999999999999,"15min":1.361},"min":{"1min":0.32300000000000001,"5min":0.27900000000000003,"15min":0.19600000000000001},"max":{"1min":7.0640000000000001,"5min":11.565,"15min":11.565},"last":0.35599999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":1.8839999999999999,"15min":1.706},"min":{"1min":0.34200000000000003,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":11.4,"5min":21.062999999999999,"15min":37.231000000000002},"last":0.50600000000000001},{"interface":"front","average":{"1min":2.532,"5min":2.1240000000000001,"15min":1.885},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":9.1630000000000003,"5min":21.128,"15min":37.381},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:21:42 2023","interfaces":[{"interface":"back","average":{"1min":2.6579999999999999,"5min":2.2709999999999999,"15min":1.6799999999999999},"min":{"1min":0.47899999999999998,"5min":0.316,"15min":0.316},"max":{"1min":19.434999999999999,"5min":19.434999999999999,"15min":19.434999999999999},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.647,"5min":1.7310000000000001,"15min":1.718},"min":{"1min":0.35999999999999999,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":4.4100000000000001,"5min":17.135999999999999,"15min":17.135999999999999},"last":0.41699999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.9390000000000001,"15min":1.9379999999999999},"min":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":10.303000000000001,"5min":17.192,"15min":115.54000000000001},"last":0.45700000000000002},{"interface":"front","average":{"1min":2.9889999999999999,"5min":1.976,"15min":1.9810000000000001},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.30499999999999999},"max":{"1min":11.442,"5min":14.321,"15min":115.691},"last":0.56000000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0939999999999999,"5min":1.744,"15min":1.7609999999999999},"min":{"1min":0.42699999999999999,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":15.972},"last":0.60099999999999998},{"interface":"front","average":{"1min":1.974,"5min":1.829,"15min":1.6240000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":6.9749999999999996,"5min":17.164000000000001,"15min":17.164000000000001},"last":0.49299999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019335,"num_pgs":65,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":959096,"kb_used_data":272304,"kb_used_omap":0,"kb_used_meta":686784,"kb_avail":92786056,"statfs":{"total":95995035648,"available":95012921344,"internally_reserved":0,"allocated":278839296,"data_stored":276860664,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":703266816},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5449999999999999,"5min":1.7589999999999999,"15min":1.387},"min":{"1min":0.443,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.0960000000000001,"5min":7.6079999999999997,"15min":13.375999999999999},"last":0.249},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.919,"15min":1.6379999999999999},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":7.6440000000000001,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.28000000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:21:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":2.048,"15min":1.1459999999999999},"min":{"1min":0.255,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.619,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.35799999999999998},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.8899999999999999,"15min":1.0640000000000001},"min":{"1min":0.28399999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.1470000000000002,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.436}]},{"osd":2,"last update":"Sat Sep 23 18:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3839999999999999,"5min":1.768,"15min":1.8500000000000001},"min":{"1min":0.34300000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.1749999999999998,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.377},{"interface":"front","average":{"1min":2.8079999999999998,"5min":1.8360000000000001,"15min":1.7649999999999999},"min":{"1min":0.36199999999999999,"5min":0.29899999999999999,"15min":0.253},"max":{"1min":17.190999999999999,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.53100000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.395,"5min":1.5529999999999999,"15min":1.9630000000000001},"min":{"1min":0.379,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.7519999999999998,"5min":11.130000000000001,"15min":29.154},"last":0.47199999999999998},{"interface":"front","average":{"1min":2.6230000000000002,"5min":1.8300000000000001,"15min":1.9930000000000001},"min":{"1min":0.376,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":12.099,"5min":13.470000000000001,"15min":29.216000000000001},"last":0.47999999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:20:45 2023","interfaces":[{"interface":"back","average":{"1min":2.0819999999999999,"5min":2.3170000000000002,"15min":2.7829999999999999},"min":{"1min":0.253,"5min":0.253,"15min":0.253},"max":{"1min":9.5009999999999994,"5min":27.675000000000001,"15min":27.675000000000001},"last":0.32600000000000001},{"interface":"front","average":{"1min":1.1000000000000001,"5min":1.6499999999999999,"15min":2.5609999999999999},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":3.8359999999999999,"5min":13.938000000000001,"15min":32.640000000000001},"last":0.49399999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.8819999999999999,"15min":1.4099999999999999},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":3.9750000000000001,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.1890000000000001,"5min":2.0499999999999998,"15min":1.619},"min":{"1min":0.311,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.4969999999999999,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.41899999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.7770000000000001,"5min":2.5430000000000001,"15min":1.8360000000000001},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":18.283000000000001,"5min":19.298999999999999,"15min":19.298999999999999},"last":0.44500000000000001},{"interface":"front","average":{"1min":1.5760000000000001,"5min":1.7869999999999999,"15min":1.5409999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.28399999999999997},"max":{"1min":3.786,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.51000000000000001}]}]},{"osd":5,"up_from":420,"seq":1803886264432,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":584132,"kb_used_data":218556,"kb_used_omap":9,"kb_used_meta":365558,"kb_avail":93161020,"statfs":{"total":95995035648,"available":95396884480,"internally_reserved":0,"allocated":223801344,"data_stored":221822902,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":374331837},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.2290000000000001,"15min":0.93200000000000005},"min":{"1min":0.21299999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.867,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.35499999999999998},{"interface":"front","average":{"1min":2.137,"5min":1.8,"15min":1.6459999999999999},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.621,"5min":20.937999999999999,"15min":20.937999999999999},"last":0.47499999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:21:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":1.9570000000000001,"15min":1.383},"min":{"1min":0.28999999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.752000000000001,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.379},{"interface":"front","average":{"1min":2.1640000000000001,"5min":1.9930000000000001,"15min":1.2290000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.4969999999999999,"5min":23.268999999999998,"15min":23.268999999999998},"last":0.41599999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.6160000000000001,"15min":1.611},"min":{"1min":0.29699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.1150000000000002,"5min":10.705,"15min":10.705},"last":0.48999999999999999},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.8520000000000001,"15min":2.1280000000000001},"min":{"1min":0.39600000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.7560000000000002,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.39000000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.762,"15min":1.359},"min":{"1min":0.35599999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.0060000000000002,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":0.317},{"interface":"front","average":{"1min":2.2200000000000002,"5min":2.1150000000000002,"15min":1.7130000000000001},"min":{"1min":0.372,"5min":0.33500000000000002,"15min":0.313},"max":{"1min":10.664999999999999,"5min":20.035,"15min":20.035},"last":0.33500000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":2.1480000000000001,"15min":2.0499999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.2270000000000003,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.36599999999999999},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1179999999999999,"15min":1.6639999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7919999999999998,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.434}]},{"osd":6,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.8340000000000001,"15min":1.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.2189999999999999,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.40899999999999997},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.3319999999999999,"15min":1.7370000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.6920000000000002,"5min":12.504,"15min":12.504},"last":0.44800000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.23,"15min":1.804},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.195,"5min":12.641,"15min":13.506},"last":0.39800000000000002},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.8500000000000001,"15min":1.5600000000000001},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":5.0839999999999996,"5min":13.138,"15min":13.138},"last":0.46400000000000002}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":276955136,"data_stored":276955136,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":649003008,"data_stored":649003008,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":271843328,"data_stored":271843328,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":214564864,"data_stored":214564864,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":10,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":280821760,"data_stored":280821760,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:21:49.664 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:49.771 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1690996435' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:21:49.771 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:49 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:21:49.771 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:49 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2938541316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:49.772 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:49.773 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.6 dump_historic_ops 2023-09-23T18:21:49.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1690996435' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:21:49.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:49 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]: dispatch 2023-09-23T18:21:49.888 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:49 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2938541316' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:49.890 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:49.894 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:49.895 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:49.997 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:50.029 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:50.063 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:50.066 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:50.067 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:50.168 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:50.197 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:50.332 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:50.335 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:50.335 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:50.369 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:21:50.436 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:50.443 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T18:21:50.443 INFO:teuthology.task.print:**** done end radosbench.yaml 2023-09-23T18:21:50.443 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T18:21:50.443 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T18:21:50.444 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T18:21:50.444 DEBUG:teuthology.orchestra.run.smithi055:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:21:50.462 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:21:50] "GET /metrics HTTP/1.1" 200 36681 "" "Prometheus/2.43.0" 2023-09-23T18:21:50.466 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:50.508 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:21:50.509 INFO:teuthology.orchestra.run.smithi055.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T18:21:50.509 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:21:50.552 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:50.556 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:50.557 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:50.575 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T18:21:50.575 DEBUG:teuthology.orchestra.run.smithi055:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T18:21:50.632 INFO:tasks.workunit:timeout=3h 2023-09-23T18:21:50.632 INFO:tasks.workunit:cleanup=True 2023-09-23T18:21:50.633 DEBUG:teuthology.orchestra.run.smithi055:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T18:21:50.657 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:50.704 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:50.711 INFO:tasks.workunit.client.0.smithi055.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T18:21:50.740 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:50.743 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-mon[184174]: from='client.56776 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:50.743 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-mon[184174]: pgmap v1521: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 11521/81 objects degraded (14223.457%); 0 B/s, 63 objects/s recovering 2023-09-23T18:21:50.743 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-mon[184174]: failed to encode map e627 with expected crc 2023-09-23T18:21:50.743 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:21:50.743 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-mon[184174]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T18:21:50.744 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-mon[179668]: from='client.56776 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:50.744 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-mon[179668]: pgmap v1521: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 11521/81 objects degraded (14223.457%); 0 B/s, 63 objects/s recovering 2023-09-23T18:21:50.744 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-mon[179668]: failed to encode map e627 with expected crc 2023-09-23T18:21:50.744 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:21:50.744 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:50 smithi055 ceph-mon[179668]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T18:21:50.745 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:50.745 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:50.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:50 smithi099 ceph-mon[167076]: from='client.56776 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:50.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:50 smithi099 ceph-mon[167076]: pgmap v1521: 119 pgs: 1 active+recovering+undersized+degraded+remapped, 3 active+recovery_wait+undersized+degraded+remapped, 115 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 11521/81 objects degraded (14223.457%); 0 B/s, 63 objects/s recovering 2023-09-23T18:21:50.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:50 smithi099 ceph-mon[167076]: failed to encode map e627 with expected crc 2023-09-23T18:21:50.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:50 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_5", "pool2": "unique_pool_5", "yes_i_really_really_mean_it": true}]': finished 2023-09-23T18:21:50.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:50 smithi099 ceph-mon[167076]: osdmap e627: 8 total, 8 up, 6 in 2023-09-23T18:21:50.845 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:50.877 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:51.161 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:51.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:51.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:51.266 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:51.296 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:51.379 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:51.382 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:51.383 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:51.483 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:51.512 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:51.597 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:51.601 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:51.601 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:51.659 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[179668]: failed to encode map e627 with expected crc 2023-09-23T18:21:51.659 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[179668]: failed to encode map e628 with expected crc 2023-09-23T18:21:51.659 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[179668]: osdmap e628: 8 total, 8 up, 6 in 2023-09-23T18:21:51.659 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[179668]: failed to encode map e628 with expected crc 2023-09-23T18:21:51.660 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2230437053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:51.660 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:51.663 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:21:51.663 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:21:51.667 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:21:51.702 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:51.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:51 smithi099 ceph-mon[167076]: failed to encode map e627 with expected crc 2023-09-23T18:21:51.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:51 smithi099 ceph-mon[167076]: failed to encode map e628 with expected crc 2023-09-23T18:21:51.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:51 smithi099 ceph-mon[167076]: osdmap e628: 8 total, 8 up, 6 in 2023-09-23T18:21:51.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:51 smithi099 ceph-mon[167076]: failed to encode map e628 with expected crc 2023-09-23T18:21:51.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:51 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2230437053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:51.727 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:51.730 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:51.815 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:51.818 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:51.819 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:51.890 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:21:51.917 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[184174]: failed to encode map e627 with expected crc 2023-09-23T18:21:51.917 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[184174]: failed to encode map e628 with expected crc 2023-09-23T18:21:51.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[184174]: osdmap e628: 8 total, 8 up, 6 in 2023-09-23T18:21:51.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[184174]: failed to encode map e628 with expected crc 2023-09-23T18:21:51.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2230437053' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:21:51.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:21:51.920 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:51.948 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:51.982 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:51.985 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:51.986 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:52.015 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_ops_in_flight 2023-09-23T18:21:52.087 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:52.124 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:52.157 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:52.160 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:52.160 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:52.261 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:52.290 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:52.324 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:52.326 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:52.327 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:52.428 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:52.454 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:52.488 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:52.490 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:52.491 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:52.498 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:52.592 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:52.618 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:52.651 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:52.652 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:21:52.654 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:52.655 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:52.664 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:21:52.665 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[179668]: pgmap v1524: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 13/81 objects degraded (16.049%) 2023-09-23T18:21:52.665 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:21:52.665 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[179668]: failed to encode map e629 with expected crc 2023-09-23T18:21:52.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2230437053' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:52.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[179668]: osdmap e629: 8 total, 8 up, 6 in 2023-09-23T18:21:52.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[179668]: failed to encode map e629 with expected crc 2023-09-23T18:21:52.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[179668]: 1.c scrub starts 2023-09-23T18:21:52.666 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[179668]: 1.c scrub ok 2023-09-23T18:21:52.755 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:52.787 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:52 smithi099 ceph-mon[167076]: pgmap v1524: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 13/81 objects degraded (16.049%) 2023-09-23T18:21:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:52 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:21:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:52 smithi099 ceph-mon[167076]: failed to encode map e629 with expected crc 2023-09-23T18:21:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2230437053' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:52 smithi099 ceph-mon[167076]: osdmap e629: 8 total, 8 up, 6 in 2023-09-23T18:21:52.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:52 smithi099 ceph-mon[167076]: failed to encode map e629 with expected crc 2023-09-23T18:21:52.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:52 smithi099 ceph-mon[167076]: 1.c scrub starts 2023-09-23T18:21:52.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:52 smithi099 ceph-mon[167076]: 1.c scrub ok 2023-09-23T18:21:52.832 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:52.838 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:52.838 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:21:52.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[184174]: pgmap v1524: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.3 MiB data, 5.2 GiB used, 531 GiB / 536 GiB avail; 13/81 objects degraded (16.049%) 2023-09-23T18:21:52.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:21:52.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[184174]: failed to encode map e629 with expected crc 2023-09-23T18:21:52.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2230437053' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:21:52.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[184174]: osdmap e629: 8 total, 8 up, 6 in 2023-09-23T18:21:52.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[184174]: failed to encode map e629 with expected crc 2023-09-23T18:21:52.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[184174]: 1.c scrub starts 2023-09-23T18:21:52.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:52 smithi055 ceph-mon[184174]: 1.c scrub ok 2023-09-23T18:21:52.939 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:52.966 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:53.370 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:53.373 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:53.374 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:53.427 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:53.475 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:53.508 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:53.653 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:53.693 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:53.693 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:53.792 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:53 smithi055 ceph-mon[179668]: Health check update: Degraded data redundancy: 13/81 objects degraded (16.049%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:21:53.793 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:53 smithi055 ceph-mon[184174]: Health check update: Degraded data redundancy: 13/81 objects degraded (16.049%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:21:53.794 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:53.802 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:21:53.825 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:21:53.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:53 smithi099 ceph-mon[167076]: Health check update: Degraded data redundancy: 13/81 objects degraded (16.049%), 1 pg degraded, 1 pg undersized (PG_DEGRADED) 2023-09-23T18:21:53.834 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:53.834 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:21:53.834 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:21:53.850 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:21:53.874 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:53.874 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:21:53.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:53.878 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:53.895 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:21:53.915 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:21:53.935 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:21:53.955 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:21:53.957 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:21:53.957 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:21:53.957 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:21:53.958 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:21:53.958 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:21:53.958 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:21:53.958 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:21:53.958 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:21:53.978 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:54.009 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:54.061 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:21:54.158 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:54.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:54.161 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:54.195 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_blocked_ops 2023-09-23T18:21:54.263 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:54.292 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:54.683 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:54.686 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:54.687 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:54.788 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:54.815 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:54.826 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:21:54.850 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:54.853 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:54.853 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:54.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:54 smithi055 ceph-mon[179668]: pgmap v1526: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 13/81 objects degraded (16.049%) 2023-09-23T18:21:54.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:54 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3328208309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:54.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:54 smithi055 ceph-mon[184174]: pgmap v1526: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 13/81 objects degraded (16.049%) 2023-09-23T18:21:54.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:54 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3328208309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:54.954 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:21:54.978 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:54 smithi099 ceph-mon[167076]: pgmap v1526: 103 pgs: 1 active+recovery_wait+undersized+degraded+remapped, 102 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 13/81 objects degraded (16.049%) 2023-09-23T18:21:54.978 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:54 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3328208309' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:21:54.980 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:55.227 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:55.481 DEBUG:teuthology.orchestra.run.smithi055:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T18:21:55.499 INFO:tasks.workunit.client.0.smithi055.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T18:21:55.501 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T18:21:55.501 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T18:21:55.576 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T18:21:55.625 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T18:21:55.667 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T18:21:55.668 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T18:21:55.668 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T18:21:55.711 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T18:21:55.733 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T18:21:55.734 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T18:21:55.798 INFO:tasks.workunit:Running workunits matching cls/test_cls_rbd.sh on client.0... 2023-09-23T18:21:55.799 INFO:tasks.workunit:Running workunit cls/test_cls_rbd.sh... 2023-09-23T18:21:55.799 DEBUG:teuthology.orchestra.run.smithi055:workunit test cls/test_cls_rbd.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/cls/test_cls_rbd.sh 2023-09-23T18:21:55.806 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:21:55.808 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:21:55.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:55 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/81 objects degraded (16.049%), 1 pg degraded, 1 pg undersized) 2023-09-23T18:21:55.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:55 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:21:55.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:55 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/81 objects degraded (16.049%), 1 pg degraded, 1 pg undersized) 2023-09-23T18:21:55.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:55 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:21:55.931 INFO:tasks.workunit.client.0.smithi055.stdout:Running main() from gmock_main.cc 2023-09-23T18:21:55.931 INFO:tasks.workunit.client.0.smithi055.stdout:[==========] Running 70 tests from 1 test suite. 2023-09-23T18:21:55.931 INFO:tasks.workunit.client.0.smithi055.stdout:[----------] Global test environment set-up. 2023-09-23T18:21:55.931 INFO:tasks.workunit.client.0.smithi055.stdout:[----------] 70 tests from TestClsRbd 2023-09-23T18:21:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:55 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 13/81 objects degraded (16.049%), 1 pg degraded, 1 pg undersized) 2023-09-23T18:21:56.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:55 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:21:56.118 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:21:56.122 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:56.123 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:21:56.223 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:56.255 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:56.293 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:56.297 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:56.297 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:56.398 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:21:56.428 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:56.592 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:21:56.596 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:56.596 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:21:56.614 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1527,"stamp":"2023-09-23T18:21:55.223064+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":4471460,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":5959,"num_read_kb":12160,"num_write":911,"num_write_kb":12762,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":411,"num_bytes_recovered":19422857,"num_keys_recovered":40,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6022,"ondisk_log_size":6022,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":356,"num_osds":6,"num_per_pool_osds":6,"num_per_pool_omap_osds":6,"kb":562470912,"kb_used":5366688,"kb_used_data":1679920,"kb_used_omap":40,"kb_used_meta":3686679,"kb_avail":557104224,"statfs":{"total":575970213888,"available":570474725376,"internally_reserved":0,"allocated":1720238080,"data_stored":1705084003,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":41670,"internal_metadata":3775159610},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":78,"apply_latency_ms":78,"commit_latency_ns":78000000,"apply_latency_ns":78000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":-11521,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":-1,"num_read_kb":-1,"num_write":-206808,"num_write_kb":-6617793,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":-11621,"num_bytes_recovered":-482803712,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"6.001272"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":915,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.473964+0000","last_change":"2023-09-23T18:17:21.786098+0000","last_active":"2023-09-23T18:21:44.473964+0000","last_peered":"2023-09-23T18:21:44.473964+0000","last_clean":"2023-09-23T18:21:44.473964+0000","last_became_active":"2023-09-23T18:17:21.784157+0000","last_became_peered":"2023-09-23T18:17:21.784157+0000","last_unstale":"2023-09-23T18:21:44.473964+0000","last_undegraded":"2023-09-23T18:21:44.473964+0000","last_fullsized":"2023-09-23T18:21:44.473964+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:13:37.476250+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":1,"num_bytes_recovered":262688,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":288,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T18:21:50.679825+0000","last_change":"2023-09-23T18:18:33.327326+0000","last_active":"2023-09-23T18:21:50.679825+0000","last_peered":"2023-09-23T18:21:50.679825+0000","last_clean":"2023-09-23T18:21:50.679825+0000","last_became_active":"2023-09-23T18:18:33.326825+0000","last_became_peered":"2023-09-23T18:18:33.326825+0000","last_unstale":"2023-09-23T18:21:50.679825+0000","last_undegraded":"2023-09-23T18:21:50.679825+0000","last_fullsized":"2023-09-23T18:21:50.679825+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:31:49.155926+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":45,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.676914+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:21:49.676914+0000","last_peered":"2023-09-23T18:21:49.676914+0000","last_clean":"2023-09-23T18:21:49.676914+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:21:49.676914+0000","last_undegraded":"2023-09-23T18:21:49.676914+0000","last_fullsized":"2023-09-23T18:21:49.676914+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":921,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474304+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:21:44.474304+0000","last_peered":"2023-09-23T18:21:44.474304+0000","last_clean":"2023-09-23T18:21:44.474304+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:21:44.474304+0000","last_undegraded":"2023-09-23T18:21:44.474304+0000","last_fullsized":"2023-09-23T18:21:44.474304+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":369,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.675821+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:21:49.675821+0000","last_peered":"2023-09-23T18:21:49.675821+0000","last_clean":"2023-09-23T18:21:49.675821+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:21:49.675821+0000","last_undegraded":"2023-09-23T18:21:49.675821+0000","last_fullsized":"2023-09-23T18:21:49.675821+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":688,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.449504+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:21:44.449504+0000","last_peered":"2023-09-23T18:21:44.449504+0000","last_clean":"2023-09-23T18:21:44.449504+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:21:44.449504+0000","last_undegraded":"2023-09-23T18:21:44.449504+0000","last_fullsized":"2023-09-23T18:21:44.449504+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":938,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475960+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:21:44.475960+0000","last_peered":"2023-09-23T18:21:44.475960+0000","last_clean":"2023-09-23T18:21:44.475960+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:21:44.475960+0000","last_undegraded":"2023-09-23T18:21:44.475960+0000","last_fullsized":"2023-09-23T18:21:44.475960+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":45,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.674582+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:21:49.674582+0000","last_peered":"2023-09-23T18:21:49.674582+0000","last_clean":"2023-09-23T18:21:49.674582+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:21:49.674582+0000","last_undegraded":"2023-09-23T18:21:49.674582+0000","last_fullsized":"2023-09-23T18:21:49.674582+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":203,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.478178+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:21:44.478178+0000","last_peered":"2023-09-23T18:21:44.478178+0000","last_clean":"2023-09-23T18:21:44.478178+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:21:44.478178+0000","last_undegraded":"2023-09-23T18:21:44.478178+0000","last_fullsized":"2023-09-23T18:21:44.478178+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1061,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475370+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:21:44.475370+0000","last_peered":"2023-09-23T18:21:44.475370+0000","last_clean":"2023-09-23T18:21:44.475370+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:21:44.475370+0000","last_undegraded":"2023-09-23T18:21:44.475370+0000","last_fullsized":"2023-09-23T18:21:44.475370+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":137,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474937+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:21:44.474937+0000","last_peered":"2023-09-23T18:21:44.474937+0000","last_clean":"2023-09-23T18:21:44.474937+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:21:44.474937+0000","last_undegraded":"2023-09-23T18:21:44.474937+0000","last_fullsized":"2023-09-23T18:21:44.474937+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":218,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.674955+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:21:49.674955+0000","last_peered":"2023-09-23T18:21:49.674955+0000","last_clean":"2023-09-23T18:21:49.674955+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:21:49.674955+0000","last_undegraded":"2023-09-23T18:21:49.674955+0000","last_fullsized":"2023-09-23T18:21:49.674955+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":838,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474092+0000","last_change":"2023-09-23T18:17:52.390584+0000","last_active":"2023-09-23T18:21:44.474092+0000","last_peered":"2023-09-23T18:21:44.474092+0000","last_clean":"2023-09-23T18:21:44.474092+0000","last_became_active":"2023-09-23T18:17:52.386810+0000","last_became_peered":"2023-09-23T18:17:52.386810+0000","last_unstale":"2023-09-23T18:21:44.474092+0000","last_undegraded":"2023-09-23T18:21:44.474092+0000","last_fullsized":"2023-09-23T18:21:44.474092+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:00:50.081413+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":173,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474501+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:21:44.474501+0000","last_peered":"2023-09-23T18:21:44.474501+0000","last_clean":"2023-09-23T18:21:44.474501+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:21:44.474501+0000","last_undegraded":"2023-09-23T18:21:44.474501+0000","last_fullsized":"2023-09-23T18:21:44.474501+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":172,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474427+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:21:44.474427+0000","last_peered":"2023-09-23T18:21:44.474427+0000","last_clean":"2023-09-23T18:21:44.474427+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:21:44.474427+0000","last_undegraded":"2023-09-23T18:21:44.474427+0000","last_fullsized":"2023-09-23T18:21:44.474427+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":972,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475021+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:21:44.475021+0000","last_peered":"2023-09-23T18:21:44.475021+0000","last_clean":"2023-09-23T18:21:44.475021+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:21:44.475021+0000","last_undegraded":"2023-09-23T18:21:44.475021+0000","last_fullsized":"2023-09-23T18:21:44.475021+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":423,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.680621+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:21:49.680621+0000","last_peered":"2023-09-23T18:21:49.680621+0000","last_clean":"2023-09-23T18:21:49.680621+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:21:49.680621+0000","last_undegraded":"2023-09-23T18:21:49.680621+0000","last_fullsized":"2023-09-23T18:21:49.680621+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":440,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.477108+0000","last_change":"2023-09-23T18:18:33.325975+0000","last_active":"2023-09-23T18:21:44.477108+0000","last_peered":"2023-09-23T18:21:44.477108+0000","last_clean":"2023-09-23T18:21:44.477108+0000","last_became_active":"2023-09-23T18:18:33.325638+0000","last_became_peered":"2023-09-23T18:18:33.325638+0000","last_unstale":"2023-09-23T18:21:44.477108+0000","last_undegraded":"2023-09-23T18:21:44.477108+0000","last_fullsized":"2023-09-23T18:21:44.477108+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:09:02.802599+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":1003,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T18:21:50.667930+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:21:50.667930+0000","last_peered":"2023-09-23T18:21:50.667930+0000","last_clean":"2023-09-23T18:21:50.667930+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:21:50.667930+0000","last_undegraded":"2023-09-23T18:21:50.667930+0000","last_fullsized":"2023-09-23T18:21:50.667930+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":179,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475997+0000","last_change":"2023-09-23T18:18:33.340624+0000","last_active":"2023-09-23T18:21:44.475997+0000","last_peered":"2023-09-23T18:21:44.475997+0000","last_clean":"2023-09-23T18:21:44.475997+0000","last_became_active":"2023-09-23T18:18:33.325687+0000","last_became_peered":"2023-09-23T18:18:33.325687+0000","last_unstale":"2023-09-23T18:21:44.475997+0000","last_undegraded":"2023-09-23T18:21:44.475997+0000","last_fullsized":"2023-09-23T18:21:44.475997+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:20:08.974603+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":4734,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":446,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.477822+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:21:44.477822+0000","last_peered":"2023-09-23T18:21:44.477822+0000","last_clean":"2023-09-23T18:21:44.477822+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:21:44.477822+0000","last_undegraded":"2023-09-23T18:21:44.477822+0000","last_fullsized":"2023-09-23T18:21:44.477822+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":853,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476774+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:21:44.476774+0000","last_peered":"2023-09-23T18:21:44.476774+0000","last_clean":"2023-09-23T18:21:44.476774+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:21:44.476774+0000","last_undegraded":"2023-09-23T18:21:44.476774+0000","last_fullsized":"2023-09-23T18:21:44.476774+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":446,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.477737+0000","last_change":"2023-09-23T18:17:52.387942+0000","last_active":"2023-09-23T18:21:44.477737+0000","last_peered":"2023-09-23T18:21:44.477737+0000","last_clean":"2023-09-23T18:21:44.477737+0000","last_became_active":"2023-09-23T18:17:52.387272+0000","last_became_peered":"2023-09-23T18:17:52.387272+0000","last_unstale":"2023-09-23T18:21:44.477737+0000","last_undegraded":"2023-09-23T18:21:44.477737+0000","last_fullsized":"2023-09-23T18:21:44.477737+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:15:52.969677+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":136,"num_keys_recovered":40,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":124,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474117+0000","last_change":"2023-09-23T18:18:07.528218+0000","last_active":"2023-09-23T18:21:44.474117+0000","last_peered":"2023-09-23T18:21:44.474117+0000","last_clean":"2023-09-23T18:21:44.474117+0000","last_became_active":"2023-09-23T18:18:07.527878+0000","last_became_peered":"2023-09-23T18:18:07.527878+0000","last_unstale":"2023-09-23T18:21:44.474117+0000","last_undegraded":"2023-09-23T18:21:44.474117+0000","last_fullsized":"2023-09-23T18:21:44.474117+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:03:48.846630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":824,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.473774+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:21:44.473774+0000","last_peered":"2023-09-23T18:21:44.473774+0000","last_clean":"2023-09-23T18:21:44.473774+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:21:44.473774+0000","last_undegraded":"2023-09-23T18:21:44.473774+0000","last_fullsized":"2023-09-23T18:21:44.473774+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":143,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.448607+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:21:44.448607+0000","last_peered":"2023-09-23T18:21:44.448607+0000","last_clean":"2023-09-23T18:21:44.448607+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:21:44.448607+0000","last_undegraded":"2023-09-23T18:21:44.448607+0000","last_fullsized":"2023-09-23T18:21:44.448607+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":137,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475629+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:21:44.475629+0000","last_peered":"2023-09-23T18:21:44.475629+0000","last_clean":"2023-09-23T18:21:44.475629+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:21:44.475629+0000","last_undegraded":"2023-09-23T18:21:44.475629+0000","last_fullsized":"2023-09-23T18:21:44.475629+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":764,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.676572+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:21:49.676572+0000","last_peered":"2023-09-23T18:21:49.676572+0000","last_clean":"2023-09-23T18:21:49.676572+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:21:49.676572+0000","last_undegraded":"2023-09-23T18:21:49.676572+0000","last_fullsized":"2023-09-23T18:21:49.676572+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":214,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474279+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:21:44.474279+0000","last_peered":"2023-09-23T18:21:44.474279+0000","last_clean":"2023-09-23T18:21:44.474279+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:21:44.474279+0000","last_undegraded":"2023-09-23T18:21:44.474279+0000","last_fullsized":"2023-09-23T18:21:44.474279+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":227,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.682697+0000","last_change":"2023-09-23T18:18:33.321235+0000","last_active":"2023-09-23T18:21:49.682697+0000","last_peered":"2023-09-23T18:21:49.682697+0000","last_clean":"2023-09-23T18:21:49.682697+0000","last_became_active":"2023-09-23T18:18:33.320845+0000","last_became_peered":"2023-09-23T18:18:33.320845+0000","last_unstale":"2023-09-23T18:21:49.682697+0000","last_undegraded":"2023-09-23T18:21:49.682697+0000","last_fullsized":"2023-09-23T18:21:49.682697+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:20.339300+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":858,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476665+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:21:44.476665+0000","last_peered":"2023-09-23T18:21:44.476665+0000","last_clean":"2023-09-23T18:21:44.476665+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:21:44.476665+0000","last_undegraded":"2023-09-23T18:21:44.476665+0000","last_fullsized":"2023-09-23T18:21:44.476665+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":767,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474935+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:21:44.474935+0000","last_peered":"2023-09-23T18:21:44.474935+0000","last_clean":"2023-09-23T18:21:44.474935+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:21:44.474935+0000","last_undegraded":"2023-09-23T18:21:44.474935+0000","last_fullsized":"2023-09-23T18:21:44.474935+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":45,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.681740+0000","last_change":"2023-09-23T18:18:46.435910+0000","last_active":"2023-09-23T18:21:49.681740+0000","last_peered":"2023-09-23T18:21:49.681740+0000","last_clean":"2023-09-23T18:21:49.681740+0000","last_became_active":"2023-09-23T18:18:46.435519+0000","last_became_peered":"2023-09-23T18:18:46.435519+0000","last_unstale":"2023-09-23T18:21:49.681740+0000","last_undegraded":"2023-09-23T18:21:49.681740+0000","last_fullsized":"2023-09-23T18:21:49.681740+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:47.675409+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":892,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475589+0000","last_change":"2023-09-23T18:17:52.389124+0000","last_active":"2023-09-23T18:21:44.475589+0000","last_peered":"2023-09-23T18:21:44.475589+0000","last_clean":"2023-09-23T18:21:44.475589+0000","last_became_active":"2023-09-23T18:17:52.388806+0000","last_became_peered":"2023-09-23T18:17:52.388806+0000","last_unstale":"2023-09-23T18:21:44.475589+0000","last_undegraded":"2023-09-23T18:21:44.475589+0000","last_fullsized":"2023-09-23T18:21:44.475589+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:14.206412+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":265,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T18:21:50.680304+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:21:50.680304+0000","last_peered":"2023-09-23T18:21:50.680304+0000","last_clean":"2023-09-23T18:21:50.680304+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:21:50.680304+0000","last_undegraded":"2023-09-23T18:21:50.680304+0000","last_fullsized":"2023-09-23T18:21:50.680304+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":137,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.473767+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:21:44.473767+0000","last_peered":"2023-09-23T18:21:44.473767+0000","last_clean":"2023-09-23T18:21:44.473767+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:21:44.473767+0000","last_undegraded":"2023-09-23T18:21:44.473767+0000","last_fullsized":"2023-09-23T18:21:44.473767+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":748,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.675433+0000","last_change":"2023-09-23T18:19:34.407955+0000","last_active":"2023-09-23T18:21:49.675433+0000","last_peered":"2023-09-23T18:21:49.675433+0000","last_clean":"2023-09-23T18:21:49.675433+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:21:49.675433+0000","last_undegraded":"2023-09-23T18:21:49.675433+0000","last_fullsized":"2023-09-23T18:21:49.675433+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:01.015845+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:01.015845+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":true,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"queued for scrub","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":8945,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":7,"num_bytes_recovered":2375208,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":266,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T18:21:50.749164+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:21:50.749164+0000","last_peered":"2023-09-23T18:21:50.749164+0000","last_clean":"2023-09-23T18:21:50.749164+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:21:50.749164+0000","last_undegraded":"2023-09-23T18:21:50.749164+0000","last_fullsized":"2023-09-23T18:21:50.749164+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":245,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T18:21:50.766011+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:21:50.766011+0000","last_peered":"2023-09-23T18:21:50.766011+0000","last_clean":"2023-09-23T18:21:50.766011+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:21:50.766011+0000","last_undegraded":"2023-09-23T18:21:50.766011+0000","last_fullsized":"2023-09-23T18:21:50.766011+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":642,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.478112+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:21:44.478112+0000","last_peered":"2023-09-23T18:21:44.478112+0000","last_clean":"2023-09-23T18:21:44.478112+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:21:44.478112+0000","last_undegraded":"2023-09-23T18:21:44.478112+0000","last_fullsized":"2023-09-23T18:21:44.478112+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":552,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475461+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:21:44.475461+0000","last_peered":"2023-09-23T18:21:44.475461+0000","last_clean":"2023-09-23T18:21:44.475461+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:21:44.475461+0000","last_undegraded":"2023-09-23T18:21:44.475461+0000","last_fullsized":"2023-09-23T18:21:44.475461+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":203,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476974+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:21:44.476974+0000","last_peered":"2023-09-23T18:21:44.476974+0000","last_clean":"2023-09-23T18:21:44.476974+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:21:44.476974+0000","last_undegraded":"2023-09-23T18:21:44.476974+0000","last_fullsized":"2023-09-23T18:21:44.476974+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":659,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.682767+0000","last_change":"2023-09-23T18:17:54.492869+0000","last_active":"2023-09-23T18:21:49.682767+0000","last_peered":"2023-09-23T18:21:49.682767+0000","last_clean":"2023-09-23T18:21:49.682767+0000","last_became_active":"2023-09-23T18:17:52.384303+0000","last_became_peered":"2023-09-23T18:17:52.384303+0000","last_unstale":"2023-09-23T18:21:49.682767+0000","last_undegraded":"2023-09-23T18:21:49.682767+0000","last_fullsized":"2023-09-23T18:21:49.682767+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:14.630169+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,1],"acting":[3,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":151,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474369+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:21:44.474369+0000","last_peered":"2023-09-23T18:21:44.474369+0000","last_clean":"2023-09-23T18:21:44.474369+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:21:44.474369+0000","last_undegraded":"2023-09-23T18:21:44.474369+0000","last_fullsized":"2023-09-23T18:21:44.474369+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":464,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.451003+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:21:44.451003+0000","last_peered":"2023-09-23T18:21:44.451003+0000","last_clean":"2023-09-23T18:21:44.451003+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:21:44.451003+0000","last_undegraded":"2023-09-23T18:21:44.451003+0000","last_fullsized":"2023-09-23T18:21:44.451003+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1109,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476726+0000","last_change":"2023-09-23T18:16:56.376439+0000","last_active":"2023-09-23T18:21:44.476726+0000","last_peered":"2023-09-23T18:21:44.476726+0000","last_clean":"2023-09-23T18:21:44.476726+0000","last_became_active":"2023-09-23T18:16:56.375867+0000","last_became_peered":"2023-09-23T18:16:56.375867+0000","last_unstale":"2023-09-23T18:21:44.476726+0000","last_undegraded":"2023-09-23T18:21:44.476726+0000","last_fullsized":"2023-09-23T18:21:44.476726+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:26:00.011070+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":365,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.676741+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:21:49.676741+0000","last_peered":"2023-09-23T18:21:49.676741+0000","last_clean":"2023-09-23T18:21:49.676741+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:21:49.676741+0000","last_undegraded":"2023-09-23T18:21:49.676741+0000","last_fullsized":"2023-09-23T18:21:49.676741+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":171,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475607+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:21:44.475607+0000","last_peered":"2023-09-23T18:21:44.475607+0000","last_clean":"2023-09-23T18:21:44.475607+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:21:44.475607+0000","last_undegraded":"2023-09-23T18:21:44.475607+0000","last_fullsized":"2023-09-23T18:21:44.475607+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":896,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.473960+0000","last_change":"2023-09-23T18:18:49.372222+0000","last_active":"2023-09-23T18:21:44.473960+0000","last_peered":"2023-09-23T18:21:44.473960+0000","last_clean":"2023-09-23T18:21:44.473960+0000","last_became_active":"2023-09-23T18:18:49.371904+0000","last_became_peered":"2023-09-23T18:18:49.371904+0000","last_unstale":"2023-09-23T18:21:44.473960+0000","last_undegraded":"2023-09-23T18:21:44.473960+0000","last_fullsized":"2023-09-23T18:21:44.473960+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:30:45.174208+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":179,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476894+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:21:44.476894+0000","last_peered":"2023-09-23T18:21:44.476894+0000","last_clean":"2023-09-23T18:21:44.476894+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:21:44.476894+0000","last_undegraded":"2023-09-23T18:21:44.476894+0000","last_fullsized":"2023-09-23T18:21:44.476894+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":478,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475724+0000","last_change":"2023-09-23T18:17:52.388016+0000","last_active":"2023-09-23T18:21:44.475724+0000","last_peered":"2023-09-23T18:21:44.475724+0000","last_clean":"2023-09-23T18:21:44.475724+0000","last_became_active":"2023-09-23T18:17:52.386413+0000","last_became_peered":"2023-09-23T18:17:52.386413+0000","last_unstale":"2023-09-23T18:21:44.475724+0000","last_undegraded":"2023-09-23T18:21:44.475724+0000","last_fullsized":"2023-09-23T18:21:44.475724+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:27:01.048069+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":898,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474390+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:21:44.474390+0000","last_peered":"2023-09-23T18:21:44.474390+0000","last_clean":"2023-09-23T18:21:44.474390+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:21:44.474390+0000","last_undegraded":"2023-09-23T18:21:44.474390+0000","last_fullsized":"2023-09-23T18:21:44.474390+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":463,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476693+0000","last_change":"2023-09-23T18:17:52.384955+0000","last_active":"2023-09-23T18:21:44.476693+0000","last_peered":"2023-09-23T18:21:44.476693+0000","last_clean":"2023-09-23T18:21:44.476693+0000","last_became_active":"2023-09-23T18:17:52.384356+0000","last_became_peered":"2023-09-23T18:17:52.384356+0000","last_unstale":"2023-09-23T18:21:44.476693+0000","last_undegraded":"2023-09-23T18:21:44.476693+0000","last_fullsized":"2023-09-23T18:21:44.476693+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:31.587653+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":239,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476076+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:21:44.476076+0000","last_peered":"2023-09-23T18:21:44.476076+0000","last_clean":"2023-09-23T18:21:44.476076+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:21:44.476076+0000","last_undegraded":"2023-09-23T18:21:44.476076+0000","last_fullsized":"2023-09-23T18:21:44.476076+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":879,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476440+0000","last_change":"2023-09-23T18:17:05.615168+0000","last_active":"2023-09-23T18:21:44.476440+0000","last_peered":"2023-09-23T18:21:44.476440+0000","last_clean":"2023-09-23T18:21:44.476440+0000","last_became_active":"2023-09-23T18:17:05.614847+0000","last_became_peered":"2023-09-23T18:17:05.614847+0000","last_unstale":"2023-09-23T18:21:44.476440+0000","last_undegraded":"2023-09-23T18:21:44.476440+0000","last_fullsized":"2023-09-23T18:21:44.476440+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:22:17.403025+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":45,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.674766+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:21:49.674766+0000","last_peered":"2023-09-23T18:21:49.674766+0000","last_clean":"2023-09-23T18:21:49.674766+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:21:49.674766+0000","last_undegraded":"2023-09-23T18:21:49.674766+0000","last_fullsized":"2023-09-23T18:21:49.674766+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":358,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T18:21:50.665305+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:21:50.665305+0000","last_peered":"2023-09-23T18:21:50.665305+0000","last_clean":"2023-09-23T18:21:50.665305+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:21:50.665305+0000","last_undegraded":"2023-09-23T18:21:50.665305+0000","last_fullsized":"2023-09-23T18:21:50.665305+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":738,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.478257+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:21:44.478257+0000","last_peered":"2023-09-23T18:21:44.478257+0000","last_clean":"2023-09-23T18:21:44.478257+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:21:44.478257+0000","last_undegraded":"2023-09-23T18:21:44.478257+0000","last_fullsized":"2023-09-23T18:21:44.478257+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"85'124","reported_seq":1017,"reported_epoch":629,"state":"active+clean","last_fresh":"2023-09-23T18:21:51.667042+0000","last_change":"2023-09-23T18:21:51.667042+0000","last_active":"2023-09-23T18:21:51.667042+0000","last_peered":"2023-09-23T18:21:51.667042+0000","last_clean":"2023-09-23T18:21:51.667042+0000","last_became_active":"2023-09-23T18:21:51.666429+0000","last_became_peered":"2023-09-23T18:21:51.666429+0000","last_unstale":"2023-09-23T18:21:51.667042+0000","last_undegraded":"2023-09-23T18:21:51.667042+0000","last_fullsized":"2023-09-23T18:21:51.667042+0000","mapping_epoch":628,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":629,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":124,"log_dups_size":0,"ondisk_log_size":124,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:03:24.607620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00051456099999999997,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":647,"num_read_kb":2787,"num_write":124,"num_write_kb":2203,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":65,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":161,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475707+0000","last_change":"2023-09-23T18:18:46.439625+0000","last_active":"2023-09-23T18:21:44.475707+0000","last_peered":"2023-09-23T18:21:44.475707+0000","last_clean":"2023-09-23T18:21:44.475707+0000","last_became_active":"2023-09-23T18:18:46.439241+0000","last_became_peered":"2023-09-23T18:18:46.439241+0000","last_unstale":"2023-09-23T18:21:44.475707+0000","last_undegraded":"2023-09-23T18:21:44.475707+0000","last_fullsized":"2023-09-23T18:21:44.475707+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:35:42.916805+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":141,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476337+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:21:44.476337+0000","last_peered":"2023-09-23T18:21:44.476337+0000","last_clean":"2023-09-23T18:21:44.476337+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:21:44.476337+0000","last_undegraded":"2023-09-23T18:21:44.476337+0000","last_fullsized":"2023-09-23T18:21:44.476337+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":810,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476869+0000","last_change":"2023-09-23T18:17:05.621817+0000","last_active":"2023-09-23T18:21:44.476869+0000","last_peered":"2023-09-23T18:21:44.476869+0000","last_clean":"2023-09-23T18:21:44.476869+0000","last_became_active":"2023-09-23T18:17:05.621026+0000","last_became_peered":"2023-09-23T18:17:05.621026+0000","last_unstale":"2023-09-23T18:21:44.476869+0000","last_undegraded":"2023-09-23T18:21:44.476869+0000","last_fullsized":"2023-09-23T18:21:44.476869+0000","mapping_epoch":536,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":537,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:56.662560+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"84'190","reported_seq":1511,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476246+0000","last_change":"2023-09-23T18:17:21.789104+0000","last_active":"2023-09-23T18:21:44.476246+0000","last_peered":"2023-09-23T18:21:44.476246+0000","last_clean":"2023-09-23T18:21:44.476246+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:21:44.476246+0000","last_undegraded":"2023-09-23T18:21:44.476246+0000","last_fullsized":"2023-09-23T18:21:44.476246+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":190,"log_dups_size":0,"ondisk_log_size":190,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00042115400000000003,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1852,"num_read_kb":2664,"num_write":143,"num_write_kb":1089,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":151,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.477781+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:21:44.477781+0000","last_peered":"2023-09-23T18:21:44.477781+0000","last_clean":"2023-09-23T18:21:44.477781+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:21:44.477781+0000","last_undegraded":"2023-09-23T18:21:44.477781+0000","last_fullsized":"2023-09-23T18:21:44.477781+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":451,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475857+0000","last_change":"2023-09-23T18:18:46.438163+0000","last_active":"2023-09-23T18:21:44.475857+0000","last_peered":"2023-09-23T18:21:44.475857+0000","last_clean":"2023-09-23T18:21:44.475857+0000","last_became_active":"2023-09-23T18:18:46.437805+0000","last_became_peered":"2023-09-23T18:18:46.437805+0000","last_unstale":"2023-09-23T18:21:44.475857+0000","last_undegraded":"2023-09-23T18:21:44.475857+0000","last_fullsized":"2023-09-23T18:21:44.475857+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:19:57.644538+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,3],"acting":[6,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":835,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476537+0000","last_change":"2023-09-23T18:17:52.383971+0000","last_active":"2023-09-23T18:21:44.476537+0000","last_peered":"2023-09-23T18:21:44.476537+0000","last_clean":"2023-09-23T18:21:44.476537+0000","last_became_active":"2023-09-23T18:17:52.383592+0000","last_became_peered":"2023-09-23T18:17:52.383592+0000","last_unstale":"2023-09-23T18:21:44.476537+0000","last_undegraded":"2023-09-23T18:21:44.476537+0000","last_fullsized":"2023-09-23T18:21:44.476537+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:05:27.061949+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,4],"acting":[5,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"85'88","reported_seq":71,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.675263+0000","last_change":"2023-09-23T18:20:35.016554+0000","last_active":"2023-09-23T18:21:49.675263+0000","last_peered":"2023-09-23T18:21:49.675263+0000","last_clean":"2023-09-23T18:21:49.675263+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:21:49.675263+0000","last_undegraded":"2023-09-23T18:21:49.675263+0000","last_fullsized":"2023-09-23T18:21:49.675263+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":88,"log_dups_size":0,"ondisk_log_size":88,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:12.231005+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.000665977,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":380,"num_read_kb":529,"num_write":85,"num_write_kb":4290,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":420,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475225+0000","last_change":"2023-09-23T18:17:21.774337+0000","last_active":"2023-09-23T18:21:44.475225+0000","last_peered":"2023-09-23T18:21:44.475225+0000","last_clean":"2023-09-23T18:21:44.475225+0000","last_became_active":"2023-09-23T18:17:21.774016+0000","last_became_peered":"2023-09-23T18:17:21.774016+0000","last_unstale":"2023-09-23T18:21:44.475225+0000","last_undegraded":"2023-09-23T18:21:44.475225+0000","last_fullsized":"2023-09-23T18:21:44.475225+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:27.559305+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,0],"acting":[6,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":163,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474859+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:21:44.474859+0000","last_peered":"2023-09-23T18:21:44.474859+0000","last_clean":"2023-09-23T18:21:44.474859+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:21:44.474859+0000","last_undegraded":"2023-09-23T18:21:44.474859+0000","last_fullsized":"2023-09-23T18:21:44.474859+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"85'251","reported_seq":1524,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474135+0000","last_change":"2023-09-23T18:17:08.342368+0000","last_active":"2023-09-23T18:21:44.474135+0000","last_peered":"2023-09-23T18:21:44.474135+0000","last_clean":"2023-09-23T18:21:44.474135+0000","last_became_active":"2023-09-23T18:17:08.341780+0000","last_became_peered":"2023-09-23T18:17:08.341780+0000","last_unstale":"2023-09-23T18:21:44.474135+0000","last_undegraded":"2023-09-23T18:21:44.474135+0000","last_fullsized":"2023-09-23T18:21:44.474135+0000","mapping_epoch":538,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":251,"log_dups_size":0,"ondisk_log_size":251,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:13:27.053039+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001253812,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1611,"num_read_kb":3474,"num_write":202,"num_write_kb":2666,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":42,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":577,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475091+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:21:44.475091+0000","last_peered":"2023-09-23T18:21:44.475091+0000","last_clean":"2023-09-23T18:21:44.475091+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:21:44.475091+0000","last_undegraded":"2023-09-23T18:21:44.475091+0000","last_fullsized":"2023-09-23T18:21:44.475091+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":137,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476057+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:21:44.476057+0000","last_peered":"2023-09-23T18:21:44.476057+0000","last_clean":"2023-09-23T18:21:44.476057+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:21:44.476057+0000","last_undegraded":"2023-09-23T18:21:44.476057+0000","last_fullsized":"2023-09-23T18:21:44.476057+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":172,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475956+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:21:44.475956+0000","last_peered":"2023-09-23T18:21:44.475956+0000","last_clean":"2023-09-23T18:21:44.475956+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:21:44.475956+0000","last_undegraded":"2023-09-23T18:21:44.475956+0000","last_fullsized":"2023-09-23T18:21:44.475956+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":825,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.477865+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:21:44.477865+0000","last_peered":"2023-09-23T18:21:44.477865+0000","last_clean":"2023-09-23T18:21:44.477865+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:21:44.477865+0000","last_undegraded":"2023-09-23T18:21:44.477865+0000","last_fullsized":"2023-09-23T18:21:44.477865+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"84'87","reported_seq":142,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.674889+0000","last_change":"2023-09-23T18:17:21.760134+0000","last_active":"2023-09-23T18:21:49.674889+0000","last_peered":"2023-09-23T18:21:49.674889+0000","last_clean":"2023-09-23T18:21:49.674889+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:21:49.674889+0000","last_undegraded":"2023-09-23T18:21:49.674889+0000","last_fullsized":"2023-09-23T18:21:49.674889+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00039679400000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":344,"num_read_kb":622,"num_write":90,"num_write_kb":224,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":45,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.675168+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:21:49.675168+0000","last_peered":"2023-09-23T18:21:49.675168+0000","last_clean":"2023-09-23T18:21:49.675168+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:21:49.675168+0000","last_undegraded":"2023-09-23T18:21:49.675168+0000","last_fullsized":"2023-09-23T18:21:49.675168+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":642,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474540+0000","last_change":"2023-09-23T18:18:33.335204+0000","last_active":"2023-09-23T18:21:44.474540+0000","last_peered":"2023-09-23T18:21:44.474540+0000","last_clean":"2023-09-23T18:21:44.474540+0000","last_became_active":"2023-09-23T18:18:33.334777+0000","last_became_peered":"2023-09-23T18:18:33.334777+0000","last_unstale":"2023-09-23T18:21:44.474540+0000","last_undegraded":"2023-09-23T18:21:44.474540+0000","last_fullsized":"2023-09-23T18:21:44.474540+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:31:58.444957+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":925,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476286+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:21:44.476286+0000","last_peered":"2023-09-23T18:21:44.476286+0000","last_clean":"2023-09-23T18:21:44.476286+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:21:44.476286+0000","last_undegraded":"2023-09-23T18:21:44.476286+0000","last_fullsized":"2023-09-23T18:21:44.476286+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"84'84","reported_seq":66,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.449108+0000","last_change":"2023-09-23T18:18:33.347213+0000","last_active":"2023-09-23T18:21:44.449108+0000","last_peered":"2023-09-23T18:21:44.449108+0000","last_clean":"2023-09-23T18:21:44.449108+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:21:44.449108+0000","last_undegraded":"2023-09-23T18:21:44.449108+0000","last_fullsized":"2023-09-23T18:21:44.449108+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":84,"log_dups_size":0,"ondisk_log_size":84,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00033496699999999998,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":393,"num_read_kb":445,"num_write":82,"num_write_kb":105,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":151,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474248+0000","last_change":"2023-09-23T18:18:33.335198+0000","last_active":"2023-09-23T18:21:44.474248+0000","last_peered":"2023-09-23T18:21:44.474248+0000","last_clean":"2023-09-23T18:21:44.474248+0000","last_became_active":"2023-09-23T18:18:33.334817+0000","last_became_peered":"2023-09-23T18:18:33.334817+0000","last_unstale":"2023-09-23T18:21:44.474248+0000","last_undegraded":"2023-09-23T18:21:44.474248+0000","last_fullsized":"2023-09-23T18:21:44.474248+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:13:48.536364+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":151,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474051+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:21:44.474051+0000","last_peered":"2023-09-23T18:21:44.474051+0000","last_clean":"2023-09-23T18:21:44.474051+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:21:44.474051+0000","last_undegraded":"2023-09-23T18:21:44.474051+0000","last_fullsized":"2023-09-23T18:21:44.474051+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":925,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475809+0000","last_change":"2023-09-23T18:17:52.388385+0000","last_active":"2023-09-23T18:21:44.475809+0000","last_peered":"2023-09-23T18:21:44.475809+0000","last_clean":"2023-09-23T18:21:44.475809+0000","last_became_active":"2023-09-23T18:17:52.387687+0000","last_became_peered":"2023-09-23T18:17:52.387687+0000","last_unstale":"2023-09-23T18:21:44.475809+0000","last_undegraded":"2023-09-23T18:21:44.475809+0000","last_fullsized":"2023-09-23T18:21:44.475809+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:53:12.000573+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":113,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474649+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:21:44.474649+0000","last_peered":"2023-09-23T18:21:44.474649+0000","last_clean":"2023-09-23T18:21:44.474649+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:21:44.474649+0000","last_undegraded":"2023-09-23T18:21:44.474649+0000","last_fullsized":"2023-09-23T18:21:44.474649+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"85'65","reported_seq":406,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476199+0000","last_change":"2023-09-23T18:17:20.787970+0000","last_active":"2023-09-23T18:21:44.476199+0000","last_peered":"2023-09-23T18:21:44.476199+0000","last_clean":"2023-09-23T18:21:44.476199+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:21:44.476199+0000","last_undegraded":"2023-09-23T18:21:44.476199+0000","last_fullsized":"2023-09-23T18:21:44.476199+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":65,"log_dups_size":0,"ondisk_log_size":65,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00034731900000000001,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":296,"num_read_kb":1244,"num_write":34,"num_write_kb":1033,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":45,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.674685+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:21:49.674685+0000","last_peered":"2023-09-23T18:21:49.674685+0000","last_clean":"2023-09-23T18:21:49.674685+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:21:49.674685+0000","last_undegraded":"2023-09-23T18:21:49.674685+0000","last_fullsized":"2023-09-23T18:21:49.674685+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":761,"reported_epoch":627,"state":"active+clean","last_fresh":"2023-09-23T18:21:49.675517+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:21:49.675517+0000","last_peered":"2023-09-23T18:21:49.675517+0000","last_clean":"2023-09-23T18:21:49.675517+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:21:49.675517+0000","last_undegraded":"2023-09-23T18:21:49.675517+0000","last_fullsized":"2023-09-23T18:21:49.675517+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"84'87","reported_seq":1063,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.478424+0000","last_change":"2023-09-23T18:17:21.843056+0000","last_active":"2023-09-23T18:21:44.478424+0000","last_peered":"2023-09-23T18:21:44.478424+0000","last_clean":"2023-09-23T18:21:44.478424+0000","last_became_active":"2023-09-23T18:17:21.777753+0000","last_became_peered":"2023-09-23T18:17:21.777753+0000","last_unstale":"2023-09-23T18:21:44.478424+0000","last_undegraded":"2023-09-23T18:21:44.478424+0000","last_fullsized":"2023-09-23T18:21:44.478424+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":87,"log_dups_size":0,"ondisk_log_size":87,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:58:46.611306+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047293900000000002,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":429,"num_read_kb":381,"num_write":86,"num_write_kb":1086,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":60,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":54,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.449684+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:21:44.449684+0000","last_peered":"2023-09-23T18:21:44.449684+0000","last_clean":"2023-09-23T18:21:44.449684+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:21:44.449684+0000","last_undegraded":"2023-09-23T18:21:44.449684+0000","last_fullsized":"2023-09-23T18:21:44.449684+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":417,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476779+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:21:44.476779+0000","last_peered":"2023-09-23T18:21:44.476779+0000","last_clean":"2023-09-23T18:21:44.476779+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:21:44.476779+0000","last_undegraded":"2023-09-23T18:21:44.476779+0000","last_fullsized":"2023-09-23T18:21:44.476779+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":657,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474507+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:21:44.474507+0000","last_peered":"2023-09-23T18:21:44.474507+0000","last_clean":"2023-09-23T18:21:44.474507+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:21:44.474507+0000","last_undegraded":"2023-09-23T18:21:44.474507+0000","last_fullsized":"2023-09-23T18:21:44.474507+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":50,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.450847+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:21:44.450847+0000","last_peered":"2023-09-23T18:21:44.450847+0000","last_clean":"2023-09-23T18:21:44.450847+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:21:44.450847+0000","last_undegraded":"2023-09-23T18:21:44.450847+0000","last_fullsized":"2023-09-23T18:21:44.450847+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":702,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T18:21:50.665828+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:21:50.665828+0000","last_peered":"2023-09-23T18:21:50.665828+0000","last_clean":"2023-09-23T18:21:50.665828+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:21:50.665828+0000","last_undegraded":"2023-09-23T18:21:50.665828+0000","last_fullsized":"2023-09-23T18:21:50.665828+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":657,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476117+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:21:44.476117+0000","last_peered":"2023-09-23T18:21:44.476117+0000","last_clean":"2023-09-23T18:21:44.476117+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:21:44.476117+0000","last_undegraded":"2023-09-23T18:21:44.476117+0000","last_fullsized":"2023-09-23T18:21:44.476117+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":145,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474295+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:21:44.474295+0000","last_peered":"2023-09-23T18:21:44.474295+0000","last_clean":"2023-09-23T18:21:44.474295+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:21:44.474295+0000","last_undegraded":"2023-09-23T18:21:44.474295+0000","last_fullsized":"2023-09-23T18:21:44.474295+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":263,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T18:21:50.679970+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:21:50.679970+0000","last_peered":"2023-09-23T18:21:50.679970+0000","last_clean":"2023-09-23T18:21:50.679970+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:21:50.679970+0000","last_undegraded":"2023-09-23T18:21:50.679970+0000","last_fullsized":"2023-09-23T18:21:50.679970+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":131,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475544+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:21:44.475544+0000","last_peered":"2023-09-23T18:21:44.475544+0000","last_clean":"2023-09-23T18:21:44.475544+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:21:44.475544+0000","last_undegraded":"2023-09-23T18:21:44.475544+0000","last_fullsized":"2023-09-23T18:21:44.475544+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":246,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T18:21:50.679904+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:21:50.679904+0000","last_peered":"2023-09-23T18:21:50.679904+0000","last_clean":"2023-09-23T18:21:50.679904+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:21:50.679904+0000","last_undegraded":"2023-09-23T18:21:50.679904+0000","last_fullsized":"2023-09-23T18:21:50.679904+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1019,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475521+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:21:44.475521+0000","last_peered":"2023-09-23T18:21:44.475521+0000","last_clean":"2023-09-23T18:21:44.475521+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:21:44.475521+0000","last_undegraded":"2023-09-23T18:21:44.475521+0000","last_fullsized":"2023-09-23T18:21:44.475521+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":524,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476243+0000","last_change":"2023-09-23T18:18:46.438178+0000","last_active":"2023-09-23T18:21:44.476243+0000","last_peered":"2023-09-23T18:21:44.476243+0000","last_clean":"2023-09-23T18:21:44.476243+0000","last_became_active":"2023-09-23T18:18:46.437799+0000","last_became_peered":"2023-09-23T18:18:46.437799+0000","last_unstale":"2023-09-23T18:21:44.476243+0000","last_undegraded":"2023-09-23T18:21:44.476243+0000","last_fullsized":"2023-09-23T18:21:44.476243+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:50.512455+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":161,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.476819+0000","last_change":"2023-09-23T18:18:46.434234+0000","last_active":"2023-09-23T18:21:44.476819+0000","last_peered":"2023-09-23T18:21:44.476819+0000","last_clean":"2023-09-23T18:21:44.476819+0000","last_became_active":"2023-09-23T18:18:46.433596+0000","last_became_peered":"2023-09-23T18:18:46.433596+0000","last_unstale":"2023-09-23T18:21:44.476819+0000","last_undegraded":"2023-09-23T18:21:44.476819+0000","last_fullsized":"2023-09-23T18:21:44.476819+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:00:38.919261+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":958,"reported_epoch":628,"state":"active+clean","last_fresh":"2023-09-23T18:21:50.665054+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:21:50.665054+0000","last_peered":"2023-09-23T18:21:50.665054+0000","last_clean":"2023-09-23T18:21:50.665054+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:21:50.665054+0000","last_undegraded":"2023-09-23T18:21:50.665054+0000","last_fullsized":"2023-09-23T18:21:50.665054+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":149,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.474785+0000","last_change":"2023-09-23T18:18:33.352625+0000","last_active":"2023-09-23T18:21:44.474785+0000","last_peered":"2023-09-23T18:21:44.474785+0000","last_clean":"2023-09-23T18:21:44.474785+0000","last_became_active":"2023-09-23T18:18:33.333320+0000","last_became_peered":"2023-09-23T18:18:33.333320+0000","last_unstale":"2023-09-23T18:21:44.474785+0000","last_undegraded":"2023-09-23T18:21:44.474785+0000","last_fullsized":"2023-09-23T18:21:44.474785+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:32.541824+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1662,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":670,"reported_epoch":626,"state":"active+clean","last_fresh":"2023-09-23T18:21:44.475010+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:21:44.475010+0000","last_peered":"2023-09-23T18:21:44.475010+0000","last_clean":"2023-09-23T18:21:44.475010+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:21:44.475010+0000","last_undegraded":"2023-09-23T18:21:44.475010+0000","last_fullsized":"2023-09-23T18:21:44.475010+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":92,"num_bytes_recovered":7688,"num_keys_recovered":40,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":271633,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":7,"num_read_kb":14,"num_write":4,"num_write_kb":37,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2637896,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":4194323,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":5952,"num_read_kb":12146,"num_write":846,"num_write_kb":12696,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":311,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":16789504,"data_stored":16777273,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":976,"ondisk_log_size":976,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001591,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669527,"num_pgs":68,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":906788,"kb_used_data":279580,"kb_used_omap":7,"kb_used_meta":627192,"kb_avail":92838364,"statfs":{"total":95995035648,"available":95066484736,"internally_reserved":0,"allocated":286289920,"data_stored":284298860,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":642245412},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":20,"apply_latency_ms":20,"commit_latency_ns":20000000,"apply_latency_ns":20000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:16 2023","interfaces":[{"interface":"back","average":{"1min":3.0169999999999999,"5min":2.194,"15min":1.5620000000000001},"min":{"1min":0.255,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":12.207000000000001,"5min":24.472999999999999,"15min":24.472999999999999},"last":0.57099999999999995},{"interface":"front","average":{"1min":2.375,"5min":2.4220000000000002,"15min":1.833},"min":{"1min":0.255,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":12.255000000000001,"5min":22.832999999999998,"15min":22.832999999999998},"last":0.36299999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.3290000000000002,"5min":2.258,"15min":1.5629999999999999},"min":{"1min":0.307,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":6.7309999999999999,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.39700000000000002},{"interface":"front","average":{"1min":2.9580000000000002,"5min":2.2490000000000001,"15min":1.5549999999999999},"min":{"1min":0.27600000000000002,"5min":0.27600000000000002,"15min":0.27600000000000002},"max":{"1min":12.273,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.55500000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.522,"5min":1.579,"15min":1.345},"min":{"1min":0.33500000000000002,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":5.7919999999999998,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.60599999999999998},{"interface":"front","average":{"1min":2.3029999999999999,"5min":2.0600000000000001,"15min":1.5580000000000001},"min":{"1min":0.39700000000000002,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":6.0730000000000004,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.438}]},{"osd":3,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.4990000000000001,"5min":3.0419999999999998,"15min":2.1150000000000002},"min":{"1min":0.34999999999999998,"5min":0.28699999999999998,"15min":0.24099999999999999},"max":{"1min":5.8170000000000002,"5min":103.348,"15min":103.348},"last":0.58799999999999997},{"interface":"front","average":{"1min":2.5600000000000001,"5min":3.6190000000000002,"15min":2.4209999999999998},"min":{"1min":0.33000000000000002,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":7.4530000000000003,"5min":103.40300000000001,"15min":103.40300000000001},"last":0.46600000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:21:16 2023","interfaces":[{"interface":"back","average":{"1min":0.97199999999999998,"5min":1.7390000000000001,"15min":1.341},"min":{"1min":0.28100000000000003,"5min":0.25700000000000001,"15min":0.23999999999999999},"max":{"1min":6.0300000000000002,"5min":8.2400000000000002,"15min":8.6059999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":1.2270000000000001,"5min":1.661,"15min":1.389},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.27000000000000002},"max":{"1min":5.7649999999999997,"5min":7.8319999999999999,"15min":9.2260000000000009},"last":0.51400000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.6219999999999999,"5min":2.1099999999999999,"15min":1.9990000000000001},"min":{"1min":0.32500000000000001,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":18.417000000000002,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.38500000000000001},{"interface":"front","average":{"1min":2.6960000000000002,"5min":2.3399999999999999,"15min":2.3780000000000001},"min":{"1min":0.36299999999999999,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":18.454999999999998,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.48799999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:20:54 2023","interfaces":[{"interface":"back","average":{"1min":1.619,"5min":2.1059999999999999,"15min":1.5580000000000001},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.26800000000000002},"max":{"1min":6.1500000000000004,"5min":10.743,"15min":13.460000000000001},"last":0.52900000000000003},{"interface":"front","average":{"1min":2.0830000000000002,"5min":2.0529999999999999,"15min":1.603},"min":{"1min":0.40300000000000002,"5min":0.38500000000000001,"15min":0.28499999999999998},"max":{"1min":8.6890000000000001,"5min":11.244,"15min":12.404},"last":0.54500000000000004}]}]},{"osd":1,"up_from":545,"seq":2340757176376,"num_pgs":60,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":879224,"kb_used_data":627184,"kb_used_omap":13,"kb_used_meta":252018,"kb_avail":92865928,"statfs":{"total":95995035648,"available":95094710272,"internally_reserved":0,"allocated":642236416,"data_stored":639130146,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":258067085},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":3,"apply_latency_ms":3,"commit_latency_ns":3000000,"apply_latency_ns":3000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":1.74,"5min":1.286,"15min":0.68899999999999995},"min":{"1min":0.23400000000000001,"5min":0.189,"15min":0.189},"max":{"1min":5.0359999999999996,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.41999999999999998},{"interface":"front","average":{"1min":2.1070000000000002,"5min":1.3280000000000001,"15min":0.76900000000000002},"min":{"1min":0.372,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":6.5650000000000004,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.33600000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":1.4450000000000001,"15min":0.78900000000000003},"min":{"1min":0.26000000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":11.02,"5min":11.02,"15min":11.02},"last":0.28899999999999998},{"interface":"front","average":{"1min":1.5249999999999999,"5min":0.97599999999999998,"15min":0.66200000000000003},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":5.1440000000000001,"5min":5.1440000000000001,"15min":5.1440000000000001},"last":0.51000000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":3.9870000000000001,"5min":1.7609999999999999,"15min":0.84499999999999997},"min":{"1min":0.51400000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":14.962,"5min":14.962,"15min":14.962},"last":0.31},{"interface":"front","average":{"1min":2.008,"5min":1.5760000000000001,"15min":0.86599999999999999},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":8.3399999999999999,"5min":19.762,"15min":19.762},"last":0.35699999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":1.782,"5min":1.421,"15min":1.351},"min":{"1min":0.41899999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":5.0759999999999996,"5min":12.663,"15min":12.663},"last":0.38800000000000001},{"interface":"front","average":{"1min":3.952,"5min":2.2669999999999999,"15min":1.7649999999999999},"min":{"1min":0.36799999999999999,"5min":0.317,"15min":0.317},"max":{"1min":11.353999999999999,"5min":13.222,"15min":13.222},"last":0.40500000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.359,"5min":1.732,"15min":1.0589999999999999},"min":{"1min":0.36799999999999999,"5min":0.308,"15min":0.308},"max":{"1min":6.3460000000000001,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.437},{"interface":"front","average":{"1min":2.3450000000000002,"5min":2.0409999999999999,"15min":1.216},"min":{"1min":0.40699999999999997,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":10.926,"5min":23.204000000000001,"15min":23.204000000000001},"last":0.45700000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":2.0489999999999999,"5min":1.7270000000000001,"15min":1.0820000000000001},"min":{"1min":0.45700000000000002,"5min":0.372,"15min":0.372},"max":{"1min":5.1680000000000001,"5min":23.443000000000001,"15min":23.443000000000001},"last":0.47699999999999998},{"interface":"front","average":{"1min":3.5059999999999998,"5min":2.2919999999999998,"15min":1.2809999999999999},"min":{"1min":0.49399999999999999,"5min":0.378,"15min":0.378},"max":{"1min":11.307,"5min":26.177,"15min":26.177},"last":0.502}]},{"osd":7,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":3.649,"5min":1.956,"15min":1.1060000000000001},"min":{"1min":0.42799999999999999,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":15.01,"5min":15.01,"15min":15.01},"last":0.52400000000000002},{"interface":"front","average":{"1min":2.931,"5min":1.899,"15min":1.1699999999999999},"min":{"1min":0.45200000000000001,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":10.988,"5min":10.988,"15min":10.988},"last":0.48799999999999999}]}]},{"osd":0,"up_from":93,"seq":399431958872,"num_pgs":62,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1232488,"kb_used_data":283040,"kb_used_omap":7,"kb_used_meta":949432,"kb_avail":92512664,"statfs":{"total":95995035648,"available":94732967936,"internally_reserved":0,"allocated":289832960,"data_stored":286898920,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":972219108},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":19,"apply_latency_ms":19,"commit_latency_ns":19000000,"apply_latency_ns":19000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:21:29 2023","interfaces":[{"interface":"back","average":{"1min":1.917,"5min":1.363,"15min":0.66700000000000004},"min":{"1min":0.19800000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":5.5090000000000003,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.33200000000000002},{"interface":"front","average":{"1min":3.3130000000000002,"5min":1.8360000000000001,"15min":0.95999999999999996},"min":{"1min":0.33800000000000002,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":22.978000000000002,"5min":22.978000000000002,"15min":22.978000000000002},"last":0.50700000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:21:04 2023","interfaces":[{"interface":"back","average":{"1min":3.008,"5min":2.0880000000000001,"15min":1.4419999999999999},"min":{"1min":0.40200000000000002,"5min":0.185,"15min":0.185},"max":{"1min":10.013,"5min":18.550999999999998,"15min":18.550999999999998},"last":0.42699999999999999},{"interface":"front","average":{"1min":3.2160000000000002,"5min":1.7470000000000001,"15min":1.097},"min":{"1min":0.33600000000000002,"5min":0.22,"15min":0.22},"max":{"1min":23.161999999999999,"5min":23.161999999999999,"15min":23.161999999999999},"last":0.60999999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:21:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.927,"15min":1.6890000000000001},"min":{"1min":0.24199999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.173999999999999,"5min":15.173999999999999,"15min":15.173999999999999},"last":0.67400000000000004},{"interface":"front","average":{"1min":2.23,"5min":2.0350000000000001,"15min":1.718},"min":{"1min":0.32100000000000001,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":15.135,"5min":15.135,"15min":15.135},"last":0.38}]},{"osd":4,"last update":"Sat Sep 23 18:21:14 2023","interfaces":[{"interface":"back","average":{"1min":2.1949999999999998,"5min":2.1019999999999999,"15min":1.982},"min":{"1min":0.39900000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":6.1379999999999999,"5min":17.741,"15min":57.746000000000002},"last":0.45500000000000002},{"interface":"front","average":{"1min":1.966,"5min":1.9099999999999999,"15min":1.792},"min":{"1min":0.30099999999999999,"5min":0.27600000000000002,"15min":0.25600000000000001},"max":{"1min":4.9089999999999998,"5min":15.079000000000001,"15min":57.816000000000003},"last":0.52600000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:21:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.73,"15min":1.466},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.3979999999999997,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.48899999999999999},{"interface":"front","average":{"1min":1.4610000000000001,"5min":2.3399999999999999,"15min":1.6279999999999999},"min":{"1min":0.35499999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.7359999999999998,"5min":15.047000000000001,"15min":15.047000000000001},"last":0.47999999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:21:14 2023","interfaces":[{"interface":"back","average":{"1min":2.2050000000000001,"5min":1.9379999999999999,"15min":2.0299999999999998},"min":{"1min":0.40300000000000002,"5min":0.377,"15min":0.32000000000000001},"max":{"1min":5.4610000000000003,"5min":7.6900000000000004,"15min":58.457999999999998},"last":0.63600000000000001},{"interface":"front","average":{"1min":3.7629999999999999,"5min":2.657,"15min":2.1190000000000002},"min":{"1min":0.54100000000000004,"5min":0.373,"15min":0.315},"max":{"1min":22.893000000000001,"5min":22.893000000000001,"15min":58.354999999999997},"last":0.56200000000000006}]},{"osd":7,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":3.431,"5min":2.2480000000000002,"15min":1.7889999999999999},"min":{"1min":0.49199999999999999,"5min":0.378,"15min":0.34499999999999997},"max":{"1min":22.945,"5min":22.945,"15min":22.945},"last":0.58199999999999996},{"interface":"front","average":{"1min":2.2330000000000001,"5min":1.714,"15min":1.851},"min":{"1min":0.47099999999999997,"5min":0.371,"15min":0.36199999999999999},"max":{"1min":6.9249999999999998,"5min":7.7519999999999998,"15min":10.404},"last":0.59099999999999997}]}]},{"osd":2,"up_from":0,"seq":2070174236758,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":3,"up_from":237,"seq":1017907249387,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":813984,"kb_used_data":12568,"kb_used_omap":3,"kb_used_meta":801404,"kb_avail":92931168,"statfs":{"total":95995035648,"available":95161516032,"internally_reserved":0,"allocated":12869632,"data_stored":9721189,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":820638440},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0329999999999999,"5min":1.5800000000000001,"15min":1.8089999999999999},"min":{"1min":0.20799999999999999,"5min":0.192,"15min":0.183},"max":{"1min":12.472,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.20300000000000001},{"interface":"front","average":{"1min":2.6760000000000002,"5min":1.405,"15min":2.004},"min":{"1min":0.23799999999999999,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":11.468,"5min":11.468,"15min":194.661},"last":0.49099999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:21:27 2023","interfaces":[{"interface":"back","average":{"1min":1.994,"5min":1.754,"15min":0.93100000000000005},"min":{"1min":0.249,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":9.1189999999999998,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.28000000000000003},{"interface":"front","average":{"1min":1.9710000000000001,"5min":2.766,"15min":1.3919999999999999},"min":{"1min":0.23699999999999999,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":6.9050000000000002,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.374}]},{"osd":2,"last update":"Sat Sep 23 18:21:02 2023","interfaces":[{"interface":"back","average":{"1min":2.5569999999999999,"5min":1.526,"15min":1.3740000000000001},"min":{"1min":0.29899999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":9.2430000000000003,"5min":12.151999999999999,"15min":12.151999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":1.78,"5min":1.3839999999999999,"15min":1.361},"min":{"1min":0.32300000000000001,"5min":0.27900000000000003,"15min":0.19600000000000001},"max":{"1min":7.0640000000000001,"5min":11.565,"15min":11.565},"last":0.32800000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.9729999999999999,"5min":1.8839999999999999,"15min":1.706},"min":{"1min":0.34200000000000003,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":11.4,"5min":21.062999999999999,"15min":37.231000000000002},"last":0.44800000000000001},{"interface":"front","average":{"1min":2.532,"5min":2.1240000000000001,"15min":1.885},"min":{"1min":0.34300000000000003,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":9.1630000000000003,"5min":21.128,"15min":37.381},"last":0.52000000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:21:42 2023","interfaces":[{"interface":"back","average":{"1min":2.6579999999999999,"5min":2.2709999999999999,"15min":1.6799999999999999},"min":{"1min":0.47899999999999998,"5min":0.316,"15min":0.316},"max":{"1min":19.434999999999999,"5min":19.434999999999999,"15min":19.434999999999999},"last":0.47899999999999998},{"interface":"front","average":{"1min":1.647,"5min":1.7310000000000001,"15min":1.718},"min":{"1min":0.35999999999999999,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":4.4100000000000001,"5min":17.135999999999999,"15min":17.135999999999999},"last":0.41299999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":2.7349999999999999,"5min":1.9390000000000001,"15min":1.9379999999999999},"min":{"1min":0.39300000000000002,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":10.303000000000001,"5min":17.192,"15min":115.54000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":2.9889999999999999,"5min":1.976,"15min":1.9810000000000001},"min":{"1min":0.35699999999999998,"5min":0.35599999999999998,"15min":0.30499999999999999},"max":{"1min":11.442,"5min":14.321,"15min":115.691},"last":0.58299999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:20:50 2023","interfaces":[{"interface":"back","average":{"1min":3.0939999999999999,"5min":1.744,"15min":1.7609999999999999},"min":{"1min":0.42699999999999999,"5min":0.37,"15min":0.35799999999999998},"max":{"1min":9.1929999999999996,"5min":9.1929999999999996,"15min":15.972},"last":0.55800000000000005},{"interface":"front","average":{"1min":1.974,"5min":1.829,"15min":1.6240000000000001},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":6.9749999999999996,"5min":17.164000000000001,"15min":17.164000000000001},"last":0.50700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019337,"num_pgs":53,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":957924,"kb_used_data":268572,"kb_used_omap":0,"kb_used_meta":689344,"kb_avail":92787228,"statfs":{"total":95995035648,"available":95014121472,"internally_reserved":0,"allocated":275017728,"data_stored":273026460,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":705888256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":17,"apply_latency_ms":17,"commit_latency_ns":17000000,"apply_latency_ns":17000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.5449999999999999,"5min":1.7589999999999999,"15min":1.387},"min":{"1min":0.443,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":6.0960000000000001,"5min":7.6079999999999997,"15min":13.375999999999999},"last":0.57199999999999995},{"interface":"front","average":{"1min":2.5950000000000002,"5min":1.919,"15min":1.6379999999999999},"min":{"1min":0.28100000000000003,"5min":0.23999999999999999,"15min":0.23000000000000001},"max":{"1min":7.6440000000000001,"5min":10.890000000000001,"15min":10.890000000000001},"last":0.33800000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:21:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3490000000000002,"5min":2.048,"15min":1.1459999999999999},"min":{"1min":0.255,"5min":0.24299999999999999,"15min":0.24299999999999999},"max":{"1min":10.619,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.50800000000000001},{"interface":"front","average":{"1min":1.7629999999999999,"5min":1.8899999999999999,"15min":1.0640000000000001},"min":{"1min":0.28399999999999997,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":6.1470000000000002,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.52100000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:21:00 2023","interfaces":[{"interface":"back","average":{"1min":2.3839999999999999,"5min":1.768,"15min":1.8500000000000001},"min":{"1min":0.34300000000000003,"5min":0.255,"15min":0.255},"max":{"1min":6.1749999999999998,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.47499999999999998},{"interface":"front","average":{"1min":2.8079999999999998,"5min":1.8360000000000001,"15min":1.7649999999999999},"min":{"1min":0.36199999999999999,"5min":0.29899999999999999,"15min":0.253},"max":{"1min":17.190999999999999,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.41999999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:21:46 2023","interfaces":[{"interface":"back","average":{"1min":7.149,"5min":2.7650000000000001,"15min":2.2559999999999998},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.23799999999999999},"max":{"1min":74.781999999999996,"5min":74.781999999999996,"15min":74.781999999999996},"last":0.54400000000000004},{"interface":"front","average":{"1min":5.6630000000000003,"5min":2.593,"15min":2.2090000000000001},"min":{"1min":0.307,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":72.152000000000001,"5min":72.152000000000001,"15min":72.152000000000001},"last":0.495}]},{"osd":5,"last update":"Sat Sep 23 18:21:46 2023","interfaces":[{"interface":"back","average":{"1min":5.9109999999999996,"5min":2.9039999999999999,"15min":2.9660000000000002},"min":{"1min":0.28399999999999997,"5min":0.253,"15min":0.253},"max":{"1min":91.432000000000002,"5min":91.432000000000002,"15min":91.432000000000002},"last":0.38600000000000001},{"interface":"front","average":{"1min":6.2249999999999996,"5min":2.5710000000000002,"15min":2.7530000000000001},"min":{"1min":0.312,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":91.626999999999995,"5min":91.626999999999995,"15min":91.626999999999995},"last":0.55500000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.8819999999999999,"15min":1.4099999999999999},"min":{"1min":0.36899999999999999,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":3.9750000000000001,"5min":18.315000000000001,"15min":18.315000000000001},"last":0.45800000000000002},{"interface":"front","average":{"1min":2.1890000000000001,"5min":2.0499999999999998,"15min":1.619},"min":{"1min":0.311,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":7.4969999999999999,"5min":18.431000000000001,"15min":18.431000000000001},"last":0.60299999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:21:08 2023","interfaces":[{"interface":"back","average":{"1min":2.7770000000000001,"5min":2.5430000000000001,"15min":1.8360000000000001},"min":{"1min":0.34699999999999998,"5min":0.32100000000000001,"15min":0.26000000000000001},"max":{"1min":18.283000000000001,"5min":19.298999999999999,"15min":19.298999999999999},"last":0.44600000000000001},{"interface":"front","average":{"1min":1.5760000000000001,"5min":1.7869999999999999,"15min":1.5409999999999999},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.28399999999999997},"max":{"1min":3.786,"5min":12.800000000000001,"15min":12.800000000000001},"last":0.53200000000000003}]}]},{"osd":5,"up_from":420,"seq":1803886264433,"num_pgs":54,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":576280,"kb_used_data":208976,"kb_used_omap":9,"kb_used_meta":367286,"kb_avail":93168872,"statfs":{"total":95995035648,"available":95404924928,"internally_reserved":0,"allocated":213991424,"data_stored":212008428,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":376101309},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":19,"apply_latency_ms":19,"commit_latency_ns":19000000,"apply_latency_ns":19000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.409,"5min":1.2290000000000001,"15min":0.93200000000000005},"min":{"1min":0.21299999999999999,"5min":0.20899999999999999,"15min":0.20899999999999999},"max":{"1min":6.867,"5min":7.2320000000000002,"15min":7.2320000000000002},"last":0.33200000000000002},{"interface":"front","average":{"1min":2.137,"5min":1.8,"15min":1.6459999999999999},"min":{"1min":0.245,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":10.621,"5min":20.937999999999999,"15min":20.937999999999999},"last":0.47699999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:21:24 2023","interfaces":[{"interface":"back","average":{"1min":2.3479999999999999,"5min":1.9570000000000001,"15min":1.383},"min":{"1min":0.28999999999999998,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":12.752000000000001,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.28100000000000003},{"interface":"front","average":{"1min":2.1640000000000001,"5min":1.9930000000000001,"15min":1.2290000000000001},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":7.4969999999999999,"5min":23.268999999999998,"15min":23.268999999999998},"last":0.318}]},{"osd":2,"last update":"Sat Sep 23 18:20:55 2023","interfaces":[{"interface":"back","average":{"1min":1.544,"5min":1.6160000000000001,"15min":1.611},"min":{"1min":0.29699999999999999,"5min":0.21199999999999999,"15min":0.21199999999999999},"max":{"1min":5.1150000000000002,"5min":10.705,"15min":10.705},"last":0.371},{"interface":"front","average":{"1min":1.7949999999999999,"5min":1.8520000000000001,"15min":2.1280000000000001},"min":{"1min":0.39600000000000002,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.7560000000000002,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.34599999999999997}]},{"osd":3,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.1480000000000001,"5min":1.762,"15min":1.359},"min":{"1min":0.35599999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":6.0060000000000002,"5min":8.8780000000000001,"15min":8.8780000000000001},"last":0.46400000000000002},{"interface":"front","average":{"1min":2.2200000000000002,"5min":2.1150000000000002,"15min":1.7130000000000001},"min":{"1min":0.372,"5min":0.33500000000000002,"15min":0.313},"max":{"1min":10.664999999999999,"5min":20.035,"15min":20.035},"last":0.40100000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":1.762,"5min":2.1480000000000001,"15min":2.0499999999999998},"min":{"1min":0.23799999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":8.2270000000000003,"5min":12.747999999999999,"15min":12.747999999999999},"last":0.38900000000000001},{"interface":"front","average":{"1min":2.1120000000000001,"5min":2.1179999999999999,"15min":1.6639999999999999},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":7.7919999999999998,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.42799999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":0.94899999999999995,"5min":1.8340000000000001,"15min":1.2669999999999999},"min":{"1min":0.28199999999999997,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":3.2189999999999999,"5min":9.0350000000000001,"15min":9.0350000000000001},"last":0.44900000000000001},{"interface":"front","average":{"1min":2.2509999999999999,"5min":2.3319999999999999,"15min":1.7370000000000001},"min":{"1min":0.39700000000000002,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":7.6920000000000002,"5min":12.504,"15min":12.504},"last":0.36299999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:20:51 2023","interfaces":[{"interface":"back","average":{"1min":2.052,"5min":2.23,"15min":1.804},"min":{"1min":0.29499999999999998,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":10.195,"5min":12.641,"15min":13.506},"last":0.41799999999999998},{"interface":"front","average":{"1min":1.5640000000000001,"5min":1.8500000000000001,"15min":1.5600000000000001},"min":{"1min":0.45400000000000001,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":5.0839999999999996,"5min":13.138,"15min":13.138},"last":0.50900000000000001}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":5434,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":5392,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:21:56.620 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T18:21:56.620 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:21:56.621 INFO:tasks.thrashosds.thrasher:Adding osd 2 2023-09-23T18:21:56.621 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 2 2023-09-23T18:21:56.697 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:21:56.743 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:56.776 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:21:56.840 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:21:56.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:56.846 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:21:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:56 smithi055 ceph-mon[179668]: pgmap v1527: 103 pgs: 103 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-09-23T18:21:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:56 smithi055 ceph-mon[179668]: from='client.37440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:56.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:56 smithi055 ceph-mon[184174]: pgmap v1527: 103 pgs: 103 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-09-23T18:21:56.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:56 smithi055 ceph-mon[184174]: from='client.37440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:56.947 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:56.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:56 smithi099 ceph-mon[167076]: pgmap v1527: 103 pgs: 103 active+clean; 4.3 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail 2023-09-23T18:21:56.985 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:56 smithi099 ceph-mon[167076]: from='client.37440 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:21:56.985 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:57.020 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:57.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:57.024 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:57.125 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:21:57.162 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:57.293 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:57.489 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:21:57.492 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:57.493 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:21:57.593 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:57.623 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:57.658 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:57.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:57.662 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:57.667 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:21:57.763 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:21:57.785 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.get_all_features 2023-09-23T18:21:57.791 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.get_all_features (6 ms) 2023-09-23T18:21:57.791 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.copyup 2023-09-23T18:21:57.792 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:21:57.800 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:57.845 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:21:57.849 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:57.849 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:21:57.940 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:21:57.940 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:21:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: "complaint_time": 30, 2023-09-23T18:21:57.941 INFO:teuthology.orchestra.run.smithi099.stdout: "num_blocked_ops": 0 2023-09-23T18:21:57.941 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:21:57.950 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:21:57.981 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.copyup (190 ms) 2023-09-23T18:21:57.981 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.sparse_copyup 2023-09-23T18:21:57.983 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:57.997 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.sparse_copyup (16 ms) 2023-09-23T18:21:57.998 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.get_and_set_id 2023-09-23T18:21:58.006 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.get_and_set_id (9 ms) 2023-09-23T18:21:58.006 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.add_remove_child 2023-09-23T18:21:58.029 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.add_remove_child (22 ms) 2023-09-23T18:21:58.029 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.directory_methods 2023-09-23T18:21:58.061 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.directory_methods (32 ms) 2023-09-23T18:21:58.061 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.create 2023-09-23T18:21:58.075 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.create (14 ms) 2023-09-23T18:21:58.075 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.get_features 2023-09-23T18:21:58.076 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.get_features (2 ms) 2023-09-23T18:21:58.076 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.get_object_prefix 2023-09-23T18:21:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:57 smithi099 ceph-mon[167076]: failed to encode map e630 with expected crc 2023-09-23T18:21:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:57 smithi099 ceph-mon[167076]: osdmap e630: 8 total, 8 up, 6 in 2023-09-23T18:21:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:57 smithi099 ceph-mon[167076]: failed to encode map e630 with expected crc 2023-09-23T18:21:58.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/510782728' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi055-461319-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:21:58.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:57 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi055-461319-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:21:58.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:57 smithi099 ceph-mon[167076]: failed to encode map e630 with expected crc 2023-09-23T18:21:58.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:57 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2863191772' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:58.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:57 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:58.078 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.get_object_prefix (2 ms) 2023-09-23T18:21:58.079 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.get_create_timestamp 2023-09-23T18:21:58.080 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.get_create_timestamp (2 ms) 2023-09-23T18:21:58.080 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.get_access_timestamp 2023-09-23T18:21:58.082 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.get_access_timestamp (2 ms) 2023-09-23T18:21:58.082 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.get_modify_timestamp 2023-09-23T18:21:58.083 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.get_modify_timestamp (1 ms) 2023-09-23T18:21:58.083 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.get_data_pool 2023-09-23T18:21:58.086 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.get_data_pool (3 ms) 2023-09-23T18:21:58.087 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.get_size 2023-09-23T18:21:58.090 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.get_size (4 ms) 2023-09-23T18:21:58.091 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.set_size 2023-09-23T18:21:58.096 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.set_size (6 ms) 2023-09-23T18:21:58.096 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.protection_status 2023-09-23T18:21:58.114 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.protection_status (17 ms) 2023-09-23T18:21:58.114 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.snapshot_limits 2023-09-23T18:21:58.121 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.snapshot_limits (8 ms) 2023-09-23T18:21:58.121 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.parents_v1 2023-09-23T18:21:58.164 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.parents_v1 (42 ms) 2023-09-23T18:21:58.164 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.parents_v2 2023-09-23T18:21:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[179668]: failed to encode map e630 with expected crc 2023-09-23T18:21:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[179668]: osdmap e630: 8 total, 8 up, 6 in 2023-09-23T18:21:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[179668]: failed to encode map e630 with expected crc 2023-09-23T18:21:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/510782728' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi055-461319-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:21:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi055-461319-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:21:58.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[179668]: failed to encode map e630 with expected crc 2023-09-23T18:21:58.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2863191772' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:58.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[184174]: failed to encode map e630 with expected crc 2023-09-23T18:21:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[184174]: osdmap e630: 8 total, 8 up, 6 in 2023-09-23T18:21:58.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[184174]: failed to encode map e630 with expected crc 2023-09-23T18:21:58.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/510782728' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi055-461319-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:21:58.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi055-461319-1","app": "rados","yes_i_really_mean_it": true}]: dispatch 2023-09-23T18:21:58.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[184174]: failed to encode map e630 with expected crc 2023-09-23T18:21:58.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2863191772' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:58.183 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:57 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:21:58.194 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.parents_v2 (31 ms) 2023-09-23T18:21:58.195 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.snapshots 2023-09-23T18:21:58.220 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.snapshots (26 ms) 2023-09-23T18:21:58.221 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.snapid_race 2023-09-23T18:21:58.227 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.snapid_race (7 ms) 2023-09-23T18:21:58.228 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.stripingv2 2023-09-23T18:21:58.237 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.stripingv2 (10 ms) 2023-09-23T18:21:58.238 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.object_map_save 2023-09-23T18:21:58.241 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.object_map_save (4 ms) 2023-09-23T18:21:58.241 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.object_map_resize 2023-09-23T18:21:58.249 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.object_map_resize (8 ms) 2023-09-23T18:21:58.250 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.object_map_update 2023-09-23T18:21:58.255 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.object_map_update (6 ms) 2023-09-23T18:21:58.255 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.object_map_load_enoent 2023-09-23T18:21:58.255 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.object_map_load_enoent (0 ms) 2023-09-23T18:21:58.256 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.object_map_snap_add 2023-09-23T18:21:58.264 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.object_map_snap_add (9 ms) 2023-09-23T18:21:58.264 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.object_map_snap_remove 2023-09-23T18:21:58.269 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.object_map_snap_remove (5 ms) 2023-09-23T18:21:58.269 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.flags 2023-09-23T18:21:58.274 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.flags (5 ms) 2023-09-23T18:21:58.275 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.metadata 2023-09-23T18:21:58.305 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.metadata (31 ms) 2023-09-23T18:21:58.305 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.set_features 2023-09-23T18:21:58.313 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.set_features (8 ms) 2023-09-23T18:21:58.313 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.mirror 2023-09-23T18:21:58.353 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.mirror (39 ms) 2023-09-23T18:21:58.353 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.mirror_image 2023-09-23T18:21:58.369 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.mirror_image (17 ms) 2023-09-23T18:21:58.370 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.mirror_image_status 2023-09-23T18:21:58.387 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.7 dump_historic_ops 2023-09-23T18:21:58.999 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:21:59.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:58 smithi099 ceph-mon[167076]: pgmap v1529: 135 pgs: 32 unknown, 103 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T18:21:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:58 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:58 smithi099 ceph-mon[167076]: failed to encode map e631 with expected crc 2023-09-23T18:21:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:58 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi055-461319-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T18:21:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:58 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:21:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:58 smithi099 ceph-mon[167076]: osdmap e631: 8 total, 8 up, 6 in 2023-09-23T18:21:59.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:58 smithi099 ceph-mon[167076]: failed to encode map e631 with expected crc 2023-09-23T18:21:59.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[179668]: pgmap v1529: 135 pgs: 32 unknown, 103 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T18:21:59.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:59.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[179668]: failed to encode map e631 with expected crc 2023-09-23T18:21:59.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi055-461319-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T18:21:59.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:21:59.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[179668]: osdmap e631: 8 total, 8 up, 6 in 2023-09-23T18:21:59.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[179668]: failed to encode map e631 with expected crc 2023-09-23T18:21:59.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[184174]: pgmap v1529: 135 pgs: 32 unknown, 103 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 0 B/s, 0 objects/s recovering 2023-09-23T18:21:59.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:21:59.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[184174]: failed to encode map e631 with expected crc 2023-09-23T18:21:59.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable","pool": "test-rados-api-smithi055-461319-1","app": "rados","yes_i_really_mean_it": true}]': finished 2023-09-23T18:21:59.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:21:59.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[184174]: osdmap e631: 8 total, 8 up, 6 in 2023-09-23T18:21:59.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:58 smithi055 ceph-mon[184174]: failed to encode map e631 with expected crc 2023-09-23T18:21:59.336 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:21:59.340 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:59.341 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:21:59.442 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:21:59.475 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:59.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:21:59.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:59.724 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:21:59.825 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:21:59.860 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:21:59.902 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:21:59.905 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:21:59.905 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:00.006 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:00.011 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:59 smithi099 ceph-mon[167076]: failed to encode map e631 with expected crc 2023-09-23T18:22:00.011 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:59 smithi099 ceph-mon[167076]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.011 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:21:59 smithi099 ceph-mon[167076]: osdmap e632: 8 total, 8 up, 6 in 2023-09-23T18:22:00.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:59 smithi055 ceph-mon[184174]: failed to encode map e631 with expected crc 2023-09-23T18:22:00.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:59 smithi055 ceph-mon[184174]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:21:59 smithi055 ceph-mon[184174]: osdmap e632: 8 total, 8 up, 6 in 2023-09-23T18:22:00.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:59 smithi055 ceph-mon[179668]: failed to encode map e631 with expected crc 2023-09-23T18:22:00.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:59 smithi055 ceph-mon[179668]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:21:59 smithi055 ceph-mon[179668]: osdmap e632: 8 total, 8 up, 6 in 2023-09-23T18:22:00.038 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:00.219 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: "size": 20, 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 600, 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [ 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:177035 8.c 8:3702eed5:::benchmark_data_smithi055_346788_object66443:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e419)", 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:12:31.274617+0000", 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 568.94350017199997, 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.1493567709999999, 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:22:00.220 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 177035 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274617+0000", 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.221 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274617+0000", 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274616+0000", 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274624+0000", 2023-09-23T18:22:00.222 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7940000000000003e-06 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274644+0000", 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.0441999999999998e-05 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.274652+0000", 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.5569999999999998e-06 2023-09-23T18:22:00.223 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.279258+0000", 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046061360000000003 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.279386+0000", 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000127995 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.224 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3", 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.279497+0000", 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00011157100000000001 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.291211+0000", 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.011713889 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.225 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295830+0000", 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0046190240000000002 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:31.295843+0000", 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3239e-05 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.307974+0000", 2023-09-23T18:22:00.226 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0121305490000001 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:32.307989+0000", 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.5268e-05 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.423885+0000", 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1158961300000001 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.227 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:12:33.423974+0000", 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.8610999999999996e-05 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:208693 8.c 8:3f12a3a6:::benchmark_data_smithi055_346788_object98101:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:11.759582+0000", 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 528.45853566899996, 2023-09-23T18:22:00.228 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0444234290000001, 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 208693 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759582+0000", 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.229 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759582+0000", 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759580+0000", 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.230 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759590+0000", 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8640000000000008e-06 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759602+0000", 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.1640999999999999e-05 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759608+0000", 2023-09-23T18:22:00.231 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.64e-06 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759679+0000", 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.1308000000000002e-05 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759809+0000", 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000129696 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.232 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759940+0000", 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013043700000000001 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765188+0000", 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.005248862 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.233 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.768203+0000", 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.003014652 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776965+0000", 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.008761385 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776972+0000", 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0049999999999998e-06 2023-09-23T18:22:00.234 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.803922+0000", 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0269501240000001 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804005+0000", 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3491000000000004e-05 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.235 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:208696 8.c 8:3a4f2983:::benchmark_data_smithi055_346788_object98104:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:11.759625+0000", 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 528.45849249499997, 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0444810689999999, 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:22:00.236 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 208696 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759625+0000", 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759625+0000", 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.237 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759624+0000", 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759628+0000", 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.3349999999999998e-06 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.238 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759635+0000", 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9949999999999999e-06 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.759639+0000", 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5170000000000001e-06 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.762500+0000", 2023-09-23T18:22:00.239 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0028612569999999999 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.762647+0000", 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014658100000000001 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.762784+0000", 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013712300000000001 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.240 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.769536+0000", 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0067522110000000002 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.773643+0000", 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0041063560000000002 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776999+0000", 2023-09-23T18:22:00.241 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033565219999999998 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.777005+0000", 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6810000000000003e-06 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804039+0000", 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0270342690000001 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.242 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804106+0000", 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.6701999999999999e-05 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:208697 8.c 8:3dd33f7b:::benchmark_data_smithi055_346788_object98105:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e434)", 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:11.765262+0000", 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 528.45285512299995, 2023-09-23T18:22:00.243 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.03891529, 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 208697 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765262+0000", 2023-09-23T18:22:00.244 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765262+0000", 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765261+0000", 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.245 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765268+0000", 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2810000000000003e-06 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765281+0000", 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.2964e-05 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.246 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765288+0000", 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.392e-06 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765340+0000", 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2036999999999997e-05 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765485+0000", 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014538800000000001 2023-09-23T18:22:00.247 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.765650+0000", 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00016466500000000001 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.773678+0000", 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0080284060000000001 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.248 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.773706+0000", 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.7759e-05 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.777028+0000", 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033222540000000002 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.777033+0000", 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.074e-06 2023-09-23T18:22:00.249 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804136+0000", 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0271029860000001 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804178+0000", 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.1449999999999998e-05 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.250 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:208709 8.c 8:3b5457b5:::benchmark_data_smithi055_346788_object98117:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e435)", 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:11.776398+0000", 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 528.441719648, 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0613724870000001, 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:22:00.251 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 208709 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776398+0000", 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776398+0000", 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.252 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776396+0000", 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776405+0000", 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2940000000000006e-06 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.253 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776415+0000", 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0898e-05 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.776423+0000", 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.9240000000000007e-06 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.777529+0000", 2023-09-23T18:22:00.254 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001105214 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.777533+0000", 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4610000000000001e-06 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804203+0000", 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0266699610000001 2023-09-23T18:22:00.255 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804332+0000", 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012890199999999999 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.804416+0000", 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3702999999999995e-05 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.256 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.830547+0000", 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.026131879 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837552+0000", 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070046869999999999 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837676+0000", 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012416900000000001 2023-09-23T18:22:00.257 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837730+0000", 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3810999999999997e-05 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837770+0000", 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.0074999999999998e-05 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.258 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.45923.0:208711 8.c 8:3f16d1b8:::benchmark_data_smithi055_346788_object98119:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e435)", 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:13:11.778108+0000", 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 528.44000901300001, 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.059774424, 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.45923", 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:22:00.259 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 208711 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778108+0000", 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778108+0000", 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.260 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778107+0000", 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778113+0000", 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0769999999999998e-06 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.261 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778123+0000", 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0713000000000001e-05 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778129+0000", 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8549999999999998e-06 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778241+0000", 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000112157 2023-09-23T18:22:00.262 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:11.778248+0000", 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.8850000000000001e-06 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.806861+0000", 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0286126259999999 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.263 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.807010+0000", 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014955099999999999 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3,5", 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.807149+0000", 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013890800000000001 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.830553+0000", 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.023403943999999999 2023-09-23T18:22:00.264 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837614+0000", 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0070608600000000004 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837807+0000", 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00019263 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.265 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837851+0000", 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.4180000000000001e-05 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:13:12.837883+0000", 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.1934000000000003e-05 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.266 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121618 9.7 9:e91f6bd2:::benchmark_data_smithi055_386467_object16958:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.648257+0000", 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 383.56986001899998, 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0535740570000001, 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121618 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.267 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648257+0000", 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648257+0000", 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.268 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648257+0000", 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648260+0000", 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.5859999999999999e-06 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648266+0000", 2023-09-23T18:22:00.269 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.2899999999999999e-06 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648270+0000", 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.3370000000000001e-06 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648396+0000", 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012564 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.270 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648520+0000", 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012453099999999999 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648655+0000", 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00013492199999999999 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.653810+0000", 2023-09-23T18:22:00.271 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0051547549999999996 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.658227+0000", 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0044168380000000002 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.669168+0000", 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.010940755999999999 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.272 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.700842+0000", 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.031674536000000003 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.700851+0000", 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0480000000000004e-06 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.273 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701731+0000", 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.000879589 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701831+0000", 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010064 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.274 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121620 9.7 9:e0b3f972:::benchmark_data_smithi055_386467_object16960:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.648304+0000", 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 383.56981306, 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.053641834, 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121620 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.275 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648304+0000", 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648304+0000", 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.276 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648304+0000", 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648314+0000", 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5780000000000005e-06 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648323+0000", 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5899999999999997e-06 2023-09-23T18:22:00.277 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.648328+0000", 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.7620000000000003e-06 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.650968+0000", 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026406110000000002 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.278 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.651091+0000", 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012274000000000001 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,5,6", 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.651209+0000", 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000117797 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.653817+0000", 2023-09-23T18:22:00.279 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0026075460000000001 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.662443+0000", 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.008626791 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.669204+0000", 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.006760856 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.280 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.700923+0000", 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.031719134000000003 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.700931+0000", 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7749999999999993e-06 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701873+0000", 2023-09-23T18:22:00.281 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.000942032 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701946+0000", 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2977000000000003e-05 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121631 9.8 9:16d41ba8:::benchmark_data_smithi055_386467_object16971:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T18:22:00.282 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.670993+0000", 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 383.54712412700002, 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0587921170000001, 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121631 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.283 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.670993+0000", 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.670993+0000", 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.670993+0000", 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:00.284 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.670995+0000", 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 2.3199999999999998e-06 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.671000+0000", 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.3129999999999998e-06 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.285 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.671004+0000", 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.7419999999999999e-06 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.680659+0000", 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0096546359999999994 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.680813+0000", 2023-09-23T18:22:00.286 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00015390400000000001 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 0,1,5", 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.680960+0000", 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00014744199999999999 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.701143+0000", 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.020182319000000001 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.287 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.701148+0000", 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7579999999999998e-06 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701792+0000", 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.000643347 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:22:00.288 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.701893+0000", 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010093400000000001 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.729712+0000", 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.027819884999999999 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.729785+0000", 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.2902000000000001e-05 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.289 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121635 9.3 9:c6494341:::benchmark_data_smithi055_386467_object16975:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e497)", 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.672050+0000", 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 383.54606768100001, 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0872817100000001, 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "reached pg", 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:22:00.290 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121635 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672050+0000", 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672050+0000", 2023-09-23T18:22:00.291 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672049+0000", 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672053+0000", 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5929999999999999e-06 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.292 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672063+0000", 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.0599999999999997e-06 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672067+0000", 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.5820000000000003e-06 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672104+0000", 2023-09-23T18:22:00.293 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.6903000000000002e-05 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672187+0000", 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.2817000000000003e-05 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 1,3", 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.672255+0000", 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.7957e-05 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.294 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.686140+0000", 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.013884728000000001 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.686149+0000", 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.5000000000000005e-06 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746343+0000", 2023-09-23T18:22:00.295 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0601943570000001 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for missing object", 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746452+0000", 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000108996 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.759262+0000", 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012809123 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.296 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.759331+0000", 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.9881000000000004e-05 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121632 9.a 9:5963b4ad:::benchmark_data_smithi055_386467_object16972:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.687284+0000", 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 383.53083345699997, 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0798843389999999, 2023-09-23T18:22:00.297 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121632 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687284+0000", 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.298 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687284+0000", 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687282+0000", 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.299 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687288+0000", 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.9460000000000003e-06 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687299+0000", 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0729e-05 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687305+0000", 2023-09-23T18:22:00.300 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1149999999999999e-06 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.688304+0000", 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00099935900000000001 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.688312+0000", 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.7109999999999995e-06 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.301 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746824+0000", 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0585122039999999 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.748540+0000", 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.001715507 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:22:00.302 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.748766+0000", 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00022613200000000001 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.754863+0000", 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0060968660000000003 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.763930+0000", 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.009067238 2023-09-23T18:22:00.303 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767002+0000", 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0030723460000000001 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767053+0000", 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.0414000000000002e-05 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.304 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767168+0000", 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000115481 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121637 9.a 9:5e2e6388:::benchmark_data_smithi055_386467_object16977:head [delete] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.687315+0000", 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 383.53080226200001, 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0801113259999999, 2023-09-23T18:22:00.305 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121637 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687315+0000", 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.306 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687315+0000", 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687315+0000", 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.307 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687318+0000", 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 3.0319999999999999e-06 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687324+0000", 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 6.1299999999999998e-06 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.687328+0000", 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.2169999999999996e-06 2023-09-23T18:22:00.308 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.688333+0000", 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0010054479999999999 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.688339+0000", 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.6810000000000003e-06 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.309 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.750580+0000", 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0622411700000001 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.750704+0000", 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00012343400000000001 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.750826+0000", 2023-09-23T18:22:00.310 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000122033 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.759396+0000", 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085699620000000004 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.763962+0000", 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045667279999999999 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.311 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767271+0000", 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0033081130000000001 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767328+0000", 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.7719000000000001e-05 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767426+0000", 2023-09-23T18:22:00.312 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.8108000000000003e-05 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121638 9.3 9:c51846a3:::benchmark_data_smithi055_386467_object16978:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.695896+0000", 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 383.52222142800002, 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0712138659999999, 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:00.313 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121638 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695896+0000", 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.314 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695896+0000", 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695894+0000", 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.9999981 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695902+0000", 2023-09-23T18:22:00.315 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.6270000000000006e-06 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695916+0000", 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.3502999999999999e-05 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695923+0000", 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 7.0049999999999998e-06 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.316 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695982+0000", 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.8718999999999998e-05 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.695991+0000", 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.1819999999999995e-06 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.317 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746475+0000", 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0504841819999999 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746584+0000", 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010859 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 3", 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.746783+0000", 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.000199481 2023-09-23T18:22:00.318 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.759330+0000", 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.012547045999999999 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.766963+0000", 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0076329800000000001 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:22:00.319 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767016+0000", 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.2680999999999999e-05 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.767110+0000", 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 9.3968000000000006e-05 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.320 INFO:teuthology.orchestra.run.smithi099.stdout: "description": "osd_op(client.36552.0:121643 9.a 9:511d9150:::benchmark_data_smithi055_386467_object16983:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e498)", 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: "initiated_at": "2023-09-23T18:15:36.723162+0000", 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: "age": 383.494954925, 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0495704269999999, 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: "type_data": { 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: "client_info": { 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: "client": "client.36552", 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: "client_addr": "172.21.15.55:0/341999805", 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: "tid": 121643 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: "events": [ 2023-09-23T18:22:00.321 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "initiated", 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723162+0000", 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "throttled", 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723162+0000", 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "header_read", 2023-09-23T18:22:00.322 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723162+0000", 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4294967295.999999 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "all_read", 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723167+0000", 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.1429999999999997e-06 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "dispatched", 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723177+0000", 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0135e-05 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.323 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "queued_for_pg", 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723182+0000", 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9010000000000002e-06 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723231+0000", 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 4.9101000000000003e-05 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for peered", 2023-09-23T18:22:00.324 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:36.723239+0000", 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 8.3499999999999997e-06 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "reached_pg", 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.752774+0000", 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 1.0295345709999999 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "started", 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.752875+0000", 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010171 2023-09-23T18:22:00.325 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "waiting for subops from 5,6", 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.752981+0000", 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010507599999999999 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "op_commit", 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.759403+0000", 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0064220290000000001 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.326 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.763992+0000", 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0045892930000000004 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.772571+0000", 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.0085792590000000005 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "commit_sent", 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.772627+0000", 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 5.5909999999999998e-05 2023-09-23T18:22:00.327 INFO:teuthology.orchestra.run.smithi099.stdout: }, 2023-09-23T18:22:00.328 INFO:teuthology.orchestra.run.smithi099.stdout: { 2023-09-23T18:22:00.328 INFO:teuthology.orchestra.run.smithi099.stdout: "event": "done", 2023-09-23T18:22:00.328 INFO:teuthology.orchestra.run.smithi099.stdout: "time": "2023-09-23T18:15:37.772733+0000", 2023-09-23T18:22:00.328 INFO:teuthology.orchestra.run.smithi099.stdout: "duration": 0.00010581000000000001 2023-09-23T18:22:00.328 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.328 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.328 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.328 INFO:teuthology.orchestra.run.smithi099.stdout: } 2023-09-23T18:22:00.328 INFO:teuthology.orchestra.run.smithi099.stdout: ] 2023-09-23T18:22:00.328 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:22:00.650 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_ops_in_flight 2023-09-23T18:22:00.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:22:00] "GET /metrics HTTP/1.1" 200 34250 "" "Prometheus/2.43.0" 2023-09-23T18:22:00.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-mon[184174]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-mon[184174]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-mon[184174]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-mon[184174]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-mon[184174]: pgmap v1532: 135 pgs: 1 creating+peering, 30 unknown, 104 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.3 MiB/s rd, 677 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T18:22:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-mon[179668]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-mon[179668]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-mon[179668]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-mon[179668]: failed to encode map e632 with expected crc 2023-09-23T18:22:00.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:00 smithi055 ceph-mon[179668]: pgmap v1532: 135 pgs: 1 creating+peering, 30 unknown, 104 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.3 MiB/s rd, 677 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T18:22:00.964 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:00.968 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:00.969 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:00.995 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.mirror_image_status (2626 ms) 2023-09-23T18:22:00.995 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.mirror_image_map 2023-09-23T18:22:01.070 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:01.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:00 smithi099 ceph-mon[167076]: failed to encode map e632 with expected crc 2023-09-23T18:22:01.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:00 smithi099 ceph-mon[167076]: failed to encode map e632 with expected crc 2023-09-23T18:22:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:00 smithi099 ceph-mon[167076]: failed to encode map e632 with expected crc 2023-09-23T18:22:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:00 smithi099 ceph-mon[167076]: failed to encode map e632 with expected crc 2023-09-23T18:22:01.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:00 smithi099 ceph-mon[167076]: pgmap v1532: 135 pgs: 1 creating+peering, 30 unknown, 104 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.3 MiB/s rd, 677 KiB/s wr, 2 op/s; 0 B/s, 0 objects/s recovering 2023-09-23T18:22:01.085 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.mirror_image_map (90 ms) 2023-09-23T18:22:01.085 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.mirror_instances 2023-09-23T18:22:01.095 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.mirror_instances (10 ms) 2023-09-23T18:22:01.096 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.mirror_snapshot 2023-09-23T18:22:01.101 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:01.108 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.mirror_snapshot (13 ms) 2023-09-23T18:22:01.109 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_dir_list 2023-09-23T18:22:01.111 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_dir_list (3 ms) 2023-09-23T18:22:01.111 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_dir_add 2023-09-23T18:22:01.115 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_dir_add (4 ms) 2023-09-23T18:22:01.115 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.dir_add_already_existing 2023-09-23T18:22:01.118 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.dir_add_already_existing (3 ms) 2023-09-23T18:22:01.118 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_dir_rename 2023-09-23T18:22:01.127 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_dir_rename (9 ms) 2023-09-23T18:22:01.128 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_dir_remove 2023-09-23T18:22:01.131 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_dir_remove (4 ms) 2023-09-23T18:22:01.131 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_dir_remove_missing 2023-09-23T18:22:01.135 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_dir_remove_missing (4 ms) 2023-09-23T18:22:01.135 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_image_add 2023-09-23T18:22:01.139 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_image_add (4 ms) 2023-09-23T18:22:01.139 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_image_remove 2023-09-23T18:22:01.142 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_image_remove (3 ms) 2023-09-23T18:22:01.142 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_image_list 2023-09-23T18:22:01.145 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_image_list (3 ms) 2023-09-23T18:22:01.145 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_image_clean 2023-09-23T18:22:01.149 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_image_clean (4 ms) 2023-09-23T18:22:01.150 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.image_group_add 2023-09-23T18:22:01.152 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.image_group_add (3 ms) 2023-09-23T18:22:01.152 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.image_group_remove 2023-09-23T18:22:01.155 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.image_group_remove (3 ms) 2023-09-23T18:22:01.156 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.image_group_get 2023-09-23T18:22:01.158 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.image_group_get (3 ms) 2023-09-23T18:22:01.158 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_name 2023-09-23T18:22:01.160 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_snap_set_empty_name (2 ms) 2023-09-23T18:22:01.160 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_snap_set_empty_id 2023-09-23T18:22:01.162 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_snap_set_empty_id (2 ms) 2023-09-23T18:22:01.162 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_id 2023-09-23T18:22:01.165 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_id (3 ms) 2023-09-23T18:22:01.165 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_snap_set_duplicate_name 2023-09-23T18:22:01.168 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_snap_set_duplicate_name (3 ms) 2023-09-23T18:22:01.168 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_snap_set 2023-09-23T18:22:01.170 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_snap_set (2 ms) 2023-09-23T18:22:01.170 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_snap_list 2023-09-23T18:22:01.173 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_snap_list (3 ms) 2023-09-23T18:22:01.173 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_snap_list_max_return 2023-09-23T18:22:01.188 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_snap_list_max_return (15 ms) 2023-09-23T18:22:01.188 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_snap_list_max_read 2023-09-23T18:22:01.400 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_snap_list_max_read (212 ms) 2023-09-23T18:22:01.401 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_snap_remove 2023-09-23T18:22:01.404 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_snap_remove (4 ms) 2023-09-23T18:22:01.404 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.group_snap_get_by_id 2023-09-23T18:22:01.406 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.group_snap_get_by_id (2 ms) 2023-09-23T18:22:01.406 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.trash_methods 2023-09-23T18:22:01.413 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.trash_methods (7 ms) 2023-09-23T18:22:01.414 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.op_features 2023-09-23T18:22:01.421 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.op_features (8 ms) 2023-09-23T18:22:01.421 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.clone_parent 2023-09-23T18:22:01.444 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.clone_parent (22 ms) 2023-09-23T18:22:01.444 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.clone_parent_ns 2023-09-23T18:22:01.454 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.clone_parent_ns (10 ms) 2023-09-23T18:22:01.454 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.clone_child 2023-09-23T18:22:01.465 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.clone_child (11 ms) 2023-09-23T18:22:01.465 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.namespace_methods 2023-09-23T18:22:01.472 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.namespace_methods (6 ms) 2023-09-23T18:22:01.472 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.migration 2023-09-23T18:22:01.482 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.migration (10 ms) 2023-09-23T18:22:01.482 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.migration_v1 2023-09-23T18:22:01.496 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:01.497 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.migration_v1 (15 ms) 2023-09-23T18:22:01.497 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.assert_snapc_seq 2023-09-23T18:22:01.499 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:01.500 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:01.601 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:01.629 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:01.665 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:01.667 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:01.668 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:01.769 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:01.796 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:01.822 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:22:01.831 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:01.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:01.835 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:01.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:01.962 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:01.995 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:01.998 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:01.998 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:02.056 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:02.099 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:02.125 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:01 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:22:01.822+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e633 definitely_dead 0 2023-09-23T18:22:02.129 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:02.372 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:02.376 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:02.376 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:02.382 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:22:02.402 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:22:02.422 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:22:02.441 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:22:02.460 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:22:02.477 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:02.478 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:22:02.501 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:22:02.509 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:02.524 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:22:02.525 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:22:02.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:22:02.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:22:02.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:22:02.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:22:02.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:22:02.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:22:02.526 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:22:02.647 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:02.649 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:02.650 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:02.750 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:02.780 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:02.821 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.assert_snapc_seq (1323 ms) 2023-09-23T18:22:02.821 INFO:tasks.workunit.client.0.smithi055.stdout:[ RUN ] TestClsRbd.sparsify 2023-09-23T18:22:02.831 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.2. 2023-09-23T18:22:02.868 INFO:tasks.workunit.client.0.smithi055.stdout:[ OK ] TestClsRbd.sparsify (48 ms) 2023-09-23T18:22:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[184174]: pgmap v1533: 135 pgs: 1 creating+peering, 28 unknown, 106 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.1 MiB/s wr, 1.03k op/s 2023-09-23T18:22:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[184174]: failed to encode map e633 with expected crc 2023-09-23T18:22:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[184174]: osdmap e633: 8 total, 8 up, 6 in 2023-09-23T18:22:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[184174]: failed to encode map e633 with expected crc 2023-09-23T18:22:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1347755691' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T18:22:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T18:22:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[184174]: failed to encode map e633 with expected crc 2023-09-23T18:22:02.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1950620218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[179668]: pgmap v1533: 135 pgs: 1 creating+peering, 28 unknown, 106 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.1 MiB/s wr, 1.03k op/s 2023-09-23T18:22:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[179668]: failed to encode map e633 with expected crc 2023-09-23T18:22:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[179668]: osdmap e633: 8 total, 8 up, 6 in 2023-09-23T18:22:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[179668]: failed to encode map e633 with expected crc 2023-09-23T18:22:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1347755691' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T18:22:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T18:22:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[179668]: failed to encode map e633 with expected crc 2023-09-23T18:22:02.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1950620218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:03.054 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:03.056 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:03.057 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:03.156 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:03.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:03.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:02 smithi099 ceph-mon[167076]: pgmap v1533: 135 pgs: 1 creating+peering, 28 unknown, 106 active+clean; 4.6 MiB data, 5.1 GiB used, 531 GiB / 536 GiB avail; 1.9 MiB/s rd, 1.1 MiB/s wr, 1.03k op/s 2023-09-23T18:22:03.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:02 smithi099 ceph-mon[167076]: failed to encode map e633 with expected crc 2023-09-23T18:22:03.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:02 smithi099 ceph-mon[167076]: osdmap e633: 8 total, 8 up, 6 in 2023-09-23T18:22:03.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:02 smithi099 ceph-mon[167076]: failed to encode map e633 with expected crc 2023-09-23T18:22:03.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1347755691' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T18:22:03.187 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:02 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["2"]}]: dispatch 2023-09-23T18:22:03.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:02 smithi099 ceph-mon[167076]: failed to encode map e633 with expected crc 2023-09-23T18:22:03.188 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1950620218' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:03.191 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:03.227 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:03.230 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:03.231 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:03.331 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:03.361 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:03.452 INFO:tasks.workunit.client.0.smithi055.stdout:[----------] 70 tests from TestClsRbd (5083 ms total) 2023-09-23T18:22:03.452 INFO:tasks.workunit.client.0.smithi055.stdout: 2023-09-23T18:22:03.452 INFO:tasks.workunit.client.0.smithi055.stdout:[----------] Global test environment tear-down 2023-09-23T18:22:03.453 INFO:tasks.workunit.client.0.smithi055.stdout:[==========] 70 tests from 1 test suite ran. (7521 ms total) 2023-09-23T18:22:03.453 INFO:tasks.workunit.client.0.smithi055.stdout:[ PASSED ] 70 tests. 2023-09-23T18:22:03.461 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T18:22:03.461 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T18:22:03.538 INFO:tasks.workunit:Stopping ['cls/test_cls_rbd.sh'] on client.0... 2023-09-23T18:22:03.538 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T18:22:03.597 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:03.603 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:03.603 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:03.704 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:22:03.733 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:03.754 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:22:03.754 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:22:03.886 DEBUG:teuthology.parallel:result is None 2023-09-23T18:22:03.887 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T18:22:03.955 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T18:22:03.955 DEBUG:teuthology.orchestra.run.smithi055:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:22:04.013 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T18:22:04.013 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T18:22:04.013 INFO:teuthology.task.print:**** done cls/test_cls_rbd.sh 5-workload 2023-09-23T18:22:04.013 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T18:22:04.014 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T18:22:04.014 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T18:22:04.014 DEBUG:teuthology.orchestra.run.smithi055:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:22:04.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[179668]: failed to encode map e634 with expected crc 2023-09-23T18:22:04.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T18:22:04.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[179668]: osdmap e634: 8 total, 8 up, 7 in 2023-09-23T18:22:04.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[179668]: failed to encode map e634 with expected crc 2023-09-23T18:22:04.016 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[179668]: pgmap v1536: 135 pgs: 1 peering, 1 unknown, 133 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 1.40k op/s 2023-09-23T18:22:04.017 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[179668]: failed to encode map e634 with expected crc 2023-09-23T18:22:04.017 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[179668]: failed to encode map e635 with expected crc 2023-09-23T18:22:04.017 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[179668]: osdmap e635: 8 total, 8 up, 7 in 2023-09-23T18:22:04.017 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[179668]: failed to encode map e635 with expected crc 2023-09-23T18:22:04.017 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[179668]: failed to encode map e635 with expected crc 2023-09-23T18:22:04.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[184174]: failed to encode map e634 with expected crc 2023-09-23T18:22:04.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T18:22:04.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[184174]: osdmap e634: 8 total, 8 up, 7 in 2023-09-23T18:22:04.017 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[184174]: failed to encode map e634 with expected crc 2023-09-23T18:22:04.018 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[184174]: pgmap v1536: 135 pgs: 1 peering, 1 unknown, 133 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 1.40k op/s 2023-09-23T18:22:04.018 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[184174]: failed to encode map e634 with expected crc 2023-09-23T18:22:04.018 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[184174]: failed to encode map e635 with expected crc 2023-09-23T18:22:04.018 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[184174]: osdmap e635: 8 total, 8 up, 7 in 2023-09-23T18:22:04.018 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[184174]: failed to encode map e635 with expected crc 2023-09-23T18:22:04.018 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:03 smithi055 ceph-mon[184174]: failed to encode map e635 with expected crc 2023-09-23T18:22:04.021 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:22:04.024 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:04.025 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:22:04.035 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:22:04.035 INFO:teuthology.orchestra.run.smithi055.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T18:22:04.035 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:22:04.093 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T18:22:04.094 DEBUG:teuthology.orchestra.run.smithi055:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T18:22:04.126 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:22:04.151 INFO:tasks.workunit:timeout=3h 2023-09-23T18:22:04.151 INFO:tasks.workunit:cleanup=True 2023-09-23T18:22:04.152 DEBUG:teuthology.orchestra.run.smithi055:> rm -rf /home/ubuntu/cephtest/clone.client.0 && git clone --depth 1 --branch quincy https://github.com/ceph/ceph.git /home/ubuntu/cephtest/clone.client.0 && cd /home/ubuntu/cephtest/clone.client.0 2023-09-23T18:22:04.161 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:04.180 INFO:tasks.workunit.client.0.smithi055.stderr:Cloning into '/home/ubuntu/cephtest/clone.client.0'... 2023-09-23T18:22:04.196 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:22:04.199 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:04.199 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:22:04.300 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:04.315 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_blocked_ops 2023-09-23T18:22:04.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:03 smithi099 ceph-mon[167076]: failed to encode map e634 with expected crc 2023-09-23T18:22:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:03 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["2"]}]': finished 2023-09-23T18:22:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:03 smithi099 ceph-mon[167076]: osdmap e634: 8 total, 8 up, 7 in 2023-09-23T18:22:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:03 smithi099 ceph-mon[167076]: failed to encode map e634 with expected crc 2023-09-23T18:22:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:03 smithi099 ceph-mon[167076]: pgmap v1536: 135 pgs: 1 peering, 1 unknown, 133 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 2.4 MiB/s rd, 1.3 MiB/s wr, 1.40k op/s 2023-09-23T18:22:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:03 smithi099 ceph-mon[167076]: failed to encode map e634 with expected crc 2023-09-23T18:22:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:03 smithi099 ceph-mon[167076]: failed to encode map e635 with expected crc 2023-09-23T18:22:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:03 smithi099 ceph-mon[167076]: osdmap e635: 8 total, 8 up, 7 in 2023-09-23T18:22:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:03 smithi099 ceph-mon[167076]: failed to encode map e635 with expected crc 2023-09-23T18:22:04.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:03 smithi099 ceph-mon[167076]: failed to encode map e635 with expected crc 2023-09-23T18:22:04.332 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:04.367 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:04.370 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:04.371 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:04.471 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:04.494 INFO:tasks.thrashosds.thrasher:Added osd 2 2023-09-23T18:22:04.501 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:04.545 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:04.549 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:04.549 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:04.650 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:04.677 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:04.711 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:04.713 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:04.714 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:04.814 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:04.840 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:04.873 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:04.876 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:04.876 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:04.977 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:05.000 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:04 smithi099 ceph-mon[167076]: failed to encode map e635 with expected crc 2023-09-23T18:22:05.000 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:04 smithi099 ceph-mon[167076]: failed to encode map e635 with expected crc 2023-09-23T18:22:05.000 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:04 smithi099 ceph-mon[167076]: failed to encode map e636 with expected crc 2023-09-23T18:22:05.000 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:04 smithi099 ceph-mon[167076]: osdmap e636: 8 total, 8 up, 7 in 2023-09-23T18:22:05.002 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:05.034 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:05.037 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:05.037 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:05.138 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:05.165 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:04 smithi055 ceph-mon[179668]: failed to encode map e635 with expected crc 2023-09-23T18:22:05.165 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:04 smithi055 ceph-mon[179668]: failed to encode map e635 with expected crc 2023-09-23T18:22:05.166 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:04 smithi055 ceph-mon[179668]: failed to encode map e636 with expected crc 2023-09-23T18:22:05.166 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:04 smithi055 ceph-mon[179668]: osdmap e636: 8 total, 8 up, 7 in 2023-09-23T18:22:05.166 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:04 smithi055 ceph-mon[184174]: failed to encode map e635 with expected crc 2023-09-23T18:22:05.166 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:04 smithi055 ceph-mon[184174]: failed to encode map e635 with expected crc 2023-09-23T18:22:05.166 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:04 smithi055 ceph-mon[184174]: failed to encode map e636 with expected crc 2023-09-23T18:22:05.166 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:04 smithi055 ceph-mon[184174]: osdmap e636: 8 total, 8 up, 7 in 2023-09-23T18:22:05.169 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:05.170 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:05.191 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:22:05.207 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:05.211 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:05.211 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:05.312 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:05.338 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:05.371 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:05.374 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:05.374 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:05.468 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:22:05.475 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:05.500 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:05.533 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:05.536 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:05.536 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:05.637 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:05.666 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:05.699 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:05.702 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:05.702 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:05.803 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:05.831 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:05.922 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:05.924 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:05.925 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:06.026 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:06.054 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:06.087 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:06.089 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:06.090 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:06.119 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[184174]: failed to encode map e636 with expected crc 2023-09-23T18:22:06.119 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[184174]: failed to encode map e636 with expected crc 2023-09-23T18:22:06.119 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[184174]: pgmap v1539: 103 pgs: 2 remapped+peering, 101 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1160149309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[184174]: failed to encode map e637 with expected crc 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1160149309' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[184174]: osdmap e637: 8 total, 8 up, 7 in 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[179668]: failed to encode map e636 with expected crc 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[179668]: failed to encode map e636 with expected crc 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[179668]: pgmap v1539: 103 pgs: 2 remapped+peering, 101 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1160149309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:06.120 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[179668]: failed to encode map e637 with expected crc 2023-09-23T18:22:06.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1160149309' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:06.121 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:05 smithi055 ceph-mon[179668]: osdmap e637: 8 total, 8 up, 7 in 2023-09-23T18:22:06.190 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:06.207 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:06.225 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:06.329 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:05 smithi099 ceph-mon[167076]: failed to encode map e636 with expected crc 2023-09-23T18:22:06.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:05 smithi099 ceph-mon[167076]: failed to encode map e636 with expected crc 2023-09-23T18:22:06.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:05 smithi099 ceph-mon[167076]: pgmap v1539: 103 pgs: 2 remapped+peering, 101 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:22:06.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1160149309' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:06.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:05 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:06.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:05 smithi099 ceph-mon[167076]: failed to encode map e637 with expected crc 2023-09-23T18:22:06.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1160149309' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:06.330 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:05 smithi099 ceph-mon[167076]: osdmap e637: 8 total, 8 up, 7 in 2023-09-23T18:22:06.335 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:06.339 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:06.339 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:06.440 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:06.471 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:06.503 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:06.506 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:06.506 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:06.607 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:06.633 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:06.666 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:06.668 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:06.669 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:06.757 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:22:06.757 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:22:06.770 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:06.795 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:06.828 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:06.830 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:06.831 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:06.931 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:06.964 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:06 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:22:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:06 smithi099 ceph-mon[167076]: failed to encode map e638 with expected crc 2023-09-23T18:22:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:06 smithi099 ceph-mon[167076]: osdmap e638: 8 total, 8 up, 7 in 2023-09-23T18:22:07.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:06 smithi099 ceph-mon[167076]: failed to encode map e638 with expected crc 2023-09-23T18:22:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:22:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[179668]: failed to encode map e638 with expected crc 2023-09-23T18:22:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[179668]: osdmap e638: 8 total, 8 up, 7 in 2023-09-23T18:22:07.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[179668]: failed to encode map e638 with expected crc 2023-09-23T18:22:07.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:22:07.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:07.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:07.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[184174]: failed to encode map e638 with expected crc 2023-09-23T18:22:07.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[184174]: osdmap e638: 8 total, 8 up, 7 in 2023-09-23T18:22:07.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:06 smithi055 ceph-mon[184174]: failed to encode map e638 with expected crc 2023-09-23T18:22:07.378 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:07.381 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:07.382 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:07.441 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.3 dump_historic_ops 2023-09-23T18:22:07.483 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:07.513 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:07.616 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:07.619 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:07.619 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:07.720 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:07.752 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:07.788 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:07.791 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:07.791 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:07.892 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:22:07.920 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:07.929 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:22:08.057 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:22:08.061 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:08.061 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:22:08.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:07 smithi099 ceph-mon[167076]: failed to encode map e638 with expected crc 2023-09-23T18:22:08.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:07 smithi099 ceph-mon[167076]: pgmap v1542: 103 pgs: 3 peering, 2 remapped+peering, 98 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 1.2 KiB/s, 2 keys/s, 3 objects/s recovering 2023-09-23T18:22:08.117 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:22:08.144 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:22:08.162 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:08.164 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:22:08.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:07 smithi055 ceph-mon[184174]: failed to encode map e638 with expected crc 2023-09-23T18:22:08.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:07 smithi055 ceph-mon[184174]: pgmap v1542: 103 pgs: 3 peering, 2 remapped+peering, 98 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 1.2 KiB/s, 2 keys/s, 3 objects/s recovering 2023-09-23T18:22:08.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:07 smithi055 ceph-mon[179668]: failed to encode map e638 with expected crc 2023-09-23T18:22:08.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:07 smithi055 ceph-mon[179668]: pgmap v1542: 103 pgs: 3 peering, 2 remapped+peering, 98 active+clean; 4.6 MiB data, 5.2 GiB used, 621 GiB / 626 GiB avail; 1.2 KiB/s, 2 keys/s, 3 objects/s recovering 2023-09-23T18:22:08.187 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:22:08.194 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:08.207 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:22:08.226 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:22:08.231 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:08.234 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:08.234 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:08.264 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:22:08.285 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:22:08.286 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:22:08.286 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:22:08.286 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:22:08.287 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:22:08.287 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:22:08.287 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:22:08.287 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:22:08.287 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:22:08.335 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:08.363 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:08.378 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:08.397 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:08.400 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:08.401 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:08.501 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:08.528 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:08.560 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:08.563 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:08.563 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:08.664 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:22:08.695 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:08.717 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:08.831 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:22:08.834 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:08.834 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:22:08.906 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:08 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3422742258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:08.906 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:08 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3422742258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:08.935 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:08.959 DEBUG:teuthology.orchestra.run.smithi055:> cd -- /home/ubuntu/cephtest/clone.client.0/qa/workunits && if test -e Makefile ; then make ; fi && find -executable -type f -printf '%P\0' >/home/ubuntu/cephtest/workunits.list.client.0 2023-09-23T18:22:08.962 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:08.977 INFO:tasks.workunit.client.0.smithi055.stdout:for d in direct_io fs ; do ( cd $d ; make all ) ; done 2023-09-23T18:22:08.979 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T18:22:08.979 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE direct_io_test.c -o direct_io_test 2023-09-23T18:22:08.994 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:08.996 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:08.997 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:09.027 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_sync_io.c -o test_sync_io 2023-09-23T18:22:09.077 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_short_dio_read.c -o test_short_dio_read 2023-09-23T18:22:09.098 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:09.128 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:09.137 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/direct_io' 2023-09-23T18:22:09.139 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Entering directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T18:22:09.139 INFO:tasks.workunit.client.0.smithi055.stdout:cc -Wall -Wextra -D_GNU_SOURCE test_o_trunc.c -o test_o_trunc 2023-09-23T18:22:09.203 INFO:tasks.workunit.client.0.smithi055.stdout:make[1]: Leaving directory '/home/ubuntu/cephtest/clone.client.0/qa/workunits/fs' 2023-09-23T18:22:09.206 DEBUG:teuthology.orchestra.run.smithi055:> set -ex 2023-09-23T18:22:09.206 DEBUG:teuthology.orchestra.run.smithi055:> dd if=/home/ubuntu/cephtest/workunits.list.client.0 of=/dev/stdout 2023-09-23T18:22:09.266 INFO:tasks.workunit:Running workunits matching rbd/import_export.sh on client.0... 2023-09-23T18:22:09.267 INFO:tasks.workunit:Running workunit rbd/import_export.sh... 2023-09-23T18:22:09.268 DEBUG:teuthology.orchestra.run.smithi055:workunit test rbd/import_export.sh> mkdir -p -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && cd -- /home/ubuntu/cephtest/mnt.0/client.0/tmp && CEPH_CLI_TEST_DUP_COMMAND=1 CEPH_REF=quincy TESTDIR="/home/ubuntu/cephtest" CEPH_ARGS="--cluster ceph" CEPH_ID="0" PATH=$PATH:/usr/sbin CEPH_BASE=/home/ubuntu/cephtest/clone.client.0 CEPH_ROOT=/home/ubuntu/cephtest/clone.client.0 CEPH_MNT=/home/ubuntu/cephtest/mnt.0 RBD_CREATE_ARGS=--new-format adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage timeout 3h /home/ubuntu/cephtest/clone.client.0/qa/workunits/rbd/import_export.sh 2023-09-23T18:22:09.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:08 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3422742258' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:09.327 INFO:tasks.workunit.client.0.smithi055.stderr:+ export RBD_FORCE_ALLOW_V1=1 2023-09-23T18:22:09.327 INFO:tasks.workunit.client.0.smithi055.stderr:+ RBD_FORCE_ALLOW_V1=1 2023-09-23T18:22:09.327 INFO:tasks.workunit.client.0.smithi055.stderr:+ TMPDIR=/tmp/rbd_import_export_463999 2023-09-23T18:22:09.327 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm -rf /tmp/rbd_import_export_463999 2023-09-23T18:22:09.328 INFO:tasks.workunit.client.0.smithi055.stderr:+ mkdir /tmp/rbd_import_export_463999 2023-09-23T18:22:09.329 INFO:tasks.workunit.client.0.smithi055.stderr:+ trap 'rm -rf /tmp/rbd_import_export_463999' INT TERM EXIT 2023-09-23T18:22:09.329 INFO:tasks.workunit.client.0.smithi055.stderr:+ mkdir foo.463999 2023-09-23T18:22:09.330 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import foo.463999 foo.dir 2023-09-23T18:22:09.407 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: cannot import a directory 2023-09-23T18:22:09.408 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 0% complete...failed. 2023-09-23T18:22:09.408 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: import failed: (21) Is a directory 2023-09-23T18:22:09.410 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T18:22:09.410 INFO:tasks.workunit.client.0.smithi055.stderr:+ rmdir foo.463999 2023-09-23T18:22:09.411 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/sh of=/tmp/rbd_import_export_463999/img bs=1k count=1 seek=10 2023-09-23T18:22:09.411 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records in 2023-09-23T18:22:09.412 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records out 2023-09-23T18:22:09.412 INFO:tasks.workunit.client.0.smithi055.stderr:1024 bytes (1.0 kB, 1.0 KiB) copied, 4.5678e-05 s, 22.4 MB/s 2023-09-23T18:22:09.412 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_463999/img bs=1k count=10 seek=100 2023-09-23T18:22:09.412 INFO:tasks.workunit.client.0.smithi055.stderr:10+0 records in 2023-09-23T18:22:09.413 INFO:tasks.workunit.client.0.smithi055.stderr:10+0 records out 2023-09-23T18:22:09.413 INFO:tasks.workunit.client.0.smithi055.stderr:10240 bytes (10 kB, 10 KiB) copied, 5.7928e-05 s, 177 MB/s 2023-09-23T18:22:09.413 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/rm of=/tmp/rbd_import_export_463999/img bs=1k count=100 seek=1000 2023-09-23T18:22:09.413 INFO:tasks.workunit.client.0.smithi055.stderr:70+1 records in 2023-09-23T18:22:09.413 INFO:tasks.workunit.client.0.smithi055.stderr:70+1 records out 2023-09-23T18:22:09.414 INFO:tasks.workunit.client.0.smithi055.stderr:72048 bytes (72 kB, 70 KiB) copied, 0.000277687 s, 259 MB/s 2023-09-23T18:22:09.414 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/ls of=/tmp/rbd_import_export_463999/img bs=1k seek=10000 2023-09-23T18:22:09.415 INFO:tasks.workunit.client.0.smithi055.stderr:139+1 records in 2023-09-23T18:22:09.415 INFO:tasks.workunit.client.0.smithi055.stderr:139+1 records out 2023-09-23T18:22:09.415 INFO:tasks.workunit.client.0.smithi055.stderr:143224 bytes (143 kB, 140 KiB) copied, 0.00059353 s, 241 MB/s 2023-09-23T18:22:09.415 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/ln of=/tmp/rbd_import_export_463999/img bs=1k seek=100000 2023-09-23T18:22:09.416 INFO:tasks.workunit.client.0.smithi055.stderr:70+1 records in 2023-09-23T18:22:09.416 INFO:tasks.workunit.client.0.smithi055.stderr:70+1 records out 2023-09-23T18:22:09.416 INFO:tasks.workunit.client.0.smithi055.stderr:72168 bytes (72 kB, 70 KiB) copied, 0.000207111 s, 348 MB/s 2023-09-23T18:22:09.416 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/grep of=/tmp/rbd_import_export_463999/img bs=1k seek=1000000 2023-09-23T18:22:09.417 INFO:tasks.workunit.client.0.smithi055.stderr:189+1 records in 2023-09-23T18:22:09.417 INFO:tasks.workunit.client.0.smithi055.stderr:189+1 records out 2023-09-23T18:22:09.417 INFO:tasks.workunit.client.0.smithi055.stderr:193768 bytes (194 kB, 189 KiB) copied, 0.000476046 s, 407 MB/s 2023-09-23T18:22:09.418 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm testimg 2023-09-23T18:22:09.455 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 0% complete...failed. 2023-09-23T18:22:09.455 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T18:22:09.458 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T18:22:09.459 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format /tmp/rbd_import_export_463999/img testimg 2023-09-23T18:22:09.495 INFO:tasks.thrashosds.thrasher:in_osds: [1, 0, 4, 3, 6, 5, 2] out_osds: [7] dead_osds: [] live_osds: [0, 4, 6, 7, 3, 5, 2, 1] 2023-09-23T18:22:09.495 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T18:22:09.565 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:09.569 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:09.569 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:09.670 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:09.708 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:09.894 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:09 smithi055 ceph-mon[184174]: pgmap v1543: 103 pgs: 3 peering, 100 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 824 B/s, 1 keys/s, 5 objects/s recovering 2023-09-23T18:22:09.895 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:09 smithi055 ceph-mon[179668]: pgmap v1543: 103 pgs: 3 peering, 100 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 824 B/s, 1 keys/s, 5 objects/s recovering 2023-09-23T18:22:10.074 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 1% complete... Importing image: 2% complete... Importing image: 3% complete... Importing image: 4% complete... Importing image: 5% complete... Importing image: 6% complete... Importing image: 7% complete... Importing image: 8% complete... Importing image: 9% complete... Importing image: 10% complete... Importing image: 11% complete... Importing image: 12% complete... Importing image: 13% complete... Importing image: 14% complete... Importing image: 15% complete... Importing image: 16% complete... Importing image: 17% complete... Importing image: 18% complete... Importing image: 19% complete... Importing image: 20% complete... Importing image: 21% complete... Importing image: 22% complete... Importing image: 23% complete... Importing image: 24% complete... Importing image: 25% complete... Importing image: 26% complete... Importing image: 27% complete... Importing image: 28% complete... Importing image: 29% complete... Importing image: 30% complete... Importing image: 31% complete... Importing image: 32% complete... Importing image: 33% complete... Importing image: 34% complete... Importing image: 35% complete... Importing image: 36% complete... Importing image: 37% complete... Importing image: 38% complete... Importing image: 39% complete... Importing image: 40% complete... Importing image: 41% complete... Importing image: 42% complete... Importing image: 43% complete... Importing image: 44% complete... Importing image: 45% complete... Importing image: 46% complete... Importing image: 47% complete... Importing image: 48% complete... Importing image: 49% complete... Importing image: 50% complete... Importing image: 51% complete... Importing image: 52% complete... Importing image: 53% complete... Importing image: 54% complete... Importing image: 55% complete... Importing image: 56% complete... Importing image: 57% complete... Importing image: 58% complete... Importing image: 59% complete... Importing image: 60% complete... Importing image: 61% complete... Importing image: 62% complete... Importing image: 63% complete... Importing image: 64% complete... Importing image: 65% complete... Importing image: 66% complete... Importing image: 67% complete... Importing image: 68% complete... Importing image: 69% complete... Importing image: 70% complete... Importing image: 71% complete... Importing image: 72% complete... Importing image: 73% complete... Importing image: 74% complete... Importing image: 75% complete... Importing image: 76% complete... Importing image: 77% complete... Importing image: 78% complete... Importing image: 79% complete... Importing image: 80% complete... Importing image: 81% complete... Importing image: 82% complete... Importing image: 83% complete... Importing image: 84% complete... Importing image: 85% complete... Importing image: 86% complete... Importing image: 87% complete... Importing image: 88% complete... Importing image: 89% complete... Importing image: 90% complete... Importing image: 91% complete... Importing image: 92% complete... Importing image: 93% complete... Importing image: 94% complete... Importing image: 95% complete... Importing image: 96% complete... Importing image: 97% complete... Importing image: 98% complete... Importing image: 99% complete... Importing image: 100% complete...done. 2023-09-23T18:22:10.075 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:22:10.078 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg /tmp/rbd_import_export_463999/img2 2023-09-23T18:22:10.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:09 smithi099 ceph-mon[167076]: pgmap v1543: 103 pgs: 3 peering, 100 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 824 B/s, 1 keys/s, 5 objects/s recovering 2023-09-23T18:22:10.406 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:10.410 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:10.411 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:10.452 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:10.455 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg - 2023-09-23T18:22:10.512 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:10.544 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:10.580 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:10.583 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:10.583 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:10.658 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:22:10.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:22:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:22:10] "GET /metrics HTTP/1.1" 200 34230 "" "Prometheus/2.43.0" 2023-09-23T18:22:10.684 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:10.717 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:10.947 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:10 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:22:10.948 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:10 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:22:11.219 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:11.223 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm testimg 2023-09-23T18:22:11.319 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T18:22:11.322 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp /tmp/rbd_import_export_463999/img /tmp/rbd_import_export_463999/img2 2023-09-23T18:22:11.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:10 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:22:11.605 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:11.610 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:11.610 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:11.711 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:11.749 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:12.053 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:22:12.053 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:22:12.053 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:22:12.053 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:22:12.053 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.45923.0:173525 8.8 8:11c377cb:::benchmark_data_smithi055_346788_object62933:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e418)", 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:12:27.592880+0000", 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 584.458920745, 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.27587297500000002, 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.45923", 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/1491551194", 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 173525 2023-09-23T18:22:12.054 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592880+0000", 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592880+0000", 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.055 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592879+0000", 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592884+0000", 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4500000000000003e-06 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592895+0000", 2023-09-23T18:22:12.056 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0366e-05 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.592901+0000", 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1600000000000003e-06 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.653327+0000", 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.060425660999999999 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.057 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for degraded object", 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.653398+0000", 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.1755000000000007e-05 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.758477+0000", 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10507913000000001 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.758540+0000", 2023-09-23T18:22:12.058 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2232999999999999e-05 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 4,7", 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.758626+0000", 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6231999999999997e-05 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.764380+0000", 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0057542340000000004 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.059 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.765155+0000", 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00077486000000000005 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.866705+0000", 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10154997 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.060 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.866758+0000", 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3186000000000002e-05 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:12:27.868753+0000", 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019952580000000002 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.061 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:35553 10.5 10:a3327171:::benchmark_data_smithi055_425050_object35552:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:17.646314+0000", 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 234.40548707900001, 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18390158100000001, 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 35553 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.062 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646314+0000", 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646314+0000", 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.063 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646314+0000", 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646337+0000", 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3941999999999999e-05 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646345+0000", 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9670000000000001e-06 2023-09-23T18:22:12.064 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.646351+0000", 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6500000000000001e-06 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647569+0000", 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0012183230000000001 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.065 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647780+0000", 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021045300000000001 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.647914+0000", 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013460100000000001 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.648993+0000", 2023-09-23T18:22:12.066 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001078411 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.653792+0000", 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047994989999999996 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830093+0000", 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.176300967 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.067 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830163+0000", 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9437999999999995e-05 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830216+0000", 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2889999999999997e-05 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.068 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:35557 10.5 10:a4583da5:::benchmark_data_smithi055_425050_object35556:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:17.651246+0000", 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 234.40055492799999, 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.17918794399999999, 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.069 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 35557 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651246+0000", 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651246+0000", 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.070 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651245+0000", 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651268+0000", 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3459999999999999e-05 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.071 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651278+0000", 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.5980000000000003e-06 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651284+0000", 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.003e-06 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651342+0000", 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8736000000000001e-05 2023-09-23T18:22:12.072 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651537+0000", 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019446700000000001 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.651716+0000", 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017865599999999999 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.073 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.653287+0000", 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0015712199999999999 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.655801+0000", 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.002514077 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830339+0000", 2023-09-23T18:22:12.074 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.174538633 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830395+0000", 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5553999999999997e-05 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.830434+0000", 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9175999999999999e-05 2023-09-23T18:22:12.075 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67163 10.c 10.eeb40c6c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.431909+0000", 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 206.619891911, 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99054996900000003, 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.076 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67163 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431909+0000", 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431909+0000", 2023-09-23T18:22:12.077 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431908+0000", 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431935+0000", 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6780000000000001e-05 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.078 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431946+0000", 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1302e-05 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.431954+0000", 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4279999999999996e-06 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.079 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.432168+0000", 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00021302499999999999 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.432177+0000", 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.7329999999999998e-06 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.422443+0000", 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.99026528199999997 2023-09-23T18:22:12.080 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.422459+0000", 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6671999999999999e-05 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67177 10.c 10.f65fed0c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.442552+0000", 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 206.60924949, 2023-09-23T18:22:12.081 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.98041694300000004, 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67177 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442552+0000", 2023-09-23T18:22:12.082 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442552+0000", 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442551+0000", 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.083 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442575+0000", 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4281999999999999e-05 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442583+0000", 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3440000000000003e-06 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.442588+0000", 2023-09-23T18:22:12.084 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5520000000000004e-06 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.444556+0000", 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019680460000000002 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.444566+0000", 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9280000000000005e-06 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.085 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.422959+0000", 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97839311500000004 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.422969+0000", 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1409999999999994e-06 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.086 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67189 10.c 10.7b7f43bc (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.452426+0000", 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 206.599375439, 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97057495000000005, 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67189 2023-09-23T18:22:12.087 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452426+0000", 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452426+0000", 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.088 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452424+0000", 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452447+0000", 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3203000000000001e-05 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452465+0000", 2023-09-23T18:22:12.089 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7722000000000001e-05 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452474+0000", 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7290000000000006e-06 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452525+0000", 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0986000000000003e-05 2023-09-23T18:22:12.090 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.452537+0000", 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.206e-05 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.422993+0000", 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.97045598 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.091 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.423001+0000", 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8860000000000004e-06 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67195 10.c 10.e21a187c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.459821+0000", 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 206.59198059400001, 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96321096799999995, 2023-09-23T18:22:12.092 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67195 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459821+0000", 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.093 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459821+0000", 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459819+0000", 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.094 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459840+0000", 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1480000000000001e-05 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459855+0000", 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4151000000000001e-05 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459867+0000", 2023-09-23T18:22:12.095 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2549e-05 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459924+0000", 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6986e-05 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.459938+0000", 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3679000000000001e-05 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.096 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.423024+0000", 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96308652900000002 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.423032+0000", 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.199e-06 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.097 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67196 10.c 10.cbe8354c (undecoded) ondisk+write+known_if_redirected+supports_pool_eio e594)", 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.460546+0000", 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 206.59125549199999, 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96306485200000003, 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "reached pg", 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67196 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.098 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460546+0000", 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460546+0000", 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.099 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460545+0000", 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460590+0000", 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5454999999999998e-05 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460608+0000", 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7227999999999998e-05 2023-09-23T18:22:12.100 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460613+0000", 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5670000000000002e-06 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460655+0000", 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2304e-05 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.101 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.460662+0000", 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0920000000000003e-06 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.423601+0000", 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.96293968399999996 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:46.423611+0000", 2023-09-23T18:22:12.102 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2909999999999996e-06 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:111865 10.5 10:ad610951:::benchmark_data_smithi055_425050_object8459:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:19:42.351369+0000", 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 149.700432202, 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.18157957399999999, 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.103 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 111865 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351369+0000", 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.104 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351369+0000", 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351367+0000", 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.105 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351375+0000", 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4490000000000002e-06 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351390+0000", 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5296999999999999e-05 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351400+0000", 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8679999999999994e-06 2023-09-23T18:22:12.106 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351455+0000", 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5569000000000003e-05 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351570+0000", 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011479199999999999 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.107 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351671+0000", 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000101233 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.363540+0000", 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011868408 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:12.108 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.364077+0000", 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00053702899999999996 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.532870+0000", 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.16879335200000001 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.532909+0000", 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8705000000000003e-05 2023-09-23T18:22:12.109 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.532949+0000", 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9758000000000001e-05 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:111868 10.5 10:a6bcee6d:::benchmark_data_smithi055_425050_object8462:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e606)", 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:19:42.351434+0000", 2023-09-23T18:22:12.110 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 149.70036741300001, 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.181727735, 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 111868 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.111 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351434+0000", 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351434+0000", 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351433+0000", 2023-09-23T18:22:12.112 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351437+0000", 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.1520000000000002e-06 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351446+0000", 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2210000000000003e-06 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.113 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.351454+0000", 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8769999999999993e-06 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.354487+0000", 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030321990000000002 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.114 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.354608+0000", 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012146699999999999 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.354723+0000", 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114887 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.364081+0000", 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0093583480000000007 2023-09-23T18:22:12.115 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.526329+0000", 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.162247373 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.533101+0000", 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.006772246 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.116 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.533132+0000", 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0898999999999997e-05 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:19:42.533161+0000", 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.9728000000000001e-05 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.117 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:200203 10.3 e621/595 10:ccd11fc8:::benchmark_data_smithi055_425050_object96797:head v 621'12634, mlcod=609'8266)", 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:28.025349+0000", 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 44.026452347000003, 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.061315044999999999, 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.025349+0000", 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.118 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.025349+0000", 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.025348+0000", 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.119 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.025353+0000", 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5260000000000004e-06 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.025363+0000", 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.9429999999999995e-06 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.025368+0000", 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.7509999999999999e-06 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.120 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.048294+0000", 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.022926406 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.048337+0000", 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2898000000000002e-05 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.086630+0000", 2023-09-23T18:22:12.121 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.038293451999999999 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.086664+0000", 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3436000000000003e-05 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:200207 10.f e621/574 10:fa4edeb1:::benchmark_data_smithi055_425050_object96801:head v 621'12425, mlcod=621'12423)", 2023-09-23T18:22:12.122 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:28.028241+0000", 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 44.023560474999996, 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.070162941000000006, 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.028241+0000", 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.123 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.028241+0000", 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.028240+0000", 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.028246+0000", 2023-09-23T18:22:12.124 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0290000000000002e-06 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.028259+0000", 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3209e-05 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.028271+0000", 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1888e-05 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.125 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.038355+0000", 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010083695 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.038382+0000", 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.7029000000000002e-05 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.098359+0000", 2023-09-23T18:22:12.126 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.059977432999999997 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.098404+0000", 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.4379000000000003e-05 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:200218 10.7 e621/595 10:ea8ed5c5:::benchmark_data_smithi055_425050_object96812:head v 621'12297, mlcod=0'0)", 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:28.038766+0000", 2023-09-23T18:22:12.127 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 44.013035213000002, 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.065962594999999999, 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.038766+0000", 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.128 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.038766+0000", 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.038765+0000", 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.038774+0000", 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7409999999999998e-06 2023-09-23T18:22:12.129 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.038789+0000", 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5252000000000001e-05 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.038797+0000", 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.8830000000000006e-06 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.130 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.041000+0000", 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0022028210000000002 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.041026+0000", 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6434000000000001e-05 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.104676+0000", 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.063649648000000003 2023-09-23T18:22:12.131 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:28.104729+0000", 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2608000000000002e-05 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:201060 10.5 10:a0e544fa:::benchmark_data_smithi055_425050_object97654:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:29.177403+0000", 2023-09-23T18:22:12.132 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 42.874398002, 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.117175695, 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 201060 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.133 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.177403+0000", 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.177403+0000", 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.177402+0000", 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.134 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.177409+0000", 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9650000000000002e-06 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.177424+0000", 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5007999999999999e-05 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.135 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.177435+0000", 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1056000000000001e-05 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.177491+0000", 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6725000000000003e-05 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.177627+0000", 2023-09-23T18:22:12.136 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013572899999999999 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.177755+0000", 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012780699999999999 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.180670+0000", 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0029147589999999998 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.137 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.189697+0000", 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0090276499999999999 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.294454+0000", 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10475670300000001 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.294524+0000", 2023-09-23T18:22:12.138 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.9560999999999993e-05 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.294579+0000", 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5235000000000002e-05 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:201071 10.3 e621/595 10:c9c54b11:::benchmark_data_smithi055_425050_object97665:head v 621'12677, mlcod=609'8328)", 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:29.189979+0000", 2023-09-23T18:22:12.139 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 42.861821728999999, 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.095073874000000003, 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.189979+0000", 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.140 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.189979+0000", 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.189979+0000", 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.189986+0000", 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.4989999999999997e-06 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.141 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.189996+0000", 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0095000000000001e-05 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.190003+0000", 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4570000000000002e-06 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.142 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.190041+0000", 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8039e-05 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.190064+0000", 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2926999999999999e-05 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.284992+0000", 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.094928203000000003 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.143 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.285053+0000", 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1325e-05 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:201073 10.5 10:ad6b62ee:::benchmark_data_smithi055_425050_object97667:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e621)", 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:29.193299+0000", 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 42.858502428000001, 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.104586048, 2023-09-23T18:22:12.144 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 201073 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.193299+0000", 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.145 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.193299+0000", 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.193297+0000", 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.146 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.193305+0000", 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0339999999999997e-06 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.193319+0000", 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4528e-05 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.193329+0000", 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.8130000000000007e-06 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.147 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.193378+0000", 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.9138000000000003e-05 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.193475+0000", 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6859000000000002e-05 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.193559+0000", 2023-09-23T18:22:12.148 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.4237e-05 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.285801+0000", 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.092241100000000006 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.285812+0000", 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1229999999999999e-05 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.149 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.297778+0000", 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.01196642 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.297841+0000", 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3041999999999997e-05 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.297885+0000", 2023-09-23T18:22:12.150 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3522000000000001e-05 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.46799.0:201074 10.7 e621/595 10:e06f5bb4:::benchmark_data_smithi055_425050_object97668:head v 621'12361, mlcod=0'0)", 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:29.194264+0000", 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 42.857537551, 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10429574699999999, 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.151 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.194264+0000", 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.194264+0000", 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.152 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.194262+0000", 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.194271+0000", 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.0499999999999992e-06 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.282898+0000", 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.088627808000000002 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.153 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.282909+0000", 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1165000000000001e-05 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.282958+0000", 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8174000000000003e-05 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.154 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.282995+0000", 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6953999999999999e-05 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.298523+0000", 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015528643999999999 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.298559+0000", 2023-09-23T18:22:12.155 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6077e-05 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop_reply(client.46799.0:201073 10.5 e621/574 ondisk, result = 0)", 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:29.196075+0000", 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 42.855725772, 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.089737905000000007, 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:22:12.156 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.196075+0000", 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.196075+0000", 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.157 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.196075+0000", 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.196080+0000", 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.977e-06 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.284799+0000", 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.088718854 2023-09-23T18:22:12.158 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.284807+0000", 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8820000000000007e-06 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.285780+0000", 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00097285500000000001 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.159 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.285792+0000", 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1449e-05 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:29.285813+0000", 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.1571000000000001e-05 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.160 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:203443 10.5 10:a4989fe2:::benchmark_data_smithi055_425050_object100037:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:32.120671+0000", 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 39.931130205000002, 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.057060899999999998, 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 203443 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.161 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120671+0000", 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120671+0000", 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.162 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120669+0000", 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120676+0000", 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3170000000000004e-06 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120691+0000", 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4409e-05 2023-09-23T18:22:12.163 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120700+0000", 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3989999999999993e-06 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.132386+0000", 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.011685442000000001 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.164 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.132540+0000", 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000154893 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.132681+0000", 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014080200000000001 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:12.165 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.142574+0000", 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0098929329999999996 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.143548+0000", 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000973426 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.177635+0000", 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.034086992000000003 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.166 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.177690+0000", 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.4954000000000003e-05 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.177732+0000", 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2243000000000002e-05 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.167 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:203447 10.5 10:ab27038a:::benchmark_data_smithi055_425050_object100041:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e622)", 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:21:32.120719+0000", 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 39.931082586999999, 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.057133953000000001, 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 203447 2023-09-23T18:22:12.168 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120719+0000", 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120719+0000", 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.169 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120718+0000", 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120722+0000", 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.5580000000000001e-06 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120733+0000", 2023-09-23T18:22:12.170 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1620999999999999e-05 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.120741+0000", 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.289e-06 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.135891+0000", 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.015150785 2023-09-23T18:22:12.171 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.136037+0000", 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014598700000000001 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,5", 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.136163+0000", 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012526700000000001 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:12.172 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.142581+0000", 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0064182830000000003 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.143574+0000", 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0009932579999999999 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.177770+0000", 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.034195950000000003 2023-09-23T18:22:12.173 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.177815+0000", 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5352999999999998e-05 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:21:32.177853+0000", 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.7112e-05 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.174 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.175 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:12.175 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:12.175 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:22:12.175 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp /tmp/rbd_import_export_463999/img /tmp/rbd_import_export_463999/img3 2023-09-23T18:22:12.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:11 smithi055 ceph-mon[179668]: pgmap v1544: 103 pgs: 3 peering, 100 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 703 B/s, 1 keys/s, 4 objects/s recovering 2023-09-23T18:22:12.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:11 smithi055 ceph-mon[184174]: pgmap v1544: 103 pgs: 3 peering, 100 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 703 B/s, 1 keys/s, 4 objects/s recovering 2023-09-23T18:22:12.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:11 smithi099 ceph-mon[167076]: pgmap v1544: 103 pgs: 3 peering, 100 active+clean; 4.6 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 703 B/s, 1 keys/s, 4 objects/s recovering 2023-09-23T18:22:12.632 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/img2 /tmp/rbd_import_export_463999/img3 2023-09-23T18:22:12.860 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format - testimg 2023-09-23T18:22:13.271 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T18:22:13.271 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:22:13.274 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg /tmp/rbd_import_export_463999/img2 2023-09-23T18:22:13.576 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:13.579 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg - 2023-09-23T18:22:14.161 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:14.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:14.165 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:14.267 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:14.298 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:14.335 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:14.338 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:14.338 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:14.390 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 1% complete... Exporting image: 2% complete... Exporting image: 3% complete... Exporting image: 4% complete... Exporting image: 5% complete... Exporting image: 6% complete... Exporting image: 7% complete... Exporting image: 8% complete... Exporting image: 9% complete... Exporting image: 10% complete... Exporting image: 11% complete... Exporting image: 12% complete... Exporting image: 13% complete... Exporting image: 14% complete... Exporting image: 15% complete... Exporting image: 16% complete... Exporting image: 17% complete... Exporting image: 18% complete... Exporting image: 19% complete... Exporting image: 20% complete... Exporting image: 21% complete... Exporting image: 22% complete... Exporting image: 23% complete... Exporting image: 24% complete... Exporting image: 25% complete... Exporting image: 26% complete... Exporting image: 27% complete... Exporting image: 28% complete... Exporting image: 29% complete... Exporting image: 30% complete... Exporting image: 31% complete... Exporting image: 32% complete... Exporting image: 33% complete... Exporting image: 34% complete... Exporting image: 35% complete... Exporting image: 36% complete... Exporting image: 37% complete... Exporting image: 38% complete... Exporting image: 39% complete... Exporting image: 40% complete... Exporting image: 41% complete... Exporting image: 42% complete... Exporting image: 43% complete... Exporting image: 44% complete... Exporting image: 45% complete... Exporting image: 46% complete... Exporting image: 47% complete... Exporting image: 48% complete... Exporting image: 49% complete... Exporting image: 50% complete... Exporting image: 51% complete... Exporting image: 52% complete... Exporting image: 53% complete... Exporting image: 54% complete... Exporting image: 55% complete... Exporting image: 56% complete... Exporting image: 57% complete... Exporting image: 58% complete... Exporting image: 59% complete... Exporting image: 60% complete... Exporting image: 61% complete... Exporting image: 62% complete... Exporting image: 63% complete... Exporting image: 64% complete... Exporting image: 65% complete... Exporting image: 66% complete... Exporting image: 67% complete... Exporting image: 68% complete... Exporting image: 69% complete... Exporting image: 70% complete... Exporting image: 71% complete... Exporting image: 72% complete... Exporting image: 73% complete... Exporting image: 74% complete... Exporting image: 75% complete... Exporting image: 76% complete... Exporting image: 77% complete... Exporting image: 78% complete... Exporting image: 79% complete... Exporting image: 80% complete... Exporting image: 81% complete... Exporting image: 82% complete... Exporting image: 83% complete... Exporting image: 84% complete... Exporting image: 85% complete... Exporting image: 86% complete... Exporting image: 87% complete... Exporting image: 88% complete... Exporting image: 89% complete... Exporting image: 90% complete... Exporting image: 91% complete... Exporting image: 92% complete... Exporting image: 93% complete... Exporting image: 94% complete... Exporting image: 95% complete... Exporting image: 96% complete... Exporting image: 97% complete... Exporting image: 98% complete... Exporting image: 99% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:14.393 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm testimg 2023-09-23T18:22:14.439 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:14.469 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:14.497 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 1% complete... Removing image: 2% complete... Removing image: 3% complete... Removing image: 4% complete... Removing image: 5% complete... Removing image: 6% complete... Removing image: 7% complete... Removing image: 8% complete... Removing image: 9% complete... Removing image: 10% complete... Removing image: 11% complete... Removing image: 12% complete... Removing image: 13% complete... Removing image: 14% complete... Removing image: 15% complete... Removing image: 16% complete... Removing image: 17% complete... Removing image: 18% complete... Removing image: 19% complete... Removing image: 20% complete... Removing image: 21% complete... Removing image: 22% complete... Removing image: 23% complete... Removing image: 24% complete... Removing image: 25% complete... Removing image: 26% complete... Removing image: 27% complete... Removing image: 28% complete... Removing image: 29% complete... Removing image: 30% complete... Removing image: 31% complete... Removing image: 32% complete... Removing image: 33% complete... Removing image: 34% complete... Removing image: 35% complete... Removing image: 36% complete... Removing image: 37% complete... Removing image: 38% complete... Removing image: 39% complete... Removing image: 40% complete... Removing image: 41% complete... Removing image: 42% complete... Removing image: 43% complete... Removing image: 44% complete... Removing image: 45% complete... Removing image: 46% complete... Removing image: 47% complete... Removing image: 48% complete... Removing image: 49% complete... Removing image: 50% complete... Removing image: 51% complete... Removing image: 52% complete... Removing image: 53% complete... Removing image: 54% complete... Removing image: 55% complete... Removing image: 56% complete... Removing image: 57% complete... Removing image: 58% complete... Removing image: 59% complete... Removing image: 60% complete... Removing image: 61% complete... Removing image: 62% complete... Removing image: 63% complete... Removing image: 64% complete... Removing image: 65% complete... Removing image: 66% complete... Removing image: 67% complete... Removing image: 68% complete... Removing image: 69% complete... Removing image: 70% complete... Removing image: 71% complete... Removing image: 72% complete... Removing image: 73% complete... Removing image: 74% complete... Removing image: 75% complete... Removing image: 76% complete... Removing image: 77% complete... Removing image: 78% complete... Removing image: 79% complete... Removing image: 80% complete... Removing image: 81% complete... Removing image: 82% complete... Removing image: 83% complete... Removing image: 84% complete... Removing image: 85% complete... Removing image: 86% complete... Removing image: 87% complete... Removing image: 88% complete... Removing image: 89% complete... Removing image: 90% complete... Removing image: 91% complete... Removing image: 92% complete... Removing image: 93% complete... Removing image: 94% complete... Removing image: 95% complete... Removing image: 96% complete... Removing image: 97% complete... Removing image: 98% complete... Removing image: 99% complete... Removing image: 100% complete...done. 2023-09-23T18:22:14.500 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp /tmp/rbd_import_export_463999/img /tmp/rbd_import_export_463999/img2 2023-09-23T18:22:14.505 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:14.507 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:14.508 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:14.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:14 smithi099 ceph-mon[167076]: pgmap v1545: 103 pgs: 103 active+clean; 7.3 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 137 KiB/s rd, 63 KiB/s wr, 23 op/s; 595 B/s, 1 keys/s, 3 objects/s recovering 2023-09-23T18:22:14.608 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:14.641 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:14.648 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_ops_in_flight 2023-09-23T18:22:14.792 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:14.794 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:14 smithi055 ceph-mon[184174]: pgmap v1545: 103 pgs: 103 active+clean; 7.3 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 137 KiB/s rd, 63 KiB/s wr, 23 op/s; 595 B/s, 1 keys/s, 3 objects/s recovering 2023-09-23T18:22:14.795 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:14 smithi055 ceph-mon[179668]: pgmap v1545: 103 pgs: 103 active+clean; 7.3 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 137 KiB/s rd, 63 KiB/s wr, 23 op/s; 595 B/s, 1 keys/s, 3 objects/s recovering 2023-09-23T18:22:14.795 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:14.796 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:14.896 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:14.908 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:14.917 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:14.923 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:14.957 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:14.960 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:14.961 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:15.061 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:15.087 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:15.123 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:15.127 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:15.127 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:15.214 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp /tmp/rbd_import_export_463999/img /tmp/rbd_import_export_463999/img3 2023-09-23T18:22:15.228 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:15.247 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:22:15.256 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:15.384 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:15.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:15.387 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:15.473 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:22:15.488 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:15.512 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4113462729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:22:15.516 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:15.525 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4113462729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:22:15.525 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4113462729' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:22:15.550 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:15.553 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:15.554 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:15.654 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:15.690 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:15.766 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/img /tmp/rbd_import_export_463999/img2 /tmp/rbd_import_export_463999/img3 2023-09-23T18:22:15.833 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:15.836 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:15.836 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:15.937 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:15.966 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:16.002 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:16.005 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:16.006 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:16.056 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd help export 2023-09-23T18:22:16.057 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -q export-format 2023-09-23T18:22:16.079 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/bin/dd of=/tmp/rbd_import_export_463999/img bs=1k count=10 seek=100 2023-09-23T18:22:16.080 INFO:tasks.workunit.client.0.smithi055.stderr:10+0 records in 2023-09-23T18:22:16.080 INFO:tasks.workunit.client.0.smithi055.stderr:10+0 records out 2023-09-23T18:22:16.080 INFO:tasks.workunit.client.0.smithi055.stderr:10240 bytes (10 kB, 10 KiB) copied, 7.2923e-05 s, 140 MB/s 2023-09-23T18:22:16.080 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format /tmp/rbd_import_export_463999/img testimg 2023-09-23T18:22:16.107 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:16.134 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:16.142 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T18:22:16.143 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:22:16.146 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap create testimg@snap 2023-09-23T18:22:16.170 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:16.174 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:16.174 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:16.275 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:16.304 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:16.390 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:16.394 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:16.395 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:16.478 INFO:teuthology.orchestra.run.smithi099.stdout:{ 2023-09-23T18:22:16.479 INFO:teuthology.orchestra.run.smithi099.stdout: "ops": [], 2023-09-23T18:22:16.479 INFO:teuthology.orchestra.run.smithi099.stdout: "num_ops": 0 2023-09-23T18:22:16.479 INFO:teuthology.orchestra.run.smithi099.stdout:} 2023-09-23T18:22:16.484 INFO:tasks.workunit.client.0.smithi055.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T18:22:16.490 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd image-meta set testimg key1 value1 2023-09-23T18:22:16.495 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:16.522 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:16.546 INFO:tasks.workunit.client.0.smithi055.stderr:++ rbd image-meta list testimg 2023-09-23T18:22:16.559 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:16.562 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:16.562 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:16.587 INFO:tasks.workunit.client.0.smithi055.stderr:+ IMAGEMETA_BEFORE='There is 1 metadatum on this image: 2023-09-23T18:22:16.587 INFO:tasks.workunit.client.0.smithi055.stderr: 2023-09-23T18:22:16.587 INFO:tasks.workunit.client.0.smithi055.stderr:Key Value 2023-09-23T18:22:16.587 INFO:tasks.workunit.client.0.smithi055.stderr:key1 value1' 2023-09-23T18:22:16.587 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_463999/img_v2 2023-09-23T18:22:16.630 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:16.635 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_463999/img_v2 testimg_import 2023-09-23T18:22:16.663 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:16.683 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:16 smithi099 ceph-mon[167076]: pgmap v1546: 103 pgs: 103 active+clean; 8.1 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 137 KiB/s rd, 73 KiB/s wr, 28 op/s; 488 B/s, 1 keys/s, 2 objects/s recovering 2023-09-23T18:22:16.684 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:16 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:16.684 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:16 smithi099 ceph-mon[167076]: failed to encode map e639 with expected crc 2023-09-23T18:22:16.684 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:16 smithi099 ceph-mon[167076]: failed to encode map e639 with expected crc 2023-09-23T18:22:16.684 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:16 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4113462729' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:22:16.684 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:16 smithi099 ceph-mon[167076]: osdmap e639: 8 total, 8 up, 7 in 2023-09-23T18:22:16.684 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:16 smithi099 ceph-mon[167076]: failed to encode map e639 with expected crc 2023-09-23T18:22:16.691 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:16.707 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[179668]: pgmap v1546: 103 pgs: 103 active+clean; 8.1 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 137 KiB/s rd, 73 KiB/s wr, 28 op/s; 488 B/s, 1 keys/s, 2 objects/s recovering 2023-09-23T18:22:16.707 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:16.707 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[179668]: failed to encode map e639 with expected crc 2023-09-23T18:22:16.707 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[179668]: failed to encode map e639 with expected crc 2023-09-23T18:22:16.707 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4113462729' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:22:16.707 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[179668]: osdmap e639: 8 total, 8 up, 7 in 2023-09-23T18:22:16.707 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[179668]: failed to encode map e639 with expected crc 2023-09-23T18:22:16.708 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[184174]: pgmap v1546: 103 pgs: 103 active+clean; 8.1 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 137 KiB/s rd, 73 KiB/s wr, 28 op/s; 488 B/s, 1 keys/s, 2 objects/s recovering 2023-09-23T18:22:16.708 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:16.708 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[184174]: failed to encode map e639 with expected crc 2023-09-23T18:22:16.708 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[184174]: failed to encode map e639 with expected crc 2023-09-23T18:22:16.708 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4113462729' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:22:16.708 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[184174]: osdmap e639: 8 total, 8 up, 7 in 2023-09-23T18:22:16.708 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:16 smithi055 ceph-mon[184174]: failed to encode map e639 with expected crc 2023-09-23T18:22:16.874 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:16.877 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:16.877 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:16.908 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_blocked_ops 2023-09-23T18:22:16.935 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:22:16.954 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:22:16.973 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:22:16.978 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:22:16.992 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:22:17.008 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:17.012 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:22:17.035 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:22:17.049 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:22:17.051 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:17.052 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:22:17.057 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:22:17.075 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:22:17.077 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:22:17.077 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:22:17.077 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:22:17.077 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:22:17.077 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:22:17.078 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:22:17.078 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:22:17.078 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:22:17.152 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:17.179 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:17.212 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:17.215 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:17.216 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:17.316 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:17.346 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:17.433 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:17.436 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:17.437 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:17.495 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T18:22:17.515 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 50% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T18:22:17.518 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import 2023-09-23T18:22:17.538 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:17.540 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:22:17.558 INFO:tasks.workunit.client.0.smithi055.stdout:rbd image 'testimg_import': 2023-09-23T18:22:17.558 INFO:tasks.workunit.client.0.smithi055.stdout: size 110 KiB in 1 objects 2023-09-23T18:22:17.558 INFO:tasks.workunit.client.0.smithi055.stdout: order 22 (4 MiB objects) 2023-09-23T18:22:17.558 INFO:tasks.workunit.client.0.smithi055.stdout: snapshot_count: 1 2023-09-23T18:22:17.558 INFO:tasks.workunit.client.0.smithi055.stdout: id: 92c7c0cb1c07 2023-09-23T18:22:17.558 INFO:tasks.workunit.client.0.smithi055.stdout: block_name_prefix: rbd_data.92c7c0cb1c07 2023-09-23T18:22:17.558 INFO:tasks.workunit.client.0.smithi055.stdout: format: 2 2023-09-23T18:22:17.559 INFO:tasks.workunit.client.0.smithi055.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T18:22:17.559 INFO:tasks.workunit.client.0.smithi055.stdout: op_features: 2023-09-23T18:22:17.559 INFO:tasks.workunit.client.0.smithi055.stdout: flags: 2023-09-23T18:22:17.559 INFO:tasks.workunit.client.0.smithi055.stdout: create_timestamp: Sat Sep 23 18:22:16 2023 2023-09-23T18:22:17.559 INFO:tasks.workunit.client.0.smithi055.stdout: access_timestamp: Sat Sep 23 18:22:16 2023 2023-09-23T18:22:17.559 INFO:tasks.workunit.client.0.smithi055.stdout: modify_timestamp: Sat Sep 23 18:22:16 2023 2023-09-23T18:22:17.562 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import@snap 2023-09-23T18:22:17.565 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:17.606 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[179668]: failed to encode map e639 with expected crc 2023-09-23T18:22:17.606 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[179668]: failed to encode map e640 with expected crc 2023-09-23T18:22:17.606 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[179668]: osdmap e640: 8 total, 8 up, 7 in 2023-09-23T18:22:17.606 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[179668]: failed to encode map e640 with expected crc 2023-09-23T18:22:17.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2334595643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:17.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1911275811' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:22:17.607 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:22:17.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[184174]: failed to encode map e639 with expected crc 2023-09-23T18:22:17.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[184174]: failed to encode map e640 with expected crc 2023-09-23T18:22:17.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[184174]: osdmap e640: 8 total, 8 up, 7 in 2023-09-23T18:22:17.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[184174]: failed to encode map e640 with expected crc 2023-09-23T18:22:17.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2334595643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:17.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1911275811' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:22:17.607 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:17 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:22:17.614 INFO:tasks.workunit.client.0.smithi055.stdout:rbd image 'testimg_import': 2023-09-23T18:22:17.614 INFO:tasks.workunit.client.0.smithi055.stdout: size 110 KiB in 1 objects 2023-09-23T18:22:17.614 INFO:tasks.workunit.client.0.smithi055.stdout: order 22 (4 MiB objects) 2023-09-23T18:22:17.614 INFO:tasks.workunit.client.0.smithi055.stdout: snapshot_count: 1 2023-09-23T18:22:17.614 INFO:tasks.workunit.client.0.smithi055.stdout: id: 92c7c0cb1c07 2023-09-23T18:22:17.614 INFO:tasks.workunit.client.0.smithi055.stdout: block_name_prefix: rbd_data.92c7c0cb1c07 2023-09-23T18:22:17.614 INFO:tasks.workunit.client.0.smithi055.stdout: format: 2 2023-09-23T18:22:17.615 INFO:tasks.workunit.client.0.smithi055.stdout: features: layering, exclusive-lock, object-map, fast-diff, deep-flatten 2023-09-23T18:22:17.615 INFO:tasks.workunit.client.0.smithi055.stdout: op_features: 2023-09-23T18:22:17.615 INFO:tasks.workunit.client.0.smithi055.stdout: flags: 2023-09-23T18:22:17.615 INFO:tasks.workunit.client.0.smithi055.stdout: create_timestamp: Sat Sep 23 18:22:16 2023 2023-09-23T18:22:17.615 INFO:tasks.workunit.client.0.smithi055.stdout: access_timestamp: Sat Sep 23 18:22:16 2023 2023-09-23T18:22:17.615 INFO:tasks.workunit.client.0.smithi055.stdout: modify_timestamp: Sat Sep 23 18:22:16 2023 2023-09-23T18:22:17.615 INFO:tasks.workunit.client.0.smithi055.stdout: protected: False 2023-09-23T18:22:17.617 INFO:tasks.workunit.client.0.smithi055.stderr:++ rbd image-meta list testimg_import 2023-09-23T18:22:17.657 INFO:tasks.workunit.client.0.smithi055.stderr:+ IMAGEMETA_AFTER='There is 1 metadatum on this image: 2023-09-23T18:22:17.657 INFO:tasks.workunit.client.0.smithi055.stderr: 2023-09-23T18:22:17.657 INFO:tasks.workunit.client.0.smithi055.stderr:Key Value 2023-09-23T18:22:17.657 INFO:tasks.workunit.client.0.smithi055.stderr:key1 value1' 2023-09-23T18:22:17.657 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 'There is 1 metadatum on this image: 2023-09-23T18:22:17.657 INFO:tasks.workunit.client.0.smithi055.stderr: 2023-09-23T18:22:17.657 INFO:tasks.workunit.client.0.smithi055.stderr:Key Value 2023-09-23T18:22:17.657 INFO:tasks.workunit.client.0.smithi055.stderr:key1 value1' = 'There is 1 metadatum on this image: 2023-09-23T18:22:17.657 INFO:tasks.workunit.client.0.smithi055.stderr: 2023-09-23T18:22:17.658 INFO:tasks.workunit.client.0.smithi055.stderr:Key Value 2023-09-23T18:22:17.658 INFO:tasks.workunit.client.0.smithi055.stderr:key1 value1' ']' 2023-09-23T18:22:17.658 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg_import /tmp/rbd_import_export_463999/img_import 2023-09-23T18:22:17.666 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:17.669 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:17.670 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:17.691 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete...done. 2023-09-23T18:22:17.694 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_463999/img /tmp/rbd_import_export_463999/img_import 2023-09-23T18:22:17.694 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_463999/img /tmp/rbd_import_export_463999/img_import 2023-09-23T18:22:17.695 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/img --format %b 2023-09-23T18:22:17.696 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=24 2023-09-23T18:22:17.696 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/img_import --format %b 2023-09-23T18:22:17.697 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=224 2023-09-23T18:22:17.697 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=200 2023-09-23T18:22:17.697 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=200 2023-09-23T18:22:17.697 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 200 -ge 0 -a 200 -lt 4096 2023-09-23T18:22:17.698 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg@snap /tmp/rbd_import_export_463999/img_snap 2023-09-23T18:22:17.733 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete...done. 2023-09-23T18:22:17.736 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export testimg_import@snap /tmp/rbd_import_export_463999/img_snap_import 2023-09-23T18:22:17.770 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:17.771 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete...done. 2023-09-23T18:22:17.774 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_463999/img_snap /tmp/rbd_import_export_463999/img_snap_import 2023-09-23T18:22:17.774 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_463999/img_snap /tmp/rbd_import_export_463999/img_snap_import 2023-09-23T18:22:17.775 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/img_snap --format %b 2023-09-23T18:22:17.776 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=224 2023-09-23T18:22:17.776 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/img_snap_import --format %b 2023-09-23T18:22:17.777 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=224 2023-09-23T18:22:17.778 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T18:22:17.778 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T18:22:17.778 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T18:22:17.778 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/img_v2 2023-09-23T18:22:17.778 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/img_import 2023-09-23T18:22:17.779 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/img_snap 2023-09-23T18:22:17.779 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/img_snap_import 2023-09-23T18:22:17.780 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap rm testimg_import@snap 2023-09-23T18:22:17.781 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:17 smithi099 ceph-mon[167076]: failed to encode map e639 with expected crc 2023-09-23T18:22:17.782 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:17 smithi099 ceph-mon[167076]: failed to encode map e640 with expected crc 2023-09-23T18:22:17.782 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:17 smithi099 ceph-mon[167076]: osdmap e640: 8 total, 8 up, 7 in 2023-09-23T18:22:17.782 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:17 smithi099 ceph-mon[167076]: failed to encode map e640 with expected crc 2023-09-23T18:22:17.782 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2334595643' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:17.782 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1911275811' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:22:17.782 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:17 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:22:17.797 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:17.841 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:17.844 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:17.844 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:17.900 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T18:22:17.945 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:18.010 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:18.054 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:18.057 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:18.057 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:18.158 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:18.186 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:18.220 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:18.223 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:18.223 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:18.324 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:18.354 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:18.455 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:18.457 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:18.458 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:18.480 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[179668]: failed to encode map e640 with expected crc 2023-09-23T18:22:18.480 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[179668]: pgmap v1549: 103 pgs: 103 active+clean; 5.4 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 297 KiB/s rd, 131 KiB/s wr, 107 op/s 2023-09-23T18:22:18.481 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[179668]: failed to encode map e641 with expected crc 2023-09-23T18:22:18.481 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[179668]: failed to encode map e641 with expected crc 2023-09-23T18:22:18.481 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:22:18.481 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[179668]: osdmap e641: 8 total, 8 up, 7 in 2023-09-23T18:22:18.501 INFO:tasks.workunit.client.0.smithi055.stderr: Removing snap: 100% complete...done. 2023-09-23T18:22:18.508 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg_import 2023-09-23T18:22:18.559 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:18.647 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:18 smithi099 ceph-mon[167076]: failed to encode map e640 with expected crc 2023-09-23T18:22:18.647 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:18 smithi099 ceph-mon[167076]: pgmap v1549: 103 pgs: 103 active+clean; 5.4 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 297 KiB/s rd, 131 KiB/s wr, 107 op/s 2023-09-23T18:22:18.647 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:18 smithi099 ceph-mon[167076]: failed to encode map e641 with expected crc 2023-09-23T18:22:18.647 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:18 smithi099 ceph-mon[167076]: failed to encode map e641 with expected crc 2023-09-23T18:22:18.647 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:18 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:22:18.648 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:18 smithi099 ceph-mon[167076]: osdmap e641: 8 total, 8 up, 7 in 2023-09-23T18:22:18.648 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:18 smithi099 ceph-mon[167076]: failed to encode map e641 with expected crc 2023-09-23T18:22:18.648 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:18.649 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T18:22:18.649 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap rm testimg@snap 2023-09-23T18:22:18.652 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:22:18.676 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:18.712 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:18.715 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:18.716 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:22:18.779 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:22:18.787 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[184174]: failed to encode map e640 with expected crc 2023-09-23T18:22:18.789 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[184174]: pgmap v1549: 103 pgs: 103 active+clean; 5.4 MiB data, 5.1 GiB used, 621 GiB / 626 GiB avail; 297 KiB/s rd, 131 KiB/s wr, 107 op/s 2023-09-23T18:22:18.790 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[184174]: failed to encode map e641 with expected crc 2023-09-23T18:22:18.791 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[184174]: failed to encode map e641 with expected crc 2023-09-23T18:22:18.791 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:22:18.791 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[184174]: osdmap e641: 8 total, 8 up, 7 in 2023-09-23T18:22:18.791 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[184174]: failed to encode map e641 with expected crc 2023-09-23T18:22:18.791 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:18 smithi055 ceph-mon[179668]: failed to encode map e641 with expected crc 2023-09-23T18:22:18.816 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:18.845 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:18.883 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:18.886 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:18.886 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:18.987 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:19.015 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:19.158 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:19.161 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:19.161 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:19.186 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.5 dump_historic_ops 2023-09-23T18:22:19.262 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:19.293 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:19.328 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:19.331 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:19.331 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:19.432 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:19.458 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:19.493 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:19.496 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:19.496 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:19.503 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:19.515 INFO:tasks.workunit.client.0.smithi055.stderr: Removing snap: 100% complete...done. 2023-09-23T18:22:19.528 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:22:19.529 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm testimg 2023-09-23T18:22:19.597 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:19.624 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:19.634 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T18:22:19.637 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --order 20 /tmp/rbd_import_export_463999/img testimg 2023-09-23T18:22:19.658 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:19.661 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:19.662 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:19.698 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T18:22:19.698 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:22:19.701 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_463999/img_v2 2023-09-23T18:22:19.741 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:19.744 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_463999/img_v2 testimg_import 2023-09-23T18:22:19.763 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:22:19.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:19 smithi055 ceph-mon[184174]: failed to encode map e641 with expected crc 2023-09-23T18:22:19.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:19 smithi055 ceph-mon[184174]: failed to encode map e642 with expected crc 2023-09-23T18:22:19.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:19 smithi055 ceph-mon[184174]: osdmap e642: 8 total, 8 up, 7 in 2023-09-23T18:22:19.801 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:19 smithi055 ceph-mon[184174]: failed to encode map e642 with expected crc 2023-09-23T18:22:19.801 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:19 smithi055 ceph-mon[179668]: failed to encode map e641 with expected crc 2023-09-23T18:22:19.802 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:19 smithi055 ceph-mon[179668]: failed to encode map e642 with expected crc 2023-09-23T18:22:19.802 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:19 smithi055 ceph-mon[179668]: osdmap e642: 8 total, 8 up, 7 in 2023-09-23T18:22:19.802 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:19 smithi055 ceph-mon[179668]: failed to encode map e642 with expected crc 2023-09-23T18:22:19.807 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:19.819 INFO:teuthology.orchestra.run.smithi099.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.b/config 2023-09-23T18:22:19.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:19 smithi099 ceph-mon[167076]: failed to encode map e641 with expected crc 2023-09-23T18:22:19.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:19 smithi099 ceph-mon[167076]: failed to encode map e642 with expected crc 2023-09-23T18:22:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:19 smithi099 ceph-mon[167076]: osdmap e642: 8 total, 8 up, 7 in 2023-09-23T18:22:19.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:19 smithi099 ceph-mon[167076]: failed to encode map e642 with expected crc 2023-09-23T18:22:19.833 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T18:22:19.838 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import 2023-09-23T18:22:19.838 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep order 2023-09-23T18:22:19.839 INFO:tasks.workunit.client.0.smithi055.stderr:+ awk '{print $2}' 2023-09-23T18:22:19.839 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 20 2023-09-23T18:22:19.880 INFO:tasks.workunit.client.0.smithi055.stdout:20 2023-09-23T18:22:19.881 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/img_v2 2023-09-23T18:22:19.881 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg_import 2023-09-23T18:22:19.896 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:22:19.900 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:19.901 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:22:19.978 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T18:22:19.987 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg 2023-09-23T18:22:20.002 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:20.031 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:20.109 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T18:22:20.112 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --image-feature layering /tmp/rbd_import_export_463999/img testimg 2023-09-23T18:22:20.165 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T18:22:20.168 INFO:tasks.workunit.client.0.smithi055.stderr:++ rbd info testimg 2023-09-23T18:22:20.168 INFO:tasks.workunit.client.0.smithi055.stderr:++ grep features 2023-09-23T18:22:20.206 INFO:tasks.workunit.client.0.smithi055.stderr:+ FEATURES_BEFORE=' features: layering 2023-09-23T18:22:20.206 INFO:tasks.workunit.client.0.smithi055.stderr: op_features: ' 2023-09-23T18:22:20.206 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_463999/img_v2 2023-09-23T18:22:20.244 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:20.247 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_463999/img_v2 testimg_import 2023-09-23T18:22:20.274 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:20.277 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:20.278 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:20.305 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T18:22:20.309 INFO:tasks.workunit.client.0.smithi055.stderr:++ rbd info testimg_import 2023-09-23T18:22:20.309 INFO:tasks.workunit.client.0.smithi055.stderr:++ grep features 2023-09-23T18:22:20.346 INFO:tasks.workunit.client.0.smithi055.stderr:+ FEATURES_AFTER=' features: layering 2023-09-23T18:22:20.346 INFO:tasks.workunit.client.0.smithi055.stderr: op_features: ' 2023-09-23T18:22:20.346 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' ' features: layering 2023-09-23T18:22:20.346 INFO:tasks.workunit.client.0.smithi055.stderr: op_features: ' '!=' ' features: layering 2023-09-23T18:22:20.346 INFO:tasks.workunit.client.0.smithi055.stderr: op_features: ' ']' 2023-09-23T18:22:20.347 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/img_v2 2023-09-23T18:22:20.347 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg_import 2023-09-23T18:22:20.359 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:22:20] "GET /metrics HTTP/1.1" 200 34230 "" "Prometheus/2.43.0" 2023-09-23T18:22:20.379 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:20.408 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:20.430 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T18:22:20.433 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg 2023-09-23T18:22:20.496 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:20.513 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:20.516 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:20.516 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:20.517 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T18:22:20.522 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T18:22:20.525 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --stripe-count 1000 --stripe-unit 4096 /tmp/rbd_import_export_463999/img testimg 2023-09-23T18:22:20.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:20 smithi099 ceph-mon[167076]: failed to encode map e642 with expected crc 2023-09-23T18:22:20.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:20 smithi099 ceph-mon[167076]: pgmap v1552: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 317 KiB/s rd, 85 KiB/s wr, 158 op/s 2023-09-23T18:22:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:20 smithi099 ceph-mon[167076]: osdmap e643: 8 total, 8 up, 7 in 2023-09-23T18:22:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:20 smithi099 ceph-mon[167076]: failed to encode map e643 with expected crc 2023-09-23T18:22:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:20 smithi099 ceph-mon[167076]: failed to encode map e643 with expected crc 2023-09-23T18:22:20.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:20 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/825364435' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:22:20.591 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T18:22:20.594 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_463999/img_v2 2023-09-23T18:22:20.617 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:20.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[179668]: failed to encode map e642 with expected crc 2023-09-23T18:22:20.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[179668]: pgmap v1552: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 317 KiB/s rd, 85 KiB/s wr, 158 op/s 2023-09-23T18:22:20.632 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[179668]: osdmap e643: 8 total, 8 up, 7 in 2023-09-23T18:22:20.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[179668]: failed to encode map e643 with expected crc 2023-09-23T18:22:20.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[179668]: failed to encode map e643 with expected crc 2023-09-23T18:22:20.633 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/825364435' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:22:20.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[184174]: failed to encode map e642 with expected crc 2023-09-23T18:22:20.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[184174]: pgmap v1552: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 317 KiB/s rd, 85 KiB/s wr, 158 op/s 2023-09-23T18:22:20.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[184174]: osdmap e643: 8 total, 8 up, 7 in 2023-09-23T18:22:20.633 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[184174]: failed to encode map e643 with expected crc 2023-09-23T18:22:20.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[184174]: failed to encode map e643 with expected crc 2023-09-23T18:22:20.634 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:20 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/825364435' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:22:20.639 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:20.643 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_463999/img_v2 testimg_import 2023-09-23T18:22:20.647 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:20.712 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T18:22:20.716 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import 2023-09-23T18:22:20.717 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 'stripe unit' 2023-09-23T18:22:20.717 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -Ei '(4 KiB|4096)' 2023-09-23T18:22:20.721 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:20.724 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:20.725 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:20.754 INFO:tasks.workunit.client.0.smithi055.stdout: stripe unit: 4 KiB 2023-09-23T18:22:20.754 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import 2023-09-23T18:22:20.755 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 'stripe count' 2023-09-23T18:22:20.755 INFO:tasks.workunit.client.0.smithi055.stderr:+ awk '{print $3}' 2023-09-23T18:22:20.755 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 1000 2023-09-23T18:22:20.793 INFO:tasks.workunit.client.0.smithi055.stdout:1000 2023-09-23T18:22:20.793 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/img_v2 2023-09-23T18:22:20.793 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg_import 2023-09-23T18:22:20.825 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:20.859 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:20.891 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T18:22:20.894 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg 2023-09-23T18:22:20.896 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:20.899 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:20.900 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:20.991 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 3% complete... Removing image: 7% complete... Removing image: 10% complete... Removing image: 14% complete... Removing image: 17% complete... Removing image: 21% complete... Removing image: 25% complete... Removing image: 28% complete... Removing image: 32% complete... Removing image: 35% complete... Removing image: 39% complete... Removing image: 42% complete... Removing image: 46% complete... Removing image: 50% complete... Removing image: 53% complete... Removing image: 57% complete... Removing image: 60% complete... Removing image: 64% complete... Removing image: 67% complete... Removing image: 71% complete... Removing image: 75% complete... Removing image: 78% complete... Removing image: 82% complete... Removing image: 85% complete... Removing image: 89% complete... Removing image: 92% complete... Removing image: 96% complete... Removing image: 100% complete...done. 2023-09-23T18:22:20.994 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --image-format=2 /tmp/rbd_import_export_463999/img testimg 2023-09-23T18:22:21.000 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:21.027 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:21.031 INFO:teuthology.orchestra.run.smithi099.stdout:ERROR: (22) Invalid argument 2023-09-23T18:22:21.031 INFO:teuthology.orchestra.run.smithi099.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:22:21.055 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T18:22:21.058 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap create testimg@snap1 2023-09-23T18:22:21.064 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:21.067 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:21.067 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:21.137 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T18:22:21.167 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:21.194 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:21.363 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_ops_in_flight 2023-09-23T18:22:21.366 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:21.369 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:21.370 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:21.470 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:21.500 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:21.523 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:22:21.542 INFO:tasks.workunit.client.0.smithi055.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T18:22:21.550 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap create testimg@snap2 2023-09-23T18:22:21.627 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:21.630 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:21.631 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:21.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[179668]: failed to encode map e643 with expected crc 2023-09-23T18:22:21.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/825364435' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:22:21.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[179668]: osdmap e644: 8 total, 8 up, 7 in 2023-09-23T18:22:21.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[179668]: failed to encode map e644 with expected crc 2023-09-23T18:22:21.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/138187645' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:21.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:21.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:21.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[184174]: failed to encode map e643 with expected crc 2023-09-23T18:22:21.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/825364435' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:22:21.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[184174]: osdmap e644: 8 total, 8 up, 7 in 2023-09-23T18:22:21.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[184174]: failed to encode map e644 with expected crc 2023-09-23T18:22:21.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/138187645' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:21.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:21.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:21.731 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:21.762 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:21 smithi099 ceph-mon[167076]: failed to encode map e643 with expected crc 2023-09-23T18:22:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/825364435' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:22:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:21 smithi099 ceph-mon[167076]: osdmap e644: 8 total, 8 up, 7 in 2023-09-23T18:22:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:21 smithi099 ceph-mon[167076]: failed to encode map e644 with expected crc 2023-09-23T18:22:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/138187645' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:21.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:21.947 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:21.949 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:21.949 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:22.051 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:22:22.085 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:22.125 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:22:22.129 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:22.129 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:22:22.230 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:22.261 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:22.297 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:22.299 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:22.300 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:22.389 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:22:22.397 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:22.398 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:22.401 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:22.410 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:22:22.437 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:22.439 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:22:22.463 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:22:22.483 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:22:22.505 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:22:22.525 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:22.527 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:22:22.529 INFO:tasks.workunit.client.0.smithi055.stderr: Creating snap: 10% complete... Creating snap: 100% complete...done. 2023-09-23T18:22:22.529 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:22.530 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:22.537 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap protect testimg@snap2 2023-09-23T18:22:22.549 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:22:22.553 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:22:22.553 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:22:22.554 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:22:22.554 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:22:22.554 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:22:22.554 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:22:22.554 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:22:22.554 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:22:22.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:22 smithi099 ceph-mon[167076]: failed to encode map e644 with expected crc 2023-09-23T18:22:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:22 smithi099 ceph-mon[167076]: pgmap v1555: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 73 KiB/s rd, 2.0 KiB/s wr, 35 op/s 2023-09-23T18:22:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:22 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:22:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:22 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:22:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/138187645' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:22 smithi099 ceph-mon[167076]: osdmap e645: 8 total, 8 up, 7 in 2023-09-23T18:22:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:22 smithi099 ceph-mon[167076]: failed to encode map e645 with expected crc 2023-09-23T18:22:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:22 smithi099 ceph-mon[167076]: failed to encode map e645 with expected crc 2023-09-23T18:22:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:22.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:22 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/833912480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:22.582 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export --export-format 2 testimg /tmp/rbd_import_export_463999/snap_protect 2023-09-23T18:22:22.623 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 33% complete... Exporting image: 100% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:22.626 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --export-format 2 /tmp/rbd_import_export_463999/snap_protect testimg_import 2023-09-23T18:22:22.631 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:22.658 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:22.694 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:22.697 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:22.698 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:22.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[179668]: failed to encode map e644 with expected crc 2023-09-23T18:22:22.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[179668]: pgmap v1555: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 73 KiB/s rd, 2.0 KiB/s wr, 35 op/s 2023-09-23T18:22:22.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:22:22.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:22:22.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/138187645' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:22.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[179668]: osdmap e645: 8 total, 8 up, 7 in 2023-09-23T18:22:22.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[179668]: failed to encode map e645 with expected crc 2023-09-23T18:22:22.739 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[179668]: failed to encode map e645 with expected crc 2023-09-23T18:22:22.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:22.740 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/833912480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:22.740 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[184174]: failed to encode map e644 with expected crc 2023-09-23T18:22:22.740 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[184174]: pgmap v1555: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 73 KiB/s rd, 2.0 KiB/s wr, 35 op/s 2023-09-23T18:22:22.740 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:22:22.740 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:22:22.741 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/138187645' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:22.741 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[184174]: osdmap e645: 8 total, 8 up, 7 in 2023-09-23T18:22:22.741 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[184174]: failed to encode map e645 with expected crc 2023-09-23T18:22:22.741 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[184174]: failed to encode map e645 with expected crc 2023-09-23T18:22:22.741 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:22.741 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:22 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/833912480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:22.798 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:22.825 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:22.858 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:22.861 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:22.861 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:22.962 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:22.991 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:23.377 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:23.380 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:23.380 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:23.481 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:23.511 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:23.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[179668]: failed to encode map e645 with expected crc 2023-09-23T18:22:23.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[179668]: osdmap e646: 8 total, 8 up, 7 in 2023-09-23T18:22:23.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[179668]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[179668]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[179668]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.526 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[179668]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.527 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[184174]: failed to encode map e645 with expected crc 2023-09-23T18:22:23.527 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[184174]: osdmap e646: 8 total, 8 up, 7 in 2023-09-23T18:22:23.527 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[184174]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.527 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[184174]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.527 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[184174]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.527 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:23 smithi055 ceph-mon[184174]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.551 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:23.554 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:23.554 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:23.655 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:23.685 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:23.719 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:23.722 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:23.722 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:23.823 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:23.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:23 smithi099 ceph-mon[167076]: failed to encode map e645 with expected crc 2023-09-23T18:22:23.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:23 smithi099 ceph-mon[167076]: osdmap e646: 8 total, 8 up, 7 in 2023-09-23T18:22:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:23 smithi099 ceph-mon[167076]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:23 smithi099 ceph-mon[167076]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:23 smithi099 ceph-mon[167076]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:23 smithi099 ceph-mon[167076]: failed to encode map e646 with expected crc 2023-09-23T18:22:23.851 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:24.021 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set noscrub 2023-09-23T18:22:24.244 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:24.244 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:22:24.244 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:22:24.244 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:22:24.244 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:22:24.247 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:24.248 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:24.349 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:24.389 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:24.426 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:24.429 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:24.429 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:24.503 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:24 smithi055 ceph-mon[184174]: failed to encode map e646 with expected crc 2023-09-23T18:22:24.530 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:24.552 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 33% complete... Importing image: 66% complete... Importing image: 100% complete... Importing image: 100% complete...done. 2023-09-23T18:22:24.555 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import@snap1 2023-09-23T18:22:24.555 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 'protected: False' 2023-09-23T18:22:24.567 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:24.600 INFO:tasks.workunit.client.0.smithi055.stdout: protected: False 2023-09-23T18:22:24.600 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd info testimg_import@snap2 2023-09-23T18:22:24.600 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep 'protected: True' 2023-09-23T18:22:24.641 INFO:tasks.workunit.client.0.smithi055.stdout: protected: True 2023-09-23T18:22:24.641 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/snap_protect 2023-09-23T18:22:24.642 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap unprotect testimg@snap2 2023-09-23T18:22:24.656 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:24.659 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:24.659 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:24.687 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap unprotect testimg_import@snap2 2023-09-23T18:22:24.733 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap purge testimg 2023-09-23T18:22:24.747 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_blocked_ops 2023-09-23T18:22:24.760 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:22:24.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:24 smithi055 ceph-mon[184174]: pgmap v1558: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 368 KiB/s rd, 45 KiB/s wr, 485 op/s 2023-09-23T18:22:24.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:24 smithi055 ceph-mon[184174]: osdmap e647: 8 total, 8 up, 7 in 2023-09-23T18:22:24.817 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:24 smithi055 ceph-mon[184174]: failed to encode map e647 with expected crc 2023-09-23T18:22:24.818 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:24 smithi055 ceph-mon[179668]: failed to encode map e646 with expected crc 2023-09-23T18:22:24.818 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:24 smithi055 ceph-mon[179668]: pgmap v1558: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 368 KiB/s rd, 45 KiB/s wr, 485 op/s 2023-09-23T18:22:24.818 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:24 smithi055 ceph-mon[179668]: osdmap e647: 8 total, 8 up, 7 in 2023-09-23T18:22:24.818 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:24 smithi055 ceph-mon[179668]: failed to encode map e647 with expected crc 2023-09-23T18:22:24.820 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:24.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:24 smithi099 ceph-mon[167076]: failed to encode map e646 with expected crc 2023-09-23T18:22:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:24 smithi099 ceph-mon[167076]: pgmap v1558: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 368 KiB/s rd, 45 KiB/s wr, 485 op/s 2023-09-23T18:22:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:24 smithi099 ceph-mon[167076]: osdmap e647: 8 total, 8 up, 7 in 2023-09-23T18:22:24.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:24 smithi099 ceph-mon[167076]: failed to encode map e647 with expected crc 2023-09-23T18:22:24.858 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:24.861 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:22:24.864 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:24.865 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:22:24.966 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:24.995 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:25.029 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:25.031 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:25.032 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:25.133 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:25.161 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:25.176 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=false 2023-09-23T18:22:25.195 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:25.198 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:25.198 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:25.299 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:25.332 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:25.548 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T18:22:25.650 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:25.653 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:25.656 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[184174]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[184174]: osdmap e648: 8 total, 8 up, 7 in 2023-09-23T18:22:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[184174]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[184174]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[184174]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3356324084' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:22:25.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[179668]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[179668]: osdmap e648: 8 total, 8 up, 7 in 2023-09-23T18:22:25.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[179668]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[179668]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[179668]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:25 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3356324084' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:22:25.754 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:25.757 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:25.798 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:25 smithi099 ceph-mon[167076]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:25 smithi099 ceph-mon[167076]: osdmap e648: 8 total, 8 up, 7 in 2023-09-23T18:22:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:25 smithi099 ceph-mon[167076]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:25 smithi099 ceph-mon[167076]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:25 smithi099 ceph-mon[167076]: failed to encode map e648 with expected crc 2023-09-23T18:22:25.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:25 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3356324084' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:22:25.867 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:25.870 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:25.870 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:25.971 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:25.998 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:26.032 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:26.034 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:26.035 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:26.135 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:26.162 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:26.194 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:26.197 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:26.197 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:26.298 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:26.324 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:26.357 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:26.359 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:26.360 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:26.460 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:26.488 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T18:22:26.497 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:26.529 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:26.570 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is set 2023-09-23T18:22:26.572 INFO:tasks.workunit.client.0.smithi055.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T18:22:26.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:26 smithi099 ceph-mon[167076]: pgmap v1561: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 439 KiB/s rd, 51 KiB/s wr, 581 op/s 2023-09-23T18:22:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:26 smithi099 ceph-mon[167076]: failed to encode map e649 with expected crc 2023-09-23T18:22:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3356324084' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:22:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:26 smithi099 ceph-mon[167076]: osdmap e649: 8 total, 8 up, 7 in 2023-09-23T18:22:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:26 smithi099 ceph-mon[167076]: failed to encode map e649 with expected crc 2023-09-23T18:22:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:26 smithi099 ceph-mon[167076]: failed to encode map e649 with expected crc 2023-09-23T18:22:26.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4023810569' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:22:26.582 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd snap purge testimg_import 2023-09-23T18:22:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[179668]: pgmap v1561: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 439 KiB/s rd, 51 KiB/s wr, 581 op/s 2023-09-23T18:22:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[179668]: failed to encode map e649 with expected crc 2023-09-23T18:22:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3356324084' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:22:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[179668]: osdmap e649: 8 total, 8 up, 7 in 2023-09-23T18:22:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[179668]: failed to encode map e649 with expected crc 2023-09-23T18:22:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[179668]: failed to encode map e649 with expected crc 2023-09-23T18:22:26.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4023810569' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:22:26.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[184174]: pgmap v1561: 103 pgs: 103 active+clean; 4.7 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 439 KiB/s rd, 51 KiB/s wr, 581 op/s 2023-09-23T18:22:26.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[184174]: failed to encode map e649 with expected crc 2023-09-23T18:22:26.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3356324084' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:22:26.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[184174]: osdmap e649: 8 total, 8 up, 7 in 2023-09-23T18:22:26.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[184174]: failed to encode map e649 with expected crc 2023-09-23T18:22:26.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[184174]: failed to encode map e649 with expected crc 2023-09-23T18:22:26.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4023810569' entity='client.admin' cmd=[{"prefix": "osd set", "key": "noscrub"}]: dispatch 2023-09-23T18:22:26.836 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:26.839 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:26.839 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:26.940 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:26.968 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:27.000 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:27.003 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:27.003 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:27.104 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:27.130 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:27.163 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:27.165 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:27.166 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:27.267 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:27.297 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:27.382 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:27.384 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:27.385 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:27.486 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:27.515 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:27.652 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:27.654 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:27 smithi055 ceph-mon[179668]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:27.654 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:27 smithi055 ceph-mon[179668]: failed to encode map e650 with expected crc 2023-09-23T18:22:27.654 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4023810569' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:22:27.654 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:27 smithi055 ceph-mon[179668]: osdmap e650: 8 total, 8 up, 7 in 2023-09-23T18:22:27.654 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:27 smithi055 ceph-mon[179668]: failed to encode map e650 with expected crc 2023-09-23T18:22:27.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:27 smithi055 ceph-mon[184174]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:27.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:27 smithi055 ceph-mon[184174]: failed to encode map e650 with expected crc 2023-09-23T18:22:27.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4023810569' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:22:27.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:27 smithi055 ceph-mon[184174]: osdmap e650: 8 total, 8 up, 7 in 2023-09-23T18:22:27.655 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:27 smithi055 ceph-mon[184174]: failed to encode map e650 with expected crc 2023-09-23T18:22:27.656 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:27.656 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:27.753 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:22:27.753 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:22:27.753 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:22:27.753 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:22:27.753 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:22:27.757 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:27.787 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:27 smithi099 ceph-mon[167076]: Health check failed: noscrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:27 smithi099 ceph-mon[167076]: failed to encode map e650 with expected crc 2023-09-23T18:22:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4023810569' entity='client.admin' cmd='[{"prefix": "osd set", "key": "noscrub"}]': finished 2023-09-23T18:22:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:27 smithi099 ceph-mon[167076]: osdmap e650: 8 total, 8 up, 7 in 2023-09-23T18:22:27.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:27 smithi099 ceph-mon[167076]: failed to encode map e650 with expected crc 2023-09-23T18:22:27.923 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:27.925 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:27.927 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:27.928 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:28.029 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:28.056 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:28.089 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:28.092 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:28.092 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:28.193 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3 2023-09-23T18:22:28.226 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.2 dump_historic_ops 2023-09-23T18:22:28.227 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:28.267 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3" found: no such container 2023-09-23T18:22:28.270 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:28.271 INFO:tasks.cephadm.osd.3:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.3' 2023-09-23T18:22:28.371 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0 2023-09-23T18:22:28.403 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:28.448 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0" found: no such container 2023-09-23T18:22:28.452 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:28.453 INFO:tasks.cephadm.osd.0:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.0' 2023-09-23T18:22:28.473 INFO:tasks.workunit.client.0.smithi055.stderr: Removing all snapshots: 50% complete... Removing all snapshots: 100% complete... Removing all snapshots: 100% complete...done. 2023-09-23T18:22:28.486 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg 2023-09-23T18:22:28.529 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:22:28.553 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:22:28.553 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:28.575 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:22:28.582 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:28.600 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:22:28.600 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T18:22:28.604 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd remove testimg_import 2023-09-23T18:22:28.618 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:28.620 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:22:28.621 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:28.622 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:28.641 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:22:28.663 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:22:28.687 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:22:28.688 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'false' 2023-09-23T18:22:28.688 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'false' 2023-09-23T18:22:28.688 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'false' 2023-09-23T18:22:28.688 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'false' 2023-09-23T18:22:28.689 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'false' 2023-09-23T18:22:28.689 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'false' 2023-09-23T18:22:28.689 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'false' 2023-09-23T18:22:28.689 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'false' 2023-09-23T18:22:28.698 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T18:22:28.702 INFO:tasks.workunit.client.0.smithi055.stderr:+ tiered=0 2023-09-23T18:22:28.702 INFO:tasks.workunit.client.0.smithi055.stderr:+ ceph osd dump 2023-09-23T18:22:28.702 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep '^pool' 2023-09-23T18:22:28.702 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep ''\''rbd'\''' 2023-09-23T18:22:28.702 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep tier 2023-09-23T18:22:28.723 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:28.749 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:28.781 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:28.784 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:28.784 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:28.885 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4 2023-09-23T18:22:28.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:28 smithi099 ceph-mon[167076]: pgmap v1564: 103 pgs: 103 active+clean; 4.8 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 236 KiB/s rd, 17 KiB/s wr, 307 op/s 2023-09-23T18:22:28.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:28 smithi099 ceph-mon[167076]: failed to encode map e651 with expected crc 2023-09-23T18:22:28.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:28 smithi099 ceph-mon[167076]: osdmap e651: 8 total, 8 up, 7 in 2023-09-23T18:22:28.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:28 smithi099 ceph-mon[167076]: failed to encode map e651 with expected crc 2023-09-23T18:22:28.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:28 smithi099 ceph-mon[167076]: osdmap e652: 8 total, 8 up, 7 in 2023-09-23T18:22:28.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:28 smithi099 ceph-mon[167076]: failed to encode map e652 with expected crc 2023-09-23T18:22:28.909 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:28 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2048702480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:28.911 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:28.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[179668]: pgmap v1564: 103 pgs: 103 active+clean; 4.8 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 236 KiB/s rd, 17 KiB/s wr, 307 op/s 2023-09-23T18:22:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[179668]: failed to encode map e651 with expected crc 2023-09-23T18:22:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[179668]: osdmap e651: 8 total, 8 up, 7 in 2023-09-23T18:22:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[179668]: failed to encode map e651 with expected crc 2023-09-23T18:22:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[179668]: osdmap e652: 8 total, 8 up, 7 in 2023-09-23T18:22:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[179668]: failed to encode map e652 with expected crc 2023-09-23T18:22:28.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2048702480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:28.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[184174]: pgmap v1564: 103 pgs: 103 active+clean; 4.8 MiB data, 5.0 GiB used, 621 GiB / 626 GiB avail; 236 KiB/s rd, 17 KiB/s wr, 307 op/s 2023-09-23T18:22:28.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[184174]: failed to encode map e651 with expected crc 2023-09-23T18:22:28.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[184174]: osdmap e651: 8 total, 8 up, 7 in 2023-09-23T18:22:28.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[184174]: failed to encode map e651 with expected crc 2023-09-23T18:22:28.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[184174]: osdmap e652: 8 total, 8 up, 7 in 2023-09-23T18:22:28.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[184174]: failed to encode map e652 with expected crc 2023-09-23T18:22:28.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:28 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2048702480' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:28.944 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4" found: no such container 2023-09-23T18:22:28.946 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:28.947 INFO:tasks.cephadm.osd.4:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.4' 2023-09-23T18:22:29.047 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:29.073 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:29.106 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:29.108 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:29.109 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:29.210 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:29.219 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd set nodeep-scrub 2023-09-23T18:22:29.241 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/dev/urandom bs=1M seek=1 count=1 of=/tmp/rbd_import_export_463999/sparse1 2023-09-23T18:22:29.244 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:29.248 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records in 2023-09-23T18:22:29.248 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records out 2023-09-23T18:22:29.248 INFO:tasks.workunit.client.0.smithi055.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00557519 s, 188 MB/s 2023-09-23T18:22:29.248 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/dev/urandom bs=1M count=1 of=/tmp/rbd_import_export_463999/sparse2 2023-09-23T18:22:29.255 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records in 2023-09-23T18:22:29.255 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records out 2023-09-23T18:22:29.255 INFO:tasks.workunit.client.0.smithi055.stderr:1048576 bytes (1.0 MB, 1.0 MiB) copied, 0.00575482 s, 182 MB/s 2023-09-23T18:22:29.255 INFO:tasks.workunit.client.0.smithi055.stderr:+ truncate /tmp/rbd_import_export_463999/sparse2 -s 2M 2023-09-23T18:22:29.256 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse1 2023-09-23T18:22:29.300 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 0% complete...failed. 2023-09-23T18:22:29.300 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T18:22:29.303 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T18:22:29.303 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_463999/sparse1 2023-09-23T18:22:29.373 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T18:22:29.374 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:22:29.374 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:22:29.379 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd ls -l 2023-09-23T18:22:29.379 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T18:22:29.379 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep sparse1 2023-09-23T18:22:29.382 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:29.387 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:29.388 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:29.417 INFO:tasks.workunit.client.0.smithi055.stdout:sparse1 2 MiB 2 2023-09-23T18:22:29.418 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse1 2023-09-23T18:22:29.418 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse1 2023-09-23T18:22:29.418 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse1 2023-09-23T18:22:29.418 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T18:22:29.418 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T18:22:29.457 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.ba3b2693a7c0 2023-09-23T18:22:29.457 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.ba3b2693a7c0 2023-09-23T18:22:29.457 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse1 2023-09-23T18:22:29.457 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse1 2023-09-23T18:22:29.457 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.ba3b2693a7c0\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T18:22:29.458 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T18:22:29.458 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse1 2023-09-23T18:22:29.459 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T18:22:29.459 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T18:22:29.462 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:29.489 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2 2023-09-23T18:22:29.517 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T18:22:29.517 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T18:22:29.517 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T18:22:29.518 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T18:22:29.518 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T18:22:29.527 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:29.559 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects=1 2023-09-23T18:22:29.559 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 1 2023-09-23T18:22:29.559 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T18:22:29.559 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export sparse1 /tmp/rbd_import_export_463999/sparse1.out 2023-09-23T18:22:29.598 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:29.602 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_463999/sparse1 /tmp/rbd_import_export_463999/sparse1.out 2023-09-23T18:22:29.602 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_463999/sparse1 /tmp/rbd_import_export_463999/sparse1.out 2023-09-23T18:22:29.604 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/sparse1 --format %b 2023-09-23T18:22:29.605 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=2048 2023-09-23T18:22:29.606 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/sparse1.out --format %b 2023-09-23T18:22:29.609 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=2048 2023-09-23T18:22:29.610 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T18:22:29.610 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T18:22:29.610 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T18:22:29.610 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/sparse1.out 2023-09-23T18:22:29.610 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse1 2023-09-23T18:22:29.702 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T18:22:29.704 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse2 2023-09-23T18:22:29.744 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 0% complete...failed. 2023-09-23T18:22:29.744 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: delete error: (2) No such file or directory 2023-09-23T18:22:29.747 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T18:22:29.747 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_463999/sparse2 2023-09-23T18:22:29.819 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 50% complete... Importing image: 100% complete...done. 2023-09-23T18:22:29.820 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:22:29.820 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:22:29.822 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd ls -l 2023-09-23T18:22:29.823 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep sparse2 2023-09-23T18:22:29.823 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -Ei '(2 MiB|2048k)' 2023-09-23T18:22:29.861 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:29 smithi055 ceph-mon[184174]: failed to encode map e652 with expected crc 2023-09-23T18:22:29.861 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:29 smithi055 ceph-mon[184174]: failed to encode map e652 with expected crc 2023-09-23T18:22:29.861 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1533620492' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:22:29.861 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1533620492' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:22:29.861 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:29 smithi055 ceph-mon[179668]: failed to encode map e652 with expected crc 2023-09-23T18:22:29.861 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:29 smithi055 ceph-mon[179668]: failed to encode map e652 with expected crc 2023-09-23T18:22:29.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1533620492' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:22:29.862 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1533620492' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:22:29.868 INFO:tasks.workunit.client.0.smithi055.stdout:sparse2 2 MiB 2 2023-09-23T18:22:29.869 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse2 2023-09-23T18:22:29.869 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse2 2023-09-23T18:22:29.869 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse2 2023-09-23T18:22:29.870 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T18:22:29.870 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T18:22:29.911 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.ba68d6bbc867 2023-09-23T18:22:29.912 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.ba68d6bbc867 2023-09-23T18:22:29.912 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse2 2023-09-23T18:22:29.912 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse2 2023-09-23T18:22:29.912 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.ba68d6bbc867\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T18:22:29.912 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T18:22:29.912 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse2 2023-09-23T18:22:29.913 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T18:22:29.913 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T18:22:29.922 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2" found: no such container 2023-09-23T18:22:29.926 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:29.926 INFO:tasks.cephadm.osd.2:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.2' 2023-09-23T18:22:29.950 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T18:22:29.950 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T18:22:29.950 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T18:22:29.950 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T18:22:29.950 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T18:22:29.983 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects=0 2023-09-23T18:22:29.983 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 0 2023-09-23T18:22:29.984 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T18:22:29.984 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export sparse2 /tmp/rbd_import_export_463999/sparse2.out 2023-09-23T18:22:30.021 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 50% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:30.025 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_463999/sparse2 /tmp/rbd_import_export_463999/sparse2.out 2023-09-23T18:22:30.025 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_463999/sparse2 /tmp/rbd_import_export_463999/sparse2.out 2023-09-23T18:22:30.027 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:30.028 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/sparse2 --format %b 2023-09-23T18:22:30.028 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=2048 2023-09-23T18:22:30.028 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/sparse2.out --format %b 2023-09-23T18:22:30.029 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=2048 2023-09-23T18:22:30.030 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T18:22:30.030 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T18:22:30.030 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T18:22:30.030 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/sparse2.out 2023-09-23T18:22:30.031 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse2 2023-09-23T18:22:30.052 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:29 smithi099 ceph-mon[167076]: failed to encode map e652 with expected crc 2023-09-23T18:22:30.053 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:29 smithi099 ceph-mon[167076]: failed to encode map e652 with expected crc 2023-09-23T18:22:30.053 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1533620492' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:22:30.053 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1533620492' entity='client.admin' cmd=[{"prefix": "osd dump"}]: dispatch 2023-09-23T18:22:30.055 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:30.090 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:30.093 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:30.093 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:30.140 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 50% complete... Removing image: 100% complete...done. 2023-09-23T18:22:30.143 INFO:tasks.workunit.client.0.smithi055.stderr:+ truncate /tmp/rbd_import_export_463999/sparse1 -s 10M 2023-09-23T18:22:30.144 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format --order 20 - sparse1 2023-09-23T18:22:30.194 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:30.220 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:30.236 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T18:22:30.237 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:22:30.237 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:22:30.240 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd ls -l 2023-09-23T18:22:30.240 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep sparse1 2023-09-23T18:22:30.240 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -Ei '(10 MiB|10240k)' 2023-09-23T18:22:30.254 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:30.256 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:30.257 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:30.275 INFO:tasks.workunit.client.0.smithi055.stdout:sparse1 10 MiB 2 2023-09-23T18:22:30.276 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse1 2023-09-23T18:22:30.276 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse1 2023-09-23T18:22:30.276 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse1 2023-09-23T18:22:30.277 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T18:22:30.277 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T18:22:30.314 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.ba9b6f4a181c 2023-09-23T18:22:30.315 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.ba9b6f4a181c 2023-09-23T18:22:30.315 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse1 2023-09-23T18:22:30.316 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse1 2023-09-23T18:22:30.316 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.ba9b6f4a181c\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T18:22:30.316 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T18:22:30.316 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse1 2023-09-23T18:22:30.317 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T18:22:30.317 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T18:22:30.354 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T18:22:30.354 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T18:22:30.354 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T18:22:30.354 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T18:22:30.354 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T18:22:30.357 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:30.383 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:30.390 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects=1 2023-09-23T18:22:30.390 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 1 2023-09-23T18:22:30.390 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T18:22:30.390 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export sparse1 /tmp/rbd_import_export_463999/sparse1.out 2023-09-23T18:22:30.396 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:30.414 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:30.417 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:30.417 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:30.439 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 10% complete... Exporting image: 20% complete... Exporting image: 30% complete... Exporting image: 40% complete... Exporting image: 50% complete... Exporting image: 60% complete... Exporting image: 70% complete... Exporting image: 80% complete... Exporting image: 90% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:30.443 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_463999/sparse1 /tmp/rbd_import_export_463999/sparse1.out 2023-09-23T18:22:30.443 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_463999/sparse1 /tmp/rbd_import_export_463999/sparse1.out 2023-09-23T18:22:30.451 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/sparse1 --format %b 2023-09-23T18:22:30.453 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=2048 2023-09-23T18:22:30.453 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/sparse1.out --format %b 2023-09-23T18:22:30.454 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=2048 2023-09-23T18:22:30.454 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T18:22:30.454 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T18:22:30.454 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T18:22:30.454 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/sparse1.out 2023-09-23T18:22:30.455 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse1 2023-09-23T18:22:30.518 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:30.544 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:30.552 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T18:22:30.555 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/dev/urandom bs=2M count=1 of=/tmp/rbd_import_export_463999/sparse2 oflag=append conv=notrunc 2023-09-23T18:22:30.567 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records in 2023-09-23T18:22:30.567 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records out 2023-09-23T18:22:30.567 INFO:tasks.workunit.client.0.smithi055.stderr:2097152 bytes (2.1 MB, 2.0 MiB) copied, 0.0106303 s, 197 MB/s 2023-09-23T18:22:30.567 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format --order 20 - sparse2 2023-09-23T18:22:30.576 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:30.579 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:30.580 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:30.604 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T18:22:30.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:30 smithi055 ceph-mon[184174]: pgmap v1567: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 150 KiB/s rd, 11 KiB/s wr, 195 op/s 2023-09-23T18:22:30.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3993218360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:22:30.623 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:30 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:22:30.623 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:22:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:22:30] "GET /metrics HTTP/1.1" 200 34223 "" "Prometheus/2.43.0" 2023-09-23T18:22:30.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:30 smithi055 ceph-mon[179668]: pgmap v1567: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 150 KiB/s rd, 11 KiB/s wr, 195 op/s 2023-09-23T18:22:30.623 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3993218360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:22:30.624 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:30 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:22:30.680 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:30.681 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T18:22:30.684 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:22:30.684 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:22:30.687 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd ls -l 2023-09-23T18:22:30.687 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep sparse2 2023-09-23T18:22:30.687 INFO:tasks.workunit.client.0.smithi055.stderr:+ grep -Ei '(4 MiB|4096k)' 2023-09-23T18:22:30.715 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:30.728 INFO:tasks.workunit.client.0.smithi055.stdout:sparse2 4 MiB 2 2023-09-23T18:22:30.729 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse2 2023-09-23T18:22:30.729 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse2 2023-09-23T18:22:30.729 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse2 2023-09-23T18:22:30.729 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T18:22:30.730 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T18:22:30.772 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.94a458d39288 2023-09-23T18:22:30.772 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.94a458d39288 2023-09-23T18:22:30.773 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse2 2023-09-23T18:22:30.773 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.94a458d39288\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T18:22:30.773 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse2 2023-09-23T18:22:30.773 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T18:22:30.773 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse2 2023-09-23T18:22:30.774 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T18:22:30.774 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T18:22:30.815 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T18:22:30.816 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T18:22:30.816 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T18:22:30.816 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T18:22:30.816 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T18:22:30.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:30 smithi099 ceph-mon[167076]: pgmap v1567: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 150 KiB/s rd, 11 KiB/s wr, 195 op/s 2023-09-23T18:22:30.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3993218360' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:22:30.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:30 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:22:30.854 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects='0 2023-09-23T18:22:30.854 INFO:tasks.workunit.client.0.smithi055.stderr:2 2023-09-23T18:22:30.854 INFO:tasks.workunit.client.0.smithi055.stderr:3' 2023-09-23T18:22:30.854 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 0 2 3 2023-09-23T18:22:30.854 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o '0 2 3' = '0 2 3' ']' 2023-09-23T18:22:30.854 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export sparse2 /tmp/rbd_import_export_463999/sparse2.out 2023-09-23T18:22:30.868 INFO:teuthology.orchestra.run.smithi055.stdout:ERROR: (22) Invalid argument 2023-09-23T18:22:30.868 INFO:teuthology.orchestra.run.smithi055.stdout:op_tracker tracking is not enabled now, so no ops are tracked currently, even those get stuck. Please enable "osd_enable_op_tracker", and the tracker will start to track new ops received afterwards. 2023-09-23T18:22:30.893 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 25% complete... Exporting image: 50% complete... Exporting image: 75% complete... Exporting image: 100% complete...done. 2023-09-23T18:22:30.897 INFO:tasks.workunit.client.0.smithi055.stderr:+ compare_files_and_ondisk_sizes /tmp/rbd_import_export_463999/sparse2 /tmp/rbd_import_export_463999/sparse2.out 2023-09-23T18:22:30.897 INFO:tasks.workunit.client.0.smithi055.stderr:+ cmp -l /tmp/rbd_import_export_463999/sparse2 /tmp/rbd_import_export_463999/sparse2.out 2023-09-23T18:22:30.900 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/sparse2 --format %b 2023-09-23T18:22:30.901 INFO:tasks.workunit.client.0.smithi055.stderr:+ origsize=6144 2023-09-23T18:22:30.902 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/sparse2.out --format %b 2023-09-23T18:22:30.903 INFO:tasks.workunit.client.0.smithi055.stderr:+ exportsize=6144 2023-09-23T18:22:30.903 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T18:22:30.903 INFO:tasks.workunit.client.0.smithi055.stderr:+ difference=0 2023-09-23T18:22:30.903 INFO:tasks.workunit.client.0.smithi055.stderr:+ test 0 -ge 0 -a 0 -lt 4096 2023-09-23T18:22:30.903 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/sparse2.out 2023-09-23T18:22:30.904 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse2 2023-09-23T18:22:30.999 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 25% complete... Removing image: 50% complete... Removing image: 75% complete... Removing image: 100% complete...done. 2023-09-23T18:22:31.002 INFO:tasks.workunit.client.0.smithi055.stderr:+ echo 'partially-sparse file imports to partially-sparse image' 2023-09-23T18:22:31.003 INFO:tasks.workunit.client.0.smithi055.stdout:partially-sparse file imports to partially-sparse image 2023-09-23T18:22:31.003 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format --order 20 /tmp/rbd_import_export_463999/sparse1 sparse 2023-09-23T18:22:31.067 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 10% complete... Importing image: 20% complete... Importing image: 30% complete... Importing image: 40% complete... Importing image: 50% complete... Importing image: 60% complete... Importing image: 70% complete... Importing image: 80% complete... Importing image: 90% complete... Importing image: 100% complete...done. 2023-09-23T18:22:31.068 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:22:31.068 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --order is deprecated, use --object-size 2023-09-23T18:22:31.071 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse 2023-09-23T18:22:31.071 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse 2023-09-23T18:22:31.072 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse 2023-09-23T18:22:31.072 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T18:22:31.072 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T18:22:31.113 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.baf5d57784df 2023-09-23T18:22:31.114 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.baf5d57784df 2023-09-23T18:22:31.114 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse 2023-09-23T18:22:31.114 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse 2023-09-23T18:22:31.114 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.baf5d57784df\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T18:22:31.115 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T18:22:31.115 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse 2023-09-23T18:22:31.115 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T18:22:31.116 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T18:22:31.151 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T18:22:31.152 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T18:22:31.152 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T18:22:31.152 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T18:22:31.152 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T18:22:31.187 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects=1 2023-09-23T18:22:31.187 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 1 2023-09-23T18:22:31.187 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o 1 = 1 ']' 2023-09-23T18:22:31.187 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse 2023-09-23T18:22:31.202 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T18:22:31.279 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 10% complete... Removing image: 20% complete... Removing image: 30% complete... Removing image: 40% complete... Removing image: 50% complete... Removing image: 60% complete... Removing image: 70% complete... Removing image: 80% complete... Removing image: 90% complete... Removing image: 100% complete...done. 2023-09-23T18:22:31.282 INFO:tasks.workunit.client.0.smithi055.stdout:zeros import through stdin to sparse image 2023-09-23T18:22:31.282 INFO:tasks.workunit.client.0.smithi055.stderr:+ echo 'zeros import through stdin to sparse image' 2023-09-23T18:22:31.283 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/dev/zero bs=1M count=4 2023-09-23T18:22:31.283 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd import --new-format - sparse 2023-09-23T18:22:31.292 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:31.296 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:31.296 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:31.331 INFO:tasks.workunit.client.0.smithi055.stderr:4+0 records in 2023-09-23T18:22:31.331 INFO:tasks.workunit.client.0.smithi055.stderr:4+0 records out 2023-09-23T18:22:31.331 INFO:tasks.workunit.client.0.smithi055.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0481218 s, 87.2 MB/s 2023-09-23T18:22:31.342 INFO:tasks.workunit.client.0.smithi055.stderr: Importing image: 100% complete...done. 2023-09-23T18:22:31.343 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:22:31.346 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse 2023-09-23T18:22:31.346 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse 2023-09-23T18:22:31.346 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse 2023-09-23T18:22:31.347 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T18:22:31.347 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T18:22:31.385 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.94f2a4bcb121 2023-09-23T18:22:31.386 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.94f2a4bcb121 2023-09-23T18:22:31.386 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse 2023-09-23T18:22:31.386 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse 2023-09-23T18:22:31.386 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.94f2a4bcb121\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T18:22:31.387 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T18:22:31.389 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse 2023-09-23T18:22:31.389 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T18:22:31.389 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T18:22:31.397 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:31.427 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:31.428 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T18:22:31.428 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T18:22:31.428 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T18:22:31.429 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T18:22:31.429 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T18:22:31.477 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:31.478 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects= 2023-09-23T18:22:31.478 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 2023-09-23T18:22:31.478 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o '' = '' ']' 2023-09-23T18:22:31.478 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse 2023-09-23T18:22:31.480 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:31.480 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:31.504 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'osd.*' injectargs --osd_enable_op_tracker=true 2023-09-23T18:22:31.546 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_ops_in_flight 2023-09-23T18:22:31.578 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T18:22:31.581 INFO:tasks.workunit.client.0.smithi055.stderr:+ echo 'zeros export to sparse file' 2023-09-23T18:22:31.581 INFO:tasks.workunit.client.0.smithi055.stdout:zeros export to sparse file 2023-09-23T18:22:31.581 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd create --new-format sparse --size 4 2023-09-23T18:22:31.581 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:31.609 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:31.640 INFO:tasks.workunit.client.0.smithi055.stderr:rbd: --new-format is deprecated, use --image-format 2 2023-09-23T18:22:31.643 INFO:tasks.workunit.client.0.smithi055.stderr:++ rbd info sparse 2023-09-23T18:22:31.643 INFO:tasks.workunit.client.0.smithi055.stderr:++ grep block_name_prefix 2023-09-23T18:22:31.644 INFO:tasks.workunit.client.0.smithi055.stderr:++ awk '{print $NF;}' 2023-09-23T18:22:31.645 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:31.648 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:31.648 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:31.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[184174]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:22:31.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[184174]: osdmap e653: 8 total, 8 up, 7 in 2023-09-23T18:22:31.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[184174]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[184174]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[184174]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[184174]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[179668]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:22:31.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[179668]: osdmap e653: 8 total, 8 up, 7 in 2023-09-23T18:22:31.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[179668]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[179668]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[179668]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:31 smithi055 ceph-mon[179668]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.686 INFO:tasks.workunit.client.0.smithi055.stderr:+ prefix=rbd_data.e06283f0087d 2023-09-23T18:22:31.686 INFO:tasks.workunit.client.0.smithi055.stderr:+ dd if=/dev/zero bs=4M count=1 2023-09-23T18:22:31.687 INFO:tasks.workunit.client.0.smithi055.stderr:++ get_image_data_pool sparse 2023-09-23T18:22:31.687 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse 2023-09-23T18:22:31.687 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse 2023-09-23T18:22:31.687 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep 'data_pool: ' 2023-09-23T18:22:31.687 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk -F: '{ print $NF }' 2023-09-23T18:22:31.733 INFO:tasks.workunit.client.0.smithi055.stderr:++ data_pool= 2023-09-23T18:22:31.733 INFO:tasks.workunit.client.0.smithi055.stderr:++ '[' -z ']' 2023-09-23T18:22:31.733 INFO:tasks.workunit.client.0.smithi055.stderr:++ data_pool=rbd 2023-09-23T18:22:31.733 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo rbd 2023-09-23T18:22:31.733 INFO:tasks.workunit.client.0.smithi055.stderr:+ rados -p rbd put rbd_data.e06283f0087d.000000000000 - 2023-09-23T18:22:31.749 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:31.766 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records in 2023-09-23T18:22:31.766 INFO:tasks.workunit.client.0.smithi055.stderr:1+0 records out 2023-09-23T18:22:31.766 INFO:tasks.workunit.client.0.smithi055.stderr:4194304 bytes (4.2 MB, 4.0 MiB) copied, 0.0787157 s, 53.3 MB/s 2023-09-23T18:22:31.778 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:31.786 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects sparse 2023-09-23T18:22:31.786 INFO:tasks.workunit.client.0.smithi055.stderr:++ image=sparse 2023-09-23T18:22:31.787 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rbd info sparse 2023-09-23T18:22:31.787 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep block_name_prefix 2023-09-23T18:22:31.787 INFO:tasks.workunit.client.0.smithi055.stderr:+++ awk '{print $NF;}' 2023-09-23T18:22:31.811 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:31.814 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:31.814 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:31.834 INFO:tasks.workunit.client.0.smithi055.stderr:++ prefix=rbd_data.e06283f0087d 2023-09-23T18:22:31.834 INFO:tasks.workunit.client.0.smithi055.stderr:+++ grep rbd_data.e06283f0087d 2023-09-23T18:22:31.834 INFO:tasks.workunit.client.0.smithi055.stderr:++++ get_image_data_pool sparse 2023-09-23T18:22:31.835 INFO:tasks.workunit.client.0.smithi055.stderr:++++ image=sparse 2023-09-23T18:22:31.835 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sort -u 2023-09-23T18:22:31.835 INFO:tasks.workunit.client.0.smithi055.stderr:+++ sed -e 's/rbd_data.e06283f0087d\.//' -e 's/^0*\([0-9a-f]\)/\1/' 2023-09-23T18:22:31.837 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ grep 'data_pool: ' 2023-09-23T18:22:31.840 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ awk -F: '{ print $NF }' 2023-09-23T18:22:31.840 INFO:tasks.workunit.client.0.smithi055.stderr:+++++ rbd info sparse 2023-09-23T18:22:31.880 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool= 2023-09-23T18:22:31.881 INFO:tasks.workunit.client.0.smithi055.stderr:++++ '[' -z ']' 2023-09-23T18:22:31.881 INFO:tasks.workunit.client.0.smithi055.stderr:++++ data_pool=rbd 2023-09-23T18:22:31.881 INFO:tasks.workunit.client.0.smithi055.stderr:++++ echo rbd 2023-09-23T18:22:31.881 INFO:tasks.workunit.client.0.smithi055.stderr:+++ rados ls -p rbd 2023-09-23T18:22:31.915 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:31.927 INFO:tasks.workunit.client.0.smithi055.stderr:++ objects=0 2023-09-23T18:22:31.927 INFO:tasks.workunit.client.0.smithi055.stderr:++ echo 0 2023-09-23T18:22:31.927 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 -eq 1 -o 0 = 0 ']' 2023-09-23T18:22:31.927 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/sparse 2023-09-23T18:22:31.928 INFO:tasks.workunit.client.0.smithi055.stderr:rm: cannot remove '/tmp/rbd_import_export_463999/sparse': No such file or directory 2023-09-23T18:22:31.928 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T18:22:31.928 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd export sparse /tmp/rbd_import_export_463999/sparse 2023-09-23T18:22:31.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:31 smithi099 ceph-mon[167076]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:31 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:22:31.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:31 smithi099 ceph-mon[167076]: osdmap e653: 8 total, 8 up, 7 in 2023-09-23T18:22:31.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:31 smithi099 ceph-mon[167076]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:31 smithi099 ceph-mon[167076]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.938 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:31 smithi099 ceph-mon[167076]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.939 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:31 smithi099 ceph-mon[167076]: failed to encode map e653 with expected crc 2023-09-23T18:22:31.941 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:31.964 INFO:tasks.workunit.client.0.smithi055.stderr: Exporting image: 100% complete...done. 2023-09-23T18:22:31.968 INFO:tasks.workunit.client.0.smithi055.stderr:++ stat /tmp/rbd_import_export_463999/sparse --format=%b 2023-09-23T18:22:31.969 INFO:tasks.workunit.client.0.smithi055.stderr:+ '[' 0 = 0 ']' 2023-09-23T18:22:31.969 INFO:tasks.workunit.client.0.smithi055.stderr:+ rbd rm sparse 2023-09-23T18:22:31.973 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:31.976 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:31.976 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:32.072 INFO:tasks.workunit.client.0.smithi055.stderr: Removing image: 100% complete...done. 2023-09-23T18:22:32.075 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm /tmp/rbd_import_export_463999/sparse /tmp/rbd_import_export_463999/sparse1 /tmp/rbd_import_export_463999/sparse2 /tmp/rbd_import_export_463999/sparse3 2023-09-23T18:22:32.077 DEBUG:teuthology.orchestra.run.smithi055:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1 2023-09-23T18:22:32.078 INFO:tasks.workunit.client.0.smithi055.stderr:rm: cannot remove '/tmp/rbd_import_export_463999/sparse3': No such file or directory 2023-09-23T18:22:32.078 INFO:tasks.workunit.client.0.smithi055.stderr:+ true 2023-09-23T18:22:32.079 INFO:tasks.workunit.client.0.smithi055.stdout:OK 2023-09-23T18:22:32.080 INFO:tasks.workunit.client.0.smithi055.stderr:+ echo OK 2023-09-23T18:22:32.080 INFO:tasks.workunit.client.0.smithi055.stderr:+ rm -rf /tmp/rbd_import_export_463999 2023-09-23T18:22:32.080 INFO:teuthology.orchestra.run:Running command with timeout 3600 2023-09-23T18:22:32.080 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0/tmp 2023-09-23T18:22:32.098 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:32.113 INFO:teuthology.orchestra.run.smithi055.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:32.133 INFO:tasks.workunit:Stopping ['rbd/import_export.sh'] on client.0... 2023-09-23T18:22:32.134 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -rf -- /home/ubuntu/cephtest/workunits.list.client.0 /home/ubuntu/cephtest/clone.client.0 2023-09-23T18:22:32.202 INFO:teuthology.orchestra.run.smithi055.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1" found: no such container 2023-09-23T18:22:32.206 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:32.206 INFO:tasks.cephadm.osd.1:Ignoring exception while sending signal: Command failed on smithi055 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.1' 2023-09-23T18:22:32.307 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7 2023-09-23T18:22:32.333 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:32.366 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7" found: no such container 2023-09-23T18:22:32.368 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:32.369 INFO:tasks.cephadm.osd.7:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.7' 2023-09-23T18:22:32.428 DEBUG:teuthology.parallel:result is None 2023-09-23T18:22:32.428 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -rf -- /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T18:22:32.463 INFO:tasks.workunit:Deleted dir /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T18:22:32.463 DEBUG:teuthology.orchestra.run.smithi055:> rmdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:22:32.469 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5 2023-09-23T18:22:32.495 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:32.505 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:32.522 INFO:tasks.workunit:Deleted artificial mount point /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T18:22:32.522 INFO:teuthology.task.sequential:In sequential, running task print... 2023-09-23T18:22:32.523 INFO:teuthology.task.print:**** done rbd/import_export.sh 5-workload 2023-09-23T18:22:32.523 INFO:teuthology.task.sequential:In sequential, running task workunit... 2023-09-23T18:22:32.523 INFO:tasks.workunit:Pulling workunits from ref quincy 2023-09-23T18:22:32.523 INFO:tasks.workunit:Making a separate scratch dir for every client... 2023-09-23T18:22:32.523 DEBUG:teuthology.orchestra.run.smithi055:> stat -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:22:32.527 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5" found: no such container 2023-09-23T18:22:32.530 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:32.531 INFO:tasks.cephadm.osd.5:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.5' 2023-09-23T18:22:32.581 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:22:32.582 INFO:teuthology.orchestra.run.smithi055.stderr:stat: cannot statx '/home/ubuntu/cephtest/mnt.0': No such file or directory 2023-09-23T18:22:32.582 DEBUG:teuthology.orchestra.run.smithi055:> mkdir -- /home/ubuntu/cephtest/mnt.0 2023-09-23T18:22:32.630 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is set 2023-09-23T18:22:32.631 DEBUG:teuthology.orchestra.run.smithi099:> sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6 2023-09-23T18:22:32.648 INFO:tasks.workunit:Created dir /home/ubuntu/cephtest/mnt.0 2023-09-23T18:22:32.648 DEBUG:teuthology.orchestra.run.smithi055:> cd -- /home/ubuntu/cephtest/mnt.0 && mkdir -- client.0 2023-09-23T18:22:32.657 INFO:teuthology.orchestra.run.smithi099.stderr:Emulate Docker CLI using podman. Create /etc/containers/nodocker to quiet msg. 2023-09-23T18:22:32.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:32 smithi055 ceph-mon[179668]: pgmap v1569: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 28 KiB/s rd, 1.5 KiB/s wr, 40 op/s 2023-09-23T18:22:32.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:32 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:22:32.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:32 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3309260744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:32.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:32 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:32.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:32 smithi055 ceph-mon[184174]: pgmap v1569: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 28 KiB/s rd, 1.5 KiB/s wr, 40 op/s 2023-09-23T18:22:32.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:32 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:22:32.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:32 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3309260744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:32.682 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:32 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:32.688 INFO:teuthology.orchestra.run.smithi099.stderr:Error: no container with name or ID "ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6" found: no such container 2023-09-23T18:22:32.691 DEBUG:teuthology.orchestra.run:got remote process result: 125 2023-09-23T18:22:32.692 INFO:tasks.cephadm.osd.6:Ignoring exception while sending signal: Command failed on smithi099 with status 125: 'sudo docker kill -s 1 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd.6' 2023-09-23T18:22:32.701 INFO:tasks.thrashosds:joining thrashosds 2023-09-23T18:22:33.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:32 smithi099 ceph-mon[167076]: pgmap v1569: 103 pgs: 103 active+clean; 4.8 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 28 KiB/s rd, 1.5 KiB/s wr, 40 op/s 2023-09-23T18:22:33.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:32 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:22:33.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:32 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3309260744' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:33.079 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:32 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd set", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:33.410 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:33.853 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[179668]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:33.853 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[179668]: failed to encode map e654 with expected crc 2023-09-23T18:22:33.853 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:33.853 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[179668]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T18:22:33.853 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[179668]: failed to encode map e654 with expected crc 2023-09-23T18:22:33.853 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[179668]: failed to encode map e654 with expected crc 2023-09-23T18:22:33.853 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[179668]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T18:22:33.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[184174]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:33.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[184174]: failed to encode map e654 with expected crc 2023-09-23T18:22:33.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:33.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[184174]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T18:22:33.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[184174]: failed to encode map e654 with expected crc 2023-09-23T18:22:33.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[184174]: failed to encode map e654 with expected crc 2023-09-23T18:22:33.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:33 smithi055 ceph-mon[184174]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T18:22:34.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:33 smithi099 ceph-mon[167076]: Health check update: noscrub,nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:34.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:33 smithi099 ceph-mon[167076]: failed to encode map e654 with expected crc 2023-09-23T18:22:34.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:33 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd set", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:34.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:33 smithi099 ceph-mon[167076]: osdmap e654: 8 total, 8 up, 7 in 2023-09-23T18:22:34.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:33 smithi099 ceph-mon[167076]: failed to encode map e654 with expected crc 2023-09-23T18:22:34.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:33 smithi099 ceph-mon[167076]: failed to encode map e654 with expected crc 2023-09-23T18:22:34.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:33 smithi099 ceph-mon[167076]: osdmap e655: 8 total, 8 up, 7 in 2023-09-23T18:22:34.616 INFO:teuthology.orchestra.run.smithi055.stdout:osd.0: {} 2023-09-23T18:22:34.635 INFO:teuthology.orchestra.run.smithi055.stdout:osd.1: {} 2023-09-23T18:22:34.645 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T18:22:34.658 INFO:teuthology.orchestra.run.smithi055.stdout:osd.2: {} 2023-09-23T18:22:34.678 INFO:teuthology.orchestra.run.smithi055.stdout:osd.3: {} 2023-09-23T18:22:34.698 INFO:teuthology.orchestra.run.smithi055.stdout:osd.4: {} 2023-09-23T18:22:34.721 INFO:teuthology.orchestra.run.smithi055.stdout:osd.5: {} 2023-09-23T18:22:34.741 INFO:teuthology.orchestra.run.smithi055.stdout:osd.6: {} 2023-09-23T18:22:34.766 INFO:teuthology.orchestra.run.smithi055.stdout:osd.7: {} 2023-09-23T18:22:34.767 INFO:teuthology.orchestra.run.smithi055.stderr:osd.0: osd_enable_op_tracker = 'true' 2023-09-23T18:22:34.767 INFO:teuthology.orchestra.run.smithi055.stderr:osd.1: osd_enable_op_tracker = 'true' 2023-09-23T18:22:34.767 INFO:teuthology.orchestra.run.smithi055.stderr:osd.2: osd_enable_op_tracker = 'true' 2023-09-23T18:22:34.767 INFO:teuthology.orchestra.run.smithi055.stderr:osd.3: osd_enable_op_tracker = 'true' 2023-09-23T18:22:34.767 INFO:teuthology.orchestra.run.smithi055.stderr:osd.4: osd_enable_op_tracker = 'true' 2023-09-23T18:22:34.768 INFO:teuthology.orchestra.run.smithi055.stderr:osd.5: osd_enable_op_tracker = 'true' 2023-09-23T18:22:34.768 INFO:teuthology.orchestra.run.smithi055.stderr:osd.6: osd_enable_op_tracker = 'true' 2023-09-23T18:22:34.768 INFO:teuthology.orchestra.run.smithi055.stderr:osd.7: osd_enable_op_tracker = 'true' 2023-09-23T18:22:34.855 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[184174]: failed to encode map e654 with expected crc 2023-09-23T18:22:34.855 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[184174]: failed to encode map e654 with expected crc 2023-09-23T18:22:34.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[184174]: pgmap v1571: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 998 KiB/s rd, 921 KiB/s wr, 358 op/s 2023-09-23T18:22:34.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[184174]: failed to encode map e655 with expected crc 2023-09-23T18:22:34.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[184174]: failed to encode map e655 with expected crc 2023-09-23T18:22:34.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[184174]: failed to encode map e655 with expected crc 2023-09-23T18:22:34.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2788680063' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:22:34.856 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/904474317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:34.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[179668]: failed to encode map e654 with expected crc 2023-09-23T18:22:34.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[179668]: failed to encode map e654 with expected crc 2023-09-23T18:22:34.856 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[179668]: pgmap v1571: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 998 KiB/s rd, 921 KiB/s wr, 358 op/s 2023-09-23T18:22:34.857 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[179668]: failed to encode map e655 with expected crc 2023-09-23T18:22:34.857 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[179668]: failed to encode map e655 with expected crc 2023-09-23T18:22:34.857 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[179668]: failed to encode map e655 with expected crc 2023-09-23T18:22:34.857 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2788680063' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:22:34.857 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:34 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/904474317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:34 smithi099 ceph-mon[167076]: failed to encode map e654 with expected crc 2023-09-23T18:22:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:34 smithi099 ceph-mon[167076]: failed to encode map e654 with expected crc 2023-09-23T18:22:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:34 smithi099 ceph-mon[167076]: pgmap v1571: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 998 KiB/s rd, 921 KiB/s wr, 358 op/s 2023-09-23T18:22:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:34 smithi099 ceph-mon[167076]: failed to encode map e655 with expected crc 2023-09-23T18:22:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:34 smithi099 ceph-mon[167076]: failed to encode map e655 with expected crc 2023-09-23T18:22:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:34 smithi099 ceph-mon[167076]: failed to encode map e655 with expected crc 2023-09-23T18:22:35.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2788680063' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:22:35.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:34 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/904474317' entity='client.admin' cmd=[{"prefix": "osd ls"}]: dispatch 2023-09-23T18:22:35.345 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T18:22:35.651 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[179668]: failed to encode map e656 with expected crc 2023-09-23T18:22:35.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2788680063' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:22:35.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[179668]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T18:22:35.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[179668]: failed to encode map e656 with expected crc 2023-09-23T18:22:35.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[179668]: failed to encode map e655 with expected crc 2023-09-23T18:22:35.652 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[179668]: failed to encode map e656 with expected crc 2023-09-23T18:22:35.855 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset noscrub 2023-09-23T18:22:35.858 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:22:35.858 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:22:35.858 INFO:teuthology.orchestra.run.smithi055.stdout: "num_ops": 0 2023-09-23T18:22:35.858 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:22:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[184174]: failed to encode map e656 with expected crc 2023-09-23T18:22:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2788680063' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:22:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[184174]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T18:22:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[184174]: failed to encode map e656 with expected crc 2023-09-23T18:22:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[184174]: failed to encode map e655 with expected crc 2023-09-23T18:22:35.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:35 smithi055 ceph-mon[184174]: failed to encode map e656 with expected crc 2023-09-23T18:22:36.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:35 smithi099 ceph-mon[167076]: failed to encode map e656 with expected crc 2023-09-23T18:22:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:35 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2788680063' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:22:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:35 smithi099 ceph-mon[167076]: osdmap e656: 8 total, 8 up, 7 in 2023-09-23T18:22:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:35 smithi099 ceph-mon[167076]: failed to encode map e656 with expected crc 2023-09-23T18:22:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:35 smithi099 ceph-mon[167076]: failed to encode map e655 with expected crc 2023-09-23T18:22:36.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:35 smithi099 ceph-mon[167076]: failed to encode map e656 with expected crc 2023-09-23T18:22:36.328 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_blocked_ops 2023-09-23T18:22:36.411 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:36.652 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:36.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:36 smithi055 ceph-mon[179668]: pgmap v1574: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 445 op/s 2023-09-23T18:22:36.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:36 smithi055 ceph-mon[184174]: pgmap v1574: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 445 op/s 2023-09-23T18:22:36.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:37.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:36 smithi099 ceph-mon[167076]: pgmap v1574: 103 pgs: 103 active+clean; 4.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 1.6 MiB/s rd, 1.5 MiB/s wr, 445 op/s 2023-09-23T18:22:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:37.574 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:38.389 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[184174]: pgmap v1575: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 427 op/s 2023-09-23T18:22:38.389 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.389 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4038256221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:22:38.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:22:38.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.390 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[179668]: pgmap v1575: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 427 op/s 2023-09-23T18:22:38.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4038256221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:22:38.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:22:38.390 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.391 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:38 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.474 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T18:22:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:38 smithi099 ceph-mon[167076]: pgmap v1575: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 1.3 MiB/s rd, 1.9 MiB/s wr, 427 op/s 2023-09-23T18:22:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:38 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:38 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4038256221' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:22:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:22:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:38 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:38 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:38.885 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T18:22:39.502 INFO:teuthology.orchestra.run.smithi055.stderr:noscrub is unset 2023-09-23T18:22:39.746 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:39 smithi055 ceph-mon[179668]: failed to encode map e657 with expected crc 2023-09-23T18:22:39.746 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:39 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:22:39.747 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:39 smithi055 ceph-mon[179668]: osdmap e657: 8 total, 8 up, 7 in 2023-09-23T18:22:39.747 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:39 smithi055 ceph-mon[179668]: failed to encode map e657 with expected crc 2023-09-23T18:22:39.747 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:39 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2645422935' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:22:39.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:39 smithi055 ceph-mon[184174]: failed to encode map e657 with expected crc 2023-09-23T18:22:39.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:39 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:22:39.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:39 smithi055 ceph-mon[184174]: osdmap e657: 8 total, 8 up, 7 in 2023-09-23T18:22:39.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:39 smithi055 ceph-mon[184174]: failed to encode map e657 with expected crc 2023-09-23T18:22:39.747 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:39 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2645422935' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:22:39.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:39 smithi099 ceph-mon[167076]: failed to encode map e657 with expected crc 2023-09-23T18:22:39.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:39 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:22:39.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:39 smithi099 ceph-mon[167076]: osdmap e657: 8 total, 8 up, 7 in 2023-09-23T18:22:39.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:39 smithi099 ceph-mon[167076]: failed to encode map e657 with expected crc 2023-09-23T18:22:39.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:39 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2645422935' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "noscrub"}]: dispatch 2023-09-23T18:22:39.843 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:22:39.843 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [], 2023-09-23T18:22:39.843 INFO:teuthology.orchestra.run.smithi055.stdout: "complaint_time": 30, 2023-09-23T18:22:39.843 INFO:teuthology.orchestra.run.smithi055.stdout: "num_blocked_ops": 0 2023-09-23T18:22:39.843 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:22:40.351 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd unset nodeep-scrub 2023-09-23T18:22:40.366 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph daemon osd.1 dump_historic_ops 2023-09-23T18:22:40.409 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:40.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:22:40] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2023-09-23T18:22:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[179668]: pgmap v1577: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 426 KiB/s rd, 1.0 MiB/s wr, 120 op/s 2023-09-23T18:22:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[179668]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[179668]: failed to encode map e658 with expected crc 2023-09-23T18:22:40.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2645422935' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:22:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[179668]: osdmap e658: 8 total, 8 up, 7 in 2023-09-23T18:22:40.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[179668]: failed to encode map e658 with expected crc 2023-09-23T18:22:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[184174]: pgmap v1577: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 426 KiB/s rd, 1.0 MiB/s wr, 120 op/s 2023-09-23T18:22:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[184174]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[184174]: failed to encode map e658 with expected crc 2023-09-23T18:22:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2645422935' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:22:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[184174]: osdmap e658: 8 total, 8 up, 7 in 2023-09-23T18:22:40.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:40 smithi055 ceph-mon[184174]: failed to encode map e658 with expected crc 2023-09-23T18:22:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:40 smithi099 ceph-mon[167076]: pgmap v1577: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 621 GiB / 626 GiB avail; 426 KiB/s rd, 1.0 MiB/s wr, 120 op/s 2023-09-23T18:22:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:40 smithi099 ceph-mon[167076]: Health check update: nodeep-scrub flag(s) set (OSDMAP_FLAGS) 2023-09-23T18:22:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:40 smithi099 ceph-mon[167076]: failed to encode map e658 with expected crc 2023-09-23T18:22:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:40 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2645422935' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "noscrub"}]': finished 2023-09-23T18:22:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:40 smithi099 ceph-mon[167076]: osdmap e658: 8 total, 8 up, 7 in 2023-09-23T18:22:40.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:40 smithi099 ceph-mon[167076]: failed to encode map e658 with expected crc 2023-09-23T18:22:41.291 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:41.469 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:41.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:41 smithi099 ceph-mon[167076]: failed to encode map e658 with expected crc 2023-09-23T18:22:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:41 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:41.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:41 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:41.740 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:41 smithi055 ceph-mon[184174]: failed to encode map e658 with expected crc 2023-09-23T18:22:41.741 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:41 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:41.741 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:41 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:41.741 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:41 smithi055 ceph-mon[179668]: failed to encode map e658 with expected crc 2023-09-23T18:22:41.741 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:41 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:41.741 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:41 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[179668]: pgmap v1579: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 60 KiB/s rd, 623 KiB/s wr, 75 op/s 2023-09-23T18:22:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:42.814 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:42.815 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:22:42.815 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:22:42.815 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[184174]: pgmap v1579: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 60 KiB/s rd, 623 KiB/s wr, 75 op/s 2023-09-23T18:22:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:42.815 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:42.816 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:22:42.816 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:22:42.816 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:42.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:42 smithi099 ceph-mon[167076]: pgmap v1579: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 60 KiB/s rd, 623 KiB/s wr, 75 op/s 2023-09-23T18:22:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:22:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:22:42.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:43.261 INFO:teuthology.orchestra.run.smithi055.stdout:{ 2023-09-23T18:22:43.261 INFO:teuthology.orchestra.run.smithi055.stdout: "size": 20, 2023-09-23T18:22:43.261 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 600, 2023-09-23T18:22:43.261 INFO:teuthology.orchestra.run.smithi055.stdout: "ops": [ 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201423 9.e e552/545 9:73127b0d:::benchmark_data_smithi055_386467_object96763:head v 552'12847, mlcod=552'12845)", 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 316.36821864400002, 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.101742522, 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:22:43.262 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890586+0000", 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.263 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890590+0000", 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0280000000000002e-06 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890596+0000", 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.0299999999999999e-06 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.264 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890601+0000", 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0660000000000003e-06 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890652+0000", 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1104999999999999e-05 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890674+0000", 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2274000000000001e-05 2023-09-23T18:22:43.265 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992278+0000", 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.1016035 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992329+0000", 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0816999999999998e-05 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.266 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201426 9.e e552/545 9:7aa9c16e:::benchmark_data_smithi055_386467_object96766:head v 552'12848, mlcod=552'12845)", 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 316.36818953099998, 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.101739732, 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.267 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.268 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890615+0000", 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890619+0000", 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0570000000000003e-06 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890625+0000", 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1970000000000001e-06 2023-09-23T18:22:43.269 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.890630+0000", 2023-09-23T18:22:43.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.0289999999999996e-06 2023-09-23T18:22:43.270 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.270 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.270 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.270 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892129+0000", 2023-09-23T18:22:43.270 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014991309999999999 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892151+0000", 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2036e-05 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992332+0000", 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10018059 2023-09-23T18:22:43.271 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992355+0000", 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2989e-05 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201428 9.3 e552/545 9:ce8adaf4:::benchmark_data_smithi055_386467_object96768:head v 552'12637, mlcod=551'12635)", 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.892727+0000", 2023-09-23T18:22:43.272 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 316.36607766899999, 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10402589299999999, 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892727+0000", 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.273 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892727+0000", 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892726+0000", 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892732+0000", 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6280000000000002e-06 2023-09-23T18:22:43.274 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892740+0000", 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.2050000000000002e-06 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892745+0000", 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.8590000000000004e-06 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.275 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892789+0000", 2023-09-23T18:22:43.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3699000000000003e-05 2023-09-23T18:22:43.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.277 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.277 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.277 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.892812+0000", 2023-09-23T18:22:43.277 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.368e-05 2023-09-23T18:22:43.277 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.996720+0000", 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.10390730099999999 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.996753+0000", 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3195000000000003e-05 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.278 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "pg_scan(get_digest 9.0 MIN-MIN e 552/545)", 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 316.36555413899998, 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.126230497, 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "started", 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.279 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:22:43.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.280 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.280 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.280 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:22:43.280 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.280 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.280 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.280 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893250+0000", 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893254+0000", 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3780000000000003e-06 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.281 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992277+0000", 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.099022751000000006 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992290+0000", 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3297e-05 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992361+0000", 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.0791999999999996e-05 2023-09-23T18:22:43.282 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992377+0000", 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.6331e-05 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:27.019481+0000", 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.027103558 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.283 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_repop(client.36552.0:201433 9.e e552/545 9:772bf683:::benchmark_data_smithi055_386467_object96773:head v 552'12849, mlcod=552'12845)", 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:17:26.893949+0000", 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 316.36485611299997, 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.130621881, 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.284 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893949+0000", 2023-09-23T18:22:43.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893949+0000", 2023-09-23T18:22:43.285 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.285 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.285 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.285 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.285 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893948+0000", 2023-09-23T18:22:43.286 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.286 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.286 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.286 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.286 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.893952+0000", 2023-09-23T18:22:43.286 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3640000000000001e-06 2023-09-23T18:22:43.286 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.286 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992298+0000", 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.098345455999999998 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992310+0000", 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.1708e-05 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.287 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992386+0000", 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6688999999999997e-05 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:26.992407+0000", 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0678000000000001e-05 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:27.024537+0000", 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.032130134999999997 2023-09-23T18:22:43.288 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:17:27.024570+0000", 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3327999999999997e-05 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:35571 10.2 10:446a4c6c:::benchmark_data_smithi055_425050_object35570:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e584)", 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:17.660369+0000", 2023-09-23T18:22:43.289 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 265.59843588799998, 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.169126159, 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 35571 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.290 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660369+0000", 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660369+0000", 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660367+0000", 2023-09-23T18:22:43.291 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660393+0000", 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5347e-05 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660411+0000", 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7903000000000001e-05 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.292 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.293 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660417+0000", 2023-09-23T18:22:43.293 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9630000000000003e-06 2023-09-23T18:22:43.293 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.293 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.293 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660468+0000", 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1572e-05 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660588+0000", 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000119657 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:22:43.294 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.660772+0000", 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000184167 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.662259+0000", 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014871649999999999 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.662719+0000", 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00046007099999999999 2023-09-23T18:22:43.295 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.829386+0000", 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.16666637400000001 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.829449+0000", 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.3450999999999999e-05 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.296 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:17.829495+0000", 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5933999999999998e-05 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55937 10.4 10:2db53d3f:::benchmark_data_smithi055_425050_object55936:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.319675+0000", 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 249.93912984599999, 2023-09-23T18:22:43.297 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3796218790000001, 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55937 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.298 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319675+0000", 2023-09-23T18:22:43.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.299 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.299 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.299 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319675+0000", 2023-09-23T18:22:43.299 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.299 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319673+0000", 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319716+0000", 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2919e-05 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.300 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319731+0000", 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.5237e-05 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.319743+0000", 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.225e-05 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.301 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321777+0000", 2023-09-23T18:22:43.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0020343399999999999 2023-09-23T18:22:43.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:43.302 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321791+0000", 2023-09-23T18:22:43.302 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3355e-05 2023-09-23T18:22:43.302 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.302 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.302 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.303 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.686788+0000", 2023-09-23T18:22:43.303 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.364997741 2023-09-23T18:22:43.303 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.303 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.303 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.687744+0000", 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00095536700000000004 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.687877+0000", 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000132908 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.304 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693069+0000", 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0051921600000000004 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693082+0000", 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.314e-05 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693115+0000", 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3355999999999997e-05 2023-09-23T18:22:43.305 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697881+0000", 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0047652900000000002 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697954+0000", 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.3436999999999995e-05 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.306 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.699297+0000", 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00134252 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55942 10.4 10:25b9d4ec:::benchmark_data_smithi055_425050_object55941:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.321753+0000", 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 249.93705125100001, 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3789243330000001, 2023-09-23T18:22:43.307 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55942 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321753+0000", 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.308 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321753+0000", 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321753+0000", 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.309 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321791+0000", 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.8318000000000001e-05 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321805+0000", 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.4208e-05 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321815+0000", 2023-09-23T18:22:43.310 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.6930000000000002e-06 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321860+0000", 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5293000000000002e-05 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321869+0000", 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.1400000000000006e-06 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.311 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.690763+0000", 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.368893624 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.690928+0000", 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00016521400000000001 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.312 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.691049+0000", 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00012102799999999999 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693150+0000", 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021011099999999998 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697641+0000", 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0044906629999999998 2023-09-23T18:22:43.313 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697713+0000", 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.2111000000000001e-05 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.699368+0000", 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016551000000000001 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.314 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.699427+0000", 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.859e-05 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.700678+0000", 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001251114 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.315 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55950 10.4 10:2586d6ac:::benchmark_data_smithi055_425050_object55949:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.321829+0000", 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 249.936975244, 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3802366020000001, 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55950 2023-09-23T18:22:43.316 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321829+0000", 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321829+0000", 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.317 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321829+0000", 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321853+0000", 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3898999999999999e-05 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.318 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321861+0000", 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9009999999999994e-06 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321867+0000", 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.387e-06 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321900+0000", 2023-09-23T18:22:43.319 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.3447e-05 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321910+0000", 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.2629999999999992e-06 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693287+0000", 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3713770679999999 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.320 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693442+0000", 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015501600000000001 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.693626+0000", 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018425900000000001 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.321 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697760+0000", 2023-09-23T18:22:43.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0041337860000000004 2023-09-23T18:22:43.322 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.322 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.322 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.322 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697773+0000", 2023-09-23T18:22:43.322 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.358e-05 2023-09-23T18:22:43.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.323 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.323 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.323 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.697826+0000", 2023-09-23T18:22:43.323 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2553999999999999e-05 2023-09-23T18:22:43.323 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.700869+0000", 2023-09-23T18:22:43.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0030429720000000001 2023-09-23T18:22:43.324 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.324 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.324 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.324 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.700906+0000", 2023-09-23T18:22:43.324 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6801999999999998e-05 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.702066+0000", 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001160276 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55965 10.4 10:2d25bc98:::benchmark_data_smithi055_425050_object55964:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:22:43.325 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.321878+0000", 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 249.93692697200001, 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3949194659999999, 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55965 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.326 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321878+0000", 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321878+0000", 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321877+0000", 2023-09-23T18:22:43.327 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321900+0000", 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.2762999999999999e-05 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321909+0000", 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.9379999999999998e-06 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.328 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321915+0000", 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1809999999999998e-06 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321958+0000", 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3473999999999997e-05 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:43.329 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321967+0000", 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.6070000000000008e-06 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.695651+0000", 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.373683609 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.695822+0000", 2023-09-23T18:22:43.330 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00017090399999999999 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.695961+0000", 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00013964700000000001 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.699322+0000", 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0033606560000000001 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.331 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.700745+0000", 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001422942 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.700778+0000", 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.2872e-05 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.332 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.715581+0000", 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.014802996000000001 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.715626+0000", 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5742999999999999e-05 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.716797+0000", 2023-09-23T18:22:43.333 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0011706889999999999 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55971 10.4 10:231c3e9f:::benchmark_data_smithi055_425050_object55970:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.321925+0000", 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 249.936880014, 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3957034830000001, 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.334 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55971 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321925+0000", 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.335 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321925+0000", 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321924+0000", 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321950+0000", 2023-09-23T18:22:43.336 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.6506999999999999e-05 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321956+0000", 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.801e-06 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321962+0000", 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5999999999999997e-06 2023-09-23T18:22:43.337 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322005+0000", 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.2996e-05 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322013+0000", 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.9500000000000001e-06 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.338 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.702122+0000", 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.380109322 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.702307+0000", 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018535200000000001 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.702450+0000", 2023-09-23T18:22:43.339 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00014205199999999999 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.707495+0000", 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0050456310000000001 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.709169+0000", 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0016743249999999999 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.340 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.709180+0000", 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0181e-05 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.716863+0000", 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0076837950000000002 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.341 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.716904+0000", 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0802e-05 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.717628+0000", 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00072384800000000003 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.342 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:55990 10.4 10:25823440:::benchmark_data_smithi055_425050_object55989:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] RETRY=1 ondisk+retry+write+known_if_redirected+supports_pool_eio e589)", 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:33.321970+0000", 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 249.93683493399999, 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3964889010000001, 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 55990 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.343 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321970+0000", 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321970+0000", 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.344 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.321969+0000", 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322021+0000", 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.2144000000000001e-05 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322027+0000", 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8889999999999999e-06 2023-09-23T18:22:43.345 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322031+0000", 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9609999999999999e-06 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322070+0000", 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.9218999999999998e-05 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.346 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for peered", 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:33.322078+0000", 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.5560000000000002e-06 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.705051+0000", 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3829731540000001 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.705240+0000", 2023-09-23T18:22:43.347 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00018871599999999999 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.705435+0000", 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019560700000000001 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.709196+0000", 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0037605849999999999 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.348 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.709232+0000", 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.6625000000000003e-05 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.715537+0000", 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0063041709999999999 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.349 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.717683+0000", 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0021465159999999998 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.717729+0000", 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5457000000000002e-05 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:34.718459+0000", 2023-09-23T18:22:43.350 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00072998799999999999 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67069 10.4 10:25914e1c:::benchmark_data_smithi055_425050_object67068:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.197763+0000", 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 238.061041163, 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.130859224, 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.351 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67069 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197763+0000", 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.352 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197763+0000", 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197762+0000", 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197792+0000", 2023-09-23T18:22:43.353 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.0354000000000001e-05 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197813+0000", 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.0839000000000002e-05 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.197823+0000", 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0178e-05 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.354 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.202503+0000", 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0046796060000000002 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.202697+0000", 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000193697 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.355 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.202901+0000", 2023-09-23T18:22:43.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020478099999999999 2023-09-23T18:22:43.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.207446+0000", 2023-09-23T18:22:43.356 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045442390000000003 2023-09-23T18:22:43.356 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.356 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.356 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.356 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.211087+0000", 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0036408669999999999 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.215890+0000", 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0048037890000000001 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.327070+0000", 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.111180026 2023-09-23T18:22:43.357 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.327133+0000", 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2765e-05 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.328623+0000", 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0014895959999999999 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.358 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67093 10.4 10:270b9556:::benchmark_data_smithi055_425050_object67092:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.219838+0000", 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 238.03896650199999, 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.11352698899999999, 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.359 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67093 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219838+0000", 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219838+0000", 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.360 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219836+0000", 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999976 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219865+0000", 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.8988000000000001e-05 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.361 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219882+0000", 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.7547999999999999e-05 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.219895+0000", 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2412000000000001e-05 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.220160+0000", 2023-09-23T18:22:43.362 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00026516400000000002 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.220284+0000", 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000124404 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.220399+0000", 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000114948 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.363 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.227313+0000", 2023-09-23T18:22:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0069139500000000003 2023-09-23T18:22:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.227354+0000", 2023-09-23T18:22:43.364 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.0824000000000001e-05 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.227379+0000", 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.4864000000000001e-05 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.331996+0000", 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.104617342 2023-09-23T18:22:43.365 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.332057+0000", 2023-09-23T18:22:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1183e-05 2023-09-23T18:22:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.366 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.333365+0000", 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0013077360000000001 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67102 10.4 10:2528425f:::benchmark_data_smithi055_425050_object67101:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.230801+0000", 2023-09-23T18:22:43.367 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 238.028003149, 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.103525952, 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67102 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.368 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230801+0000", 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230801+0000", 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230800+0000", 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:43.369 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230826+0000", 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5854999999999999e-05 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230839+0000", 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2743e-05 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.370 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.230845+0000", 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.1840000000000004e-06 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.231280+0000", 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00043508799999999998 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.231471+0000", 2023-09-23T18:22:43.371 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00019095900000000001 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.231713+0000", 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000242248 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.239490+0000", 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0077772129999999998 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.372 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.239536+0000", 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.6171999999999998e-05 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.239556+0000", 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.9157999999999999e-05 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.373 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.333433+0000", 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.093877304999999994 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.333489+0000", 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.6233999999999997e-05 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.334327+0000", 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00083832600000000002 2023-09-23T18:22:43.374 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67112 10.4 10:2686efcb:::benchmark_data_smithi055_425050_object67111:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.243689+0000", 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 238.015115923, 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.093717647000000001, 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.375 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67112 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243689+0000", 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.376 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243689+0000", 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243687+0000", 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999986 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243713+0000", 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5333999999999998e-05 2023-09-23T18:22:43.377 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243726+0000", 2023-09-23T18:22:43.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3794999999999999e-05 2023-09-23T18:22:43.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.378 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.378 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.378 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.243735+0000", 2023-09-23T18:22:43.378 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.32e-06 2023-09-23T18:22:43.378 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.247533+0000", 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.003798413 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.247651+0000", 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000118192 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.379 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.247763+0000", 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011155899999999999 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.251684+0000", 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0039215109999999999 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.251753+0000", 2023-09-23T18:22:43.380 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.8285000000000001e-05 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.259065+0000", 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0073120659999999999 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.336340+0000", 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.077275132999999996 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.381 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.336403+0000", 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.2880999999999996e-05 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.337406+0000", 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001003557 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.382 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67113 10.4 10:2c8fad6a:::benchmark_data_smithi055_425050_object67112:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.244958+0000", 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 238.013846607, 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.093448478000000001, 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.383 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67113 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244958+0000", 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244958+0000", 2023-09-23T18:22:43.384 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244957+0000", 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244982+0000", 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.5035e-05 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.385 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244992+0000", 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 9.3449999999999995e-06 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.244999+0000", 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 7.6410000000000002e-06 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.386 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.249567+0000", 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0045676069999999996 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.249721+0000", 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00015449600000000001 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.249834+0000", 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000112039 2023-09-23T18:22:43.387 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.251740+0000", 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001906196 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.251859+0000", 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00011937799999999999 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.388 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.259121+0000", 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0072623979999999998 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.337468+0000", 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.078346247999999993 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.337526+0000", 2023-09-23T18:22:43.389 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.8489000000000002e-05 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.338406+0000", 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00088027600000000004 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:67137 10.4 10:217dc21f:::benchmark_data_smithi055_425050_object67136:head [set-alloc-hint object_size 65536 write_size 65536,write 0~65536] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e593)", 2023-09-23T18:22:43.390 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:18:45.340575+0000", 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 237.91822977499999, 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.094444057999999997, 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 67137 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.391 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340575+0000", 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340575+0000", 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340574+0000", 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999995 2023-09-23T18:22:43.392 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340598+0000", 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 2.3149000000000001e-05 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340610+0000", 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.2245e-05 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.393 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.340615+0000", 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.519e-06 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.348604+0000", 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0079891879999999995 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.348811+0000", 2023-09-23T18:22:43.394 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00020653699999999999 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,3,4", 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.349042+0000", 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00023097600000000001 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.353098+0000", 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0040564579999999998 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.395 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.354411+0000", 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001312192 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.356938+0000", 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0025268809999999999 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.396 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.432957+0000", 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.076019171999999996 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.433021+0000", 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.4101000000000002e-05 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:18:45.435019+0000", 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0019981259999999998 2023-09-23T18:22:43.397 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:138262 10.2 10:4c48af53:::benchmark_data_smithi055_425050_object34856:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e610)", 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:20:13.013930+0000", 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 150.244874619, 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.118017868, 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.398 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 138262 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.013930+0000", 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.399 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.013930+0000", 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.013928+0000", 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.9999981 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.013935+0000", 2023-09-23T18:22:43.400 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 6.883e-06 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.013949+0000", 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.3647e-05 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.013957+0000", 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.7299999999999994e-06 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.401 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.014013+0000", 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.5384999999999999e-05 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.014117+0000", 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.000104472 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:22:43.402 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.014224+0000", 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.00010692 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.016872+0000", 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0026482150000000002 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.027111+0000", 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.010238300000000001 2023-09-23T18:22:43.403 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.131851+0000", 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.104740473 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.131903+0000", 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.1644999999999998e-05 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.404 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.405 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.131948+0000", 2023-09-23T18:22:43.405 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.5031000000000003e-05 2023-09-23T18:22:43.405 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.405 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.405 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.405 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: "description": "osd_op(client.46799.0:138273 10.2 10:46208d4a:::benchmark_data_smithi055_425050_object34867:head [delete] snapc 0=[] ondisk+write+known_if_redirected+supports_pool_eio e610)", 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: "initiated_at": "2023-09-23T18:20:13.029033+0000", 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: "age": 150.22977182299999, 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.108698564, 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: "type_data": { 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: "flag_point": "commit sent; apply or cleanup", 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: "client_info": { 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: "client": "client.46799", 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: "client_addr": "172.21.15.55:0/985932536", 2023-09-23T18:22:43.406 INFO:teuthology.orchestra.run.smithi055.stdout: "tid": 138273 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: "events": [ 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "initiated", 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.029033+0000", 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "throttled", 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.029033+0000", 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.407 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "header_read", 2023-09-23T18:22:43.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.029032+0000", 2023-09-23T18:22:43.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4294967295.999999 2023-09-23T18:22:43.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.408 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "all_read", 2023-09-23T18:22:43.408 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.029038+0000", 2023-09-23T18:22:43.408 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.801e-06 2023-09-23T18:22:43.408 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.408 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "dispatched", 2023-09-23T18:22:43.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.029048+0000", 2023-09-23T18:22:43.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 1.0152e-05 2023-09-23T18:22:43.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.409 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.409 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "queued_for_pg", 2023-09-23T18:22:43.409 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.029054+0000", 2023-09-23T18:22:43.409 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.9399999999999999e-06 2023-09-23T18:22:43.409 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.410 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "reached_pg", 2023-09-23T18:22:43.410 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.029088+0000", 2023-09-23T18:22:43.410 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 3.4042999999999999e-05 2023-09-23T18:22:43.410 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.410 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "started", 2023-09-23T18:22:43.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.029176+0000", 2023-09-23T18:22:43.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8721999999999995e-05 2023-09-23T18:22:43.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.411 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "waiting for subops from 0,4", 2023-09-23T18:22:43.411 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.029265+0000", 2023-09-23T18:22:43.411 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 8.8786e-05 2023-09-23T18:22:43.411 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.411 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "op_commit", 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.031147+0000", 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.0018815710000000001 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.033084+0000", 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.001937507 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "sub_op_commit_rec", 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.137634+0000", 2023-09-23T18:22:43.412 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 0.104550017 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "commit_sent", 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.137688+0000", 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 5.3584000000000001e-05 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: }, 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: { 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: "event": "done", 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: "time": "2023-09-23T18:20:13.137731+0000", 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: "duration": 4.3541000000000002e-05 2023-09-23T18:22:43.413 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.414 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.414 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.414 INFO:teuthology.orchestra.run.smithi055.stdout: } 2023-09-23T18:22:43.414 INFO:teuthology.orchestra.run.smithi055.stdout: ] 2023-09-23T18:22:43.414 INFO:teuthology.orchestra.run.smithi055.stdout:} 2023-09-23T18:22:43.524 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T18:22:43.814 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:43 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3009186402' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:22:43.815 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:43 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3009186402' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:22:43.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:43 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3009186402' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:22:44.054 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T18:22:44.054 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:22:44.527 INFO:teuthology.orchestra.run.smithi055.stderr:nodeep-scrub is unset 2023-09-23T18:22:44.571 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:44 smithi099 ceph-mon[167076]: pgmap v1580: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 49 KiB/s rd, 514 KiB/s wr, 62 op/s 2023-09-23T18:22:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:44 smithi099 ceph-mon[167076]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3009186402' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:22:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:44 smithi099 ceph-mon[167076]: osdmap e659: 8 total, 8 up, 7 in 2023-09-23T18:22:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:44 smithi099 ceph-mon[167076]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:44 smithi099 ceph-mon[167076]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:44 smithi099 ceph-mon[167076]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:44 smithi099 ceph-mon[167076]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:44 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2104447149' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:44.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:44 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:44.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[184174]: pgmap v1580: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 49 KiB/s rd, 514 KiB/s wr, 62 op/s 2023-09-23T18:22:44.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[184174]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3009186402' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:22:44.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[184174]: osdmap e659: 8 total, 8 up, 7 in 2023-09-23T18:22:44.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[184174]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[184174]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[184174]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[184174]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2104447149' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:44.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:44.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[179668]: pgmap v1580: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 49 KiB/s rd, 514 KiB/s wr, 62 op/s 2023-09-23T18:22:44.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[179668]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3009186402' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:22:44.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[179668]: osdmap e659: 8 total, 8 up, 7 in 2023-09-23T18:22:44.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[179668]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[179668]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[179668]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[179668]: failed to encode map e659 with expected crc 2023-09-23T18:22:44.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2104447149' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:44.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:44 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd unset", "key": "nodeep-scrub"}]: dispatch 2023-09-23T18:22:45.520 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:22:45.520 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:22:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:22:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:22:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: osdmap e660: 8 total, 8 up, 7 in 2023-09-23T18:22:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:45 smithi099 ceph-mon[167076]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: osdmap e660: 8 total, 8 up, 7 in 2023-09-23T18:22:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[179668]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: Health check cleared: OSDMAP_FLAGS (was: nodeep-scrub flag(s) set) 2023-09-23T18:22:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:22:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd unset", "key": "nodeep-scrub"}]': finished 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: osdmap e660: 8 total, 8 up, 7 in 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:45.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:45 smithi055 ceph-mon[184174]: failed to encode map e660 with expected crc 2023-09-23T18:22:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:46 smithi099 ceph-mon[167076]: pgmap v1583: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:22:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:46 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:22:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:46 smithi099 ceph-mon[167076]: failed to encode map e661 with expected crc 2023-09-23T18:22:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:46 smithi099 ceph-mon[167076]: osdmap e661: 8 total, 8 up, 7 in 2023-09-23T18:22:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:46 smithi099 ceph-mon[167076]: failed to encode map e661 with expected crc 2023-09-23T18:22:46.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:46 smithi099 ceph-mon[167076]: failed to encode map e661 with expected crc 2023-09-23T18:22:46.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[179668]: pgmap v1583: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:22:46.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:22:46.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[179668]: failed to encode map e661 with expected crc 2023-09-23T18:22:46.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[179668]: osdmap e661: 8 total, 8 up, 7 in 2023-09-23T18:22:46.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[179668]: failed to encode map e661 with expected crc 2023-09-23T18:22:46.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[179668]: failed to encode map e661 with expected crc 2023-09-23T18:22:46.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[184174]: pgmap v1583: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:22:46.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:22:46.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[184174]: failed to encode map e661 with expected crc 2023-09-23T18:22:46.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[184174]: osdmap e661: 8 total, 8 up, 7 in 2023-09-23T18:22:46.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[184174]: failed to encode map e661 with expected crc 2023-09-23T18:22:46.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:46 smithi055 ceph-mon[184174]: failed to encode map e661 with expected crc 2023-09-23T18:22:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:47 smithi055 ceph-mon[184174]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:47 smithi055 ceph-mon[184174]: osdmap e662: 8 total, 8 up, 7 in 2023-09-23T18:22:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:47 smithi055 ceph-mon[184174]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:47 smithi055 ceph-mon[184174]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:47 smithi055 ceph-mon[184174]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:47 smithi055 ceph-mon[179668]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:47 smithi055 ceph-mon[179668]: osdmap e662: 8 total, 8 up, 7 in 2023-09-23T18:22:47.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:47 smithi055 ceph-mon[179668]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:47 smithi055 ceph-mon[179668]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.681 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:47 smithi055 ceph-mon[179668]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:47 smithi099 ceph-mon[167076]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:47 smithi099 ceph-mon[167076]: osdmap e662: 8 total, 8 up, 7 in 2023-09-23T18:22:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:47 smithi099 ceph-mon[167076]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:47 smithi099 ceph-mon[167076]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:47 smithi099 ceph-mon[167076]: failed to encode map e662 with expected crc 2023-09-23T18:22:47.922 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:22:47.931 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:22:48.305 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1587,"stamp":"2023-09-23T18:22:47.559922+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":464,"num_bytes_recovered":17315659,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":302,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":312,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4988348,"kb_used_data":1092100,"kb_used_omap":62,"kb_used_meta":3896129,"kb_avail":651227716,"statfs":{"total":671965249536,"available":666857181184,"internally_reserved":0,"allocated":1118310400,"data_stored":1099612847,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":64275,"internal_metadata":3989636333},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":31,"num_bytes_recovered":263242,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.056966"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":962,"reported_epoch":661,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:22:45.552442+0000","last_change":"2023-09-23T18:22:45.552649+0000","last_active":"2023-09-23T18:22:45.552442+0000","last_peered":"2023-09-23T18:22:45.552442+0000","last_clean":"2023-09-23T18:22:39.500492+0000","last_became_active":"2023-09-23T18:22:45.548325+0000","last_became_peered":"2023-09-23T18:22:45.548325+0000","last_unstale":"2023-09-23T18:22:45.552442+0000","last_undegraded":"2023-09-23T18:22:45.552442+0000","last_fullsized":"2023-09-23T18:22:45.539208+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:13.640598+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,1],"avail_no_missing":["5","0","1","2"],"object_location_counts":[{"shards":"1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":331,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.545247+0000","last_change":"2023-09-23T18:22:44.528866+0000","last_active":"2023-09-23T18:22:45.545247+0000","last_peered":"2023-09-23T18:22:45.545247+0000","last_clean":"2023-09-23T18:22:45.545247+0000","last_became_active":"2023-09-23T18:22:44.528499+0000","last_became_peered":"2023-09-23T18:22:44.528499+0000","last_unstale":"2023-09-23T18:22:45.545247+0000","last_undegraded":"2023-09-23T18:22:45.545247+0000","last_fullsized":"2023-09-23T18:22:45.545247+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:26.288744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512987+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:22:44.512987+0000","last_peered":"2023-09-23T18:22:44.512987+0000","last_clean":"2023-09-23T18:22:44.512987+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:22:44.512987+0000","last_undegraded":"2023-09-23T18:22:44.512987+0000","last_fullsized":"2023-09-23T18:22:44.512987+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":954,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515593+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:22:44.515593+0000","last_peered":"2023-09-23T18:22:44.515593+0000","last_clean":"2023-09-23T18:22:44.515593+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:22:44.515593+0000","last_undegraded":"2023-09-23T18:22:44.515593+0000","last_fullsized":"2023-09-23T18:22:44.515593+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":402,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514671+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:22:44.514671+0000","last_peered":"2023-09-23T18:22:44.514671+0000","last_clean":"2023-09-23T18:22:44.514671+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:22:44.514671+0000","last_undegraded":"2023-09-23T18:22:44.514671+0000","last_fullsized":"2023-09-23T18:22:44.514671+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":721,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.509087+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:22:43.509087+0000","last_peered":"2023-09-23T18:22:43.509087+0000","last_clean":"2023-09-23T18:22:43.509087+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:22:43.509087+0000","last_undegraded":"2023-09-23T18:22:43.509087+0000","last_fullsized":"2023-09-23T18:22:43.509087+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":971,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513513+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:22:44.513513+0000","last_peered":"2023-09-23T18:22:44.513513+0000","last_clean":"2023-09-23T18:22:44.513513+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:22:44.513513+0000","last_undegraded":"2023-09-23T18:22:44.513513+0000","last_fullsized":"2023-09-23T18:22:44.513513+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512080+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:22:44.512080+0000","last_peered":"2023-09-23T18:22:44.512080+0000","last_clean":"2023-09-23T18:22:44.512080+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:22:44.512080+0000","last_undegraded":"2023-09-23T18:22:44.512080+0000","last_fullsized":"2023-09-23T18:22:44.512080+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":236,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516913+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:22:44.516913+0000","last_peered":"2023-09-23T18:22:44.516913+0000","last_clean":"2023-09-23T18:22:44.516913+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:22:44.516913+0000","last_undegraded":"2023-09-23T18:22:44.516913+0000","last_fullsized":"2023-09-23T18:22:44.516913+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1094,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513069+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:22:44.513069+0000","last_peered":"2023-09-23T18:22:44.513069+0000","last_clean":"2023-09-23T18:22:44.513069+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:22:44.513069+0000","last_undegraded":"2023-09-23T18:22:44.513069+0000","last_fullsized":"2023-09-23T18:22:44.513069+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":170,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513333+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:22:44.513333+0000","last_peered":"2023-09-23T18:22:44.513333+0000","last_clean":"2023-09-23T18:22:44.513333+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:22:44.513333+0000","last_undegraded":"2023-09-23T18:22:44.513333+0000","last_fullsized":"2023-09-23T18:22:44.513333+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":251,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513028+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:22:44.513028+0000","last_peered":"2023-09-23T18:22:44.513028+0000","last_clean":"2023-09-23T18:22:44.513028+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:22:44.513028+0000","last_undegraded":"2023-09-23T18:22:44.513028+0000","last_fullsized":"2023-09-23T18:22:44.513028+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":837,"reported_epoch":657,"state":"active+clean","last_fresh":"2023-09-23T18:22:38.517753+0000","last_change":"2023-09-23T18:22:06.467889+0000","last_active":"2023-09-23T18:22:38.517753+0000","last_peered":"2023-09-23T18:22:38.517753+0000","last_clean":"2023-09-23T18:22:38.517753+0000","last_became_active":"2023-09-23T18:22:06.467521+0000","last_became_peered":"2023-09-23T18:22:06.467521+0000","last_unstale":"2023-09-23T18:22:38.517753+0000","last_undegraded":"2023-09-23T18:22:38.517753+0000","last_fullsized":"2023-09-23T18:22:38.517753+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:07.204206+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":206,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512839+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:22:44.512839+0000","last_peered":"2023-09-23T18:22:44.512839+0000","last_clean":"2023-09-23T18:22:44.512839+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:22:44.512839+0000","last_undegraded":"2023-09-23T18:22:44.512839+0000","last_fullsized":"2023-09-23T18:22:44.512839+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":205,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514365+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:22:44.514365+0000","last_peered":"2023-09-23T18:22:44.514365+0000","last_clean":"2023-09-23T18:22:44.514365+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:22:44.514365+0000","last_undegraded":"2023-09-23T18:22:44.514365+0000","last_fullsized":"2023-09-23T18:22:44.514365+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":1005,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513638+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:22:44.513638+0000","last_peered":"2023-09-23T18:22:44.513638+0000","last_clean":"2023-09-23T18:22:44.513638+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:22:44.513638+0000","last_undegraded":"2023-09-23T18:22:44.513638+0000","last_fullsized":"2023-09-23T18:22:44.513638+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":456,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513558+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:22:44.513558+0000","last_peered":"2023-09-23T18:22:44.513558+0000","last_clean":"2023-09-23T18:22:44.513558+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:22:44.513558+0000","last_undegraded":"2023-09-23T18:22:44.513558+0000","last_fullsized":"2023-09-23T18:22:44.513558+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":483,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516017+0000","last_change":"2023-09-23T18:22:03.468342+0000","last_active":"2023-09-23T18:22:44.516017+0000","last_peered":"2023-09-23T18:22:44.516017+0000","last_clean":"2023-09-23T18:22:44.516017+0000","last_became_active":"2023-09-23T18:22:03.467804+0000","last_became_peered":"2023-09-23T18:22:03.467804+0000","last_unstale":"2023-09-23T18:22:44.516017+0000","last_undegraded":"2023-09-23T18:22:44.516017+0000","last_fullsized":"2023-09-23T18:22:44.516017+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:17.762831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":1034,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T18:22:39.498138+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:22:39.498138+0000","last_peered":"2023-09-23T18:22:39.498138+0000","last_clean":"2023-09-23T18:22:39.498138+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:22:39.498138+0000","last_undegraded":"2023-09-23T18:22:39.498138+0000","last_fullsized":"2023-09-23T18:22:39.498138+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":232,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516384+0000","last_change":"2023-09-23T18:22:03.479912+0000","last_active":"2023-09-23T18:22:44.516384+0000","last_peered":"2023-09-23T18:22:44.516384+0000","last_clean":"2023-09-23T18:22:44.516384+0000","last_became_active":"2023-09-23T18:22:03.464545+0000","last_became_peered":"2023-09-23T18:22:03.464545+0000","last_unstale":"2023-09-23T18:22:44.516384+0000","last_undegraded":"2023-09-23T18:22:44.516384+0000","last_fullsized":"2023-09-23T18:22:44.516384+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:50.025694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":479,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515985+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:22:44.515985+0000","last_peered":"2023-09-23T18:22:44.515985+0000","last_clean":"2023-09-23T18:22:44.515985+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:22:44.515985+0000","last_undegraded":"2023-09-23T18:22:44.515985+0000","last_fullsized":"2023-09-23T18:22:44.515985+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":886,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513283+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:22:44.513283+0000","last_peered":"2023-09-23T18:22:44.513283+0000","last_clean":"2023-09-23T18:22:44.513283+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:22:44.513283+0000","last_undegraded":"2023-09-23T18:22:44.513283+0000","last_fullsized":"2023-09-23T18:22:44.513283+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":495,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515712+0000","last_change":"2023-09-23T18:22:03.475293+0000","last_active":"2023-09-23T18:22:44.515712+0000","last_peered":"2023-09-23T18:22:44.515712+0000","last_clean":"2023-09-23T18:22:44.515712+0000","last_became_active":"2023-09-23T18:22:03.465603+0000","last_became_peered":"2023-09-23T18:22:03.465603+0000","last_unstale":"2023-09-23T18:22:44.515712+0000","last_undegraded":"2023-09-23T18:22:44.515712+0000","last_fullsized":"2023-09-23T18:22:44.515712+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:11.349439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":105,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.509910+0000","last_change":"2023-09-23T18:22:03.464544+0000","last_active":"2023-09-23T18:22:43.509910+0000","last_peered":"2023-09-23T18:22:43.509910+0000","last_clean":"2023-09-23T18:22:43.509910+0000","last_became_active":"2023-09-23T18:22:03.464227+0000","last_became_peered":"2023-09-23T18:22:03.464227+0000","last_unstale":"2023-09-23T18:22:43.509910+0000","last_undegraded":"2023-09-23T18:22:43.509910+0000","last_fullsized":"2023-09-23T18:22:43.509910+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:56.368635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":857,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514280+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:22:44.514280+0000","last_peered":"2023-09-23T18:22:44.514280+0000","last_clean":"2023-09-23T18:22:44.514280+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:22:44.514280+0000","last_undegraded":"2023-09-23T18:22:44.514280+0000","last_fullsized":"2023-09-23T18:22:44.514280+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":176,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.508723+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:22:43.508723+0000","last_peered":"2023-09-23T18:22:43.508723+0000","last_clean":"2023-09-23T18:22:43.508723+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:22:43.508723+0000","last_undegraded":"2023-09-23T18:22:43.508723+0000","last_fullsized":"2023-09-23T18:22:43.508723+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":170,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513402+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:22:44.513402+0000","last_peered":"2023-09-23T18:22:44.513402+0000","last_clean":"2023-09-23T18:22:44.513402+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:22:44.513402+0000","last_undegraded":"2023-09-23T18:22:44.513402+0000","last_fullsized":"2023-09-23T18:22:44.513402+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":797,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512324+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:22:44.512324+0000","last_peered":"2023-09-23T18:22:44.512324+0000","last_clean":"2023-09-23T18:22:44.512324+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:22:44.512324+0000","last_undegraded":"2023-09-23T18:22:44.512324+0000","last_fullsized":"2023-09-23T18:22:44.512324+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":247,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514273+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:22:44.514273+0000","last_peered":"2023-09-23T18:22:44.514273+0000","last_clean":"2023-09-23T18:22:44.514273+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:22:44.514273+0000","last_undegraded":"2023-09-23T18:22:44.514273+0000","last_fullsized":"2023-09-23T18:22:44.514273+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":269,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514420+0000","last_change":"2023-09-23T18:22:03.464607+0000","last_active":"2023-09-23T18:22:44.514420+0000","last_peered":"2023-09-23T18:22:44.514420+0000","last_clean":"2023-09-23T18:22:44.514420+0000","last_became_active":"2023-09-23T18:22:03.464333+0000","last_became_peered":"2023-09-23T18:22:03.464333+0000","last_unstale":"2023-09-23T18:22:44.514420+0000","last_undegraded":"2023-09-23T18:22:44.514420+0000","last_fullsized":"2023-09-23T18:22:44.514420+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:51.062916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":891,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514205+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:22:44.514205+0000","last_peered":"2023-09-23T18:22:44.514205+0000","last_clean":"2023-09-23T18:22:44.514205+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:22:44.514205+0000","last_undegraded":"2023-09-23T18:22:44.514205+0000","last_fullsized":"2023-09-23T18:22:44.514205+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":800,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514831+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:22:44.514831+0000","last_peered":"2023-09-23T18:22:44.514831+0000","last_clean":"2023-09-23T18:22:44.514831+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:22:44.514831+0000","last_undegraded":"2023-09-23T18:22:44.514831+0000","last_fullsized":"2023-09-23T18:22:44.514831+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":87,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513355+0000","last_change":"2023-09-23T18:22:03.468997+0000","last_active":"2023-09-23T18:22:44.513355+0000","last_peered":"2023-09-23T18:22:44.513355+0000","last_clean":"2023-09-23T18:22:44.513355+0000","last_became_active":"2023-09-23T18:22:03.468301+0000","last_became_peered":"2023-09-23T18:22:03.468301+0000","last_unstale":"2023-09-23T18:22:44.513355+0000","last_undegraded":"2023-09-23T18:22:44.513355+0000","last_fullsized":"2023-09-23T18:22:44.513355+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:32:17.742551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":910,"reported_epoch":662,"state":"peering","last_fresh":"2023-09-23T18:22:46.560855+0000","last_change":"2023-09-23T18:22:46.561691+0000","last_active":"2023-09-23T18:22:46.560818+0000","last_peered":"2023-09-23T18:22:45.549690+0000","last_clean":"2023-09-23T18:17:50.936418+0000","last_became_active":"2023-09-23T18:22:45.549353+0000","last_became_peered":"2023-09-23T18:22:45.549353+0000","last_unstale":"2023-09-23T18:22:46.560855+0000","last_undegraded":"2023-09-23T18:22:46.560855+0000","last_fullsized":"2023-09-23T18:22:46.560855+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:05.452199+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":298,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.546257+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:22:45.546257+0000","last_peered":"2023-09-23T18:22:45.546257+0000","last_clean":"2023-09-23T18:22:45.546257+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:22:45.546257+0000","last_undegraded":"2023-09-23T18:22:45.546257+0000","last_fullsized":"2023-09-23T18:22:45.546257+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":170,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514387+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:22:44.514387+0000","last_peered":"2023-09-23T18:22:44.514387+0000","last_clean":"2023-09-23T18:22:44.514387+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:22:44.514387+0000","last_undegraded":"2023-09-23T18:22:44.514387+0000","last_fullsized":"2023-09-23T18:22:44.514387+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":793,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514045+0000","last_change":"2023-09-23T18:21:51.851401+0000","last_active":"2023-09-23T18:22:44.514045+0000","last_peered":"2023-09-23T18:22:44.514045+0000","last_clean":"2023-09-23T18:22:44.514045+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:22:44.514045+0000","last_undegraded":"2023-09-23T18:22:44.514045+0000","last_fullsized":"2023-09-23T18:22:44.514045+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:21:51.851269+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:21:51.851269+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:48.305008+0000","scrub_duration":0.001933539,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":299,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.546237+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:22:45.546237+0000","last_peered":"2023-09-23T18:22:45.546237+0000","last_clean":"2023-09-23T18:22:45.546237+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:22:45.546237+0000","last_undegraded":"2023-09-23T18:22:45.546237+0000","last_fullsized":"2023-09-23T18:22:45.546237+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":278,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.544150+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:22:45.544150+0000","last_peered":"2023-09-23T18:22:45.544150+0000","last_clean":"2023-09-23T18:22:45.544150+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:22:45.544150+0000","last_undegraded":"2023-09-23T18:22:45.544150+0000","last_fullsized":"2023-09-23T18:22:45.544150+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":675,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516796+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:22:44.516796+0000","last_peered":"2023-09-23T18:22:44.516796+0000","last_clean":"2023-09-23T18:22:44.516796+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:22:44.516796+0000","last_undegraded":"2023-09-23T18:22:44.516796+0000","last_fullsized":"2023-09-23T18:22:44.516796+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":585,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513180+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:22:44.513180+0000","last_peered":"2023-09-23T18:22:44.513180+0000","last_clean":"2023-09-23T18:22:44.513180+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:22:44.513180+0000","last_undegraded":"2023-09-23T18:22:44.513180+0000","last_fullsized":"2023-09-23T18:22:44.513180+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":236,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515842+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:22:44.515842+0000","last_peered":"2023-09-23T18:22:44.515842+0000","last_clean":"2023-09-23T18:22:44.515842+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:22:44.515842+0000","last_undegraded":"2023-09-23T18:22:44.515842+0000","last_fullsized":"2023-09-23T18:22:44.515842+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":701,"reported_epoch":661,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:22:45.550767+0000","last_change":"2023-09-23T18:22:45.550653+0000","last_active":"2023-09-23T18:22:45.550767+0000","last_peered":"2023-09-23T18:22:45.550767+0000","last_clean":"2023-09-23T18:22:40.227293+0000","last_became_active":"2023-09-23T18:22:45.550653+0000","last_became_peered":"2023-09-23T18:22:45.550653+0000","last_unstale":"2023-09-23T18:22:45.550767+0000","last_undegraded":"2023-09-23T18:22:45.550767+0000","last_fullsized":"2023-09-23T18:22:45.543763+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:33.107993+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4],"avail_no_missing":["3","1","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":184,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513631+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:22:44.513631+0000","last_peered":"2023-09-23T18:22:44.513631+0000","last_clean":"2023-09-23T18:22:44.513631+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:22:44.513631+0000","last_undegraded":"2023-09-23T18:22:44.513631+0000","last_fullsized":"2023-09-23T18:22:44.513631+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":498,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.511786+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:22:43.511786+0000","last_peered":"2023-09-23T18:22:43.511786+0000","last_clean":"2023-09-23T18:22:43.511786+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:22:43.511786+0000","last_undegraded":"2023-09-23T18:22:43.511786+0000","last_fullsized":"2023-09-23T18:22:43.511786+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1152,"reported_epoch":661,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:22:45.543394+0000","last_change":"2023-09-23T18:22:45.543277+0000","last_active":"2023-09-23T18:22:45.543394+0000","last_peered":"2023-09-23T18:22:45.543394+0000","last_clean":"2023-09-23T18:22:39.500356+0000","last_became_active":"2023-09-23T18:22:45.543277+0000","last_became_peered":"2023-09-23T18:22:45.543277+0000","last_unstale":"2023-09-23T18:22:45.543394+0000","last_undegraded":"2023-09-23T18:22:45.543394+0000","last_fullsized":"2023-09-23T18:22:45.539213+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:28.644193+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","2","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":399,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513662+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:22:44.513662+0000","last_peered":"2023-09-23T18:22:44.513662+0000","last_clean":"2023-09-23T18:22:44.513662+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:22:44.513662+0000","last_undegraded":"2023-09-23T18:22:44.513662+0000","last_fullsized":"2023-09-23T18:22:44.513662+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":204,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516130+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:22:44.516130+0000","last_peered":"2023-09-23T18:22:44.516130+0000","last_clean":"2023-09-23T18:22:44.516130+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:22:44.516130+0000","last_undegraded":"2023-09-23T18:22:44.516130+0000","last_fullsized":"2023-09-23T18:22:44.516130+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":949,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514576+0000","last_change":"2023-09-23T18:22:06.466040+0000","last_active":"2023-09-23T18:22:44.514576+0000","last_peered":"2023-09-23T18:22:44.514576+0000","last_clean":"2023-09-23T18:22:44.514576+0000","last_became_active":"2023-09-23T18:22:06.465158+0000","last_became_peered":"2023-09-23T18:22:06.465158+0000","last_unstale":"2023-09-23T18:22:44.514576+0000","last_undegraded":"2023-09-23T18:22:44.514576+0000","last_fullsized":"2023-09-23T18:22:44.514576+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:29.587874+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":212,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515914+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:22:44.515914+0000","last_peered":"2023-09-23T18:22:44.515914+0000","last_clean":"2023-09-23T18:22:44.515914+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:22:44.515914+0000","last_undegraded":"2023-09-23T18:22:44.515914+0000","last_fullsized":"2023-09-23T18:22:44.515914+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":456,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.546207+0000","last_change":"2023-09-23T18:22:44.527193+0000","last_active":"2023-09-23T18:22:45.546207+0000","last_peered":"2023-09-23T18:22:45.546207+0000","last_clean":"2023-09-23T18:22:45.546207+0000","last_became_active":"2023-09-23T18:22:44.526903+0000","last_became_peered":"2023-09-23T18:22:44.526903+0000","last_unstale":"2023-09-23T18:22:45.546207+0000","last_undegraded":"2023-09-23T18:22:45.546207+0000","last_fullsized":"2023-09-23T18:22:45.546207+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:09.126029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":931,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515692+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:22:44.515692+0000","last_peered":"2023-09-23T18:22:44.515692+0000","last_clean":"2023-09-23T18:22:44.515692+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:22:44.515692+0000","last_undegraded":"2023-09-23T18:22:44.515692+0000","last_fullsized":"2023-09-23T18:22:44.515692+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":466,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.544371+0000","last_change":"2023-09-23T18:22:03.465235+0000","last_active":"2023-09-23T18:22:45.544371+0000","last_peered":"2023-09-23T18:22:45.544371+0000","last_clean":"2023-09-23T18:22:45.544371+0000","last_became_active":"2023-09-23T18:22:03.464906+0000","last_became_peered":"2023-09-23T18:22:03.464906+0000","last_unstale":"2023-09-23T18:22:45.544371+0000","last_undegraded":"2023-09-23T18:22:45.544371+0000","last_fullsized":"2023-09-23T18:22:45.544371+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:06:46.429598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":272,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516744+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:22:44.516744+0000","last_peered":"2023-09-23T18:22:44.516744+0000","last_clean":"2023-09-23T18:22:44.516744+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:22:44.516744+0000","last_undegraded":"2023-09-23T18:22:44.516744+0000","last_fullsized":"2023-09-23T18:22:44.516744+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":932,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513958+0000","last_change":"2023-09-23T18:22:06.468619+0000","last_active":"2023-09-23T18:22:44.513958+0000","last_peered":"2023-09-23T18:22:44.513958+0000","last_clean":"2023-09-23T18:22:44.513958+0000","last_became_active":"2023-09-23T18:22:06.468197+0000","last_became_peered":"2023-09-23T18:22:06.468197+0000","last_unstale":"2023-09-23T18:22:44.513958+0000","last_undegraded":"2023-09-23T18:22:44.513958+0000","last_fullsized":"2023-09-23T18:22:44.513958+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:10.149437+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":78,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512693+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:22:44.512693+0000","last_peered":"2023-09-23T18:22:44.512693+0000","last_clean":"2023-09-23T18:22:44.512693+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:22:44.512693+0000","last_undegraded":"2023-09-23T18:22:44.512693+0000","last_fullsized":"2023-09-23T18:22:44.512693+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":388,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T18:22:39.498605+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:22:39.498605+0000","last_peered":"2023-09-23T18:22:39.498605+0000","last_clean":"2023-09-23T18:22:39.498605+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:22:39.498605+0000","last_undegraded":"2023-09-23T18:22:39.498605+0000","last_fullsized":"2023-09-23T18:22:39.498605+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":771,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516996+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:22:44.516996+0000","last_peered":"2023-09-23T18:22:44.516996+0000","last_clean":"2023-09-23T18:22:44.516996+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:22:44.516996+0000","last_undegraded":"2023-09-23T18:22:44.516996+0000","last_fullsized":"2023-09-23T18:22:44.516996+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"653'291","reported_seq":1543,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T18:22:39.497772+0000","last_change":"2023-09-23T18:22:28.465602+0000","last_active":"2023-09-23T18:22:39.497772+0000","last_peered":"2023-09-23T18:22:39.497772+0000","last_clean":"2023-09-23T18:22:39.497772+0000","last_became_active":"2023-09-23T18:22:06.466146+0000","last_became_peered":"2023-09-23T18:22:06.466146+0000","last_unstale":"2023-09-23T18:22:39.497772+0000","last_undegraded":"2023-09-23T18:22:39.497772+0000","last_fullsized":"2023-09-23T18:22:39.497772+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":291,"log_dups_size":0,"ondisk_log_size":291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:22.022031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047292100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1326,"num_read_kb":5772,"num_write":300,"num_write_kb":4517,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":204,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514839+0000","last_change":"2023-09-23T18:22:03.462589+0000","last_active":"2023-09-23T18:22:44.514839+0000","last_peered":"2023-09-23T18:22:44.514839+0000","last_clean":"2023-09-23T18:22:44.514839+0000","last_became_active":"2023-09-23T18:22:03.462239+0000","last_became_peered":"2023-09-23T18:22:03.462239+0000","last_unstale":"2023-09-23T18:22:44.514839+0000","last_undegraded":"2023-09-23T18:22:44.514839+0000","last_fullsized":"2023-09-23T18:22:44.514839+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.020770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":174,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513865+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:22:44.513865+0000","last_peered":"2023-09-23T18:22:44.513865+0000","last_clean":"2023-09-23T18:22:44.513865+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:22:44.513865+0000","last_undegraded":"2023-09-23T18:22:44.513865+0000","last_fullsized":"2023-09-23T18:22:44.513865+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":863,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513643+0000","last_change":"2023-09-23T18:22:06.465690+0000","last_active":"2023-09-23T18:22:44.513643+0000","last_peered":"2023-09-23T18:22:44.513643+0000","last_clean":"2023-09-23T18:22:44.513643+0000","last_became_active":"2023-09-23T18:22:06.465148+0000","last_became_peered":"2023-09-23T18:22:06.465148+0000","last_unstale":"2023-09-23T18:22:44.513643+0000","last_undegraded":"2023-09-23T18:22:44.513643+0000","last_fullsized":"2023-09-23T18:22:44.513643+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:00.043030+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"653'323","reported_seq":2260,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513775+0000","last_change":"2023-09-23T18:22:28.473090+0000","last_active":"2023-09-23T18:22:44.513775+0000","last_peered":"2023-09-23T18:22:44.513775+0000","last_clean":"2023-09-23T18:22:44.513775+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:22:44.513775+0000","last_undegraded":"2023-09-23T18:22:44.513775+0000","last_fullsized":"2023-09-23T18:22:44.513775+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":323,"log_dups_size":0,"ondisk_log_size":323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001426075,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3093,"num_read_kb":5799,"num_write":265,"num_write_kb":3199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":184,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515869+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:22:44.515869+0000","last_peered":"2023-09-23T18:22:44.515869+0000","last_clean":"2023-09-23T18:22:44.515869+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:22:44.515869+0000","last_undegraded":"2023-09-23T18:22:44.515869+0000","last_fullsized":"2023-09-23T18:22:44.515869+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":413,"reported_epoch":659,"state":"peering","last_fresh":"2023-09-23T18:22:43.513526+0000","last_change":"2023-09-23T18:18:45.410660+0000","last_active":"2023-09-23T18:18:45.410409+0000","last_peered":"2023-09-23T18:18:37.455533+0000","last_clean":"2023-09-23T18:18:37.455533+0000","last_became_active":"2023-09-23T18:17:21.777074+0000","last_became_peered":"2023-09-23T18:17:21.777074+0000","last_unstale":"2023-09-23T18:22:43.513526+0000","last_undegraded":"2023-09-23T18:22:43.513526+0000","last_fullsized":"2023-09-23T18:22:43.513526+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:44.617263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":687,"reported_epoch":660,"state":"remapped+peering","last_fresh":"2023-09-23T18:22:44.510038+0000","last_change":"2023-09-23T18:22:44.510038+0000","last_active":"2023-09-23T18:11:39.495758+0000","last_peered":"2023-09-23T18:11:33.615512+0000","last_clean":"2023-09-23T18:11:33.615512+0000","last_became_active":"2023-09-23T18:11:30.565763+0000","last_became_peered":"2023-09-23T18:11:30.565763+0000","last_unstale":"2023-09-23T18:22:44.510038+0000","last_undegraded":"2023-09-23T18:22:44.510038+0000","last_fullsized":"2023-09-23T18:22:44.510038+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:12.113443+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"653'200","reported_seq":475,"reported_epoch":660,"state":"remapped+peering","last_fresh":"2023-09-23T18:22:44.509405+0000","last_change":"2023-09-23T18:22:44.509405+0000","last_active":"2023-09-23T18:22:44.509405+0000","last_peered":"2023-09-23T18:22:31.064269+0000","last_clean":"2023-09-23T18:22:31.064269+0000","last_became_active":"2023-09-23T18:18:46.432359+0000","last_became_peered":"2023-09-23T18:18:46.432359+0000","last_unstale":"2023-09-23T18:22:44.509405+0000","last_undegraded":"2023-09-23T18:22:44.509405+0000","last_fullsized":"2023-09-23T18:22:44.509405+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":613,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:45.768007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0022753629999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1032,"num_read_kb":1283,"num_write":194,"num_write_kb":5509,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":48,"num_bytes_recovered":16777216,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":463,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.519424+0000","last_change":"2023-09-23T18:22:44.518872+0000","last_active":"2023-09-23T18:22:44.519424+0000","last_peered":"2023-09-23T18:22:44.519424+0000","last_clean":"2023-09-23T18:22:44.519424+0000","last_became_active":"2023-09-23T18:22:44.518360+0000","last_became_peered":"2023-09-23T18:22:44.518360+0000","last_unstale":"2023-09-23T18:22:44.519424+0000","last_undegraded":"2023-09-23T18:22:44.519424+0000","last_fullsized":"2023-09-23T18:22:44.519424+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:43.986310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":196,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513310+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:22:44.513310+0000","last_peered":"2023-09-23T18:22:44.513310+0000","last_clean":"2023-09-23T18:22:44.513310+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:22:44.513310+0000","last_undegraded":"2023-09-23T18:22:44.513310+0000","last_fullsized":"2023-09-23T18:22:44.513310+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"653'474","reported_seq":2262,"reported_epoch":661,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:22:45.617550+0000","last_change":"2023-09-23T18:22:45.617748+0000","last_active":"2023-09-23T18:22:45.617550+0000","last_peered":"2023-09-23T18:22:45.617550+0000","last_clean":"2023-09-23T18:22:39.500537+0000","last_became_active":"2023-09-23T18:22:45.555209+0000","last_became_peered":"2023-09-23T18:22:45.555209+0000","last_unstale":"2023-09-23T18:22:45.617550+0000","last_undegraded":"2023-09-23T18:22:45.617550+0000","last_fullsized":"2023-09-23T18:22:45.539663+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":474,"log_dups_size":0,"ondisk_log_size":474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:49.153587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014671619999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2565,"num_read_kb":5830,"num_write":410,"num_write_kb":4052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,3],"avail_no_missing":["5","2","3","6"],"object_location_counts":[{"shards":"3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":610,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515026+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:22:44.515026+0000","last_peered":"2023-09-23T18:22:44.515026+0000","last_clean":"2023-09-23T18:22:44.515026+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:22:44.515026+0000","last_undegraded":"2023-09-23T18:22:44.515026+0000","last_fullsized":"2023-09-23T18:22:44.515026+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":170,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513686+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:22:44.513686+0000","last_peered":"2023-09-23T18:22:44.513686+0000","last_clean":"2023-09-23T18:22:44.513686+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:22:44.513686+0000","last_undegraded":"2023-09-23T18:22:44.513686+0000","last_fullsized":"2023-09-23T18:22:44.513686+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":205,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.517780+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:22:44.517780+0000","last_peered":"2023-09-23T18:22:44.517780+0000","last_clean":"2023-09-23T18:22:44.517780+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:22:44.517780+0000","last_undegraded":"2023-09-23T18:22:44.517780+0000","last_fullsized":"2023-09-23T18:22:44.517780+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":858,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516140+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:22:44.516140+0000","last_peered":"2023-09-23T18:22:44.516140+0000","last_clean":"2023-09-23T18:22:44.516140+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:22:44.516140+0000","last_undegraded":"2023-09-23T18:22:44.516140+0000","last_fullsized":"2023-09-23T18:22:44.516140+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"653'156","reported_seq":453,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512940+0000","last_change":"2023-09-23T18:22:28.481047+0000","last_active":"2023-09-23T18:22:44.512940+0000","last_peered":"2023-09-23T18:22:44.512940+0000","last_clean":"2023-09-23T18:22:44.512940+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:22:44.512940+0000","last_undegraded":"2023-09-23T18:22:44.512940+0000","last_fullsized":"2023-09-23T18:22:44.512940+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":156,"log_dups_size":0,"ondisk_log_size":156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504954,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":805,"num_read_kb":2049,"num_write":144,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":78,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514613+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:22:44.514613+0000","last_peered":"2023-09-23T18:22:44.514613+0000","last_clean":"2023-09-23T18:22:44.514613+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:22:44.514613+0000","last_undegraded":"2023-09-23T18:22:44.514613+0000","last_fullsized":"2023-09-23T18:22:44.514613+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":607,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.519254+0000","last_change":"2023-09-23T18:22:44.518222+0000","last_active":"2023-09-23T18:22:44.519254+0000","last_peered":"2023-09-23T18:22:44.519254+0000","last_clean":"2023-09-23T18:22:44.519254+0000","last_became_active":"2023-09-23T18:22:44.517848+0000","last_became_peered":"2023-09-23T18:22:44.517848+0000","last_unstale":"2023-09-23T18:22:44.519254+0000","last_undegraded":"2023-09-23T18:22:44.519254+0000","last_fullsized":"2023-09-23T18:22:44.519254+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:40.879308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":958,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516997+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:22:44.516997+0000","last_peered":"2023-09-23T18:22:44.516997+0000","last_clean":"2023-09-23T18:22:44.516997+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:22:44.516997+0000","last_undegraded":"2023-09-23T18:22:44.516997+0000","last_fullsized":"2023-09-23T18:22:44.516997+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"653'181","reported_seq":616,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.509057+0000","last_change":"2023-09-23T18:22:28.468611+0000","last_active":"2023-09-23T18:22:44.509057+0000","last_peered":"2023-09-23T18:22:44.509057+0000","last_clean":"2023-09-23T18:22:44.509057+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:22:44.509057+0000","last_undegraded":"2023-09-23T18:22:44.509057+0000","last_fullsized":"2023-09-23T18:22:44.509057+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001526466,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1349,"num_read_kb":1261,"num_write":144,"num_write_kb":4223,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":194,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.520250+0000","last_change":"2023-09-23T18:22:44.518634+0000","last_active":"2023-09-23T18:22:44.520250+0000","last_peered":"2023-09-23T18:22:44.520250+0000","last_clean":"2023-09-23T18:22:44.520250+0000","last_became_active":"2023-09-23T18:22:44.518286+0000","last_became_peered":"2023-09-23T18:22:44.518286+0000","last_unstale":"2023-09-23T18:22:44.520250+0000","last_undegraded":"2023-09-23T18:22:44.520250+0000","last_fullsized":"2023-09-23T18:22:44.520250+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:05.309943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":184,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513540+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:22:44.513540+0000","last_peered":"2023-09-23T18:22:44.513540+0000","last_clean":"2023-09-23T18:22:44.513540+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:22:44.513540+0000","last_undegraded":"2023-09-23T18:22:44.513540+0000","last_fullsized":"2023-09-23T18:22:44.513540+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":968,"reported_epoch":661,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:22:45.546784+0000","last_change":"2023-09-23T18:22:45.546653+0000","last_active":"2023-09-23T18:22:45.546784+0000","last_peered":"2023-09-23T18:22:45.546784+0000","last_clean":"2023-09-23T18:22:39.751536+0000","last_became_active":"2023-09-23T18:22:45.546653+0000","last_became_peered":"2023-09-23T18:22:45.546653+0000","last_unstale":"2023-09-23T18:22:45.546784+0000","last_undegraded":"2023-09-23T18:22:45.546784+0000","last_fullsized":"2023-09-23T18:22:45.541492+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.138528+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4],"avail_no_missing":["6","2","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":146,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513194+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:22:44.513194+0000","last_peered":"2023-09-23T18:22:44.513194+0000","last_clean":"2023-09-23T18:22:44.513194+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:22:44.513194+0000","last_undegraded":"2023-09-23T18:22:44.513194+0000","last_fullsized":"2023-09-23T18:22:44.513194+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"653'188","reported_seq":940,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516844+0000","last_change":"2023-09-23T18:22:28.475214+0000","last_active":"2023-09-23T18:22:44.516844+0000","last_peered":"2023-09-23T18:22:44.516844+0000","last_clean":"2023-09-23T18:22:44.516844+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:22:44.516844+0000","last_undegraded":"2023-09-23T18:22:44.516844+0000","last_fullsized":"2023-09-23T18:22:44.516844+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":188,"log_dups_size":0,"ondisk_log_size":188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055658400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1067,"num_read_kb":2442,"num_write":105,"num_write_kb":1356,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":79,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.541414+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:22:45.541414+0000","last_peered":"2023-09-23T18:22:45.541414+0000","last_clean":"2023-09-23T18:22:45.541414+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:22:45.541414+0000","last_undegraded":"2023-09-23T18:22:45.541414+0000","last_fullsized":"2023-09-23T18:22:45.541414+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":794,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514140+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:22:44.514140+0000","last_peered":"2023-09-23T18:22:44.514140+0000","last_clean":"2023-09-23T18:22:44.514140+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:22:44.514140+0000","last_undegraded":"2023-09-23T18:22:44.514140+0000","last_fullsized":"2023-09-23T18:22:44.514140+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"652'137","reported_seq":1316,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.517164+0000","last_change":"2023-09-23T18:22:28.475379+0000","last_active":"2023-09-23T18:22:44.517164+0000","last_peered":"2023-09-23T18:22:44.517164+0000","last_clean":"2023-09-23T18:22:44.517164+0000","last_became_active":"2023-09-23T18:22:03.468424+0000","last_became_peered":"2023-09-23T18:22:03.468424+0000","last_unstale":"2023-09-23T18:22:44.517164+0000","last_undegraded":"2023-09-23T18:22:44.517164+0000","last_fullsized":"2023-09-23T18:22:44.517164+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:46.671634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044965899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":925,"num_write":128,"num_write_kb":1260,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":87,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.510109+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:22:43.510109+0000","last_peered":"2023-09-23T18:22:43.510109+0000","last_clean":"2023-09-23T18:22:43.510109+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:22:43.510109+0000","last_undegraded":"2023-09-23T18:22:43.510109+0000","last_fullsized":"2023-09-23T18:22:43.510109+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":450,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513168+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:22:44.513168+0000","last_peered":"2023-09-23T18:22:44.513168+0000","last_clean":"2023-09-23T18:22:44.513168+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:22:44.513168+0000","last_undegraded":"2023-09-23T18:22:44.513168+0000","last_fullsized":"2023-09-23T18:22:44.513168+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":690,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514459+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:22:44.514459+0000","last_peered":"2023-09-23T18:22:44.514459+0000","last_clean":"2023-09-23T18:22:44.514459+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:22:44.514459+0000","last_undegraded":"2023-09-23T18:22:44.514459+0000","last_fullsized":"2023-09-23T18:22:44.514459+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":83,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.511633+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:22:43.511633+0000","last_peered":"2023-09-23T18:22:43.511633+0000","last_clean":"2023-09-23T18:22:43.511633+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:22:43.511633+0000","last_undegraded":"2023-09-23T18:22:43.511633+0000","last_fullsized":"2023-09-23T18:22:43.511633+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":731,"reported_epoch":656,"state":"active+clean","last_fresh":"2023-09-23T18:22:34.762268+0000","last_change":"2023-09-23T18:18:46.436895+0000","last_active":"2023-09-23T18:22:34.762268+0000","last_peered":"2023-09-23T18:22:34.762268+0000","last_clean":"2023-09-23T18:22:34.762268+0000","last_became_active":"2023-09-23T18:18:46.434229+0000","last_became_peered":"2023-09-23T18:18:46.434229+0000","last_unstale":"2023-09-23T18:22:34.762268+0000","last_undegraded":"2023-09-23T18:22:34.762268+0000","last_fullsized":"2023-09-23T18:22:34.762268+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:26:47.967983+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,0],"acting":[1,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":690,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516679+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:22:44.516679+0000","last_peered":"2023-09-23T18:22:44.516679+0000","last_clean":"2023-09-23T18:22:44.516679+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:22:44.516679+0000","last_undegraded":"2023-09-23T18:22:44.516679+0000","last_fullsized":"2023-09-23T18:22:44.516679+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":178,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513346+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:22:44.513346+0000","last_peered":"2023-09-23T18:22:44.513346+0000","last_clean":"2023-09-23T18:22:44.513346+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:22:44.513346+0000","last_undegraded":"2023-09-23T18:22:44.513346+0000","last_fullsized":"2023-09-23T18:22:44.513346+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":296,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.545245+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:22:45.545245+0000","last_peered":"2023-09-23T18:22:45.545245+0000","last_clean":"2023-09-23T18:22:45.545245+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:22:45.545245+0000","last_undegraded":"2023-09-23T18:22:45.545245+0000","last_fullsized":"2023-09-23T18:22:45.545245+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":164,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513297+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:22:44.513297+0000","last_peered":"2023-09-23T18:22:44.513297+0000","last_clean":"2023-09-23T18:22:44.513297+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:22:44.513297+0000","last_undegraded":"2023-09-23T18:22:44.513297+0000","last_fullsized":"2023-09-23T18:22:44.513297+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":279,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.545257+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:22:45.545257+0000","last_peered":"2023-09-23T18:22:45.545257+0000","last_clean":"2023-09-23T18:22:45.545257+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:22:45.545257+0000","last_undegraded":"2023-09-23T18:22:45.545257+0000","last_fullsized":"2023-09-23T18:22:45.545257+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1052,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516028+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:22:44.516028+0000","last_peered":"2023-09-23T18:22:44.516028+0000","last_clean":"2023-09-23T18:22:44.516028+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:22:44.516028+0000","last_undegraded":"2023-09-23T18:22:44.516028+0000","last_fullsized":"2023-09-23T18:22:44.516028+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":567,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.522694+0000","last_change":"2023-09-23T18:22:44.520133+0000","last_active":"2023-09-23T18:22:44.522694+0000","last_peered":"2023-09-23T18:22:44.522694+0000","last_clean":"2023-09-23T18:22:44.522694+0000","last_became_active":"2023-09-23T18:22:44.519765+0000","last_became_peered":"2023-09-23T18:22:44.519765+0000","last_unstale":"2023-09-23T18:22:44.522694+0000","last_undegraded":"2023-09-23T18:22:44.522694+0000","last_fullsized":"2023-09-23T18:22:44.522694+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:02.736149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.541741+0000","last_change":"2023-09-23T18:22:44.519760+0000","last_active":"2023-09-23T18:22:45.541741+0000","last_peered":"2023-09-23T18:22:45.541741+0000","last_clean":"2023-09-23T18:22:45.541741+0000","last_became_active":"2023-09-23T18:22:44.519006+0000","last_became_peered":"2023-09-23T18:22:44.519006+0000","last_unstale":"2023-09-23T18:22:45.541741+0000","last_undegraded":"2023-09-23T18:22:45.541741+0000","last_fullsized":"2023-09-23T18:22:45.541741+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:02.871094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":989,"reported_epoch":658,"state":"active+clean","last_fresh":"2023-09-23T18:22:39.497928+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:22:39.497928+0000","last_peered":"2023-09-23T18:22:39.497928+0000","last_clean":"2023-09-23T18:22:39.497928+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:22:39.497928+0000","last_undegraded":"2023-09-23T18:22:39.497928+0000","last_fullsized":"2023-09-23T18:22:39.497928+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":198,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.528984+0000","last_change":"2023-09-23T18:22:44.528526+0000","last_active":"2023-09-23T18:22:44.528984+0000","last_peered":"2023-09-23T18:22:44.528984+0000","last_clean":"2023-09-23T18:22:44.528984+0000","last_became_active":"2023-09-23T18:22:44.518445+0000","last_became_peered":"2023-09-23T18:22:44.518445+0000","last_unstale":"2023-09-23T18:22:44.528984+0000","last_undegraded":"2023-09-23T18:22:44.528984+0000","last_fullsized":"2023-09-23T18:22:44.528984+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:17.761911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":703,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514924+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:22:44.514924+0000","last_peered":"2023-09-23T18:22:44.514924+0000","last_clean":"2023-09-23T18:22:44.514924+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:22:44.514924+0000","last_undegraded":"2023-09-23T18:22:44.514924+0000","last_fullsized":"2023-09-23T18:22:44.514924+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":13010,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":15980,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":88,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":362,"num_bytes_recovered":16777273,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":22,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001601,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669538,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807536,"kb_used_data":174248,"kb_used_omap":7,"kb_used_meta":633272,"kb_avail":92937616,"statfs":{"total":95995035648,"available":95168118784,"internally_reserved":0,"allocated":178429952,"data_stored":176424787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":648471332},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:22 2023","interfaces":[{"interface":"back","average":{"1min":4.6609999999999996,"5min":2.7709999999999999,"15min":1.8129999999999999},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":79.203999999999994,"5min":79.203999999999994,"15min":79.203999999999994},"last":0.57199999999999995},{"interface":"front","average":{"1min":4.0229999999999997,"5min":2.7810000000000001,"15min":1.9570000000000001},"min":{"1min":0.252,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":78.034000000000006,"5min":78.034000000000006,"15min":78.034000000000006},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":2.1240000000000001,"15min":1.518},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.8480000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.627},{"interface":"front","average":{"1min":0.58399999999999996,"5min":2.1240000000000001,"15min":1.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.27,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.54800000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.518,"15min":1.369},"min":{"1min":0.33500000000000002,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":7.734,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.8120000000000001,"5min":2.0209999999999999,"15min":1.5920000000000001},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":8.2690000000000001,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.66500000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":4.6050000000000004,"5min":3.601,"15min":2.3479999999999999},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.24099999999999999},"max":{"1min":78.643000000000001,"5min":103.348,"15min":103.348},"last":0.61399999999999999},{"interface":"front","average":{"1min":5.2629999999999999,"5min":4.298,"15min":2.6930000000000001},"min":{"1min":0.32100000000000001,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":78.347999999999999,"5min":103.40300000000001,"15min":103.40300000000001},"last":0.60099999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":1.3220000000000001,"15min":1.3400000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23999999999999999},"max":{"1min":1.103,"5min":6.2800000000000002,"15min":8.6059999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.46100000000000002,"5min":1.2769999999999999,"15min":1.375},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":0.91200000000000003,"5min":7.8319999999999999,"15min":9.2260000000000009},"last":0.372}]},{"osd":5,"last update":"Sat Sep 23 18:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.8280000000000001,"15min":1.925},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":5.0460000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.9990000000000001,"15min":2.258},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0489999999999999,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.41699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.6930000000000001,"15min":1.603},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.26800000000000002},"max":{"1min":6.1799999999999997,"5min":10.743,"15min":13.460000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.593,"15min":1.6200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.28499999999999998},"max":{"1min":5.0990000000000002,"5min":8.6890000000000001,"15min":12.404},"last":0.52700000000000002}]}]},{"osd":1,"up_from":545,"seq":2340757176386,"num_pgs":59,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":691408,"kb_used_data":443464,"kb_used_omap":13,"kb_used_meta":247922,"kb_avail":93053744,"statfs":{"total":95995035648,"available":95287033856,"internally_reserved":0,"allocated":454107136,"data_stored":450899143,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":253872781},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.331,"15min":0.70399999999999996},"min":{"1min":0.23100000000000001,"5min":0.189,"15min":0.189},"max":{"1min":4.5030000000000001,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.28199999999999997},{"interface":"front","average":{"1min":0.38400000000000001,"5min":1.3069999999999999,"15min":0.76200000000000001},"min":{"1min":0.28399999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.51600000000000001,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.39200000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":1.4339999999999999,"15min":0.78500000000000003},"min":{"1min":0.28899999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.61899999999999999,"5min":11.02,"15min":11.02},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.98799999999999999,"15min":0.66600000000000004},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.234,"5min":5.1440000000000001,"15min":5.1440000000000001},"last":0.34300000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":1.7689999999999999,"15min":0.84699999999999998},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0980000000000001,"5min":14.962,"15min":14.962},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.577,"15min":0.86699999999999999},"min":{"1min":0.35499999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.82199999999999995,"5min":19.762,"15min":19.762},"last":0.435}]},{"osd":4,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.304,"15min":1.3120000000000001},"min":{"1min":0.38,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.532,"5min":12.663,"15min":12.663},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.1230000000000002,"15min":1.7170000000000001},"min":{"1min":0.39000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":4.7320000000000002,"5min":13.222,"15min":13.222},"last":0.752}]},{"osd":5,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.7350000000000001,"15min":1.0600000000000001},"min":{"1min":0.373,"5min":0.308,"15min":0.308},"max":{"1min":3.6760000000000002,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.877},{"interface":"front","average":{"1min":0.85699999999999998,"5min":2.052,"15min":1.22},"min":{"1min":0.42699999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.556,"5min":23.204000000000001,"15min":23.204000000000001},"last":0.70499999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.736,"15min":1.085},"min":{"1min":0.46999999999999997,"5min":0.372,"15min":0.372},"max":{"1min":3.5659999999999998,"5min":23.443000000000001,"15min":23.443000000000001},"last":0.90100000000000002},{"interface":"front","average":{"1min":0.84499999999999997,"5min":2.306,"15min":1.286},"min":{"1min":0.42499999999999999,"5min":0.378,"15min":0.378},"max":{"1min":4.8959999999999999,"5min":26.177,"15min":26.177},"last":0.71699999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.9399999999999999,"15min":1.1000000000000001},"min":{"1min":0.47399999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.052,"5min":15.01,"15min":15.01},"last":0.73699999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.867,"15min":1.159},"min":{"1min":0.46899999999999997,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78400000000000003,"5min":10.988,"15min":10.988},"last":0.84399999999999997}]}]},{"osd":0,"up_from":93,"seq":399431958883,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1163040,"kb_used_data":180056,"kb_used_omap":7,"kb_used_meta":982968,"kb_avail":92582112,"statfs":{"total":95995035648,"available":94804082688,"internally_reserved":0,"allocated":184377344,"data_stored":181337770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":1006559972},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.3839999999999999,"15min":0.67400000000000004},"min":{"1min":0.251,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":1.2170000000000001,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.63},{"interface":"front","average":{"1min":0.51100000000000001,"5min":1.8340000000000001,"15min":0.95999999999999996},"min":{"1min":0.28299999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.853,"5min":22.978000000000002,"15min":22.978000000000002},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:22:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":2.355,"15min":1.4790000000000001},"min":{"1min":0.25,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":9.8300000000000001,"5min":18.550999999999998,"15min":18.550999999999998},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.925,"15min":1.1419999999999999},"min":{"1min":0.31900000000000001,"5min":0.22500000000000001,"15min":0.22},"max":{"1min":6.2220000000000004,"5min":23.161999999999999,"15min":23.161999999999999},"last":0.50700000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:21:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.927,"15min":1.6890000000000001},"min":{"1min":0.24199999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.173999999999999,"5min":15.173999999999999,"15min":15.173999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.23,"5min":2.0350000000000001,"15min":1.718},"min":{"1min":0.32100000000000001,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":15.135,"5min":15.135,"15min":15.135},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:22:15 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.944,"15min":1.9299999999999999},"min":{"1min":0.33300000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9260000000000002,"5min":17.741,"15min":57.746000000000002},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.899,"15min":1.7150000000000001},"min":{"1min":0.42999999999999999,"5min":0.29199999999999998,"15min":0.25600000000000001},"max":{"1min":6.3860000000000001,"5min":15.079000000000001,"15min":57.816000000000003},"last":0.77600000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:21:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.73,"15min":1.466},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.3979999999999997,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.4610000000000001,"5min":2.3399999999999999,"15min":1.6279999999999999},"min":{"1min":0.35499999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.7359999999999998,"5min":15.047000000000001,"15min":15.047000000000001},"last":0.68600000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.8089999999999999,"15min":1.9870000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.32000000000000001},"max":{"1min":6.0439999999999996,"5min":7.6900000000000004,"15min":58.457999999999998},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.284,"5min":2.645,"15min":2.0739999999999998},"min":{"1min":0.47599999999999998,"5min":0.375,"15min":0.315},"max":{"1min":4.4470000000000001,"5min":22.893000000000001,"15min":58.354999999999997},"last":0.70199999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:21:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":2.2130000000000001,"15min":1.825},"min":{"1min":0.46200000000000002,"5min":0.45000000000000001,"15min":0.34499999999999997},"max":{"1min":6.1200000000000001,"5min":22.945,"15min":22.945},"last":0.65800000000000003},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.7410000000000001,"15min":1.851},"min":{"1min":0.42799999999999999,"5min":0.371,"15min":0.36199999999999999},"max":{"1min":5.4320000000000004,"5min":7.7519999999999998,"15min":10.404},"last":0.79600000000000004}]}]},{"osd":2,"up_from":482,"seq":2070174236768,"num_pgs":14,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":156652,"kb_used_data":8420,"kb_used_omap":22,"kb_used_meta":148201,"kb_avail":93588500,"statfs":{"total":95995035648,"available":95834624000,"internally_reserved":0,"allocated":8622080,"data_stored":5440356,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22605,"internal_metadata":151758771},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.5980000000000001,"15min":0.90900000000000003},"min":{"1min":0.24299999999999999,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":4.7060000000000004,"5min":9.6359999999999992,"15min":9.6359999999999992},"last":0.39300000000000002},{"interface":"front","average":{"1min":2.9289999999999998,"5min":1.964,"15min":1.286},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.3390000000000004,"5min":13.472,"15min":13.472},"last":0.46600000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.4219999999999999,"15min":0.80800000000000005},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.1320000000000001,"5min":13.538,"15min":13.538},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.504,"5min":1.681,"15min":0.98399999999999999},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.1499999999999999,"5min":16.567,"15min":16.567},"last":0.45500000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":2.1640000000000001,"15min":1.3919999999999999},"min":{"1min":0.40300000000000002,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":8.093,"5min":15.926,"15min":15.926},"last":0.53700000000000003},{"interface":"front","average":{"1min":1.841,"5min":1.754,"15min":1.409},"min":{"1min":0.28199999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":13.247999999999999,"5min":16.619,"15min":16.619},"last":0.42599999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:22:03 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.667,"15min":1.512},"min":{"1min":0.375,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":4.6539999999999999,"5min":9.532,"15min":11.334},"last":0.65500000000000003},{"interface":"front","average":{"1min":1.601,"5min":1.5589999999999999,"15min":1.1599999999999999},"min":{"1min":0.40000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.5229999999999997,"5min":6.367,"15min":11.244},"last":0.70799999999999996}]},{"osd":5,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.714,"15min":2.0339999999999998},"min":{"1min":0.38300000000000001,"5min":0.34799999999999998,"15min":0.308},"max":{"1min":9.4960000000000004,"5min":9.4960000000000004,"15min":11.852},"last":0.68100000000000005},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.6890000000000001,"15min":1.5980000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32600000000000001},"max":{"1min":4.6719999999999997,"5min":8.3849999999999998,"15min":10.942},"last":0.75800000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:22:03 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":2.0569999999999999,"15min":1.966},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":29.489000000000001,"5min":29.489000000000001,"15min":29.489000000000001},"last":0.67000000000000004},{"interface":"front","average":{"1min":3.048,"5min":2.1080000000000001,"15min":2.3759999999999999},"min":{"1min":0.42899999999999999,"5min":0.39600000000000002,"15min":0.38700000000000001},"max":{"1min":29.408999999999999,"5min":29.408999999999999,"15min":29.408999999999999},"last":0.77200000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":1.671,"15min":3.839},"min":{"1min":0.495,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":6.9779999999999998,"5min":13.106,"15min":76.966999999999999},"last":0.73099999999999998},{"interface":"front","average":{"1min":1.746,"5min":1.724,"15min":4.0179999999999998},"min":{"1min":0.45500000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":9.5619999999999994,"5min":9.5619999999999994,"15min":76.936999999999998},"last":0.63600000000000001}]}]},{"osd":3,"up_from":237,"seq":1017907249398,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818452,"kb_used_data":12940,"kb_used_omap":3,"kb_used_meta":805500,"kb_avail":92926700,"statfs":{"total":95995035648,"available":95156940800,"internally_reserved":0,"allocated":13250560,"data_stored":9997021,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":824832744},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.841,"15min":1.8700000000000001},"min":{"1min":0.20100000000000001,"5min":0.192,"15min":0.183},"max":{"1min":6.3360000000000003,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":1.925,"5min":1.7270000000000001,"15min":2.0590000000000002},"min":{"1min":0.311,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":8.9440000000000008,"5min":11.468,"15min":194.661},"last":0.247}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.847,"15min":0.96199999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":8.9629999999999992,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.7839999999999998,"15min":1.3979999999999999},"min":{"1min":0.29099999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.5600000000000001,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:22:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2090000000000001,"5min":2.0369999999999999,"15min":1.5009999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.21099999999999999},"max":{"1min":18.722999999999999,"5min":18.722999999999999,"15min":18.722999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.6699999999999999,"15min":1.3999999999999999},"min":{"1min":0.32800000000000001,"5min":0.27900000000000003,"15min":0.19600000000000001},"max":{"1min":7.0579999999999998,"5min":11.565,"15min":11.565},"last":0.314}]},{"osd":4,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":1.76,"15min":1.7330000000000001},"min":{"1min":0.36899999999999999,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":4.4089999999999998,"5min":20.850000000000001,"15min":37.231000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.98,"5min":1.99,"15min":1.9570000000000001},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":7.6840000000000002,"5min":20.786000000000001,"15min":37.381},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:21:42 2023","interfaces":[{"interface":"back","average":{"1min":2.6579999999999999,"5min":2.2709999999999999,"15min":1.6799999999999999},"min":{"1min":0.47899999999999998,"5min":0.316,"15min":0.316},"max":{"1min":19.434999999999999,"5min":19.434999999999999,"15min":19.434999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.647,"5min":1.7310000000000001,"15min":1.718},"min":{"1min":0.35999999999999999,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":4.4100000000000001,"5min":17.135999999999999,"15min":17.135999999999999},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.944,"15min":1.9570000000000001},"min":{"1min":0.45700000000000002,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":4.4489999999999998,"5min":17.192,"15min":115.54000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.268,"5min":1.7390000000000001,"15min":2.0030000000000001},"min":{"1min":0.42599999999999999,"5min":0.35699999999999998,"15min":0.30499999999999999},"max":{"1min":4.3840000000000003,"5min":11.442,"15min":115.691},"last":0.42899999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.7729999999999999,"15min":1.8160000000000001},"min":{"1min":0.39500000000000002,"5min":0.38700000000000001,"15min":0.35799999999999998},"max":{"1min":7.7679999999999998,"5min":9.1929999999999996,"15min":15.972},"last":0.44},{"interface":"front","average":{"1min":1.589,"5min":1.835,"15min":1.6599999999999999},"min":{"1min":0.49299999999999999,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":4.2279999999999998,"5min":17.164000000000001,"15min":17.164000000000001},"last":0.50700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019348,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":874612,"kb_used_data":169260,"kb_used_omap":0,"kb_used_meta":705344,"kb_avail":92870540,"statfs":{"total":95995035648,"available":95099432960,"internally_reserved":0,"allocated":173322240,"data_stored":171313325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":722272256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":3.8959999999999999,"5min":2.242,"15min":1.5209999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":72.754999999999995,"5min":72.754999999999995,"15min":72.754999999999995},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.9460000000000002,"5min":2.3500000000000001,"15min":1.7789999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":71.533000000000001,"5min":71.533000000000001,"15min":71.533000000000001},"last":0.55000000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:22:24 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":2.052,"15min":1.147},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.1399999999999997,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.9740000000000002,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.72499999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.655,"15min":1.77},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":4.0030000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.6399999999999999,"15min":1.702},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.253},"max":{"1min":3.3839999999999999,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.67600000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":2.6419999999999999,"15min":2.169},"min":{"1min":0.40200000000000002,"5min":0.32000000000000001,"15min":0.23799999999999999},"max":{"1min":0.80300000000000005,"5min":74.781999999999996,"15min":74.781999999999996},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":2.5670000000000002,"15min":2.1309999999999998},"min":{"1min":0.38600000000000001,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":0.78900000000000003,"5min":72.152000000000001,"15min":72.152000000000001},"last":0.77700000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":2.3460000000000001,"15min":2.7869999999999999},"min":{"1min":0.29199999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.73899999999999999,"5min":91.432000000000002,"15min":91.432000000000002},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":2.121,"15min":2.5640000000000001},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":0.70199999999999996,"5min":91.626999999999995,"15min":91.626999999999995},"last":0.70199999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":5.141,"5min":2.4329999999999998,"15min":1.7090000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":91.688999999999993,"5min":91.688999999999993,"15min":91.688999999999993},"last":0.66200000000000003},{"interface":"front","average":{"1min":5.1420000000000003,"5min":2.6120000000000001,"15min":1.9159999999999999},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":91.837000000000003,"5min":91.837000000000003,"15min":91.837000000000003},"last":0.76200000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":1.798,"15min":1.835},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":3.6800000000000002,"5min":18.283000000000001,"15min":19.298999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.4470000000000001,"15min":1.5529999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.28399999999999997},"max":{"1min":4.4269999999999996,"5min":9.4730000000000008,"15min":12.800000000000001},"last":0.80500000000000005}]}]},{"osd":5,"up_from":420,"seq":1803886264444,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":476648,"kb_used_data":103712,"kb_used_omap":9,"kb_used_meta":372918,"kb_avail":93268504,"statfs":{"total":95995035648,"available":95506948096,"internally_reserved":0,"allocated":106201088,"data_stored":104200445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":381868477},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":4.7930000000000001,"5min":1.889,"15min":1.2090000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":77.162000000000006,"5min":77.162000000000006,"15min":77.162000000000006},"last":0.46800000000000003},{"interface":"front","average":{"1min":4.9450000000000003,"5min":2.4169999999999998,"15min":1.8720000000000001},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":75.784999999999997,"5min":75.784999999999997,"15min":75.784999999999997},"last":0.81399999999999995}]},{"osd":1,"last update":"Sat Sep 23 18:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.998,"15min":1.3959999999999999},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.4819999999999993,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.133,"15min":1.2749999999999999},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.2759999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":0.69599999999999995}]},{"osd":2,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.615,"15min":1.611},"min":{"1min":0.31,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":8.8789999999999996,"5min":10.705,"15min":10.705},"last":0.752},{"interface":"front","average":{"1min":1.302,"5min":1.673,"15min":2.0640000000000001},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.3390000000000004,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.85699999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":5.4400000000000004,"5min":2.5369999999999999,"15min":1.6579999999999999},"min":{"1min":0.317,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":79.016999999999996,"5min":79.016999999999996,"15min":79.016999999999996},"last":0.83899999999999997},{"interface":"front","average":{"1min":5.2060000000000004,"5min":2.7810000000000001,"15min":1.9590000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.313},"max":{"1min":76.188999999999993,"5min":76.188999999999993,"15min":76.188999999999993},"last":0.77000000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.718,"15min":2.016},"min":{"1min":0.29199999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.9930000000000003,"5min":8.2270000000000003,"15min":12.747999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.7689999999999999,"15min":1.647},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":8.5559999999999992,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.66200000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.3340000000000001,"15min":1.2929999999999999},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.7859999999999996,"5min":6.7859999999999996,"15min":9.0350000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.119,"5min":1.859,"15min":1.716},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.0890000000000004,"5min":12.504,"15min":12.504},"last":0.92400000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.704,"15min":1.806},"min":{"1min":0.33700000000000002,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":6.9569999999999999,"5min":10.528,"15min":13.506},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.494,"15min":1.5860000000000001},"min":{"1min":0.39900000000000002,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":12.702,"5min":12.702,"15min":13.138},"last":0.88}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":8392704,"data_stored":8388627,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4768,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:22:48.312 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:22:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:48 smithi099 ceph-mon[167076]: pgmap v1586: 103 pgs: 5 active+undersized+remapped, 2 remapped+peering, 1 peering, 95 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 64 KiB/s, 7 objects/s recovering 2023-09-23T18:22:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:48 smithi099 ceph-mon[167076]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:48 smithi099 ceph-mon[167076]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:48 smithi099 ceph-mon[167076]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:48 smithi099 ceph-mon[167076]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:48 smithi099 ceph-mon[167076]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:48 smithi099 ceph-mon[167076]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:48 smithi099 ceph-mon[167076]: osdmap e663: 8 total, 8 up, 7 in 2023-09-23T18:22:48.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:48 smithi099 ceph-mon[167076]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:48 smithi099 ceph-mon[167076]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.920 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:48.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[179668]: pgmap v1586: 103 pgs: 5 active+undersized+remapped, 2 remapped+peering, 1 peering, 95 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 64 KiB/s, 7 objects/s recovering 2023-09-23T18:22:48.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[179668]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[179668]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[179668]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[179668]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[179668]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[179668]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[179668]: osdmap e663: 8 total, 8 up, 7 in 2023-09-23T18:22:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[179668]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[179668]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[184174]: pgmap v1586: 103 pgs: 5 active+undersized+remapped, 2 remapped+peering, 1 peering, 95 active+clean; 8.6 MiB data, 4.8 GiB used, 621 GiB / 626 GiB avail; 64 KiB/s, 7 objects/s recovering 2023-09-23T18:22:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[184174]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[184174]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[184174]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[184174]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[184174]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[184174]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[184174]: osdmap e663: 8 total, 8 up, 7 in 2023-09-23T18:22:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[184174]: failed to encode map e663 with expected crc 2023-09-23T18:22:48.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:48 smithi055 ceph-mon[184174]: failed to encode map e663 with expected crc 2023-09-23T18:22:49.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:49 smithi055 ceph-mon[179668]: from='client.38259 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:22:49.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:49 smithi055 ceph-mon[184174]: from='client.38259 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:22:49.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:49 smithi099 ceph-mon[167076]: from='client.38259 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:22:50.240 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:22:50.242 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:22:50.562 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:22:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:22:50] "GET /metrics HTTP/1.1" 200 34232 "" "Prometheus/2.43.0" 2023-09-23T18:22:50.570 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1588,"stamp":"2023-09-23T18:22:49.235513+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":484,"num_bytes_recovered":21509963,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":303,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":329,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4969516,"kb_used_data":1073268,"kb_used_omap":62,"kb_used_meta":3896129,"kb_avail":651246548,"statfs":{"total":671965249536,"available":666876465152,"internally_reserved":0,"allocated":1099026432,"data_stored":1080290757,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":64275,"internal_metadata":3989636333},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":2,"apply_latency_ms":2,"commit_latency_ns":2000000,"apply_latency_ns":2000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":51,"num_bytes_recovered":4457546,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"4.729011"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":962,"reported_epoch":661,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:22:45.552442+0000","last_change":"2023-09-23T18:22:45.552649+0000","last_active":"2023-09-23T18:22:45.552442+0000","last_peered":"2023-09-23T18:22:45.552442+0000","last_clean":"2023-09-23T18:22:39.500492+0000","last_became_active":"2023-09-23T18:22:45.548325+0000","last_became_peered":"2023-09-23T18:22:45.548325+0000","last_unstale":"2023-09-23T18:22:45.552442+0000","last_undegraded":"2023-09-23T18:22:45.552442+0000","last_fullsized":"2023-09-23T18:22:45.539208+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:00:13.640598+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,1],"avail_no_missing":["5","0","1","2"],"object_location_counts":[{"shards":"1,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":331,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.545247+0000","last_change":"2023-09-23T18:22:44.528866+0000","last_active":"2023-09-23T18:22:45.545247+0000","last_peered":"2023-09-23T18:22:45.545247+0000","last_clean":"2023-09-23T18:22:45.545247+0000","last_became_active":"2023-09-23T18:22:44.528499+0000","last_became_peered":"2023-09-23T18:22:44.528499+0000","last_unstale":"2023-09-23T18:22:45.545247+0000","last_undegraded":"2023-09-23T18:22:45.545247+0000","last_fullsized":"2023-09-23T18:22:45.545247+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:26.288744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":78,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512987+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:22:44.512987+0000","last_peered":"2023-09-23T18:22:44.512987+0000","last_clean":"2023-09-23T18:22:44.512987+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:22:44.512987+0000","last_undegraded":"2023-09-23T18:22:44.512987+0000","last_fullsized":"2023-09-23T18:22:44.512987+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":954,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515593+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:22:44.515593+0000","last_peered":"2023-09-23T18:22:44.515593+0000","last_clean":"2023-09-23T18:22:44.515593+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:22:44.515593+0000","last_undegraded":"2023-09-23T18:22:44.515593+0000","last_fullsized":"2023-09-23T18:22:44.515593+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":402,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514671+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:22:44.514671+0000","last_peered":"2023-09-23T18:22:44.514671+0000","last_clean":"2023-09-23T18:22:44.514671+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:22:44.514671+0000","last_undegraded":"2023-09-23T18:22:44.514671+0000","last_fullsized":"2023-09-23T18:22:44.514671+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":721,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.509087+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:22:43.509087+0000","last_peered":"2023-09-23T18:22:43.509087+0000","last_clean":"2023-09-23T18:22:43.509087+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:22:43.509087+0000","last_undegraded":"2023-09-23T18:22:43.509087+0000","last_fullsized":"2023-09-23T18:22:43.509087+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":971,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513513+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:22:44.513513+0000","last_peered":"2023-09-23T18:22:44.513513+0000","last_clean":"2023-09-23T18:22:44.513513+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:22:44.513513+0000","last_undegraded":"2023-09-23T18:22:44.513513+0000","last_fullsized":"2023-09-23T18:22:44.513513+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":78,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512080+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:22:44.512080+0000","last_peered":"2023-09-23T18:22:44.512080+0000","last_clean":"2023-09-23T18:22:44.512080+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:22:44.512080+0000","last_undegraded":"2023-09-23T18:22:44.512080+0000","last_fullsized":"2023-09-23T18:22:44.512080+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":236,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516913+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:22:44.516913+0000","last_peered":"2023-09-23T18:22:44.516913+0000","last_clean":"2023-09-23T18:22:44.516913+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:22:44.516913+0000","last_undegraded":"2023-09-23T18:22:44.516913+0000","last_fullsized":"2023-09-23T18:22:44.516913+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1094,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513069+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:22:44.513069+0000","last_peered":"2023-09-23T18:22:44.513069+0000","last_clean":"2023-09-23T18:22:44.513069+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:22:44.513069+0000","last_undegraded":"2023-09-23T18:22:44.513069+0000","last_fullsized":"2023-09-23T18:22:44.513069+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":170,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513333+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:22:44.513333+0000","last_peered":"2023-09-23T18:22:44.513333+0000","last_clean":"2023-09-23T18:22:44.513333+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:22:44.513333+0000","last_undegraded":"2023-09-23T18:22:44.513333+0000","last_fullsized":"2023-09-23T18:22:44.513333+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":251,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513028+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:22:44.513028+0000","last_peered":"2023-09-23T18:22:44.513028+0000","last_clean":"2023-09-23T18:22:44.513028+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:22:44.513028+0000","last_undegraded":"2023-09-23T18:22:44.513028+0000","last_fullsized":"2023-09-23T18:22:44.513028+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":842,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.565289+0000","last_change":"2023-09-23T18:22:06.467889+0000","last_active":"2023-09-23T18:22:46.565289+0000","last_peered":"2023-09-23T18:22:46.565289+0000","last_clean":"2023-09-23T18:22:46.565289+0000","last_became_active":"2023-09-23T18:22:06.467521+0000","last_became_peered":"2023-09-23T18:22:06.467521+0000","last_unstale":"2023-09-23T18:22:46.565289+0000","last_undegraded":"2023-09-23T18:22:46.565289+0000","last_fullsized":"2023-09-23T18:22:46.565289+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:07.204206+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":206,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512839+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:22:44.512839+0000","last_peered":"2023-09-23T18:22:44.512839+0000","last_clean":"2023-09-23T18:22:44.512839+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:22:44.512839+0000","last_undegraded":"2023-09-23T18:22:44.512839+0000","last_fullsized":"2023-09-23T18:22:44.512839+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":205,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514365+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:22:44.514365+0000","last_peered":"2023-09-23T18:22:44.514365+0000","last_clean":"2023-09-23T18:22:44.514365+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:22:44.514365+0000","last_undegraded":"2023-09-23T18:22:44.514365+0000","last_fullsized":"2023-09-23T18:22:44.514365+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":1005,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513638+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:22:44.513638+0000","last_peered":"2023-09-23T18:22:44.513638+0000","last_clean":"2023-09-23T18:22:44.513638+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:22:44.513638+0000","last_undegraded":"2023-09-23T18:22:44.513638+0000","last_fullsized":"2023-09-23T18:22:44.513638+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":456,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513558+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:22:44.513558+0000","last_peered":"2023-09-23T18:22:44.513558+0000","last_clean":"2023-09-23T18:22:44.513558+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:22:44.513558+0000","last_undegraded":"2023-09-23T18:22:44.513558+0000","last_fullsized":"2023-09-23T18:22:44.513558+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":483,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516017+0000","last_change":"2023-09-23T18:22:03.468342+0000","last_active":"2023-09-23T18:22:44.516017+0000","last_peered":"2023-09-23T18:22:44.516017+0000","last_clean":"2023-09-23T18:22:44.516017+0000","last_became_active":"2023-09-23T18:22:03.467804+0000","last_became_peered":"2023-09-23T18:22:03.467804+0000","last_unstale":"2023-09-23T18:22:44.516017+0000","last_undegraded":"2023-09-23T18:22:44.516017+0000","last_fullsized":"2023-09-23T18:22:44.516017+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:17.762831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":1038,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558902+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:22:46.558902+0000","last_peered":"2023-09-23T18:22:46.558902+0000","last_clean":"2023-09-23T18:22:46.558902+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:22:46.558902+0000","last_undegraded":"2023-09-23T18:22:46.558902+0000","last_fullsized":"2023-09-23T18:22:46.558902+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":232,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516384+0000","last_change":"2023-09-23T18:22:03.479912+0000","last_active":"2023-09-23T18:22:44.516384+0000","last_peered":"2023-09-23T18:22:44.516384+0000","last_clean":"2023-09-23T18:22:44.516384+0000","last_became_active":"2023-09-23T18:22:03.464545+0000","last_became_peered":"2023-09-23T18:22:03.464545+0000","last_unstale":"2023-09-23T18:22:44.516384+0000","last_undegraded":"2023-09-23T18:22:44.516384+0000","last_fullsized":"2023-09-23T18:22:44.516384+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:50.025694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":479,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515985+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:22:44.515985+0000","last_peered":"2023-09-23T18:22:44.515985+0000","last_clean":"2023-09-23T18:22:44.515985+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:22:44.515985+0000","last_undegraded":"2023-09-23T18:22:44.515985+0000","last_fullsized":"2023-09-23T18:22:44.515985+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":886,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513283+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:22:44.513283+0000","last_peered":"2023-09-23T18:22:44.513283+0000","last_clean":"2023-09-23T18:22:44.513283+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:22:44.513283+0000","last_undegraded":"2023-09-23T18:22:44.513283+0000","last_fullsized":"2023-09-23T18:22:44.513283+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":495,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515712+0000","last_change":"2023-09-23T18:22:03.475293+0000","last_active":"2023-09-23T18:22:44.515712+0000","last_peered":"2023-09-23T18:22:44.515712+0000","last_clean":"2023-09-23T18:22:44.515712+0000","last_became_active":"2023-09-23T18:22:03.465603+0000","last_became_peered":"2023-09-23T18:22:03.465603+0000","last_unstale":"2023-09-23T18:22:44.515712+0000","last_undegraded":"2023-09-23T18:22:44.515712+0000","last_fullsized":"2023-09-23T18:22:44.515712+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:11.349439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":105,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.509910+0000","last_change":"2023-09-23T18:22:03.464544+0000","last_active":"2023-09-23T18:22:43.509910+0000","last_peered":"2023-09-23T18:22:43.509910+0000","last_clean":"2023-09-23T18:22:43.509910+0000","last_became_active":"2023-09-23T18:22:03.464227+0000","last_became_peered":"2023-09-23T18:22:03.464227+0000","last_unstale":"2023-09-23T18:22:43.509910+0000","last_undegraded":"2023-09-23T18:22:43.509910+0000","last_fullsized":"2023-09-23T18:22:43.509910+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:56.368635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":857,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514280+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:22:44.514280+0000","last_peered":"2023-09-23T18:22:44.514280+0000","last_clean":"2023-09-23T18:22:44.514280+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:22:44.514280+0000","last_undegraded":"2023-09-23T18:22:44.514280+0000","last_fullsized":"2023-09-23T18:22:44.514280+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":176,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.508723+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:22:43.508723+0000","last_peered":"2023-09-23T18:22:43.508723+0000","last_clean":"2023-09-23T18:22:43.508723+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:22:43.508723+0000","last_undegraded":"2023-09-23T18:22:43.508723+0000","last_fullsized":"2023-09-23T18:22:43.508723+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":170,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513402+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:22:44.513402+0000","last_peered":"2023-09-23T18:22:44.513402+0000","last_clean":"2023-09-23T18:22:44.513402+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:22:44.513402+0000","last_undegraded":"2023-09-23T18:22:44.513402+0000","last_fullsized":"2023-09-23T18:22:44.513402+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":797,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512324+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:22:44.512324+0000","last_peered":"2023-09-23T18:22:44.512324+0000","last_clean":"2023-09-23T18:22:44.512324+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:22:44.512324+0000","last_undegraded":"2023-09-23T18:22:44.512324+0000","last_fullsized":"2023-09-23T18:22:44.512324+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":247,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514273+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:22:44.514273+0000","last_peered":"2023-09-23T18:22:44.514273+0000","last_clean":"2023-09-23T18:22:44.514273+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:22:44.514273+0000","last_undegraded":"2023-09-23T18:22:44.514273+0000","last_fullsized":"2023-09-23T18:22:44.514273+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":269,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514420+0000","last_change":"2023-09-23T18:22:03.464607+0000","last_active":"2023-09-23T18:22:44.514420+0000","last_peered":"2023-09-23T18:22:44.514420+0000","last_clean":"2023-09-23T18:22:44.514420+0000","last_became_active":"2023-09-23T18:22:03.464333+0000","last_became_peered":"2023-09-23T18:22:03.464333+0000","last_unstale":"2023-09-23T18:22:44.514420+0000","last_undegraded":"2023-09-23T18:22:44.514420+0000","last_fullsized":"2023-09-23T18:22:44.514420+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:51.062916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":891,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514205+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:22:44.514205+0000","last_peered":"2023-09-23T18:22:44.514205+0000","last_clean":"2023-09-23T18:22:44.514205+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:22:44.514205+0000","last_undegraded":"2023-09-23T18:22:44.514205+0000","last_fullsized":"2023-09-23T18:22:44.514205+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":800,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514831+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:22:44.514831+0000","last_peered":"2023-09-23T18:22:44.514831+0000","last_clean":"2023-09-23T18:22:44.514831+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:22:44.514831+0000","last_undegraded":"2023-09-23T18:22:44.514831+0000","last_fullsized":"2023-09-23T18:22:44.514831+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":87,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513355+0000","last_change":"2023-09-23T18:22:03.468997+0000","last_active":"2023-09-23T18:22:44.513355+0000","last_peered":"2023-09-23T18:22:44.513355+0000","last_clean":"2023-09-23T18:22:44.513355+0000","last_became_active":"2023-09-23T18:22:03.468301+0000","last_became_peered":"2023-09-23T18:22:03.468301+0000","last_unstale":"2023-09-23T18:22:44.513355+0000","last_undegraded":"2023-09-23T18:22:44.513355+0000","last_fullsized":"2023-09-23T18:22:44.513355+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:32:17.742551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":910,"reported_epoch":662,"state":"peering","last_fresh":"2023-09-23T18:22:46.560855+0000","last_change":"2023-09-23T18:22:46.561691+0000","last_active":"2023-09-23T18:22:46.560818+0000","last_peered":"2023-09-23T18:22:45.549690+0000","last_clean":"2023-09-23T18:17:50.936418+0000","last_became_active":"2023-09-23T18:22:45.549353+0000","last_became_peered":"2023-09-23T18:22:45.549353+0000","last_unstale":"2023-09-23T18:22:46.560855+0000","last_undegraded":"2023-09-23T18:22:46.560855+0000","last_fullsized":"2023-09-23T18:22:46.560855+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:28:05.452199+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[1,2,6],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":298,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.546257+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:22:45.546257+0000","last_peered":"2023-09-23T18:22:45.546257+0000","last_clean":"2023-09-23T18:22:45.546257+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:22:45.546257+0000","last_undegraded":"2023-09-23T18:22:45.546257+0000","last_fullsized":"2023-09-23T18:22:45.546257+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":170,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514387+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:22:44.514387+0000","last_peered":"2023-09-23T18:22:44.514387+0000","last_clean":"2023-09-23T18:22:44.514387+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:22:44.514387+0000","last_undegraded":"2023-09-23T18:22:44.514387+0000","last_fullsized":"2023-09-23T18:22:44.514387+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":793,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514045+0000","last_change":"2023-09-23T18:21:51.851401+0000","last_active":"2023-09-23T18:22:44.514045+0000","last_peered":"2023-09-23T18:22:44.514045+0000","last_clean":"2023-09-23T18:22:44.514045+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:22:44.514045+0000","last_undegraded":"2023-09-23T18:22:44.514045+0000","last_fullsized":"2023-09-23T18:22:44.514045+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:21:51.851269+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:21:51.851269+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:48.305008+0000","scrub_duration":0.001933539,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":299,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.546237+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:22:45.546237+0000","last_peered":"2023-09-23T18:22:45.546237+0000","last_clean":"2023-09-23T18:22:45.546237+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:22:45.546237+0000","last_undegraded":"2023-09-23T18:22:45.546237+0000","last_fullsized":"2023-09-23T18:22:45.546237+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":278,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.544150+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:22:45.544150+0000","last_peered":"2023-09-23T18:22:45.544150+0000","last_clean":"2023-09-23T18:22:45.544150+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:22:45.544150+0000","last_undegraded":"2023-09-23T18:22:45.544150+0000","last_fullsized":"2023-09-23T18:22:45.544150+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":675,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516796+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:22:44.516796+0000","last_peered":"2023-09-23T18:22:44.516796+0000","last_clean":"2023-09-23T18:22:44.516796+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:22:44.516796+0000","last_undegraded":"2023-09-23T18:22:44.516796+0000","last_fullsized":"2023-09-23T18:22:44.516796+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":585,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513180+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:22:44.513180+0000","last_peered":"2023-09-23T18:22:44.513180+0000","last_clean":"2023-09-23T18:22:44.513180+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:22:44.513180+0000","last_undegraded":"2023-09-23T18:22:44.513180+0000","last_fullsized":"2023-09-23T18:22:44.513180+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":236,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515842+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:22:44.515842+0000","last_peered":"2023-09-23T18:22:44.515842+0000","last_clean":"2023-09-23T18:22:44.515842+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:22:44.515842+0000","last_undegraded":"2023-09-23T18:22:44.515842+0000","last_fullsized":"2023-09-23T18:22:44.515842+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":701,"reported_epoch":661,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:22:45.550767+0000","last_change":"2023-09-23T18:22:45.550653+0000","last_active":"2023-09-23T18:22:45.550767+0000","last_peered":"2023-09-23T18:22:45.550767+0000","last_clean":"2023-09-23T18:22:40.227293+0000","last_became_active":"2023-09-23T18:22:45.550653+0000","last_became_peered":"2023-09-23T18:22:45.550653+0000","last_unstale":"2023-09-23T18:22:45.550767+0000","last_undegraded":"2023-09-23T18:22:45.550767+0000","last_fullsized":"2023-09-23T18:22:45.543763+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:30:33.107993+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4],"avail_no_missing":["3","1","2","4"],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":184,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513631+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:22:44.513631+0000","last_peered":"2023-09-23T18:22:44.513631+0000","last_clean":"2023-09-23T18:22:44.513631+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:22:44.513631+0000","last_undegraded":"2023-09-23T18:22:44.513631+0000","last_fullsized":"2023-09-23T18:22:44.513631+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":498,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.511786+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:22:43.511786+0000","last_peered":"2023-09-23T18:22:43.511786+0000","last_clean":"2023-09-23T18:22:43.511786+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:22:43.511786+0000","last_undegraded":"2023-09-23T18:22:43.511786+0000","last_fullsized":"2023-09-23T18:22:43.511786+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1152,"reported_epoch":661,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:22:45.543394+0000","last_change":"2023-09-23T18:22:45.543277+0000","last_active":"2023-09-23T18:22:45.543394+0000","last_peered":"2023-09-23T18:22:45.543394+0000","last_clean":"2023-09-23T18:22:39.500356+0000","last_became_active":"2023-09-23T18:22:45.543277+0000","last_became_peered":"2023-09-23T18:22:45.543277+0000","last_unstale":"2023-09-23T18:22:45.543394+0000","last_undegraded":"2023-09-23T18:22:45.543394+0000","last_fullsized":"2023-09-23T18:22:45.539213+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:51:28.644193+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3],"avail_no_missing":["5","2","3","6"],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":399,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513662+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:22:44.513662+0000","last_peered":"2023-09-23T18:22:44.513662+0000","last_clean":"2023-09-23T18:22:44.513662+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:22:44.513662+0000","last_undegraded":"2023-09-23T18:22:44.513662+0000","last_fullsized":"2023-09-23T18:22:44.513662+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":204,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516130+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:22:44.516130+0000","last_peered":"2023-09-23T18:22:44.516130+0000","last_clean":"2023-09-23T18:22:44.516130+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:22:44.516130+0000","last_undegraded":"2023-09-23T18:22:44.516130+0000","last_fullsized":"2023-09-23T18:22:44.516130+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":949,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514576+0000","last_change":"2023-09-23T18:22:06.466040+0000","last_active":"2023-09-23T18:22:44.514576+0000","last_peered":"2023-09-23T18:22:44.514576+0000","last_clean":"2023-09-23T18:22:44.514576+0000","last_became_active":"2023-09-23T18:22:06.465158+0000","last_became_peered":"2023-09-23T18:22:06.465158+0000","last_unstale":"2023-09-23T18:22:44.514576+0000","last_undegraded":"2023-09-23T18:22:44.514576+0000","last_fullsized":"2023-09-23T18:22:44.514576+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:29.587874+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":212,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515914+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:22:44.515914+0000","last_peered":"2023-09-23T18:22:44.515914+0000","last_clean":"2023-09-23T18:22:44.515914+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:22:44.515914+0000","last_undegraded":"2023-09-23T18:22:44.515914+0000","last_fullsized":"2023-09-23T18:22:44.515914+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":456,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.546207+0000","last_change":"2023-09-23T18:22:44.527193+0000","last_active":"2023-09-23T18:22:45.546207+0000","last_peered":"2023-09-23T18:22:45.546207+0000","last_clean":"2023-09-23T18:22:45.546207+0000","last_became_active":"2023-09-23T18:22:44.526903+0000","last_became_peered":"2023-09-23T18:22:44.526903+0000","last_unstale":"2023-09-23T18:22:45.546207+0000","last_undegraded":"2023-09-23T18:22:45.546207+0000","last_fullsized":"2023-09-23T18:22:45.546207+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:09.126029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":931,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515692+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:22:44.515692+0000","last_peered":"2023-09-23T18:22:44.515692+0000","last_clean":"2023-09-23T18:22:44.515692+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:22:44.515692+0000","last_undegraded":"2023-09-23T18:22:44.515692+0000","last_fullsized":"2023-09-23T18:22:44.515692+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":466,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.544371+0000","last_change":"2023-09-23T18:22:03.465235+0000","last_active":"2023-09-23T18:22:45.544371+0000","last_peered":"2023-09-23T18:22:45.544371+0000","last_clean":"2023-09-23T18:22:45.544371+0000","last_became_active":"2023-09-23T18:22:03.464906+0000","last_became_peered":"2023-09-23T18:22:03.464906+0000","last_unstale":"2023-09-23T18:22:45.544371+0000","last_undegraded":"2023-09-23T18:22:45.544371+0000","last_fullsized":"2023-09-23T18:22:45.544371+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:06:46.429598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":272,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516744+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:22:44.516744+0000","last_peered":"2023-09-23T18:22:44.516744+0000","last_clean":"2023-09-23T18:22:44.516744+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:22:44.516744+0000","last_undegraded":"2023-09-23T18:22:44.516744+0000","last_fullsized":"2023-09-23T18:22:44.516744+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":932,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513958+0000","last_change":"2023-09-23T18:22:06.468619+0000","last_active":"2023-09-23T18:22:44.513958+0000","last_peered":"2023-09-23T18:22:44.513958+0000","last_clean":"2023-09-23T18:22:44.513958+0000","last_became_active":"2023-09-23T18:22:06.468197+0000","last_became_peered":"2023-09-23T18:22:06.468197+0000","last_unstale":"2023-09-23T18:22:44.513958+0000","last_undegraded":"2023-09-23T18:22:44.513958+0000","last_fullsized":"2023-09-23T18:22:44.513958+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:10.149437+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":78,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512693+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:22:44.512693+0000","last_peered":"2023-09-23T18:22:44.512693+0000","last_clean":"2023-09-23T18:22:44.512693+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:22:44.512693+0000","last_undegraded":"2023-09-23T18:22:44.512693+0000","last_fullsized":"2023-09-23T18:22:44.512693+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":392,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558148+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:22:46.558148+0000","last_peered":"2023-09-23T18:22:46.558148+0000","last_clean":"2023-09-23T18:22:46.558148+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:22:46.558148+0000","last_undegraded":"2023-09-23T18:22:46.558148+0000","last_fullsized":"2023-09-23T18:22:46.558148+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":771,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516996+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:22:44.516996+0000","last_peered":"2023-09-23T18:22:44.516996+0000","last_clean":"2023-09-23T18:22:44.516996+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:22:44.516996+0000","last_undegraded":"2023-09-23T18:22:44.516996+0000","last_fullsized":"2023-09-23T18:22:44.516996+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"653'291","reported_seq":1547,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557793+0000","last_change":"2023-09-23T18:22:28.465602+0000","last_active":"2023-09-23T18:22:46.557793+0000","last_peered":"2023-09-23T18:22:46.557793+0000","last_clean":"2023-09-23T18:22:46.557793+0000","last_became_active":"2023-09-23T18:22:06.466146+0000","last_became_peered":"2023-09-23T18:22:06.466146+0000","last_unstale":"2023-09-23T18:22:46.557793+0000","last_undegraded":"2023-09-23T18:22:46.557793+0000","last_fullsized":"2023-09-23T18:22:46.557793+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":291,"log_dups_size":0,"ondisk_log_size":291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:22.022031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047292100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1326,"num_read_kb":5772,"num_write":300,"num_write_kb":4517,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":204,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514839+0000","last_change":"2023-09-23T18:22:03.462589+0000","last_active":"2023-09-23T18:22:44.514839+0000","last_peered":"2023-09-23T18:22:44.514839+0000","last_clean":"2023-09-23T18:22:44.514839+0000","last_became_active":"2023-09-23T18:22:03.462239+0000","last_became_peered":"2023-09-23T18:22:03.462239+0000","last_unstale":"2023-09-23T18:22:44.514839+0000","last_undegraded":"2023-09-23T18:22:44.514839+0000","last_fullsized":"2023-09-23T18:22:44.514839+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.020770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":174,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513865+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:22:44.513865+0000","last_peered":"2023-09-23T18:22:44.513865+0000","last_clean":"2023-09-23T18:22:44.513865+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:22:44.513865+0000","last_undegraded":"2023-09-23T18:22:44.513865+0000","last_fullsized":"2023-09-23T18:22:44.513865+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":863,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513643+0000","last_change":"2023-09-23T18:22:06.465690+0000","last_active":"2023-09-23T18:22:44.513643+0000","last_peered":"2023-09-23T18:22:44.513643+0000","last_clean":"2023-09-23T18:22:44.513643+0000","last_became_active":"2023-09-23T18:22:06.465148+0000","last_became_peered":"2023-09-23T18:22:06.465148+0000","last_unstale":"2023-09-23T18:22:44.513643+0000","last_undegraded":"2023-09-23T18:22:44.513643+0000","last_fullsized":"2023-09-23T18:22:44.513643+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:00.043030+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"653'323","reported_seq":2260,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513775+0000","last_change":"2023-09-23T18:22:28.473090+0000","last_active":"2023-09-23T18:22:44.513775+0000","last_peered":"2023-09-23T18:22:44.513775+0000","last_clean":"2023-09-23T18:22:44.513775+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:22:44.513775+0000","last_undegraded":"2023-09-23T18:22:44.513775+0000","last_fullsized":"2023-09-23T18:22:44.513775+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":323,"log_dups_size":0,"ondisk_log_size":323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001426075,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3093,"num_read_kb":5799,"num_write":265,"num_write_kb":3199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":184,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515869+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:22:44.515869+0000","last_peered":"2023-09-23T18:22:44.515869+0000","last_clean":"2023-09-23T18:22:44.515869+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:22:44.515869+0000","last_undegraded":"2023-09-23T18:22:44.515869+0000","last_fullsized":"2023-09-23T18:22:44.515869+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":425,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557567+0000","last_change":"2023-09-23T18:22:44.519917+0000","last_active":"2023-09-23T18:22:46.557567+0000","last_peered":"2023-09-23T18:22:46.557567+0000","last_clean":"2023-09-23T18:22:46.557567+0000","last_became_active":"2023-09-23T18:22:44.519386+0000","last_became_peered":"2023-09-23T18:22:44.519386+0000","last_unstale":"2023-09-23T18:22:46.557567+0000","last_undegraded":"2023-09-23T18:22:46.557567+0000","last_fullsized":"2023-09-23T18:22:46.557567+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:44.617263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":687,"reported_epoch":660,"state":"remapped+peering","last_fresh":"2023-09-23T18:22:44.510038+0000","last_change":"2023-09-23T18:22:44.510038+0000","last_active":"2023-09-23T18:11:39.495758+0000","last_peered":"2023-09-23T18:11:33.615512+0000","last_clean":"2023-09-23T18:11:33.615512+0000","last_became_active":"2023-09-23T18:11:30.565763+0000","last_became_peered":"2023-09-23T18:11:30.565763+0000","last_unstale":"2023-09-23T18:22:44.510038+0000","last_undegraded":"2023-09-23T18:22:44.510038+0000","last_fullsized":"2023-09-23T18:22:44.510038+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:06:12.113443+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"653'200","reported_seq":488,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.570312+0000","last_change":"2023-09-23T18:22:47.569585+0000","last_active":"2023-09-23T18:22:47.570312+0000","last_peered":"2023-09-23T18:22:47.570312+0000","last_clean":"2023-09-23T18:22:47.570312+0000","last_became_active":"2023-09-23T18:22:47.568716+0000","last_became_peered":"2023-09-23T18:22:47.568716+0000","last_unstale":"2023-09-23T18:22:47.570312+0000","last_undegraded":"2023-09-23T18:22:47.570312+0000","last_fullsized":"2023-09-23T18:22:47.570312+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:16.049107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0022753629999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1032,"num_read_kb":1283,"num_write":194,"num_write_kb":5509,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":20971520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":463,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.519424+0000","last_change":"2023-09-23T18:22:44.518872+0000","last_active":"2023-09-23T18:22:44.519424+0000","last_peered":"2023-09-23T18:22:44.519424+0000","last_clean":"2023-09-23T18:22:44.519424+0000","last_became_active":"2023-09-23T18:22:44.518360+0000","last_became_peered":"2023-09-23T18:22:44.518360+0000","last_unstale":"2023-09-23T18:22:44.519424+0000","last_undegraded":"2023-09-23T18:22:44.519424+0000","last_fullsized":"2023-09-23T18:22:44.519424+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:43.986310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":196,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513310+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:22:44.513310+0000","last_peered":"2023-09-23T18:22:44.513310+0000","last_clean":"2023-09-23T18:22:44.513310+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:22:44.513310+0000","last_undegraded":"2023-09-23T18:22:44.513310+0000","last_fullsized":"2023-09-23T18:22:44.513310+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"653'474","reported_seq":2262,"reported_epoch":661,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:22:45.617550+0000","last_change":"2023-09-23T18:22:45.617748+0000","last_active":"2023-09-23T18:22:45.617550+0000","last_peered":"2023-09-23T18:22:45.617550+0000","last_clean":"2023-09-23T18:22:39.500537+0000","last_became_active":"2023-09-23T18:22:45.555209+0000","last_became_peered":"2023-09-23T18:22:45.555209+0000","last_unstale":"2023-09-23T18:22:45.617550+0000","last_undegraded":"2023-09-23T18:22:45.617550+0000","last_fullsized":"2023-09-23T18:22:45.539663+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":539,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":474,"log_dups_size":0,"ondisk_log_size":474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:25:49.153587+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014671619999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2565,"num_read_kb":5830,"num_write":410,"num_write_kb":4052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,3],"avail_no_missing":["5","2","3","6"],"object_location_counts":[{"shards":"3,5","objects":1}],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":610,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.515026+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:22:44.515026+0000","last_peered":"2023-09-23T18:22:44.515026+0000","last_clean":"2023-09-23T18:22:44.515026+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:22:44.515026+0000","last_undegraded":"2023-09-23T18:22:44.515026+0000","last_fullsized":"2023-09-23T18:22:44.515026+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":170,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513686+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:22:44.513686+0000","last_peered":"2023-09-23T18:22:44.513686+0000","last_clean":"2023-09-23T18:22:44.513686+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:22:44.513686+0000","last_undegraded":"2023-09-23T18:22:44.513686+0000","last_fullsized":"2023-09-23T18:22:44.513686+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":205,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.517780+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:22:44.517780+0000","last_peered":"2023-09-23T18:22:44.517780+0000","last_clean":"2023-09-23T18:22:44.517780+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:22:44.517780+0000","last_undegraded":"2023-09-23T18:22:44.517780+0000","last_fullsized":"2023-09-23T18:22:44.517780+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":858,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516140+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:22:44.516140+0000","last_peered":"2023-09-23T18:22:44.516140+0000","last_clean":"2023-09-23T18:22:44.516140+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:22:44.516140+0000","last_undegraded":"2023-09-23T18:22:44.516140+0000","last_fullsized":"2023-09-23T18:22:44.516140+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"653'156","reported_seq":453,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.512940+0000","last_change":"2023-09-23T18:22:28.481047+0000","last_active":"2023-09-23T18:22:44.512940+0000","last_peered":"2023-09-23T18:22:44.512940+0000","last_clean":"2023-09-23T18:22:44.512940+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:22:44.512940+0000","last_undegraded":"2023-09-23T18:22:44.512940+0000","last_fullsized":"2023-09-23T18:22:44.512940+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":156,"log_dups_size":0,"ondisk_log_size":156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504954,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":805,"num_read_kb":2049,"num_write":144,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":78,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514613+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:22:44.514613+0000","last_peered":"2023-09-23T18:22:44.514613+0000","last_clean":"2023-09-23T18:22:44.514613+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:22:44.514613+0000","last_undegraded":"2023-09-23T18:22:44.514613+0000","last_fullsized":"2023-09-23T18:22:44.514613+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":607,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.519254+0000","last_change":"2023-09-23T18:22:44.518222+0000","last_active":"2023-09-23T18:22:44.519254+0000","last_peered":"2023-09-23T18:22:44.519254+0000","last_clean":"2023-09-23T18:22:44.519254+0000","last_became_active":"2023-09-23T18:22:44.517848+0000","last_became_peered":"2023-09-23T18:22:44.517848+0000","last_unstale":"2023-09-23T18:22:44.519254+0000","last_undegraded":"2023-09-23T18:22:44.519254+0000","last_fullsized":"2023-09-23T18:22:44.519254+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:40.879308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":958,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516997+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:22:44.516997+0000","last_peered":"2023-09-23T18:22:44.516997+0000","last_clean":"2023-09-23T18:22:44.516997+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:22:44.516997+0000","last_undegraded":"2023-09-23T18:22:44.516997+0000","last_fullsized":"2023-09-23T18:22:44.516997+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"653'181","reported_seq":616,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.509057+0000","last_change":"2023-09-23T18:22:28.468611+0000","last_active":"2023-09-23T18:22:44.509057+0000","last_peered":"2023-09-23T18:22:44.509057+0000","last_clean":"2023-09-23T18:22:44.509057+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:22:44.509057+0000","last_undegraded":"2023-09-23T18:22:44.509057+0000","last_fullsized":"2023-09-23T18:22:44.509057+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001526466,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1349,"num_read_kb":1261,"num_write":144,"num_write_kb":4223,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":194,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.520250+0000","last_change":"2023-09-23T18:22:44.518634+0000","last_active":"2023-09-23T18:22:44.520250+0000","last_peered":"2023-09-23T18:22:44.520250+0000","last_clean":"2023-09-23T18:22:44.520250+0000","last_became_active":"2023-09-23T18:22:44.518286+0000","last_became_peered":"2023-09-23T18:22:44.518286+0000","last_unstale":"2023-09-23T18:22:44.520250+0000","last_undegraded":"2023-09-23T18:22:44.520250+0000","last_fullsized":"2023-09-23T18:22:44.520250+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:05.309943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":184,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513540+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:22:44.513540+0000","last_peered":"2023-09-23T18:22:44.513540+0000","last_clean":"2023-09-23T18:22:44.513540+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:22:44.513540+0000","last_undegraded":"2023-09-23T18:22:44.513540+0000","last_fullsized":"2023-09-23T18:22:44.513540+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":968,"reported_epoch":661,"state":"active+undersized+remapped","last_fresh":"2023-09-23T18:22:45.546784+0000","last_change":"2023-09-23T18:22:45.546653+0000","last_active":"2023-09-23T18:22:45.546784+0000","last_peered":"2023-09-23T18:22:45.546784+0000","last_clean":"2023-09-23T18:22:39.751536+0000","last_became_active":"2023-09-23T18:22:45.546653+0000","last_became_peered":"2023-09-23T18:22:45.546653+0000","last_unstale":"2023-09-23T18:22:45.546784+0000","last_undegraded":"2023-09-23T18:22:45.546784+0000","last_fullsized":"2023-09-23T18:22:45.541492+0000","mapping_epoch":660,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:59:33.138528+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4],"avail_no_missing":["6","2","4","5"],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":146,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513194+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:22:44.513194+0000","last_peered":"2023-09-23T18:22:44.513194+0000","last_clean":"2023-09-23T18:22:44.513194+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:22:44.513194+0000","last_undegraded":"2023-09-23T18:22:44.513194+0000","last_fullsized":"2023-09-23T18:22:44.513194+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"653'188","reported_seq":940,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516844+0000","last_change":"2023-09-23T18:22:28.475214+0000","last_active":"2023-09-23T18:22:44.516844+0000","last_peered":"2023-09-23T18:22:44.516844+0000","last_clean":"2023-09-23T18:22:44.516844+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:22:44.516844+0000","last_undegraded":"2023-09-23T18:22:44.516844+0000","last_fullsized":"2023-09-23T18:22:44.516844+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":188,"log_dups_size":0,"ondisk_log_size":188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055658400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1067,"num_read_kb":2442,"num_write":105,"num_write_kb":1356,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":79,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.541414+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:22:45.541414+0000","last_peered":"2023-09-23T18:22:45.541414+0000","last_clean":"2023-09-23T18:22:45.541414+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:22:45.541414+0000","last_undegraded":"2023-09-23T18:22:45.541414+0000","last_fullsized":"2023-09-23T18:22:45.541414+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":794,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514140+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:22:44.514140+0000","last_peered":"2023-09-23T18:22:44.514140+0000","last_clean":"2023-09-23T18:22:44.514140+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:22:44.514140+0000","last_undegraded":"2023-09-23T18:22:44.514140+0000","last_fullsized":"2023-09-23T18:22:44.514140+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"652'137","reported_seq":1316,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.517164+0000","last_change":"2023-09-23T18:22:28.475379+0000","last_active":"2023-09-23T18:22:44.517164+0000","last_peered":"2023-09-23T18:22:44.517164+0000","last_clean":"2023-09-23T18:22:44.517164+0000","last_became_active":"2023-09-23T18:22:03.468424+0000","last_became_peered":"2023-09-23T18:22:03.468424+0000","last_unstale":"2023-09-23T18:22:44.517164+0000","last_undegraded":"2023-09-23T18:22:44.517164+0000","last_fullsized":"2023-09-23T18:22:44.517164+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:46.671634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044965899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":925,"num_write":128,"num_write_kb":1260,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":87,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.510109+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:22:43.510109+0000","last_peered":"2023-09-23T18:22:43.510109+0000","last_clean":"2023-09-23T18:22:43.510109+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:22:43.510109+0000","last_undegraded":"2023-09-23T18:22:43.510109+0000","last_fullsized":"2023-09-23T18:22:43.510109+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":450,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513168+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:22:44.513168+0000","last_peered":"2023-09-23T18:22:44.513168+0000","last_clean":"2023-09-23T18:22:44.513168+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:22:44.513168+0000","last_undegraded":"2023-09-23T18:22:44.513168+0000","last_fullsized":"2023-09-23T18:22:44.513168+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":690,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514459+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:22:44.514459+0000","last_peered":"2023-09-23T18:22:44.514459+0000","last_clean":"2023-09-23T18:22:44.514459+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:22:44.514459+0000","last_undegraded":"2023-09-23T18:22:44.514459+0000","last_fullsized":"2023-09-23T18:22:44.514459+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":83,"reported_epoch":659,"state":"active+clean","last_fresh":"2023-09-23T18:22:43.511633+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:22:43.511633+0000","last_peered":"2023-09-23T18:22:43.511633+0000","last_clean":"2023-09-23T18:22:43.511633+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:22:43.511633+0000","last_undegraded":"2023-09-23T18:22:43.511633+0000","last_fullsized":"2023-09-23T18:22:43.511633+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.564029+0000","last_change":"2023-09-23T18:22:44.520691+0000","last_active":"2023-09-23T18:22:47.564029+0000","last_peered":"2023-09-23T18:22:47.564029+0000","last_clean":"2023-09-23T18:22:47.564029+0000","last_became_active":"2023-09-23T18:22:44.520082+0000","last_became_peered":"2023-09-23T18:22:44.520082+0000","last_unstale":"2023-09-23T18:22:47.564029+0000","last_undegraded":"2023-09-23T18:22:47.564029+0000","last_fullsized":"2023-09-23T18:22:47.564029+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:29.126035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":690,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516679+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:22:44.516679+0000","last_peered":"2023-09-23T18:22:44.516679+0000","last_clean":"2023-09-23T18:22:44.516679+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:22:44.516679+0000","last_undegraded":"2023-09-23T18:22:44.516679+0000","last_fullsized":"2023-09-23T18:22:44.516679+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":178,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513346+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:22:44.513346+0000","last_peered":"2023-09-23T18:22:44.513346+0000","last_clean":"2023-09-23T18:22:44.513346+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:22:44.513346+0000","last_undegraded":"2023-09-23T18:22:44.513346+0000","last_fullsized":"2023-09-23T18:22:44.513346+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":296,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.545245+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:22:45.545245+0000","last_peered":"2023-09-23T18:22:45.545245+0000","last_clean":"2023-09-23T18:22:45.545245+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:22:45.545245+0000","last_undegraded":"2023-09-23T18:22:45.545245+0000","last_fullsized":"2023-09-23T18:22:45.545245+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":164,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.513297+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:22:44.513297+0000","last_peered":"2023-09-23T18:22:44.513297+0000","last_clean":"2023-09-23T18:22:44.513297+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:22:44.513297+0000","last_undegraded":"2023-09-23T18:22:44.513297+0000","last_fullsized":"2023-09-23T18:22:44.513297+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":279,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.545257+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:22:45.545257+0000","last_peered":"2023-09-23T18:22:45.545257+0000","last_clean":"2023-09-23T18:22:45.545257+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:22:45.545257+0000","last_undegraded":"2023-09-23T18:22:45.545257+0000","last_fullsized":"2023-09-23T18:22:45.545257+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1052,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.516028+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:22:44.516028+0000","last_peered":"2023-09-23T18:22:44.516028+0000","last_clean":"2023-09-23T18:22:44.516028+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:22:44.516028+0000","last_undegraded":"2023-09-23T18:22:44.516028+0000","last_fullsized":"2023-09-23T18:22:44.516028+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":567,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.522694+0000","last_change":"2023-09-23T18:22:44.520133+0000","last_active":"2023-09-23T18:22:44.522694+0000","last_peered":"2023-09-23T18:22:44.522694+0000","last_clean":"2023-09-23T18:22:44.522694+0000","last_became_active":"2023-09-23T18:22:44.519765+0000","last_became_peered":"2023-09-23T18:22:44.519765+0000","last_unstale":"2023-09-23T18:22:44.522694+0000","last_undegraded":"2023-09-23T18:22:44.522694+0000","last_fullsized":"2023-09-23T18:22:44.522694+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:02.736149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":133,"reported_epoch":661,"state":"active+clean","last_fresh":"2023-09-23T18:22:45.541741+0000","last_change":"2023-09-23T18:22:44.519760+0000","last_active":"2023-09-23T18:22:45.541741+0000","last_peered":"2023-09-23T18:22:45.541741+0000","last_clean":"2023-09-23T18:22:45.541741+0000","last_became_active":"2023-09-23T18:22:44.519006+0000","last_became_peered":"2023-09-23T18:22:44.519006+0000","last_unstale":"2023-09-23T18:22:45.541741+0000","last_undegraded":"2023-09-23T18:22:45.541741+0000","last_fullsized":"2023-09-23T18:22:45.541741+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:02.871094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":993,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.559268+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:22:46.559268+0000","last_peered":"2023-09-23T18:22:46.559268+0000","last_clean":"2023-09-23T18:22:46.559268+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:22:46.559268+0000","last_undegraded":"2023-09-23T18:22:46.559268+0000","last_fullsized":"2023-09-23T18:22:46.559268+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":198,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.528984+0000","last_change":"2023-09-23T18:22:44.528526+0000","last_active":"2023-09-23T18:22:44.528984+0000","last_peered":"2023-09-23T18:22:44.528984+0000","last_clean":"2023-09-23T18:22:44.528984+0000","last_became_active":"2023-09-23T18:22:44.518445+0000","last_became_peered":"2023-09-23T18:22:44.518445+0000","last_unstale":"2023-09-23T18:22:44.528984+0000","last_undegraded":"2023-09-23T18:22:44.528984+0000","last_fullsized":"2023-09-23T18:22:44.528984+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:17.761911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":703,"reported_epoch":660,"state":"active+clean","last_fresh":"2023-09-23T18:22:44.514924+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:22:44.514924+0000","last_peered":"2023-09-23T18:22:44.514924+0000","last_clean":"2023-09-23T18:22:44.514924+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:22:44.514924+0000","last_undegraded":"2023-09-23T18:22:44.514924+0000","last_fullsized":"2023-09-23T18:22:44.514924+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":13010,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2048000,"data_stored":2033792,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":88,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":382,"num_bytes_recovered":20971577,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":23,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001602,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669538,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":807536,"kb_used_data":174248,"kb_used_omap":7,"kb_used_meta":633272,"kb_avail":92937616,"statfs":{"total":95995035648,"available":95168118784,"internally_reserved":0,"allocated":178429952,"data_stored":176424787,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":648471332},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:22 2023","interfaces":[{"interface":"back","average":{"1min":4.6609999999999996,"5min":2.7709999999999999,"15min":1.8129999999999999},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":79.203999999999994,"5min":79.203999999999994,"15min":79.203999999999994},"last":0.57199999999999995},{"interface":"front","average":{"1min":4.0229999999999997,"5min":2.7810000000000001,"15min":1.9570000000000001},"min":{"1min":0.252,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":78.034000000000006,"5min":78.034000000000006,"15min":78.034000000000006},"last":0.50900000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":2.1240000000000001,"15min":1.518},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.8480000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.627},{"interface":"front","average":{"1min":0.58399999999999996,"5min":2.1240000000000001,"15min":1.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.27,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.54800000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.518,"15min":1.369},"min":{"1min":0.33500000000000002,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":7.734,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.65000000000000002},{"interface":"front","average":{"1min":1.8120000000000001,"5min":2.0209999999999999,"15min":1.5920000000000001},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":8.2690000000000001,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.66500000000000004}]},{"osd":3,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":4.6050000000000004,"5min":3.601,"15min":2.3479999999999999},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.24099999999999999},"max":{"1min":78.643000000000001,"5min":103.348,"15min":103.348},"last":0.61399999999999999},{"interface":"front","average":{"1min":5.2629999999999999,"5min":4.298,"15min":2.6930000000000001},"min":{"1min":0.32100000000000001,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":78.347999999999999,"5min":103.40300000000001,"15min":103.40300000000001},"last":0.60099999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":1.3220000000000001,"15min":1.3400000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23999999999999999},"max":{"1min":1.103,"5min":6.2800000000000002,"15min":8.6059999999999999},"last":0.34200000000000003},{"interface":"front","average":{"1min":0.46100000000000002,"5min":1.2769999999999999,"15min":1.375},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":0.91200000000000003,"5min":7.8319999999999999,"15min":9.2260000000000009},"last":0.372}]},{"osd":5,"last update":"Sat Sep 23 18:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.8280000000000001,"15min":1.925},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":5.0460000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.9990000000000001,"15min":2.258},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0489999999999999,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.41699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.6930000000000001,"15min":1.603},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.26800000000000002},"max":{"1min":6.1799999999999997,"5min":10.743,"15min":13.460000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.593,"15min":1.6200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.28499999999999998},"max":{"1min":5.0990000000000002,"5min":8.6890000000000001,"15min":12.404},"last":0.52700000000000002}]}]},{"osd":1,"up_from":545,"seq":2340757176387,"num_pgs":58,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":668156,"kb_used_data":420212,"kb_used_omap":13,"kb_used_meta":247922,"kb_avail":93076996,"statfs":{"total":95995035648,"available":95310843904,"internally_reserved":0,"allocated":430297088,"data_stored":427077303,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":253872781},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.331,"15min":0.70399999999999996},"min":{"1min":0.23100000000000001,"5min":0.189,"15min":0.189},"max":{"1min":4.5030000000000001,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.68700000000000006},{"interface":"front","average":{"1min":0.38400000000000001,"5min":1.3069999999999999,"15min":0.76200000000000001},"min":{"1min":0.28399999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.51600000000000001,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.53000000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":1.4339999999999999,"15min":0.78500000000000003},"min":{"1min":0.28899999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.61899999999999999,"5min":11.02,"15min":11.02},"last":0.50800000000000001},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.98799999999999999,"15min":0.66600000000000004},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.234,"5min":5.1440000000000001,"15min":5.1440000000000001},"last":0.61399999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":1.7689999999999999,"15min":0.84699999999999998},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0980000000000001,"5min":14.962,"15min":14.962},"last":0.65300000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.577,"15min":0.86699999999999999},"min":{"1min":0.35499999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.82199999999999995,"5min":19.762,"15min":19.762},"last":0.63400000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.304,"15min":1.3120000000000001},"min":{"1min":0.38,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.532,"5min":12.663,"15min":12.663},"last":0.77700000000000002},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.1230000000000002,"15min":1.7170000000000001},"min":{"1min":0.39000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":4.7320000000000002,"5min":13.222,"15min":13.222},"last":0.78900000000000003}]},{"osd":5,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.7350000000000001,"15min":1.0600000000000001},"min":{"1min":0.373,"5min":0.308,"15min":0.308},"max":{"1min":3.6760000000000002,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.72999999999999998},{"interface":"front","average":{"1min":0.85699999999999998,"5min":2.052,"15min":1.22},"min":{"1min":0.42699999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.556,"5min":23.204000000000001,"15min":23.204000000000001},"last":0.85499999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.736,"15min":1.085},"min":{"1min":0.46999999999999997,"5min":0.372,"15min":0.372},"max":{"1min":3.5659999999999998,"5min":23.443000000000001,"15min":23.443000000000001},"last":0.93100000000000005},{"interface":"front","average":{"1min":0.84499999999999997,"5min":2.306,"15min":1.286},"min":{"1min":0.42499999999999999,"5min":0.378,"15min":0.378},"max":{"1min":4.8959999999999999,"5min":26.177,"15min":26.177},"last":0.89800000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.9399999999999999,"15min":1.1000000000000001},"min":{"1min":0.47399999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.052,"5min":15.01,"15min":15.01},"last":0.82299999999999995},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.867,"15min":1.159},"min":{"1min":0.46899999999999997,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78400000000000003,"5min":10.988,"15min":10.988},"last":0.91300000000000003}]}]},{"osd":0,"up_from":93,"seq":399431958883,"num_pgs":47,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1163040,"kb_used_data":180056,"kb_used_omap":7,"kb_used_meta":982968,"kb_avail":92582112,"statfs":{"total":95995035648,"available":94804082688,"internally_reserved":0,"allocated":184377344,"data_stored":181337770,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":1006559972},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.3839999999999999,"15min":0.67400000000000004},"min":{"1min":0.251,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":1.2170000000000001,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.63},{"interface":"front","average":{"1min":0.51100000000000001,"5min":1.8340000000000001,"15min":0.95999999999999996},"min":{"1min":0.28299999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.853,"5min":22.978000000000002,"15min":22.978000000000002},"last":0.54400000000000004}]},{"osd":2,"last update":"Sat Sep 23 18:22:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":2.355,"15min":1.4790000000000001},"min":{"1min":0.25,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":9.8300000000000001,"5min":18.550999999999998,"15min":18.550999999999998},"last":0.45400000000000001},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.925,"15min":1.1419999999999999},"min":{"1min":0.31900000000000001,"5min":0.22500000000000001,"15min":0.22},"max":{"1min":6.2220000000000004,"5min":23.161999999999999,"15min":23.161999999999999},"last":0.50700000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:21:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.927,"15min":1.6890000000000001},"min":{"1min":0.24199999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.173999999999999,"5min":15.173999999999999,"15min":15.173999999999999},"last":0.56100000000000005},{"interface":"front","average":{"1min":2.23,"5min":2.0350000000000001,"15min":1.718},"min":{"1min":0.32100000000000001,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":15.135,"5min":15.135,"15min":15.135},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:22:15 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.944,"15min":1.9299999999999999},"min":{"1min":0.33300000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9260000000000002,"5min":17.741,"15min":57.746000000000002},"last":0.72599999999999998},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.899,"15min":1.7150000000000001},"min":{"1min":0.42999999999999999,"5min":0.29199999999999998,"15min":0.25600000000000001},"max":{"1min":6.3860000000000001,"5min":15.079000000000001,"15min":57.816000000000003},"last":0.77600000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:21:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.73,"15min":1.466},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.3979999999999997,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.76000000000000001},{"interface":"front","average":{"1min":1.4610000000000001,"5min":2.3399999999999999,"15min":1.6279999999999999},"min":{"1min":0.35499999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.7359999999999998,"5min":15.047000000000001,"15min":15.047000000000001},"last":0.68600000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.8089999999999999,"15min":1.9870000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.32000000000000001},"max":{"1min":6.0439999999999996,"5min":7.6900000000000004,"15min":58.457999999999998},"last":0.67000000000000004},{"interface":"front","average":{"1min":1.284,"5min":2.645,"15min":2.0739999999999998},"min":{"1min":0.47599999999999998,"5min":0.375,"15min":0.315},"max":{"1min":4.4470000000000001,"5min":22.893000000000001,"15min":58.354999999999997},"last":0.70199999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:21:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":2.2130000000000001,"15min":1.825},"min":{"1min":0.46200000000000002,"5min":0.45000000000000001,"15min":0.34499999999999997},"max":{"1min":6.1200000000000001,"5min":22.945,"15min":22.945},"last":0.65800000000000003},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.7410000000000001,"15min":1.851},"min":{"1min":0.42799999999999999,"5min":0.371,"15min":0.36199999999999999},"max":{"1min":5.4320000000000004,"5min":7.7519999999999998,"15min":10.404},"last":0.79600000000000004}]}]},{"osd":2,"up_from":482,"seq":2070174236769,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161072,"kb_used_data":12840,"kb_used_omap":22,"kb_used_meta":148201,"kb_avail":93584080,"statfs":{"total":95995035648,"available":95830097920,"internally_reserved":0,"allocated":13148160,"data_stored":9940106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22605,"internal_metadata":151758771},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.5980000000000001,"15min":0.90900000000000003},"min":{"1min":0.24299999999999999,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":4.7060000000000004,"5min":9.6359999999999992,"15min":9.6359999999999992},"last":0.46000000000000002},{"interface":"front","average":{"1min":2.9289999999999998,"5min":1.964,"15min":1.286},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.3390000000000004,"5min":13.472,"15min":13.472},"last":0.41599999999999998}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.4219999999999999,"15min":0.80800000000000005},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.1320000000000001,"5min":13.538,"15min":13.538},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.504,"5min":1.681,"15min":0.98399999999999999},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.1499999999999999,"5min":16.567,"15min":16.567},"last":0.438}]},{"osd":3,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":2.1640000000000001,"15min":1.3919999999999999},"min":{"1min":0.40300000000000002,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":8.093,"5min":15.926,"15min":15.926},"last":0.53600000000000003},{"interface":"front","average":{"1min":1.841,"5min":1.754,"15min":1.409},"min":{"1min":0.28199999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":13.247999999999999,"5min":16.619,"15min":16.619},"last":0.52400000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:22:03 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.667,"15min":1.512},"min":{"1min":0.375,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":4.6539999999999999,"5min":9.532,"15min":11.334},"last":0.66000000000000003},{"interface":"front","average":{"1min":1.601,"5min":1.5589999999999999,"15min":1.1599999999999999},"min":{"1min":0.40000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.5229999999999997,"5min":6.367,"15min":11.244},"last":0.51100000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.714,"15min":2.0339999999999998},"min":{"1min":0.38300000000000001,"5min":0.34799999999999998,"15min":0.308},"max":{"1min":9.4960000000000004,"5min":9.4960000000000004,"15min":11.852},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.6890000000000001,"15min":1.5980000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32600000000000001},"max":{"1min":4.6719999999999997,"5min":8.3849999999999998,"15min":10.942},"last":0.58399999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:22:03 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":2.0569999999999999,"15min":1.966},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":29.489000000000001,"5min":29.489000000000001,"15min":29.489000000000001},"last":0.55500000000000005},{"interface":"front","average":{"1min":3.048,"5min":2.1080000000000001,"15min":2.3759999999999999},"min":{"1min":0.42899999999999999,"5min":0.39600000000000002,"15min":0.38700000000000001},"max":{"1min":29.408999999999999,"5min":29.408999999999999,"15min":29.408999999999999},"last":0.755}]},{"osd":7,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":1.671,"15min":3.839},"min":{"1min":0.495,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":6.9779999999999998,"5min":13.106,"15min":76.966999999999999},"last":0.80500000000000005},{"interface":"front","average":{"1min":1.746,"5min":1.724,"15min":4.0179999999999998},"min":{"1min":0.45500000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":9.5619999999999994,"5min":9.5619999999999994,"15min":76.936999999999998},"last":0.73399999999999999}]}]},{"osd":3,"up_from":237,"seq":1017907249398,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818452,"kb_used_data":12940,"kb_used_omap":3,"kb_used_meta":805500,"kb_avail":92926700,"statfs":{"total":95995035648,"available":95156940800,"internally_reserved":0,"allocated":13250560,"data_stored":9997021,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":824832744},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.841,"15min":1.8700000000000001},"min":{"1min":0.20100000000000001,"5min":0.192,"15min":0.183},"max":{"1min":6.3360000000000003,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.27100000000000002},{"interface":"front","average":{"1min":1.925,"5min":1.7270000000000001,"15min":2.0590000000000002},"min":{"1min":0.311,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":8.9440000000000008,"5min":11.468,"15min":194.661},"last":0.247}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.847,"15min":0.96199999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":8.9629999999999992,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.25600000000000001},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.7839999999999998,"15min":1.3979999999999999},"min":{"1min":0.29099999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.5600000000000001,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.36699999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:22:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2090000000000001,"5min":2.0369999999999999,"15min":1.5009999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.21099999999999999},"max":{"1min":18.722999999999999,"5min":18.722999999999999,"15min":18.722999999999999},"last":0.35199999999999998},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.6699999999999999,"15min":1.3999999999999999},"min":{"1min":0.32800000000000001,"5min":0.27900000000000003,"15min":0.19600000000000001},"max":{"1min":7.0579999999999998,"5min":11.565,"15min":11.565},"last":0.314}]},{"osd":4,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":1.76,"15min":1.7330000000000001},"min":{"1min":0.36899999999999999,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":4.4089999999999998,"5min":20.850000000000001,"15min":37.231000000000002},"last":0.39300000000000002},{"interface":"front","average":{"1min":1.98,"5min":1.99,"15min":1.9570000000000001},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":7.6840000000000002,"5min":20.786000000000001,"15min":37.381},"last":0.38500000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:21:42 2023","interfaces":[{"interface":"back","average":{"1min":2.6579999999999999,"5min":2.2709999999999999,"15min":1.6799999999999999},"min":{"1min":0.47899999999999998,"5min":0.316,"15min":0.316},"max":{"1min":19.434999999999999,"5min":19.434999999999999,"15min":19.434999999999999},"last":0.40799999999999997},{"interface":"front","average":{"1min":1.647,"5min":1.7310000000000001,"15min":1.718},"min":{"1min":0.35999999999999999,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":4.4100000000000001,"5min":17.135999999999999,"15min":17.135999999999999},"last":0.47399999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.944,"15min":1.9570000000000001},"min":{"1min":0.45700000000000002,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":4.4489999999999998,"5min":17.192,"15min":115.54000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":1.268,"5min":1.7390000000000001,"15min":2.0030000000000001},"min":{"1min":0.42599999999999999,"5min":0.35699999999999998,"15min":0.30499999999999999},"max":{"1min":4.3840000000000003,"5min":11.442,"15min":115.691},"last":0.42899999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.7729999999999999,"15min":1.8160000000000001},"min":{"1min":0.39500000000000002,"5min":0.38700000000000001,"15min":0.35799999999999998},"max":{"1min":7.7679999999999998,"5min":9.1929999999999996,"15min":15.972},"last":0.44},{"interface":"front","average":{"1min":1.589,"5min":1.835,"15min":1.6599999999999999},"min":{"1min":0.49299999999999999,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":4.2279999999999998,"5min":17.164000000000001,"15min":17.164000000000001},"last":0.50700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849019348,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":874612,"kb_used_data":169260,"kb_used_omap":0,"kb_used_meta":705344,"kb_avail":92870540,"statfs":{"total":95995035648,"available":95099432960,"internally_reserved":0,"allocated":173322240,"data_stored":171313325,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":722272256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":3.8959999999999999,"5min":2.242,"15min":1.5209999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":72.754999999999995,"5min":72.754999999999995,"15min":72.754999999999995},"last":0.49099999999999999},{"interface":"front","average":{"1min":3.9460000000000002,"5min":2.3500000000000001,"15min":1.7789999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":71.533000000000001,"5min":71.533000000000001,"15min":71.533000000000001},"last":0.55000000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:22:24 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":2.052,"15min":1.147},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.1399999999999997,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.9740000000000002,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.72499999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.655,"15min":1.77},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":4.0030000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.60899999999999999},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.6399999999999999,"15min":1.702},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.253},"max":{"1min":3.3839999999999999,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.67600000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":2.6419999999999999,"15min":2.169},"min":{"1min":0.40200000000000002,"5min":0.32000000000000001,"15min":0.23799999999999999},"max":{"1min":0.80300000000000005,"5min":74.781999999999996,"15min":74.781999999999996},"last":0.64900000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":2.5670000000000002,"15min":2.1309999999999998},"min":{"1min":0.38600000000000001,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":0.78900000000000003,"5min":72.152000000000001,"15min":72.152000000000001},"last":0.77700000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":2.3460000000000001,"15min":2.7869999999999999},"min":{"1min":0.29199999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.73899999999999999,"5min":91.432000000000002,"15min":91.432000000000002},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.50700000000000001,"5min":2.121,"15min":2.5640000000000001},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":0.70199999999999996,"5min":91.626999999999995,"15min":91.626999999999995},"last":0.70199999999999996}]},{"osd":6,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":5.141,"5min":2.4329999999999998,"15min":1.7090000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":91.688999999999993,"5min":91.688999999999993,"15min":91.688999999999993},"last":0.66200000000000003},{"interface":"front","average":{"1min":5.1420000000000003,"5min":2.6120000000000001,"15min":1.9159999999999999},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":91.837000000000003,"5min":91.837000000000003,"15min":91.837000000000003},"last":0.76200000000000001}]},{"osd":7,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":1.798,"15min":1.835},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":3.6800000000000002,"5min":18.283000000000001,"15min":19.298999999999999},"last":0.74399999999999999},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.4470000000000001,"15min":1.5529999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.28399999999999997},"max":{"1min":4.4269999999999996,"5min":9.4730000000000008,"15min":12.800000000000001},"last":0.80500000000000005}]}]},{"osd":5,"up_from":420,"seq":1803886264444,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":476648,"kb_used_data":103712,"kb_used_omap":9,"kb_used_meta":372918,"kb_avail":93268504,"statfs":{"total":95995035648,"available":95506948096,"internally_reserved":0,"allocated":106201088,"data_stored":104200445,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":381868477},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":4.7930000000000001,"5min":1.889,"15min":1.2090000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":77.162000000000006,"5min":77.162000000000006,"15min":77.162000000000006},"last":0.46800000000000003},{"interface":"front","average":{"1min":4.9450000000000003,"5min":2.4169999999999998,"15min":1.8720000000000001},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":75.784999999999997,"5min":75.784999999999997,"15min":75.784999999999997},"last":0.81399999999999995}]},{"osd":1,"last update":"Sat Sep 23 18:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.998,"15min":1.3959999999999999},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.4819999999999993,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.79500000000000004},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.133,"15min":1.2749999999999999},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.2759999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":0.69599999999999995}]},{"osd":2,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.615,"15min":1.611},"min":{"1min":0.31,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":8.8789999999999996,"5min":10.705,"15min":10.705},"last":0.752},{"interface":"front","average":{"1min":1.302,"5min":1.673,"15min":2.0640000000000001},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.3390000000000004,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.85699999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":5.4400000000000004,"5min":2.5369999999999999,"15min":1.6579999999999999},"min":{"1min":0.317,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":79.016999999999996,"5min":79.016999999999996,"15min":79.016999999999996},"last":0.83899999999999997},{"interface":"front","average":{"1min":5.2060000000000004,"5min":2.7810000000000001,"15min":1.9590000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.313},"max":{"1min":76.188999999999993,"5min":76.188999999999993,"15min":76.188999999999993},"last":0.77000000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.718,"15min":2.016},"min":{"1min":0.29199999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.9930000000000003,"5min":8.2270000000000003,"15min":12.747999999999999},"last":0.71899999999999997},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.7689999999999999,"15min":1.647},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":8.5559999999999992,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.66200000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.3340000000000001,"15min":1.2929999999999999},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.7859999999999996,"5min":6.7859999999999996,"15min":9.0350000000000001},"last":0.56299999999999994},{"interface":"front","average":{"1min":1.119,"5min":1.859,"15min":1.716},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.0890000000000004,"5min":12.504,"15min":12.504},"last":0.92400000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.704,"15min":1.806},"min":{"1min":0.33700000000000002,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":6.9569999999999999,"5min":10.528,"15min":13.506},"last":0.58199999999999996},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.494,"15min":1.5860000000000001},"min":{"1min":0.39900000000000002,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":12.702,"5min":12.702,"15min":13.138},"last":0.88}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:22:50.576 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph status --format=json 2023-09-23T18:22:50.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:50 smithi099 ceph-mon[167076]: pgmap v1588: 103 pgs: 5 active+undersized+remapped, 1 remapped+peering, 1 peering, 96 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 921 KiB/s, 10 objects/s recovering 2023-09-23T18:22:50.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:50 smithi055 ceph-mon[184174]: pgmap v1588: 103 pgs: 5 active+undersized+remapped, 1 remapped+peering, 1 peering, 96 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 921 KiB/s, 10 objects/s recovering 2023-09-23T18:22:50.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:50 smithi055 ceph-mon[179668]: pgmap v1588: 103 pgs: 5 active+undersized+remapped, 1 remapped+peering, 1 peering, 96 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 921 KiB/s, 10 objects/s recovering 2023-09-23T18:22:51.186 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:51 smithi055 ceph-mon[179668]: from='client.38265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:22:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:51.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:51 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:51.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:51 smithi055 ceph-mon[184174]: from='client.38265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:22:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:51.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:51 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:51 smithi099 ceph-mon[167076]: from='client.38265 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:22:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:52.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:51 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:22:52.706 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:22:52.706 INFO:teuthology.orchestra.run.smithi055.stdout:{"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","health":{"status":"HEALTH_OK","checks":{},"mutes":[]},"election_epoch":30,"quorum":[0,1,2],"quorum_names":["a","b","c"],"quorum_age":1862,"monmap":{"epoch":4,"min_mon_release_name":"reef","num_mons":3},"osdmap":{"epoch":663,"num_osds":8,"num_up_osds":8,"osd_up_since":1695493039,"num_in_osds":7,"osd_in_since":1695493322,"num_remapped_pgs":0},"pgmap":{"pgs_by_state":[{"state_name":"active+clean","count":97},{"state_name":"active+undersized+remapped","count":5},{"state_name":"peering","count":1}],"num_pgs":103,"num_pools":4,"num_objects":29,"data_bytes":8984499,"bytes_used":5078716416,"bytes_avail":666886533120,"bytes_total":671965249536,"inactive_pgs_ratio":0.0097087379544973373,"recovering_objects_per_sec":8,"recovering_bytes_per_sec":742744,"recovering_keys_per_sec":0,"num_objects_recovered":51,"num_bytes_recovered":4457546,"num_keys_recovered":0},"fsmap":{"epoch":9,"id":1,"up":1,"in":1,"max":1,"by_rank":[{"filesystem_id":1,"rank":0,"name":"foo.smithi055.exjftp","status":"up:active","gid":14658}],"up:standby":1},"mgrmap":{"available":true,"num_standbys":1,"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"services":{"dashboard":"https://172.21.15.55:8443/","prometheus":"http://172.21.15.55:9283/"}},"servicemap":{"epoch":34,"modified":"2023-09-23T18:21:27.214787+0000","services":{}},"progress_events":{"17520f84-0b13-444e-9552-f8b3c9f4329a":{"message":"Global Recovery Event (0s)\n [............................] ","progress":0,"add_to_ceph_s":true},"7bafb739-309c-4d56-8a22-310847f0ac26":{"message":"Upgrade to 18.2.0-460-gbb0daeff (4m)\n [============................] (remaining: 5m)","progress":0.44999998807907104,"add_to_ceph_s":true}}} 2023-09-23T18:22:52.918 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:52 smithi055 ceph-mon[179668]: pgmap v1589: 103 pgs: 5 active+undersized+remapped, 1 peering, 97 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 725 KiB/s, 8 objects/s recovering 2023-09-23T18:22:52.918 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:52 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:22:52.918 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:52 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:22:52.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:52 smithi055 ceph-mon[184174]: pgmap v1589: 103 pgs: 5 active+undersized+remapped, 1 peering, 97 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 725 KiB/s, 8 objects/s recovering 2023-09-23T18:22:52.918 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:52 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:22:52.919 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:52 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:22:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:52 smithi099 ceph-mon[167076]: pgmap v1589: 103 pgs: 5 active+undersized+remapped, 1 peering, 97 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 725 KiB/s, 8 objects/s recovering 2023-09-23T18:22:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:52 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:22:53.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:52 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:22:53.249 INFO:tasks.cephadm.ceph_manager.ceph:making progress, resetting timeout 2023-09-23T18:22:53.249 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:22:53.594 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:53 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2407047009' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:22:53.860 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:22:53.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:53 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2407047009' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:22:54.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:53 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2407047009' entity='client.admin' cmd=[{"prefix": "status", "format": "json"}]: dispatch 2023-09-23T18:22:54.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:54 smithi055 ceph-mon[179668]: pgmap v1590: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 565 KiB/s, 6 objects/s recovering 2023-09-23T18:22:54.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:54 smithi055 ceph-mon[184174]: pgmap v1590: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 565 KiB/s, 6 objects/s recovering 2023-09-23T18:22:55.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:54 smithi099 ceph-mon[167076]: pgmap v1590: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 565 KiB/s, 6 objects/s recovering 2023-09-23T18:22:55.298 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:22:55.300 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:22:55.720 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1591,"stamp":"2023-09-23T18:22:55.237092+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":484,"num_bytes_recovered":21509963,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":316,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4913804,"kb_used_data":1017428,"kb_used_omap":62,"kb_used_meta":3896257,"kb_avail":651302260,"statfs":{"total":671965249536,"available":666933514240,"internally_reserved":0,"allocated":1041846272,"data_stored":1023099099,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":64275,"internal_metadata":3989767405},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"8.686615"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":974,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.571570+0000","last_change":"2023-09-23T18:22:47.571570+0000","last_active":"2023-09-23T18:22:47.571570+0000","last_peered":"2023-09-23T18:22:47.571570+0000","last_clean":"2023-09-23T18:22:47.571570+0000","last_became_active":"2023-09-23T18:22:47.571095+0000","last_became_peered":"2023-09-23T18:22:47.571095+0000","last_unstale":"2023-09-23T18:22:47.571570+0000","last_undegraded":"2023-09-23T18:22:47.571570+0000","last_fullsized":"2023-09-23T18:22:47.571570+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:47.718943+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":332,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561067+0000","last_change":"2023-09-23T18:22:44.528866+0000","last_active":"2023-09-23T18:22:46.561067+0000","last_peered":"2023-09-23T18:22:46.561067+0000","last_clean":"2023-09-23T18:22:46.561067+0000","last_became_active":"2023-09-23T18:22:44.528499+0000","last_became_peered":"2023-09-23T18:22:44.528499+0000","last_unstale":"2023-09-23T18:22:46.561067+0000","last_undegraded":"2023-09-23T18:22:46.561067+0000","last_fullsized":"2023-09-23T18:22:46.561067+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:26.288744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":80,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558455+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:22:46.558455+0000","last_peered":"2023-09-23T18:22:46.558455+0000","last_clean":"2023-09-23T18:22:46.558455+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:22:46.558455+0000","last_undegraded":"2023-09-23T18:22:46.558455+0000","last_fullsized":"2023-09-23T18:22:46.558455+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":956,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556429+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:22:46.556429+0000","last_peered":"2023-09-23T18:22:46.556429+0000","last_clean":"2023-09-23T18:22:46.556429+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:22:46.556429+0000","last_undegraded":"2023-09-23T18:22:46.556429+0000","last_fullsized":"2023-09-23T18:22:46.556429+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":404,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.559596+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:22:46.559596+0000","last_peered":"2023-09-23T18:22:46.559596+0000","last_clean":"2023-09-23T18:22:46.559596+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:22:46.559596+0000","last_undegraded":"2023-09-23T18:22:46.559596+0000","last_fullsized":"2023-09-23T18:22:46.559596+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":725,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.567554+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:22:47.567554+0000","last_peered":"2023-09-23T18:22:47.567554+0000","last_clean":"2023-09-23T18:22:47.567554+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:22:47.567554+0000","last_undegraded":"2023-09-23T18:22:47.567554+0000","last_fullsized":"2023-09-23T18:22:47.567554+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":973,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556166+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:22:46.556166+0000","last_peered":"2023-09-23T18:22:46.556166+0000","last_clean":"2023-09-23T18:22:46.556166+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:22:46.556166+0000","last_undegraded":"2023-09-23T18:22:46.556166+0000","last_fullsized":"2023-09-23T18:22:46.556166+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557446+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:22:46.557446+0000","last_peered":"2023-09-23T18:22:46.557446+0000","last_clean":"2023-09-23T18:22:46.557446+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:22:46.557446+0000","last_undegraded":"2023-09-23T18:22:46.557446+0000","last_fullsized":"2023-09-23T18:22:46.557446+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":238,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558352+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:22:46.558352+0000","last_peered":"2023-09-23T18:22:46.558352+0000","last_clean":"2023-09-23T18:22:46.558352+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:22:46.558352+0000","last_undegraded":"2023-09-23T18:22:46.558352+0000","last_fullsized":"2023-09-23T18:22:46.558352+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1096,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557657+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:22:46.557657+0000","last_peered":"2023-09-23T18:22:46.557657+0000","last_clean":"2023-09-23T18:22:46.557657+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:22:46.557657+0000","last_undegraded":"2023-09-23T18:22:46.557657+0000","last_fullsized":"2023-09-23T18:22:46.557657+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":172,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557196+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:22:46.557196+0000","last_peered":"2023-09-23T18:22:46.557196+0000","last_clean":"2023-09-23T18:22:46.557196+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:22:46.557196+0000","last_undegraded":"2023-09-23T18:22:46.557196+0000","last_fullsized":"2023-09-23T18:22:46.557196+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":253,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558136+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:22:46.558136+0000","last_peered":"2023-09-23T18:22:46.558136+0000","last_clean":"2023-09-23T18:22:46.558136+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:22:46.558136+0000","last_undegraded":"2023-09-23T18:22:46.558136+0000","last_fullsized":"2023-09-23T18:22:46.558136+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":842,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.565289+0000","last_change":"2023-09-23T18:22:06.467889+0000","last_active":"2023-09-23T18:22:46.565289+0000","last_peered":"2023-09-23T18:22:46.565289+0000","last_clean":"2023-09-23T18:22:46.565289+0000","last_became_active":"2023-09-23T18:22:06.467521+0000","last_became_peered":"2023-09-23T18:22:06.467521+0000","last_unstale":"2023-09-23T18:22:46.565289+0000","last_undegraded":"2023-09-23T18:22:46.565289+0000","last_fullsized":"2023-09-23T18:22:46.565289+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:07.204206+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":208,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557482+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:22:46.557482+0000","last_peered":"2023-09-23T18:22:46.557482+0000","last_clean":"2023-09-23T18:22:46.557482+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:22:46.557482+0000","last_undegraded":"2023-09-23T18:22:46.557482+0000","last_fullsized":"2023-09-23T18:22:46.557482+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":207,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557313+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:22:46.557313+0000","last_peered":"2023-09-23T18:22:46.557313+0000","last_clean":"2023-09-23T18:22:46.557313+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:22:46.557313+0000","last_undegraded":"2023-09-23T18:22:46.557313+0000","last_fullsized":"2023-09-23T18:22:46.557313+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":1007,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557294+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:22:46.557294+0000","last_peered":"2023-09-23T18:22:46.557294+0000","last_clean":"2023-09-23T18:22:46.557294+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:22:46.557294+0000","last_undegraded":"2023-09-23T18:22:46.557294+0000","last_fullsized":"2023-09-23T18:22:46.557294+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":458,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558372+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:22:46.558372+0000","last_peered":"2023-09-23T18:22:46.558372+0000","last_clean":"2023-09-23T18:22:46.558372+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:22:46.558372+0000","last_undegraded":"2023-09-23T18:22:46.558372+0000","last_fullsized":"2023-09-23T18:22:46.558372+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":485,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557201+0000","last_change":"2023-09-23T18:22:03.468342+0000","last_active":"2023-09-23T18:22:46.557201+0000","last_peered":"2023-09-23T18:22:46.557201+0000","last_clean":"2023-09-23T18:22:46.557201+0000","last_became_active":"2023-09-23T18:22:03.467804+0000","last_became_peered":"2023-09-23T18:22:03.467804+0000","last_unstale":"2023-09-23T18:22:46.557201+0000","last_undegraded":"2023-09-23T18:22:46.557201+0000","last_fullsized":"2023-09-23T18:22:46.557201+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:17.762831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":1038,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558902+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:22:46.558902+0000","last_peered":"2023-09-23T18:22:46.558902+0000","last_clean":"2023-09-23T18:22:46.558902+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:22:46.558902+0000","last_undegraded":"2023-09-23T18:22:46.558902+0000","last_fullsized":"2023-09-23T18:22:46.558902+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":234,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557520+0000","last_change":"2023-09-23T18:22:03.479912+0000","last_active":"2023-09-23T18:22:46.557520+0000","last_peered":"2023-09-23T18:22:46.557520+0000","last_clean":"2023-09-23T18:22:46.557520+0000","last_became_active":"2023-09-23T18:22:03.464545+0000","last_became_peered":"2023-09-23T18:22:03.464545+0000","last_unstale":"2023-09-23T18:22:46.557520+0000","last_undegraded":"2023-09-23T18:22:46.557520+0000","last_fullsized":"2023-09-23T18:22:46.557520+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:50.025694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":481,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557148+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:22:46.557148+0000","last_peered":"2023-09-23T18:22:46.557148+0000","last_clean":"2023-09-23T18:22:46.557148+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:22:46.557148+0000","last_undegraded":"2023-09-23T18:22:46.557148+0000","last_fullsized":"2023-09-23T18:22:46.557148+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":888,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556113+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:22:46.556113+0000","last_peered":"2023-09-23T18:22:46.556113+0000","last_clean":"2023-09-23T18:22:46.556113+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:22:46.556113+0000","last_undegraded":"2023-09-23T18:22:46.556113+0000","last_fullsized":"2023-09-23T18:22:46.556113+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":497,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556905+0000","last_change":"2023-09-23T18:22:03.475293+0000","last_active":"2023-09-23T18:22:46.556905+0000","last_peered":"2023-09-23T18:22:46.556905+0000","last_clean":"2023-09-23T18:22:46.556905+0000","last_became_active":"2023-09-23T18:22:03.465603+0000","last_became_peered":"2023-09-23T18:22:03.465603+0000","last_unstale":"2023-09-23T18:22:46.556905+0000","last_undegraded":"2023-09-23T18:22:46.556905+0000","last_fullsized":"2023-09-23T18:22:46.556905+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:11.349439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":108,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.564362+0000","last_change":"2023-09-23T18:22:03.464544+0000","last_active":"2023-09-23T18:22:46.564362+0000","last_peered":"2023-09-23T18:22:46.564362+0000","last_clean":"2023-09-23T18:22:46.564362+0000","last_became_active":"2023-09-23T18:22:03.464227+0000","last_became_peered":"2023-09-23T18:22:03.464227+0000","last_unstale":"2023-09-23T18:22:46.564362+0000","last_undegraded":"2023-09-23T18:22:46.564362+0000","last_fullsized":"2023-09-23T18:22:46.564362+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:56.368635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":859,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.555904+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:22:46.555904+0000","last_peered":"2023-09-23T18:22:46.555904+0000","last_clean":"2023-09-23T18:22:46.555904+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:22:46.555904+0000","last_undegraded":"2023-09-23T18:22:46.555904+0000","last_fullsized":"2023-09-23T18:22:46.555904+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":179,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.562704+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:22:46.562704+0000","last_peered":"2023-09-23T18:22:46.562704+0000","last_clean":"2023-09-23T18:22:46.562704+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:22:46.562704+0000","last_undegraded":"2023-09-23T18:22:46.562704+0000","last_fullsized":"2023-09-23T18:22:46.562704+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":172,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556626+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:22:46.556626+0000","last_peered":"2023-09-23T18:22:46.556626+0000","last_clean":"2023-09-23T18:22:46.556626+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:22:46.556626+0000","last_undegraded":"2023-09-23T18:22:46.556626+0000","last_fullsized":"2023-09-23T18:22:46.556626+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":799,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.560384+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:22:46.560384+0000","last_peered":"2023-09-23T18:22:46.560384+0000","last_clean":"2023-09-23T18:22:46.560384+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:22:46.560384+0000","last_undegraded":"2023-09-23T18:22:46.560384+0000","last_fullsized":"2023-09-23T18:22:46.560384+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":249,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557231+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:22:46.557231+0000","last_peered":"2023-09-23T18:22:46.557231+0000","last_clean":"2023-09-23T18:22:46.557231+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:22:46.557231+0000","last_undegraded":"2023-09-23T18:22:46.557231+0000","last_fullsized":"2023-09-23T18:22:46.557231+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":271,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558602+0000","last_change":"2023-09-23T18:22:03.464607+0000","last_active":"2023-09-23T18:22:46.558602+0000","last_peered":"2023-09-23T18:22:46.558602+0000","last_clean":"2023-09-23T18:22:46.558602+0000","last_became_active":"2023-09-23T18:22:03.464333+0000","last_became_peered":"2023-09-23T18:22:03.464333+0000","last_unstale":"2023-09-23T18:22:46.558602+0000","last_undegraded":"2023-09-23T18:22:46.558602+0000","last_fullsized":"2023-09-23T18:22:46.558602+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:51.062916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":893,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556529+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:22:46.556529+0000","last_peered":"2023-09-23T18:22:46.556529+0000","last_clean":"2023-09-23T18:22:46.556529+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:22:46.556529+0000","last_undegraded":"2023-09-23T18:22:46.556529+0000","last_fullsized":"2023-09-23T18:22:46.556529+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":802,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556519+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:22:46.556519+0000","last_peered":"2023-09-23T18:22:46.556519+0000","last_clean":"2023-09-23T18:22:46.556519+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:22:46.556519+0000","last_undegraded":"2023-09-23T18:22:46.556519+0000","last_fullsized":"2023-09-23T18:22:46.556519+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.560049+0000","last_change":"2023-09-23T18:22:03.468997+0000","last_active":"2023-09-23T18:22:46.560049+0000","last_peered":"2023-09-23T18:22:46.560049+0000","last_clean":"2023-09-23T18:22:46.560049+0000","last_became_active":"2023-09-23T18:22:03.468301+0000","last_became_peered":"2023-09-23T18:22:03.468301+0000","last_unstale":"2023-09-23T18:22:46.560049+0000","last_undegraded":"2023-09-23T18:22:46.560049+0000","last_fullsized":"2023-09-23T18:22:46.560049+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:32:17.742551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":919,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.574504+0000","last_change":"2023-09-23T18:22:47.574504+0000","last_active":"2023-09-23T18:22:47.574504+0000","last_peered":"2023-09-23T18:22:47.574504+0000","last_clean":"2023-09-23T18:22:47.574504+0000","last_became_active":"2023-09-23T18:22:47.574095+0000","last_became_peered":"2023-09-23T18:22:47.574095+0000","last_unstale":"2023-09-23T18:22:47.574504+0000","last_undegraded":"2023-09-23T18:22:47.574504+0000","last_fullsized":"2023-09-23T18:22:47.574504+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:13.813532+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":299,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.562539+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:22:46.562539+0000","last_peered":"2023-09-23T18:22:46.562539+0000","last_clean":"2023-09-23T18:22:46.562539+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:22:46.562539+0000","last_undegraded":"2023-09-23T18:22:46.562539+0000","last_fullsized":"2023-09-23T18:22:46.562539+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":172,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.555882+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:22:46.555882+0000","last_peered":"2023-09-23T18:22:46.555882+0000","last_clean":"2023-09-23T18:22:46.555882+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:22:46.555882+0000","last_undegraded":"2023-09-23T18:22:46.555882+0000","last_fullsized":"2023-09-23T18:22:46.555882+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":795,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.559529+0000","last_change":"2023-09-23T18:21:51.851401+0000","last_active":"2023-09-23T18:22:46.559529+0000","last_peered":"2023-09-23T18:22:46.559529+0000","last_clean":"2023-09-23T18:22:46.559529+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:22:46.559529+0000","last_undegraded":"2023-09-23T18:22:46.559529+0000","last_fullsized":"2023-09-23T18:22:46.559529+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:21:51.851269+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:21:51.851269+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:48.305008+0000","scrub_duration":0.001933539,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":300,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.562705+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:22:46.562705+0000","last_peered":"2023-09-23T18:22:46.562705+0000","last_clean":"2023-09-23T18:22:46.562705+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:22:46.562705+0000","last_undegraded":"2023-09-23T18:22:46.562705+0000","last_fullsized":"2023-09-23T18:22:46.562705+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":279,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561306+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:22:46.561306+0000","last_peered":"2023-09-23T18:22:46.561306+0000","last_clean":"2023-09-23T18:22:46.561306+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:22:46.561306+0000","last_undegraded":"2023-09-23T18:22:46.561306+0000","last_fullsized":"2023-09-23T18:22:46.561306+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":677,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558263+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:22:46.558263+0000","last_peered":"2023-09-23T18:22:46.558263+0000","last_clean":"2023-09-23T18:22:46.558263+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:22:46.558263+0000","last_undegraded":"2023-09-23T18:22:46.558263+0000","last_fullsized":"2023-09-23T18:22:46.558263+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":587,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556446+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:22:46.556446+0000","last_peered":"2023-09-23T18:22:46.556446+0000","last_clean":"2023-09-23T18:22:46.556446+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:22:46.556446+0000","last_undegraded":"2023-09-23T18:22:46.556446+0000","last_fullsized":"2023-09-23T18:22:46.556446+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":238,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556898+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:22:46.556898+0000","last_peered":"2023-09-23T18:22:46.556898+0000","last_clean":"2023-09-23T18:22:46.556898+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:22:46.556898+0000","last_undegraded":"2023-09-23T18:22:46.556898+0000","last_fullsized":"2023-09-23T18:22:46.556898+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":712,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.573659+0000","last_change":"2023-09-23T18:22:47.573659+0000","last_active":"2023-09-23T18:22:47.573659+0000","last_peered":"2023-09-23T18:22:47.573659+0000","last_clean":"2023-09-23T18:22:47.573659+0000","last_became_active":"2023-09-23T18:22:47.573173+0000","last_became_peered":"2023-09-23T18:22:47.573173+0000","last_unstale":"2023-09-23T18:22:47.573659+0000","last_undegraded":"2023-09-23T18:22:47.573659+0000","last_fullsized":"2023-09-23T18:22:47.573659+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:15.423306+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":186,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556793+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:22:46.556793+0000","last_peered":"2023-09-23T18:22:46.556793+0000","last_clean":"2023-09-23T18:22:46.556793+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:22:46.556793+0000","last_undegraded":"2023-09-23T18:22:46.556793+0000","last_fullsized":"2023-09-23T18:22:46.556793+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":501,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.563334+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:22:46.563334+0000","last_peered":"2023-09-23T18:22:46.563334+0000","last_clean":"2023-09-23T18:22:46.563334+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:22:46.563334+0000","last_undegraded":"2023-09-23T18:22:46.563334+0000","last_fullsized":"2023-09-23T18:22:46.563334+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1164,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.571678+0000","last_change":"2023-09-23T18:22:47.571678+0000","last_active":"2023-09-23T18:22:47.571678+0000","last_peered":"2023-09-23T18:22:47.571678+0000","last_clean":"2023-09-23T18:22:47.571678+0000","last_became_active":"2023-09-23T18:22:47.571151+0000","last_became_peered":"2023-09-23T18:22:47.571151+0000","last_unstale":"2023-09-23T18:22:47.571678+0000","last_undegraded":"2023-09-23T18:22:47.571678+0000","last_fullsized":"2023-09-23T18:22:47.571678+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:46.934447+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":401,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561103+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:22:46.561103+0000","last_peered":"2023-09-23T18:22:46.561103+0000","last_clean":"2023-09-23T18:22:46.561103+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:22:46.561103+0000","last_undegraded":"2023-09-23T18:22:46.561103+0000","last_fullsized":"2023-09-23T18:22:46.561103+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":206,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558902+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:22:46.558902+0000","last_peered":"2023-09-23T18:22:46.558902+0000","last_clean":"2023-09-23T18:22:46.558902+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:22:46.558902+0000","last_undegraded":"2023-09-23T18:22:46.558902+0000","last_fullsized":"2023-09-23T18:22:46.558902+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":951,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556042+0000","last_change":"2023-09-23T18:22:06.466040+0000","last_active":"2023-09-23T18:22:46.556042+0000","last_peered":"2023-09-23T18:22:46.556042+0000","last_clean":"2023-09-23T18:22:46.556042+0000","last_became_active":"2023-09-23T18:22:06.465158+0000","last_became_peered":"2023-09-23T18:22:06.465158+0000","last_unstale":"2023-09-23T18:22:46.556042+0000","last_undegraded":"2023-09-23T18:22:46.556042+0000","last_fullsized":"2023-09-23T18:22:46.556042+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:29.587874+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556809+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:22:46.556809+0000","last_peered":"2023-09-23T18:22:46.556809+0000","last_clean":"2023-09-23T18:22:46.556809+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:22:46.556809+0000","last_undegraded":"2023-09-23T18:22:46.556809+0000","last_fullsized":"2023-09-23T18:22:46.556809+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":457,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.562477+0000","last_change":"2023-09-23T18:22:44.527193+0000","last_active":"2023-09-23T18:22:46.562477+0000","last_peered":"2023-09-23T18:22:46.562477+0000","last_clean":"2023-09-23T18:22:46.562477+0000","last_became_active":"2023-09-23T18:22:44.526903+0000","last_became_peered":"2023-09-23T18:22:44.526903+0000","last_unstale":"2023-09-23T18:22:46.562477+0000","last_undegraded":"2023-09-23T18:22:46.562477+0000","last_fullsized":"2023-09-23T18:22:46.562477+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:09.126029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":933,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556525+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:22:46.556525+0000","last_peered":"2023-09-23T18:22:46.556525+0000","last_clean":"2023-09-23T18:22:46.556525+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:22:46.556525+0000","last_undegraded":"2023-09-23T18:22:46.556525+0000","last_fullsized":"2023-09-23T18:22:46.556525+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":467,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561149+0000","last_change":"2023-09-23T18:22:03.465235+0000","last_active":"2023-09-23T18:22:46.561149+0000","last_peered":"2023-09-23T18:22:46.561149+0000","last_clean":"2023-09-23T18:22:46.561149+0000","last_became_active":"2023-09-23T18:22:03.464906+0000","last_became_peered":"2023-09-23T18:22:03.464906+0000","last_unstale":"2023-09-23T18:22:46.561149+0000","last_undegraded":"2023-09-23T18:22:46.561149+0000","last_fullsized":"2023-09-23T18:22:46.561149+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:06:46.429598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":274,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556564+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:22:46.556564+0000","last_peered":"2023-09-23T18:22:46.556564+0000","last_clean":"2023-09-23T18:22:46.556564+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:22:46.556564+0000","last_undegraded":"2023-09-23T18:22:46.556564+0000","last_fullsized":"2023-09-23T18:22:46.556564+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":934,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557370+0000","last_change":"2023-09-23T18:22:06.468619+0000","last_active":"2023-09-23T18:22:46.557370+0000","last_peered":"2023-09-23T18:22:46.557370+0000","last_clean":"2023-09-23T18:22:46.557370+0000","last_became_active":"2023-09-23T18:22:06.468197+0000","last_became_peered":"2023-09-23T18:22:06.468197+0000","last_unstale":"2023-09-23T18:22:46.557370+0000","last_undegraded":"2023-09-23T18:22:46.557370+0000","last_fullsized":"2023-09-23T18:22:46.557370+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:10.149437+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":80,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557792+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:22:46.557792+0000","last_peered":"2023-09-23T18:22:46.557792+0000","last_clean":"2023-09-23T18:22:46.557792+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:22:46.557792+0000","last_undegraded":"2023-09-23T18:22:46.557792+0000","last_fullsized":"2023-09-23T18:22:46.557792+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":392,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558148+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:22:46.558148+0000","last_peered":"2023-09-23T18:22:46.558148+0000","last_clean":"2023-09-23T18:22:46.558148+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:22:46.558148+0000","last_undegraded":"2023-09-23T18:22:46.558148+0000","last_fullsized":"2023-09-23T18:22:46.558148+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":773,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558449+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:22:46.558449+0000","last_peered":"2023-09-23T18:22:46.558449+0000","last_clean":"2023-09-23T18:22:46.558449+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:22:46.558449+0000","last_undegraded":"2023-09-23T18:22:46.558449+0000","last_fullsized":"2023-09-23T18:22:46.558449+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"653'291","reported_seq":1547,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557793+0000","last_change":"2023-09-23T18:22:28.465602+0000","last_active":"2023-09-23T18:22:46.557793+0000","last_peered":"2023-09-23T18:22:46.557793+0000","last_clean":"2023-09-23T18:22:46.557793+0000","last_became_active":"2023-09-23T18:22:06.466146+0000","last_became_peered":"2023-09-23T18:22:06.466146+0000","last_unstale":"2023-09-23T18:22:46.557793+0000","last_undegraded":"2023-09-23T18:22:46.557793+0000","last_fullsized":"2023-09-23T18:22:46.557793+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":291,"log_dups_size":0,"ondisk_log_size":291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:22.022031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047292100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1326,"num_read_kb":5772,"num_write":300,"num_write_kb":4517,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":206,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556897+0000","last_change":"2023-09-23T18:22:03.462589+0000","last_active":"2023-09-23T18:22:46.556897+0000","last_peered":"2023-09-23T18:22:46.556897+0000","last_clean":"2023-09-23T18:22:46.556897+0000","last_became_active":"2023-09-23T18:22:03.462239+0000","last_became_peered":"2023-09-23T18:22:03.462239+0000","last_unstale":"2023-09-23T18:22:46.556897+0000","last_undegraded":"2023-09-23T18:22:46.556897+0000","last_fullsized":"2023-09-23T18:22:46.556897+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.020770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":176,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557276+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:22:46.557276+0000","last_peered":"2023-09-23T18:22:46.557276+0000","last_clean":"2023-09-23T18:22:46.557276+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:22:46.557276+0000","last_undegraded":"2023-09-23T18:22:46.557276+0000","last_fullsized":"2023-09-23T18:22:46.557276+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":865,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556229+0000","last_change":"2023-09-23T18:22:06.465690+0000","last_active":"2023-09-23T18:22:46.556229+0000","last_peered":"2023-09-23T18:22:46.556229+0000","last_clean":"2023-09-23T18:22:46.556229+0000","last_became_active":"2023-09-23T18:22:06.465148+0000","last_became_peered":"2023-09-23T18:22:06.465148+0000","last_unstale":"2023-09-23T18:22:46.556229+0000","last_undegraded":"2023-09-23T18:22:46.556229+0000","last_fullsized":"2023-09-23T18:22:46.556229+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:00.043030+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"653'323","reported_seq":2262,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557184+0000","last_change":"2023-09-23T18:22:28.473090+0000","last_active":"2023-09-23T18:22:46.557184+0000","last_peered":"2023-09-23T18:22:46.557184+0000","last_clean":"2023-09-23T18:22:46.557184+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:22:46.557184+0000","last_undegraded":"2023-09-23T18:22:46.557184+0000","last_fullsized":"2023-09-23T18:22:46.557184+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":323,"log_dups_size":0,"ondisk_log_size":323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001426075,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3093,"num_read_kb":5799,"num_write":265,"num_write_kb":3199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":186,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557004+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:22:46.557004+0000","last_peered":"2023-09-23T18:22:46.557004+0000","last_clean":"2023-09-23T18:22:46.557004+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:22:46.557004+0000","last_undegraded":"2023-09-23T18:22:46.557004+0000","last_fullsized":"2023-09-23T18:22:46.557004+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":425,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557567+0000","last_change":"2023-09-23T18:22:44.519917+0000","last_active":"2023-09-23T18:22:46.557567+0000","last_peered":"2023-09-23T18:22:46.557567+0000","last_clean":"2023-09-23T18:22:46.557567+0000","last_became_active":"2023-09-23T18:22:44.519386+0000","last_became_peered":"2023-09-23T18:22:44.519386+0000","last_unstale":"2023-09-23T18:22:46.557567+0000","last_undegraded":"2023-09-23T18:22:46.557567+0000","last_fullsized":"2023-09-23T18:22:46.557567+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:44.617263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":705,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.573442+0000","last_change":"2023-09-23T18:22:47.573442+0000","last_active":"2023-09-23T18:22:47.573442+0000","last_peered":"2023-09-23T18:22:47.573442+0000","last_clean":"2023-09-23T18:22:47.573442+0000","last_became_active":"2023-09-23T18:22:47.573059+0000","last_became_peered":"2023-09-23T18:22:47.573059+0000","last_unstale":"2023-09-23T18:22:47.573442+0000","last_undegraded":"2023-09-23T18:22:47.573442+0000","last_fullsized":"2023-09-23T18:22:47.573442+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:43.254558+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"653'200","reported_seq":488,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.570312+0000","last_change":"2023-09-23T18:22:47.569585+0000","last_active":"2023-09-23T18:22:47.570312+0000","last_peered":"2023-09-23T18:22:47.570312+0000","last_clean":"2023-09-23T18:22:47.570312+0000","last_became_active":"2023-09-23T18:22:47.568716+0000","last_became_peered":"2023-09-23T18:22:47.568716+0000","last_unstale":"2023-09-23T18:22:47.570312+0000","last_undegraded":"2023-09-23T18:22:47.570312+0000","last_fullsized":"2023-09-23T18:22:47.570312+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:16.049107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0022753629999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1032,"num_read_kb":1283,"num_write":194,"num_write_kb":5509,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":20971520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":465,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557119+0000","last_change":"2023-09-23T18:22:44.518872+0000","last_active":"2023-09-23T18:22:46.557119+0000","last_peered":"2023-09-23T18:22:46.557119+0000","last_clean":"2023-09-23T18:22:46.557119+0000","last_became_active":"2023-09-23T18:22:44.518360+0000","last_became_peered":"2023-09-23T18:22:44.518360+0000","last_unstale":"2023-09-23T18:22:46.557119+0000","last_undegraded":"2023-09-23T18:22:46.557119+0000","last_fullsized":"2023-09-23T18:22:46.557119+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:43.986310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":198,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557102+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:22:46.557102+0000","last_peered":"2023-09-23T18:22:46.557102+0000","last_clean":"2023-09-23T18:22:46.557102+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:22:46.557102+0000","last_undegraded":"2023-09-23T18:22:46.557102+0000","last_fullsized":"2023-09-23T18:22:46.557102+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"653'474","reported_seq":2274,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.573054+0000","last_change":"2023-09-23T18:22:47.573054+0000","last_active":"2023-09-23T18:22:47.573054+0000","last_peered":"2023-09-23T18:22:47.573054+0000","last_clean":"2023-09-23T18:22:47.573054+0000","last_became_active":"2023-09-23T18:22:47.572246+0000","last_became_peered":"2023-09-23T18:22:47.572246+0000","last_unstale":"2023-09-23T18:22:47.573054+0000","last_undegraded":"2023-09-23T18:22:47.573054+0000","last_fullsized":"2023-09-23T18:22:47.573054+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":474,"log_dups_size":0,"ondisk_log_size":474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:41.513401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014671619999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2565,"num_read_kb":5830,"num_write":410,"num_write_kb":4052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":612,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556798+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:22:46.556798+0000","last_peered":"2023-09-23T18:22:46.556798+0000","last_clean":"2023-09-23T18:22:46.556798+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:22:46.556798+0000","last_undegraded":"2023-09-23T18:22:46.556798+0000","last_fullsized":"2023-09-23T18:22:46.556798+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":172,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556370+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:22:46.556370+0000","last_peered":"2023-09-23T18:22:46.556370+0000","last_clean":"2023-09-23T18:22:46.556370+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:22:46.556370+0000","last_undegraded":"2023-09-23T18:22:46.556370+0000","last_fullsized":"2023-09-23T18:22:46.556370+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":207,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558681+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:22:46.558681+0000","last_peered":"2023-09-23T18:22:46.558681+0000","last_clean":"2023-09-23T18:22:46.558681+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:22:46.558681+0000","last_undegraded":"2023-09-23T18:22:46.558681+0000","last_fullsized":"2023-09-23T18:22:46.558681+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":860,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557250+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:22:46.557250+0000","last_peered":"2023-09-23T18:22:46.557250+0000","last_clean":"2023-09-23T18:22:46.557250+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:22:46.557250+0000","last_undegraded":"2023-09-23T18:22:46.557250+0000","last_fullsized":"2023-09-23T18:22:46.557250+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"653'156","reported_seq":455,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558033+0000","last_change":"2023-09-23T18:22:28.481047+0000","last_active":"2023-09-23T18:22:46.558033+0000","last_peered":"2023-09-23T18:22:46.558033+0000","last_clean":"2023-09-23T18:22:46.558033+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:22:46.558033+0000","last_undegraded":"2023-09-23T18:22:46.558033+0000","last_fullsized":"2023-09-23T18:22:46.558033+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":156,"log_dups_size":0,"ondisk_log_size":156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504954,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":805,"num_read_kb":2049,"num_write":144,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":80,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558841+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:22:46.558841+0000","last_peered":"2023-09-23T18:22:46.558841+0000","last_clean":"2023-09-23T18:22:46.558841+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:22:46.558841+0000","last_undegraded":"2023-09-23T18:22:46.558841+0000","last_fullsized":"2023-09-23T18:22:46.558841+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":610,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.563158+0000","last_change":"2023-09-23T18:22:44.518222+0000","last_active":"2023-09-23T18:22:46.563158+0000","last_peered":"2023-09-23T18:22:46.563158+0000","last_clean":"2023-09-23T18:22:46.563158+0000","last_became_active":"2023-09-23T18:22:44.517848+0000","last_became_peered":"2023-09-23T18:22:44.517848+0000","last_unstale":"2023-09-23T18:22:46.563158+0000","last_undegraded":"2023-09-23T18:22:46.563158+0000","last_fullsized":"2023-09-23T18:22:46.563158+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:40.879308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":960,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557033+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:22:46.557033+0000","last_peered":"2023-09-23T18:22:46.557033+0000","last_clean":"2023-09-23T18:22:46.557033+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:22:46.557033+0000","last_undegraded":"2023-09-23T18:22:46.557033+0000","last_fullsized":"2023-09-23T18:22:46.557033+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"653'181","reported_seq":619,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.562954+0000","last_change":"2023-09-23T18:22:28.468611+0000","last_active":"2023-09-23T18:22:46.562954+0000","last_peered":"2023-09-23T18:22:46.562954+0000","last_clean":"2023-09-23T18:22:46.562954+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:22:46.562954+0000","last_undegraded":"2023-09-23T18:22:46.562954+0000","last_fullsized":"2023-09-23T18:22:46.562954+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001526466,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1349,"num_read_kb":1261,"num_write":144,"num_write_kb":4223,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":196,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556527+0000","last_change":"2023-09-23T18:22:44.518634+0000","last_active":"2023-09-23T18:22:46.556527+0000","last_peered":"2023-09-23T18:22:46.556527+0000","last_clean":"2023-09-23T18:22:46.556527+0000","last_became_active":"2023-09-23T18:22:44.518286+0000","last_became_peered":"2023-09-23T18:22:44.518286+0000","last_unstale":"2023-09-23T18:22:46.556527+0000","last_undegraded":"2023-09-23T18:22:46.556527+0000","last_fullsized":"2023-09-23T18:22:46.556527+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:05.309943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":186,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556284+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:22:46.556284+0000","last_peered":"2023-09-23T18:22:46.556284+0000","last_clean":"2023-09-23T18:22:46.556284+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:22:46.556284+0000","last_undegraded":"2023-09-23T18:22:46.556284+0000","last_fullsized":"2023-09-23T18:22:46.556284+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":980,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.573464+0000","last_change":"2023-09-23T18:22:47.573464+0000","last_active":"2023-09-23T18:22:47.573464+0000","last_peered":"2023-09-23T18:22:47.573464+0000","last_clean":"2023-09-23T18:22:47.573464+0000","last_became_active":"2023-09-23T18:22:47.572988+0000","last_became_peered":"2023-09-23T18:22:47.572988+0000","last_unstale":"2023-09-23T18:22:47.573464+0000","last_undegraded":"2023-09-23T18:22:47.573464+0000","last_fullsized":"2023-09-23T18:22:47.573464+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.224856+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":148,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556861+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:22:46.556861+0000","last_peered":"2023-09-23T18:22:46.556861+0000","last_clean":"2023-09-23T18:22:46.556861+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:22:46.556861+0000","last_undegraded":"2023-09-23T18:22:46.556861+0000","last_fullsized":"2023-09-23T18:22:46.556861+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"653'188","reported_seq":942,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556793+0000","last_change":"2023-09-23T18:22:28.475214+0000","last_active":"2023-09-23T18:22:46.556793+0000","last_peered":"2023-09-23T18:22:46.556793+0000","last_clean":"2023-09-23T18:22:46.556793+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:22:46.556793+0000","last_undegraded":"2023-09-23T18:22:46.556793+0000","last_fullsized":"2023-09-23T18:22:46.556793+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":188,"log_dups_size":0,"ondisk_log_size":188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055658400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1067,"num_read_kb":2442,"num_write":105,"num_write_kb":1356,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":81,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557536+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:22:46.557536+0000","last_peered":"2023-09-23T18:22:46.557536+0000","last_clean":"2023-09-23T18:22:46.557536+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:22:46.557536+0000","last_undegraded":"2023-09-23T18:22:46.557536+0000","last_fullsized":"2023-09-23T18:22:46.557536+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":796,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.559653+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:22:46.559653+0000","last_peered":"2023-09-23T18:22:46.559653+0000","last_clean":"2023-09-23T18:22:46.559653+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:22:46.559653+0000","last_undegraded":"2023-09-23T18:22:46.559653+0000","last_fullsized":"2023-09-23T18:22:46.559653+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"652'137","reported_seq":1318,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558629+0000","last_change":"2023-09-23T18:22:28.475379+0000","last_active":"2023-09-23T18:22:46.558629+0000","last_peered":"2023-09-23T18:22:46.558629+0000","last_clean":"2023-09-23T18:22:46.558629+0000","last_became_active":"2023-09-23T18:22:03.468424+0000","last_became_peered":"2023-09-23T18:22:03.468424+0000","last_unstale":"2023-09-23T18:22:46.558629+0000","last_undegraded":"2023-09-23T18:22:46.558629+0000","last_fullsized":"2023-09-23T18:22:46.558629+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:46.671634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044965899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":925,"num_write":128,"num_write_kb":1260,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":90,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.563369+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:22:46.563369+0000","last_peered":"2023-09-23T18:22:46.563369+0000","last_clean":"2023-09-23T18:22:46.563369+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:22:46.563369+0000","last_undegraded":"2023-09-23T18:22:46.563369+0000","last_fullsized":"2023-09-23T18:22:46.563369+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":452,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556001+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:22:46.556001+0000","last_peered":"2023-09-23T18:22:46.556001+0000","last_clean":"2023-09-23T18:22:46.556001+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:22:46.556001+0000","last_undegraded":"2023-09-23T18:22:46.556001+0000","last_fullsized":"2023-09-23T18:22:46.556001+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":692,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558085+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:22:46.558085+0000","last_peered":"2023-09-23T18:22:46.558085+0000","last_clean":"2023-09-23T18:22:46.558085+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:22:46.558085+0000","last_undegraded":"2023-09-23T18:22:46.558085+0000","last_fullsized":"2023-09-23T18:22:46.558085+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":86,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.563168+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:22:46.563168+0000","last_peered":"2023-09-23T18:22:46.563168+0000","last_clean":"2023-09-23T18:22:46.563168+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:22:46.563168+0000","last_undegraded":"2023-09-23T18:22:46.563168+0000","last_fullsized":"2023-09-23T18:22:46.563168+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.564029+0000","last_change":"2023-09-23T18:22:44.520691+0000","last_active":"2023-09-23T18:22:47.564029+0000","last_peered":"2023-09-23T18:22:47.564029+0000","last_clean":"2023-09-23T18:22:47.564029+0000","last_became_active":"2023-09-23T18:22:44.520082+0000","last_became_peered":"2023-09-23T18:22:44.520082+0000","last_unstale":"2023-09-23T18:22:47.564029+0000","last_undegraded":"2023-09-23T18:22:47.564029+0000","last_fullsized":"2023-09-23T18:22:47.564029+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:29.126035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":692,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556685+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:22:46.556685+0000","last_peered":"2023-09-23T18:22:46.556685+0000","last_clean":"2023-09-23T18:22:46.556685+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:22:46.556685+0000","last_undegraded":"2023-09-23T18:22:46.556685+0000","last_fullsized":"2023-09-23T18:22:46.556685+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":180,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556628+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:22:46.556628+0000","last_peered":"2023-09-23T18:22:46.556628+0000","last_clean":"2023-09-23T18:22:46.556628+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:22:46.556628+0000","last_undegraded":"2023-09-23T18:22:46.556628+0000","last_fullsized":"2023-09-23T18:22:46.556628+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":297,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561410+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:22:46.561410+0000","last_peered":"2023-09-23T18:22:46.561410+0000","last_clean":"2023-09-23T18:22:46.561410+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:22:46.561410+0000","last_undegraded":"2023-09-23T18:22:46.561410+0000","last_fullsized":"2023-09-23T18:22:46.561410+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":166,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556537+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:22:46.556537+0000","last_peered":"2023-09-23T18:22:46.556537+0000","last_clean":"2023-09-23T18:22:46.556537+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:22:46.556537+0000","last_undegraded":"2023-09-23T18:22:46.556537+0000","last_fullsized":"2023-09-23T18:22:46.556537+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":280,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561182+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:22:46.561182+0000","last_peered":"2023-09-23T18:22:46.561182+0000","last_clean":"2023-09-23T18:22:46.561182+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:22:46.561182+0000","last_undegraded":"2023-09-23T18:22:46.561182+0000","last_fullsized":"2023-09-23T18:22:46.561182+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1054,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557110+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:22:46.557110+0000","last_peered":"2023-09-23T18:22:46.557110+0000","last_clean":"2023-09-23T18:22:46.557110+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:22:46.557110+0000","last_undegraded":"2023-09-23T18:22:46.557110+0000","last_fullsized":"2023-09-23T18:22:46.557110+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":569,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556931+0000","last_change":"2023-09-23T18:22:44.520133+0000","last_active":"2023-09-23T18:22:46.556931+0000","last_peered":"2023-09-23T18:22:46.556931+0000","last_clean":"2023-09-23T18:22:46.556931+0000","last_became_active":"2023-09-23T18:22:44.519765+0000","last_became_peered":"2023-09-23T18:22:44.519765+0000","last_unstale":"2023-09-23T18:22:46.556931+0000","last_undegraded":"2023-09-23T18:22:46.556931+0000","last_fullsized":"2023-09-23T18:22:46.556931+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:02.736149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558734+0000","last_change":"2023-09-23T18:22:44.519760+0000","last_active":"2023-09-23T18:22:46.558734+0000","last_peered":"2023-09-23T18:22:46.558734+0000","last_clean":"2023-09-23T18:22:46.558734+0000","last_became_active":"2023-09-23T18:22:44.519006+0000","last_became_peered":"2023-09-23T18:22:44.519006+0000","last_unstale":"2023-09-23T18:22:46.558734+0000","last_undegraded":"2023-09-23T18:22:46.558734+0000","last_fullsized":"2023-09-23T18:22:46.558734+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:02.871094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":993,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.559268+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:22:46.559268+0000","last_peered":"2023-09-23T18:22:46.559268+0000","last_clean":"2023-09-23T18:22:46.559268+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:22:46.559268+0000","last_undegraded":"2023-09-23T18:22:46.559268+0000","last_fullsized":"2023-09-23T18:22:46.559268+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":200,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557018+0000","last_change":"2023-09-23T18:22:44.528526+0000","last_active":"2023-09-23T18:22:46.557018+0000","last_peered":"2023-09-23T18:22:46.557018+0000","last_clean":"2023-09-23T18:22:46.557018+0000","last_became_active":"2023-09-23T18:22:44.518445+0000","last_became_peered":"2023-09-23T18:22:44.518445+0000","last_unstale":"2023-09-23T18:22:46.557018+0000","last_undegraded":"2023-09-23T18:22:46.557018+0000","last_fullsized":"2023-09-23T18:22:46.557018+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:17.761911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":705,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556635+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:22:46.556635+0000","last_peered":"2023-09-23T18:22:46.556635+0000","last_clean":"2023-09-23T18:22:46.556635+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:22:46.556635+0000","last_undegraded":"2023-09-23T18:22:46.556635+0000","last_fullsized":"2023-09-23T18:22:46.556635+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":13010,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":382,"num_bytes_recovered":20971577,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001603,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669539,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":799936,"kb_used_data":166584,"kb_used_omap":7,"kb_used_meta":633336,"kb_avail":92945216,"statfs":{"total":95995035648,"available":95175901184,"internally_reserved":0,"allocated":170582016,"data_stored":168576055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":648536868},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:22 2023","interfaces":[{"interface":"back","average":{"1min":4.6609999999999996,"5min":2.7709999999999999,"15min":1.8129999999999999},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":79.203999999999994,"5min":79.203999999999994,"15min":79.203999999999994},"last":0.85299999999999998},{"interface":"front","average":{"1min":4.0229999999999997,"5min":2.7810000000000001,"15min":1.9570000000000001},"min":{"1min":0.252,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":78.034000000000006,"5min":78.034000000000006,"15min":78.034000000000006},"last":0.65900000000000003}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":2.1240000000000001,"15min":1.518},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.8480000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.77400000000000002},{"interface":"front","average":{"1min":0.58399999999999996,"5min":2.1240000000000001,"15min":1.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.27,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.63400000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.518,"15min":1.369},"min":{"1min":0.33500000000000002,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":7.734,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.69299999999999995},{"interface":"front","average":{"1min":1.8120000000000001,"5min":2.0209999999999999,"15min":1.5920000000000001},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":8.2690000000000001,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.73399999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":4.6050000000000004,"5min":3.601,"15min":2.3479999999999999},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.24099999999999999},"max":{"1min":78.643000000000001,"5min":103.348,"15min":103.348},"last":0.752},{"interface":"front","average":{"1min":5.2629999999999999,"5min":4.298,"15min":2.6930000000000001},"min":{"1min":0.32100000000000001,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":78.347999999999999,"5min":103.40300000000001,"15min":103.40300000000001},"last":0.83499999999999996}]},{"osd":4,"last update":"Sat Sep 23 18:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":1.3220000000000001,"15min":1.3400000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23999999999999999},"max":{"1min":1.103,"5min":6.2800000000000002,"15min":8.6059999999999999},"last":0.71299999999999997},{"interface":"front","average":{"1min":0.46100000000000002,"5min":1.2769999999999999,"15min":1.375},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":0.91200000000000003,"5min":7.8319999999999999,"15min":9.2260000000000009},"last":0.52300000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:21:51 2023","interfaces":[{"interface":"back","average":{"1min":0.84699999999999998,"5min":1.8280000000000001,"15min":1.925},"min":{"1min":0.29599999999999999,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":5.0460000000000003,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.71499999999999997,"5min":1.9990000000000001,"15min":2.258},"min":{"1min":0.32000000000000001,"5min":0.28599999999999998,"15min":0.28599999999999998},"max":{"1min":3.0489999999999999,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.67300000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.6930000000000001,"15min":1.603},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.26800000000000002},"max":{"1min":6.1799999999999997,"5min":10.743,"15min":13.460000000000001},"last":0.78700000000000003},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.593,"15min":1.6200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.28499999999999998},"max":{"1min":5.0990000000000002,"5min":8.6890000000000001,"15min":12.404},"last":0.81599999999999995}]}]},{"osd":1,"up_from":545,"seq":2340757176388,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":648956,"kb_used_data":401012,"kb_used_omap":13,"kb_used_meta":247922,"kb_avail":93096196,"statfs":{"total":95995035648,"available":95330504704,"internally_reserved":0,"allocated":410636288,"data_stored":407416503,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":253872781},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.331,"15min":0.70399999999999996},"min":{"1min":0.23100000000000001,"5min":0.189,"15min":0.189},"max":{"1min":4.5030000000000001,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.245},{"interface":"front","average":{"1min":0.38400000000000001,"5min":1.3069999999999999,"15min":0.76200000000000001},"min":{"1min":0.28399999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.51600000000000001,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.26400000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":1.4339999999999999,"15min":0.78500000000000003},"min":{"1min":0.28899999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.61899999999999999,"5min":11.02,"15min":11.02},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.98799999999999999,"15min":0.66600000000000004},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.234,"5min":5.1440000000000001,"15min":5.1440000000000001},"last":0.38900000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":1.7689999999999999,"15min":0.84699999999999998},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0980000000000001,"5min":14.962,"15min":14.962},"last":0.32700000000000001},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.577,"15min":0.86699999999999999},"min":{"1min":0.35499999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.82199999999999995,"5min":19.762,"15min":19.762},"last":0.34200000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.304,"15min":1.3120000000000001},"min":{"1min":0.38,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.532,"5min":12.663,"15min":12.663},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.1230000000000002,"15min":1.7170000000000001},"min":{"1min":0.39000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":4.7320000000000002,"5min":13.222,"15min":13.222},"last":0.55800000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.7350000000000001,"15min":1.0600000000000001},"min":{"1min":0.373,"5min":0.308,"15min":0.308},"max":{"1min":3.6760000000000002,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.85699999999999998,"5min":2.052,"15min":1.22},"min":{"1min":0.42699999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.556,"5min":23.204000000000001,"15min":23.204000000000001},"last":0.625}]},{"osd":6,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.736,"15min":1.085},"min":{"1min":0.46999999999999997,"5min":0.372,"15min":0.372},"max":{"1min":3.5659999999999998,"5min":23.443000000000001,"15min":23.443000000000001},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.84499999999999997,"5min":2.306,"15min":1.286},"min":{"1min":0.42499999999999999,"5min":0.378,"15min":0.378},"max":{"1min":4.8959999999999999,"5min":26.177,"15min":26.177},"last":0.59999999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.9399999999999999,"15min":1.1000000000000001},"min":{"1min":0.47399999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.052,"5min":15.01,"15min":15.01},"last":0.64000000000000001},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.867,"15min":1.159},"min":{"1min":0.46899999999999997,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78400000000000003,"5min":10.988,"15min":10.988},"last":0.57299999999999995}]}]},{"osd":0,"up_from":93,"seq":399431958884,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1153208,"kb_used_data":170224,"kb_used_omap":7,"kb_used_meta":982968,"kb_avail":92591944,"statfs":{"total":95995035648,"available":94814150656,"internally_reserved":0,"allocated":174309376,"data_stored":171268704,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":1006559972},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":1,"apply_latency_ms":1,"commit_latency_ns":1000000,"apply_latency_ns":1000000},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.3839999999999999,"15min":0.67400000000000004},"min":{"1min":0.251,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":1.2170000000000001,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.32300000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":1.8340000000000001,"15min":0.95999999999999996},"min":{"1min":0.28299999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.853,"5min":22.978000000000002,"15min":22.978000000000002},"last":0.39900000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:22:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":2.355,"15min":1.4790000000000001},"min":{"1min":0.25,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":9.8300000000000001,"5min":18.550999999999998,"15min":18.550999999999998},"last":0.36099999999999999},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.925,"15min":1.1419999999999999},"min":{"1min":0.31900000000000001,"5min":0.22500000000000001,"15min":0.22},"max":{"1min":6.2220000000000004,"5min":23.161999999999999,"15min":23.161999999999999},"last":0.46200000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:21:48 2023","interfaces":[{"interface":"back","average":{"1min":2.8559999999999999,"5min":1.927,"15min":1.6890000000000001},"min":{"1min":0.24199999999999999,"5min":0.20000000000000001,"15min":0.20000000000000001},"max":{"1min":15.173999999999999,"5min":15.173999999999999,"15min":15.173999999999999},"last":0.47599999999999998},{"interface":"front","average":{"1min":2.23,"5min":2.0350000000000001,"15min":1.718},"min":{"1min":0.32100000000000001,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":15.135,"5min":15.135,"15min":15.135},"last":0.42999999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:22:15 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.944,"15min":1.9299999999999999},"min":{"1min":0.33300000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9260000000000002,"5min":17.741,"15min":57.746000000000002},"last":0.58099999999999996},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.899,"15min":1.7150000000000001},"min":{"1min":0.42999999999999999,"5min":0.29199999999999998,"15min":0.25600000000000001},"max":{"1min":6.3860000000000001,"5min":15.079000000000001,"15min":57.816000000000003},"last":0.68000000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:21:48 2023","interfaces":[{"interface":"back","average":{"1min":1.3640000000000001,"5min":1.73,"15min":1.466},"min":{"1min":0.30199999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":5.3979999999999997,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.76900000000000002},{"interface":"front","average":{"1min":1.4610000000000001,"5min":2.3399999999999999,"15min":1.6279999999999999},"min":{"1min":0.35499999999999998,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":7.7359999999999998,"5min":15.047000000000001,"15min":15.047000000000001},"last":0.73099999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.8089999999999999,"15min":1.9870000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.32000000000000001},"max":{"1min":6.0439999999999996,"5min":7.6900000000000004,"15min":58.457999999999998},"last":0.746},{"interface":"front","average":{"1min":1.284,"5min":2.645,"15min":2.0739999999999998},"min":{"1min":0.47599999999999998,"5min":0.375,"15min":0.315},"max":{"1min":4.4470000000000001,"5min":22.893000000000001,"15min":58.354999999999997},"last":0.69799999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:21:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":2.2130000000000001,"15min":1.825},"min":{"1min":0.46200000000000002,"5min":0.45000000000000001,"15min":0.34499999999999997},"max":{"1min":6.1200000000000001,"5min":22.945,"15min":22.945},"last":0.63700000000000001},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.7410000000000001,"15min":1.851},"min":{"1min":0.42799999999999999,"5min":0.371,"15min":0.36199999999999999},"max":{"1min":5.4320000000000004,"5min":7.7519999999999998,"15min":10.404},"last":0.71399999999999997}]}]},{"osd":2,"up_from":482,"seq":2070174236770,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161072,"kb_used_data":12840,"kb_used_omap":22,"kb_used_meta":148201,"kb_avail":93584080,"statfs":{"total":95995035648,"available":95830097920,"internally_reserved":0,"allocated":13148160,"data_stored":9940106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22605,"internal_metadata":151758771},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.5980000000000001,"15min":0.90900000000000003},"min":{"1min":0.24299999999999999,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":4.7060000000000004,"5min":9.6359999999999992,"15min":9.6359999999999992},"last":0.40600000000000003},{"interface":"front","average":{"1min":2.9289999999999998,"5min":1.964,"15min":1.286},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.3390000000000004,"5min":13.472,"15min":13.472},"last":0.51900000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.4219999999999999,"15min":0.80800000000000005},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.1320000000000001,"5min":13.538,"15min":13.538},"last":0.48499999999999999},{"interface":"front","average":{"1min":0.504,"5min":1.681,"15min":0.98399999999999999},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.1499999999999999,"5min":16.567,"15min":16.567},"last":0.502}]},{"osd":3,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":2.1640000000000001,"15min":1.3919999999999999},"min":{"1min":0.40300000000000002,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":8.093,"5min":15.926,"15min":15.926},"last":0.54200000000000004},{"interface":"front","average":{"1min":1.841,"5min":1.754,"15min":1.409},"min":{"1min":0.28199999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":13.247999999999999,"5min":16.619,"15min":16.619},"last":0.46700000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:22:03 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.667,"15min":1.512},"min":{"1min":0.375,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":4.6539999999999999,"5min":9.532,"15min":11.334},"last":0.60599999999999998},{"interface":"front","average":{"1min":1.601,"5min":1.5589999999999999,"15min":1.1599999999999999},"min":{"1min":0.40000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.5229999999999997,"5min":6.367,"15min":11.244},"last":0.754}]},{"osd":5,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.714,"15min":2.0339999999999998},"min":{"1min":0.38300000000000001,"5min":0.34799999999999998,"15min":0.308},"max":{"1min":9.4960000000000004,"5min":9.4960000000000004,"15min":11.852},"last":0.79700000000000004},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.6890000000000001,"15min":1.5980000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32600000000000001},"max":{"1min":4.6719999999999997,"5min":8.3849999999999998,"15min":10.942},"last":0.71799999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:22:03 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":2.0569999999999999,"15min":1.966},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":29.489000000000001,"5min":29.489000000000001,"15min":29.489000000000001},"last":0.66300000000000003},{"interface":"front","average":{"1min":3.048,"5min":2.1080000000000001,"15min":2.3759999999999999},"min":{"1min":0.42899999999999999,"5min":0.39600000000000002,"15min":0.38700000000000001},"max":{"1min":29.408999999999999,"5min":29.408999999999999,"15min":29.408999999999999},"last":0.81100000000000005}]},{"osd":7,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":1.671,"15min":3.839},"min":{"1min":0.495,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":6.9779999999999998,"5min":13.106,"15min":76.966999999999999},"last":0.78500000000000003},{"interface":"front","average":{"1min":1.746,"5min":1.724,"15min":4.0179999999999998},"min":{"1min":0.45500000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":9.5619999999999994,"5min":9.5619999999999994,"15min":76.936999999999998},"last":0.68799999999999994}]}]},{"osd":3,"up_from":237,"seq":1017907249399,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818476,"kb_used_data":12964,"kb_used_omap":3,"kb_used_meta":805500,"kb_avail":92926676,"statfs":{"total":95995035648,"available":95156916224,"internally_reserved":0,"allocated":13275136,"data_stored":10013585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":824832744},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.7869999999999999,"5min":1.841,"15min":1.8700000000000001},"min":{"1min":0.20100000000000001,"5min":0.192,"15min":0.183},"max":{"1min":6.3360000000000003,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.29599999999999999},{"interface":"front","average":{"1min":1.925,"5min":1.7270000000000001,"15min":2.0590000000000002},"min":{"1min":0.311,"5min":0.217,"15min":0.20999999999999999},"max":{"1min":8.9440000000000008,"5min":11.468,"15min":194.661},"last":0.42099999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.847,"15min":0.96199999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":8.9629999999999992,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.7839999999999998,"15min":1.3979999999999999},"min":{"1min":0.29099999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.5600000000000001,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.40600000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:22:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2090000000000001,"5min":2.0369999999999999,"15min":1.5009999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.21099999999999999},"max":{"1min":18.722999999999999,"5min":18.722999999999999,"15min":18.722999999999999},"last":0.44700000000000001},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.6699999999999999,"15min":1.3999999999999999},"min":{"1min":0.32800000000000001,"5min":0.27900000000000003,"15min":0.19600000000000001},"max":{"1min":7.0579999999999998,"5min":11.565,"15min":11.565},"last":0.371}]},{"osd":4,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.3340000000000001,"5min":1.76,"15min":1.7330000000000001},"min":{"1min":0.36899999999999999,"5min":0.29499999999999998,"15min":0.26000000000000001},"max":{"1min":4.4089999999999998,"5min":20.850000000000001,"15min":37.231000000000002},"last":0.71799999999999997},{"interface":"front","average":{"1min":1.98,"5min":1.99,"15min":1.9570000000000001},"min":{"1min":0.34399999999999997,"5min":0.317,"15min":0.29199999999999998},"max":{"1min":7.6840000000000002,"5min":20.786000000000001,"15min":37.381},"last":0.77800000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":2.0710000000000002,"15min":1.633},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":0.77800000000000002,"5min":19.434999999999999,"15min":19.434999999999999},"last":0.67300000000000004},{"interface":"front","average":{"1min":0.57699999999999996,"5min":1.671,"15min":1.643},"min":{"1min":0.41299999999999998,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":0.86899999999999999,"5min":17.135999999999999,"15min":17.135999999999999},"last":0.56100000000000005}]},{"osd":6,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.367,"5min":1.944,"15min":1.9570000000000001},"min":{"1min":0.45700000000000002,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":4.4489999999999998,"5min":17.192,"15min":115.54000000000001},"last":0.57999999999999996},{"interface":"front","average":{"1min":1.268,"5min":1.7390000000000001,"15min":2.0030000000000001},"min":{"1min":0.42599999999999999,"5min":0.35699999999999998,"15min":0.30499999999999999},"max":{"1min":4.3840000000000003,"5min":11.442,"15min":115.691},"last":0.79000000000000004}]},{"osd":7,"last update":"Sat Sep 23 18:21:52 2023","interfaces":[{"interface":"back","average":{"1min":1.8120000000000001,"5min":1.7729999999999999,"15min":1.8160000000000001},"min":{"1min":0.39500000000000002,"5min":0.38700000000000001,"15min":0.35799999999999998},"max":{"1min":7.7679999999999998,"5min":9.1929999999999996,"15min":15.972},"last":0.74299999999999999},{"interface":"front","average":{"1min":1.589,"5min":1.835,"15min":1.6599999999999999},"min":{"1min":0.49299999999999999,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":4.2279999999999998,"5min":17.164000000000001,"15min":17.164000000000001},"last":0.621}]}]},{"osd":4,"up_from":30,"seq":128849019349,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":865028,"kb_used_data":159676,"kb_used_omap":0,"kb_used_meta":705344,"kb_avail":92880124,"statfs":{"total":95995035648,"available":95109246976,"internally_reserved":0,"allocated":163508224,"data_stored":161498513,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":722272256},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":3.8959999999999999,"5min":2.242,"15min":1.5209999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":72.754999999999995,"5min":72.754999999999995,"15min":72.754999999999995},"last":0.32700000000000001},{"interface":"front","average":{"1min":3.9460000000000002,"5min":2.3500000000000001,"15min":1.7789999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":71.533000000000001,"5min":71.533000000000001,"15min":71.533000000000001},"last":0.497}]},{"osd":1,"last update":"Sat Sep 23 18:22:24 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":2.052,"15min":1.147},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.1399999999999997,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.58299999999999996},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.9740000000000002,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.42799999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.655,"15min":1.77},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":4.0030000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.6399999999999999,"15min":1.702},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.253},"max":{"1min":3.3839999999999999,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.35599999999999998}]},{"osd":3,"last update":"Sat Sep 23 18:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":2.6419999999999999,"15min":2.169},"min":{"1min":0.40200000000000002,"5min":0.32000000000000001,"15min":0.23799999999999999},"max":{"1min":0.80300000000000005,"5min":74.781999999999996,"15min":74.781999999999996},"last":0.39300000000000002},{"interface":"front","average":{"1min":0.58899999999999997,"5min":2.5670000000000002,"15min":2.1309999999999998},"min":{"1min":0.38600000000000001,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":0.78900000000000003,"5min":72.152000000000001,"15min":72.152000000000001},"last":0.55900000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":2.3460000000000001,"15min":2.7869999999999999},"min":{"1min":0.29199999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.73899999999999999,"5min":91.432000000000002,"15min":91.432000000000002},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.50700000000000001,"5min":2.121,"15min":2.5640000000000001},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":0.70199999999999996,"5min":91.626999999999995,"15min":91.626999999999995},"last":0.622}]},{"osd":6,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":5.141,"5min":2.4329999999999998,"15min":1.7090000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":91.688999999999993,"5min":91.688999999999993,"15min":91.688999999999993},"last":0.54400000000000004},{"interface":"front","average":{"1min":5.1420000000000003,"5min":2.6120000000000001,"15min":1.9159999999999999},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":91.837000000000003,"5min":91.837000000000003,"15min":91.837000000000003},"last":0.51800000000000002}]},{"osd":7,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":1.798,"15min":1.835},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":3.6800000000000002,"5min":18.283000000000001,"15min":19.298999999999999},"last":0.64500000000000002},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.4470000000000001,"15min":1.5529999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.28399999999999997},"max":{"1min":4.4269999999999996,"5min":9.4730000000000008,"15min":12.800000000000001},"last":0.67100000000000004}]}]},{"osd":5,"up_from":420,"seq":1803886264445,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":467128,"kb_used_data":94128,"kb_used_omap":9,"kb_used_meta":372982,"kb_avail":93278024,"statfs":{"total":95995035648,"available":95516696576,"internally_reserved":0,"allocated":96387072,"data_stored":94385633,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":381934013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":4.7930000000000001,"5min":1.889,"15min":1.2090000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":77.162000000000006,"5min":77.162000000000006,"15min":77.162000000000006},"last":0.54800000000000004},{"interface":"front","average":{"1min":4.9450000000000003,"5min":2.4169999999999998,"15min":1.8720000000000001},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":75.784999999999997,"5min":75.784999999999997,"15min":75.784999999999997},"last":0.443}]},{"osd":1,"last update":"Sat Sep 23 18:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.998,"15min":1.3959999999999999},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.4819999999999993,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.60899999999999999},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.133,"15min":1.2749999999999999},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.2759999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":0.52100000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.615,"15min":1.611},"min":{"1min":0.31,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":8.8789999999999996,"5min":10.705,"15min":10.705},"last":0.47999999999999998},{"interface":"front","average":{"1min":1.302,"5min":1.673,"15min":2.0640000000000001},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.3390000000000004,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.56699999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":5.4400000000000004,"5min":2.5369999999999999,"15min":1.6579999999999999},"min":{"1min":0.317,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":79.016999999999996,"5min":79.016999999999996,"15min":79.016999999999996},"last":0.58399999999999996},{"interface":"front","average":{"1min":5.2060000000000004,"5min":2.7810000000000001,"15min":1.9590000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.313},"max":{"1min":76.188999999999993,"5min":76.188999999999993,"15min":76.188999999999993},"last":0.62}]},{"osd":4,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.718,"15min":2.016},"min":{"1min":0.29199999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.9930000000000003,"5min":8.2270000000000003,"15min":12.747999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.7689999999999999,"15min":1.647},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":8.5559999999999992,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.66100000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.3340000000000001,"15min":1.2929999999999999},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.7859999999999996,"5min":6.7859999999999996,"15min":9.0350000000000001},"last":0.63500000000000001},{"interface":"front","average":{"1min":1.119,"5min":1.859,"15min":1.716},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.0890000000000004,"5min":12.504,"15min":12.504},"last":0.59799999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.704,"15min":1.806},"min":{"1min":0.33700000000000002,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":6.9569999999999999,"5min":10.528,"15min":13.506},"last":0.53300000000000003},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.494,"15min":1.5860000000000001},"min":{"1min":0.39900000000000002,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":12.702,"5min":12.702,"15min":13.138},"last":0.68999999999999995}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:22:56.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:56 smithi055 ceph-mon[184174]: pgmap v1591: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 472 KiB/s, 2 objects/s recovering 2023-09-23T18:22:56.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:56 smithi055 ceph-mon[184174]: from='client.48041 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:22:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:56 smithi055 ceph-mon[179668]: pgmap v1591: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 472 KiB/s, 2 objects/s recovering 2023-09-23T18:22:56.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:56 smithi055 ceph-mon[179668]: from='client.48041 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:22:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:56 smithi099 ceph-mon[167076]: pgmap v1591: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 472 KiB/s, 2 objects/s recovering 2023-09-23T18:22:57.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:56 smithi099 ceph-mon[167076]: from='client.48041 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:22:58.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:58 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:58.430 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:22:58 smithi055 ceph-mon[179668]: pgmap v1592: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 409 KiB/s, 1 objects/s recovering 2023-09-23T18:22:58.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:58 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:58.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:22:58 smithi055 ceph-mon[184174]: pgmap v1592: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 409 KiB/s, 1 objects/s recovering 2023-09-23T18:22:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:58 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:22:58.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:22:58 smithi099 ceph-mon[167076]: pgmap v1592: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 621 GiB / 626 GiB avail; 409 KiB/s, 1 objects/s recovering 2023-09-23T18:22:58.730 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:22:59.354 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:00.668 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:23:00.670 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:23:00.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:23:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:23:00] "GET /metrics HTTP/1.1" 200 34237 "" "Prometheus/2.43.0" 2023-09-23T18:23:00.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:00 smithi055 ceph-mon[179668]: pgmap v1593: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 351 KiB/s, 1 objects/s recovering 2023-09-23T18:23:00.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:00 smithi055 ceph-mon[184174]: pgmap v1593: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 351 KiB/s, 1 objects/s recovering 2023-09-23T18:23:00.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:00 smithi099 ceph-mon[167076]: pgmap v1593: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail; 351 KiB/s, 1 objects/s recovering 2023-09-23T18:23:01.001 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1593,"stamp":"2023-09-23T18:22:59.238180+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":484,"num_bytes_recovered":21509963,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":315,"num_osds":7,"num_per_pool_osds":7,"num_per_pool_omap_osds":7,"kb":656216064,"kb_used":4856268,"kb_used_data":959828,"kb_used_omap":62,"kb_used_meta":3896321,"kb_avail":651359796,"statfs":{"total":671965249536,"available":666992431104,"internally_reserved":0,"allocated":982863872,"data_stored":964116699,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":64275,"internal_metadata":3989832941},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":20,"num_bytes_recovered":4194304,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"11.678257"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":974,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.571570+0000","last_change":"2023-09-23T18:22:47.571570+0000","last_active":"2023-09-23T18:22:47.571570+0000","last_peered":"2023-09-23T18:22:47.571570+0000","last_clean":"2023-09-23T18:22:47.571570+0000","last_became_active":"2023-09-23T18:22:47.571095+0000","last_became_peered":"2023-09-23T18:22:47.571095+0000","last_unstale":"2023-09-23T18:22:47.571570+0000","last_undegraded":"2023-09-23T18:22:47.571570+0000","last_fullsized":"2023-09-23T18:22:47.571570+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:34:47.718943+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,1],"acting":[5,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":332,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561067+0000","last_change":"2023-09-23T18:22:44.528866+0000","last_active":"2023-09-23T18:22:46.561067+0000","last_peered":"2023-09-23T18:22:46.561067+0000","last_clean":"2023-09-23T18:22:46.561067+0000","last_became_active":"2023-09-23T18:22:44.528499+0000","last_became_peered":"2023-09-23T18:22:44.528499+0000","last_unstale":"2023-09-23T18:22:46.561067+0000","last_undegraded":"2023-09-23T18:22:46.561067+0000","last_fullsized":"2023-09-23T18:22:46.561067+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:26.288744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":80,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558455+0000","last_change":"2023-09-23T18:18:46.439723+0000","last_active":"2023-09-23T18:22:46.558455+0000","last_peered":"2023-09-23T18:22:46.558455+0000","last_clean":"2023-09-23T18:22:46.558455+0000","last_became_active":"2023-09-23T18:18:46.439036+0000","last_became_peered":"2023-09-23T18:18:46.439036+0000","last_unstale":"2023-09-23T18:22:46.558455+0000","last_undegraded":"2023-09-23T18:22:46.558455+0000","last_fullsized":"2023-09-23T18:22:46.558455+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:53:53.714267+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":956,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556429+0000","last_change":"2023-09-23T18:18:36.393733+0000","last_active":"2023-09-23T18:22:46.556429+0000","last_peered":"2023-09-23T18:22:46.556429+0000","last_clean":"2023-09-23T18:22:46.556429+0000","last_became_active":"2023-09-23T18:18:36.393312+0000","last_became_peered":"2023-09-23T18:18:36.393312+0000","last_unstale":"2023-09-23T18:22:46.556429+0000","last_undegraded":"2023-09-23T18:22:46.556429+0000","last_fullsized":"2023-09-23T18:22:46.556429+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:21:12.601470+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":405,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.566041+0000","last_change":"2023-09-23T18:17:20.747533+0000","last_active":"2023-09-23T18:22:47.566041+0000","last_peered":"2023-09-23T18:22:47.566041+0000","last_clean":"2023-09-23T18:22:47.566041+0000","last_became_active":"2023-09-23T18:17:20.747198+0000","last_became_peered":"2023-09-23T18:17:20.747198+0000","last_unstale":"2023-09-23T18:22:47.566041+0000","last_undegraded":"2023-09-23T18:22:47.566041+0000","last_fullsized":"2023-09-23T18:22:47.566041+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:49.156322+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":725,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.567554+0000","last_change":"2023-09-23T18:18:46.432020+0000","last_active":"2023-09-23T18:22:47.567554+0000","last_peered":"2023-09-23T18:22:47.567554+0000","last_clean":"2023-09-23T18:22:47.567554+0000","last_became_active":"2023-09-23T18:18:46.431635+0000","last_became_peered":"2023-09-23T18:18:46.431635+0000","last_unstale":"2023-09-23T18:22:47.567554+0000","last_undegraded":"2023-09-23T18:22:47.567554+0000","last_fullsized":"2023-09-23T18:22:47.567554+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:46:22.269111+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":973,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556166+0000","last_change":"2023-09-23T18:17:52.384070+0000","last_active":"2023-09-23T18:22:46.556166+0000","last_peered":"2023-09-23T18:22:46.556166+0000","last_clean":"2023-09-23T18:22:46.556166+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:22:46.556166+0000","last_undegraded":"2023-09-23T18:22:46.556166+0000","last_fullsized":"2023-09-23T18:22:46.556166+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:11:52.443252+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":80,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557446+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:22:46.557446+0000","last_peered":"2023-09-23T18:22:46.557446+0000","last_clean":"2023-09-23T18:22:46.557446+0000","last_became_active":"2023-09-23T18:18:46.426308+0000","last_became_peered":"2023-09-23T18:18:46.426308+0000","last_unstale":"2023-09-23T18:22:46.557446+0000","last_undegraded":"2023-09-23T18:22:46.557446+0000","last_fullsized":"2023-09-23T18:22:46.557446+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:47.384608+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":238,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558352+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:22:46.558352+0000","last_peered":"2023-09-23T18:22:46.558352+0000","last_clean":"2023-09-23T18:22:46.558352+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:22:46.558352+0000","last_undegraded":"2023-09-23T18:22:46.558352+0000","last_fullsized":"2023-09-23T18:22:46.558352+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":1096,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557657+0000","last_change":"2023-09-23T18:17:21.786502+0000","last_active":"2023-09-23T18:22:46.557657+0000","last_peered":"2023-09-23T18:22:46.557657+0000","last_clean":"2023-09-23T18:22:46.557657+0000","last_became_active":"2023-09-23T18:17:21.784737+0000","last_became_peered":"2023-09-23T18:17:21.784737+0000","last_unstale":"2023-09-23T18:22:46.557657+0000","last_undegraded":"2023-09-23T18:22:46.557657+0000","last_fullsized":"2023-09-23T18:22:46.557657+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:11:44.380163+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":172,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557196+0000","last_change":"2023-09-23T18:18:46.440420+0000","last_active":"2023-09-23T18:22:46.557196+0000","last_peered":"2023-09-23T18:22:46.557196+0000","last_clean":"2023-09-23T18:22:46.557196+0000","last_became_active":"2023-09-23T18:18:46.439979+0000","last_became_peered":"2023-09-23T18:18:46.439979+0000","last_unstale":"2023-09-23T18:22:46.557196+0000","last_undegraded":"2023-09-23T18:22:46.557196+0000","last_fullsized":"2023-09-23T18:22:46.557196+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:36:40.195075+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":253,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558136+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:22:46.558136+0000","last_peered":"2023-09-23T18:22:46.558136+0000","last_clean":"2023-09-23T18:22:46.558136+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:22:46.558136+0000","last_undegraded":"2023-09-23T18:22:46.558136+0000","last_fullsized":"2023-09-23T18:22:46.558136+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":842,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.565289+0000","last_change":"2023-09-23T18:22:06.467889+0000","last_active":"2023-09-23T18:22:46.565289+0000","last_peered":"2023-09-23T18:22:46.565289+0000","last_clean":"2023-09-23T18:22:46.565289+0000","last_became_active":"2023-09-23T18:22:06.467521+0000","last_became_peered":"2023-09-23T18:22:06.467521+0000","last_unstale":"2023-09-23T18:22:46.565289+0000","last_undegraded":"2023-09-23T18:22:46.565289+0000","last_fullsized":"2023-09-23T18:22:46.565289+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:13:07.204206+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,1],"acting":[2,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":208,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557482+0000","last_change":"2023-09-23T18:18:46.429645+0000","last_active":"2023-09-23T18:22:46.557482+0000","last_peered":"2023-09-23T18:22:46.557482+0000","last_clean":"2023-09-23T18:22:46.557482+0000","last_became_active":"2023-09-23T18:18:46.429282+0000","last_became_peered":"2023-09-23T18:18:46.429282+0000","last_unstale":"2023-09-23T18:22:46.557482+0000","last_undegraded":"2023-09-23T18:22:46.557482+0000","last_fullsized":"2023-09-23T18:22:46.557482+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:57:32.787744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,3],"acting":[5,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":207,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557313+0000","last_change":"2023-09-23T18:16:53.307713+0000","last_active":"2023-09-23T18:22:46.557313+0000","last_peered":"2023-09-23T18:22:46.557313+0000","last_clean":"2023-09-23T18:22:46.557313+0000","last_became_active":"2023-09-23T18:16:53.306327+0000","last_became_peered":"2023-09-23T18:16:53.306327+0000","last_unstale":"2023-09-23T18:22:46.557313+0000","last_undegraded":"2023-09-23T18:22:46.557313+0000","last_fullsized":"2023-09-23T18:22:46.557313+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:56:37.561998+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":1007,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557294+0000","last_change":"2023-09-23T18:18:36.376678+0000","last_active":"2023-09-23T18:22:46.557294+0000","last_peered":"2023-09-23T18:22:46.557294+0000","last_clean":"2023-09-23T18:22:46.557294+0000","last_became_active":"2023-09-23T18:18:36.376244+0000","last_became_peered":"2023-09-23T18:18:36.376244+0000","last_unstale":"2023-09-23T18:22:46.557294+0000","last_undegraded":"2023-09-23T18:22:46.557294+0000","last_fullsized":"2023-09-23T18:22:46.557294+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:07:56.562064+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":458,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558372+0000","last_change":"2023-09-23T18:16:53.332923+0000","last_active":"2023-09-23T18:22:46.558372+0000","last_peered":"2023-09-23T18:22:46.558372+0000","last_clean":"2023-09-23T18:22:46.558372+0000","last_became_active":"2023-09-23T18:16:53.320703+0000","last_became_peered":"2023-09-23T18:16:53.320703+0000","last_unstale":"2023-09-23T18:22:46.558372+0000","last_undegraded":"2023-09-23T18:22:46.558372+0000","last_fullsized":"2023-09-23T18:22:46.558372+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:17:07.473085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":18,"num_bytes_recovered":810,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,0],"acting":[3,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":485,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557201+0000","last_change":"2023-09-23T18:22:03.468342+0000","last_active":"2023-09-23T18:22:46.557201+0000","last_peered":"2023-09-23T18:22:46.557201+0000","last_clean":"2023-09-23T18:22:46.557201+0000","last_became_active":"2023-09-23T18:22:03.467804+0000","last_became_peered":"2023-09-23T18:22:03.467804+0000","last_unstale":"2023-09-23T18:22:46.557201+0000","last_undegraded":"2023-09-23T18:22:46.557201+0000","last_fullsized":"2023-09-23T18:22:46.557201+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:51:17.762831+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":1038,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558902+0000","last_change":"2023-09-23T18:18:49.354483+0000","last_active":"2023-09-23T18:22:46.558902+0000","last_peered":"2023-09-23T18:22:46.558902+0000","last_clean":"2023-09-23T18:22:46.558902+0000","last_became_active":"2023-09-23T18:18:49.354123+0000","last_became_peered":"2023-09-23T18:18:49.354123+0000","last_unstale":"2023-09-23T18:22:46.558902+0000","last_undegraded":"2023-09-23T18:22:46.558902+0000","last_fullsized":"2023-09-23T18:22:46.558902+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:04:20.346211+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":234,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557520+0000","last_change":"2023-09-23T18:22:03.479912+0000","last_active":"2023-09-23T18:22:46.557520+0000","last_peered":"2023-09-23T18:22:46.557520+0000","last_clean":"2023-09-23T18:22:46.557520+0000","last_became_active":"2023-09-23T18:22:03.464545+0000","last_became_peered":"2023-09-23T18:22:03.464545+0000","last_unstale":"2023-09-23T18:22:46.557520+0000","last_undegraded":"2023-09-23T18:22:46.557520+0000","last_fullsized":"2023-09-23T18:22:46.557520+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:50.025694+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":9468,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":481,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557148+0000","last_change":"2023-09-23T18:17:52.387576+0000","last_active":"2023-09-23T18:22:46.557148+0000","last_peered":"2023-09-23T18:22:46.557148+0000","last_clean":"2023-09-23T18:22:46.557148+0000","last_became_active":"2023-09-23T18:17:52.386284+0000","last_became_peered":"2023-09-23T18:17:52.386284+0000","last_unstale":"2023-09-23T18:22:46.557148+0000","last_undegraded":"2023-09-23T18:22:46.557148+0000","last_fullsized":"2023-09-23T18:22:46.557148+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:43.225949+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":888,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556113+0000","last_change":"2023-09-23T18:18:49.363744+0000","last_active":"2023-09-23T18:22:46.556113+0000","last_peered":"2023-09-23T18:22:46.556113+0000","last_clean":"2023-09-23T18:22:46.556113+0000","last_became_active":"2023-09-23T18:18:49.363451+0000","last_became_peered":"2023-09-23T18:18:49.363451+0000","last_unstale":"2023-09-23T18:22:46.556113+0000","last_undegraded":"2023-09-23T18:22:46.556113+0000","last_fullsized":"2023-09-23T18:22:46.556113+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:53.934927+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,3],"acting":[5,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":497,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556905+0000","last_change":"2023-09-23T18:22:03.475293+0000","last_active":"2023-09-23T18:22:46.556905+0000","last_peered":"2023-09-23T18:22:46.556905+0000","last_clean":"2023-09-23T18:22:46.556905+0000","last_became_active":"2023-09-23T18:22:03.465603+0000","last_became_peered":"2023-09-23T18:22:03.465603+0000","last_unstale":"2023-09-23T18:22:46.556905+0000","last_undegraded":"2023-09-23T18:22:46.556905+0000","last_fullsized":"2023-09-23T18:22:46.556905+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:13:11.349439+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":108,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.564362+0000","last_change":"2023-09-23T18:22:03.464544+0000","last_active":"2023-09-23T18:22:46.564362+0000","last_peered":"2023-09-23T18:22:46.564362+0000","last_clean":"2023-09-23T18:22:46.564362+0000","last_became_active":"2023-09-23T18:22:03.464227+0000","last_became_peered":"2023-09-23T18:22:03.464227+0000","last_unstale":"2023-09-23T18:22:46.564362+0000","last_undegraded":"2023-09-23T18:22:46.564362+0000","last_fullsized":"2023-09-23T18:22:46.564362+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:56.368635+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":859,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.555904+0000","last_change":"2023-09-23T18:16:56.351909+0000","last_active":"2023-09-23T18:22:46.555904+0000","last_peered":"2023-09-23T18:22:46.555904+0000","last_clean":"2023-09-23T18:22:46.555904+0000","last_became_active":"2023-09-23T18:16:56.351534+0000","last_became_peered":"2023-09-23T18:16:56.351534+0000","last_unstale":"2023-09-23T18:22:46.555904+0000","last_undegraded":"2023-09-23T18:22:46.555904+0000","last_fullsized":"2023-09-23T18:22:46.555904+0000","mapping_epoch":530,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":531,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:41:10.142502+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":179,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.562704+0000","last_change":"2023-09-23T18:18:46.432100+0000","last_active":"2023-09-23T18:22:46.562704+0000","last_peered":"2023-09-23T18:22:46.562704+0000","last_clean":"2023-09-23T18:22:46.562704+0000","last_became_active":"2023-09-23T18:18:46.431807+0000","last_became_peered":"2023-09-23T18:18:46.431807+0000","last_unstale":"2023-09-23T18:22:46.562704+0000","last_undegraded":"2023-09-23T18:22:46.562704+0000","last_fullsized":"2023-09-23T18:22:46.562704+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:26:56.346445+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,1],"acting":[0,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":172,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556626+0000","last_change":"2023-09-23T18:16:53.302931+0000","last_active":"2023-09-23T18:22:46.556626+0000","last_peered":"2023-09-23T18:22:46.556626+0000","last_clean":"2023-09-23T18:22:46.556626+0000","last_became_active":"2023-09-23T18:16:53.302474+0000","last_became_peered":"2023-09-23T18:16:53.302474+0000","last_unstale":"2023-09-23T18:22:46.556626+0000","last_undegraded":"2023-09-23T18:22:46.556626+0000","last_fullsized":"2023-09-23T18:22:46.556626+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:39:00.867388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":799,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.560384+0000","last_change":"2023-09-23T18:18:49.372374+0000","last_active":"2023-09-23T18:22:46.560384+0000","last_peered":"2023-09-23T18:22:46.560384+0000","last_clean":"2023-09-23T18:22:46.560384+0000","last_became_active":"2023-09-23T18:18:49.371962+0000","last_became_peered":"2023-09-23T18:18:49.371962+0000","last_unstale":"2023-09-23T18:22:46.560384+0000","last_undegraded":"2023-09-23T18:22:46.560384+0000","last_fullsized":"2023-09-23T18:22:46.560384+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:26:40.974424+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":249,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557231+0000","last_change":"2023-09-23T18:17:52.389812+0000","last_active":"2023-09-23T18:22:46.557231+0000","last_peered":"2023-09-23T18:22:46.557231+0000","last_clean":"2023-09-23T18:22:46.557231+0000","last_became_active":"2023-09-23T18:17:52.386461+0000","last_became_peered":"2023-09-23T18:17:52.386461+0000","last_unstale":"2023-09-23T18:22:46.557231+0000","last_undegraded":"2023-09-23T18:22:46.557231+0000","last_fullsized":"2023-09-23T18:22:46.557231+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:59:09.907765+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":271,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558602+0000","last_change":"2023-09-23T18:22:03.464607+0000","last_active":"2023-09-23T18:22:46.558602+0000","last_peered":"2023-09-23T18:22:46.558602+0000","last_clean":"2023-09-23T18:22:46.558602+0000","last_became_active":"2023-09-23T18:22:03.464333+0000","last_became_peered":"2023-09-23T18:22:03.464333+0000","last_unstale":"2023-09-23T18:22:46.558602+0000","last_undegraded":"2023-09-23T18:22:46.558602+0000","last_fullsized":"2023-09-23T18:22:46.558602+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:49:51.062916+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,2],"acting":[3,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":893,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556529+0000","last_change":"2023-09-23T18:18:10.553331+0000","last_active":"2023-09-23T18:22:46.556529+0000","last_peered":"2023-09-23T18:22:46.556529+0000","last_clean":"2023-09-23T18:22:46.556529+0000","last_became_active":"2023-09-23T18:18:10.553009+0000","last_became_peered":"2023-09-23T18:18:10.553009+0000","last_unstale":"2023-09-23T18:22:46.556529+0000","last_undegraded":"2023-09-23T18:22:46.556529+0000","last_fullsized":"2023-09-23T18:22:46.556529+0000","mapping_epoch":582,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":583,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:44:11.502339+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,1],"acting":[6,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":802,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556519+0000","last_change":"2023-09-23T18:18:33.357683+0000","last_active":"2023-09-23T18:22:46.556519+0000","last_peered":"2023-09-23T18:22:46.556519+0000","last_clean":"2023-09-23T18:22:46.556519+0000","last_became_active":"2023-09-23T18:18:33.329434+0000","last_became_peered":"2023-09-23T18:18:33.329434+0000","last_unstale":"2023-09-23T18:22:46.556519+0000","last_undegraded":"2023-09-23T18:22:46.556519+0000","last_fullsized":"2023-09-23T18:22:46.556519+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:23.905796+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":24,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":89,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.560049+0000","last_change":"2023-09-23T18:22:03.468997+0000","last_active":"2023-09-23T18:22:46.560049+0000","last_peered":"2023-09-23T18:22:46.560049+0000","last_clean":"2023-09-23T18:22:46.560049+0000","last_became_active":"2023-09-23T18:22:03.468301+0000","last_became_peered":"2023-09-23T18:22:03.468301+0000","last_unstale":"2023-09-23T18:22:46.560049+0000","last_undegraded":"2023-09-23T18:22:46.560049+0000","last_fullsized":"2023-09-23T18:22:46.560049+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:32:17.742551+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,2],"acting":[3,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":919,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.574504+0000","last_change":"2023-09-23T18:22:47.574504+0000","last_active":"2023-09-23T18:22:47.574504+0000","last_peered":"2023-09-23T18:22:47.574504+0000","last_clean":"2023-09-23T18:22:47.574504+0000","last_became_active":"2023-09-23T18:22:47.574095+0000","last_became_peered":"2023-09-23T18:22:47.574095+0000","last_unstale":"2023-09-23T18:22:47.574504+0000","last_undegraded":"2023-09-23T18:22:47.574504+0000","last_fullsized":"2023-09-23T18:22:47.574504+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:42:13.813532+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":299,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.562539+0000","last_change":"2023-09-23T18:18:33.362111+0000","last_active":"2023-09-23T18:22:46.562539+0000","last_peered":"2023-09-23T18:22:46.562539+0000","last_clean":"2023-09-23T18:22:46.562539+0000","last_became_active":"2023-09-23T18:18:33.327843+0000","last_became_peered":"2023-09-23T18:18:33.327843+0000","last_unstale":"2023-09-23T18:22:46.562539+0000","last_undegraded":"2023-09-23T18:22:46.562539+0000","last_fullsized":"2023-09-23T18:22:46.562539+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:43:49.829876+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":66,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,1],"acting":[4,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":172,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.555882+0000","last_change":"2023-09-23T18:16:53.311139+0000","last_active":"2023-09-23T18:22:46.555882+0000","last_peered":"2023-09-23T18:22:46.555882+0000","last_clean":"2023-09-23T18:22:46.555882+0000","last_became_active":"2023-09-23T18:16:53.310854+0000","last_became_peered":"2023-09-23T18:16:53.310854+0000","last_unstale":"2023-09-23T18:22:46.555882+0000","last_undegraded":"2023-09-23T18:22:46.555882+0000","last_fullsized":"2023-09-23T18:22:46.555882+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:28.950795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,4],"acting":[5,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":795,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.559529+0000","last_change":"2023-09-23T18:21:51.851401+0000","last_active":"2023-09-23T18:22:46.559529+0000","last_peered":"2023-09-23T18:22:46.559529+0000","last_clean":"2023-09-23T18:22:46.559529+0000","last_became_active":"2023-09-23T18:19:34.407669+0000","last_became_peered":"2023-09-23T18:19:34.407669+0000","last_unstale":"2023-09-23T18:22:46.559529+0000","last_undegraded":"2023-09-23T18:22:46.559529+0000","last_fullsized":"2023-09-23T18:22:46.559529+0000","mapping_epoch":605,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":606,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:21:51.851269+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:21:51.851269+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:10:48.305008+0000","scrub_duration":0.001933539,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":300,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.562705+0000","last_change":"2023-09-23T18:18:33.347903+0000","last_active":"2023-09-23T18:22:46.562705+0000","last_peered":"2023-09-23T18:22:46.562705+0000","last_clean":"2023-09-23T18:22:46.562705+0000","last_became_active":"2023-09-23T18:18:33.327603+0000","last_became_peered":"2023-09-23T18:18:33.327603+0000","last_unstale":"2023-09-23T18:22:46.562705+0000","last_undegraded":"2023-09-23T18:22:46.562705+0000","last_fullsized":"2023-09-23T18:22:46.562705+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:49.160244+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,0],"acting":[4,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":279,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561306+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:22:46.561306+0000","last_peered":"2023-09-23T18:22:46.561306+0000","last_clean":"2023-09-23T18:22:46.561306+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:22:46.561306+0000","last_undegraded":"2023-09-23T18:22:46.561306+0000","last_fullsized":"2023-09-23T18:22:46.561306+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":677,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558263+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:22:46.558263+0000","last_peered":"2023-09-23T18:22:46.558263+0000","last_clean":"2023-09-23T18:22:46.558263+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:22:46.558263+0000","last_undegraded":"2023-09-23T18:22:46.558263+0000","last_fullsized":"2023-09-23T18:22:46.558263+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":587,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556446+0000","last_change":"2023-09-23T18:16:53.322112+0000","last_active":"2023-09-23T18:22:46.556446+0000","last_peered":"2023-09-23T18:22:46.556446+0000","last_clean":"2023-09-23T18:22:46.556446+0000","last_became_active":"2023-09-23T18:16:53.302104+0000","last_became_peered":"2023-09-23T18:16:53.302104+0000","last_unstale":"2023-09-23T18:22:46.556446+0000","last_undegraded":"2023-09-23T18:22:46.556446+0000","last_fullsized":"2023-09-23T18:22:46.556446+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:45.844123+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,6,3],"acting":[5,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":238,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556898+0000","last_change":"2023-09-23T18:18:07.536531+0000","last_active":"2023-09-23T18:22:46.556898+0000","last_peered":"2023-09-23T18:22:46.556898+0000","last_clean":"2023-09-23T18:22:46.556898+0000","last_became_active":"2023-09-23T18:18:07.536277+0000","last_became_peered":"2023-09-23T18:18:07.536277+0000","last_unstale":"2023-09-23T18:22:46.556898+0000","last_undegraded":"2023-09-23T18:22:46.556898+0000","last_fullsized":"2023-09-23T18:22:46.556898+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:58.902374+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,5],"acting":[6,4,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":712,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.573659+0000","last_change":"2023-09-23T18:22:47.573659+0000","last_active":"2023-09-23T18:22:47.573659+0000","last_peered":"2023-09-23T18:22:47.573659+0000","last_clean":"2023-09-23T18:22:47.573659+0000","last_became_active":"2023-09-23T18:22:47.573173+0000","last_became_peered":"2023-09-23T18:22:47.573173+0000","last_unstale":"2023-09-23T18:22:47.573659+0000","last_undegraded":"2023-09-23T18:22:47.573659+0000","last_fullsized":"2023-09-23T18:22:47.573659+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:42:15.423306+0000","scrub_duration":0.002063086,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,4,2],"acting":[3,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":186,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556793+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:22:46.556793+0000","last_peered":"2023-09-23T18:22:46.556793+0000","last_clean":"2023-09-23T18:22:46.556793+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:22:46.556793+0000","last_undegraded":"2023-09-23T18:22:46.556793+0000","last_fullsized":"2023-09-23T18:22:46.556793+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":501,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.563334+0000","last_change":"2023-09-23T18:17:52.383960+0000","last_active":"2023-09-23T18:22:46.563334+0000","last_peered":"2023-09-23T18:22:46.563334+0000","last_clean":"2023-09-23T18:22:46.563334+0000","last_became_active":"2023-09-23T18:17:52.383642+0000","last_became_peered":"2023-09-23T18:17:52.383642+0000","last_unstale":"2023-09-23T18:22:46.563334+0000","last_undegraded":"2023-09-23T18:22:46.563334+0000","last_fullsized":"2023-09-23T18:22:46.563334+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:00:22.510495+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1164,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.571678+0000","last_change":"2023-09-23T18:22:47.571678+0000","last_active":"2023-09-23T18:22:47.571678+0000","last_peered":"2023-09-23T18:22:47.571678+0000","last_clean":"2023-09-23T18:22:47.571678+0000","last_became_active":"2023-09-23T18:22:47.571151+0000","last_became_peered":"2023-09-23T18:22:47.571151+0000","last_unstale":"2023-09-23T18:22:47.571678+0000","last_undegraded":"2023-09-23T18:22:47.571678+0000","last_fullsized":"2023-09-23T18:22:47.571678+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:06:46.934447+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":401,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561103+0000","last_change":"2023-09-23T18:18:33.349638+0000","last_active":"2023-09-23T18:22:46.561103+0000","last_peered":"2023-09-23T18:22:46.561103+0000","last_clean":"2023-09-23T18:22:46.561103+0000","last_became_active":"2023-09-23T18:18:33.327176+0000","last_became_peered":"2023-09-23T18:18:33.327176+0000","last_unstale":"2023-09-23T18:22:46.561103+0000","last_undegraded":"2023-09-23T18:22:46.561103+0000","last_fullsized":"2023-09-23T18:22:46.561103+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:37:39.504388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":206,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558902+0000","last_change":"2023-09-23T18:17:52.390775+0000","last_active":"2023-09-23T18:22:46.558902+0000","last_peered":"2023-09-23T18:22:46.558902+0000","last_clean":"2023-09-23T18:22:46.558902+0000","last_became_active":"2023-09-23T18:17:52.390407+0000","last_became_peered":"2023-09-23T18:17:52.390407+0000","last_unstale":"2023-09-23T18:22:46.558902+0000","last_undegraded":"2023-09-23T18:22:46.558902+0000","last_fullsized":"2023-09-23T18:22:46.558902+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:42:27.708548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":951,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556042+0000","last_change":"2023-09-23T18:22:06.466040+0000","last_active":"2023-09-23T18:22:46.556042+0000","last_peered":"2023-09-23T18:22:46.556042+0000","last_clean":"2023-09-23T18:22:46.556042+0000","last_became_active":"2023-09-23T18:22:06.465158+0000","last_became_peered":"2023-09-23T18:22:06.465158+0000","last_unstale":"2023-09-23T18:22:46.556042+0000","last_undegraded":"2023-09-23T18:22:46.556042+0000","last_fullsized":"2023-09-23T18:22:46.556042+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:29.587874+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":214,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556809+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:22:46.556809+0000","last_peered":"2023-09-23T18:22:46.556809+0000","last_clean":"2023-09-23T18:22:46.556809+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:22:46.556809+0000","last_undegraded":"2023-09-23T18:22:46.556809+0000","last_fullsized":"2023-09-23T18:22:46.556809+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":457,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.562477+0000","last_change":"2023-09-23T18:22:44.527193+0000","last_active":"2023-09-23T18:22:46.562477+0000","last_peered":"2023-09-23T18:22:46.562477+0000","last_clean":"2023-09-23T18:22:46.562477+0000","last_became_active":"2023-09-23T18:22:44.526903+0000","last_became_peered":"2023-09-23T18:22:44.526903+0000","last_unstale":"2023-09-23T18:22:46.562477+0000","last_undegraded":"2023-09-23T18:22:46.562477+0000","last_fullsized":"2023-09-23T18:22:46.562477+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:09.126029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":933,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556525+0000","last_change":"2023-09-23T18:17:21.788696+0000","last_active":"2023-09-23T18:22:46.556525+0000","last_peered":"2023-09-23T18:22:46.556525+0000","last_clean":"2023-09-23T18:22:46.556525+0000","last_became_active":"2023-09-23T18:17:21.786196+0000","last_became_peered":"2023-09-23T18:17:21.786196+0000","last_unstale":"2023-09-23T18:22:46.556525+0000","last_undegraded":"2023-09-23T18:22:46.556525+0000","last_fullsized":"2023-09-23T18:22:46.556525+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:05:54.107827+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":467,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561149+0000","last_change":"2023-09-23T18:22:03.465235+0000","last_active":"2023-09-23T18:22:46.561149+0000","last_peered":"2023-09-23T18:22:46.561149+0000","last_clean":"2023-09-23T18:22:46.561149+0000","last_became_active":"2023-09-23T18:22:03.464906+0000","last_became_peered":"2023-09-23T18:22:03.464906+0000","last_unstale":"2023-09-23T18:22:46.561149+0000","last_undegraded":"2023-09-23T18:22:46.561149+0000","last_fullsized":"2023-09-23T18:22:46.561149+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:06:46.429598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":274,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556564+0000","last_change":"2023-09-23T18:17:52.389993+0000","last_active":"2023-09-23T18:22:46.556564+0000","last_peered":"2023-09-23T18:22:46.556564+0000","last_clean":"2023-09-23T18:22:46.556564+0000","last_became_active":"2023-09-23T18:17:52.389663+0000","last_became_peered":"2023-09-23T18:17:52.389663+0000","last_unstale":"2023-09-23T18:22:46.556564+0000","last_undegraded":"2023-09-23T18:22:46.556564+0000","last_fullsized":"2023-09-23T18:22:46.556564+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:45:30.747862+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":934,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557370+0000","last_change":"2023-09-23T18:22:06.468619+0000","last_active":"2023-09-23T18:22:46.557370+0000","last_peered":"2023-09-23T18:22:46.557370+0000","last_clean":"2023-09-23T18:22:46.557370+0000","last_became_active":"2023-09-23T18:22:06.468197+0000","last_became_peered":"2023-09-23T18:22:06.468197+0000","last_unstale":"2023-09-23T18:22:46.557370+0000","last_undegraded":"2023-09-23T18:22:46.557370+0000","last_fullsized":"2023-09-23T18:22:46.557370+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:10.149437+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":80,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557792+0000","last_change":"2023-09-23T18:18:46.439205+0000","last_active":"2023-09-23T18:22:46.557792+0000","last_peered":"2023-09-23T18:22:46.557792+0000","last_clean":"2023-09-23T18:22:46.557792+0000","last_became_active":"2023-09-23T18:18:46.438920+0000","last_became_peered":"2023-09-23T18:18:46.438920+0000","last_unstale":"2023-09-23T18:22:46.557792+0000","last_undegraded":"2023-09-23T18:22:46.557792+0000","last_fullsized":"2023-09-23T18:22:46.557792+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:31.312904+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":392,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558148+0000","last_change":"2023-09-23T18:17:52.384951+0000","last_active":"2023-09-23T18:22:46.558148+0000","last_peered":"2023-09-23T18:22:46.558148+0000","last_clean":"2023-09-23T18:22:46.558148+0000","last_became_active":"2023-09-23T18:17:52.384539+0000","last_became_peered":"2023-09-23T18:17:52.384539+0000","last_unstale":"2023-09-23T18:22:46.558148+0000","last_undegraded":"2023-09-23T18:22:46.558148+0000","last_fullsized":"2023-09-23T18:22:46.558148+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:00:10.074284+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":773,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558449+0000","last_change":"2023-09-23T18:18:49.359762+0000","last_active":"2023-09-23T18:22:46.558449+0000","last_peered":"2023-09-23T18:22:46.558449+0000","last_clean":"2023-09-23T18:22:46.558449+0000","last_became_active":"2023-09-23T18:18:49.359418+0000","last_became_peered":"2023-09-23T18:18:49.359418+0000","last_unstale":"2023-09-23T18:22:46.558449+0000","last_undegraded":"2023-09-23T18:22:46.558449+0000","last_fullsized":"2023-09-23T18:22:46.558449+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:09:38.350740+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,5],"acting":[6,3,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"653'291","reported_seq":1547,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557793+0000","last_change":"2023-09-23T18:22:28.465602+0000","last_active":"2023-09-23T18:22:46.557793+0000","last_peered":"2023-09-23T18:22:46.557793+0000","last_clean":"2023-09-23T18:22:46.557793+0000","last_became_active":"2023-09-23T18:22:06.466146+0000","last_became_peered":"2023-09-23T18:22:06.466146+0000","last_unstale":"2023-09-23T18:22:46.557793+0000","last_undegraded":"2023-09-23T18:22:46.557793+0000","last_fullsized":"2023-09-23T18:22:46.557793+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":291,"log_dups_size":0,"ondisk_log_size":291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:57:22.022031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047292100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1326,"num_read_kb":5772,"num_write":300,"num_write_kb":4517,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":78,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,2],"acting":[1,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":206,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556897+0000","last_change":"2023-09-23T18:22:03.462589+0000","last_active":"2023-09-23T18:22:46.556897+0000","last_peered":"2023-09-23T18:22:46.556897+0000","last_clean":"2023-09-23T18:22:46.556897+0000","last_became_active":"2023-09-23T18:22:03.462239+0000","last_became_peered":"2023-09-23T18:22:03.462239+0000","last_unstale":"2023-09-23T18:22:46.556897+0000","last_undegraded":"2023-09-23T18:22:46.556897+0000","last_fullsized":"2023-09-23T18:22:46.556897+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.020770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":176,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557276+0000","last_change":"2023-09-23T18:17:21.788579+0000","last_active":"2023-09-23T18:22:46.557276+0000","last_peered":"2023-09-23T18:22:46.557276+0000","last_clean":"2023-09-23T18:22:46.557276+0000","last_became_active":"2023-09-23T18:17:21.783987+0000","last_became_peered":"2023-09-23T18:17:21.783987+0000","last_unstale":"2023-09-23T18:22:46.557276+0000","last_undegraded":"2023-09-23T18:22:46.557276+0000","last_fullsized":"2023-09-23T18:22:46.557276+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:04:10.781273+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,4],"acting":[5,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":865,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556229+0000","last_change":"2023-09-23T18:22:06.465690+0000","last_active":"2023-09-23T18:22:46.556229+0000","last_peered":"2023-09-23T18:22:46.556229+0000","last_clean":"2023-09-23T18:22:46.556229+0000","last_became_active":"2023-09-23T18:22:06.465148+0000","last_became_peered":"2023-09-23T18:22:06.465148+0000","last_unstale":"2023-09-23T18:22:46.556229+0000","last_undegraded":"2023-09-23T18:22:46.556229+0000","last_fullsized":"2023-09-23T18:22:46.556229+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:54:00.043030+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"653'323","reported_seq":2262,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557184+0000","last_change":"2023-09-23T18:22:28.473090+0000","last_active":"2023-09-23T18:22:46.557184+0000","last_peered":"2023-09-23T18:22:46.557184+0000","last_clean":"2023-09-23T18:22:46.557184+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:22:46.557184+0000","last_undegraded":"2023-09-23T18:22:46.557184+0000","last_fullsized":"2023-09-23T18:22:46.557184+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":323,"log_dups_size":0,"ondisk_log_size":323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001426075,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3093,"num_read_kb":5799,"num_write":265,"num_write_kb":3199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":186,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557004+0000","last_change":"2023-09-23T18:17:52.387757+0000","last_active":"2023-09-23T18:22:46.557004+0000","last_peered":"2023-09-23T18:22:46.557004+0000","last_clean":"2023-09-23T18:22:46.557004+0000","last_became_active":"2023-09-23T18:17:52.386331+0000","last_became_peered":"2023-09-23T18:17:52.386331+0000","last_unstale":"2023-09-23T18:22:46.557004+0000","last_undegraded":"2023-09-23T18:22:46.557004+0000","last_fullsized":"2023-09-23T18:22:46.557004+0000","mapping_epoch":571,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":572,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:49:31.737575+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":425,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557567+0000","last_change":"2023-09-23T18:22:44.519917+0000","last_active":"2023-09-23T18:22:46.557567+0000","last_peered":"2023-09-23T18:22:46.557567+0000","last_clean":"2023-09-23T18:22:46.557567+0000","last_became_active":"2023-09-23T18:22:44.519386+0000","last_became_peered":"2023-09-23T18:22:44.519386+0000","last_unstale":"2023-09-23T18:22:46.557567+0000","last_undegraded":"2023-09-23T18:22:46.557567+0000","last_fullsized":"2023-09-23T18:22:46.557567+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:45:44.617263+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":705,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.573442+0000","last_change":"2023-09-23T18:22:47.573442+0000","last_active":"2023-09-23T18:22:47.573442+0000","last_peered":"2023-09-23T18:22:47.573442+0000","last_clean":"2023-09-23T18:22:47.573442+0000","last_became_active":"2023-09-23T18:22:47.573059+0000","last_became_peered":"2023-09-23T18:22:47.573059+0000","last_unstale":"2023-09-23T18:22:47.573442+0000","last_undegraded":"2023-09-23T18:22:47.573442+0000","last_fullsized":"2023-09-23T18:22:47.573442+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:20:43.254558+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,2],"acting":[0,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.1","version":"653'200","reported_seq":488,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.570312+0000","last_change":"2023-09-23T18:22:47.569585+0000","last_active":"2023-09-23T18:22:47.570312+0000","last_peered":"2023-09-23T18:22:47.570312+0000","last_clean":"2023-09-23T18:22:47.570312+0000","last_became_active":"2023-09-23T18:22:47.568716+0000","last_became_peered":"2023-09-23T18:22:47.568716+0000","last_unstale":"2023-09-23T18:22:47.570312+0000","last_undegraded":"2023-09-23T18:22:47.570312+0000","last_fullsized":"2023-09-23T18:22:47.570312+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:58:16.049107+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0022753629999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1032,"num_read_kb":1283,"num_write":194,"num_write_kb":5509,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":68,"num_bytes_recovered":20971520,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":465,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557119+0000","last_change":"2023-09-23T18:22:44.518872+0000","last_active":"2023-09-23T18:22:46.557119+0000","last_peered":"2023-09-23T18:22:46.557119+0000","last_clean":"2023-09-23T18:22:46.557119+0000","last_became_active":"2023-09-23T18:22:44.518360+0000","last_became_peered":"2023-09-23T18:22:44.518360+0000","last_unstale":"2023-09-23T18:22:46.557119+0000","last_undegraded":"2023-09-23T18:22:46.557119+0000","last_fullsized":"2023-09-23T18:22:46.557119+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:43.986310+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,2],"acting":[6,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":198,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557102+0000","last_change":"2023-09-23T18:18:33.332775+0000","last_active":"2023-09-23T18:22:46.557102+0000","last_peered":"2023-09-23T18:22:46.557102+0000","last_clean":"2023-09-23T18:22:46.557102+0000","last_became_active":"2023-09-23T18:18:33.331184+0000","last_became_peered":"2023-09-23T18:18:33.331184+0000","last_unstale":"2023-09-23T18:22:46.557102+0000","last_undegraded":"2023-09-23T18:22:46.557102+0000","last_fullsized":"2023-09-23T18:22:46.557102+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:40:22.161054+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.3","version":"653'474","reported_seq":2274,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.573054+0000","last_change":"2023-09-23T18:22:47.573054+0000","last_active":"2023-09-23T18:22:47.573054+0000","last_peered":"2023-09-23T18:22:47.573054+0000","last_clean":"2023-09-23T18:22:47.573054+0000","last_became_active":"2023-09-23T18:22:47.572246+0000","last_became_peered":"2023-09-23T18:22:47.572246+0000","last_unstale":"2023-09-23T18:22:47.573054+0000","last_undegraded":"2023-09-23T18:22:47.573054+0000","last_fullsized":"2023-09-23T18:22:47.573054+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":474,"log_dups_size":0,"ondisk_log_size":474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:23:41.513401+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014671619999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2565,"num_read_kb":5830,"num_write":410,"num_write_kb":4052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":612,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556798+0000","last_change":"2023-09-23T18:18:36.363214+0000","last_active":"2023-09-23T18:22:46.556798+0000","last_peered":"2023-09-23T18:22:46.556798+0000","last_clean":"2023-09-23T18:22:46.556798+0000","last_became_active":"2023-09-23T18:18:36.362873+0000","last_became_peered":"2023-09-23T18:18:36.362873+0000","last_unstale":"2023-09-23T18:22:46.556798+0000","last_undegraded":"2023-09-23T18:22:46.556798+0000","last_fullsized":"2023-09-23T18:22:46.556798+0000","mapping_epoch":591,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":592,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:16:39.048805+0000","scrub_duration":0.0038807780000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,1],"acting":[6,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":172,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556370+0000","last_change":"2023-09-23T18:18:46.440242+0000","last_active":"2023-09-23T18:22:46.556370+0000","last_peered":"2023-09-23T18:22:46.556370+0000","last_clean":"2023-09-23T18:22:46.556370+0000","last_became_active":"2023-09-23T18:18:46.439799+0000","last_became_peered":"2023-09-23T18:18:46.439799+0000","last_unstale":"2023-09-23T18:22:46.556370+0000","last_undegraded":"2023-09-23T18:22:46.556370+0000","last_fullsized":"2023-09-23T18:22:46.556370+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:06:55.861056+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":207,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558681+0000","last_change":"2023-09-23T18:16:53.317182+0000","last_active":"2023-09-23T18:22:46.558681+0000","last_peered":"2023-09-23T18:22:46.558681+0000","last_clean":"2023-09-23T18:22:46.558681+0000","last_became_active":"2023-09-23T18:16:53.316026+0000","last_became_peered":"2023-09-23T18:16:53.316026+0000","last_unstale":"2023-09-23T18:22:46.558681+0000","last_undegraded":"2023-09-23T18:22:46.558681+0000","last_fullsized":"2023-09-23T18:22:46.558681+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:17:27.938651+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,0],"acting":[6,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":860,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557250+0000","last_change":"2023-09-23T18:17:20.907328+0000","last_active":"2023-09-23T18:22:46.557250+0000","last_peered":"2023-09-23T18:22:46.557250+0000","last_clean":"2023-09-23T18:22:46.557250+0000","last_became_active":"2023-09-23T18:17:20.907014+0000","last_became_peered":"2023-09-23T18:17:20.907014+0000","last_unstale":"2023-09-23T18:22:46.557250+0000","last_undegraded":"2023-09-23T18:22:46.557250+0000","last_fullsized":"2023-09-23T18:22:46.557250+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:32:35.829911+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.0","version":"653'156","reported_seq":456,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.565453+0000","last_change":"2023-09-23T18:22:28.481047+0000","last_active":"2023-09-23T18:22:47.565453+0000","last_peered":"2023-09-23T18:22:47.565453+0000","last_clean":"2023-09-23T18:22:47.565453+0000","last_became_active":"2023-09-23T18:17:21.759503+0000","last_became_peered":"2023-09-23T18:17:21.759503+0000","last_unstale":"2023-09-23T18:22:47.565453+0000","last_undegraded":"2023-09-23T18:22:47.565453+0000","last_fullsized":"2023-09-23T18:22:47.565453+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":156,"log_dups_size":0,"ondisk_log_size":156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:12:18.473739+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504954,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":805,"num_read_kb":2049,"num_write":144,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,0],"acting":[3,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":81,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.565925+0000","last_change":"2023-09-23T18:18:46.438675+0000","last_active":"2023-09-23T18:22:47.565925+0000","last_peered":"2023-09-23T18:22:47.565925+0000","last_clean":"2023-09-23T18:22:47.565925+0000","last_became_active":"2023-09-23T18:18:46.438339+0000","last_became_peered":"2023-09-23T18:18:46.438339+0000","last_unstale":"2023-09-23T18:22:47.565925+0000","last_undegraded":"2023-09-23T18:22:47.565925+0000","last_fullsized":"2023-09-23T18:22:47.565925+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:32.800388+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":610,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.563158+0000","last_change":"2023-09-23T18:22:44.518222+0000","last_active":"2023-09-23T18:22:46.563158+0000","last_peered":"2023-09-23T18:22:46.563158+0000","last_clean":"2023-09-23T18:22:46.563158+0000","last_became_active":"2023-09-23T18:22:44.517848+0000","last_became_peered":"2023-09-23T18:22:44.517848+0000","last_unstale":"2023-09-23T18:22:46.563158+0000","last_undegraded":"2023-09-23T18:22:46.563158+0000","last_fullsized":"2023-09-23T18:22:46.563158+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:47:40.879308+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":960,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557033+0000","last_change":"2023-09-23T18:18:49.369659+0000","last_active":"2023-09-23T18:22:46.557033+0000","last_peered":"2023-09-23T18:22:46.557033+0000","last_clean":"2023-09-23T18:22:46.557033+0000","last_became_active":"2023-09-23T18:18:49.369295+0000","last_became_peered":"2023-09-23T18:18:49.369295+0000","last_unstale":"2023-09-23T18:22:46.557033+0000","last_undegraded":"2023-09-23T18:22:46.557033+0000","last_fullsized":"2023-09-23T18:22:46.557033+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:39:05.857426+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"653'181","reported_seq":619,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.562954+0000","last_change":"2023-09-23T18:22:28.468611+0000","last_active":"2023-09-23T18:22:46.562954+0000","last_peered":"2023-09-23T18:22:46.562954+0000","last_clean":"2023-09-23T18:22:46.562954+0000","last_became_active":"2023-09-23T18:18:33.327788+0000","last_became_peered":"2023-09-23T18:18:33.327788+0000","last_unstale":"2023-09-23T18:22:46.562954+0000","last_undegraded":"2023-09-23T18:22:46.562954+0000","last_fullsized":"2023-09-23T18:22:46.562954+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:57:28.551510+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001526466,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1349,"num_read_kb":1261,"num_write":144,"num_write_kb":4223,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":21,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":196,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556527+0000","last_change":"2023-09-23T18:22:44.518634+0000","last_active":"2023-09-23T18:22:46.556527+0000","last_peered":"2023-09-23T18:22:46.556527+0000","last_clean":"2023-09-23T18:22:46.556527+0000","last_became_active":"2023-09-23T18:22:44.518286+0000","last_became_peered":"2023-09-23T18:22:44.518286+0000","last_unstale":"2023-09-23T18:22:46.556527+0000","last_undegraded":"2023-09-23T18:22:46.556527+0000","last_fullsized":"2023-09-23T18:22:46.556527+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:43:05.309943+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,2],"acting":[5,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":186,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556284+0000","last_change":"2023-09-23T18:17:21.793568+0000","last_active":"2023-09-23T18:22:46.556284+0000","last_peered":"2023-09-23T18:22:46.556284+0000","last_clean":"2023-09-23T18:22:46.556284+0000","last_became_active":"2023-09-23T18:17:21.793266+0000","last_became_peered":"2023-09-23T18:17:21.793266+0000","last_unstale":"2023-09-23T18:22:46.556284+0000","last_undegraded":"2023-09-23T18:22:46.556284+0000","last_fullsized":"2023-09-23T18:22:46.556284+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:35:43.183187+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,1],"acting":[5,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":980,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.573464+0000","last_change":"2023-09-23T18:22:47.573464+0000","last_active":"2023-09-23T18:22:47.573464+0000","last_peered":"2023-09-23T18:22:47.573464+0000","last_clean":"2023-09-23T18:22:47.573464+0000","last_became_active":"2023-09-23T18:22:47.572988+0000","last_became_peered":"2023-09-23T18:22:47.572988+0000","last_unstale":"2023-09-23T18:22:47.573464+0000","last_undegraded":"2023-09-23T18:22:47.573464+0000","last_fullsized":"2023-09-23T18:22:47.573464+0000","mapping_epoch":662,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":663,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:30:54.224856+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,2],"acting":[6,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":148,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556861+0000","last_change":"2023-09-23T18:16:53.335333+0000","last_active":"2023-09-23T18:22:46.556861+0000","last_peered":"2023-09-23T18:22:46.556861+0000","last_clean":"2023-09-23T18:22:46.556861+0000","last_became_active":"2023-09-23T18:16:53.305925+0000","last_became_peered":"2023-09-23T18:16:53.305925+0000","last_unstale":"2023-09-23T18:22:46.556861+0000","last_undegraded":"2023-09-23T18:22:46.556861+0000","last_fullsized":"2023-09-23T18:22:46.556861+0000","mapping_epoch":527,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":528,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:12.584465+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,0],"acting":[5,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.6","version":"653'188","reported_seq":942,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556793+0000","last_change":"2023-09-23T18:22:28.475214+0000","last_active":"2023-09-23T18:22:46.556793+0000","last_peered":"2023-09-23T18:22:46.556793+0000","last_clean":"2023-09-23T18:22:46.556793+0000","last_became_active":"2023-09-23T18:17:20.787561+0000","last_became_peered":"2023-09-23T18:17:20.787561+0000","last_unstale":"2023-09-23T18:22:46.556793+0000","last_undegraded":"2023-09-23T18:22:46.556793+0000","last_fullsized":"2023-09-23T18:22:46.556793+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":188,"log_dups_size":0,"ondisk_log_size":188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:26.971942+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055658400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1067,"num_read_kb":2442,"num_write":105,"num_write_kb":1356,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,4],"acting":[6,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":81,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557536+0000","last_change":"2023-09-23T18:18:46.426766+0000","last_active":"2023-09-23T18:22:46.557536+0000","last_peered":"2023-09-23T18:22:46.557536+0000","last_clean":"2023-09-23T18:22:46.557536+0000","last_became_active":"2023-09-23T18:18:46.426409+0000","last_became_peered":"2023-09-23T18:18:46.426409+0000","last_unstale":"2023-09-23T18:22:46.557536+0000","last_undegraded":"2023-09-23T18:22:46.557536+0000","last_fullsized":"2023-09-23T18:22:46.557536+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:04:19.420504+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":797,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.565217+0000","last_change":"2023-09-23T18:18:49.365648+0000","last_active":"2023-09-23T18:22:47.565217+0000","last_peered":"2023-09-23T18:22:47.565217+0000","last_clean":"2023-09-23T18:22:47.565217+0000","last_became_active":"2023-09-23T18:18:49.365241+0000","last_became_peered":"2023-09-23T18:18:49.365241+0000","last_unstale":"2023-09-23T18:22:47.565217+0000","last_undegraded":"2023-09-23T18:22:47.565217+0000","last_fullsized":"2023-09-23T18:22:47.565217+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:11:07.663171+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"652'137","reported_seq":1318,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558629+0000","last_change":"2023-09-23T18:22:28.475379+0000","last_active":"2023-09-23T18:22:46.558629+0000","last_peered":"2023-09-23T18:22:46.558629+0000","last_clean":"2023-09-23T18:22:46.558629+0000","last_became_active":"2023-09-23T18:22:03.468424+0000","last_became_peered":"2023-09-23T18:22:03.468424+0000","last_unstale":"2023-09-23T18:22:46.558629+0000","last_undegraded":"2023-09-23T18:22:46.558629+0000","last_fullsized":"2023-09-23T18:22:46.558629+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:57:46.671634+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044965899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":925,"num_write":128,"num_write_kb":1260,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":70,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,2,3],"acting":[6,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":91,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.567864+0000","last_change":"2023-09-23T18:18:33.342733+0000","last_active":"2023-09-23T18:22:47.567864+0000","last_peered":"2023-09-23T18:22:47.567864+0000","last_clean":"2023-09-23T18:22:47.567864+0000","last_became_active":"2023-09-23T18:18:33.328761+0000","last_became_peered":"2023-09-23T18:18:33.328761+0000","last_unstale":"2023-09-23T18:22:47.567864+0000","last_undegraded":"2023-09-23T18:22:47.567864+0000","last_fullsized":"2023-09-23T18:22:47.567864+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:18:18.538258+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":452,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556001+0000","last_change":"2023-09-23T18:17:42.001394+0000","last_active":"2023-09-23T18:22:46.556001+0000","last_peered":"2023-09-23T18:22:46.556001+0000","last_clean":"2023-09-23T18:22:46.556001+0000","last_became_active":"2023-09-23T18:17:42.001070+0000","last_became_peered":"2023-09-23T18:17:42.001070+0000","last_unstale":"2023-09-23T18:22:46.556001+0000","last_undegraded":"2023-09-23T18:22:46.556001+0000","last_fullsized":"2023-09-23T18:22:46.556001+0000","mapping_epoch":566,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":567,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:23:50.613770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":692,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558085+0000","last_change":"2023-09-23T18:18:49.358334+0000","last_active":"2023-09-23T18:22:46.558085+0000","last_peered":"2023-09-23T18:22:46.558085+0000","last_clean":"2023-09-23T18:22:46.558085+0000","last_became_active":"2023-09-23T18:18:49.358093+0000","last_became_peered":"2023-09-23T18:18:49.358093+0000","last_unstale":"2023-09-23T18:22:46.558085+0000","last_undegraded":"2023-09-23T18:22:46.558085+0000","last_fullsized":"2023-09-23T18:22:46.558085+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:42:01.203721+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,0],"acting":[6,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":86,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.563168+0000","last_change":"2023-09-23T18:18:33.328498+0000","last_active":"2023-09-23T18:22:46.563168+0000","last_peered":"2023-09-23T18:22:46.563168+0000","last_clean":"2023-09-23T18:22:46.563168+0000","last_became_active":"2023-09-23T18:18:33.328081+0000","last_became_peered":"2023-09-23T18:18:33.328081+0000","last_unstale":"2023-09-23T18:22:46.563168+0000","last_undegraded":"2023-09-23T18:22:46.563168+0000","last_fullsized":"2023-09-23T18:22:46.563168+0000","mapping_epoch":588,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":589,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:56:56.167368+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,1],"acting":[0,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":14,"reported_epoch":663,"state":"active+clean","last_fresh":"2023-09-23T18:22:47.564029+0000","last_change":"2023-09-23T18:22:44.520691+0000","last_active":"2023-09-23T18:22:47.564029+0000","last_peered":"2023-09-23T18:22:47.564029+0000","last_clean":"2023-09-23T18:22:47.564029+0000","last_became_active":"2023-09-23T18:22:44.520082+0000","last_became_peered":"2023-09-23T18:22:44.520082+0000","last_unstale":"2023-09-23T18:22:47.564029+0000","last_undegraded":"2023-09-23T18:22:47.564029+0000","last_fullsized":"2023-09-23T18:22:47.564029+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:58:29.126035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":692,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556685+0000","last_change":"2023-09-23T18:18:49.378346+0000","last_active":"2023-09-23T18:22:46.556685+0000","last_peered":"2023-09-23T18:22:46.556685+0000","last_clean":"2023-09-23T18:22:46.556685+0000","last_became_active":"2023-09-23T18:18:49.378003+0000","last_became_peered":"2023-09-23T18:18:49.378003+0000","last_unstale":"2023-09-23T18:22:46.556685+0000","last_undegraded":"2023-09-23T18:22:46.556685+0000","last_fullsized":"2023-09-23T18:22:46.556685+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:05:31.151945+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,4],"acting":[6,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":180,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556628+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:22:46.556628+0000","last_peered":"2023-09-23T18:22:46.556628+0000","last_clean":"2023-09-23T18:22:46.556628+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:22:46.556628+0000","last_undegraded":"2023-09-23T18:22:46.556628+0000","last_fullsized":"2023-09-23T18:22:46.556628+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":297,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561410+0000","last_change":"2023-09-23T18:17:20.768678+0000","last_active":"2023-09-23T18:22:46.561410+0000","last_peered":"2023-09-23T18:22:46.561410+0000","last_clean":"2023-09-23T18:22:46.561410+0000","last_became_active":"2023-09-23T18:17:20.764440+0000","last_became_peered":"2023-09-23T18:17:20.764440+0000","last_unstale":"2023-09-23T18:22:46.561410+0000","last_undegraded":"2023-09-23T18:22:46.561410+0000","last_fullsized":"2023-09-23T18:22:46.561410+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:52:31.258097+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":166,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556537+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:22:46.556537+0000","last_peered":"2023-09-23T18:22:46.556537+0000","last_clean":"2023-09-23T18:22:46.556537+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:22:46.556537+0000","last_undegraded":"2023-09-23T18:22:46.556537+0000","last_fullsized":"2023-09-23T18:22:46.556537+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":280,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.561182+0000","last_change":"2023-09-23T18:13:39.230076+0000","last_active":"2023-09-23T18:22:46.561182+0000","last_peered":"2023-09-23T18:22:46.561182+0000","last_clean":"2023-09-23T18:22:46.561182+0000","last_became_active":"2023-09-23T18:13:39.229747+0000","last_became_peered":"2023-09-23T18:13:39.229747+0000","last_unstale":"2023-09-23T18:22:46.561182+0000","last_undegraded":"2023-09-23T18:22:46.561182+0000","last_fullsized":"2023-09-23T18:22:46.561182+0000","mapping_epoch":456,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":457,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:32:11.115476+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":1054,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557110+0000","last_change":"2023-09-23T18:17:21.777478+0000","last_active":"2023-09-23T18:22:46.557110+0000","last_peered":"2023-09-23T18:22:46.557110+0000","last_clean":"2023-09-23T18:22:46.557110+0000","last_became_active":"2023-09-23T18:17:21.777157+0000","last_became_peered":"2023-09-23T18:17:21.777157+0000","last_unstale":"2023-09-23T18:22:46.557110+0000","last_undegraded":"2023-09-23T18:22:46.557110+0000","last_fullsized":"2023-09-23T18:22:46.557110+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:45:06.026656+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,5],"acting":[6,1,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":569,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556931+0000","last_change":"2023-09-23T18:22:44.520133+0000","last_active":"2023-09-23T18:22:46.556931+0000","last_peered":"2023-09-23T18:22:46.556931+0000","last_clean":"2023-09-23T18:22:46.556931+0000","last_became_active":"2023-09-23T18:22:44.519765+0000","last_became_peered":"2023-09-23T18:22:44.519765+0000","last_unstale":"2023-09-23T18:22:46.556931+0000","last_undegraded":"2023-09-23T18:22:46.556931+0000","last_fullsized":"2023-09-23T18:22:46.556931+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:54:02.736149+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,2],"acting":[6,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":135,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.558734+0000","last_change":"2023-09-23T18:22:44.519760+0000","last_active":"2023-09-23T18:22:46.558734+0000","last_peered":"2023-09-23T18:22:46.558734+0000","last_clean":"2023-09-23T18:22:46.558734+0000","last_became_active":"2023-09-23T18:22:44.519006+0000","last_became_peered":"2023-09-23T18:22:44.519006+0000","last_unstale":"2023-09-23T18:22:46.558734+0000","last_undegraded":"2023-09-23T18:22:46.558734+0000","last_fullsized":"2023-09-23T18:22:46.558734+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:01:02.871094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":993,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.559268+0000","last_change":"2023-09-23T18:18:49.367141+0000","last_active":"2023-09-23T18:22:46.559268+0000","last_peered":"2023-09-23T18:22:46.559268+0000","last_clean":"2023-09-23T18:22:46.559268+0000","last_became_active":"2023-09-23T18:18:49.366736+0000","last_became_peered":"2023-09-23T18:18:49.366736+0000","last_unstale":"2023-09-23T18:22:46.559268+0000","last_undegraded":"2023-09-23T18:22:46.559268+0000","last_fullsized":"2023-09-23T18:22:46.559268+0000","mapping_epoch":597,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":598,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.598490+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.598490+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:04.748497+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":200,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.557018+0000","last_change":"2023-09-23T18:22:44.528526+0000","last_active":"2023-09-23T18:22:46.557018+0000","last_peered":"2023-09-23T18:22:46.557018+0000","last_clean":"2023-09-23T18:22:46.557018+0000","last_became_active":"2023-09-23T18:22:44.518445+0000","last_became_peered":"2023-09-23T18:22:44.518445+0000","last_unstale":"2023-09-23T18:22:46.557018+0000","last_undegraded":"2023-09-23T18:22:46.557018+0000","last_fullsized":"2023-09-23T18:22:46.557018+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:33:17.761911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":2216,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":705,"reported_epoch":662,"state":"active+clean","last_fresh":"2023-09-23T18:22:46.556635+0000","last_change":"2023-09-23T18:18:07.536293+0000","last_active":"2023-09-23T18:22:46.556635+0000","last_peered":"2023-09-23T18:22:46.556635+0000","last_clean":"2023-09-23T18:22:46.556635+0000","last_became_active":"2023-09-23T18:18:07.535725+0000","last_became_peered":"2023-09-23T18:18:07.535725+0000","last_unstale":"2023-09-23T18:22:46.556635+0000","last_undegraded":"2023-09-23T18:22:46.556635+0000","last_fullsized":"2023-09-23T18:22:46.556635+0000","mapping_epoch":579,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":580,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:27:39.016224+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,3],"acting":[6,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":100,"num_bytes_recovered":13010,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":102400,"data_stored":16534,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":525376,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":382,"num_bytes_recovered":20971577,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":0,"seq":919123001604,"num_pgs":0,"num_osds":0,"num_per_pool_osds":0,"num_per_pool_omap_osds":0,"kb":0,"kb_used":0,"kb_used_data":0,"kb_used_omap":0,"kb_used_meta":0,"kb_avail":0,"statfs":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},{"osd":6,"up_from":163,"seq":700079669540,"num_pgs":44,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":790336,"kb_used_data":156984,"kb_used_omap":7,"kb_used_meta":633336,"kb_avail":92954816,"statfs":{"total":95995035648,"available":95185731584,"internally_reserved":0,"allocated":160751616,"data_stored":158745655,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":648536868},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:22 2023","interfaces":[{"interface":"back","average":{"1min":4.6609999999999996,"5min":2.7709999999999999,"15min":1.8129999999999999},"min":{"1min":0.245,"5min":0.23000000000000001,"15min":0.22500000000000001},"max":{"1min":79.203999999999994,"5min":79.203999999999994,"15min":79.203999999999994},"last":0.51500000000000001},{"interface":"front","average":{"1min":4.0229999999999997,"5min":2.7810000000000001,"15min":1.9570000000000001},"min":{"1min":0.252,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":78.034000000000006,"5min":78.034000000000006,"15min":78.034000000000006},"last":0.54000000000000004}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.54700000000000004,"5min":2.1240000000000001,"15min":1.518},"min":{"1min":0.28799999999999998,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.8480000000000001,"5min":22.972000000000001,"15min":22.972000000000001},"last":0.59099999999999997},{"interface":"front","average":{"1min":0.58399999999999996,"5min":2.1240000000000001,"15min":1.5129999999999999},"min":{"1min":0.26600000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":3.27,"5min":22.664000000000001,"15min":22.664000000000001},"last":0.42299999999999999}]},{"osd":2,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.581,"5min":1.518,"15min":1.369},"min":{"1min":0.33500000000000002,"5min":0.26800000000000002,"15min":0.25700000000000001},"max":{"1min":7.734,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.57399999999999995},{"interface":"front","average":{"1min":1.8120000000000001,"5min":2.0209999999999999,"15min":1.5920000000000001},"min":{"1min":0.34399999999999997,"5min":0.29699999999999999,"15min":0.28100000000000003},"max":{"1min":8.2690000000000001,"5min":14.771000000000001,"15min":14.771000000000001},"last":0.68899999999999995}]},{"osd":3,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":4.6050000000000004,"5min":3.601,"15min":2.3479999999999999},"min":{"1min":0.33600000000000002,"5min":0.28699999999999998,"15min":0.24099999999999999},"max":{"1min":78.643000000000001,"5min":103.348,"15min":103.348},"last":0.67000000000000004},{"interface":"front","average":{"1min":5.2629999999999999,"5min":4.298,"15min":2.6930000000000001},"min":{"1min":0.32100000000000001,"5min":0.28999999999999998,"15min":0.28699999999999998},"max":{"1min":78.347999999999999,"5min":103.40300000000001,"15min":103.40300000000001},"last":0.65400000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:22:22 2023","interfaces":[{"interface":"back","average":{"1min":0.42999999999999999,"5min":1.3220000000000001,"15min":1.3400000000000001},"min":{"1min":0.24199999999999999,"5min":0.24199999999999999,"15min":0.23999999999999999},"max":{"1min":1.103,"5min":6.2800000000000002,"15min":8.6059999999999999},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.46100000000000002,"5min":1.2769999999999999,"15min":1.375},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":0.91200000000000003,"5min":7.8319999999999999,"15min":9.2260000000000009},"last":0.623}]},{"osd":5,"last update":"Sat Sep 23 18:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":1.4219999999999999,"15min":1.827},"min":{"1min":0.376,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":0.64600000000000002,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.56200000000000006},{"interface":"front","average":{"1min":0.53400000000000003,"5min":1.4930000000000001,"15min":2.1259999999999999},"min":{"1min":0.40200000000000002,"5min":0.32000000000000001,"15min":0.28599999999999998},"max":{"1min":0.70599999999999996,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:21:54 2023","interfaces":[{"interface":"back","average":{"1min":1.1919999999999999,"5min":1.6930000000000001,"15min":1.603},"min":{"1min":0.36499999999999999,"5min":0.34899999999999998,"15min":0.26800000000000002},"max":{"1min":6.1799999999999997,"5min":10.743,"15min":13.460000000000001},"last":0.64000000000000001},{"interface":"front","average":{"1min":1.0209999999999999,"5min":1.593,"15min":1.6200000000000001},"min":{"1min":0.35599999999999998,"5min":0.35599999999999998,"15min":0.28499999999999998},"max":{"1min":5.0990000000000002,"5min":8.6890000000000001,"15min":12.404},"last":0.71499999999999997}]}]},{"osd":1,"up_from":545,"seq":2340757176389,"num_pgs":56,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":629756,"kb_used_data":381812,"kb_used_omap":13,"kb_used_meta":247922,"kb_avail":93115396,"statfs":{"total":95995035648,"available":95350165504,"internally_reserved":0,"allocated":390975488,"data_stored":387755703,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":253872781},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.61699999999999999,"5min":1.331,"15min":0.70399999999999996},"min":{"1min":0.23100000000000001,"5min":0.189,"15min":0.189},"max":{"1min":4.5030000000000001,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.38700000000000001},{"interface":"front","average":{"1min":0.38400000000000001,"5min":1.3069999999999999,"15min":0.76200000000000001},"min":{"1min":0.28399999999999997,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.51600000000000001,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.432}]},{"osd":2,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.40699999999999997,"5min":1.4339999999999999,"15min":0.78500000000000003},"min":{"1min":0.28899999999999998,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.61899999999999999,"5min":11.02,"15min":11.02},"last":0.47699999999999998},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.98799999999999999,"15min":0.66600000000000004},"min":{"1min":0.249,"5min":0.249,"15min":0.249},"max":{"1min":2.234,"5min":5.1440000000000001,"15min":5.1440000000000001},"last":0.501}]},{"osd":3,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42899999999999999,"5min":1.7689999999999999,"15min":0.84699999999999998},"min":{"1min":0.26600000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":1.0980000000000001,"5min":14.962,"15min":14.962},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.51800000000000002,"5min":1.577,"15min":0.86699999999999999},"min":{"1min":0.35499999999999998,"5min":0.318,"15min":0.318},"max":{"1min":0.82199999999999995,"5min":19.762,"15min":19.762},"last":0.64700000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73199999999999998,"5min":1.304,"15min":1.3120000000000001},"min":{"1min":0.38,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":3.532,"5min":12.663,"15min":12.663},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.79700000000000004,"5min":2.1230000000000002,"15min":1.7170000000000001},"min":{"1min":0.39000000000000001,"5min":0.317,"15min":0.317},"max":{"1min":4.7320000000000002,"5min":13.222,"15min":13.222},"last":0.629}]},{"osd":5,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.73699999999999999,"5min":1.7350000000000001,"15min":1.0600000000000001},"min":{"1min":0.373,"5min":0.308,"15min":0.308},"max":{"1min":3.6760000000000002,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.85699999999999998,"5min":2.052,"15min":1.22},"min":{"1min":0.42699999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":4.556,"5min":23.204000000000001,"15min":23.204000000000001},"last":0.72699999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.80200000000000005,"5min":1.736,"15min":1.085},"min":{"1min":0.46999999999999997,"5min":0.372,"15min":0.372},"max":{"1min":3.5659999999999998,"5min":23.443000000000001,"15min":23.443000000000001},"last":0.69199999999999995},{"interface":"front","average":{"1min":0.84499999999999997,"5min":2.306,"15min":1.286},"min":{"1min":0.42499999999999999,"5min":0.378,"15min":0.378},"max":{"1min":4.8959999999999999,"5min":26.177,"15min":26.177},"last":0.57699999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.9399999999999999,"15min":1.1000000000000001},"min":{"1min":0.47399999999999998,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":1.052,"5min":15.01,"15min":15.01},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.867,"15min":1.159},"min":{"1min":0.46899999999999997,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.78400000000000003,"5min":10.988,"15min":10.988},"last":0.77400000000000002}]}]},{"osd":0,"up_from":93,"seq":399431958885,"num_pgs":42,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1143608,"kb_used_data":160624,"kb_used_omap":7,"kb_used_meta":982968,"kb_avail":92601544,"statfs":{"total":95995035648,"available":94823981056,"internally_reserved":0,"allocated":164478976,"data_stored":161438304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":1006559972},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:22:31 2023","interfaces":[{"interface":"back","average":{"1min":0.42599999999999999,"5min":1.3839999999999999,"15min":0.67400000000000004},"min":{"1min":0.251,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":1.2170000000000001,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.39100000000000001},{"interface":"front","average":{"1min":0.51100000000000001,"5min":1.8340000000000001,"15min":0.95999999999999996},"min":{"1min":0.28299999999999997,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":1.853,"5min":22.978000000000002,"15min":22.978000000000002},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:22:06 2023","interfaces":[{"interface":"back","average":{"1min":1.6830000000000001,"5min":2.355,"15min":1.4790000000000001},"min":{"1min":0.25,"5min":0.19800000000000001,"15min":0.185},"max":{"1min":9.8300000000000001,"5min":18.550999999999998,"15min":18.550999999999998},"last":0.36299999999999999},{"interface":"front","average":{"1min":1.4390000000000001,"5min":1.925,"15min":1.1419999999999999},"min":{"1min":0.31900000000000001,"5min":0.22500000000000001,"15min":0.22},"max":{"1min":6.2220000000000004,"5min":23.161999999999999,"15min":23.161999999999999},"last":0.379}]},{"osd":3,"last update":"Sat Sep 23 18:22:51 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":1.9299999999999999,"15min":1.577},"min":{"1min":0.314,"5min":0.24199999999999999,"15min":0.20000000000000001},"max":{"1min":0.88900000000000001,"5min":15.173999999999999,"15min":15.173999999999999},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.55300000000000005,"5min":2.048,"15min":1.6539999999999999},"min":{"1min":0.34399999999999997,"5min":0.26600000000000001,"15min":0.26400000000000001},"max":{"1min":0.98299999999999998,"5min":15.135,"15min":15.135},"last":0.44700000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:22:15 2023","interfaces":[{"interface":"back","average":{"1min":0.89800000000000002,"5min":1.944,"15min":1.9299999999999999},"min":{"1min":0.33300000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":2.9260000000000002,"5min":17.741,"15min":57.746000000000002},"last":0.55300000000000005},{"interface":"front","average":{"1min":1.0960000000000001,"5min":1.899,"15min":1.7150000000000001},"min":{"1min":0.42999999999999999,"5min":0.29199999999999998,"15min":0.25600000000000001},"max":{"1min":6.3860000000000001,"5min":15.079000000000001,"15min":57.816000000000003},"last":0.46100000000000002}]},{"osd":5,"last update":"Sat Sep 23 18:22:51 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":1.5600000000000001,"15min":1.4419999999999999},"min":{"1min":0.42099999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":1.1910000000000001,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.42099999999999999},{"interface":"front","average":{"1min":0.73399999999999999,"5min":2.1720000000000002,"15min":1.6080000000000001},"min":{"1min":0.46700000000000003,"5min":0.35499999999999998,"15min":0.32200000000000001},"max":{"1min":1.5680000000000001,"5min":15.047000000000001,"15min":15.047000000000001},"last":0.50600000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:22:15 2023","interfaces":[{"interface":"back","average":{"1min":1.046,"5min":1.8089999999999999,"15min":1.9870000000000001},"min":{"1min":0.36099999999999999,"5min":0.36099999999999999,"15min":0.32000000000000001},"max":{"1min":6.0439999999999996,"5min":7.6900000000000004,"15min":58.457999999999998},"last":0.57499999999999996},{"interface":"front","average":{"1min":1.284,"5min":2.645,"15min":2.0739999999999998},"min":{"1min":0.47599999999999998,"5min":0.375,"15min":0.315},"max":{"1min":4.4470000000000001,"5min":22.893000000000001,"15min":58.354999999999997},"last":0.48999999999999999}]},{"osd":7,"last update":"Sat Sep 23 18:21:57 2023","interfaces":[{"interface":"back","average":{"1min":1.9750000000000001,"5min":2.2130000000000001,"15min":1.825},"min":{"1min":0.46200000000000002,"5min":0.45000000000000001,"15min":0.34499999999999997},"max":{"1min":6.1200000000000001,"5min":22.945,"15min":22.945},"last":0.40699999999999997},{"interface":"front","average":{"1min":1.2010000000000001,"5min":1.7410000000000001,"15min":1.851},"min":{"1min":0.42799999999999999,"5min":0.371,"15min":0.36199999999999999},"max":{"1min":5.4320000000000004,"5min":7.7519999999999998,"15min":10.404},"last":0.52200000000000002}]}]},{"osd":2,"up_from":482,"seq":2070174236771,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161072,"kb_used_data":12840,"kb_used_omap":22,"kb_used_meta":148201,"kb_avail":93584080,"statfs":{"total":95995035648,"available":95830097920,"internally_reserved":0,"allocated":13148160,"data_stored":9940106,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22605,"internal_metadata":151758771},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":1.385,"5min":1.5980000000000001,"15min":0.90900000000000003},"min":{"1min":0.24299999999999999,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":4.7060000000000004,"5min":9.6359999999999992,"15min":9.6359999999999992},"last":0.35299999999999998},{"interface":"front","average":{"1min":2.9289999999999998,"5min":1.964,"15min":1.286},"min":{"1min":0.25900000000000001,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":8.3390000000000004,"5min":13.472,"15min":13.472},"last":0.26100000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.4219999999999999,"15min":0.80800000000000005},"min":{"1min":0.26100000000000001,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":2.1320000000000001,"5min":13.538,"15min":13.538},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.504,"5min":1.681,"15min":0.98399999999999999},"min":{"1min":0.26300000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":1.1499999999999999,"5min":16.567,"15min":16.567},"last":0.33900000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":2.4470000000000001,"5min":2.1640000000000001,"15min":1.3919999999999999},"min":{"1min":0.40300000000000002,"5min":0.28499999999999998,"15min":0.224},"max":{"1min":8.093,"5min":15.926,"15min":15.926},"last":0.32300000000000001},{"interface":"front","average":{"1min":1.841,"5min":1.754,"15min":1.409},"min":{"1min":0.28199999999999997,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":13.247999999999999,"5min":16.619,"15min":16.619},"last":0.37}]},{"osd":4,"last update":"Sat Sep 23 18:22:03 2023","interfaces":[{"interface":"back","average":{"1min":0.98999999999999999,"5min":1.667,"15min":1.512},"min":{"1min":0.375,"5min":0.31,"15min":0.30299999999999999},"max":{"1min":4.6539999999999999,"5min":9.532,"15min":11.334},"last":0.51900000000000002},{"interface":"front","average":{"1min":1.601,"5min":1.5589999999999999,"15min":1.1599999999999999},"min":{"1min":0.40000000000000002,"5min":0.31900000000000001,"15min":0.31900000000000001},"max":{"1min":5.5229999999999997,"5min":6.367,"15min":11.244},"last":0.59299999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":1.5449999999999999,"5min":1.714,"15min":2.0339999999999998},"min":{"1min":0.38300000000000001,"5min":0.34799999999999998,"15min":0.308},"max":{"1min":9.4960000000000004,"5min":9.4960000000000004,"15min":11.852},"last":0.46000000000000002},{"interface":"front","average":{"1min":1.3320000000000001,"5min":1.6890000000000001,"15min":1.5980000000000001},"min":{"1min":0.34200000000000003,"5min":0.34200000000000003,"15min":0.32600000000000001},"max":{"1min":4.6719999999999997,"5min":8.3849999999999998,"15min":10.942},"last":0.63700000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:22:03 2023","interfaces":[{"interface":"back","average":{"1min":2.9359999999999999,"5min":2.0569999999999999,"15min":1.966},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":29.489000000000001,"5min":29.489000000000001,"15min":29.489000000000001},"last":0.54000000000000004},{"interface":"front","average":{"1min":3.048,"5min":2.1080000000000001,"15min":2.3759999999999999},"min":{"1min":0.42899999999999999,"5min":0.39600000000000002,"15min":0.38700000000000001},"max":{"1min":29.408999999999999,"5min":29.408999999999999,"15min":29.408999999999999},"last":0.47699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:22:01 2023","interfaces":[{"interface":"back","average":{"1min":2.024,"5min":1.671,"15min":3.839},"min":{"1min":0.495,"5min":0.35499999999999998,"15min":0.35499999999999998},"max":{"1min":6.9779999999999998,"5min":13.106,"15min":76.966999999999999},"last":0.57599999999999996},{"interface":"front","average":{"1min":1.746,"5min":1.724,"15min":4.0179999999999998},"min":{"1min":0.45500000000000002,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":9.5619999999999994,"5min":9.5619999999999994,"15min":76.936999999999998},"last":0.55400000000000005}]}]},{"osd":3,"up_from":237,"seq":1017907249400,"num_pgs":51,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818476,"kb_used_data":12964,"kb_used_omap":3,"kb_used_meta":805500,"kb_avail":92926676,"statfs":{"total":95995035648,"available":95156916224,"internally_reserved":0,"allocated":13275136,"data_stored":10013585,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":824832744},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":1.891,"15min":1.859},"min":{"1min":0.24199999999999999,"5min":0.192,"15min":0.183},"max":{"1min":3.4169999999999998,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.27300000000000002},{"interface":"front","average":{"1min":0.35699999999999998,"5min":1.7250000000000001,"15min":1.9550000000000001},"min":{"1min":0.19700000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.55900000000000005,"5min":11.468,"15min":194.661},"last":0.33100000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:22:30 2023","interfaces":[{"interface":"back","average":{"1min":0.98599999999999999,"5min":1.847,"15min":0.96199999999999997},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":8.9629999999999992,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.39500000000000002},{"interface":"front","average":{"1min":0.79600000000000004,"5min":2.7839999999999998,"15min":1.3979999999999999},"min":{"1min":0.29099999999999998,"5min":0.23699999999999999,"15min":0.23699999999999999},"max":{"1min":3.5600000000000001,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.37}]},{"osd":2,"last update":"Sat Sep 23 18:22:04 2023","interfaces":[{"interface":"back","average":{"1min":3.2090000000000001,"5min":2.0369999999999999,"15min":1.5009999999999999},"min":{"1min":0.249,"5min":0.249,"15min":0.21099999999999999},"max":{"1min":18.722999999999999,"5min":18.722999999999999,"15min":18.722999999999999},"last":0.314},{"interface":"front","average":{"1min":1.9319999999999999,"5min":1.6699999999999999,"15min":1.3999999999999999},"min":{"1min":0.32800000000000001,"5min":0.27900000000000003,"15min":0.19600000000000001},"max":{"1min":7.0579999999999998,"5min":11.565,"15min":11.565},"last":0.41799999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.58999999999999997,"5min":1.75,"15min":1.6539999999999999},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":0.98699999999999999,"5min":20.850000000000001,"15min":37.231000000000002},"last":0.52500000000000002},{"interface":"front","average":{"1min":0.56599999999999995,"5min":1.8839999999999999,"15min":1.931},"min":{"1min":0.32700000000000001,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":0.84399999999999997,"5min":20.786000000000001,"15min":37.381},"last":0.64100000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:22:47 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":2.0710000000000002,"15min":1.633},"min":{"1min":0.38200000000000001,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":0.77800000000000002,"5min":19.434999999999999,"15min":19.434999999999999},"last":0.56999999999999995},{"interface":"front","average":{"1min":0.57699999999999996,"5min":1.671,"15min":1.643},"min":{"1min":0.41299999999999998,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":0.86899999999999999,"5min":17.135999999999999,"15min":17.135999999999999},"last":0.499}]},{"osd":6,"last update":"Sat Sep 23 18:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.64100000000000001,"5min":1.784,"15min":1.9490000000000001},"min":{"1min":0.41699999999999998,"5min":0.36699999999999999,"15min":0.32200000000000001},"max":{"1min":0.91100000000000003,"5min":17.192,"15min":115.54000000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.6240000000000001,"15min":1.929},"min":{"1min":0.36899999999999999,"5min":0.35699999999999998,"15min":0.30499999999999999},"max":{"1min":1.0169999999999999,"5min":11.442,"15min":115.691},"last":0.504}]},{"osd":7,"last update":"Sat Sep 23 18:22:52 2023","interfaces":[{"interface":"back","average":{"1min":0.61499999999999999,"5min":1.7729999999999999,"15min":1.7889999999999999},"min":{"1min":0.41999999999999998,"5min":0.38700000000000001,"15min":0.35799999999999998},"max":{"1min":0.91100000000000003,"5min":9.1929999999999996,"15min":15.972},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.66000000000000003,"5min":1.772,"15min":1.6479999999999999},"min":{"1min":0.48299999999999998,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":1.032,"5min":17.164000000000001,"15min":17.164000000000001},"last":0.55900000000000005}]}]},{"osd":4,"up_from":30,"seq":128849019350,"num_pgs":50,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":855492,"kb_used_data":150076,"kb_used_omap":0,"kb_used_meta":705408,"kb_avail":92889660,"statfs":{"total":95995035648,"available":95119011840,"internally_reserved":0,"allocated":153677824,"data_stored":151668113,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":722337792},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":3.8959999999999999,"5min":2.242,"15min":1.5209999999999999},"min":{"1min":0.22700000000000001,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":72.754999999999995,"5min":72.754999999999995,"15min":72.754999999999995},"last":0.51300000000000001},{"interface":"front","average":{"1min":3.9460000000000002,"5min":2.3500000000000001,"15min":1.7789999999999999},"min":{"1min":0.23899999999999999,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":71.533000000000001,"5min":71.533000000000001,"15min":71.533000000000001},"last":0.58899999999999997}]},{"osd":1,"last update":"Sat Sep 23 18:22:24 2023","interfaces":[{"interface":"back","average":{"1min":0.71599999999999997,"5min":2.052,"15min":1.147},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":4.1399999999999997,"5min":44.146999999999998,"15min":44.146999999999998},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.68200000000000005,"5min":1.8959999999999999,"15min":1.0660000000000001},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":3.9740000000000002,"5min":43.497999999999998,"15min":43.497999999999998},"last":0.60399999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:22:05 2023","interfaces":[{"interface":"back","average":{"1min":0.68700000000000006,"5min":1.655,"15min":1.77},"min":{"1min":0.31,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":4.0030000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.55900000000000005},{"interface":"front","average":{"1min":0.78600000000000003,"5min":1.6399999999999999,"15min":1.702},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.253},"max":{"1min":3.3839999999999999,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.45700000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":2.6419999999999999,"15min":2.169},"min":{"1min":0.40200000000000002,"5min":0.32000000000000001,"15min":0.23799999999999999},"max":{"1min":0.80300000000000005,"5min":74.781999999999996,"15min":74.781999999999996},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.58899999999999997,"5min":2.5670000000000002,"15min":2.1309999999999998},"min":{"1min":0.38600000000000001,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":0.78900000000000003,"5min":72.152000000000001,"15min":72.152000000000001},"last":0.67900000000000005}]},{"osd":5,"last update":"Sat Sep 23 18:22:46 2023","interfaces":[{"interface":"back","average":{"1min":0.49399999999999999,"5min":2.3460000000000001,"15min":2.7869999999999999},"min":{"1min":0.29199999999999998,"5min":0.253,"15min":0.253},"max":{"1min":0.73899999999999999,"5min":91.432000000000002,"15min":91.432000000000002},"last":0.53700000000000003},{"interface":"front","average":{"1min":0.50700000000000001,"5min":2.121,"15min":2.5640000000000001},"min":{"1min":0.34200000000000003,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":0.70199999999999996,"5min":91.626999999999995,"15min":91.626999999999995},"last":0.48299999999999998}]},{"osd":6,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":5.141,"5min":2.4329999999999998,"15min":1.7090000000000001},"min":{"1min":0.30099999999999999,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":91.688999999999993,"5min":91.688999999999993,"15min":91.688999999999993},"last":0.57599999999999996},{"interface":"front","average":{"1min":5.1420000000000003,"5min":2.6120000000000001,"15min":1.9159999999999999},"min":{"1min":0.34399999999999997,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":91.837000000000003,"5min":91.837000000000003,"15min":91.837000000000003},"last":0.66200000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:22:09 2023","interfaces":[{"interface":"back","average":{"1min":0.68999999999999995,"5min":1.798,"15min":1.835},"min":{"1min":0.34999999999999998,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":3.6800000000000002,"5min":18.283000000000001,"15min":19.298999999999999},"last":0.63900000000000001},{"interface":"front","average":{"1min":0.85099999999999998,"5min":1.4470000000000001,"15min":1.5529999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.28399999999999997},"max":{"1min":4.4269999999999996,"5min":9.4730000000000008,"15min":12.800000000000001},"last":0.70799999999999996}]}]},{"osd":5,"up_from":420,"seq":1803886264446,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":457528,"kb_used_data":84528,"kb_used_omap":9,"kb_used_meta":372982,"kb_avail":93287624,"statfs":{"total":95995035648,"available":95526526976,"internally_reserved":0,"allocated":86556672,"data_stored":84555233,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":381934013},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":4.7930000000000001,"5min":1.889,"15min":1.2090000000000001},"min":{"1min":0.20499999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":77.162000000000006,"5min":77.162000000000006,"15min":77.162000000000006},"last":0.71499999999999997},{"interface":"front","average":{"1min":4.9450000000000003,"5min":2.4169999999999998,"15min":1.8720000000000001},"min":{"1min":0.23999999999999999,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":75.784999999999997,"5min":75.784999999999997,"15min":75.784999999999997},"last":0.52200000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:22:25 2023","interfaces":[{"interface":"back","average":{"1min":1.3029999999999999,"5min":1.998,"15min":1.3959999999999999},"min":{"1min":0.23300000000000001,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":9.4819999999999993,"5min":23.946000000000002,"15min":23.946000000000002},"last":0.752},{"interface":"front","average":{"1min":1.5449999999999999,"5min":2.133,"15min":1.2749999999999999},"min":{"1min":0.27300000000000002,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":9.2759999999999998,"5min":23.268999999999998,"15min":23.268999999999998},"last":0.59799999999999998}]},{"osd":2,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.611,"5min":1.615,"15min":1.611},"min":{"1min":0.31,"5min":0.26200000000000001,"15min":0.21199999999999999},"max":{"1min":8.8789999999999996,"5min":10.705,"15min":10.705},"last":0.67900000000000005},{"interface":"front","average":{"1min":1.302,"5min":1.673,"15min":2.0640000000000001},"min":{"1min":0.32300000000000001,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":4.3390000000000004,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.57499999999999996}]},{"osd":3,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":5.4400000000000004,"5min":2.5369999999999999,"15min":1.6579999999999999},"min":{"1min":0.317,"5min":0.26200000000000001,"15min":0.25600000000000001},"max":{"1min":79.016999999999996,"5min":79.016999999999996,"15min":79.016999999999996},"last":0.63800000000000001},{"interface":"front","average":{"1min":5.2060000000000004,"5min":2.7810000000000001,"15min":1.9590000000000001},"min":{"1min":0.32900000000000001,"5min":0.32900000000000001,"15min":0.313},"max":{"1min":76.188999999999993,"5min":76.188999999999993,"15min":76.188999999999993},"last":0.61899999999999999}]},{"osd":4,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.4750000000000001,"5min":1.718,"15min":2.016},"min":{"1min":0.29199999999999998,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":6.9930000000000003,"5min":8.2270000000000003,"15min":12.747999999999999},"last":0.45200000000000001},{"interface":"front","average":{"1min":1.1200000000000001,"5min":1.7689999999999999,"15min":1.647},"min":{"1min":0.254,"5min":0.254,"15min":0.254},"max":{"1min":8.5559999999999992,"5min":9.0690000000000008,"15min":9.0690000000000008},"last":0.42899999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.238,"5min":1.3340000000000001,"15min":1.2929999999999999},"min":{"1min":0.29499999999999998,"5min":0.26400000000000001,"15min":0.26400000000000001},"max":{"1min":6.7859999999999996,"5min":6.7859999999999996,"15min":9.0350000000000001},"last":0.53800000000000003},{"interface":"front","average":{"1min":1.119,"5min":1.859,"15min":1.716},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":5.0890000000000004,"5min":12.504,"15min":12.504},"last":0.47299999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:21:55 2023","interfaces":[{"interface":"back","average":{"1min":1.393,"5min":1.704,"15min":1.806},"min":{"1min":0.33700000000000002,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":6.9569999999999999,"5min":10.528,"15min":13.506},"last":0.51000000000000001},{"interface":"front","average":{"1min":1.6830000000000001,"5min":1.494,"15min":1.5860000000000001},"min":{"1min":0.39900000000000002,"5min":0.35299999999999998,"15min":0.29799999999999999},"max":{"1min":12.702,"5min":12.702,"15min":13.138},"last":0.79900000000000004}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4198400,"data_stored":4194323,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":12288,"data_stored":666,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":5322,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":160,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":20480,"data_stored":4838,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":22,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:23:01.261 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T18:23:01.262 INFO:tasks.thrashosds.thrasher:choose_action: min_in 4 min_out 0 min_live 2 min_dead 0 chance_down 0.40 2023-09-23T18:23:01.262 INFO:tasks.thrashosds.thrasher:Removing osd 0, in_osds are: [1, 0, 4, 3, 6, 5, 2] 2023-09-23T18:23:01.262 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd out 0 2023-09-23T18:23:01.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:01 smithi055 ceph-mon[179668]: from='client.38283 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:23:01.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:01 smithi055 ceph-mon[184174]: from='client.38283 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:23:01.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:01 smithi099 ceph-mon[167076]: from='client.38283 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:23:01.875 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:02.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:02 smithi055 ceph-mon[184174]: pgmap v1594: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:23:02.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:02 smithi055 ceph-mon[179668]: pgmap v1594: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:23:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:02 smithi099 ceph-mon[167076]: pgmap v1594: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:23:03.512 INFO:teuthology.orchestra.run.smithi055.stderr:marked out osd.0. 2023-09-23T18:23:03.658 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:03 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/177337020' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T18:23:03.658 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:03 smithi055 ceph-mon[184174]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T18:23:03.658 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:03 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:23:03.287+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e663 definitely_dead 0 2023-09-23T18:23:03.658 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:03 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/177337020' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T18:23:03.658 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:03 smithi055 ceph-mon[179668]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T18:23:03.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:03 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/177337020' entity='client.admin' cmd=[{"prefix": "osd out", "ids": ["0"]}]: dispatch 2023-09-23T18:23:03.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:03 smithi099 ceph-mon[167076]: Client client.admin marked osd.0 out, while it was still marked up 2023-09-23T18:23:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:04 smithi099 ceph-mon[167076]: pgmap v1595: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:23:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:04 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/177337020' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T18:23:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:04 smithi099 ceph-mon[167076]: osdmap e664: 8 total, 8 up, 6 in 2023-09-23T18:23:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:04 smithi099 ceph-mon[167076]: failed to encode map e664 with expected crc 2023-09-23T18:23:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:04 smithi099 ceph-mon[167076]: failed to encode map e664 with expected crc 2023-09-23T18:23:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:04 smithi099 ceph-mon[167076]: failed to encode map e664 with expected crc 2023-09-23T18:23:04.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[179668]: pgmap v1595: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:23:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/177337020' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T18:23:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[179668]: osdmap e664: 8 total, 8 up, 6 in 2023-09-23T18:23:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[179668]: failed to encode map e664 with expected crc 2023-09-23T18:23:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[179668]: failed to encode map e664 with expected crc 2023-09-23T18:23:04.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[179668]: failed to encode map e664 with expected crc 2023-09-23T18:23:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[184174]: pgmap v1595: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 621 GiB / 626 GiB avail 2023-09-23T18:23:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/177337020' entity='client.admin' cmd='[{"prefix": "osd out", "ids": ["0"]}]': finished 2023-09-23T18:23:04.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[184174]: osdmap e664: 8 total, 8 up, 6 in 2023-09-23T18:23:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[184174]: failed to encode map e664 with expected crc 2023-09-23T18:23:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[184174]: failed to encode map e664 with expected crc 2023-09-23T18:23:04.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:04 smithi055 ceph-mon[184174]: failed to encode map e664 with expected crc 2023-09-23T18:23:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e664 with expected crc 2023-09-23T18:23:05.974 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e664 with expected crc 2023-09-23T18:23:05.974 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.974 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: osdmap e665: 8 total, 8 up, 6 in 2023-09-23T18:23:05.974 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.974 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.974 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.974 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.975 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.975 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.975 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.975 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.975 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:05 smithi099 ceph-mon[167076]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e664 with expected crc 2023-09-23T18:23:05.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e664 with expected crc 2023-09-23T18:23:05.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: osdmap e665: 8 total, 8 up, 6 in 2023-09-23T18:23:05.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.976 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.977 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.977 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.977 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.977 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.977 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.977 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[184174]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.977 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e664 with expected crc 2023-09-23T18:23:05.977 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e664 with expected crc 2023-09-23T18:23:05.978 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.978 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: osdmap e665: 8 total, 8 up, 6 in 2023-09-23T18:23:05.978 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.978 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.978 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.978 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.978 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.978 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.978 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.978 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e665 with expected crc 2023-09-23T18:23:05.979 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:05 smithi055 ceph-mon[179668]: failed to encode map e665 with expected crc 2023-09-23T18:23:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:06 smithi099 ceph-mon[167076]: pgmap v1598: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail 2023-09-23T18:23:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:06 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:23:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:06 smithi099 ceph-mon[167076]: failed to encode map e666 with expected crc 2023-09-23T18:23:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:06 smithi099 ceph-mon[167076]: osdmap e666: 8 total, 8 up, 6 in 2023-09-23T18:23:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:06 smithi099 ceph-mon[167076]: failed to encode map e666 with expected crc 2023-09-23T18:23:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:06 smithi099 ceph-mon[167076]: failed to encode map e666 with expected crc 2023-09-23T18:23:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[179668]: pgmap v1598: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail 2023-09-23T18:23:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:23:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[179668]: failed to encode map e666 with expected crc 2023-09-23T18:23:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[179668]: osdmap e666: 8 total, 8 up, 6 in 2023-09-23T18:23:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[179668]: failed to encode map e666 with expected crc 2023-09-23T18:23:06.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[179668]: failed to encode map e666 with expected crc 2023-09-23T18:23:06.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[184174]: pgmap v1598: 103 pgs: 3 peering, 100 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail 2023-09-23T18:23:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:23:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[184174]: failed to encode map e666 with expected crc 2023-09-23T18:23:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[184174]: osdmap e666: 8 total, 8 up, 6 in 2023-09-23T18:23:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[184174]: failed to encode map e666 with expected crc 2023-09-23T18:23:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[184174]: failed to encode map e666 with expected crc 2023-09-23T18:23:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:07 smithi099 ceph-mon[167076]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:07 smithi099 ceph-mon[167076]: osdmap e667: 8 total, 8 up, 6 in 2023-09-23T18:23:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:07 smithi099 ceph-mon[167076]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:07 smithi099 ceph-mon[167076]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:07 smithi099 ceph-mon[167076]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:07 smithi099 ceph-mon[167076]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:07 smithi099 ceph-mon[167076]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:07 smithi099 ceph-mon[167076]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[184174]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[184174]: osdmap e667: 8 total, 8 up, 6 in 2023-09-23T18:23:07.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[184174]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[184174]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[184174]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[184174]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[184174]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[184174]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[179668]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[179668]: osdmap e667: 8 total, 8 up, 6 in 2023-09-23T18:23:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[179668]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[179668]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[179668]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[179668]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[179668]: failed to encode map e667 with expected crc 2023-09-23T18:23:07.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:07 smithi055 ceph-mon[179668]: failed to encode map e667 with expected crc 2023-09-23T18:23:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:08 smithi099 ceph-mon[167076]: pgmap v1601: 103 pgs: 10 active+undersized+remapped, 2 remapped+peering, 3 peering, 88 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 80 KiB/s, 6 objects/s recovering 2023-09-23T18:23:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:08 smithi099 ceph-mon[167076]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:08 smithi099 ceph-mon[167076]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:08 smithi099 ceph-mon[167076]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:08 smithi099 ceph-mon[167076]: osdmap e668: 8 total, 8 up, 6 in 2023-09-23T18:23:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:08 smithi099 ceph-mon[167076]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:08 smithi099 ceph-mon[167076]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:08 smithi099 ceph-mon[167076]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:08 smithi099 ceph-mon[167076]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.925 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T18:23:08.925 INFO:tasks.thrashosds.thrasher:Adding osd 7 2023-09-23T18:23:08.926 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 7 2023-09-23T18:23:08.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[179668]: pgmap v1601: 103 pgs: 10 active+undersized+remapped, 2 remapped+peering, 3 peering, 88 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 80 KiB/s, 6 objects/s recovering 2023-09-23T18:23:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[179668]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[179668]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[179668]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[179668]: osdmap e668: 8 total, 8 up, 6 in 2023-09-23T18:23:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[179668]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[179668]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[179668]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[179668]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[184174]: pgmap v1601: 103 pgs: 10 active+undersized+remapped, 2 remapped+peering, 3 peering, 88 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 80 KiB/s, 6 objects/s recovering 2023-09-23T18:23:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[184174]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[184174]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[184174]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[184174]: osdmap e668: 8 total, 8 up, 6 in 2023-09-23T18:23:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[184174]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[184174]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[184174]: failed to encode map e668 with expected crc 2023-09-23T18:23:08.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:08 smithi055 ceph-mon[184174]: failed to encode map e668 with expected crc 2023-09-23T18:23:09.561 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:09.854 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:09 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:23:09.854 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:09 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:23:09.854 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:09 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:23:09.855 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:09 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:23:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:09 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg inactive, 1 pg peering) 2023-09-23T18:23:10.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:09 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:23:10.572 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:23:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:23:10] "GET /metrics HTTP/1.1" 200 34219 "" "Prometheus/2.43.0" 2023-09-23T18:23:10.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:10 smithi055 ceph-mon[179668]: pgmap v1603: 103 pgs: 10 active+undersized+remapped, 2 remapped+peering, 91 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 932 KiB/s, 14 objects/s recovering 2023-09-23T18:23:10.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:10 smithi055 ceph-mon[184174]: pgmap v1603: 103 pgs: 10 active+undersized+remapped, 2 remapped+peering, 91 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 932 KiB/s, 14 objects/s recovering 2023-09-23T18:23:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:10 smithi099 ceph-mon[167076]: pgmap v1603: 103 pgs: 10 active+undersized+remapped, 2 remapped+peering, 91 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 932 KiB/s, 14 objects/s recovering 2023-09-23T18:23:11.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:23:10.958+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e668 definitely_dead 0 2023-09-23T18:23:11.625 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.7. 2023-09-23T18:23:11.916 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:11 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/879175101' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T18:23:11.916 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:11 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/879175101' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T18:23:12.013 INFO:tasks.thrashosds.thrasher:Added osd 7 2023-09-23T18:23:12.013 INFO:tasks.thrashosds.thrasher:inning osd 2023-09-23T18:23:12.014 INFO:tasks.thrashosds.thrasher:Adding osd 0 2023-09-23T18:23:12.014 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd in 0 2023-09-23T18:23:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:11 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/879175101' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["7"]}]: dispatch 2023-09-23T18:23:12.667 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:12.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[184174]: pgmap v1604: 103 pgs: 5 active+undersized+remapped, 1 remapped+peering, 97 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 736 KiB/s, 11 objects/s recovering 2023-09-23T18:23:12.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[184174]: failed to encode map e669 with expected crc 2023-09-23T18:23:12.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/879175101' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T18:23:12.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[184174]: osdmap e669: 8 total, 8 up, 7 in 2023-09-23T18:23:12.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[184174]: failed to encode map e669 with expected crc 2023-09-23T18:23:12.872 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[184174]: failed to encode map e669 with expected crc 2023-09-23T18:23:12.873 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[179668]: pgmap v1604: 103 pgs: 5 active+undersized+remapped, 1 remapped+peering, 97 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 736 KiB/s, 11 objects/s recovering 2023-09-23T18:23:12.873 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[179668]: failed to encode map e669 with expected crc 2023-09-23T18:23:12.873 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/879175101' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T18:23:12.873 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[179668]: osdmap e669: 8 total, 8 up, 7 in 2023-09-23T18:23:12.873 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[179668]: failed to encode map e669 with expected crc 2023-09-23T18:23:12.873 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:12 smithi055 ceph-mon[179668]: failed to encode map e669 with expected crc 2023-09-23T18:23:13.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:12 smithi099 ceph-mon[167076]: pgmap v1604: 103 pgs: 5 active+undersized+remapped, 1 remapped+peering, 97 active+clean; 8.6 MiB data, 3.5 GiB used, 533 GiB / 536 GiB avail; 736 KiB/s, 11 objects/s recovering 2023-09-23T18:23:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:12 smithi099 ceph-mon[167076]: failed to encode map e669 with expected crc 2023-09-23T18:23:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:12 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/879175101' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["7"]}]': finished 2023-09-23T18:23:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:12 smithi099 ceph-mon[167076]: osdmap e669: 8 total, 8 up, 7 in 2023-09-23T18:23:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:12 smithi099 ceph-mon[167076]: failed to encode map e669 with expected crc 2023-09-23T18:23:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:12 smithi099 ceph-mon[167076]: failed to encode map e669 with expected crc 2023-09-23T18:23:13.929 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[184174]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[184174]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[184174]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[184174]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[184174]: osdmap e670: 8 total, 8 up, 7 in 2023-09-23T18:23:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[184174]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[184174]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[184174]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[184174]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[179668]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[179668]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[179668]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[179668]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[179668]: osdmap e670: 8 total, 8 up, 7 in 2023-09-23T18:23:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[179668]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[179668]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[179668]: failed to encode map e670 with expected crc 2023-09-23T18:23:13.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:13 smithi055 ceph-mon[179668]: failed to encode map e670 with expected crc 2023-09-23T18:23:14.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:13 smithi099 ceph-mon[167076]: failed to encode map e670 with expected crc 2023-09-23T18:23:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:13 smithi099 ceph-mon[167076]: failed to encode map e670 with expected crc 2023-09-23T18:23:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:13 smithi099 ceph-mon[167076]: failed to encode map e670 with expected crc 2023-09-23T18:23:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:13 smithi099 ceph-mon[167076]: failed to encode map e670 with expected crc 2023-09-23T18:23:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:13 smithi099 ceph-mon[167076]: osdmap e670: 8 total, 8 up, 7 in 2023-09-23T18:23:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:13 smithi099 ceph-mon[167076]: failed to encode map e670 with expected crc 2023-09-23T18:23:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:13 smithi099 ceph-mon[167076]: failed to encode map e670 with expected crc 2023-09-23T18:23:14.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:13 smithi099 ceph-mon[167076]: failed to encode map e670 with expected crc 2023-09-23T18:23:14.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:13 smithi099 ceph-mon[167076]: failed to encode map e670 with expected crc 2023-09-23T18:23:14.429 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:23:14.148+0000 7fb80f4d6700 -1 mon.a@0(leader).osd e671 definitely_dead 0 2023-09-23T18:23:14.669 INFO:teuthology.orchestra.run.smithi055.stderr:marked in osd.0. 2023-09-23T18:23:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: pgmap v1607: 103 pgs: 3 remapped+peering, 1 peering, 99 active+clean; 8.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 1.3 MiB/s, 9 objects/s recovering 2023-09-23T18:23:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:23:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: osdmap e671: 8 total, 8 up, 7 in 2023-09-23T18:23:14.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2202755885' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T18:23:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: pgmap v1607: 103 pgs: 3 remapped+peering, 1 peering, 99 active+clean; 8.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 1.3 MiB/s, 9 objects/s recovering 2023-09-23T18:23:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:23:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: osdmap e671: 8 total, 8 up, 7 in 2023-09-23T18:23:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: failed to encode map e671 with expected crc 2023-09-23T18:23:14.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:14 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2202755885' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T18:23:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: pgmap v1607: 103 pgs: 3 remapped+peering, 1 peering, 99 active+clean; 8.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 1.3 MiB/s, 9 objects/s recovering 2023-09-23T18:23:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg inactive, 2 pgs peering (PG_AVAILABILITY) 2023-09-23T18:23:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: failed to encode map e671 with expected crc 2023-09-23T18:23:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: failed to encode map e671 with expected crc 2023-09-23T18:23:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: failed to encode map e671 with expected crc 2023-09-23T18:23:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: failed to encode map e671 with expected crc 2023-09-23T18:23:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: failed to encode map e671 with expected crc 2023-09-23T18:23:15.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: failed to encode map e671 with expected crc 2023-09-23T18:23:15.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: failed to encode map e671 with expected crc 2023-09-23T18:23:15.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: failed to encode map e671 with expected crc 2023-09-23T18:23:15.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: osdmap e671: 8 total, 8 up, 7 in 2023-09-23T18:23:15.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: failed to encode map e671 with expected crc 2023-09-23T18:23:15.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:14 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2202755885' entity='client.admin' cmd=[{"prefix": "osd in", "ids": ["0"]}]: dispatch 2023-09-23T18:23:15.135 INFO:tasks.thrashosds.thrasher:Added osd 0 2023-09-23T18:23:15.136 INFO:tasks.thrashosds.thrasher:fixing pg num pool .mgr 2023-09-23T18:23:15.136 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd pool set .mgr pgp_num 51 2023-09-23T18:23:15.753 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[179668]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2202755885' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T18:23:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[179668]: osdmap e672: 8 total, 8 up, 8 in 2023-09-23T18:23:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[179668]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[179668]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[179668]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[179668]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[179668]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[179668]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[179668]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[184174]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2202755885' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T18:23:15.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[184174]: osdmap e672: 8 total, 8 up, 8 in 2023-09-23T18:23:15.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[184174]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[184174]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[184174]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[184174]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[184174]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[184174]: failed to encode map e672 with expected crc 2023-09-23T18:23:15.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:15 smithi055 ceph-mon[184174]: failed to encode map e672 with expected crc 2023-09-23T18:23:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:15 smithi099 ceph-mon[167076]: failed to encode map e672 with expected crc 2023-09-23T18:23:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2202755885' entity='client.admin' cmd='[{"prefix": "osd in", "ids": ["0"]}]': finished 2023-09-23T18:23:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:15 smithi099 ceph-mon[167076]: osdmap e672: 8 total, 8 up, 8 in 2023-09-23T18:23:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:15 smithi099 ceph-mon[167076]: failed to encode map e672 with expected crc 2023-09-23T18:23:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:15 smithi099 ceph-mon[167076]: failed to encode map e672 with expected crc 2023-09-23T18:23:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:15 smithi099 ceph-mon[167076]: failed to encode map e672 with expected crc 2023-09-23T18:23:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:15 smithi099 ceph-mon[167076]: failed to encode map e672 with expected crc 2023-09-23T18:23:16.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:15 smithi099 ceph-mon[167076]: failed to encode map e672 with expected crc 2023-09-23T18:23:16.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:15 smithi099 ceph-mon[167076]: failed to encode map e672 with expected crc 2023-09-23T18:23:16.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:15 smithi099 ceph-mon[167076]: failed to encode map e672 with expected crc 2023-09-23T18:23:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: pgmap v1610: 103 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 activating+remapped, 3 activating+undersized+remapped, 1 active+undersized+remapped, 3 remapped+peering, 1 peering, 91 active+clean; 8.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 51/87 objects degraded (58.621%); 22 B/s, 0 objects/s recovering 2023-09-23T18:23:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: Health check failed: Degraded data redundancy: 51/87 objects degraded (58.621%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:23:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T18:23:16.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[184174]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: pgmap v1610: 103 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 activating+remapped, 3 activating+undersized+remapped, 1 active+undersized+remapped, 3 remapped+peering, 1 peering, 91 active+clean; 8.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 51/87 objects degraded (58.621%); 22 B/s, 0 objects/s recovering 2023-09-23T18:23:16.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: Health check failed: Degraded data redundancy: 51/87 objects degraded (58.621%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:23:16.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:16.932 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:16 smithi055 ceph-mon[179668]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: pgmap v1610: 103 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 activating+remapped, 3 activating+undersized+remapped, 1 active+undersized+remapped, 3 remapped+peering, 1 peering, 91 active+clean; 8.6 MiB data, 3.9 GiB used, 622 GiB / 626 GiB avail; 51/87 objects degraded (58.621%); 22 B/s, 0 objects/s recovering 2023-09-23T18:23:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: Health check failed: Degraded data redundancy: 51/87 objects degraded (58.621%), 3 pgs degraded (PG_DEGRADED) 2023-09-23T18:23:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: osdmap e673: 8 total, 8 up, 8 in 2023-09-23T18:23:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:16 smithi099 ceph-mon[167076]: failed to encode map e673 with expected crc 2023-09-23T18:23:17.164 INFO:teuthology.orchestra.run.smithi055.stderr:Error EINVAL: specified pgp_num 51 > pg_num 31 2023-09-23T18:23:17.511 DEBUG:teuthology.orchestra.run:got remote process result: 22 2023-09-23T18:23:17.512 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph -- tell 'mon.*' injectargs --mon_osd_down_out_interval=600 2023-09-23T18:23:17.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T18:23:17.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:17.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1918638982' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T18:23:17.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T18:23:17.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: failed to encode map e674 with expected crc 2023-09-23T18:23:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:17.932 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:17 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1918638982' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T18:23:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: failed to encode map e674 with expected crc 2023-09-23T18:23:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: failed to encode map e674 with expected crc 2023-09-23T18:23:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: osdmap e674: 8 total, 8 up, 8 in 2023-09-23T18:23:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: failed to encode map e674 with expected crc 2023-09-23T18:23:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: failed to encode map e674 with expected crc 2023-09-23T18:23:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: failed to encode map e674 with expected crc 2023-09-23T18:23:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: failed to encode map e674 with expected crc 2023-09-23T18:23:18.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: failed to encode map e674 with expected crc 2023-09-23T18:23:18.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: failed to encode map e674 with expected crc 2023-09-23T18:23:18.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:18.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:17 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1918638982' entity='client.admin' cmd=[{"prefix": "osd pool set", "pool": ".mgr", "var": "pgp_num", "val": "51"}]: dispatch 2023-09-23T18:23:18.129 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: pgmap v1613: 103 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 activating+remapped, 3 activating+undersized+remapped, 1 active+undersized+remapped, 11 remapped+peering, 4 peering, 80 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 51/87 objects degraded (58.621%); 0 B/s, 5 objects/s recovering 2023-09-23T18:23:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T18:23:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:18 smithi099 ceph-mon[167076]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.179 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: pgmap v1613: 103 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 activating+remapped, 3 activating+undersized+remapped, 1 active+undersized+remapped, 11 remapped+peering, 4 peering, 80 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 51/87 objects degraded (58.621%); 0 B/s, 5 objects/s recovering 2023-09-23T18:23:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T18:23:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[184174]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: pgmap v1613: 103 pgs: 2 activating+undersized+degraded+remapped, 1 active+undersized+degraded+remapped, 1 activating+remapped, 3 activating+undersized+remapped, 1 active+undersized+remapped, 11 remapped+peering, 4 peering, 80 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 51/87 objects degraded (58.621%); 0 B/s, 5 objects/s recovering 2023-09-23T18:23:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: osdmap e675: 8 total, 8 up, 8 in 2023-09-23T18:23:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: failed to encode map e675 with expected crc 2023-09-23T18:23:19.182 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:18 smithi055 ceph-mon[179668]: failed to encode map e675 with expected crc 2023-09-23T18:23:20.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:19 smithi099 ceph-mon[167076]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:19 smithi099 ceph-mon[167076]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:19 smithi099 ceph-mon[167076]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:19 smithi099 ceph-mon[167076]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T18:23:20.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:19 smithi099 ceph-mon[167076]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:19 smithi099 ceph-mon[167076]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:19 smithi099 ceph-mon[167076]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[184174]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[184174]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[184174]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[184174]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T18:23:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[184174]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[184174]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[184174]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[179668]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[179668]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[179668]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[179668]: osdmap e676: 8 total, 8 up, 8 in 2023-09-23T18:23:20.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[179668]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[179668]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:19 smithi055 ceph-mon[179668]: failed to encode map e676 with expected crc 2023-09-23T18:23:20.546 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:23:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:23:20] "GET /metrics HTTP/1.1" 200 34219 "" "Prometheus/2.43.0" 2023-09-23T18:23:20.796 INFO:teuthology.orchestra.run.smithi055.stdout:mon.a: {} 2023-09-23T18:23:20.806 INFO:teuthology.orchestra.run.smithi055.stdout:mon.b: {} 2023-09-23T18:23:20.816 INFO:teuthology.orchestra.run.smithi055.stdout:mon.c: {} 2023-09-23T18:23:20.817 INFO:teuthology.orchestra.run.smithi055.stderr:mon.a: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T18:23:20.817 INFO:teuthology.orchestra.run.smithi055.stderr:mon.b: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T18:23:20.818 INFO:teuthology.orchestra.run.smithi055.stderr:mon.c: mon_osd_down_out_interval = '600' (not observed, change may require restart) 2023-09-23T18:23:20.844 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:20 smithi055 ceph-mon[179668]: pgmap v1616: 103 pgs: 8 remapped+peering, 6 peering, 89 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s, 44 objects/s recovering 2023-09-23T18:23:20.844 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:20 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 3 pgs peering) 2023-09-23T18:23:20.844 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:20 smithi055 ceph-mon[179668]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 51/87 objects degraded (58.621%), 3 pgs degraded) 2023-09-23T18:23:20.844 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:20 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:23:20.844 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:20 smithi055 ceph-mon[184174]: pgmap v1616: 103 pgs: 8 remapped+peering, 6 peering, 89 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s, 44 objects/s recovering 2023-09-23T18:23:20.844 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:20 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 3 pgs peering) 2023-09-23T18:23:20.844 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:20 smithi055 ceph-mon[184174]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 51/87 objects degraded (58.621%), 3 pgs degraded) 2023-09-23T18:23:20.845 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:20 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:23:21.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:20 smithi099 ceph-mon[167076]: pgmap v1616: 103 pgs: 8 remapped+peering, 6 peering, 89 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 3.1 MiB/s, 44 objects/s recovering 2023-09-23T18:23:21.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:20 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 5 pgs inactive, 3 pgs peering) 2023-09-23T18:23:21.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:20 smithi099 ceph-mon[167076]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 51/87 objects degraded (58.621%), 3 pgs degraded) 2023-09-23T18:23:21.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:20 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:23:21.211 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 0 1 2023-09-23T18:23:21.817 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:22.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:21 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/258179730' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:23:22.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:22.004 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:22.004 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:21 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/258179730' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:23:22.004 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:22.004 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:21 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/258179730' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-09-23T18:23:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:22.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:22 smithi099 ceph-mon[167076]: pgmap v1617: 103 pgs: 3 remapped+peering, 6 peering, 94 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s, 31 objects/s recovering 2023-09-23T18:23:23.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:22 smithi055 ceph-mon[179668]: pgmap v1617: 103 pgs: 3 remapped+peering, 6 peering, 94 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s, 31 objects/s recovering 2023-09-23T18:23:23.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:22 smithi055 ceph-mon[184174]: pgmap v1617: 103 pgs: 3 remapped+peering, 6 peering, 94 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 2.2 MiB/s, 31 objects/s recovering 2023-09-23T18:23:23.753 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.0 to 1 (10000) 2023-09-23T18:23:24.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:23 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3046872764' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:23:24.149 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:23 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3046872764' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:23:24.149 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:23 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3046872764' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:23:24.179 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 0 1 2023-09-23T18:23:24.790 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:25.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[179668]: pgmap v1618: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s, 19 objects/s recovering 2023-09-23T18:23:25.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[179668]: failed to encode map e677 with expected crc 2023-09-23T18:23:25.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[179668]: failed to encode map e677 with expected crc 2023-09-23T18:23:25.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3046872764' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:23:25.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[179668]: osdmap e677: 8 total, 8 up, 8 in 2023-09-23T18:23:25.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[179668]: failed to encode map e677 with expected crc 2023-09-23T18:23:25.012 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[184174]: pgmap v1618: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s, 19 objects/s recovering 2023-09-23T18:23:25.012 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[184174]: failed to encode map e677 with expected crc 2023-09-23T18:23:25.012 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[184174]: failed to encode map e677 with expected crc 2023-09-23T18:23:25.012 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3046872764' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:23:25.012 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[184174]: osdmap e677: 8 total, 8 up, 8 in 2023-09-23T18:23:25.012 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:24 smithi055 ceph-mon[184174]: failed to encode map e677 with expected crc 2023-09-23T18:23:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:24 smithi099 ceph-mon[167076]: pgmap v1618: 103 pgs: 103 active+clean; 8.6 MiB data, 4.9 GiB used, 710 GiB / 715 GiB avail; 1.9 MiB/s, 19 objects/s recovering 2023-09-23T18:23:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:24 smithi099 ceph-mon[167076]: failed to encode map e677 with expected crc 2023-09-23T18:23:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:24 smithi099 ceph-mon[167076]: failed to encode map e677 with expected crc 2023-09-23T18:23:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:24 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3046872764' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:23:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:24 smithi099 ceph-mon[167076]: osdmap e677: 8 total, 8 up, 8 in 2023-09-23T18:23:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:24 smithi099 ceph-mon[167076]: failed to encode map e677 with expected crc 2023-09-23T18:23:26.764 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.0 primary-affinity to 1 (8655362) 2023-09-23T18:23:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:26 smithi099 ceph-mon[167076]: pgmap v1620: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s, 16 objects/s recovering 2023-09-23T18:23:27.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:26 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3628504771' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:23:27.078 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:26 smithi055 ceph-mon[179668]: pgmap v1620: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s, 16 objects/s recovering 2023-09-23T18:23:27.078 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:26 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3628504771' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:23:27.078 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:26 smithi055 ceph-mon[184174]: pgmap v1620: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 1.6 MiB/s, 16 objects/s recovering 2023-09-23T18:23:27.078 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:26 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3628504771' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]: dispatch 2023-09-23T18:23:27.158 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 4 1 2023-09-23T18:23:27.768 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:28.034 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[179668]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3628504771' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:23:28.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[179668]: osdmap e678: 8 total, 8 up, 8 in 2023-09-23T18:23:28.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[179668]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[179668]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[179668]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.035 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:28.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[184174]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3628504771' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:23:28.035 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[184174]: osdmap e678: 8 total, 8 up, 8 in 2023-09-23T18:23:28.036 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[184174]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.036 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[184174]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.036 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[184174]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.036 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:27 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:27 smithi099 ceph-mon[167076]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3628504771' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 0, "weight": 1.0}]': finished 2023-09-23T18:23:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:27 smithi099 ceph-mon[167076]: osdmap e678: 8 total, 8 up, 8 in 2023-09-23T18:23:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:27 smithi099 ceph-mon[167076]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:27 smithi099 ceph-mon[167076]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:27 smithi099 ceph-mon[167076]: failed to encode map e678 with expected crc 2023-09-23T18:23:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:27 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:28 smithi099 ceph-mon[167076]: pgmap v1622: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 40 KiB/s, 0 objects/s recovering 2023-09-23T18:23:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:28 smithi099 ceph-mon[167076]: failed to encode map e679 with expected crc 2023-09-23T18:23:29.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:28 smithi099 ceph-mon[167076]: osdmap e679: 8 total, 8 up, 8 in 2023-09-23T18:23:29.078 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:28 smithi099 ceph-mon[167076]: failed to encode map e679 with expected crc 2023-09-23T18:23:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:28 smithi055 ceph-mon[179668]: pgmap v1622: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 40 KiB/s, 0 objects/s recovering 2023-09-23T18:23:29.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:28 smithi055 ceph-mon[179668]: failed to encode map e679 with expected crc 2023-09-23T18:23:29.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:28 smithi055 ceph-mon[179668]: osdmap e679: 8 total, 8 up, 8 in 2023-09-23T18:23:29.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:28 smithi055 ceph-mon[179668]: failed to encode map e679 with expected crc 2023-09-23T18:23:29.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:28 smithi055 ceph-mon[184174]: pgmap v1622: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail; 40 KiB/s, 0 objects/s recovering 2023-09-23T18:23:29.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:28 smithi055 ceph-mon[184174]: failed to encode map e679 with expected crc 2023-09-23T18:23:29.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:28 smithi055 ceph-mon[184174]: osdmap e679: 8 total, 8 up, 8 in 2023-09-23T18:23:29.182 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:28 smithi055 ceph-mon[184174]: failed to encode map e679 with expected crc 2023-09-23T18:23:29.802 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.4 to 1 (10000) 2023-09-23T18:23:30.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2420851660' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:23:30.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2420851660' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:23:30.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2420851660' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:23:30.131 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 4 1 2023-09-23T18:23:30.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:23:30] "GET /metrics HTTP/1.1" 200 34246 "" "Prometheus/2.43.0" 2023-09-23T18:23:30.743 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:30 smithi099 ceph-mon[167076]: pgmap v1624: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:30 smithi099 ceph-mon[167076]: failed to encode map e680 with expected crc 2023-09-23T18:23:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2420851660' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:23:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:30 smithi099 ceph-mon[167076]: osdmap e680: 8 total, 8 up, 8 in 2023-09-23T18:23:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:30 smithi099 ceph-mon[167076]: failed to encode map e680 with expected crc 2023-09-23T18:23:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:30 smithi099 ceph-mon[167076]: failed to encode map e680 with expected crc 2023-09-23T18:23:31.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[179668]: pgmap v1624: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:31.129 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[179668]: failed to encode map e680 with expected crc 2023-09-23T18:23:31.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2420851660' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:23:31.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[179668]: osdmap e680: 8 total, 8 up, 8 in 2023-09-23T18:23:31.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[179668]: failed to encode map e680 with expected crc 2023-09-23T18:23:31.130 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[179668]: failed to encode map e680 with expected crc 2023-09-23T18:23:31.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[184174]: pgmap v1624: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:31.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[184174]: failed to encode map e680 with expected crc 2023-09-23T18:23:31.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2420851660' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:23:31.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[184174]: osdmap e680: 8 total, 8 up, 8 in 2023-09-23T18:23:31.130 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[184174]: failed to encode map e680 with expected crc 2023-09-23T18:23:31.131 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:30 smithi055 ceph-mon[184174]: failed to encode map e680 with expected crc 2023-09-23T18:23:32.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:31 smithi099 ceph-mon[167076]: failed to encode map e680 with expected crc 2023-09-23T18:23:32.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:31 smithi099 ceph-mon[167076]: failed to encode map e680 with expected crc 2023-09-23T18:23:32.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:31 smithi055 ceph-mon[179668]: failed to encode map e680 with expected crc 2023-09-23T18:23:32.181 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:31 smithi055 ceph-mon[179668]: failed to encode map e680 with expected crc 2023-09-23T18:23:32.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:31 smithi055 ceph-mon[184174]: failed to encode map e680 with expected crc 2023-09-23T18:23:32.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:31 smithi055 ceph-mon[184174]: failed to encode map e680 with expected crc 2023-09-23T18:23:32.828 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.4 primary-affinity to 1 (8655362) 2023-09-23T18:23:33.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:32 smithi055 ceph-mon[179668]: pgmap v1626: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:33.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:32 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3927456144' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:23:33.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:32 smithi055 ceph-mon[184174]: pgmap v1626: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:33.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:32 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3927456144' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:23:33.296 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 6 1 2023-09-23T18:23:33.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:32 smithi099 ceph-mon[167076]: pgmap v1626: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:33.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:32 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3927456144' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]: dispatch 2023-09-23T18:23:33.908 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:34.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:33 smithi055 ceph-mon[179668]: failed to encode map e681 with expected crc 2023-09-23T18:23:34.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:33 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3927456144' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:23:34.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:33 smithi055 ceph-mon[179668]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T18:23:34.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:33 smithi055 ceph-mon[179668]: pgmap v1628: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:34.111 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:33 smithi055 ceph-mon[179668]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T18:23:34.111 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:33 smithi055 ceph-mon[184174]: failed to encode map e681 with expected crc 2023-09-23T18:23:34.112 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:33 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3927456144' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:23:34.112 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:33 smithi055 ceph-mon[184174]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T18:23:34.112 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:33 smithi055 ceph-mon[184174]: pgmap v1628: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:34.112 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:33 smithi055 ceph-mon[184174]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T18:23:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:33 smithi099 ceph-mon[167076]: failed to encode map e681 with expected crc 2023-09-23T18:23:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:33 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3927456144' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 4, "weight": 1.0}]': finished 2023-09-23T18:23:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:33 smithi099 ceph-mon[167076]: osdmap e681: 8 total, 8 up, 8 in 2023-09-23T18:23:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:33 smithi099 ceph-mon[167076]: pgmap v1628: 103 pgs: 103 active+clean; 8.6 MiB data, 4.8 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:34.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:33 smithi099 ceph-mon[167076]: osdmap e682: 8 total, 8 up, 8 in 2023-09-23T18:23:35.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:34 smithi055 ceph-mon[179668]: failed to encode map e682 with expected crc 2023-09-23T18:23:35.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:34 smithi055 ceph-mon[179668]: failed to encode map e682 with expected crc 2023-09-23T18:23:35.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:34 smithi055 ceph-mon[184174]: failed to encode map e682 with expected crc 2023-09-23T18:23:35.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:34 smithi055 ceph-mon[184174]: failed to encode map e682 with expected crc 2023-09-23T18:23:35.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:34 smithi099 ceph-mon[167076]: failed to encode map e682 with expected crc 2023-09-23T18:23:35.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:34 smithi099 ceph-mon[167076]: failed to encode map e682 with expected crc 2023-09-23T18:23:35.876 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.6 to 1 (10000) 2023-09-23T18:23:36.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:35 smithi055 ceph-mon[179668]: pgmap v1630: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:36.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:35 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/933435730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:23:36.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:35 smithi055 ceph-mon[184174]: pgmap v1630: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:36.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:35 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/933435730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:23:36.271 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 6 1 2023-09-23T18:23:36.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:35 smithi099 ceph-mon[167076]: pgmap v1630: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 710 GiB / 715 GiB avail 2023-09-23T18:23:36.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:35 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/933435730' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:23:36.883 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:37.116 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:36 smithi055 ceph-mon[184174]: failed to encode map e683 with expected crc 2023-09-23T18:23:37.116 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:36 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/933435730' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:23:37.116 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:36 smithi055 ceph-mon[184174]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T18:23:37.116 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:37.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:36 smithi055 ceph-mon[179668]: failed to encode map e683 with expected crc 2023-09-23T18:23:37.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:36 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/933435730' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:23:37.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:36 smithi055 ceph-mon[179668]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T18:23:37.116 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:37.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:36 smithi099 ceph-mon[167076]: failed to encode map e683 with expected crc 2023-09-23T18:23:37.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:36 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/933435730' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:23:37.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:36 smithi099 ceph-mon[167076]: osdmap e683: 8 total, 8 up, 8 in 2023-09-23T18:23:37.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:38.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:37 smithi055 ceph-mon[179668]: pgmap v1632: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:38.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:37 smithi055 ceph-mon[184174]: pgmap v1632: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:38.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:37 smithi099 ceph-mon[167076]: pgmap v1632: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:38.902 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.6 primary-affinity to 1 (8655362) 2023-09-23T18:23:39.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:38 smithi055 ceph-mon[179668]: 1.1d scrub starts 2023-09-23T18:23:39.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:38 smithi055 ceph-mon[179668]: 1.1d scrub ok 2023-09-23T18:23:39.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:38 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1525764387' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:23:39.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:38 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:23:39.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:38 smithi055 ceph-mon[184174]: 1.1d scrub starts 2023-09-23T18:23:39.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:38 smithi055 ceph-mon[184174]: 1.1d scrub ok 2023-09-23T18:23:39.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:38 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1525764387' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:23:39.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:38 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:23:39.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:38 smithi099 ceph-mon[167076]: 1.1d scrub starts 2023-09-23T18:23:39.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:38 smithi099 ceph-mon[167076]: 1.1d scrub ok 2023-09-23T18:23:39.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:38 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1525764387' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:23:39.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:38 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]: dispatch 2023-09-23T18:23:39.335 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 7 1 2023-09-23T18:23:39.948 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:40.165 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:39 smithi055 ceph-mon[179668]: failed to encode map e684 with expected crc 2023-09-23T18:23:40.166 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:39 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:23:40.166 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:39 smithi055 ceph-mon[179668]: osdmap e684: 8 total, 8 up, 8 in 2023-09-23T18:23:40.166 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:39 smithi055 ceph-mon[179668]: failed to encode map e684 with expected crc 2023-09-23T18:23:40.166 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:39 smithi055 ceph-mon[179668]: pgmap v1634: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 7.3 KiB/s, 0 objects/s recovering 2023-09-23T18:23:40.166 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:39 smithi055 ceph-mon[184174]: failed to encode map e684 with expected crc 2023-09-23T18:23:40.166 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:39 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:23:40.166 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:39 smithi055 ceph-mon[184174]: osdmap e684: 8 total, 8 up, 8 in 2023-09-23T18:23:40.167 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:39 smithi055 ceph-mon[184174]: failed to encode map e684 with expected crc 2023-09-23T18:23:40.167 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:39 smithi055 ceph-mon[184174]: pgmap v1634: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 7.3 KiB/s, 0 objects/s recovering 2023-09-23T18:23:40.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:39 smithi099 ceph-mon[167076]: failed to encode map e684 with expected crc 2023-09-23T18:23:40.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:39 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 6, "weight": 1.0}]': finished 2023-09-23T18:23:40.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:39 smithi099 ceph-mon[167076]: osdmap e684: 8 total, 8 up, 8 in 2023-09-23T18:23:40.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:39 smithi099 ceph-mon[167076]: failed to encode map e684 with expected crc 2023-09-23T18:23:40.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:39 smithi099 ceph-mon[167076]: pgmap v1634: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 3.5 KiB/s rd, 9.2 KiB/s wr, 2 op/s; 7.3 KiB/s, 0 objects/s recovering 2023-09-23T18:23:40.420 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:23:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:23:40] "GET /metrics HTTP/1.1" 200 34217 "" "Prometheus/2.43.0" 2023-09-23T18:23:41.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:40 smithi055 ceph-mon[179668]: failed to encode map e684 with expected crc 2023-09-23T18:23:41.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:40 smithi055 ceph-mon[179668]: failed to encode map e684 with expected crc 2023-09-23T18:23:41.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:40 smithi055 ceph-mon[184174]: failed to encode map e684 with expected crc 2023-09-23T18:23:41.181 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:40 smithi055 ceph-mon[184174]: failed to encode map e684 with expected crc 2023-09-23T18:23:41.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:40 smithi099 ceph-mon[167076]: failed to encode map e684 with expected crc 2023-09-23T18:23:41.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:40 smithi099 ceph-mon[167076]: failed to encode map e684 with expected crc 2023-09-23T18:23:41.949 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.7 to 1 (10000) 2023-09-23T18:23:42.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:41 smithi055 ceph-mon[179668]: pgmap v1635: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.8 KiB/s rd, 7.4 KiB/s wr, 2 op/s; 5.9 KiB/s, 0 objects/s recovering 2023-09-23T18:23:42.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:41 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4039199030' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:23:42.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:41 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:23:42.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:41 smithi055 ceph-mon[184174]: pgmap v1635: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.8 KiB/s rd, 7.4 KiB/s wr, 2 op/s; 5.9 KiB/s, 0 objects/s recovering 2023-09-23T18:23:42.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:41 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4039199030' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:23:42.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:41 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:23:42.326 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:41 smithi099 ceph-mon[167076]: pgmap v1635: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 2.8 KiB/s rd, 7.4 KiB/s wr, 2 op/s; 5.9 KiB/s, 0 objects/s recovering 2023-09-23T18:23:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:41 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4039199030' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:23:42.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:41 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:23:42.412 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 7 1 2023-09-23T18:23:42.937 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:43.201 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:23:43.201 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[179668]: osdmap e685: 8 total, 8 up, 8 in 2023-09-23T18:23:43.202 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[179668]: failed to encode map e685 with expected crc 2023-09-23T18:23:43.202 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:23:43.202 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.202 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.202 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.202 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.202 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:23:43.202 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[184174]: osdmap e685: 8 total, 8 up, 8 in 2023-09-23T18:23:43.202 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[184174]: failed to encode map e685 with expected crc 2023-09-23T18:23:43.202 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:23:43.203 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.203 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.203 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.203 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:42 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:42 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:23:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:42 smithi099 ceph-mon[167076]: osdmap e685: 8 total, 8 up, 8 in 2023-09-23T18:23:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:42 smithi099 ceph-mon[167076]: failed to encode map e685 with expected crc 2023-09-23T18:23:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-09-23T18:23:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:43.327 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:42 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:44.477 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:44 smithi055 ceph-mon[184174]: pgmap v1637: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 641 B/s, 0 objects/s recovering 2023-09-23T18:23:44.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:44 smithi099 ceph-mon[167076]: pgmap v1637: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 641 B/s, 0 objects/s recovering 2023-09-23T18:23:44.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:44 smithi055 ceph-mon[179668]: pgmap v1637: 103 pgs: 103 active+clean; 8.6 MiB data, 4.7 GiB used, 711 GiB / 715 GiB avail; 641 B/s, 0 objects/s recovering 2023-09-23T18:23:45.508 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.7 primary-affinity to 1 (8655362) 2023-09-23T18:23:45.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:45 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2702594393' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:23:45.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:45 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2702594393' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:23:45.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:45 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2702594393' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]: dispatch 2023-09-23T18:23:46.217 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 3 1 2023-09-23T18:23:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:46 smithi099 ceph-mon[167076]: pgmap v1638: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 591 B/s, 0 objects/s recovering 2023-09-23T18:23:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:46 smithi099 ceph-mon[167076]: failed to encode map e686 with expected crc 2023-09-23T18:23:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:46 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2702594393' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:23:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:46 smithi099 ceph-mon[167076]: osdmap e686: 8 total, 8 up, 8 in 2023-09-23T18:23:46.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:46 smithi099 ceph-mon[167076]: failed to encode map e686 with expected crc 2023-09-23T18:23:46.834 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:46.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:46 smithi055 ceph-mon[179668]: pgmap v1638: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 591 B/s, 0 objects/s recovering 2023-09-23T18:23:46.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:46 smithi055 ceph-mon[179668]: failed to encode map e686 with expected crc 2023-09-23T18:23:46.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:46 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2702594393' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:23:46.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:46 smithi055 ceph-mon[179668]: osdmap e686: 8 total, 8 up, 8 in 2023-09-23T18:23:46.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:46 smithi055 ceph-mon[179668]: failed to encode map e686 with expected crc 2023-09-23T18:23:46.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:46 smithi055 ceph-mon[184174]: pgmap v1638: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail; 591 B/s, 0 objects/s recovering 2023-09-23T18:23:46.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:46 smithi055 ceph-mon[184174]: failed to encode map e686 with expected crc 2023-09-23T18:23:46.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:46 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2702594393' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 7, "weight": 1.0}]': finished 2023-09-23T18:23:46.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:46 smithi055 ceph-mon[184174]: osdmap e686: 8 total, 8 up, 8 in 2023-09-23T18:23:46.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:46 smithi055 ceph-mon[184174]: failed to encode map e686 with expected crc 2023-09-23T18:23:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:47 smithi055 ceph-mon[179668]: failed to encode map e687 with expected crc 2023-09-23T18:23:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:47 smithi055 ceph-mon[179668]: osdmap e687: 8 total, 8 up, 8 in 2023-09-23T18:23:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:47 smithi055 ceph-mon[179668]: failed to encode map e687 with expected crc 2023-09-23T18:23:47.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:47 smithi055 ceph-mon[179668]: failed to encode map e687 with expected crc 2023-09-23T18:23:47.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:47 smithi055 ceph-mon[184174]: failed to encode map e687 with expected crc 2023-09-23T18:23:47.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:47 smithi055 ceph-mon[184174]: osdmap e687: 8 total, 8 up, 8 in 2023-09-23T18:23:47.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:47 smithi055 ceph-mon[184174]: failed to encode map e687 with expected crc 2023-09-23T18:23:47.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:47 smithi055 ceph-mon[184174]: failed to encode map e687 with expected crc 2023-09-23T18:23:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:47 smithi099 ceph-mon[167076]: failed to encode map e687 with expected crc 2023-09-23T18:23:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:47 smithi099 ceph-mon[167076]: osdmap e687: 8 total, 8 up, 8 in 2023-09-23T18:23:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:47 smithi099 ceph-mon[167076]: failed to encode map e687 with expected crc 2023-09-23T18:23:47.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:47 smithi099 ceph-mon[167076]: failed to encode map e687 with expected crc 2023-09-23T18:23:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:48 smithi099 ceph-mon[167076]: pgmap v1641: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:48 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:48 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:48 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:48 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:48 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:23:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:48 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:23:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:48 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:48 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2345298070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:48.828 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:48 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[184174]: pgmap v1641: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:23:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:23:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2345298070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[179668]: pgmap v1641: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2345298070' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:48.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:48 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:49.272 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.3 to 1 (10000) 2023-09-23T18:23:49.586 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 3 1 2023-09-23T18:23:50.196 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:50 smithi099 ceph-mon[167076]: pgmap v1642: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:50 smithi099 ceph-mon[167076]: failed to encode map e688 with expected crc 2023-09-23T18:23:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:50 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:23:50.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:50 smithi099 ceph-mon[167076]: osdmap e688: 8 total, 8 up, 8 in 2023-09-23T18:23:50.579 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:50 smithi055 ceph-mon[179668]: pgmap v1642: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:50.579 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:50 smithi055 ceph-mon[179668]: failed to encode map e688 with expected crc 2023-09-23T18:23:50.579 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:50 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:23:50.579 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:50 smithi055 ceph-mon[179668]: osdmap e688: 8 total, 8 up, 8 in 2023-09-23T18:23:50.579 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:23:50 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:23:50] "GET /metrics HTTP/1.1" 200 34217 "" "Prometheus/2.43.0" 2023-09-23T18:23:50.579 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:50 smithi055 ceph-mon[184174]: pgmap v1642: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:50.580 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:50 smithi055 ceph-mon[184174]: failed to encode map e688 with expected crc 2023-09-23T18:23:50.580 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:50 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:23:50.580 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:50 smithi055 ceph-mon[184174]: osdmap e688: 8 total, 8 up, 8 in 2023-09-23T18:23:52.289 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.3 primary-affinity to 1 (8655362) 2023-09-23T18:23:52.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:52 smithi099 ceph-mon[167076]: pgmap v1644: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:52 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:52 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/163682084' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:52.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:52 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:52.591 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:52 smithi055 ceph-mon[184174]: pgmap v1644: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:52.591 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:52 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:52.591 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:52 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/163682084' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:52.591 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:52 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:52.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:52 smithi055 ceph-mon[179668]: pgmap v1644: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:52.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:52 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:23:52.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:52 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/163682084' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:52.591 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:52 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]: dispatch 2023-09-23T18:23:52.666 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 5 1 2023-09-23T18:23:53.274 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:53 smithi099 ceph-mon[167076]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:53 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:23:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:53 smithi099 ceph-mon[167076]: osdmap e689: 8 total, 8 up, 8 in 2023-09-23T18:23:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:53 smithi099 ceph-mon[167076]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:53 smithi099 ceph-mon[167076]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:53 smithi099 ceph-mon[167076]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[179668]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:23:53.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[179668]: osdmap e689: 8 total, 8 up, 8 in 2023-09-23T18:23:53.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[179668]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[179668]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.668 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[179668]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[184174]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 3, "weight": 1.0}]': finished 2023-09-23T18:23:53.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[184174]: osdmap e689: 8 total, 8 up, 8 in 2023-09-23T18:23:53.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[184174]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[184174]: failed to encode map e689 with expected crc 2023-09-23T18:23:53.669 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:53 smithi055 ceph-mon[184174]: failed to encode map e689 with expected crc 2023-09-23T18:23:54.302 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[179668]: pgmap v1646: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:54.302 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[179668]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[179668]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[179668]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[179668]: osdmap e690: 8 total, 8 up, 8 in 2023-09-23T18:23:54.303 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[179668]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:54 smithi099 ceph-mon[167076]: pgmap v1646: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:54 smithi099 ceph-mon[167076]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:54 smithi099 ceph-mon[167076]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:54 smithi099 ceph-mon[167076]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:54 smithi099 ceph-mon[167076]: osdmap e690: 8 total, 8 up, 8 in 2023-09-23T18:23:54.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:54 smithi099 ceph-mon[167076]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.679 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[184174]: pgmap v1646: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:54.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[184174]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[184174]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[184174]: failed to encode map e690 with expected crc 2023-09-23T18:23:54.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[184174]: osdmap e690: 8 total, 8 up, 8 in 2023-09-23T18:23:54.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:54 smithi055 ceph-mon[184174]: failed to encode map e690 with expected crc 2023-09-23T18:23:55.319 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.5 to 1 (10000) 2023-09-23T18:23:55.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:55 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/327987655' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:23:55.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:55 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/327987655' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:23:55.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:55 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/327987655' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:23:55.745 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 5 1 2023-09-23T18:23:56.358 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:56.566 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:56 smithi055 ceph-mon[179668]: pgmap v1648: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:56.566 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:56 smithi055 ceph-mon[179668]: failed to encode map e691 with expected crc 2023-09-23T18:23:56.566 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:56 smithi055 ceph-mon[179668]: failed to encode map e691 with expected crc 2023-09-23T18:23:56.567 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:56 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/327987655' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:23:56.567 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:56 smithi055 ceph-mon[179668]: osdmap e691: 8 total, 8 up, 8 in 2023-09-23T18:23:56.567 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:56 smithi055 ceph-mon[184174]: pgmap v1648: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:56.567 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:56 smithi055 ceph-mon[184174]: failed to encode map e691 with expected crc 2023-09-23T18:23:56.567 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:56 smithi055 ceph-mon[184174]: failed to encode map e691 with expected crc 2023-09-23T18:23:56.567 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:56 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/327987655' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:23:56.567 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:56 smithi055 ceph-mon[184174]: osdmap e691: 8 total, 8 up, 8 in 2023-09-23T18:23:56.576 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:56 smithi099 ceph-mon[167076]: pgmap v1648: 103 pgs: 103 active+clean; 8.6 MiB data, 4.6 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:56.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:56 smithi099 ceph-mon[167076]: failed to encode map e691 with expected crc 2023-09-23T18:23:56.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:56 smithi099 ceph-mon[167076]: failed to encode map e691 with expected crc 2023-09-23T18:23:56.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:56 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/327987655' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:23:56.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:56 smithi099 ceph-mon[167076]: osdmap e691: 8 total, 8 up, 8 in 2023-09-23T18:23:57.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:57 smithi055 ceph-mon[179668]: failed to encode map e691 with expected crc 2023-09-23T18:23:57.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:57 smithi055 ceph-mon[179668]: failed to encode map e691 with expected crc 2023-09-23T18:23:57.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:57 smithi055 ceph-mon[179668]: failed to encode map e691 with expected crc 2023-09-23T18:23:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:57 smithi055 ceph-mon[184174]: failed to encode map e691 with expected crc 2023-09-23T18:23:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:57 smithi055 ceph-mon[184174]: failed to encode map e691 with expected crc 2023-09-23T18:23:57.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:57 smithi055 ceph-mon[184174]: failed to encode map e691 with expected crc 2023-09-23T18:23:57.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:57 smithi099 ceph-mon[167076]: failed to encode map e691 with expected crc 2023-09-23T18:23:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:57 smithi099 ceph-mon[167076]: failed to encode map e691 with expected crc 2023-09-23T18:23:57.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:57 smithi099 ceph-mon[167076]: failed to encode map e691 with expected crc 2023-09-23T18:23:58.369 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.5 primary-affinity to 1 (8655362) 2023-09-23T18:23:58.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:58 smithi055 ceph-mon[179668]: pgmap v1650: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:58.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:58 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1518673872' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:23:58.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:58 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:23:58.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:58 smithi055 ceph-mon[184174]: pgmap v1650: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:58.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:58 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1518673872' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:23:58.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:58 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:23:58.808 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 2 1 2023-09-23T18:23:58.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:58 smithi099 ceph-mon[167076]: pgmap v1650: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:23:58.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:58 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1518673872' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:23:58.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:58 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]: dispatch 2023-09-23T18:23:59.420 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:23:59.618 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:59 smithi055 ceph-mon[184174]: failed to encode map e692 with expected crc 2023-09-23T18:23:59.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:59 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:23:59.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:59 smithi055 ceph-mon[184174]: osdmap e692: 8 total, 8 up, 8 in 2023-09-23T18:23:59.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:59 smithi055 ceph-mon[184174]: failed to encode map e692 with expected crc 2023-09-23T18:23:59.619 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:23:59 smithi055 ceph-mon[184174]: failed to encode map e692 with expected crc 2023-09-23T18:23:59.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:59 smithi055 ceph-mon[179668]: failed to encode map e692 with expected crc 2023-09-23T18:23:59.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:59 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:23:59.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:59 smithi055 ceph-mon[179668]: osdmap e692: 8 total, 8 up, 8 in 2023-09-23T18:23:59.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:59 smithi055 ceph-mon[179668]: failed to encode map e692 with expected crc 2023-09-23T18:23:59.619 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:23:59 smithi055 ceph-mon[179668]: failed to encode map e692 with expected crc 2023-09-23T18:23:59.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:59 smithi099 ceph-mon[167076]: failed to encode map e692 with expected crc 2023-09-23T18:23:59.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:59 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 5, "weight": 1.0}]': finished 2023-09-23T18:23:59.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:59 smithi099 ceph-mon[167076]: osdmap e692: 8 total, 8 up, 8 in 2023-09-23T18:23:59.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:59 smithi099 ceph-mon[167076]: failed to encode map e692 with expected crc 2023-09-23T18:23:59.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:23:59 smithi099 ceph-mon[167076]: failed to encode map e692 with expected crc 2023-09-23T18:24:00.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:00 smithi055 ceph-mon[179668]: failed to encode map e692 with expected crc 2023-09-23T18:24:00.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:00 smithi055 ceph-mon[179668]: pgmap v1652: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:00.681 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:00 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:24:00] "GET /metrics HTTP/1.1" 200 34216 "" "Prometheus/2.43.0" 2023-09-23T18:24:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:00 smithi055 ceph-mon[184174]: failed to encode map e692 with expected crc 2023-09-23T18:24:00.681 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:00 smithi055 ceph-mon[184174]: pgmap v1652: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:00.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:00 smithi099 ceph-mon[167076]: failed to encode map e692 with expected crc 2023-09-23T18:24:01.009 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:00 smithi099 ceph-mon[167076]: pgmap v1652: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:01.375 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.2 to 1 (10000) 2023-09-23T18:24:01.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:01 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1633753068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:24:01.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:01 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1633753068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:24:01.770 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 2 1 2023-09-23T18:24:01.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:01 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1633753068' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:24:02.380 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:02.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[179668]: pgmap v1653: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:02.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[179668]: failed to encode map e693 with expected crc 2023-09-23T18:24:02.635 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1633753068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:24:02.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[179668]: osdmap e693: 8 total, 8 up, 8 in 2023-09-23T18:24:02.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[179668]: failed to encode map e693 with expected crc 2023-09-23T18:24:02.636 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[179668]: failed to encode map e693 with expected crc 2023-09-23T18:24:02.636 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[184174]: pgmap v1653: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:02.636 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[184174]: failed to encode map e693 with expected crc 2023-09-23T18:24:02.636 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1633753068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:24:02.636 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[184174]: osdmap e693: 8 total, 8 up, 8 in 2023-09-23T18:24:02.636 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[184174]: failed to encode map e693 with expected crc 2023-09-23T18:24:02.636 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:02 smithi055 ceph-mon[184174]: failed to encode map e693 with expected crc 2023-09-23T18:24:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:02 smithi099 ceph-mon[167076]: pgmap v1653: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:02 smithi099 ceph-mon[167076]: failed to encode map e693 with expected crc 2023-09-23T18:24:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:02 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1633753068' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:24:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:02 smithi099 ceph-mon[167076]: osdmap e693: 8 total, 8 up, 8 in 2023-09-23T18:24:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:02 smithi099 ceph-mon[167076]: failed to encode map e693 with expected crc 2023-09-23T18:24:02.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:02 smithi099 ceph-mon[167076]: failed to encode map e693 with expected crc 2023-09-23T18:24:03.509 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:03 smithi055 ceph-mon[184174]: failed to encode map e693 with expected crc 2023-09-23T18:24:03.509 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:03 smithi055 ceph-mon[179668]: failed to encode map e693 with expected crc 2023-09-23T18:24:03.826 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:03 smithi099 ceph-mon[167076]: failed to encode map e693 with expected crc 2023-09-23T18:24:04.489 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.2 primary-affinity to 1 (8655362) 2023-09-23T18:24:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:04 smithi099 ceph-mon[167076]: pgmap v1655: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:04.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:04 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2073139828' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:24:04.880 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:04 smithi055 ceph-mon[184174]: pgmap v1655: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:04.880 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:04 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2073139828' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:24:04.880 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:04 smithi055 ceph-mon[179668]: pgmap v1655: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:04.881 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:04 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2073139828' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]: dispatch 2023-09-23T18:24:04.933 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd reweight 1 1 2023-09-23T18:24:05.543 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:05.774 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2073139828' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:24:05.774 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[179668]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T18:24:05.774 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[179668]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.774 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[179668]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.774 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[179668]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.774 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[179668]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.774 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[179668]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.775 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[179668]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.775 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[179668]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T18:24:05.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2073139828' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:24:05.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[184174]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T18:24:05.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[184174]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[184174]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[184174]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[184174]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.775 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[184174]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.776 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[184174]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.776 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:05 smithi055 ceph-mon[184174]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T18:24:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:05 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2073139828' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 2, "weight": 1.0}]': finished 2023-09-23T18:24:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:05 smithi099 ceph-mon[167076]: osdmap e694: 8 total, 8 up, 8 in 2023-09-23T18:24:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:05 smithi099 ceph-mon[167076]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:05 smithi099 ceph-mon[167076]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:05 smithi099 ceph-mon[167076]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:05 smithi099 ceph-mon[167076]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:05 smithi099 ceph-mon[167076]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:05 smithi099 ceph-mon[167076]: failed to encode map e694 with expected crc 2023-09-23T18:24:05.829 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:05 smithi099 ceph-mon[167076]: osdmap e695: 8 total, 8 up, 8 in 2023-09-23T18:24:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:06 smithi099 ceph-mon[167076]: pgmap v1657: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:06 smithi099 ceph-mon[167076]: failed to encode map e695 with expected crc 2023-09-23T18:24:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:06 smithi099 ceph-mon[167076]: failed to encode map e695 with expected crc 2023-09-23T18:24:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:06 smithi099 ceph-mon[167076]: failed to encode map e695 with expected crc 2023-09-23T18:24:06.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:06 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:06 smithi055 ceph-mon[184174]: pgmap v1657: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:06 smithi055 ceph-mon[184174]: failed to encode map e695 with expected crc 2023-09-23T18:24:06.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:06 smithi055 ceph-mon[184174]: failed to encode map e695 with expected crc 2023-09-23T18:24:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:06 smithi055 ceph-mon[184174]: failed to encode map e695 with expected crc 2023-09-23T18:24:06.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:06 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:06.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:06 smithi055 ceph-mon[179668]: pgmap v1657: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:06.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:06 smithi055 ceph-mon[179668]: failed to encode map e695 with expected crc 2023-09-23T18:24:06.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:06 smithi055 ceph-mon[179668]: failed to encode map e695 with expected crc 2023-09-23T18:24:06.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:06 smithi055 ceph-mon[179668]: failed to encode map e695 with expected crc 2023-09-23T18:24:06.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:06 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:07.555 INFO:teuthology.orchestra.run.smithi055.stderr:reweighted osd.1 to 1 (10000) 2023-09-23T18:24:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:07 smithi099 ceph-mon[167076]: failed to encode map e695 with expected crc 2023-09-23T18:24:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:07 smithi099 ceph-mon[167076]: failed to encode map e695 with expected crc 2023-09-23T18:24:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:07 smithi099 ceph-mon[167076]: failed to encode map e695 with expected crc 2023-09-23T18:24:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]: dispatch 2023-09-23T18:24:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:07 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T18:24:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:07 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/4205565409' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:07.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:07 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:07.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[179668]: failed to encode map e695 with expected crc 2023-09-23T18:24:07.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[179668]: failed to encode map e695 with expected crc 2023-09-23T18:24:07.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[179668]: failed to encode map e695 with expected crc 2023-09-23T18:24:07.897 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]: dispatch 2023-09-23T18:24:07.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T18:24:07.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/4205565409' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:07.898 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:07.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[184174]: failed to encode map e695 with expected crc 2023-09-23T18:24:07.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[184174]: failed to encode map e695 with expected crc 2023-09-23T18:24:07.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[184174]: failed to encode map e695 with expected crc 2023-09-23T18:24:07.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]: dispatch 2023-09-23T18:24:07.898 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]: dispatch 2023-09-23T18:24:07.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/4205565409' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:07.899 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:07 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:07.994 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd primary-affinity 1 1 2023-09-23T18:24:08.606 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:08.824 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:08.824 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.824 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.824 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.824 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.824 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2023-09-23T18:24:08.825 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T18:24:08.825 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:24:08.825 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: osdmap e696: 8 total, 8 up, 8 in 2023-09-23T18:24:08.825 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.825 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[184174]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.825 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:08.825 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.825 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.825 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.826 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.826 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2023-09-23T18:24:08.826 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T18:24:08.826 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:24:08.826 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: osdmap e696: 8 total, 8 up, 8 in 2023-09-23T18:24:08.826 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.826 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:08 smithi055 ceph-mon[179668]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: pgmap v1659: 103 pgs: 103 active+clean; 8.6 MiB data, 4.5 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.17", "id": [3, 2]}]': finished 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.1b", "id": [1, 2]}]': finished 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd reweight", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: osdmap e696: 8 total, 8 up, 8 in 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: failed to encode map e696 with expected crc 2023-09-23T18:24:08.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:08 smithi099 ceph-mon[167076]: failed to encode map e696 with expected crc 2023-09-23T18:24:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:09 smithi099 ceph-mon[167076]: failed to encode map e697 with expected crc 2023-09-23T18:24:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:09 smithi099 ceph-mon[167076]: osdmap e697: 8 total, 8 up, 8 in 2023-09-23T18:24:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:09 smithi099 ceph-mon[167076]: failed to encode map e697 with expected crc 2023-09-23T18:24:09.827 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:09 smithi099 ceph-mon[167076]: failed to encode map e697 with expected crc 2023-09-23T18:24:09.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:09 smithi055 ceph-mon[179668]: failed to encode map e697 with expected crc 2023-09-23T18:24:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:09 smithi055 ceph-mon[179668]: osdmap e697: 8 total, 8 up, 8 in 2023-09-23T18:24:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:09 smithi055 ceph-mon[179668]: failed to encode map e697 with expected crc 2023-09-23T18:24:09.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:09 smithi055 ceph-mon[179668]: failed to encode map e697 with expected crc 2023-09-23T18:24:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:09 smithi055 ceph-mon[184174]: failed to encode map e697 with expected crc 2023-09-23T18:24:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:09 smithi055 ceph-mon[184174]: osdmap e697: 8 total, 8 up, 8 in 2023-09-23T18:24:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:09 smithi055 ceph-mon[184174]: failed to encode map e697 with expected crc 2023-09-23T18:24:09.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:09 smithi055 ceph-mon[184174]: failed to encode map e697 with expected crc 2023-09-23T18:24:10.601 INFO:teuthology.orchestra.run.smithi055.stderr:set osd.1 primary-affinity to 1 (8655362) 2023-09-23T18:24:10.629 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:10 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:24:10] "GET /metrics HTTP/1.1" 200 34213 "" "Prometheus/2.43.0" 2023-09-23T18:24:10.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:10 smithi055 ceph-mon[179668]: pgmap v1662: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s, 12 objects/s recovering 2023-09-23T18:24:10.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:10 smithi055 ceph-mon[179668]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:24:10.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:10 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2920830205' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:10.630 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:10 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:10.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:10 smithi055 ceph-mon[184174]: pgmap v1662: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s, 12 objects/s recovering 2023-09-23T18:24:10.630 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:10 smithi055 ceph-mon[184174]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:24:10.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:10 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2920830205' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:10.631 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:10 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:10.960 INFO:tasks.thrashosds.thrasher:joining the do_sighup greenlet 2023-09-23T18:24:10.960 INFO:tasks.thrashosds.thrasher:joining the do_optrack_toggle greenlet 2023-09-23T18:24:10.960 INFO:tasks.thrashosds.thrasher:joining the do_dump_ops greenlet 2023-09-23T18:24:10.960 INFO:tasks.thrashosds.thrasher:joining the do_noscrub_toggle greenlet 2023-09-23T18:24:10.960 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-09-23T18:24:10.960 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T18:24:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:10 smithi099 ceph-mon[167076]: pgmap v1662: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.5 MiB/s, 12 objects/s recovering 2023-09-23T18:24:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:10 smithi099 ceph-mon[167076]: Health check failed: Reduced data availability: 1 pg peering (PG_AVAILABILITY) 2023-09-23T18:24:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:10 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2920830205' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:11.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:10 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]: dispatch 2023-09-23T18:24:11.573 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:11.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[179668]: failed to encode map e698 with expected crc 2023-09-23T18:24:11.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[179668]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:24:11.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[179668]: osdmap e698: 8 total, 8 up, 8 in 2023-09-23T18:24:11.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[179668]: failed to encode map e698 with expected crc 2023-09-23T18:24:11.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[179668]: failed to encode map e698 with expected crc 2023-09-23T18:24:11.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[179668]: failed to encode map e698 with expected crc 2023-09-23T18:24:11.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[179668]: osdmap e699: 8 total, 8 up, 8 in 2023-09-23T18:24:11.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[184174]: failed to encode map e698 with expected crc 2023-09-23T18:24:11.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[184174]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:24:11.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[184174]: osdmap e698: 8 total, 8 up, 8 in 2023-09-23T18:24:11.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[184174]: failed to encode map e698 with expected crc 2023-09-23T18:24:11.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[184174]: failed to encode map e698 with expected crc 2023-09-23T18:24:11.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[184174]: failed to encode map e698 with expected crc 2023-09-23T18:24:11.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:11 smithi055 ceph-mon[184174]: osdmap e699: 8 total, 8 up, 8 in 2023-09-23T18:24:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:11 smithi099 ceph-mon[167076]: failed to encode map e698 with expected crc 2023-09-23T18:24:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:11 smithi099 ceph-mon[167076]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd primary-affinity", "id": 1, "weight": 1.0}]': finished 2023-09-23T18:24:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:11 smithi099 ceph-mon[167076]: osdmap e698: 8 total, 8 up, 8 in 2023-09-23T18:24:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:11 smithi099 ceph-mon[167076]: failed to encode map e698 with expected crc 2023-09-23T18:24:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:11 smithi099 ceph-mon[167076]: failed to encode map e698 with expected crc 2023-09-23T18:24:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:11 smithi099 ceph-mon[167076]: failed to encode map e698 with expected crc 2023-09-23T18:24:12.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:11 smithi099 ceph-mon[167076]: osdmap e699: 8 total, 8 up, 8 in 2023-09-23T18:24:12.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:12 smithi055 ceph-mon[179668]: pgmap v1664: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s, 10 objects/s recovering 2023-09-23T18:24:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:12 smithi055 ceph-mon[179668]: failed to encode map e699 with expected crc 2023-09-23T18:24:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:12 smithi055 ceph-mon[179668]: failed to encode map e699 with expected crc 2023-09-23T18:24:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:12 smithi055 ceph-mon[179668]: failed to encode map e699 with expected crc 2023-09-23T18:24:12.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:12 smithi055 ceph-mon[179668]: failed to encode map e699 with expected crc 2023-09-23T18:24:12.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:12 smithi055 ceph-mon[184174]: pgmap v1664: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s, 10 objects/s recovering 2023-09-23T18:24:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:12 smithi055 ceph-mon[184174]: failed to encode map e699 with expected crc 2023-09-23T18:24:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:12 smithi055 ceph-mon[184174]: failed to encode map e699 with expected crc 2023-09-23T18:24:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:12 smithi055 ceph-mon[184174]: failed to encode map e699 with expected crc 2023-09-23T18:24:12.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:12 smithi055 ceph-mon[184174]: failed to encode map e699 with expected crc 2023-09-23T18:24:12.963 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:24:12.964 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":699,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","created":"2023-09-23T17:40:13.381005+0000","modified":"2023-09-23T18:24:11.576664+0000","last_up_change":"2023-09-23T18:17:19.702897+0000","last_in_change":"2023-09-23T18:23:14.625785+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":28,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T17:43:05.158393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"324","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"197","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":1.809999942779541,"score_stable":1.809999942779541,"optimal_score":1,"raw_score_acting":1.809999942779541,"raw_score_stable":1.809999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T17:45:10.969164+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"652","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":652,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T17:45:49.622328+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T17:45:50.526600+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"99e7cae6-389e-461a-8a86-a5789673908b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":89,"up_from":93,"up_thru":678,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6803","nonce":3605839049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6805","nonce":3605839049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6809","nonce":3605839049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6807","nonce":3605839049}]},"public_addr":"172.21.15.55:6803/3605839049","cluster_addr":"172.21.15.55:6805/3605839049","heartbeat_back_addr":"172.21.15.55:6809/3605839049","heartbeat_front_addr":"172.21.15.55:6807/3605839049","state":["exists","up"]},{"osd":1,"uuid":"bfb0da01-7a06-430b-b973-209f22ac30d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":98,"last_clean_end":497,"up_from":545,"up_thru":698,"down_at":498,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":3845695292},{"type":"v1","addr":"172.21.15.55:6811","nonce":3845695292}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":3845695292},{"type":"v1","addr":"172.21.15.55:6813","nonce":3845695292}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":3845695292},{"type":"v1","addr":"172.21.15.55:6817","nonce":3845695292}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":3845695292},{"type":"v1","addr":"172.21.15.55:6815","nonce":3845695292}]},"public_addr":"172.21.15.55:6811/3845695292","cluster_addr":"172.21.15.55:6813/3845695292","heartbeat_back_addr":"172.21.15.55:6817/3845695292","heartbeat_front_addr":"172.21.15.55:6815/3845695292","state":["exists","up"]},{"osd":2,"uuid":"f766d604-db54-4a09-9993-fe1a3ce0048e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":103,"last_clean_end":461,"up_from":482,"up_thru":696,"down_at":462,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1022522207},{"type":"v1","addr":"172.21.15.55:6819","nonce":1022522207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1022522207},{"type":"v1","addr":"172.21.15.55:6821","nonce":1022522207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1022522207},{"type":"v1","addr":"172.21.15.55:6825","nonce":1022522207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1022522207},{"type":"v1","addr":"172.21.15.55:6823","nonce":1022522207}]},"public_addr":"172.21.15.55:6819/1022522207","cluster_addr":"172.21.15.55:6821/1022522207","heartbeat_back_addr":"172.21.15.55:6825/1022522207","heartbeat_front_addr":"172.21.15.55:6823/1022522207","state":["exists","up"]},{"osd":3,"uuid":"9f7bc62d-c839-4d05-89c7-98e5abd0a121","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":108,"last_clean_end":141,"up_from":237,"up_thru":689,"down_at":142,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":958494807},{"type":"v1","addr":"172.21.15.55:6827","nonce":958494807}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":958494807},{"type":"v1","addr":"172.21.15.55:6829","nonce":958494807}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":958494807},{"type":"v1","addr":"172.21.15.55:6833","nonce":958494807}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":958494807},{"type":"v1","addr":"172.21.15.55:6831","nonce":958494807}]},"public_addr":"172.21.15.55:6827/958494807","cluster_addr":"172.21.15.55:6829/958494807","heartbeat_back_addr":"172.21.15.55:6833/958494807","heartbeat_front_addr":"172.21.15.55:6831/958494807","state":["exists","up"]},{"osd":4,"uuid":"a9c0950d-2618-4a47-ad13-035e2bd227f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":696,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6801","nonce":2570729307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6803","nonce":2570729307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6807","nonce":2570729307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6805","nonce":2570729307}]},"public_addr":"172.21.15.99:6801/2570729307","cluster_addr":"172.21.15.99:6803/2570729307","heartbeat_back_addr":"172.21.15.99:6807/2570729307","heartbeat_front_addr":"172.21.15.99:6805/2570729307","state":["exists","up"]},{"osd":5,"uuid":"64ac7fc4-adc3-4e7f-8512-a458d0092f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":366,"up_from":420,"up_thru":675,"down_at":367,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":4029064001},{"type":"v1","addr":"172.21.15.99:6809","nonce":4029064001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":4029064001},{"type":"v1","addr":"172.21.15.99:6811","nonce":4029064001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":4029064001},{"type":"v1","addr":"172.21.15.99:6815","nonce":4029064001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":4029064001},{"type":"v1","addr":"172.21.15.99:6813","nonce":4029064001}]},"public_addr":"172.21.15.99:6809/4029064001","cluster_addr":"172.21.15.99:6811/4029064001","heartbeat_back_addr":"172.21.15.99:6815/4029064001","heartbeat_front_addr":"172.21.15.99:6813/4029064001","state":["exists","up"]},{"osd":6,"uuid":"9be63962-c98b-4df9-8781-270556e51119","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":45,"last_clean_end":113,"up_from":163,"up_thru":675,"down_at":114,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":370560019},{"type":"v1","addr":"172.21.15.99:6817","nonce":370560019}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":370560019},{"type":"v1","addr":"172.21.15.99:6819","nonce":370560019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":370560019},{"type":"v1","addr":"172.21.15.99:6823","nonce":370560019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":370560019},{"type":"v1","addr":"172.21.15.99:6821","nonce":370560019}]},"public_addr":"172.21.15.99:6817/370560019","cluster_addr":"172.21.15.99:6819/370560019","heartbeat_back_addr":"172.21.15.99:6823/370560019","heartbeat_front_addr":"172.21.15.99:6821/370560019","state":["exists","up"]},{"osd":7,"uuid":"ea011d8c-d9f8-4619-9ac0-575813be808a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":53,"last_clean_end":149,"up_from":214,"up_thru":686,"down_at":150,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":179726820},{"type":"v1","addr":"172.21.15.99:6825","nonce":179726820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":179726820},{"type":"v1","addr":"172.21.15.99:6827","nonce":179726820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":179726820},{"type":"v1","addr":"172.21.15.99:6831","nonce":179726820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":179726820},{"type":"v1","addr":"172.21.15.99:6829","nonce":179726820}]},"public_addr":"172.21.15.99:6825/179726820","cluster_addr":"172.21.15.99:6827/179726820","heartbeat_back_addr":"172.21.15.99:6831/179726820","heartbeat_front_addr":"172.21.15.99:6829/179726820","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:22.099373+0000","dead_epoch":89},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:42.018683+0000","dead_epoch":497},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:01.856304+0000","dead_epoch":461},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:22.152206+0000","dead_epoch":141},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:41.385241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:01.725749+0000","dead_epoch":366},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:21.176273+0000","dead_epoch":112},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:41.516585+0000","dead_epoch":149}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.12","mappings":[{"from":5,"to":3}]},{"pgid":"4.a","mappings":[{"from":1,"to":4}]},{"pgid":"4.17","mappings":[{"from":3,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":3}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/4290529019":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/1663054712":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6800/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/658967358":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/573100440":"2023-09-24T17:47:42.374356+0000","172.21.15.55:6801/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/3781541742":"2023-09-24T17:49:06.276090+0000","172.21.15.99:0/1490815752":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6800/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6800/32209804":"2023-09-24T17:47:42.374356+0000","172.21.15.55:0/591596032":"2023-09-24T17:41:12.624150+0000","172.21.15.99:0/3564577630":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/1826983981":"2023-09-24T17:40:48.304146+0000","172.21.15.99:6835/2823676616":"2023-09-24T17:48:26.829766+0000","172.21.15.99:0/2950334811":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/880894918":"2023-09-24T17:47:42.374356+0000","172.21.15.55:0/3452781072":"2023-09-24T17:49:06.276090+0000","172.21.15.99:0/2680034432":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6801/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1355725764":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/2064251389":"2023-09-24T17:47:42.374356+0000","172.21.15.55:0/880528732":"2023-09-24T17:40:48.304146+0000","172.21.15.99:0/3129059336":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6801/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/3953275940":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/3757490612":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/2173356416":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1552377843":"2023-09-24T17:49:06.276090+0000","172.21.15.55:6801/2816080264":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/238250704":"2023-09-24T17:41:12.624150+0000","172.21.15.99:6834/2823676616":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6800/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/2880364962":"2023-09-24T17:47:42.374356+0000","172.21.15.55:6801/32209804":"2023-09-24T17:47:42.374356+0000","172.21.15.55:0/2453190559":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/218576037":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6800/2816080264":"2023-09-24T17:49:06.276090+0000","172.21.15.99:0/1538541481":"2023-09-24T17:48:26.829766+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T18:24:13.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:12 smithi099 ceph-mon[167076]: pgmap v1664: 103 pgs: 1 peering, 102 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s, 10 objects/s recovering 2023-09-23T18:24:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:12 smithi099 ceph-mon[167076]: failed to encode map e699 with expected crc 2023-09-23T18:24:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:12 smithi099 ceph-mon[167076]: failed to encode map e699 with expected crc 2023-09-23T18:24:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:12 smithi099 ceph-mon[167076]: failed to encode map e699 with expected crc 2023-09-23T18:24:13.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:12 smithi099 ceph-mon[167076]: failed to encode map e699 with expected crc 2023-09-23T18:24:13.342 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-09-23T18:24:13.342 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd dump --format=json 2023-09-23T18:24:13.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:13 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/815657641' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:24:13.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:13 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/815657641' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:24:13.952 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:14.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:13 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/815657641' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:24:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:14 smithi055 ceph-mon[184174]: pgmap v1666: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s, 10 objects/s recovering 2023-09-23T18:24:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:14 smithi055 ceph-mon[184174]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:24:14.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:14 smithi055 ceph-mon[184174]: Cluster is now healthy 2023-09-23T18:24:14.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:14 smithi055 ceph-mon[179668]: pgmap v1666: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s, 10 objects/s recovering 2023-09-23T18:24:14.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:14 smithi055 ceph-mon[179668]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:24:14.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:14 smithi055 ceph-mon[179668]: Cluster is now healthy 2023-09-23T18:24:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:14 smithi099 ceph-mon[167076]: pgmap v1666: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail; 2.1 MiB/s, 10 objects/s recovering 2023-09-23T18:24:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:14 smithi099 ceph-mon[167076]: Health check cleared: PG_AVAILABILITY (was: Reduced data availability: 1 pg peering) 2023-09-23T18:24:15.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:14 smithi099 ceph-mon[167076]: Cluster is now healthy 2023-09-23T18:24:15.372 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:24:15.372 INFO:teuthology.orchestra.run.smithi055.stdout:{"epoch":699,"fsid":"2b70c030-5a38-11ee-8db4-212e2dc638e7","created":"2023-09-23T17:40:13.381005+0000","modified":"2023-09-23T18:24:11.576664+0000","last_up_change":"2023-09-23T18:17:19.702897+0000","last_in_change":"2023-09-23T18:23:14.625785+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":28,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":11,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","allow_crimson":false,"pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-09-23T17:43:05.158393+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":31,"pg_placement_num":31,"pg_placement_num_target":31,"pg_num_target":31,"pg_num_pending":31,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"324","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"197","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}},"read_balance":{"score_acting":1.809999942779541,"score_stable":1.809999942779541,"optimal_score":1,"raw_score_acting":1.809999942779541,"raw_score_stable":1.809999942779541,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":2,"pool_name":"rbd","create_time":"2023-09-23T17:45:10.969164+0000","flags":8193,"flags_names":"hashpspool,selfmanaged_snaps","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"652","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":27,"snap_epoch":652,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}},"read_balance":{"score_acting":2,"score_stable":2,"optimal_score":1,"raw_score_acting":2,"raw_score_stable":2,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-09-23T17:45:49.622328+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"66","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}},"read_balance":{"score_acting":1.5,"score_stable":1.5,"optimal_score":1,"raw_score_acting":1.5,"raw_score_stable":1.5,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-09-23T17:45:50.526600+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"67","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}},"read_balance":{"score_acting":2.25,"score_stable":2.25,"optimal_score":1,"raw_score_acting":2.25,"raw_score_stable":2.25,"primary_affinity_weighted":1,"average_primary_affinity":1,"average_primary_affinity_weighted":1}}],"osds":[{"osd":0,"uuid":"99e7cae6-389e-461a-8a86-a5789673908b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":8,"last_clean_end":89,"up_from":93,"up_thru":678,"down_at":90,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6802","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6803","nonce":3605839049}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6804","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6805","nonce":3605839049}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6808","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6809","nonce":3605839049}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6806","nonce":3605839049},{"type":"v1","addr":"172.21.15.55:6807","nonce":3605839049}]},"public_addr":"172.21.15.55:6803/3605839049","cluster_addr":"172.21.15.55:6805/3605839049","heartbeat_back_addr":"172.21.15.55:6809/3605839049","heartbeat_front_addr":"172.21.15.55:6807/3605839049","state":["exists","up"]},{"osd":1,"uuid":"bfb0da01-7a06-430b-b973-209f22ac30d4","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":98,"last_clean_end":497,"up_from":545,"up_thru":698,"down_at":498,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6810","nonce":3845695292},{"type":"v1","addr":"172.21.15.55:6811","nonce":3845695292}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6812","nonce":3845695292},{"type":"v1","addr":"172.21.15.55:6813","nonce":3845695292}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6816","nonce":3845695292},{"type":"v1","addr":"172.21.15.55:6817","nonce":3845695292}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6814","nonce":3845695292},{"type":"v1","addr":"172.21.15.55:6815","nonce":3845695292}]},"public_addr":"172.21.15.55:6811/3845695292","cluster_addr":"172.21.15.55:6813/3845695292","heartbeat_back_addr":"172.21.15.55:6817/3845695292","heartbeat_front_addr":"172.21.15.55:6815/3845695292","state":["exists","up"]},{"osd":2,"uuid":"f766d604-db54-4a09-9993-fe1a3ce0048e","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":103,"last_clean_end":461,"up_from":482,"up_thru":696,"down_at":462,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6818","nonce":1022522207},{"type":"v1","addr":"172.21.15.55:6819","nonce":1022522207}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6820","nonce":1022522207},{"type":"v1","addr":"172.21.15.55:6821","nonce":1022522207}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6824","nonce":1022522207},{"type":"v1","addr":"172.21.15.55:6825","nonce":1022522207}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6822","nonce":1022522207},{"type":"v1","addr":"172.21.15.55:6823","nonce":1022522207}]},"public_addr":"172.21.15.55:6819/1022522207","cluster_addr":"172.21.15.55:6821/1022522207","heartbeat_back_addr":"172.21.15.55:6825/1022522207","heartbeat_front_addr":"172.21.15.55:6823/1022522207","state":["exists","up"]},{"osd":3,"uuid":"9f7bc62d-c839-4d05-89c7-98e5abd0a121","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":108,"last_clean_end":141,"up_from":237,"up_thru":689,"down_at":142,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6826","nonce":958494807},{"type":"v1","addr":"172.21.15.55:6827","nonce":958494807}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6828","nonce":958494807},{"type":"v1","addr":"172.21.15.55:6829","nonce":958494807}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6832","nonce":958494807},{"type":"v1","addr":"172.21.15.55:6833","nonce":958494807}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.55:6830","nonce":958494807},{"type":"v1","addr":"172.21.15.55:6831","nonce":958494807}]},"public_addr":"172.21.15.55:6827/958494807","cluster_addr":"172.21.15.55:6829/958494807","heartbeat_back_addr":"172.21.15.55:6833/958494807","heartbeat_front_addr":"172.21.15.55:6831/958494807","state":["exists","up"]},{"osd":4,"uuid":"a9c0950d-2618-4a47-ad13-035e2bd227f1","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":696,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6800","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6801","nonce":2570729307}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6802","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6803","nonce":2570729307}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6806","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6807","nonce":2570729307}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6804","nonce":2570729307},{"type":"v1","addr":"172.21.15.99:6805","nonce":2570729307}]},"public_addr":"172.21.15.99:6801/2570729307","cluster_addr":"172.21.15.99:6803/2570729307","heartbeat_back_addr":"172.21.15.99:6807/2570729307","heartbeat_front_addr":"172.21.15.99:6805/2570729307","state":["exists","up"]},{"osd":5,"uuid":"64ac7fc4-adc3-4e7f-8512-a458d0092f49","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":38,"last_clean_end":366,"up_from":420,"up_thru":675,"down_at":367,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6808","nonce":4029064001},{"type":"v1","addr":"172.21.15.99:6809","nonce":4029064001}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6810","nonce":4029064001},{"type":"v1","addr":"172.21.15.99:6811","nonce":4029064001}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6814","nonce":4029064001},{"type":"v1","addr":"172.21.15.99:6815","nonce":4029064001}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6812","nonce":4029064001},{"type":"v1","addr":"172.21.15.99:6813","nonce":4029064001}]},"public_addr":"172.21.15.99:6809/4029064001","cluster_addr":"172.21.15.99:6811/4029064001","heartbeat_back_addr":"172.21.15.99:6815/4029064001","heartbeat_front_addr":"172.21.15.99:6813/4029064001","state":["exists","up"]},{"osd":6,"uuid":"9be63962-c98b-4df9-8781-270556e51119","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":45,"last_clean_end":113,"up_from":163,"up_thru":675,"down_at":114,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6816","nonce":370560019},{"type":"v1","addr":"172.21.15.99:6817","nonce":370560019}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6818","nonce":370560019},{"type":"v1","addr":"172.21.15.99:6819","nonce":370560019}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6822","nonce":370560019},{"type":"v1","addr":"172.21.15.99:6823","nonce":370560019}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6820","nonce":370560019},{"type":"v1","addr":"172.21.15.99:6821","nonce":370560019}]},"public_addr":"172.21.15.99:6817/370560019","cluster_addr":"172.21.15.99:6819/370560019","heartbeat_back_addr":"172.21.15.99:6823/370560019","heartbeat_front_addr":"172.21.15.99:6821/370560019","state":["exists","up"]},{"osd":7,"uuid":"ea011d8c-d9f8-4619-9ac0-575813be808a","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":53,"last_clean_end":149,"up_from":214,"up_thru":686,"down_at":150,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6824","nonce":179726820},{"type":"v1","addr":"172.21.15.99:6825","nonce":179726820}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6826","nonce":179726820},{"type":"v1","addr":"172.21.15.99:6827","nonce":179726820}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6830","nonce":179726820},{"type":"v1","addr":"172.21.15.99:6831","nonce":179726820}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.99:6828","nonce":179726820},{"type":"v1","addr":"172.21.15.99:6829","nonce":179726820}]},"public_addr":"172.21.15.99:6825/179726820","cluster_addr":"172.21.15.99:6827/179726820","heartbeat_back_addr":"172.21.15.99:6831/179726820","heartbeat_front_addr":"172.21.15.99:6829/179726820","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:22.099373+0000","dead_epoch":89},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:42:42.018683+0000","dead_epoch":497},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:01.856304+0000","dead_epoch":461},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138322906710015,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:22.152206+0000","dead_epoch":141},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:43:41.385241+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:01.725749+0000","dead_epoch":366},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:21.176273+0000","dead_epoch":112},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-09-23T17:44:41.516585+0000","dead_epoch":149}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"1.12","mappings":[{"from":5,"to":3}]},{"pgid":"4.a","mappings":[{"from":1,"to":4}]},{"pgid":"4.17","mappings":[{"from":3,"to":2}]},{"pgid":"4.18","mappings":[{"from":1,"to":3}]},{"pgid":"4.1b","mappings":[{"from":1,"to":2}]}],"pg_upmap_primaries":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.55:0/1552377843":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/1663054712":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/4290529019":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/2173356416":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.99:0/3564577630":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/591596032":"2023-09-24T17:41:12.624150+0000","172.21.15.55:0/1826983981":"2023-09-24T17:40:48.304146+0000","172.21.15.99:6835/2823676616":"2023-09-24T17:48:26.829766+0000","172.21.15.99:0/2950334811":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/880894918":"2023-09-24T17:47:42.374356+0000","172.21.15.99:0/1490815752":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/1355725764":"2023-09-24T17:49:06.276090+0000","172.21.15.55:6801/106254940":"2023-09-24T17:41:12.624150+0000","172.21.15.55:6800/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:6800/32209804":"2023-09-24T17:47:42.374356+0000","172.21.15.99:0/2680034432":"2023-09-24T17:48:26.829766+0000","172.21.15.55:0/3452781072":"2023-09-24T17:49:06.276090+0000","172.21.15.99:0/3129059336":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6801/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/3953275940":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/3757490612":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/658967358":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/573100440":"2023-09-24T17:47:42.374356+0000","172.21.15.55:6800/470581910":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/2880364962":"2023-09-24T17:47:42.374356+0000","172.21.15.55:0/218576037":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/238250704":"2023-09-24T17:41:12.624150+0000","172.21.15.99:6834/2823676616":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6801/2816080264":"2023-09-24T17:49:06.276090+0000","172.21.15.55:0/880528732":"2023-09-24T17:40:48.304146+0000","172.21.15.55:0/2064251389":"2023-09-24T17:47:42.374356+0000","172.21.15.55:6801/2971878511":"2023-09-24T17:45:12.276860+0000","172.21.15.55:0/3781541742":"2023-09-24T17:49:06.276090+0000","172.21.15.99:0/1538541481":"2023-09-24T17:48:26.829766+0000","172.21.15.55:6800/2816080264":"2023-09-24T17:49:06.276090+0000","172.21.15.55:6801/32209804":"2023-09-24T17:47:42.374356+0000","172.21.15.55:0/2453190559":"2023-09-24T17:49:06.276090+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[{"pool":11,"snaps":[{"begin":2,"length":1}]}],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-09-23T18:24:15.680 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:15 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/791876378' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:24:15.680 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:15 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/791876378' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:24:15.813 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.0 flush_pg_stats 2023-09-23T18:24:15.813 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.1 flush_pg_stats 2023-09-23T18:24:15.814 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.2 flush_pg_stats 2023-09-23T18:24:15.814 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.3 flush_pg_stats 2023-09-23T18:24:15.815 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.4 flush_pg_stats 2023-09-23T18:24:15.815 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.5 flush_pg_stats 2023-09-23T18:24:15.816 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.6 flush_pg_stats 2023-09-23T18:24:15.816 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph tell osd.7 flush_pg_stats 2023-09-23T18:24:16.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:15 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/791876378' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-09-23T18:24:16.534 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:16.620 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:16.633 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:16.656 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:16.747 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:16.748 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:16.874 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:16.879 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:16.915 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:16 smithi055 ceph-mon[184174]: pgmap v1667: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:16.915 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:16 smithi055 ceph-mon[179668]: pgmap v1667: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:17.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:16 smithi099 ceph-mon[167076]: pgmap v1667: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:18.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:18 smithi055 ceph-mon[179668]: pgmap v1668: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:18.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:18 smithi055 ceph-mon[184174]: pgmap v1668: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:19.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:18 smithi099 ceph-mon[167076]: pgmap v1668: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:20.572 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:20 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:24:20] "GET /metrics HTTP/1.1" 200 34213 "" "Prometheus/2.43.0" 2023-09-23T18:24:20.876 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:20 smithi055 ceph-mon[179668]: pgmap v1669: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:20.877 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:20 smithi055 ceph-mon[184174]: pgmap v1669: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:21.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:20 smithi099 ceph-mon[167076]: pgmap v1669: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:21.478 INFO:teuthology.orchestra.run.smithi055.stdout:1017907249417 2023-09-23T18:24:21.478 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.3 2023-09-23T18:24:21.678 INFO:teuthology.orchestra.run.smithi055.stdout:399431958903 2023-09-23T18:24:21.679 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.0 2023-09-23T18:24:21.891 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:21 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:21.891 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:21 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:22.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:21 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:22.652 INFO:teuthology.orchestra.run.smithi055.stdout:700079669558 2023-09-23T18:24:22.652 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.6 2023-09-23T18:24:22.766 INFO:teuthology.orchestra.run.smithi055.stdout:128849019367 2023-09-23T18:24:22.766 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.4 2023-09-23T18:24:22.896 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:22 smithi055 ceph-mon[179668]: pgmap v1670: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:22.896 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:22 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:24:22.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:22 smithi055 ceph-mon[184174]: pgmap v1670: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:22.896 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:22 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:24:23.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:22 smithi099 ceph-mon[167076]: pgmap v1670: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:23.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:22 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' 2023-09-23T18:24:23.610 INFO:teuthology.orchestra.run.smithi055.stdout:919123001621 2023-09-23T18:24:23.610 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.7 2023-09-23T18:24:23.673 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:23.995 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:24.852 INFO:teuthology.orchestra.run.smithi055.stdout:2070174236789 2023-09-23T18:24:24.853 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.2 2023-09-23T18:24:24.854 INFO:teuthology.orchestra.run.smithi055.stdout:1803886264464 2023-09-23T18:24:24.854 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.5 2023-09-23T18:24:24.882 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:24 smithi055 ceph-mon[179668]: pgmap v1671: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:24.883 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:24 smithi055 ceph-mon[184174]: pgmap v1671: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:24.922 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:25.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:24 smithi099 ceph-mon[167076]: pgmap v1671: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:25.281 INFO:teuthology.orchestra.run.smithi055.stdout:2340757176406 2023-09-23T18:24:25.282 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph osd last-stat-seq osd.1 2023-09-23T18:24:25.430 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:26.209 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:26.209 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:26.479 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:27.021 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:26 smithi055 ceph-mon[179668]: pgmap v1672: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:27.022 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:26 smithi055 ceph-mon[184174]: pgmap v1672: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:27.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:26 smithi099 ceph-mon[167076]: pgmap v1672: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:27.203 INFO:teuthology.orchestra.run.smithi055.stdout:399431958904 2023-09-23T18:24:27.638 INFO:teuthology.orchestra.run.smithi055.stdout:1017907249418 2023-09-23T18:24:27.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1268496940' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:24:27.931 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:27 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2576397140' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:24:27.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1268496940' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:24:27.931 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:27 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2576397140' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:24:27.999 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:28.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1268496940' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-09-23T18:24:28.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:27 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2576397140' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-09-23T18:24:28.487 INFO:tasks.cephadm.ceph_manager.ceph:need seq 399431958903 got 399431958904 for osd.0 2023-09-23T18:24:28.487 DEBUG:teuthology.parallel:result is None 2023-09-23T18:24:29.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:28 smithi099 ceph-mon[167076]: pgmap v1673: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:29.079 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:28 smithi055 ceph-mon[179668]: pgmap v1673: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:29.079 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:28 smithi055 ceph-mon[184174]: pgmap v1673: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:29.084 INFO:teuthology.orchestra.run.smithi055.stdout:128849019368 2023-09-23T18:24:29.147 INFO:teuthology.orchestra.run.smithi055.stdout:700079669559 2023-09-23T18:24:29.537 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1017907249417 got 1017907249418 for osd.3 2023-09-23T18:24:29.538 DEBUG:teuthology.parallel:result is None 2023-09-23T18:24:29.929 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3919278007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:24:29.930 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:29 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/3190762914' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:24:29.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3919278007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:24:29.930 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:29 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/3190762914' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:24:30.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3919278007' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-09-23T18:24:30.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:29 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/3190762914' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-09-23T18:24:30.099 INFO:teuthology.orchestra.run.smithi055.stdout:2070174236790 2023-09-23T18:24:30.408 INFO:teuthology.orchestra.run.smithi055.stdout:2340757176408 2023-09-23T18:24:30.679 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:30 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:24:30] "GET /metrics HTTP/1.1" 200 34213 "" "Prometheus/2.43.0" 2023-09-23T18:24:30.918 INFO:tasks.cephadm.ceph_manager.ceph:need seq 700079669558 got 700079669559 for osd.6 2023-09-23T18:24:30.918 DEBUG:teuthology.parallel:result is None 2023-09-23T18:24:30.921 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849019367 got 128849019368 for osd.4 2023-09-23T18:24:30.921 DEBUG:teuthology.parallel:result is None 2023-09-23T18:24:30.949 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:30 smithi055 ceph-mon[179668]: pgmap v1674: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:30.949 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/815816036' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:24:30.949 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:30 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/1691038830' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:24:30.950 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:30 smithi055 ceph-mon[184174]: pgmap v1674: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:30.950 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/815816036' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:24:30.950 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:30 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/1691038830' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:24:31.011 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2340757176406 got 2340757176408 for osd.1 2023-09-23T18:24:31.011 DEBUG:teuthology.parallel:result is None 2023-09-23T18:24:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:30 smithi099 ceph-mon[167076]: pgmap v1674: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/815816036' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-09-23T18:24:31.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:30 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/1691038830' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-09-23T18:24:31.118 INFO:tasks.cephadm.ceph_manager.ceph:need seq 2070174236789 got 2070174236790 for osd.2 2023-09-23T18:24:31.118 DEBUG:teuthology.parallel:result is None 2023-09-23T18:24:31.485 INFO:teuthology.orchestra.run.smithi055.stdout:919123001623 2023-09-23T18:24:31.890 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:31 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2337218668' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:24:31.907 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:31 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2337218668' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:24:31.908 INFO:tasks.cephadm.ceph_manager.ceph:need seq 919123001621 got 919123001623 for osd.7 2023-09-23T18:24:31.908 DEBUG:teuthology.parallel:result is None 2023-09-23T18:24:32.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:31 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2337218668' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-09-23T18:24:32.472 INFO:teuthology.orchestra.run.smithi055.stdout:1803886264466 2023-09-23T18:24:32.822 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:32 smithi055 ceph-mon[179668]: pgmap v1675: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:32.822 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:32 smithi055 ceph-mon[179668]: from='client.? 172.21.15.55:0/2691590453' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:24:32.823 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:32 smithi055 ceph-mon[184174]: pgmap v1675: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:32.823 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:32 smithi055 ceph-mon[184174]: from='client.? 172.21.15.55:0/2691590453' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:24:32.824 INFO:tasks.cephadm.ceph_manager.ceph:need seq 1803886264464 got 1803886264466 for osd.5 2023-09-23T18:24:32.824 DEBUG:teuthology.parallel:result is None 2023-09-23T18:24:32.824 INFO:tasks.cephadm.ceph_manager.ceph:waiting for recovery to complete 2023-09-23T18:24:32.825 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:24:33.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:32 smithi099 ceph-mon[167076]: pgmap v1675: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:33.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:32 smithi099 ceph-mon[167076]: from='client.? 172.21.15.55:0/2691590453' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-09-23T18:24:33.434 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:34.834 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:24:34.836 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:24:35.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:34 smithi099 ceph-mon[167076]: pgmap v1676: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:35.132 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:34 smithi055 ceph-mon[179668]: pgmap v1676: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:35.132 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:34 smithi055 ceph-mon[184174]: pgmap v1676: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:35.193 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1676,"stamp":"2023-09-23T18:24:33.262768+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":697,"num_bytes_recovered":39210717,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":310,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4599624,"kb_used_data":192584,"kb_used_omap":76,"kb_used_meta":4406899,"kb_avail":745361592,"statfs":{"total":767960285184,"available":763250270208,"internally_reserved":0,"allocated":197206016,"data_stored":175761265,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78507,"internal_metadata":4512664917},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003389"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":1034,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585587+0000","last_change":"2023-09-23T18:23:15.701666+0000","last_active":"2023-09-23T18:24:10.585587+0000","last_peered":"2023-09-23T18:24:10.585587+0000","last_clean":"2023-09-23T18:24:10.585587+0000","last_became_active":"2023-09-23T18:23:15.700793+0000","last_became_peered":"2023-09-23T18:23:15.700793+0000","last_unstale":"2023-09-23T18:24:10.585587+0000","last_undegraded":"2023-09-23T18:24:10.585587+0000","last_fullsized":"2023-09-23T18:24:10.585587+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:50.372583+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":368,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608340+0000","last_change":"2023-09-23T18:22:44.528866+0000","last_active":"2023-09-23T18:24:10.608340+0000","last_peered":"2023-09-23T18:24:10.608340+0000","last_clean":"2023-09-23T18:24:10.608340+0000","last_became_active":"2023-09-23T18:22:44.528499+0000","last_became_peered":"2023-09-23T18:22:44.528499+0000","last_unstale":"2023-09-23T18:24:10.608340+0000","last_undegraded":"2023-09-23T18:24:10.608340+0000","last_fullsized":"2023-09-23T18:24:10.608340+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:26.288744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":498,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593014+0000","last_change":"2023-09-23T18:23:53.309716+0000","last_active":"2023-09-23T18:24:10.593014+0000","last_peered":"2023-09-23T18:24:10.593014+0000","last_clean":"2023-09-23T18:24:10.593014+0000","last_became_active":"2023-09-23T18:23:53.309419+0000","last_became_peered":"2023-09-23T18:23:53.309419+0000","last_unstale":"2023-09-23T18:24:10.593014+0000","last_undegraded":"2023-09-23T18:24:10.593014+0000","last_fullsized":"2023-09-23T18:24:10.593014+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:01.509606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":1032,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585759+0000","last_change":"2023-09-23T18:23:18.705964+0000","last_active":"2023-09-23T18:24:10.585759+0000","last_peered":"2023-09-23T18:24:10.585759+0000","last_clean":"2023-09-23T18:24:10.585759+0000","last_became_active":"2023-09-23T18:23:18.705607+0000","last_became_peered":"2023-09-23T18:23:18.705607+0000","last_unstale":"2023-09-23T18:24:10.585759+0000","last_undegraded":"2023-09-23T18:24:10.585759+0000","last_fullsized":"2023-09-23T18:24:10.585759+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:54.370381+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":288,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593981+0000","last_change":"2023-09-23T18:24:11.592707+0000","last_active":"2023-09-23T18:24:11.593981+0000","last_peered":"2023-09-23T18:24:11.593981+0000","last_clean":"2023-09-23T18:24:11.593981+0000","last_became_active":"2023-09-23T18:24:11.592366+0000","last_became_peered":"2023-09-23T18:24:11.592366+0000","last_unstale":"2023-09-23T18:24:11.593981+0000","last_undegraded":"2023-09-23T18:24:11.593981+0000","last_fullsized":"2023-09-23T18:24:11.593981+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:55.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":46,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596010+0000","last_change":"2023-09-23T18:23:27.789164+0000","last_active":"2023-09-23T18:24:10.596010+0000","last_peered":"2023-09-23T18:24:10.596010+0000","last_clean":"2023-09-23T18:24:10.596010+0000","last_became_active":"2023-09-23T18:23:27.788868+0000","last_became_peered":"2023-09-23T18:23:27.788868+0000","last_unstale":"2023-09-23T18:24:10.596010+0000","last_undegraded":"2023-09-23T18:24:10.596010+0000","last_fullsized":"2023-09-23T18:24:10.596010+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:45.134497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":991,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608485+0000","last_change":"2023-09-23T18:23:15.706818+0000","last_active":"2023-09-23T18:24:10.608485+0000","last_peered":"2023-09-23T18:24:10.608485+0000","last_clean":"2023-09-23T18:24:10.608485+0000","last_became_active":"2023-09-23T18:23:15.706430+0000","last_became_peered":"2023-09-23T18:23:15.706430+0000","last_unstale":"2023-09-23T18:24:10.608485+0000","last_undegraded":"2023-09-23T18:24:10.608485+0000","last_fullsized":"2023-09-23T18:24:10.608485+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:19.880623+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":132,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593691+0000","last_change":"2023-09-23T18:23:53.307443+0000","last_active":"2023-09-23T18:24:10.593691+0000","last_peered":"2023-09-23T18:24:10.593691+0000","last_clean":"2023-09-23T18:24:10.593691+0000","last_became_active":"2023-09-23T18:23:53.306809+0000","last_became_peered":"2023-09-23T18:23:53.306809+0000","last_unstale":"2023-09-23T18:24:10.593691+0000","last_undegraded":"2023-09-23T18:24:10.593691+0000","last_fullsized":"2023-09-23T18:24:10.593691+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:26.431718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":274,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605891+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:24:10.605891+0000","last_peered":"2023-09-23T18:24:10.605891+0000","last_clean":"2023-09-23T18:24:10.605891+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:24:10.605891+0000","last_undegraded":"2023-09-23T18:24:10.605891+0000","last_fullsized":"2023-09-23T18:24:10.605891+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":692,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605978+0000","last_change":"2023-09-23T18:23:15.696286+0000","last_active":"2023-09-23T18:24:10.605978+0000","last_peered":"2023-09-23T18:24:10.605978+0000","last_clean":"2023-09-23T18:24:10.605978+0000","last_became_active":"2023-09-23T18:23:15.695903+0000","last_became_peered":"2023-09-23T18:23:15.695903+0000","last_unstale":"2023-09-23T18:24:10.605978+0000","last_undegraded":"2023-09-23T18:24:10.605978+0000","last_fullsized":"2023-09-23T18:24:10.605978+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:02.891204+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":225,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.598004+0000","last_change":"2023-09-23T18:23:27.788735+0000","last_active":"2023-09-23T18:24:10.598004+0000","last_peered":"2023-09-23T18:24:10.598004+0000","last_clean":"2023-09-23T18:24:10.598004+0000","last_became_active":"2023-09-23T18:23:27.788405+0000","last_became_peered":"2023-09-23T18:23:27.788405+0000","last_unstale":"2023-09-23T18:24:10.598004+0000","last_undegraded":"2023-09-23T18:24:10.598004+0000","last_fullsized":"2023-09-23T18:24:10.598004+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:17.598909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":290,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.594040+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:24:10.594040+0000","last_peered":"2023-09-23T18:24:10.594040+0000","last_clean":"2023-09-23T18:24:10.594040+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:24:10.594040+0000","last_undegraded":"2023-09-23T18:24:10.594040+0000","last_fullsized":"2023-09-23T18:24:10.594040+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":862,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.643341+0000","last_change":"2023-09-23T18:24:05.497052+0000","last_active":"2023-09-23T18:24:11.643341+0000","last_peered":"2023-09-23T18:24:11.643341+0000","last_clean":"2023-09-23T18:24:11.643341+0000","last_became_active":"2023-09-23T18:24:05.496700+0000","last_became_peered":"2023-09-23T18:24:05.496700+0000","last_unstale":"2023-09-23T18:24:11.643341+0000","last_undegraded":"2023-09-23T18:24:11.643341+0000","last_fullsized":"2023-09-23T18:24:11.643341+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:07.451298+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":254,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586210+0000","last_change":"2023-09-23T18:23:12.646379+0000","last_active":"2023-09-23T18:24:10.586210+0000","last_peered":"2023-09-23T18:24:10.586210+0000","last_clean":"2023-09-23T18:24:10.586210+0000","last_became_active":"2023-09-23T18:23:12.646006+0000","last_became_peered":"2023-09-23T18:23:12.646006+0000","last_unstale":"2023-09-23T18:24:10.586210+0000","last_undegraded":"2023-09-23T18:24:10.586210+0000","last_fullsized":"2023-09-23T18:24:10.586210+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:33.403421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":298,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596325+0000","last_change":"2023-09-23T18:23:53.308722+0000","last_active":"2023-09-23T18:24:10.596325+0000","last_peered":"2023-09-23T18:24:10.596325+0000","last_clean":"2023-09-23T18:24:10.596325+0000","last_became_active":"2023-09-23T18:23:53.308425+0000","last_became_peered":"2023-09-23T18:23:53.308425+0000","last_unstale":"2023-09-23T18:24:10.596325+0000","last_undegraded":"2023-09-23T18:24:10.596325+0000","last_fullsized":"2023-09-23T18:24:10.596325+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:36.073008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":1093,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586285+0000","last_change":"2023-09-23T18:23:18.704918+0000","last_active":"2023-09-23T18:24:10.586285+0000","last_peered":"2023-09-23T18:24:10.586285+0000","last_clean":"2023-09-23T18:24:10.586285+0000","last_became_active":"2023-09-23T18:23:18.704443+0000","last_became_peered":"2023-09-23T18:23:18.704443+0000","last_unstale":"2023-09-23T18:24:10.586285+0000","last_undegraded":"2023-09-23T18:24:10.586285+0000","last_fullsized":"2023-09-23T18:24:10.586285+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:56:12.850187+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":45,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605825+0000","last_change":"2023-09-23T18:23:12.672052+0000","last_active":"2023-09-23T18:24:10.605825+0000","last_peered":"2023-09-23T18:24:10.605825+0000","last_clean":"2023-09-23T18:24:10.605825+0000","last_became_active":"2023-09-23T18:23:12.645704+0000","last_became_peered":"2023-09-23T18:23:12.645704+0000","last_unstale":"2023-09-23T18:24:10.605825+0000","last_undegraded":"2023-09-23T18:24:10.605825+0000","last_fullsized":"2023-09-23T18:24:10.605825+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:34.094770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":990,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":460,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.644733+0000","last_change":"2023-09-23T18:24:11.644260+0000","last_active":"2023-09-23T18:24:11.644733+0000","last_peered":"2023-09-23T18:24:11.644733+0000","last_clean":"2023-09-23T18:24:11.644733+0000","last_became_active":"2023-09-23T18:24:11.643918+0000","last_became_peered":"2023-09-23T18:24:11.643918+0000","last_unstale":"2023-09-23T18:24:11.644733+0000","last_undegraded":"2023-09-23T18:24:11.644733+0000","last_fullsized":"2023-09-23T18:24:11.644733+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:14.921661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":1096,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.591673+0000","last_change":"2023-09-23T18:24:11.591673+0000","last_active":"2023-09-23T18:24:11.591673+0000","last_peered":"2023-09-23T18:24:11.591673+0000","last_clean":"2023-09-23T18:24:11.591673+0000","last_became_active":"2023-09-23T18:24:11.591320+0000","last_became_peered":"2023-09-23T18:24:11.591320+0000","last_unstale":"2023-09-23T18:24:11.591673+0000","last_undegraded":"2023-09-23T18:24:11.591673+0000","last_fullsized":"2023-09-23T18:24:11.591673+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:45.793533+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":447,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608587+0000","last_change":"2023-09-23T18:23:33.840584+0000","last_active":"2023-09-23T18:24:10.608587+0000","last_peered":"2023-09-23T18:24:10.608587+0000","last_clean":"2023-09-23T18:24:10.608587+0000","last_became_active":"2023-09-23T18:23:33.840167+0000","last_became_peered":"2023-09-23T18:23:33.840167+0000","last_unstale":"2023-09-23T18:24:10.608587+0000","last_undegraded":"2023-09-23T18:24:10.608587+0000","last_fullsized":"2023-09-23T18:24:10.608587+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:19.850055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":440,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608727+0000","last_change":"2023-09-23T18:23:33.843086+0000","last_active":"2023-09-23T18:24:10.608727+0000","last_peered":"2023-09-23T18:24:10.608727+0000","last_clean":"2023-09-23T18:24:10.608727+0000","last_became_active":"2023-09-23T18:23:33.842456+0000","last_became_peered":"2023-09-23T18:23:33.842456+0000","last_unstale":"2023-09-23T18:24:10.608727+0000","last_undegraded":"2023-09-23T18:24:10.608727+0000","last_fullsized":"2023-09-23T18:24:10.608727+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:52.924489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":954,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596989+0000","last_change":"2023-09-23T18:23:27.793962+0000","last_active":"2023-09-23T18:24:10.596989+0000","last_peered":"2023-09-23T18:24:10.596989+0000","last_clean":"2023-09-23T18:24:10.596989+0000","last_became_active":"2023-09-23T18:23:27.791134+0000","last_became_peered":"2023-09-23T18:23:27.791134+0000","last_unstale":"2023-09-23T18:24:10.596989+0000","last_undegraded":"2023-09-23T18:24:10.596989+0000","last_fullsized":"2023-09-23T18:24:10.596989+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:47.069746+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":45,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.604746+0000","last_change":"2023-09-23T18:23:12.652460+0000","last_active":"2023-09-23T18:24:10.604746+0000","last_peered":"2023-09-23T18:24:10.604746+0000","last_clean":"2023-09-23T18:24:10.604746+0000","last_became_active":"2023-09-23T18:23:12.637983+0000","last_became_peered":"2023-09-23T18:23:12.637983+0000","last_unstale":"2023-09-23T18:24:10.604746+0000","last_undegraded":"2023-09-23T18:24:10.604746+0000","last_fullsized":"2023-09-23T18:24:10.604746+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:42:45.669563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":33,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.594194+0000","last_change":"2023-09-23T18:23:53.306077+0000","last_active":"2023-09-23T18:24:10.594194+0000","last_peered":"2023-09-23T18:24:10.594194+0000","last_clean":"2023-09-23T18:24:10.594194+0000","last_became_active":"2023-09-23T18:23:53.305594+0000","last_became_peered":"2023-09-23T18:23:53.305594+0000","last_unstale":"2023-09-23T18:24:10.594194+0000","last_undegraded":"2023-09-23T18:24:10.594194+0000","last_fullsized":"2023-09-23T18:24:10.594194+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:05.212893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":915,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585393+0000","last_change":"2023-09-23T18:23:15.701886+0000","last_active":"2023-09-23T18:24:10.585393+0000","last_peered":"2023-09-23T18:24:10.585393+0000","last_clean":"2023-09-23T18:24:10.585393+0000","last_became_active":"2023-09-23T18:23:15.701306+0000","last_became_peered":"2023-09-23T18:23:15.701306+0000","last_unstale":"2023-09-23T18:24:10.585393+0000","last_undegraded":"2023-09-23T18:24:10.585393+0000","last_fullsized":"2023-09-23T18:24:10.585393+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:36.144212+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":46,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596811+0000","last_change":"2023-09-23T18:23:27.790518+0000","last_active":"2023-09-23T18:24:10.596811+0000","last_peered":"2023-09-23T18:24:10.596811+0000","last_clean":"2023-09-23T18:24:10.596811+0000","last_became_active":"2023-09-23T18:23:27.789716+0000","last_became_peered":"2023-09-23T18:23:27.789716+0000","last_unstale":"2023-09-23T18:24:10.596811+0000","last_undegraded":"2023-09-23T18:24:10.596811+0000","last_fullsized":"2023-09-23T18:24:10.596811+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:26.760633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":218,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586265+0000","last_change":"2023-09-23T18:23:12.646863+0000","last_active":"2023-09-23T18:24:10.586265+0000","last_peered":"2023-09-23T18:24:10.586265+0000","last_clean":"2023-09-23T18:24:10.586265+0000","last_became_active":"2023-09-23T18:23:12.646501+0000","last_became_peered":"2023-09-23T18:23:12.646501+0000","last_unstale":"2023-09-23T18:24:10.586265+0000","last_undegraded":"2023-09-23T18:24:10.586265+0000","last_fullsized":"2023-09-23T18:24:10.586265+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.839398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":925,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.594089+0000","last_change":"2023-09-23T18:24:11.593422+0000","last_active":"2023-09-23T18:24:11.594089+0000","last_peered":"2023-09-23T18:24:11.594089+0000","last_clean":"2023-09-23T18:24:11.594089+0000","last_became_active":"2023-09-23T18:24:11.592470+0000","last_became_peered":"2023-09-23T18:24:11.592470+0000","last_unstale":"2023-09-23T18:24:11.594089+0000","last_undegraded":"2023-09-23T18:24:11.594089+0000","last_fullsized":"2023-09-23T18:24:11.594089+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:39.773928+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":296,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.604833+0000","last_change":"2023-09-23T18:23:46.831702+0000","last_active":"2023-09-23T18:24:10.604833+0000","last_peered":"2023-09-23T18:24:10.604833+0000","last_clean":"2023-09-23T18:24:10.604833+0000","last_became_active":"2023-09-23T18:23:46.831430+0000","last_became_peered":"2023-09-23T18:23:46.831430+0000","last_unstale":"2023-09-23T18:24:10.604833+0000","last_undegraded":"2023-09-23T18:24:10.604833+0000","last_fullsized":"2023-09-23T18:24:10.604833+0000","mapping_epoch":686,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":687,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:56.532410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":331,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596108+0000","last_change":"2023-09-23T18:23:27.789041+0000","last_active":"2023-09-23T18:24:10.596108+0000","last_peered":"2023-09-23T18:24:10.596108+0000","last_clean":"2023-09-23T18:24:10.596108+0000","last_became_active":"2023-09-23T18:23:27.788471+0000","last_became_peered":"2023-09-23T18:23:27.788471+0000","last_unstale":"2023-09-23T18:24:10.596108+0000","last_undegraded":"2023-09-23T18:24:10.596108+0000","last_fullsized":"2023-09-23T18:24:10.596108+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:26.454700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":816,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593416+0000","last_change":"2023-09-23T18:23:53.308352+0000","last_active":"2023-09-23T18:24:10.593416+0000","last_peered":"2023-09-23T18:24:10.593416+0000","last_clean":"2023-09-23T18:24:10.593416+0000","last_became_active":"2023-09-23T18:23:53.308040+0000","last_became_peered":"2023-09-23T18:23:53.308040+0000","last_unstale":"2023-09-23T18:24:10.593416+0000","last_undegraded":"2023-09-23T18:24:10.593416+0000","last_fullsized":"2023-09-23T18:24:10.593416+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:27.222024+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":878,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605562+0000","last_change":"2023-09-23T18:23:15.712056+0000","last_active":"2023-09-23T18:24:10.605562+0000","last_peered":"2023-09-23T18:24:10.605562+0000","last_clean":"2023-09-23T18:24:10.605562+0000","last_became_active":"2023-09-23T18:23:15.690682+0000","last_became_peered":"2023-09-23T18:23:15.690682+0000","last_unstale":"2023-09-23T18:24:10.605562+0000","last_undegraded":"2023-09-23T18:24:10.605562+0000","last_fullsized":"2023-09-23T18:24:10.605562+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:18.252430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596649+0000","last_change":"2023-09-23T18:24:08.744171+0000","last_active":"2023-09-23T18:24:10.596649+0000","last_peered":"2023-09-23T18:24:10.596649+0000","last_clean":"2023-09-23T18:24:10.596649+0000","last_became_active":"2023-09-23T18:24:08.743847+0000","last_became_peered":"2023-09-23T18:24:08.743847+0000","last_unstale":"2023-09-23T18:24:10.596649+0000","last_undegraded":"2023-09-23T18:24:10.596649+0000","last_fullsized":"2023-09-23T18:24:10.596649+0000","mapping_epoch":696,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":697,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:27.473003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":692,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606034+0000","last_change":"2023-09-23T18:23:15.706446+0000","last_active":"2023-09-23T18:24:10.606034+0000","last_peered":"2023-09-23T18:24:10.606034+0000","last_clean":"2023-09-23T18:24:10.606034+0000","last_became_active":"2023-09-23T18:23:15.706089+0000","last_became_peered":"2023-09-23T18:23:15.706089+0000","last_unstale":"2023-09-23T18:24:10.606034+0000","last_undegraded":"2023-09-23T18:24:10.606034+0000","last_fullsized":"2023-09-23T18:24:10.606034+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:02.005632+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":57,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606579+0000","last_change":"2023-09-23T18:23:15.721550+0000","last_active":"2023-09-23T18:24:10.606579+0000","last_peered":"2023-09-23T18:24:10.606579+0000","last_clean":"2023-09-23T18:24:10.606579+0000","last_became_active":"2023-09-23T18:23:15.700750+0000","last_became_peered":"2023-09-23T18:23:15.700750+0000","last_unstale":"2023-09-23T18:24:10.606579+0000","last_undegraded":"2023-09-23T18:24:10.606579+0000","last_fullsized":"2023-09-23T18:24:10.606579+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:03.517711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":218,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585437+0000","last_change":"2023-09-23T18:23:12.640414+0000","last_active":"2023-09-23T18:24:10.585437+0000","last_peered":"2023-09-23T18:24:10.585437+0000","last_clean":"2023-09-23T18:24:10.585437+0000","last_became_active":"2023-09-23T18:23:12.639849+0000","last_became_peered":"2023-09-23T18:23:12.639849+0000","last_unstale":"2023-09-23T18:24:10.585437+0000","last_undegraded":"2023-09-23T18:24:10.585437+0000","last_fullsized":"2023-09-23T18:24:10.585437+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:34.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":874,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.594927+0000","last_change":"2023-09-23T18:23:18.704895+0000","last_active":"2023-09-23T18:24:10.594927+0000","last_peered":"2023-09-23T18:24:10.594927+0000","last_clean":"2023-09-23T18:24:10.594927+0000","last_became_active":"2023-09-23T18:23:18.704488+0000","last_became_peered":"2023-09-23T18:23:18.704488+0000","last_unstale":"2023-09-23T18:24:10.594927+0000","last_undegraded":"2023-09-23T18:24:10.594927+0000","last_fullsized":"2023-09-23T18:24:10.594927+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:21:51.851269+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:21:51.851269+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:44.733888+0000","scrub_duration":0.001933539,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":43,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606273+0000","last_change":"2023-09-23T18:23:12.656626+0000","last_active":"2023-09-23T18:24:10.606273+0000","last_peered":"2023-09-23T18:24:10.606273+0000","last_clean":"2023-09-23T18:24:10.606273+0000","last_became_active":"2023-09-23T18:23:12.645323+0000","last_became_peered":"2023-09-23T18:23:12.645323+0000","last_unstale":"2023-09-23T18:24:10.606273+0000","last_undegraded":"2023-09-23T18:24:10.606273+0000","last_fullsized":"2023-09-23T18:24:10.606273+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:28.109499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":315,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608142+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:24:10.608142+0000","last_peered":"2023-09-23T18:24:10.608142+0000","last_clean":"2023-09-23T18:24:10.608142+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:24:10.608142+0000","last_undegraded":"2023-09-23T18:24:10.608142+0000","last_fullsized":"2023-09-23T18:24:10.608142+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":713,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605782+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:24:10.605782+0000","last_peered":"2023-09-23T18:24:10.605782+0000","last_clean":"2023-09-23T18:24:10.605782+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:24:10.605782+0000","last_undegraded":"2023-09-23T18:24:10.605782+0000","last_fullsized":"2023-09-23T18:24:10.605782+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":43,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605903+0000","last_change":"2023-09-23T18:23:12.668907+0000","last_active":"2023-09-23T18:24:10.605903+0000","last_peered":"2023-09-23T18:24:10.605903+0000","last_clean":"2023-09-23T18:24:10.605903+0000","last_became_active":"2023-09-23T18:23:12.646244+0000","last_became_peered":"2023-09-23T18:23:12.646244+0000","last_unstale":"2023-09-23T18:24:10.605903+0000","last_undegraded":"2023-09-23T18:24:10.605903+0000","last_fullsized":"2023-09-23T18:24:10.605903+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:07:50.424121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":284,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606672+0000","last_change":"2023-09-23T18:23:12.639745+0000","last_active":"2023-09-23T18:24:10.606672+0000","last_peered":"2023-09-23T18:24:10.606672+0000","last_clean":"2023-09-23T18:24:10.606672+0000","last_became_active":"2023-09-23T18:23:12.639398+0000","last_became_peered":"2023-09-23T18:23:12.639398+0000","last_unstale":"2023-09-23T18:24:10.606672+0000","last_undegraded":"2023-09-23T18:24:10.606672+0000","last_fullsized":"2023-09-23T18:24:10.606672+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:19.476844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":886,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608896+0000","last_change":"2023-09-23T18:23:33.839754+0000","last_active":"2023-09-23T18:24:10.608896+0000","last_peered":"2023-09-23T18:24:10.608896+0000","last_clean":"2023-09-23T18:24:10.608896+0000","last_became_active":"2023-09-23T18:23:33.839201+0000","last_became_peered":"2023-09-23T18:23:33.839201+0000","last_unstale":"2023-09-23T18:24:10.608896+0000","last_undegraded":"2023-09-23T18:24:10.608896+0000","last_fullsized":"2023-09-23T18:24:10.608896+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:08.161741+0000","scrub_duration":0.007530826,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":222,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586126+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:24:10.586126+0000","last_peered":"2023-09-23T18:24:10.586126+0000","last_clean":"2023-09-23T18:24:10.586126+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:24:10.586126+0000","last_undegraded":"2023-09-23T18:24:10.586126+0000","last_fullsized":"2023-09-23T18:24:10.586126+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":37,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.600047+0000","last_change":"2023-09-23T18:23:15.699232+0000","last_active":"2023-09-23T18:24:10.600047+0000","last_peered":"2023-09-23T18:24:10.600047+0000","last_clean":"2023-09-23T18:24:10.600047+0000","last_became_active":"2023-09-23T18:23:15.698945+0000","last_became_peered":"2023-09-23T18:23:15.698945+0000","last_unstale":"2023-09-23T18:24:10.600047+0000","last_undegraded":"2023-09-23T18:24:10.600047+0000","last_fullsized":"2023-09-23T18:24:10.600047+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:01.104606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1188,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.604979+0000","last_change":"2023-09-23T18:23:15.694144+0000","last_active":"2023-09-23T18:24:10.604979+0000","last_peered":"2023-09-23T18:24:10.604979+0000","last_clean":"2023-09-23T18:24:10.604979+0000","last_became_active":"2023-09-23T18:23:15.693764+0000","last_became_peered":"2023-09-23T18:23:15.693764+0000","last_unstale":"2023-09-23T18:24:10.604979+0000","last_undegraded":"2023-09-23T18:24:10.604979+0000","last_fullsized":"2023-09-23T18:24:10.604979+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:44.221828+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":460,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.592543+0000","last_change":"2023-09-23T18:23:53.309838+0000","last_active":"2023-09-23T18:24:10.592543+0000","last_peered":"2023-09-23T18:24:10.592543+0000","last_clean":"2023-09-23T18:24:10.592543+0000","last_became_active":"2023-09-23T18:23:53.309561+0000","last_became_peered":"2023-09-23T18:23:53.309561+0000","last_unstale":"2023-09-23T18:24:10.592543+0000","last_undegraded":"2023-09-23T18:24:10.592543+0000","last_fullsized":"2023-09-23T18:24:10.592543+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:54.966938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":236,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608460+0000","last_change":"2023-09-23T18:23:33.842661+0000","last_active":"2023-09-23T18:24:10.608460+0000","last_peered":"2023-09-23T18:24:10.608460+0000","last_clean":"2023-09-23T18:24:10.608460+0000","last_became_active":"2023-09-23T18:23:33.842267+0000","last_became_peered":"2023-09-23T18:23:33.842267+0000","last_unstale":"2023-09-23T18:24:10.608460+0000","last_undegraded":"2023-09-23T18:24:10.608460+0000","last_fullsized":"2023-09-23T18:24:10.608460+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:21.603948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":987,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585594+0000","last_change":"2023-09-23T18:22:06.466040+0000","last_active":"2023-09-23T18:24:10.585594+0000","last_peered":"2023-09-23T18:24:10.585594+0000","last_clean":"2023-09-23T18:24:10.585594+0000","last_became_active":"2023-09-23T18:22:06.465158+0000","last_became_peered":"2023-09-23T18:22:06.465158+0000","last_unstale":"2023-09-23T18:24:10.585594+0000","last_undegraded":"2023-09-23T18:24:10.585594+0000","last_fullsized":"2023-09-23T18:24:10.585594+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:29.587874+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":250,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605920+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:24:10.605920+0000","last_peered":"2023-09-23T18:24:10.605920+0000","last_clean":"2023-09-23T18:24:10.605920+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:24:10.605920+0000","last_undegraded":"2023-09-23T18:24:10.605920+0000","last_fullsized":"2023-09-23T18:24:10.605920+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":493,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608981+0000","last_change":"2023-09-23T18:22:44.527193+0000","last_active":"2023-09-23T18:24:10.608981+0000","last_peered":"2023-09-23T18:24:10.608981+0000","last_clean":"2023-09-23T18:24:10.608981+0000","last_became_active":"2023-09-23T18:22:44.526903+0000","last_became_peered":"2023-09-23T18:22:44.526903+0000","last_unstale":"2023-09-23T18:24:10.608981+0000","last_undegraded":"2023-09-23T18:24:10.608981+0000","last_fullsized":"2023-09-23T18:24:10.608981+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:09.126029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":916,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593951+0000","last_change":"2023-09-23T18:24:11.592309+0000","last_active":"2023-09-23T18:24:11.593951+0000","last_peered":"2023-09-23T18:24:11.593951+0000","last_clean":"2023-09-23T18:24:11.593951+0000","last_became_active":"2023-09-23T18:24:11.591723+0000","last_became_peered":"2023-09-23T18:24:11.591723+0000","last_unstale":"2023-09-23T18:24:11.593951+0000","last_undegraded":"2023-09-23T18:24:11.593951+0000","last_fullsized":"2023-09-23T18:24:11.593951+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:33.935341+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":501,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608331+0000","last_change":"2023-09-23T18:23:33.840098+0000","last_active":"2023-09-23T18:24:10.608331+0000","last_peered":"2023-09-23T18:24:10.608331+0000","last_clean":"2023-09-23T18:24:10.608331+0000","last_became_active":"2023-09-23T18:23:33.839509+0000","last_became_peered":"2023-09-23T18:23:33.839509+0000","last_unstale":"2023-09-23T18:24:10.608331+0000","last_undegraded":"2023-09-23T18:24:10.608331+0000","last_fullsized":"2023-09-23T18:24:10.608331+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:48.488620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":233,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608908+0000","last_change":"2023-09-23T18:23:33.843126+0000","last_active":"2023-09-23T18:24:10.608908+0000","last_peered":"2023-09-23T18:24:10.608908+0000","last_clean":"2023-09-23T18:24:10.608908+0000","last_became_active":"2023-09-23T18:23:33.842811+0000","last_became_peered":"2023-09-23T18:23:33.842811+0000","last_unstale":"2023-09-23T18:24:10.608908+0000","last_undegraded":"2023-09-23T18:24:10.608908+0000","last_fullsized":"2023-09-23T18:24:10.608908+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:18.853788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":970,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585775+0000","last_change":"2023-09-23T18:22:06.468619+0000","last_active":"2023-09-23T18:24:10.585775+0000","last_peered":"2023-09-23T18:24:10.585775+0000","last_clean":"2023-09-23T18:24:10.585775+0000","last_became_active":"2023-09-23T18:22:06.468197+0000","last_became_peered":"2023-09-23T18:22:06.468197+0000","last_unstale":"2023-09-23T18:24:10.585775+0000","last_undegraded":"2023-09-23T18:24:10.585775+0000","last_fullsized":"2023-09-23T18:24:10.585775+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:10.149437+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":126,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593886+0000","last_change":"2023-09-23T18:23:12.639264+0000","last_active":"2023-09-23T18:24:10.593886+0000","last_peered":"2023-09-23T18:24:10.593886+0000","last_clean":"2023-09-23T18:24:10.593886+0000","last_became_active":"2023-09-23T18:23:12.638919+0000","last_became_peered":"2023-09-23T18:23:12.638919+0000","last_unstale":"2023-09-23T18:24:10.593886+0000","last_undegraded":"2023-09-23T18:24:10.593886+0000","last_fullsized":"2023-09-23T18:24:10.593886+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:42:06.409381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":420,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593952+0000","last_change":"2023-09-23T18:24:11.591698+0000","last_active":"2023-09-23T18:24:11.593952+0000","last_peered":"2023-09-23T18:24:11.593952+0000","last_clean":"2023-09-23T18:24:11.593952+0000","last_became_active":"2023-09-23T18:24:11.591424+0000","last_became_peered":"2023-09-23T18:24:11.591424+0000","last_unstale":"2023-09-23T18:24:11.593952+0000","last_undegraded":"2023-09-23T18:24:11.593952+0000","last_fullsized":"2023-09-23T18:24:11.593952+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:30.613497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":829,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606291+0000","last_change":"2023-09-23T18:23:15.694981+0000","last_active":"2023-09-23T18:24:10.606291+0000","last_peered":"2023-09-23T18:24:10.606291+0000","last_clean":"2023-09-23T18:24:10.606291+0000","last_became_active":"2023-09-23T18:23:15.694607+0000","last_became_peered":"2023-09-23T18:23:15.694607+0000","last_unstale":"2023-09-23T18:24:10.606291+0000","last_undegraded":"2023-09-23T18:24:10.606291+0000","last_fullsized":"2023-09-23T18:24:10.606291+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:52.640350+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"653'291","reported_seq":1684,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.594063+0000","last_change":"2023-09-23T18:24:11.592902+0000","last_active":"2023-09-23T18:24:11.594063+0000","last_peered":"2023-09-23T18:24:11.594063+0000","last_clean":"2023-09-23T18:24:11.594063+0000","last_became_active":"2023-09-23T18:24:11.592020+0000","last_became_peered":"2023-09-23T18:24:11.592020+0000","last_unstale":"2023-09-23T18:24:11.594063+0000","last_undegraded":"2023-09-23T18:24:11.594063+0000","last_fullsized":"2023-09-23T18:24:11.594063+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":291,"log_dups_size":0,"ondisk_log_size":291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:40.158357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047292100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1326,"num_read_kb":5772,"num_write":300,"num_write_kb":4517,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":242,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586479+0000","last_change":"2023-09-23T18:22:03.462589+0000","last_active":"2023-09-23T18:24:10.586479+0000","last_peered":"2023-09-23T18:24:10.586479+0000","last_clean":"2023-09-23T18:24:10.586479+0000","last_became_active":"2023-09-23T18:22:03.462239+0000","last_became_peered":"2023-09-23T18:22:03.462239+0000","last_unstale":"2023-09-23T18:24:10.586479+0000","last_undegraded":"2023-09-23T18:24:10.586479+0000","last_fullsized":"2023-09-23T18:24:10.586479+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.020770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":275,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593970+0000","last_change":"2023-09-23T18:24:11.591973+0000","last_active":"2023-09-23T18:24:11.593970+0000","last_peered":"2023-09-23T18:24:11.593970+0000","last_clean":"2023-09-23T18:24:11.593970+0000","last_became_active":"2023-09-23T18:24:11.591608+0000","last_became_peered":"2023-09-23T18:24:11.591608+0000","last_unstale":"2023-09-23T18:24:11.593970+0000","last_undegraded":"2023-09-23T18:24:11.593970+0000","last_fullsized":"2023-09-23T18:24:11.593970+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:42:02.296074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":941,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585568+0000","last_change":"2023-09-23T18:23:18.701192+0000","last_active":"2023-09-23T18:24:10.585568+0000","last_peered":"2023-09-23T18:24:10.585568+0000","last_clean":"2023-09-23T18:24:10.585568+0000","last_became_active":"2023-09-23T18:23:18.700482+0000","last_became_peered":"2023-09-23T18:23:18.700482+0000","last_unstale":"2023-09-23T18:24:10.585568+0000","last_undegraded":"2023-09-23T18:24:10.585568+0000","last_fullsized":"2023-09-23T18:24:10.585568+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:30.748035+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"653'323","reported_seq":2298,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585612+0000","last_change":"2023-09-23T18:22:28.473090+0000","last_active":"2023-09-23T18:24:10.585612+0000","last_peered":"2023-09-23T18:24:10.585612+0000","last_clean":"2023-09-23T18:24:10.585612+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:24:10.585612+0000","last_undegraded":"2023-09-23T18:24:10.585612+0000","last_fullsized":"2023-09-23T18:24:10.585612+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":323,"log_dups_size":0,"ondisk_log_size":323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001426075,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3093,"num_read_kb":5799,"num_write":265,"num_write_kb":3199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":239,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608533+0000","last_change":"2023-09-23T18:23:33.843234+0000","last_active":"2023-09-23T18:24:10.608533+0000","last_peered":"2023-09-23T18:24:10.608533+0000","last_clean":"2023-09-23T18:24:10.608533+0000","last_became_active":"2023-09-23T18:23:33.842353+0000","last_became_peered":"2023-09-23T18:23:33.842353+0000","last_unstale":"2023-09-23T18:24:10.608533+0000","last_undegraded":"2023-09-23T18:24:10.608533+0000","last_fullsized":"2023-09-23T18:24:10.608533+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:47.177729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":460,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.645801+0000","last_change":"2023-09-23T18:24:11.645801+0000","last_active":"2023-09-23T18:24:11.645801+0000","last_peered":"2023-09-23T18:24:11.645801+0000","last_clean":"2023-09-23T18:24:11.645801+0000","last_became_active":"2023-09-23T18:24:11.644474+0000","last_became_peered":"2023-09-23T18:24:11.644474+0000","last_unstale":"2023-09-23T18:24:11.645801+0000","last_undegraded":"2023-09-23T18:24:11.645801+0000","last_fullsized":"2023-09-23T18:24:11.645801+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:20.338293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":742,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605721+0000","last_change":"2023-09-23T18:23:46.520844+0000","last_active":"2023-09-23T18:24:10.605721+0000","last_peered":"2023-09-23T18:24:10.605721+0000","last_clean":"2023-09-23T18:24:10.605721+0000","last_became_active":"2023-09-23T18:23:46.520494+0000","last_became_peered":"2023-09-23T18:23:46.520494+0000","last_unstale":"2023-09-23T18:24:10.605721+0000","last_undegraded":"2023-09-23T18:24:10.605721+0000","last_fullsized":"2023-09-23T18:24:10.605721+0000","mapping_epoch":686,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":687,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:03.685739+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"653'200","reported_seq":634,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596938+0000","last_change":"2023-09-23T18:24:05.490508+0000","last_active":"2023-09-23T18:24:10.596938+0000","last_peered":"2023-09-23T18:24:10.596938+0000","last_clean":"2023-09-23T18:24:10.596938+0000","last_became_active":"2023-09-23T18:24:05.489857+0000","last_became_peered":"2023-09-23T18:24:05.489857+0000","last_unstale":"2023-09-23T18:24:10.596938+0000","last_undegraded":"2023-09-23T18:24:10.596938+0000","last_fullsized":"2023-09-23T18:24:10.596938+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:30.566706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0022753629999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1032,"num_read_kb":1283,"num_write":194,"num_write_kb":5509,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":29360128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":467,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.645159+0000","last_change":"2023-09-23T18:24:11.644700+0000","last_active":"2023-09-23T18:24:11.645159+0000","last_peered":"2023-09-23T18:24:11.645159+0000","last_clean":"2023-09-23T18:24:11.645159+0000","last_became_active":"2023-09-23T18:24:11.644359+0000","last_became_peered":"2023-09-23T18:24:11.644359+0000","last_unstale":"2023-09-23T18:24:11.645159+0000","last_undegraded":"2023-09-23T18:24:11.645159+0000","last_fullsized":"2023-09-23T18:24:11.645159+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:31.725247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":263,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593932+0000","last_change":"2023-09-23T18:24:11.591624+0000","last_active":"2023-09-23T18:24:11.593932+0000","last_peered":"2023-09-23T18:24:11.593932+0000","last_clean":"2023-09-23T18:24:11.593932+0000","last_became_active":"2023-09-23T18:24:11.591276+0000","last_became_peered":"2023-09-23T18:24:11.591276+0000","last_unstale":"2023-09-23T18:24:11.593932+0000","last_undegraded":"2023-09-23T18:24:11.593932+0000","last_fullsized":"2023-09-23T18:24:11.593932+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:57.325094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"653'474","reported_seq":2388,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585726+0000","last_change":"2023-09-23T18:23:15.703265+0000","last_active":"2023-09-23T18:24:10.585726+0000","last_peered":"2023-09-23T18:24:10.585726+0000","last_clean":"2023-09-23T18:24:10.585726+0000","last_became_active":"2023-09-23T18:23:15.701955+0000","last_became_peered":"2023-09-23T18:23:15.701955+0000","last_unstale":"2023-09-23T18:24:10.585726+0000","last_undegraded":"2023-09-23T18:24:10.585726+0000","last_fullsized":"2023-09-23T18:24:10.585726+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":474,"log_dups_size":0,"ondisk_log_size":474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:03.254370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014671619999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2565,"num_read_kb":5830,"num_write":410,"num_write_kb":4052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":690,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605329+0000","last_change":"2023-09-23T18:23:18.709442+0000","last_active":"2023-09-23T18:24:10.605329+0000","last_peered":"2023-09-23T18:24:10.605329+0000","last_clean":"2023-09-23T18:24:10.605329+0000","last_became_active":"2023-09-23T18:23:18.709023+0000","last_became_peered":"2023-09-23T18:23:18.709023+0000","last_unstale":"2023-09-23T18:24:10.605329+0000","last_undegraded":"2023-09-23T18:24:10.605329+0000","last_fullsized":"2023-09-23T18:24:10.605329+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:59.988881+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593329+0000","last_change":"2023-09-23T18:23:53.308051+0000","last_active":"2023-09-23T18:24:10.593329+0000","last_peered":"2023-09-23T18:24:10.593329+0000","last_clean":"2023-09-23T18:24:10.593329+0000","last_became_active":"2023-09-23T18:23:53.307760+0000","last_became_peered":"2023-09-23T18:23:53.307760+0000","last_unstale":"2023-09-23T18:24:10.593329+0000","last_undegraded":"2023-09-23T18:24:10.593329+0000","last_fullsized":"2023-09-23T18:24:10.593329+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:23.973027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":263,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606149+0000","last_change":"2023-09-23T18:23:12.647798+0000","last_active":"2023-09-23T18:24:10.606149+0000","last_peered":"2023-09-23T18:24:10.606149+0000","last_clean":"2023-09-23T18:24:10.606149+0000","last_became_active":"2023-09-23T18:23:12.647441+0000","last_became_peered":"2023-09-23T18:23:12.647441+0000","last_unstale":"2023-09-23T18:24:10.606149+0000","last_undegraded":"2023-09-23T18:24:10.606149+0000","last_fullsized":"2023-09-23T18:24:10.606149+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:13.708809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":707,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.595799+0000","last_change":"2023-09-23T18:24:11.595281+0000","last_active":"2023-09-23T18:24:11.595799+0000","last_peered":"2023-09-23T18:24:11.595799+0000","last_clean":"2023-09-23T18:24:11.595799+0000","last_became_active":"2023-09-23T18:24:11.594913+0000","last_became_peered":"2023-09-23T18:24:11.594913+0000","last_unstale":"2023-09-23T18:24:11.595799+0000","last_undegraded":"2023-09-23T18:24:11.595799+0000","last_fullsized":"2023-09-23T18:24:11.595799+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:05.054956+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"653'156","reported_seq":524,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605423+0000","last_change":"2023-09-23T18:23:18.708902+0000","last_active":"2023-09-23T18:24:10.605423+0000","last_peered":"2023-09-23T18:24:10.605423+0000","last_clean":"2023-09-23T18:24:10.605423+0000","last_became_active":"2023-09-23T18:23:18.708208+0000","last_became_peered":"2023-09-23T18:23:18.708208+0000","last_unstale":"2023-09-23T18:24:10.605423+0000","last_undegraded":"2023-09-23T18:24:10.605423+0000","last_fullsized":"2023-09-23T18:24:10.605423+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":156,"log_dups_size":0,"ondisk_log_size":156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:57.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504954,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":805,"num_read_kb":2049,"num_write":144,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":132,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.597054+0000","last_change":"2023-09-23T18:23:27.794628+0000","last_active":"2023-09-23T18:24:10.597054+0000","last_peered":"2023-09-23T18:24:10.597054+0000","last_clean":"2023-09-23T18:24:10.597054+0000","last_became_active":"2023-09-23T18:23:27.793171+0000","last_became_peered":"2023-09-23T18:23:27.793171+0000","last_unstale":"2023-09-23T18:24:10.597054+0000","last_undegraded":"2023-09-23T18:24:10.597054+0000","last_fullsized":"2023-09-23T18:24:10.597054+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:09.526134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":37,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596313+0000","last_change":"2023-09-23T18:23:15.679226+0000","last_active":"2023-09-23T18:24:10.596313+0000","last_peered":"2023-09-23T18:24:10.596313+0000","last_clean":"2023-09-23T18:24:10.596313+0000","last_became_active":"2023-09-23T18:23:15.678888+0000","last_became_peered":"2023-09-23T18:23:15.678888+0000","last_unstale":"2023-09-23T18:24:10.596313+0000","last_undegraded":"2023-09-23T18:24:10.596313+0000","last_fullsized":"2023-09-23T18:24:10.596313+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:29.362618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":1036,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606587+0000","last_change":"2023-09-23T18:23:18.706776+0000","last_active":"2023-09-23T18:24:10.606587+0000","last_peered":"2023-09-23T18:24:10.606587+0000","last_clean":"2023-09-23T18:24:10.606587+0000","last_became_active":"2023-09-23T18:23:18.706386+0000","last_became_peered":"2023-09-23T18:23:18.706386+0000","last_unstale":"2023-09-23T18:24:10.606587+0000","last_undegraded":"2023-09-23T18:24:10.606587+0000","last_fullsized":"2023-09-23T18:24:10.606587+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:35.053952+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"653'181","reported_seq":683,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606087+0000","last_change":"2023-09-23T18:23:18.708971+0000","last_active":"2023-09-23T18:24:10.606087+0000","last_peered":"2023-09-23T18:24:10.606087+0000","last_clean":"2023-09-23T18:24:10.606087+0000","last_became_active":"2023-09-23T18:23:18.708188+0000","last_became_peered":"2023-09-23T18:23:18.708188+0000","last_unstale":"2023-09-23T18:24:10.606087+0000","last_undegraded":"2023-09-23T18:24:10.606087+0000","last_fullsized":"2023-09-23T18:24:10.606087+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:38.993938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001526466,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1349,"num_read_kb":1261,"num_write":144,"num_write_kb":4223,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":281,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.646196+0000","last_change":"2023-09-23T18:24:11.645815+0000","last_active":"2023-09-23T18:24:11.646196+0000","last_peered":"2023-09-23T18:24:11.646196+0000","last_clean":"2023-09-23T18:24:11.646196+0000","last_became_active":"2023-09-23T18:24:11.643987+0000","last_became_peered":"2023-09-23T18:24:11.643987+0000","last_unstale":"2023-09-23T18:24:11.646196+0000","last_undegraded":"2023-09-23T18:24:11.646196+0000","last_fullsized":"2023-09-23T18:24:11.646196+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:40.308962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":249,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.599809+0000","last_change":"2023-09-23T18:23:27.782160+0000","last_active":"2023-09-23T18:24:10.599809+0000","last_peered":"2023-09-23T18:24:10.599809+0000","last_clean":"2023-09-23T18:24:10.599809+0000","last_became_active":"2023-09-23T18:23:27.781726+0000","last_became_peered":"2023-09-23T18:23:27.781726+0000","last_unstale":"2023-09-23T18:24:10.599809+0000","last_undegraded":"2023-09-23T18:24:10.599809+0000","last_fullsized":"2023-09-23T18:24:10.599809+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:51.328287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":1037,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608871+0000","last_change":"2023-09-23T18:23:33.843634+0000","last_active":"2023-09-23T18:24:10.608871+0000","last_peered":"2023-09-23T18:24:10.608871+0000","last_clean":"2023-09-23T18:24:10.608871+0000","last_became_active":"2023-09-23T18:23:33.843279+0000","last_became_peered":"2023-09-23T18:23:33.843279+0000","last_unstale":"2023-09-23T18:24:10.608871+0000","last_undegraded":"2023-09-23T18:24:10.608871+0000","last_fullsized":"2023-09-23T18:24:10.608871+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:24.463121+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":297,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593708+0000","last_change":"2023-09-23T18:23:53.305574+0000","last_active":"2023-09-23T18:24:10.593708+0000","last_peered":"2023-09-23T18:24:10.593708+0000","last_clean":"2023-09-23T18:24:10.593708+0000","last_became_active":"2023-09-23T18:23:53.305221+0000","last_became_peered":"2023-09-23T18:23:53.305221+0000","last_unstale":"2023-09-23T18:24:10.593708+0000","last_undegraded":"2023-09-23T18:24:10.593708+0000","last_fullsized":"2023-09-23T18:24:10.593708+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:05.291632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"653'188","reported_seq":936,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.596005+0000","last_change":"2023-09-23T18:24:11.595534+0000","last_active":"2023-09-23T18:24:11.596005+0000","last_peered":"2023-09-23T18:24:11.596005+0000","last_clean":"2023-09-23T18:24:11.596005+0000","last_became_active":"2023-09-23T18:24:11.595006+0000","last_became_peered":"2023-09-23T18:24:11.595006+0000","last_unstale":"2023-09-23T18:24:11.596005+0000","last_undegraded":"2023-09-23T18:24:11.596005+0000","last_fullsized":"2023-09-23T18:24:11.596005+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":188,"log_dups_size":0,"ondisk_log_size":188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:28.658622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055658400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1067,"num_read_kb":2442,"num_write":105,"num_write_kb":1356,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":100,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593783+0000","last_change":"2023-09-23T18:23:53.307641+0000","last_active":"2023-09-23T18:24:10.593783+0000","last_peered":"2023-09-23T18:24:10.593783+0000","last_clean":"2023-09-23T18:24:10.593783+0000","last_became_active":"2023-09-23T18:23:53.307132+0000","last_became_peered":"2023-09-23T18:23:53.307132+0000","last_unstale":"2023-09-23T18:24:10.593783+0000","last_undegraded":"2023-09-23T18:24:10.593783+0000","last_fullsized":"2023-09-23T18:24:10.593783+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:10.036096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":850,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.595041+0000","last_change":"2023-09-23T18:23:15.692819+0000","last_active":"2023-09-23T18:24:10.595041+0000","last_peered":"2023-09-23T18:24:10.595041+0000","last_clean":"2023-09-23T18:24:10.595041+0000","last_became_active":"2023-09-23T18:23:15.692460+0000","last_became_peered":"2023-09-23T18:23:15.692460+0000","last_unstale":"2023-09-23T18:24:10.595041+0000","last_undegraded":"2023-09-23T18:24:10.595041+0000","last_fullsized":"2023-09-23T18:24:10.595041+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:00.270208+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"652'137","reported_seq":1410,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608406+0000","last_change":"2023-09-23T18:23:15.695165+0000","last_active":"2023-09-23T18:24:10.608406+0000","last_peered":"2023-09-23T18:24:10.608406+0000","last_clean":"2023-09-23T18:24:10.608406+0000","last_became_active":"2023-09-23T18:23:15.694501+0000","last_became_peered":"2023-09-23T18:23:15.694501+0000","last_unstale":"2023-09-23T18:24:10.608406+0000","last_undegraded":"2023-09-23T18:24:10.608406+0000","last_fullsized":"2023-09-23T18:24:10.608406+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:17.791654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044965899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":925,"num_write":128,"num_write_kb":1260,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":41,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596224+0000","last_change":"2023-09-23T18:23:15.707579+0000","last_active":"2023-09-23T18:24:10.596224+0000","last_peered":"2023-09-23T18:24:10.596224+0000","last_clean":"2023-09-23T18:24:10.596224+0000","last_became_active":"2023-09-23T18:23:15.699673+0000","last_became_peered":"2023-09-23T18:23:15.699673+0000","last_unstale":"2023-09-23T18:24:10.596224+0000","last_undegraded":"2023-09-23T18:24:10.596224+0000","last_fullsized":"2023-09-23T18:24:10.596224+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:55:15.419839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":437,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608217+0000","last_change":"2023-09-23T18:23:33.841290+0000","last_active":"2023-09-23T18:24:10.608217+0000","last_peered":"2023-09-23T18:24:10.608217+0000","last_clean":"2023-09-23T18:24:10.608217+0000","last_became_active":"2023-09-23T18:23:33.840983+0000","last_became_peered":"2023-09-23T18:23:33.840983+0000","last_unstale":"2023-09-23T18:24:10.608217+0000","last_undegraded":"2023-09-23T18:24:10.608217+0000","last_fullsized":"2023-09-23T18:24:10.608217+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:48.677598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":630,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.595240+0000","last_change":"2023-09-23T18:23:53.307537+0000","last_active":"2023-09-23T18:24:10.595240+0000","last_peered":"2023-09-23T18:24:10.595240+0000","last_clean":"2023-09-23T18:24:10.595240+0000","last_became_active":"2023-09-23T18:23:53.307253+0000","last_became_peered":"2023-09-23T18:23:53.307253+0000","last_unstale":"2023-09-23T18:24:10.595240+0000","last_undegraded":"2023-09-23T18:24:10.595240+0000","last_fullsized":"2023-09-23T18:24:10.595240+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:28.080598+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":46,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.599716+0000","last_change":"2023-09-23T18:23:27.787712+0000","last_active":"2023-09-23T18:24:10.599716+0000","last_peered":"2023-09-23T18:24:10.599716+0000","last_clean":"2023-09-23T18:24:10.599716+0000","last_became_active":"2023-09-23T18:23:27.787370+0000","last_became_peered":"2023-09-23T18:23:27.787370+0000","last_unstale":"2023-09-23T18:24:10.599716+0000","last_undegraded":"2023-09-23T18:24:10.599716+0000","last_fullsized":"2023-09-23T18:24:10.599716+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:39.524762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":675,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596469+0000","last_change":"2023-09-23T18:24:05.487558+0000","last_active":"2023-09-23T18:24:10.596469+0000","last_peered":"2023-09-23T18:24:10.596469+0000","last_clean":"2023-09-23T18:24:10.596469+0000","last_became_active":"2023-09-23T18:24:05.487266+0000","last_became_peered":"2023-09-23T18:24:05.487266+0000","last_unstale":"2023-09-23T18:24:10.596469+0000","last_undegraded":"2023-09-23T18:24:10.596469+0000","last_fullsized":"2023-09-23T18:24:10.596469+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:23:41.504763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":630,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.594394+0000","last_change":"2023-09-23T18:23:53.308220+0000","last_active":"2023-09-23T18:24:10.594394+0000","last_peered":"2023-09-23T18:24:10.594394+0000","last_clean":"2023-09-23T18:24:10.594394+0000","last_became_active":"2023-09-23T18:23:53.307884+0000","last_became_peered":"2023-09-23T18:23:53.307884+0000","last_unstale":"2023-09-23T18:24:10.594394+0000","last_undegraded":"2023-09-23T18:24:10.594394+0000","last_fullsized":"2023-09-23T18:24:10.594394+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:11.834311+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":216,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585908+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:24:10.585908+0000","last_peered":"2023-09-23T18:24:10.585908+0000","last_clean":"2023-09-23T18:24:10.585908+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:24:10.585908+0000","last_undegraded":"2023-09-23T18:24:10.585908+0000","last_fullsized":"2023-09-23T18:24:10.585908+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":343,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608534+0000","last_change":"2023-09-23T18:24:08.582024+0000","last_active":"2023-09-23T18:24:10.608534+0000","last_peered":"2023-09-23T18:24:10.608534+0000","last_clean":"2023-09-23T18:24:10.608534+0000","last_became_active":"2023-09-23T18:24:08.581698+0000","last_became_peered":"2023-09-23T18:24:08.581698+0000","last_unstale":"2023-09-23T18:24:10.608534+0000","last_undegraded":"2023-09-23T18:24:10.608534+0000","last_fullsized":"2023-09-23T18:24:10.608534+0000","mapping_epoch":696,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":697,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:22.813974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":202,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586325+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:24:10.586325+0000","last_peered":"2023-09-23T18:24:10.586325+0000","last_clean":"2023-09-23T18:24:10.586325+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:24:10.586325+0000","last_undegraded":"2023-09-23T18:24:10.586325+0000","last_fullsized":"2023-09-23T18:24:10.586325+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":343,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608344+0000","last_change":"2023-09-23T18:23:33.839503+0000","last_active":"2023-09-23T18:24:10.608344+0000","last_peered":"2023-09-23T18:24:10.608344+0000","last_clean":"2023-09-23T18:24:10.608344+0000","last_became_active":"2023-09-23T18:23:33.838993+0000","last_became_peered":"2023-09-23T18:23:33.838993+0000","last_unstale":"2023-09-23T18:24:10.608344+0000","last_undegraded":"2023-09-23T18:24:10.608344+0000","last_fullsized":"2023-09-23T18:24:10.608344+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:08.671873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":698,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.607390+0000","last_change":"2023-09-23T18:23:15.704773+0000","last_active":"2023-09-23T18:24:10.607390+0000","last_peered":"2023-09-23T18:24:10.607390+0000","last_clean":"2023-09-23T18:24:10.607390+0000","last_became_active":"2023-09-23T18:23:15.704418+0000","last_became_peered":"2023-09-23T18:23:15.704418+0000","last_unstale":"2023-09-23T18:24:10.607390+0000","last_undegraded":"2023-09-23T18:24:10.607390+0000","last_fullsized":"2023-09-23T18:24:10.607390+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:53.408914+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":504,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.594573+0000","last_change":"2023-09-23T18:23:53.308023+0000","last_active":"2023-09-23T18:24:10.594573+0000","last_peered":"2023-09-23T18:24:10.594573+0000","last_clean":"2023-09-23T18:24:10.594573+0000","last_became_active":"2023-09-23T18:23:53.307691+0000","last_became_peered":"2023-09-23T18:23:53.307691+0000","last_unstale":"2023-09-23T18:24:10.594573+0000","last_undegraded":"2023-09-23T18:24:10.594573+0000","last_fullsized":"2023-09-23T18:24:10.594573+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:55:27.932947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":168,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.595446+0000","last_change":"2023-09-23T18:23:53.305705+0000","last_active":"2023-09-23T18:24:10.595446+0000","last_peered":"2023-09-23T18:24:10.595446+0000","last_clean":"2023-09-23T18:24:10.595446+0000","last_became_active":"2023-09-23T18:23:53.305403+0000","last_became_peered":"2023-09-23T18:23:53.305403+0000","last_unstale":"2023-09-23T18:24:10.595446+0000","last_undegraded":"2023-09-23T18:24:10.595446+0000","last_fullsized":"2023-09-23T18:24:10.595446+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:43.513337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":1020,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593143+0000","last_change":"2023-09-23T18:24:11.593143+0000","last_active":"2023-09-23T18:24:11.593143+0000","last_peered":"2023-09-23T18:24:11.593143+0000","last_clean":"2023-09-23T18:24:11.593143+0000","last_became_active":"2023-09-23T18:24:11.592852+0000","last_became_peered":"2023-09-23T18:24:11.592852+0000","last_unstale":"2023-09-23T18:24:11.593143+0000","last_undegraded":"2023-09-23T18:24:11.593143+0000","last_fullsized":"2023-09-23T18:24:11.593143+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:23:37.422771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:23:37.422771+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:48:27.536323+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":259,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.597979+0000","last_change":"2023-09-23T18:23:27.787952+0000","last_active":"2023-09-23T18:24:10.597979+0000","last_peered":"2023-09-23T18:24:10.597979+0000","last_clean":"2023-09-23T18:24:10.597979+0000","last_became_active":"2023-09-23T18:23:27.787603+0000","last_became_peered":"2023-09-23T18:23:27.787603+0000","last_unstale":"2023-09-23T18:24:10.597979+0000","last_undegraded":"2023-09-23T18:24:10.597979+0000","last_fullsized":"2023-09-23T18:24:10.597979+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:45.743031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":654,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606915+0000","last_change":"2023-09-23T18:23:33.843335+0000","last_active":"2023-09-23T18:24:10.606915+0000","last_peered":"2023-09-23T18:24:10.606915+0000","last_clean":"2023-09-23T18:24:10.606915+0000","last_became_active":"2023-09-23T18:23:33.842525+0000","last_became_peered":"2023-09-23T18:23:33.842525+0000","last_unstale":"2023-09-23T18:24:10.606915+0000","last_undegraded":"2023-09-23T18:24:10.606915+0000","last_fullsized":"2023-09-23T18:24:10.606915+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:07:07.884918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":123,"num_bytes_recovered":18500,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1443424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":569,"num_bytes_recovered":37748793,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001624,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":514880,"kb_used_data":11832,"kb_used_omap":13,"kb_used_meta":503026,"kb_avail":93230272,"statfs":{"total":95995035648,"available":95467798528,"internally_reserved":0,"allocated":12115968,"data_stored":10065726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":515098728},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.867,"15min":1.508},"min":{"1min":0.35299999999999998,"5min":0.224,"15min":0.20200000000000001},"max":{"1min":0.76800000000000002,"5min":33.344000000000001,"15min":33.344000000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.50700000000000001,"5min":1.641,"15min":1.583},"min":{"1min":0.34799999999999998,"5min":0.25800000000000001,"15min":0.217},"max":{"1min":0.80800000000000005,"5min":33.220999999999997,"15min":33.220999999999997},"last":0.45700000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:23:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.5209999999999999,"15min":1.075},"min":{"1min":0.379,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":0.80200000000000005,"5min":9.6929999999999996,"15min":9.6929999999999996},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.0389999999999999,"15min":0.92900000000000005},"min":{"1min":0.32700000000000001,"5min":0.28399999999999997,"15min":0.247},"max":{"1min":0.78900000000000003,"5min":6.077,"15min":6.077},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:24:01 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.139,"15min":1.214},"min":{"1min":0.40799999999999997,"5min":0.33300000000000002,"15min":0.27100000000000002},"max":{"1min":0.83399999999999996,"5min":6.5389999999999997,"15min":7.8979999999999997},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.55400000000000005,"5min":1.2450000000000001,"15min":1.605},"min":{"1min":0.34300000000000003,"5min":0.34000000000000002,"15min":0.248},"max":{"1min":0.755,"5min":8.5120000000000005,"15min":8.5120000000000005},"last":0.501}]},{"osd":3,"last update":"Sat Sep 23 18:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":1.292,"15min":2.101},"min":{"1min":0.35899999999999999,"5min":0.32100000000000001,"15min":0.27100000000000002},"max":{"1min":0.72599999999999998,"5min":6.032,"15min":54.622999999999998},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.622,"5min":1.5609999999999999,"15min":1.99},"min":{"1min":0.38500000000000001,"5min":0.375,"15min":0.249},"max":{"1min":0.872,"5min":16.297999999999998,"15min":52.475999999999999},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:24:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":1.246,"15min":1.4750000000000001},"min":{"1min":0.42899999999999999,"5min":0.26200000000000001,"15min":0.23999999999999999},"max":{"1min":0.72499999999999998,"5min":19.151,"15min":29.782},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.623,"5min":1,"15min":1.4419999999999999},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.26200000000000001},"max":{"1min":0.85099999999999998,"5min":6.6959999999999997,"15min":40.277000000000001},"last":0.72899999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.079,"15min":1.5189999999999999},"min":{"1min":0.35899999999999999,"5min":0.29099999999999998,"15min":0.26800000000000002},"max":{"1min":0.84399999999999997,"5min":8.0129999999999999,"15min":15.587999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.3899999999999999,"15min":2.0579999999999998},"min":{"1min":0.40100000000000002,"5min":0.29299999999999998,"15min":0.253},"max":{"1min":0.77800000000000002,"5min":11.433,"15min":16.221},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:24:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.425,"15min":1.6459999999999999},"min":{"1min":0.496,"5min":0.35599999999999998,"15min":0.28299999999999997},"max":{"1min":0.877,"5min":12.481,"15min":15.847},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.97399999999999998,"15min":1.6259999999999999},"min":{"1min":0.44700000000000001,"5min":0.33600000000000002,"15min":0.253},"max":{"1min":0.81799999999999995,"5min":8.6120000000000001,"15min":16.975999999999999},"last":0.58399999999999996}]}]},{"osd":6,"up_from":163,"seq":700079669560,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":642292,"kb_used_data":7788,"kb_used_omap":7,"kb_used_meta":634488,"kb_avail":93102860,"statfs":{"total":95995035648,"available":95337328640,"internally_reserved":0,"allocated":7974912,"data_stored":5936287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":649716516},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":2.2639999999999998,"15min":1.7809999999999999},"min":{"1min":0.39100000000000001,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":0.745,"5min":79.203999999999994,"15min":79.203999999999994},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":2.0030000000000001,"15min":1.9019999999999999},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.23300000000000001},"max":{"1min":0.80500000000000005,"5min":78.034000000000006,"15min":78.034000000000006},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:24:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.274,"15min":1.4319999999999999},"min":{"1min":0.34999999999999998,"5min":0.28799999999999998,"15min":0.26600000000000001},"max":{"1min":0.90000000000000002,"5min":11.726000000000001,"15min":22.972000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.4239999999999999,"15min":1.429},"min":{"1min":0.32800000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.232,"5min":12.336,"15min":22.664000000000001},"last":0.64000000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.226,"15min":1.286},"min":{"1min":0.38600000000000001,"5min":0.32900000000000001,"15min":0.25700000000000001},"max":{"1min":0.875,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.85299999999999998},{"interface":"front","average":{"1min":0.621,"5min":1.4610000000000001,"15min":1.5049999999999999},"min":{"1min":0.437,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":1.071,"5min":8.2690000000000001,"15min":14.771000000000001},"last":0.80300000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":3.2770000000000001,"15min":2.25},"min":{"1min":0.39300000000000002,"5min":0.32500000000000001,"15min":0.24099999999999999},"max":{"1min":1.004,"5min":103.348,"15min":103.348},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":3.7370000000000001,"15min":2.6059999999999999},"min":{"1min":0.44600000000000001,"5min":0.32100000000000001,"15min":0.28699999999999998},"max":{"1min":1.0249999999999999,"5min":103.40300000000001,"15min":103.40300000000001},"last":0.66700000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.76500000000000001,"15min":1.294},"min":{"1min":0.34499999999999997,"5min":0.24199999999999999,"15min":0.23999999999999999},"max":{"1min":0.81899999999999995,"5min":6.0730000000000004,"15min":8.6059999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.92900000000000005,"15min":1.3089999999999999},"min":{"1min":0.42399999999999999,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":1.048,"5min":7.8319999999999999,"15min":9.2260000000000009},"last":0.76400000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.177,"15min":1.734},"min":{"1min":0.372,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":0.96299999999999997,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.63400000000000001,"5min":1.1899999999999999,"15min":2.0009999999999999},"min":{"1min":0.40100000000000002,"5min":0.32000000000000001,"15min":0.28599999999999998},"max":{"1min":1.165,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.78500000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.0980000000000001,"15min":1.5960000000000001},"min":{"1min":0.45200000000000001,"5min":0.34899999999999998,"15min":0.28000000000000003},"max":{"1min":1.0840000000000001,"5min":6.4320000000000004,"15min":13.460000000000001},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.66700000000000004,"5min":1.1799999999999999,"15min":1.609},"min":{"1min":0.36099999999999999,"5min":0.35599999999999998,"15min":0.318},"max":{"1min":1.125,"5min":8.6890000000000001,"15min":12.404},"last":0.81299999999999994}]}]},{"osd":1,"up_from":545,"seq":2340757176408,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":361844,"kb_used_data":109804,"kb_used_omap":13,"kb_used_meta":252018,"kb_avail":93383308,"statfs":{"total":95995035648,"available":95624507392,"internally_reserved":0,"allocated":112439296,"data_stored":109068614,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":258067085},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:23:32 2023","interfaces":[{"interface":"back","average":{"1min":0.38,"5min":1.329,"15min":0.70299999999999996},"min":{"1min":0.23799999999999999,"5min":0.19900000000000001,"15min":0.189},"max":{"1min":0.68700000000000006,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.432},{"interface":"front","average":{"1min":0.36799999999999999,"5min":1.282,"15min":0.754},"min":{"1min":0.26400000000000001,"5min":0.20999999999999999,"15min":0.20999999999999999},"max":{"1min":0.55500000000000005,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.40300000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:23:32 2023","interfaces":[{"interface":"back","average":{"1min":0.378,"5min":1.4179999999999999,"15min":0.78000000000000003},"min":{"1min":0.25600000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.59199999999999997,"5min":11.02,"15min":11.02},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.39400000000000002,"5min":0.96599999999999997,"15min":0.65800000000000003},"min":{"1min":0.27900000000000003,"5min":0.249,"15min":0.249},"max":{"1min":0.61399999999999999,"5min":5.1440000000000001,"15min":5.1440000000000001},"last":0.63600000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:23:32 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":1.772,"15min":0.84799999999999998},"min":{"1min":0.30599999999999999,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":0.65300000000000002,"5min":14.962,"15min":14.962},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.502,"5min":1.575,"15min":0.86599999999999999},"min":{"1min":0.34200000000000003,"5min":0.318,"15min":0.318},"max":{"1min":1.6120000000000001,"5min":19.762,"15min":19.762},"last":0.496}]},{"osd":4,"last update":"Sat Sep 23 18:23:32 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":1.1599999999999999,"15min":1.264},"min":{"1min":0.40799999999999997,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":0.95799999999999996,"5min":7.6609999999999996,"15min":12.663},"last":0.53500000000000003},{"interface":"front","average":{"1min":0.624,"5min":1.9450000000000001,"15min":1.657},"min":{"1min":0.47499999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.84999999999999998,"5min":11.353999999999999,"15min":13.222},"last":0.58999999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:23:32 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":1.7150000000000001,"15min":1.0529999999999999},"min":{"1min":0.45800000000000002,"5min":0.308,"15min":0.308},"max":{"1min":0.88700000000000001,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.65300000000000002,"5min":2.0219999999999998,"15min":1.21},"min":{"1min":0.499,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.91000000000000003,"5min":23.204000000000001,"15min":23.204000000000001},"last":0.61699999999999999}]},{"osd":6,"last update":"Sat Sep 23 18:23:32 2023","interfaces":[{"interface":"back","average":{"1min":0.65300000000000002,"5min":1.714,"15min":1.0780000000000001},"min":{"1min":0.42899999999999999,"5min":0.372,"15min":0.372},"max":{"1min":0.93100000000000005,"5min":23.443000000000001,"15min":23.443000000000001},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.64800000000000002,"5min":2.2799999999999998,"15min":1.2769999999999999},"min":{"1min":0.495,"5min":0.38900000000000001,"15min":0.378},"max":{"1min":0.90000000000000002,"5min":26.177,"15min":26.177},"last":0.70799999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:23:32 2023","interfaces":[{"interface":"back","average":{"1min":0.64300000000000002,"5min":1.9319999999999999,"15min":1.0980000000000001},"min":{"1min":0.436,"5min":0.42299999999999999,"15min":0.40000000000000002},"max":{"1min":0.871,"5min":15.01,"15min":15.01},"last":0.68600000000000005},{"interface":"front","average":{"1min":0.67200000000000004,"5min":1.8400000000000001,"15min":1.1499999999999999},"min":{"1min":0.51800000000000002,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.94399999999999995,"5min":10.988,"15min":10.988},"last":0.57699999999999996}]}]},{"osd":0,"up_from":93,"seq":399431958905,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000200,"kb_used_data":17216,"kb_used_omap":7,"kb_used_meta":982968,"kb_avail":92744952,"statfs":{"total":95995035648,"available":94970830848,"internally_reserved":0,"allocated":17629184,"data_stored":14474527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":1006559972},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":1.3999999999999999,"15min":0.68000000000000005},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":0.65500000000000003,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.28899999999999998},{"interface":"front","average":{"1min":0.436,"5min":1.8169999999999999,"15min":0.95399999999999996},"min":{"1min":0.29999999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.71899999999999997,"5min":22.978000000000002,"15min":22.978000000000002},"last":0.39100000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:24:11 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":1.8919999999999999,"15min":1.387},"min":{"1min":0.27900000000000003,"5min":0.25,"15min":0.185},"max":{"1min":0.57099999999999995,"5min":15.002000000000001,"15min":18.550999999999998},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.673,"15min":1.0960000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.22},"max":{"1min":0.55900000000000005,"5min":23.161999999999999,"15min":23.161999999999999},"last":0.33200000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.8360000000000001,"15min":1.512},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.20000000000000001},"max":{"1min":0.72599999999999998,"5min":15.173999999999999,"15min":15.173999999999999},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.496,"5min":1.9710000000000001,"15min":1.5860000000000001},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":0.78200000000000003,"5min":15.135,"15min":15.135},"last":0.40899999999999997}]},{"osd":4,"last update":"Sat Sep 23 18:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.6020000000000001,"15min":1.831},"min":{"1min":0.44600000000000001,"5min":0.33300000000000002,"15min":0.30099999999999999},"max":{"1min":0.85599999999999998,"5min":17.741,"15min":57.746000000000002},"last":0.55100000000000005},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.6439999999999999,"15min":1.607},"min":{"1min":0.51600000000000001,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":0.93300000000000005,"5min":15.079000000000001,"15min":57.816000000000003},"last":0.45200000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":1.484,"15min":1.429},"min":{"1min":0.42599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.5830000000000002,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.78600000000000003,"5min":2.1110000000000002,"15min":1.5920000000000001},"min":{"1min":0.50600000000000001,"5min":0.35499999999999998,"15min":0.32200000000000001},"max":{"1min":2.2240000000000002,"5min":15.047000000000001,"15min":15.047000000000001},"last":0.52300000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":1.4430000000000001,"15min":1.6719999999999999},"min":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.32000000000000001},"max":{"1min":0.94599999999999995,"5min":7.6900000000000004,"15min":9.1170000000000009},"last":0.57099999999999995},{"interface":"front","average":{"1min":0.69599999999999995,"5min":2.2069999999999999,"15min":1.778},"min":{"1min":0.48699999999999999,"5min":0.442,"15min":0.315},"max":{"1min":1.0089999999999999,"5min":22.893000000000001,"15min":22.893000000000001},"last":0.53500000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:23:58 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.0659999999999998,"15min":1.7290000000000001},"min":{"1min":0.436,"5min":0.40699999999999997,"15min":0.36599999999999999},"max":{"1min":2.7370000000000001,"5min":22.945,"15min":22.945},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.71799999999999997,"5min":1.669,"15min":1.7130000000000001},"min":{"1min":0.47399999999999998,"5min":0.42799999999999999,"15min":0.36199999999999999},"max":{"1min":1.0029999999999999,"5min":7.7519999999999998,"15min":10.404},"last":0.51300000000000001}]}]},{"osd":2,"up_from":482,"seq":2070174236791,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161484,"kb_used_data":13252,"kb_used_omap":22,"kb_used_meta":148201,"kb_avail":93583668,"statfs":{"total":95995035648,"available":95829676032,"internally_reserved":0,"allocated":13570048,"data_stored":10233567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22605,"internal_metadata":151758771},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":1.484,"15min":0.88200000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19700000000000001},"max":{"1min":0.56000000000000005,"5min":9.6359999999999992,"15min":9.6359999999999992},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.34399999999999997,"5min":1.7769999999999999,"15min":1.202},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20100000000000001},"max":{"1min":0.49299999999999999,"5min":13.472,"15min":13.472},"last":0.38200000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":1.4039999999999999,"15min":0.80200000000000005},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54200000000000004,"5min":13.538,"15min":13.538},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.40000000000000002,"5min":1.6339999999999999,"15min":0.96799999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.20999999999999999},"max":{"1min":0.66300000000000003,"5min":16.567,"15min":16.567},"last":0.44800000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":1.9750000000000001,"15min":1.3089999999999999},"min":{"1min":0.30199999999999999,"5min":0.29399999999999998,"15min":0.224},"max":{"1min":0.623,"5min":15.926,"15min":15.926},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.39400000000000002,"5min":1.7,"15min":1.2949999999999999},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.23499999999999999},"max":{"1min":0.66000000000000003,"5min":16.619,"15min":16.619},"last":0.40500000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:24:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.349,"15min":1.399},"min":{"1min":0.435,"5min":0.375,"15min":0.30299999999999999},"max":{"1min":0.90500000000000003,"5min":9.532,"15min":11.334},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.262,"15min":1.121},"min":{"1min":0.51300000000000001,"5min":0.33300000000000002,"15min":0.31900000000000001},"max":{"1min":0.89400000000000002,"5min":5.5229999999999997,"15min":11.244},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.4330000000000001,"15min":1.8049999999999999},"min":{"1min":0.40999999999999998,"5min":0.36899999999999999,"15min":0.308},"max":{"1min":0.94799999999999995,"5min":9.4960000000000004,"15min":11.852},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.4650000000000001,"15min":1.4630000000000001},"min":{"1min":0.47099999999999997,"5min":0.34200000000000003,"15min":0.32600000000000001},"max":{"1min":0.81200000000000006,"5min":8.3849999999999998,"15min":10.942},"last":0.58599999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:24:12 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.7170000000000001,"15min":1.7849999999999999},"min":{"1min":0.46200000000000002,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":0.79800000000000004,"5min":29.489000000000001,"15min":29.489000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":1.734,"15min":2.133},"min":{"1min":0.45500000000000002,"5min":0.39700000000000002,"15min":0.38700000000000001},"max":{"1min":0.92300000000000004,"5min":29.408999999999999,"15min":29.408999999999999},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":1.419,"15min":3.2210000000000001},"min":{"1min":0.42999999999999999,"5min":0.40999999999999998,"15min":0.35499999999999998},"max":{"1min":0.88,"5min":13.106,"15min":76.966999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.4470000000000001,"15min":3.359},"min":{"1min":0.442,"5min":0.442,"15min":0.41699999999999998},"max":{"1min":0.84899999999999998,"5min":9.5619999999999994,"15min":76.936999999999998},"last":0.66800000000000004}]}]},{"osd":3,"up_from":237,"seq":1017907249420,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818912,"kb_used_data":13400,"kb_used_omap":3,"kb_used_meta":805500,"kb_avail":92926240,"statfs":{"total":95995035648,"available":95156469760,"internally_reserved":0,"allocated":13721600,"data_stored":10312138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":824832744},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":1.8200000000000001,"15min":1.7989999999999999},"min":{"1min":0.23200000000000001,"5min":0.192,"15min":0.183},"max":{"1min":0.52200000000000002,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.37,"5min":1.603,"15min":1.871},"min":{"1min":0.25900000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.50700000000000001,"5min":11.468,"15min":194.661},"last":0.35999999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:23:31 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":1.8169999999999999,"15min":0.95199999999999996},"min":{"1min":0.23699999999999999,"5min":0.218,"15min":0.218},"max":{"1min":0.49199999999999999,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":2.7269999999999999,"15min":1.379},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.875,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.438}]},{"osd":2,"last update":"Sat Sep 23 18:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":1.944,"15min":1.395},"min":{"1min":0.313,"5min":0.249,"15min":0.21099999999999999},"max":{"1min":0.58199999999999996,"5min":18.722999999999999,"15min":18.722999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.45100000000000001,"5min":1.5580000000000001,"15min":1.2789999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.19600000000000001},"max":{"1min":0.64400000000000002,"5min":11.565,"15min":11.565},"last":0.41699999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.5089999999999999,"15min":1.486},"min":{"1min":0.438,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":0.81899999999999995,"5min":11.4,"15min":21.062999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.5880000000000001,"15min":1.7490000000000001},"min":{"1min":0.45300000000000001,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":0.753,"5min":9.1630000000000003,"15min":21.128},"last":0.752}]},{"osd":5,"last update":"Sat Sep 23 18:23:48 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":2.0230000000000001,"15min":1.5900000000000001},"min":{"1min":0.441,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":0.77100000000000002,"5min":19.434999999999999,"15min":19.434999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.5780000000000001,"15min":1.569},"min":{"1min":0.443,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":0.73199999999999998,"5min":17.135999999999999,"15min":17.135999999999999},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.6890000000000001,"15min":1.518},"min":{"1min":0.48199999999999998,"5min":0.39300000000000002,"15min":0.32700000000000001},"max":{"1min":0.76000000000000001,"5min":17.192,"15min":17.192},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.5469999999999999,"15min":1.502},"min":{"1min":0.45500000000000002,"5min":0.35699999999999998,"15min":0.30499999999999999},"max":{"1min":0.75,"5min":11.442,"15min":14.321},"last":0.69399999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.6799999999999999,"15min":1.7370000000000001},"min":{"1min":0.45700000000000002,"5min":0.38700000000000001,"15min":0.35799999999999998},"max":{"1min":0.79500000000000004,"5min":9.1929999999999996,"15min":15.972},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.7,"15min":1.599},"min":{"1min":0.54600000000000004,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":0.81299999999999994,"5min":17.164000000000001,"15min":17.164000000000001},"last":0.74199999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019370,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":718204,"kb_used_data":11572,"kb_used_omap":0,"kb_used_meta":706624,"kb_avail":93026948,"statfs":{"total":95995035648,"available":95259594752,"internally_reserved":0,"allocated":11849728,"data_stored":9803291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":723582976},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:14 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.903,"15min":1.46},"min":{"1min":0.33400000000000002,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":0.77700000000000002,"5min":72.754999999999995,"15min":72.754999999999995},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.966,"15min":1.7390000000000001},"min":{"1min":0.36799999999999999,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":0.85399999999999998,"5min":71.533000000000001,"15min":71.533000000000001},"last":0.63100000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:24:27 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.22,"15min":1.1279999999999999},"min":{"1min":0.435,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.73499999999999999,"5min":10.619,"15min":44.146999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.623,"5min":1.093,"15min":1.0580000000000001},"min":{"1min":0.46400000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.88700000000000001,"5min":7.5700000000000003,"15min":43.497999999999998},"last":0.40400000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:24:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.3620000000000001,"15min":1.5920000000000001},"min":{"1min":0.45100000000000001,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":0.76000000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.2969999999999999,"15min":1.5569999999999999},"min":{"1min":0.40000000000000002,"5min":0.27000000000000002,"15min":0.253},"max":{"1min":0.89300000000000002,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.67700000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":2.4380000000000002,"15min":2.0950000000000002},"min":{"1min":0.39300000000000002,"5min":0.32000000000000001,"15min":0.23799999999999999},"max":{"1min":0.81200000000000006,"5min":74.781999999999996,"15min":74.781999999999996},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.622,"5min":2.335,"15min":2.0510000000000002},"min":{"1min":0.42699999999999999,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":0.81799999999999995,"5min":72.152000000000001,"15min":72.152000000000001},"last":0.58599999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":2.0169999999999999,"15min":2.613},"min":{"1min":0.39900000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.76600000000000001,"5min":91.432000000000002,"15min":91.432000000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.9610000000000001,"15min":2.379},"min":{"1min":0.44700000000000001,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":0.745,"5min":91.626999999999995,"15min":91.626999999999995},"last":0.45600000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:24:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.8560000000000001,"15min":1.6859999999999999},"min":{"1min":0.432,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":0.79600000000000004,"5min":91.688999999999993,"15min":91.688999999999993},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.65800000000000003,"5min":2.0920000000000001,"15min":1.891},"min":{"1min":0.504,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":0.90200000000000002,"5min":91.837000000000003,"15min":91.837000000000003},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:24:14 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":1.329,"15min":1.821},"min":{"1min":0.45500000000000002,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":0.96999999999999997,"5min":18.283000000000001,"15min":19.298999999999999},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.1080000000000001,"15min":1.5569999999999999},"min":{"1min":0.46999999999999997,"5min":0.36399999999999999,"15min":0.28399999999999997},"max":{"1min":1.0329999999999999,"5min":9.4730000000000008,"15min":12.800000000000001},"last":0.69899999999999995}]}]},{"osd":5,"up_from":420,"seq":1803886264466,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":381808,"kb_used_data":7720,"kb_used_omap":9,"kb_used_meta":374070,"kb_avail":93363344,"statfs":{"total":95995035648,"available":95604064256,"internally_reserved":0,"allocated":7905280,"data_stored":5867115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":383048125},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":1.7230000000000001,"15min":1.1990000000000001},"min":{"1min":0.30399999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.86899999999999999,"5min":77.162000000000006,"15min":77.162000000000006},"last":0.437},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.2229999999999999,"15min":1.738},"min":{"1min":0.33900000000000002,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":0.73299999999999998,"5min":75.784999999999997,"15min":75.784999999999997},"last":0.61699999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:24:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.4199999999999999,"15min":1.331},"min":{"1min":0.48599999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.70799999999999996,"5min":12.752000000000001,"15min":23.946000000000002},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":1.4139999999999999,"15min":1.2410000000000001},"min":{"1min":0.35099999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.746,"5min":9.2759999999999998,"15min":23.268999999999998},"last":0.56100000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.361,"15min":1.476},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.21199999999999999},"max":{"1min":0.84699999999999998,"5min":10.705,"15min":10.705},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.3740000000000001,"15min":1.8440000000000001},"min":{"1min":0.39900000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":0.879,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.65900000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":2.3319999999999999,"15min":1.613},"min":{"1min":0.41899999999999998,"5min":0.317,"15min":0.25600000000000001},"max":{"1min":0.93999999999999995,"5min":79.016999999999996,"15min":79.016999999999996},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.63200000000000001,"5min":2.4870000000000001,"15min":1.843},"min":{"1min":0.46200000000000002,"5min":0.32900000000000001,"15min":0.313},"max":{"1min":0.92800000000000005,"5min":76.188999999999993,"15min":76.188999999999993},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":1.3089999999999999,"15min":1.8129999999999999},"min":{"1min":0.30499999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.65900000000000003,"5min":8.2270000000000003,"15min":12.747999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.3149999999999999,"15min":1.5349999999999999},"min":{"1min":0.34399999999999997,"5min":0.254,"15min":0.254},"max":{"1min":0.89900000000000002,"5min":8.5559999999999992,"15min":9.0690000000000008},"last":0.58799999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.038,"15min":1.252},"min":{"1min":0.32100000000000001,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":0.97399999999999998,"5min":6.7859999999999996,"15min":9.0350000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.3580000000000001,"15min":1.5980000000000001},"min":{"1min":0.37,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.96099999999999997,"5min":7.8410000000000002,"15min":12.504},"last":0.69999999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.351,"15min":1.6930000000000001},"min":{"1min":0.36799999999999999,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":0.75800000000000001,"5min":10.528,"15min":13.506},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.69699999999999995,"5min":1.1839999999999999,"15min":1.5009999999999999},"min":{"1min":0.44700000000000001,"5min":0.36099999999999999,"15min":0.29799999999999999},"max":{"1min":1.0720000000000001,"5min":12.702,"15min":13.138},"last":0.68200000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:24:35.201 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 -- ceph pg dump --format=json 2023-09-23T18:24:35.809 INFO:teuthology.orchestra.run.smithi055.stderr:Inferring config /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/mon.a/config 2023-09-23T18:24:36.011 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:35 smithi055 ceph-mon[179668]: from='client.48290 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:24:36.011 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:35 smithi055 ceph-mon[184174]: from='client.48290 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:24:36.076 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:35 smithi099 ceph-mon[167076]: from='client.48290 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-09-23T18:24:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:36 smithi099 ceph-mon[167076]: pgmap v1677: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:37.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:36 smithi099 ceph-mon[167076]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:37.179 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:36 smithi055 ceph-mon[179668]: pgmap v1677: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:37.180 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:36 smithi055 ceph-mon[179668]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:37.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:36 smithi055 ceph-mon[184174]: pgmap v1677: 103 pgs: 103 active+clean; 8.6 MiB data, 4.4 GiB used, 711 GiB / 715 GiB avail 2023-09-23T18:24:37.180 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:36 smithi055 ceph-mon[184174]: from='mgr.25793 172.21.15.55:0/2943628677' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-09-23T18:24:37.201 INFO:teuthology.orchestra.run.smithi055.stdout: 2023-09-23T18:24:37.209 INFO:teuthology.orchestra.run.smithi055.stderr:dumped all 2023-09-23T18:24:37.568 INFO:teuthology.orchestra.run.smithi055.stdout:{"pg_ready":true,"pg_map":{"version":1677,"stamp":"2023-09-23T18:24:35.263275+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":8984499,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1751,"num_write_kb":25419,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":697,"num_bytes_recovered":39210717,"num_keys_recovered":50,"num_objects_omap":16,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":6996,"ondisk_log_size":6996,"up":309,"acting":309,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":310,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":4590024,"kb_used_data":182984,"kb_used_omap":76,"kb_used_meta":4406899,"kb_avail":745371192,"statfs":{"total":767960285184,"available":763260100608,"internally_reserved":0,"allocated":187375616,"data_stored":165930865,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":78507,"internal_metadata":4512664917},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.003265"},"pg_stats":[{"pgid":"1.18","version":"89'161","reported_seq":1034,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585587+0000","last_change":"2023-09-23T18:23:15.701666+0000","last_active":"2023-09-23T18:24:10.585587+0000","last_peered":"2023-09-23T18:24:10.585587+0000","last_clean":"2023-09-23T18:24:10.585587+0000","last_became_active":"2023-09-23T18:23:15.700793+0000","last_became_peered":"2023-09-23T18:23:15.700793+0000","last_unstale":"2023-09-23T18:24:10.585587+0000","last_undegraded":"2023-09-23T18:24:10.585587+0000","last_fullsized":"2023-09-23T18:24:10.585587+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:50.194394+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:50.194394+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:53:50.372583+0000","scrub_duration":0.023647478999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":262688,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":788064,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.1a","version":"67'1","reported_seq":368,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608340+0000","last_change":"2023-09-23T18:22:44.528866+0000","last_active":"2023-09-23T18:24:10.608340+0000","last_peered":"2023-09-23T18:24:10.608340+0000","last_clean":"2023-09-23T18:24:10.608340+0000","last_became_active":"2023-09-23T18:22:44.528499+0000","last_became_peered":"2023-09-23T18:22:44.528499+0000","last_unstale":"2023-09-23T18:24:10.608340+0000","last_undegraded":"2023-09-23T18:24:10.608340+0000","last_fullsized":"2023-09-23T18:24:10.608340+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:01:26.288744+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":498,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593014+0000","last_change":"2023-09-23T18:23:53.309716+0000","last_active":"2023-09-23T18:24:10.593014+0000","last_peered":"2023-09-23T18:24:10.593014+0000","last_clean":"2023-09-23T18:24:10.593014+0000","last_became_active":"2023-09-23T18:23:53.309419+0000","last_became_peered":"2023-09-23T18:23:53.309419+0000","last_unstale":"2023-09-23T18:24:10.593014+0000","last_undegraded":"2023-09-23T18:24:10.593014+0000","last_fullsized":"2023-09-23T18:24:10.593014+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:05:01.509606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1b","version":"89'161","reported_seq":1032,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585759+0000","last_change":"2023-09-23T18:23:18.705964+0000","last_active":"2023-09-23T18:24:10.585759+0000","last_peered":"2023-09-23T18:24:10.585759+0000","last_clean":"2023-09-23T18:24:10.585759+0000","last_became_active":"2023-09-23T18:23:18.705607+0000","last_became_peered":"2023-09-23T18:23:18.705607+0000","last_unstale":"2023-09-23T18:24:10.585759+0000","last_undegraded":"2023-09-23T18:24:10.585759+0000","last_fullsized":"2023-09-23T18:24:10.585759+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:39.327099+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:39.327099+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:41:54.370381+0000","scrub_duration":0.001408082,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,0],"acting":[5,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":288,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593981+0000","last_change":"2023-09-23T18:24:11.592707+0000","last_active":"2023-09-23T18:24:11.593981+0000","last_peered":"2023-09-23T18:24:11.593981+0000","last_clean":"2023-09-23T18:24:11.593981+0000","last_became_active":"2023-09-23T18:24:11.592366+0000","last_became_peered":"2023-09-23T18:24:11.592366+0000","last_unstale":"2023-09-23T18:24:11.593981+0000","last_undegraded":"2023-09-23T18:24:11.593981+0000","last_fullsized":"2023-09-23T18:24:11.593981+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:01:55.735609+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":46,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596010+0000","last_change":"2023-09-23T18:23:27.789164+0000","last_active":"2023-09-23T18:24:10.596010+0000","last_peered":"2023-09-23T18:24:10.596010+0000","last_clean":"2023-09-23T18:24:10.596010+0000","last_became_active":"2023-09-23T18:23:27.788868+0000","last_became_peered":"2023-09-23T18:23:27.788868+0000","last_unstale":"2023-09-23T18:24:10.596010+0000","last_undegraded":"2023-09-23T18:24:10.596010+0000","last_fullsized":"2023-09-23T18:24:10.596010+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:38:45.134497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.1a","version":"89'161","reported_seq":991,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608485+0000","last_change":"2023-09-23T18:23:15.706818+0000","last_active":"2023-09-23T18:24:10.608485+0000","last_peered":"2023-09-23T18:24:10.608485+0000","last_clean":"2023-09-23T18:24:10.608485+0000","last_became_active":"2023-09-23T18:23:15.706430+0000","last_became_peered":"2023-09-23T18:23:15.706430+0000","last_unstale":"2023-09-23T18:24:10.608485+0000","last_undegraded":"2023-09-23T18:24:10.608485+0000","last_fullsized":"2023-09-23T18:24:10.608485+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:27.208891+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:27.208891+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:53:19.880623+0000","scrub_duration":0.006436247,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,1],"acting":[4,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":132,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593691+0000","last_change":"2023-09-23T18:23:53.307443+0000","last_active":"2023-09-23T18:24:10.593691+0000","last_peered":"2023-09-23T18:24:10.593691+0000","last_clean":"2023-09-23T18:24:10.593691+0000","last_became_active":"2023-09-23T18:23:53.306809+0000","last_became_peered":"2023-09-23T18:23:53.306809+0000","last_unstale":"2023-09-23T18:24:10.593691+0000","last_undegraded":"2023-09-23T18:24:10.593691+0000","last_fullsized":"2023-09-23T18:24:10.593691+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:22:26.431718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":274,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605891+0000","last_change":"2023-09-23T18:17:21.832487+0000","last_active":"2023-09-23T18:24:10.605891+0000","last_peered":"2023-09-23T18:24:10.605891+0000","last_clean":"2023-09-23T18:24:10.605891+0000","last_became_active":"2023-09-23T18:17:21.831953+0000","last_became_peered":"2023-09-23T18:17:21.831953+0000","last_unstale":"2023-09-23T18:24:10.605891+0000","last_undegraded":"2023-09-23T18:24:10.605891+0000","last_fullsized":"2023-09-23T18:24:10.605891+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:03:53.702136+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.15","version":"89'161","reported_seq":692,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605978+0000","last_change":"2023-09-23T18:23:15.696286+0000","last_active":"2023-09-23T18:24:10.605978+0000","last_peered":"2023-09-23T18:24:10.605978+0000","last_clean":"2023-09-23T18:24:10.605978+0000","last_became_active":"2023-09-23T18:23:15.695903+0000","last_became_peered":"2023-09-23T18:23:15.695903+0000","last_unstale":"2023-09-23T18:24:10.605978+0000","last_undegraded":"2023-09-23T18:24:10.605978+0000","last_fullsized":"2023-09-23T18:24:10.605978+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:56.388529+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:56.388529+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:02.891204+0000","scrub_duration":0.003981587,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,1],"acting":[7,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":225,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.598004+0000","last_change":"2023-09-23T18:23:27.788735+0000","last_active":"2023-09-23T18:24:10.598004+0000","last_peered":"2023-09-23T18:24:10.598004+0000","last_clean":"2023-09-23T18:24:10.598004+0000","last_became_active":"2023-09-23T18:23:27.788405+0000","last_became_peered":"2023-09-23T18:23:27.788405+0000","last_unstale":"2023-09-23T18:24:10.598004+0000","last_undegraded":"2023-09-23T18:24:10.598004+0000","last_fullsized":"2023-09-23T18:24:10.598004+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:55:17.598909+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":290,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.594040+0000","last_change":"2023-09-23T18:17:20.799761+0000","last_active":"2023-09-23T18:24:10.594040+0000","last_peered":"2023-09-23T18:24:10.594040+0000","last_clean":"2023-09-23T18:24:10.594040+0000","last_became_active":"2023-09-23T18:17:20.799545+0000","last_became_peered":"2023-09-23T18:17:20.799545+0000","last_unstale":"2023-09-23T18:24:10.594040+0000","last_undegraded":"2023-09-23T18:24:10.594040+0000","last_fullsized":"2023-09-23T18:24:10.594040+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:02:45.696193+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.14","version":"89'161","reported_seq":862,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.643341+0000","last_change":"2023-09-23T18:24:05.497052+0000","last_active":"2023-09-23T18:24:11.643341+0000","last_peered":"2023-09-23T18:24:11.643341+0000","last_clean":"2023-09-23T18:24:11.643341+0000","last_became_active":"2023-09-23T18:24:05.496700+0000","last_became_peered":"2023-09-23T18:24:05.496700+0000","last_unstale":"2023-09-23T18:24:11.643341+0000","last_undegraded":"2023-09-23T18:24:11.643341+0000","last_fullsized":"2023-09-23T18:24:11.643341+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:23.589768+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:23.589768+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:45:07.451298+0000","scrub_duration":0.00144658,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,4],"acting":[2,7,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":254,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586210+0000","last_change":"2023-09-23T18:23:12.646379+0000","last_active":"2023-09-23T18:24:10.586210+0000","last_peered":"2023-09-23T18:24:10.586210+0000","last_clean":"2023-09-23T18:24:10.586210+0000","last_became_active":"2023-09-23T18:23:12.646006+0000","last_became_peered":"2023-09-23T18:23:12.646006+0000","last_unstale":"2023-09-23T18:24:10.586210+0000","last_undegraded":"2023-09-23T18:24:10.586210+0000","last_fullsized":"2023-09-23T18:24:10.586210+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:50:33.403421+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":298,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596325+0000","last_change":"2023-09-23T18:23:53.308722+0000","last_active":"2023-09-23T18:24:10.596325+0000","last_peered":"2023-09-23T18:24:10.596325+0000","last_clean":"2023-09-23T18:24:10.596325+0000","last_became_active":"2023-09-23T18:23:53.308425+0000","last_became_peered":"2023-09-23T18:23:53.308425+0000","last_unstale":"2023-09-23T18:24:10.596325+0000","last_undegraded":"2023-09-23T18:24:10.596325+0000","last_fullsized":"2023-09-23T18:24:10.596325+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:35:36.073008+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.17","version":"89'161","reported_seq":1093,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586285+0000","last_change":"2023-09-23T18:23:18.704918+0000","last_active":"2023-09-23T18:24:10.586285+0000","last_peered":"2023-09-23T18:24:10.586285+0000","last_clean":"2023-09-23T18:24:10.586285+0000","last_became_active":"2023-09-23T18:23:18.704443+0000","last_became_peered":"2023-09-23T18:23:18.704443+0000","last_unstale":"2023-09-23T18:24:10.586285+0000","last_undegraded":"2023-09-23T18:24:10.586285+0000","last_fullsized":"2023-09-23T18:24:10.586285+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:49.188565+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:49.188565+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:56:12.850187+0000","scrub_duration":0.0061041350000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,6],"acting":[5,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.15","version":"67'3","reported_seq":45,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605825+0000","last_change":"2023-09-23T18:23:12.672052+0000","last_active":"2023-09-23T18:24:10.605825+0000","last_peered":"2023-09-23T18:24:10.605825+0000","last_clean":"2023-09-23T18:24:10.605825+0000","last_became_active":"2023-09-23T18:23:12.645704+0000","last_became_peered":"2023-09-23T18:23:12.645704+0000","last_unstale":"2023-09-23T18:24:10.605825+0000","last_undegraded":"2023-09-23T18:24:10.605825+0000","last_fullsized":"2023-09-23T18:24:10.605825+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":3,"log_dups_size":0,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:25:34.094770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":22,"num_bytes_recovered":990,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":460,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.644733+0000","last_change":"2023-09-23T18:24:11.644260+0000","last_active":"2023-09-23T18:24:11.644733+0000","last_peered":"2023-09-23T18:24:11.644733+0000","last_clean":"2023-09-23T18:24:11.644733+0000","last_became_active":"2023-09-23T18:24:11.643918+0000","last_became_peered":"2023-09-23T18:24:11.643918+0000","last_unstale":"2023-09-23T18:24:11.644733+0000","last_undegraded":"2023-09-23T18:24:11.644733+0000","last_fullsized":"2023-09-23T18:24:11.644733+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:14.921661+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.16","version":"89'161","reported_seq":1097,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593941+0000","last_change":"2023-09-23T18:24:11.591673+0000","last_active":"2023-09-23T18:24:11.593941+0000","last_peered":"2023-09-23T18:24:11.593941+0000","last_clean":"2023-09-23T18:24:11.593941+0000","last_became_active":"2023-09-23T18:24:11.591320+0000","last_became_peered":"2023-09-23T18:24:11.591320+0000","last_unstale":"2023-09-23T18:24:11.593941+0000","last_undegraded":"2023-09-23T18:24:11.593941+0000","last_fullsized":"2023-09-23T18:24:11.593941+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:43.794457+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:43.794457+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:56:45.793533+0000","scrub_duration":0.0072157009999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,3],"acting":[1,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"89'22","reported_seq":447,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608587+0000","last_change":"2023-09-23T18:23:33.840584+0000","last_active":"2023-09-23T18:24:10.608587+0000","last_peered":"2023-09-23T18:24:10.608587+0000","last_clean":"2023-09-23T18:24:10.608587+0000","last_became_active":"2023-09-23T18:23:33.840167+0000","last_became_peered":"2023-09-23T18:23:33.840167+0000","last_unstale":"2023-09-23T18:24:10.608587+0000","last_undegraded":"2023-09-23T18:24:10.608587+0000","last_fullsized":"2023-09-23T18:24:10.608587+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":22,"log_dups_size":0,"ondisk_log_size":22,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:03:19.850055+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":4734,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":22,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":12,"num_bytes_recovered":14202,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":440,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608727+0000","last_change":"2023-09-23T18:23:33.843086+0000","last_active":"2023-09-23T18:24:10.608727+0000","last_peered":"2023-09-23T18:24:10.608727+0000","last_clean":"2023-09-23T18:24:10.608727+0000","last_became_active":"2023-09-23T18:23:33.842456+0000","last_became_peered":"2023-09-23T18:23:33.842456+0000","last_unstale":"2023-09-23T18:24:10.608727+0000","last_undegraded":"2023-09-23T18:24:10.608727+0000","last_fullsized":"2023-09-23T18:24:10.608727+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:41:52.924489+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.11","version":"89'161","reported_seq":954,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596989+0000","last_change":"2023-09-23T18:23:27.793962+0000","last_active":"2023-09-23T18:24:10.596989+0000","last_peered":"2023-09-23T18:24:10.596989+0000","last_clean":"2023-09-23T18:24:10.596989+0000","last_became_active":"2023-09-23T18:23:27.791134+0000","last_became_peered":"2023-09-23T18:23:27.791134+0000","last_unstale":"2023-09-23T18:24:10.596989+0000","last_undegraded":"2023-09-23T18:24:10.596989+0000","last_fullsized":"2023-09-23T18:24:10.596989+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:47.181409+0000","last_deep_scrub":"89'161","last_deep_scrub_stamp":"2023-09-23T18:09:24.869576+0000","last_clean_scrub_stamp":"2023-09-23T18:16:47.181409+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:47.069746+0000","scrub_duration":0.012026096999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.13","version":"67'2","reported_seq":45,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.604746+0000","last_change":"2023-09-23T18:23:12.652460+0000","last_active":"2023-09-23T18:24:10.604746+0000","last_peered":"2023-09-23T18:24:10.604746+0000","last_clean":"2023-09-23T18:24:10.604746+0000","last_became_active":"2023-09-23T18:23:12.637983+0000","last_became_peered":"2023-09-23T18:23:12.637983+0000","last_unstale":"2023-09-23T18:24:10.604746+0000","last_undegraded":"2023-09-23T18:24:10.604746+0000","last_fullsized":"2023-09-23T18:24:10.604746+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:42:45.669563+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":170,"num_keys_recovered":50,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":33,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.594194+0000","last_change":"2023-09-23T18:23:53.306077+0000","last_active":"2023-09-23T18:24:10.594194+0000","last_peered":"2023-09-23T18:24:10.594194+0000","last_clean":"2023-09-23T18:24:10.594194+0000","last_became_active":"2023-09-23T18:23:53.305594+0000","last_became_peered":"2023-09-23T18:23:53.305594+0000","last_unstale":"2023-09-23T18:24:10.594194+0000","last_undegraded":"2023-09-23T18:24:10.594194+0000","last_fullsized":"2023-09-23T18:24:10.594194+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:07:05.212893+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.10","version":"89'161","reported_seq":915,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585393+0000","last_change":"2023-09-23T18:23:15.701886+0000","last_active":"2023-09-23T18:24:10.585393+0000","last_peered":"2023-09-23T18:24:10.585393+0000","last_clean":"2023-09-23T18:24:10.585393+0000","last_became_active":"2023-09-23T18:23:15.701306+0000","last_became_peered":"2023-09-23T18:23:15.701306+0000","last_unstale":"2023-09-23T18:24:10.585393+0000","last_undegraded":"2023-09-23T18:24:10.585393+0000","last_fullsized":"2023-09-23T18:24:10.585393+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:13:54.269442+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:13:54.269442+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:05:36.144212+0000","scrub_duration":0.0044808850000000004,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":46,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596811+0000","last_change":"2023-09-23T18:23:27.790518+0000","last_active":"2023-09-23T18:24:10.596811+0000","last_peered":"2023-09-23T18:24:10.596811+0000","last_clean":"2023-09-23T18:24:10.596811+0000","last_became_active":"2023-09-23T18:23:27.789716+0000","last_became_peered":"2023-09-23T18:23:27.789716+0000","last_unstale":"2023-09-23T18:24:10.596811+0000","last_undegraded":"2023-09-23T18:24:10.596811+0000","last_fullsized":"2023-09-23T18:24:10.596811+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:43:26.760633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":218,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586265+0000","last_change":"2023-09-23T18:23:12.646863+0000","last_active":"2023-09-23T18:24:10.586265+0000","last_peered":"2023-09-23T18:24:10.586265+0000","last_clean":"2023-09-23T18:24:10.586265+0000","last_became_active":"2023-09-23T18:23:12.646501+0000","last_became_peered":"2023-09-23T18:23:12.646501+0000","last_unstale":"2023-09-23T18:24:10.586265+0000","last_undegraded":"2023-09-23T18:24:10.586265+0000","last_fullsized":"2023-09-23T18:24:10.586265+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:50:55.839398+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.13","version":"89'161","reported_seq":925,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.594089+0000","last_change":"2023-09-23T18:24:11.593422+0000","last_active":"2023-09-23T18:24:11.594089+0000","last_peered":"2023-09-23T18:24:11.594089+0000","last_clean":"2023-09-23T18:24:11.594089+0000","last_became_active":"2023-09-23T18:24:11.592470+0000","last_became_peered":"2023-09-23T18:24:11.592470+0000","last_unstale":"2023-09-23T18:24:11.594089+0000","last_undegraded":"2023-09-23T18:24:11.594089+0000","last_fullsized":"2023-09-23T18:24:11.594089+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:44.768363+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:44.768363+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:15:39.773928+0000","scrub_duration":0.009148537,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":296,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.604833+0000","last_change":"2023-09-23T18:23:46.831702+0000","last_active":"2023-09-23T18:24:10.604833+0000","last_peered":"2023-09-23T18:24:10.604833+0000","last_clean":"2023-09-23T18:24:10.604833+0000","last_became_active":"2023-09-23T18:23:46.831430+0000","last_became_peered":"2023-09-23T18:23:46.831430+0000","last_unstale":"2023-09-23T18:24:10.604833+0000","last_undegraded":"2023-09-23T18:24:10.604833+0000","last_fullsized":"2023-09-23T18:24:10.604833+0000","mapping_epoch":686,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":687,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:49:56.532410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":331,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596108+0000","last_change":"2023-09-23T18:23:27.789041+0000","last_active":"2023-09-23T18:24:10.596108+0000","last_peered":"2023-09-23T18:24:10.596108+0000","last_clean":"2023-09-23T18:24:10.596108+0000","last_became_active":"2023-09-23T18:23:27.788471+0000","last_became_peered":"2023-09-23T18:23:27.788471+0000","last_unstale":"2023-09-23T18:24:10.596108+0000","last_undegraded":"2023-09-23T18:24:10.596108+0000","last_fullsized":"2023-09-23T18:24:10.596108+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:09:26.454700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.12","version":"89'161","reported_seq":816,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593416+0000","last_change":"2023-09-23T18:23:53.308352+0000","last_active":"2023-09-23T18:24:10.593416+0000","last_peered":"2023-09-23T18:24:10.593416+0000","last_clean":"2023-09-23T18:24:10.593416+0000","last_became_active":"2023-09-23T18:23:53.308040+0000","last_became_peered":"2023-09-23T18:23:53.308040+0000","last_unstale":"2023-09-23T18:24:10.593416+0000","last_undegraded":"2023-09-23T18:24:10.593416+0000","last_fullsized":"2023-09-23T18:24:10.593416+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:28.210590+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:28.210590+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:10:27.222024+0000","scrub_duration":0.0041571619999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"67'8","reported_seq":878,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605562+0000","last_change":"2023-09-23T18:23:15.712056+0000","last_active":"2023-09-23T18:24:10.605562+0000","last_peered":"2023-09-23T18:24:10.605562+0000","last_clean":"2023-09-23T18:24:10.605562+0000","last_became_active":"2023-09-23T18:23:15.690682+0000","last_became_peered":"2023-09-23T18:23:15.690682+0000","last_unstale":"2023-09-23T18:24:10.605562+0000","last_undegraded":"2023-09-23T18:24:10.605562+0000","last_fullsized":"2023-09-23T18:24:10.605562+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":8,"log_dups_size":0,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:18.252430+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":32,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":13,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596649+0000","last_change":"2023-09-23T18:24:08.744171+0000","last_active":"2023-09-23T18:24:10.596649+0000","last_peered":"2023-09-23T18:24:10.596649+0000","last_clean":"2023-09-23T18:24:10.596649+0000","last_became_active":"2023-09-23T18:24:08.743847+0000","last_became_peered":"2023-09-23T18:24:08.743847+0000","last_unstale":"2023-09-23T18:24:10.596649+0000","last_undegraded":"2023-09-23T18:24:10.596649+0000","last_fullsized":"2023-09-23T18:24:10.596649+0000","mapping_epoch":696,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":697,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:24:27.473003+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,1],"acting":[2,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.d","version":"89'161","reported_seq":692,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606034+0000","last_change":"2023-09-23T18:23:15.706446+0000","last_active":"2023-09-23T18:24:10.606034+0000","last_peered":"2023-09-23T18:24:10.606034+0000","last_clean":"2023-09-23T18:24:10.606034+0000","last_became_active":"2023-09-23T18:23:15.706089+0000","last_became_peered":"2023-09-23T18:23:15.706089+0000","last_unstale":"2023-09-23T18:24:10.606034+0000","last_undegraded":"2023-09-23T18:24:10.606034+0000","last_fullsized":"2023-09-23T18:24:10.606034+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:58.454062+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:58.454062+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:10:02.005632+0000","scrub_duration":0.0050227839999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"67'1","reported_seq":57,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606579+0000","last_change":"2023-09-23T18:23:15.721550+0000","last_active":"2023-09-23T18:24:10.606579+0000","last_peered":"2023-09-23T18:24:10.606579+0000","last_clean":"2023-09-23T18:24:10.606579+0000","last_became_active":"2023-09-23T18:23:15.700750+0000","last_became_peered":"2023-09-23T18:23:15.700750+0000","last_unstale":"2023-09-23T18:24:10.606579+0000","last_undegraded":"2023-09-23T18:24:10.606579+0000","last_fullsized":"2023-09-23T18:24:10.606579+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:11:03.517711+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":88,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":218,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585437+0000","last_change":"2023-09-23T18:23:12.640414+0000","last_active":"2023-09-23T18:24:10.585437+0000","last_peered":"2023-09-23T18:24:10.585437+0000","last_clean":"2023-09-23T18:24:10.585437+0000","last_became_active":"2023-09-23T18:23:12.639849+0000","last_became_peered":"2023-09-23T18:23:12.639849+0000","last_unstale":"2023-09-23T18:24:10.585437+0000","last_undegraded":"2023-09-23T18:24:10.585437+0000","last_fullsized":"2023-09-23T18:24:10.585437+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:02:34.282521+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.c","version":"89'161","reported_seq":874,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.594927+0000","last_change":"2023-09-23T18:23:18.704895+0000","last_active":"2023-09-23T18:24:10.594927+0000","last_peered":"2023-09-23T18:24:10.594927+0000","last_clean":"2023-09-23T18:24:10.594927+0000","last_became_active":"2023-09-23T18:23:18.704488+0000","last_became_peered":"2023-09-23T18:23:18.704488+0000","last_unstale":"2023-09-23T18:24:10.594927+0000","last_undegraded":"2023-09-23T18:24:10.594927+0000","last_fullsized":"2023-09-23T18:24:10.594927+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:21:51.851269+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:21:51.851269+0000","objects_scrubbed":1,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:03:44.733888+0000","scrub_duration":0.001933539,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":327680,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":2,"num_bytes_recovered":655360,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.e","version":"67'2","reported_seq":43,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606273+0000","last_change":"2023-09-23T18:23:12.656626+0000","last_active":"2023-09-23T18:24:10.606273+0000","last_peered":"2023-09-23T18:24:10.606273+0000","last_clean":"2023-09-23T18:24:10.606273+0000","last_became_active":"2023-09-23T18:23:12.645323+0000","last_became_peered":"2023-09-23T18:23:12.645323+0000","last_unstale":"2023-09-23T18:24:10.606273+0000","last_undegraded":"2023-09-23T18:24:10.606273+0000","last_fullsized":"2023-09-23T18:24:10.606273+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:19:28.109499+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":315,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608142+0000","last_change":"2023-09-23T18:18:46.438449+0000","last_active":"2023-09-23T18:24:10.608142+0000","last_peered":"2023-09-23T18:24:10.608142+0000","last_clean":"2023-09-23T18:24:10.608142+0000","last_became_active":"2023-09-23T18:18:46.438095+0000","last_became_peered":"2023-09-23T18:18:46.438095+0000","last_unstale":"2023-09-23T18:24:10.608142+0000","last_undegraded":"2023-09-23T18:24:10.608142+0000","last_fullsized":"2023-09-23T18:24:10.608142+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:59:24.328202+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,3],"acting":[4,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.f","version":"89'161","reported_seq":713,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605782+0000","last_change":"2023-09-23T18:18:04.932513+0000","last_active":"2023-09-23T18:24:10.605782+0000","last_peered":"2023-09-23T18:24:10.605782+0000","last_clean":"2023-09-23T18:24:10.605782+0000","last_became_active":"2023-09-23T18:16:11.305599+0000","last_became_peered":"2023-09-23T18:16:11.305599+0000","last_unstale":"2023-09-23T18:24:10.605782+0000","last_undegraded":"2023-09-23T18:24:10.605782+0000","last_fullsized":"2023-09-23T18:24:10.605782+0000","mapping_epoch":512,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":513,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:04.932424+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:04.932424+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:20.422769+0000","scrub_duration":0.0029136000000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,4],"acting":[6,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"67'1","reported_seq":43,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605903+0000","last_change":"2023-09-23T18:23:12.668907+0000","last_active":"2023-09-23T18:24:10.605903+0000","last_peered":"2023-09-23T18:24:10.605903+0000","last_clean":"2023-09-23T18:24:10.605903+0000","last_became_active":"2023-09-23T18:23:12.646244+0000","last_became_peered":"2023-09-23T18:23:12.646244+0000","last_unstale":"2023-09-23T18:24:10.605903+0000","last_undegraded":"2023-09-23T18:24:10.605903+0000","last_fullsized":"2023-09-23T18:24:10.605903+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:07:50.424121+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":4,"num_bytes_recovered":280,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":284,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606672+0000","last_change":"2023-09-23T18:23:12.639745+0000","last_active":"2023-09-23T18:24:10.606672+0000","last_peered":"2023-09-23T18:24:10.606672+0000","last_clean":"2023-09-23T18:24:10.606672+0000","last_became_active":"2023-09-23T18:23:12.639398+0000","last_became_peered":"2023-09-23T18:23:12.639398+0000","last_unstale":"2023-09-23T18:24:10.606672+0000","last_undegraded":"2023-09-23T18:24:10.606672+0000","last_fullsized":"2023-09-23T18:24:10.606672+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:46:19.476844+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.e","version":"89'161","reported_seq":886,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608896+0000","last_change":"2023-09-23T18:23:33.839754+0000","last_active":"2023-09-23T18:24:10.608896+0000","last_peered":"2023-09-23T18:24:10.608896+0000","last_clean":"2023-09-23T18:24:10.608896+0000","last_became_active":"2023-09-23T18:23:33.839201+0000","last_became_peered":"2023-09-23T18:23:33.839201+0000","last_unstale":"2023-09-23T18:24:10.608896+0000","last_undegraded":"2023-09-23T18:24:10.608896+0000","last_fullsized":"2023-09-23T18:24:10.608896+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:17:54.492748+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:17:54.492748+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:08.161741+0000","scrub_duration":0.007530826,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,3],"acting":[4,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":222,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586126+0000","last_change":"2023-09-23T18:18:46.439904+0000","last_active":"2023-09-23T18:24:10.586126+0000","last_peered":"2023-09-23T18:24:10.586126+0000","last_clean":"2023-09-23T18:24:10.586126+0000","last_became_active":"2023-09-23T18:18:46.439357+0000","last_became_peered":"2023-09-23T18:18:46.439357+0000","last_unstale":"2023-09-23T18:24:10.586126+0000","last_undegraded":"2023-09-23T18:24:10.586126+0000","last_fullsized":"2023-09-23T18:24:10.586126+0000","mapping_epoch":594,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":595,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:21.799326+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":37,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.600047+0000","last_change":"2023-09-23T18:23:15.699232+0000","last_active":"2023-09-23T18:24:10.600047+0000","last_peered":"2023-09-23T18:24:10.600047+0000","last_clean":"2023-09-23T18:24:10.600047+0000","last_became_active":"2023-09-23T18:23:15.698945+0000","last_became_peered":"2023-09-23T18:23:15.698945+0000","last_unstale":"2023-09-23T18:24:10.600047+0000","last_undegraded":"2023-09-23T18:24:10.600047+0000","last_fullsized":"2023-09-23T18:24:10.600047+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:43:01.104606+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.9","version":"89'161","reported_seq":1188,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.604979+0000","last_change":"2023-09-23T18:23:15.694144+0000","last_active":"2023-09-23T18:24:10.604979+0000","last_peered":"2023-09-23T18:24:10.604979+0000","last_clean":"2023-09-23T18:24:10.604979+0000","last_became_active":"2023-09-23T18:23:15.693764+0000","last_became_peered":"2023-09-23T18:23:15.693764+0000","last_unstale":"2023-09-23T18:24:10.604979+0000","last_undegraded":"2023-09-23T18:24:10.604979+0000","last_fullsized":"2023-09-23T18:24:10.604979+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:55.369314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:55.369314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:23:44.221828+0000","scrub_duration":0.0064757219999999997,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,2],"acting":[7,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.b","version":"67'2","reported_seq":460,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.592543+0000","last_change":"2023-09-23T18:23:53.309838+0000","last_active":"2023-09-23T18:24:10.592543+0000","last_peered":"2023-09-23T18:24:10.592543+0000","last_clean":"2023-09-23T18:24:10.592543+0000","last_became_active":"2023-09-23T18:23:53.309561+0000","last_became_peered":"2023-09-23T18:23:53.309561+0000","last_unstale":"2023-09-23T18:24:10.592543+0000","last_undegraded":"2023-09-23T18:24:10.592543+0000","last_fullsized":"2023-09-23T18:24:10.592543+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:44:54.966938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":236,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608460+0000","last_change":"2023-09-23T18:23:33.842661+0000","last_active":"2023-09-23T18:24:10.608460+0000","last_peered":"2023-09-23T18:24:10.608460+0000","last_clean":"2023-09-23T18:24:10.608460+0000","last_became_active":"2023-09-23T18:23:33.842267+0000","last_became_peered":"2023-09-23T18:23:33.842267+0000","last_unstale":"2023-09-23T18:24:10.608460+0000","last_undegraded":"2023-09-23T18:24:10.608460+0000","last_fullsized":"2023-09-23T18:24:10.608460+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:55:21.603948+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.8","version":"89'161","reported_seq":987,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585594+0000","last_change":"2023-09-23T18:22:06.466040+0000","last_active":"2023-09-23T18:24:10.585594+0000","last_peered":"2023-09-23T18:24:10.585594+0000","last_clean":"2023-09-23T18:24:10.585594+0000","last_became_active":"2023-09-23T18:22:06.465158+0000","last_became_peered":"2023-09-23T18:22:06.465158+0000","last_unstale":"2023-09-23T18:24:10.585594+0000","last_undegraded":"2023-09-23T18:24:10.585594+0000","last_fullsized":"2023-09-23T18:24:10.585594+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.715942+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.715942+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:42:29.587874+0000","scrub_duration":0.0067410869999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,3],"acting":[5,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":250,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605920+0000","last_change":"2023-09-23T18:17:20.800613+0000","last_active":"2023-09-23T18:24:10.605920+0000","last_peered":"2023-09-23T18:24:10.605920+0000","last_clean":"2023-09-23T18:24:10.605920+0000","last_became_active":"2023-09-23T18:17:20.799494+0000","last_became_peered":"2023-09-23T18:17:20.799494+0000","last_unstale":"2023-09-23T18:24:10.605920+0000","last_undegraded":"2023-09-23T18:24:10.605920+0000","last_fullsized":"2023-09-23T18:24:10.605920+0000","mapping_epoch":545,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":546,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:28:16.836002+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":493,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608981+0000","last_change":"2023-09-23T18:22:44.527193+0000","last_active":"2023-09-23T18:24:10.608981+0000","last_peered":"2023-09-23T18:24:10.608981+0000","last_clean":"2023-09-23T18:24:10.608981+0000","last_became_active":"2023-09-23T18:22:44.526903+0000","last_became_peered":"2023-09-23T18:22:44.526903+0000","last_unstale":"2023-09-23T18:24:10.608981+0000","last_undegraded":"2023-09-23T18:24:10.608981+0000","last_fullsized":"2023-09-23T18:24:10.608981+0000","mapping_epoch":659,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":660,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:09:09.126029+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.b","version":"89'161","reported_seq":916,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593951+0000","last_change":"2023-09-23T18:24:11.592309+0000","last_active":"2023-09-23T18:24:11.593951+0000","last_peered":"2023-09-23T18:24:11.593951+0000","last_clean":"2023-09-23T18:24:11.593951+0000","last_became_active":"2023-09-23T18:24:11.591723+0000","last_became_peered":"2023-09-23T18:24:11.591723+0000","last_unstale":"2023-09-23T18:24:11.593951+0000","last_undegraded":"2023-09-23T18:24:11.593951+0000","last_fullsized":"2023-09-23T18:24:11.593951+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":171,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.620165+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.620165+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:44:33.935341+0000","scrub_duration":0.0047349269999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,5],"acting":[1,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":501,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608331+0000","last_change":"2023-09-23T18:23:33.840098+0000","last_active":"2023-09-23T18:24:10.608331+0000","last_peered":"2023-09-23T18:24:10.608331+0000","last_clean":"2023-09-23T18:24:10.608331+0000","last_became_active":"2023-09-23T18:23:33.839509+0000","last_became_peered":"2023-09-23T18:23:33.839509+0000","last_unstale":"2023-09-23T18:24:10.608331+0000","last_undegraded":"2023-09-23T18:24:10.608331+0000","last_fullsized":"2023-09-23T18:24:10.608331+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:38:48.488620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":233,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608908+0000","last_change":"2023-09-23T18:23:33.843126+0000","last_active":"2023-09-23T18:24:10.608908+0000","last_peered":"2023-09-23T18:24:10.608908+0000","last_clean":"2023-09-23T18:24:10.608908+0000","last_became_active":"2023-09-23T18:23:33.842811+0000","last_became_peered":"2023-09-23T18:23:33.842811+0000","last_unstale":"2023-09-23T18:24:10.608908+0000","last_undegraded":"2023-09-23T18:24:10.608908+0000","last_fullsized":"2023-09-23T18:24:10.608908+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:06:18.853788+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.a","version":"89'161","reported_seq":970,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585775+0000","last_change":"2023-09-23T18:22:06.468619+0000","last_active":"2023-09-23T18:24:10.585775+0000","last_peered":"2023-09-23T18:24:10.585775+0000","last_clean":"2023-09-23T18:24:10.585775+0000","last_became_active":"2023-09-23T18:22:06.468197+0000","last_became_peered":"2023-09-23T18:22:06.468197+0000","last_unstale":"2023-09-23T18:24:10.585775+0000","last_undegraded":"2023-09-23T18:24:10.585775+0000","last_fullsized":"2023-09-23T18:24:10.585775+0000","mapping_epoch":637,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":638,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:46.190308+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:46.190308+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:20:10.149437+0000","scrub_duration":0.0067918559999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,6],"acting":[5,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":126,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593886+0000","last_change":"2023-09-23T18:23:12.639264+0000","last_active":"2023-09-23T18:24:10.593886+0000","last_peered":"2023-09-23T18:24:10.593886+0000","last_clean":"2023-09-23T18:24:10.593886+0000","last_became_active":"2023-09-23T18:23:12.638919+0000","last_became_peered":"2023-09-23T18:23:12.638919+0000","last_unstale":"2023-09-23T18:24:10.593886+0000","last_undegraded":"2023-09-23T18:24:10.593886+0000","last_fullsized":"2023-09-23T18:24:10.593886+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:42:06.409381+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":420,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593952+0000","last_change":"2023-09-23T18:24:11.591698+0000","last_active":"2023-09-23T18:24:11.593952+0000","last_peered":"2023-09-23T18:24:11.593952+0000","last_clean":"2023-09-23T18:24:11.593952+0000","last_became_active":"2023-09-23T18:24:11.591424+0000","last_became_peered":"2023-09-23T18:24:11.591424+0000","last_unstale":"2023-09-23T18:24:11.593952+0000","last_undegraded":"2023-09-23T18:24:11.593952+0000","last_fullsized":"2023-09-23T18:24:11.593952+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:26:30.613497+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.7","version":"89'161","reported_seq":829,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606291+0000","last_change":"2023-09-23T18:23:15.694981+0000","last_active":"2023-09-23T18:24:10.606291+0000","last_peered":"2023-09-23T18:24:10.606291+0000","last_clean":"2023-09-23T18:24:10.606291+0000","last_became_active":"2023-09-23T18:23:15.694607+0000","last_became_peered":"2023-09-23T18:23:15.694607+0000","last_unstale":"2023-09-23T18:24:10.606291+0000","last_undegraded":"2023-09-23T18:24:10.606291+0000","last_fullsized":"2023-09-23T18:24:10.606291+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:00.732635+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:00.732635+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:25:52.640350+0000","scrub_duration":0.0017860409999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,3,7],"acting":[6,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.4","version":"653'291","reported_seq":1684,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.594063+0000","last_change":"2023-09-23T18:24:11.592902+0000","last_active":"2023-09-23T18:24:11.594063+0000","last_peered":"2023-09-23T18:24:11.594063+0000","last_clean":"2023-09-23T18:24:11.594063+0000","last_became_active":"2023-09-23T18:24:11.592020+0000","last_became_peered":"2023-09-23T18:24:11.592020+0000","last_unstale":"2023-09-23T18:24:11.594063+0000","last_undegraded":"2023-09-23T18:24:11.594063+0000","last_fullsized":"2023-09-23T18:24:11.594063+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":291,"log_dups_size":0,"ondisk_log_size":291,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:34:40.158357+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00047292100000000001,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1326,"num_read_kb":5772,"num_write":300,"num_write_kb":4517,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":124,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":242,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586479+0000","last_change":"2023-09-23T18:22:03.462589+0000","last_active":"2023-09-23T18:24:10.586479+0000","last_peered":"2023-09-23T18:24:10.586479+0000","last_clean":"2023-09-23T18:24:10.586479+0000","last_became_active":"2023-09-23T18:22:03.462239+0000","last_became_peered":"2023-09-23T18:22:03.462239+0000","last_unstale":"2023-09-23T18:24:10.586479+0000","last_undegraded":"2023-09-23T18:24:10.586479+0000","last_fullsized":"2023-09-23T18:24:10.586479+0000","mapping_epoch":634,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":635,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:38:03.020770+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":275,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593970+0000","last_change":"2023-09-23T18:24:11.591973+0000","last_active":"2023-09-23T18:24:11.593970+0000","last_peered":"2023-09-23T18:24:11.593970+0000","last_clean":"2023-09-23T18:24:11.593970+0000","last_became_active":"2023-09-23T18:24:11.591608+0000","last_became_peered":"2023-09-23T18:24:11.591608+0000","last_unstale":"2023-09-23T18:24:11.593970+0000","last_undegraded":"2023-09-23T18:24:11.593970+0000","last_fullsized":"2023-09-23T18:24:11.593970+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:42:02.296074+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.1","version":"89'161","reported_seq":941,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585568+0000","last_change":"2023-09-23T18:23:18.701192+0000","last_active":"2023-09-23T18:24:10.585568+0000","last_peered":"2023-09-23T18:24:10.585568+0000","last_clean":"2023-09-23T18:24:10.585568+0000","last_became_active":"2023-09-23T18:23:18.700482+0000","last_became_peered":"2023-09-23T18:23:18.700482+0000","last_unstale":"2023-09-23T18:24:10.585568+0000","last_undegraded":"2023-09-23T18:24:10.585568+0000","last_fullsized":"2023-09-23T18:24:10.585568+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:04.204813+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:04.204813+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:54:30.748035+0000","scrub_duration":0.0014467530000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,0,2],"acting":[5,0,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.2","version":"653'323","reported_seq":2298,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585612+0000","last_change":"2023-09-23T18:22:28.473090+0000","last_active":"2023-09-23T18:24:10.585612+0000","last_peered":"2023-09-23T18:24:10.585612+0000","last_clean":"2023-09-23T18:24:10.585612+0000","last_became_active":"2023-09-23T18:17:21.785354+0000","last_became_peered":"2023-09-23T18:17:21.785354+0000","last_unstale":"2023-09-23T18:24:10.585612+0000","last_undegraded":"2023-09-23T18:24:10.585612+0000","last_fullsized":"2023-09-23T18:24:10.585612+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":323,"log_dups_size":0,"ondisk_log_size":323,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:14:19.293612+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001426075,"stat_sum":{"num_bytes":19,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":3093,"num_read_kb":5799,"num_write":265,"num_write_kb":3199,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":33,"num_bytes_recovered":57,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":239,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608533+0000","last_change":"2023-09-23T18:23:33.843234+0000","last_active":"2023-09-23T18:24:10.608533+0000","last_peered":"2023-09-23T18:24:10.608533+0000","last_clean":"2023-09-23T18:24:10.608533+0000","last_became_active":"2023-09-23T18:23:33.842353+0000","last_became_peered":"2023-09-23T18:23:33.842353+0000","last_unstale":"2023-09-23T18:24:10.608533+0000","last_undegraded":"2023-09-23T18:24:10.608533+0000","last_fullsized":"2023-09-23T18:24:10.608533+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:28:47.177729+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":460,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.645801+0000","last_change":"2023-09-23T18:24:11.645801+0000","last_active":"2023-09-23T18:24:11.645801+0000","last_peered":"2023-09-23T18:24:11.645801+0000","last_clean":"2023-09-23T18:24:11.645801+0000","last_became_active":"2023-09-23T18:24:11.644474+0000","last_became_peered":"2023-09-23T18:24:11.644474+0000","last_unstale":"2023-09-23T18:24:11.645801+0000","last_undegraded":"2023-09-23T18:24:11.645801+0000","last_fullsized":"2023-09-23T18:24:11.645801+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:50:20.338293+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"1.2","version":"89'161","reported_seq":742,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605721+0000","last_change":"2023-09-23T18:23:46.520844+0000","last_active":"2023-09-23T18:24:10.605721+0000","last_peered":"2023-09-23T18:24:10.605721+0000","last_clean":"2023-09-23T18:24:10.605721+0000","last_became_active":"2023-09-23T18:23:46.520494+0000","last_became_peered":"2023-09-23T18:23:46.520494+0000","last_unstale":"2023-09-23T18:24:10.605721+0000","last_undegraded":"2023-09-23T18:24:10.605721+0000","last_fullsized":"2023-09-23T18:24:10.605721+0000","mapping_epoch":686,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":687,"parent":"0.0","parent_split_bits":4,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:14.334662+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:14.334662+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:20:03.685739+0000","scrub_duration":0.003854592,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.1","version":"653'200","reported_seq":634,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596938+0000","last_change":"2023-09-23T18:24:05.490508+0000","last_active":"2023-09-23T18:24:10.596938+0000","last_peered":"2023-09-23T18:24:10.596938+0000","last_clean":"2023-09-23T18:24:10.596938+0000","last_became_active":"2023-09-23T18:24:05.489857+0000","last_became_peered":"2023-09-23T18:24:05.489857+0000","last_unstale":"2023-09-23T18:24:10.596938+0000","last_undegraded":"2023-09-23T18:24:10.596938+0000","last_fullsized":"2023-09-23T18:24:10.596938+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":200,"log_dups_size":0,"ondisk_log_size":200,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:30.566706+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0022753629999999999,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1032,"num_read_kb":1283,"num_write":194,"num_write_kb":5509,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":108,"num_bytes_recovered":29360128,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":467,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.645159+0000","last_change":"2023-09-23T18:24:11.644700+0000","last_active":"2023-09-23T18:24:11.645159+0000","last_peered":"2023-09-23T18:24:11.645159+0000","last_clean":"2023-09-23T18:24:11.645159+0000","last_became_active":"2023-09-23T18:24:11.644359+0000","last_became_peered":"2023-09-23T18:24:11.644359+0000","last_unstale":"2023-09-23T18:24:11.645159+0000","last_undegraded":"2023-09-23T18:24:11.645159+0000","last_fullsized":"2023-09-23T18:24:11.645159+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:29:31.725247+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":263,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593932+0000","last_change":"2023-09-23T18:24:11.591624+0000","last_active":"2023-09-23T18:24:11.593932+0000","last_peered":"2023-09-23T18:24:11.593932+0000","last_clean":"2023-09-23T18:24:11.593932+0000","last_became_active":"2023-09-23T18:24:11.591276+0000","last_became_peered":"2023-09-23T18:24:11.591276+0000","last_unstale":"2023-09-23T18:24:11.593932+0000","last_undegraded":"2023-09-23T18:24:11.593932+0000","last_fullsized":"2023-09-23T18:24:11.593932+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:22:57.325094+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"653'474","reported_seq":2388,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585726+0000","last_change":"2023-09-23T18:23:15.703265+0000","last_active":"2023-09-23T18:24:10.585726+0000","last_peered":"2023-09-23T18:24:10.585726+0000","last_clean":"2023-09-23T18:24:10.585726+0000","last_became_active":"2023-09-23T18:23:15.701955+0000","last_became_peered":"2023-09-23T18:23:15.701955+0000","last_unstale":"2023-09-23T18:24:10.585726+0000","last_undegraded":"2023-09-23T18:24:10.585726+0000","last_fullsized":"2023-09-23T18:24:10.585726+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":474,"log_dups_size":0,"ondisk_log_size":474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:29:03.254370+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.0014671619999999999,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":2565,"num_read_kb":5830,"num_write":410,"num_write_kb":4052,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":98,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"89'161","reported_seq":690,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605329+0000","last_change":"2023-09-23T18:23:18.709442+0000","last_active":"2023-09-23T18:24:10.605329+0000","last_peered":"2023-09-23T18:24:10.605329+0000","last_clean":"2023-09-23T18:24:10.605329+0000","last_became_active":"2023-09-23T18:23:18.709023+0000","last_became_peered":"2023-09-23T18:23:18.709023+0000","last_unstale":"2023-09-23T18:24:10.605329+0000","last_undegraded":"2023-09-23T18:24:10.605329+0000","last_fullsized":"2023-09-23T18:24:10.605329+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:18:05.942044+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:18:05.942044+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T04:14:59.988881+0000","scrub_duration":0.0014660739999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":117,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593329+0000","last_change":"2023-09-23T18:23:53.308051+0000","last_active":"2023-09-23T18:24:10.593329+0000","last_peered":"2023-09-23T18:24:10.593329+0000","last_clean":"2023-09-23T18:24:10.593329+0000","last_became_active":"2023-09-23T18:23:53.307760+0000","last_became_peered":"2023-09-23T18:23:53.307760+0000","last_unstale":"2023-09-23T18:24:10.593329+0000","last_undegraded":"2023-09-23T18:24:10.593329+0000","last_fullsized":"2023-09-23T18:24:10.593329+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:45:23.973027+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":263,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606149+0000","last_change":"2023-09-23T18:23:12.647798+0000","last_active":"2023-09-23T18:24:10.606149+0000","last_peered":"2023-09-23T18:24:10.606149+0000","last_clean":"2023-09-23T18:24:10.606149+0000","last_became_active":"2023-09-23T18:23:12.647441+0000","last_became_peered":"2023-09-23T18:23:12.647441+0000","last_unstale":"2023-09-23T18:24:10.606149+0000","last_undegraded":"2023-09-23T18:24:10.606149+0000","last_fullsized":"2023-09-23T18:24:10.606149+0000","mapping_epoch":669,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":670,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:56:13.708809+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"1.3","version":"89'161","reported_seq":707,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.595799+0000","last_change":"2023-09-23T18:24:11.595281+0000","last_active":"2023-09-23T18:24:11.595799+0000","last_peered":"2023-09-23T18:24:11.595799+0000","last_clean":"2023-09-23T18:24:11.595799+0000","last_became_active":"2023-09-23T18:24:11.594913+0000","last_became_peered":"2023-09-23T18:24:11.594913+0000","last_unstale":"2023-09-23T18:24:11.595799+0000","last_undegraded":"2023-09-23T18:24:11.595799+0000","last_fullsized":"2023-09-23T18:24:11.595799+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:09:16.643371+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:09:16.643371+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:47:05.054956+0000","scrub_duration":0.001672574,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.0","version":"653'156","reported_seq":524,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.605423+0000","last_change":"2023-09-23T18:23:18.708902+0000","last_active":"2023-09-23T18:24:10.605423+0000","last_peered":"2023-09-23T18:24:10.605423+0000","last_clean":"2023-09-23T18:24:10.605423+0000","last_became_active":"2023-09-23T18:23:18.708208+0000","last_became_peered":"2023-09-23T18:23:18.708208+0000","last_unstale":"2023-09-23T18:24:10.605423+0000","last_undegraded":"2023-09-23T18:24:10.605423+0000","last_fullsized":"2023-09-23T18:24:10.605423+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":156,"log_dups_size":0,"ondisk_log_size":156,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:00:57.541302+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001504954,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":805,"num_read_kb":2049,"num_write":144,"num_write_kb":1274,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":62,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":132,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.597054+0000","last_change":"2023-09-23T18:23:27.794628+0000","last_active":"2023-09-23T18:24:10.597054+0000","last_peered":"2023-09-23T18:24:10.597054+0000","last_clean":"2023-09-23T18:24:10.597054+0000","last_became_active":"2023-09-23T18:23:27.793171+0000","last_became_peered":"2023-09-23T18:23:27.793171+0000","last_unstale":"2023-09-23T18:24:10.597054+0000","last_undegraded":"2023-09-23T18:24:10.597054+0000","last_fullsized":"2023-09-23T18:24:10.597054+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:56:09.526134+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":37,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596313+0000","last_change":"2023-09-23T18:23:15.679226+0000","last_active":"2023-09-23T18:24:10.596313+0000","last_peered":"2023-09-23T18:24:10.596313+0000","last_clean":"2023-09-23T18:24:10.596313+0000","last_became_active":"2023-09-23T18:23:15.678888+0000","last_became_peered":"2023-09-23T18:23:15.678888+0000","last_unstale":"2023-09-23T18:24:10.596313+0000","last_undegraded":"2023-09-23T18:24:10.596313+0000","last_fullsized":"2023-09-23T18:24:10.596313+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:29.362618+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.6","version":"89'161","reported_seq":1036,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606587+0000","last_change":"2023-09-23T18:23:18.706776+0000","last_active":"2023-09-23T18:24:10.606587+0000","last_peered":"2023-09-23T18:24:10.606587+0000","last_clean":"2023-09-23T18:24:10.606587+0000","last_became_active":"2023-09-23T18:23:18.706386+0000","last_became_peered":"2023-09-23T18:23:18.706386+0000","last_unstale":"2023-09-23T18:24:10.606587+0000","last_undegraded":"2023-09-23T18:24:10.606587+0000","last_fullsized":"2023-09-23T18:24:10.606587+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:06:03.805314+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:06:03.805314+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:13:35.053952+0000","scrub_duration":0.0057233129999999998,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,3],"acting":[6,0,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.5","version":"653'181","reported_seq":683,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606087+0000","last_change":"2023-09-23T18:23:18.708971+0000","last_active":"2023-09-23T18:24:10.606087+0000","last_peered":"2023-09-23T18:24:10.606087+0000","last_clean":"2023-09-23T18:24:10.606087+0000","last_became_active":"2023-09-23T18:23:18.708188+0000","last_became_peered":"2023-09-23T18:23:18.708188+0000","last_unstale":"2023-09-23T18:24:10.606087+0000","last_undegraded":"2023-09-23T18:24:10.606087+0000","last_fullsized":"2023-09-23T18:24:10.606087+0000","mapping_epoch":675,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":676,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":181,"log_dups_size":0,"ondisk_log_size":181,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:05:38.993938+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.001526466,"stat_sum":{"num_bytes":4194304,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":1349,"num_read_kb":1261,"num_write":144,"num_write_kb":4223,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":47,"num_bytes_recovered":8388608,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":281,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.646196+0000","last_change":"2023-09-23T18:24:11.645815+0000","last_active":"2023-09-23T18:24:11.646196+0000","last_peered":"2023-09-23T18:24:11.646196+0000","last_clean":"2023-09-23T18:24:11.646196+0000","last_became_active":"2023-09-23T18:24:11.643987+0000","last_became_peered":"2023-09-23T18:24:11.643987+0000","last_unstale":"2023-09-23T18:24:11.646196+0000","last_undegraded":"2023-09-23T18:24:11.646196+0000","last_fullsized":"2023-09-23T18:24:11.646196+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:03:40.308962+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":249,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.599809+0000","last_change":"2023-09-23T18:23:27.782160+0000","last_active":"2023-09-23T18:24:10.599809+0000","last_peered":"2023-09-23T18:24:10.599809+0000","last_clean":"2023-09-23T18:24:10.599809+0000","last_became_active":"2023-09-23T18:23:27.781726+0000","last_became_peered":"2023-09-23T18:23:27.781726+0000","last_unstale":"2023-09-23T18:24:10.599809+0000","last_undegraded":"2023-09-23T18:24:10.599809+0000","last_fullsized":"2023-09-23T18:24:10.599809+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:20:51.328287+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"1.5","version":"89'161","reported_seq":1037,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608871+0000","last_change":"2023-09-23T18:23:33.843634+0000","last_active":"2023-09-23T18:24:10.608871+0000","last_peered":"2023-09-23T18:24:10.608871+0000","last_clean":"2023-09-23T18:24:10.608871+0000","last_became_active":"2023-09-23T18:23:33.843279+0000","last_became_peered":"2023-09-23T18:23:33.843279+0000","last_unstale":"2023-09-23T18:24:10.608871+0000","last_undegraded":"2023-09-23T18:24:10.608871+0000","last_fullsized":"2023-09-23T18:24:10.608871+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:05:54.350883+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:05:54.350883+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:23:24.463121+0000","scrub_duration":0.00385747,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,6],"acting":[4,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.7","version":"67'2","reported_seq":297,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593708+0000","last_change":"2023-09-23T18:23:53.305574+0000","last_active":"2023-09-23T18:24:10.593708+0000","last_peered":"2023-09-23T18:24:10.593708+0000","last_clean":"2023-09-23T18:24:10.593708+0000","last_became_active":"2023-09-23T18:23:53.305221+0000","last_became_peered":"2023-09-23T18:23:53.305221+0000","last_unstale":"2023-09-23T18:24:10.593708+0000","last_undegraded":"2023-09-23T18:24:10.593708+0000","last_fullsized":"2023-09-23T18:24:10.593708+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T03:10:05.291632+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":3,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"653'188","reported_seq":936,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.596005+0000","last_change":"2023-09-23T18:24:11.595534+0000","last_active":"2023-09-23T18:24:11.596005+0000","last_peered":"2023-09-23T18:24:11.596005+0000","last_clean":"2023-09-23T18:24:11.596005+0000","last_became_active":"2023-09-23T18:24:11.595006+0000","last_became_peered":"2023-09-23T18:24:11.595006+0000","last_unstale":"2023-09-23T18:24:11.596005+0000","last_undegraded":"2023-09-23T18:24:11.596005+0000","last_fullsized":"2023-09-23T18:24:11.596005+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":188,"log_dups_size":0,"ondisk_log_size":188,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:58:28.658622+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00055658400000000005,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":1067,"num_read_kb":2442,"num_write":105,"num_write_kb":1356,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":100,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.593783+0000","last_change":"2023-09-23T18:23:53.307641+0000","last_active":"2023-09-23T18:24:10.593783+0000","last_peered":"2023-09-23T18:24:10.593783+0000","last_clean":"2023-09-23T18:24:10.593783+0000","last_became_active":"2023-09-23T18:23:53.307132+0000","last_became_peered":"2023-09-23T18:23:53.307132+0000","last_unstale":"2023-09-23T18:24:10.593783+0000","last_undegraded":"2023-09-23T18:24:10.593783+0000","last_fullsized":"2023-09-23T18:24:10.593783+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T21:33:10.036096+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.4","version":"89'161","reported_seq":850,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.595041+0000","last_change":"2023-09-23T18:23:15.692819+0000","last_active":"2023-09-23T18:24:10.595041+0000","last_peered":"2023-09-23T18:24:10.595041+0000","last_clean":"2023-09-23T18:24:10.595041+0000","last_became_active":"2023-09-23T18:23:15.692460+0000","last_became_peered":"2023-09-23T18:23:15.692460+0000","last_unstale":"2023-09-23T18:24:10.595041+0000","last_undegraded":"2023-09-23T18:24:10.595041+0000","last_fullsized":"2023-09-23T18:24:10.595041+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":148,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":3,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:10.352133+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:10.352133+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T20:07:00.270208+0000","scrub_duration":0.0014339649999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,7],"acting":[3,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.7","version":"652'137","reported_seq":1410,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608406+0000","last_change":"2023-09-23T18:23:15.695165+0000","last_active":"2023-09-23T18:24:10.608406+0000","last_peered":"2023-09-23T18:24:10.608406+0000","last_clean":"2023-09-23T18:24:10.608406+0000","last_became_active":"2023-09-23T18:23:15.694501+0000","last_became_peered":"2023-09-23T18:23:15.694501+0000","last_unstale":"2023-09-23T18:24:10.608406+0000","last_undegraded":"2023-09-23T18:24:10.608406+0000","last_fullsized":"2023-09-23T18:24:10.608406+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":58,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:11.019330+0000","last_clean_scrub_stamp":"2023-09-23T17:45:11.019330+0000","objects_scrubbed":0,"log_size":137,"log_dups_size":0,"ondisk_log_size":137,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:41:17.791654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0.00044965899999999999,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":713,"num_read_kb":925,"num_write":128,"num_write_kb":1260,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":87,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"67'1","reported_seq":41,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596224+0000","last_change":"2023-09-23T18:23:15.707579+0000","last_active":"2023-09-23T18:24:10.596224+0000","last_peered":"2023-09-23T18:24:10.596224+0000","last_clean":"2023-09-23T18:24:10.596224+0000","last_became_active":"2023-09-23T18:23:15.699673+0000","last_became_peered":"2023-09-23T18:23:15.699673+0000","last_unstale":"2023-09-23T18:24:10.596224+0000","last_undegraded":"2023-09-23T18:24:10.596224+0000","last_fullsized":"2023-09-23T18:24:10.596224+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:55:15.419839+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":8,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":437,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608217+0000","last_change":"2023-09-23T18:23:33.841290+0000","last_active":"2023-09-23T18:24:10.608217+0000","last_peered":"2023-09-23T18:24:10.608217+0000","last_clean":"2023-09-23T18:24:10.608217+0000","last_became_active":"2023-09-23T18:23:33.840983+0000","last_became_peered":"2023-09-23T18:23:33.840983+0000","last_unstale":"2023-09-23T18:24:10.608217+0000","last_undegraded":"2023-09-23T18:24:10.608217+0000","last_fullsized":"2023-09-23T18:24:10.608217+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T23:18:48.677598+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.19","version":"89'161","reported_seq":630,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.595240+0000","last_change":"2023-09-23T18:23:53.307537+0000","last_active":"2023-09-23T18:24:10.595240+0000","last_peered":"2023-09-23T18:24:10.595240+0000","last_clean":"2023-09-23T18:24:10.595240+0000","last_became_active":"2023-09-23T18:23:53.307253+0000","last_became_peered":"2023-09-23T18:23:53.307253+0000","last_unstale":"2023-09-23T18:24:10.595240+0000","last_undegraded":"2023-09-23T18:24:10.595240+0000","last_fullsized":"2023-09-23T18:24:10.595240+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:48.316483+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:48.316483+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:08:28.080598+0000","scrub_duration":0.020242349999999999,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,6],"acting":[3,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"67'1","reported_seq":46,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.599716+0000","last_change":"2023-09-23T18:23:27.787712+0000","last_active":"2023-09-23T18:24:10.599716+0000","last_peered":"2023-09-23T18:24:10.599716+0000","last_clean":"2023-09-23T18:24:10.599716+0000","last_became_active":"2023-09-23T18:23:27.787370+0000","last_became_peered":"2023-09-23T18:23:27.787370+0000","last_unstale":"2023-09-23T18:24:10.599716+0000","last_undegraded":"2023-09-23T18:24:10.599716+0000","last_fullsized":"2023-09-23T18:24:10.599716+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:48:39.524762+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":675,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.596469+0000","last_change":"2023-09-23T18:24:05.487558+0000","last_active":"2023-09-23T18:24:10.596469+0000","last_peered":"2023-09-23T18:24:10.596469+0000","last_clean":"2023-09-23T18:24:10.596469+0000","last_became_active":"2023-09-23T18:24:05.487266+0000","last_became_peered":"2023-09-23T18:24:05.487266+0000","last_unstale":"2023-09-23T18:24:10.596469+0000","last_undegraded":"2023-09-23T18:24:10.596469+0000","last_fullsized":"2023-09-23T18:24:10.596469+0000","mapping_epoch":694,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":695,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:23:41.504763+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"1.1e","version":"89'161","reported_seq":630,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.594394+0000","last_change":"2023-09-23T18:23:53.308220+0000","last_active":"2023-09-23T18:24:10.594394+0000","last_peered":"2023-09-23T18:24:10.594394+0000","last_clean":"2023-09-23T18:24:10.594394+0000","last_became_active":"2023-09-23T18:23:53.307884+0000","last_became_peered":"2023-09-23T18:23:53.307884+0000","last_unstale":"2023-09-23T18:24:10.594394+0000","last_undegraded":"2023-09-23T18:24:10.594394+0000","last_fullsized":"2023-09-23T18:24:10.594394+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:16:52.216646+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:16:52.216646+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T22:07:11.834311+0000","scrub_duration":0.0063893520000000001,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,7],"acting":[3,6,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1c","version":"67'2","reported_seq":216,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.585908+0000","last_change":"2023-09-23T18:17:21.793768+0000","last_active":"2023-09-23T18:24:10.585908+0000","last_peered":"2023-09-23T18:24:10.585908+0000","last_clean":"2023-09-23T18:24:10.585908+0000","last_became_active":"2023-09-23T18:17:21.793377+0000","last_became_peered":"2023-09-23T18:17:21.793377+0000","last_unstale":"2023-09-23T18:24:10.585908+0000","last_undegraded":"2023-09-23T18:24:10.585908+0000","last_fullsized":"2023-09-23T18:24:10.585908+0000","mapping_epoch":546,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":547,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:13:49.814727+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":343,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608534+0000","last_change":"2023-09-23T18:24:08.582024+0000","last_active":"2023-09-23T18:24:10.608534+0000","last_peered":"2023-09-23T18:24:10.608534+0000","last_clean":"2023-09-23T18:24:10.608534+0000","last_became_active":"2023-09-23T18:24:08.581698+0000","last_became_peered":"2023-09-23T18:24:08.581698+0000","last_unstale":"2023-09-23T18:24:10.608534+0000","last_undegraded":"2023-09-23T18:24:10.608534+0000","last_fullsized":"2023-09-23T18:24:10.608534+0000","mapping_epoch":696,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":697,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:25:22.813974+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,2],"acting":[4,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"67'2","reported_seq":202,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.586325+0000","last_change":"2023-09-23T18:16:08.311956+0000","last_active":"2023-09-23T18:24:10.586325+0000","last_peered":"2023-09-23T18:24:10.586325+0000","last_clean":"2023-09-23T18:24:10.586325+0000","last_became_active":"2023-09-23T18:16:08.297955+0000","last_became_peered":"2023-09-23T18:16:08.297955+0000","last_unstale":"2023-09-23T18:24:10.586325+0000","last_undegraded":"2023-09-23T18:24:10.586325+0000","last_fullsized":"2023-09-23T18:24:10.586325+0000","mapping_epoch":509,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":510,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":2,"log_dups_size":0,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T01:34:47.849758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":343,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.608344+0000","last_change":"2023-09-23T18:23:33.839503+0000","last_active":"2023-09-23T18:24:10.608344+0000","last_peered":"2023-09-23T18:24:10.608344+0000","last_clean":"2023-09-23T18:24:10.608344+0000","last_became_active":"2023-09-23T18:23:33.838993+0000","last_became_peered":"2023-09-23T18:23:33.838993+0000","last_unstale":"2023-09-23T18:24:10.608344+0000","last_undegraded":"2023-09-23T18:24:10.608344+0000","last_fullsized":"2023-09-23T18:24:10.608344+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T05:40:08.671873+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"1.1c","version":"89'161","reported_seq":698,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.607390+0000","last_change":"2023-09-23T18:23:15.704773+0000","last_active":"2023-09-23T18:24:10.607390+0000","last_peered":"2023-09-23T18:24:10.607390+0000","last_clean":"2023-09-23T18:24:10.607390+0000","last_became_active":"2023-09-23T18:23:15.704418+0000","last_became_peered":"2023-09-23T18:23:15.704418+0000","last_unstale":"2023-09-23T18:24:10.607390+0000","last_undegraded":"2023-09-23T18:24:10.607390+0000","last_fullsized":"2023-09-23T18:24:10.607390+0000","mapping_epoch":672,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":673,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:01:34.224656+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:01:34.224656+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T00:57:53.408914+0000","scrub_duration":0.006700302,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,1],"acting":[7,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1e","version":"67'1","reported_seq":504,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.594573+0000","last_change":"2023-09-23T18:23:53.308023+0000","last_active":"2023-09-23T18:24:10.594573+0000","last_peered":"2023-09-23T18:24:10.594573+0000","last_clean":"2023-09-23T18:24:10.594573+0000","last_became_active":"2023-09-23T18:23:53.307691+0000","last_became_peered":"2023-09-23T18:23:53.307691+0000","last_unstale":"2023-09-23T18:24:10.594573+0000","last_undegraded":"2023-09-23T18:24:10.594573+0000","last_fullsized":"2023-09-23T18:24:10.594573+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":1,"log_dups_size":0,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T17:55:27.932947+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":168,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.595446+0000","last_change":"2023-09-23T18:23:53.305705+0000","last_active":"2023-09-23T18:24:10.595446+0000","last_peered":"2023-09-23T18:24:10.595446+0000","last_clean":"2023-09-23T18:24:10.595446+0000","last_became_active":"2023-09-23T18:23:53.305403+0000","last_became_peered":"2023-09-23T18:23:53.305403+0000","last_unstale":"2023-09-23T18:24:10.595446+0000","last_undegraded":"2023-09-23T18:24:10.595446+0000","last_fullsized":"2023-09-23T18:24:10.595446+0000","mapping_epoch":689,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":690,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-25T02:24:43.513337+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.1d","version":"89'161","reported_seq":1020,"reported_epoch":699,"state":"active+clean","last_fresh":"2023-09-23T18:24:11.593143+0000","last_change":"2023-09-23T18:24:11.593143+0000","last_active":"2023-09-23T18:24:11.593143+0000","last_peered":"2023-09-23T18:24:11.593143+0000","last_clean":"2023-09-23T18:24:11.593143+0000","last_became_active":"2023-09-23T18:24:11.592852+0000","last_became_peered":"2023-09-23T18:24:11.592852+0000","last_unstale":"2023-09-23T18:24:11.593143+0000","last_undegraded":"2023-09-23T18:24:11.593143+0000","last_fullsized":"2023-09-23T18:24:11.593143+0000","mapping_epoch":698,"log_start":"0'0","ondisk_log_start":"0'0","created":197,"last_epoch_clean":699,"parent":"0.0","parent_split_bits":5,"last_scrub":"89'161","last_scrub_stamp":"2023-09-23T18:23:37.422771+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:43:05.158540+0000","last_clean_scrub_stamp":"2023-09-23T18:23:37.422771+0000","objects_scrubbed":0,"log_size":161,"log_dups_size":0,"ondisk_log_size":161,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":1,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T19:48:27.536323+0000","scrub_duration":0.001648416,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.1f","version":"67'4","reported_seq":259,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.597979+0000","last_change":"2023-09-23T18:23:27.787952+0000","last_active":"2023-09-23T18:24:10.597979+0000","last_peered":"2023-09-23T18:24:10.597979+0000","last_clean":"2023-09-23T18:24:10.597979+0000","last_became_active":"2023-09-23T18:23:27.787603+0000","last_became_peered":"2023-09-23T18:23:27.787603+0000","last_unstale":"2023-09-23T18:24:10.597979+0000","last_undegraded":"2023-09-23T18:24:10.597979+0000","last_fullsized":"2023-09-23T18:24:10.597979+0000","mapping_epoch":678,"log_start":"0'0","ondisk_log_start":"0'0","created":63,"last_epoch_clean":679,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:50.074605+0000","last_clean_scrub_stamp":"2023-09-23T17:45:50.074605+0000","objects_scrubbed":0,"log_size":4,"log_dups_size":0,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:40:45.743031+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":10,"num_bytes_recovered":2770,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":654,"reported_epoch":698,"state":"active+clean","last_fresh":"2023-09-23T18:24:10.606915+0000","last_change":"2023-09-23T18:23:33.843335+0000","last_active":"2023-09-23T18:24:10.606915+0000","last_peered":"2023-09-23T18:24:10.606915+0000","last_clean":"2023-09-23T18:24:10.606915+0000","last_became_active":"2023-09-23T18:23:33.842525+0000","last_became_peered":"2023-09-23T18:23:33.842525+0000","last_unstale":"2023-09-23T18:24:10.606915+0000","last_undegraded":"2023-09-23T18:24:10.606915+0000","last_fullsized":"2023-09-23T18:24:10.606915+0000","mapping_epoch":681,"log_start":"0'0","ondisk_log_start":"0'0","created":64,"last_epoch_clean":682,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-09-23T17:45:51.086592+0000","last_clean_scrub_stamp":"2023-09-23T17:45:51.086592+0000","objects_scrubbed":0,"log_size":0,"log_dups_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-09-24T18:07:07.884918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,3],"acting":[4,6,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":5504,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":61,"num_write_kb":29,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":123,"num_bytes_recovered":18500,"num_keys_recovered":50,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":16512,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":71710,"internal_metadata":0},"log_size":55,"ondisk_log_size":55,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":31,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":5,"num_bytes_recovered":1443424,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":4991,"ondisk_log_size":4991,"up":93,"acting":93,"num_store_stats":8},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":8388627,"num_objects":5,"num_object_clones":0,"num_object_copies":15,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":5,"num_whiteouts":0,"num_read":11950,"num_read_kb":25361,"num_write":1690,"num_write_kb":25390,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":569,"num_bytes_recovered":37748793,"num_keys_recovered":0,"num_objects_omap":2,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":25178112,"data_stored":25165881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":6797,"internal_metadata":0},"log_size":1950,"ondisk_log_size":1950,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":214,"seq":919123001624,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":514880,"kb_used_data":11832,"kb_used_omap":13,"kb_used_meta":503026,"kb_avail":93230272,"statfs":{"total":95995035648,"available":95467798528,"internally_reserved":0,"allocated":12115968,"data_stored":10065726,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":14232,"internal_metadata":515098728},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:01 2023","interfaces":[{"interface":"back","average":{"1min":0.56299999999999994,"5min":1.867,"15min":1.508},"min":{"1min":0.35299999999999998,"5min":0.224,"15min":0.20200000000000001},"max":{"1min":0.76800000000000002,"5min":33.344000000000001,"15min":33.344000000000001},"last":0.65500000000000003},{"interface":"front","average":{"1min":0.50700000000000001,"5min":1.641,"15min":1.583},"min":{"1min":0.34799999999999998,"5min":0.25800000000000001,"15min":0.217},"max":{"1min":0.80800000000000005,"5min":33.220999999999997,"15min":33.220999999999997},"last":0.45700000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:23:34 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":1.5209999999999999,"15min":1.075},"min":{"1min":0.379,"5min":0.27000000000000002,"15min":0.22600000000000001},"max":{"1min":0.80200000000000005,"5min":9.6929999999999996,"15min":9.6929999999999996},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.0389999999999999,"15min":0.92900000000000005},"min":{"1min":0.32700000000000001,"5min":0.28399999999999997,"15min":0.247},"max":{"1min":0.78900000000000003,"5min":6.077,"15min":6.077},"last":0.51900000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:24:01 2023","interfaces":[{"interface":"back","average":{"1min":0.55200000000000005,"5min":1.139,"15min":1.214},"min":{"1min":0.40799999999999997,"5min":0.33300000000000002,"15min":0.27100000000000002},"max":{"1min":0.83399999999999996,"5min":6.5389999999999997,"15min":7.8979999999999997},"last":0.56299999999999994},{"interface":"front","average":{"1min":0.55400000000000005,"5min":1.2450000000000001,"15min":1.605},"min":{"1min":0.34300000000000003,"5min":0.34000000000000002,"15min":0.248},"max":{"1min":0.755,"5min":8.5120000000000005,"15min":8.5120000000000005},"last":0.501}]},{"osd":3,"last update":"Sat Sep 23 18:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":1.292,"15min":2.101},"min":{"1min":0.35899999999999999,"5min":0.32100000000000001,"15min":0.27100000000000002},"max":{"1min":0.72599999999999998,"5min":6.032,"15min":54.622999999999998},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.622,"5min":1.5609999999999999,"15min":1.99},"min":{"1min":0.38500000000000001,"5min":0.375,"15min":0.249},"max":{"1min":0.872,"5min":16.297999999999998,"15min":52.475999999999999},"last":0.60399999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:24:01 2023","interfaces":[{"interface":"back","average":{"1min":0.54400000000000004,"5min":1.246,"15min":1.4750000000000001},"min":{"1min":0.42899999999999999,"5min":0.26200000000000001,"15min":0.23999999999999999},"max":{"1min":0.72499999999999998,"5min":19.151,"15min":29.782},"last":0.59299999999999997},{"interface":"front","average":{"1min":0.623,"5min":1,"15min":1.4419999999999999},"min":{"1min":0.39600000000000002,"5min":0.30499999999999999,"15min":0.26200000000000001},"max":{"1min":0.85099999999999998,"5min":6.6959999999999997,"15min":40.277000000000001},"last":0.72899999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.65800000000000003,"5min":1.079,"15min":1.5189999999999999},"min":{"1min":0.35899999999999999,"5min":0.29099999999999998,"15min":0.26800000000000002},"max":{"1min":0.84399999999999997,"5min":8.0129999999999999,"15min":15.587999999999999},"last":0.68799999999999994},{"interface":"front","average":{"1min":0.59399999999999997,"5min":1.3899999999999999,"15min":2.0579999999999998},"min":{"1min":0.40100000000000002,"5min":0.29299999999999998,"15min":0.253},"max":{"1min":0.77800000000000002,"5min":11.433,"15min":16.221},"last":0.54000000000000004}]},{"osd":6,"last update":"Sat Sep 23 18:24:01 2023","interfaces":[{"interface":"back","average":{"1min":0.66800000000000004,"5min":1.425,"15min":1.6459999999999999},"min":{"1min":0.496,"5min":0.35599999999999998,"15min":0.28299999999999997},"max":{"1min":0.877,"5min":12.481,"15min":15.847},"last":0.70899999999999996},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.97399999999999998,"15min":1.6259999999999999},"min":{"1min":0.44700000000000001,"5min":0.33600000000000002,"15min":0.253},"max":{"1min":0.81799999999999995,"5min":8.6120000000000001,"15min":16.975999999999999},"last":0.58399999999999996}]}]},{"osd":6,"up_from":163,"seq":700079669560,"num_pgs":40,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":642292,"kb_used_data":7788,"kb_used_omap":7,"kb_used_meta":634488,"kb_avail":93102860,"statfs":{"total":95995035648,"available":95337328640,"internally_reserved":0,"allocated":7974912,"data_stored":5936287,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":649716516},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":2.2639999999999998,"15min":1.7809999999999999},"min":{"1min":0.39100000000000001,"5min":0.245,"15min":0.22500000000000001},"max":{"1min":0.745,"5min":79.203999999999994,"15min":79.203999999999994},"last":0.48299999999999998},{"interface":"front","average":{"1min":0.56100000000000005,"5min":2.0030000000000001,"15min":1.9019999999999999},"min":{"1min":0.34599999999999997,"5min":0.252,"15min":0.23300000000000001},"max":{"1min":0.80500000000000005,"5min":78.034000000000006,"15min":78.034000000000006},"last":0.51700000000000002}]},{"osd":1,"last update":"Sat Sep 23 18:24:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":1.274,"15min":1.4319999999999999},"min":{"1min":0.34999999999999998,"5min":0.28799999999999998,"15min":0.26600000000000001},"max":{"1min":0.90000000000000002,"5min":11.726000000000001,"15min":22.972000000000001},"last":0.60399999999999998},{"interface":"front","average":{"1min":0.59499999999999997,"5min":1.4239999999999999,"15min":1.429},"min":{"1min":0.32800000000000001,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":1.232,"5min":12.336,"15min":22.664000000000001},"last":0.64000000000000001}]},{"osd":2,"last update":"Sat Sep 23 18:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.226,"15min":1.286},"min":{"1min":0.38600000000000001,"5min":0.32900000000000001,"15min":0.25700000000000001},"max":{"1min":0.875,"5min":8.0559999999999992,"15min":8.0559999999999992},"last":0.85299999999999998},{"interface":"front","average":{"1min":0.621,"5min":1.4610000000000001,"15min":1.5049999999999999},"min":{"1min":0.437,"5min":0.34000000000000002,"15min":0.28100000000000003},"max":{"1min":1.071,"5min":8.2690000000000001,"15min":14.771000000000001},"last":0.80300000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.57399999999999995,"5min":3.2770000000000001,"15min":2.25},"min":{"1min":0.39300000000000002,"5min":0.32500000000000001,"15min":0.24099999999999999},"max":{"1min":1.004,"5min":103.348,"15min":103.348},"last":0.69399999999999995},{"interface":"front","average":{"1min":0.59599999999999997,"5min":3.7370000000000001,"15min":2.6059999999999999},"min":{"1min":0.44600000000000001,"5min":0.32100000000000001,"15min":0.28699999999999998},"max":{"1min":1.0249999999999999,"5min":103.40300000000001,"15min":103.40300000000001},"last":0.66700000000000004}]},{"osd":4,"last update":"Sat Sep 23 18:24:24 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.76500000000000001,"15min":1.294},"min":{"1min":0.34499999999999997,"5min":0.24199999999999999,"15min":0.23999999999999999},"max":{"1min":0.81899999999999995,"5min":6.0730000000000004,"15min":8.6059999999999999},"last":0.57699999999999996},{"interface":"front","average":{"1min":0.65900000000000003,"5min":0.92900000000000005,"15min":1.3089999999999999},"min":{"1min":0.42399999999999999,"5min":0.27900000000000003,"15min":0.27000000000000002},"max":{"1min":1.048,"5min":7.8319999999999999,"15min":9.2260000000000009},"last":0.76400000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:23:52 2023","interfaces":[{"interface":"back","average":{"1min":0.55800000000000005,"5min":1.177,"15min":1.734},"min":{"1min":0.372,"5min":0.29599999999999999,"15min":0.28899999999999998},"max":{"1min":0.96299999999999997,"5min":18.417000000000002,"15min":18.417000000000002},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.63400000000000001,"5min":1.1899999999999999,"15min":2.0009999999999999},"min":{"1min":0.40100000000000002,"5min":0.32000000000000001,"15min":0.28599999999999998},"max":{"1min":1.165,"5min":18.454999999999998,"15min":18.454999999999998},"last":0.78500000000000003}]},{"osd":7,"last update":"Sat Sep 23 18:23:59 2023","interfaces":[{"interface":"back","average":{"1min":0.64000000000000001,"5min":1.0980000000000001,"15min":1.5960000000000001},"min":{"1min":0.45200000000000001,"5min":0.34899999999999998,"15min":0.28000000000000003},"max":{"1min":1.0840000000000001,"5min":6.4320000000000004,"15min":13.460000000000001},"last":0.82599999999999996},{"interface":"front","average":{"1min":0.66700000000000004,"5min":1.1799999999999999,"15min":1.609},"min":{"1min":0.36099999999999999,"5min":0.35599999999999998,"15min":0.318},"max":{"1min":1.125,"5min":8.6890000000000001,"15min":12.404},"last":0.81299999999999994}]}]},{"osd":1,"up_from":545,"seq":2340757176409,"num_pgs":43,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":352244,"kb_used_data":100204,"kb_used_omap":13,"kb_used_meta":252018,"kb_avail":93392908,"statfs":{"total":95995035648,"available":95634337792,"internally_reserved":0,"allocated":102608896,"data_stored":99238214,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":258067085},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:33 2023","interfaces":[{"interface":"back","average":{"1min":0.38300000000000001,"5min":1.05,"15min":0.70299999999999996},"min":{"1min":0.25600000000000001,"5min":0.19900000000000001,"15min":0.189},"max":{"1min":0.53100000000000003,"5min":9.0299999999999994,"15min":9.0299999999999994},"last":0.378},{"interface":"front","average":{"1min":0.372,"5min":1.1080000000000001,"15min":0.746},"min":{"1min":0.26500000000000001,"5min":0.26400000000000001,"15min":0.20999999999999999},"max":{"1min":0.53000000000000003,"5min":14.952999999999999,"15min":14.952999999999999},"last":0.499}]},{"osd":2,"last update":"Sat Sep 23 18:24:33 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":1.194,"15min":0.77800000000000002},"min":{"1min":0.25700000000000001,"5min":0.25600000000000001,"15min":0.25600000000000001},"max":{"1min":0.71499999999999997,"5min":11.02,"15min":11.02},"last":0.45900000000000002},{"interface":"front","average":{"1min":0.42299999999999999,"5min":0.79600000000000004,"15min":0.65300000000000002},"min":{"1min":0.28000000000000003,"5min":0.249,"15min":0.249},"max":{"1min":0.63600000000000001,"5min":5.1440000000000001,"15min":5.1440000000000001},"last":0.42799999999999999}]},{"osd":3,"last update":"Sat Sep 23 18:24:33 2023","interfaces":[{"interface":"back","average":{"1min":0.40200000000000002,"5min":1.415,"15min":0.84899999999999998},"min":{"1min":0.315,"5min":0.26600000000000001,"15min":0.25800000000000001},"max":{"1min":0.57799999999999996,"5min":14.962,"15min":14.962},"last":0.443},{"interface":"front","average":{"1min":0.47599999999999998,"5min":1.222,"15min":0.86399999999999999},"min":{"1min":0.36899999999999999,"5min":0.318,"15min":0.318},"max":{"1min":0.752,"5min":19.762,"15min":19.762},"last":0.52500000000000002}]},{"osd":4,"last update":"Sat Sep 23 18:24:33 2023","interfaces":[{"interface":"back","average":{"1min":0.58099999999999996,"5min":0.98199999999999998,"15min":1.2150000000000001},"min":{"1min":0.38800000000000001,"5min":0.311,"15min":0.28499999999999998},"max":{"1min":0.71699999999999997,"5min":5.0759999999999996,"15min":12.663},"last":0.67500000000000004},{"interface":"front","average":{"1min":0.62,"5min":1.679,"15min":1.5980000000000001},"min":{"1min":0.38100000000000001,"5min":0.34399999999999997,"15min":0.317},"max":{"1min":0.81599999999999995,"5min":11.353999999999999,"15min":13.222},"last":0.56899999999999995}]},{"osd":5,"last update":"Sat Sep 23 18:24:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":1.5289999999999999,"15min":1.0449999999999999},"min":{"1min":0.40899999999999997,"5min":0.32200000000000001,"15min":0.308},"max":{"1min":0.71699999999999997,"5min":23.190999999999999,"15min":23.190999999999999},"last":0.63500000000000001},{"interface":"front","average":{"1min":0.64800000000000002,"5min":1.7350000000000001,"15min":1.1990000000000001},"min":{"1min":0.48499999999999999,"5min":0.40699999999999997,"15min":0.40699999999999997},"max":{"1min":0.78100000000000003,"5min":23.204000000000001,"15min":23.204000000000001},"last":0.66400000000000003}]},{"osd":6,"last update":"Sat Sep 23 18:24:33 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.4450000000000001,"15min":1.0669999999999999},"min":{"1min":0.39300000000000002,"5min":0.372,"15min":0.372},"max":{"1min":0.752,"5min":23.443000000000001,"15min":23.443000000000001},"last":0.60699999999999998},{"interface":"front","average":{"1min":0.66200000000000003,"5min":1.891,"15min":1.2689999999999999},"min":{"1min":0.41299999999999998,"5min":0.41299999999999998,"15min":0.378},"max":{"1min":0.81200000000000006,"5min":26.177,"15min":26.177},"last":0.71499999999999997}]},{"osd":7,"last update":"Sat Sep 23 18:24:33 2023","interfaces":[{"interface":"back","average":{"1min":0.629,"5min":1.591,"15min":1.0940000000000001},"min":{"1min":0.436,"5min":0.42299999999999999,"15min":0.40000000000000002},"max":{"1min":0.76100000000000001,"5min":15.01,"15min":15.01},"last":0.61699999999999999},{"interface":"front","average":{"1min":0.64300000000000002,"5min":1.4399999999999999,"15min":1.139},"min":{"1min":0.51700000000000002,"5min":0.42399999999999999,"15min":0.42399999999999999},"max":{"1min":0.79800000000000004,"5min":10.988,"15min":10.988},"last":0.70299999999999996}]}]},{"osd":0,"up_from":93,"seq":399431958906,"num_pgs":36,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":1000200,"kb_used_data":17216,"kb_used_omap":7,"kb_used_meta":982968,"kb_avail":92744952,"statfs":{"total":95995035648,"available":94970830848,"internally_reserved":0,"allocated":17629184,"data_stored":14474527,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":1006559972},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sat Sep 23 18:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.40000000000000002,"5min":1.3999999999999999,"15min":0.68000000000000005},"min":{"1min":0.221,"5min":0.19800000000000001,"15min":0.19700000000000001},"max":{"1min":0.65500000000000003,"5min":8.2309999999999999,"15min":8.2309999999999999},"last":0.249},{"interface":"front","average":{"1min":0.436,"5min":1.8169999999999999,"15min":0.95399999999999996},"min":{"1min":0.29999999999999999,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.71899999999999997,"5min":22.978000000000002,"15min":22.978000000000002},"last":0.33000000000000002}]},{"osd":2,"last update":"Sat Sep 23 18:24:11 2023","interfaces":[{"interface":"back","average":{"1min":0.39800000000000002,"5min":1.8919999999999999,"15min":1.387},"min":{"1min":0.27900000000000003,"5min":0.25,"15min":0.185},"max":{"1min":0.57099999999999995,"5min":15.002000000000001,"15min":18.550999999999998},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.40999999999999998,"5min":1.673,"15min":1.0960000000000001},"min":{"1min":0.23999999999999999,"5min":0.23999999999999999,"15min":0.22},"max":{"1min":0.55900000000000005,"5min":23.161999999999999,"15min":23.161999999999999},"last":0.27800000000000002}]},{"osd":3,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.45300000000000001,"5min":1.8360000000000001,"15min":1.512},"min":{"1min":0.26800000000000002,"5min":0.24199999999999999,"15min":0.20000000000000001},"max":{"1min":0.72599999999999998,"5min":15.173999999999999,"15min":15.173999999999999},"last":0.32000000000000001},{"interface":"front","average":{"1min":0.496,"5min":1.9710000000000001,"15min":1.5860000000000001},"min":{"1min":0.33400000000000002,"5min":0.28699999999999998,"15min":0.26400000000000001},"max":{"1min":0.78200000000000003,"5min":15.135,"15min":15.135},"last":0.35099999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.63,"5min":1.6020000000000001,"15min":1.831},"min":{"1min":0.44600000000000001,"5min":0.33300000000000002,"15min":0.30099999999999999},"max":{"1min":0.85599999999999998,"5min":17.741,"15min":57.746000000000002},"last":0.63800000000000001},{"interface":"front","average":{"1min":0.66300000000000003,"5min":1.6439999999999999,"15min":1.607},"min":{"1min":0.51600000000000001,"5min":0.30099999999999999,"15min":0.25600000000000001},"max":{"1min":0.93300000000000005,"5min":15.079000000000001,"15min":57.816000000000003},"last":0.72699999999999998}]},{"osd":5,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.78100000000000003,"5min":1.484,"15min":1.429},"min":{"1min":0.42599999999999999,"5min":0.30199999999999999,"15min":0.30199999999999999},"max":{"1min":2.5830000000000002,"5min":7.5999999999999996,"15min":7.9660000000000002},"last":0.73899999999999999},{"interface":"front","average":{"1min":0.78600000000000003,"5min":2.1110000000000002,"15min":1.5920000000000001},"min":{"1min":0.50600000000000001,"5min":0.35499999999999998,"15min":0.32200000000000001},"max":{"1min":2.2240000000000002,"5min":15.047000000000001,"15min":15.047000000000001},"last":0.59099999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:24:22 2023","interfaces":[{"interface":"back","average":{"1min":0.67500000000000004,"5min":1.4430000000000001,"15min":1.6719999999999999},"min":{"1min":0.40400000000000003,"5min":0.36099999999999999,"15min":0.32000000000000001},"max":{"1min":0.94599999999999995,"5min":7.6900000000000004,"15min":9.1170000000000009},"last":0.70399999999999996},{"interface":"front","average":{"1min":0.69599999999999995,"5min":2.2069999999999999,"15min":1.778},"min":{"1min":0.48699999999999999,"5min":0.442,"15min":0.315},"max":{"1min":1.0089999999999999,"5min":22.893000000000001,"15min":22.893000000000001},"last":0.628}]},{"osd":7,"last update":"Sat Sep 23 18:23:58 2023","interfaces":[{"interface":"back","average":{"1min":0.85699999999999998,"5min":2.0659999999999998,"15min":1.7290000000000001},"min":{"1min":0.436,"5min":0.40699999999999997,"15min":0.36599999999999999},"max":{"1min":2.7370000000000001,"5min":22.945,"15min":22.945},"last":0.65100000000000002},{"interface":"front","average":{"1min":0.71799999999999997,"5min":1.669,"15min":1.7130000000000001},"min":{"1min":0.47399999999999998,"5min":0.42799999999999999,"15min":0.36199999999999999},"max":{"1min":1.0029999999999999,"5min":7.7519999999999998,"15min":10.404},"last":0.78900000000000003}]}]},{"osd":2,"up_from":482,"seq":2070174236791,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":161484,"kb_used_data":13252,"kb_used_omap":22,"kb_used_meta":148201,"kb_avail":93583668,"statfs":{"total":95995035648,"available":95829676032,"internally_reserved":0,"allocated":13570048,"data_stored":10233567,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":22605,"internal_metadata":151758771},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.34100000000000003,"5min":1.484,"15min":0.88200000000000001},"min":{"1min":0.21299999999999999,"5min":0.21299999999999999,"15min":0.19700000000000001},"max":{"1min":0.56000000000000005,"5min":9.6359999999999992,"15min":9.6359999999999992},"last":0.35999999999999999},{"interface":"front","average":{"1min":0.34399999999999997,"5min":1.7769999999999999,"15min":1.202},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.20100000000000001},"max":{"1min":0.49299999999999999,"5min":13.472,"15min":13.472},"last":0.38200000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:23:33 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":1.4039999999999999,"15min":0.80200000000000005},"min":{"1min":0.29199999999999998,"5min":0.26100000000000001,"15min":0.26100000000000001},"max":{"1min":0.54200000000000004,"5min":13.538,"15min":13.538},"last":0.29399999999999998},{"interface":"front","average":{"1min":0.40000000000000002,"5min":1.6339999999999999,"15min":0.96799999999999997},"min":{"1min":0.249,"5min":0.249,"15min":0.20999999999999999},"max":{"1min":0.66300000000000003,"5min":16.567,"15min":16.567},"last":0.44800000000000001}]},{"osd":3,"last update":"Sat Sep 23 18:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.41199999999999998,"5min":1.9750000000000001,"15min":1.3089999999999999},"min":{"1min":0.30199999999999999,"5min":0.29399999999999998,"15min":0.224},"max":{"1min":0.623,"5min":15.926,"15min":15.926},"last":0.33100000000000002},{"interface":"front","average":{"1min":0.39400000000000002,"5min":1.7,"15min":1.2949999999999999},"min":{"1min":0.314,"5min":0.27600000000000002,"15min":0.23499999999999999},"max":{"1min":0.66000000000000003,"5min":16.619,"15min":16.619},"last":0.40500000000000003}]},{"osd":4,"last update":"Sat Sep 23 18:24:12 2023","interfaces":[{"interface":"back","average":{"1min":0.63100000000000001,"5min":1.349,"15min":1.399},"min":{"1min":0.435,"5min":0.375,"15min":0.30299999999999999},"max":{"1min":0.90500000000000003,"5min":9.532,"15min":11.334},"last":0.59899999999999998},{"interface":"front","average":{"1min":0.65200000000000002,"5min":1.262,"15min":1.121},"min":{"1min":0.51300000000000001,"5min":0.33300000000000002,"15min":0.31900000000000001},"max":{"1min":0.89400000000000002,"5min":5.5229999999999997,"15min":11.244},"last":0.63800000000000001}]},{"osd":5,"last update":"Sat Sep 23 18:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60999999999999999,"5min":1.4330000000000001,"15min":1.8049999999999999},"min":{"1min":0.40999999999999998,"5min":0.36899999999999999,"15min":0.308},"max":{"1min":0.94799999999999995,"5min":9.4960000000000004,"15min":11.852},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.61399999999999999,"5min":1.4650000000000001,"15min":1.4630000000000001},"min":{"1min":0.47099999999999997,"5min":0.34200000000000003,"15min":0.32600000000000001},"max":{"1min":0.81200000000000006,"5min":8.3849999999999998,"15min":10.942},"last":0.58599999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:24:12 2023","interfaces":[{"interface":"back","average":{"1min":0.60299999999999998,"5min":1.7170000000000001,"15min":1.7849999999999999},"min":{"1min":0.46200000000000002,"5min":0.35799999999999998,"15min":0.33900000000000002},"max":{"1min":0.79800000000000004,"5min":29.489000000000001,"15min":29.489000000000001},"last":0.61499999999999999},{"interface":"front","average":{"1min":0.63200000000000001,"5min":1.734,"15min":2.133},"min":{"1min":0.45500000000000002,"5min":0.39700000000000002,"15min":0.38700000000000001},"max":{"1min":0.92300000000000004,"5min":29.408999999999999,"15min":29.408999999999999},"last":0.57299999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:24:03 2023","interfaces":[{"interface":"back","average":{"1min":0.64200000000000002,"5min":1.419,"15min":3.2210000000000001},"min":{"1min":0.42999999999999999,"5min":0.40999999999999998,"15min":0.35499999999999998},"max":{"1min":0.88,"5min":13.106,"15min":76.966999999999999},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.61199999999999999,"5min":1.4470000000000001,"15min":3.359},"min":{"1min":0.442,"5min":0.442,"15min":0.41699999999999998},"max":{"1min":0.84899999999999998,"5min":9.5619999999999994,"15min":76.936999999999998},"last":0.66800000000000004}]}]},{"osd":3,"up_from":237,"seq":1017907249420,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":818912,"kb_used_data":13400,"kb_used_omap":3,"kb_used_meta":805500,"kb_avail":92926240,"statfs":{"total":95995035648,"available":95156469760,"internally_reserved":0,"allocated":13721600,"data_stored":10312138,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":824832744},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.34999999999999998,"5min":1.8200000000000001,"15min":1.7989999999999999},"min":{"1min":0.23200000000000001,"5min":0.192,"15min":0.183},"max":{"1min":0.52200000000000002,"5min":17.036000000000001,"15min":195.69800000000001},"last":0.33700000000000002},{"interface":"front","average":{"1min":0.37,"5min":1.603,"15min":1.871},"min":{"1min":0.25900000000000001,"5min":0.19700000000000001,"15min":0.19700000000000001},"max":{"1min":0.50700000000000001,"5min":11.468,"15min":194.661},"last":0.35999999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:23:31 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":1.8169999999999999,"15min":0.95199999999999996},"min":{"1min":0.23699999999999999,"5min":0.218,"15min":0.218},"max":{"1min":0.49199999999999999,"5min":21.777000000000001,"15min":21.777000000000001},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.41999999999999998,"5min":2.7269999999999999,"15min":1.379},"min":{"1min":0.22900000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.875,"5min":78.980000000000004,"15min":78.980000000000004},"last":0.438}]},{"osd":2,"last update":"Sat Sep 23 18:24:09 2023","interfaces":[{"interface":"back","average":{"1min":0.41999999999999998,"5min":1.944,"15min":1.395},"min":{"1min":0.313,"5min":0.249,"15min":0.21099999999999999},"max":{"1min":0.58199999999999996,"5min":18.722999999999999,"15min":18.722999999999999},"last":0.45500000000000002},{"interface":"front","average":{"1min":0.45100000000000001,"5min":1.5580000000000001,"15min":1.2789999999999999},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.19600000000000001},"max":{"1min":0.64400000000000002,"5min":11.565,"15min":11.565},"last":0.41699999999999998}]},{"osd":4,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.60399999999999998,"5min":1.5089999999999999,"15min":1.486},"min":{"1min":0.438,"5min":0.27300000000000002,"15min":0.26000000000000001},"max":{"1min":0.81899999999999995,"5min":11.4,"15min":21.062999999999999},"last":0.59699999999999998},{"interface":"front","average":{"1min":0.61099999999999999,"5min":1.5880000000000001,"15min":1.7490000000000001},"min":{"1min":0.45300000000000001,"5min":0.32700000000000001,"15min":0.29199999999999998},"max":{"1min":0.753,"5min":9.1630000000000003,"15min":21.128},"last":0.752}]},{"osd":5,"last update":"Sat Sep 23 18:23:48 2023","interfaces":[{"interface":"back","average":{"1min":0.61599999999999999,"5min":2.0230000000000001,"15min":1.5900000000000001},"min":{"1min":0.441,"5min":0.35099999999999998,"15min":0.316},"max":{"1min":0.77100000000000002,"5min":19.434999999999999,"15min":19.434999999999999},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.60699999999999998,"5min":1.5780000000000001,"15min":1.569},"min":{"1min":0.443,"5min":0.32800000000000001,"15min":0.32000000000000001},"max":{"1min":0.73199999999999998,"5min":17.135999999999999,"15min":17.135999999999999},"last":0.63100000000000001}]},{"osd":6,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.65700000000000003,"5min":1.6890000000000001,"15min":1.518},"min":{"1min":0.48199999999999998,"5min":0.39300000000000002,"15min":0.32700000000000001},"max":{"1min":0.76000000000000001,"5min":17.192,"15min":17.192},"last":0.66000000000000003},{"interface":"front","average":{"1min":0.61499999999999999,"5min":1.5469999999999999,"15min":1.502},"min":{"1min":0.45500000000000002,"5min":0.35699999999999998,"15min":0.30499999999999999},"max":{"1min":0.75,"5min":11.442,"15min":14.321},"last":0.69399999999999995}]},{"osd":7,"last update":"Sat Sep 23 18:23:53 2023","interfaces":[{"interface":"back","average":{"1min":0.63200000000000001,"5min":1.6799999999999999,"15min":1.7370000000000001},"min":{"1min":0.45700000000000002,"5min":0.38700000000000001,"15min":0.35799999999999998},"max":{"1min":0.79500000000000004,"5min":9.1929999999999996,"15min":15.972},"last":0.64600000000000002},{"interface":"front","average":{"1min":0.68799999999999994,"5min":1.7,"15min":1.599},"min":{"1min":0.54600000000000004,"5min":0.41699999999999998,"15min":0.34899999999999998},"max":{"1min":0.81299999999999994,"5min":17.164000000000001,"15min":17.164000000000001},"last":0.74199999999999999}]}]},{"osd":4,"up_from":30,"seq":128849019370,"num_pgs":45,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":718204,"kb_used_data":11572,"kb_used_omap":0,"kb_used_meta":706624,"kb_avail":93026948,"statfs":{"total":95995035648,"available":95259594752,"internally_reserved":0,"allocated":11849728,"data_stored":9803291,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":723582976},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:14 2023","interfaces":[{"interface":"back","average":{"1min":0.55000000000000004,"5min":1.903,"15min":1.46},"min":{"1min":0.33400000000000002,"5min":0.22700000000000001,"15min":0.19700000000000001},"max":{"1min":0.77700000000000002,"5min":72.754999999999995,"15min":72.754999999999995},"last":0.34300000000000003},{"interface":"front","average":{"1min":0.57399999999999995,"5min":1.966,"15min":1.7390000000000001},"min":{"1min":0.36799999999999999,"5min":0.23899999999999999,"15min":0.23000000000000001},"max":{"1min":0.85399999999999998,"5min":71.533000000000001,"15min":71.533000000000001},"last":0.63100000000000001}]},{"osd":1,"last update":"Sat Sep 23 18:24:27 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.22,"15min":1.1279999999999999},"min":{"1min":0.435,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.73499999999999999,"5min":10.619,"15min":44.146999999999998},"last":0.41699999999999998},{"interface":"front","average":{"1min":0.623,"5min":1.093,"15min":1.0580000000000001},"min":{"1min":0.46400000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.88700000000000001,"5min":7.5700000000000003,"15min":43.497999999999998},"last":0.40400000000000003}]},{"osd":2,"last update":"Sat Sep 23 18:24:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":1.3620000000000001,"15min":1.5920000000000001},"min":{"1min":0.45100000000000001,"5min":0.29799999999999999,"15min":0.255},"max":{"1min":0.76000000000000001,"5min":11.896000000000001,"15min":11.896000000000001},"last":0.44400000000000001},{"interface":"front","average":{"1min":0.59999999999999998,"5min":1.2969999999999999,"15min":1.5569999999999999},"min":{"1min":0.40000000000000002,"5min":0.27000000000000002,"15min":0.253},"max":{"1min":0.89300000000000002,"5min":17.190999999999999,"15min":17.190999999999999},"last":0.67700000000000005}]},{"osd":3,"last update":"Sat Sep 23 18:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":2.4380000000000002,"15min":2.0950000000000002},"min":{"1min":0.39300000000000002,"5min":0.32000000000000001,"15min":0.23799999999999999},"max":{"1min":0.81200000000000006,"5min":74.781999999999996,"15min":74.781999999999996},"last":0.47599999999999998},{"interface":"front","average":{"1min":0.622,"5min":2.335,"15min":2.0510000000000002},"min":{"1min":0.42699999999999999,"5min":0.307,"15min":0.30599999999999999},"max":{"1min":0.81799999999999995,"5min":72.152000000000001,"15min":72.152000000000001},"last":0.58599999999999997}]},{"osd":5,"last update":"Sat Sep 23 18:23:47 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":2.0169999999999999,"15min":2.613},"min":{"1min":0.39900000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.76600000000000001,"5min":91.432000000000002,"15min":91.432000000000002},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.57199999999999995,"5min":1.9610000000000001,"15min":2.379},"min":{"1min":0.44700000000000001,"5min":0.27800000000000002,"15min":0.26300000000000001},"max":{"1min":0.745,"5min":91.626999999999995,"15min":91.626999999999995},"last":0.45600000000000002}]},{"osd":6,"last update":"Sat Sep 23 18:24:14 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":1.8560000000000001,"15min":1.6859999999999999},"min":{"1min":0.432,"5min":0.30099999999999999,"15min":0.25},"max":{"1min":0.79600000000000004,"5min":91.688999999999993,"15min":91.688999999999993},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.65800000000000003,"5min":2.0920000000000001,"15min":1.891},"min":{"1min":0.504,"5min":0.311,"15min":0.27500000000000002},"max":{"1min":0.90200000000000002,"5min":91.837000000000003,"15min":91.837000000000003},"last":0.60699999999999998}]},{"osd":7,"last update":"Sat Sep 23 18:24:14 2023","interfaces":[{"interface":"back","average":{"1min":0.68300000000000005,"5min":1.329,"15min":1.821},"min":{"1min":0.45500000000000002,"5min":0.34699999999999998,"15min":0.26000000000000001},"max":{"1min":0.96999999999999997,"5min":18.283000000000001,"15min":19.298999999999999},"last":0.65600000000000003},{"interface":"front","average":{"1min":0.71699999999999997,"5min":1.1080000000000001,"15min":1.5569999999999999},"min":{"1min":0.46999999999999997,"5min":0.36399999999999999,"15min":0.28399999999999997},"max":{"1min":1.0329999999999999,"5min":9.4730000000000008,"15min":12.800000000000001},"last":0.69899999999999995}]}]},{"osd":5,"up_from":420,"seq":1803886264466,"num_pgs":32,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":381808,"kb_used_data":7720,"kb_used_omap":9,"kb_used_meta":374070,"kb_avail":93363344,"statfs":{"total":95995035648,"available":95604064256,"internally_reserved":0,"allocated":7905280,"data_stored":5867115,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":383048125},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":1.7230000000000001,"15min":1.1990000000000001},"min":{"1min":0.30399999999999999,"5min":0.20499999999999999,"15min":0.20499999999999999},"max":{"1min":0.86899999999999999,"5min":77.162000000000006,"15min":77.162000000000006},"last":0.437},{"interface":"front","average":{"1min":0.57499999999999996,"5min":2.2229999999999999,"15min":1.738},"min":{"1min":0.33900000000000002,"5min":0.23999999999999999,"15min":0.23499999999999999},"max":{"1min":0.73299999999999998,"5min":75.784999999999997,"15min":75.784999999999997},"last":0.61699999999999999}]},{"osd":1,"last update":"Sat Sep 23 18:24:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":1.4199999999999999,"15min":1.331},"min":{"1min":0.48599999999999999,"5min":0.23200000000000001,"15min":0.23200000000000001},"max":{"1min":0.70799999999999996,"5min":12.752000000000001,"15min":23.946000000000002},"last":0.52700000000000002},{"interface":"front","average":{"1min":0.58799999999999997,"5min":1.4139999999999999,"15min":1.2410000000000001},"min":{"1min":0.35099999999999998,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.746,"5min":9.2759999999999998,"15min":23.268999999999998},"last":0.56100000000000005}]},{"osd":2,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":1.361,"15min":1.476},"min":{"1min":0.36199999999999999,"5min":0.28699999999999998,"15min":0.21199999999999999},"max":{"1min":0.84699999999999998,"5min":10.705,"15min":10.705},"last":0.40100000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":1.3740000000000001,"15min":1.8440000000000001},"min":{"1min":0.39900000000000002,"5min":0.32300000000000001,"15min":0.30199999999999999},"max":{"1min":0.879,"5min":7.8099999999999996,"15min":7.8099999999999996},"last":0.65900000000000003}]},{"osd":3,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.64600000000000002,"5min":2.3319999999999999,"15min":1.613},"min":{"1min":0.41899999999999998,"5min":0.317,"15min":0.25600000000000001},"max":{"1min":0.93999999999999995,"5min":79.016999999999996,"15min":79.016999999999996},"last":0.72099999999999997},{"interface":"front","average":{"1min":0.63200000000000001,"5min":2.4870000000000001,"15min":1.843},"min":{"1min":0.46200000000000002,"5min":0.32900000000000001,"15min":0.313},"max":{"1min":0.92800000000000005,"5min":76.188999999999993,"15min":76.188999999999993},"last":0.50600000000000001}]},{"osd":4,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.49099999999999999,"5min":1.3089999999999999,"15min":1.8129999999999999},"min":{"1min":0.30499999999999999,"5min":0.23799999999999999,"15min":0.23799999999999999},"max":{"1min":0.65900000000000003,"5min":8.2270000000000003,"15min":12.747999999999999},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.58599999999999997,"5min":1.3149999999999999,"15min":1.5349999999999999},"min":{"1min":0.34399999999999997,"5min":0.254,"15min":0.254},"max":{"1min":0.89900000000000002,"5min":8.5559999999999992,"15min":9.0690000000000008},"last":0.58799999999999997}]},{"osd":6,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":1.038,"15min":1.252},"min":{"1min":0.32100000000000001,"5min":0.28100000000000003,"15min":0.26400000000000001},"max":{"1min":0.97399999999999998,"5min":6.7859999999999996,"15min":9.0350000000000001},"last":0.64400000000000002},{"interface":"front","average":{"1min":0.58299999999999996,"5min":1.3580000000000001,"15min":1.5980000000000001},"min":{"1min":0.37,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.96099999999999997,"5min":7.8410000000000002,"15min":12.504},"last":0.69999999999999996}]},{"osd":7,"last update":"Sat Sep 23 18:24:00 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":1.351,"15min":1.6930000000000001},"min":{"1min":0.36799999999999999,"5min":0.29499999999999998,"15min":0.28599999999999998},"max":{"1min":0.75800000000000001,"5min":10.528,"15min":13.506},"last":0.48399999999999999},{"interface":"front","average":{"1min":0.69699999999999995,"5min":1.1839999999999999,"15min":1.5009999999999999},"min":{"1min":0.44700000000000001,"5min":0.36099999999999999,"15min":0.29799999999999999},"max":{"1min":1.0720000000000001,"5min":12.702,"15min":13.138},"last":0.68200000000000005}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":327680,"data_stored":327680,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":266240,"data_stored":262688,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8388608,"data_stored":8388608,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":4498,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":19,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":4194304,"data_stored":4194304,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":2299,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7452,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":13683,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":18107,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":3352,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":4880,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":9795,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":4804,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":7388,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":4950,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":11933,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-09-23T18:24:37.578 INFO:tasks.cephadm.ceph_manager.ceph:recovered! 2023-09-23T18:24:37.579 ERROR:teuthology.run_tasks:Saw exception from tasks. Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T18:24:37.649 ERROR:teuthology.util.sentry: Sentry event: https://sentry.ceph.com/organizations/ceph/?query=8ccd226a2e034608a31269aa82ea231b Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 105, in run_tasks manager = run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 56, in task p.spawn(_run_spawned, ctx, confg, taskname) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 84, in __exit__ for result in self: File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 98, in __next__ resurrect_traceback(result) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 30, in resurrect_traceback raise exc.exc_info[1] File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/parallel.py", line 23, in capture_traceback return func(*args, **kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/parallel.py", line 64, in _run_spawned mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=config) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/sequential.py", line 47, in task mgr = run_tasks.run_one_task(taskname, ctx=ctx, config=confg) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 83, in run_one_task return task(**kwargs) File "/home/teuthworker/src/github.com_ceph_ceph_bb0daeff97f020f686596fca6eba20c310e26fb7/qa/tasks/workunit.py", line 119, in task assert 'client' in role AssertionError 2023-09-23T18:24:37.651 DEBUG:teuthology.run_tasks:Unwinding manager cephadm 2023-09-23T18:24:37.661 INFO:tasks.cephadm:Teardown begin 2023-09-23T18:24:37.662 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:37.714 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T18:24:37.763 INFO:tasks.cephadm:Cleaning up testdir ceph.* files... 2023-09-23T18:24:37.763 DEBUG:teuthology.orchestra.run.smithi055:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T18:24:37.781 DEBUG:teuthology.orchestra.run.smithi099:> rm -f /home/ubuntu/cephtest/seed.ceph.conf /home/ubuntu/cephtest/ceph.pub 2023-09-23T18:24:37.799 INFO:tasks.cephadm:Stopping all daemons... 2023-09-23T18:24:37.799 INFO:tasks.cephadm.mon.a:Stopping mon.a... 2023-09-23T18:24:37.799 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.a 2023-09-23T18:24:38.173 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:37 smithi055 systemd[1]: Stopping Ceph mon.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:24:38.428 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:38 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:24:38.171+0000 7fb814ce1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:24:38.428 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:38 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a[179624]: 2023-09-23T18:24:38.171+0000 7fb814ce1700 -1 mon.a@0(leader) e4 *** Got Signal Terminated *** 2023-09-23T18:24:38.428 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:38 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:38] ENGINE Bus STOPPING 2023-09-23T18:24:38.428 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:38 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T18:24:38.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:38 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:38] ENGINE Bus STOPPED 2023-09-23T18:24:38.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:38 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:38] ENGINE Bus STARTING 2023-09-23T18:24:38.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:38 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:38] ENGINE Serving on http://:::9283 2023-09-23T18:24:38.429 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:38 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:38] ENGINE Bus STARTED 2023-09-23T18:24:38.679 INFO:journalctl@ceph.mon.a.smithi055.stdout:Sep 23 18:24:38 smithi055 bash[484189]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-a 2023-09-23T18:24:38.764 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.a.service' 2023-09-23T18:24:38.807 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:24:38.807 INFO:tasks.cephadm.mon.a:Stopped mon.a 2023-09-23T18:24:38.808 INFO:tasks.cephadm.mon.b:Stopping mon.c... 2023-09-23T18:24:38.808 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.c 2023-09-23T18:24:39.177 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:38 smithi055 systemd[1]: Stopping Ceph mon.c for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:24:39.177 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:39 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-c[184147]: 2023-09-23T18:24:39.175+0000 7f1c7bc88700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.c -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:24:39.430 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:39 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-c[184147]: 2023-09-23T18:24:39.175+0000 7f1c7bc88700 -1 mon.c@2(peon) e4 *** Got Signal Terminated *** 2023-09-23T18:24:39.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:39 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:39] ENGINE Bus STOPPING 2023-09-23T18:24:39.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:39 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:39] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-09-23T18:24:39.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:39 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:39] ENGINE Bus STOPPED 2023-09-23T18:24:39.430 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:39 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:39] ENGINE Bus STARTING 2023-09-23T18:24:39.431 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:39 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:39] ENGINE Serving on http://:::9283 2023-09-23T18:24:39.431 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:39 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: [23/Sep/2023:18:24:39] ENGINE Bus STARTED 2023-09-23T18:24:39.738 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.c.service' 2023-09-23T18:24:39.763 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:39 smithi055 bash[484351]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-c 2023-09-23T18:24:39.763 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:39 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.c.service: Succeeded. 2023-09-23T18:24:39.763 INFO:journalctl@ceph.mon.c.smithi055.stdout:Sep 23 18:24:39 smithi055 systemd[1]: Stopped Ceph mon.c for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:24:39.778 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:24:39.778 INFO:tasks.cephadm.mon.b:Stopped mon.c 2023-09-23T18:24:39.778 INFO:tasks.cephadm.mon.b:Stopping mon.b... 2023-09-23T18:24:39.778 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.b 2023-09-23T18:24:40.077 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:39 smithi099 systemd[1]: Stopping Ceph mon.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:24:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-b[167052]: 2023-09-23T18:24:40.142+0000 7fef466c3700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mon -n mon.b -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:24:40.577 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-b[167052]: 2023-09-23T18:24:40.142+0000 7fef466c3700 -1 mon.b@1(peon) e4 *** Got Signal Terminated *** 2023-09-23T18:24:40.680 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:40 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: ::ffff:172.21.15.99 - - [23/Sep/2023:18:24:40] "GET /metrics HTTP/1.1" 200 34215 "" "Prometheus/2.43.0" 2023-09-23T18:24:40.881 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.b.service' 2023-09-23T18:24:40.906 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:40 smithi099 bash[305932]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mon-b 2023-09-23T18:24:40.906 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:40 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mon.b.service: Succeeded. 2023-09-23T18:24:40.906 INFO:journalctl@ceph.mon.b.smithi099.stdout:Sep 23 18:24:40 smithi099 systemd[1]: Stopped Ceph mon.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:24:40.921 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:24:40.921 INFO:tasks.cephadm.mon.b:Stopped mon.b 2023-09-23T18:24:40.922 INFO:tasks.cephadm.mgr.y:Stopping mgr.y... 2023-09-23T18:24:40.922 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.y 2023-09-23T18:24:41.280 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:40 smithi055 systemd[1]: Stopping Ceph mgr.y for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:24:41.280 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:41 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y[167623]: 2023-09-23T18:24:41.278+0000 7f3fb5002700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-mgr -n mgr.y -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:24:41.822 INFO:journalctl@ceph.mgr.y.smithi055.stdout:Sep 23 18:24:41 smithi055 bash[484502]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-y 2023-09-23T18:24:41.829 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.y.service' 2023-09-23T18:24:41.869 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:24:41.869 INFO:tasks.cephadm.mgr.y:Stopped mgr.y 2023-09-23T18:24:41.869 INFO:tasks.cephadm.mgr.x:Stopping mgr.x... 2023-09-23T18:24:41.869 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.x 2023-09-23T18:24:42.321 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 18:24:41 smithi099 systemd[1]: Stopping Ceph mgr.x for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:24:42.576 INFO:journalctl@ceph.mgr.x.smithi099.stdout:Sep 23 18:24:42 smithi099 bash[306074]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-mgr-x 2023-09-23T18:24:42.764 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@mgr.x.service' 2023-09-23T18:24:42.802 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:24:42.802 INFO:tasks.cephadm.mgr.x:Stopped mgr.x 2023-09-23T18:24:42.802 INFO:tasks.cephadm.osd.0:Stopping osd.0... 2023-09-23T18:24:42.802 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.0 2023-09-23T18:24:43.180 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 18:24:42 smithi055 systemd[1]: Stopping Ceph osd.0 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:24:44.179 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 18:24:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[194956]: 2023-09-23T18:24:43.922+0000 7f1c21fea700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.0 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:24:44.180 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 18:24:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[194956]: 2023-09-23T18:24:43.922+0000 7f1c21fea700 -1 osd.0 699 *** Got signal Terminated *** 2023-09-23T18:24:44.180 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 18:24:43 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0[194956]: 2023-09-23T18:24:43.922+0000 7f1c21fea700 -1 osd.0 699 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:24:49.930 INFO:journalctl@ceph.osd.0.smithi055.stdout:Sep 23 18:24:49 smithi055 bash[484643]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-0 2023-09-23T18:24:51.725 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.0.service' 2023-09-23T18:24:51.767 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:24:51.767 INFO:tasks.cephadm.osd.0:Stopped osd.0 2023-09-23T18:24:51.767 INFO:tasks.cephadm.osd.1:Stopping osd.1... 2023-09-23T18:24:51.767 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.1 2023-09-23T18:24:52.145 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:24:51 smithi055 systemd[1]: Stopping Ceph osd.1 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:24:52.429 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:24:52 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[415375]: 2023-09-23T18:24:52.144+0000 7f5f371d1700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.1 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:24:52.430 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:24:52 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[415375]: 2023-09-23T18:24:52.144+0000 7f5f371d1700 -1 osd.1 699 *** Got signal Terminated *** 2023-09-23T18:24:52.430 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:24:52 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1[415375]: 2023-09-23T18:24:52.144+0000 7f5f371d1700 -1 osd.1 699 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:24:58.139 INFO:journalctl@ceph.osd.1.smithi055.stdout:Sep 23 18:24:58 smithi055 bash[484911]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-1 2023-09-23T18:24:59.416 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.1.service' 2023-09-23T18:24:59.457 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:24:59.457 INFO:tasks.cephadm.osd.1:Stopped osd.1 2023-09-23T18:24:59.458 INFO:tasks.cephadm.osd.2:Stopping osd.2... 2023-09-23T18:24:59.458 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.2 2023-09-23T18:24:59.679 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:24:59 smithi055 systemd[1]: Stopping Ceph osd.2 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:00.180 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:24:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[394395]: 2023-09-23T18:24:59.824+0000 7f180db6f700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.2 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:25:00.180 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:24:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[394395]: 2023-09-23T18:24:59.824+0000 7f180db6f700 -1 osd.2 699 *** Got signal Terminated *** 2023-09-23T18:25:00.180 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:24:59 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2[394395]: 2023-09-23T18:24:59.824+0000 7f180db6f700 -1 osd.2 699 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:25:05.930 INFO:journalctl@ceph.osd.2.smithi055.stdout:Sep 23 18:25:05 smithi055 bash[485180]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-2 2023-09-23T18:25:06.940 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.2.service' 2023-09-23T18:25:06.983 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:25:06.983 INFO:tasks.cephadm.osd.2:Stopped osd.2 2023-09-23T18:25:06.984 INFO:tasks.cephadm.osd.3:Stopping osd.3... 2023-09-23T18:25:06.984 DEBUG:teuthology.orchestra.run.smithi055:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.3 2023-09-23T18:25:07.368 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:25:07 smithi055 systemd[1]: Stopping Ceph osd.3 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:07.679 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:25:07 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[287212]: 2023-09-23T18:25:07.367+0000 7f2501504700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.3 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:25:07.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:25:07 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[287212]: 2023-09-23T18:25:07.367+0000 7f2501504700 -1 osd.3 699 *** Got signal Terminated *** 2023-09-23T18:25:07.680 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:25:07 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3[287212]: 2023-09-23T18:25:07.367+0000 7f2501504700 -1 osd.3 699 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:25:10.077 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:09 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:09.764+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:11.076 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:10 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:10.745+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:12.011 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:11 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:11.708+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:12.011 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:11 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:11.678+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:12.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:12.011+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:13.000 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:12.707+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:13.001 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:12 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:12.722+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:13.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:13.000+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:13.430 INFO:journalctl@ceph.osd.3.smithi055.stdout:Sep 23 18:25:13 smithi055 bash[485460]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-3 2023-09-23T18:25:14.001 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:13.716+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:14.001 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:13 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:13.682+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:14.257 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:14.000+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:14.577 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:14.257+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:14.625 DEBUG:teuthology.orchestra.run.smithi055:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.3.service' 2023-09-23T18:25:14.666 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:25:14.667 INFO:tasks.cephadm.osd.3:Stopped osd.3 2023-09-23T18:25:14.667 INFO:tasks.cephadm.osd.4:Stopping osd.4... 2023-09-23T18:25:14.667 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.4 2023-09-23T18:25:15.007 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:14 smithi099 systemd[1]: Stopping Ceph osd.4 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:15.008 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:14.709+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:15.008 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:14 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:14.698+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:15.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:15 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:15.251+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:15.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:15 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:15.006+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:15.327 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:15 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:15.027+0000 7fbd3a19e700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.4 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:25:15.328 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:15 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:15.027+0000 7fbd3a19e700 -1 osd.4 699 *** Got signal Terminated *** 2023-09-23T18:25:15.328 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:15 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:15.027+0000 7fbd3a19e700 -1 osd.4 699 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:25:16.056 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:15 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:15.741+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:16.057 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:15 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:15.673+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:16.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:16 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:16.245+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:16.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:16 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:16.056+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:17.046 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:16 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:16.715+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:17.046 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:16 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:16.641+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:17.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:17 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:17.237+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:17.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:17 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:17.046+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:18.000 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:17 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:17.667+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:18.001 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:17 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:17.618+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:18.267 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:18.000+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:18.577 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:18.267+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:18.577 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:18.267+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:53.839457+0000 front 2023-09-23T18:24:53.839527+0000 (oldest deadline 2023-09-23T18:25:17.939275+0000) 2023-09-23T18:25:18.961 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:18.618+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:18.962 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:18.696+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:18.962 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:18.697+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:54.946700+0000 front 2023-09-23T18:24:54.946717+0000 (oldest deadline 2023-09-23T18:25:17.846592+0000) 2023-09-23T18:25:19.257 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:18 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:18.961+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:19.573 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:19 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:19.257+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:19.574 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:19 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:19.257+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:53.839457+0000 front 2023-09-23T18:24:53.839527+0000 (oldest deadline 2023-09-23T18:25:17.939275+0000) 2023-09-23T18:25:19.827 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:19 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:19.675+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:44.945338+0000 front 2023-09-23T18:24:44.945241+0000 (oldest deadline 2023-09-23T18:25:09.645332+0000) 2023-09-23T18:25:19.827 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:19 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4[105803]: 2023-09-23T18:25:19.675+0000 7fbd37f9f700 -1 osd.4 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:54.946700+0000 front 2023-09-23T18:24:54.946717+0000 (oldest deadline 2023-09-23T18:25:17.846592+0000) 2023-09-23T18:25:19.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:19 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:19.574+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:19.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:19 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:19.574+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:20.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:20 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:20.214+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:20.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:20 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:20.214+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:53.839457+0000 front 2023-09-23T18:24:53.839527+0000 (oldest deadline 2023-09-23T18:25:17.939275+0000) 2023-09-23T18:25:20.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:19 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:19.996+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:20.328 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:19 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:19.996+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:20.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:20 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:20.563+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:20.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:20 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:20.563+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:21.163 INFO:journalctl@ceph.osd.4.smithi099.stdout:Sep 23 18:25:20 smithi099 bash[306219]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-4 2023-09-23T18:25:21.164 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:21 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:21.032+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:21.164 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:21 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:21.032+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:21.543 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:21 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:21.163+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:21.543 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:21 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:21.163+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:53.839457+0000 front 2023-09-23T18:24:53.839527+0000 (oldest deadline 2023-09-23T18:25:17.939275+0000) 2023-09-23T18:25:21.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:21 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:21.542+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:21.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:21 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:21.542+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:22.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:22 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:22.062+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:22.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:22 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:22.062+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:22.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:22 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:22.124+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:22.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:22 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:22.124+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:53.839457+0000 front 2023-09-23T18:24:53.839527+0000 (oldest deadline 2023-09-23T18:25:17.939275+0000) 2023-09-23T18:25:22.341 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.4.service' 2023-09-23T18:25:22.382 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:25:22.382 INFO:tasks.cephadm.osd.4:Stopped osd.4 2023-09-23T18:25:22.382 INFO:tasks.cephadm.osd.5:Stopping osd.5... 2023-09-23T18:25:22.383 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.5 2023-09-23T18:25:22.758 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:22 smithi099 systemd[1]: Stopping Ceph osd.5 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:22.759 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:22 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:22.758+0000 7fecb09a9700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.5 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:25:22.759 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:22 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:22.578+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:22.759 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:22 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:22.578+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:23.061 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:22 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:22.758+0000 7fecb09a9700 -1 osd.5 699 *** Got signal Terminated *** 2023-09-23T18:25:23.062 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:22 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:22.758+0000 7fecb09a9700 -1 osd.5 699 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:25:23.326 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:23 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:23.083+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:23.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:23 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:23.083+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:53.839457+0000 front 2023-09-23T18:24:53.839527+0000 (oldest deadline 2023-09-23T18:25:17.939275+0000) 2023-09-23T18:25:23.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:23 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:23.062+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:23.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:23 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:23.062+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:23.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:23 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:23.561+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:23.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:23 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:23.561+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:24.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:24.045+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:24.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:24.045+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:24.329 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:24.100+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:24.329 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:24.100+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:53.839457+0000 front 2023-09-23T18:24:53.839527+0000 (oldest deadline 2023-09-23T18:25:17.939275+0000) 2023-09-23T18:25:24.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:24.548+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:24.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:24 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:24.548+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:25.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:25.029+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:25.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:25.029+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:25.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:25.126+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:25.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:25.126+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:53.839457+0000 front 2023-09-23T18:24:53.839527+0000 (oldest deadline 2023-09-23T18:25:17.939275+0000) 2023-09-23T18:25:26.040 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:25.590+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:26.041 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:25 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:25.590+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:26.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:26.171+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:26.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:26.171+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:53.839457+0000 front 2023-09-23T18:24:53.839527+0000 (oldest deadline 2023-09-23T18:25:17.939275+0000) 2023-09-23T18:25:26.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:26.171+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.840367+0000 front 2023-09-23T18:25:03.840390+0000 (oldest deadline 2023-09-23T18:25:26.140140+0000) 2023-09-23T18:25:26.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:26.040+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:26.328 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:26.040+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:27.000 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:26.583+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:27.000 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:26.583+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:27.000 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:26 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:26.583+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.042791+0000 front 2023-09-23T18:25:03.042804+0000 (oldest deadline 2023-09-23T18:25:26.542658+0000) 2023-09-23T18:25:27.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:27.188+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:47.939166+0000 front 2023-09-23T18:24:47.939097+0000 (oldest deadline 2023-09-23T18:25:13.838875+0000) 2023-09-23T18:25:27.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:27.188+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:53.839457+0000 front 2023-09-23T18:24:53.839527+0000 (oldest deadline 2023-09-23T18:25:17.939275+0000) 2023-09-23T18:25:27.327 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5[251355]: 2023-09-23T18:25:27.188+0000 7fecadfa9700 -1 osd.5 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.840367+0000 front 2023-09-23T18:25:03.840390+0000 (oldest deadline 2023-09-23T18:25:26.140140+0000) 2023-09-23T18:25:27.328 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:26.999+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:27.328 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:26.999+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:27.328 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:26.999+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:27.962 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:27.607+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:27.962 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:27.607+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:27.962 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:27.607+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.042791+0000 front 2023-09-23T18:25:03.042804+0000 (oldest deadline 2023-09-23T18:25:26.542658+0000) 2023-09-23T18:25:28.326 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:27.961+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:28.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:27.961+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:28.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:27 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:27.961+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:28.827 INFO:journalctl@ceph.osd.5.smithi099.stdout:Sep 23 18:25:28 smithi099 bash[306489]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-5 2023-09-23T18:25:28.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:28.630+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:28.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:28.630+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:28.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:28.630+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.042791+0000 front 2023-09-23T18:25:03.042804+0000 (oldest deadline 2023-09-23T18:25:26.542658+0000) 2023-09-23T18:25:29.262 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:28.938+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:29.262 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:28.938+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:29.262 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:28 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:28.938+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:29.905 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:29 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:29.655+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:29.905 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:29 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:29.656+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:29.905 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:29 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:29.656+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.042791+0000 front 2023-09-23T18:25:03.042804+0000 (oldest deadline 2023-09-23T18:25:26.542658+0000) 2023-09-23T18:25:29.906 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:29 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:29.891+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:29.906 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:29 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:29.891+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:29.906 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:29 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:29.891+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:29.911 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.5.service' 2023-09-23T18:25:29.951 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:25:29.952 INFO:tasks.cephadm.osd.5:Stopped osd.5 2023-09-23T18:25:29.952 INFO:tasks.cephadm.osd.6:Stopping osd.6... 2023-09-23T18:25:29.952 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.6 2023-09-23T18:25:30.313 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:30 smithi099 systemd[1]: Stopping Ceph osd.6 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:30.576 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:30.311+0000 7f8a8c810700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.6 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:25:30.577 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:30.311+0000 7f8a8c810700 -1 osd.6 699 *** Got signal Terminated *** 2023-09-23T18:25:30.577 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:30.311+0000 7f8a8c810700 -1 osd.6 699 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:25:30.917 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:30.621+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:30.917 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:30.621+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:30.917 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:30.621+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.042791+0000 front 2023-09-23T18:25:03.042804+0000 (oldest deadline 2023-09-23T18:25:26.542658+0000) 2023-09-23T18:25:31.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:30.916+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:31.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:30.916+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:31.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:30 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:30.916+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:31.906 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:31 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:31.591+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:31.907 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:31 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:31.591+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:31.907 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:31 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:31.591+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.042791+0000 front 2023-09-23T18:25:03.042804+0000 (oldest deadline 2023-09-23T18:25:26.542658+0000) 2023-09-23T18:25:32.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:31 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:31.905+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:32.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:31 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:31.905+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:32.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:31 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:31.905+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:32.826 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:32.557+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:32.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:32.557+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:32.827 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:32.557+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.042791+0000 front 2023-09-23T18:25:03.042804+0000 (oldest deadline 2023-09-23T18:25:26.542658+0000) 2023-09-23T18:25:33.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:32.873+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:33.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:32.873+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:33.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:32 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:32.873+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:33.867 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:33.593+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:33.867 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:33.593+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:33.867 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:33.593+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.042791+0000 front 2023-09-23T18:25:03.042804+0000 (oldest deadline 2023-09-23T18:25:26.542658+0000) 2023-09-23T18:25:34.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:33.866+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:34.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:33.866+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:34.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:33 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:33.866+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:34.903 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:34.578+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.839715+0000 front 2023-09-23T18:24:46.839615+0000 (oldest deadline 2023-09-23T18:25:10.939743+0000) 2023-09-23T18:25:34.903 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:34.578+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:55.040781+0000 front 2023-09-23T18:24:55.040770+0000 (oldest deadline 2023-09-23T18:25:19.140699+0000) 2023-09-23T18:25:34.903 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6[188402]: 2023-09-23T18:25:34.578+0000 7f8a87e0c700 -1 osd.6 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:25:03.042791+0000 front 2023-09-23T18:25:03.042804+0000 (oldest deadline 2023-09-23T18:25:26.542658+0000) 2023-09-23T18:25:35.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:34.902+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:35.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:34.902+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:35.327 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:34 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:34.902+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:36.155 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:35 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:35.863+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:36.155 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:35 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:35.863+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:36.155 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:35 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:35.863+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:36.155 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:35 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:35.863+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2023-09-23T18:25:09.804286+0000 front 2023-09-23T18:25:09.804379+0000 (oldest deadline 2023-09-23T18:25:35.104191+0000) 2023-09-23T18:25:36.577 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:36 smithi099 bash[306750]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-6 2023-09-23T18:25:37.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:36 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:36.843+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:37.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:36 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:36.843+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:37.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:36 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:36.843+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:37.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:36 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:36.843+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2023-09-23T18:25:09.804286+0000 front 2023-09-23T18:25:09.804379+0000 (oldest deadline 2023-09-23T18:25:35.104191+0000) 2023-09-23T18:25:37.683 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.6.service' 2023-09-23T18:25:37.710 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:37 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.6.service: Succeeded. 2023-09-23T18:25:37.710 INFO:journalctl@ceph.osd.6.smithi099.stdout:Sep 23 18:25:37 smithi099 systemd[1]: Stopped Ceph osd.6 for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:25:37.727 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:25:37.727 INFO:tasks.cephadm.osd.6:Stopped osd.6 2023-09-23T18:25:37.727 INFO:tasks.cephadm.osd.7:Stopping osd.7... 2023-09-23T18:25:37.727 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.7 2023-09-23T18:25:38.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:37 smithi099 systemd[1]: Stopping Ceph osd.7 for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:38.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:37 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:37.809+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:38.078 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:37 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:37.809+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:38.078 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:37 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:37.809+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:38.078 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:37 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:37.809+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2023-09-23T18:25:09.804286+0000 front 2023-09-23T18:25:09.804379+0000 (oldest deadline 2023-09-23T18:25:35.104191+0000) 2023-09-23T18:25:38.577 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:38.109+0000 7f8235db2700 -1 received signal: Terminated from /dev/init -- /usr/bin/ceph-osd -n osd.7 -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false (PID: 1) UID: 0 2023-09-23T18:25:38.577 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:38.109+0000 7f8235db2700 -1 osd.7 699 *** Got signal Terminated *** 2023-09-23T18:25:38.577 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:38.109+0000 7f8235db2700 -1 osd.7 699 *** Immediate shutdown (osd_fast_shutdown=true) *** 2023-09-23T18:25:39.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:38.762+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:39.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:38.762+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:39.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:38.762+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:39.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:38 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:38.762+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2023-09-23T18:25:09.804286+0000 front 2023-09-23T18:25:09.804379+0000 (oldest deadline 2023-09-23T18:25:35.104191+0000) 2023-09-23T18:25:40.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:39.791+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:40.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:39.792+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:40.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:39.792+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:40.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:39 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:39.792+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2023-09-23T18:25:09.804286+0000 front 2023-09-23T18:25:09.804379+0000 (oldest deadline 2023-09-23T18:25:35.104191+0000) 2023-09-23T18:25:40.757 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:40.388Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=node msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=node-exporter\": dial tcp 172.21.15.55:8765: connect: connection refused" 2023-09-23T18:25:40.757 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:40.388Z caller=refresh.go:99 level=error component="discovery manager notify" discovery=http config=config-0 msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=alertmanager\": dial tcp 172.21.15.55:8765: connect: connection refused" 2023-09-23T18:25:40.757 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:40.388Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=mgr-prometheus\": dial tcp 172.21.15.55:8765: connect: connection refused" 2023-09-23T18:25:40.757 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:40.389Z caller=refresh.go:99 level=error component="discovery manager scrape" discovery=http config=ceph-exporter msg="Unable to refresh target groups" err="Get \"http://172.21.15.55:8765/sd/prometheus/sd-config?service=ceph-exporter\": dial tcp 172.21.15.55:8765: connect: connection refused" 2023-09-23T18:25:41.076 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:40.756+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:41.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:40.756+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:41.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:40.756+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:41.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:40 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:40.756+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2023-09-23T18:25:09.804286+0000 front 2023-09-23T18:25:09.804379+0000 (oldest deadline 2023-09-23T18:25:35.104191+0000) 2023-09-23T18:25:42.076 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:41 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:41.720+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:42.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:41 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:41.720+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:42.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:41 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:41.720+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:42.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:41 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:41.720+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2023-09-23T18:25:09.804286+0000 front 2023-09-23T18:25:09.804379+0000 (oldest deadline 2023-09-23T18:25:35.104191+0000) 2023-09-23T18:25:43.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:42.677+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6806 osd.0 since back 2023-09-23T18:24:46.401457+0000 front 2023-09-23T18:24:46.401569+0000 (oldest deadline 2023-09-23T18:25:11.101399+0000) 2023-09-23T18:25:43.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:42.677+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6814 osd.1 since back 2023-09-23T18:24:57.002481+0000 front 2023-09-23T18:24:57.002440+0000 (oldest deadline 2023-09-23T18:25:19.302338+0000) 2023-09-23T18:25:43.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:42.677+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6822 osd.2 since back 2023-09-23T18:24:59.302938+0000 front 2023-09-23T18:24:59.302920+0000 (oldest deadline 2023-09-23T18:25:26.903299+0000) 2023-09-23T18:25:43.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:42.677+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.55:6830 osd.3 since back 2023-09-23T18:25:09.804286+0000 front 2023-09-23T18:25:09.804379+0000 (oldest deadline 2023-09-23T18:25:35.104191+0000) 2023-09-23T18:25:43.077 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:42 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7[203342]: 2023-09-23T18:25:42.677+0000 7f822f3aa700 -1 osd.7 699 heartbeat_check: no reply from 172.21.15.99:6804 osd.4 since back 2023-09-23T18:25:18.004955+0000 front 2023-09-23T18:25:18.004922+0000 (oldest deadline 2023-09-23T18:25:42.005542+0000) 2023-09-23T18:25:43.890 INFO:journalctl@ceph.osd.7.smithi099.stdout:Sep 23 18:25:43 smithi099 bash[307024]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-osd-7 2023-09-23T18:25:45.150 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@osd.7.service' 2023-09-23T18:25:45.190 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:25:45.190 INFO:tasks.cephadm.osd.7:Stopped osd.7 2023-09-23T18:25:45.190 INFO:tasks.cephadm.prometheus.a:Stopping prometheus.a... 2023-09-23T18:25:45.191 DEBUG:teuthology.orchestra.run.smithi099:> sudo systemctl stop ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@prometheus.a 2023-09-23T18:25:45.562 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 systemd[1]: Stopping Ceph prometheus.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:45.819 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.562Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-09-23T18:25:45.819 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.562Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-09-23T18:25:45.819 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.562Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-09-23T18:25:45.819 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.562Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-09-23T18:25:45.820 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.562Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-09-23T18:25:45.820 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.562Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-09-23T18:25:45.820 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.562Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-09-23T18:25:45.820 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.562Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-09-23T18:25:45.820 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.562Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-09-23T18:25:45.820 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.564Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-09-23T18:25:45.820 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.564Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-09-23T18:25:45.820 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a[154732]: ts=2023-09-23T18:25:45.564Z caller=main.go:1147 level=info msg="See you next time!" 2023-09-23T18:25:45.821 INFO:journalctl@ceph.prometheus.a.smithi099.stdout:Sep 23 18:25:45 smithi099 bash[307289]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-prometheus-a 2023-09-23T18:25:46.009 DEBUG:teuthology.orchestra.run.smithi099:> sudo pkill -f 'journalctl -f -n 0 -u ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@prometheus.a.service' 2023-09-23T18:25:46.045 DEBUG:teuthology.orchestra.run:got remote process result: None 2023-09-23T18:25:46.045 INFO:tasks.cephadm.prometheus.a:Stopped prometheus.a 2023-09-23T18:25:46.046 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T18:25:47.680 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 18:25:47 smithi055 systemd[1]: Stopping Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:48.166 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 18:25:47 smithi055 ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a[166572]: ts=2023-09-23T18:25:47.813Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-09-23T18:25:48.429 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 18:25:48 smithi055 bash[485954]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-alertmanager-a 2023-09-23T18:25:48.430 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 18:25:48 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@alertmanager.a.service: Succeeded. 2023-09-23T18:25:48.430 INFO:journalctl@ceph.alertmanager.a.smithi055.stdout:Sep 23 18:25:48 smithi055 systemd[1]: Stopped Ceph alertmanager.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:25:48.887 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 18:25:48 smithi055 systemd[1]: Stopping Ceph node-exporter.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:49.141 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 18:25:49 smithi055 bash[486077]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-a 2023-09-23T18:25:49.142 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 18:25:49 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@node-exporter.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T18:25:49.680 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 18:25:49 smithi055 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@node-exporter.a.service: Failed with result 'exit-code'. 2023-09-23T18:25:49.680 INFO:journalctl@ceph.node-exporter.a.smithi055.stdout:Sep 23 18:25:49 smithi055 systemd[1]: Stopped Ceph node-exporter.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:25:56.449 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 --force --keep-logs 2023-09-23T18:25:58.077 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 18:25:57 smithi099 systemd[1]: Stopping Ceph grafana.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:58.534 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 18:25:58 smithi099 bash[307652]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-grafana-a 2023-09-23T18:25:58.821 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 18:25:58 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@grafana.a.service: Main process exited, code=exited, status=143/n/a 2023-09-23T18:25:58.821 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 18:25:58 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@grafana.a.service: Failed with result 'exit-code'. 2023-09-23T18:25:58.821 INFO:journalctl@ceph.grafana.a.smithi099.stdout:Sep 23 18:25:58 smithi099 systemd[1]: Stopped Ceph grafana.a for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:25:59.559 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 18:25:58 smithi099 systemd[1]: Stopping Ceph node-exporter.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7... 2023-09-23T18:25:59.560 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 18:25:59 smithi099 bash[307776]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7-node-exporter-b 2023-09-23T18:25:59.560 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 18:25:59 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@node-exporter.b.service: Main process exited, code=exited, status=143/n/a 2023-09-23T18:25:59.826 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 18:25:59 smithi099 systemd[1]: ceph-2b70c030-5a38-11ee-8db4-212e2dc638e7@node-exporter.b.service: Failed with result 'exit-code'. 2023-09-23T18:25:59.827 INFO:journalctl@ceph.node-exporter.b.smithi099.stdout:Sep 23 18:25:59 smithi099 systemd[1]: Stopped Ceph node-exporter.b for 2b70c030-5a38-11ee-8db4-212e2dc638e7. 2023-09-23T18:26:02.025 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T18:26:02.063 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f /etc/ceph/ceph.conf /etc/ceph/ceph.client.admin.keyring 2023-09-23T18:26:02.093 INFO:tasks.cephadm:Archiving crash dumps... 2023-09-23T18:26:02.095 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763/remote/smithi055/crash 2023-09-23T18:26:02.096 DEBUG:teuthology.orchestra.run.smithi055:> sudo tar c -f - -C /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T18:26:02.155 INFO:teuthology.orchestra.run.smithi055.stderr:tar: /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T18:26:02.156 INFO:teuthology.orchestra.run.smithi055.stderr:tar: Error is not recoverable: exiting now 2023-09-23T18:26:02.157 DEBUG:teuthology.misc:Transferring archived files from smithi099:/var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/crash to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763/remote/smithi099/crash 2023-09-23T18:26:02.158 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/crash -- . 2023-09-23T18:26:02.217 INFO:teuthology.orchestra.run.smithi099.stderr:tar: /var/lib/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/crash: Cannot open: No such file or directory 2023-09-23T18:26:02.217 INFO:teuthology.orchestra.run.smithi099.stderr:tar: Error is not recoverable: exiting now 2023-09-23T18:26:02.217 INFO:tasks.cephadm:Checking cluster log for badness... 2023-09-23T18:26:02.218 DEBUG:teuthology.orchestra.run.smithi055:> sudo egrep '\[ERR\]|\[WRN\]|\[SEC\]' /var/log/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/ceph.log | egrep -v '\(MDS_ALL_DOWN\)' | egrep -v '\(MDS_UP_LESS_THAN_MAX\)' | head -n 1 2023-09-23T18:26:02.256 INFO:teuthology.orchestra.run.smithi055.stderr:grep: /var/log/ceph/2b70c030-5a38-11ee-8db4-212e2dc638e7/ceph.log: No such file or directory 2023-09-23T18:26:02.259 INFO:tasks.cephadm:Compressing logs... 2023-09-23T18:26:02.259 DEBUG:teuthology.orchestra.run.smithi055:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T18:26:02.302 DEBUG:teuthology.orchestra.run.smithi099:> sudo find /var/log/ceph /var/log/rbd-target-api -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T18:26:02.340 INFO:teuthology.orchestra.run.smithi099.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T18:26:02.341 INFO:teuthology.orchestra.run.smithi055.stderr:find: ‘/var/log/rbd-target-api’: No such file or directory 2023-09-23T18:36:36.383 INFO:tasks.cephadm:Archiving logs... 2023-09-23T18:36:36.407 DEBUG:teuthology.misc:Transferring archived files from smithi055:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763/remote/smithi055/log 2023-09-23T18:36:36.408 DEBUG:teuthology.orchestra.run.smithi055:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T18:37:36.995 DEBUG:teuthology.misc:Transferring archived files from smithi099:/var/log/ceph to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763/remote/smithi099/log 2023-09-23T18:37:36.996 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /var/log/ceph -- . 2023-09-23T18:38:58.571 INFO:tasks.cephadm:Removing cluster... 2023-09-23T18:38:58.572 DEBUG:teuthology.orchestra.run.smithi055:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 --force 2023-09-23T18:39:00.112 DEBUG:teuthology.orchestra.run.smithi099:> sudo /home/ubuntu/cephtest/cephadm rm-cluster --fsid 2b70c030-5a38-11ee-8db4-212e2dc638e7 --force 2023-09-23T18:39:01.811 INFO:tasks.cephadm:Removing cephadm ... 2023-09-23T18:39:01.812 DEBUG:teuthology.orchestra.run.smithi055:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T18:39:01.834 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf /home/ubuntu/cephtest/cephadm 2023-09-23T18:39:01.865 INFO:tasks.cephadm:Teardown complete 2023-09-23T18:39:01.865 DEBUG:teuthology.run_tasks:Unwinding manager install 2023-09-23T18:39:01.879 INFO:teuthology.task.install.util:Removing shipped files: /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer... 2023-09-23T18:39:01.880 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T18:39:01.882 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f -- /home/ubuntu/cephtest/valgrind.supp /usr/bin/daemon-helper /usr/bin/adjust-ulimits /usr/bin/stdin-killer 2023-09-23T18:39:02.019 DEBUG:teuthology.run_tasks:Unwinding manager clock 2023-09-23T18:39:02.029 INFO:teuthology.task.clock:Checking final clock skew... 2023-09-23T18:39:02.029 DEBUG:teuthology.orchestra.run.smithi055:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T18:39:02.032 DEBUG:teuthology.orchestra.run.smithi099:> PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-09-23T18:39:02.047 INFO:teuthology.orchestra.run.smithi055.stderr:bash: ntpq: command not found 2023-09-23T18:39:02.057 INFO:teuthology.orchestra.run.smithi099.stderr:bash: ntpq: command not found 2023-09-23T18:39:02.078 INFO:teuthology.orchestra.run.smithi099.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T18:39:02.078 INFO:teuthology.orchestra.run.smithi099.stdout:=============================================================================== 2023-09-23T18:39:02.078 INFO:teuthology.orchestra.run.smithi099.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 10 -701us[ -701us] +/- 74ms 2023-09-23T18:39:02.078 INFO:teuthology.orchestra.run.smithi099.stdout:^* hv02.front.sepia.ceph.com 3 6 377 12 -1034us[-1036us] +/- 56ms 2023-09-23T18:39:02.078 INFO:teuthology.orchestra.run.smithi099.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 10 +2264us[+2264us] +/- 81ms 2023-09-23T18:39:02.079 INFO:teuthology.orchestra.run.smithi099.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T18:39:02.096 INFO:teuthology.orchestra.run.smithi055.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-09-23T18:39:02.096 INFO:teuthology.orchestra.run.smithi055.stdout:=============================================================================== 2023-09-23T18:39:02.096 INFO:teuthology.orchestra.run.smithi055.stdout:^+ hv01.front.sepia.ceph.com 2 6 377 9 -661us[ -661us] +/- 74ms 2023-09-23T18:39:02.096 INFO:teuthology.orchestra.run.smithi055.stdout:^* hv02.front.sepia.ceph.com 3 6 377 9 -975us[ -975us] +/- 56ms 2023-09-23T18:39:02.097 INFO:teuthology.orchestra.run.smithi055.stdout:^+ hv03.front.sepia.ceph.com 3 6 377 10 +2319us[+2319us] +/- 81ms 2023-09-23T18:39:02.097 INFO:teuthology.orchestra.run.smithi055.stdout:^? hv04.front.sepia.ceph.com 0 10 0 - +0ns[ +0ns] +/- 0ns 2023-09-23T18:39:02.098 DEBUG:teuthology.run_tasks:Unwinding manager ansible.cephlab 2023-09-23T18:39:02.108 INFO:teuthology.task.ansible:Skipping ansible cleanup... 2023-09-23T18:39:02.108 DEBUG:teuthology.run_tasks:Unwinding manager selinux 2023-09-23T18:39:02.176 DEBUG:teuthology.orchestra.run.smithi055:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T18:39:02.311 DEBUG:teuthology.orchestra.run.smithi099:> mkdir /home/ubuntu/cephtest/archive/audit && sudo cp /var/log/audit/audit.log /home/ubuntu/cephtest/archive/audit && sudo chown $USER /home/ubuntu/cephtest/archive/audit/audit.log && gzip /home/ubuntu/cephtest/archive/audit/audit.log 2023-09-23T18:39:02.440 DEBUG:teuthology.orchestra.run.smithi055:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T18:39:02.477 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:02.477 DEBUG:teuthology.orchestra.run.smithi099:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-09-23T18:39:02.509 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:02.510 DEBUG:teuthology.run_tasks:Unwinding manager pcp 2023-09-23T18:39:02.521 DEBUG:teuthology.run_tasks:Unwinding manager internal.timer 2023-09-23T18:39:02.568 INFO:teuthology.task.internal:Duration was 4447.583625 seconds 2023-09-23T18:39:02.569 DEBUG:teuthology.run_tasks:Unwinding manager internal.syslog 2023-09-23T18:39:02.716 INFO:teuthology.task.internal.syslog:Shutting down syslog monitoring... 2023-09-23T18:39:02.716 DEBUG:teuthology.orchestra.run.smithi055:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T18:39:02.719 DEBUG:teuthology.orchestra.run.smithi099:> sudo rm -f -- /etc/rsyslog.d/80-cephtest.conf && sudo service rsyslog restart 2023-09-23T18:39:02.796 INFO:teuthology.orchestra.run.smithi055.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T18:39:02.826 INFO:teuthology.orchestra.run.smithi099.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-09-23T18:39:03.205 INFO:teuthology.task.internal.syslog:Checking logs for errors... 2023-09-23T18:39:03.205 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi055.front.sepia.ceph.com 2023-09-23T18:39:03.205 DEBUG:teuthology.orchestra.run.smithi055:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T18:39:03.232 DEBUG:teuthology.task.internal.syslog:Checking ubuntu@smithi099.front.sepia.ceph.com 2023-09-23T18:39:03.233 DEBUG:teuthology.orchestra.run.smithi099:> egrep --binary-files=text '\bBUG\b|\bINFO\b|\bDEADLOCK\b' /home/ubuntu/cephtest/archive/syslog/kern.log | grep -v 'task .* blocked for more than .* seconds' | grep -v 'lockdep is turned off' | grep -v 'trying to register non-static key' | grep -v 'DEBUG: fsize' | grep -v CRON | grep -v 'BUG: bad unlock balance detected' | grep -v 'inconsistent lock state' | grep -v '*** DEADLOCK ***' | grep -v 'INFO: possible irq lock inversion dependency detected' | grep -v 'INFO: NMI handler (perf_event_nmi_handler) took too long to run' | grep -v 'INFO: recovery required on readonly' | grep -v 'ceph-create-keys: INFO' | grep -v INFO:ceph-create-keys | grep -v 'Loaded datasource DataSourceOpenStack' | grep -v 'container-storage-setup: INFO: Volume group backing root filesystem could not be determined' | egrep -v '\bsalt-master\b|\bsalt-minion\b|\bsalt-api\b' | grep -v ceph-crash | egrep -v '\btcmu-runner\b.*\bINFO\b' | head -n 1 2023-09-23T18:39:03.273 INFO:teuthology.task.internal.syslog:Compressing syslogs... 2023-09-23T18:39:03.273 DEBUG:teuthology.orchestra.run.smithi055:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T18:39:03.276 DEBUG:teuthology.orchestra.run.smithi099:> find /home/ubuntu/cephtest/archive/syslog -name '*.log' -print0 | sudo xargs -0 --no-run-if-empty -- gzip -- 2023-09-23T18:39:03.463 INFO:teuthology.task.internal.syslog:Gathering journactl -b0... 2023-09-23T18:39:03.464 DEBUG:teuthology.orchestra.run.smithi055:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T18:39:03.466 DEBUG:teuthology.orchestra.run.smithi099:> sudo journalctl -b0 | gzip -9 > /home/ubuntu/cephtest/archive/syslog/journalctl-b0.gz 2023-09-23T18:39:03.848 DEBUG:teuthology.run_tasks:Unwinding manager internal.sudo 2023-09-23T18:39:03.859 INFO:teuthology.task.internal:Restoring /etc/sudoers... 2023-09-23T18:39:03.859 DEBUG:teuthology.orchestra.run.smithi055:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T18:39:03.903 DEBUG:teuthology.orchestra.run.smithi099:> sudo mv -f /etc/sudoers.orig.teuthology /etc/sudoers 2023-09-23T18:39:03.948 DEBUG:teuthology.run_tasks:Unwinding manager internal.coredump 2023-09-23T18:39:03.959 DEBUG:teuthology.orchestra.run.smithi055:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T18:39:03.962 DEBUG:teuthology.orchestra.run.smithi099:> sudo sysctl -w kernel.core_pattern=core && sudo bash -c 'for f in `find /home/ubuntu/cephtest/archive/coredump -type f`; do file $f | grep -q systemd-sysusers && rm $f || true ; done' && rmdir --ignore-fail-on-non-empty -- /home/ubuntu/cephtest/archive/coredump 2023-09-23T18:39:03.987 INFO:teuthology.orchestra.run.smithi055.stdout:kernel.core_pattern = core 2023-09-23T18:39:03.996 INFO:teuthology.orchestra.run.smithi099.stdout:kernel.core_pattern = core 2023-09-23T18:39:04.038 DEBUG:teuthology.orchestra.run.smithi055:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T18:39:04.065 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.066 DEBUG:teuthology.orchestra.run.smithi099:> test -e /home/ubuntu/cephtest/archive/coredump 2023-09-23T18:39:04.094 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.095 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive 2023-09-23T18:39:04.107 INFO:teuthology.task.internal:Transferring archived files... 2023-09-23T18:39:04.108 DEBUG:teuthology.misc:Transferring archived files from smithi055:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763/remote/smithi055 2023-09-23T18:39:04.108 DEBUG:teuthology.orchestra.run.smithi055:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T18:39:04.173 DEBUG:teuthology.misc:Transferring archived files from smithi099:/home/ubuntu/cephtest/archive to /home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763/remote/smithi099 2023-09-23T18:39:04.173 DEBUG:teuthology.orchestra.run.smithi099:> sudo tar c -f - -C /home/ubuntu/cephtest/archive -- . 2023-09-23T18:39:04.219 INFO:teuthology.task.internal:Removing archive directory... 2023-09-23T18:39:04.220 DEBUG:teuthology.orchestra.run.smithi055:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T18:39:04.222 DEBUG:teuthology.orchestra.run.smithi099:> rm -rf -- /home/ubuntu/cephtest/archive 2023-09-23T18:39:04.266 DEBUG:teuthology.run_tasks:Unwinding manager internal.archive_upload 2023-09-23T18:39:04.279 INFO:teuthology.task.internal:Not uploading archives. 2023-09-23T18:39:04.279 DEBUG:teuthology.run_tasks:Unwinding manager internal.base 2023-09-23T18:39:04.312 INFO:teuthology.task.internal:Tidying up after the test... 2023-09-23T18:39:04.313 DEBUG:teuthology.orchestra.run.smithi055:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T18:39:04.315 DEBUG:teuthology.orchestra.run.smithi099:> find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest 2023-09-23T18:39:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: 262307 4 drwxr-xr-x 3 ubuntu ubuntu 4096 Sep 23 18:39 /home/ubuntu/cephtest 2023-09-23T18:39:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: 3538945 4 drwxrwxr-x 3 ubuntu ubuntu 4096 Sep 23 18:22 /home/ubuntu/cephtest/mnt.0 2023-09-23T18:39:04.330 INFO:teuthology.orchestra.run.smithi055.stdout: 3538955 4 drwxrwxr-x 2 ubuntu ubuntu 4096 Sep 23 18:22 /home/ubuntu/cephtest/mnt.0/client.0 2023-09-23T18:39:04.331 INFO:teuthology.orchestra.run.smithi099.stdout: 262320 4 drwxr-xr-x 2 ubuntu ubuntu 4096 Sep 23 18:39 /home/ubuntu/cephtest 2023-09-23T18:39:04.337 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-09-23T18:39:04.337 INFO:teuthology.orchestra.run.smithi055.stderr:rmdir: failed to remove '/home/ubuntu/cephtest': Directory not empty 2023-09-23T18:39:04.337 ERROR:teuthology.run_tasks:Manager failed: internal.base Traceback (most recent call last): File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/run_tasks.py", line 154, in run_tasks suppress = manager.__exit__(*exc_info) File "/usr/lib/python3.8/contextlib.py", line 120, in __exit__ next(self.gen) File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/task/internal/__init__.py", line 53, in base run.wait( File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 479, in wait proc.wait() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 161, in wait self._raise_for_status() File "/home/teuthworker/src/git.ceph.com_teuthology_54e62bcbac4e53d9685e08328b790d3b20d71cae/teuthology/orchestra/run.py", line 181, in _raise_for_status raise CommandFailedError( teuthology.exceptions.CommandFailedError: Command failed on smithi055 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T18:39:04.344 DEBUG:teuthology.run_tasks:Unwinding manager kernel 2023-09-23T18:39:04.361 DEBUG:teuthology.run_tasks:Unwinding manager console_log 2023-09-23T18:39:04.411 DEBUG:teuthology.run_tasks:Exception was not quenched, exiting: CommandFailedError: Command failed on smithi055 with status 1: 'find /home/ubuntu/cephtest -ls ; rmdir -- /home/ubuntu/cephtest' 2023-09-23T18:39:04.412 INFO:teuthology.nuke:Checking targets against current locks 2023-09-23T18:39:04.442 DEBUG:teuthology.nuke:shortname: smithi055 2023-09-23T18:39:04.442 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T18:39:04.466 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi055.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 17:20:13.686066', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T18:39:04.469 DEBUG:teuthology.nuke:shortname: smithi099 2023-09-23T18:39:04.470 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-09-23T18:39:04.493 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi099.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-09-23_14:23:10-upgrade:quincy-x-reef-distro-default-smithi/7401763', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'rhel', 'os_version': '8.6', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-09-23 17:20:13.687547', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLVvN3+LmzPZ4ixiwyCkS3KZLZ9R6MOPm+kPtaJbl8fExpzbk8TudXCcJWHBL7DGPWFvye2jT6sei9eiX0EUQYs='} 2023-09-23T18:39:04.518 INFO:teuthology.orchestra.console.smithi055:Power off 2023-09-23T18:39:04.518 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T18:39:04.541 DEBUG:teuthology.orchestra.console.smithi055:power off output: Chassis Power Control: Down/Off 2023-09-23T18:39:04.542 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T18:39:04.558 INFO:teuthology.orchestra.console.smithi099:Power off 2023-09-23T18:39:04.558 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power off 2023-09-23T18:39:04.566 DEBUG:teuthology.orchestra.console.smithi055:check power output: Chassis Power is on 2023-09-23T18:39:04.581 DEBUG:teuthology.orchestra.console.smithi099:power off output: Chassis Power Control: Down/Off 2023-09-23T18:39:04.581 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T18:39:04.602 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is on 2023-09-23T18:39:08.567 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T18:39:08.603 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T18:39:08.679 DEBUG:teuthology.orchestra.console.smithi055:check power output: Chassis Power is on 2023-09-23T18:39:08.711 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is on 2023-09-23T18:39:12.682 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T18:39:12.712 DEBUG:teuthology.orchestra.console.smithi099:pexpect command: ipmitool -H smithi099.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T18:39:12.794 DEBUG:teuthology.orchestra.console.smithi055:check power output: Chassis Power is on 2023-09-23T18:39:12.823 DEBUG:teuthology.orchestra.console.smithi099:check power output: Chassis Power is off 2023-09-23T18:39:12.924 INFO:teuthology.orchestra.console.smithi099:Power off completed 2023-09-23T18:39:16.798 DEBUG:teuthology.orchestra.console.smithi055:pexpect command: ipmitool -H smithi055.ipmi.sepia.ceph.com -I lanplus -U inktank -P ApGNXcA7 power status 2023-09-23T18:39:16.908 DEBUG:teuthology.orchestra.console.smithi055:check power output: Chassis Power is off 2023-09-23T18:39:17.009 INFO:teuthology.orchestra.console.smithi055:Power off completed 2023-09-23T18:39:17.113 INFO:teuthology.run:Summary data: description: upgrade:quincy-x/stress-split/{0-distro/rhel_8.6_container_tools_rhel8 0-roles 1-start 2-first-half-tasks/rbd-import-export 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} duration: 4447.583624839783 failure_reason: '' flavor: default owner: scheduled_teuthology@teuthology sentry_event: https://sentry.ceph.com/organizations/ceph/?query=8ccd226a2e034608a31269aa82ea231b status: fail success: false 2023-09-23T18:39:17.113 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-09-23T18:39:17.195 INFO:teuthology.run:FAIL